sparkforensics-server 0.3.0 → 0.5.0

package/public/docs/assets/tuning-reference_bottleneck-cold-start.md.CEuy-72y.js

@@ -1,7 +0,0 @@
-import{_ as t,o as a,c as s,a5 as o}from"./chunks/framework.DSg0KOwT.js";const r="/docs/assets/cold-start-timeline.DxC_Sc7w.svg",i="/docs/assets/cold-start-timeline.dark.CZ17YcAG.svg",m=JSON.parse('{"title":"Cold Start","description":"","frontmatter":{"title":"Cold Start"},"headers":[],"relativePath":"tuning-reference/bottleneck-cold-start.md","filePath":"tuning-reference/bottleneck-cold-start.md"}'),n={name:"tuning-reference/bottleneck-cold-start.md"};function l(c,e,h,d,f,p){return a(),s("div",null,[...e[0]||(e[0]=[o('<h1 id="bottleneck-cold-start" tabindex="-1">Cold Start <a class="header-anchor" href="#bottleneck-cold-start" aria-label="Permalink to &quot;Cold Start {#bottleneck-cold-start}&quot;">​</a></h1><p><span class="tag">COLD</span></p><h2 id="what-it-is" tabindex="-1">What it is <a class="header-anchor" href="#what-it-is" aria-label="Permalink to &quot;What it is&quot;">​</a></h2><p>Cold start is the gap between when a Spark application starts and when it actually begins doing work: the delay before the <a href="./spark-architecture.html">DAGScheduler</a> hands off the first stage for execution. In the event log, that moment is marked by <code>SparkListenerStageSubmitted</code>, whose payload is a direct serialization of a <code>StageInfo</code>: the event name plus the <code>stageInfo</code> and any properties, with no task-level data attached<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup>. That shape matches what the event records. The DAGScheduler doesn&#39;t pre-schedule the whole DAG upfront (it reacts to stage completions, unlocking child stages one at a time), and once it has carved the DAG into stages and handed a <code>TaskSet</code> for the next stage to the TaskScheduler, only then does the TaskScheduler assign individual tasks to executors<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup>. <code>SparkListenerStageSubmitted</code> therefore captures the moment the DAGScheduler creates/submits that <code>TaskSet</code>, before any task has actually been placed on an executor.</p><h2 id="how-it-s-detected" tabindex="-1">How it&#39;s detected <a class="header-anchor" href="#how-it-s-detected" aria-label="Permalink to &quot;How it&#39;s detected&quot;">​</a></h2><p><code>firstStageSubmittedAt − app.startTime</code> &gt; 30 s → Warning. <code>firstStageSubmittedAt</code> is a synthetic field: the timestamp of the first <code>SparkListenerStageSubmitted</code> event in the event log.</p><img class="light-only" src="'+r+'" alt="A timeline from app.startTime through the gap where executors are acquired and no tasks run to firstStageSubmittedAt, with that gap marked as the cold-start delay."><img class="dark-only" src="'+i+`" alt="A timeline from app.startTime through the gap where executors are acquired and no tasks run to firstStageSubmittedAt, with that gap marked as the cold-start delay."><h2 id="why-it-matters" tabindex="-1">Why it matters <a class="header-anchor" href="#why-it-matters" aria-label="Permalink to &quot;Why it matters&quot;">​</a></h2><p>Because the event carries only stage-level metadata and fires ahead of any executor task placement<sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup><sup class="footnote-ref"><a href="#fn2" id="fnref2:1">[2:1]</a></sup>, a large gap here reflects time spent before the DAGScheduler could submit work at all, not time spent inside tasks. It&#39;s a signal about scheduling/startup latency, distinct from the per-task metrics that describe work once tasks are actually running on executors.</p><h2 id="how-to-fix-it" tabindex="-1">How to fix it <a class="header-anchor" href="#how-to-fix-it" aria-label="Permalink to &quot;How to fix it&quot;">​</a></h2><p>Dynamic-allocation ramp-up isn&#39;t directly tied to <code>firstStageSubmittedAt</code> timing, but the configuration below governs how many executors an application has available at launch, and how reliably that number can grow:</p><ul><li>Size executors deliberately instead of defaulting to one large executor per node. For a 10-node, 16-core/64GB cluster targeting ~5 cores per executor, the standard derivation reserves 1 core per node for daemons (15 usable cores/node, 150 total), divides by 5 cores/executor (30), and subtracts 1 for the YARN ApplicationMaster: 29 executors × 5 cores × ~18GB<sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup>. Cloudera&#39;s equivalent 6-node walkthrough lands on the same shape: 17 executors × 5 cores × 19G, rather than one fat executor per node<sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup>.</li><li>If dynamic allocation is enabled, its ability to add executors at all depends on one of a few supporting mechanisms being turned on. One such mechanism is <code>spark.dynamicAllocation.shuffleTracking.enabled</code> (default <code>true</code> since Spark 3.0), which &quot;enables shuffle file tracking for executors, which allows dynamic allocation without the need for an external shuffle service,&quot; and &quot;will try to keep alive executors that are storing shuffle data for active jobs&quot;<sup class="footnote-ref"><a href="#fn5" id="fnref5">[5]</a></sup>. Without shuffle tracking or an external shuffle service configured, dynamic allocation cannot be enabled at all<sup class="footnote-ref"><a href="#fn5" id="fnref5:1">[5:1]</a></sup>.</li></ul><p>Deliberate executor sizing plus the dynamic-allocation precondition (values from the 16-core/64GB-node derivation above):</p><div class="language-properties vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">properties</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># ~5 cores per executor instead of one fat executor per node</span></span>
-<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.executor.cores</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=5</span></span>
-<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.executor.memory</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=18g</span></span>
-<span class="line"></span>
-<span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Dynamic allocation, with its required shuffle-tracking precondition (default true since Spark 3.0)</span></span>
-<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.dynamicAllocation.enabled</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=true</span></span>
-<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.dynamicAllocation.shuffleTracking.enabled</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=true</span></span></code></pre></div><h2 id="limitations-false-positive-risk" tabindex="-1">Limitations / false-positive risk <a class="header-anchor" href="#limitations-false-positive-risk" aria-label="Permalink to &quot;Limitations / false-positive risk&quot;">​</a></h2><p>Some startup latency is unavoidable: a cluster still has to provision before it can run anything. The signal measures the gap before the first stage is submitted, and that gap can legitimately vary by cluster manager, so a warning here doesn&#39;t always mean something is wrong.</p><h2 id="related" tabindex="-1">Related <a class="header-anchor" href="#related" aria-label="Permalink to &quot;Related&quot;">​</a></h2><ul><li><strong>Executor sizing &amp; dynamic allocation:</strong> <a href="./cluster-config.html">Cluster Tuning</a></li></ul><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><a href="https://raw.githubusercontent.com/apache/spark/v3.5.0/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala" target="_blank" rel="noreferrer">JsonProtocol.scala</a> <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><a href="https://luminousmen.com/post/spark-anatomy-of-spark-application" target="_blank" rel="noreferrer">Anatomy of a Spark Application</a> <a href="#fnref2" class="footnote-backref">↩︎</a> <a href="#fnref2:1" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><a href="https://raw.githubusercontent.com/spoddutur/spark-notes/master/distribution_of_executors_cores_and_memory_for_spark_application.md" target="_blank" rel="noreferrer">Distribution of Executors, Cores and Memory for a Spark Application</a> <a href="#fnref3" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><a href="https://blog.cloudera.com/how-to-tune-your-apache-spark-jobs-part-2/" target="_blank" rel="noreferrer">How to Tune Your Apache Spark Jobs (Part 2)</a> <a href="#fnref4" class="footnote-backref">↩︎</a></p></li><li id="fn5" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration (Spark)</a> <a href="#fnref5" class="footnote-backref">↩︎</a> <a href="#fnref5:1" class="footnote-backref">↩︎</a></p></li></ol></section>`,21)])])}const k=t(n,[["render",l]]);export{m as __pageData,k as default};

package/public/docs/assets/tuning-reference_bottleneck-cold-start.md.CEuy-72y.lean.js

@@ -1 +0,0 @@
-import{_ as t,o as a,c as s,a5 as o}from"./chunks/framework.DSg0KOwT.js";const r="/docs/assets/cold-start-timeline.DxC_Sc7w.svg",i="/docs/assets/cold-start-timeline.dark.CZ17YcAG.svg",m=JSON.parse('{"title":"Cold Start","description":"","frontmatter":{"title":"Cold Start"},"headers":[],"relativePath":"tuning-reference/bottleneck-cold-start.md","filePath":"tuning-reference/bottleneck-cold-start.md"}'),n={name:"tuning-reference/bottleneck-cold-start.md"};function l(c,e,h,d,f,p){return a(),s("div",null,[...e[0]||(e[0]=[o("",21)])])}const k=t(n,[["render",l]]);export{m as __pageData,k as default};

package/public/docs/assets/tuning-reference_bottleneck-duplicate-plan-subtree.md.CIohQDfn.js

@@ -1 +0,0 @@
-import{_ as t,a}from"./chunks/duplicate-plan-subtree.dark.Cdp70QhV.js";import{_ as r,o as s,c as o,a5 as n}from"./chunks/framework.DSg0KOwT.js";const b=JSON.parse('{"title":"Duplicate Plan Subtree","description":"","frontmatter":{"title":"Duplicate Plan Subtree"},"headers":[],"relativePath":"tuning-reference/bottleneck-duplicate-plan-subtree.md","filePath":"tuning-reference/bottleneck-duplicate-plan-subtree.md"}'),i={name:"tuning-reference/bottleneck-duplicate-plan-subtree.md"};function l(h,e,c,f,u,d){return s(),o("div",null,[...e[0]||(e[0]=[n('<h1 id="bottleneck-duplicate-plan-subtree" tabindex="-1">Duplicate Plan Subtree <a class="header-anchor" href="#bottleneck-duplicate-plan-subtree" aria-label="Permalink to &quot;Duplicate Plan Subtree {#bottleneck-duplicate-plan-subtree}&quot;">​</a></h1><p><span class="tag">DUPPLAN</span></p><h2 id="what-it-is" tabindex="-1">What it is <a class="header-anchor" href="#what-it-is" aria-label="Permalink to &quot;What it is&quot;">​</a></h2><p>The physical plan is the form Spark actually runs: after optimizing the logical plan, the engine produces a plan that &quot;specifies how the logical plan will execute on the cluster,&quot; compiling the query into a series of RDDs and transformations<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup>. A duplicated subtree is the same source read, together with the operators stacked above it, appearing in more than one branch of that plan. When the same logic sits in two different branches of a pipeline, you are &quot;basically signing up to recompute everything from scratch multiple times,&quot; because every action on an uncached DataFrame starts back at the source and rebuilds all the intermediate steps<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup>.</p><h2 id="how-it-s-detected" tabindex="-1">How it&#39;s detected <a class="header-anchor" href="#how-it-s-detected" aria-label="Permalink to &quot;How it&#39;s detected&quot;">​</a></h2><p>Exchanges are the giveaway. A <a href="./shuffle.html">shuffle</a> or broadcast is an exchange, and these are the points where Spark&#39;s pipelining stops: &quot;Spark operators are often pipelined and executed in parallel processes. However, a shuffle or broadcast exchange breaks this pipeline. We call them materialization points and use the term &#39;query stages&#39; to denote subsections bounded by these materialization points&quot;<sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup>. The signal is the same source-plus-operators subtree feeding more than one branch across those materialization boundaries, which is the shape that gets recomputed unless something intervenes.</p><img class="light-only" src="'+t+'" alt="Before and after: the same scan and operators feed two branches and run twice, then a single shared node from exchange reuse or a cache feeds both consumers."><img class="dark-only" src="'+a+'" alt="Before and after: the same scan and operators feed two branches and run twice, then a single shared node from exchange reuse or a cache feeds both consumers."><table tabindex="0"><thead><tr><th>Signal</th><th>What it points to</th></tr></thead><tbody><tr><td>Identical scan + operators in two or more branches</td><td>Same subtree recomputed per branch</td></tr><tr><td>A materialization point (shuffle/broadcast) below the repeat</td><td>The pipeline breaks and the branch restarts from source</td></tr></tbody></table><h2 id="why-it-matters" tabindex="-1">Why it matters <a class="header-anchor" href="#why-it-matters" aria-label="Permalink to &quot;Why it matters&quot;">​</a></h2><p>Left alone, a repeated subtree is repeated work. Each branch re-reads the source and re-runs everything above it, so a subtree that shows up twice is roughly paid for twice<sup class="footnote-ref"><a href="#fn2" id="fnref2:1">[2:1]</a></sup>. That cost lands on exactly the expensive parts of a plan (scans and shuffles) rather than on cheap row-level operators.</p><h2 id="how-to-fix-it" tabindex="-1">How to fix it <a class="header-anchor" href="#how-to-fix-it" aria-label="Permalink to &quot;How to fix it&quot;">​</a></h2><p>Two mechanisms let a plan reuse an identical subtree instead of rebuilding it.</p><ul><li>Exchange reuse leans on the fact that a shuffle already writes its output to disk. Spark &quot;always executes shuffles by first having the &#39;source&#39; tasks ... write shuffle files to their local disks,&quot; so &quot;running a new job over data that&#39;s already been shuffled does not rerun the &#39;source&#39; side of the shuffle. Because the shuffle files were already written to disk earlier, Spark knows that it can use them to run the later stages of the job, and it need not redo the earlier ones. In the Spark UI and logs, you will see the pre-shuffle stages marked as &#39;skipped&#39;&quot;<sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup>. When the same materialized exchange feeds two consumers, the second reads those existing shuffle files instead of recomputing the subtree beneath it.</li><li><code>cache()</code> / <code>persist()</code> truncate recomputation by materializing the result and keying future lookups on it. Persisting &quot;means materializing an RDD (usually by storing it in memory on the executors) for reuse during the current job&quot;<sup class="footnote-ref"><a href="#fn2" id="fnref2:2">[2:2]</a></sup>, and in the Structured API the lookup is plan-keyed: &quot;caching is done based on the physical plan. This means that we effectively store the physical plan as our key ... and perform a lookup prior to the execution of a Structured job&quot;<sup class="footnote-ref"><a href="#fn1" id="fnref1:2">[1:2]</a></sup>. On a hit, Spark returns the stored data rather than rebuilding intermediate steps from the source<sup class="footnote-ref"><a href="#fn2" id="fnref2:3">[2:3]</a></sup>.</li></ul><blockquote><p><strong>PySpark:</strong> if the repeated subtree is a DataFrame you reuse across branches, <code>df.cache()</code> (or <code>df.persist()</code>) before the branches split lets both read the materialized result instead of recomputing it.</p></blockquote><h2 id="confidence" tabindex="-1">Confidence <a class="header-anchor" href="#confidence" aria-label="Permalink to &quot;Confidence&quot;">​</a></h2><p>Medium. Inferring a duplicated subtree from the static plan is a medium-confidence signal that requires runtime validation: whether a repeated subtree actually turns into avoided work depends on runtime facts the plan never shows, such as cache hits, retained shuffle files, and eviction<sup class="footnote-ref"><a href="#fn2" id="fnref2:4">[2:4]</a></sup>.</p><h2 id="limitations-false-positive-risk" tabindex="-1">Limitations / false-positive risk <a class="header-anchor" href="#limitations-false-positive-risk" aria-label="Permalink to &quot;Limitations / false-positive risk&quot;">​</a></h2><p>Two subtrees that look identical in the printed plan are not always the same recomputed work. The cache &quot;is tied to the analyzed logical plan, not the final optimized one,&quot; so semantically identical queries with different analyzed plans miss the cache and recompute even though &quot;the optimizer will eventually produce the same physical plan&quot;<sup class="footnote-ref"><a href="#fn2" id="fnref2:5">[2:5]</a></sup>. <a href="./caching.html">Caching</a> is also lazy and often partial: it happens &quot;only as they are accessed,&quot; &quot;the only blocks that get cached are the ones Spark was forced to touch,&quot; and Spark gives &quot;no built-in visibility into how much of your DataFrame is actually cached, or how much has already been evicted&quot;<sup class="footnote-ref"><a href="#fn2" id="fnref2:6">[2:6]</a></sup>. Plan-level inference can therefore over-count, flagging repetition that either never materializes or is only partly reused.</p><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><a href="https://www.oreilly.com/library/view/spark-the-definitive/9781491912201/" target="_blank" rel="noreferrer">Spark: The Definitive Guide</a>, Chambers &amp; Zaharia, chs. 4, 15, 19 <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a> <a href="#fnref1:2" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><a href="https://luminousmen.com/post/explaining-the-mechanics-of-spark-caching" target="_blank" rel="noreferrer">Explaining the mechanics of Spark caching</a> <a href="#fnref2" class="footnote-backref">↩︎</a> <a href="#fnref2:1" class="footnote-backref">↩︎</a> <a href="#fnref2:2" class="footnote-backref">↩︎</a> <a href="#fnref2:3" class="footnote-backref">↩︎</a> <a href="#fnref2:4" class="footnote-backref">↩︎</a> <a href="#fnref2:5" class="footnote-backref">↩︎</a> <a href="#fnref2:6" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><a href="https://www.databricks.com/blog/2020/05/29/adaptive-query-execution-speeding-up-spark-sql-at-runtime.html" target="_blank" rel="noreferrer">Adaptive Query Execution: Speeding Up Spark SQL at Runtime</a> <a href="#fnref3" class="footnote-backref">↩︎</a></p></li></ol></section>',21)])])}const g=r(i,[["render",l]]);export{b as __pageData,g as default};

package/public/docs/assets/tuning-reference_bottleneck-duplicate-plan-subtree.md.CIohQDfn.lean.js

@@ -1 +0,0 @@
-import{_ as t,a}from"./chunks/duplicate-plan-subtree.dark.Cdp70QhV.js";import{_ as r,o as s,c as o,a5 as n}from"./chunks/framework.DSg0KOwT.js";const b=JSON.parse('{"title":"Duplicate Plan Subtree","description":"","frontmatter":{"title":"Duplicate Plan Subtree"},"headers":[],"relativePath":"tuning-reference/bottleneck-duplicate-plan-subtree.md","filePath":"tuning-reference/bottleneck-duplicate-plan-subtree.md"}'),i={name:"tuning-reference/bottleneck-duplicate-plan-subtree.md"};function l(h,e,c,f,u,d){return s(),o("div",null,[...e[0]||(e[0]=[n("",21)])])}const g=r(i,[["render",l]]);export{b as __pageData,g as default};

package/public/docs/assets/tuning-reference_bottleneck-failures.md.CAsFf3RR.lean.js

@@ -1 +0,0 @@
-import{_ as a,a as t}from"./chunks/retry-escalation-ladder.dark.DHipdJgZ.js";import{_ as o,o as s,c as r,a5 as i}from"./chunks/framework.DSg0KOwT.js";const k=JSON.parse('{"title":"Task Failures","description":"","frontmatter":{"title":"Task Failures"},"headers":[],"relativePath":"tuning-reference/bottleneck-failures.md","filePath":"tuning-reference/bottleneck-failures.md"}'),n={name:"tuning-reference/bottleneck-failures.md"};function l(c,e,d,f,h,u){return s(),r("div",null,[...e[0]||(e[0]=[i("",29)])])}const b=o(n,[["render",l]]);export{k as __pageData,b as default};

package/public/docs/assets/tuning-reference_bottleneck-gc.md.B4T3kJYV.js

@@ -1,6 +0,0 @@
-import{_ as a,o as t,c as o,a5 as s}from"./chunks/framework.DSg0KOwT.js";const p=JSON.parse('{"title":"GC Pressure","description":"","frontmatter":{"title":"GC Pressure"},"headers":[],"relativePath":"tuning-reference/bottleneck-gc.md","filePath":"tuning-reference/bottleneck-gc.md"}'),r={name:"tuning-reference/bottleneck-gc.md"};function i(n,e,l,c,h,d){return t(),o("div",null,[...e[0]||(e[0]=[s(`<h1 id="bottleneck-gc" tabindex="-1">GC Pressure <a class="header-anchor" href="#bottleneck-gc" aria-label="Permalink to &quot;GC Pressure {#bottleneck-gc}&quot;">​</a></h1><p><span class="tag">GC</span></p><h2 id="what-it-is" tabindex="-1">What it is <a class="header-anchor" href="#what-it-is" aria-label="Permalink to &quot;What it is&quot;">​</a></h2><p>GC pressure describes how much of an executor&#39;s time goes to JVM garbage collection instead of running task code. Spark tracks this directly in its per-task metrics: <code>jvmGCTime</code> sits alongside <code>executorRunTime</code>, the elapsed wall-clock time the executor spent running the task (as opposed to <code>executorCpuTime</code>, which measures CPU time specifically).<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup> <code>jvmGCTime</code> is not additive on top of that duration: it&#39;s defined as the elapsed time the JVM spent in garbage collection <em>while executing the task</em>, so GC pauses fall inside the same wall-clock window <code>executorRunTime</code> already measures, not outside it. Summing the two would double-count the GC pauses and overstate how long the task actually ran.<sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup></p><h2 id="how-it-s-detected" tabindex="-1">How it&#39;s detected <a class="header-anchor" href="#how-it-s-detected" aria-label="Permalink to &quot;How it&#39;s detected&quot;">​</a></h2><p>Because <code>jvmGCTime</code> sits inside <code>executorRunTime</code> rather than alongside it, the ratio between the two gives a bounded read on how much of a task&#39;s wall-clock time went to garbage collection. Both directions below apply once a stage&#39;s <code>executorRunTime</code> reaches 10 seconds, a floor that keeps short stages from reading as noise:</p><table tabindex="0"><thead><tr><th>Signal (gcPct = jvmGCTime / executorRunTime)</th><th>Fires when</th></tr></thead><tbody><tr><td>High GC</td><td>&gt; 10%</td></tr><tr><td>Low GC (cost signal)</td><td>&lt; 5%</td></tr></tbody></table><p>A stage lands in at most one of these two opposite-direction bands at a time. Severity for both tracks the estimated recoverable time as a share of the app&#39;s total runtime: ≥2% is critical, ≥0.5% is warning, anything smaller is info. Low GC signals a possible memory over-provisioning cost rather than time lost to garbage collection.</p><h2 id="why-it-matters" tabindex="-1">Why it matters <a class="header-anchor" href="#why-it-matters" aria-label="Permalink to &quot;Why it matters&quot;">​</a></h2><p>As gcPct climbs past these thresholds, a growing share of every task&#39;s wall-clock time is consumed by garbage collection instead of actual computation, so the executor&#39;s useful throughput drops even while it appears busy. Two causes show up repeatedly: oversized executors, and an on-heap memory split pushed too far toward execution/storage.</p><p>Sizing an executor with too many cores relative to its heap is one documented trigger: a fat-executor configuration using all 16 cores of a node was observed to not only hurt HDFS throughput but also &quot;result in excessive garbage [collection].&quot;<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup></p><p>Pushing <code>spark.memory.fraction</code> too far toward its upper end has a similar effect through a different path. Its complement, <code>1 − spark.memory.fraction</code>, is untracked &quot;User Memory&quot; reserved for UDFs, Python/Arrow glue, and native library buffers; Spark doesn&#39;t manage this region at all, and starving it (which is what raising <code>spark.memory.fraction</code> toward something like 0.9 does) leads to &quot;GC pressure or random OOMs,&quot; with no warning before the failure.<sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup></p><h2 id="how-to-fix-it" tabindex="-1">How to fix it <a class="header-anchor" href="#how-to-fix-it" aria-label="Permalink to &quot;How to fix it&quot;">​</a></h2><ul><li>Keep cores per executor down rather than packing an entire node&#39;s cores onto one JVM (the common ~5-cores-per-executor guideline). That avoids the excessive-garbage-collection failure mode documented for fat-executor configurations.<sup class="footnote-ref"><a href="#fn2" id="fnref2:1">[2:1]</a></sup></li><li>Don&#39;t raise <code>spark.memory.fraction</code> past a comfortable range chasing more execution/storage memory: doing so starves the untracked user-memory region and risks the same GC-pressure/OOM failure mode.<sup class="footnote-ref"><a href="#fn3" id="fnref3:1">[3:1]</a></sup></li><li>Move Tungsten&#39;s execution and storage buffers off the JVM heap with <code>spark.memory.offHeap.enabled=true</code> (and a positive <code>spark.memory.offHeap.size</code>, which is required whenever off-heap is enabled<sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup>). Off-heap buffers are invisible to the garbage collector, so fewer and smaller live objects need to be tracked, scanned, and copied on the heap, reducing both the frequency and duration of GC pauses.<sup class="footnote-ref"><a href="#fn3" id="fnref3:2">[3:2]</a></sup></li><li>Tune or switch the GC collector on large heaps. G1GC is Spark&#39;s default collector since 4.0 (which defaults to JDK 17), and large executor heaps may need <code>-XX:G1HeapRegionSize</code> raised as well.<sup class="footnote-ref"><a href="#fn5" id="fnref5">[5]</a></sup> G1&#39;s stop-the-world evacuation and full-GC pauses can still spike under heavy old-generation pressure on large heaps: one documented 88GB-heap benchmark saw a job&#39;s pause spike to nearly 100 seconds under default G1 settings, and only tuning <code>InitiatingHeapOccupancyPercent</code>, <code>ConcGCThreads</code>, and RSet-update parameters brought G1 back ahead of Parallel/CMS on both throughput and latency.<sup class="footnote-ref"><a href="#fn6" id="fnref6">[6]</a></sup> ZGC is a documented alternative built for pause-time control: it performs its expensive work concurrently, without stopping application threads for more than a millisecond, with pause times designed to stay independent of heap size from a few hundred megabytes up to 16TB.<sup class="footnote-ref"><a href="#fn7" id="fnref7">[7]</a></sup></li></ul><p>Move buffers off-heap and give G1 room on large heaps (sizes are examples; set to your workload):</p><div class="language-properties vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">properties</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Move Tungsten execution/storage buffers off the JVM heap so the GC has fewer objects to scan</span></span>
-<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.memory.offHeap.enabled</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=true</span></span>
-<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.memory.offHeap.size</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=2g            </span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># must be &gt; 0 whenever off-heap is enabled; 2g is an example</span></span>
-<span class="line"></span>
-<span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># G1 is the default collector since Spark 4.0 (JDK 17); raise the region size on large heaps</span></span>
-<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.executor.extraJavaOptions</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=-XX:+UseG1GC -XX:</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">G1HeapRegionSize</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=16m   </span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># 16m is an example</span></span></code></pre></div><h2 id="confidence" tabindex="-1">Confidence <a class="header-anchor" href="#confidence" aria-label="Permalink to &quot;Confidence&quot;">​</a></h2><p>The warning and critical thresholds on gcPct are validated: they read a bounded ratio of <code>jvmGCTime</code> to <code>executorRunTime</code>, both first-class per-task metrics Spark records directly,<sup class="footnote-ref"><a href="#fn1" id="fnref1:2">[1:2]</a></sup> and crossing 10% or 20% marks a real, measurable share of wall-clock time lost to garbage collection.</p><p><span class="tag">EXPERIMENTAL</span> A separate signal treats suspiciously <em>low</em> GC time as a possible cost-model or noise-floor indicator, and that one is an unsourced heuristic with no external basis, so treat it as exploratory rather than a confirmed diagnostic.</p><h2 id="limitations-false-positive-risk" tabindex="-1">Limitations / false-positive risk <a class="header-anchor" href="#limitations-false-positive-risk" aria-label="Permalink to &quot;Limitations / false-positive risk&quot;">​</a></h2><p>A high gcPct can reflect a transient memory spike (a single skewed task, a brief burst of large allocations) rather than chronic pressure, so a threshold crossing on one stage does not by itself prove a sustained sizing problem; corroborate against the executor&#39;s behavior over the whole run. The low-GC signal has no external basis at all, and a near-zero reading is at least as likely to mean the workload simply is not allocation-heavy as to mean anything is wrong.</p><h2 id="related" tabindex="-1">Related <a class="header-anchor" href="#related" aria-label="Permalink to &quot;Related&quot;">​</a></h2><ul><li><strong>Why it happens:</strong> <a href="./memory-model.html">Memory Management</a></li><li><strong>Executor sizing:</strong> <a href="./cluster-config.html">Cluster Tuning</a></li></ul><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/monitoring.html#spark-history-server" target="_blank" rel="noreferrer">Monitoring and Instrumentation</a> <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a> <a href="#fnref1:2" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><a href="https://raw.githubusercontent.com/spoddutur/spark-notes/master/distribution_of_executors_cores_and_memory_for_spark_application.md" target="_blank" rel="noreferrer">Distribution of Executors, Cores, and Memory for a Spark Application</a> <a href="#fnref2" class="footnote-backref">↩︎</a> <a href="#fnref2:1" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><a href="https://luminousmen.com/post/dive-into-spark-memory" target="_blank" rel="noreferrer">Diving into Spark Memory Management</a> <a href="#fnref3" class="footnote-backref">↩︎</a> <a href="#fnref3:1" class="footnote-backref">↩︎</a> <a href="#fnref3:2" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration (Spark)</a> <a href="#fnref4" class="footnote-backref">↩︎</a></p></li><li id="fn5" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/tuning.html" target="_blank" rel="noreferrer">Spark Tuning Guide</a> <a href="#fnref5" class="footnote-backref">↩︎</a></p></li><li id="fn6" class="footnote-item"><p><a href="https://www.databricks.com/blog/2015/05/28/tuning-java-garbage-collection-for-spark-applications.html" target="_blank" rel="noreferrer">Tuning Java Garbage Collection for Apache Spark Applications</a> <a href="#fnref6" class="footnote-backref">↩︎</a></p></li><li id="fn7" class="footnote-item"><p><a href="https://wiki.openjdk.org/display/zgc" target="_blank" rel="noreferrer">The Z Garbage Collector (ZGC)</a> <a href="#fnref7" class="footnote-backref">↩︎</a></p></li></ol></section>`,25)])])}const u=a(r,[["render",i]]);export{p as __pageData,u as default};

package/public/docs/assets/tuning-reference_bottleneck-gc.md.B4T3kJYV.lean.js

@@ -1 +0,0 @@
-import{_ as a,o as t,c as o,a5 as s}from"./chunks/framework.DSg0KOwT.js";const p=JSON.parse('{"title":"GC Pressure","description":"","frontmatter":{"title":"GC Pressure"},"headers":[],"relativePath":"tuning-reference/bottleneck-gc.md","filePath":"tuning-reference/bottleneck-gc.md"}'),r={name:"tuning-reference/bottleneck-gc.md"};function i(n,e,l,c,h,d){return t(),o("div",null,[...e[0]||(e[0]=[s("",25)])])}const u=a(r,[["render",i]]);export{p as __pageData,u as default};

package/public/docs/assets/tuning-reference_bottleneck-job-failure-rate.md.BaJl__1W.js

@@ -1,8 +0,0 @@
-import{_ as t,a}from"./chunks/retry-escalation-ladder.dark.DHipdJgZ.js";import{_ as s,o,c as r,a5 as i}from"./chunks/framework.DSg0KOwT.js";const b=JSON.parse('{"title":"Job Failure Rate","description":"","frontmatter":{"title":"Job Failure Rate"},"headers":[],"relativePath":"tuning-reference/bottleneck-job-failure-rate.md","filePath":"tuning-reference/bottleneck-job-failure-rate.md"}'),n={name:"tuning-reference/bottleneck-job-failure-rate.md"};function l(c,e,u,f,d,h){return o(),r("div",null,[...e[0]||(e[0]=[i('<h1 id="bottleneck-job-failure-rate" tabindex="-1">Job Failure Rate <a class="header-anchor" href="#bottleneck-job-failure-rate" aria-label="Permalink to &quot;Job Failure Rate {#bottleneck-job-failure-rate}&quot;">​</a></h1><p><span class="tag">FAIL-RATE</span></p><h2 id="what-it-is" tabindex="-1">What it is <a class="header-anchor" href="#what-it-is" aria-label="Permalink to &quot;What it is&quot;">​</a></h2><p>Not every task retry threatens the job. <code>spark.task.maxFailures</code> (default <code>4</code>) counts &quot;continuous failures of any particular task before giving up on the job. The total number of failures spread across different tasks will not cause the job to fail; a particular task has to fail this number of attempts continuously. If any attempt succeeds, the failure count for the task will be reset&quot;<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup>. A task that fails once and then succeeds on retry resets its counter and never counts toward job abandonment: that&#39;s a transient retry, and by itself it&#39;s harmless. Whole-job abandonment is a different, layered escalation above that: it only happens once an unbroken run of failures on the same task reaches the configured limit (3 retries allowed by default, since allowed retries = value − 1), or once higher-level retry budgets above the task level are themselves exhausted.</p><h2 id="how-it-s-detected" tabindex="-1">How it&#39;s detected <a class="header-anchor" href="#how-it-s-detected" aria-label="Permalink to &quot;How it&#39;s detected&quot;">​</a></h2><p>The layers above task-level retries are what turn isolated failures into a failed job. At the stage level, <code>spark.stage.maxConsecutiveAttempts</code> (default <code>4</code>) bounds &quot;the number of consecutive stage attempts allowed before a stage is aborted&quot;<sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup>. By default, <code>spark.stage.ignoreDecommissionFetchFailure</code> (<code>true</code>, since 3.4.0) excludes fetch failures caused by graceful executor decommission from counting toward that limit, so a decommission-triggered <code>FetchFailed</code> doesn&#39;t push a stage toward abortion the way a genuine repeated fetch failure would<sup class="footnote-ref"><a href="#fn1" id="fnref1:2">[1:2]</a></sup>. On YARN and Kubernetes there&#39;s a further application-level ceiling: <code>spark.executor.maxNumFailures</code> (default <code>numExecutors * 2</code>, minimum 3, since 3.5.0) is &quot;the maximum number of executor failures before failing the application,&quot; while <code>spark.executor.failuresValidityInterval</code> (since 3.5.0) lets failures spaced far enough apart be &quot;considered independent and not accumulate towards the attempt count&quot;<sup class="footnote-ref"><a href="#fn1" id="fnref1:3">[1:3]</a></sup>. The TaskScheduler owns retries at the task level, but it is &quot;the DAGScheduler that ultimately declares the job to have failed&quot;<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup> once those retry budgets are exhausted, and because &quot;a Spark job corresponds to one action&quot; with a fixed DAG once that action is called<sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup>, an aborted stage cascades directly into failure of the job built on it.</p><img class="light-only" src="'+t+'" alt="How a failure escalates from a task retry up through stage resubmission, the executor failure ceiling, and a failed application attempt before the job is aborted."><img class="dark-only" src="'+a+`" alt="How a failure escalates from a task retry up through stage resubmission, the executor failure ceiling, and a failed application attempt before the job is aborted."><h2 id="why-it-matters" tabindex="-1">Why it matters <a class="header-anchor" href="#why-it-matters" aria-label="Permalink to &quot;Why it matters&quot;">​</a></h2><p>A rising failure rate that traces back to exhausted retry budgets, rather than to a single one-off task failure, points at a systemic problem (an unstable executor, a genuinely broken fetch path, or a resource ceiling being hit repeatedly) since none of these budgets trip on a single transient hiccup. Job-level failure is also distinct from application-attempt failure: on YARN, &quot;each application may have multiple attempts,&quot; and the history server displays failed attempts alongside &quot;any ongoing incomplete attempt or the final successful attempt&quot;<sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup>: a separate, higher layer of retry (driver/application restart) above the in-job task and stage retries covered here.</p><h2 id="how-to-fix-it" tabindex="-1">How to fix it <a class="header-anchor" href="#how-to-fix-it" aria-label="Permalink to &quot;How to fix it&quot;">​</a></h2><ul><li>Check which budget was exhausted before treating a job failure as a single root cause: a single task failing <code>spark.task.maxFailures</code> times consecutively, a stage being resubmitted until <code>spark.stage.maxConsecutiveAttempts</code> is reached and aborted, or (on YARN/Kubernetes) cumulative distinct executor failures exceeding <code>spark.executor.maxNumFailures</code><sup class="footnote-ref"><a href="#fn1" id="fnref1:4">[1:4]</a></sup> each point at a different underlying problem.</li><li>If failures are decommission-triggered <code>FetchFailed</code>s during a normal scale-down, confirm <code>spark.stage.ignoreDecommissionFetchFailure</code> is enabled (default <code>true</code> since 3.4.0) so they aren&#39;t inflating the stage-abort count<sup class="footnote-ref"><a href="#fn1" id="fnref1:5">[1:5]</a></sup>.</li><li>On YARN/Kubernetes, if unrelated executor failures spread far apart in time are tripping <code>spark.executor.maxNumFailures</code>, <code>spark.executor.failuresValidityInterval</code> (since 3.5.0) can let sufficiently-spaced failures stop accumulating toward the same count<sup class="footnote-ref"><a href="#fn1" id="fnref1:6">[1:6]</a></sup>.</li><li>See also <a href="./bottleneck-retry-waste.html">retry waste</a>: the same executor-loss and <code>FetchFailed</code> causes that eventually exhaust these budgets and fail a job are, below that threshold, also the source of wasted executor time on jobs that ultimately succeed.</li></ul><blockquote><p><strong>PySpark:</strong> these are session-level configs, settable without a <code>spark-submit</code> flag: <code>spark.conf.set(&quot;spark.task.maxFailures&quot;, &quot;4&quot;)</code>, and the stage/executor-level equivalents the same way.</p></blockquote><p>The retry-budget ladder: defaults shown; raise a budget only once you know which layer is tripping:</p><div class="language-properties vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">properties</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Task level: consecutive failures of one task before the job is abandoned (default 4 =&gt; 3 retries)</span></span>
-<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.task.maxFailures</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=4</span></span>
-<span class="line"></span>
-<span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Stage level: consecutive stage attempts before the stage is aborted (default 4)</span></span>
-<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.stage.maxConsecutiveAttempts</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=4</span></span>
-<span class="line"></span>
-<span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Don&#39;t count graceful-decommission fetch failures toward the stage-abort limit (default true since Spark 3.4)</span></span>
-<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.stage.ignoreDecommissionFetchFailure</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=true</span></span></code></pre></div><h2 id="limitations-false-positive-risk" tabindex="-1">Limitations / false-positive risk <a class="header-anchor" href="#limitations-false-positive-risk" aria-label="Permalink to &quot;Limitations / false-positive risk&quot;">​</a></h2><p>A nonzero job-failure rate can be dominated by a single repeatedly-failing job rather than a systemic issue, so the rate alone doesn&#39;t tell you whether the problem is broad or isolated. Retried jobs that eventually succeed still count toward attempts, which can inflate the rate above what the eventual outcomes justify.</p><h2 id="related" tabindex="-1">Related <a class="header-anchor" href="#related" aria-label="Permalink to &quot;Related&quot;">​</a></h2><ul><li><strong>Retry budgets &amp; executor stability:</strong> <a href="./cluster-config.html">Cluster Tuning</a></li><li><strong>Avoiding the failures upstream:</strong> <a href="./anti-patterns.html">Anti-Patterns</a></li></ul><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration (Spark)</a> <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a> <a href="#fnref1:2" class="footnote-backref">↩︎</a> <a href="#fnref1:3" class="footnote-backref">↩︎</a> <a href="#fnref1:4" class="footnote-backref">↩︎</a> <a href="#fnref1:5" class="footnote-backref">↩︎</a> <a href="#fnref1:6" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><a href="https://luminousmen.com/post/spark-anatomy-of-spark-application" target="_blank" rel="noreferrer">Spark: Anatomy of Spark Application</a> <a href="#fnref2" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><em>High Performance Spark, 2nd Edition</em>, Karau, Polak &amp; Warren, ch. 2 <a href="#fnref3" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/monitoring.html#spark-history-server" target="_blank" rel="noreferrer">Monitoring and Instrumentation</a> <a href="#fnref4" class="footnote-backref">↩︎</a></p></li></ol></section>`,21)])])}const k=s(n,[["render",l]]);export{b as __pageData,k as default};

package/public/docs/assets/tuning-reference_bottleneck-job-failure-rate.md.BaJl__1W.lean.js

@@ -1 +0,0 @@
-import{_ as t,a}from"./chunks/retry-escalation-ladder.dark.DHipdJgZ.js";import{_ as s,o,c as r,a5 as i}from"./chunks/framework.DSg0KOwT.js";const b=JSON.parse('{"title":"Job Failure Rate","description":"","frontmatter":{"title":"Job Failure Rate"},"headers":[],"relativePath":"tuning-reference/bottleneck-job-failure-rate.md","filePath":"tuning-reference/bottleneck-job-failure-rate.md"}'),n={name:"tuning-reference/bottleneck-job-failure-rate.md"};function l(c,e,u,f,d,h){return o(),r("div",null,[...e[0]||(e[0]=[i("",21)])])}const k=s(n,[["render",l]]);export{b as __pageData,k as default};

package/public/docs/assets/tuning-reference_bottleneck-memory-utilization.md.DbP-SJZc.js

@@ -1,7 +0,0 @@
-import{_ as a,o as t,c as o,a5 as s}from"./chunks/framework.DSg0KOwT.js";const p=JSON.parse('{"title":"Memory Utilization","description":"","frontmatter":{"title":"Memory Utilization"},"headers":[],"relativePath":"tuning-reference/bottleneck-memory-utilization.md","filePath":"tuning-reference/bottleneck-memory-utilization.md"}'),r={name:"tuning-reference/bottleneck-memory-utilization.md"};function n(i,e,l,c,d,f){return t(),o("div",null,[...e[0]||(e[0]=[s(`<h1 id="bottleneck-memory-utilization" tabindex="-1">Memory Utilization <a class="header-anchor" href="#bottleneck-memory-utilization" aria-label="Permalink to &quot;Memory Utilization {#bottleneck-memory-utilization}&quot;">​</a></h1><p><span class="tag">MEM</span></p><h2 id="what-it-is" tabindex="-1">What it is <a class="header-anchor" href="#what-it-is" aria-label="Permalink to &quot;What it is&quot;">​</a></h2><p>An executor is a single JVM process that gets a fixed memory allocation when the application starts, and it holds that whole allocation for its entire lifetime, whether or not it has work to run<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup>. Two things can leave that memory band poorly used: cores sitting idle inside a held executor, and a held executor whose memory drains of active work but is never released. This finding covers both.</p><p>An executor&#39;s core count is its concurrency ceiling. <code>spark.executor.cores</code> sets how many tasks it can run at once, so <code>--executor-cores 5</code> caps that executor at five concurrent tasks<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup>. The scheduler turns cores into task slots from <code>spark.executor.cores</code> and <code>spark.task.cpus</code> (minimum 1), with <code>spark.task.cpus</code> defaulting to one core per task<sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup>. Cores go idle whenever there are fewer runnable tasks than slots: a stage with fewer partitions than the total slots across executors leaves slots empty, and so does the tail of a stage where a <a href="./bottleneck-straggler.html">straggler</a> or two keep running after their peers finish. Raising <code>spark.task.cpus</code> above 1 has the same effect from the other side, since each task then reserves several cores and fewer run side by side. Through all of it the JVM keeps its fixed heap<sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup>.</p><h2 id="how-it-s-detected" tabindex="-1">How it&#39;s detected <a class="header-anchor" href="#how-it-s-detected" aria-label="Permalink to &quot;How it&#39;s detected&quot;">​</a></h2><table tabindex="0"><thead><tr><th>Rule</th><th>Signal</th><th>Status</th></tr></thead><tbody><tr><td>idleCores</td><td>Task parallelism below allocated cores while the executor is held</td><td>Validated</td></tr><tr><td>memoryBand</td><td>A whole executor&#39;s memory band held with little or no active work (dynamic-allocation idle timeouts)</td><td>Validated</td></tr><tr><td>wasteModel</td><td>Peak used memory well below allocated memory (unused headroom)</td><td>Experimental</td></tr></tbody></table><h2 id="why-it-matters" tabindex="-1">Why it matters <a class="header-anchor" href="#why-it-matters" aria-label="Permalink to &quot;Why it matters&quot;">​</a></h2><p>A held executor is allocation you pay for regardless of how busy it is. On YARN that is the memory YARN grants the container; on Kubernetes it is the pod memory limit<sup class="footnote-ref"><a href="#fn1" id="fnref1:2">[1:2]</a></sup>. When cores idle or a whole executor lingers with nothing to do, that fixed band is reserved without returning work, so the cost lands whether or not tasks are running.</p><h2 id="how-to-fix-it" tabindex="-1">How to fix it <a class="header-anchor" href="#how-to-fix-it" aria-label="Permalink to &quot;How to fix it&quot;">​</a></h2><p>Keep executors from being oversized in the first place. The guidance is against packing every core of a node into one fat executor: assigning all 16 cores of a node to a single executor hurts HDFS throughput and drives excessive <a href="./bottleneck-gc.html">garbage collection</a>, so aim for a balance between tiny (one core per executor) and fat (one executor per node) sizing<sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup>. On YARN, leave cores for the OS and Hadoop daemons instead of handing 100% of a node to Spark containers<sup class="footnote-ref"><a href="#fn2" id="fnref2:1">[2:1]</a></sup>.</p><p>For the held memory band, lean on <a href="./cluster-config.html">dynamic allocation</a> to reclaim executors once the work drains. It requests executors when tasks back up and frees them when they go idle<sup class="footnote-ref"><a href="#fn2" id="fnref2:2">[2:2]</a></sup>. Executors are added in rounds once tasks have been pending for <code>spark.dynamicAllocation.schedulerBacklogTimeout</code> (default 1s), then again every <code>spark.dynamicAllocation.sustainedSchedulerBacklogTimeout</code> while the backlog holds<sup class="footnote-ref"><a href="#fn5" id="fnref5">[5]</a></sup>. On the release side, an executor is removed after it has been idle longer than <code>spark.dynamicAllocation.executorIdleTimeout</code><sup class="footnote-ref"><a href="#fn5" id="fnref5:1">[5:1]</a></sup>.</p><p>Cached data is the trap here. By default an executor holding <a href="./caching.html">cached blocks</a> is never removed, governed by <code>spark.dynamicAllocation.cachedExecutorIdleTimeout</code>, whose default is infinity<sup class="footnote-ref"><a href="#fn3" id="fnref3:1">[3:1]</a></sup>. Such an executor keeps its full memory band indefinitely with no active tasks unless you set that timeout to a finite value, or turn on <code>spark.shuffle.service.fetch.rdd.enabled</code> so executors holding only disk-persisted blocks are treated as idle after <code>spark.dynamicAllocation.executorIdleTimeout</code> and released<sup class="footnote-ref"><a href="#fn5" id="fnref5:2">[5:2]</a></sup>. To avoid holding barely-used executors at all, lower <code>spark.dynamicAllocation.executorAllocationRatio</code> (default 1.0, full parallelism) toward 0.5, since with small tasks full-parallelism allocation can request executors that never do any work<sup class="footnote-ref"><a href="#fn3" id="fnref3:2">[3:2]</a></sup>.</p><div class="language-properties vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">properties</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Balanced executor sizing, not one fat executor per node</span></span>
-<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.executor.cores</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=5</span></span>
-<span class="line"></span>
-<span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Reclaim idle executors; give cached holders a finite timeout</span></span>
-<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.dynamicAllocation.enabled</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=true</span></span>
-<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.dynamicAllocation.cachedExecutorIdleTimeout</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=300s</span></span>
-<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.dynamicAllocation.executorAllocationRatio</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=0.5</span></span></code></pre></div><h3 id="reading-the-wasted-memory-estimate-experimental" tabindex="-1">Reading the wasted-memory estimate <span class="tag">EXPERIMENTAL</span> <a class="header-anchor" href="#reading-the-wasted-memory-estimate-experimental" aria-label="Permalink to &quot;Reading the wasted-memory estimate &lt;span class=&quot;tag&quot;&gt;EXPERIMENTAL&lt;/span&gt;&quot;">​</a></h3><p>The wasteModel rule estimates unused (&quot;wasted&quot;) allocated memory from the gap between an executor&#39;s peak used memory and its allocated total. Treat it as a rough buffer heuristic, not a measurement. Peak used memory is a high-water mark rather than a ceiling: Spark records it under <code>peakMemoryMetrics.*</code>, and each figure is the maximum its pool ever reached, so peak sits at or below the allocated total by construction and the space between them is the executor&#39;s unused headroom<sup class="footnote-ref"><a href="#fn6" id="fnref6">[6]</a></sup>. Turning that headroom into a wasted number stays approximate for concrete reasons. The peak heap figure counts garbage, since <code>JVMHeapMemory</code> is the peak used heap including &quot;the amount of memory occupied by both live objects and garbage objects that have not been collected,&quot; so it overstates the live footprint<sup class="footnote-ref"><a href="#fn6" id="fnref6:1">[6:1]</a></sup>. The region boundaries also move: <code>totalOnHeapStorageMemory</code> and <code>totalOffHeapStorageMemory</code> &quot;can vary over time, depending on the MemoryManager implementation,&quot; so there is no single fixed allocated-to-storage value to subtract a peak from<sup class="footnote-ref"><a href="#fn6" id="fnref6:2">[6:2]</a></sup>. Peak metrics only reach the event log when <code>spark.eventLog.logStageExecutorMetrics</code> is true<sup class="footnote-ref"><a href="#fn6" id="fnref6:3">[6:3]</a></sup>, so without that setting there is nothing to estimate from. Use the number as a hint that an executor may be oversized, then confirm against sizing before acting on it.</p><h2 id="confidence" tabindex="-1">Confidence <a class="header-anchor" href="#confidence" aria-label="Permalink to &quot;Confidence&quot;">​</a></h2><p>The idleCores and memoryBand rules are validated: they rest on documented Spark concurrency and dynamic-allocation behavior. The wasteModel estimate is low-confidence and experimental. It is a buffer heuristic derived from peak-versus-allocated sampling, not an exact accounting of unused memory, so it should steer investigation rather than settle it.</p><h2 id="limitations-false-positive-risk" tabindex="-1">Limitations / false-positive risk <a class="header-anchor" href="#limitations-false-positive-risk" aria-label="Permalink to &quot;Limitations / false-positive risk&quot;">​</a></h2><p>Idle cores are not always waste. The tail of a stage legitimately leaves slots empty while a straggler or two finish<sup class="footnote-ref"><a href="#fn3" id="fnref3:3">[3:3]</a></sup>, so a snapshot of low parallelism can reflect a normal straggler tail rather than chronic under-utilization. The wasted-memory estimate depends on peak-memory sampling and is only as good as that sampling: the peak includes uncollected garbage, the managed-region boundaries shift under the MemoryManager, and the figures are absent entirely unless <code>spark.eventLog.logStageExecutorMetrics</code> is enabled<sup class="footnote-ref"><a href="#fn6" id="fnref6:4">[6:4]</a></sup>.</p><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><a href="https://luminousmen.com/post/dive-into-spark-memory" target="_blank" rel="noreferrer">Dive into Spark memory</a> <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a> <a href="#fnref1:2" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><a href="https://blog.cloudera.com/how-to-tune-your-apache-spark-jobs-part-2/" target="_blank" rel="noreferrer">How to Tune Your Apache Spark Jobs (Part 2)</a> <a href="#fnref2" class="footnote-backref">↩︎</a> <a href="#fnref2:1" class="footnote-backref">↩︎</a> <a href="#fnref2:2" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration — Spark</a> <a href="#fnref3" class="footnote-backref">↩︎</a> <a href="#fnref3:1" class="footnote-backref">↩︎</a> <a href="#fnref3:2" class="footnote-backref">↩︎</a> <a href="#fnref3:3" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><a href="https://raw.githubusercontent.com/spoddutur/spark-notes/master/distribution_of_executors_cores_and_memory_for_spark_application.md" target="_blank" rel="noreferrer">Distribution of executors, cores and memory for a Spark application</a> <a href="#fnref4" class="footnote-backref">↩︎</a></p></li><li id="fn5" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/job-scheduling.html" target="_blank" rel="noreferrer">Job Scheduling — Spark</a> <a href="#fnref5" class="footnote-backref">↩︎</a> <a href="#fnref5:1" class="footnote-backref">↩︎</a> <a href="#fnref5:2" class="footnote-backref">↩︎</a></p></li><li id="fn6" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/monitoring.html" target="_blank" rel="noreferrer">Monitoring — Spark</a> <a href="#fnref6" class="footnote-backref">↩︎</a> <a href="#fnref6:1" class="footnote-backref">↩︎</a> <a href="#fnref6:2" class="footnote-backref">↩︎</a> <a href="#fnref6:3" class="footnote-backref">↩︎</a> <a href="#fnref6:4" class="footnote-backref">↩︎</a></p></li></ol></section>`,22)])])}const u=a(r,[["render",n]]);export{p as __pageData,u as default};

package/public/docs/assets/tuning-reference_bottleneck-memory-utilization.md.DbP-SJZc.lean.js

@@ -1 +0,0 @@
-import{_ as a,o as t,c as o,a5 as s}from"./chunks/framework.DSg0KOwT.js";const p=JSON.parse('{"title":"Memory Utilization","description":"","frontmatter":{"title":"Memory Utilization"},"headers":[],"relativePath":"tuning-reference/bottleneck-memory-utilization.md","filePath":"tuning-reference/bottleneck-memory-utilization.md"}'),r={name:"tuning-reference/bottleneck-memory-utilization.md"};function n(i,e,l,c,d,f){return t(),o("div",null,[...e[0]||(e[0]=[s("",22)])])}const u=a(r,[["render",n]]);export{p as __pageData,u as default};

package/public/docs/assets/tuning-reference_bottleneck-retry-waste.md.D5JMjVOt.js

@@ -1 +0,0 @@
-import{_ as t,a}from"./chunks/retry-escalation-ladder.dark.DHipdJgZ.js";import{_ as r,o,c as s,a5 as i}from"./chunks/framework.DSg0KOwT.js";const k=JSON.parse('{"title":"Retry Waste","description":"","frontmatter":{"title":"Retry Waste"},"headers":[],"relativePath":"tuning-reference/bottleneck-retry-waste.md","filePath":"tuning-reference/bottleneck-retry-waste.md"}'),n={name:"tuning-reference/bottleneck-retry-waste.md"};function l(c,e,f,h,u,d){return o(),s("div",null,[...e[0]||(e[0]=[i('<h1 id="bottleneck-retry-waste" tabindex="-1">Retry Waste <a class="header-anchor" href="#bottleneck-retry-waste" aria-label="Permalink to &quot;Retry Waste {#bottleneck-retry-waste}&quot;">​</a></h1><p><span class="tag">RETRY</span></p><h2 id="what-it-is" tabindex="-1">What it is <a class="header-anchor" href="#what-it-is" aria-label="Permalink to &quot;What it is&quot;">​</a></h2><p>When a task attempt is superseded by a later retry, every bit of executor time the abandoned attempt spent counts for nothing. Every attempt, successful or not, accrues the standard task metrics: <code>executorRunTime</code> (&quot;elapsed time the executor spent running this task,&quot; including time fetching shuffle data) and <code>executorCpuTime</code><sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup>, but only the winning attempt&#39;s output survives. The TaskScheduler is what triggers the do-over: &quot;if an executor dies or a task throws an exception, the TaskScheduler resubmits the task to another executor, respecting Spark&#39;s task locality preferences&quot;<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup>. Whatever the discarded attempt had already run is gone; none of it carries forward into the job&#39;s result.</p><img class="light-only" src="'+t+'" alt="Where a superseded retry sits in the escalation ladder, below the stage and application budgets that a job crosses only when the same failures keep recurring."><img class="dark-only" src="'+a+'" alt="Where a superseded retry sits in the escalation ladder, below the stage and application budgets that a job crosses only when the same failures keep recurring."><h2 id="how-it-s-detected" tabindex="-1">How it&#39;s detected <a class="header-anchor" href="#how-it-s-detected" aria-label="Permalink to &quot;How it&#39;s detected&quot;">​</a></h2><p>A stage&#39;s superseded task attempts are the signal: a later retry supersedes an earlier attempt after <a href="./bottleneck-failures.html">an executor is lost</a> (<code>ExecutorLostFailure</code>) or after a shuffle <code>FetchFailed</code>, since &quot;if an executor dies or a task throws an exception, the TaskScheduler resubmits the task to another executor, respecting Spark&#39;s task locality preferences&quot;<sup class="footnote-ref"><a href="#fn2" id="fnref2:1">[2:1]</a></sup>. Because only the winning attempt&#39;s output survives, the abandoned attempt&#39;s <code>executorRunTime</code> (the elapsed executor time defined above) is the wasted time.</p><p>The two causes are handled differently by Spark&#39;s own machinery, which is how they can be told apart after the fact. An <code>ExecutorLostFailure</code> removes the executor (and any shuffle map output it had already written) from the pool outright<sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup>. A shuffle <code>FetchFailed</code>, by contrast, is a read-side failure against a still-alive remote executor, and Spark&#39;s own exclusion machinery treats it as a distinct category from a general executor loss: <code>spark.excludeOnFailure.killExcludedExecutors</code> governs whether Spark kills executors &quot;excluded on fetch failure or excluded for the entire application&quot;<sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup>, a separate bucket from executor loss in Spark&#39;s accounting. That distinction (executor-loss bucket vs. fetch-failure bucket) is the basis for identifying which cause dominated a given stage&#39;s retry waste.</p><h2 id="why-it-matters" tabindex="-1">Why it matters <a class="header-anchor" href="#why-it-matters" aria-label="Permalink to &quot;Why it matters&quot;">​</a></h2><p>A lost executor can waste more than just the in-flight task&#39;s time: &quot;dynamic allocation may remove an executor before the shuffle completes, in which case the shuffle files written by that executor must be recomputed unnecessarily&quot;<sup class="footnote-ref"><a href="#fn3" id="fnref3:1">[3:1]</a></sup>, so prior map-output work from that same executor may need redoing too. A <code>FetchFailed</code> doesn&#39;t carry that same risk, since the remote executor stays alive and only the one failed read is lost<sup class="footnote-ref"><a href="#fn4" id="fnref4:1">[4:1]</a></sup>.</p><h2 id="how-to-fix-it" tabindex="-1">How to fix it <a class="header-anchor" href="#how-to-fix-it" aria-label="Permalink to &quot;How to fix it&quot;">​</a></h2><ul><li>Check the retried-attempt count and accumulated <code>executorRunTime</code>/<code>executorCpuTime</code> on the abandoned attempts directly, rather than trusting the stage&#39;s final wall-clock duration: a clean-looking stage can still be hiding significant wasted compute<sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup>.</li><li>If the dominant cause is executor loss, investigate node-level stability (OOM-kills, node death, aggressive dynamic-allocation deallocation) rather than the task logic itself, since the lost executor&#39;s prior shuffle-write work may also need recomputing<sup class="footnote-ref"><a href="#fn3" id="fnref3:2">[3:2]</a></sup>.</li><li>If the dominant cause is <code>FetchFailed</code>, check whether <code>spark.excludeOnFailure.killExcludedExecutors</code> is causing repeated exclusion churn on otherwise-live executors, and treat it separately from outright executor loss<sup class="footnote-ref"><a href="#fn4" id="fnref4:2">[4:2]</a></sup>.</li><li>See also <a href="./bottleneck-job-failure-rate.html">job failure rate</a>: retry waste can pile up quietly on a job that ultimately succeeds, while the same underlying causes (exhausted at a higher threshold) are what push a job to fail outright.</li></ul><blockquote><p><strong>PySpark:</strong> there&#39;s no attempt-level API to recover a discarded attempt&#39;s <code>executorRunTime</code>: pull it from the event log or history server UI&#39;s per-stage task list, filtering for tasks whose attempt number is greater than zero.</p></blockquote><p>The preventive lever: keep an executor&#39;s shuffle output available if it is lost, so retries don&#39;t recompute prior map work<sup class="footnote-ref"><a href="#fn5" id="fnref5">[5]</a></sup>:</p><div class="language-properties vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">properties</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.dynamicAllocation.shuffleTracking.enabled</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=true</span></span></code></pre></div><h2 id="limitations-false-positive-risk" tabindex="-1">Limitations / false-positive risk <a class="header-anchor" href="#limitations-false-positive-risk" aria-label="Permalink to &quot;Limitations / false-positive risk&quot;">​</a></h2><p>Some retry waste is unavoidable: transient cluster faults will always cost a few superseded attempts, and no tuning drives that to zero. The metric attributes burned executor time to the abandoned attempts, so its accuracy tracks the failure cause. When a lost executor also forces recompute of prior map output, the per-attempt <code>executorRunTime</code> undercounts the true cost; when an attempt fails almost immediately, it overcounts the compute actually lost. Read the number as a directional signal, not an exact ledger.</p><h2 id="related" tabindex="-1">Related <a class="header-anchor" href="#related" aria-label="Permalink to &quot;Related&quot;">​</a></h2><ul><li><strong>Shuffle recompute on executor loss:</strong> <a href="./shuffle.html">Shuffle</a></li><li><strong>Executor stability &amp; dynamic allocation:</strong> <a href="./cluster-config.html">Cluster Tuning</a></li></ul><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/monitoring.html" target="_blank" rel="noreferrer">Monitoring and Instrumentation</a> <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><a href="https://luminousmen.com/post/spark-anatomy-of-spark-application" target="_blank" rel="noreferrer">Spark: Anatomy of Spark Application</a> <a href="#fnref2" class="footnote-backref">↩︎</a> <a href="#fnref2:1" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/job-scheduling.html" target="_blank" rel="noreferrer">Job Scheduling (Spark)</a> <a href="#fnref3" class="footnote-backref">↩︎</a> <a href="#fnref3:1" class="footnote-backref">↩︎</a> <a href="#fnref3:2" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration (Spark)</a> <a href="#fnref4" class="footnote-backref">↩︎</a> <a href="#fnref4:1" class="footnote-backref">↩︎</a> <a href="#fnref4:2" class="footnote-backref">↩︎</a></p></li><li id="fn5" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration (Spark)</a> <a href="#fnref5" class="footnote-backref">↩︎</a></p></li></ol></section>',22)])])}const g=r(n,[["render",l]]);export{k as __pageData,g as default};

package/public/docs/assets/tuning-reference_bottleneck-retry-waste.md.D5JMjVOt.lean.js

@@ -1 +0,0 @@
-import{_ as t,a}from"./chunks/retry-escalation-ladder.dark.DHipdJgZ.js";import{_ as r,o,c as s,a5 as i}from"./chunks/framework.DSg0KOwT.js";const k=JSON.parse('{"title":"Retry Waste","description":"","frontmatter":{"title":"Retry Waste"},"headers":[],"relativePath":"tuning-reference/bottleneck-retry-waste.md","filePath":"tuning-reference/bottleneck-retry-waste.md"}'),n={name:"tuning-reference/bottleneck-retry-waste.md"};function l(c,e,f,h,u,d){return o(),s("div",null,[...e[0]||(e[0]=[i("",22)])])}const g=r(n,[["render",l]]);export{k as __pageData,g as default};

package/public/docs/assets/tuning-reference_bottleneck-shuffle.md.DDMQVOW6.lean.js

@@ -1 +0,0 @@
-import{_ as a,o as t,c as s,a5 as o}from"./chunks/framework.DSg0KOwT.js";const p=JSON.parse('{"title":"Shuffle I/O","description":"","frontmatter":{"title":"Shuffle I/O"},"headers":[],"relativePath":"tuning-reference/bottleneck-shuffle.md","filePath":"tuning-reference/bottleneck-shuffle.md"}'),r={name:"tuning-reference/bottleneck-shuffle.md"};function i(n,e,f,l,c,d){return t(),s("div",null,[...e[0]||(e[0]=[o("",34)])])}const u=a(r,[["render",i]]);export{p as __pageData,u as default};

package/public/docs/assets/tuning-reference_bottleneck-skew.md.B0MzhWcX.lean.js

@@ -1 +0,0 @@
-import{_ as e,o as s,c as t,a5 as i}from"./chunks/framework.DSg0KOwT.js";const f=JSON.parse('{"title":"Task Skew","description":"","frontmatter":{"title":"Task Skew"},"headers":[],"relativePath":"tuning-reference/bottleneck-skew.md","filePath":"tuning-reference/bottleneck-skew.md"}'),n={name:"tuning-reference/bottleneck-skew.md"};function o(r,a,l,h,p,d){return s(),t("div",null,[...a[0]||(a[0]=[i("",28)])])}const c=e(n,[["render",o]]);export{f as __pageData,c as default};

package/public/docs/assets/tuning-reference_bottleneck-slow-host.md.BlIo6UDW.lean.js

@@ -1 +0,0 @@
-import{_ as t,o as a,c as s,a5 as o}from"./chunks/framework.DSg0KOwT.js";const p=JSON.parse('{"title":"Slow Host","description":"","frontmatter":{"title":"Slow Host"},"headers":[],"relativePath":"tuning-reference/bottleneck-slow-host.md","filePath":"tuning-reference/bottleneck-slow-host.md"}'),i={name:"tuning-reference/bottleneck-slow-host.md"};function n(r,e,l,h,c,d){return a(),s("div",null,[...e[0]||(e[0]=[o("",27)])])}const u=t(i,[["render",n]]);export{p as __pageData,u as default};

package/public/docs/assets/tuning-reference_bottleneck-small-files.md.DHGMTmPs.js

@@ -1,5 +0,0 @@
-import{_ as t,o as a,c as s,a5 as i}from"./chunks/framework.DSg0KOwT.js";const p=JSON.parse('{"title":"Small Files","description":"","frontmatter":{"title":"Small Files"},"headers":[],"relativePath":"tuning-reference/bottleneck-small-files.md","filePath":"tuning-reference/bottleneck-small-files.md"}'),o={name:"tuning-reference/bottleneck-small-files.md"};function n(r,e,l,f,h,c){return a(),s("div",null,[...e[0]||(e[0]=[i(`<h1 id="bottleneck-small-files" tabindex="-1">Small Files <a class="header-anchor" href="#bottleneck-small-files" aria-label="Permalink to &quot;Small Files {#bottleneck-small-files}&quot;">​</a></h1><p><span class="tag">SMALLFILES</span></p><h2 id="what-it-is" tabindex="-1">What it is <a class="header-anchor" href="#what-it-is" aria-label="Permalink to &quot;What it is&quot;">​</a></h2><p>The small-files problem is the cost of managing a large count of tiny files. Writing many small files runs up significant metadata overhead, and Spark handles that pattern badly; distributed filesystems such as HDFS handle it badly too, which is why it earned the name &quot;small file problem&quot;<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup>. The same trap shows up on the read side. Push <code>spark.sql.files.maxPartitionBytes</code> (default 128 MB, aligned to the <a href="./data-formats.html">Parquet</a> block size) too low and Spark carves the input into many small partition files, piling on disk I/O and the filesystem overhead of opening, closing, and listing directories, all of which are slow on a distributed store<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup><sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup>.</p><p>Spark writes one file per output partition, so a job left with 200 partitions writes 200 files and one left with 3000 writes 3000, even when many of those files hold only a handful of rows. That count then punishes every downstream job forced to read thousands of tiny files<sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup>. The opposite extreme is not free either: files that are too large make it inefficient to read a whole block when you only need a few rows<sup class="footnote-ref"><a href="#fn5" id="fnref5">[5]</a></sup>.</p><h2 id="how-it-s-detected" tabindex="-1">How it&#39;s detected <a class="header-anchor" href="#how-it-s-detected" aria-label="Permalink to &quot;How it&#39;s detected&quot;">​</a></h2><p>Each SQL plan node carries file-count and file-size metrics Spark reports directly: <code>number of files read</code>/<code>size of files read</code> on the read side, <code>number of written files</code>/<code>written output</code> on the write side. A node reads as a small-files problem once its file count on a given side is high and the resulting average file size is small.</p><table tabindex="0"><thead><tr><th>Signal (per plan node, checked separately for read and write)</th><th>Fires when</th></tr></thead><tbody><tr><td>File count</td><td>&gt; 100</td></tr><tr><td>Average file size (bytes ÷ file count)</td><td>&lt; 3 MB</td></tr></tbody></table><p>Both conditions have to hold together, so a node with thousands of files that are each big enough, or a handful of genuinely tiny ones, doesn&#39;t register. The signal comes entirely from those file-count and file-size metrics, independent of <code>spark.sql.files.maxPartitionBytes</code>.</p><h2 id="why-it-matters" tabindex="-1">Why it matters <a class="header-anchor" href="#why-it-matters" aria-label="Permalink to &quot;Why it matters&quot;">​</a></h2><p>Every extra file is another open, close, and directory-list operation, and on a distributed filesystem those are not cheap<sup class="footnote-ref"><a href="#fn2" id="fnref2:1">[2:1]</a></sup>. The count compounds downstream: a stage that scatters 3000 tiny files hands the next job 3000 files to read back, so the metadata tax is paid twice over<sup class="footnote-ref"><a href="#fn4" id="fnref4:1">[4:1]</a></sup>. None of it is a single dramatic failure. The waste is spread thin across the file count, which is exactly why it goes unnoticed until listing and I/O start to dominate.</p><h2 id="how-to-fix-it" tabindex="-1">How to fix it <a class="header-anchor" href="#how-to-fix-it" aria-label="Permalink to &quot;How to fix it&quot;">​</a></h2><p>Repartition right before the write so the output lands in fewer, better-sized files.</p><ul><li><code>coalesce()</code> is a shuffle-free merge: it fuses existing partitions with no data movement, so <code>df.coalesce(100).write.parquet(...)</code> collapses 3000 tiny files into 100 reasonable ones<sup class="footnote-ref"><a href="#fn4" id="fnref4:2">[4:2]</a></sup>. It does not rebalance, though, so uneven inputs stay uneven once lumped together, and pushing it to <code>coalesce(1)</code> kills parallelism by forcing one executor to do all the work<sup class="footnote-ref"><a href="#fn4" id="fnref4:3">[4:3]</a></sup>.</li><li><code>repartition()</code> is a full reshuffle that buys even distribution at the cost of that <a href="./shuffle.html">shuffle</a>. Reach for it when the distribution itself needs fixing, not just the file count<sup class="footnote-ref"><a href="#fn4" id="fnref4:4">[4:4]</a></sup>.</li></ul><blockquote><p><strong>PySpark:</strong> both are one-line calls on a DataFrame: <code>df.coalesce(100)</code> for a shuffle-free merge, or <code>df.repartition(100)</code> when the data also needs rebalancing.</p></blockquote><div class="language-python vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">python</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Collapse many tiny output files without a shuffle (distribution already even)</span></span>
-<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">df.coalesce(</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">100</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">).write.parquet(path)</span></span>
-<span class="line"></span>
-<span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Full reshuffle to a target count when the distribution itself needs fixing</span></span>
-<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">df.repartition(</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">100</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">).write.parquet(path)</span></span></code></pre></div><p>For runtime control, <a href="./aqe.html">Adaptive Query Execution</a> can merge tiny shuffle partitions on its own. With <code>spark.sql.adaptive.enabled</code> and <code>spark.sql.adaptive.coalescePartitions.enabled</code> (default true) set, AQE coalesces contiguous shuffle partitions toward <code>spark.sql.adaptive.advisoryPartitionSizeInBytes</code> (default 64 MB)<sup class="footnote-ref"><a href="#fn4" id="fnref4:5">[4:5]</a></sup><sup class="footnote-ref"><a href="#fn5" id="fnref5:1">[5:1]</a></sup>. Just mind the scope: AQE only engages after the first shuffle, so it fixes shuffle-output partition counts but will not repair input-side partitioning or a bad file layout on disk<sup class="footnote-ref"><a href="#fn6" id="fnref6">[6]</a></sup>.</p><h2 id="confidence" tabindex="-1">Confidence <a class="header-anchor" href="#confidence" aria-label="Permalink to &quot;Confidence&quot;">​</a></h2><p>The core inference, that one output file per partition times a high partition count yields many tiny files, is grounded directly in Spark&#39;s write behavior<sup class="footnote-ref"><a href="#fn4" id="fnref4:6">[4:6]</a></sup>, and the mitigation levers (<code>coalesce()</code>, <code>repartition()</code>, and AQE coalescing) are documented Spark features<sup class="footnote-ref"><a href="#fn4" id="fnref4:7">[4:7]</a></sup><sup class="footnote-ref"><a href="#fn5" id="fnref5:2">[5:2]</a></sup>.</p><h2 id="limitations-false-positive-risk" tabindex="-1">Limitations / false-positive risk <a class="header-anchor" href="#limitations-false-positive-risk" aria-label="Permalink to &quot;Limitations / false-positive risk&quot;">​</a></h2><p>A high file count can be perfectly legitimate: partitioned output deliberately fans data across many files by partition column, so a large count there is by design, not a defect. And AQE coalescing is not a cure-all here, since it only affects post-shuffle partitions and leaves the input file layout untouched<sup class="footnote-ref"><a href="#fn6" id="fnref6:1">[6:1]</a></sup>. The signal reflects the shape, not intent, so confirm the write is not an intended partitioned layout before acting.</p><h2 id="related" tabindex="-1">Related <a class="header-anchor" href="#related" aria-label="Permalink to &quot;Related&quot;">​</a></h2><ul><li><strong>Partition sizing:</strong> <a href="./partitioning.html">Partitioning</a></li><li><strong>Too many small tasks:</strong> <a href="./bottleneck-tiny-tasks.html">Tiny Tasks</a></li></ul><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><em>Spark: The Definitive Guide</em>, Chambers &amp; Zaharia, ch. 19 <a href="#fnref1" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><em>Learning Spark, 2nd Edition</em>, Damji, Wenig, Das, Lee, ch. 7 <a href="#fnref2" class="footnote-backref">↩︎</a> <a href="#fnref2:1" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><a href="https://raw.githubusercontent.com/apache/spark/v3.5.0/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala" target="_blank" rel="noreferrer">SQLConf.scala</a> <a href="#fnref3" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><a href="https://luminousmen.com/post/spark-partitions" target="_blank" rel="noreferrer">Spark Partitions</a> <a href="#fnref4" class="footnote-backref">↩︎</a> <a href="#fnref4:1" class="footnote-backref">↩︎</a> <a href="#fnref4:2" class="footnote-backref">↩︎</a> <a href="#fnref4:3" class="footnote-backref">↩︎</a> <a href="#fnref4:4" class="footnote-backref">↩︎</a> <a href="#fnref4:5" class="footnote-backref">↩︎</a> <a href="#fnref4:6" class="footnote-backref">↩︎</a> <a href="#fnref4:7" class="footnote-backref">↩︎</a></p></li><li id="fn5" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/sql-performance-tuning.html" target="_blank" rel="noreferrer">Performance Tuning (Spark SQL, DataFrames and Datasets Guide)</a> <a href="#fnref5" class="footnote-backref">↩︎</a> <a href="#fnref5:1" class="footnote-backref">↩︎</a> <a href="#fnref5:2" class="footnote-backref">↩︎</a></p></li><li id="fn6" class="footnote-item"><p><a href="https://luminousmen.com/post/the-apache-spark-optimization-checklist" target="_blank" rel="noreferrer">The Apache Spark Optimization Checklist</a> <a href="#fnref6" class="footnote-backref">↩︎</a> <a href="#fnref6:1" class="footnote-backref">↩︎</a></p></li></ol></section>`,25)])])}const u=t(o,[["render",n]]);export{p as __pageData,u as default};

package/public/docs/assets/tuning-reference_bottleneck-small-files.md.DHGMTmPs.lean.js

@@ -1 +0,0 @@
-import{_ as t,o as a,c as s,a5 as i}from"./chunks/framework.DSg0KOwT.js";const p=JSON.parse('{"title":"Small Files","description":"","frontmatter":{"title":"Small Files"},"headers":[],"relativePath":"tuning-reference/bottleneck-small-files.md","filePath":"tuning-reference/bottleneck-small-files.md"}'),o={name:"tuning-reference/bottleneck-small-files.md"};function n(r,e,l,f,h,c){return a(),s("div",null,[...e[0]||(e[0]=[i("",25)])])}const u=t(o,[["render",n]]);export{p as __pageData,u as default};

package/public/docs/assets/tuning-reference_bottleneck-spill.md.PNH7mITt.js

@@ -1,6 +0,0 @@
-import{_ as a,o as t,c as o,a5 as s}from"./chunks/framework.DSg0KOwT.js";const i="/docs/assets/spill-classification.BU2euYDO.svg",r="/docs/assets/spill-classification.dark.D7i1M40d.svg",u=JSON.parse('{"title":"Memory / Disk Spill","description":"","frontmatter":{"title":"Memory / Disk Spill"},"headers":[],"relativePath":"tuning-reference/bottleneck-spill.md","filePath":"tuning-reference/bottleneck-spill.md"}'),n={name:"tuning-reference/bottleneck-spill.md"};function l(c,e,f,h,p,d){return t(),o("div",null,[...e[0]||(e[0]=[s('<h1 id="bottleneck-spill" tabindex="-1">Memory / Disk Spill <a class="header-anchor" href="#bottleneck-spill" aria-label="Permalink to &quot;Memory / Disk Spill {#bottleneck-spill}&quot;">​</a></h1><p><span class="tag">SPILL</span></p><h2 id="what-it-is" tabindex="-1">What it is <a class="header-anchor" href="#what-it-is" aria-label="Permalink to &quot;What it is&quot;">​</a></h2><p>Spill happens when a task&#39;s share of execution memory runs out during a sort or hash aggregation. Each task gets its own <code>TaskMemoryManager</code>, which arbitrates the shared execution pool across every task running concurrently on an executor; with <code>n</code> tasks running concurrently, each is allowed to allocate somewhere between <code>1/(2n)</code> and <code>1/n</code> of total execution memory: a soft limit, not a hard stop.<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup> When a task&#39;s sort or hash-aggregation operator keeps requesting execution-memory pages it can&#39;t get, Spark doesn&#39;t fail immediately: it blocks the requesting task, triggers a spill of that task&#39;s in-memory data structure to disk, or, in the worst case, throws an <code>OutOfMemoryError</code>.<sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup></p><p>The spill path runs through Spark&#39;s map/shuffle I/O machinery: shuffle partitions created by wide transformations like <code>groupBy()</code> or <code>join()</code> spill to the executors&#39; local disks, at the location set by <code>spark.local.directory</code>.<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup> SQL physical operators apply the same idea with their own guardrails: <code>SortMergeJoinExec</code>&#39;s in-memory buffer and the cartesian-product operator&#39;s buffer both spill once they cross a configured row-count threshold, which by default is <code>spark.shuffle.spill.numElementsForceSpillThreshold</code>.<sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup> Whatever the trigger, the resulting spilled bytes are exposed on the task metrics as <code>memoryBytesSpilled</code>, visible in the Spark UI and event log.<sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup></p><h2 id="how-it-s-detected" tabindex="-1">How it&#39;s detected <a class="header-anchor" href="#how-it-s-detected" aria-label="Permalink to &quot;How it&#39;s detected&quot;">​</a></h2><p>Detection: any <code>memoryBytesSpilled &gt; 0</code> → Warning. The classification is the actionable signal.</p><p>Spill classification:</p><ul><li><code>skew</code>: ≥ 80% of tasks have zero spill; fix: address <a href="./bottleneck-skew.html">task skew</a>, not memory.</li><li><code>volume</code>: &lt; 20% of tasks have zero spill; fix: more partitions or more memory.</li><li><code>unclassified</code>: neither condition; treat as volume.</li></ul><img class="light-only" src="'+i+'" alt="How a nonzero memoryBytesSpilled is classified as skew, volume, or unclassified from the share of tasks with zero spill, and the fix each classification points to."><img class="dark-only" src="'+r+`" alt="How a nonzero memoryBytesSpilled is classified as skew, volume, or unclassified from the share of tasks with zero spill, and the fix each classification points to."><h2 id="why-it-matters" tabindex="-1">Why it matters <a class="header-anchor" href="#why-it-matters" aria-label="Permalink to &quot;Why it matters&quot;">​</a></h2><p>Spill is the fallback Spark reaches for once a task can no longer get the execution memory it&#39;s asking for: rather than failing outright, it blocks the task, writes its buffered data to disk, or (if that&#39;s not enough) throws an <code>OutOfMemoryError</code>.<sup class="footnote-ref"><a href="#fn1" id="fnref1:2">[1:2]</a></sup> A task that&#39;s spilling has already given up pure in-memory speed to keep running at all, which is why the skew-vs-volume split above is the actionable part of the signal: the two failure shapes call for opposite fixes.</p><h2 id="how-to-fix-it" tabindex="-1">How to fix it <a class="header-anchor" href="#how-to-fix-it" aria-label="Permalink to &quot;How to fix it&quot;">​</a></h2><ul><li><strong>Skew</strong>: When only a few tasks spill, the problem is the shape of the data, not the size of the memory pool. <code>repartition()</code> is the tool for this: reach for it specifically to fix a lopsided partition distribution or to increase partition count<sup class="footnote-ref"><a href="#fn5" id="fnref5">[5]</a></sup>, in contrast to <code>coalesce()</code>, which never rebalances skewed data because it just stacks existing partitions together: partitions that were uneven going in are still uneven coming out.<sup class="footnote-ref"><a href="#fn6" id="fnref6">[6]</a></sup> For skewed joins, <a href="./aqe.html">Adaptive Query Execution</a> can detect and split<sup class="footnote-ref"><a href="#fn7" id="fnref7">[7]</a></sup> oversized partitions automatically: a partition counts as skewed once it&#39;s larger than <code>spark.sql.adaptive.skewJoin.skewedPartitionFactor</code> (default <code>5.0</code>) times the median partition size, and also larger than <code>spark.sql.adaptive.skewJoin.skewedPartitionThresholdInBytes</code> (default <code>256 MB</code>).<sup class="footnote-ref"><a href="#fn8" id="fnref8">[8]</a></sup><sup class="footnote-ref"><a href="#fn9" id="fnref9">[9]</a></sup> Before AQE existed, the manual equivalent was salting: adding a prefix to the skewed keys to make the same key look different, then adjusting the data distribution accordingly.<sup class="footnote-ref"><a href="#fn7" id="fnref7:1">[7:1]</a></sup> Databricks&#39; <code>/*+ SKEW(...) */</code> hint lets you name the skewed relation and specific key values directly, so the planner targets just those keys instead of relying on automatic detection.<sup class="footnote-ref"><a href="#fn10" id="fnref10">[10]</a></sup></li><li><strong>Volume</strong>: When most tasks spill, the fix is more room to work with: more partitions, more memory, or both. <code>repartition(n)</code> guarantees exactly <code>n</code> output partitions via a hash shuffle<sup class="footnote-ref"><a href="#fn11" id="fnref11">[11]</a></sup>, and since Spark&#39;s per-task startup overhead is low (unlike MapReduce&#39;s), the general bias is to err toward more partitions rather than fewer.<sup class="footnote-ref"><a href="#fn12" id="fnref12">[12]</a></sup><code>spark.sql.files.maxPartitionBytes</code> (default <code>128 MB</code>) is the equivalent lever on the input side, governing how much file data gets packed into each input partition before a shuffle even happens.<sup class="footnote-ref"><a href="#fn6" id="fnref6:1">[6:1]</a></sup> On the memory side, execution memory is a fraction of the JVM heap. <code>spark.memory.fraction</code> (default <code>0.6</code>) sizes the shared execution/storage region as <code>(JVM heap − 300 MiB) × spark.memory.fraction</code><sup class="footnote-ref"><a href="#fn13" id="fnref13">[13]</a></sup>. Because each task&#39;s slice of that region is capped by the <code>TaskMemoryManager</code> at between <code>1/(2n)</code> and <code>1/n</code> of the total,<sup class="footnote-ref"><a href="#fn1" id="fnref1:3">[1:3]</a></sup> giving executors more memory, or running fewer concurrent tasks on each one, directly raises the ceiling before spill kicks in.</li></ul><p>For the <strong>volume</strong> case, give tasks more room: defaults shown, comments say which way to move:</p><div class="language-properties vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">properties</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Create more, smaller input partitions before the shuffle (default 128m)</span></span>
-<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.sql.files.maxPartitionBytes</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=128m</span></span>
-<span class="line"></span>
-<span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Execution/storage share of (JVM heap - 300MiB) (default 0.6);</span></span>
-<span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># raising it grows execution memory but starves the untracked user-memory region</span></span>
-<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.memory.fraction</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=0.6</span></span></code></pre></div><p>For the <strong>skew</strong> case (only a few tasks spill), the fix is <code>df.repartition(n)</code>, not more memory: see <a href="./partitioning.html">Partitioning</a>.</p><h2 id="confidence" tabindex="-1">Confidence <a class="header-anchor" href="#confidence" aria-label="Permalink to &quot;Confidence&quot;">​</a></h2><p>The core detection (any <code>memoryBytesSpilled &gt; 0</code>) and the skew-vs-volume classification are validated: both the skew branch (most tasks spill nothing, so the shape of the data is the problem) and the volume branch (nearly every task spills, so the pool is too small) map to a well-understood fix, and neither needs further validation. <span class="tag">EXPERIMENTAL</span> When a spill matches neither shape, it falls back to a low-confidence unclassified finding that still requires validation, because there is no confirmed cause to act on; the page defaults it to the volume remedy, but that is a guess, not a diagnosis.</p><h2 id="limitations-false-positive-risk" tabindex="-1">Limitations / false-positive risk <a class="header-anchor" href="#limitations-false-positive-risk" aria-label="Permalink to &quot;Limitations / false-positive risk&quot;">​</a></h2><p>Some spill is normal. A large aggregation or join can spill by design once its working set outgrows execution memory, and a small spill on a healthy stage rarely repays the effort of chasing it. The raw <code>memoryBytesSpilled &gt; 0</code> trigger fires on both the actionable and the routine cases, so treat a small spill as informational until the skew-vs-volume split says otherwise.</p><h2 id="related" tabindex="-1">Related <a class="header-anchor" href="#related" aria-label="Permalink to &quot;Related&quot;">​</a></h2><ul><li><strong>Why it happens:</strong> <a href="./memory-model.html">Memory Management</a>, <a href="./partitioning.html">Partitioning</a></li></ul><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><a href="https://luminousmen.com/post/dive-into-spark-memory" target="_blank" rel="noreferrer">Diving into Spark Memory Management</a> <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a> <a href="#fnref1:2" class="footnote-backref">↩︎</a> <a href="#fnref1:3" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><em>Learning Spark, 2nd Edition</em>, Damji, Wenig, Das, Lee, ch. 7 <a href="#fnref2" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><a href="https://raw.githubusercontent.com/apache/spark/v3.5.0/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala" target="_blank" rel="noreferrer">SQLConf.scala</a> <a href="#fnref3" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/monitoring.html#spark-history-server" target="_blank" rel="noreferrer">Monitoring and Instrumentation</a> <a href="#fnref4" class="footnote-backref">↩︎</a></p></li><li id="fn5" class="footnote-item"><p><em>Spark: The Definitive Guide</em>, Chambers &amp; Zaharia, ch. 19 <a href="#fnref5" class="footnote-backref">↩︎</a></p></li><li id="fn6" class="footnote-item"><p><a href="https://luminousmen.com/post/spark-partitions" target="_blank" rel="noreferrer">Spark Partitions</a> <a href="#fnref6" class="footnote-backref">↩︎</a> <a href="#fnref6:1" class="footnote-backref">↩︎</a></p></li><li id="fn7" class="footnote-item"><p><a href="https://issues.apache.org/jira/browse/SPARK-29544" target="_blank" rel="noreferrer">SPARK-29544: Optimize Skewed Join in SQL Adaptive Execution</a> <a href="#fnref7" class="footnote-backref">↩︎</a> <a href="#fnref7:1" class="footnote-backref">↩︎</a></p></li><li id="fn8" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration (Spark)</a> <a href="#fnref8" class="footnote-backref">↩︎</a></p></li><li id="fn9" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/sql-performance-tuning.html" target="_blank" rel="noreferrer">Performance Tuning (Spark SQL, DataFrames and Datasets Guide)</a> <a href="#fnref9" class="footnote-backref">↩︎</a></p></li><li id="fn10" class="footnote-item"><p><a href="https://docs.databricks.com/aws/en/archive/legacy/skew-join" target="_blank" rel="noreferrer">Skew Join Optimization</a> <a href="#fnref10" class="footnote-backref">↩︎</a></p></li><li id="fn11" class="footnote-item"><p><a href="https://raw.githubusercontent.com/apache/spark/v3.5.0/core/src/main/scala/org/apache/spark/rdd/RDD.scala" target="_blank" rel="noreferrer">RDD.scala</a> <a href="#fnref11" class="footnote-backref">↩︎</a></p></li><li id="fn12" class="footnote-item"><p><a href="https://blog.cloudera.com/how-to-tune-your-apache-spark-jobs-part-2/" target="_blank" rel="noreferrer">How to Tune Your Apache Spark Jobs (Part 2): Cloudera Engineering Blog</a> <a href="#fnref12" class="footnote-backref">↩︎</a></p></li><li id="fn13" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/tuning.html" target="_blank" rel="noreferrer">Spark Tuning Guide</a> <a href="#fnref13" class="footnote-backref">↩︎</a></p></li></ol></section>`,26)])])}const k=a(n,[["render",l]]);export{u as __pageData,k as default};

package/public/docs/assets/tuning-reference_bottleneck-spill.md.PNH7mITt.lean.js

@@ -1 +0,0 @@
-import{_ as a,o as t,c as o,a5 as s}from"./chunks/framework.DSg0KOwT.js";const i="/docs/assets/spill-classification.BU2euYDO.svg",r="/docs/assets/spill-classification.dark.D7i1M40d.svg",u=JSON.parse('{"title":"Memory / Disk Spill","description":"","frontmatter":{"title":"Memory / Disk Spill"},"headers":[],"relativePath":"tuning-reference/bottleneck-spill.md","filePath":"tuning-reference/bottleneck-spill.md"}'),n={name:"tuning-reference/bottleneck-spill.md"};function l(c,e,f,h,p,d){return t(),o("div",null,[...e[0]||(e[0]=[s("",26)])])}const k=a(n,[["render",l]]);export{u as __pageData,k as default};

package/public/docs/assets/tuning-reference_bottleneck-straggler.md.DIbXdAcQ.js

@@ -1,7 +0,0 @@
-import{_ as t,o as a,c as s,a5 as o}from"./chunks/framework.DSg0KOwT.js";const p=JSON.parse('{"title":"Stragglers","description":"","frontmatter":{"title":"Stragglers"},"headers":[],"relativePath":"tuning-reference/bottleneck-straggler.md","filePath":"tuning-reference/bottleneck-straggler.md"}'),i={name:"tuning-reference/bottleneck-straggler.md"};function r(n,e,l,h,c,d){return a(),s("div",null,[...e[0]||(e[0]=[o(`<h1 id="bottleneck-straggler" tabindex="-1">Stragglers <a class="header-anchor" href="#bottleneck-straggler" aria-label="Permalink to &quot;Stragglers {#bottleneck-straggler}&quot;">​</a></h1><p><span class="tag">STRAG</span></p><h2 id="what-it-is" tabindex="-1">What it is <a class="header-anchor" href="#what-it-is" aria-label="Permalink to &quot;What it is&quot;">​</a></h2><p>A straggler is one task (or a small handful of tasks) that runs far longer than the rest of the tasks in its stage, even when the stage is otherwise healthy. Because a stage only completes once its last task finishes, a single straggler holds up the whole stage, and every other executor sits idle waiting for it to catch up.</p><h2 id="how-it-s-detected" tabindex="-1">How it&#39;s detected <a class="header-anchor" href="#how-it-s-detected" aria-label="Permalink to &quot;How it&#39;s detected&quot;">​</a></h2><p>A task counts as a straggler when speculative tasks fired for it, or when more than 5% of the tasks in a stage run at least 4× the stage&#39;s median task duration; this rule only applies once a stage has at least 10 tasks.</p><p>&quot;Duration&quot; here is <code>executorRunTime</code>: elapsed wall-clock time, not CPU time, and it already includes any time the task spent blocked fetching shuffle data<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup>. That matters when tracking down why a task is slow: a long <code>executorRunTime</code> doesn&#39;t necessarily mean more compute happened. A <a href="./bottleneck-gc.html">garbage-collection pause</a> is counted inside it rather than added on top (<code>jvmGCTime</code> is a subset of <code>executorRunTime</code>, not additive<sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup>), and a task waiting on a remote shuffle block it needs next shows up in <code>shuffleReadMetrics.fetchWaitTime</code>, which only counts genuine blocking time, not blocks being prefetched in the background<sup class="footnote-ref"><a href="#fn1" id="fnref1:2">[1:2]</a></sup>.</p><p>The &quot;speculative tasks fired&quot; half of the rule is Spark&#39;s own detector: once <code>spark.speculation.quantile</code> (default <code>0.9</code>) of a stage&#39;s tasks finish, Spark compares each remaining task&#39;s duration against <code>spark.speculation.multiplier</code> (default <code>3</code>) times the median of the tasks that already finished, subject to a <code>spark.speculation.minTaskRuntime</code> floor (default <code>100ms</code>) so short tasks aren&#39;t flagged just for being slower than a tiny median<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup>. Speculation itself is off by default (<code>spark.speculation</code> defaults to <code>false</code>), so this half of the rule only fires on stages where it&#39;s been turned on<sup class="footnote-ref"><a href="#fn2" id="fnref2:1">[2:1]</a></sup>.</p><p>Watch for overlap with <a href="./bottleneck-skew.html">task skew</a>: an unevenly distributed key sends one partition far more data than its peers, and that partition&#39;s task will trip this same duration threshold even though the underlying cause is data volume, not a slow host or a GC pause. Check the skew signals before assuming the latter.</p><h2 id="why-it-matters" tabindex="-1">Why it matters <a class="header-anchor" href="#why-it-matters" aria-label="Permalink to &quot;Why it matters&quot;">​</a></h2><p>A straggler wastes cluster capacity the same way a skewed stage does: every executor other than the one running the slow task finishes early and idles, while total job time still tracks the single slowest task. Because a straggler&#39;s duration can be inflated by a GC pause<sup class="footnote-ref"><a href="#fn1" id="fnref1:3">[1:3]</a></sup> or a slow <a href="./shuffle.html">shuffle</a> fetch<sup class="footnote-ref"><a href="#fn1" id="fnref1:4">[1:4]</a></sup> rather than genuinely more work, it&#39;s worth checking those angles (and whether the real cause is skew rather than anything task-local) before assuming a hardware explanation.</p><h2 id="how-to-fix-it" tabindex="-1">How to fix it <a class="header-anchor" href="#how-to-fix-it" aria-label="Permalink to &quot;How to fix it&quot;">​</a></h2><ul><li>Enable speculative execution: <code>spark.speculation</code> is <code>false</code> by default, so nothing reruns a slow task automatically until it&#39;s turned on<sup class="footnote-ref"><a href="#fn2" id="fnref2:2">[2:2]</a></sup>.</li><li>Tune the trigger: <code>spark.speculation.quantile</code> (default <code>0.9</code>) sets how much of the stage must finish before speculation kicks in, and <code>spark.speculation.multiplier</code> (default <code>3</code>) sets how many times slower than the median a task must be. For stages with very few tasks, <code>spark.speculation.task.duration.threshold</code> (available since 3.0.0) gives an absolute-duration trigger instead of relying on the median<sup class="footnote-ref"><a href="#fn2" id="fnref2:3">[2:3]</a></sup>.</li><li>Since Spark 3.4, <code>spark.speculation.efficiency.enabled</code> (default <code>true</code>) adds a filter so a task is only speculated if its data-process rate is below the stage average (times <code>spark.speculation.efficiency.processRateMultiplier</code>, default <code>0.75</code>) or its duration exceeds <code>spark.speculation.efficiency.longRunTaskFactor</code> (default <code>2</code>) times the same time threshold. This avoids wasting a duplicate task slot on work that&#39;s simply doing more, not running slower<sup class="footnote-ref"><a href="#fn2" id="fnref2:4">[2:4]</a></sup>.</li><li>If the cause is really a skewed key, treat it as <a href="./bottleneck-skew.html">task skew</a> instead: AQE&#39;s skew-join optimization automatically splits a partition once it&#39;s larger than <code>spark.sql.adaptive.skewJoin.skewedPartitionFactor</code> (default <code>5.0</code>) times the median partition size and above <code>spark.sql.adaptive.skewJoin.skewedPartitionThresholdInBytes</code> (default <code>256 MB</code>)<sup class="footnote-ref"><a href="#fn2" id="fnref2:5">[2:5]</a></sup><sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup>. Manual salting (appending a random prefix to the skewed key so it spreads across more partitions) is the pre-AQE fallback, though the design doc behind AQE&#39;s skew handling calls salting and other manual approaches limited compared to the automatic option<sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup>.</li></ul><blockquote><p><strong>PySpark:</strong> speculation settings can be set on the session directly, no <code>spark-submit</code> flag needed: <code>spark.conf.set(&quot;spark.speculation&quot;, &quot;true&quot;)</code>, then the quantile/multiplier equivalents the same way.</p></blockquote><p>Speculation config: defaults shown, plus the absolute-duration trigger for small stages:</p><div class="language-properties vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">properties</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Speculatively relaunch straggler tasks (OFF by default)</span></span>
-<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.speculation</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=true</span></span>
-<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.speculation.quantile</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=0.9          </span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># default; portion of tasks finished before speculation begins</span></span>
-<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.speculation.multiplier</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=3          </span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># default; multiple of median duration that marks a task slow</span></span>
-<span class="line"></span>
-<span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Absolute-duration trigger for stages with very few tasks (since Spark 3.0)</span></span>
-<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.speculation.task.duration.threshold</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=10s   </span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># 10s is an example</span></span></code></pre></div><h2 id="confidence" tabindex="-1">Confidence <a class="header-anchor" href="#confidence" aria-label="Permalink to &quot;Confidence&quot;">​</a></h2><p>Validated.</p><h2 id="limitations-false-positive-risk" tabindex="-1">Limitations / false-positive risk <a class="header-anchor" href="#limitations-false-positive-risk" aria-label="Permalink to &quot;Limitations / false-positive risk&quot;">​</a></h2><p>The 4x-median rule flags a slow task, but slow is not the same as broken. The same threshold trips on a skewed key that simply has more data to process, or on a task that spent its time in a GC pause rather than doing extra work, so a flagged task is not automatically a slow host. Small stages make this worse: with only a handful of tasks the median is unstable, and one moderately slow task can look like a straggler against a median computed from too few peers.</p><h2 id="bottleneck-speculation-waste" tabindex="-1">Speculation waste <a class="header-anchor" href="#bottleneck-speculation-waste" aria-label="Permalink to &quot;Speculation waste {#bottleneck-speculation-waste}&quot;">​</a></h2><p><span class="tag">SPEC</span></p><p>A straggler is reported by whichever signal, straggler share or speculative-task count, best explains it. Speculation waste is a separate, narrower signal: the executor time speculative attempts burned without confirming a genuine straggler, whether the speculative copy lost the race to the original attempt or the other way around. Either way, the losing attempt&#39;s executor time is pure waste.</p><h3 id="how-it-s-detected-1" tabindex="-1">How it&#39;s detected <a class="header-anchor" href="#how-it-s-detected-1" aria-label="Permalink to &quot;How it&#39;s detected&quot;">​</a></h3><table tabindex="0"><thead><tr><th>Signal</th><th>Fires when</th></tr></thead><tbody><tr><td>Wasted speculative attempts in a stage</td><td>≥ 5</td></tr><tr><td>Wasted executor time from those attempts</td><td>≥ 60 seconds</td></tr></tbody></table><p>Both conditions have to hold together. Severity tracks the estimated recoverable time as a share of the app&#39;s total runtime, the same model used across this page: ≥2% is critical, ≥0.5% is warning, anything smaller is info.</p><h3 id="why-it-matters-1" tabindex="-1">Why it matters <a class="header-anchor" href="#why-it-matters-1" aria-label="Permalink to &quot;Why it matters&quot;">​</a></h3><p>Every wasted speculative attempt occupies an executor slot that could have run other work, so a stage generating a lot of speculative waste is trading cluster capacity for copies that never pay off.</p><h3 id="how-to-fix-it-1" tabindex="-1">How to fix it <a class="header-anchor" href="#how-to-fix-it-1" aria-label="Permalink to &quot;How to fix it&quot;">​</a></h3><p>If task durations are naturally variable rather than genuine stragglers, speculation is firing too eagerly: tune <code>spark.speculation.multiplier</code> (require a bigger gap from the median before speculating) or <code>spark.speculation.quantile</code> (wait for more of the stage to finish first) so fewer ordinary slow tasks get speculated in the first place.</p><h2 id="related" tabindex="-1">Related <a class="header-anchor" href="#related" aria-label="Permalink to &quot;Related&quot;">​</a></h2><ul><li><strong>When the real cause is a skewed key:</strong> <a href="./partitioning.html">Partitioning</a>, <a href="./aqe.html">Adaptive Query Execution</a></li></ul><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/monitoring.html#spark-history-server" target="_blank" rel="noreferrer">Monitoring and Instrumentation</a> <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a> <a href="#fnref1:2" class="footnote-backref">↩︎</a> <a href="#fnref1:3" class="footnote-backref">↩︎</a> <a href="#fnref1:4" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration (Spark)</a> <a href="#fnref2" class="footnote-backref">↩︎</a> <a href="#fnref2:1" class="footnote-backref">↩︎</a> <a href="#fnref2:2" class="footnote-backref">↩︎</a> <a href="#fnref2:3" class="footnote-backref">↩︎</a> <a href="#fnref2:4" class="footnote-backref">↩︎</a> <a href="#fnref2:5" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/sql-performance-tuning.html#optimizing-skew-join" target="_blank" rel="noreferrer">Optimizing Skew Join (Spark SQL, DataFrames and Datasets Guide)</a> <a href="#fnref3" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><a href="https://issues.apache.org/jira/browse/SPARK-29544" target="_blank" rel="noreferrer">SPARK-29544: Optimize Skewed Join at Runtime with New Adaptive Execution</a> <a href="#fnref4" class="footnote-backref">↩︎</a></p></li></ol></section>`,34)])])}const u=t(i,[["render",r]]);export{p as __pageData,u as default};

package/public/docs/assets/tuning-reference_bottleneck-straggler.md.DIbXdAcQ.lean.js

@@ -1 +0,0 @@
-import{_ as t,o as a,c as s,a5 as o}from"./chunks/framework.DSg0KOwT.js";const p=JSON.parse('{"title":"Stragglers","description":"","frontmatter":{"title":"Stragglers"},"headers":[],"relativePath":"tuning-reference/bottleneck-straggler.md","filePath":"tuning-reference/bottleneck-straggler.md"}'),i={name:"tuning-reference/bottleneck-straggler.md"};function r(n,e,l,h,c,d){return a(),s("div",null,[...e[0]||(e[0]=[o("",34)])])}const u=t(i,[["render",r]]);export{p as __pageData,u as default};

package/public/docs/assets/tuning-reference_bottleneck-tiny-tasks.md.CB05v3zh.js

@@ -1,8 +0,0 @@
-import{_ as t,o as a,c as s,a5 as i}from"./chunks/framework.DSg0KOwT.js";const p=JSON.parse('{"title":"Tiny Tasks","description":"","frontmatter":{"title":"Tiny Tasks"},"headers":[],"relativePath":"tuning-reference/bottleneck-tiny-tasks.md","filePath":"tuning-reference/bottleneck-tiny-tasks.md"}'),o={name:"tuning-reference/bottleneck-tiny-tasks.md"};function n(r,e,l,h,f,d){return a(),s("div",null,[...e[0]||(e[0]=[i(`<h1 id="bottleneck-tiny-tasks" tabindex="-1">Tiny Tasks <a class="header-anchor" href="#bottleneck-tiny-tasks" aria-label="Permalink to &quot;Tiny Tasks {#bottleneck-tiny-tasks}&quot;">​</a></h1><p><span class="tag">TINY</span></p><h2 id="what-it-is" tabindex="-1">What it is <a class="header-anchor" href="#what-it-is" aria-label="Permalink to &quot;What it is&quot;">​</a></h2><p>Every task pays a fixed placement and serialization cost before it does any real work. Spark schedules around data locality rather than moving data to code: &quot;Spark builds its scheduling around this general principle&quot; that shipping serialized code is cheaper than shipping data, preferring <code>PROCESS_LOCAL</code> locality down to <code>ANY</code> and waiting a configurable timeout for a busy CPU to free up before shipping data to a farther executor<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup>. On top of that placement cost, every task also pays a serialization cost for its closure and, if not using Kryo, for its data: Java&#39;s default <code>ObjectOutputStream</code>-based serialization &quot;is flexible but often quite slow, and leads to large serialized formats,&quot; while Kryo is &quot;significantly faster and more compact than Java serialization (often as much as 10x)&quot;<sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup>. None of that overhead is large per task: Spark&#39;s own guidance leans toward more, smaller tasks rather than fewer, larger ones, unlike MapReduce: &quot;it&#39;s almost always better to err on the side of a larger number of tasks (and thus partitions)... The difference stems from the fact that MapReduce has a high startup overhead for tasks, while Spark does not&quot;<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup>. But once partitions get small enough, that per-task overhead stops being negligible and starts to dominate.</p><h2 id="how-it-s-detected" tabindex="-1">How it&#39;s detected <a class="header-anchor" href="#how-it-s-detected" aria-label="Permalink to &quot;How it&#39;s detected&quot;">​</a></h2><p>The failure mode is partitions so small that scheduling and serialization overhead outweighs the actual work. With Spark&#39;s default of 200 shuffle partitions applied to a dataset of only a few megabytes, &quot;those 200 partitions will each get like ten rows. Tasks become microscopic. Most of your CPUs will just be sitting there doing nothing, wasting cluster hours&quot;<sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup>. The same small-partition penalty shows up on the <a href="./bottleneck-shuffle.html">shuffle-read side</a>: production measurements found &quot;the average shuffle block size is only 10s of KBs, which leads to delayed shuffle data fetch,&quot; and the shuffle reduce stages with the largest fetch delays (over 30 seconds per task) were consistently the ones with small block sizes<sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup>. A high task count paired with very short median task duration, and/or very small shuffle block sizes on the read side, are the signals to look for.</p><h2 id="why-it-matters" tabindex="-1">Why it matters <a class="header-anchor" href="#why-it-matters" aria-label="Permalink to &quot;Why it matters&quot;">​</a></h2><p>Tiny tasks waste cluster capacity even though no single task is a straggler the way <a href="./bottleneck-skew.html">skewed</a> or <a href="./bottleneck-straggler.html">straggler</a> tasks are: the cost here is spread evenly across thousands of tasks, each individually cheap but collectively adding up in scheduling and serialization overhead, while cores that could be doing other useful work sit mostly idle between task launches<sup class="footnote-ref"><a href="#fn3" id="fnref3:1">[3:1]</a></sup>.</p><h2 id="how-to-fix-it" tabindex="-1">How to fix it <a class="header-anchor" href="#how-to-fix-it" aria-label="Permalink to &quot;How to fix it&quot;">​</a></h2><ul><li><code>coalesce()</code> merges existing partitions without a shuffle (&quot;no data movement, no shuffle&quot;) and is typically used right before a write, e.g. <code>df.coalesce(100).write.parquet(...)</code>, to collapse many tiny output files into a manageable number cheaply<sup class="footnote-ref"><a href="#fn3" id="fnref3:2">[3:2]</a></sup>. Because it only stacks partitions together rather than redistributing data, it doesn&#39;t fix an uneven underlying distribution (uneven input partitions stay uneven, just grouped) and pushing it too far (<code>coalesce(1)</code>) kills parallelism by funneling all work onto one executor while the rest sit idle<sup class="footnote-ref"><a href="#fn3" id="fnref3:3">[3:3]</a></sup>.</li><li><code>repartition()</code> performs a full reshuffle, giving control and rebalancing that Spark won&#39;t do on its own: with <code>df.repartition(200)</code>, &quot;you&#39;re paying for predictability&quot;<sup class="footnote-ref"><a href="#fn3" id="fnref3:4">[3:4]</a></sup>. Use it when the partitioning itself needs to be fixed or rebalanced, not merely reduced in count, accepting the shuffle cost to get there.</li><li>There&#39;s a middle option, <code>coalesce(N, shuffle=True)</code>, which &quot;acts more like a repartition, but leaning toward reduction... not free (you pay for the shuffle cost) but you get better distribution and fewer partitions&quot;<sup class="footnote-ref"><a href="#fn3" id="fnref3:5">[3:5]</a></sup>.</li><li>Rule of thumb: reach for <code>coalesce()</code> when only the partition count needs to shrink and the existing distribution is already reasonably even (e.g. collapsing output files); reach for <code>repartition()</code> when the distribution itself is the problem.</li></ul><blockquote><p><strong>PySpark:</strong> both are one-line calls on a DataFrame: <code>df.coalesce(100)</code> for a shuffle-free merge, <code>df.repartition(200)</code> for a full reshuffle, or <code>df.coalesce(100, shuffle=True)</code> for the shuffled middle option.</p></blockquote><p>The three sizing calls, side by side:</p><div class="language-python vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">python</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Collapse many tiny output files without a shuffle (use when distribution is already even)</span></span>
-<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">df.coalesce(</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">100</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">).write.parquet(path)</span></span>
-<span class="line"></span>
-<span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Full reshuffle to a target count — use when the distribution itself needs rebalancing</span></span>
-<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">df </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">=</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> df.repartition(</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">200</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">)</span></span>
-<span class="line"></span>
-<span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Middle ground: reduce partition count but still rebalance (pays a shuffle)</span></span>
-<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">df </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">=</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> df.coalesce(</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">100</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70;">shuffle</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">=</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">True</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">)</span></span></code></pre></div><h2 id="limitations-false-positive-risk" tabindex="-1">Limitations / false-positive risk <a class="header-anchor" href="#limitations-false-positive-risk" aria-label="Permalink to &quot;Limitations / false-positive risk&quot;">​</a></h2><p>Very short tasks are not always a problem. A tiny final stage can be perfectly fine, and one that just writes a small result set doesn&#39;t need to be resized. The scheduling-overhead penalty only bites when tiny tasks dominate the stage, so treat a handful of short tasks as noise rather than a finding.</p><h2 id="related" tabindex="-1">Related <a class="header-anchor" href="#related" aria-label="Permalink to &quot;Related&quot;">​</a></h2><ul><li><strong>Partition sizing:</strong> <a href="./partitioning.html">Partitioning</a></li></ul><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/tuning.html" target="_blank" rel="noreferrer">Tuning (Spark)</a> <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><a href="https://blog.cloudera.com/how-to-tune-your-apache-spark-jobs-part-2/" target="_blank" rel="noreferrer">How to Tune Your Apache Spark Jobs (Part 2): Cloudera</a> <a href="#fnref2" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><a href="https://luminousmen.com/post/spark-partitions" target="_blank" rel="noreferrer">Spark Partitions</a> <a href="#fnref3" class="footnote-backref">↩︎</a> <a href="#fnref3:1" class="footnote-backref">↩︎</a> <a href="#fnref3:2" class="footnote-backref">↩︎</a> <a href="#fnref3:3" class="footnote-backref">↩︎</a> <a href="#fnref3:4" class="footnote-backref">↩︎</a> <a href="#fnref3:5" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><a href="https://issues.apache.org/jira/browse/SPARK-30602" target="_blank" rel="noreferrer">SPARK-30602</a> <a href="#fnref4" class="footnote-backref">↩︎</a></p></li></ol></section>`,19)])])}const u=t(o,[["render",n]]);export{p as __pageData,u as default};

package/public/docs/assets/tuning-reference_bottleneck-tiny-tasks.md.CB05v3zh.lean.js

@@ -1 +0,0 @@
-import{_ as t,o as a,c as s,a5 as i}from"./chunks/framework.DSg0KOwT.js";const p=JSON.parse('{"title":"Tiny Tasks","description":"","frontmatter":{"title":"Tiny Tasks"},"headers":[],"relativePath":"tuning-reference/bottleneck-tiny-tasks.md","filePath":"tuning-reference/bottleneck-tiny-tasks.md"}'),o={name:"tuning-reference/bottleneck-tiny-tasks.md"};function n(r,e,l,h,f,d){return a(),s("div",null,[...e[0]||(e[0]=[i("",19)])])}const u=t(o,[["render",n]]);export{p as __pageData,u as default};

package/public/docs/assets/tuning-reference_bottleneck-utilization.md.DqC_iGkw.lean.js

@@ -1 +0,0 @@
-import{_ as t,a}from"./chunks/duplicate-plan-subtree.dark.Cdp70QhV.js";import{_ as o,o as s,c as r,a5 as i}from"./chunks/framework.DSg0KOwT.js";const k=JSON.parse('{"title":"Executor Utilization","description":"","frontmatter":{"title":"Executor Utilization"},"headers":[],"relativePath":"tuning-reference/bottleneck-utilization.md","filePath":"tuning-reference/bottleneck-utilization.md"}'),n={name:"tuning-reference/bottleneck-utilization.md"};function l(c,e,f,h,d,p){return s(),r("div",null,[...e[0]||(e[0]=[i("",44)])])}const g=o(n,[["render",l]]);export{k as __pageData,g as default};

package/public/docs/assets/tuning-reference_caching.md.B7aQ8asB.js

@@ -1 +0,0 @@
-import{_ as a,o,c as t,a5 as r}from"./chunks/framework.DSg0KOwT.js";const s="/docs/assets/cache-lifecycle.rEOVYQNU.svg",n="/docs/assets/cache-lifecycle.dark.B-hS7AgU.svg",m=JSON.parse('{"title":"Caching & Persistence","description":"","frontmatter":{"title":"Caching & Persistence"},"headers":[],"relativePath":"tuning-reference/caching.md","filePath":"tuning-reference/caching.md"}'),f={name:"tuning-reference/caching.md"};function c(i,e,l,h,d,p){return o(),t("div",null,[...e[0]||(e[0]=[r('<h1 id="caching" tabindex="-1">Caching &amp; Persistence <a class="header-anchor" href="#caching" aria-label="Permalink to &quot;Caching &amp; Persistence {#caching}&quot;">​</a></h1><h2 id="how-persisting-and-checkpointing-differ" tabindex="-1">How persisting and checkpointing differ <a class="header-anchor" href="#how-persisting-and-checkpointing-differ" aria-label="Permalink to &quot;How persisting and checkpointing differ&quot;">​</a></h2><p>Spark&#39;s <code>persist()</code> is the general mechanism for keeping a DataFrame&#39;s already-computed partitions around instead of recomputing them from source on every action; <code>cache()</code> is shorthand for <code>persist(StorageLevel.MEMORY_AND_DISK)</code>: it calls persist with that one fixed level internally<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup>, whereas persist() lets you choose any storage level (in-memory vs on-disk, serialized vs deserialized, replicated or not)<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup>. When the level passed to persist() is exactly <code>MEMORY_AND_DISK</code>, the two calls are identical<sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup>.</p><p>Each storage level is defined by five attributes: <code>useDisk</code>, <code>useMemory</code>, <code>useOffHeap</code>, <code>deserialized</code>, and <code>replication</code><sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup>. Under <code>MEMORY_AND_DISK</code>, Spark stores data directly as objects in memory and serializes only the portion that doesn&#39;t fit, writing that overflow to disk<sup class="footnote-ref"><a href="#fn2" id="fnref2:1">[2:1]</a></sup>. <code>MEMORY_AND_DISK_SER</code> behaves the same way except the data kept in memory is also serialized (data written to disk is always serialized under either level)<sup class="footnote-ref"><a href="#fn2" id="fnref2:2">[2:2]</a></sup>. Serialized byte streams use less memory than deserialized JVM objects, which carry structural overhead, but reading them back costs more CPU than reading deserialized objects directly<sup class="footnote-ref"><a href="#fn3" id="fnref3:1">[3:1]</a></sup><sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup>.</p><p>Checkpointing is a related but distinct mechanism. Instead of persisting, it writes an RDD&#39;s partitions to an external, reliable store (HDFS, S3) and drops the lineage, the dependency chain Spark would otherwise use to recompute it<sup class="footnote-ref"><a href="#fn3" id="fnref3:2">[3:2]</a></sup><sup class="footnote-ref"><a href="#fn5" id="fnref5">[5]</a></sup>. That truncation doesn&#39;t happen the instant <code>.checkpoint()</code> is called; the call only marks the RDD, and the actual truncation runs later, in <code>doCheckpoint()</code>, after a job using the RDD completes. At that point the RDD is already materialized, and its dependencies and old parents are cleared<sup class="footnote-ref"><a href="#fn6" id="fnref6">[6]</a></sup>. A related call, <code>localCheckpoint()</code>, also truncates lineage using Spark&#39;s caching layer, but trades fault tolerance for speed: its data lives in ephemeral local executor storage rather than a reliable filesystem, so losing an executor mid-computation can make that data permanently unrecoverable<sup class="footnote-ref"><a href="#fn6" id="fnref6:1">[6:1]</a></sup>.</p><h2 id="when-caching-pays-off" tabindex="-1">When caching pays off <a class="header-anchor" href="#when-caching-pays-off" aria-label="Permalink to &quot;When caching pays off&quot;">​</a></h2><p>Because of how that materialization works, caching earns its keep only in specific situations. It&#39;s worth reaching for when the same DataFrame is scanned or recomputed more than once downstream: repeated queries against the same base data, iterative ML training loops, or interactive exploration where a dataset feeds multiple branches<sup class="footnote-ref"><a href="#fn2" id="fnref2:3">[2:3]</a></sup><sup class="footnote-ref"><a href="#fn7" id="fnref7">[7]</a></sup>. In one book benchmark, caching a 10M-row DataFrame and materializing it with <code>count()</code> cut a subsequent <code>count()</code> from 5.11s to 0.44s, roughly a 12x speedup<sup class="footnote-ref"><a href="#fn2" id="fnref2:4">[2:4]</a></sup>.</p><p>A few signals point the other way, toward caching having no effect or actively hurting:</p><ul><li><strong>Single-use data.</strong> If a dataset is processed only once downstream, caching just adds serialization and storage-bookkeeping cost for no reuse benefit<sup class="footnote-ref"><a href="#fn7" id="fnref7:1">[7:1]</a></sup>.</li><li><strong>Data too big to fit.</strong> Caching is all-or-nothing per partition: a DataFrame can be &quot;fractionally cached&quot; across its partitions, but individual partitions can&#39;t be split. With room for only 4.5 of 8 partitions, exactly 4 get cached; the uncached remainder is recomputed on every access, which can end up slower than not caching at all<sup class="footnote-ref"><a href="#fn2" id="fnref2:5">[2:5]</a></sup>.</li><li><strong>Partial materialization.</strong> <code>cache()</code>/<code>persist()</code> are lazy: nothing is cached until an action forces a full pass. <code>count()</code> forces a genuine full pass and materializes every partition, but an action like <code>take(1)</code> computes and caches only the one partition Catalyst needs, so a later full scan still recomputes most of the data. The same applies to <code>take(10)</code>, <code>limit(100)</code>, or any other partial or filtered scan: only the blocks Spark was forced to touch get cached, and the rest stays lazily uncomputed<sup class="footnote-ref"><a href="#fn2" id="fnref2:6">[2:6]</a></sup><sup class="footnote-ref"><a href="#fn1" id="fnref1:2">[1:2]</a></sup>.</li><li><strong>Plan mismatch.</strong> Caching wraps the <em>analyzed</em> (pre-optimization) logical plan in an <code>InMemoryRelation</code>. A logically-equivalent query written differently produces a different analyzed plan, so Spark can silently miss the cache and recompute from scratch even though the optimized plan would have been identical<sup class="footnote-ref"><a href="#fn1" id="fnref1:3">[1:3]</a></sup>.</li><li><strong>Optimizer limitations.</strong> Caching freezes Catalyst&#39;s optimization opportunities at the cached point. It can, for example, block <a href="./data-formats.html">predicate pushdown</a> once data is served from the in-memory cache instead of the source<sup class="footnote-ref"><a href="#fn5" id="fnref5:1">[5:1]</a></sup>.</li></ul><h2 id="why-cached-blocks-don-t-stick-around" tabindex="-1">Why cached blocks don&#39;t stick around <a class="header-anchor" href="#why-cached-blocks-don-t-stick-around" aria-label="Permalink to &quot;Why cached blocks don&#39;t stick around&quot;">​</a></h2><p>Materializing a cache is only half the story, though. A <code>cache()</code> followed by <code>count()</code> guarantees a materialization pass over every partition at that moment: caching happens locally and incrementally, with each executor&#39;s BlockManager storing only the blocks it computes, as it computes them, and no centralized &quot;cache the whole dataset&quot; step<sup class="footnote-ref"><a href="#fn1" id="fnref1:4">[1:4]</a></sup>. It does not guarantee those partitions stay cached afterward. Two things can undo it:</p><ul><li><strong>Executor loss.</strong> If a block was cached on an executor that later goes away, the cached data goes with it, unless a replicated storage level such as <code>MEMORY_AND_DISK_2</code> was used<sup class="footnote-ref"><a href="#fn1" id="fnref1:5">[1:5]</a></sup>. This matters specifically for <a href="./cluster-config.html">dynamic allocation</a>: Spark&#39;s documentation is explicit that caching together with dynamic allocation &quot;is NOT safe,&quot; because reclaiming idle executors takes their cached blocks with them<sup class="footnote-ref"><a href="#fn6" id="fnref6:2">[6:2]</a></sup>.</li><li><strong>Memory pressure eviction.</strong> Cached blocks can be evicted by other operations&#39; memory demands after materialization, independent of any executor loss<sup class="footnote-ref"><a href="#fn1" id="fnref1:6">[1:6]</a></sup>.</li></ul><p>Eviction itself is governed by LRU: Spark removes the least-recently-used cached blocks when storage memory is under pressure<sup class="footnote-ref"><a href="#fn3" id="fnref3:3">[3:3]</a></sup><sup class="footnote-ref"><a href="#fn1" id="fnref1:7">[1:7]</a></sup><sup class="footnote-ref"><a href="#fn8" id="fnref8">[8]</a></sup>. The trigger is memory contention between Spark&#39;s <a href="./memory-model.html">unified Execution and Storage regions</a>: under the <code>UnifiedMemoryManager</code>, Execution has priority, so if a task needs execution memory for a shuffle, join, aggregation, or sort and Storage is occupying that space, Spark evicts cached blocks to free it, and this can happen at any time, not only the next time the cache is accessed<sup class="footnote-ref"><a href="#fn9" id="fnref9">[9]</a></sup><sup class="footnote-ref"><a href="#fn1" id="fnref1:8">[1:8]</a></sup>. What happens to an evicted block depends on its storage level, not on a separate spill decision: a disk-backed level (<code>MEMORY_AND_DISK</code>, <code>MEMORY_AND_DISK_SER</code>) spills the block to disk and reads it back on next use, at the cost of disk I/O; a memory-only level (<code>MEMORY_ONLY</code>) simply drops the block and recomputes it from source when needed again<sup class="footnote-ref"><a href="#fn2" id="fnref2:7">[2:7]</a></sup><sup class="footnote-ref"><a href="#fn10" id="fnref10">[10]</a></sup><sup class="footnote-ref"><a href="#fn3" id="fnref3:4">[3:4]</a></sup><sup class="footnote-ref"><a href="#fn1" id="fnref1:9">[1:9]</a></sup>.</p><img class="light-only" src="'+s+'" alt="A cached block moves to evicted under memory pressure or lost when an executor goes away, then either spills to disk under MEMORY_AND_DISK or recomputes from lineage under MEMORY_ONLY."><img class="dark-only" src="'+n+'" alt="A cached block moves to evicted under memory pressure or lost when an executor goes away, then either spills to disk under MEMORY_AND_DISK or recomputes from lineage under MEMORY_ONLY."><p>There&#39;s no hard limit on how many DataFrames can be cached simultaneously; the constraint is aggregate storage memory, not a count of objects<sup class="footnote-ref"><a href="#fn2" id="fnref2:8">[2:8]</a></sup>. That budget is fraction-based configuration translated into a live byte allowance, not a fixed absolute constant: <code>spark.memory.fraction</code> (default 0.6) sets the fraction of (heap minus a 300MB reserved region) used for the combined Execution+Storage region, and <code>spark.memory.storageFraction</code> (default 0.5) sets the portion of that region immune to eviction. Lowering either makes <a href="./bottleneck-spill.html">spills</a> and evictions more frequent<sup class="footnote-ref"><a href="#fn4" id="fnref4:1">[4:1]</a></sup><sup class="footnote-ref"><a href="#fn9" id="fnref9:1">[9:1]</a></sup><sup class="footnote-ref"><a href="#fn1" id="fnref1:10">[1:10]</a></sup>.</p><h2 id="habits-that-keep-caching-effective" tabindex="-1">Habits that keep caching effective <a class="header-anchor" href="#habits-that-keep-caching-effective" aria-label="Permalink to &quot;Habits that keep caching effective&quot;">​</a></h2><p>A few habits keep caching effective despite how easily any of that goes wrong:</p><ul><li><strong>Materialize deliberately.</strong> Because caching is lazy, follow <code>cache()</code>/<code>persist()</code> with an action that forces a full pass (<code>count()</code> is the standard choice) rather than assuming the call alone did the work<sup class="footnote-ref"><a href="#fn2" id="fnref2:9">[2:9]</a></sup><sup class="footnote-ref"><a href="#fn7" id="fnref7:2">[7:2]</a></sup>.</li><li><strong>Match the storage level to the constraint you&#39;re solving.</strong> <code>cache()</code> / <code>MEMORY_AND_DISK</code> is a reasonable default: Spark keeps deserialized objects in memory and only serializes the overflow to disk. Reach for <code>MEMORY_AND_DISK_SER</code> when memory pressure is the binding constraint and you can afford the extra CPU cost of deserializing on read<sup class="footnote-ref"><a href="#fn2" id="fnref2:10">[2:10]</a></sup><sup class="footnote-ref"><a href="#fn4" id="fnref4:2">[4:2]</a></sup><sup class="footnote-ref"><a href="#fn3" id="fnref3:5">[3:5]</a></sup>.</li><li><strong>Protect cached data from dynamic allocation.</strong> If executors holding cached blocks might be reclaimed as idle, raise <code>spark.dynamicAllocation.cachedExecutorIdleTimeout</code> so they aren&#39;t pulled out from under the cache<sup class="footnote-ref"><a href="#fn6" id="fnref6:3">[6:3]</a></sup>, or use a replicated storage level (e.g., <code>MEMORY_AND_DISK_2</code>) so losing one executor doesn&#39;t lose the data<sup class="footnote-ref"><a href="#fn1" id="fnref1:11">[1:11]</a></sup>.</li><li><strong>Reach for checkpointing, not persisting, for genuine lineage truncation with fault tolerance</strong> across a long transformation chain, since it writes to a reliable external filesystem rather than relying on executor-local memory or disk<sup class="footnote-ref"><a href="#fn3" id="fnref3:6">[3:6]</a></sup><sup class="footnote-ref"><a href="#fn5" id="fnref5:2">[5:2]</a></sup>. Avoid <code>localCheckpoint()</code> under dynamic allocation for the same reason caching is unsafe there: its data lives in ephemeral local executor storage that can vanish along with a reclaimed executor<sup class="footnote-ref"><a href="#fn6" id="fnref6:4">[6:4]</a></sup>. Also set a checkpoint directory via <code>SparkContext.setCheckpointDir</code> before calling <code>.checkpoint()</code>; without one, the call throws immediately rather than silently doing nothing<sup class="footnote-ref"><a href="#fn6" id="fnref6:5">[6:5]</a></sup>.</li><li><strong>Don&#39;t cache data that won&#39;t benefit:</strong> single-use datasets, datasets that don&#39;t fit in available memory, or datasets you only ever touch through partial actions (<code>take</code>, <code>limit</code>). In each of these cases the caching overhead isn&#39;t paid back<sup class="footnote-ref"><a href="#fn7" id="fnref7:3">[7:3]</a></sup><sup class="footnote-ref"><a href="#fn2" id="fnref2:11">[2:11]</a></sup>.</li></ul><h2 id="sources" tabindex="-1">Sources <a class="header-anchor" href="#sources" aria-label="Permalink to &quot;Sources&quot;">​</a></h2><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><a href="https://luminousmen.com/post/explaining-the-mechanics-of-spark-caching" target="_blank" rel="noreferrer">Explaining the Mechanics of Spark Caching</a> <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a> <a href="#fnref1:2" class="footnote-backref">↩︎</a> <a href="#fnref1:3" class="footnote-backref">↩︎</a> <a href="#fnref1:4" class="footnote-backref">↩︎</a> <a href="#fnref1:5" class="footnote-backref">↩︎</a> <a href="#fnref1:6" class="footnote-backref">↩︎</a> <a href="#fnref1:7" class="footnote-backref">↩︎</a> <a href="#fnref1:8" class="footnote-backref">↩︎</a> <a href="#fnref1:9" class="footnote-backref">↩︎</a> <a href="#fnref1:10" class="footnote-backref">↩︎</a> <a href="#fnref1:11" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><em>Learning Spark, 2nd Edition</em>, Damji, Wenig, Das &amp; Lee, ch. 7 — Optimizing and Tuning Spark Applications <a href="#fnref2" class="footnote-backref">↩︎</a> <a href="#fnref2:1" class="footnote-backref">↩︎</a> <a href="#fnref2:2" class="footnote-backref">↩︎</a> <a href="#fnref2:3" class="footnote-backref">↩︎</a> <a href="#fnref2:4" class="footnote-backref">↩︎</a> <a href="#fnref2:5" class="footnote-backref">↩︎</a> <a href="#fnref2:6" class="footnote-backref">↩︎</a> <a href="#fnref2:7" class="footnote-backref">↩︎</a> <a href="#fnref2:8" class="footnote-backref">↩︎</a> <a href="#fnref2:9" class="footnote-backref">↩︎</a> <a href="#fnref2:10" class="footnote-backref">↩︎</a> <a href="#fnref2:11" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><em>High Performance Spark, 2nd Edition</em>, Karau, Polak &amp; Warren, ch. 7 — Effective Transformations <a href="#fnref3" class="footnote-backref">↩︎</a> <a href="#fnref3:1" class="footnote-backref">↩︎</a> <a href="#fnref3:2" class="footnote-backref">↩︎</a> <a href="#fnref3:3" class="footnote-backref">↩︎</a> <a href="#fnref3:4" class="footnote-backref">↩︎</a> <a href="#fnref3:5" class="footnote-backref">↩︎</a> <a href="#fnref3:6" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration — Spark</a> <a href="#fnref4" class="footnote-backref">↩︎</a> <a href="#fnref4:1" class="footnote-backref">↩︎</a> <a href="#fnref4:2" class="footnote-backref">↩︎</a></p></li><li id="fn5" class="footnote-item"><p><a href="https://luminousmen.com/post/spark-tips-caching" target="_blank" rel="noreferrer">Spark Tips: Caching</a> <a href="#fnref5" class="footnote-backref">↩︎</a> <a href="#fnref5:1" class="footnote-backref">↩︎</a> <a href="#fnref5:2" class="footnote-backref">↩︎</a></p></li><li id="fn6" class="footnote-item"><p><a href="https://raw.githubusercontent.com/apache/spark/v3.5.0/core/src/main/scala/org/apache/spark/rdd/RDD.scala" target="_blank" rel="noreferrer">RDD.scala</a> <a href="#fnref6" class="footnote-backref">↩︎</a> <a href="#fnref6:1" class="footnote-backref">↩︎</a> <a href="#fnref6:2" class="footnote-backref">↩︎</a> <a href="#fnref6:3" class="footnote-backref">↩︎</a> <a href="#fnref6:4" class="footnote-backref">↩︎</a> <a href="#fnref6:5" class="footnote-backref">↩︎</a></p></li><li id="fn7" class="footnote-item"><p><em>Spark: The Definitive Guide</em>, Chambers &amp; Zaharia, ch. 19 — Performance Tuning <a href="#fnref7" class="footnote-backref">↩︎</a> <a href="#fnref7:1" class="footnote-backref">↩︎</a> <a href="#fnref7:2" class="footnote-backref">↩︎</a> <a href="#fnref7:3" class="footnote-backref">↩︎</a></p></li><li id="fn8" class="footnote-item"><p><a href="https://raw.githubusercontent.com/spoddutur/spark-notes/master/task_memory_management_in_spark.md" target="_blank" rel="noreferrer">Task Memory Management in Spark</a> <a href="#fnref8" class="footnote-backref">↩︎</a></p></li><li id="fn9" class="footnote-item"><p><a href="https://luminousmen.com/post/dive-into-spark-memory" target="_blank" rel="noreferrer">Dive into Spark Memory</a> <a href="#fnref9" class="footnote-backref">↩︎</a> <a href="#fnref9:1" class="footnote-backref">↩︎</a></p></li><li id="fn10" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/rdd-programming-guide.html" target="_blank" rel="noreferrer">RDD Programming Guide</a> <a href="#fnref10" class="footnote-backref">↩︎</a></p></li></ol></section>',22)])])}const g=a(f,[["render",c]]);export{m as __pageData,g as default};

package/public/docs/assets/tuning-reference_caching.md.B7aQ8asB.lean.js

@@ -1 +0,0 @@
-import{_ as a,o,c as t,a5 as r}from"./chunks/framework.DSg0KOwT.js";const s="/docs/assets/cache-lifecycle.rEOVYQNU.svg",n="/docs/assets/cache-lifecycle.dark.B-hS7AgU.svg",m=JSON.parse('{"title":"Caching & Persistence","description":"","frontmatter":{"title":"Caching & Persistence"},"headers":[],"relativePath":"tuning-reference/caching.md","filePath":"tuning-reference/caching.md"}'),f={name:"tuning-reference/caching.md"};function c(i,e,l,h,d,p){return o(),t("div",null,[...e[0]||(e[0]=[r("",22)])])}const g=a(f,[["render",c]]);export{m as __pageData,g as default};