sparkforensics-server 0.2.0 → 0.2.3

package/public/docs/assets/tuning-reference_bottleneck-utilization.md.DqC_iGkw.js

@@ -0,0 +1,5 @@
+import{_ as t,a}from"./chunks/duplicate-plan-subtree.dark.Cdp70QhV.js";import{_ as o,o as s,c as r,a5 as i}from"./chunks/framework.DSg0KOwT.js";const k=JSON.parse('{"title":"Executor Utilization","description":"","frontmatter":{"title":"Executor Utilization"},"headers":[],"relativePath":"tuning-reference/bottleneck-utilization.md","filePath":"tuning-reference/bottleneck-utilization.md"}'),n={name:"tuning-reference/bottleneck-utilization.md"};function l(c,e,f,h,d,p){return s(),r("div",null,[...e[0]||(e[0]=[i(`<h1 id="bottleneck-utilization" tabindex="-1">Executor Utilization <a class="header-anchor" href="#bottleneck-utilization" aria-label="Permalink to &quot;Executor Utilization {#bottleneck-utilization}&quot;">​</a></h1><p><span class="tag">UTIL</span></p><h2 id="what-it-is" tabindex="-1">What it is <a class="header-anchor" href="#what-it-is" aria-label="Permalink to &quot;What it is&quot;">​</a></h2><p>Executor utilization measures how much of the cluster&#39;s allocated core-time a job actually keeps busy. When busy core-time trails the core-time available across the run, the cluster is holding compute (cores and memory) that isn&#39;t running any tasks.</p><h2 id="how-it-s-detected" tabindex="-1">How it&#39;s detected <a class="header-anchor" href="#how-it-s-detected" aria-label="Permalink to &quot;How it&#39;s detected&quot;">​</a></h2><p>The signal is the ratio of busy executor core-time to the core-time available over the run: peak concurrent cores multiplied by the app&#39;s duration.</p><table tabindex="0"><thead><tr><th>Signal</th><th>Fires when</th></tr></thead><tbody><tr><td>busy core-time / available core-time</td><td>&lt; 60%</td></tr></tbody></table><p>A finding here always reports at the info level.</p><h2 id="why-it-matters" tabindex="-1">Why it matters <a class="header-anchor" href="#why-it-matters" aria-label="Permalink to &quot;Why it matters&quot;">​</a></h2><p>Idle executors are allocation you&#39;re paying for without getting work done. One common cause is too little task parallelism to occupy the allocated cores: the guidance is to keep at least as many partitions as there are cores across the executors, so no core sits idle<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup>. Parallelism can also be lost by accident rather than by under-partitioning upfront: because <code>coalesce</code> is a narrow transformation, reducing partition count with it forces the <em>entire</em> upstream stage down to the reduced parallelism, not just the coalesce step, trading a shuffle for lost concurrency<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup>. Separately, under dynamic allocation, a workload with many small tasks can end up over-provisioned: by default it requests enough executors to maximize parallelism for the task count, and with small tasks that can mean some executors &quot;might not even do any work,&quot; wasting resources on allocation overhead<sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup>.</p><h2 id="how-to-fix-it" tabindex="-1">How to fix it <a class="header-anchor" href="#how-to-fix-it" aria-label="Permalink to &quot;How to fix it&quot;">​</a></h2><ul><li>Reach for <code>repartition</code> (not <code>coalesce</code>) when you need to raise partition count or rebalance data: <code>coalesce</code> only avoids a shuffle when shrinking partition count, and doing so also drags the entire upstream stage down to the reduced parallelism<sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup><sup class="footnote-ref"><a href="#fn2" id="fnref2:1">[2:1]</a></sup>.</li><li>Size partitions so the task count is at least the core count available across executors, to avoid leaving cores idle<sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup>.</li><li>Under dynamic allocation, lower <code>spark.dynamicAllocation.executorAllocationRatio</code> below its default of <code>1.0</code> to scale back the number of executors requested for workloads made up of many small tasks<sup class="footnote-ref"><a href="#fn3" id="fnref3:1">[3:1]</a></sup>.</li><li>Dynamic allocation requires shuffle tracking, the external shuffle service, or shuffle-block decommissioning to be enabled as a precondition<sup class="footnote-ref"><a href="#fn3" id="fnref3:2">[3:2]</a></sup>; without one of them, an executor removed mid-shuffle takes its unfetched shuffle files with it, forcing a recompute<sup class="footnote-ref"><a href="#fn5" id="fnref5">[5]</a></sup>, which undercuts using dynamic allocation to shed idle executors in the first place. <code>spark.dynamicAllocation.shuffleTracking.enabled</code> defaults to <code>true</code> since Spark 3.0, satisfying that precondition without needing a separate external shuffle service<sup class="footnote-ref"><a href="#fn3" id="fnref3:3">[3:3]</a></sup>.</li></ul><p>Restore parallelism after a heavy filter, and rein in over-provisioning for many-small-task jobs:</p><div class="language-python vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">python</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># repartition (not coalesce) rebalances and can raise partition count; aim &gt;= total executor cores</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">df </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">=</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> df.filter(heavy_predicate).repartition(spark.sparkContext.defaultParallelism)</span></span>
+<span class="line"></span>
+<span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Scale back executors requested for many-small-task workloads (default ratio 1.0)</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">spark.conf.set(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;spark.dynamicAllocation.executorAllocationRatio&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;0.5&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">)  </span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># 0.5 is an example</span></span></code></pre></div><h2 id="limitations-false-positive-risk" tabindex="-1">Limitations / false-positive risk <a class="header-anchor" href="#limitations-false-positive-risk" aria-label="Permalink to &quot;Limitations / false-positive risk&quot;">​</a></h2><p>A low average-to-peak ratio isn&#39;t always waste. A bursty or I/O-bound job legitimately holds executors while tasks wait on external systems rather than burning cores, and the ratio is sensitive to short stages, where a brief spike in allocation skews the average without meaning the cluster was genuinely idle.</p><h2 id="bottleneck-core-locality" tabindex="-1">Core locality <a class="header-anchor" href="#bottleneck-core-locality" aria-label="Permalink to &quot;Core locality {#bottleneck-core-locality}&quot;">​</a></h2><p><span class="tag">LOCAL</span></p><p>Idle cores are one half of wasted capacity; this is the other half. A core running a task without process- or node-local data placement is still busy, but that task now has to fetch its input across the network or from a different process instead of reading it in place, work the core wouldn&#39;t have to do at all if it were scheduled on data it already holds.</p><h3 id="how-it-s-detected-1" tabindex="-1">How it&#39;s detected <a class="header-anchor" href="#how-it-s-detected-1" aria-label="Permalink to &quot;How it&#39;s detected&quot;">​</a></h3><p>The share of tasks that ran RACK_LOCAL or ANY, out of every task with a recorded locality level, is the non-local task share. NO_PREF tasks stay in the denominator only, since shuffle-read stages legitimately report that level without it indicating a placement problem. This applies once a run has logged at least 50 total tasks.</p><table tabindex="0"><thead><tr><th>Signal</th><th>Warning</th><th>Critical</th></tr></thead><tbody><tr><td>Non-local task share</td><td>≥ 15%</td><td>≥ 35%</td></tr></tbody></table><h3 id="why-it-matters-1" tabindex="-1">Why it matters <a class="header-anchor" href="#why-it-matters-1" aria-label="Permalink to &quot;Why it matters&quot;">​</a></h3><p>A task denied process- or node-local placement pulls its input over the network or through inter-process I/O instead of reading it from local memory or disk, adding latency to every task that lands that way. Spread across a whole run, a high non-local share adds up to a meaningful share of total task time spent moving data that a better-placed schedule wouldn&#39;t have had to move.</p><h3 id="how-to-fix-it-1" tabindex="-1">How to fix it <a class="header-anchor" href="#how-to-fix-it-1" aria-label="Permalink to &quot;How to fix it&quot;">​</a></h3><ul><li>Check <code>spark.locality.wait</code> (default 3s) and its per-level overrides (<code>.process</code>/<code>.node</code>/<code>.rack</code>): a wait set too short gives Spark less time to find a local slot before it falls back to a less-local one.</li><li>Check executor and data colocation: if the executors are running far from where the data actually lives (a different rack, a different zone), no amount of locality-wait tuning fixes a placement that isn&#39;t available to begin with.</li></ul><h2 id="bottleneck-caching-opportunity" tabindex="-1">Caching opportunity <a class="header-anchor" href="#bottleneck-caching-opportunity" aria-label="Permalink to &quot;Caching opportunity {#bottleneck-caching-opportunity}&quot;">​</a></h2><p><span class="tag">CACHE</span></p><p>When the same DataFrame, RDD, or input is scanned more than once, low utilization can trace back to repeated recomputation rather than idle cores. Spark keeps nothing between actions: transformations only build a DAG, and once an action finishes its intermediate results are discarded<sup class="footnote-ref"><a href="#fn6" id="fnref6">[6]</a></sup>. Call a second action on the same logic and Spark re-runs the whole DAG from the source, which can mean re-reading a terabyte from S3, re-reading Kafka, or repeating expensive decompression<sup class="footnote-ref"><a href="#fn6" id="fnref6:1">[6:1]</a></sup>. Fork that logic into two pipeline branches and you sign up to recompute everything twice<sup class="footnote-ref"><a href="#fn6" id="fnref6:2">[6:2]</a></sup>.</p><h3 id="how-it-s-detected-2" tabindex="-1">How it&#39;s detected <a class="header-anchor" href="#how-it-s-detected-2" aria-label="Permalink to &quot;How it&#39;s detected&quot;">​</a></h3><p>An input relation that recurs across at least 2 SQL executions in one run is flagged by its scan format and path identity. A join or union subtree that recurs across at least 2 executions is matched by operator name, metric names, and its join or filter condition normalized to strip per-analysis ids and canonicalize commutative operand order, while columns and literal values are kept intact: two executions of the exact same join or filter still match even when Spark&#39;s internal ids differ between runs, but a genuinely different filter or join value does not. A qualifying join or union subtree is reported as one finding covering everything beneath it. Every finding here reports at the info level.</p><img class="light-only" src="`+t+'" alt="How two branches that repeat the same scan and operators each recompute it, until a shared cached or reused node lets both read one materialized result."><img class="dark-only" src="'+a+'" alt="How two branches that repeat the same scan and operators each recompute it, until a shared cached or reused node lets both read one materialized result."><p><a href="./caching.html"><code>cache()</code> and <code>persist()</code></a> are how you stop that. Persisting materializes the RDD (usually in memory on the executors) so it can be reused within the job, while Spark keeps its lineage to recompute any lost partition<sup class="footnote-ref"><a href="#fn7" id="fnref7">[7]</a></sup>. For a dataset you read repeatedly, this is one of the most useful optimizations available: it parks a DataFrame, table, or RDD in temporary storage across the executors and makes later reads fast<sup class="footnote-ref"><a href="#fn8" id="fnref8">[8]</a></sup>. Materialization is lazy and per-block, so only an action that touches every partition (<code>count()</code> or a full write) caches all of it; a subset scan like <code>take(10)</code> leaves a partial cache<sup class="footnote-ref"><a href="#fn6" id="fnref6:3">[6:3]</a></sup>.</p><p><code>cache()</code> is shorthand; <code>persist()</code> takes a <code>StorageLevel</code> and lets you pick the tradeoff:</p><table tabindex="0"><thead><tr><th>Level</th><th>Tradeoff</th></tr></thead><tbody><tr><td><code>MEMORY_ONLY</code></td><td>Fastest reads (deserialized JVM objects), but partitions that don&#39;t fit are recomputed on the fly each time<sup class="footnote-ref"><a href="#fn8" id="fnref8:1">[8:1]</a></sup>.</td></tr><tr><td><code>MEMORY_AND_DISK</code></td><td>Spills the overflow to disk instead of recomputing; Spark&#39;s default caching strategy and fine for most pipelines<sup class="footnote-ref"><a href="#fn9" id="fnref9">[9]</a></sup>. For DataFrames, <code>.cache()</code> maps to this<sup class="footnote-ref"><a href="#fn6" id="fnref6:4">[6:4]</a></sup>.</td></tr><tr><td>Serialized (<code>_SER</code>)</td><td>Byte arrays instead of raw objects: smaller footprint, slower reads. Reach for it when memory is tight<sup class="footnote-ref"><a href="#fn9" id="fnref9:1">[9:1]</a></sup>.</td></tr><tr><td>Replicated (<code>_2</code>)</td><td>A second copy for fast fault recovery instead of waiting on recomputation, at twice the space<sup class="footnote-ref"><a href="#fn10" id="fnref10">[10]</a></sup><sup class="footnote-ref"><a href="#fn9" id="fnref9:2">[9:2]</a></sup>.</td></tr></tbody></table><p>Whether to spill to disk hinges on recomputation cost: reading a block back from disk only beats recomputing it when the work that produced the data is expensive or filters out a large fraction, so the RDD guide says don&#39;t enable disk otherwise<sup class="footnote-ref"><a href="#fn10" id="fnref10:1">[10:1]</a></sup>.</p><p>Caching is not free, and several cases don&#39;t warrant it:</p><ul><li><strong>Single use.</strong> Caching adds serialization, deserialization, and storage cost, so caching data you read once only slows you down<sup class="footnote-ref"><a href="#fn8" id="fnref8:2">[8:2]</a></sup>.</li><li><strong>Data larger than storage memory,</strong> or a cheap transformation that isn&#39;t reused often regardless of size<sup class="footnote-ref"><a href="#fn11" id="fnref11">[11]</a></sup>.</li><li><strong>Memory pressure.</strong> Cache memory is memory taken from processing, and under the default spill strategy cached data can land on slower disk, so caching can cost more than just re-reading the source<sup class="footnote-ref"><a href="#fn9" id="fnref9:3">[9:3]</a></sup>.</li><li><strong>Lost optimizer freedom.</strong> Once a dataset is cached, Catalyst works on the in-memory copy and can no longer push filters down to the source<sup class="footnote-ref"><a href="#fn9" id="fnref9:4">[9:4]</a></sup>.</li></ul><p>A persisted RDD you&#39;ve stopped using still occupies memory until the app ends or eviction forces it out, so call <code>unpersist()</code> to reclaim it deliberately, which matters most on shared clusters<sup class="footnote-ref"><a href="#fn9" id="fnref9:5">[9:5]</a></sup>.</p><h2 id="related" tabindex="-1">Related <a class="header-anchor" href="#related" aria-label="Permalink to &quot;Related&quot;">​</a></h2><ul><li><strong>Task parallelism:</strong> <a href="./partitioning.html">Partitioning</a></li><li><strong>Executor sizing:</strong> <a href="./cluster-config.html">Cluster Tuning</a></li></ul><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><em>Learning Spark, 2nd Edition</em>, Damji, Wenig, Das &amp; Lee, ch. 7 <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><em>High Performance Spark, 2nd Edition</em>, Karau, Polak &amp; Warren, ch. 8 <a href="#fnref2" class="footnote-backref">↩︎</a> <a href="#fnref2:1" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration — Spark</a> <a href="#fnref3" class="footnote-backref">↩︎</a> <a href="#fnref3:1" class="footnote-backref">↩︎</a> <a href="#fnref3:2" class="footnote-backref">↩︎</a> <a href="#fnref3:3" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><em>Spark: The Definitive Guide</em>, Chambers &amp; Zaharia, ch. 19 <a href="#fnref4" class="footnote-backref">↩︎</a></p></li><li id="fn5" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/job-scheduling.html" target="_blank" rel="noreferrer">Job Scheduling — Dynamic Resource Allocation</a> <a href="#fnref5" class="footnote-backref">↩︎</a></p></li><li id="fn6" class="footnote-item"><p><a href="https://luminousmen.com/post/explaining-the-mechanics-of-spark-caching" target="_blank" rel="noreferrer">Explaining the Mechanics of Spark Caching</a> <a href="#fnref6" class="footnote-backref">↩︎</a> <a href="#fnref6:1" class="footnote-backref">↩︎</a> <a href="#fnref6:2" class="footnote-backref">↩︎</a> <a href="#fnref6:3" class="footnote-backref">↩︎</a> <a href="#fnref6:4" class="footnote-backref">↩︎</a></p></li><li id="fn7" class="footnote-item"><p><em>High Performance Spark, 2nd Edition</em>, Karau, Polak &amp; Warren, ch. 2 <a href="#fnref7" class="footnote-backref">↩︎</a></p></li><li id="fn8" class="footnote-item"><p><em>Spark: The Definitive Guide</em>, Chambers &amp; Zaharia, ch. 19 <a href="#fnref8" class="footnote-backref">↩︎</a> <a href="#fnref8:1" class="footnote-backref">↩︎</a> <a href="#fnref8:2" class="footnote-backref">↩︎</a></p></li><li id="fn9" class="footnote-item"><p><a href="https://luminousmen.com/post/spark-tips-caching" target="_blank" rel="noreferrer">Spark Tips: Caching</a> <a href="#fnref9" class="footnote-backref">↩︎</a> <a href="#fnref9:1" class="footnote-backref">↩︎</a> <a href="#fnref9:2" class="footnote-backref">↩︎</a> <a href="#fnref9:3" class="footnote-backref">↩︎</a> <a href="#fnref9:4" class="footnote-backref">↩︎</a> <a href="#fnref9:5" class="footnote-backref">↩︎</a></p></li><li id="fn10" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/rdd-programming-guide.html" target="_blank" rel="noreferrer">RDD Programming Guide — Spark</a> <a href="#fnref10" class="footnote-backref">↩︎</a> <a href="#fnref10:1" class="footnote-backref">↩︎</a></p></li><li id="fn11" class="footnote-item"><p><em>Learning Spark, 2nd Edition</em>, Damji, Wenig, Das &amp; Lee, ch. 7 <a href="#fnref11" class="footnote-backref">↩︎</a></p></li></ol></section>',44)])])}const g=o(n,[["render",l]]);export{k as __pageData,g as default};

package/public/docs/assets/tuning-reference_bottleneck-utilization.md.DqC_iGkw.lean.js

@@ -0,0 +1 @@
+import{_ as t,a}from"./chunks/duplicate-plan-subtree.dark.Cdp70QhV.js";import{_ as o,o as s,c as r,a5 as i}from"./chunks/framework.DSg0KOwT.js";const k=JSON.parse('{"title":"Executor Utilization","description":"","frontmatter":{"title":"Executor Utilization"},"headers":[],"relativePath":"tuning-reference/bottleneck-utilization.md","filePath":"tuning-reference/bottleneck-utilization.md"}'),n={name:"tuning-reference/bottleneck-utilization.md"};function l(c,e,f,h,d,p){return s(),r("div",null,[...e[0]||(e[0]=[i("",44)])])}const g=o(n,[["render",l]]);export{k as __pageData,g as default};

package/public/docs/assets/{tuning-reference_cluster-config.md.ZVmDGsQ3.js → tuning-reference_cluster-config.md.CD6vzYtD.js}

@@ -1 +1 @@
-import{_ as o,o as a,c as t,a5 as r}from"./chunks/framework.DSg0KOwT.js";const s="/docs/assets/container-memory.DIO0AnIm.svg",n="/docs/assets/container-memory.dark.CP-5zuCl.svg",m=JSON.parse('{"title":"Cluster Tuning","description":"","frontmatter":{"title":"Cluster Tuning"},"headers":[],"relativePath":"tuning-reference/cluster-config.md","filePath":"tuning-reference/cluster-config.md"}'),c={name:"tuning-reference/cluster-config.md"};function f(i,e,l,u,d,h){return a(),t("div",null,[...e[0]||(e[0]=[r('<h1 id="cluster-config" tabindex="-1">Cluster Tuning <a class="header-anchor" href="#cluster-config" aria-label="Permalink to &quot;Cluster Tuning {#cluster-config}&quot;">​</a></h1><h2 id="sizing-executors-and-containers" tabindex="-1">Sizing executors and containers <a class="header-anchor" href="#sizing-executors-and-containers" aria-label="Permalink to &quot;Sizing executors and containers&quot;">​</a></h2><p>Cluster tuning is the set of decisions that map an application&#39;s cores, memory, and executor count onto the underlying cluster (YARN or Kubernetes) plus a handful of related knobs (data locality, dynamic allocation, per-task CPU reservation) that all interact with that sizing decision.</p><p>The starting point is executor sizing. Working through a concrete example (a 10-node cluster with 16 cores and 64GB RAM per node), the derivation runs: assign 5 cores per executor; reserve about 1 core per node for Hadoop/YARN/OS daemons, leaving 15 usable cores per node (150 total); dividing by 5 cores per executor gives 30 executors, minus 1 reserved for the YARN ApplicationMaster (29); with 3 executors per node, memory per executor is 64GB / 3 ≈ 21GB, and after subtracting about 7% for YARN memory overhead that comes out to roughly 18GB usable, a recommended shape of 29 executors × 5 cores × 18GB<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup>. Cloudera&#39;s own worked example, on a 6-node cluster with the same 16-core / 64GB-per-node hardware, lands on the same shape: <code>--num-executors 17 --executor-cores 5 --executor-memory 19G</code>, rather than one &quot;fat&quot; executor per node (<code>--num-executors 6 --executor-cores 15 --executor-memory 63G</code>)<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup>. Both sources treat the resulting task count (executor-cores × num-executors) as the single most important tuning lever, since Spark cannot compensate for too little parallelism on its own<sup class="footnote-ref"><a href="#fn2" id="fnref2:1">[2:1]</a></sup>. <code>spark.executor.cores</code> is the config that controls concurrent tasks per executor; it defaults to 1 in YARN mode, or to all available cores on the worker in standalone mode<sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup>, and <code>spark.executor.memory</code> sets the JVM heap size per executor<sup class="footnote-ref"><a href="#fn2" id="fnref2:2">[2:2]</a></sup>. Independent of the sizing formula above, the Spark tuning guide recommends targeting 2–3 tasks per CPU core across the cluster<sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup>.</p><p>On top of executor memory sits container memory. Spark computes the total container/pod allocation as <code>executorMemoryMiB + memoryOverheadMiB + memoryOffHeapMiB + pysparkMemToUseMiB</code>. On YARN this is what gets allocated for the container, on Kubernetes it becomes the pod memory limit<sup class="footnote-ref"><a href="#fn5" id="fnref5">[5]</a></sup>. <code>spark.executor.memoryOverhead</code> itself defaults to <code>max(0.1 * executorMemory, 384MB)</code><sup class="footnote-ref"><a href="#fn5" id="fnref5:1">[5:1]</a></sup><sup class="footnote-ref"><a href="#fn3" id="fnref3:1">[3:1]</a></sup>.</p><img class="light-only" src="'+s+'" alt="Container memory sums executor heap, memory overhead, off-heap size, and pyspark memory into the requested container size; the resource manager grants a container of that size and OOMKills the executor when its actual runtime footprint spills past that limit."><img class="dark-only" src="'+n+'" alt="Container memory sums executor heap, memory overhead, off-heap size, and pyspark memory into the requested container size; the resource manager grants a container of that size and OOMKills the executor when its actual runtime footprint spills past that limit."><blockquote><p><strong>PySpark:</strong> <code>spark.executor.pyspark.memory</code> is only added as its own container-memory term when it&#39;s set explicitly; otherwise PySpark&#39;s memory use is folded into the general overhead budget rather than tracked separately.<sup class="footnote-ref"><a href="#fn5" id="fnref5:2">[5:2]</a></sup></p></blockquote><p>On Kubernetes specifically, <code>spark.kubernetes.executor.request.cores</code> and <code>spark.kubernetes.executor.limit.cores</code> take priority over <code>spark.executor.cores</code> for the pod&#39;s CPU request/limit sent to the Kubernetes scheduler<sup class="footnote-ref"><a href="#fn6" id="fnref6">[6]</a></sup>, while <code>spark.executor.cores</code> remains the config Spark itself uses to size the number of concurrent task slots<sup class="footnote-ref"><a href="#fn3" id="fnref3:2">[3:2]</a></sup>.</p><p>Two more knobs round out the picture. <code>spark.locality.wait</code> (default <code>3s</code>) controls how long a task waits for a data-local placement before Spark gives up and schedules it less locally, stepping through the same wait across process-local → node-local → rack-local → any; each level can be overridden independently via <code>spark.locality.wait.process</code>, <code>.node</code>, and <code>.rack</code>, all of which default to the base wait when left unset<sup class="footnote-ref"><a href="#fn3" id="fnref3:3">[3:3]</a></sup>. Dynamic allocation lets Spark add and remove executors as work changes, but it requires one of several supporting mechanisms: an external shuffle service, shuffle tracking (<code>spark.dynamicAllocation.shuffleTracking.enabled</code>, default <code>true</code> since Spark 3.0), shuffle-block decommission, or the experimental sort-IO plugin<sup class="footnote-ref"><a href="#fn3" id="fnref3:4">[3:4]</a></sup>. Removing an executor can otherwise destroy shuffle state it&#39;s holding. Finally, <code>spark.task.cpus</code> (default <code>1</code>) sets how many cores each task reserves; the number of concurrent task slots per executor is derived jointly from it and <code>spark.executor.cores</code>, roughly <code>spark.executor.cores / spark.task.cpus</code><sup class="footnote-ref"><a href="#fn3" id="fnref3:5">[3:5]</a></sup>.</p><h2 id="spotting-a-bad-sizing-decision" tabindex="-1">Spotting a bad sizing decision <a class="header-anchor" href="#spotting-a-bad-sizing-decision" aria-label="Permalink to &quot;Spotting a bad sizing decision&quot;">​</a></h2><p>Getting that sizing wrong shows up in a handful of specific symptoms. The clearest sign of an under-sized executor layout is HDFS throughput dropping under load: &quot;HDFS client has trouble with tons of concurrent threads. It was observed that HDFS achieves full write throughput with ~5 tasks per executor&quot;<sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup>. The &quot;fat executor&quot; case (one executor per node, using all 16 cores) shows the failure mode directly: &quot;with all 16 cores per executor... HDFS throughput will hurt and it&#39;ll result in excessive garbage [collection]&quot;<sup class="footnote-ref"><a href="#fn1" id="fnref1:2">[1:2]</a></sup>.</p><p>On the memory side, an executor or pod that dies with no Spark-level error is a strong signal that off-heap memory was never added into the container budget. Enabling <code>spark.memory.offHeap.enabled=true</code> with <code>spark.memory.offHeap.size=1g</code> on top of an 8G executor with default overhead (819MB) means real usage is 8192+819+1024 = 10,035MB, while the container was only granted 8192+819 = 9,011MB. The executor gets killed by YARN, or OOMKilled by the Kubernetes kubelet, with no warning from Spark itself<sup class="footnote-ref"><a href="#fn5" id="fnref5:3">[5:3]</a></sup>.</p><p>With dynamic allocation on, unexplained shuffle recomputation is a detectable symptom of executors being reclaimed mid-shuffle: &quot;In the event of stragglers... dynamic allocation may remove an executor before the shuffle completes, in which case the shuffle files written by that executor must be recomputed unnecessarily&quot;<sup class="footnote-ref"><a href="#fn6" id="fnref6:1">[6:1]</a></sup>. Jobs that abort with a serialized-result-size error are hitting the <code>spark.driver.maxResultSize</code> guardrail rather than a driver heap exhaustion: &quot;Jobs will be aborted if the total size [of serialized action results] is above this limit&quot;<sup class="footnote-ref"><a href="#fn3" id="fnref3:6">[3:6]</a></sup>. And on the resource-waste side, executors that are provisioned but never assigned work are a sign that dynamic allocation is targeting full parallelism against a workload made of many small tasks<sup class="footnote-ref"><a href="#fn3" id="fnref3:7">[3:7]</a></sup>.</p><h2 id="what-a-bad-sizing-decision-costs" tabindex="-1">What a bad sizing decision costs <a class="header-anchor" href="#what-a-bad-sizing-decision-costs" aria-label="Permalink to &quot;What a bad sizing decision costs&quot;">​</a></h2><p>Each symptom above carries a specific price tag. Task count (<code>executor-cores × num-executors</code>) is treated by both cited sources as the single most important tuning lever, since Spark cannot compensate for too little parallelism on its own<sup class="footnote-ref"><a href="#fn2" id="fnref2:3">[2:3]</a></sup>. Going the other way, cramming too many cores into one executor degrades HDFS throughput and drives up garbage collection<sup class="footnote-ref"><a href="#fn1" id="fnref1:3">[1:3]</a></sup>.</p><p>Container-memory misconfiguration matters because the failure is silent from Spark&#39;s point of view: off-heap memory that isn&#39;t accounted for in <code>spark.executor.memoryOverhead</code> causes the executor to actually use more memory than the container/pod was granted, so it gets killed externally (by YARN or the kubelet) with no Spark-level diagnostic to point at the real cause<sup class="footnote-ref"><a href="#fn5" id="fnref5:4">[5:4]</a></sup>.</p><p>Dynamic allocation&#39;s interaction with shuffle state matters because, before dynamic allocation existed, an executor exiting alongside its application meant all its state could be safely discarded; with dynamic allocation, the application keeps running after an executor is explicitly removed, so any later need for that executor&#39;s state forces a recompute<sup class="footnote-ref"><a href="#fn6" id="fnref6:2">[6:2]</a></sup>. That is exactly why Spark needs &quot;a mechanism to decommission an executor gracefully by preserving its state before removing it&quot;<sup class="footnote-ref"><a href="#fn6" id="fnref6:3">[6:3]</a></sup>. Without shuffle tracking, an external shuffle service, or shuffle-block decommission enabled, dynamic allocation either can&#39;t be turned on at all, or, if it&#39;s active regardless, an executor holding unpreserved shuffle output that gets removed forces exactly that unnecessary recompute<sup class="footnote-ref"><a href="#fn3" id="fnref3:8">[3:8]</a></sup><sup class="footnote-ref"><a href="#fn6" id="fnref6:4">[6:4]</a></sup>.</p><p>On the driver side, <code>spark.driver.maxResultSize</code> and <code>spark.driver.memory</code> are separate budgets that still interact: whether a high <code>maxResultSize</code> actually causes an out-of-memory error &quot;depends on spark.driver.memory and memory overhead of objects in JVM&quot;<sup class="footnote-ref"><a href="#fn3" id="fnref3:9">[3:9]</a></sup>. Raising one without considering the other doesn&#39;t fully protect the driver.</p><p>Finally, over-provisioning executors against a small-task workload wastes cluster resources: &quot;with small tasks this setting can waste a lot of resources due to executor allocation overhead, as some executor might not even do any work&quot;<sup class="footnote-ref"><a href="#fn3" id="fnref3:10">[3:10]</a></sup>.</p><h2 id="sizing-the-cluster-correctly" tabindex="-1">Sizing the cluster correctly <a class="header-anchor" href="#sizing-the-cluster-correctly" aria-label="Permalink to &quot;Sizing the cluster correctly&quot;">​</a></h2><p>Avoiding those costs starts from a formula, not one fat executor per node. Apply the balanced-executor formula instead; the worked examples above give shapes like 29 × 5 × 18GB for a 10-node/16-core/64GB cluster, or 17 × 5 × 19G for a 6-node cluster with the same per-node hardware<sup class="footnote-ref"><a href="#fn1" id="fnref1:4">[1:4]</a></sup><sup class="footnote-ref"><a href="#fn2" id="fnref2:4">[2:4]</a></sup>. Keep <code>--executor-cores</code> at or below roughly 5 so HDFS client concurrency stays in the range where it sustains full write throughput<sup class="footnote-ref"><a href="#fn1" id="fnref1:5">[1:5]</a></sup>.</p><p>When enabling off-heap memory, account for <code>spark.memory.offHeap.size</code> in the container/pod memory budget explicitly; <code>spark.executor.memoryOverhead</code>&#39;s default of <code>max(0.1 * executorMemory, 384MB)</code> does not include it<sup class="footnote-ref"><a href="#fn5" id="fnref5:5">[5:5]</a></sup>. If you&#39;re still referencing the older <code>spark.yarn.executor.memoryOverhead</code> name, note it was removed in Spark 3.0<sup class="footnote-ref"><a href="#fn5" id="fnref5:6">[5:6]</a></sup>.</p><p>On Kubernetes, remember that <code>spark.kubernetes.executor.request.cores</code> and <code>.limit.cores</code> govern the pod&#39;s CPU request/limit and take priority over <code>spark.executor.cores</code> for that purpose<sup class="footnote-ref"><a href="#fn6" id="fnref6:5">[6:5]</a></sup>, while <code>spark.executor.cores</code> separately governs Spark&#39;s own task-slot count<sup class="footnote-ref"><a href="#fn3" id="fnref3:11">[3:11]</a></sup>. The two need to be reasoned about together rather than assumed to be redundant.</p><p>Tune <code>spark.locality.wait</code> (and its per-level overrides) upward when tasks are long-running and locality is poor, since the default is tuned for typical workloads<sup class="footnote-ref"><a href="#fn3" id="fnref3:12">[3:12]</a></sup>; set <code>spark.locality.wait.node</code> to <code>0</code> to skip straight to rack locality when node locality isn&#39;t achievable<sup class="footnote-ref"><a href="#fn3" id="fnref3:13">[3:13]</a></sup>.</p><p>Turn on shuffle tracking (<code>spark.dynamicAllocation.shuffleTracking.enabled</code>, default <code>true</code> since Spark 3.0) or the external shuffle service so dynamic allocation can reclaim executors without losing shuffle output or forcing recomputation<sup class="footnote-ref"><a href="#fn3" id="fnref3:14">[3:14]</a></sup><sup class="footnote-ref"><a href="#fn6" id="fnref6:6">[6:6]</a></sup>. The external shuffle service additionally lets persisted RDD blocks survive executor removal when <code>spark.shuffle.service.fetch.rdd.enabled</code> is set, and executors holding cached blocks are, by default, never removed at all, tunable via <code>spark.dynamicAllocation.cachedExecutorIdleTimeout</code><sup class="footnote-ref"><a href="#fn6" id="fnref6:7">[6:7]</a></sup>.</p><p>Set <code>spark.driver.maxResultSize</code> as a guardrail on serialized action-result size, and size <code>spark.driver.memory</code> with it in mind rather than in isolation<sup class="footnote-ref"><a href="#fn3" id="fnref3:15">[3:15]</a></sup>. Use <code>spark.dynamicAllocation.executorAllocationRatio</code> (default <code>1.0</code>, added in 2.4.0) to scale down over-allocation when tasks are small; for example, a value of <code>0.5</code> halves the target executor count dynamic allocation would otherwise compute<sup class="footnote-ref"><a href="#fn3" id="fnref3:16">[3:16]</a></sup>. Raise <code>spark.task.cpus</code> above its default of <code>1</code> when a task needs more than one core; doing so proportionally reduces concurrent task slots per executor (<code>spark.executor.cores / spark.task.cpus</code>)<sup class="footnote-ref"><a href="#fn3" id="fnref3:17">[3:17]</a></sup>.</p><h2 id="sources" tabindex="-1">Sources <a class="header-anchor" href="#sources" aria-label="Permalink to &quot;Sources&quot;">​</a></h2><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><a href="https://raw.githubusercontent.com/spoddutur/spark-notes/master/distribution_of_executors_cores_and_memory_for_spark_application.md" target="_blank" rel="noreferrer">Distribution of Executors, Cores and Memory for a Spark Application</a> <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a> <a href="#fnref1:2" class="footnote-backref">↩︎</a> <a href="#fnref1:3" class="footnote-backref">↩︎</a> <a href="#fnref1:4" class="footnote-backref">↩︎</a> <a href="#fnref1:5" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><a href="https://blog.cloudera.com/how-to-tune-your-apache-spark-jobs-part-2/" target="_blank" rel="noreferrer">How to Tune Your Apache Spark Jobs (Part 2)</a> <a href="#fnref2" class="footnote-backref">↩︎</a> <a href="#fnref2:1" class="footnote-backref">↩︎</a> <a href="#fnref2:2" class="footnote-backref">↩︎</a> <a href="#fnref2:3" class="footnote-backref">↩︎</a> <a href="#fnref2:4" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration — Spark</a> <a href="#fnref3" class="footnote-backref">↩︎</a> <a href="#fnref3:1" class="footnote-backref">↩︎</a> <a href="#fnref3:2" class="footnote-backref">↩︎</a> <a href="#fnref3:3" class="footnote-backref">↩︎</a> <a href="#fnref3:4" class="footnote-backref">↩︎</a> <a href="#fnref3:5" class="footnote-backref">↩︎</a> <a href="#fnref3:6" class="footnote-backref">↩︎</a> <a href="#fnref3:7" class="footnote-backref">↩︎</a> <a href="#fnref3:8" class="footnote-backref">↩︎</a> <a href="#fnref3:9" class="footnote-backref">↩︎</a> <a href="#fnref3:10" class="footnote-backref">↩︎</a> <a href="#fnref3:11" class="footnote-backref">↩︎</a> <a href="#fnref3:12" class="footnote-backref">↩︎</a> <a href="#fnref3:13" class="footnote-backref">↩︎</a> <a href="#fnref3:14" class="footnote-backref">↩︎</a> <a href="#fnref3:15" class="footnote-backref">↩︎</a> <a href="#fnref3:16" class="footnote-backref">↩︎</a> <a href="#fnref3:17" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/tuning.html" target="_blank" rel="noreferrer">Spark Tuning Guide</a> <a href="#fnref4" class="footnote-backref">↩︎</a></p></li><li id="fn5" class="footnote-item"><p><a href="https://luminousmen.com/post/dive-into-spark-memory" target="_blank" rel="noreferrer">Dive into Spark Memory</a> <a href="#fnref5" class="footnote-backref">↩︎</a> <a href="#fnref5:1" class="footnote-backref">↩︎</a> <a href="#fnref5:2" class="footnote-backref">↩︎</a> <a href="#fnref5:3" class="footnote-backref">↩︎</a> <a href="#fnref5:4" class="footnote-backref">↩︎</a> <a href="#fnref5:5" class="footnote-backref">↩︎</a> <a href="#fnref5:6" class="footnote-backref">↩︎</a></p></li><li id="fn6" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/job-scheduling.html" target="_blank" rel="noreferrer">Job Scheduling — Dynamic Resource Allocation</a> <a href="#fnref6" class="footnote-backref">↩︎</a> <a href="#fnref6:1" class="footnote-backref">↩︎</a> <a href="#fnref6:2" class="footnote-backref">↩︎</a> <a href="#fnref6:3" class="footnote-backref">↩︎</a> <a href="#fnref6:4" class="footnote-backref">↩︎</a> <a href="#fnref6:5" class="footnote-backref">↩︎</a> <a href="#fnref6:6" class="footnote-backref">↩︎</a> <a href="#fnref6:7" class="footnote-backref">↩︎</a></p></li></ol></section>',30)])])}const k=o(c,[["render",f]]);export{m as __pageData,k as default};
+import{_ as o,o as t,c as a,a5 as r}from"./chunks/framework.DSg0KOwT.js";const s="/docs/assets/container-memory.DIO0AnIm.svg",n="/docs/assets/container-memory.dark.CP-5zuCl.svg",m=JSON.parse('{"title":"Cluster Tuning","description":"","frontmatter":{"title":"Cluster Tuning"},"headers":[],"relativePath":"tuning-reference/cluster-config.md","filePath":"tuning-reference/cluster-config.md"}'),c={name:"tuning-reference/cluster-config.md"};function i(f,e,l,d,u,h){return t(),a("div",null,[...e[0]||(e[0]=[r('<h1 id="cluster-config" tabindex="-1">Cluster Tuning <a class="header-anchor" href="#cluster-config" aria-label="Permalink to &quot;Cluster Tuning {#cluster-config}&quot;">​</a></h1><h2 id="sizing-executors-and-containers" tabindex="-1">Sizing executors and containers <a class="header-anchor" href="#sizing-executors-and-containers" aria-label="Permalink to &quot;Sizing executors and containers&quot;">​</a></h2><p>Cluster tuning is the set of decisions that map an application&#39;s cores, memory, and executor count onto the underlying cluster (YARN or Kubernetes) plus a handful of related knobs (data locality, dynamic allocation, per-task CPU reservation) that all interact with that sizing decision.</p><p>The starting point is executor sizing. Working through a concrete example (a 10-node cluster with 16 cores and 64GB RAM per node), the derivation runs: assign 5 cores per executor; reserve about 1 core per node for Hadoop/YARN/OS daemons, leaving 15 usable cores per node (150 total); dividing by 5 cores per executor gives 30 executors, minus 1 reserved for the YARN ApplicationMaster (29); with 3 executors per node, memory per executor is 64GB / 3 ≈ 21GB, and after subtracting about 7% for YARN memory overhead that comes out to roughly 18GB usable, a recommended shape of 29 executors × 5 cores × 18GB<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup>. Cloudera&#39;s own worked example, on a 6-node cluster with the same 16-core / 64GB-per-node hardware, lands on the same shape: <code>--num-executors 17 --executor-cores 5 --executor-memory 19G</code>, rather than one &quot;fat&quot; executor per node (<code>--num-executors 6 --executor-cores 15 --executor-memory 63G</code>)<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup>. Both sources treat the resulting task count (executor-cores × num-executors) as the single most important tuning lever, since Spark cannot compensate for too little parallelism on its own<sup class="footnote-ref"><a href="#fn2" id="fnref2:1">[2:1]</a></sup>. <code>spark.executor.cores</code> is the config that controls concurrent tasks per executor; it defaults to 1 in YARN mode, or to all available cores on the worker in standalone mode<sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup>, and <code>spark.executor.memory</code> sets the JVM heap size per executor<sup class="footnote-ref"><a href="#fn2" id="fnref2:2">[2:2]</a></sup>. Independent of the sizing formula above, the Spark tuning guide recommends targeting 2–3 tasks per CPU core across the cluster<sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup>.</p><p>On top of executor memory sits container memory. Spark computes the total container/pod allocation as <code>executorMemoryMiB + memoryOverheadMiB + memoryOffHeapMiB + pysparkMemToUseMiB</code>. On YARN this is what gets allocated for the container, on Kubernetes it becomes the pod memory limit<sup class="footnote-ref"><a href="#fn5" id="fnref5">[5]</a></sup>. <code>spark.executor.memoryOverhead</code> itself defaults to <code>max(0.1 * executorMemory, 384MB)</code><sup class="footnote-ref"><a href="#fn5" id="fnref5:1">[5:1]</a></sup><sup class="footnote-ref"><a href="#fn3" id="fnref3:1">[3:1]</a></sup>.</p><img class="light-only" src="'+s+'" alt="Container memory sums executor heap, memory overhead, off-heap size, and pyspark memory into the requested container size; the resource manager grants a container of that size and OOMKills the executor when its actual runtime footprint spills past that limit."><img class="dark-only" src="'+n+'" alt="Container memory sums executor heap, memory overhead, off-heap size, and pyspark memory into the requested container size; the resource manager grants a container of that size and OOMKills the executor when its actual runtime footprint spills past that limit."><blockquote><p><strong>PySpark:</strong> <code>spark.executor.pyspark.memory</code> is only added as its own container-memory term when it&#39;s set explicitly; otherwise PySpark&#39;s memory use is folded into the general overhead budget rather than tracked separately.<sup class="footnote-ref"><a href="#fn5" id="fnref5:2">[5:2]</a></sup></p></blockquote><p>On Kubernetes specifically, <code>spark.kubernetes.executor.request.cores</code> and <code>spark.kubernetes.executor.limit.cores</code> take priority over <code>spark.executor.cores</code> for the pod&#39;s CPU request/limit sent to the Kubernetes scheduler<sup class="footnote-ref"><a href="#fn6" id="fnref6">[6]</a></sup>, while <code>spark.executor.cores</code> remains the config Spark itself uses to size the number of concurrent task slots<sup class="footnote-ref"><a href="#fn3" id="fnref3:2">[3:2]</a></sup>.</p><p>Two more knobs round out the picture. <code>spark.locality.wait</code> (default <code>3s</code>) controls how long a task waits for a data-local placement before Spark gives up and schedules it less locally, stepping through the same wait across process-local → node-local → rack-local → any; each level can be overridden independently via <code>spark.locality.wait.process</code>, <code>.node</code>, and <code>.rack</code>, all of which default to the base wait when left unset<sup class="footnote-ref"><a href="#fn3" id="fnref3:3">[3:3]</a></sup>. Dynamic allocation lets Spark add and remove executors as work changes, but it requires one of several supporting mechanisms: an external shuffle service, shuffle tracking (<code>spark.dynamicAllocation.shuffleTracking.enabled</code>, default <code>true</code> since Spark 3.0), shuffle-block decommission, or the experimental sort-IO plugin<sup class="footnote-ref"><a href="#fn3" id="fnref3:4">[3:4]</a></sup>. Removing an executor can otherwise destroy shuffle state it&#39;s holding. Finally, <code>spark.task.cpus</code> (default <code>1</code>) sets how many cores each task reserves; the number of concurrent task slots per executor is derived jointly from it and <code>spark.executor.cores</code>, roughly <code>spark.executor.cores / spark.task.cpus</code><sup class="footnote-ref"><a href="#fn3" id="fnref3:5">[3:5]</a></sup>.</p><h2 id="spotting-a-bad-sizing-decision" tabindex="-1">Spotting a bad sizing decision <a class="header-anchor" href="#spotting-a-bad-sizing-decision" aria-label="Permalink to &quot;Spotting a bad sizing decision&quot;">​</a></h2><p>Getting that sizing wrong shows up in a handful of specific symptoms. The clearest sign of an under-sized executor layout is HDFS throughput dropping under load: &quot;HDFS client has trouble with tons of concurrent threads. It was observed that HDFS achieves full write throughput with ~5 tasks per executor&quot;<sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup>. The &quot;fat executor&quot; case (one executor per node, using all 16 cores) shows the failure mode directly: &quot;with all 16 cores per executor... HDFS throughput will hurt and it&#39;ll result in excessive garbage [collection]&quot;<sup class="footnote-ref"><a href="#fn1" id="fnref1:2">[1:2]</a></sup>.</p><p>On the memory side, an executor or pod that dies with no Spark-level error is a strong signal that off-heap memory was never added into the container budget. Enabling <code>spark.memory.offHeap.enabled=true</code> with <code>spark.memory.offHeap.size=1g</code> on top of an 8G executor with default overhead (819MB) means real usage is 8192+819+1024 = 10,035MB, while the container was only granted 8192+819 = 9,011MB. The executor gets killed by YARN, or OOMKilled by the Kubernetes kubelet, with no warning from Spark itself<sup class="footnote-ref"><a href="#fn5" id="fnref5:3">[5:3]</a></sup>.</p><p>With dynamic allocation on, unexplained shuffle recomputation is a detectable symptom of executors being reclaimed mid-shuffle: &quot;In the event of stragglers... dynamic allocation may remove an executor before the shuffle completes, in which case the shuffle files written by that executor must be recomputed unnecessarily&quot;<sup class="footnote-ref"><a href="#fn6" id="fnref6:1">[6:1]</a></sup>. Jobs that abort with a serialized-result-size error are hitting the <code>spark.driver.maxResultSize</code> guardrail rather than a driver heap exhaustion: &quot;Jobs will be aborted if the total size [of serialized action results] is above this limit&quot;<sup class="footnote-ref"><a href="#fn3" id="fnref3:6">[3:6]</a></sup>. And on the resource-waste side, executors that are provisioned but never assigned work are a sign that dynamic allocation is targeting full parallelism against a workload made of many small tasks<sup class="footnote-ref"><a href="#fn3" id="fnref3:7">[3:7]</a></sup>.</p><h2 id="what-a-bad-sizing-decision-costs" tabindex="-1">What a bad sizing decision costs <a class="header-anchor" href="#what-a-bad-sizing-decision-costs" aria-label="Permalink to &quot;What a bad sizing decision costs&quot;">​</a></h2><p>Each symptom above carries a specific price tag. Task count (<code>executor-cores × num-executors</code>) is treated by both cited sources as the single most important tuning lever, since Spark cannot compensate for too little parallelism on its own<sup class="footnote-ref"><a href="#fn2" id="fnref2:3">[2:3]</a></sup>. Going the other way, cramming too many cores into one executor degrades HDFS throughput and drives up garbage collection<sup class="footnote-ref"><a href="#fn1" id="fnref1:3">[1:3]</a></sup>.</p><p>Container-memory misconfiguration matters because the failure is silent from Spark&#39;s point of view: off-heap memory that isn&#39;t accounted for in <code>spark.executor.memoryOverhead</code> causes the executor to actually use more memory than the container/pod was granted, so it gets killed externally (by YARN or the kubelet) with no Spark-level diagnostic to point at the real cause<sup class="footnote-ref"><a href="#fn5" id="fnref5:4">[5:4]</a></sup>.</p><p>Dynamic allocation&#39;s interaction with shuffle state matters because, before dynamic allocation existed, an executor exiting alongside its application meant all its state could be safely discarded; with dynamic allocation, the application keeps running after an executor is explicitly removed, so any later need for that executor&#39;s state forces a recompute<sup class="footnote-ref"><a href="#fn6" id="fnref6:2">[6:2]</a></sup>. That is exactly why Spark needs &quot;a mechanism to decommission an executor gracefully by preserving its state before removing it&quot;<sup class="footnote-ref"><a href="#fn6" id="fnref6:3">[6:3]</a></sup>. Without shuffle tracking, an external shuffle service, or shuffle-block decommission enabled, dynamic allocation either can&#39;t be turned on at all, or, if it&#39;s active regardless, an executor holding unpreserved shuffle output that gets removed forces exactly that unnecessary recompute<sup class="footnote-ref"><a href="#fn3" id="fnref3:8">[3:8]</a></sup><sup class="footnote-ref"><a href="#fn6" id="fnref6:4">[6:4]</a></sup>.</p><p>On the driver side, <code>spark.driver.maxResultSize</code> and <code>spark.driver.memory</code> are separate budgets that still interact: whether a high <code>maxResultSize</code> actually causes an out-of-memory error &quot;depends on spark.driver.memory and memory overhead of objects in JVM&quot;<sup class="footnote-ref"><a href="#fn3" id="fnref3:9">[3:9]</a></sup>. Raising one without considering the other doesn&#39;t fully protect the driver.</p><p>Finally, over-provisioning executors against a small-task workload wastes cluster resources: &quot;with small tasks this setting can waste a lot of resources due to executor allocation overhead, as some executor might not even do any work&quot;<sup class="footnote-ref"><a href="#fn3" id="fnref3:10">[3:10]</a></sup>.</p><h2 id="sizing-the-cluster-correctly" tabindex="-1">Sizing the cluster correctly <a class="header-anchor" href="#sizing-the-cluster-correctly" aria-label="Permalink to &quot;Sizing the cluster correctly&quot;">​</a></h2><p>Avoiding those costs starts from a formula, not one fat executor per node. Apply the balanced-executor formula instead; the worked examples above give shapes like 29 × 5 × 18GB for a 10-node/16-core/64GB cluster, or 17 × 5 × 19G for a 6-node cluster with the same per-node hardware<sup class="footnote-ref"><a href="#fn1" id="fnref1:4">[1:4]</a></sup><sup class="footnote-ref"><a href="#fn2" id="fnref2:4">[2:4]</a></sup>. Keep <code>--executor-cores</code> at or below roughly 5 so HDFS client concurrency stays in the range where it sustains full write throughput<sup class="footnote-ref"><a href="#fn1" id="fnref1:5">[1:5]</a></sup>.</p><p>When enabling off-heap memory, account for <code>spark.memory.offHeap.size</code> in the container/pod memory budget explicitly; <code>spark.executor.memoryOverhead</code>&#39;s default of <code>max(0.1 * executorMemory, 384MB)</code> does not include it<sup class="footnote-ref"><a href="#fn5" id="fnref5:5">[5:5]</a></sup>. If you&#39;re still referencing the older <code>spark.yarn.executor.memoryOverhead</code> name, note it was removed in Spark 3.0<sup class="footnote-ref"><a href="#fn5" id="fnref5:6">[5:6]</a></sup>.</p><p>On Kubernetes, remember that <code>spark.kubernetes.executor.request.cores</code> and <code>.limit.cores</code> govern the pod&#39;s CPU request/limit and take priority over <code>spark.executor.cores</code> for that purpose<sup class="footnote-ref"><a href="#fn6" id="fnref6:5">[6:5]</a></sup>, while <code>spark.executor.cores</code> separately governs Spark&#39;s own task-slot count<sup class="footnote-ref"><a href="#fn3" id="fnref3:11">[3:11]</a></sup>. The two need to be reasoned about together rather than assumed to be redundant.</p><p>Tune <code>spark.locality.wait</code> (and its per-level overrides) upward when tasks are long-running and locality is poor, since the default is tuned for typical workloads<sup class="footnote-ref"><a href="#fn3" id="fnref3:12">[3:12]</a></sup>; set <code>spark.locality.wait.node</code> to <code>0</code> to skip straight to rack locality when node locality isn&#39;t achievable<sup class="footnote-ref"><a href="#fn3" id="fnref3:13">[3:13]</a></sup>.</p><p>Turn on shuffle tracking (<code>spark.dynamicAllocation.shuffleTracking.enabled</code>, default <code>true</code> since Spark 3.0) or the external shuffle service so dynamic allocation can reclaim executors without losing shuffle output or forcing recomputation<sup class="footnote-ref"><a href="#fn3" id="fnref3:14">[3:14]</a></sup><sup class="footnote-ref"><a href="#fn6" id="fnref6:6">[6:6]</a></sup>. The external shuffle service additionally lets persisted RDD blocks survive executor removal when <code>spark.shuffle.service.fetch.rdd.enabled</code> is set, and executors holding cached blocks are, by default, never removed at all, tunable via <code>spark.dynamicAllocation.cachedExecutorIdleTimeout</code><sup class="footnote-ref"><a href="#fn6" id="fnref6:7">[6:7]</a></sup>.</p><p>Set <code>spark.driver.maxResultSize</code> as a guardrail on serialized action-result size, and size <code>spark.driver.memory</code> with it in mind rather than in isolation<sup class="footnote-ref"><a href="#fn3" id="fnref3:15">[3:15]</a></sup>. Use <code>spark.dynamicAllocation.executorAllocationRatio</code> (default <code>1.0</code>, added in 2.4.0) to scale down over-allocation when tasks are small; for example, a value of <code>0.5</code> halves the target executor count dynamic allocation would otherwise compute<sup class="footnote-ref"><a href="#fn3" id="fnref3:16">[3:16]</a></sup>. Raise <code>spark.task.cpus</code> above its default of <code>1</code> when a task needs more than one core; doing so proportionally reduces concurrent task slots per executor (<code>spark.executor.cores / spark.task.cpus</code>)<sup class="footnote-ref"><a href="#fn3" id="fnref3:17">[3:17]</a></sup>.</p><h2 id="bottleneck-autoscaling-churn" tabindex="-1">Autoscaling churn <a class="header-anchor" href="#bottleneck-autoscaling-churn" aria-label="Permalink to &quot;Autoscaling churn {#bottleneck-autoscaling-churn}&quot;">​</a></h2><p><span class="tag">CHRN</span> <span class="tag">EXPERIMENTAL</span></p><p>Dynamic allocation adds executors once tasks back up and frees them once they go idle<sup class="footnote-ref"><a href="#fn6" id="fnref6:8">[6:8]</a></sup>. Churn is a narrower failure mode inside that same mechanism: executors that get stood up and torn down again before they&#39;ve done much useful work, cycling through provisioning and JVM startup cost repeatedly instead of running tasks.</p><h3 id="how-it-s-detected" tabindex="-1">How it&#39;s detected <a class="header-anchor" href="#how-it-s-detected" aria-label="Permalink to &quot;How it&#39;s detected&quot;">​</a></h3><p>An executor counts as short-lived when its lifetime, from its <code>ExecutorAdded</code> event to either its <code>ExecutorRemoved</code> event or the end of the run if it was never removed, is under 2 minutes. This is evaluated once a run has added at least 5 executors, a floor that keeps a two- or three-executor job from registering on ordinary scale-down.</p><table tabindex="0"><thead><tr><th>Signal</th><th>Warning</th><th>Critical</th></tr></thead><tbody><tr><td>Share of added executors that are short-lived (&lt; 2 min lifetime)</td><td>&gt; 30%</td><td>&gt; 60%</td></tr></tbody></table><p>The 2-minute lifetime cutoff and the 30%/60% split are unvalidated heuristics rather than figures published in Spark&#39;s own documentation or benchmarks, so treat a finding here as a prompt to look at the executor timeline rather than a calibrated verdict.</p><h3 id="why-it-matters" tabindex="-1">Why it matters <a class="header-anchor" href="#why-it-matters" aria-label="Permalink to &quot;Why it matters&quot;">​</a></h3><p>Every short-lived executor pays the same provisioning and JVM-startup cost as a long-lived one, but returns little task work in exchange. A run that keeps flapping between scaling up and down spends more of its wall-clock time paying that repeated overhead than one that settles into a stable executor count.</p><h3 id="how-to-fix-it" tabindex="-1">How to fix it <a class="header-anchor" href="#how-to-fix-it" aria-label="Permalink to &quot;How to fix it&quot;">​</a></h3><ul><li>Raise <code>spark.dynamicAllocation.executorIdleTimeout</code> (default 60s<sup class="footnote-ref"><a href="#fn3" id="fnref3:18">[3:18]</a></sup>) so an executor survives a brief lull instead of being released the moment it goes idle, only to be requested again shortly after.</li><li>Widen the gap between <code>spark.dynamicAllocation.minExecutors</code> and <code>.maxExecutors</code><sup class="footnote-ref"><a href="#fn3" id="fnref3:19">[3:19]</a></sup>: bounds set too close together force Spark to repeatedly add and remove executors to track small fluctuations in the task backlog instead of settling into a stable range.</li></ul><h2 id="sources" tabindex="-1">Sources <a class="header-anchor" href="#sources" aria-label="Permalink to &quot;Sources&quot;">​</a></h2><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><a href="https://raw.githubusercontent.com/spoddutur/spark-notes/master/distribution_of_executors_cores_and_memory_for_spark_application.md" target="_blank" rel="noreferrer">Distribution of Executors, Cores and Memory for a Spark Application</a> <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a> <a href="#fnref1:2" class="footnote-backref">↩︎</a> <a href="#fnref1:3" class="footnote-backref">↩︎</a> <a href="#fnref1:4" class="footnote-backref">↩︎</a> <a href="#fnref1:5" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><a href="https://blog.cloudera.com/how-to-tune-your-apache-spark-jobs-part-2/" target="_blank" rel="noreferrer">How to Tune Your Apache Spark Jobs (Part 2)</a> <a href="#fnref2" class="footnote-backref">↩︎</a> <a href="#fnref2:1" class="footnote-backref">↩︎</a> <a href="#fnref2:2" class="footnote-backref">↩︎</a> <a href="#fnref2:3" class="footnote-backref">↩︎</a> <a href="#fnref2:4" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration — Spark</a> <a href="#fnref3" class="footnote-backref">↩︎</a> <a href="#fnref3:1" class="footnote-backref">↩︎</a> <a href="#fnref3:2" class="footnote-backref">↩︎</a> <a href="#fnref3:3" class="footnote-backref">↩︎</a> <a href="#fnref3:4" class="footnote-backref">↩︎</a> <a href="#fnref3:5" class="footnote-backref">↩︎</a> <a href="#fnref3:6" class="footnote-backref">↩︎</a> <a href="#fnref3:7" class="footnote-backref">↩︎</a> <a href="#fnref3:8" class="footnote-backref">↩︎</a> <a href="#fnref3:9" class="footnote-backref">↩︎</a> <a href="#fnref3:10" class="footnote-backref">↩︎</a> <a href="#fnref3:11" class="footnote-backref">↩︎</a> <a href="#fnref3:12" class="footnote-backref">↩︎</a> <a href="#fnref3:13" class="footnote-backref">↩︎</a> <a href="#fnref3:14" class="footnote-backref">↩︎</a> <a href="#fnref3:15" class="footnote-backref">↩︎</a> <a href="#fnref3:16" class="footnote-backref">↩︎</a> <a href="#fnref3:17" class="footnote-backref">↩︎</a> <a href="#fnref3:18" class="footnote-backref">↩︎</a> <a href="#fnref3:19" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/tuning.html" target="_blank" rel="noreferrer">Spark Tuning Guide</a> <a href="#fnref4" class="footnote-backref">↩︎</a></p></li><li id="fn5" class="footnote-item"><p><a href="https://luminousmen.com/post/dive-into-spark-memory" target="_blank" rel="noreferrer">Dive into Spark Memory</a> <a href="#fnref5" class="footnote-backref">↩︎</a> <a href="#fnref5:1" class="footnote-backref">↩︎</a> <a href="#fnref5:2" class="footnote-backref">↩︎</a> <a href="#fnref5:3" class="footnote-backref">↩︎</a> <a href="#fnref5:4" class="footnote-backref">↩︎</a> <a href="#fnref5:5" class="footnote-backref">↩︎</a> <a href="#fnref5:6" class="footnote-backref">↩︎</a></p></li><li id="fn6" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/job-scheduling.html" target="_blank" rel="noreferrer">Job Scheduling — Dynamic Resource Allocation</a> <a href="#fnref6" class="footnote-backref">↩︎</a> <a href="#fnref6:1" class="footnote-backref">↩︎</a> <a href="#fnref6:2" class="footnote-backref">↩︎</a> <a href="#fnref6:3" class="footnote-backref">↩︎</a> <a href="#fnref6:4" class="footnote-backref">↩︎</a> <a href="#fnref6:5" class="footnote-backref">↩︎</a> <a href="#fnref6:6" class="footnote-backref">↩︎</a> <a href="#fnref6:7" class="footnote-backref">↩︎</a> <a href="#fnref6:8" class="footnote-backref">↩︎</a></p></li></ol></section>',41)])])}const k=o(c,[["render",i]]);export{m as __pageData,k as default};

package/public/docs/assets/{tuning-reference_cluster-config.md.ZVmDGsQ3.lean.js → tuning-reference_cluster-config.md.CD6vzYtD.lean.js}

@@ -1 +1 @@
-import{_ as o,o as a,c as t,a5 as r}from"./chunks/framework.DSg0KOwT.js";const s="/docs/assets/container-memory.DIO0AnIm.svg",n="/docs/assets/container-memory.dark.CP-5zuCl.svg",m=JSON.parse('{"title":"Cluster Tuning","description":"","frontmatter":{"title":"Cluster Tuning"},"headers":[],"relativePath":"tuning-reference/cluster-config.md","filePath":"tuning-reference/cluster-config.md"}'),c={name:"tuning-reference/cluster-config.md"};function f(i,e,l,u,d,h){return a(),t("div",null,[...e[0]||(e[0]=[r("",30)])])}const k=o(c,[["render",f]]);export{m as __pageData,k as default};
+import{_ as o,o as t,c as a,a5 as r}from"./chunks/framework.DSg0KOwT.js";const s="/docs/assets/container-memory.DIO0AnIm.svg",n="/docs/assets/container-memory.dark.CP-5zuCl.svg",m=JSON.parse('{"title":"Cluster Tuning","description":"","frontmatter":{"title":"Cluster Tuning"},"headers":[],"relativePath":"tuning-reference/cluster-config.md","filePath":"tuning-reference/cluster-config.md"}'),c={name:"tuning-reference/cluster-config.md"};function i(f,e,l,d,u,h){return t(),a("div",null,[...e[0]||(e[0]=[r("",41)])])}const k=o(c,[["render",i]]);export{m as __pageData,k as default};

package/public/docs/assets/{tuning-reference_memory-model.md.DhT-n4y3.js → tuning-reference_memory-model.md.CWhvgevP.js}

@@ -1 +1 @@
-import{_ as o,o as a,c as r,a5 as t}from"./chunks/framework.DSg0KOwT.js";const s="/docs/assets/memory-regions.XHvO7jHG.svg",n="/docs/assets/memory-regions.dark.D4TP9_08.svg",f="/docs/assets/memory-borrowing.BqQRJg0u.svg",i="/docs/assets/memory-borrowing.dark.Yhh20O9C.svg",y=JSON.parse('{"title":"Memory Management","description":"","frontmatter":{"title":"Memory Management"},"headers":[],"relativePath":"tuning-reference/memory-model.md","filePath":"tuning-reference/memory-model.md"}'),c={name:"tuning-reference/memory-model.md"};function l(h,e,d,p,u,m){return a(),r("div",null,[...e[0]||(e[0]=[t('<h1 id="memory-model" tabindex="-1">Memory Management <a class="header-anchor" href="#memory-model" aria-label="Permalink to &quot;Memory Management {#memory-model}&quot;">​</a></h1><h2 id="the-unified-memory-model" tabindex="-1">The unified memory model <a class="header-anchor" href="#the-unified-memory-model" aria-label="Permalink to &quot;The unified memory model&quot;">​</a></h2><p>Since Spark 1.6, the <code>UnifiedMemoryManager</code> has replaced the earlier static split between execution and storage memory with a single shared region, referred to as M<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup><sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup>. M is carved out of the JVM heap after first setting aside 300 MiB of reserved system memory: <code>M = (JVM heap − 300 MiB) × spark.memory.fraction</code>, with <code>spark.memory.fraction</code> defaulting to 0.6<sup class="footnote-ref"><a href="#fn2" id="fnref2:1">[2:1]</a></sup><sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup>. Within M, <code>spark.memory.storageFraction</code> (default 0.5) marks off a subregion reserved for cached blocks that are immune to eviction; with defaults, this works out to roughly a 30%/30% on-heap split between execution and storage out of total executor memory<sup class="footnote-ref"><a href="#fn3" id="fnref3:1">[3:1]</a></sup>.</p><img class="light-only" src="'+s+'" alt="The executor memory layout showing reserved memory, the unified region split into execution and storage, and user memory inside the JVM heap, with the off-heap pool and memory overhead outside it."><img class="dark-only" src="'+n+'" alt="The executor memory layout showing reserved memory, the unified region split into execution and storage, and user memory inside the JVM heap, with the off-heap pool and memory overhead outside it."><p>Execution and storage borrow from each other dynamically rather than sitting behind a hard wall: when execution memory is unused, storage can acquire all of it, and vice versa<sup class="footnote-ref"><a href="#fn2" id="fnref2:2">[2:2]</a></sup><sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup>. The two are not equal peers, though. Execution always has priority, taking memory immediately and evicting cached storage blocks if necessary<sup class="footnote-ref"><a href="#fn3" id="fnref3:2">[3:2]</a></sup>.</p><p>The reserved 300 MB is a flat constant (<code>RESERVED_SYSTEM_MEMORY_BYTES</code> in the Spark source), not a formula, and it&#39;s hardcoded: there is no supported production configuration to change it. The only override is the internal <code>spark.testing.reservedMemory</code> property, which exists for Spark&#39;s own test suite rather than production tuning<sup class="footnote-ref"><a href="#fn3" id="fnref3:3">[3:3]</a></sup>. <code>spark.memory.fraction</code> is applied against heap minus that reserved 300 MB, not against total heap. The tuning guide is explicit that it &quot;expresses the size of M as a fraction of the (JVM heap space − 300MiB)&quot;<sup class="footnote-ref"><a href="#fn2" id="fnref2:3">[2:3]</a></sup>.</p><p>Off-heap memory extends the same model outside the JVM heap. When <code>spark.memory.offHeap.enabled=true</code>, Spark allocates raw off-heap buffers via <code>sun.misc.Unsafe</code> (routed through <code>jdk.internal.misc.Unsafe</code> on JDK 17+, which is why some builds need <code>--add-opens</code> flags), and that off-heap region is split into its own execution and storage pools, following the same borrowing rules as on-heap memory: execution has priority, storage gets evicted first<sup class="footnote-ref"><a href="#fn3" id="fnref3:4">[3:4]</a></sup>. Enabling it requires <code>spark.memory.offHeap.size</code> to be set to a positive value; leaving it enabled with the default size of <code>0</code> is a documented-invalid combination, not a supported way to run with off-heap &quot;on but empty&quot;<sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup>. The setting also &quot;has no impact on heap memory usage&quot;. Turning on off-heap memory does not shrink the JVM <code>-Xmx</code> for you, so the on-heap size has to be reduced manually to keep total footprint constant<sup class="footnote-ref"><a href="#fn4" id="fnref4:1">[4:1]</a></sup>. This off-heap execution memory underlies Project Tungsten: its compact binary row encoding, its explicit-memory-managed hash map for aggregations, and its cache-aware sort/join algorithms are all built to run against off-heap, GC-invisible memory<sup class="footnote-ref"><a href="#fn5" id="fnref5">[5]</a></sup><sup class="footnote-ref"><a href="#fn6" id="fnref6">[6]</a></sup>.</p><p>Executor memory overhead is a separate pool layered on top of M. <code>spark.executor.memoryOverhead</code> defaults to <code>executorMemory × spark.executor.memoryOverheadFactor</code>, floored at <code>spark.executor.minMemoryOverhead</code> (384 MiB)<sup class="footnote-ref"><a href="#fn4" id="fnref4:2">[4:2]</a></sup>. <code>spark.executor.memoryOverheadFactor</code> itself defaults to 0.10 for ordinary JVM executors, but Spark bumps that default to 0.40 specifically for Kubernetes non-JVM jobs, since those workloads tend to need more non-JVM heap space<sup class="footnote-ref"><a href="#fn4" id="fnref4:3">[4:3]</a></sup>. The legacy, YARN-only <code>spark.yarn.executor.memoryOverhead</code> property was removed in Spark 3.0 in favor of the cluster-manager-agnostic <code>spark.executor.memoryOverhead</code><sup class="footnote-ref"><a href="#fn3" id="fnref3:5">[3:5]</a></sup>.</p><blockquote><p><strong>PySpark:</strong> <code>spark.python.worker.memory</code> (default <code>512m</code>) is a soft spill threshold for a Python worker&#39;s own aggregation buffering, not a JVM-enforced cap. A PySpark worker runs as a separate OS process outside the JVM heap, so the JVM cannot police it directly<sup class="footnote-ref"><a href="#fn3" id="fnref3:6">[3:6]</a></sup>. The only setting that actually tries to bound PySpark memory per executor is <code>spark.executor.pyspark.memory</code>, and even that depends on Python&#39;s <code>resource</code> module, which isn&#39;t supported on Windows and doesn&#39;t actually limit anything on macOS<sup class="footnote-ref"><a href="#fn4" id="fnref4:4">[4:4]</a></sup>. Left unset, PySpark memory usage is folded into <code>spark.executor.memoryOverhead</code> by default<sup class="footnote-ref"><a href="#fn4" id="fnref4:5">[4:5]</a></sup>.</p></blockquote><h2 id="watching-eviction-and-spill" tabindex="-1">Watching eviction and spill <a class="header-anchor" href="#watching-eviction-and-spill" aria-label="Permalink to &quot;Watching eviction and spill&quot;">​</a></h2><p>That borrowing leaves a trace once eviction actually happens. Storage eviction under memory pressure follows an LRU (least recently used) policy<sup class="footnote-ref"><a href="#fn1" id="fnref1:2">[1:2]</a></sup><sup class="footnote-ref"><a href="#fn7" id="fnref7">[7]</a></sup>. The granularity is the block: the <code>BlockManager</code> tracks cached data as per-partition blocks (one block per RDD/DataFrame partition) and LRU operates at that level rather than evicting an entire RDD or cached table in one shot<sup class="footnote-ref"><a href="#fn8" id="fnref8">[8]</a></sup><sup class="footnote-ref"><a href="#fn9" id="fnref9">[9]</a></sup>. In practice this tends to evict the oldest partitions first, those materialized in the earliest job or stage, though lazy evaluation makes it hard to predict exactly which partitions will go ahead of time<sup class="footnote-ref"><a href="#fn9" id="fnref9:1">[9:1]</a></sup>.</p><p>What happens to an evicted block is visible in its downstream effect and depends on its <code>StorageLevel</code>. For <code>MEMORY_ONLY</code>, the block is simply dropped and recomputed from the RDD&#39;s lineage the next time it&#39;s needed. For a disk-backed level like <code>MEMORY_AND_DISK</code>, the evicted block is written to disk and read back from there instead of being recomputed<sup class="footnote-ref"><a href="#fn9" id="fnref9:2">[9:2]</a></sup><sup class="footnote-ref"><a href="#fn7" id="fnref7:1">[7:1]</a></sup>. Blocks can also be evicted well before you ever try to reuse them: <code>.cache()</code> does not guarantee a block stays resident, especially on busy clusters or long pipelines<sup class="footnote-ref"><a href="#fn7" id="fnref7:2">[7:2]</a></sup>.</p><p>On the execution side, each task gets its own <code>TaskMemoryManager</code>, which enforces a soft per-task cap: with <code>n</code> tasks running concurrently, each task is allowed to allocate somewhere between <code>1/(2n)</code> and <code>1/n</code> of total execution memory, and the first task to arrive on an idle executor typically grabs more than its later-arriving neighbors<sup class="footnote-ref"><a href="#fn3" id="fnref3:7">[3:7]</a></sup>.</p><p>When a sort or hash-aggregation operator keeps requesting execution-memory pages and can&#39;t get more, Spark doesn&#39;t fail immediately. It blocks the requesting task, spills that task&#39;s in-memory data structure to disk, or, in the worst case, throws an <code>OutOfMemoryError</code><sup class="footnote-ref"><a href="#fn3" id="fnref3:8">[3:8]</a></sup>. The spill path runs through Spark&#39;s map/shuffle I/O machinery: shuffle partitions created by wide transformations like <code>groupBy()</code> or <code>join()</code> <a href="./bottleneck-spill.html">spill</a> to the executors&#39; local disks at the location set by <code>spark.local.directory</code><sup class="footnote-ref"><a href="#fn6" id="fnref6:1">[6:1]</a></sup>. SQL physical operators apply the same idea with their own row-count guardrails: <code>sortMergeJoinExec</code>&#39;s in-memory buffer and the cartesian-product operator&#39;s buffer both spill once they cross a configured row threshold, which by default is set to the value of <code>spark.shuffle.spill.numElementsForceSpillThreshold</code><sup class="footnote-ref"><a href="#fn10" id="fnref10">[10]</a></sup>.</p><p>Whatever the trigger, the resulting spilled bytes are exposed on the task metrics as <code>memoryBytesSpilled</code>, visible in the Spark UI and event log. This is the concrete signal to watch for execution memory pressure<sup class="footnote-ref"><a href="#fn11" id="fnref11">[11]</a></sup>.</p><p>Memory-overhead misconfiguration shows up differently: it manifests as the container or pod being OOMKilled with a vague container-memory error rather than a Spark-level exception, since the enforcement happens at the YARN NodeManager or Kubernetes kubelet/cgroup layer, not inside the JVM<sup class="footnote-ref"><a href="#fn3" id="fnref3:9">[3:9]</a></sup>. Under the old default overhead factor of 0.10, non-JVM Kubernetes jobs commonly failed with &quot;Memory Overhead Exceeded&quot; errors, which is exactly why Spark bumped the Kubernetes non-JVM default to 0.40<sup class="footnote-ref"><a href="#fn4" id="fnref4:6">[4:6]</a></sup>.</p><h2 id="what-the-borrowing-costs-you" tabindex="-1">What the borrowing costs you <a class="header-anchor" href="#what-the-borrowing-costs-you" aria-label="Permalink to &quot;What the borrowing costs you&quot;">​</a></h2><p>None of this is free for whoever&#39;s counting on cached data staying put. Because execution always wins the tug-of-war over storage, <a href="./caching.html">caching</a> is never a guarantee; it&#39;s a best effort. As luminousmen-memory-management puts it: &quot;If Execution needs memory, it takes it. If Storage is using that space (cached RDDs, broadcasts), Spark starts evicting blocks. If Execution is idle, Storage can grow into that space, until Execution comes back.&quot;<sup class="footnote-ref"><a href="#fn3" id="fnref3:10">[3:10]</a></sup> That growth-then-shrink-back behavior is dynamic borrowing, not storage evicting execution; eviction is strictly one-directional. As spark-notes-task-memory-management summarizes the agreement between the two regions: &quot;keep acquiring execution memory and evict storage as you need more execution memory&quot;<sup class="footnote-ref"><a href="#fn1" id="fnref1:3">[1:3]</a></sup>, never the reverse. The same asymmetric rule carries over when off-heap memory is enabled: execution still has priority and storage still gets evicted first<sup class="footnote-ref"><a href="#fn3" id="fnref3:11">[3:11]</a></sup>. Practically, this means a cached DataFrame can silently lose blocks under memory pressure, forcing an expensive lineage recompute (for <code>MEMORY_ONLY</code>) or an extra disk round-trip (for <code>MEMORY_AND_DISK</code>) the next time it&#39;s touched.</p><img class="light-only" src="'+f+'" alt="How storage memory borrows idle execution space while execution reclaims its own space by evicting storage in one direction."><img class="dark-only" src="'+i+'" alt="How storage memory borrows idle execution space while execution reclaims its own space by evicting storage in one direction."><p>The <code>TaskMemoryManager</code>&#39;s soft per-task cap explains why spill behavior is workload-shape-dependent rather than a fixed threshold: with more tasks packed onto an executor, each one&#39;s guaranteed share of execution memory shrinks toward <code>1/(2n)</code>, making spills more likely under high task concurrency even when total execution memory hasn&#39;t changed<sup class="footnote-ref"><a href="#fn3" id="fnref3:12">[3:12]</a></sup>.</p><p>Off-heap memory&#39;s payoff is specifically about <a href="./bottleneck-gc.html">garbage collection</a>: because off-heap buffers sit outside the JVM heap, they are invisible to the garbage collector, so fewer and smaller live objects need to be tracked, scanned, and copied, which reduces both the frequency and duration of GC pauses<sup class="footnote-ref"><a href="#fn3" id="fnref3:13">[3:13]</a></sup>. Project Tungsten&#39;s off-heap hash map for aggregations was benchmarked at over 1 million operations per second in a single thread, with &quot;almost no performance degradation as memory utilization increases,&quot; unlike the JVM default <code>java.util.HashMap</code>, which eventually thrashes on GC<sup class="footnote-ref"><a href="#fn5" id="fnref5:1">[5:1]</a></sup><sup class="footnote-ref"><a href="#fn6" id="fnref6:2">[6:2]</a></sup>. The tradeoff is that off-heap memory removes the GC safety net along with the GC overhead: there&#39;s no garbage collector cleaning up if something goes wrong<sup class="footnote-ref"><a href="#fn3" id="fnref3:14">[3:14]</a></sup>.</p><p>Memory overhead matters because off-heap memory and (if unset) PySpark memory are not automatically folded into the overhead calculation the way heap memory is. Enabling <code>spark.memory.offHeap.size</code> without also raising <code>spark.executor.memoryOverhead</code> (or explicitly setting <code>spark.executor.pyspark.memory</code>) can push the executor&#39;s real footprint past what the cluster manager granted, and the process gets killed with a vague container-memory error instead of a Spark-level exception<sup class="footnote-ref"><a href="#fn3" id="fnref3:15">[3:15]</a></sup>. Worked example: with <code>--executor-memory=8G</code>, the default 10% overhead gives <code>max(0.1 × 8192 MB, 384 MB) = 819 MB</code>, so the total memory requested from the cluster manager is <code>8192 + 819 = 9011 MB</code><sup class="footnote-ref"><a href="#fn3" id="fnref3:16">[3:16]</a></sup>, useful to keep in mind when <a href="./cluster-config.html">sizing containers or pods</a> against a cluster&#39;s available capacity.</p><h2 id="tuning-the-memory-pools" tabindex="-1">Tuning the memory pools <a class="header-anchor" href="#tuning-the-memory-pools" aria-label="Permalink to &quot;Tuning the memory pools&quot;">​</a></h2><p>Most of this is adjustable, within limits. Tune <code>spark.memory.fraction</code> and <code>spark.memory.storageFraction</code> only if your workload&#39;s execution/storage balance genuinely needs to shift away from the ~30/30 default. But don&#39;t try to reclaim the 300 MB reserved region; it&#39;s a fixed, non-tunable constant in production<sup class="footnote-ref"><a href="#fn3" id="fnref3:17">[3:17]</a></sup>.</p><p>If losing cached partitions to eviction is costly, prefer a disk-backed <code>StorageLevel</code> such as <code>MEMORY_AND_DISK</code> over <code>MEMORY_ONLY</code>: an evicted block gets written to disk and read back rather than triggering a full lineage recompute<sup class="footnote-ref"><a href="#fn9" id="fnref9:3">[9:3]</a></sup><sup class="footnote-ref"><a href="#fn7" id="fnref7:3">[7:3]</a></sup>. Keep in mind <code>.cache()</code> alone is not a residency guarantee even with this choice<sup class="footnote-ref"><a href="#fn7" id="fnref7:4">[7:4]</a></sup>.</p><p>Watch <code>memoryBytesSpilled</code> in the Spark UI or history server to catch execution-memory pressure early<sup class="footnote-ref"><a href="#fn11" id="fnref11:1">[11:1]</a></sup>. If sort or hash-aggregation spills are heavy, consider giving the executor more memory or reducing the number of concurrently running tasks per executor, since the <code>TaskMemoryManager</code>&#39;s per-task guarantee shrinks as task concurrency <code>n</code> grows<sup class="footnote-ref"><a href="#fn3" id="fnref3:18">[3:18]</a></sup>. For SQL joins and cartesian products, <code>spark.shuffle.spill.numElementsForceSpillThreshold</code> governs when <code>sortMergeJoinExec</code> and the cartesian-product operator spill their buffers<sup class="footnote-ref"><a href="#fn10" id="fnref10:1">[10:1]</a></sup>.</p><p>Set <code>spark.executor.memoryOverhead</code> explicitly rather than relying on the default, especially on Kubernetes for non-JVM workloads (where the default factor is already bumped to 0.40) or whenever off-heap memory or heavy PySpark usage is in play: those aren&#39;t automatically folded into the overhead calculation, so an unadjusted overhead can lead to an OOMKilled container instead of a clean Spark-level error<sup class="footnote-ref"><a href="#fn3" id="fnref3:19">[3:19]</a></sup><sup class="footnote-ref"><a href="#fn4" id="fnref4:7">[4:7]</a></sup>. Use <code>spark.executor.memoryOverhead</code>, not the removed <code>spark.yarn.executor.memoryOverhead</code><sup class="footnote-ref"><a href="#fn3" id="fnref3:20">[3:20]</a></sup>. For capacity planning, remember the total requested from the cluster manager is executor memory plus overhead, e.g., <code>8192 + 819 = 9011 MB</code> for an 8 GB executor at the default 10% factor<sup class="footnote-ref"><a href="#fn3" id="fnref3:21">[3:21]</a></sup>.</p><p>When enabling off-heap memory, always pair <code>spark.memory.offHeap.enabled=true</code> with an explicit positive <code>spark.memory.offHeap.size</code> (never leave it enabled at the default size of <code>0</code><sup class="footnote-ref"><a href="#fn4" id="fnref4:8">[4:8]</a></sup>) and manually reduce the JVM <code>-Xmx</code> to compensate, since enabling off-heap memory does not shrink the heap for you<sup class="footnote-ref"><a href="#fn4" id="fnref4:9">[4:9]</a></sup>.</p><blockquote><p><strong>PySpark:</strong> if you need PySpark&#39;s own memory bounded rather than folded silently into the overhead, set <code>spark.executor.pyspark.memory</code> explicitly, though its enforcement relies on Python&#39;s <code>resource</code> module and won&#39;t work on Windows and won&#39;t actually limit anything on macOS<sup class="footnote-ref"><a href="#fn4" id="fnref4:10">[4:10]</a></sup>.</p></blockquote><h2 id="sources" tabindex="-1">Sources <a class="header-anchor" href="#sources" aria-label="Permalink to &quot;Sources&quot;">​</a></h2><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><a href="https://raw.githubusercontent.com/spoddutur/spark-notes/master/task_memory_management_in_spark.md" target="_blank" rel="noreferrer">Task Memory Management in Spark</a> <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a> <a href="#fnref1:2" class="footnote-backref">↩︎</a> <a href="#fnref1:3" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/tuning.html" target="_blank" rel="noreferrer">Tuning Spark</a> <a href="#fnref2" class="footnote-backref">↩︎</a> <a href="#fnref2:1" class="footnote-backref">↩︎</a> <a href="#fnref2:2" class="footnote-backref">↩︎</a> <a href="#fnref2:3" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><a href="https://luminousmen.com/post/dive-into-spark-memory" target="_blank" rel="noreferrer">Dive into Spark Memory</a> <a href="#fnref3" class="footnote-backref">↩︎</a> <a href="#fnref3:1" class="footnote-backref">↩︎</a> <a href="#fnref3:2" class="footnote-backref">↩︎</a> <a href="#fnref3:3" class="footnote-backref">↩︎</a> <a href="#fnref3:4" class="footnote-backref">↩︎</a> <a href="#fnref3:5" class="footnote-backref">↩︎</a> <a href="#fnref3:6" class="footnote-backref">↩︎</a> <a href="#fnref3:7" class="footnote-backref">↩︎</a> <a href="#fnref3:8" class="footnote-backref">↩︎</a> <a href="#fnref3:9" class="footnote-backref">↩︎</a> <a href="#fnref3:10" class="footnote-backref">↩︎</a> <a href="#fnref3:11" class="footnote-backref">↩︎</a> <a href="#fnref3:12" class="footnote-backref">↩︎</a> <a href="#fnref3:13" class="footnote-backref">↩︎</a> <a href="#fnref3:14" class="footnote-backref">↩︎</a> <a href="#fnref3:15" class="footnote-backref">↩︎</a> <a href="#fnref3:16" class="footnote-backref">↩︎</a> <a href="#fnref3:17" class="footnote-backref">↩︎</a> <a href="#fnref3:18" class="footnote-backref">↩︎</a> <a href="#fnref3:19" class="footnote-backref">↩︎</a> <a href="#fnref3:20" class="footnote-backref">↩︎</a> <a href="#fnref3:21" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration — Spark</a> <a href="#fnref4" class="footnote-backref">↩︎</a> <a href="#fnref4:1" class="footnote-backref">↩︎</a> <a href="#fnref4:2" class="footnote-backref">↩︎</a> <a href="#fnref4:3" class="footnote-backref">↩︎</a> <a href="#fnref4:4" class="footnote-backref">↩︎</a> <a href="#fnref4:5" class="footnote-backref">↩︎</a> <a href="#fnref4:6" class="footnote-backref">↩︎</a> <a href="#fnref4:7" class="footnote-backref">↩︎</a> <a href="#fnref4:8" class="footnote-backref">↩︎</a> <a href="#fnref4:9" class="footnote-backref">↩︎</a> <a href="#fnref4:10" class="footnote-backref">↩︎</a></p></li><li id="fn5" class="footnote-item"><p><a href="https://www.databricks.com/blog/2015/04/28/project-tungsten-bringing-spark-closer-to-bare-metal.html" target="_blank" rel="noreferrer">Project Tungsten: Bringing Spark Closer to Bare Metal</a> <a href="#fnref5" class="footnote-backref">↩︎</a> <a href="#fnref5:1" class="footnote-backref">↩︎</a></p></li><li id="fn6" class="footnote-item"><p><em>Learning Spark, 2nd Edition</em>, Damji, Wenig, Das, Lee, ch. 6–7 <a href="#fnref6" class="footnote-backref">↩︎</a> <a href="#fnref6:1" class="footnote-backref">↩︎</a> <a href="#fnref6:2" class="footnote-backref">↩︎</a></p></li><li id="fn7" class="footnote-item"><p><a href="https://luminousmen.com/post/explaining-the-mechanics-of-spark-caching" target="_blank" rel="noreferrer">Explaining the Mechanics of Spark Caching</a> <a href="#fnref7" class="footnote-backref">↩︎</a> <a href="#fnref7:1" class="footnote-backref">↩︎</a> <a href="#fnref7:2" class="footnote-backref">↩︎</a> <a href="#fnref7:3" class="footnote-backref">↩︎</a> <a href="#fnref7:4" class="footnote-backref">↩︎</a></p></li><li id="fn8" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/rdd-programming-guide.html" target="_blank" rel="noreferrer">RDD Programming Guide</a> <a href="#fnref8" class="footnote-backref">↩︎</a></p></li><li id="fn9" class="footnote-item"><p><em>High Performance Spark, 2nd Edition</em>, Karau, Polak &amp; Warren, ch. 7 <a href="#fnref9" class="footnote-backref">↩︎</a> <a href="#fnref9:1" class="footnote-backref">↩︎</a> <a href="#fnref9:2" class="footnote-backref">↩︎</a> <a href="#fnref9:3" class="footnote-backref">↩︎</a></p></li><li id="fn10" class="footnote-item"><p><a href="https://raw.githubusercontent.com/apache/spark/v3.5.0/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala" target="_blank" rel="noreferrer">SQLConf.scala</a> <a href="#fnref10" class="footnote-backref">↩︎</a> <a href="#fnref10:1" class="footnote-backref">↩︎</a></p></li><li id="fn11" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/monitoring.html#spark-history-server" target="_blank" rel="noreferrer">Monitoring and Instrumentation</a> <a href="#fnref11" class="footnote-backref">↩︎</a> <a href="#fnref11:1" class="footnote-backref">↩︎</a></p></li></ol></section>',34)])])}const k=o(c,[["render",l]]);export{y as __pageData,k as default};
+import{_ as o,o as a,c as t,a5 as r}from"./chunks/framework.DSg0KOwT.js";const s="/docs/assets/memory-regions.XHvO7jHG.svg",n="/docs/assets/memory-regions.dark.D4TP9_08.svg",i="/docs/assets/memory-borrowing.BqQRJg0u.svg",f="/docs/assets/memory-borrowing.dark.Yhh20O9C.svg",y=JSON.parse('{"title":"Memory Management","description":"","frontmatter":{"title":"Memory Management"},"headers":[],"relativePath":"tuning-reference/memory-model.md","filePath":"tuning-reference/memory-model.md"}'),c={name:"tuning-reference/memory-model.md"};function l(h,e,d,p,u,m){return a(),t("div",null,[...e[0]||(e[0]=[r('<h1 id="memory-model" tabindex="-1">Memory Management <a class="header-anchor" href="#memory-model" aria-label="Permalink to &quot;Memory Management {#memory-model}&quot;">​</a></h1><h2 id="the-unified-memory-model" tabindex="-1">The unified memory model <a class="header-anchor" href="#the-unified-memory-model" aria-label="Permalink to &quot;The unified memory model&quot;">​</a></h2><p>Since Spark 1.6, the <code>UnifiedMemoryManager</code> has replaced the earlier static split between execution and storage memory with a single shared region, referred to as M<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup><sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup>. M is carved out of the JVM heap after first setting aside 300 MiB of reserved system memory: <code>M = (JVM heap − 300 MiB) × spark.memory.fraction</code>, with <code>spark.memory.fraction</code> defaulting to 0.6<sup class="footnote-ref"><a href="#fn2" id="fnref2:1">[2:1]</a></sup><sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup>. Within M, <code>spark.memory.storageFraction</code> (default 0.5) marks off a subregion reserved for cached blocks that are immune to eviction; with defaults, this works out to roughly a 30%/30% on-heap split between execution and storage out of total executor memory<sup class="footnote-ref"><a href="#fn3" id="fnref3:1">[3:1]</a></sup>.</p><img class="light-only" src="'+s+'" alt="The executor memory layout showing reserved memory, the unified region split into execution and storage, and user memory inside the JVM heap, with the off-heap pool and memory overhead outside it."><img class="dark-only" src="'+n+'" alt="The executor memory layout showing reserved memory, the unified region split into execution and storage, and user memory inside the JVM heap, with the off-heap pool and memory overhead outside it."><p>Execution and storage borrow from each other dynamically rather than sitting behind a hard wall: when execution memory is unused, storage can acquire all of it, and vice versa<sup class="footnote-ref"><a href="#fn2" id="fnref2:2">[2:2]</a></sup><sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup>. The two are not equal peers, though. Execution always has priority, taking memory immediately and evicting cached storage blocks if necessary<sup class="footnote-ref"><a href="#fn3" id="fnref3:2">[3:2]</a></sup>.</p><p>The reserved 300 MB is a flat constant (<code>RESERVED_SYSTEM_MEMORY_BYTES</code> in the Spark source), not a formula, and it&#39;s hardcoded: there is no supported production configuration to change it. The only override is the internal <code>spark.testing.reservedMemory</code> property, which exists for Spark&#39;s own test suite rather than production tuning<sup class="footnote-ref"><a href="#fn3" id="fnref3:3">[3:3]</a></sup>. <code>spark.memory.fraction</code> is applied against heap minus that reserved 300 MB, not against total heap. The tuning guide is explicit that it &quot;expresses the size of M as a fraction of the (JVM heap space − 300MiB)&quot;<sup class="footnote-ref"><a href="#fn2" id="fnref2:3">[2:3]</a></sup>.</p><p>Off-heap memory extends the same model outside the JVM heap. When <code>spark.memory.offHeap.enabled=true</code>, Spark allocates raw off-heap buffers via <code>sun.misc.Unsafe</code> (routed through <code>jdk.internal.misc.Unsafe</code> on JDK 17+, which is why some builds need <code>--add-opens</code> flags), and that off-heap region is split into its own execution and storage pools, following the same borrowing rules as on-heap memory: execution has priority, storage gets evicted first<sup class="footnote-ref"><a href="#fn3" id="fnref3:4">[3:4]</a></sup>. Enabling it requires <code>spark.memory.offHeap.size</code> to be set to a positive value; leaving it enabled with the default size of <code>0</code> is a documented-invalid combination, not a supported way to run with off-heap &quot;on but empty&quot;<sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup>. The setting also &quot;has no impact on heap memory usage&quot;. Turning on off-heap memory does not shrink the JVM <code>-Xmx</code> for you, so the on-heap size has to be reduced manually to keep total footprint constant<sup class="footnote-ref"><a href="#fn4" id="fnref4:1">[4:1]</a></sup>. This off-heap execution memory underlies Project Tungsten: its compact binary row encoding, its explicit-memory-managed hash map for aggregations, and its cache-aware sort/join algorithms are all built to run against off-heap, GC-invisible memory<sup class="footnote-ref"><a href="#fn5" id="fnref5">[5]</a></sup><sup class="footnote-ref"><a href="#fn6" id="fnref6">[6]</a></sup>.</p><p>Executor memory overhead is a separate pool layered on top of M. <code>spark.executor.memoryOverhead</code> defaults to <code>executorMemory × spark.executor.memoryOverheadFactor</code>, floored at <code>spark.executor.minMemoryOverhead</code> (384 MiB)<sup class="footnote-ref"><a href="#fn4" id="fnref4:2">[4:2]</a></sup>. <code>spark.executor.memoryOverheadFactor</code> itself defaults to 0.10 for ordinary JVM executors, but Spark bumps that default to 0.40 specifically for Kubernetes non-JVM jobs, since those workloads tend to need more non-JVM heap space<sup class="footnote-ref"><a href="#fn4" id="fnref4:3">[4:3]</a></sup>. The legacy, YARN-only <code>spark.yarn.executor.memoryOverhead</code> property was removed in Spark 3.0 in favor of the cluster-manager-agnostic <code>spark.executor.memoryOverhead</code><sup class="footnote-ref"><a href="#fn3" id="fnref3:5">[3:5]</a></sup>.</p><blockquote><p><strong>PySpark:</strong> <code>spark.python.worker.memory</code> (default <code>512m</code>) is a soft spill threshold for a Python worker&#39;s own aggregation buffering, not a JVM-enforced cap. A PySpark worker runs as a separate OS process outside the JVM heap, so the JVM cannot police it directly<sup class="footnote-ref"><a href="#fn3" id="fnref3:6">[3:6]</a></sup>. The only setting that actually tries to bound PySpark memory per executor is <code>spark.executor.pyspark.memory</code>, and even that depends on Python&#39;s <code>resource</code> module, which isn&#39;t supported on Windows and doesn&#39;t actually limit anything on macOS<sup class="footnote-ref"><a href="#fn4" id="fnref4:4">[4:4]</a></sup>. Left unset, PySpark memory usage is folded into <code>spark.executor.memoryOverhead</code> by default<sup class="footnote-ref"><a href="#fn4" id="fnref4:5">[4:5]</a></sup>.</p></blockquote><h2 id="watching-eviction-and-spill" tabindex="-1">Watching eviction and spill <a class="header-anchor" href="#watching-eviction-and-spill" aria-label="Permalink to &quot;Watching eviction and spill&quot;">​</a></h2><p>That borrowing leaves a trace once eviction actually happens. Storage eviction under memory pressure follows an LRU (least recently used) policy<sup class="footnote-ref"><a href="#fn1" id="fnref1:2">[1:2]</a></sup><sup class="footnote-ref"><a href="#fn7" id="fnref7">[7]</a></sup>. The granularity is the block: the <code>BlockManager</code> tracks cached data as per-partition blocks (one block per RDD/DataFrame partition) and LRU operates at that level rather than evicting an entire RDD or cached table in one shot<sup class="footnote-ref"><a href="#fn8" id="fnref8">[8]</a></sup><sup class="footnote-ref"><a href="#fn9" id="fnref9">[9]</a></sup>. In practice this tends to evict the oldest partitions first, those materialized in the earliest job or stage, though lazy evaluation makes it hard to predict exactly which partitions will go ahead of time<sup class="footnote-ref"><a href="#fn9" id="fnref9:1">[9:1]</a></sup>.</p><p>What happens to an evicted block is visible in its downstream effect and depends on its <code>StorageLevel</code>. For <code>MEMORY_ONLY</code>, the block is simply dropped and recomputed from the RDD&#39;s lineage the next time it&#39;s needed. For a disk-backed level like <code>MEMORY_AND_DISK</code>, the evicted block is written to disk and read back from there instead of being recomputed<sup class="footnote-ref"><a href="#fn9" id="fnref9:2">[9:2]</a></sup><sup class="footnote-ref"><a href="#fn7" id="fnref7:1">[7:1]</a></sup>. Blocks can also be evicted well before you ever try to reuse them: <code>.cache()</code> does not guarantee a block stays resident, especially on busy clusters or long pipelines<sup class="footnote-ref"><a href="#fn7" id="fnref7:2">[7:2]</a></sup>.</p><p>On the execution side, each task gets its own <code>TaskMemoryManager</code>, which enforces a soft per-task cap: with <code>n</code> tasks running concurrently, each task is allowed to allocate somewhere between <code>1/(2n)</code> and <code>1/n</code> of total execution memory, and the first task to arrive on an idle executor typically grabs more than its later-arriving neighbors<sup class="footnote-ref"><a href="#fn3" id="fnref3:7">[3:7]</a></sup>.</p><p>When a sort or hash-aggregation operator keeps requesting execution-memory pages and can&#39;t get more, Spark doesn&#39;t fail immediately. It blocks the requesting task, spills that task&#39;s in-memory data structure to disk, or, in the worst case, throws an <code>OutOfMemoryError</code><sup class="footnote-ref"><a href="#fn3" id="fnref3:8">[3:8]</a></sup>. The spill path runs through Spark&#39;s map/shuffle I/O machinery: shuffle partitions created by wide transformations like <code>groupBy()</code> or <code>join()</code> <a href="./bottleneck-spill.html">spill</a> to the executors&#39; local disks at the location set by <code>spark.local.directory</code><sup class="footnote-ref"><a href="#fn6" id="fnref6:1">[6:1]</a></sup>. SQL physical operators apply the same idea with their own row-count guardrails: <code>sortMergeJoinExec</code>&#39;s in-memory buffer and the cartesian-product operator&#39;s buffer both spill once they cross a configured row threshold, which by default is set to the value of <code>spark.shuffle.spill.numElementsForceSpillThreshold</code><sup class="footnote-ref"><a href="#fn10" id="fnref10">[10]</a></sup>.</p><p>Whatever the trigger, the resulting spilled bytes are exposed on the task metrics as <code>memoryBytesSpilled</code>, visible in the Spark UI and event log. This is the concrete signal to watch for execution memory pressure<sup class="footnote-ref"><a href="#fn11" id="fnref11">[11]</a></sup>.</p><p>Memory-overhead misconfiguration shows up differently: it manifests as the container or pod being OOMKilled with a vague container-memory error rather than a Spark-level exception, since the enforcement happens at the YARN NodeManager or Kubernetes kubelet/cgroup layer, not inside the JVM<sup class="footnote-ref"><a href="#fn3" id="fnref3:9">[3:9]</a></sup>. Under the old default overhead factor of 0.10, non-JVM Kubernetes jobs commonly failed with &quot;Memory Overhead Exceeded&quot; errors, which is exactly why Spark bumped the Kubernetes non-JVM default to 0.40<sup class="footnote-ref"><a href="#fn4" id="fnref4:6">[4:6]</a></sup>.</p><h2 id="what-the-borrowing-costs-you" tabindex="-1">What the borrowing costs you <a class="header-anchor" href="#what-the-borrowing-costs-you" aria-label="Permalink to &quot;What the borrowing costs you&quot;">​</a></h2><p>None of this is free for whoever&#39;s counting on cached data staying put. Because execution always wins the tug-of-war over storage, <a href="./caching.html">caching</a> is never a guarantee; it&#39;s a best effort. As luminousmen-memory-management puts it: &quot;If Execution needs memory, it takes it. If Storage is using that space (cached RDDs, broadcasts), Spark starts evicting blocks. If Execution is idle, Storage can grow into that space, until Execution comes back.&quot;<sup class="footnote-ref"><a href="#fn3" id="fnref3:10">[3:10]</a></sup> That growth-then-shrink-back behavior is dynamic borrowing, not storage evicting execution; eviction is strictly one-directional. As spark-notes-task-memory-management summarizes the agreement between the two regions: &quot;keep acquiring execution memory and evict storage as you need more execution memory&quot;<sup class="footnote-ref"><a href="#fn1" id="fnref1:3">[1:3]</a></sup>, never the reverse. The same asymmetric rule carries over when off-heap memory is enabled: execution still has priority and storage still gets evicted first<sup class="footnote-ref"><a href="#fn3" id="fnref3:11">[3:11]</a></sup>. Practically, this means a cached DataFrame can silently lose blocks under memory pressure, forcing an expensive lineage recompute (for <code>MEMORY_ONLY</code>) or an extra disk round-trip (for <code>MEMORY_AND_DISK</code>) the next time it&#39;s touched.</p><img class="light-only" src="'+i+'" alt="How storage memory borrows idle execution space while execution reclaims its own space by evicting storage in one direction."><img class="dark-only" src="'+f+'" alt="How storage memory borrows idle execution space while execution reclaims its own space by evicting storage in one direction."><p>The <code>TaskMemoryManager</code>&#39;s soft per-task cap explains why spill behavior is workload-shape-dependent rather than a fixed threshold: with more tasks packed onto an executor, each one&#39;s guaranteed share of execution memory shrinks toward <code>1/(2n)</code>, making spills more likely under high task concurrency even when total execution memory hasn&#39;t changed<sup class="footnote-ref"><a href="#fn3" id="fnref3:12">[3:12]</a></sup>.</p><p>Off-heap memory&#39;s payoff is specifically about <a href="./bottleneck-gc.html">garbage collection</a>: because off-heap buffers sit outside the JVM heap, they are invisible to the garbage collector, so fewer and smaller live objects need to be tracked, scanned, and copied, which reduces both the frequency and duration of GC pauses<sup class="footnote-ref"><a href="#fn3" id="fnref3:13">[3:13]</a></sup>. Project Tungsten&#39;s off-heap hash map for aggregations was benchmarked at over 1 million operations per second in a single thread, with &quot;almost no performance degradation as memory utilization increases,&quot; unlike the JVM default <code>java.util.HashMap</code>, which eventually thrashes on GC<sup class="footnote-ref"><a href="#fn5" id="fnref5:1">[5:1]</a></sup><sup class="footnote-ref"><a href="#fn6" id="fnref6:2">[6:2]</a></sup>. The tradeoff is that off-heap memory removes the GC safety net along with the GC overhead: there&#39;s no garbage collector cleaning up if something goes wrong<sup class="footnote-ref"><a href="#fn3" id="fnref3:14">[3:14]</a></sup>.</p><p>Memory overhead matters because off-heap memory and (if unset) PySpark memory are not automatically folded into the overhead calculation the way heap memory is. Enabling <code>spark.memory.offHeap.size</code> without also raising <code>spark.executor.memoryOverhead</code> (or explicitly setting <code>spark.executor.pyspark.memory</code>) can push the executor&#39;s real footprint past what the cluster manager granted, and the process gets killed with a vague container-memory error instead of a Spark-level exception<sup class="footnote-ref"><a href="#fn3" id="fnref3:15">[3:15]</a></sup>. Worked example: with <code>--executor-memory=8G</code>, the default 10% overhead gives <code>max(0.1 × 8192 MB, 384 MB) = 819 MB</code>, so the total memory requested from the cluster manager is <code>8192 + 819 = 9011 MB</code><sup class="footnote-ref"><a href="#fn3" id="fnref3:16">[3:16]</a></sup>, useful to keep in mind when <a href="./cluster-config.html">sizing containers or pods</a> against a cluster&#39;s available capacity.</p><h2 id="tuning-the-memory-pools" tabindex="-1">Tuning the memory pools <a class="header-anchor" href="#tuning-the-memory-pools" aria-label="Permalink to &quot;Tuning the memory pools&quot;">​</a></h2><p>Most of this is adjustable, within limits. Tune <code>spark.memory.fraction</code> and <code>spark.memory.storageFraction</code> only if your workload&#39;s execution/storage balance genuinely needs to shift away from the ~30/30 default. But don&#39;t try to reclaim the 300 MB reserved region; it&#39;s a fixed, non-tunable constant in production<sup class="footnote-ref"><a href="#fn3" id="fnref3:17">[3:17]</a></sup>.</p><p>If losing cached partitions to eviction is costly, prefer a disk-backed <code>StorageLevel</code> such as <code>MEMORY_AND_DISK</code> over <code>MEMORY_ONLY</code>: an evicted block gets written to disk and read back rather than triggering a full lineage recompute<sup class="footnote-ref"><a href="#fn9" id="fnref9:3">[9:3]</a></sup><sup class="footnote-ref"><a href="#fn7" id="fnref7:3">[7:3]</a></sup>. Keep in mind <code>.cache()</code> alone is not a residency guarantee even with this choice<sup class="footnote-ref"><a href="#fn7" id="fnref7:4">[7:4]</a></sup>.</p><p>Watch <code>memoryBytesSpilled</code> in the Spark UI or history server to catch execution-memory pressure early<sup class="footnote-ref"><a href="#fn11" id="fnref11:1">[11:1]</a></sup>. If sort or hash-aggregation spills are heavy, consider giving the executor more memory or reducing the number of concurrently running tasks per executor, since the <code>TaskMemoryManager</code>&#39;s per-task guarantee shrinks as task concurrency <code>n</code> grows<sup class="footnote-ref"><a href="#fn3" id="fnref3:18">[3:18]</a></sup>. For SQL joins and cartesian products, <code>spark.shuffle.spill.numElementsForceSpillThreshold</code> governs when <code>sortMergeJoinExec</code> and the cartesian-product operator spill their buffers<sup class="footnote-ref"><a href="#fn10" id="fnref10:1">[10:1]</a></sup>.</p><p>Set <code>spark.executor.memoryOverhead</code> explicitly rather than relying on the default, especially on Kubernetes for non-JVM workloads (where the default factor is already bumped to 0.40) or whenever off-heap memory or heavy PySpark usage is in play: those aren&#39;t automatically folded into the overhead calculation, so an unadjusted overhead can lead to an OOMKilled container instead of a clean Spark-level error<sup class="footnote-ref"><a href="#fn3" id="fnref3:19">[3:19]</a></sup><sup class="footnote-ref"><a href="#fn4" id="fnref4:7">[4:7]</a></sup>. Use <code>spark.executor.memoryOverhead</code>, not the removed <code>spark.yarn.executor.memoryOverhead</code><sup class="footnote-ref"><a href="#fn3" id="fnref3:20">[3:20]</a></sup>. For capacity planning, remember the total requested from the cluster manager is executor memory plus overhead, e.g., <code>8192 + 819 = 9011 MB</code> for an 8 GB executor at the default 10% factor<sup class="footnote-ref"><a href="#fn3" id="fnref3:21">[3:21]</a></sup>.</p><p>When enabling off-heap memory, always pair <code>spark.memory.offHeap.enabled=true</code> with an explicit positive <code>spark.memory.offHeap.size</code> (never leave it enabled at the default size of <code>0</code><sup class="footnote-ref"><a href="#fn4" id="fnref4:8">[4:8]</a></sup>) and manually reduce the JVM <code>-Xmx</code> to compensate, since enabling off-heap memory does not shrink the heap for you<sup class="footnote-ref"><a href="#fn4" id="fnref4:9">[4:9]</a></sup>.</p><blockquote><p><strong>PySpark:</strong> if you need PySpark&#39;s own memory bounded rather than folded silently into the overhead, set <code>spark.executor.pyspark.memory</code> explicitly, though its enforcement relies on Python&#39;s <code>resource</code> module and won&#39;t work on Windows and won&#39;t actually limit anything on macOS<sup class="footnote-ref"><a href="#fn4" id="fnref4:10">[4:10]</a></sup>.</p></blockquote><h2 id="bottleneck-cache-utilization" tabindex="-1">Cache utilization <a class="header-anchor" href="#bottleneck-cache-utilization" aria-label="Permalink to &quot;Cache utilization {#bottleneck-cache-utilization}&quot;">​</a></h2><p><span class="tag">CSTOR</span></p><p>Spark&#39;s event log carries no block-access or hit-rate events, so cache utilization is read from periodic per-RDD storage snapshots instead: how much of a persisted RDD is actually resident where it was asked to be.</p><h3 id="how-it-s-detected" tabindex="-1">How it&#39;s detected <a class="header-anchor" href="#how-it-s-detected" aria-label="Permalink to &quot;How it&#39;s detected&quot;">​</a></h3><p>A persisted RDD, one whose storage level requests memory and/or disk with at least one partition actually cached, is evaluated on two independent measures and can register on both at once:</p><table tabindex="0"><thead><tr><th>Signal</th><th>Info</th><th>Warning</th></tr></thead><tbody><tr><td>Cached ratio (cached partitions ÷ total partitions)</td><td>&lt; 90%</td><td>&lt; 50%</td></tr><tr><td>Disk ratio (disk bytes ÷ (memory bytes + disk bytes)), <code>MEMORY_AND_DISK*</code> levels only</td><td>&gt; 15%</td><td>&gt; 40%</td></tr></tbody></table><p>Both ratios come from a point-in-time storage snapshot taken at stage-submission events, not a runtime block-access count, so confirm against the Spark UI&#39;s Storage tab before acting. Confidence scales with the RDD&#39;s partition count: below 10 partitions the ratio is noisy enough to call low confidence, at 50 or more it&#39;s high.</p><h3 id="why-it-matters" tabindex="-1">Why it matters <a class="header-anchor" href="#why-it-matters" aria-label="Permalink to &quot;Why it matters&quot;">​</a></h3><p>A partially cached RDD still pays the recomputation cost for its evicted share on every later read, undercutting the reason it was cached in the first place. Disk spillover under a <code>MEMORY_AND_DISK*</code> level keeps that data around instead of forcing a recompute, but a disk read is still far slower than serving it out of memory.</p><h3 id="how-to-fix-it" tabindex="-1">How to fix it <a class="header-anchor" href="#how-to-fix-it" aria-label="Permalink to &quot;How to fix it&quot;">​</a></h3><ul><li>Increase executor memory, or reduce the size of the dataset being cached, so more of it fits in the storage region without eviction.</li><li>If the RDD is only cached for the occasional narrow scan, weigh whether persisting it at all is worth the partial-cache overhead, versus recomputing the slice you actually need.</li></ul><h2 id="sources" tabindex="-1">Sources <a class="header-anchor" href="#sources" aria-label="Permalink to &quot;Sources&quot;">​</a></h2><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><a href="https://raw.githubusercontent.com/spoddutur/spark-notes/master/task_memory_management_in_spark.md" target="_blank" rel="noreferrer">Task Memory Management in Spark</a> <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a> <a href="#fnref1:2" class="footnote-backref">↩︎</a> <a href="#fnref1:3" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/tuning.html" target="_blank" rel="noreferrer">Tuning Spark</a> <a href="#fnref2" class="footnote-backref">↩︎</a> <a href="#fnref2:1" class="footnote-backref">↩︎</a> <a href="#fnref2:2" class="footnote-backref">↩︎</a> <a href="#fnref2:3" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><a href="https://luminousmen.com/post/dive-into-spark-memory" target="_blank" rel="noreferrer">Dive into Spark Memory</a> <a href="#fnref3" class="footnote-backref">↩︎</a> <a href="#fnref3:1" class="footnote-backref">↩︎</a> <a href="#fnref3:2" class="footnote-backref">↩︎</a> <a href="#fnref3:3" class="footnote-backref">↩︎</a> <a href="#fnref3:4" class="footnote-backref">↩︎</a> <a href="#fnref3:5" class="footnote-backref">↩︎</a> <a href="#fnref3:6" class="footnote-backref">↩︎</a> <a href="#fnref3:7" class="footnote-backref">↩︎</a> <a href="#fnref3:8" class="footnote-backref">↩︎</a> <a href="#fnref3:9" class="footnote-backref">↩︎</a> <a href="#fnref3:10" class="footnote-backref">↩︎</a> <a href="#fnref3:11" class="footnote-backref">↩︎</a> <a href="#fnref3:12" class="footnote-backref">↩︎</a> <a href="#fnref3:13" class="footnote-backref">↩︎</a> <a href="#fnref3:14" class="footnote-backref">↩︎</a> <a href="#fnref3:15" class="footnote-backref">↩︎</a> <a href="#fnref3:16" class="footnote-backref">↩︎</a> <a href="#fnref3:17" class="footnote-backref">↩︎</a> <a href="#fnref3:18" class="footnote-backref">↩︎</a> <a href="#fnref3:19" class="footnote-backref">↩︎</a> <a href="#fnref3:20" class="footnote-backref">↩︎</a> <a href="#fnref3:21" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration — Spark</a> <a href="#fnref4" class="footnote-backref">↩︎</a> <a href="#fnref4:1" class="footnote-backref">↩︎</a> <a href="#fnref4:2" class="footnote-backref">↩︎</a> <a href="#fnref4:3" class="footnote-backref">↩︎</a> <a href="#fnref4:4" class="footnote-backref">↩︎</a> <a href="#fnref4:5" class="footnote-backref">↩︎</a> <a href="#fnref4:6" class="footnote-backref">↩︎</a> <a href="#fnref4:7" class="footnote-backref">↩︎</a> <a href="#fnref4:8" class="footnote-backref">↩︎</a> <a href="#fnref4:9" class="footnote-backref">↩︎</a> <a href="#fnref4:10" class="footnote-backref">↩︎</a></p></li><li id="fn5" class="footnote-item"><p><a href="https://www.databricks.com/blog/2015/04/28/project-tungsten-bringing-spark-closer-to-bare-metal.html" target="_blank" rel="noreferrer">Project Tungsten: Bringing Spark Closer to Bare Metal</a> <a href="#fnref5" class="footnote-backref">↩︎</a> <a href="#fnref5:1" class="footnote-backref">↩︎</a></p></li><li id="fn6" class="footnote-item"><p><em>Learning Spark, 2nd Edition</em>, Damji, Wenig, Das, Lee, ch. 6–7 <a href="#fnref6" class="footnote-backref">↩︎</a> <a href="#fnref6:1" class="footnote-backref">↩︎</a> <a href="#fnref6:2" class="footnote-backref">↩︎</a></p></li><li id="fn7" class="footnote-item"><p><a href="https://luminousmen.com/post/explaining-the-mechanics-of-spark-caching" target="_blank" rel="noreferrer">Explaining the Mechanics of Spark Caching</a> <a href="#fnref7" class="footnote-backref">↩︎</a> <a href="#fnref7:1" class="footnote-backref">↩︎</a> <a href="#fnref7:2" class="footnote-backref">↩︎</a> <a href="#fnref7:3" class="footnote-backref">↩︎</a> <a href="#fnref7:4" class="footnote-backref">↩︎</a></p></li><li id="fn8" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/rdd-programming-guide.html" target="_blank" rel="noreferrer">RDD Programming Guide</a> <a href="#fnref8" class="footnote-backref">↩︎</a></p></li><li id="fn9" class="footnote-item"><p><em>High Performance Spark, 2nd Edition</em>, Karau, Polak &amp; Warren, ch. 7 <a href="#fnref9" class="footnote-backref">↩︎</a> <a href="#fnref9:1" class="footnote-backref">↩︎</a> <a href="#fnref9:2" class="footnote-backref">↩︎</a> <a href="#fnref9:3" class="footnote-backref">↩︎</a></p></li><li id="fn10" class="footnote-item"><p><a href="https://raw.githubusercontent.com/apache/spark/v3.5.0/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala" target="_blank" rel="noreferrer">SQLConf.scala</a> <a href="#fnref10" class="footnote-backref">↩︎</a> <a href="#fnref10:1" class="footnote-backref">↩︎</a></p></li><li id="fn11" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/monitoring.html#spark-history-server" target="_blank" rel="noreferrer">Monitoring and Instrumentation</a> <a href="#fnref11" class="footnote-backref">↩︎</a> <a href="#fnref11:1" class="footnote-backref">↩︎</a></p></li></ol></section>',45)])])}const k=o(c,[["render",l]]);export{y as __pageData,k as default};

package/public/docs/assets/{tuning-reference_memory-model.md.DhT-n4y3.lean.js → tuning-reference_memory-model.md.CWhvgevP.lean.js}

@@ -1 +1 @@
-import{_ as o,o as a,c as r,a5 as t}from"./chunks/framework.DSg0KOwT.js";const s="/docs/assets/memory-regions.XHvO7jHG.svg",n="/docs/assets/memory-regions.dark.D4TP9_08.svg",f="/docs/assets/memory-borrowing.BqQRJg0u.svg",i="/docs/assets/memory-borrowing.dark.Yhh20O9C.svg",y=JSON.parse('{"title":"Memory Management","description":"","frontmatter":{"title":"Memory Management"},"headers":[],"relativePath":"tuning-reference/memory-model.md","filePath":"tuning-reference/memory-model.md"}'),c={name:"tuning-reference/memory-model.md"};function l(h,e,d,p,u,m){return a(),r("div",null,[...e[0]||(e[0]=[t("",34)])])}const k=o(c,[["render",l]]);export{y as __pageData,k as default};
+import{_ as o,o as a,c as t,a5 as r}from"./chunks/framework.DSg0KOwT.js";const s="/docs/assets/memory-regions.XHvO7jHG.svg",n="/docs/assets/memory-regions.dark.D4TP9_08.svg",i="/docs/assets/memory-borrowing.BqQRJg0u.svg",f="/docs/assets/memory-borrowing.dark.Yhh20O9C.svg",y=JSON.parse('{"title":"Memory Management","description":"","frontmatter":{"title":"Memory Management"},"headers":[],"relativePath":"tuning-reference/memory-model.md","filePath":"tuning-reference/memory-model.md"}'),c={name:"tuning-reference/memory-model.md"};function l(h,e,d,p,u,m){return a(),t("div",null,[...e[0]||(e[0]=[r("",45)])])}const k=o(c,[["render",l]]);export{y as __pageData,k as default};

package/public/docs/assets/{user-guide_alternative-log-retrieval.md.sU3KGarf.js → user-guide_alternative-log-retrieval.md.B4tPGIal.js}

@@ -1 +1 @@
-import{_ as t,o as a,c as s,a5 as o}from"./chunks/framework.DSg0KOwT.js";const u=JSON.parse('{"title":"Alternative ways to get the logs","description":"","frontmatter":{},"headers":[],"relativePath":"user-guide/alternative-log-retrieval.md","filePath":"user-guide/alternative-log-retrieval.md"}'),i={name:"user-guide/alternative-log-retrieval.md"};function r(h,e,l,n,d,p){return a(),s("div",null,[...e[0]||(e[0]=[o('<h1 id="alternative-ways-to-get-the-logs" tabindex="-1">Alternative ways to get the logs <a class="header-anchor" href="#alternative-ways-to-get-the-logs" aria-label="Permalink to &quot;Alternative ways to get the logs&quot;">​</a></h1><p><code>--shs-base-url</code> is the direct route: point SparkForensics at the History Server and it fetches the event log itself. This page covers what to do when that direct route isn&#39;t available.</p><h2 id="when-you-need-this" tabindex="-1">When you need this <a class="header-anchor" href="#when-you-need-this" aria-label="Permalink to &quot;When you need this&quot;">​</a></h2><p>The most common case is a Spark History Server (SHS) reachable only through an SSH bastion or jump host (the recipe below is the same regardless of what&#39;s issuing the SSH session), with the event logs themselves living on Kerberized HDFS and no direct HTTP path from your machine to SHS.</p><h2 id="why-shs-base-url-won-t-work-here" tabindex="-1">Why <code>--shs-base-url</code> won&#39;t work here <a class="header-anchor" href="#why-shs-base-url-won-t-work-here" aria-label="Permalink to &quot;Why `--shs-base-url` won&#39;t work here&quot;">​</a></h2><p>The CLI&#39;s <code>--shs-base-url</code>/<code>--app-id</code> flags and the MCP tools&#39; <code>{ shsBaseUrl, appId }</code> source both make a direct HTTP request from wherever SparkForensics runs to the History Server&#39;s REST API. If that address is only reachable from inside a bastion, that request never lands: there&#39;s no <code>--via-ssh</code> flag or other way to route it through an SSH session.</p><h2 id="the-recipe" tabindex="-1">The recipe <a class="header-anchor" href="#the-recipe" aria-label="Permalink to &quot;The recipe&quot;">​</a></h2><ol><li><p>SSH into the edge node (through the bastion, or whatever gets you there).</p></li><li><p>Pull the event log off HDFS onto local disk on the edge node:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">hdfs</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> dfs</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> -get</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> /path/to/spark-events/application_XXXX_XXXX</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> /tmp/application_XXXX_XXXX</span></span></code></pre></div></li><li><p>Copy that file back to your own machine through the same bastion, e.g. with <code>scp</code> or <code>sftp</code>:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">scp</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> edge-node:/tmp/application_XXXX_XXXX</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> ./application_XXXX_XXXX</span></span></code></pre></div></li><li><p>Point SparkForensics at the local copy instead of a <code>shsBaseUrl</code>:</p><ul><li>CLI: <code>sparkforensics-analyze ./application_XXXX_XXXX</code></li><li>MCP: call a tool with <code>{ &quot;source&quot;: { &quot;path&quot;: &quot;./application_XXXX_XXXX&quot; } }</code></li></ul></li></ol><p>The file is the app&#39;s native input format either way: a single event-log file, optionally <code>.gz</code>/<code>.zstd</code>/<code>.lz4</code>/<code>.snappy</code>-compressed, or a rolling <code>eventlog_v2_*</code> directory pulled the same way. No conversion step needed.</p>',9)])])}const g=t(i,[["render",r]]);export{u as __pageData,g as default};
+import{_ as t,o as a,c as s,a5 as o}from"./chunks/framework.DSg0KOwT.js";const g=JSON.parse('{"title":"Alternative ways to get the logs","description":"","frontmatter":{},"headers":[],"relativePath":"user-guide/alternative-log-retrieval.md","filePath":"user-guide/alternative-log-retrieval.md"}'),i={name:"user-guide/alternative-log-retrieval.md"};function r(l,e,h,n,d,p){return a(),s("div",null,[...e[0]||(e[0]=[o('<h1 id="alternative-ways-to-get-the-logs" tabindex="-1">Alternative ways to get the logs <a class="header-anchor" href="#alternative-ways-to-get-the-logs" aria-label="Permalink to &quot;Alternative ways to get the logs&quot;">​</a></h1><p><code>--shs-base-url</code> is the direct route: point SparkForensics at the History Server and it fetches the event log itself. This page covers what to do when that direct route isn&#39;t available.</p><h2 id="when-you-need-this" tabindex="-1">When you need this <a class="header-anchor" href="#when-you-need-this" aria-label="Permalink to &quot;When you need this&quot;">​</a></h2><p>The most common case is a Spark History Server (SHS) reachable only through an SSH bastion or jump host (the recipe below is the same regardless of what&#39;s issuing the SSH session), with the event logs themselves living on Kerberized HDFS and no direct HTTP path from your machine to SHS.</p><h2 id="why-shs-base-url-won-t-work-here" tabindex="-1">Why <code>--shs-base-url</code> won&#39;t work here <a class="header-anchor" href="#why-shs-base-url-won-t-work-here" aria-label="Permalink to &quot;Why `--shs-base-url` won&#39;t work here&quot;">​</a></h2><p>The CLI&#39;s <code>--shs-base-url</code>/<code>--app-id</code> flags and the MCP tools&#39; <code>{ shsBaseUrl, appId }</code> source both make a direct HTTP request from wherever SparkForensics runs to the History Server&#39;s REST API. If that address is only reachable from inside a bastion, that request never lands: there&#39;s no <code>--via-ssh</code> flag or other way to route it through an SSH session.</p><h2 id="the-recipe" tabindex="-1">The recipe <a class="header-anchor" href="#the-recipe" aria-label="Permalink to &quot;The recipe&quot;">​</a></h2><ol><li><p>SSH into the edge node (through the bastion, or whatever gets you there).</p></li><li><p>Pull the event log off HDFS onto local disk on the edge node:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">hdfs</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> dfs</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> -get</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> /path/to/spark-events/application_XXXX_XXXX</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> /tmp/application_XXXX_XXXX</span></span></code></pre></div></li><li><p>Copy that file back to your own machine through the same bastion, e.g. with <code>scp</code> or <code>sftp</code>:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">scp</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> edge-node:/tmp/application_XXXX_XXXX</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> ./application_XXXX_XXXX</span></span></code></pre></div></li><li><p>Point SparkForensics at the local copy instead of a <code>shsBaseUrl</code>:</p><ul><li>Browser: drop the file onto the landing page, same as any other run (see <a href="./getting-started.html">Getting started</a>).</li><li>CLI: <code>npx sparkforensics-analyze ./application_XXXX_XXXX</code> (full flag list in <a href="./getting-started.html#ci-and-automation">Getting started</a>).</li><li>MCP: call a tool with <code>{ &quot;source&quot;: { &quot;path&quot;: &quot;./application_XXXX_XXXX&quot; } }</code> (tool reference in <a href="./mcp-tools.html">MCP tools reference</a>).</li></ul></li></ol><p>The file is the app&#39;s native input format either way: a single event-log file, optionally <code>.gz</code>/<code>.zstd</code>/<code>.lz4</code>/<code>.snappy</code>-compressed, or a rolling <code>eventlog_v2_*</code> directory pulled the same way. No conversion step needed.</p>',9)])])}const u=t(i,[["render",r]]);export{g as __pageData,u as default};

package/public/docs/assets/{user-guide_alternative-log-retrieval.md.sU3KGarf.lean.js → user-guide_alternative-log-retrieval.md.B4tPGIal.lean.js}

@@ -1 +1 @@
-import{_ as t,o as a,c as s,a5 as o}from"./chunks/framework.DSg0KOwT.js";const u=JSON.parse('{"title":"Alternative ways to get the logs","description":"","frontmatter":{},"headers":[],"relativePath":"user-guide/alternative-log-retrieval.md","filePath":"user-guide/alternative-log-retrieval.md"}'),i={name:"user-guide/alternative-log-retrieval.md"};function r(h,e,l,n,d,p){return a(),s("div",null,[...e[0]||(e[0]=[o("",9)])])}const g=t(i,[["render",r]]);export{u as __pageData,g as default};
+import{_ as t,o as a,c as s,a5 as o}from"./chunks/framework.DSg0KOwT.js";const g=JSON.parse('{"title":"Alternative ways to get the logs","description":"","frontmatter":{},"headers":[],"relativePath":"user-guide/alternative-log-retrieval.md","filePath":"user-guide/alternative-log-retrieval.md"}'),i={name:"user-guide/alternative-log-retrieval.md"};function r(l,e,h,n,d,p){return a(),s("div",null,[...e[0]||(e[0]=[o("",9)])])}const u=t(i,[["render",r]]);export{g as __pageData,u as default};

package/public/docs/assets/user-guide_getting-started.md.BJvwLEIM.js

@@ -0,0 +1,3 @@
+import{_ as a,o as t,c as o,a5 as s}from"./chunks/framework.DSg0KOwT.js";const g=JSON.parse('{"title":"Getting started","description":"","frontmatter":{},"headers":[],"relativePath":"user-guide/getting-started.md","filePath":"user-guide/getting-started.md"}'),r={name:"user-guide/getting-started.md"};function n(i,e,d,l,h,c){return t(),o("div",null,[...e[0]||(e[0]=[s(`<h1 id="getting-started" tabindex="-1">Getting started <a class="header-anchor" href="#getting-started" aria-label="Permalink to &quot;Getting started&quot;">​</a></h1><p>SparkForensics reads a Spark History Server event log and turns it into a dashboard of flagged bottlenecks. No install and no account: open the app in a browser and drop a log in.</p><h2 id="load-a-run" tabindex="-1">Load a run <a class="header-anchor" href="#load-a-run" aria-label="Permalink to &quot;Load a run&quot;">​</a></h2><p>Drop a log file onto the landing page, or click <strong>Choose file</strong> to pick one. Parsing runs in a background worker, off the browser&#39;s main thread, so a multi-hundred-megabyte event stream doesn&#39;t freeze the tab.</p><p>No log of your own yet? Click <strong>Try a sample run</strong> on the landing page to load a bundled example run and see a populated dashboard right away.</p><p>The app takes a newline-delimited JSON event log (one JSON event per line, the format Spark writes to <code>spark.eventLog.dir</code>), either plain or gzip/Zstandard/LZ4/Snappy-compressed.</p><p>Click <strong>Other sources</strong> on the landing page for two more ways in:</p><ul><li><strong>Choose rolling-log folder</strong>, for an <code>eventlog_v2_*</code> rolling directory. Drop the folder or point the picker at it; the app reassembles its parts in order before parsing.</li><li><strong>Fetch from Spark History Server</strong>, to pull an application straight from a reachable History Server. This needs <strong>local-server mode</strong> (see below) and an application ID in one of the forms Spark itself uses: <code>application_&lt;timestamp&gt;_&lt;id&gt;</code>, <code>local-&lt;timestamp&gt;</code>, <code>app-&lt;id&gt;</code>, <code>spark-&lt;id&gt;</code>, or <code>driver-&lt;id&gt;</code>. If the fetch fails, the panel names the problem (server unreachable, application not found, local server not running, and so on) and suggests what to try next.</li></ul><p>Can&#39;t reach the History Server directly (it&#39;s only reachable through an SSH bastion)? See <a href="./alternative-log-retrieval.html">Alternative ways to get the logs</a>.</p><p>A History Server export needs one manual step. <code>GET /api/v1/applications/&lt;appId&gt;/logs</code> hands back a zip, and the drop zone does not unwrap zip containers. Extract the <code>.zstd</code> event log from that zip yourself and drop the extracted file in. It needs no further decompressing.</p><p>Files you have already loaded stay listed under <strong>Recent files</strong> on the landing page.</p><h3 id="local-server-mode" tabindex="-1">Local-server mode <a class="header-anchor" href="#local-server-mode" aria-label="Permalink to &quot;Local-server mode&quot;">​</a></h3><p>The plain browser app can&#39;t fetch from a History Server itself: the browser&#39;s CORS policy blocks a cross-origin request like that, and there&#39;s no server on the other end to proxy it. Local-server mode adds that server. Run it with:</p><div class="language-sh vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">npx</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> sparkforensics-server</span></span></code></pre></div><p>It listens on <code>http://127.0.0.1:4173</code> by default and binds to localhost only. Open that URL instead of the static build, and the <strong>Fetch from Spark History Server</strong> panel works because requests now go server-to-server. Full setup and its port/environment-variable overrides are in the <a href="https://github.com/shuffle-works/sparkforensics#readme" target="_blank" rel="noreferrer">project README</a>, which also covers what changes for a static (no-server) deploy.</p><h2 id="reading-the-dashboard" tabindex="-1">Reading the dashboard <a class="header-anchor" href="#reading-the-dashboard" aria-label="Permalink to &quot;Reading the dashboard&quot;">​</a></h2><p>When parsing finishes, the dashboard shows a board of widgets, each in its own card with a title. Every widget that flags a problem uses the same convention: a colored impact dot (critical / warning / info) and an ALL-CAPS tag for the bottleneck category (<code>SKEW</code>, <code>SPILL</code>, <code>GC</code>, and so on: see <a href="./understanding-findings.html">Understanding findings</a>). Widgets list every affected stage, not just the worst one. The dot&#39;s color tracks how much run time the finding could save you rather than how unusual the metric looks, so a small-looking anomaly with a big payoff can outrank a dramatic one that would barely move your run time.</p><p>A run scorecard (wall-clock, efficiency, wastage) always shows at the top. Below it, two tabs split the rest of the board:</p><ol><li><strong>Findings</strong>: every flagged finding and its detail widget, grouped by impact band (Critical, Warning, Info). Within a band, a recommendation row for a bottleneck type collapses into a summary row when it fires more than once; clicking the summary row expands its full list, and clicking any single row or widget jumps straight to that finding. Below the impact-band groups, memory and core-usage utilization always show, even on a clean run. Widgets that found nothing fold away into a &quot;Clean checks&quot; disclosure. This is the tab you land on.</li><li><strong>Full app report</strong>: the wall-clock and executor timelines, the stage table, and the reference-only cards.</li></ol><p>Click a finding&#39;s documentation link (or the topbar&#39;s <strong>Docs</strong> button) to open the reference material in a slide-in panel beside the dashboard: the dashboard stays visible and interactive, so you can check a metric against the reference without losing your place.</p><h3 id="advanced-view" tabindex="-1">Advanced view <a class="header-anchor" href="#advanced-view" aria-label="Permalink to &quot;Advanced view&quot;">​</a></h3><p>The topbar has an <strong>Advanced view</strong> toggle. It&#39;s off by default, which keeps each widget to the finding itself and what to do about it. Turn it on to also show confidence levels, supporting evidence, and documentation links for each finding, plus a few extra table columns. Your choice is remembered across runs.</p><h3 id="the-rest-of-the-topbar" tabindex="-1">The rest of the topbar <a class="header-anchor" href="#the-rest-of-the-topbar" aria-label="Permalink to &quot;The rest of the topbar&quot;">​</a></h3><p>Once a run is loaded, the topbar also carries a few more controls. <strong>New analysis</strong> goes back to the landing page to load another run. <strong>Plan graph</strong> opens an interactive node-and-edge view of the run&#39;s SQL execution plan, filterable down to I/O operators (scan, exchange), a broader &quot;basic&quot; set, or every operator. <strong>Export evidence</strong> downloads the current run&#39;s findings as a portable Markdown or JSON report, the same shape the CLI and MCP tools produce; turn on <strong>Redact identifiers</strong> first if the report is headed outside the environment that produced it, since that pseudonymizes the app id and any host/IP tokens. <strong>Keyboard shortcuts</strong> (press <code>?</code> from anywhere) lists every shortcut. Rounding it out: a <strong>Docs</strong> link and a theme toggle.</p><h2 id="compare-two-runs" tabindex="-1">Compare two runs <a class="header-anchor" href="#compare-two-runs" aria-label="Permalink to &quot;Compare two runs&quot;">​</a></h2><p>To compare a baseline run against a candidate, say to check whether a tuning change helped, see <a href="./run-comparison.html">Run comparison mode</a>.</p><h2 id="ci-and-automation" tabindex="-1">CI and automation <a class="header-anchor" href="#ci-and-automation" aria-label="Permalink to &quot;CI and automation&quot;">​</a></h2><p>In a pipeline or a script, use the <code>analyze</code> CLI instead of the browser dashboard. It parses the same event logs and exits non-zero when a run crosses a threshold you set, so it can gate a build:</p><div class="language-sh vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">npx</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> sparkforensics-analyze</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> &lt;</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">file</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">|</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">dir</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">&gt; [--max-runtime </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">ms]</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> [--max-skew </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">ratio]</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  [--max-spill </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">gb]</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> [--max-failed-task-rate </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">pct]</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> [--min-efficiency </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">pct]</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \\</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  [--out </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">path]</span></span></code></pre></div><p>Point it at a single event log or a directory of them. Pass <code>--out</code> to also write the findings to a file.</p><p>Pass <code>--export-html &lt;dir&gt;</code> to write a self-contained HTML dashboard for the run into <code>&lt;dir&gt;</code> (which must not already exist or must be empty). Open <code>&lt;dir&gt;/index.html</code> directly in a browser over <code>file://</code>, with no server, to get the same interactive dashboard offline. This makes it easy to archive or share a run. <code>--redact</code> applies to the exported report too.</p><p>The CLI also supports fetching a run directly from a reachable Spark History Server (<code>--shs-base-url</code>/<code>--app-id</code>/<code>--attempt-id</code>) instead of a local file, comparing a candidate run against a baseline with regression gating (<code>--baseline</code>/<code>--max-regression-pct</code>/<code>--regression-metric</code>/ <code>--fail-on-introduced</code>), redacting the app id and any host/IP tokens before sharing output (<code>--redact</code>), and narrowing the findings to certain impact bands, types, or a stage (<code>--impact</code>/<code>--type</code>/<code>--stage</code>). Run it with <code>--help</code> for the full flag list.</p><p>Same caveat as above: if the History Server is only reachable through an SSH bastion, <code>--shs-base-url</code> can&#39;t reach it either: see <a href="./alternative-log-retrieval.html">Alternative ways to get the logs</a>.</p><p>Running in Airflow instead of a plain CI pipeline? See <a href="https://github.com/shuffle-works/sparkforensics-operator" target="_blank" rel="noreferrer">sparkforensics-operator</a>, an Airflow operator that wraps the CLI and acts on the result after each Spark job, so you don&#39;t have to wire up the call yourself.</p><p>Want an AI assistant to diagnose a run directly, without the dashboard or a CI gate? See <a href="./mcp-tools.html">MCP tools reference</a>.</p>`,35)])])}const u=a(r,[["render",n]]);export{g as __pageData,u as default};

package/public/docs/assets/user-guide_getting-started.md.BJvwLEIM.lean.js

@@ -0,0 +1 @@
+import{_ as a,o as t,c as o,a5 as s}from"./chunks/framework.DSg0KOwT.js";const g=JSON.parse('{"title":"Getting started","description":"","frontmatter":{},"headers":[],"relativePath":"user-guide/getting-started.md","filePath":"user-guide/getting-started.md"}'),r={name:"user-guide/getting-started.md"};function n(i,e,d,l,h,c){return t(),o("div",null,[...e[0]||(e[0]=[s("",35)])])}const u=a(r,[["render",n]]);export{g as __pageData,u as default};