npm - sparkforensics-cli - Versions diffs - 0.1.0 → 0.2.0 - Mend

sparkforensics-cli 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (360) hide show

package/export-template/docs/assets/tuning-reference_config.md.UvveiWG3.js ADDED Viewed

@@ -0,0 +1 @@

+ import{_ as a,o,c as t,a5 as r}from"./chunks/framework.DSg0KOwT.js";const u=JSON.parse('{"title":"Spark Config Quick-Reference","description":"","frontmatter":{"title":"Spark Config Quick-Reference"},"headers":[],"relativePath":"tuning-reference/config.md","filePath":"tuning-reference/config.md"}'),s={name:"tuning-reference/config.md"};function f(n,e,c,i,d,l){return o(),t("div",null,[...e[0]||(e[0]=[r('<h1 id="config" tabindex="-1">Spark Config Quick-Reference <a class="header-anchor" href="#config" aria-label="Permalink to "Spark Config Quick-Reference {#config}""></a></h1>A cross-reference of the Spark and PySpark configuration properties discussed elsewhere in this guide, plus the JVM garbage-collection flags relevant to executor tuning. Defaults and version notes below are traced to Spark's own configuration docs, its SQLConf source, and the cited literature. Where a source doesn't give a hard number (a range, a ceiling, a recommended workload profile), the table leaves it out rather than guess.<h2 id="spark-properties" tabindex="-1">Spark properties <a class="header-anchor" href="#spark-properties" aria-label="Permalink to "Spark properties""></a></h2><table tabindex="0"><thead><tr><th>Key</th><th>Default</th><th>Recommended</th><th>Notes</th></tr></thead><tbody><tr><td><code>spark.sql.shuffle.partitions</code></td><td>200 (unchanged since Spark 1.1.0)<a href="#fn1" id="fnref1">[1]</a><a href="#fn2" id="fnref2">[2]</a><a href="#fn3" id="fnref3">[3]</a></td><td></td><td>AQE doesn't override this config value; see notes below for the effective runtime count.</td></tr><tr><td><code>spark.sql.adaptive.enabled</code></td><td><code>true</code><a href="#fn4" id="fnref4">[4]</a></td><td><code>true</code></td><td></td></tr><tr><td><code>spark.sql.adaptive.coalescePartitions.enabled</code></td><td><code>true</code> (since 3.0.0)<a href="#fn1" id="fnref1:1">[1:1]</a></td><td><code>true</code></td><td>Requires AQE; merges contiguous small post-shuffle partitions instead of one task per configured partition<a href="#fn1" id="fnref1:2">[1:2]</a>.</td></tr><tr><td><code>spark.sql.adaptive.coalescePartitions.initialPartitionNum</code></td><td>unset → falls back to <code>spark.sql.shuffle.partitions</code> (200)<a href="#fn1" id="fnref1:3">[1:3]</a></td><td></td><td>Sets the partition count entering the coalescing step.</td></tr><tr><td><code>spark.sql.adaptive.coalescePartitions.parallelismFirst</code></td><td><code>true</code> (since 3.2.0)<a href="#fn1" id="fnref1:4">[1:4]</a><a href="#fn5" id="fnref5">[5]</a></td><td><code>false</code> on busy clusters<a href="#fn1" id="fnref1:5">[1:5]</a></td><td>When <code>true</code>, ignores <code>advisoryPartitionSizeInBytes</code> and derives a target from the cluster's default parallelism, respecting only the <code>minPartitionSize</code> floor.</td></tr><tr><td><code>spark.sql.adaptive.advisoryPartitionSizeInBytes</code></td><td>64MB (since 3.0.0)<a href="#fn1" id="fnref1:6">[1:6]</a><a href="#fn4" id="fnref4:1">[4:1]</a><a href="#fn5" id="fnref5:1">[5:1]</a></td><td></td><td>Ignored during coalescing when <code>parallelismFirst=true</code> (the default).</td></tr><tr><td><code>spark.sql.adaptive.coalescePartitions.minPartitionSize</code></td><td>1MB (since 3.2.0)<a href="#fn5" id="fnref5:2">[5:2]</a></td><td></td><td>Floor enforced when the target size is ignored: the default <code>parallelismFirst=true</code> case<a href="#fn1" id="fnref1:7">[1:7]</a>.</td></tr><tr><td><code>spark.memory.fraction</code></td><td>0.6<a href="#fn6" id="fnref6">[6]</a></td><td></td><td>See notes below; no hard 0–1 ceiling documented in the cited sources.</td></tr><tr><td><code>spark.memory.storageFraction</code></td><td>0.5<a href="#fn7" id="fnref7">[7]</a></td><td></td><td>A fraction of the region sized by <code>spark.memory.fraction</code>, not an independent pool; see notes below.</td></tr><tr><td><code>spark.sql.execution.arrow.pyspark.enabled</code></td><td><code>false</code><a href="#fn8" id="fnref8">[8]</a></td><td></td><td>See PySpark callout below.</td></tr><tr><td><code>spark.sql.execution.arrow.pyspark.fallback.enabled</code></td><td><code>true</code> (inherited from the deprecated <code>arrow.fallback.enabled</code>)<a href="#fn8" id="fnref8:1">[8:1]</a><a href="#fn5" id="fnref5:3">[5:3]</a><a href="#fn4" id="fnref4:2">[4:2]</a></td><td></td><td>Falls back to the non-Arrow path automatically if a conversion error occurs, before computation runs<a href="#fn8" id="fnref8:2">[8:2]</a>.</td></tr><tr><td><code>spark.dynamicAllocation.shuffleTracking.enabled</code></td><td><code>true</code> (since 3.0.0)<a href="#fn4" id="fnref4:3">[4:3]</a></td><td></td><td>Lets dynamic allocation track shuffle files per executor without an external shuffle service.</td></tr><tr><td><code>spark.dynamicAllocation.shuffleTracking.timeout</code></td><td><code>infinity</code><a href="#fn4" id="fnref4:4">[4:4]</a></td><td></td><td>Executors holding shuffle data wait for it to be garbage collected before release, by default; set a finite value if GC isn't keeping up.</td></tr><tr><td><code>spark.shuffle.service.enabled</code></td><td></td><td></td><td>External shuffle service; see <code>#config-shuffle-service</code> below.</td></tr><tr><td><code>spark.dynamicAllocation.minExecutors</code> / <code>maxExecutors</code></td><td><code>0</code> / <code>infinity</code><a href="#fn4" id="fnref4:5">[4:5]</a></td><td></td><td>Autoscale bounds; see <code>#config-autoscale-bounds</code> below.</td></tr><tr><td><code>spark.serializer</code></td><td><code>org.apache.spark.serializer.JavaSerializer</code><a href="#fn4" id="fnref4:6">[4:6]</a></td><td><code>KryoSerializer</code></td><td>See <code>#config-serializer</code> below.</td></tr><tr><td><code>spark.executor.memoryOverhead</code></td><td>greater of 10% of executor memory or 384MB<a href="#fn7" id="fnref7:1">[7:1]</a></td><td></td><td>Off-heap overhead; see <code>#config-memory-overhead</code> below.</td></tr></tbody></table><h3 id="shuffle-partitions-and-aqe-coalescing" tabindex="-1">Shuffle partitions and AQE coalescing <a class="header-anchor" href="#shuffle-partitions-and-aqe-coalescing" aria-label="Permalink to "Shuffle partitions and AQE coalescing""></a></h3>The 200 default for <code>spark.sql.shuffle.partitions</code> hasn't moved since 1.1.0, and AQE (on by default since Spark 3.0) doesn't change that config value<a href="#fn1" id="fnref1:8">[1:8]</a><a href="#fn2" id="fnref2:1">[2:1]</a><a href="#fn3" id="fnref3:1">[3:1]</a>. What it changes is the effective number of partitions used at runtime. With <code>coalescePartitions.enabled</code> also on by default, Spark merges small contiguous post-shuffle partitions instead of running one task per configured partition<a href="#fn1" id="fnref1:9">[1:9]</a>. Because <code>parallelismFirst</code> defaults to <code>true</code> since 3.2.0, that merge target usually isn't the 64MB <code>advisoryPartitionSizeInBytes</code> value; it's derived from the cluster's default parallelism, with <code>minPartitionSize</code> (1MB) as the only enforced floor<a href="#fn1" id="fnref1:10">[1:10]</a><a href="#fn5" id="fnref5:4">[5:4]</a>. Databricks' own writeup on AQE shows the reduce-task count actually shrinking based on measured data volume<a href="#fn9" id="fnref9">[9]</a>, and Learning Spark accordingly calls the static 200 default "too high for smaller or streaming workloads"<a href="#fn3" id="fnref3:2">[3:2]</a>. A related, distinct knob (<code>spark.sql.adaptive.coalescePartitions.minPartitionNum</code>) sets a minimum parallelism floor for data that's slow to compute despite being small, per High Performance Spark<a href="#fn10" id="fnref10">[10]</a>.<h3 id="spark-memory-fraction-and-spark-memory-storagefraction" tabindex="-1"><code>spark.memory.fraction</code> and <code>spark.memory.storageFraction</code> <a class="header-anchor" href="#spark-memory-fraction-and-spark-memory-storagefraction" aria-label="Permalink to "`spark.memory.fraction` and `spark.memory.storageFraction`""></a></h3><code>spark.memory.fraction</code> sizes the unified execution/storage region (M) as a fraction of (JVM heap − 300MiB); the tuning guide frames it as a knob to fit M "comfortably within the JVM's old or tenured generation" rather than stating an enforced numeric range<a href="#fn6" id="fnref6:1">[6:1]</a>. The cited sources document no failure threshold tied to a specific value like 0.9. What they do document is the risk of pushing the fraction high: the complement, <code>1 − spark.memory.fraction</code>, is untracked "User Memory" for UDFs, Python/Arrow glue, and native buffers, and starving it (which is what raising the fraction toward 0.9 does) leads to "GC pressure or random OOMs," with no warning from Spark<a href="#fn7" id="fnref7:2">[7:2]</a>.<code>spark.memory.storageFraction</code> isn't independent of that: it's a fraction of M, the same region <code>spark.memory.fraction</code> sizes<a href="#fn6" id="fnref6:2">[6:2]</a>. Inside that shared pool, execution can evict storage down to the threshold set by <code>storageFraction</code>, but storage can never evict execution<a href="#fn11" id="fnref11">[11]</a><a href="#fn6" id="fnref6:3">[6:3]</a>. With the defaults (0.6 and 0.5), that works out to execution and storage each getting 30% of usable heap; raising <code>memory.fraction</code> scales both pools at once, while <code>storageFraction</code> only re-splits the pool that's already been carved out<a href="#fn7" id="fnref7:3">[7:3]</a>.<blockquote>PySpark: <code>spark.sql.execution.arrow.pyspark.enabled</code> is off by default and governs Arrow use for <code>DataFrame.toPandas()</code> and <code>SparkSession.createDataFrame()</code> from a Pandas DataFrame or NumPy array<a href="#fn8" id="fnref8:3">[8:3]</a>. Its documented risk is a type-coverage gap, not silent data corruption: <code>ArrayType</code> of <code>TimestampType</code> is explicitly unsupported<a href="#fn4" id="fnref4:7">[4:7]</a><a href="#fn5" id="fnref5:5">[5:5]</a>, and more generally an unsupported column type raises an error rather than converting wrongly<a href="#fn8" id="fnref8:4">[8:4]</a>. <code>spark.sql.execution.arrow.pyspark.fallback.enabled</code> defaults to <code>true</code> and automatically falls back to the non-Arrow path if a conversion error occurs, before any computation runs<a href="#fn8" id="fnref8:5">[8:5]</a><a href="#fn5" id="fnref5:6">[5:6]</a><a href="#fn4" id="fnref4:8">[4:8]</a>. So the failure mode is an exception plus fallback, not a wrong result. Separately, and not specific to this Arrow config, PySpark's own type coercion has its own hazards: numeric values passed for <code>ByteType</code>/<code>ShortType</code>/<code>IntegerType</code> must fall within fixed ranges or get rejected or converted unexpectedly<a href="#fn2" id="fnref2:2">[2:2]</a>.</blockquote><h3 id="config-shuffle-service" tabindex="-1">Shuffle service <a class="header-anchor" href="#config-shuffle-service" aria-label="Permalink to "Shuffle service {#config-shuffle-service}""></a></h3>During a shuffle, an executor writes its map output to local disk and then serves fetch requests for that data itself<a href="#fn12" id="fnref12">[12]</a>. Dynamic allocation can reclaim an idle executor before a later stage has fetched all the shuffle blocks it holds (especially with stragglers, tasks that run much longer than their peers), and once that executor is gone, any stage that still needs its shuffle output gets a <code>FetchFailed</code> and has to recompute it<a href="#fn12" id="fnref12:1">[12:1]</a>. The external shuffle service fixes this by moving shuffle-block serving out of the executor process: it's a long-running process on each node, independent of any particular application's executors, and once <code>spark.shuffle.service.enabled</code> is <code>true</code>, executors fetch shuffle blocks from it instead of from each other, so an executor's shuffle output keeps being served after that executor is reclaimed<a href="#fn12" id="fnref12:2">[12:2]</a><a href="#fn4" id="fnref4:9">[4:9]</a>. Setting up dynamic allocation requires enabling this service on every worker node in addition to <code>spark.dynamicAllocation.enabled</code><a href="#fn2" id="fnref2:3">[2:3]</a>. As of Spark 3.0, shuffle-file tracking (<code>spark.dynamicAllocation.shuffleTracking.enabled</code>) is an alternative to the external shuffle service for the same safe-removal problem<a href="#fn4" id="fnref4:10">[4:10]</a>.Limitations / false-positive risk: the service can be intentionally left off when dynamic allocation relies on shuffle-file tracking instead, so a disabled setting is not automatically a misconfiguration.<h3 id="config-autoscale-bounds" tabindex="-1">Autoscale bounds <a class="header-anchor" href="#config-autoscale-bounds" aria-label="Permalink to "Autoscale bounds {#config-autoscale-bounds}""></a></h3>By default, <code>spark.dynamicAllocation.minExecutors</code> is <code>0</code> and <code>spark.dynamicAllocation.maxExecutors</code> is <code>infinity</code><a href="#fn4" id="fnref4:11">[4:11]</a>. Leaving <code>maxExecutors</code> unset therefore leaves dynamic allocation unbounded on the high end as far as Spark itself is concerned; in practice the ceiling ends up being whatever the cluster or scheduler enforces outside Spark<a href="#fn4" id="fnref4:12">[4:12]</a>. <code>spark.dynamicAllocation.initialExecutors</code> defaults to <code>minExecutors</code>, unless <code>--num-executors</code> (or <code>spark.executor.instances</code>) sets a larger starting value<a href="#fn4" id="fnref4:13">[4:13]</a>. Whatever executor count <code>executorAllocationRatio</code> computes to maximize parallelism, that target is still clamped by the <code>minExecutors</code>/<code>maxExecutors</code> floor and ceiling<a href="#fn4" id="fnref4:14">[4:14]</a>.Limitations / false-positive risk: an unbounded <code>maxExecutors</code> is often deliberate on clusters where the scheduler enforces the real ceiling, so an unset high end is not always wrong.<h3 id="config-serializer" tabindex="-1">Serializer <a class="header-anchor" href="#config-serializer" aria-label="Permalink to "Serializer {#config-serializer}""></a></h3>The default <code>spark.serializer</code> is <code>org.apache.spark.serializer.JavaSerializer</code>, which works with any <code>Serializable</code> Java object but is "quite slow"; Spark's configuration reference recommends switching to <code>KryoSerializer</code> "when speed is necessary"<a href="#fn4" id="fnref4:15">[4:15]</a>. The tuning guide is more specific: Kryo is "significantly faster and more compact than Java serialization (often as much as 10x)," though it doesn't support all <code>Serializable</code> types and needs its classes registered in advance for best performance<a href="#fn6" id="fnref6:4">[6:4]</a>. That registration requirement is the stated reason Kryo isn't the default: "The only reason Kryo is not the default is because of the custom registration requirement," and Spark recommends trying it for any network-intensive application<a href="#fn6" id="fnref6:5">[6:5]</a>. Since Spark 2.0.0, Spark internally uses Kryo regardless of this setting when shuffling RDDs of simple types, arrays of simple types, or strings, and auto-registers common Scala classes via Twitter chill's <code>AllScalaRegistrar</code><a href="#fn6" id="fnref6:6">[6:6]</a>. A practitioner summary puts Java serialization at "2-10x slower and larger on the wire" on every shuffle, with the caveat that classes left unregistered silently fall back to Java serialization unless registered via <code>spark.kryo.classesToRegister</code> or <code>spark.kryo.registrator</code><a href="#fn13" id="fnref13">[13]</a>. Persisting RDDs in serialized form is another case where Kryo can be more space-efficient than Java serialization<a href="#fn10" id="fnref10:1">[10:1]</a>.Limitations / false-positive risk: <code>JavaSerializer</code> may be kept on purpose when an application depends on <code>Serializable</code> types Kryo cannot handle, so a non-Kryo setting is not necessarily a mistake.<h3 id="config-memory-overhead" tabindex="-1">Memory overhead <a class="header-anchor" href="#config-memory-overhead" aria-label="Permalink to "Memory overhead {#config-memory-overhead}""></a></h3><code>spark.executor.memoryOverhead</code> covers everything an executor needs outside the JVM heap sized by <code>spark.executor.memory</code>: thread stacks, JIT buffers, metaspace, JNI, native libraries, and, if <code>spark.executor.pyspark.memory</code> isn't set separately, the memory used by PySpark's per-task Python worker processes<a href="#fn7" id="fnref7:4">[7:4]</a>. Left unset, Spark defaults it to the greater of 10% of executor memory or 384MB<a href="#fn7" id="fnref7:5">[7:5]</a>; the configuration reference formalizes that floor as <code>spark.executor.minMemoryOverhead</code> (default <code>384m</code>, since Spark 4.0) and the percentage as <code>spark.executor.memoryOverheadFactor</code> (default 0.10, or 0.40 for non-JVM jobs on Kubernetes, since those need more non-JVM heap space)<a href="#fn4" id="fnref4:16">[4:16]</a>. The resource manager (YARN's NodeManager, or the Kubernetes kubelet) sizes the container/pod to the sum of <code>spark.executor.memoryOverhead</code>, <code>spark.executor.memory</code>, <code>spark.memory.offHeap.size</code>, and <code>spark.executor.pyspark.memory</code><a href="#fn4" id="fnref4:17">[4:17]</a>. If off-heap memory or PySpark worker processes consume memory that isn't reflected in a correspondingly larger <code>memoryOverhead</code>, that extra usage still shows up in the process's actual RSS, which the resource manager does track. Once real usage exceeds the container's allocated size, the executor is killed by YARN or OOMKilled by the kubelet, with no Spark-level error, just an abrupt process death<a href="#fn7" id="fnref7:6">[7:6]</a>.Limitations / false-positive risk: the default overhead is adequate for many JVM-only jobs, so a value left at the default only signals trouble on off-heap-heavy or PySpark workloads that need more non-heap room.<h2 id="jvm-gc-flags" tabindex="-1">JVM GC flags <a class="header-anchor" href="#jvm-gc-flags" aria-label="Permalink to "JVM GC flags""></a></h2><table tabindex="0"><thead><tr><th>Flag</th><th>Effect</th></tr></thead><tbody><tr><td><code>-XX:+UseG1GC</code></td><td>Default collector since Spark 4.0.0, which defaults to JDK 17<a href="#fn6" id="fnref6:7">[6:7]</a>.</td></tr><tr><td><code>-XX:G1HeapRegionSize</code></td><td>May need raising alongside large executor heaps<a href="#fn6" id="fnref6:8">[6:8]</a>.</td></tr><tr><td><code>-XX:InitiatingHeapOccupancyPercent</code></td><td>Tuned (with <code>-XX:ConcGCThreads</code> and RSet-update settings) to fix a documented ~100-second G1 full-GC pause on an 88GB executor heap<a href="#fn14" id="fnref14">[14]</a>.</td></tr><tr><td><code>-XX:ConcGCThreads</code></td><td>See <code>-XX:InitiatingHeapOccupancyPercent</code> above<a href="#fn14" id="fnref14:1">[14:1]</a>.</td></tr><tr><td><code>-XX:+UseZGC</code></td><td>Concurrent, low-latency collector; sub-millisecond pause target independent of heap size, from a few hundred megabytes up to 16TB<a href="#fn15" id="fnref15">[15]</a>.</td></tr></tbody></table><h3 id="g1-vs-zgc" tabindex="-1">G1 vs. ZGC <a class="header-anchor" href="#g1-vs-zgc" aria-label="Permalink to "G1 vs. ZGC""></a></h3>G1 was designed as a CMS replacement aiming at both throughput and low latency: it partitions the heap into equal-sized regions and copies out only the live objects from collected regions rather than compacting the whole heap<a href="#fn14" id="fnref14:2">[14:2]</a>. Even so, a documented 88GB-heap Spark benchmark hit "unacceptable full GC," with one job pausing nearly 100 seconds under default G1 settings. Only after tuning <code>InitiatingHeapOccupancyPercent</code>, <code>ConcGCThreads</code>, and RSet-update parameters did G1 beat Parallel/CMS GC on both throughput and latency<a href="#fn14" id="fnref14:3">[14:3]</a>. ZGC, per the OpenJDK project page, "performs all expensive work concurrently, without stopping the execution of application threads for more than a millisecond," with pause times that stay flat as heap size scales from a few hundred megabytes up to 16TB<a href="#fn15" id="fnref15:1">[15:1]</a>.<h2 id="sources" tabindex="-1">Sources <a class="header-anchor" href="#sources" aria-label="Permalink to "Sources""></a></h2><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><a href="https://spark.apache.org/docs/latest/sql-performance-tuning.html" target="_blank" rel="noreferrer">Performance Tuning — Spark SQL, DataFrames and Datasets Guide</a> <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a> <a href="#fnref1:2" class="footnote-backref">↩︎</a> <a href="#fnref1:3" class="footnote-backref">↩︎</a> <a href="#fnref1:4" class="footnote-backref">↩︎</a> <a href="#fnref1:5" class="footnote-backref">↩︎</a> <a href="#fnref1:6" class="footnote-backref">↩︎</a> <a href="#fnref1:7" class="footnote-backref">↩︎</a> <a href="#fnref1:8" class="footnote-backref">↩︎</a> <a href="#fnref1:9" class="footnote-backref">↩︎</a> <a href="#fnref1:10" class="footnote-backref">↩︎</a></li><li id="fn2" class="footnote-item">Spark: The Definitive Guide, Chambers & Zaharia, ch. 4, ch. 10 <a href="#fnref2" class="footnote-backref">↩︎</a> <a href="#fnref2:1" class="footnote-backref">↩︎</a> <a href="#fnref2:2" class="footnote-backref">↩︎</a> <a href="#fnref2:3" class="footnote-backref">↩︎</a></li><li id="fn3" class="footnote-item">Learning Spark, 2nd Edition, Damji, Wenig, Das & Lee, ch. 7 <a href="#fnref3" class="footnote-backref">↩︎</a> <a href="#fnref3:1" class="footnote-backref">↩︎</a> <a href="#fnref3:2" class="footnote-backref">↩︎</a></li><li id="fn4" class="footnote-item"><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration — Spark</a> <a href="#fnref4" class="footnote-backref">↩︎</a> <a href="#fnref4:1" class="footnote-backref">↩︎</a> <a href="#fnref4:2" class="footnote-backref">↩︎</a> <a href="#fnref4:3" class="footnote-backref">↩︎</a> <a href="#fnref4:4" class="footnote-backref">↩︎</a> <a href="#fnref4:5" class="footnote-backref">↩︎</a> <a href="#fnref4:6" class="footnote-backref">↩︎</a> <a href="#fnref4:7" class="footnote-backref">↩︎</a> <a href="#fnref4:8" class="footnote-backref">↩︎</a> <a href="#fnref4:9" class="footnote-backref">↩︎</a> <a href="#fnref4:10" class="footnote-backref">↩︎</a> <a href="#fnref4:11" class="footnote-backref">↩︎</a> <a href="#fnref4:12" class="footnote-backref">↩︎</a> <a href="#fnref4:13" class="footnote-backref">↩︎</a> <a href="#fnref4:14" class="footnote-backref">↩︎</a> <a href="#fnref4:15" class="footnote-backref">↩︎</a> <a href="#fnref4:16" class="footnote-backref">↩︎</a> <a href="#fnref4:17" class="footnote-backref">↩︎</a></li><li id="fn5" class="footnote-item"><a href="https://raw.githubusercontent.com/apache/spark/v3.5.0/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala" target="_blank" rel="noreferrer">SQLConf.scala (Spark 3.5.0)</a> <a href="#fnref5" class="footnote-backref">↩︎</a> <a href="#fnref5:1" class="footnote-backref">↩︎</a> <a href="#fnref5:2" class="footnote-backref">↩︎</a> <a href="#fnref5:3" class="footnote-backref">↩︎</a> <a href="#fnref5:4" class="footnote-backref">↩︎</a> <a href="#fnref5:5" class="footnote-backref">↩︎</a> <a href="#fnref5:6" class="footnote-backref">↩︎</a></li><li id="fn6" class="footnote-item"><a href="https://spark.apache.org/docs/latest/tuning.html" target="_blank" rel="noreferrer">Tuning Spark</a> <a href="#fnref6" class="footnote-backref">↩︎</a> <a href="#fnref6:1" class="footnote-backref">↩︎</a> <a href="#fnref6:2" class="footnote-backref">↩︎</a> <a href="#fnref6:3" class="footnote-backref">↩︎</a> <a href="#fnref6:4" class="footnote-backref">↩︎</a> <a href="#fnref6:5" class="footnote-backref">↩︎</a> <a href="#fnref6:6" class="footnote-backref">↩︎</a> <a href="#fnref6:7" class="footnote-backref">↩︎</a> <a href="#fnref6:8" class="footnote-backref">↩︎</a></li><li id="fn7" class="footnote-item"><a href="https://luminousmen.com/post/dive-into-spark-memory" target="_blank" rel="noreferrer">Dive Into Spark Memory Management</a> <a href="#fnref7" class="footnote-backref">↩︎</a> <a href="#fnref7:1" class="footnote-backref">↩︎</a> <a href="#fnref7:2" class="footnote-backref">↩︎</a> <a href="#fnref7:3" class="footnote-backref">↩︎</a> <a href="#fnref7:4" class="footnote-backref">↩︎</a> <a href="#fnref7:5" class="footnote-backref">↩︎</a> <a href="#fnref7:6" class="footnote-backref">↩︎</a></li><li id="fn8" class="footnote-item"><a href="https://spark.apache.org/docs/3.5.8/api/python/user_guide/sql/arrow_pandas.html" target="_blank" rel="noreferrer">Apache Arrow in PySpark</a> <a href="#fnref8" class="footnote-backref">↩︎</a> <a href="#fnref8:1" class="footnote-backref">↩︎</a> <a href="#fnref8:2" class="footnote-backref">↩︎</a> <a href="#fnref8:3" class="footnote-backref">↩︎</a> <a href="#fnref8:4" class="footnote-backref">↩︎</a> <a href="#fnref8:5" class="footnote-backref">↩︎</a></li><li id="fn9" class="footnote-item"><a href="https://www.databricks.com/blog/2020/05/29/adaptive-query-execution-speeding-up-spark-sql-at-runtime.html" target="_blank" rel="noreferrer">Adaptive Query Execution: Speeding Up Spark SQL at Runtime</a> <a href="#fnref9" class="footnote-backref">↩︎</a></li><li id="fn10" class="footnote-item">High Performance Spark, 2nd Edition, Karau, Polak & Warren, ch. 5 <a href="#fnref10" class="footnote-backref">↩︎</a> <a href="#fnref10:1" class="footnote-backref">↩︎</a></li><li id="fn11" class="footnote-item"><a href="https://raw.githubusercontent.com/spoddutur/spark-notes/master/task_memory_management_in_spark.md" target="_blank" rel="noreferrer">task_memory_management_in_spark.md</a> <a href="#fnref11" class="footnote-backref">↩︎</a></li><li id="fn12" class="footnote-item"><a href="https://spark.apache.org/docs/latest/job-scheduling.html" target="_blank" rel="noreferrer">Job Scheduling — Spark</a> <a href="#fnref12" class="footnote-backref">↩︎</a> <a href="#fnref12:1" class="footnote-backref">↩︎</a> <a href="#fnref12:2" class="footnote-backref">↩︎</a></li><li id="fn13" class="footnote-item"><a href="https://luminousmen.com/post/the-apache-spark-optimization-checklist" target="_blank" rel="noreferrer">The Apache Spark Optimization Checklist</a> <a href="#fnref13" class="footnote-backref">↩︎</a></li><li id="fn14" class="footnote-item"><a href="https://www.databricks.com/blog/2015/05/28/tuning-java-garbage-collection-for-spark-applications.html" target="_blank" rel="noreferrer">Tuning Java Garbage Collection for Spark Applications</a> <a href="#fnref14" class="footnote-backref">↩︎</a> <a href="#fnref14:1" class="footnote-backref">↩︎</a> <a href="#fnref14:2" class="footnote-backref">↩︎</a> <a href="#fnref14:3" class="footnote-backref">↩︎</a></li><li id="fn15" class="footnote-item"><a href="https://wiki.openjdk.org/display/zgc" target="_blank" rel="noreferrer">ZGC — The Z Garbage Collector</a> <a href="#fnref15" class="footnote-backref">↩︎</a> <a href="#fnref15:1" class="footnote-backref">↩︎</a></li></ol></section>',29)])])}const h=a(s,[["render",f]]);export{u as __pageData,h as default};

package/export-template/docs/assets/tuning-reference_config.md.UvveiWG3.lean.js ADDED Viewed

@@ -0,0 +1 @@

package/export-template/docs/assets/tuning-reference_data-formats.md.bjCAWH3N.js ADDED Viewed

@@ -0,0 +1 @@

+ import{_ as a,o as t,c as o,a5 as r}from"./chunks/framework.DSg0KOwT.js";const s="../assets/columnar-layout.PghGeOEA.svg",n="../assets/columnar-layout.dark.BVNlz0ff.svg",m=JSON.parse('{"title":"Data Formats","description":"","frontmatter":{"title":"Data Formats"},"headers":[],"relativePath":"tuning-reference/data-formats.md","filePath":"tuning-reference/data-formats.md"}'),f={name:"tuning-reference/data-formats.md"};function i(l,e,c,d,p,h){return t(),o("div",null,[...e[0]||(e[0]=[r('<h1 id="data-formats" tabindex="-1">Data Formats <a class="header-anchor" href="#data-formats" aria-label="Permalink to "Data Formats {#data-formats}""></a></h1><h2 id="how-parquet-and-orc-lay-out-data" tabindex="-1">How Parquet and ORC lay out data <a class="header-anchor" href="#how-parquet-and-orc-lay-out-data" aria-label="Permalink to "How Parquet and ORC lay out data""></a></h2>Parquet and ORC are both self-describing, columnar file formats for storing Spark's structured data<a href="#fn1" id="fnref1">[1]</a>. They're closer than they are different: Spark: The Definitive Guide frames the choice as "for the most part, they're quite similar; the fundamental difference is that Parquet is further optimized for use with Spark, whereas ORC is further optimized for Hive"<a href="#fn1" id="fnref1:1">[1:1]</a>, and notes that ORC "has no options for reading in data because Spark understands the file format quite well"<a href="#fn1" id="fnref1:2">[1:2]</a>: Spark treats it as a format it understands natively rather than one with extra read-side knobs.Both formats group rows into chunks (row groups in Parquet, stripes in ORC), each holding every column's data for that slice, with a footer or file-level metadata section recording where each chunk lives on disk. Parquet's <code>FileMetaData</code> footer records the offset and size of every row group and column chunk, and "readers are expected to first read the file metadata to find all the column chunks they are interested in. The column chunks should then be read sequentially"<a href="#fn2" id="fnref2">[2]</a>. Because compression is applied per column chunk rather than as one continuous stream over the whole file, a reader can seek straight to a row group's footer-recorded offset and decode just that chunk, which is exactly why Parquet files compressed with Snappy stay splittable even though the Snappy stream format itself provides no split points: it has "no entropy encoder backend nor framing layer -- the latter is assumed to be handled by other parts of the system"<a href="#fn3" id="fnref3">[3]</a>.<img class="light-only" src="'+s+'" alt="A columnar file's footer holds row-group offsets and per-column min/max statistics that let a reader skip row groups whose stats exclude the predicate."><img class="dark-only" src="'+n+'" alt="A columnar file's footer holds row-group offsets and per-column min/max statistics that let a reader skip row groups whose stats exclude the predicate.">Both formats also carry the statistics that later drive predicate pushdown. Parquet records min/max values per column chunk, plus an optional page-level <code>ColumnIndex</code> that lets a reader binary-search ordered columns for matching pages<a href="#fn4" id="fnref4">[4]</a>, and optional Bloom filters for columns whose cardinality is too high for a dictionary to be practical<a href="#fn5" id="fnref5">[5]</a>. ORC's <code>ColumnStatistics</code> protobuf records row count and, for most primitive types, min/max (plus sum for numeric types); from Hive 1.1.0 it also records a <code>hasNull</code> flag used specifically by ORC's predicate pushdown to answer <code>IS NULL</code> queries<a href="#fn6" id="fnref6">[6]</a>. ORC layers this at two granularities: a <code>RowIndexEntry</code> per row group (10,000 rows by default), kept at the front of each stripe so it's read only when pushdown or seeking is actually needed<a href="#fn7" id="fnref7">[7]</a>, and file-level <code>StripeStatistics</code> that let whole stripes be skipped by predicate pushdown<a href="#fn8" id="fnref8">[8]</a>. ORC also supports Bloom filters (Hive 1.2.0+), but only evaluates them against row groups that already passed the min/max row-index check first: a second-stage filter, not an independent one<a href="#fn7" id="fnref7:1">[7:1]</a>.On compression: Parquet defaults to Snappy, while ORC's default has been Zstd since Spark 2.3.0<a href="#fn9" id="fnref9">[9]</a>. Zstandard's own manual describes it as "a fast lossless compression algorithm, targeting real-time compression scenarios at zlib-level and better compression ratios," offering regular levels 1–22 plus negative levels that trade ratio for speed<a href="#fn10" id="fnref10">[10]</a>. Spark exposes the level directly through <code>spark.io.compression.zstd.level</code> (default <code>1</code>)<a href="#fn9" id="fnref9:1">[9:1]</a>, along with <code>spark.io.compression.zstd.workers</code> for parallel compression threads (default <code>0</code>, since 4.0.0) and <code>spark.io.compression.zstd.bufferSize</code> (default 32k)<a href="#fn9" id="fnref9:2">[9:2]</a>.Parquet has one more structural knob: <code>parquet.writer.version</code>. Version 2 changes the on-disk page-header encoding, and the spec is explicit that this is a forward-incompatible change: "a reader that only understands <code>DataPageHeader</code> cannot parse <code>DataPageHeaderV2</code> pages"<a href="#fn11" id="fnref11">[11]</a>. The spec also flags that the file's own version marker "has historically been used inconsistently: writers populate 1 or 2 without a consistent relationship to the features actually used"<a href="#fn11" id="fnref11:1">[11:1]</a>, so the marker itself isn't a reliable way to tell which page format a file actually contains.<h2 id="what-shows-up-at-read-time" tabindex="-1">What shows up at read time <a class="header-anchor" href="#what-shows-up-at-read-time" aria-label="Permalink to "What shows up at read time""></a></h2>A file's layout shapes the job the moment Spark reads it. Spark's file readers derive partition counts from file metadata rather than from a fixed rule: "for structured formats like Parquet, ORC, or Avro, Spark actually reads the metadata (footers, row groups, that kind of thing) and tries to slice the file in a way that makes sense. Often you'll see one partition per row group, though Spark may merge or split depending on file sizes and configs"<a href="#fn12" id="fnref12">[12]</a>. A concrete case: a single 30 GB Parquet file with 300 row groups gets split into exactly 300 partitions<a href="#fn12" id="fnref12:1">[12:1]</a>. So a surprising partition count for a given file is usually explained by its row-group layout, separately from <code>spark.sql.files.maxPartitionBytes</code> (128 MB by default) and its <code>maxPartitionNum</code>/<code>minPartitionNum</code> companions, which govern splitting for file-based sources generally<a href="#fn13" id="fnref13">[13]</a>.The <a href="./bottleneck-small-files.html">small-files problem</a> shows up as metadata overhead rather than raw I/O cost: "when you're writing lots of small files, there's a significant metadata overhead that you incur managing all of those files. Spark especially does not do well with small files"<a href="#fn1" id="fnref1:3">[1:3]</a>. At the other extreme, oversized or misaligned row groups show up as lost locality: the Parquet spec's rationale for matching block size to row-group size is that "since an entire row group might need to be read, we want it to completely fit on one HDFS block"<a href="#fn14" id="fnref14">[14]</a>. A row group straddling a block boundary can't get that benefit.Raw (non-Parquet) gzip or zip source files reveal themselves as a single-executor bottleneck at read time, visible as one task doing dramatically more work than the rest of its stage: Spark "needs to download the whole file on one executor, unpack it on just one core, and then redistribute the partitions to the cluster nodes"<a href="#fn15" id="fnref15">[15]</a>.Version and schema mismatches surface as read failures rather than silent corruption. A file written with <code>parquet.writer.version=2</code> can't be parsed by an older Spark version, or a non-Spark engine such as Hive or Impala, if that reader only implements the original <code>DataPageHeader</code><a href="#fn11" id="fnref11:2">[11:2]</a>. Spark: The Definitive Guide frames this as a general risk: "you can still encounter problems if you're working with incompatible Parquet files. Be careful when you write out Parquet files with different versions of Spark (especially older ones) because this can cause significant headache"<a href="#fn1" id="fnref1:4">[1:4]</a>.<h2 id="what-the-layout-buys-you" tabindex="-1">What the layout buys you <a class="header-anchor" href="#what-the-layout-buys-you" aria-label="Permalink to "What the layout buys you""></a></h2>Those read-time symptoms trace back to specific tradeoffs in the layout. Splittability is what decides whether a file can be processed in parallel at all. A non-splittable raw compressed source file forces Spark onto a single core for the whole download-and-unpack step before it can redistribute anything: "as you can imagine, this becomes a huge bottleneck in your distributed processing"<a href="#fn15" id="fnref15:1">[15:1]</a>. Parquet sidesteps this at the container level: because its footer records row-group and column-chunk offsets independently of whichever codec compressed the bytes inside them, a task can seek and decode a row group on its own, so even a non-splittable codec like Snappy doesn't cost the file its parallelism<a href="#fn2" id="fnref2:1">[2:1]</a><a href="#fn3" id="fnref3:1">[3:1]</a>.Row-group and block-size alignment governs the same kind of locality at a coarser grain. The spec recommends large row groups, 512 MB–1 GB, "because larger row groups allow for larger column chunks which makes it possible to do larger sequential IO," at the cost of more write-side buffering, paired with a block size sized the same way; its own worked example is "1GB row groups, 1GB HDFS block size, 1 HDFS block per HDFS file"<a href="#fn14" id="fnref14:1">[14:1]</a>.The statistics both formats carry are what predicate pushdown runs against. Spark plans queries directly off "statistics that Spark reads directly from the underlying data source, like the counts and min/max values in the metadata of Parquet files"<a href="#fn16" id="fnref16">[16]</a>, gated by <code>spark.sql.parquet.filterPushdown</code> (default <code>true</code> since Spark 1.2.0) and, for ORC, <code>spark.sql.orc.filterPushdown</code> (default <code>true</code> since 1.4.0)<a href="#fn17" id="fnref17">[17]</a><a href="#fn18" id="fnref18">[18]</a>. A more aggressive option, <code>spark.sql.parquet.aggregatePushdown</code> (default <code>false</code>, since 3.3.0), pushes <code>MIN</code>, <code>MAX</code>, and <code>COUNT</code> down to Parquet's footer statistics directly, and throws if the needed statistic is missing from a file's footer<a href="#fn17" id="fnref17:1">[17:1]</a>. Skipping a row group or stripe this way means its bytes are never read off disk, which beats any amount of post-read filtering.Small files cost more in metadata management than their data volume would suggest, and Spark "especially does not do well" with them<a href="#fn1" id="fnref1:5">[1:5]</a>. On the scheduling side, Spark's tuning guide notes it "can efficiently support tasks as short as 200 ms" because it reuses one executor JVM across many tasks and has low task-launch cost, while separately recommending "2-3 tasks per CPU core in your cluster"<a href="#fn19" id="fnref19">[19]</a>, which implies <a href="./bottleneck-tiny-tasks.html">scheduling overhead</a> only becomes proportionally significant once tasks (and the files behind them) shrink below roughly that 200 ms floor.Compression choice trades CPU against I/O and storage. Once I/O stops being the bottleneck, paying a codec's decompression cost is a net loss: "uncompressed files are clearly outperforming compressed files. This is because uncompressed files are I/O bound, and compressed files are CPU bound, but I/O is good enough here"<a href="#fn15" id="fnref15:2">[15:2]</a>. Bzip2 illustrates the opposite failure mode: it's splittable, but compresses so aggressively that "you get very few partitions and therefore they can be poorly distributed"<a href="#fn15" id="fnref15:3">[15:3]</a>.Writer-version and schema changes matter because they fail at read time, for whoever reads the data next (not at write time, for whoever wrote it). A <code>parquet.writer.version=2</code> file is simply unreadable by any engine that only understands the original page header<a href="#fn11" id="fnref11:3">[11:3]</a>. Schema drift has its own correctness angle: for <a href="./table-formats.html">Delta Lake</a> tables, adding a column via <code>mergeSchema</code> causes existing rows, when read back, to have that new column's value read as <code>NULL</code><a href="#fn20" id="fnref20">[20]</a>, a defined outcome, but one that changes what a downstream query sees for rows written before the schema changed.<h2 id="choosing-and-tuning-the-format" tabindex="-1">Choosing and tuning the format <a class="header-anchor" href="#choosing-and-tuning-the-format" aria-label="Permalink to "Choosing and tuning the format""></a></h2>A few defaults cover most cases despite those tradeoffs. Default to Parquet. Reach for ORC specifically when the same data also has to serve Hive consumers or existing Hive ORC tables, where ORC is the better-optimized target<a href="#fn1" id="fnref1:6">[1:6]</a>.Size row groups and the underlying filesystem block together, not independently. The Parquet spec's own recommended setup is 1 GB row groups paired with a 1 GB HDFS block size, one block per file<a href="#fn14" id="fnref14:2">[14:2]</a>. Leave <code>spark.sql.files.maxPartitionBytes</code> (128 MB default) and its <code>maxPartitionNum</code>/<code>minPartitionNum</code> companions for the general file-splitting case, but expect row-group boundaries (not these settings) to be what actually decides partition count for row-group-oriented formats<a href="#fn13" id="fnref13:1">[13:1]</a><a href="#fn12" id="fnref12:2">[12:2]</a>.Leave predicate pushdown on: <code>spark.sql.parquet.filterPushdown</code> and <code>spark.sql.orc.filterPushdown</code> both default to <code>true</code> already<a href="#fn17" id="fnref17:2">[17:2]</a><a href="#fn18" id="fnref18:1">[18:1]</a>, so the main action is not disabling them, plus considering <code>spark.sql.parquet.aggregatePushdown</code> when a workload is dominated by <code>MIN</code>/<code>MAX</code>/<code>COUNT</code> over Parquet sources with complete footer statistics<a href="#fn17" id="fnref17:3">[17:3]</a>.Cap output file size directly instead of letting the small-files problem accumulate: <code>maxRecordsPerFile</code>, introduced in Spark 2.2, targets an optimum file size by capping the number of records written per file, e.g. <code>df.write.option("maxRecordsPerFile", 5000)</code><a href="#fn1" id="fnref1:7">[1:7]</a>.Pick a compression codec based on the actual bottleneck. Parquet's default, Snappy, favors speed; ORC has already defaulted to Zstd since Spark 2.3.0<a href="#fn9" id="fnref9:3">[9:3]</a>, and at least one optimization checklist recommends overriding Parquet's default to Zstd as well<a href="#fn21" id="fnref21">[21]</a>. Zstd's level is tunable through <code>spark.io.compression.zstd.level</code>: higher levels buy better compression "at the expense of more CPU and memory"<a href="#fn9" id="fnref9:4">[9:4]</a>. Avoid feeding Spark large raw <code>.gz</code>/<code>.zip</code> source files directly (unpack them before loading), since the non-splittability lives in the raw container format rather than in gzip itself; the same Definitive Guide that warns about raw gzip files elsewhere still recommends Parquet with gzip compression once gzip is wrapped inside Parquet's row-group container<a href="#fn1" id="fnref1:8">[1:8]</a>.Treat <code>spark.sql.parquet.mergeSchema</code> (default <code>false</code>) as opt-in rather than default-on: enabling it "merges schemas collected from all data files," instead of trusting a single summary file or a random file's schema<a href="#fn22" id="fnref22">[22]</a>, useful for evolving schemas, but it means every file in the dataset gets scanned for its schema. For Delta tables specifically, remember that <code>mergeSchema</code>-added columns read back as <code>NULL</code> on pre-existing rows<a href="#fn20" id="fnref20:1">[20:1]</a> before relying on it for a backfill.Don't flip <code>parquet.writer.version</code> to <code>2</code> without confirming every downstream reader of that data (an older Spark version, Hive, Impala, or anything else touching the files) actually supports <code>DataPageHeaderV2</code> first; it's a forward-incompatible page-format change, not an additive one<a href="#fn11" id="fnref11:4">[11:4]</a><a href="#fn1" id="fnref1:9">[1:9]</a>.<h2 id="sources" tabindex="-1">Sources <a class="header-anchor" href="#sources" aria-label="Permalink to "Sources""></a></h2><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item">Spark: The Definitive Guide, Chambers & Zaharia, ch. 9 <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a> <a href="#fnref1:2" class="footnote-backref">↩︎</a> <a href="#fnref1:3" class="footnote-backref">↩︎</a> <a href="#fnref1:4" class="footnote-backref">↩︎</a> <a href="#fnref1:5" class="footnote-backref">↩︎</a> <a href="#fnref1:6" class="footnote-backref">↩︎</a> <a href="#fnref1:7" class="footnote-backref">↩︎</a> <a href="#fnref1:8" class="footnote-backref">↩︎</a> <a href="#fnref1:9" class="footnote-backref">↩︎</a></li><li id="fn2" class="footnote-item"><a href="https://parquet.apache.org/_print/docs/file-format/" target="_blank" rel="noreferrer">Parquet File Format: File Metadata and Row Groups</a> <a href="#fnref2" class="footnote-backref">↩︎</a> <a href="#fnref2:1" class="footnote-backref">↩︎</a></li><li id="fn3" class="footnote-item"><a href="https://github.com/google/snappy/blob/main/format_description.txt" target="_blank" rel="noreferrer">Snappy Compressed Format Description</a> <a href="#fnref3" class="footnote-backref">↩︎</a> <a href="#fnref3:1" class="footnote-backref">↩︎</a></li><li id="fn4" class="footnote-item"><a href="https://parquet.apache.org/_print/docs/file-format/" target="_blank" rel="noreferrer">Parquet File Format: Column Index</a> <a href="#fnref4" class="footnote-backref">↩︎</a></li><li id="fn5" class="footnote-item"><a href="https://parquet.apache.org/docs/file-format/bloomfilter/" target="_blank" rel="noreferrer">Parquet File Format: Bloom Filter</a> <a href="#fnref5" class="footnote-backref">↩︎</a></li><li id="fn6" class="footnote-item"><a href="https://orc.apache.org/specification/ORCv1/" target="_blank" rel="noreferrer">ORC Specification v1: Column Statistics</a> <a href="#fnref6" class="footnote-backref">↩︎</a></li><li id="fn7" class="footnote-item"><a href="https://orc.apache.org/specification/ORCv1/" target="_blank" rel="noreferrer">ORC Specification v1: Row Index and Bloom Filters</a> <a href="#fnref7" class="footnote-backref">↩︎</a> <a href="#fnref7:1" class="footnote-backref">↩︎</a></li><li id="fn8" class="footnote-item"><a href="https://orc.apache.org/specification/ORCv1/" target="_blank" rel="noreferrer">ORC Specification v1: Stripe Statistics</a> <a href="#fnref8" class="footnote-backref">↩︎</a></li><li id="fn9" class="footnote-item"><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration: Spark</a> <a href="#fnref9" class="footnote-backref">↩︎</a> <a href="#fnref9:1" class="footnote-backref">↩︎</a> <a href="#fnref9:2" class="footnote-backref">↩︎</a> <a href="#fnref9:3" class="footnote-backref">↩︎</a> <a href="#fnref9:4" class="footnote-backref">↩︎</a></li><li id="fn10" class="footnote-item"><a href="https://facebook.github.io/zstd/zstd_manual.html" target="_blank" rel="noreferrer">Zstandard Manual</a> <a href="#fnref10" class="footnote-backref">↩︎</a></li><li id="fn11" class="footnote-item"><a href="https://parquet.apache.org/_print/docs/file-format/" target="_blank" rel="noreferrer">Parquet File Format: Data Pages (V1/V2)</a> <a href="#fnref11" class="footnote-backref">↩︎</a> <a href="#fnref11:1" class="footnote-backref">↩︎</a> <a href="#fnref11:2" class="footnote-backref">↩︎</a> <a href="#fnref11:3" class="footnote-backref">↩︎</a> <a href="#fnref11:4" class="footnote-backref">↩︎</a></li><li id="fn12" class="footnote-item"><a href="https://luminousmen.com/post/spark-partitions" target="_blank" rel="noreferrer">How Spark Determines Partitions for a File</a> <a href="#fnref12" class="footnote-backref">↩︎</a> <a href="#fnref12:1" class="footnote-backref">↩︎</a> <a href="#fnref12:2" class="footnote-backref">↩︎</a></li><li id="fn13" class="footnote-item"><a href="https://spark.apache.org/docs/latest/sql-performance-tuning.html" target="_blank" rel="noreferrer">Performance Tuning: Spark SQL, DataFrames and Datasets Guide</a> <a href="#fnref13" class="footnote-backref">↩︎</a> <a href="#fnref13:1" class="footnote-backref">↩︎</a></li><li id="fn14" class="footnote-item"><a href="https://parquet.apache.org/_print/docs/file-format/" target="_blank" rel="noreferrer">Parquet File Format: Row Group Size</a> <a href="#fnref14" class="footnote-backref">↩︎</a> <a href="#fnref14:1" class="footnote-backref">↩︎</a> <a href="#fnref14:2" class="footnote-backref">↩︎</a></li><li id="fn15" class="footnote-item"><a href="https://luminousmen.com/post/spark-tips-dont-collect-data-on-driver" target="_blank" rel="noreferrer">Spark Tips: Don't Collect Data on Driver</a> <a href="#fnref15" class="footnote-backref">↩︎</a> <a href="#fnref15:1" class="footnote-backref">↩︎</a> <a href="#fnref15:2" class="footnote-backref">↩︎</a> <a href="#fnref15:3" class="footnote-backref">↩︎</a></li><li id="fn16" class="footnote-item"><a href="https://spark.apache.org/docs/latest/sql-performance-tuning.html" target="_blank" rel="noreferrer">Performance Tuning: Spark SQL, DataFrames and Datasets Guide</a> <a href="#fnref16" class="footnote-backref">↩︎</a></li><li id="fn17" class="footnote-item"><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration: Spark</a> <a href="#fnref17" class="footnote-backref">↩︎</a> <a href="#fnref17:1" class="footnote-backref">↩︎</a> <a href="#fnref17:2" class="footnote-backref">↩︎</a> <a href="#fnref17:3" class="footnote-backref">↩︎</a></li><li id="fn18" class="footnote-item"><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration: Spark</a> <a href="#fnref18" class="footnote-backref">↩︎</a> <a href="#fnref18:1" class="footnote-backref">↩︎</a></li><li id="fn19" class="footnote-item"><a href="https://spark.apache.org/docs/latest/tuning.html" target="_blank" rel="noreferrer">Spark Tuning Guide: Level of Parallelism</a> <a href="#fnref19" class="footnote-backref">↩︎</a></li><li id="fn20" class="footnote-item">Learning Spark, 2nd Edition, Damji, Wenig, Das & Lee, ch. 9 <a href="#fnref20" class="footnote-backref">↩︎</a> <a href="#fnref20:1" class="footnote-backref">↩︎</a></li><li id="fn21" class="footnote-item"><a href="https://luminousmen.com/post/the-apache-spark-optimization-checklist" target="_blank" rel="noreferrer">The Apache Spark Optimization Checklist</a> <a href="#fnref21" class="footnote-backref">↩︎</a></li><li id="fn22" class="footnote-item"><a href="https://raw.githubusercontent.com/apache/spark/v3.5.0/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala" target="_blank" rel="noreferrer">SQLConf.scala</a> <a href="#fnref22" class="footnote-backref">↩︎</a></li></ol></section>',32)])])}const g=a(f,[["render",i]]);export{m as __pageData,g as default};

package/export-template/docs/assets/tuning-reference_data-formats.md.bjCAWH3N.lean.js ADDED Viewed

@@ -0,0 +1 @@

package/export-template/docs/assets/tuning-reference_index.md.BQ_NooMV.js ADDED Viewed

@@ -0,0 +1 @@

+ import{_ as n,C as t,o as a,c as r,E as i}from"./chunks/framework.DSg0KOwT.js";const g=JSON.parse('{"title":"Spark Tuning Reference","description":"","frontmatter":{"layout":"page","title":"Spark Tuning Reference","sidebar":false},"headers":[],"relativePath":"tuning-reference/index.md","filePath":"tuning-reference/index.md"}'),o={name:"tuning-reference/index.md"};function c(s,d,p,f,l,u){const e=t("TuningLanding");return a(),r("div",null,[i(e)])}const m=n(o,[["render",c]]);export{g as __pageData,m as default};

package/export-template/docs/assets/tuning-reference_index.md.BQ_NooMV.lean.js ADDED Viewed

@@ -0,0 +1 @@

package/export-template/docs/assets/tuning-reference_intro.md.CobD-lGB.js ADDED Viewed

@@ -0,0 +1 @@

+ import{_ as t,o as e,c as s,a5 as n}from"./chunks/framework.DSg0KOwT.js";const f=JSON.parse('{"title":"Introduction","description":"","frontmatter":{"title":"Introduction"},"headers":[],"relativePath":"tuning-reference/intro.md","filePath":"tuning-reference/intro.md"}'),r={name:"tuning-reference/intro.md"};function o(i,a,l,c,p,h){return e(),s("div",null,[...a[0]||(a[0]=[n('<h1 id="intro" tabindex="-1">Introduction <a class="header-anchor" href="#intro" aria-label="Permalink to "Introduction {#intro}""></a></h1>This guide is a hands-on reference for optimizing Spark and PySpark jobs. It covers <a href="./spark-architecture.html">Spark internals</a>, <a href="./memory-model.html">memory management</a>, <a href="./joins.html">joins</a>, <a href="./shuffle.html">shuffle</a>, <a href="./data-formats.html">data formats</a>, <a href="./pyspark.html">PySpark-specific patterns</a>, <a href="./aqe.html">Adaptive Query Execution</a> (AQE), and <a href="./cluster-config.html">cluster tuning</a>, followed by a bottleneck diagnostic reference.Read it top to bottom for a full grounding in Spark performance, or jump straight to the Bottleneck Reference section below when you already know which symptom you're chasing.<h2 id="severity-dots" tabindex="-1">Severity dots <a class="header-anchor" href="#severity-dots" aria-label="Permalink to "Severity dots""></a></h2>Each finding is marked with a severity dot:<ul><li> Info: worth knowing, not yet a problem</li><li> Warning: likely hurting job performance</li><li> Critical: actively bottlenecking the job</li></ul>Each bottleneck section below documents the exact thresholds behind these dots.<h2 id="tag-system" tabindex="-1">Tag system <a class="header-anchor" href="#tag-system" aria-label="Permalink to "Tag system""></a></h2>Every bottleneck has a short tag used throughout this reference:SKEW SHFL SPILLGC COLD UTILHOST FAIL STRAGRETRY TINY FAIL-RATESee the <a href="./bottleneck-skew.html">Bottleneck Reference</a> for what each one means, how it's detected, and how to fix it.',11)])])}const g=t(r,[["render",o]]);export{f as __pageData,g as default};

package/export-template/docs/assets/tuning-reference_intro.md.CobD-lGB.lean.js ADDED Viewed

@@ -0,0 +1 @@

package/export-template/docs/assets/tuning-reference_joins.md.BtKs_CuW.js ADDED Viewed

@@ -0,0 +1 @@

+ import{_ as a,o as t,c as o,a5 as s}from"./chunks/framework.DSg0KOwT.js";const r="../assets/join-strategy.C_FvrCEo.svg",n="../assets/join-strategy.dark.ChMLnNII.svg",b=JSON.parse('{"title":"Join Optimization","description":"","frontmatter":{"title":"Join Optimization"},"headers":[],"relativePath":"tuning-reference/joins.md","filePath":"tuning-reference/joins.md"}'),i={name:"tuning-reference/joins.md"};function f(c,e,l,h,d,p){return t(),o("div",null,[...e[0]||(e[0]=[s('<h1 id="joins" tabindex="-1">Join Optimization <a class="header-anchor" href="#joins" aria-label="Permalink to "Join Optimization {#joins}""></a></h1><h2 id="join-strategy-selection" tabindex="-1">Join strategy selection <a class="header-anchor" href="#join-strategy-selection" aria-label="Permalink to "Join strategy selection""></a></h2>Spark SQL's Catalyst optimizer picks from five physical join operators: broadcast hash join, broadcast nested loop join, shuffle hash join, shuffle sort-merge join (SMJ), and shuffle-and-replicate nested loop (cartesian) join<a href="#fn1" id="fnref1">[1]</a>. Broadcast hash join avoids a shuffle entirely by sending one small side to every executor; it requires an equi-join condition and supports every join type except full outer<a href="#fn1" id="fnref1:1">[1:1]</a>. Broadcast nested loop join relaxes the equi-join requirement (it supports non-equi conditions and every join type) at the cost of scanning one side repeatedly, so it's normally a fallback rather than a first choice<a href="#fn1" id="fnref1:2">[1:2]</a>. Unlike hand-tuned RDD joins, where the partitioner is chosen explicitly, Spark SQL's optimizer can also push down or reorder other operators automatically to make the eventual join cheaper<a href="#fn1" id="fnref1:3">[1:3]</a>.At the communication level, Spark's choice is binary: an all-to-all <a href="./shuffle.html">shuffle</a> join, or a broadcast join that replicates the small side once so every node can then join locally with no further network traffic<a href="#fn2" id="fnref2">[2]</a>. Once a join has been routed to a shuffle-based strategy (broadcast wasn't used), Spark defaults to preferring sort-merge join over shuffle hash join, a preference controlled by <code>spark.sql.join.preferSortMergeJoin</code><a href="#fn3" id="fnref3">[3]</a>. Since Spark 3.2, <a href="./aqe.html">Adaptive Query Execution</a> (AQE), on by default, re-optimizes the physical plan mid-execution using runtime statistics gathered after the shuffle actually runs, rather than relying only on pre-execution estimates<a href="#fn4" id="fnref4">[4]</a>.<img class="light-only" src="'+r+'" alt="Decision tree for choosing a physical join operator: the equi-join test splits off the nested-loop strategies, then the broadcast-threshold and preferSortMergeJoin checks select broadcast hash, sort-merge, or shuffle hash join, with AQE able to promote sort-merge to broadcast at runtime."><img class="dark-only" src="'+n+'" alt="Decision tree for choosing a physical join operator: the equi-join test splits off the nested-loop strategies, then the broadcast-threshold and preferSortMergeJoin checks select broadcast hash, sort-merge, or shuffle hash join, with AQE able to promote sort-merge to broadcast at runtime."><h2 id="reading-it-in-the-plan" tabindex="-1">Reading it in the plan <a class="header-anchor" href="#reading-it-in-the-plan" aria-label="Permalink to "Reading it in the plan""></a></h2>That choice is legible before a job even runs. The deciding signal is the relation's size relative to <code>spark.sql.autoBroadcastJoinThreshold</code>, which defaults to <code>10485760</code> bytes (10 MB), unchanged since it was introduced in Spark 1.1.0 and still the documented default in Spark 3.5<a href="#fn4" id="fnref4:1">[4:1]</a><a href="#fn5" id="fnref5">[5]</a>. That comparison is driven by table/plan size statistics rather than a fresh scan of the data: Spark consults catalog statistics (collected via <code>ANALYZE TABLE</code>, inspectable through <code>DESCRIBE EXTENDED</code>) and the cost estimates shown in <code>EXPLAIN COST</code><a href="#fn4" id="fnref4:2">[4:2]</a>. When those statistics are missing, <code>spark.sql.statistics.fallBackToHdfs</code> (default <code>false</code>) controls whether Spark falls back to on-disk file size to judge broadcast eligibility, and for partitioned tables without statistics Spark instead uses the <code>spark.sql.defaultSizeInBytes</code> placeholder<a href="#fn5" id="fnref5:1">[5:1]</a>.<blockquote>PySpark: don't guess whether a join will broadcast. Call <code>df.explain(mode="cost")</code> to see the same size estimates Catalyst used to pick the strategy.</blockquote>Because AQE re-optimizes at runtime, the physical join operator visible in a plan can change after execution starts. Two runtime-driven overrides are worth checking for in the Spark UI or an <code>EXPLAIN</code> plan: <code>spark.sql.adaptive.maxShuffledHashJoinLocalMapThreshold</code> (default <code>0</code>, disabled) can make AQE prefer shuffled hash join over sort-merge "regardless of the value of <code>spark.sql.join.preferSortMergeJoin</code>" whenever every post-shuffle partition stays under that threshold and above <code>spark.sql.adaptive.advisoryPartitionSizeInBytes</code><a href="#fn4" id="fnref4:3">[4:3]</a>; separately, AQE can convert an already-planned sort-merge join into a broadcast hash join mid-execution when the runtime statistics of either join side turn out smaller than the adaptive broadcast threshold<a href="#fn4" id="fnref4:4">[4:4]</a>.Bucketing status is also visible directly in the physical plan: a shuffle-free bucketed join only appears once both sides are bucketed to the same count, at which point the <code>Exchange</code> nodes disappear entirely<a href="#fn6" id="fnref6">[6]</a>. This is demonstrated with 4 buckets on each side<a href="#fn7" id="fnref7">[7]</a> and with 16 buckets on each side, where the plan shows <code>SelectedBucketsCount: 16 out of 16</code> on both branches of the <code>SortMergeJoin</code><a href="#fn8" id="fnref8">[8]</a>. A separate real-world account of eliminating a bucketed shuffle frames the same signal just as plainly: the giveaway is that "the right branch is missing an Exchange (i.e. shuffle)"<a href="#fn6" id="fnref6:1">[6:1]</a>. Removing the shuffle this way does not necessarily remove the sort phase: in the 16-bucket demo, a <code>Sort</code> operator still appears on each branch of the <code>SortMergeJoin</code> even with the <code>Exchange</code> gone<a href="#fn8" id="fnref8:1">[8:1]</a>.Whether Catalyst is reordering joins (rather than just individual operators) is also a config check: <code>spark.sql.cbo.enabled</code> and the separate <code>spark.sql.cbo.joinReorder.enabled</code> flag both default to <code>false</code>, so multi-way join reordering is off unless both are explicitly enabled<a href="#fn5" id="fnref5:2">[5:2]</a>. That's distinct from Catalyst's regular rule-based optimizations, which run regardless of those flags. For example, a filter written after a join in the DataFrame API was observed moved before the join (and pushed into the JDBC source) automatically, visible in the physical plan<a href="#fn9" id="fnref9">[9]</a>.<h2 id="costs-the-plan-doesn-t-show" tabindex="-1">Costs the plan doesn't show <a class="header-anchor" href="#costs-the-plan-doesn-t-show" aria-label="Permalink to "Costs the plan doesn't show""></a></h2>Picking a strategy is one thing; paying for it is another. Broadcasting isn't free on the driver side. Building a broadcast join replicates the small-side DataFrame to every worker, but that replication is preceded by collecting the DataFrame back to the driver first: an <a href="./bottleneck-broadcast-sizing.html">oversized broadcast</a>, whether chosen automatically or forced via a hint or <code>broadcast()</code> call, "can crash your driver node (because that collect is expensive)"<a href="#fn2" id="fnref2:1">[2:1]</a>. The shuffle-and-replicate nested loop (cartesian-style) strategy carries a related but distinct risk: because every partition is joined against every other partition, it has a high chance of data explosion<a href="#fn1" id="fnref1:4">[1:4]</a>.Bucketing's shuffle-free path also has a real tradeoff once bucket counts don't match exactly. Since Spark 3.1, <code>spark.sql.bucketing.coalesceBucketsInJoin.enabled</code> lets Spark coalesce the side with more buckets down to the smaller count, but only within a ratio bounded by <code>spark.sql.bucketing.coalesceBucketsInJoin.maxBucketRatio</code> (default <code>4</code>). Enabling it can still remove the shuffle, but it does so by cutting parallelism on the finer-grained side, and Spark's own docs note it "could possibly cause OOM for shuffled hash join" as a result<a href="#fn5" id="fnref5:3">[5:3]</a>. Outside that ratio, or with coalescing disabled, mismatched bucket counts get no free win at all: the join simply falls back to a normal shuffle.<h2 id="forcing-a-strategy" tabindex="-1">Forcing a strategy <a class="header-anchor" href="#forcing-a-strategy" aria-label="Permalink to "Forcing a strategy""></a></h2>When the automatic choice isn't the right one, Spark still lets you override it directly. Join hints force a specific strategy per relation, and Spark honors them even against its own size-based defaults:<ul><li>BROADCAST (also accepted as <code>BROADCASTJOIN</code>/<code>MAPJOIN</code>) forces a broadcast join with the hinted relation as the build side (broadcast hash if there's an equi-join key, broadcast nested loop otherwise), and this is honored "even if the size of table 't1' suggested by the statistics is above the configuration <code>spark.sql.autoBroadcastJoinThreshold</code>"<a href="#fn4" id="fnref4:5">[4:5]</a>.</li><li>MERGE forces a shuffle sort-merge join; it needs an equi-join on sortable keys and high cardinality to pay off, and is less prone to OOM than the hash-based options since it never builds an in-memory hash table<a href="#fn1" id="fnref1:5">[1:5]</a>.</li><li>SHUFFLE_HASH forces a shuffle hash join, supporting all join types on an equi-join key but, like MERGE, wanting high cardinality; it builds a per-partition hash table after the shuffle<a href="#fn1" id="fnref1:6">[1:6]</a>.</li><li>SHUFFLE_REPLICATE_NL forces the shuffle-and-replicate nested loop join, supporting inner and cartesian joins with equi or non-equi conditions<a href="#fn1" id="fnref1:7">[1:7]</a>.</li></ul>When both sides of a join carry conflicting hints, Spark resolves them by a fixed priority: BROADCAST over MERGE over SHUFFLE_HASH over SHUFFLE_REPLICATE_NL. If both sides carry the same BROADCAST or SHUFFLE_HASH hint, Spark still picks a build side based on join type and relation sizes<a href="#fn4" id="fnref4:6">[4:6]</a>. As with any hint, none of this is guaranteed: Spark won't honor a hinted strategy that structurally can't support the actual join type being performed<a href="#fn4" id="fnref4:7">[4:7]</a>. The threshold itself can also be turned off outright: setting <code>spark.sql.autoBroadcastJoinThreshold</code> to <code>-1</code> forces every join to fall back to shuffle sort-merge instead of ever considering broadcast<a href="#fn3" id="fnref3:1">[3:1]</a>.For repeated joins on the same key, bucketing both tables to the same bucket count removes the shuffle for free. To also remove the sort phase (not just the shuffle), the bucketed tables additionally need to be written pre-sorted on the join key, using <code>sortBy</code> alongside <code>bucketBy</code>; done that way, the merge phase has nothing left to do, since "the joined output is sorted ... because we saved the tables sorted in ascending order ... there's no need to sort during the <code>SortMergeJoin</code>," and the Spark UI shows the query going straight to <code>WholeStageCodegen</code> with no <code>Exchange</code> at all<a href="#fn3" id="fnref3:2">[3:2]</a>.<blockquote>PySpark: <code>df.write.bucketBy(n, "key").saveAsTable(...)</code> drops the shuffle; add <code>.sortBy("key")</code> on both tables to drop the sort phase too.</blockquote>Multi-way join reordering is a deliberate opt-in: enable <code>spark.sql.cbo.enabled</code> for cost-based statistics, then <code>spark.sql.cbo.joinReorder.enabled</code> for the reordering rule itself. <code>spark.sql.cbo.joinReorder.dp.threshold</code> caps the dynamic-programming enumeration at 12 joined nodes by default, and <code>spark.sql.cbo.joinReorder.dp.star.filter</code> applies star-join filter heuristics on top of it<a href="#fn5" id="fnref5:4">[5:4]</a>. A lighter-weight alternative that skips full CBO is <code>spark.sql.cbo.starSchemaDetection</code> (default <code>false</code>), which enables join reordering based on star-schema detection alone<a href="#fn5" id="fnref5:5">[5:5]</a>.For <a href="./bottleneck-skew.html">skewed join keys</a>, Spark offers two built-in alternatives to hand-rolled salting: AQE's skew-join handling (<code>spark.sql.adaptive.skewJoin.enabled</code>), which detects oversized shuffle partitions at runtime and splits them automatically, replicating if needed<a href="#fn10" id="fnref10">[10]</a><a href="#fn5" id="fnref5:6">[5:6]</a>, and Databricks' declarative <code>SKEW</code> hint, which builds a skew-aware plan without any manual salting<a href="#fn11" id="fnref11">[11]</a>. Manual salting remains the fallback where neither is available: add a random salt column to the join key on both sides so a hot key spreads across many partitions (exploding the dimension side into one row per salt value and assigning a random salt on the fact side), then join on the composite <code>(key, salt)</code> pair<a href="#fn12" id="fnref12">[12]</a>.<h2 id="sources" tabindex="-1">Sources <a class="header-anchor" href="#sources" aria-label="Permalink to "Sources""></a></h2><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item">High Performance Spark, 2nd Edition, Karau, Polak & Warren, ch. 6 <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a> <a href="#fnref1:2" class="footnote-backref">↩︎</a> <a href="#fnref1:3" class="footnote-backref">↩︎</a> <a href="#fnref1:4" class="footnote-backref">↩︎</a> <a href="#fnref1:5" class="footnote-backref">↩︎</a> <a href="#fnref1:6" class="footnote-backref">↩︎</a> <a href="#fnref1:7" class="footnote-backref">↩︎</a></li><li id="fn2" class="footnote-item">Spark: The Definitive Guide, Chambers & Zaharia, ch. 8 <a href="#fnref2" class="footnote-backref">↩︎</a> <a href="#fnref2:1" class="footnote-backref">↩︎</a></li><li id="fn3" class="footnote-item">Learning Spark, 2nd Edition, Damji, Wenig, Das, Lee, ch. 7 <a href="#fnref3" class="footnote-backref">↩︎</a> <a href="#fnref3:1" class="footnote-backref">↩︎</a> <a href="#fnref3:2" class="footnote-backref">↩︎</a></li><li id="fn4" class="footnote-item"><a href="https://spark.apache.org/docs/latest/sql-performance-tuning.html" target="_blank" rel="noreferrer">Performance Tuning — Spark SQL, DataFrames and Datasets Guide</a> <a href="#fnref4" class="footnote-backref">↩︎</a> <a href="#fnref4:1" class="footnote-backref">↩︎</a> <a href="#fnref4:2" class="footnote-backref">↩︎</a> <a href="#fnref4:3" class="footnote-backref">↩︎</a> <a href="#fnref4:4" class="footnote-backref">↩︎</a> <a href="#fnref4:5" class="footnote-backref">↩︎</a> <a href="#fnref4:6" class="footnote-backref">↩︎</a> <a href="#fnref4:7" class="footnote-backref">↩︎</a></li><li id="fn5" class="footnote-item"><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration — Spark</a> <a href="#fnref5" class="footnote-backref">↩︎</a> <a href="#fnref5:1" class="footnote-backref">↩︎</a> <a href="#fnref5:2" class="footnote-backref">↩︎</a> <a href="#fnref5:3" class="footnote-backref">↩︎</a> <a href="#fnref5:4" class="footnote-backref">↩︎</a> <a href="#fnref5:5" class="footnote-backref">↩︎</a> <a href="#fnref5:6" class="footnote-backref">↩︎</a></li><li id="fn6" class="footnote-item"><a href="https://www.taboola.com/engineering/bucket-the-shuffle-out-of-here/" target="_blank" rel="noreferrer">Bucket the Shuffle Out of Here</a> <a href="#fnref6" class="footnote-backref">↩︎</a> <a href="#fnref6:1" class="footnote-backref">↩︎</a></li><li id="fn7" class="footnote-item"><a href="https://books.japila.pl/spark-sql-internals/bucketing/" target="_blank" rel="noreferrer">Bucketing — The Internals of Spark SQL</a> <a href="#fnref7" class="footnote-backref">↩︎</a></li><li id="fn8" class="footnote-item"><a href="https://luminousmen.com/post/the-5-minute-guide-to-using-bucketing-in-pyspark" target="_blank" rel="noreferrer">The 5-Minute Guide to Using Bucketing in PySpark</a> <a href="#fnref8" class="footnote-backref">↩︎</a> <a href="#fnref8:1" class="footnote-backref">↩︎</a></li><li id="fn9" class="footnote-item"><a href="https://luminousmen.com/post/spark-tips-dataframe-api" target="_blank" rel="noreferrer">Spark Tips: DataFrame API</a> <a href="#fnref9" class="footnote-backref">↩︎</a></li><li id="fn10" class="footnote-item"><a href="https://issues.apache.org/jira/browse/SPARK-29544" target="_blank" rel="noreferrer">SPARK-29544 — Optimize Skewed Join at Runtime</a> <a href="#fnref10" class="footnote-backref">↩︎</a></li><li id="fn11" class="footnote-item"><a href="https://docs.databricks.com/aws/en/archive/legacy/skew-join" target="_blank" rel="noreferrer">Skew Join Hint</a> <a href="#fnref11" class="footnote-backref">↩︎</a></li><li id="fn12" class="footnote-item"><a href="https://luminousmen.com/post/spark-tips-partition-tuning" target="_blank" rel="noreferrer">Spark Tips: Partition Tuning</a> <a href="#fnref12" class="footnote-backref">↩︎</a></li></ol></section>',26)])])}const g=a(i,[["render",f]]);export{b as __pageData,g as default};

package/export-template/docs/assets/tuning-reference_joins.md.BtKs_CuW.lean.js ADDED Viewed

@@ -0,0 +1 @@

package/export-template/docs/assets/tuning-reference_memory-model.md.DhT-n4y3.js ADDED Viewed

@@ -0,0 +1 @@

+ import{_ as o,o as a,c as r,a5 as t}from"./chunks/framework.DSg0KOwT.js";const s="../assets/memory-regions.XHvO7jHG.svg",n="../assets/memory-regions.dark.D4TP9_08.svg",f="../assets/memory-borrowing.BqQRJg0u.svg",i="../assets/memory-borrowing.dark.Yhh20O9C.svg",y=JSON.parse('{"title":"Memory Management","description":"","frontmatter":{"title":"Memory Management"},"headers":[],"relativePath":"tuning-reference/memory-model.md","filePath":"tuning-reference/memory-model.md"}'),c={name:"tuning-reference/memory-model.md"};function l(h,e,d,p,u,m){return a(),r("div",null,[...e[0]||(e[0]=[t('<h1 id="memory-model" tabindex="-1">Memory Management <a class="header-anchor" href="#memory-model" aria-label="Permalink to "Memory Management {#memory-model}""></a></h1><h2 id="the-unified-memory-model" tabindex="-1">The unified memory model <a class="header-anchor" href="#the-unified-memory-model" aria-label="Permalink to "The unified memory model""></a></h2>Since Spark 1.6, the <code>UnifiedMemoryManager</code> has replaced the earlier static split between execution and storage memory with a single shared region, referred to as M<a href="#fn1" id="fnref1">[1]</a><a href="#fn2" id="fnref2">[2]</a>. M is carved out of the JVM heap after first setting aside 300 MiB of reserved system memory: <code>M = (JVM heap − 300 MiB) × spark.memory.fraction</code>, with <code>spark.memory.fraction</code> defaulting to 0.6<a href="#fn2" id="fnref2:1">[2:1]</a><a href="#fn3" id="fnref3">[3]</a>. Within M, <code>spark.memory.storageFraction</code> (default 0.5) marks off a subregion reserved for cached blocks that are immune to eviction; with defaults, this works out to roughly a 30%/30% on-heap split between execution and storage out of total executor memory<a href="#fn3" id="fnref3:1">[3:1]</a>.<img class="light-only" src="'+s+'" alt="The executor memory layout showing reserved memory, the unified region split into execution and storage, and user memory inside the JVM heap, with the off-heap pool and memory overhead outside it."><img class="dark-only" src="'+n+'" alt="The executor memory layout showing reserved memory, the unified region split into execution and storage, and user memory inside the JVM heap, with the off-heap pool and memory overhead outside it.">Execution and storage borrow from each other dynamically rather than sitting behind a hard wall: when execution memory is unused, storage can acquire all of it, and vice versa<a href="#fn2" id="fnref2:2">[2:2]</a><a href="#fn1" id="fnref1:1">[1:1]</a>. The two are not equal peers, though. Execution always has priority, taking memory immediately and evicting cached storage blocks if necessary<a href="#fn3" id="fnref3:2">[3:2]</a>.The reserved 300 MB is a flat constant (<code>RESERVED_SYSTEM_MEMORY_BYTES</code> in the Spark source), not a formula, and it's hardcoded: there is no supported production configuration to change it. The only override is the internal <code>spark.testing.reservedMemory</code> property, which exists for Spark's own test suite rather than production tuning<a href="#fn3" id="fnref3:3">[3:3]</a>. <code>spark.memory.fraction</code> is applied against heap minus that reserved 300 MB, not against total heap. The tuning guide is explicit that it "expresses the size of M as a fraction of the (JVM heap space − 300MiB)"<a href="#fn2" id="fnref2:3">[2:3]</a>.Off-heap memory extends the same model outside the JVM heap. When <code>spark.memory.offHeap.enabled=true</code>, Spark allocates raw off-heap buffers via <code>sun.misc.Unsafe</code> (routed through <code>jdk.internal.misc.Unsafe</code> on JDK 17+, which is why some builds need <code>--add-opens</code> flags), and that off-heap region is split into its own execution and storage pools, following the same borrowing rules as on-heap memory: execution has priority, storage gets evicted first<a href="#fn3" id="fnref3:4">[3:4]</a>. Enabling it requires <code>spark.memory.offHeap.size</code> to be set to a positive value; leaving it enabled with the default size of <code>0</code> is a documented-invalid combination, not a supported way to run with off-heap "on but empty"<a href="#fn4" id="fnref4">[4]</a>. The setting also "has no impact on heap memory usage". Turning on off-heap memory does not shrink the JVM <code>-Xmx</code> for you, so the on-heap size has to be reduced manually to keep total footprint constant<a href="#fn4" id="fnref4:1">[4:1]</a>. This off-heap execution memory underlies Project Tungsten: its compact binary row encoding, its explicit-memory-managed hash map for aggregations, and its cache-aware sort/join algorithms are all built to run against off-heap, GC-invisible memory<a href="#fn5" id="fnref5">[5]</a><a href="#fn6" id="fnref6">[6]</a>.Executor memory overhead is a separate pool layered on top of M. <code>spark.executor.memoryOverhead</code> defaults to <code>executorMemory × spark.executor.memoryOverheadFactor</code>, floored at <code>spark.executor.minMemoryOverhead</code> (384 MiB)<a href="#fn4" id="fnref4:2">[4:2]</a>. <code>spark.executor.memoryOverheadFactor</code> itself defaults to 0.10 for ordinary JVM executors, but Spark bumps that default to 0.40 specifically for Kubernetes non-JVM jobs, since those workloads tend to need more non-JVM heap space<a href="#fn4" id="fnref4:3">[4:3]</a>. The legacy, YARN-only <code>spark.yarn.executor.memoryOverhead</code> property was removed in Spark 3.0 in favor of the cluster-manager-agnostic <code>spark.executor.memoryOverhead</code><a href="#fn3" id="fnref3:5">[3:5]</a>.<blockquote>PySpark: <code>spark.python.worker.memory</code> (default <code>512m</code>) is a soft spill threshold for a Python worker's own aggregation buffering, not a JVM-enforced cap. A PySpark worker runs as a separate OS process outside the JVM heap, so the JVM cannot police it directly<a href="#fn3" id="fnref3:6">[3:6]</a>. The only setting that actually tries to bound PySpark memory per executor is <code>spark.executor.pyspark.memory</code>, and even that depends on Python's <code>resource</code> module, which isn't supported on Windows and doesn't actually limit anything on macOS<a href="#fn4" id="fnref4:4">[4:4]</a>. Left unset, PySpark memory usage is folded into <code>spark.executor.memoryOverhead</code> by default<a href="#fn4" id="fnref4:5">[4:5]</a>.</blockquote><h2 id="watching-eviction-and-spill" tabindex="-1">Watching eviction and spill <a class="header-anchor" href="#watching-eviction-and-spill" aria-label="Permalink to "Watching eviction and spill""></a></h2>That borrowing leaves a trace once eviction actually happens. Storage eviction under memory pressure follows an LRU (least recently used) policy<a href="#fn1" id="fnref1:2">[1:2]</a><a href="#fn7" id="fnref7">[7]</a>. The granularity is the block: the <code>BlockManager</code> tracks cached data as per-partition blocks (one block per RDD/DataFrame partition) and LRU operates at that level rather than evicting an entire RDD or cached table in one shot<a href="#fn8" id="fnref8">[8]</a><a href="#fn9" id="fnref9">[9]</a>. In practice this tends to evict the oldest partitions first, those materialized in the earliest job or stage, though lazy evaluation makes it hard to predict exactly which partitions will go ahead of time<a href="#fn9" id="fnref9:1">[9:1]</a>.What happens to an evicted block is visible in its downstream effect and depends on its <code>StorageLevel</code>. For <code>MEMORY_ONLY</code>, the block is simply dropped and recomputed from the RDD's lineage the next time it's needed. For a disk-backed level like <code>MEMORY_AND_DISK</code>, the evicted block is written to disk and read back from there instead of being recomputed<a href="#fn9" id="fnref9:2">[9:2]</a><a href="#fn7" id="fnref7:1">[7:1]</a>. Blocks can also be evicted well before you ever try to reuse them: <code>.cache()</code> does not guarantee a block stays resident, especially on busy clusters or long pipelines<a href="#fn7" id="fnref7:2">[7:2]</a>.On the execution side, each task gets its own <code>TaskMemoryManager</code>, which enforces a soft per-task cap: with <code>n</code> tasks running concurrently, each task is allowed to allocate somewhere between <code>1/(2n)</code> and <code>1/n</code> of total execution memory, and the first task to arrive on an idle executor typically grabs more than its later-arriving neighbors<a href="#fn3" id="fnref3:7">[3:7]</a>.When a sort or hash-aggregation operator keeps requesting execution-memory pages and can't get more, Spark doesn't fail immediately. It blocks the requesting task, spills that task's in-memory data structure to disk, or, in the worst case, throws an <code>OutOfMemoryError</code><a href="#fn3" id="fnref3:8">[3:8]</a>. The spill path runs through Spark's map/shuffle I/O machinery: shuffle partitions created by wide transformations like <code>groupBy()</code> or <code>join()</code> <a href="./bottleneck-spill.html">spill</a> to the executors' local disks at the location set by <code>spark.local.directory</code><a href="#fn6" id="fnref6:1">[6:1]</a>. SQL physical operators apply the same idea with their own row-count guardrails: <code>sortMergeJoinExec</code>'s in-memory buffer and the cartesian-product operator's buffer both spill once they cross a configured row threshold, which by default is set to the value of <code>spark.shuffle.spill.numElementsForceSpillThreshold</code><a href="#fn10" id="fnref10">[10]</a>.Whatever the trigger, the resulting spilled bytes are exposed on the task metrics as <code>memoryBytesSpilled</code>, visible in the Spark UI and event log. This is the concrete signal to watch for execution memory pressure<a href="#fn11" id="fnref11">[11]</a>.Memory-overhead misconfiguration shows up differently: it manifests as the container or pod being OOMKilled with a vague container-memory error rather than a Spark-level exception, since the enforcement happens at the YARN NodeManager or Kubernetes kubelet/cgroup layer, not inside the JVM<a href="#fn3" id="fnref3:9">[3:9]</a>. Under the old default overhead factor of 0.10, non-JVM Kubernetes jobs commonly failed with "Memory Overhead Exceeded" errors, which is exactly why Spark bumped the Kubernetes non-JVM default to 0.40<a href="#fn4" id="fnref4:6">[4:6]</a>.<h2 id="what-the-borrowing-costs-you" tabindex="-1">What the borrowing costs you <a class="header-anchor" href="#what-the-borrowing-costs-you" aria-label="Permalink to "What the borrowing costs you""></a></h2>None of this is free for whoever's counting on cached data staying put. Because execution always wins the tug-of-war over storage, <a href="./caching.html">caching</a> is never a guarantee; it's a best effort. As luminousmen-memory-management puts it: "If Execution needs memory, it takes it. If Storage is using that space (cached RDDs, broadcasts), Spark starts evicting blocks. If Execution is idle, Storage can grow into that space, until Execution comes back."<a href="#fn3" id="fnref3:10">[3:10]</a> That growth-then-shrink-back behavior is dynamic borrowing, not storage evicting execution; eviction is strictly one-directional. As spark-notes-task-memory-management summarizes the agreement between the two regions: "keep acquiring execution memory and evict storage as you need more execution memory"<a href="#fn1" id="fnref1:3">[1:3]</a>, never the reverse. The same asymmetric rule carries over when off-heap memory is enabled: execution still has priority and storage still gets evicted first<a href="#fn3" id="fnref3:11">[3:11]</a>. Practically, this means a cached DataFrame can silently lose blocks under memory pressure, forcing an expensive lineage recompute (for <code>MEMORY_ONLY</code>) or an extra disk round-trip (for <code>MEMORY_AND_DISK</code>) the next time it's touched.<img class="light-only" src="'+f+'" alt="How storage memory borrows idle execution space while execution reclaims its own space by evicting storage in one direction."><img class="dark-only" src="'+i+'" alt="How storage memory borrows idle execution space while execution reclaims its own space by evicting storage in one direction.">The <code>TaskMemoryManager</code>'s soft per-task cap explains why spill behavior is workload-shape-dependent rather than a fixed threshold: with more tasks packed onto an executor, each one's guaranteed share of execution memory shrinks toward <code>1/(2n)</code>, making spills more likely under high task concurrency even when total execution memory hasn't changed<a href="#fn3" id="fnref3:12">[3:12]</a>.Off-heap memory's payoff is specifically about <a href="./bottleneck-gc.html">garbage collection</a>: because off-heap buffers sit outside the JVM heap, they are invisible to the garbage collector, so fewer and smaller live objects need to be tracked, scanned, and copied, which reduces both the frequency and duration of GC pauses<a href="#fn3" id="fnref3:13">[3:13]</a>. Project Tungsten's off-heap hash map for aggregations was benchmarked at over 1 million operations per second in a single thread, with "almost no performance degradation as memory utilization increases," unlike the JVM default <code>java.util.HashMap</code>, which eventually thrashes on GC<a href="#fn5" id="fnref5:1">[5:1]</a><a href="#fn6" id="fnref6:2">[6:2]</a>. The tradeoff is that off-heap memory removes the GC safety net along with the GC overhead: there's no garbage collector cleaning up if something goes wrong<a href="#fn3" id="fnref3:14">[3:14]</a>.Memory overhead matters because off-heap memory and (if unset) PySpark memory are not automatically folded into the overhead calculation the way heap memory is. Enabling <code>spark.memory.offHeap.size</code> without also raising <code>spark.executor.memoryOverhead</code> (or explicitly setting <code>spark.executor.pyspark.memory</code>) can push the executor's real footprint past what the cluster manager granted, and the process gets killed with a vague container-memory error instead of a Spark-level exception<a href="#fn3" id="fnref3:15">[3:15]</a>. Worked example: with <code>--executor-memory=8G</code>, the default 10% overhead gives <code>max(0.1 × 8192 MB, 384 MB) = 819 MB</code>, so the total memory requested from the cluster manager is <code>8192 + 819 = 9011 MB</code><a href="#fn3" id="fnref3:16">[3:16]</a>, useful to keep in mind when <a href="./cluster-config.html">sizing containers or pods</a> against a cluster's available capacity.<h2 id="tuning-the-memory-pools" tabindex="-1">Tuning the memory pools <a class="header-anchor" href="#tuning-the-memory-pools" aria-label="Permalink to "Tuning the memory pools""></a></h2>Most of this is adjustable, within limits. Tune <code>spark.memory.fraction</code> and <code>spark.memory.storageFraction</code> only if your workload's execution/storage balance genuinely needs to shift away from the ~30/30 default. But don't try to reclaim the 300 MB reserved region; it's a fixed, non-tunable constant in production<a href="#fn3" id="fnref3:17">[3:17]</a>.If losing cached partitions to eviction is costly, prefer a disk-backed <code>StorageLevel</code> such as <code>MEMORY_AND_DISK</code> over <code>MEMORY_ONLY</code>: an evicted block gets written to disk and read back rather than triggering a full lineage recompute<a href="#fn9" id="fnref9:3">[9:3]</a><a href="#fn7" id="fnref7:3">[7:3]</a>. Keep in mind <code>.cache()</code> alone is not a residency guarantee even with this choice<a href="#fn7" id="fnref7:4">[7:4]</a>.Watch <code>memoryBytesSpilled</code> in the Spark UI or history server to catch execution-memory pressure early<a href="#fn11" id="fnref11:1">[11:1]</a>. If sort or hash-aggregation spills are heavy, consider giving the executor more memory or reducing the number of concurrently running tasks per executor, since the <code>TaskMemoryManager</code>'s per-task guarantee shrinks as task concurrency <code>n</code> grows<a href="#fn3" id="fnref3:18">[3:18]</a>. For SQL joins and cartesian products, <code>spark.shuffle.spill.numElementsForceSpillThreshold</code> governs when <code>sortMergeJoinExec</code> and the cartesian-product operator spill their buffers<a href="#fn10" id="fnref10:1">[10:1]</a>.Set <code>spark.executor.memoryOverhead</code> explicitly rather than relying on the default, especially on Kubernetes for non-JVM workloads (where the default factor is already bumped to 0.40) or whenever off-heap memory or heavy PySpark usage is in play: those aren't automatically folded into the overhead calculation, so an unadjusted overhead can lead to an OOMKilled container instead of a clean Spark-level error<a href="#fn3" id="fnref3:19">[3:19]</a><a href="#fn4" id="fnref4:7">[4:7]</a>. Use <code>spark.executor.memoryOverhead</code>, not the removed <code>spark.yarn.executor.memoryOverhead</code><a href="#fn3" id="fnref3:20">[3:20]</a>. For capacity planning, remember the total requested from the cluster manager is executor memory plus overhead, e.g., <code>8192 + 819 = 9011 MB</code> for an 8 GB executor at the default 10% factor<a href="#fn3" id="fnref3:21">[3:21]</a>.When enabling off-heap memory, always pair <code>spark.memory.offHeap.enabled=true</code> with an explicit positive <code>spark.memory.offHeap.size</code> (never leave it enabled at the default size of <code>0</code><a href="#fn4" id="fnref4:8">[4:8]</a>) and manually reduce the JVM <code>-Xmx</code> to compensate, since enabling off-heap memory does not shrink the heap for you<a href="#fn4" id="fnref4:9">[4:9]</a>.<blockquote>PySpark: if you need PySpark's own memory bounded rather than folded silently into the overhead, set <code>spark.executor.pyspark.memory</code> explicitly, though its enforcement relies on Python's <code>resource</code> module and won't work on Windows and won't actually limit anything on macOS<a href="#fn4" id="fnref4:10">[4:10]</a>.</blockquote><h2 id="sources" tabindex="-1">Sources <a class="header-anchor" href="#sources" aria-label="Permalink to "Sources""></a></h2><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><a href="https://raw.githubusercontent.com/spoddutur/spark-notes/master/task_memory_management_in_spark.md" target="_blank" rel="noreferrer">Task Memory Management in Spark</a> <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a> <a href="#fnref1:2" class="footnote-backref">↩︎</a> <a href="#fnref1:3" class="footnote-backref">↩︎</a></li><li id="fn2" class="footnote-item"><a href="https://spark.apache.org/docs/latest/tuning.html" target="_blank" rel="noreferrer">Tuning Spark</a> <a href="#fnref2" class="footnote-backref">↩︎</a> <a href="#fnref2:1" class="footnote-backref">↩︎</a> <a href="#fnref2:2" class="footnote-backref">↩︎</a> <a href="#fnref2:3" class="footnote-backref">↩︎</a></li><li id="fn3" class="footnote-item"><a href="https://luminousmen.com/post/dive-into-spark-memory" target="_blank" rel="noreferrer">Dive into Spark Memory</a> <a href="#fnref3" class="footnote-backref">↩︎</a> <a href="#fnref3:1" class="footnote-backref">↩︎</a> <a href="#fnref3:2" class="footnote-backref">↩︎</a> <a href="#fnref3:3" class="footnote-backref">↩︎</a> <a href="#fnref3:4" class="footnote-backref">↩︎</a> <a href="#fnref3:5" class="footnote-backref">↩︎</a> <a href="#fnref3:6" class="footnote-backref">↩︎</a> <a href="#fnref3:7" class="footnote-backref">↩︎</a> <a href="#fnref3:8" class="footnote-backref">↩︎</a> <a href="#fnref3:9" class="footnote-backref">↩︎</a> <a href="#fnref3:10" class="footnote-backref">↩︎</a> <a href="#fnref3:11" class="footnote-backref">↩︎</a> <a href="#fnref3:12" class="footnote-backref">↩︎</a> <a href="#fnref3:13" class="footnote-backref">↩︎</a> <a href="#fnref3:14" class="footnote-backref">↩︎</a> <a href="#fnref3:15" class="footnote-backref">↩︎</a> <a href="#fnref3:16" class="footnote-backref">↩︎</a> <a href="#fnref3:17" class="footnote-backref">↩︎</a> <a href="#fnref3:18" class="footnote-backref">↩︎</a> <a href="#fnref3:19" class="footnote-backref">↩︎</a> <a href="#fnref3:20" class="footnote-backref">↩︎</a> <a href="#fnref3:21" class="footnote-backref">↩︎</a></li><li id="fn4" class="footnote-item"><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration — Spark</a> <a href="#fnref4" class="footnote-backref">↩︎</a> <a href="#fnref4:1" class="footnote-backref">↩︎</a> <a href="#fnref4:2" class="footnote-backref">↩︎</a> <a href="#fnref4:3" class="footnote-backref">↩︎</a> <a href="#fnref4:4" class="footnote-backref">↩︎</a> <a href="#fnref4:5" class="footnote-backref">↩︎</a> <a href="#fnref4:6" class="footnote-backref">↩︎</a> <a href="#fnref4:7" class="footnote-backref">↩︎</a> <a href="#fnref4:8" class="footnote-backref">↩︎</a> <a href="#fnref4:9" class="footnote-backref">↩︎</a> <a href="#fnref4:10" class="footnote-backref">↩︎</a></li><li id="fn5" class="footnote-item"><a href="https://www.databricks.com/blog/2015/04/28/project-tungsten-bringing-spark-closer-to-bare-metal.html" target="_blank" rel="noreferrer">Project Tungsten: Bringing Spark Closer to Bare Metal</a> <a href="#fnref5" class="footnote-backref">↩︎</a> <a href="#fnref5:1" class="footnote-backref">↩︎</a></li><li id="fn6" class="footnote-item">Learning Spark, 2nd Edition, Damji, Wenig, Das, Lee, ch. 6–7 <a href="#fnref6" class="footnote-backref">↩︎</a> <a href="#fnref6:1" class="footnote-backref">↩︎</a> <a href="#fnref6:2" class="footnote-backref">↩︎</a></li><li id="fn7" class="footnote-item"><a href="https://luminousmen.com/post/explaining-the-mechanics-of-spark-caching" target="_blank" rel="noreferrer">Explaining the Mechanics of Spark Caching</a> <a href="#fnref7" class="footnote-backref">↩︎</a> <a href="#fnref7:1" class="footnote-backref">↩︎</a> <a href="#fnref7:2" class="footnote-backref">↩︎</a> <a href="#fnref7:3" class="footnote-backref">↩︎</a> <a href="#fnref7:4" class="footnote-backref">↩︎</a></li><li id="fn8" class="footnote-item"><a href="https://spark.apache.org/docs/latest/rdd-programming-guide.html" target="_blank" rel="noreferrer">RDD Programming Guide</a> <a href="#fnref8" class="footnote-backref">↩︎</a></li><li id="fn9" class="footnote-item">High Performance Spark, 2nd Edition, Karau, Polak & Warren, ch. 7 <a href="#fnref9" class="footnote-backref">↩︎</a> <a href="#fnref9:1" class="footnote-backref">↩︎</a> <a href="#fnref9:2" class="footnote-backref">↩︎</a> <a href="#fnref9:3" class="footnote-backref">↩︎</a></li><li id="fn10" class="footnote-item"><a href="https://raw.githubusercontent.com/apache/spark/v3.5.0/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala" target="_blank" rel="noreferrer">SQLConf.scala</a> <a href="#fnref10" class="footnote-backref">↩︎</a> <a href="#fnref10:1" class="footnote-backref">↩︎</a></li><li id="fn11" class="footnote-item"><a href="https://spark.apache.org/docs/latest/monitoring.html#spark-history-server" target="_blank" rel="noreferrer">Monitoring and Instrumentation</a> <a href="#fnref11" class="footnote-backref">↩︎</a> <a href="#fnref11:1" class="footnote-backref">↩︎</a></li></ol></section>',34)])])}const k=o(c,[["render",l]]);export{y as __pageData,k as default};

package/export-template/docs/assets/tuning-reference_memory-model.md.DhT-n4y3.lean.js ADDED Viewed

@@ -0,0 +1 @@

package/export-template/docs/assets/tuning-reference_metrics.md.mLOh7Apj.js ADDED Viewed

@@ -0,0 +1 @@

+ import{_ as t,o as a,c as r,a5 as s}from"./chunks/framework.DSg0KOwT.js";const u=JSON.parse('{"title":"Metrics Glossary","description":"","frontmatter":{"title":"Metrics Glossary"},"headers":[],"relativePath":"tuning-reference/metrics.md","filePath":"tuning-reference/metrics.md"}'),o={name:"tuning-reference/metrics.md"};function i(c,e,n,l,d,f){return a(),r("div",null,[...e[0]||(e[0]=[s('<h1 id="metrics" tabindex="-1">Metrics Glossary <a class="header-anchor" href="#metrics" aria-label="Permalink to "Metrics Glossary {#metrics}""></a></h1>Reference for every metric surfaced elsewhere in this guide: what it measures, where Spark records it in the event log, and what a problematic value looks like.<h2 id="metric-task-duration" tabindex="-1">Task duration (P50/P95/max) <a class="header-anchor" href="#metric-task-duration" aria-label="Permalink to "Task duration (P50/P95/max) {#metric-task-duration}""></a></h2>Per-task wall-clock duration, aggregated across a stage's tasks into percentiles (P50, P95, max) to surface skew and stragglers. It is computed from <code>SparkListenerTaskEnd</code> events rather than read off a single field: each task end carries its own duration, and the percentiles are derived by aggregating those values across the stage.A problematic value looks like a small fraction of tasks running far past the rest: a stage is classified as a straggler when more than 5% of tasks run at least 4x the median duration (with at least 10 tasks in the stage), or when speculative tasks were fired.<h2 id="metric-shuffle-read-bytes" tabindex="-1">Shuffle read bytes <a class="header-anchor" href="#metric-shuffle-read-bytes" aria-label="Permalink to "Shuffle read bytes {#metric-shuffle-read-bytes}""></a></h2>The volume of shuffle data a task reads from remote executors, recorded in <code>taskMetrics.shuffleReadMetrics.remoteBytesRead</code>.<h2 id="metric-shuffle-write-bytes" tabindex="-1">Shuffle write bytes <a class="header-anchor" href="#metric-shuffle-write-bytes" aria-label="Permalink to "Shuffle write bytes {#metric-shuffle-write-bytes}""></a></h2>The size of the shuffle output a task writes, recorded in <code>taskMetrics.shuffleWriteMetrics.bytesWritten</code>. Spark's own metric description calls it simply the "Number of bytes written in shuffle operations," without stating compression state on its own<a href="#fn1" id="fnref1">[1]</a>.The sort-based shuffle writer fills in the mechanics: incoming records are serialized as soon as they reach the shuffle writer and buffered in serialized form while sorting<a href="#fn2" id="fnref2">[2]</a>. When the spill compression codec supports concatenating compressed data, the final merge step concatenates the already-compressed spill partitions directly into the output file, using <code>transferTo</code> rather than decompressing and recompressing<a href="#fn2" id="fnref2:1">[2:1]</a>. So <code>bytesWritten</code> counts compressed, post-serialization bytes, and in this common fast-merge path the final written output is built directly from data that had already been spilled to disk during sorting, rather than written fresh at merge time.<h2 id="metric-memory-bytes-spilled" tabindex="-1">Memory bytes spilled <a class="header-anchor" href="#metric-memory-bytes-spilled" aria-label="Permalink to "Memory bytes spilled {#metric-memory-bytes-spilled}""></a></h2>Bytes a task spilled from in-memory structures, recorded in <code>taskMetrics.memoryBytesSpilled</code>. Any value above zero is a spill warning, with the skew-vs-volume classification (based on what fraction of a stage's tasks show zero spill) as the actionable signal.<h2 id="metric-disk-bytes-spilled" tabindex="-1">Disk bytes spilled <a class="header-anchor" href="#metric-disk-bytes-spilled" aria-label="Permalink to "Disk bytes spilled {#metric-disk-bytes-spilled}""></a></h2>The on-disk counterpart to memory spill: bytes a task spilled to disk, recorded in <code>taskMetrics.diskBytesSpilled</code>. It feeds the same spill classification as memory bytes spilled.<h2 id="metric-jvm-gc-time" tabindex="-1">JVM GC time <a class="header-anchor" href="#metric-jvm-gc-time" aria-label="Permalink to "JVM GC time {#metric-jvm-gc-time}""></a></h2>Elapsed time the JVM spent in garbage collection while a task executed, recorded in <code>taskMetrics.jvmGCTime</code> and expressed in milliseconds<a href="#fn1" id="fnref1:1">[1:1]</a>. The value is cumulative across the task's full execution window: the sum of every GC pause that occurred during that task's run, not just the most recent one<a href="#fn1" id="fnref1:2">[1:2]</a>. This matches how it's serialized: as a single scalar long, consistent with an accumulator rather than a per-GC-event log entry<a href="#fn3" id="fnref3">[3]</a>.<h2 id="metric-gcpct" tabindex="-1">gcPct <a class="header-anchor" href="#metric-gcpct" aria-label="Permalink to "gcPct {#metric-gcpct}""></a></h2>A synthetic ratio, <code>jvmGCTime / executorRunTime</code>, not a raw Spark field. It drives GC-bottleneck classification: above 10% is a warning, above 20% is critical.<h2 id="metric-executor-run-time" tabindex="-1">Executor run time <a class="header-anchor" href="#metric-executor-run-time" aria-label="Permalink to "Executor run time {#metric-executor-run-time}""></a></h2>Elapsed time the executor spent running a task, recorded in <code>taskMetrics.executorRunTime</code> and expressed in milliseconds<a href="#fn1" id="fnref1:3">[1:3]</a>. <code>TaskMetrics</code> (and therefore <code>executorRunTime</code>) is serialized as an optional part of the <code>SparkListenerTaskEnd</code> payload, not guaranteed on every task end<a href="#fn3" id="fnref3:1">[3:1]</a>. In practice it is available for failed tasks that got far enough to actually run, such as <code>ExceptionFailure</code> or <code>TaskKilled</code>. The event-log deserialization code even falls back to reading accumulator updates out of the embedded <code>TaskMetrics</code> for old, Spark-1.x-era logs, which only makes sense if the metrics object is normally populated for that failure type<a href="#fn3" id="fnref3:2">[3:2]</a>. It can be absent for reasons like <code>Resubmitted</code>, where the task attempt never completed on that executor<a href="#fn3" id="fnref3:3">[3:3]</a>.<h2 id="metric-fetch-wait-time-ratio" tabindex="-1">Fetch wait time ratio <a class="header-anchor" href="#metric-fetch-wait-time-ratio" aria-label="Permalink to "Fetch wait time ratio {#metric-fetch-wait-time-ratio}""></a></h2>A synthetic ratio, <code>fetchWaitTime / taskDuration</code>, not a raw Spark field: the time a task spent blocked waiting on remote shuffle blocks, relative to its total duration.<h2 id="metric-input-bytes" tabindex="-1">Input bytes <a class="header-anchor" href="#metric-input-bytes" aria-label="Permalink to "Input bytes {#metric-input-bytes}""></a></h2>Bytes a task read as input, recorded in <code>taskMetrics.inputMetrics.bytesRead</code>.<h2 id="metric-output-bytes" tabindex="-1">Output bytes <a class="header-anchor" href="#metric-output-bytes" aria-label="Permalink to "Output bytes {#metric-output-bytes}""></a></h2>Bytes a task wrote as output, recorded in <code>taskMetrics.outputMetrics.bytesWritten</code>.<h2 id="metric-io-ratio" tabindex="-1">I/O ratio <a class="header-anchor" href="#metric-io-ratio" aria-label="Permalink to "I/O ratio {#metric-io-ratio}""></a></h2>A synthetic ratio, <code>outputBytes / inputBytes</code>, not a raw Spark field.<h2 id="metric-peak-execution-memory" tabindex="-1">Peak execution memory <a class="header-anchor" href="#metric-peak-execution-memory" aria-label="Permalink to "Peak execution memory {#metric-peak-execution-memory}""></a></h2>Peak memory recorded in <code>taskMetrics.peakExecutionMemory</code>. This is a task-level accumulator, distinct from the separate executor-level <code>peakMemoryMetrics.OnHeapExecutionMemory</code> and <code>.OffHeapExecutionMemory</code> gauges, which report the on-heap and off-heap execution pools as two separate numbers at the executor level rather than as a single per-task figure<a href="#fn1" id="fnref1:4">[1:4]</a>.<h2 id="metric-failed-tasks" tabindex="-1">Failed tasks / failure rate <a class="header-anchor" href="#metric-failed-tasks" aria-label="Permalink to "Failed tasks / failure rate {#metric-failed-tasks}""></a></h2>Tasks whose <code>SparkListenerTaskEnd</code> reason is not <code>Success</code>. The <code>Reason</code> field holds the formatted class name of whichever <code>TaskEndReason</code> was assigned to that task end<a href="#fn3" id="fnref3:4">[3:4]</a>, and the canonical set of reasons includes <code>FetchFailed</code>, <code>ExceptionFailure</code>, <code>TaskResultLost</code>, <code>TaskKilled</code>, <code>TaskCommitDenied</code>, <code>ExecutorLostFailure</code>, and <code>UnknownReason</code><a href="#fn3" id="fnref3:5">[3:5]</a>. Because <code>TaskMetrics</code> is only an optional part of the task-end payload, it can be absent for some of these reasons: for example <code>Resubmitted</code>, where the task attempt never actually completed on that executor<a href="#fn3" id="fnref3:6">[3:6]</a>.<h2 id="metric-speculative-tasks" tabindex="-1">Speculative tasks / straggler count <a class="header-anchor" href="#metric-speculative-tasks" aria-label="Permalink to "Speculative tasks / straggler count {#metric-speculative-tasks}""></a></h2>Tasks launched as speculative retries of a slow-running task, recorded via <code>SparkListenerTaskStart</code> where <code>speculative = true</code>. Any speculative task firing (or more than 5% of a stage's tasks running at least 4x the median duration, with at least 10 tasks in the stage) is a straggler signal.<h2 id="metric-executor-count" tabindex="-1">Executor count (added/removed/concurrent) <a class="header-anchor" href="#metric-executor-count" aria-label="Permalink to "Executor count (added/removed/concurrent) {#metric-executor-count}""></a></h2>The number of executors added, removed, or concurrently running, recorded via <code>SparkListenerExecutorAdded</code> / <code>SparkListenerExecutorRemoved</code> events.<h2 id="metric-stage-duration" tabindex="-1">Stage wall-clock duration <a class="header-anchor" href="#metric-stage-duration" aria-label="Permalink to "Stage wall-clock duration {#metric-stage-duration}""></a></h2>A stage's total elapsed time, recorded from <code>SparkListenerStageCompleted</code>: <code>completionTime − submissionTime</code>.<h2 id="metric-first-stage-submitted-at" tabindex="-1">firstStageSubmittedAt <a class="header-anchor" href="#metric-first-stage-submitted-at" aria-label="Permalink to "firstStageSubmittedAt {#metric-first-stage-submitted-at}""></a></h2>A synthetic field: the timestamp of the first <code>SparkListenerStageSubmitted</code> event in the event log. The gap between this timestamp and the application's start time drives cold-start classification: more than 30 seconds is a warning.<h2 id="sources" tabindex="-1">Sources <a class="header-anchor" href="#sources" aria-label="Permalink to "Sources""></a></h2><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><a href="https://spark.apache.org/docs/latest/monitoring.html#spark-history-server" target="_blank" rel="noreferrer">Monitoring and Instrumentation</a> <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a> <a href="#fnref1:2" class="footnote-backref">↩︎</a> <a href="#fnref1:3" class="footnote-backref">↩︎</a> <a href="#fnref1:4" class="footnote-backref">↩︎</a></li><li id="fn2" class="footnote-item"><a href="https://raw.githubusercontent.com/apache/spark/v3.5.0/core/src/main/scala/org/apache/spark/shuffle/sort/SortShuffleManager.scala" target="_blank" rel="noreferrer">SortShuffleManager.scala</a> <a href="#fnref2" class="footnote-backref">↩︎</a> <a href="#fnref2:1" class="footnote-backref">↩︎</a></li><li id="fn3" class="footnote-item"><a href="https://raw.githubusercontent.com/apache/spark/v3.5.0/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala" target="_blank" rel="noreferrer">JsonProtocol.scala</a> <a href="#fnref3" class="footnote-backref">↩︎</a> <a href="#fnref3:1" class="footnote-backref">↩︎</a> <a href="#fnref3:2" class="footnote-backref">↩︎</a> <a href="#fnref3:3" class="footnote-backref">↩︎</a> <a href="#fnref3:4" class="footnote-backref">↩︎</a> <a href="#fnref3:5" class="footnote-backref">↩︎</a> <a href="#fnref3:6" class="footnote-backref">↩︎</a></li></ol></section>',43)])])}const m=t(o,[["render",i]]);export{u as __pageData,m as default};

package/export-template/docs/assets/tuning-reference_metrics.md.mLOh7Apj.lean.js ADDED Viewed

@@ -0,0 +1 @@

package/export-template/docs/assets/tuning-reference_partitioning.md.q0zKF_8X.js ADDED Viewed

@@ -0,0 +1 @@

+ import{_ as a,o as t,c as o,a5 as r}from"./chunks/framework.DSg0KOwT.js";const s="../assets/repartition-vs-coalesce.BovLRrpj.svg",n="../assets/repartition-vs-coalesce.dark.BhAczKZQ.svg",g=JSON.parse('{"title":"Partitioning","description":"","frontmatter":{"title":"Partitioning"},"headers":[],"relativePath":"tuning-reference/partitioning.md","filePath":"tuning-reference/partitioning.md"}'),i={name:"tuning-reference/partitioning.md"};function f(l,e,c,h,d,p){return t(),o("div",null,[...e[0]||(e[0]=[r('<h1 id="partitioning" tabindex="-1">Partitioning <a class="header-anchor" href="#partitioning" aria-label="Permalink to "Partitioning {#partitioning}""></a></h1><h2 id="how-partitions-get-sized-and-shuffled" tabindex="-1">How partitions get sized and shuffled <a class="header-anchor" href="#how-partitions-get-sized-and-shuffled" aria-label="Permalink to "How partitions get sized and shuffled""></a></h2>Every partition Spark creates maps to exactly one task and one thread, which is why partition count and size drive so much of a job's performance<a href="#fn1" id="fnref1">[1]</a>. The <a href="./shuffle.html">shuffle</a> side of that is governed by a single default: <code>spark.sql.shuffle.partitions</code>, which defaults to 200 and applies to every <code>join()</code>, <code>groupBy()</code>, and aggregation regardless of how much data is actually moving<a href="#fn2" id="fnref2">[2]</a><a href="#fn3" id="fnref3">[3]</a>.Two operations reshape that layout, and they are not interchangeable. <code>repartition(numPartitions)</code> "return[s] a new RDD that has exactly numPartitions partitions"<a href="#fn4" id="fnref4">[4]</a>, and it earns that guarantee by always running a full hash shuffle: records are first spread across a temporary key space starting from a randomized position, seeded per-partition via <code>XORShiftRandom</code>, so upstream data ends up distributed evenly rather than clustered by its original layout. That feeds into a <code>ShuffledRDD</code> keyed by a <code>HashPartitioner(numPartitions)</code>, then a <code>CoalescedRDD</code> lands it on exactly the requested count<a href="#fn4" id="fnref4:1">[4:1]</a>. High Performance Spark puts the same mechanics more plainly: "repartition shuffles the RDD with a hash partitioner and the given number of partitions"<a href="#fn5" id="fnref5">[5]</a>. That holds whether the new count is bigger or smaller than the current one. Repartition always performs the full shuffle just described, unlike coalesce<a href="#fn6" id="fnref6">[6]</a>.<code>coalesce</code>, left at its default, does not shuffle at all: it's a narrow transformation where each output partition is simply the union of a fixed set of parent partitions decided at plan time, not routed by data values, so the stage's task count just drops to the coalesced number<a href="#fn5" id="fnref5:1">[5:1]</a>.Neither <code>repartition</code> nor <code>coalesce</code> leaves behind a "known partitioner" the way <code>partitionBy</code> does<a href="#fn5" id="fnref5:2">[5:2]</a>. That distinction is what Spark's join planner checks before it will skip a shuffle: it only does so when both sides already carry partitioner objects it can prove equal: matching partition counts for a <code>HashPartitioner</code>, matching range bounds for a <code>RangePartitioner</code><a href="#fn5" id="fnref5:3">[5:3]</a>. Its default shuffle hash join path makes the same point from the other side: it partitions the second dataset using the same partitioner as the first specifically so matching keys land together, which only lets a later shuffle be skipped when a shared, recognized partitioner is already in place beforehand<a href="#fn5" id="fnref5:4">[5:4]</a>.There's also a hard ceiling on shuffle output, though it isn't something to design around: Spark's sort-based shuffle manager caps output partitions in serialized mode at <code>PackedRecordPointer.MAXIMUM_PARTITION_ID + 1</code>, roughly 16.8 million. The source code itself calls this "an extreme defensive programming measure," since no real shuffle comes remotely close to it<a href="#fn7" id="fnref7">[7]</a>.<h2 id="spotting-a-bad-layout" tabindex="-1">Spotting a bad layout <a class="header-anchor" href="#spotting-a-bad-layout" aria-label="Permalink to "Spotting a bad layout""></a></h2>That layout choice shows up as overhead in both directions once it's wrong: partitions that are too small flood the cluster with per-task scheduling overhead, while partitions that are too large create memory pressure and straggler tasks<a href="#fn1" id="fnref1:1">[1:1]</a>. Learning Spark 2nd Edition frames the healthy target from the parallelism side rather than an absolute number: at least as many partitions as there are cores across the executors, so no core sits idle; more partitions than cores is fine as long as it doesn't drift into the small-partition overhead regime above<a href="#fn3" id="fnref3:1">[3:1]</a>.A heavy filter is an easy-to-miss cause of imbalance: Spark doesn't shrink partition count when rows are filtered out, so 2,000 partitions holding 5% of the original rows just become 2,000 mostly-empty partitions<a href="#fn1" id="fnref1:2">[1:2]</a>.To size shuffle partitions from a job's actual behavior rather than a guess, Cloudera's tuning guide takes a stage that already ran, computes the ratio between its Shuffle Spill (Memory) and Shuffle Spill (Disk) metrics, and multiplies total shuffle write by that ratio to estimate in-memory shuffle size, then rounds the resulting partition count up rather than down<a href="#fn8" id="fnref8">[8]</a>.<a href="./bottleneck-skew.html">Skew</a> specifically has documented, numeric detection thresholds under <a href="./aqe.html">Adaptive Query Execution</a>: a partition counts as skewed if it's larger than <code>spark.sql.adaptive.skewJoin.skewedPartitionFactor</code> (default 5.0) times the median partition size, and also larger than <code>spark.sql.adaptive.skewJoin.skewedPartitionThresholdInBytes</code> (default 256 MB)<a href="#fn2" id="fnref2:1">[2:1]</a><a href="#fn9" id="fnref9">[9]</a>.On the input side, <code>spark.sql.files.maxPartitionBytes</code> (defaulting to 128 MB) governs the target chunk size when Spark splits splittable file-based sources (CSV, JSON, line-delimited text) into input partitions, based on file size<a href="#fn1" id="fnref1:3">[1:3]</a>.<h2 id="where-a-bad-layout-costs-you" tabindex="-1">Where a bad layout costs you <a class="header-anchor" href="#where-a-bad-layout-costs-you" aria-label="Permalink to "Where a bad layout costs you""></a></h2>Getting the partition count wrong costs more than the immediate slowdown suggests. Cloudera's tuning guide argues it's safer to over-provision partitions than to under-provision them, because Spark, unlike MapReduce, has low per-task startup overhead: "when in doubt, it's almost always better to err on the side of a larger number of tasks"<a href="#fn8" id="fnref8:1">[8:1]</a>.Getting <code>coalesce</code> wrong costs more than the coalesce step itself: because it's narrow, it forces the entire upstream stage to run at the reduced parallelism, not just the final step<a href="#fn5" id="fnref5:5">[5:5]</a>. Pushed too far (<code>coalesce(1)</code>), it kills parallelism outright, because coalesce doesn't rebalance data, it just stacks existing partitions together, so partitions that were uneven going in are still uneven coming out<a href="#fn1" id="fnref1:4">[1:4]</a>.Once a shuffle stage has run, its output file count is locked in: you can't change it after the fact without inserting a stage barrier, such as writing to temporary storage or calling <code>localCheckpoint()</code>, between the shuffle and the write<a href="#fn10" id="fnref10">[10]</a>.Join alignment has its own gotcha: calling <code>repartition(n, col)</code> with the same column and count on two separate DataFrames produces data that's plausibly laid out the same way, but it doesn't leave a trackable partitioner object behind. The planner has no recorded partitioner to compare, so it has no basis for treating the two sides as co-partitioned and skipping the shuffle at join time, even though the call sites look identical<a href="#fn5" id="fnref5:6">[5:6]</a>.<h2 id="fixing-the-layout" tabindex="-1">Fixing the layout <a class="header-anchor" href="#fixing-the-layout" aria-label="Permalink to "Fixing the layout""></a></h2>Fixing those costs starts with accepting there's no formula to look up. There's no single formula for the right <code>spark.sql.shuffle.partitions</code> value: it depends on data set size, core count, and executor memory, and comes down to trial and error<a href="#fn3" id="fnref3:2">[3:2]</a>. As a starting point, Learning Spark notes the default of 200 is usually too high for small or streaming workloads, where it should be pulled down toward the executor core count<a href="#fn3" id="fnref3:3">[3:3]</a>.For the repartition-vs-coalesce decision itself: reach for <code>coalesce</code> first whenever you're only reducing partition count, since it merges partitions already on the same node without a shuffle<a href="#fn6" id="fnref6:1">[6:1]</a>. Reach for <code>repartition</code> when you actually need the shuffle: increasing partition count, fixing a lopsided distribution, or preparing a DataFrame ahead of a join or a <code>cache()</code> call, where even parallelism is worth more than the shuffle cost<a href="#fn6" id="fnref6:2">[6:2]</a>. High Performance Spark reduces this to a readability rule: use <code>repartition</code> when you want a shuffle, <code>coalesce</code> when you don't, rather than leaning on coalesce's shuffle toggle to blur the line<a href="#fn5" id="fnref5:7">[5:7]</a>. There is a middle option, <code>coalesce(n, shuffle=True)</code>, which behaves more like repartition, paying for a shuffle but getting real rebalancing on the way down<a href="#fn1" id="fnref1:5">[1:5]</a>. <code>coalesce()</code> itself belongs at the tail of a pipeline, right before a write, purely to cut down <a href="./bottleneck-small-files.html">output file count</a><a href="#fn1" id="fnref1:6">[1:6]</a>; after a heavy filter has left partitions mostly empty, following up with <code>repartition(100)</code> (or similar) buys back real parallelism<a href="#fn1" id="fnref1:7">[1:7]</a>. At the SQL layer, both directions have dedicated hints for controlling output partitioning directly: <code>/*+ REPARTITION(n) */</code>, <code>/*+ REPARTITION(cols) */</code>, <code>/*+ REPARTITION_BY_RANGE(cols) */</code>, and <code>/*+ COALESCE(n) */</code><a href="#fn9" id="fnref9:1">[9:1]</a>.<img class="light-only" src="'+s+'" alt="A decision flowchart for choosing repartition versus coalesce based on whether a shuffle and a rebalance are needed."><img class="dark-only" src="'+n+'" alt="A decision flowchart for choosing repartition versus coalesce based on whether a shuffle and a rebalance are needed.">Separate from the DataFrame <code>.coalesce()</code> call, Adaptive Query Execution has its own runtime coalescing behavior: <code>spark.sql.adaptive.coalescePartitions.enabled</code> (default true) merges small, contiguous post-shuffle partitions toward a target size at runtime, correcting over-partitioning without a manual <code>.coalesce()</code> call<a href="#fn9" id="fnref9:2">[9:2]</a><a href="#fn1" id="fnref1:8">[1:8]</a>.For skewed joins specifically, salting (adding a prefix to skewed keys so the same key is treated as several different keys, then adjusting the data distribution accordingly) was one of three manual approaches used before adaptive execution existed, alongside raising <code>spark.sql.shuffle.partitions</code> and raising the broadcast hash join threshold to push a sort-merge join toward a broadcast hash join instead. All three carry "lots of limitations" and require manual processing, which is exactly the gap AQE's skew-join optimization was built to close<a href="#fn11" id="fnref11">[11]</a>. Where automatic detection isn't precise enough, Databricks' <code>/*+ SKEW(...) */</code> hint names the skewed relation and column(s), and optionally the specific skewed key values, letting the planner target just those keys directly instead of relying on automatic detection<a href="#fn12" id="fnref12">[12]</a>.For the join-alignment gotcha above, the mechanism that does reliably guarantee a shuffle can be skipped is storage-partitioned joins: both tables are physically bucketed identically at the catalog level, for example <a href="./table-formats.html">Iceberg</a> tables created with matching <code>PARTITIONED BY (bucket(...))</code> clauses. When Spark recognizes both sides report the same partitioning through <code>SupportsReportPartitioning</code>, it can drop the Exchange (shuffle) node entirely, or shuffle only one side. A plain DataFrame-level <code>repartition(col)</code> doesn't offer that guarantee, because it isn't backed by catalog-level partitioning metadata<a href="#fn9" id="fnref9:3">[9:3]</a><a href="#fn2" id="fnref2:2">[2:2]</a>.<h2 id="sources" tabindex="-1">Sources <a class="header-anchor" href="#sources" aria-label="Permalink to "Sources""></a></h2><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><a href="https://luminousmen.com/post/spark-partitions" target="_blank" rel="noreferrer">Spark Partitions</a> <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a> <a href="#fnref1:2" class="footnote-backref">↩︎</a> <a href="#fnref1:3" class="footnote-backref">↩︎</a> <a href="#fnref1:4" class="footnote-backref">↩︎</a> <a href="#fnref1:5" class="footnote-backref">↩︎</a> <a href="#fnref1:6" class="footnote-backref">↩︎</a> <a href="#fnref1:7" class="footnote-backref">↩︎</a> <a href="#fnref1:8" class="footnote-backref">↩︎</a></li><li id="fn2" class="footnote-item"><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration — Spark</a> <a href="#fnref2" class="footnote-backref">↩︎</a> <a href="#fnref2:1" class="footnote-backref">↩︎</a> <a href="#fnref2:2" class="footnote-backref">↩︎</a></li><li id="fn3" class="footnote-item">Learning Spark, 2nd Edition, Damji, Wenig, Das & Lee, ch. 7 <a href="#fnref3" class="footnote-backref">↩︎</a> <a href="#fnref3:1" class="footnote-backref">↩︎</a> <a href="#fnref3:2" class="footnote-backref">↩︎</a> <a href="#fnref3:3" class="footnote-backref">↩︎</a></li><li id="fn4" class="footnote-item"><a href="https://raw.githubusercontent.com/apache/spark/v3.5.0/core/src/main/scala/org/apache/spark/rdd/RDD.scala" target="_blank" rel="noreferrer">RDD.scala</a> <a href="#fnref4" class="footnote-backref">↩︎</a> <a href="#fnref4:1" class="footnote-backref">↩︎</a></li><li id="fn5" class="footnote-item">High Performance Spark, 2nd Edition, Karau, Polak & Warren, ch. 8 <a href="#fnref5" class="footnote-backref">↩︎</a> <a href="#fnref5:1" class="footnote-backref">↩︎</a> <a href="#fnref5:2" class="footnote-backref">↩︎</a> <a href="#fnref5:3" class="footnote-backref">↩︎</a> <a href="#fnref5:4" class="footnote-backref">↩︎</a> <a href="#fnref5:5" class="footnote-backref">↩︎</a> <a href="#fnref5:6" class="footnote-backref">↩︎</a> <a href="#fnref5:7" class="footnote-backref">↩︎</a></li><li id="fn6" class="footnote-item">Spark: The Definitive Guide, Chambers & Zaharia, ch. 19 <a href="#fnref6" class="footnote-backref">↩︎</a> <a href="#fnref6:1" class="footnote-backref">↩︎</a> <a href="#fnref6:2" class="footnote-backref">↩︎</a></li><li id="fn7" class="footnote-item"><a href="https://raw.githubusercontent.com/apache/spark/v3.5.0/core/src/main/scala/org/apache/spark/shuffle/sort/SortShuffleManager.scala" target="_blank" rel="noreferrer">SortShuffleManager.scala</a> <a href="#fnref7" class="footnote-backref">↩︎</a></li><li id="fn8" class="footnote-item"><a href="https://blog.cloudera.com/how-to-tune-your-apache-spark-jobs-part-2/" target="_blank" rel="noreferrer">How to Tune Your Apache Spark Jobs (Part 2)</a> <a href="#fnref8" class="footnote-backref">↩︎</a> <a href="#fnref8:1" class="footnote-backref">↩︎</a></li><li id="fn9" class="footnote-item"><a href="https://spark.apache.org/docs/latest/sql-performance-tuning.html" target="_blank" rel="noreferrer">Performance Tuning — Spark SQL, DataFrames and Datasets Guide</a> <a href="#fnref9" class="footnote-backref">↩︎</a> <a href="#fnref9:1" class="footnote-backref">↩︎</a> <a href="#fnref9:2" class="footnote-backref">↩︎</a> <a href="#fnref9:3" class="footnote-backref">↩︎</a></li><li id="fn10" class="footnote-item"><a href="https://luminousmen.com/post/spark-tips-partition-tuning" target="_blank" rel="noreferrer">Spark Tips: Partition Tuning</a> <a href="#fnref10" class="footnote-backref">↩︎</a></li><li id="fn11" class="footnote-item"><a href="https://issues.apache.org/jira/browse/SPARK-29544" target="_blank" rel="noreferrer">SPARK-29544 — Optimize skewed join at runtime</a> <a href="#fnref11" class="footnote-backref">↩︎</a></li><li id="fn12" class="footnote-item"><a href="https://docs.databricks.com/aws/en/archive/legacy/skew-join" target="_blank" rel="noreferrer">Skew Join (legacy)</a> <a href="#fnref12" class="footnote-backref">↩︎</a></li></ol></section>',29)])])}const m=a(i,[["render",f]]);export{g as __pageData,m as default};

package/export-template/docs/assets/tuning-reference_partitioning.md.q0zKF_8X.lean.js ADDED Viewed

@@ -0,0 +1 @@