npm - sparkforensics-cli - Versions diffs - 0.1.0 → 0.2.2 - Mend

sparkforensics-cli 0.1.0 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (361) hide show

package/export-template/docs/tuning-reference/partitioning.html ADDED Viewed

@@ -0,0 +1,25 @@
+<!DOCTYPE html>
+<html lang="en-US" dir="ltr">
+  <head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width,initial-scale=1">
+    <title>Partitioning | SparkForensics</title>
+    <meta name="description" content="Docs for using and contributing to SparkForensics">
+    <meta name="generator" content="VitePress v1.6.4">
+    <link rel="preload stylesheet" href="../assets/style.DSixAiZE.css" as="style">
+    <link rel="preload stylesheet" href="../vp-icons.css" as="style">
+    <link rel="icon" type="image/svg+xml" href="../favicon.svg">
+    <link rel="preconnect" href="https://fonts.googleapis.com">
+    <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin="">
+    <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Recursive:wght,CASL@400..700,0..1&amp;family=JetBrains+Mono:wght@400;500;600&amp;display=swap">
+    <script id="check-dark-mode">(()=>{const e=localStorage.getItem("vitepress-theme-appearance")||"auto",a=window.matchMedia("(prefers-color-scheme: dark)").matches;(!e||e==="auto"?a:e==="dark")&&document.documentElement.classList.add("dark")})();</script>
+    <script id="check-mac-os">document.documentElement.classList.toggle("mac",/Mac|iPhone|iPod|iPad/i.test(navigator.platform));</script>
+  </head>
+  <body>
+    <div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="../index.html" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="../favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="../user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="../contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="../tuning-reference/index.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Getting Started</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/intro.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Introduction</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Optimization Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/spark-architecture.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Execution Model</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/memory-model.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Management</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/partitioning.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Partitioning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/joins.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Join Optimization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/data-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Data Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/table-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Table Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/caching.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Caching & Persistence</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/pyspark.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>PySpark Specifics</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/aqe.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Adaptive Query Execution</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/cluster-config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cluster Tuning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/anti-patterns.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Anti-Patterns</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Detector Catalog</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-skew.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Skew</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle I/O</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-spill.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory / Disk Spill</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-gc.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>GC Pressure</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-cold-start.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cold Start</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Executor Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-slow-host.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Slow Host</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-failures.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Failures</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-straggler.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Stragglers</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-retry-waste.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Retry Waste</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-tiny-tasks.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Tiny Tasks</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-job-failure-rate.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Job Failure Rate</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-memory-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-duplicate-plan-subtree.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Duplicate Plan Subtree</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-small-files.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Small Files</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-broadcast-sizing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Broadcast Sizing</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Reference</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/metrics.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Metrics Glossary</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Config Quick-Reference</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_tuning-reference_partitioning" data-v-39a288b8><div><h1 id="partitioning" tabindex="-1">Partitioning <a class="header-anchor" href="#partitioning" aria-label="Permalink to &quot;Partitioning {#partitioning}&quot;"></a></h1><h2 id="how-partitions-get-sized-and-shuffled" tabindex="-1">How partitions get sized and shuffled <a class="header-anchor" href="#how-partitions-get-sized-and-shuffled" aria-label="Permalink to &quot;How partitions get sized and shuffled&quot;"></a></h2><p>Every partition Spark creates maps to exactly one task and one thread, which is why partition count and size drive so much of a job&#39;s performance<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup>. The <a href="./shuffle.html">shuffle</a> side of that is governed by a single default: <code>spark.sql.shuffle.partitions</code>, which defaults to 200 and applies to every <code>join()</code>, <code>groupBy()</code>, and aggregation regardless of how much data is actually moving<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup><sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup>.</p><p>Two operations reshape that layout, and they are not interchangeable. <code>repartition(numPartitions)</code> &quot;return[s] a new RDD that has exactly numPartitions partitions&quot;<sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup>, and it earns that guarantee by always running a full hash shuffle: records are first spread across a temporary key space starting from a randomized position, seeded per-partition via <code>XORShiftRandom</code>, so upstream data ends up distributed evenly rather than clustered by its original layout. That feeds into a <code>ShuffledRDD</code> keyed by a <code>HashPartitioner(numPartitions)</code>, then a <code>CoalescedRDD</code> lands it on exactly the requested count<sup class="footnote-ref"><a href="#fn4" id="fnref4:1">[4:1]</a></sup>. High Performance Spark puts the same mechanics more plainly: &quot;repartition shuffles the RDD with a hash partitioner and the given number of partitions&quot;<sup class="footnote-ref"><a href="#fn5" id="fnref5">[5]</a></sup>. That holds whether the new count is bigger or smaller than the current one. Repartition always performs the full shuffle just described, unlike coalesce<sup class="footnote-ref"><a href="#fn6" id="fnref6">[6]</a></sup>.</p><p><code>coalesce</code>, left at its default, does not shuffle at all: it&#39;s a narrow transformation where each output partition is simply the union of a fixed set of parent partitions decided at plan time, not routed by data values, so the stage&#39;s task count just drops to the coalesced number<sup class="footnote-ref"><a href="#fn5" id="fnref5:1">[5:1]</a></sup>.</p><p>Neither <code>repartition</code> nor <code>coalesce</code> leaves behind a &quot;known partitioner&quot; the way <code>partitionBy</code> does<sup class="footnote-ref"><a href="#fn5" id="fnref5:2">[5:2]</a></sup>. That distinction is what Spark&#39;s join planner checks before it will skip a shuffle: it only does so when both sides already carry partitioner objects it can prove equal: matching partition counts for a <code>HashPartitioner</code>, matching range bounds for a <code>RangePartitioner</code><sup class="footnote-ref"><a href="#fn5" id="fnref5:3">[5:3]</a></sup>. Its default shuffle hash join path makes the same point from the other side: it partitions the second dataset using the same partitioner as the first specifically so matching keys land together, which only lets a later shuffle be skipped when a shared, recognized partitioner is already in place beforehand<sup class="footnote-ref"><a href="#fn5" id="fnref5:4">[5:4]</a></sup>.</p><p>There&#39;s also a hard ceiling on shuffle output, though it isn&#39;t something to design around: Spark&#39;s sort-based shuffle manager caps output partitions in serialized mode at <code>PackedRecordPointer.MAXIMUM_PARTITION_ID + 1</code>, roughly 16.8 million. The source code itself calls this &quot;an extreme defensive programming measure,&quot; since no real shuffle comes remotely close to it<sup class="footnote-ref"><a href="#fn7" id="fnref7">[7]</a></sup>.</p><h2 id="spotting-a-bad-layout" tabindex="-1">Spotting a bad layout <a class="header-anchor" href="#spotting-a-bad-layout" aria-label="Permalink to &quot;Spotting a bad layout&quot;"></a></h2><p>That layout choice shows up as overhead in both directions once it&#39;s wrong: partitions that are too small flood the cluster with per-task scheduling overhead, while partitions that are too large create memory pressure and straggler tasks<sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup>. Learning Spark 2nd Edition frames the healthy target from the parallelism side rather than an absolute number: at least as many partitions as there are cores across the executors, so no core sits idle; more partitions than cores is fine as long as it doesn&#39;t drift into the small-partition overhead regime above<sup class="footnote-ref"><a href="#fn3" id="fnref3:1">[3:1]</a></sup>.</p><p>A heavy filter is an easy-to-miss cause of imbalance: Spark doesn&#39;t shrink partition count when rows are filtered out, so 2,000 partitions holding 5% of the original rows just become 2,000 mostly-empty partitions<sup class="footnote-ref"><a href="#fn1" id="fnref1:2">[1:2]</a></sup>.</p><p>To size shuffle partitions from a job&#39;s actual behavior rather than a guess, Cloudera&#39;s tuning guide takes a stage that already ran, computes the ratio between its Shuffle Spill (Memory) and Shuffle Spill (Disk) metrics, and multiplies total shuffle write by that ratio to estimate in-memory shuffle size, then rounds the resulting partition count up rather than down<sup class="footnote-ref"><a href="#fn8" id="fnref8">[8]</a></sup>.</p><p><a href="./bottleneck-skew.html">Skew</a> specifically has documented, numeric detection thresholds under <a href="./aqe.html">Adaptive Query Execution</a>: a partition counts as skewed if it&#39;s larger than <code>spark.sql.adaptive.skewJoin.skewedPartitionFactor</code> (default 5.0) times the median partition size, and also larger than <code>spark.sql.adaptive.skewJoin.skewedPartitionThresholdInBytes</code> (default 256 MB)<sup class="footnote-ref"><a href="#fn2" id="fnref2:1">[2:1]</a></sup><sup class="footnote-ref"><a href="#fn9" id="fnref9">[9]</a></sup>.</p><p>On the input side, <code>spark.sql.files.maxPartitionBytes</code> (defaulting to 128 MB) governs the target chunk size when Spark splits splittable file-based sources (CSV, JSON, line-delimited text) into input partitions, based on file size<sup class="footnote-ref"><a href="#fn1" id="fnref1:3">[1:3]</a></sup>.</p><h2 id="where-a-bad-layout-costs-you" tabindex="-1">Where a bad layout costs you <a class="header-anchor" href="#where-a-bad-layout-costs-you" aria-label="Permalink to &quot;Where a bad layout costs you&quot;"></a></h2><p>Getting the partition count wrong costs more than the immediate slowdown suggests. Cloudera&#39;s tuning guide argues it&#39;s safer to over-provision partitions than to under-provision them, because Spark, unlike MapReduce, has low per-task startup overhead: &quot;when in doubt, it&#39;s almost always better to err on the side of a larger number of tasks&quot;<sup class="footnote-ref"><a href="#fn8" id="fnref8:1">[8:1]</a></sup>.</p><p>Getting <code>coalesce</code> wrong costs more than the coalesce step itself: because it&#39;s narrow, it forces the <em>entire</em> upstream stage to run at the reduced parallelism, not just the final step<sup class="footnote-ref"><a href="#fn5" id="fnref5:5">[5:5]</a></sup>. Pushed too far (<code>coalesce(1)</code>), it kills parallelism outright, because coalesce doesn&#39;t rebalance data, it just stacks existing partitions together, so partitions that were uneven going in are still uneven coming out<sup class="footnote-ref"><a href="#fn1" id="fnref1:4">[1:4]</a></sup>.</p><p>Once a shuffle stage has run, its output file count is locked in: you can&#39;t change it after the fact without inserting a stage barrier, such as writing to temporary storage or calling <code>localCheckpoint()</code>, between the shuffle and the write<sup class="footnote-ref"><a href="#fn10" id="fnref10">[10]</a></sup>.</p><p>Join alignment has its own gotcha: calling <code>repartition(n, col)</code> with the same column and count on two separate DataFrames produces data that&#39;s plausibly laid out the same way, but it doesn&#39;t leave a trackable partitioner object behind. The planner has no recorded partitioner to compare, so it has no basis for treating the two sides as co-partitioned and skipping the shuffle at join time, even though the call sites look identical<sup class="footnote-ref"><a href="#fn5" id="fnref5:6">[5:6]</a></sup>.</p><h2 id="fixing-the-layout" tabindex="-1">Fixing the layout <a class="header-anchor" href="#fixing-the-layout" aria-label="Permalink to &quot;Fixing the layout&quot;"></a></h2><p>Fixing those costs starts with accepting there&#39;s no formula to look up. There&#39;s no single formula for the right <code>spark.sql.shuffle.partitions</code> value: it depends on data set size, core count, and executor memory, and comes down to trial and error<sup class="footnote-ref"><a href="#fn3" id="fnref3:2">[3:2]</a></sup>. As a starting point, Learning Spark notes the default of 200 is usually too high for small or streaming workloads, where it should be pulled down toward the executor core count<sup class="footnote-ref"><a href="#fn3" id="fnref3:3">[3:3]</a></sup>.</p><p>For the repartition-vs-coalesce decision itself: reach for <code>coalesce</code> first whenever you&#39;re only reducing partition count, since it merges partitions already on the same node without a shuffle<sup class="footnote-ref"><a href="#fn6" id="fnref6:1">[6:1]</a></sup>. Reach for <code>repartition</code> when you actually need the shuffle: increasing partition count, fixing a lopsided distribution, or preparing a DataFrame ahead of a join or a <code>cache()</code> call, where even parallelism is worth more than the shuffle cost<sup class="footnote-ref"><a href="#fn6" id="fnref6:2">[6:2]</a></sup>. High Performance Spark reduces this to a readability rule: use <code>repartition</code> when you want a shuffle, <code>coalesce</code> when you don&#39;t, rather than leaning on coalesce&#39;s shuffle toggle to blur the line<sup class="footnote-ref"><a href="#fn5" id="fnref5:7">[5:7]</a></sup>. There is a middle option, <code>coalesce(n, shuffle=True)</code>, which behaves more like repartition, paying for a shuffle but getting real rebalancing on the way down<sup class="footnote-ref"><a href="#fn1" id="fnref1:5">[1:5]</a></sup>. <code>coalesce()</code> itself belongs at the tail of a pipeline, right before a write, purely to cut down <a href="./bottleneck-small-files.html">output file count</a><sup class="footnote-ref"><a href="#fn1" id="fnref1:6">[1:6]</a></sup>; after a heavy filter has left partitions mostly empty, following up with <code>repartition(100)</code> (or similar) buys back real parallelism<sup class="footnote-ref"><a href="#fn1" id="fnref1:7">[1:7]</a></sup>. At the SQL layer, both directions have dedicated hints for controlling output partitioning directly: <code>/*+ REPARTITION(n) */</code>, <code>/*+ REPARTITION(cols) */</code>, <code>/*+ REPARTITION_BY_RANGE(cols) */</code>, and <code>/*+ COALESCE(n) */</code><sup class="footnote-ref"><a href="#fn9" id="fnref9:1">[9:1]</a></sup>.</p><img class="light-only" src="../assets/repartition-vs-coalesce.BovLRrpj.svg" alt="A decision flowchart for choosing repartition versus coalesce based on whether a shuffle and a rebalance are needed."><img class="dark-only" src="../assets/repartition-vs-coalesce.dark.BhAczKZQ.svg" alt="A decision flowchart for choosing repartition versus coalesce based on whether a shuffle and a rebalance are needed."><p>Separate from the DataFrame <code>.coalesce()</code> call, Adaptive Query Execution has its own runtime coalescing behavior: <code>spark.sql.adaptive.coalescePartitions.enabled</code> (default true) merges small, contiguous post-shuffle partitions toward a target size at runtime, correcting over-partitioning without a manual <code>.coalesce()</code> call<sup class="footnote-ref"><a href="#fn9" id="fnref9:2">[9:2]</a></sup><sup class="footnote-ref"><a href="#fn1" id="fnref1:8">[1:8]</a></sup>.</p><p>For skewed joins specifically, salting (adding a prefix to skewed keys so the same key is treated as several different keys, then adjusting the data distribution accordingly) was one of three manual approaches used before adaptive execution existed, alongside raising <code>spark.sql.shuffle.partitions</code> and raising the broadcast hash join threshold to push a sort-merge join toward a broadcast hash join instead. All three carry &quot;lots of limitations&quot; and require manual processing, which is exactly the gap AQE&#39;s skew-join optimization was built to close<sup class="footnote-ref"><a href="#fn11" id="fnref11">[11]</a></sup>. Where automatic detection isn&#39;t precise enough, Databricks&#39; <code>/*+ SKEW(...) */</code> hint names the skewed relation and column(s), and optionally the specific skewed key values, letting the planner target just those keys directly instead of relying on automatic detection<sup class="footnote-ref"><a href="#fn12" id="fnref12">[12]</a></sup>.</p><p>For the join-alignment gotcha above, the mechanism that does reliably guarantee a shuffle can be skipped is storage-partitioned joins: both tables are physically bucketed identically at the catalog level, for example <a href="./table-formats.html">Iceberg</a> tables created with matching <code>PARTITIONED BY (bucket(...))</code> clauses. When Spark recognizes both sides report the same partitioning through <code>SupportsReportPartitioning</code>, it can drop the Exchange (shuffle) node entirely, or shuffle only one side. A plain DataFrame-level <code>repartition(col)</code> doesn&#39;t offer that guarantee, because it isn&#39;t backed by catalog-level partitioning metadata<sup class="footnote-ref"><a href="#fn9" id="fnref9:3">[9:3]</a></sup><sup class="footnote-ref"><a href="#fn2" id="fnref2:2">[2:2]</a></sup>.</p><h2 id="sources" tabindex="-1">Sources <a class="header-anchor" href="#sources" aria-label="Permalink to &quot;Sources&quot;"></a></h2><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><a href="https://luminousmen.com/post/spark-partitions" target="_blank" rel="noreferrer">Spark Partitions</a> <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a> <a href="#fnref1:2" class="footnote-backref">↩︎</a> <a href="#fnref1:3" class="footnote-backref">↩︎</a> <a href="#fnref1:4" class="footnote-backref">↩︎</a> <a href="#fnref1:5" class="footnote-backref">↩︎</a> <a href="#fnref1:6" class="footnote-backref">↩︎</a> <a href="#fnref1:7" class="footnote-backref">↩︎</a> <a href="#fnref1:8" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration — Spark</a> <a href="#fnref2" class="footnote-backref">↩︎</a> <a href="#fnref2:1" class="footnote-backref">↩︎</a> <a href="#fnref2:2" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><em>Learning Spark, 2nd Edition</em>, Damji, Wenig, Das &amp; Lee, ch. 7 <a href="#fnref3" class="footnote-backref">↩︎</a> <a href="#fnref3:1" class="footnote-backref">↩︎</a> <a href="#fnref3:2" class="footnote-backref">↩︎</a> <a href="#fnref3:3" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><a href="https://raw.githubusercontent.com/apache/spark/v3.5.0/core/src/main/scala/org/apache/spark/rdd/RDD.scala" target="_blank" rel="noreferrer">RDD.scala</a> <a href="#fnref4" class="footnote-backref">↩︎</a> <a href="#fnref4:1" class="footnote-backref">↩︎</a></p></li><li id="fn5" class="footnote-item"><p><em>High Performance Spark, 2nd Edition</em>, Karau, Polak &amp; Warren, ch. 8 <a href="#fnref5" class="footnote-backref">↩︎</a> <a href="#fnref5:1" class="footnote-backref">↩︎</a> <a href="#fnref5:2" class="footnote-backref">↩︎</a> <a href="#fnref5:3" class="footnote-backref">↩︎</a> <a href="#fnref5:4" class="footnote-backref">↩︎</a> <a href="#fnref5:5" class="footnote-backref">↩︎</a> <a href="#fnref5:6" class="footnote-backref">↩︎</a> <a href="#fnref5:7" class="footnote-backref">↩︎</a></p></li><li id="fn6" class="footnote-item"><p><em>Spark: The Definitive Guide</em>, Chambers &amp; Zaharia, ch. 19 <a href="#fnref6" class="footnote-backref">↩︎</a> <a href="#fnref6:1" class="footnote-backref">↩︎</a> <a href="#fnref6:2" class="footnote-backref">↩︎</a></p></li><li id="fn7" class="footnote-item"><p><a href="https://raw.githubusercontent.com/apache/spark/v3.5.0/core/src/main/scala/org/apache/spark/shuffle/sort/SortShuffleManager.scala" target="_blank" rel="noreferrer">SortShuffleManager.scala</a> <a href="#fnref7" class="footnote-backref">↩︎</a></p></li><li id="fn8" class="footnote-item"><p><a href="https://blog.cloudera.com/how-to-tune-your-apache-spark-jobs-part-2/" target="_blank" rel="noreferrer">How to Tune Your Apache Spark Jobs (Part 2)</a> <a href="#fnref8" class="footnote-backref">↩︎</a> <a href="#fnref8:1" class="footnote-backref">↩︎</a></p></li><li id="fn9" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/sql-performance-tuning.html" target="_blank" rel="noreferrer">Performance Tuning — Spark SQL, DataFrames and Datasets Guide</a> <a href="#fnref9" class="footnote-backref">↩︎</a> <a href="#fnref9:1" class="footnote-backref">↩︎</a> <a href="#fnref9:2" class="footnote-backref">↩︎</a> <a href="#fnref9:3" class="footnote-backref">↩︎</a></p></li><li id="fn10" class="footnote-item"><p><a href="https://luminousmen.com/post/spark-tips-partition-tuning" target="_blank" rel="noreferrer">Spark Tips: Partition Tuning</a> <a href="#fnref10" class="footnote-backref">↩︎</a></p></li><li id="fn11" class="footnote-item"><p><a href="https://issues.apache.org/jira/browse/SPARK-29544" target="_blank" rel="noreferrer">SPARK-29544 — Optimize skewed join at runtime</a> <a href="#fnref11" class="footnote-backref">↩︎</a></p></li><li id="fn12" class="footnote-item"><p><a href="https://docs.databricks.com/aws/en/archive/legacy/skew-join" target="_blank" rel="noreferrer">Skew Join (legacy)</a> <a href="#fnref12" class="footnote-backref">↩︎</a></p></li></ol></section></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="../tuning-reference/memory-model.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>Memory Management</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="../tuning-reference/joins.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>Join Optimization</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
+  </body>
+</html>

package/export-template/docs/tuning-reference/pyspark.html ADDED Viewed

@@ -0,0 +1,30 @@
+<!DOCTYPE html>
+<html lang="en-US" dir="ltr">
+  <head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width,initial-scale=1">
+    <title>PySpark Specifics | SparkForensics</title>
+    <meta name="description" content="Docs for using and contributing to SparkForensics">
+    <meta name="generator" content="VitePress v1.6.4">
+    <link rel="preload stylesheet" href="../assets/style.DSixAiZE.css" as="style">
+    <link rel="preload stylesheet" href="../vp-icons.css" as="style">
+    <link rel="icon" type="image/svg+xml" href="../favicon.svg">
+    <link rel="preconnect" href="https://fonts.googleapis.com">
+    <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin="">
+    <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Recursive:wght,CASL@400..700,0..1&amp;family=JetBrains+Mono:wght@400;500;600&amp;display=swap">
+    <script id="check-dark-mode">(()=>{const e=localStorage.getItem("vitepress-theme-appearance")||"auto",a=window.matchMedia("(prefers-color-scheme: dark)").matches;(!e||e==="auto"?a:e==="dark")&&document.documentElement.classList.add("dark")})();</script>
+    <script id="check-mac-os">document.documentElement.classList.toggle("mac",/Mac|iPhone|iPod|iPad/i.test(navigator.platform));</script>
+  </head>
+  <body>
+    <div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="../index.html" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="../favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="../user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="../contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="../tuning-reference/index.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Getting Started</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/intro.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Introduction</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Optimization Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/spark-architecture.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Execution Model</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/memory-model.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Management</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/partitioning.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Partitioning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/joins.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Join Optimization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/data-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Data Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/table-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Table Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/caching.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Caching & Persistence</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/pyspark.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>PySpark Specifics</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/aqe.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Adaptive Query Execution</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/cluster-config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cluster Tuning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/anti-patterns.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Anti-Patterns</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Detector Catalog</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-skew.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Skew</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle I/O</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-spill.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory / Disk Spill</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-gc.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>GC Pressure</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-cold-start.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cold Start</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Executor Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-slow-host.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Slow Host</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-failures.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Failures</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-straggler.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Stragglers</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-retry-waste.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Retry Waste</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-tiny-tasks.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Tiny Tasks</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-job-failure-rate.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Job Failure Rate</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-memory-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-duplicate-plan-subtree.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Duplicate Plan Subtree</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-small-files.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Small Files</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-broadcast-sizing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Broadcast Sizing</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Reference</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/metrics.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Metrics Glossary</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Config Quick-Reference</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_tuning-reference_pyspark" data-v-39a288b8><div><h1 id="pyspark" tabindex="-1">PySpark Specifics <a class="header-anchor" href="#pyspark" aria-label="Permalink to &quot;PySpark Specifics {#pyspark}&quot;"></a></h1><h2 id="crossing-the-jvm-python-boundary" tabindex="-1">Crossing the JVM/Python boundary <a class="header-anchor" href="#crossing-the-jvm-python-boundary" aria-label="Permalink to &quot;Crossing the JVM/Python boundary&quot;"></a></h2><p>A plain PySpark UDF runs one row at a time in a separate Python process, and getting each row there costs something real: &quot;PySpark UDFs required data movement between the JVM and Python, which was quite expensive,&quot; using pickle to serialize the data across that boundary<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup>. <em>Spark: The Definitive Guide</em> breaks the cost down further. Starting the extra Python process is one expense, but &quot;the real cost is in serializing the data to Python,&quot; and once the data is over there the JVM can no longer manage that worker&#39;s memory, so JVM and Python end up competing for the same machine&#39;s RAM and the worker can fail under pressure<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup>. That&#39;s the underlying reason the same source recommends writing UDFs in Scala or Java and calling them from Python when that&#39;s practical<sup class="footnote-ref"><a href="#fn2" id="fnref2:1">[2:1]</a></sup>.</p><p>Since Spark 3.4 the row-at-a-time path can itself be Arrow-optimized without giving up row semantics: set <code>useArrow=True</code> on <code>udf()</code>, or flip the session-wide <code>spark.sql.execution.pythonUDF.arrow.enabled</code> (default <code>false</code>), and a regular Python UDF becomes an &quot;Arrow Python UDF&quot; that still runs row by row but moves data with Arrow instead of pickle<sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup><sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup>. The session config only takes effect when <code>useArrow</code> is left unset on the UDF itself<sup class="footnote-ref"><a href="#fn3" id="fnref3:1">[3:1]</a></sup>.</p><img class="light-only" src="../assets/udf-execution-models.BUFDICuG.svg" alt="Across the JVM to Python boundary a plain Python UDF pickles row-at-a-time, an Arrow-optimized UDF uses Arrow transfer with row semantics, and a pandas UDF passes whole Arrow batches with no per-row handoff."><img class="dark-only" src="../assets/udf-execution-models.dark.YTNS6GDq.svg" alt="Across the JVM to Python boundary a plain Python UDF pickles row-at-a-time, an Arrow-optimized UDF uses Arrow transfer with row semantics, and a pandas UDF passes whole Arrow batches with no per-row handoff."><p>The pandas UDF (vectorized UDF, introduced in Spark 2.3) goes further and drops the per-row JVM↔Python handoff entirely: it hands the Python worker whole Arrow batches, operated on as pandas Series or DataFrames, so there&#39;s nothing to pickle row by row<sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup>. Three shapes cover most cases. <strong>SCALAR</strong> (<code>pandas.Series, ... -&gt; pandas.Series</code>) is the direct vectorized swap-in for a row-at-a-time scalar UDF (computing <code>v + 1</code>, or <code>cubed(x)</code>); PySpark calls the function once per Arrow batch and concatenates the results back into a column<sup class="footnote-ref"><a href="#fn5" id="fnref5">[5]</a></sup><sup class="footnote-ref"><a href="#fn1" id="fnref1:2">[1:2]</a></sup>. <strong>SCALAR_ITER</strong> (<code>Iterator[Series] -&gt; Iterator[Series]</code>) works the same way internally, but takes and yields an iterator instead of a single Series, which lets a function prefetch across batches<sup class="footnote-ref"><a href="#fn3" id="fnref3:2">[3:2]</a></sup>. <strong>MAP_ITER</strong>, exposed as <code>DataFrame.mapInPandas()</code> rather than as a <code>pandas_udf</code> type, maps an iterator of whole <code>pandas.DataFrame</code> partitions to another iterator of <code>pandas.DataFrame</code>s and, unlike the other two, can change the row count<sup class="footnote-ref"><a href="#fn3" id="fnref3:3">[3:3]</a></sup>. A related grouped-map API, <code>DataFrame.groupBy().applyInPandas()</code>, splits the DataFrame into groups and runs a <code>pandas.DataFrame -&gt; pandas.DataFrame</code> function per group: split, apply, combine<sup class="footnote-ref"><a href="#fn5" id="fnref5:1">[5:1]</a></sup><sup class="footnote-ref"><a href="#fn3" id="fnref3:4">[3:4]</a></sup>.</p><p><code>spark.sql.execution.arrow.pyspark.enabled</code> isn&#39;t limited to pandas UDFs, either: it also governs Arrow-based columnar transfer for <code>DataFrame.toPandas()</code> and for <code>SparkSession.createDataFrame()</code> when given a pandas DataFrame or NumPy ndarray<sup class="footnote-ref"><a href="#fn4" id="fnref4:1">[4:1]</a></sup><sup class="footnote-ref"><a href="#fn3" id="fnref3:5">[3:5]</a></sup>. A companion flag, <code>spark.sql.execution.arrow.pyspark.fallback.enabled</code>, silently drops back to the non-Arrow path if an error occurs before computation starts<sup class="footnote-ref"><a href="#fn3" id="fnref3:6">[3:6]</a></sup><sup class="footnote-ref"><a href="#fn4" id="fnref4:2">[4:2]</a></sup>.</p><p>Two worker-level configs round out the picture. <code>spark.python.worker.reuse</code> (default <code>true</code>) keeps a fixed pool of Python worker processes alive across tasks instead of forking a fresh one each time, which also means a large broadcast variable doesn&#39;t have to cross the JVM↔Python boundary again for every task<sup class="footnote-ref"><a href="#fn4" id="fnref4:3">[4:3]</a></sup>. <code>spark.python.worker.memory</code> (default <code>512m</code>) is a per-worker, Spark-managed accounting threshold for in-worker aggregation buffering (not an OS-enforced cap), and Spark <a href="./bottleneck-spill.html">spills to disk</a> once it&#39;s exceeded<sup class="footnote-ref"><a href="#fn4" id="fnref4:4">[4:4]</a></sup>.</p><h2 id="measuring-the-gap" tabindex="-1">Measuring the gap <a class="header-anchor" href="#measuring-the-gap" aria-label="Permalink to &quot;Measuring the gap&quot;"></a></h2><p>The clearest measurement here is workload-level. Databricks&#39; introductory pandas-UDF post ran three operations (Plus One, Cumulative Probability, Subtract Mean) over a 10M-row, two-column DataFrame on a single-node Databricks Community Edition cluster, and found pandas UDFs &quot;perform much better than row-at-a-time UDFs across the board, ranging from 3x to over 100x&quot;<sup class="footnote-ref"><a href="#fn5" id="fnref5:2">[5:2]</a></sup>. If a job&#39;s Python UDFs are a suspected bottleneck, expect an order-of-magnitude gap between a row-at-a-time UDF and its pandas-UDF equivalent, not a marginal one.</p><p>The comparisons in the corpus consistently favor <code>pyspark.sql.functions</code> and vectorized code over row-at-a-time UDFs. For the &quot;plus one&quot; example, &quot;built-in column operators can perform much faster in this scenario,&quot; and the pandas UDF equivalent is &quot;much faster than the row-at-a-time version&quot; because it&#39;s vectorized over the Series<sup class="footnote-ref"><a href="#fn5" id="fnref5:3">[5:3]</a></sup>. A Python UDF that could instead be expressed with built-in column functions is worth flagging on its own.</p><p>On the memory side, the two failure modes look different and are worth telling apart. A worker that spills is showing up in Spark&#39;s own accounting: it exceeded <code>spark.python.worker.memory</code> during aggregation and wrote to disk, which is expected behavior, not a crash<sup class="footnote-ref"><a href="#fn4" id="fnref4:5">[4:5]</a></sup>. A worker that&#39;s actually killed is a container-level event: if total container memory (JVM heap plus off-heap plus the Python process) exceeds what YARN or Kubernetes allocated, &quot;Kubernetes won&#39;t hesitate&quot; and the process is &quot;OOMKilled&quot;<sup class="footnote-ref"><a href="#fn6" id="fnref6">[6]</a></sup>. That boundary is governed by <a href="./memory-model.html">executor overhead</a> and <code>spark.executor.pyspark.memory</code> settings, not by <code>worker.memory</code>.</p><h2 id="what-the-gap-costs" tabindex="-1">What the gap costs <a class="header-anchor" href="#what-the-gap-costs" aria-label="Permalink to &quot;What the gap costs&quot;"></a></h2><p>Those numbers point to a structural cost, not a tuning quirk. The JVM↔Python boundary is where a plain Python UDF pays twice: once to start the separate process, and again (the larger cost) to serialize every row across it<sup class="footnote-ref"><a href="#fn2" id="fnref2:2">[2:2]</a></sup>. Because the JVM can&#39;t manage memory inside the Python process once data has crossed over, the two runtimes end up competing for the same machine&#39;s memory, and the Python worker can fail under that pressure<sup class="footnote-ref"><a href="#fn2" id="fnref2:3">[2:3]</a></sup>. That&#39;s a correctness risk as well as a performance one, and it&#39;s why <em>Spark: The Definitive Guide</em> recommends Scala/Java UDFs called from Python over native Python UDFs wherever that&#39;s practical<sup class="footnote-ref"><a href="#fn2" id="fnref2:4">[2:4]</a></sup>.</p><p>The magnitude backs this up: Databricks measured pandas UDFs beating row-at-a-time UDFs by 3x to over 100x depending on the operation<sup class="footnote-ref"><a href="#fn5" id="fnref5:4">[5:4]</a></sup>. Skipping per-row pickling (by moving to Arrow-based row UDFs or, further, to pandas UDFs operating on whole batches) isn&#39;t a marginal tuning knob here; it changes which order of magnitude a job runs at.</p><p>The two memory configs matter for different reasons, too. <code>spark.python.worker.memory</code> only controls when Spark chooses to spill aggregation state to disk; tuning it trades disk I/O for headroom, it doesn&#39;t prevent a crash<sup class="footnote-ref"><a href="#fn4" id="fnref4:6">[4:6]</a></sup>. Actual OOM kills happen at the container boundary, and <code>spark.executor.pyspark.memory</code> (which leans on Python&#39;s <code>resource</code> module and so doesn&#39;t cap memory on macOS and doesn&#39;t exist at all on Windows) is the config that&#39;s actually in that path<sup class="footnote-ref"><a href="#fn4" id="fnref4:7">[4:7]</a></sup>. Conflating the two means tuning the wrong knob when a Python worker gets OOMKilled.</p><h2 id="closing-the-gap" tabindex="-1">Closing the gap <a class="header-anchor" href="#closing-the-gap" aria-label="Permalink to &quot;Closing the gap&quot;"></a></h2><p>Closing that gap means avoiding the boundary crossing altogether, or crossing it as cheaply as possible. Where the logic allows it, prefer a built-in <code>pyspark.sql.functions</code> expression or a Scala/Java UDF called from Python over a plain row-at-a-time Python UDF<sup class="footnote-ref"><a href="#fn2" id="fnref2:5">[2:5]</a></sup><sup class="footnote-ref"><a href="#fn5" id="fnref5:5">[5:5]</a></sup>; this is the change with the largest documented payoff, 3x to over 100x<sup class="footnote-ref"><a href="#fn5" id="fnref5:6">[5:6]</a></sup>.</p><p>Where a Python UDF is unavoidable, cut the row-at-a-time cost first by turning on Arrow for it:</p><div class="language-python vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">python</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">@udf</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70;">returnType</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">=</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&#39;int&#39;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70;">useArrow</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">=</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">True</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">)  </span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># An Arrow Python UDF</span></span>
+<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">def</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;"> arrow_slen</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(s):</span></span>
+<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">    return</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> len</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(s)</span></span></code></pre></div><p>or set the session-wide equivalent so existing UDFs pick it up without code changes.</p><blockquote><p><strong>PySpark:</strong> <code>spark.conf.set(&quot;spark.sql.execution.pythonUDF.arrow.enabled&quot;, &quot;true&quot;)</code> turns plain UDFs into Arrow-backed ones, as long as <code>useArrow</code> isn&#39;t explicitly set on the UDF itself<sup class="footnote-ref"><a href="#fn3" id="fnref3:7">[3:7]</a></sup><sup class="footnote-ref"><a href="#fn4" id="fnref4:8">[4:8]</a></sup>.</p></blockquote><p>Beyond that, reach for the vectorized APIs instead of a scalar UDF:</p><ul><li>Use a <strong>SCALAR</strong> pandas UDF (<code>pandas.Series, ... -&gt; pandas.Series</code>) as the default vectorized replacement for a row-at-a-time scalar UDF<sup class="footnote-ref"><a href="#fn5" id="fnref5:7">[5:7]</a></sup><sup class="footnote-ref"><a href="#fn1" id="fnref1:3">[1:3]</a></sup>.</li><li>Use <strong>SCALAR_ITER</strong> (<code>Iterator[Series] -&gt; Iterator[Series]</code>) when the function needs expensive one-time setup. The documented pattern initializes state once, then loops over the batch iterator reusing it, instead of re-initializing per batch<sup class="footnote-ref"><a href="#fn3" id="fnref3:8">[3:8]</a></sup>:</li></ul><div class="language-python vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">python</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">def</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;"> apply_with_state</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(iterator):</span></span>
+<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">    state </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">=</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> very_expensive_initialization()</span></span>
+<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">    for</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> batch </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">in</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> iterator:</span></span>
+<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">        yield</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> calculate_with_state(batch, state)</span></span></code></pre></div><ul><li>Use <code>DataFrame.mapInPandas()</code> when the transform needs to change row count (filtering, expansion, deduplication) rather than map one-to-one<sup class="footnote-ref"><a href="#fn3" id="fnref3:9">[3:9]</a></sup>.</li><li>Use <code>DataFrame.groupBy().applyInPandas()</code> for per-group logic that needs the whole group as state, such as subtracting a group mean or fitting a per-group regression<sup class="footnote-ref"><a href="#fn5" id="fnref5:8">[5:8]</a></sup><sup class="footnote-ref"><a href="#fn3" id="fnref3:10">[3:10]</a></sup>. Size groups with care: a full group loads into memory before the function runs, and <code>maxRecordsPerBatch</code> doesn&#39;t apply to groups, so a skewed group risks OOM<sup class="footnote-ref"><a href="#fn3" id="fnref3:11">[3:11]</a></sup>.</li></ul><p>For pandas/NumPy conversion at the driver, turn on Arrow explicitly rather than relying on defaults.</p><blockquote><p><strong>PySpark:</strong> with <code>spark.sql.execution.arrow.pyspark.enabled</code> set to <code>&quot;true&quot;</code>, <code>spark.createDataFrame(pdf)</code> and <code>df.select(&quot;*&quot;).toPandas()</code> convert through Arrow and return the same result as the non-Arrow path<sup class="footnote-ref"><a href="#fn4" id="fnref4:9">[4:9]</a></sup><sup class="footnote-ref"><a href="#fn3" id="fnref3:12">[3:12]</a></sup>. <code>spark.sql.execution.arrow.pyspark.fallback.enabled</code> keeps a safety net by falling back silently on pre-computation errors<sup class="footnote-ref"><a href="#fn3" id="fnref3:13">[3:13]</a></sup><sup class="footnote-ref"><a href="#fn4" id="fnref4:10">[4:10]</a></sup>.</p></blockquote><p>Leave <code>spark.python.worker.reuse</code> at its default (<code>true</code>) unless there&#39;s a specific reason not to; it keeps a fixed pool of Python workers alive so a large broadcast variable isn&#39;t re-shipped to Python for every task<sup class="footnote-ref"><a href="#fn4" id="fnref4:11">[4:11]</a></sup>. If a job spills at the Python-worker level, <code>spark.python.worker.memory</code> is the knob for that; if it&#39;s getting OOMKilled at the container level, look at executor overhead and <code>spark.executor.pyspark.memory</code> instead, keeping in mind the latter&#39;s <code>resource</code>-module limitations on macOS and its absence on Windows<sup class="footnote-ref"><a href="#fn4" id="fnref4:12">[4:12]</a></sup><sup class="footnote-ref"><a href="#fn6" id="fnref6:1">[6:1]</a></sup>.</p><h2 id="sources" tabindex="-1">Sources <a class="header-anchor" href="#sources" aria-label="Permalink to &quot;Sources&quot;"></a></h2><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><em>Learning Spark, 2nd Edition</em>, Damji, Wenig, Das &amp; Lee, ch. 5 <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a> <a href="#fnref1:2" class="footnote-backref">↩︎</a> <a href="#fnref1:3" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><em>Spark: The Definitive Guide</em>, Chambers &amp; Zaharia, ch. 6 <a href="#fnref2" class="footnote-backref">↩︎</a> <a href="#fnref2:1" class="footnote-backref">↩︎</a> <a href="#fnref2:2" class="footnote-backref">↩︎</a> <a href="#fnref2:3" class="footnote-backref">↩︎</a> <a href="#fnref2:4" class="footnote-backref">↩︎</a> <a href="#fnref2:5" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><a href="https://spark.apache.org/docs/3.5.8/api/python/user_guide/sql/arrow_pandas.html" target="_blank" rel="noreferrer">Apache Arrow in PySpark</a> <a href="#fnref3" class="footnote-backref">↩︎</a> <a href="#fnref3:1" class="footnote-backref">↩︎</a> <a href="#fnref3:2" class="footnote-backref">↩︎</a> <a href="#fnref3:3" class="footnote-backref">↩︎</a> <a href="#fnref3:4" class="footnote-backref">↩︎</a> <a href="#fnref3:5" class="footnote-backref">↩︎</a> <a href="#fnref3:6" class="footnote-backref">↩︎</a> <a href="#fnref3:7" class="footnote-backref">↩︎</a> <a href="#fnref3:8" class="footnote-backref">↩︎</a> <a href="#fnref3:9" class="footnote-backref">↩︎</a> <a href="#fnref3:10" class="footnote-backref">↩︎</a> <a href="#fnref3:11" class="footnote-backref">↩︎</a> <a href="#fnref3:12" class="footnote-backref">↩︎</a> <a href="#fnref3:13" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration — Spark</a> <a href="#fnref4" class="footnote-backref">↩︎</a> <a href="#fnref4:1" class="footnote-backref">↩︎</a> <a href="#fnref4:2" class="footnote-backref">↩︎</a> <a href="#fnref4:3" class="footnote-backref">↩︎</a> <a href="#fnref4:4" class="footnote-backref">↩︎</a> <a href="#fnref4:5" class="footnote-backref">↩︎</a> <a href="#fnref4:6" class="footnote-backref">↩︎</a> <a href="#fnref4:7" class="footnote-backref">↩︎</a> <a href="#fnref4:8" class="footnote-backref">↩︎</a> <a href="#fnref4:9" class="footnote-backref">↩︎</a> <a href="#fnref4:10" class="footnote-backref">↩︎</a> <a href="#fnref4:11" class="footnote-backref">↩︎</a> <a href="#fnref4:12" class="footnote-backref">↩︎</a></p></li><li id="fn5" class="footnote-item"><p><a href="https://www.databricks.com/blog/2017/10/30/introducing-vectorized-udfs-for-pyspark.html" target="_blank" rel="noreferrer">Introducing Pandas UDF for PySpark</a> <a href="#fnref5" class="footnote-backref">↩︎</a> <a href="#fnref5:1" class="footnote-backref">↩︎</a> <a href="#fnref5:2" class="footnote-backref">↩︎</a> <a href="#fnref5:3" class="footnote-backref">↩︎</a> <a href="#fnref5:4" class="footnote-backref">↩︎</a> <a href="#fnref5:5" class="footnote-backref">↩︎</a> <a href="#fnref5:6" class="footnote-backref">↩︎</a> <a href="#fnref5:7" class="footnote-backref">↩︎</a> <a href="#fnref5:8" class="footnote-backref">↩︎</a></p></li><li id="fn6" class="footnote-item"><p><a href="https://luminousmen.com/post/dive-into-spark-memory" target="_blank" rel="noreferrer">Dive into Spark memory management</a> <a href="#fnref6" class="footnote-backref">↩︎</a> <a href="#fnref6:1" class="footnote-backref">↩︎</a></p></li></ol></section></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="../tuning-reference/caching.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>Caching & Persistence</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="../tuning-reference/aqe.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>Adaptive Query Execution</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
+  </body>
+</html>

package/export-template/docs/tuning-reference/shuffle.html ADDED Viewed

@@ -0,0 +1,25 @@
+<!DOCTYPE html>
+<html lang="en-US" dir="ltr">
+  <head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width,initial-scale=1">
+    <title>Shuffle | SparkForensics</title>
+    <meta name="description" content="Docs for using and contributing to SparkForensics">
+    <meta name="generator" content="VitePress v1.6.4">
+    <link rel="preload stylesheet" href="../assets/style.DSixAiZE.css" as="style">
+    <link rel="preload stylesheet" href="../vp-icons.css" as="style">
+    <link rel="icon" type="image/svg+xml" href="../favicon.svg">
+    <link rel="preconnect" href="https://fonts.googleapis.com">
+    <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin="">
+    <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Recursive:wght,CASL@400..700,0..1&amp;family=JetBrains+Mono:wght@400;500;600&amp;display=swap">
+    <script id="check-dark-mode">(()=>{const e=localStorage.getItem("vitepress-theme-appearance")||"auto",a=window.matchMedia("(prefers-color-scheme: dark)").matches;(!e||e==="auto"?a:e==="dark")&&document.documentElement.classList.add("dark")})();</script>
+    <script id="check-mac-os">document.documentElement.classList.toggle("mac",/Mac|iPhone|iPod|iPad/i.test(navigator.platform));</script>
+  </head>
+  <body>
+    <div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="../index.html" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="../favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="../user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="../contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="../tuning-reference/index.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Getting Started</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/intro.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Introduction</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Optimization Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/spark-architecture.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Execution Model</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/memory-model.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Management</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/partitioning.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Partitioning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/joins.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Join Optimization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/data-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Data Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/table-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Table Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/caching.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Caching & Persistence</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/pyspark.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>PySpark Specifics</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/aqe.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Adaptive Query Execution</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/cluster-config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cluster Tuning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/anti-patterns.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Anti-Patterns</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Detector Catalog</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-skew.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Skew</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle I/O</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-spill.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory / Disk Spill</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-gc.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>GC Pressure</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-cold-start.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cold Start</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Executor Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-slow-host.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Slow Host</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-failures.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Failures</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-straggler.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Stragglers</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-retry-waste.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Retry Waste</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-tiny-tasks.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Tiny Tasks</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-job-failure-rate.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Job Failure Rate</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-memory-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-duplicate-plan-subtree.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Duplicate Plan Subtree</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-small-files.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Small Files</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-broadcast-sizing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Broadcast Sizing</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Reference</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/metrics.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Metrics Glossary</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Config Quick-Reference</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_tuning-reference_shuffle" data-v-39a288b8><div><h1 id="shuffle" tabindex="-1">Shuffle <a class="header-anchor" href="#shuffle" aria-label="Permalink to &quot;Shuffle {#shuffle}&quot;"></a></h1><h2 id="how-the-shuffle-works" tabindex="-1">How the shuffle works <a class="header-anchor" href="#how-the-shuffle-works" aria-label="Permalink to &quot;How the shuffle works&quot;"></a></h2><p>A shuffle is Spark&#39;s mechanism for exchanging, sorting, grouping, and merging data across executors whenever an operation needs rows that share a key to land on the same partition. At the DataFrame/SQL level, the classic wide transformations that trigger one are <code>groupBy()</code>, <code>join()</code>, <code>agg()</code>, <code>sortBy()</code>, and <code>reduceByKey()</code>-style aggregations. Any join implementation that isn&#39;t a <a href="./joins.html">broadcast join</a> (shuffle hash join, shuffle sort-merge join, or shuffle-and-replicated nested loop / Cartesian product join) also requires a shuffle<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup>. At the RDD layer underneath, the operations that can cause a shuffle are <a href="./partitioning.html">repartitioning</a> (<code>repartition</code>, <code>coalesce</code>), <code>&#39;ByKey</code> operations other than counting (<code>groupByKey</code>, <code>reduceByKey</code>), and join-family operations (<code>cogroup</code>, <code>join</code>)<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup>.</p><p>Spark can skip the shuffle outright in a few well-defined cases where it already knows the data layout. Storage Partition Join (SPJ) avoids the shuffle phase entirely when Spark can use the partitioning already reported by a compatible V2 data source (<code>spark.sql.sources.v2.bucketing.enabled</code>, default <code>true</code> since 3.3.0), generalizing bucket joins to functions registered in a <code>FunctionCatalog</code><sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup>. Classic Hive-style bucketing gets the same effect: once both sides of a join are bucketed and sorted the same way, the physical plan shows no <code>Exchange</code> operator at all<sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup>. <code>DataFrameWriter.partitionBy</code>, by contrast, does not trigger a shuffle by itself on write, though it can still leave you with a large number of small output files<sup class="footnote-ref"><a href="#fn5" id="fnref5">[5]</a></sup>.</p><p>Modern Spark ships a single shuffle manager, <code>SortShuffleManager</code> (3.5 source). Incoming records are sorted by their target partition id and written to one map output file per task; reducers then fetch contiguous regions of that file for their share of the map output, spilling sorted subsets to disk and merging them if the data doesn&#39;t fit in memory<sup class="footnote-ref"><a href="#fn6" id="fnref6">[6]</a></sup>. Internally it has two write paths: a &quot;serialized sorting&quot; path used when there&#39;s no map-side combine, the serializer supports relocation of serialized values (Kryo or Spark SQL&#39;s custom serializers), and the shuffle produces at most 16,777,216 output partitions; and a &quot;deserialized sorting&quot; path used for everything else<sup class="footnote-ref"><a href="#fn6" id="fnref6:1">[6:1]</a></sup>. Framed against the older hash-based approach (where each map task produces a separate output file per reduce task rather than one sorted file), sort-based shuffle pays a sorting cost but scales better once shuffle data gets large, which is why Spark and Flink both moved to it and pair it with an external shuffle service<sup class="footnote-ref"><a href="#fn7" id="fnref7">[7]</a></sup>.</p><img class="light-only" src="../assets/shuffle-map-reduce.KuOEZVmg.svg" alt="Map tasks sorting records by target partition id into one output file each while reduce tasks fetch their partition blocks from every map output."><img class="dark-only" src="../assets/shuffle-map-reduce.dark.BgQZnFSb.svg" alt="Map tasks sorting records by target partition id into one output file each while reduce tasks fetch their partition blocks from every map output."><p>Two pieces of supporting infrastructure sit alongside the core shuffle manager. The external shuffle service (<code>spark.shuffle.service.enabled</code>, default <code>false</code>) preserves the shuffle files written by executors so they can be safely removed, or so shuffle fetches continue even after an executor failure; enabling it requires standing up the service separately, since the flag alone isn&#39;t enough<sup class="footnote-ref"><a href="#fn8" id="fnref8">[8]</a></sup>. It&#39;s a long-running process on each cluster node, independent of any particular Spark application or its executors: once enabled, executors fetch shuffle files from this service instead of from each other, so shuffle state written by an executor keeps being served after that executor&#39;s own lifetime ends<sup class="footnote-ref"><a href="#fn9" id="fnref9">[9]</a></sup>. Push-based shuffle, designed under SPARK-30602 and based on LinkedIn&#39;s &quot;Magnet&quot; shuffle service, goes further: it changes the reduce side from a pull model (reducers requesting many small, randomly-ordered blocks from wherever each mapper wrote them) into a push-and-merge model, where mapper-generated blocks are pushed to remote Magnet shuffle services that opportunistically merge them into large per-partition chunks before the reduce stage starts<sup class="footnote-ref"><a href="#fn7" id="fnref7:1">[7:1]</a></sup>.</p><h2 id="reading-exchange-nodes" tabindex="-1">Reading Exchange nodes <a class="header-anchor" href="#reading-exchange-nodes" aria-label="Permalink to &quot;Reading Exchange nodes&quot;"></a></h2><p>Whether an operation shuffles doesn&#39;t have to be guesswork: shuffles show up in the physical plan as <code>Exchange</code> nodes, and reading the plan is the most direct way to check. <code>groupBy()</code> on a DataFrame is a wide transformation and normally requires an <code>Exchange</code> so all rows for a key land on the same partition<sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup>; that node disappears only when Spark already knows the data is co-partitioned by the grouping key. The clearest documented case is a table bucketed and sorted on the join/group key, where the physical plan drops the <code>Exchange</code> because the data was already shuffled at write time<sup class="footnote-ref"><a href="#fn10" id="fnref10">[10]</a></sup>. SPJ produces the same absence of an <code>Exchange</code> for join queries against compatible V2 sources<sup class="footnote-ref"><a href="#fn3" id="fnref3:1">[3:1]</a></sup>.</p><p><code>DataFrame.repartition(key)</code> is worth watching for separately: it&#39;s a one-time, explicit shuffle, but if you cache the result, subsequent joins on that same key skip their own shuffle because the data is already partitioned that way<sup class="footnote-ref"><a href="#fn5" id="fnref5:1">[5:1]</a></sup>. Don&#39;t mistake <code>partitionBy</code> on write for the same thing: it changes the on-disk directory layout but is &quot;not the equivalent&quot; of an in-memory repartition, so <code>df.groupBy(&#39;key&#39;).sum()</code> over data written with <code>partitionBy</code> still requires an <code>Exchange</code><sup class="footnote-ref"><a href="#fn5" id="fnref5:2">[5:2]</a></sup>.</p><p>Deduplication is a case where there&#39;s no avoiding the shuffle regardless of how you write it: Spark SQL&#39;s <code>dropDuplicates</code> selects unique rows the same way RDD-level <code>distinct</code> does, and both &quot;can require a shuffle&quot;<sup class="footnote-ref"><a href="#fn11" id="fnref11">[11]</a></sup>. The only documented difference between them is key flexibility, not shuffle volume: unlike RDD <code>distinct</code>, DataFrame <code>dropDuplicates</code> can optionally drop rows based on only a subset of columns (e.g., <code>dropDuplicates(List(&quot;id&quot;))</code>) rather than requiring uniqueness across the whole row<sup class="footnote-ref"><a href="#fn11" id="fnref11:1">[11:1]</a></sup>.</p><h2 id="where-the-cost-accumulates" tabindex="-1">Where the cost accumulates <a class="header-anchor" href="#where-the-cost-accumulates" aria-label="Permalink to &quot;Where the cost accumulates&quot;"></a></h2><p>Confirming a shuffle happened is the easy part; pricing it out is the harder one. Shuffle cost shows up in several places at once, and Spark&#39;s shuffle-tuning knobs largely exist to manage it. Inside <code>SortShuffleManager</code>, when there are fewer than <code>spark.shuffle.sort.bypassMergeThreshold</code> reduce partitions (default 200)<sup class="footnote-ref"><a href="#fn8" id="fnref8:1">[8:1]</a></sup> and no map-side aggregation is needed, Spark takes a bypass path: it writes <code>numPartitions</code> files directly per map task and concatenates them at the end, rather than merge-sorting spilled files. This avoids serializing and deserializing the data twice, which the normal merge-sort path would otherwise pay; the trade-off is having multiple files open at once, and more memory allocated to buffers<sup class="footnote-ref"><a href="#fn6" id="fnref6:2">[6:2]</a></sup>. That trade stays cheap only while the reduce-side partition count is small.</p><p>Reducers also pay a fixed memory cost for every map output fetched in parallel, since each needs a buffer to receive it; that&#39;s what <code>spark.reducer.maxSizeInFlight</code> (default 48m) caps<sup class="footnote-ref"><a href="#fn8" id="fnref8:2">[8:2]</a></sup>. And map tasks pay an I/O cost writing shuffle files: a larger <code>spark.shuffle.file.buffer</code> (default 32k) reduces the number of disk seeks and system calls needed to create those intermediate files<sup class="footnote-ref"><a href="#fn8" id="fnref8:3">[8:3]</a></sup>.</p><p>External shuffle infrastructure matters most once <a href="./cluster-config.html">dynamic allocation</a> is in play: without the external shuffle service, an executor decommissioned mid-shuffle (say, because of stragglers) would force its shuffle files to be recomputed from scratch once it&#39;s removed<sup class="footnote-ref"><a href="#fn9" id="fnref9:1">[9:1]</a></sup>. Push-based shuffle exists because, as shuffle data volume grows, the number of blocks a classic shuffle produces grows quadratically (mappers × reducers) while individual block sizes shrink to only tens of KB, which is inefficient for HDD-backed random reads<sup class="footnote-ref"><a href="#fn7" id="fnref7:2">[7:2]</a></sup>. Magnet&#39;s push-and-merge model improves disk I/O efficiency by converting many small random reads into large sequential reads of pre-merged chunks, and improves reducer data locality since merged output can be co-located with the reduce tasks that consume it; the push itself is decoupled from the mappers, so it doesn&#39;t add to map task runtime or fail map tasks if a push fails, and it&#39;s best-effort, so reducers can fetch a mix of merged and unmerged blocks<sup class="footnote-ref"><a href="#fn7" id="fnref7:3">[7:3]</a></sup>. LinkedIn&#39;s own production rollout, on a cluster running 30K+ Spark applications shuffling roughly 5 PB/day, reported a large reduction in shuffle read wait time and a corresponding drop in total executor runtime after enabling it on complex production flows<sup class="footnote-ref"><a href="#fn7" id="fnref7:4">[7:4]</a></sup>.</p><h2 id="tuning-or-avoiding-the-shuffle" tabindex="-1">Tuning or avoiding the shuffle <a class="header-anchor" href="#tuning-or-avoiding-the-shuffle" aria-label="Permalink to &quot;Tuning or avoiding the shuffle&quot;"></a></h2><p>The levers split into two groups: avoiding a shuffle, and tuning the one you can&#39;t avoid. Where possible, avoid the shuffle rather than tune it: bucket and sort tables on the join/group key so the physical plan drops the <code>Exchange</code> entirely<sup class="footnote-ref"><a href="#fn10" id="fnref10:1">[10:1]</a></sup><sup class="footnote-ref"><a href="#fn4" id="fnref4:1">[4:1]</a></sup>, or lean on Storage Partition Join for compatible V2 sources<sup class="footnote-ref"><a href="#fn3" id="fnref3:2">[3:2]</a></sup>. If you need to reuse a partitioning scheme across several joins, <code>repartition(key)</code> once and cache the result rather than relying on <code>partitionBy</code>, which only affects the on-disk layout and doesn&#39;t exempt later aggregations from their own <code>Exchange</code><sup class="footnote-ref"><a href="#fn5" id="fnref5:3">[5:3]</a></sup>.</p><blockquote><p><strong>PySpark:</strong> <code>dropDuplicates(subset)</code> still shuffles like <code>distinct()</code>, but lets you dedupe on a subset of columns instead of the whole row; use <code>df.dropDuplicates([&quot;id&quot;])</code> when uniqueness only needs to hold on a few key columns<sup class="footnote-ref"><a href="#fn11" id="fnref11:2">[11:2]</a></sup>.</p></blockquote><p>For shuffles you can&#39;t avoid, the main tuning levers are:</p><ul><li><code>spark.shuffle.sort.bypassMergeThreshold</code> (default 200): raise or lower the reduce-partition-count cutoff for the bypass (hash-style) write path described above<sup class="footnote-ref"><a href="#fn8" id="fnref8:4">[8:4]</a></sup>.</li><li><code>spark.shuffle.compress</code> (default <code>true</code>): compresses map output files using whatever codec <code>spark.io.compression.codec</code> names (default <code>lz4</code>; <code>lzf</code>, <code>snappy</code>, and <code>zstd</code> are also available)<sup class="footnote-ref"><a href="#fn8" id="fnref8:5">[8:5]</a></sup>. <code>spark.shuffle.spill.compress</code> (also default <code>true</code>) separately governs compression of spilled shuffle data, using the same codec setting<sup class="footnote-ref"><a href="#fn8" id="fnref8:6">[8:6]</a></sup>.</li><li><code>spark.reducer.maxSizeInFlight</code> (default 48m): raising it lets more data move in flight at the cost of per-task memory; lowering it saves memory at the cost of more fetch rounds<sup class="footnote-ref"><a href="#fn8" id="fnref8:7">[8:7]</a></sup>.</li><li><code>spark.shuffle.file.buffer</code> (default 32k): Learning Spark&#39;s tuning guidance recommends bumping it to 1 MB for large jobs, trading some per-task memory for fewer disk seeks and syscalls during the shuffle-write phase<sup class="footnote-ref"><a href="#fn1" id="fnref1:2">[1:2]</a></sup>.</li><li><code>spark.shuffle.service.enabled</code> (default <code>false</code>): turn on the external shuffle service so executors can be safely removed under dynamic allocation without losing shuffle state. Dynamic allocation&#39;s documentation lists shuffle tracking (<code>spark.dynamicAllocation.shuffleTracking.enabled</code>), shuffle-block decommissioning, and a custom <code>ShuffleDataIO</code> plugin backed by reliable storage as alternatives to enabling the service outright<sup class="footnote-ref"><a href="#fn8" id="fnref8:8">[8:8]</a></sup>.</li><li>Push-based shuffle: enable it with the paired server/client flags added in Spark 3.2.0: <code>spark.shuffle.push.server.mergedShuffleFileManagerImpl</code> on the server side, and <code>spark.shuffle.push.enabled=true</code> on the client side (both disabled by default; the client flag only takes effect together with the server-side one). It&#39;s currently only supported for Spark on YARN with the external shuffle service enabled<sup class="footnote-ref"><a href="#fn8" id="fnref8:9">[8:9]</a></sup>.</li></ul><h2 id="sources" tabindex="-1">Sources <a class="header-anchor" href="#sources" aria-label="Permalink to &quot;Sources&quot;"></a></h2><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><em>Learning Spark, 2nd Edition</em>, Damji, Wenig, Das, Lee, ch. 7: Optimizing and Tuning Spark Applications <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a> <a href="#fnref1:2" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/rdd-programming-guide.html" target="_blank" rel="noreferrer">RDD Programming Guide</a> <a href="#fnref2" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/sql-performance-tuning.html" target="_blank" rel="noreferrer">Performance Tuning: Spark SQL, DataFrames and Datasets Guide</a> <a href="#fnref3" class="footnote-backref">↩︎</a> <a href="#fnref3:1" class="footnote-backref">↩︎</a> <a href="#fnref3:2" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><a href="https://books.japila.pl/spark-sql-internals/bucketing/" target="_blank" rel="noreferrer">Bucketing (The Internals of Spark SQL)</a> <a href="#fnref4" class="footnote-backref">↩︎</a> <a href="#fnref4:1" class="footnote-backref">↩︎</a></p></li><li id="fn5" class="footnote-item"><p><a href="https://luminousmen.com/post/spark-tips-partition-tuning" target="_blank" rel="noreferrer">Spark Tips: Partition Tuning</a> <a href="#fnref5" class="footnote-backref">↩︎</a> <a href="#fnref5:1" class="footnote-backref">↩︎</a> <a href="#fnref5:2" class="footnote-backref">↩︎</a> <a href="#fnref5:3" class="footnote-backref">↩︎</a></p></li><li id="fn6" class="footnote-item"><p><a href="https://raw.githubusercontent.com/apache/spark/v3.5.0/core/src/main/scala/org/apache/spark/shuffle/sort/SortShuffleManager.scala" target="_blank" rel="noreferrer">SortShuffleManager.scala</a> <a href="#fnref6" class="footnote-backref">↩︎</a> <a href="#fnref6:1" class="footnote-backref">↩︎</a> <a href="#fnref6:2" class="footnote-backref">↩︎</a></p></li><li id="fn7" class="footnote-item"><p><a href="https://issues.apache.org/jira/browse/SPARK-30602" target="_blank" rel="noreferrer">SPARK-30602: Support push-based shuffle to improve shuffle efficiency (Magnet)</a> <a href="#fnref7" class="footnote-backref">↩︎</a> <a href="#fnref7:1" class="footnote-backref">↩︎</a> <a href="#fnref7:2" class="footnote-backref">↩︎</a> <a href="#fnref7:3" class="footnote-backref">↩︎</a> <a href="#fnref7:4" class="footnote-backref">↩︎</a></p></li><li id="fn8" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration (Spark)</a> <a href="#fnref8" class="footnote-backref">↩︎</a> <a href="#fnref8:1" class="footnote-backref">↩︎</a> <a href="#fnref8:2" class="footnote-backref">↩︎</a> <a href="#fnref8:3" class="footnote-backref">↩︎</a> <a href="#fnref8:4" class="footnote-backref">↩︎</a> <a href="#fnref8:5" class="footnote-backref">↩︎</a> <a href="#fnref8:6" class="footnote-backref">↩︎</a> <a href="#fnref8:7" class="footnote-backref">↩︎</a> <a href="#fnref8:8" class="footnote-backref">↩︎</a> <a href="#fnref8:9" class="footnote-backref">↩︎</a></p></li><li id="fn9" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/job-scheduling.html" target="_blank" rel="noreferrer">Job Scheduling</a> <a href="#fnref9" class="footnote-backref">↩︎</a> <a href="#fnref9:1" class="footnote-backref">↩︎</a></p></li><li id="fn10" class="footnote-item"><p><a href="https://www.taboola.com/engineering/bucket-the-shuffle-out-of-here/" target="_blank" rel="noreferrer">Bucket the Shuffle Out of Here (Taboola Engineering)</a> <a href="#fnref10" class="footnote-backref">↩︎</a> <a href="#fnref10:1" class="footnote-backref">↩︎</a></p></li><li id="fn11" class="footnote-item"><p><em>High Performance Spark, 2nd Edition</em>, Karau, Polak &amp; Warren, ch. 5: DataFrames, Datasets, and Spark SQL <a href="#fnref11" class="footnote-backref">↩︎</a> <a href="#fnref11:1" class="footnote-backref">↩︎</a> <a href="#fnref11:2" class="footnote-backref">↩︎</a></p></li></ol></section></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="../tuning-reference/joins.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>Join Optimization</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="../tuning-reference/data-formats.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>Data Formats</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
+  </body>
+</html>