npm - sparkforensics-cli - Versions diffs - 0.1.0 → 0.2.2 - Mend

sparkforensics-cli 0.1.0 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (361) hide show

package/export-template/docs/tuning-reference/joins.html ADDED Viewed

@@ -0,0 +1,25 @@
+<!DOCTYPE html>
+<html lang="en-US" dir="ltr">
+  <head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width,initial-scale=1">
+    <title>Join Optimization | SparkForensics</title>
+    <meta name="description" content="Docs for using and contributing to SparkForensics">
+    <meta name="generator" content="VitePress v1.6.4">
+    <link rel="preload stylesheet" href="../assets/style.DSixAiZE.css" as="style">
+    <link rel="preload stylesheet" href="../vp-icons.css" as="style">
+    <link rel="icon" type="image/svg+xml" href="../favicon.svg">
+    <link rel="preconnect" href="https://fonts.googleapis.com">
+    <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin="">
+    <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Recursive:wght,CASL@400..700,0..1&amp;family=JetBrains+Mono:wght@400;500;600&amp;display=swap">
+    <script id="check-dark-mode">(()=>{const e=localStorage.getItem("vitepress-theme-appearance")||"auto",a=window.matchMedia("(prefers-color-scheme: dark)").matches;(!e||e==="auto"?a:e==="dark")&&document.documentElement.classList.add("dark")})();</script>
+    <script id="check-mac-os">document.documentElement.classList.toggle("mac",/Mac|iPhone|iPod|iPad/i.test(navigator.platform));</script>
+  </head>
+  <body>
+    <div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="../index.html" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="../favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="../user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="../contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="../tuning-reference/index.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Getting Started</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/intro.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Introduction</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Optimization Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/spark-architecture.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Execution Model</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/memory-model.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Management</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/partitioning.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Partitioning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/joins.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Join Optimization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/data-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Data Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/table-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Table Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/caching.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Caching & Persistence</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/pyspark.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>PySpark Specifics</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/aqe.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Adaptive Query Execution</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/cluster-config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cluster Tuning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/anti-patterns.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Anti-Patterns</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Detector Catalog</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-skew.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Skew</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle I/O</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-spill.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory / Disk Spill</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-gc.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>GC Pressure</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-cold-start.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cold Start</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Executor Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-slow-host.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Slow Host</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-failures.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Failures</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-straggler.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Stragglers</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-retry-waste.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Retry Waste</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-tiny-tasks.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Tiny Tasks</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-job-failure-rate.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Job Failure Rate</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-memory-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-duplicate-plan-subtree.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Duplicate Plan Subtree</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-small-files.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Small Files</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-broadcast-sizing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Broadcast Sizing</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Reference</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/metrics.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Metrics Glossary</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Config Quick-Reference</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_tuning-reference_joins" data-v-39a288b8><div><h1 id="joins" tabindex="-1">Join Optimization <a class="header-anchor" href="#joins" aria-label="Permalink to &quot;Join Optimization {#joins}&quot;"></a></h1><h2 id="join-strategy-selection" tabindex="-1">Join strategy selection <a class="header-anchor" href="#join-strategy-selection" aria-label="Permalink to &quot;Join strategy selection&quot;"></a></h2><p>Spark SQL&#39;s Catalyst optimizer picks from five physical join operators: broadcast hash join, broadcast nested loop join, shuffle hash join, shuffle sort-merge join (SMJ), and shuffle-and-replicate nested loop (cartesian) join<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup>. Broadcast hash join avoids a shuffle entirely by sending one small side to every executor; it requires an equi-join condition and supports every join type except full outer<sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup>. Broadcast nested loop join relaxes the equi-join requirement (it supports non-equi conditions and every join type) at the cost of scanning one side repeatedly, so it&#39;s normally a fallback rather than a first choice<sup class="footnote-ref"><a href="#fn1" id="fnref1:2">[1:2]</a></sup>. Unlike hand-tuned RDD joins, where the partitioner is chosen explicitly, Spark SQL&#39;s optimizer can also push down or reorder other operators automatically to make the eventual join cheaper<sup class="footnote-ref"><a href="#fn1" id="fnref1:3">[1:3]</a></sup>.</p><p>At the communication level, Spark&#39;s choice is binary: an all-to-all <a href="./shuffle.html">shuffle</a> join, or a broadcast join that replicates the small side once so every node can then join locally with no further network traffic<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup>. Once a join has been routed to a shuffle-based strategy (broadcast wasn&#39;t used), Spark defaults to preferring sort-merge join over shuffle hash join, a preference controlled by <code>spark.sql.join.preferSortMergeJoin</code><sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup>. Since Spark 3.2, <a href="./aqe.html">Adaptive Query Execution</a> (AQE), on by default, re-optimizes the physical plan mid-execution using runtime statistics gathered after the shuffle actually runs, rather than relying only on pre-execution estimates<sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup>.</p><img class="light-only" src="../assets/join-strategy.C_FvrCEo.svg" alt="Decision tree for choosing a physical join operator: the equi-join test splits off the nested-loop strategies, then the broadcast-threshold and preferSortMergeJoin checks select broadcast hash, sort-merge, or shuffle hash join, with AQE able to promote sort-merge to broadcast at runtime."><img class="dark-only" src="../assets/join-strategy.dark.ChMLnNII.svg" alt="Decision tree for choosing a physical join operator: the equi-join test splits off the nested-loop strategies, then the broadcast-threshold and preferSortMergeJoin checks select broadcast hash, sort-merge, or shuffle hash join, with AQE able to promote sort-merge to broadcast at runtime."><h2 id="reading-it-in-the-plan" tabindex="-1">Reading it in the plan <a class="header-anchor" href="#reading-it-in-the-plan" aria-label="Permalink to &quot;Reading it in the plan&quot;"></a></h2><p>That choice is legible before a job even runs. The deciding signal is the relation&#39;s size relative to <code>spark.sql.autoBroadcastJoinThreshold</code>, which defaults to <code>10485760</code> bytes (10 MB), unchanged since it was introduced in Spark 1.1.0 and still the documented default in Spark 3.5<sup class="footnote-ref"><a href="#fn4" id="fnref4:1">[4:1]</a></sup><sup class="footnote-ref"><a href="#fn5" id="fnref5">[5]</a></sup>. That comparison is driven by table/plan size statistics rather than a fresh scan of the data: Spark consults catalog statistics (collected via <code>ANALYZE TABLE</code>, inspectable through <code>DESCRIBE EXTENDED</code>) and the cost estimates shown in <code>EXPLAIN COST</code><sup class="footnote-ref"><a href="#fn4" id="fnref4:2">[4:2]</a></sup>. When those statistics are missing, <code>spark.sql.statistics.fallBackToHdfs</code> (default <code>false</code>) controls whether Spark falls back to on-disk file size to judge broadcast eligibility, and for partitioned tables without statistics Spark instead uses the <code>spark.sql.defaultSizeInBytes</code> placeholder<sup class="footnote-ref"><a href="#fn5" id="fnref5:1">[5:1]</a></sup>.</p><blockquote><p><strong>PySpark:</strong> don&#39;t guess whether a join will broadcast. Call <code>df.explain(mode=&quot;cost&quot;)</code> to see the same size estimates Catalyst used to pick the strategy.</p></blockquote><p>Because AQE re-optimizes at runtime, the physical join operator visible in a plan can change after execution starts. Two runtime-driven overrides are worth checking for in the Spark UI or an <code>EXPLAIN</code> plan: <code>spark.sql.adaptive.maxShuffledHashJoinLocalMapThreshold</code> (default <code>0</code>, disabled) can make AQE prefer shuffled hash join over sort-merge &quot;regardless of the value of <code>spark.sql.join.preferSortMergeJoin</code>&quot; whenever every post-shuffle partition stays under that threshold and above <code>spark.sql.adaptive.advisoryPartitionSizeInBytes</code><sup class="footnote-ref"><a href="#fn4" id="fnref4:3">[4:3]</a></sup>; separately, AQE can convert an already-planned sort-merge join into a broadcast hash join mid-execution when the runtime statistics of either join side turn out smaller than the adaptive broadcast threshold<sup class="footnote-ref"><a href="#fn4" id="fnref4:4">[4:4]</a></sup>.</p><p>Bucketing status is also visible directly in the physical plan: a shuffle-free bucketed join only appears once both sides are bucketed to the same count, at which point the <code>Exchange</code> nodes disappear entirely<sup class="footnote-ref"><a href="#fn6" id="fnref6">[6]</a></sup>. This is demonstrated with 4 buckets on each side<sup class="footnote-ref"><a href="#fn7" id="fnref7">[7]</a></sup> and with 16 buckets on each side, where the plan shows <code>SelectedBucketsCount: 16 out of 16</code> on both branches of the <code>SortMergeJoin</code><sup class="footnote-ref"><a href="#fn8" id="fnref8">[8]</a></sup>. A separate real-world account of eliminating a bucketed shuffle frames the same signal just as plainly: the giveaway is that &quot;the right branch is missing an Exchange (i.e. shuffle)&quot;<sup class="footnote-ref"><a href="#fn6" id="fnref6:1">[6:1]</a></sup>. Removing the shuffle this way does not necessarily remove the sort phase: in the 16-bucket demo, a <code>Sort</code> operator still appears on each branch of the <code>SortMergeJoin</code> even with the <code>Exchange</code> gone<sup class="footnote-ref"><a href="#fn8" id="fnref8:1">[8:1]</a></sup>.</p><p>Whether Catalyst is reordering joins (rather than just individual operators) is also a config check: <code>spark.sql.cbo.enabled</code> and the separate <code>spark.sql.cbo.joinReorder.enabled</code> flag both default to <code>false</code>, so multi-way join reordering is off unless both are explicitly enabled<sup class="footnote-ref"><a href="#fn5" id="fnref5:2">[5:2]</a></sup>. That&#39;s distinct from Catalyst&#39;s regular rule-based optimizations, which run regardless of those flags. For example, a filter written after a join in the DataFrame API was observed moved before the join (and pushed into the JDBC source) automatically, visible in the physical plan<sup class="footnote-ref"><a href="#fn9" id="fnref9">[9]</a></sup>.</p><h2 id="costs-the-plan-doesn-t-show" tabindex="-1">Costs the plan doesn&#39;t show <a class="header-anchor" href="#costs-the-plan-doesn-t-show" aria-label="Permalink to &quot;Costs the plan doesn&#39;t show&quot;"></a></h2><p>Picking a strategy is one thing; paying for it is another. Broadcasting isn&#39;t free on the driver side. Building a broadcast join replicates the small-side DataFrame to every worker, but that replication is preceded by collecting the DataFrame back to the driver first: an <a href="./bottleneck-broadcast-sizing.html">oversized broadcast</a>, whether chosen automatically or forced via a hint or <code>broadcast()</code> call, &quot;can crash your driver node (because that collect is expensive)&quot;<sup class="footnote-ref"><a href="#fn2" id="fnref2:1">[2:1]</a></sup>. The shuffle-and-replicate nested loop (cartesian-style) strategy carries a related but distinct risk: because every partition is joined against every other partition, it has a high chance of data explosion<sup class="footnote-ref"><a href="#fn1" id="fnref1:4">[1:4]</a></sup>.</p><p>Bucketing&#39;s shuffle-free path also has a real tradeoff once bucket counts don&#39;t match exactly. Since Spark 3.1, <code>spark.sql.bucketing.coalesceBucketsInJoin.enabled</code> lets Spark coalesce the side with more buckets down to the smaller count, but only within a ratio bounded by <code>spark.sql.bucketing.coalesceBucketsInJoin.maxBucketRatio</code> (default <code>4</code>). Enabling it can still remove the shuffle, but it does so by cutting parallelism on the finer-grained side, and Spark&#39;s own docs note it &quot;could possibly cause OOM for shuffled hash join&quot; as a result<sup class="footnote-ref"><a href="#fn5" id="fnref5:3">[5:3]</a></sup>. Outside that ratio, or with coalescing disabled, mismatched bucket counts get no free win at all: the join simply falls back to a normal shuffle.</p><h2 id="forcing-a-strategy" tabindex="-1">Forcing a strategy <a class="header-anchor" href="#forcing-a-strategy" aria-label="Permalink to &quot;Forcing a strategy&quot;"></a></h2><p>When the automatic choice isn&#39;t the right one, Spark still lets you override it directly. Join hints force a specific strategy per relation, and Spark honors them even against its own size-based defaults:</p><ul><li><strong>BROADCAST</strong> (also accepted as <code>BROADCASTJOIN</code>/<code>MAPJOIN</code>) forces a broadcast join with the hinted relation as the build side (broadcast hash if there&#39;s an equi-join key, broadcast nested loop otherwise), and this is honored &quot;even if the size of table &#39;t1&#39; suggested by the statistics is above the configuration <code>spark.sql.autoBroadcastJoinThreshold</code>&quot;<sup class="footnote-ref"><a href="#fn4" id="fnref4:5">[4:5]</a></sup>.</li><li><strong>MERGE</strong> forces a shuffle sort-merge join; it needs an equi-join on sortable keys and high cardinality to pay off, and is less prone to OOM than the hash-based options since it never builds an in-memory hash table<sup class="footnote-ref"><a href="#fn1" id="fnref1:5">[1:5]</a></sup>.</li><li><strong>SHUFFLE_HASH</strong> forces a shuffle hash join, supporting all join types on an equi-join key but, like MERGE, wanting high cardinality; it builds a per-partition hash table after the shuffle<sup class="footnote-ref"><a href="#fn1" id="fnref1:6">[1:6]</a></sup>.</li><li><strong>SHUFFLE_REPLICATE_NL</strong> forces the shuffle-and-replicate nested loop join, supporting inner and cartesian joins with equi or non-equi conditions<sup class="footnote-ref"><a href="#fn1" id="fnref1:7">[1:7]</a></sup>.</li></ul><p>When both sides of a join carry conflicting hints, Spark resolves them by a fixed priority: BROADCAST over MERGE over SHUFFLE_HASH over SHUFFLE_REPLICATE_NL. If both sides carry the <em>same</em> BROADCAST or SHUFFLE_HASH hint, Spark still picks a build side based on join type and relation sizes<sup class="footnote-ref"><a href="#fn4" id="fnref4:6">[4:6]</a></sup>. As with any hint, none of this is guaranteed: Spark won&#39;t honor a hinted strategy that structurally can&#39;t support the actual join type being performed<sup class="footnote-ref"><a href="#fn4" id="fnref4:7">[4:7]</a></sup>. The threshold itself can also be turned off outright: setting <code>spark.sql.autoBroadcastJoinThreshold</code> to <code>-1</code> forces every join to fall back to shuffle sort-merge instead of ever considering broadcast<sup class="footnote-ref"><a href="#fn3" id="fnref3:1">[3:1]</a></sup>.</p><p>For repeated joins on the same key, bucketing both tables to the <em>same</em> bucket count removes the shuffle for free. To also remove the sort phase (not just the shuffle), the bucketed tables additionally need to be written pre-sorted on the join key, using <code>sortBy</code> alongside <code>bucketBy</code>; done that way, the merge phase has nothing left to do, since &quot;the joined output is sorted ... because we saved the tables sorted in ascending order ... there&#39;s no need to sort during the <code>SortMergeJoin</code>,&quot; and the Spark UI shows the query going straight to <code>WholeStageCodegen</code> with no <code>Exchange</code> at all<sup class="footnote-ref"><a href="#fn3" id="fnref3:2">[3:2]</a></sup>.</p><blockquote><p><strong>PySpark:</strong> <code>df.write.bucketBy(n, &quot;key&quot;).saveAsTable(...)</code> drops the shuffle; add <code>.sortBy(&quot;key&quot;)</code> on both tables to drop the sort phase too.</p></blockquote><p>Multi-way join reordering is a deliberate opt-in: enable <code>spark.sql.cbo.enabled</code> for cost-based statistics, then <code>spark.sql.cbo.joinReorder.enabled</code> for the reordering rule itself. <code>spark.sql.cbo.joinReorder.dp.threshold</code> caps the dynamic-programming enumeration at 12 joined nodes by default, and <code>spark.sql.cbo.joinReorder.dp.star.filter</code> applies star-join filter heuristics on top of it<sup class="footnote-ref"><a href="#fn5" id="fnref5:4">[5:4]</a></sup>. A lighter-weight alternative that skips full CBO is <code>spark.sql.cbo.starSchemaDetection</code> (default <code>false</code>), which enables join reordering based on star-schema detection alone<sup class="footnote-ref"><a href="#fn5" id="fnref5:5">[5:5]</a></sup>.</p><p>For <a href="./bottleneck-skew.html">skewed join keys</a>, Spark offers two built-in alternatives to hand-rolled salting: AQE&#39;s skew-join handling (<code>spark.sql.adaptive.skewJoin.enabled</code>), which detects oversized shuffle partitions at runtime and splits them automatically, replicating if needed<sup class="footnote-ref"><a href="#fn10" id="fnref10">[10]</a></sup><sup class="footnote-ref"><a href="#fn5" id="fnref5:6">[5:6]</a></sup>, and Databricks&#39; declarative <code>SKEW</code> hint, which builds a skew-aware plan without any manual salting<sup class="footnote-ref"><a href="#fn11" id="fnref11">[11]</a></sup>. Manual salting remains the fallback where neither is available: add a random salt column to the join key on both sides so a hot key spreads across many partitions (exploding the dimension side into one row per salt value and assigning a random salt on the fact side), then join on the composite <code>(key, salt)</code> pair<sup class="footnote-ref"><a href="#fn12" id="fnref12">[12]</a></sup>.</p><h2 id="sources" tabindex="-1">Sources <a class="header-anchor" href="#sources" aria-label="Permalink to &quot;Sources&quot;"></a></h2><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><em>High Performance Spark, 2nd Edition</em>, Karau, Polak &amp; Warren, ch. 6 <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a> <a href="#fnref1:2" class="footnote-backref">↩︎</a> <a href="#fnref1:3" class="footnote-backref">↩︎</a> <a href="#fnref1:4" class="footnote-backref">↩︎</a> <a href="#fnref1:5" class="footnote-backref">↩︎</a> <a href="#fnref1:6" class="footnote-backref">↩︎</a> <a href="#fnref1:7" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><em>Spark: The Definitive Guide</em>, Chambers &amp; Zaharia, ch. 8 <a href="#fnref2" class="footnote-backref">↩︎</a> <a href="#fnref2:1" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><em>Learning Spark, 2nd Edition</em>, Damji, Wenig, Das, Lee, ch. 7 <a href="#fnref3" class="footnote-backref">↩︎</a> <a href="#fnref3:1" class="footnote-backref">↩︎</a> <a href="#fnref3:2" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/sql-performance-tuning.html" target="_blank" rel="noreferrer">Performance Tuning — Spark SQL, DataFrames and Datasets Guide</a> <a href="#fnref4" class="footnote-backref">↩︎</a> <a href="#fnref4:1" class="footnote-backref">↩︎</a> <a href="#fnref4:2" class="footnote-backref">↩︎</a> <a href="#fnref4:3" class="footnote-backref">↩︎</a> <a href="#fnref4:4" class="footnote-backref">↩︎</a> <a href="#fnref4:5" class="footnote-backref">↩︎</a> <a href="#fnref4:6" class="footnote-backref">↩︎</a> <a href="#fnref4:7" class="footnote-backref">↩︎</a></p></li><li id="fn5" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration — Spark</a> <a href="#fnref5" class="footnote-backref">↩︎</a> <a href="#fnref5:1" class="footnote-backref">↩︎</a> <a href="#fnref5:2" class="footnote-backref">↩︎</a> <a href="#fnref5:3" class="footnote-backref">↩︎</a> <a href="#fnref5:4" class="footnote-backref">↩︎</a> <a href="#fnref5:5" class="footnote-backref">↩︎</a> <a href="#fnref5:6" class="footnote-backref">↩︎</a></p></li><li id="fn6" class="footnote-item"><p><a href="https://www.taboola.com/engineering/bucket-the-shuffle-out-of-here/" target="_blank" rel="noreferrer">Bucket the Shuffle Out of Here</a> <a href="#fnref6" class="footnote-backref">↩︎</a> <a href="#fnref6:1" class="footnote-backref">↩︎</a></p></li><li id="fn7" class="footnote-item"><p><a href="https://books.japila.pl/spark-sql-internals/bucketing/" target="_blank" rel="noreferrer">Bucketing — The Internals of Spark SQL</a> <a href="#fnref7" class="footnote-backref">↩︎</a></p></li><li id="fn8" class="footnote-item"><p><a href="https://luminousmen.com/post/the-5-minute-guide-to-using-bucketing-in-pyspark" target="_blank" rel="noreferrer">The 5-Minute Guide to Using Bucketing in PySpark</a> <a href="#fnref8" class="footnote-backref">↩︎</a> <a href="#fnref8:1" class="footnote-backref">↩︎</a></p></li><li id="fn9" class="footnote-item"><p><a href="https://luminousmen.com/post/spark-tips-dataframe-api" target="_blank" rel="noreferrer">Spark Tips: DataFrame API</a> <a href="#fnref9" class="footnote-backref">↩︎</a></p></li><li id="fn10" class="footnote-item"><p><a href="https://issues.apache.org/jira/browse/SPARK-29544" target="_blank" rel="noreferrer">SPARK-29544 — Optimize Skewed Join at Runtime</a> <a href="#fnref10" class="footnote-backref">↩︎</a></p></li><li id="fn11" class="footnote-item"><p><a href="https://docs.databricks.com/aws/en/archive/legacy/skew-join" target="_blank" rel="noreferrer">Skew Join Hint</a> <a href="#fnref11" class="footnote-backref">↩︎</a></p></li><li id="fn12" class="footnote-item"><p><a href="https://luminousmen.com/post/spark-tips-partition-tuning" target="_blank" rel="noreferrer">Spark Tips: Partition Tuning</a> <a href="#fnref12" class="footnote-backref">↩︎</a></p></li></ol></section></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="../tuning-reference/partitioning.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>Partitioning</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="../tuning-reference/shuffle.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>Shuffle</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
+  </body>
+</html>

package/export-template/docs/tuning-reference/memory-model.html ADDED Viewed

@@ -0,0 +1,25 @@
+<!DOCTYPE html>
+<html lang="en-US" dir="ltr">
+  <head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width,initial-scale=1">
+    <title>Memory Management | SparkForensics</title>
+    <meta name="description" content="Docs for using and contributing to SparkForensics">
+    <meta name="generator" content="VitePress v1.6.4">
+    <link rel="preload stylesheet" href="../assets/style.DSixAiZE.css" as="style">
+    <link rel="preload stylesheet" href="../vp-icons.css" as="style">
+    <link rel="icon" type="image/svg+xml" href="../favicon.svg">
+    <link rel="preconnect" href="https://fonts.googleapis.com">
+    <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin="">
+    <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Recursive:wght,CASL@400..700,0..1&amp;family=JetBrains+Mono:wght@400;500;600&amp;display=swap">
+    <script id="check-dark-mode">(()=>{const e=localStorage.getItem("vitepress-theme-appearance")||"auto",a=window.matchMedia("(prefers-color-scheme: dark)").matches;(!e||e==="auto"?a:e==="dark")&&document.documentElement.classList.add("dark")})();</script>
+    <script id="check-mac-os">document.documentElement.classList.toggle("mac",/Mac|iPhone|iPod|iPad/i.test(navigator.platform));</script>
+  </head>
+  <body>
+    <div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="../index.html" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="../favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="../user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="../contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="../tuning-reference/index.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Getting Started</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/intro.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Introduction</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Optimization Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/spark-architecture.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Execution Model</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/memory-model.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Management</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/partitioning.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Partitioning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/joins.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Join Optimization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/data-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Data Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/table-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Table Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/caching.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Caching & Persistence</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/pyspark.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>PySpark Specifics</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/aqe.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Adaptive Query Execution</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/cluster-config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cluster Tuning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/anti-patterns.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Anti-Patterns</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Detector Catalog</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-skew.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Skew</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle I/O</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-spill.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory / Disk Spill</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-gc.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>GC Pressure</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-cold-start.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cold Start</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Executor Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-slow-host.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Slow Host</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-failures.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Failures</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-straggler.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Stragglers</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-retry-waste.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Retry Waste</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-tiny-tasks.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Tiny Tasks</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-job-failure-rate.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Job Failure Rate</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-memory-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-duplicate-plan-subtree.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Duplicate Plan Subtree</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-small-files.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Small Files</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-broadcast-sizing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Broadcast Sizing</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Reference</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/metrics.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Metrics Glossary</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Config Quick-Reference</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_tuning-reference_memory-model" data-v-39a288b8><div><h1 id="memory-model" tabindex="-1">Memory Management <a class="header-anchor" href="#memory-model" aria-label="Permalink to &quot;Memory Management {#memory-model}&quot;"></a></h1><h2 id="the-unified-memory-model" tabindex="-1">The unified memory model <a class="header-anchor" href="#the-unified-memory-model" aria-label="Permalink to &quot;The unified memory model&quot;"></a></h2><p>Since Spark 1.6, the <code>UnifiedMemoryManager</code> has replaced the earlier static split between execution and storage memory with a single shared region, referred to as M<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup><sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup>. M is carved out of the JVM heap after first setting aside 300 MiB of reserved system memory: <code>M = (JVM heap − 300 MiB) × spark.memory.fraction</code>, with <code>spark.memory.fraction</code> defaulting to 0.6<sup class="footnote-ref"><a href="#fn2" id="fnref2:1">[2:1]</a></sup><sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup>. Within M, <code>spark.memory.storageFraction</code> (default 0.5) marks off a subregion reserved for cached blocks that are immune to eviction; with defaults, this works out to roughly a 30%/30% on-heap split between execution and storage out of total executor memory<sup class="footnote-ref"><a href="#fn3" id="fnref3:1">[3:1]</a></sup>.</p><img class="light-only" src="../assets/memory-regions.XHvO7jHG.svg" alt="The executor memory layout showing reserved memory, the unified region split into execution and storage, and user memory inside the JVM heap, with the off-heap pool and memory overhead outside it."><img class="dark-only" src="../assets/memory-regions.dark.D4TP9_08.svg" alt="The executor memory layout showing reserved memory, the unified region split into execution and storage, and user memory inside the JVM heap, with the off-heap pool and memory overhead outside it."><p>Execution and storage borrow from each other dynamically rather than sitting behind a hard wall: when execution memory is unused, storage can acquire all of it, and vice versa<sup class="footnote-ref"><a href="#fn2" id="fnref2:2">[2:2]</a></sup><sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup>. The two are not equal peers, though. Execution always has priority, taking memory immediately and evicting cached storage blocks if necessary<sup class="footnote-ref"><a href="#fn3" id="fnref3:2">[3:2]</a></sup>.</p><p>The reserved 300 MB is a flat constant (<code>RESERVED_SYSTEM_MEMORY_BYTES</code> in the Spark source), not a formula, and it&#39;s hardcoded: there is no supported production configuration to change it. The only override is the internal <code>spark.testing.reservedMemory</code> property, which exists for Spark&#39;s own test suite rather than production tuning<sup class="footnote-ref"><a href="#fn3" id="fnref3:3">[3:3]</a></sup>. <code>spark.memory.fraction</code> is applied against heap minus that reserved 300 MB, not against total heap. The tuning guide is explicit that it &quot;expresses the size of M as a fraction of the (JVM heap space − 300MiB)&quot;<sup class="footnote-ref"><a href="#fn2" id="fnref2:3">[2:3]</a></sup>.</p><p>Off-heap memory extends the same model outside the JVM heap. When <code>spark.memory.offHeap.enabled=true</code>, Spark allocates raw off-heap buffers via <code>sun.misc.Unsafe</code> (routed through <code>jdk.internal.misc.Unsafe</code> on JDK 17+, which is why some builds need <code>--add-opens</code> flags), and that off-heap region is split into its own execution and storage pools, following the same borrowing rules as on-heap memory: execution has priority, storage gets evicted first<sup class="footnote-ref"><a href="#fn3" id="fnref3:4">[3:4]</a></sup>. Enabling it requires <code>spark.memory.offHeap.size</code> to be set to a positive value; leaving it enabled with the default size of <code>0</code> is a documented-invalid combination, not a supported way to run with off-heap &quot;on but empty&quot;<sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup>. The setting also &quot;has no impact on heap memory usage&quot;. Turning on off-heap memory does not shrink the JVM <code>-Xmx</code> for you, so the on-heap size has to be reduced manually to keep total footprint constant<sup class="footnote-ref"><a href="#fn4" id="fnref4:1">[4:1]</a></sup>. This off-heap execution memory underlies Project Tungsten: its compact binary row encoding, its explicit-memory-managed hash map for aggregations, and its cache-aware sort/join algorithms are all built to run against off-heap, GC-invisible memory<sup class="footnote-ref"><a href="#fn5" id="fnref5">[5]</a></sup><sup class="footnote-ref"><a href="#fn6" id="fnref6">[6]</a></sup>.</p><p>Executor memory overhead is a separate pool layered on top of M. <code>spark.executor.memoryOverhead</code> defaults to <code>executorMemory × spark.executor.memoryOverheadFactor</code>, floored at <code>spark.executor.minMemoryOverhead</code> (384 MiB)<sup class="footnote-ref"><a href="#fn4" id="fnref4:2">[4:2]</a></sup>. <code>spark.executor.memoryOverheadFactor</code> itself defaults to 0.10 for ordinary JVM executors, but Spark bumps that default to 0.40 specifically for Kubernetes non-JVM jobs, since those workloads tend to need more non-JVM heap space<sup class="footnote-ref"><a href="#fn4" id="fnref4:3">[4:3]</a></sup>. The legacy, YARN-only <code>spark.yarn.executor.memoryOverhead</code> property was removed in Spark 3.0 in favor of the cluster-manager-agnostic <code>spark.executor.memoryOverhead</code><sup class="footnote-ref"><a href="#fn3" id="fnref3:5">[3:5]</a></sup>.</p><blockquote><p><strong>PySpark:</strong> <code>spark.python.worker.memory</code> (default <code>512m</code>) is a soft spill threshold for a Python worker&#39;s own aggregation buffering, not a JVM-enforced cap. A PySpark worker runs as a separate OS process outside the JVM heap, so the JVM cannot police it directly<sup class="footnote-ref"><a href="#fn3" id="fnref3:6">[3:6]</a></sup>. The only setting that actually tries to bound PySpark memory per executor is <code>spark.executor.pyspark.memory</code>, and even that depends on Python&#39;s <code>resource</code> module, which isn&#39;t supported on Windows and doesn&#39;t actually limit anything on macOS<sup class="footnote-ref"><a href="#fn4" id="fnref4:4">[4:4]</a></sup>. Left unset, PySpark memory usage is folded into <code>spark.executor.memoryOverhead</code> by default<sup class="footnote-ref"><a href="#fn4" id="fnref4:5">[4:5]</a></sup>.</p></blockquote><h2 id="watching-eviction-and-spill" tabindex="-1">Watching eviction and spill <a class="header-anchor" href="#watching-eviction-and-spill" aria-label="Permalink to &quot;Watching eviction and spill&quot;"></a></h2><p>That borrowing leaves a trace once eviction actually happens. Storage eviction under memory pressure follows an LRU (least recently used) policy<sup class="footnote-ref"><a href="#fn1" id="fnref1:2">[1:2]</a></sup><sup class="footnote-ref"><a href="#fn7" id="fnref7">[7]</a></sup>. The granularity is the block: the <code>BlockManager</code> tracks cached data as per-partition blocks (one block per RDD/DataFrame partition) and LRU operates at that level rather than evicting an entire RDD or cached table in one shot<sup class="footnote-ref"><a href="#fn8" id="fnref8">[8]</a></sup><sup class="footnote-ref"><a href="#fn9" id="fnref9">[9]</a></sup>. In practice this tends to evict the oldest partitions first, those materialized in the earliest job or stage, though lazy evaluation makes it hard to predict exactly which partitions will go ahead of time<sup class="footnote-ref"><a href="#fn9" id="fnref9:1">[9:1]</a></sup>.</p><p>What happens to an evicted block is visible in its downstream effect and depends on its <code>StorageLevel</code>. For <code>MEMORY_ONLY</code>, the block is simply dropped and recomputed from the RDD&#39;s lineage the next time it&#39;s needed. For a disk-backed level like <code>MEMORY_AND_DISK</code>, the evicted block is written to disk and read back from there instead of being recomputed<sup class="footnote-ref"><a href="#fn9" id="fnref9:2">[9:2]</a></sup><sup class="footnote-ref"><a href="#fn7" id="fnref7:1">[7:1]</a></sup>. Blocks can also be evicted well before you ever try to reuse them: <code>.cache()</code> does not guarantee a block stays resident, especially on busy clusters or long pipelines<sup class="footnote-ref"><a href="#fn7" id="fnref7:2">[7:2]</a></sup>.</p><p>On the execution side, each task gets its own <code>TaskMemoryManager</code>, which enforces a soft per-task cap: with <code>n</code> tasks running concurrently, each task is allowed to allocate somewhere between <code>1/(2n)</code> and <code>1/n</code> of total execution memory, and the first task to arrive on an idle executor typically grabs more than its later-arriving neighbors<sup class="footnote-ref"><a href="#fn3" id="fnref3:7">[3:7]</a></sup>.</p><p>When a sort or hash-aggregation operator keeps requesting execution-memory pages and can&#39;t get more, Spark doesn&#39;t fail immediately. It blocks the requesting task, spills that task&#39;s in-memory data structure to disk, or, in the worst case, throws an <code>OutOfMemoryError</code><sup class="footnote-ref"><a href="#fn3" id="fnref3:8">[3:8]</a></sup>. The spill path runs through Spark&#39;s map/shuffle I/O machinery: shuffle partitions created by wide transformations like <code>groupBy()</code> or <code>join()</code> <a href="./bottleneck-spill.html">spill</a> to the executors&#39; local disks at the location set by <code>spark.local.directory</code><sup class="footnote-ref"><a href="#fn6" id="fnref6:1">[6:1]</a></sup>. SQL physical operators apply the same idea with their own row-count guardrails: <code>sortMergeJoinExec</code>&#39;s in-memory buffer and the cartesian-product operator&#39;s buffer both spill once they cross a configured row threshold, which by default is set to the value of <code>spark.shuffle.spill.numElementsForceSpillThreshold</code><sup class="footnote-ref"><a href="#fn10" id="fnref10">[10]</a></sup>.</p><p>Whatever the trigger, the resulting spilled bytes are exposed on the task metrics as <code>memoryBytesSpilled</code>, visible in the Spark UI and event log. This is the concrete signal to watch for execution memory pressure<sup class="footnote-ref"><a href="#fn11" id="fnref11">[11]</a></sup>.</p><p>Memory-overhead misconfiguration shows up differently: it manifests as the container or pod being OOMKilled with a vague container-memory error rather than a Spark-level exception, since the enforcement happens at the YARN NodeManager or Kubernetes kubelet/cgroup layer, not inside the JVM<sup class="footnote-ref"><a href="#fn3" id="fnref3:9">[3:9]</a></sup>. Under the old default overhead factor of 0.10, non-JVM Kubernetes jobs commonly failed with &quot;Memory Overhead Exceeded&quot; errors, which is exactly why Spark bumped the Kubernetes non-JVM default to 0.40<sup class="footnote-ref"><a href="#fn4" id="fnref4:6">[4:6]</a></sup>.</p><h2 id="what-the-borrowing-costs-you" tabindex="-1">What the borrowing costs you <a class="header-anchor" href="#what-the-borrowing-costs-you" aria-label="Permalink to &quot;What the borrowing costs you&quot;"></a></h2><p>None of this is free for whoever&#39;s counting on cached data staying put. Because execution always wins the tug-of-war over storage, <a href="./caching.html">caching</a> is never a guarantee; it&#39;s a best effort. As luminousmen-memory-management puts it: &quot;If Execution needs memory, it takes it. If Storage is using that space (cached RDDs, broadcasts), Spark starts evicting blocks. If Execution is idle, Storage can grow into that space, until Execution comes back.&quot;<sup class="footnote-ref"><a href="#fn3" id="fnref3:10">[3:10]</a></sup> That growth-then-shrink-back behavior is dynamic borrowing, not storage evicting execution; eviction is strictly one-directional. As spark-notes-task-memory-management summarizes the agreement between the two regions: &quot;keep acquiring execution memory and evict storage as you need more execution memory&quot;<sup class="footnote-ref"><a href="#fn1" id="fnref1:3">[1:3]</a></sup>, never the reverse. The same asymmetric rule carries over when off-heap memory is enabled: execution still has priority and storage still gets evicted first<sup class="footnote-ref"><a href="#fn3" id="fnref3:11">[3:11]</a></sup>. Practically, this means a cached DataFrame can silently lose blocks under memory pressure, forcing an expensive lineage recompute (for <code>MEMORY_ONLY</code>) or an extra disk round-trip (for <code>MEMORY_AND_DISK</code>) the next time it&#39;s touched.</p><img class="light-only" src="../assets/memory-borrowing.BqQRJg0u.svg" alt="How storage memory borrows idle execution space while execution reclaims its own space by evicting storage in one direction."><img class="dark-only" src="../assets/memory-borrowing.dark.Yhh20O9C.svg" alt="How storage memory borrows idle execution space while execution reclaims its own space by evicting storage in one direction."><p>The <code>TaskMemoryManager</code>&#39;s soft per-task cap explains why spill behavior is workload-shape-dependent rather than a fixed threshold: with more tasks packed onto an executor, each one&#39;s guaranteed share of execution memory shrinks toward <code>1/(2n)</code>, making spills more likely under high task concurrency even when total execution memory hasn&#39;t changed<sup class="footnote-ref"><a href="#fn3" id="fnref3:12">[3:12]</a></sup>.</p><p>Off-heap memory&#39;s payoff is specifically about <a href="./bottleneck-gc.html">garbage collection</a>: because off-heap buffers sit outside the JVM heap, they are invisible to the garbage collector, so fewer and smaller live objects need to be tracked, scanned, and copied, which reduces both the frequency and duration of GC pauses<sup class="footnote-ref"><a href="#fn3" id="fnref3:13">[3:13]</a></sup>. Project Tungsten&#39;s off-heap hash map for aggregations was benchmarked at over 1 million operations per second in a single thread, with &quot;almost no performance degradation as memory utilization increases,&quot; unlike the JVM default <code>java.util.HashMap</code>, which eventually thrashes on GC<sup class="footnote-ref"><a href="#fn5" id="fnref5:1">[5:1]</a></sup><sup class="footnote-ref"><a href="#fn6" id="fnref6:2">[6:2]</a></sup>. The tradeoff is that off-heap memory removes the GC safety net along with the GC overhead: there&#39;s no garbage collector cleaning up if something goes wrong<sup class="footnote-ref"><a href="#fn3" id="fnref3:14">[3:14]</a></sup>.</p><p>Memory overhead matters because off-heap memory and (if unset) PySpark memory are not automatically folded into the overhead calculation the way heap memory is. Enabling <code>spark.memory.offHeap.size</code> without also raising <code>spark.executor.memoryOverhead</code> (or explicitly setting <code>spark.executor.pyspark.memory</code>) can push the executor&#39;s real footprint past what the cluster manager granted, and the process gets killed with a vague container-memory error instead of a Spark-level exception<sup class="footnote-ref"><a href="#fn3" id="fnref3:15">[3:15]</a></sup>. Worked example: with <code>--executor-memory=8G</code>, the default 10% overhead gives <code>max(0.1 × 8192 MB, 384 MB) = 819 MB</code>, so the total memory requested from the cluster manager is <code>8192 + 819 = 9011 MB</code><sup class="footnote-ref"><a href="#fn3" id="fnref3:16">[3:16]</a></sup>, useful to keep in mind when <a href="./cluster-config.html">sizing containers or pods</a> against a cluster&#39;s available capacity.</p><h2 id="tuning-the-memory-pools" tabindex="-1">Tuning the memory pools <a class="header-anchor" href="#tuning-the-memory-pools" aria-label="Permalink to &quot;Tuning the memory pools&quot;"></a></h2><p>Most of this is adjustable, within limits. Tune <code>spark.memory.fraction</code> and <code>spark.memory.storageFraction</code> only if your workload&#39;s execution/storage balance genuinely needs to shift away from the ~30/30 default. But don&#39;t try to reclaim the 300 MB reserved region; it&#39;s a fixed, non-tunable constant in production<sup class="footnote-ref"><a href="#fn3" id="fnref3:17">[3:17]</a></sup>.</p><p>If losing cached partitions to eviction is costly, prefer a disk-backed <code>StorageLevel</code> such as <code>MEMORY_AND_DISK</code> over <code>MEMORY_ONLY</code>: an evicted block gets written to disk and read back rather than triggering a full lineage recompute<sup class="footnote-ref"><a href="#fn9" id="fnref9:3">[9:3]</a></sup><sup class="footnote-ref"><a href="#fn7" id="fnref7:3">[7:3]</a></sup>. Keep in mind <code>.cache()</code> alone is not a residency guarantee even with this choice<sup class="footnote-ref"><a href="#fn7" id="fnref7:4">[7:4]</a></sup>.</p><p>Watch <code>memoryBytesSpilled</code> in the Spark UI or history server to catch execution-memory pressure early<sup class="footnote-ref"><a href="#fn11" id="fnref11:1">[11:1]</a></sup>. If sort or hash-aggregation spills are heavy, consider giving the executor more memory or reducing the number of concurrently running tasks per executor, since the <code>TaskMemoryManager</code>&#39;s per-task guarantee shrinks as task concurrency <code>n</code> grows<sup class="footnote-ref"><a href="#fn3" id="fnref3:18">[3:18]</a></sup>. For SQL joins and cartesian products, <code>spark.shuffle.spill.numElementsForceSpillThreshold</code> governs when <code>sortMergeJoinExec</code> and the cartesian-product operator spill their buffers<sup class="footnote-ref"><a href="#fn10" id="fnref10:1">[10:1]</a></sup>.</p><p>Set <code>spark.executor.memoryOverhead</code> explicitly rather than relying on the default, especially on Kubernetes for non-JVM workloads (where the default factor is already bumped to 0.40) or whenever off-heap memory or heavy PySpark usage is in play: those aren&#39;t automatically folded into the overhead calculation, so an unadjusted overhead can lead to an OOMKilled container instead of a clean Spark-level error<sup class="footnote-ref"><a href="#fn3" id="fnref3:19">[3:19]</a></sup><sup class="footnote-ref"><a href="#fn4" id="fnref4:7">[4:7]</a></sup>. Use <code>spark.executor.memoryOverhead</code>, not the removed <code>spark.yarn.executor.memoryOverhead</code><sup class="footnote-ref"><a href="#fn3" id="fnref3:20">[3:20]</a></sup>. For capacity planning, remember the total requested from the cluster manager is executor memory plus overhead, e.g., <code>8192 + 819 = 9011 MB</code> for an 8 GB executor at the default 10% factor<sup class="footnote-ref"><a href="#fn3" id="fnref3:21">[3:21]</a></sup>.</p><p>When enabling off-heap memory, always pair <code>spark.memory.offHeap.enabled=true</code> with an explicit positive <code>spark.memory.offHeap.size</code> (never leave it enabled at the default size of <code>0</code><sup class="footnote-ref"><a href="#fn4" id="fnref4:8">[4:8]</a></sup>) and manually reduce the JVM <code>-Xmx</code> to compensate, since enabling off-heap memory does not shrink the heap for you<sup class="footnote-ref"><a href="#fn4" id="fnref4:9">[4:9]</a></sup>.</p><blockquote><p><strong>PySpark:</strong> if you need PySpark&#39;s own memory bounded rather than folded silently into the overhead, set <code>spark.executor.pyspark.memory</code> explicitly, though its enforcement relies on Python&#39;s <code>resource</code> module and won&#39;t work on Windows and won&#39;t actually limit anything on macOS<sup class="footnote-ref"><a href="#fn4" id="fnref4:10">[4:10]</a></sup>.</p></blockquote><h2 id="sources" tabindex="-1">Sources <a class="header-anchor" href="#sources" aria-label="Permalink to &quot;Sources&quot;"></a></h2><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><a href="https://raw.githubusercontent.com/spoddutur/spark-notes/master/task_memory_management_in_spark.md" target="_blank" rel="noreferrer">Task Memory Management in Spark</a> <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a> <a href="#fnref1:2" class="footnote-backref">↩︎</a> <a href="#fnref1:3" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/tuning.html" target="_blank" rel="noreferrer">Tuning Spark</a> <a href="#fnref2" class="footnote-backref">↩︎</a> <a href="#fnref2:1" class="footnote-backref">↩︎</a> <a href="#fnref2:2" class="footnote-backref">↩︎</a> <a href="#fnref2:3" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><a href="https://luminousmen.com/post/dive-into-spark-memory" target="_blank" rel="noreferrer">Dive into Spark Memory</a> <a href="#fnref3" class="footnote-backref">↩︎</a> <a href="#fnref3:1" class="footnote-backref">↩︎</a> <a href="#fnref3:2" class="footnote-backref">↩︎</a> <a href="#fnref3:3" class="footnote-backref">↩︎</a> <a href="#fnref3:4" class="footnote-backref">↩︎</a> <a href="#fnref3:5" class="footnote-backref">↩︎</a> <a href="#fnref3:6" class="footnote-backref">↩︎</a> <a href="#fnref3:7" class="footnote-backref">↩︎</a> <a href="#fnref3:8" class="footnote-backref">↩︎</a> <a href="#fnref3:9" class="footnote-backref">↩︎</a> <a href="#fnref3:10" class="footnote-backref">↩︎</a> <a href="#fnref3:11" class="footnote-backref">↩︎</a> <a href="#fnref3:12" class="footnote-backref">↩︎</a> <a href="#fnref3:13" class="footnote-backref">↩︎</a> <a href="#fnref3:14" class="footnote-backref">↩︎</a> <a href="#fnref3:15" class="footnote-backref">↩︎</a> <a href="#fnref3:16" class="footnote-backref">↩︎</a> <a href="#fnref3:17" class="footnote-backref">↩︎</a> <a href="#fnref3:18" class="footnote-backref">↩︎</a> <a href="#fnref3:19" class="footnote-backref">↩︎</a> <a href="#fnref3:20" class="footnote-backref">↩︎</a> <a href="#fnref3:21" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration — Spark</a> <a href="#fnref4" class="footnote-backref">↩︎</a> <a href="#fnref4:1" class="footnote-backref">↩︎</a> <a href="#fnref4:2" class="footnote-backref">↩︎</a> <a href="#fnref4:3" class="footnote-backref">↩︎</a> <a href="#fnref4:4" class="footnote-backref">↩︎</a> <a href="#fnref4:5" class="footnote-backref">↩︎</a> <a href="#fnref4:6" class="footnote-backref">↩︎</a> <a href="#fnref4:7" class="footnote-backref">↩︎</a> <a href="#fnref4:8" class="footnote-backref">↩︎</a> <a href="#fnref4:9" class="footnote-backref">↩︎</a> <a href="#fnref4:10" class="footnote-backref">↩︎</a></p></li><li id="fn5" class="footnote-item"><p><a href="https://www.databricks.com/blog/2015/04/28/project-tungsten-bringing-spark-closer-to-bare-metal.html" target="_blank" rel="noreferrer">Project Tungsten: Bringing Spark Closer to Bare Metal</a> <a href="#fnref5" class="footnote-backref">↩︎</a> <a href="#fnref5:1" class="footnote-backref">↩︎</a></p></li><li id="fn6" class="footnote-item"><p><em>Learning Spark, 2nd Edition</em>, Damji, Wenig, Das, Lee, ch. 6–7 <a href="#fnref6" class="footnote-backref">↩︎</a> <a href="#fnref6:1" class="footnote-backref">↩︎</a> <a href="#fnref6:2" class="footnote-backref">↩︎</a></p></li><li id="fn7" class="footnote-item"><p><a href="https://luminousmen.com/post/explaining-the-mechanics-of-spark-caching" target="_blank" rel="noreferrer">Explaining the Mechanics of Spark Caching</a> <a href="#fnref7" class="footnote-backref">↩︎</a> <a href="#fnref7:1" class="footnote-backref">↩︎</a> <a href="#fnref7:2" class="footnote-backref">↩︎</a> <a href="#fnref7:3" class="footnote-backref">↩︎</a> <a href="#fnref7:4" class="footnote-backref">↩︎</a></p></li><li id="fn8" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/rdd-programming-guide.html" target="_blank" rel="noreferrer">RDD Programming Guide</a> <a href="#fnref8" class="footnote-backref">↩︎</a></p></li><li id="fn9" class="footnote-item"><p><em>High Performance Spark, 2nd Edition</em>, Karau, Polak &amp; Warren, ch. 7 <a href="#fnref9" class="footnote-backref">↩︎</a> <a href="#fnref9:1" class="footnote-backref">↩︎</a> <a href="#fnref9:2" class="footnote-backref">↩︎</a> <a href="#fnref9:3" class="footnote-backref">↩︎</a></p></li><li id="fn10" class="footnote-item"><p><a href="https://raw.githubusercontent.com/apache/spark/v3.5.0/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala" target="_blank" rel="noreferrer">SQLConf.scala</a> <a href="#fnref10" class="footnote-backref">↩︎</a> <a href="#fnref10:1" class="footnote-backref">↩︎</a></p></li><li id="fn11" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/monitoring.html#spark-history-server" target="_blank" rel="noreferrer">Monitoring and Instrumentation</a> <a href="#fnref11" class="footnote-backref">↩︎</a> <a href="#fnref11:1" class="footnote-backref">↩︎</a></p></li></ol></section></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="../tuning-reference/spark-architecture.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>Spark Execution Model</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="../tuning-reference/partitioning.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>Partitioning</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
+  </body>
+</html>

package/export-template/docs/tuning-reference/metrics.html ADDED Viewed

@@ -0,0 +1,25 @@
+<!DOCTYPE html>
+<html lang="en-US" dir="ltr">
+  <head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width,initial-scale=1">
+    <title>Metrics Glossary | SparkForensics</title>
+    <meta name="description" content="Docs for using and contributing to SparkForensics">
+    <meta name="generator" content="VitePress v1.6.4">
+    <link rel="preload stylesheet" href="../assets/style.DSixAiZE.css" as="style">
+    <link rel="preload stylesheet" href="../vp-icons.css" as="style">
+    <link rel="icon" type="image/svg+xml" href="../favicon.svg">
+    <link rel="preconnect" href="https://fonts.googleapis.com">
+    <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin="">
+    <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Recursive:wght,CASL@400..700,0..1&amp;family=JetBrains+Mono:wght@400;500;600&amp;display=swap">
+    <script id="check-dark-mode">(()=>{const e=localStorage.getItem("vitepress-theme-appearance")||"auto",a=window.matchMedia("(prefers-color-scheme: dark)").matches;(!e||e==="auto"?a:e==="dark")&&document.documentElement.classList.add("dark")})();</script>
+    <script id="check-mac-os">document.documentElement.classList.toggle("mac",/Mac|iPhone|iPod|iPad/i.test(navigator.platform));</script>
+  </head>
+  <body>
+    <div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="../index.html" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="../favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="../user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="../contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="../tuning-reference/index.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Getting Started</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/intro.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Introduction</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Optimization Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/spark-architecture.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Execution Model</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/memory-model.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Management</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/partitioning.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Partitioning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/joins.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Join Optimization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/data-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Data Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/table-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Table Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/caching.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Caching & Persistence</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/pyspark.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>PySpark Specifics</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/aqe.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Adaptive Query Execution</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/cluster-config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cluster Tuning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/anti-patterns.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Anti-Patterns</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Detector Catalog</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-skew.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Skew</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle I/O</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-spill.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory / Disk Spill</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-gc.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>GC Pressure</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-cold-start.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cold Start</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Executor Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-slow-host.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Slow Host</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-failures.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Failures</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-straggler.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Stragglers</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-retry-waste.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Retry Waste</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-tiny-tasks.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Tiny Tasks</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-job-failure-rate.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Job Failure Rate</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-memory-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-duplicate-plan-subtree.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Duplicate Plan Subtree</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-small-files.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Small Files</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/bottleneck-broadcast-sizing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Broadcast Sizing</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Reference</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/metrics.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Metrics Glossary</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="../tuning-reference/config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Config Quick-Reference</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_tuning-reference_metrics" data-v-39a288b8><div><h1 id="metrics" tabindex="-1">Metrics Glossary <a class="header-anchor" href="#metrics" aria-label="Permalink to &quot;Metrics Glossary {#metrics}&quot;"></a></h1><p>Reference for every metric surfaced elsewhere in this guide: what it measures, where Spark records it in the event log, and what a problematic value looks like.</p><h2 id="metric-task-duration" tabindex="-1">Task duration (P50/P95/max) <a class="header-anchor" href="#metric-task-duration" aria-label="Permalink to &quot;Task duration (P50/P95/max) {#metric-task-duration}&quot;"></a></h2><p>Per-task wall-clock duration, aggregated across a stage&#39;s tasks into percentiles (P50, P95, max) to surface skew and stragglers. It is computed from <code>SparkListenerTaskEnd</code> events rather than read off a single field: each task end carries its own duration, and the percentiles are derived by aggregating those values across the stage.</p><p>A problematic value looks like a small fraction of tasks running far past the rest: a stage is classified as a straggler when more than 5% of tasks run at least 4x the median duration (with at least 10 tasks in the stage), or when speculative tasks were fired.</p><h2 id="metric-shuffle-read-bytes" tabindex="-1">Shuffle read bytes <a class="header-anchor" href="#metric-shuffle-read-bytes" aria-label="Permalink to &quot;Shuffle read bytes {#metric-shuffle-read-bytes}&quot;"></a></h2><p>The volume of shuffle data a task reads from remote executors, recorded in <code>taskMetrics.shuffleReadMetrics.remoteBytesRead</code>.</p><h2 id="metric-shuffle-write-bytes" tabindex="-1">Shuffle write bytes <a class="header-anchor" href="#metric-shuffle-write-bytes" aria-label="Permalink to &quot;Shuffle write bytes {#metric-shuffle-write-bytes}&quot;"></a></h2><p>The size of the shuffle output a task writes, recorded in <code>taskMetrics.shuffleWriteMetrics.bytesWritten</code>. Spark&#39;s own metric description calls it simply the &quot;Number of bytes written in shuffle operations,&quot; without stating compression state on its own<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup>.</p><p>The sort-based shuffle writer fills in the mechanics: incoming records are serialized as soon as they reach the shuffle writer and buffered in serialized form while sorting<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup>. When the spill compression codec supports concatenating compressed data, the final merge step concatenates the already-compressed spill partitions directly into the output file, using <code>transferTo</code> rather than decompressing and recompressing<sup class="footnote-ref"><a href="#fn2" id="fnref2:1">[2:1]</a></sup>. So <code>bytesWritten</code> counts compressed, post-serialization bytes, and in this common fast-merge path the final written output is built directly from data that had already been spilled to disk during sorting, rather than written fresh at merge time.</p><h2 id="metric-memory-bytes-spilled" tabindex="-1">Memory bytes spilled <a class="header-anchor" href="#metric-memory-bytes-spilled" aria-label="Permalink to &quot;Memory bytes spilled {#metric-memory-bytes-spilled}&quot;"></a></h2><p>Bytes a task spilled from in-memory structures, recorded in <code>taskMetrics.memoryBytesSpilled</code>. Any value above zero is a spill warning, with the skew-vs-volume classification (based on what fraction of a stage&#39;s tasks show zero spill) as the actionable signal.</p><h2 id="metric-disk-bytes-spilled" tabindex="-1">Disk bytes spilled <a class="header-anchor" href="#metric-disk-bytes-spilled" aria-label="Permalink to &quot;Disk bytes spilled {#metric-disk-bytes-spilled}&quot;"></a></h2><p>The on-disk counterpart to memory spill: bytes a task spilled to disk, recorded in <code>taskMetrics.diskBytesSpilled</code>. It feeds the same spill classification as memory bytes spilled.</p><h2 id="metric-jvm-gc-time" tabindex="-1">JVM GC time <a class="header-anchor" href="#metric-jvm-gc-time" aria-label="Permalink to &quot;JVM GC time {#metric-jvm-gc-time}&quot;"></a></h2><p>Elapsed time the JVM spent in garbage collection while a task executed, recorded in <code>taskMetrics.jvmGCTime</code> and expressed in milliseconds<sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup>. The value is cumulative across the task&#39;s full execution window: the sum of every GC pause that occurred during that task&#39;s run, not just the most recent one<sup class="footnote-ref"><a href="#fn1" id="fnref1:2">[1:2]</a></sup>. This matches how it&#39;s serialized: as a single scalar long, consistent with an accumulator rather than a per-GC-event log entry<sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup>.</p><h2 id="metric-gcpct" tabindex="-1">gcPct <a class="header-anchor" href="#metric-gcpct" aria-label="Permalink to &quot;gcPct {#metric-gcpct}&quot;"></a></h2><p>A synthetic ratio, <code>jvmGCTime / executorRunTime</code>, not a raw Spark field. It drives GC-bottleneck classification: above 10% is a warning, above 20% is critical.</p><h2 id="metric-executor-run-time" tabindex="-1">Executor run time <a class="header-anchor" href="#metric-executor-run-time" aria-label="Permalink to &quot;Executor run time {#metric-executor-run-time}&quot;"></a></h2><p>Elapsed time the executor spent running a task, recorded in <code>taskMetrics.executorRunTime</code> and expressed in milliseconds<sup class="footnote-ref"><a href="#fn1" id="fnref1:3">[1:3]</a></sup>. <code>TaskMetrics</code> (and therefore <code>executorRunTime</code>) is serialized as an optional part of the <code>SparkListenerTaskEnd</code> payload, not guaranteed on every task end<sup class="footnote-ref"><a href="#fn3" id="fnref3:1">[3:1]</a></sup>. In practice it is available for failed tasks that got far enough to actually run, such as <code>ExceptionFailure</code> or <code>TaskKilled</code>. The event-log deserialization code even falls back to reading accumulator updates out of the embedded <code>TaskMetrics</code> for old, Spark-1.x-era logs, which only makes sense if the metrics object is normally populated for that failure type<sup class="footnote-ref"><a href="#fn3" id="fnref3:2">[3:2]</a></sup>. It can be absent for reasons like <code>Resubmitted</code>, where the task attempt never completed on that executor<sup class="footnote-ref"><a href="#fn3" id="fnref3:3">[3:3]</a></sup>.</p><h2 id="metric-fetch-wait-time-ratio" tabindex="-1">Fetch wait time ratio <a class="header-anchor" href="#metric-fetch-wait-time-ratio" aria-label="Permalink to &quot;Fetch wait time ratio {#metric-fetch-wait-time-ratio}&quot;"></a></h2><p>A synthetic ratio, <code>fetchWaitTime / taskDuration</code>, not a raw Spark field: the time a task spent blocked waiting on remote shuffle blocks, relative to its total duration.</p><h2 id="metric-input-bytes" tabindex="-1">Input bytes <a class="header-anchor" href="#metric-input-bytes" aria-label="Permalink to &quot;Input bytes {#metric-input-bytes}&quot;"></a></h2><p>Bytes a task read as input, recorded in <code>taskMetrics.inputMetrics.bytesRead</code>.</p><h2 id="metric-output-bytes" tabindex="-1">Output bytes <a class="header-anchor" href="#metric-output-bytes" aria-label="Permalink to &quot;Output bytes {#metric-output-bytes}&quot;"></a></h2><p>Bytes a task wrote as output, recorded in <code>taskMetrics.outputMetrics.bytesWritten</code>.</p><h2 id="metric-io-ratio" tabindex="-1">I/O ratio <a class="header-anchor" href="#metric-io-ratio" aria-label="Permalink to &quot;I/O ratio {#metric-io-ratio}&quot;"></a></h2><p>A synthetic ratio, <code>outputBytes / inputBytes</code>, not a raw Spark field.</p><h2 id="metric-peak-execution-memory" tabindex="-1">Peak execution memory <a class="header-anchor" href="#metric-peak-execution-memory" aria-label="Permalink to &quot;Peak execution memory {#metric-peak-execution-memory}&quot;"></a></h2><p>Peak memory recorded in <code>taskMetrics.peakExecutionMemory</code>. This is a task-level accumulator, distinct from the separate executor-level <code>peakMemoryMetrics.OnHeapExecutionMemory</code> and <code>.OffHeapExecutionMemory</code> gauges, which report the on-heap and off-heap execution pools as two separate numbers at the executor level rather than as a single per-task figure<sup class="footnote-ref"><a href="#fn1" id="fnref1:4">[1:4]</a></sup>.</p><h2 id="metric-failed-tasks" tabindex="-1">Failed tasks / failure rate <a class="header-anchor" href="#metric-failed-tasks" aria-label="Permalink to &quot;Failed tasks / failure rate {#metric-failed-tasks}&quot;"></a></h2><p>Tasks whose <code>SparkListenerTaskEnd</code> reason is not <code>Success</code>. The <code>Reason</code> field holds the formatted class name of whichever <code>TaskEndReason</code> was assigned to that task end<sup class="footnote-ref"><a href="#fn3" id="fnref3:4">[3:4]</a></sup>, and the canonical set of reasons includes <code>FetchFailed</code>, <code>ExceptionFailure</code>, <code>TaskResultLost</code>, <code>TaskKilled</code>, <code>TaskCommitDenied</code>, <code>ExecutorLostFailure</code>, and <code>UnknownReason</code><sup class="footnote-ref"><a href="#fn3" id="fnref3:5">[3:5]</a></sup>. Because <code>TaskMetrics</code> is only an optional part of the task-end payload, it can be absent for some of these reasons: for example <code>Resubmitted</code>, where the task attempt never actually completed on that executor<sup class="footnote-ref"><a href="#fn3" id="fnref3:6">[3:6]</a></sup>.</p><h2 id="metric-speculative-tasks" tabindex="-1">Speculative tasks / straggler count <a class="header-anchor" href="#metric-speculative-tasks" aria-label="Permalink to &quot;Speculative tasks / straggler count {#metric-speculative-tasks}&quot;"></a></h2><p>Tasks launched as speculative retries of a slow-running task, recorded via <code>SparkListenerTaskStart</code> where <code>speculative = true</code>. Any speculative task firing (or more than 5% of a stage&#39;s tasks running at least 4x the median duration, with at least 10 tasks in the stage) is a straggler signal.</p><h2 id="metric-executor-count" tabindex="-1">Executor count (added/removed/concurrent) <a class="header-anchor" href="#metric-executor-count" aria-label="Permalink to &quot;Executor count (added/removed/concurrent) {#metric-executor-count}&quot;"></a></h2><p>The number of executors added, removed, or concurrently running, recorded via <code>SparkListenerExecutorAdded</code> / <code>SparkListenerExecutorRemoved</code> events.</p><h2 id="metric-stage-duration" tabindex="-1">Stage wall-clock duration <a class="header-anchor" href="#metric-stage-duration" aria-label="Permalink to &quot;Stage wall-clock duration {#metric-stage-duration}&quot;"></a></h2><p>A stage&#39;s total elapsed time, recorded from <code>SparkListenerStageCompleted</code>: <code>completionTime − submissionTime</code>.</p><h2 id="metric-first-stage-submitted-at" tabindex="-1">firstStageSubmittedAt <a class="header-anchor" href="#metric-first-stage-submitted-at" aria-label="Permalink to &quot;firstStageSubmittedAt {#metric-first-stage-submitted-at}&quot;"></a></h2><p>A synthetic field: the timestamp of the first <code>SparkListenerStageSubmitted</code> event in the event log. The gap between this timestamp and the application&#39;s start time drives cold-start classification: more than 30 seconds is a warning.</p><h2 id="sources" tabindex="-1">Sources <a class="header-anchor" href="#sources" aria-label="Permalink to &quot;Sources&quot;"></a></h2><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/monitoring.html#spark-history-server" target="_blank" rel="noreferrer">Monitoring and Instrumentation</a> <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a> <a href="#fnref1:2" class="footnote-backref">↩︎</a> <a href="#fnref1:3" class="footnote-backref">↩︎</a> <a href="#fnref1:4" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><a href="https://raw.githubusercontent.com/apache/spark/v3.5.0/core/src/main/scala/org/apache/spark/shuffle/sort/SortShuffleManager.scala" target="_blank" rel="noreferrer">SortShuffleManager.scala</a> <a href="#fnref2" class="footnote-backref">↩︎</a> <a href="#fnref2:1" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><a href="https://raw.githubusercontent.com/apache/spark/v3.5.0/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala" target="_blank" rel="noreferrer">JsonProtocol.scala</a> <a href="#fnref3" class="footnote-backref">↩︎</a> <a href="#fnref3:1" class="footnote-backref">↩︎</a> <a href="#fnref3:2" class="footnote-backref">↩︎</a> <a href="#fnref3:3" class="footnote-backref">↩︎</a> <a href="#fnref3:4" class="footnote-backref">↩︎</a> <a href="#fnref3:5" class="footnote-backref">↩︎</a> <a href="#fnref3:6" class="footnote-backref">↩︎</a></p></li></ol></section></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="../tuning-reference/bottleneck-broadcast-sizing.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>Broadcast Sizing</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="../tuning-reference/config.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>Spark Config Quick-Reference</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
+  </body>
+</html>