sparkforensics-server 0.2.2 → 0.2.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (213) hide show
  1. package/package.json +1 -1
  2. package/public/assets/{AutoscalingChurn-B96FKx97.js → AutoscalingChurn-BOiUUfuY.js} +1 -1
  3. package/public/assets/CacheUtilization-DwX7lisd.js +1 -0
  4. package/public/assets/CachingOpportunity-DUE3GKpx.js +1 -0
  5. package/public/assets/{ColdStart-w8QYTNKo.js → ColdStart-ZHxPrJmI.js} +1 -1
  6. package/public/assets/ConfigAudit-BkVnq2XE.js +1 -0
  7. package/public/assets/{CoreUsageArea-o3kqQwyi.js → CoreUsageArea-DobrSXk2.js} +1 -1
  8. package/public/assets/Dashboard-y3_9iQJo.js +1 -0
  9. package/public/assets/{DuplicatePlanSubtree-Cm_UU3lY.js → DuplicatePlanSubtree-BSQZFeNq.js} +1 -1
  10. package/public/assets/{DurationHistogram-ltF-7M0A.js → DurationHistogram-DlQv4ZCB.js} +1 -1
  11. package/public/assets/{EmptyState-BCA3xioa.js → EmptyState-DqnSxTjJ.js} +1 -1
  12. package/public/assets/{ExecutorUtilization-Cn69i0uq.js → ExecutorUtilization-CuQUxaCd.js} +1 -1
  13. package/public/assets/GcPressure-CLT5hkMJ.js +1 -0
  14. package/public/assets/{ImpactEstimate-BGt-4WX5.js → ImpactEstimate-Df_laG-n.js} +1 -1
  15. package/public/assets/{IncompleteRun-s6uQ8OLf.js → IncompleteRun-CpIkS4XZ.js} +1 -1
  16. package/public/assets/{JobFailures-DJ2dDvPA.js → JobFailures-DKbvYc4J.js} +1 -1
  17. package/public/assets/{MemoryUtilization-C9G7P94X.js → MemoryUtilization-C5kHmXIb.js} +1 -1
  18. package/public/assets/{OverBroadcast-DYauoYWQ.js → OverBroadcast-C5pF66aq.js} +1 -1
  19. package/public/assets/PartitionSizing-BvEJQ5M1.js +1 -0
  20. package/public/assets/{PlanExplorer-_2oB5ouy.js → PlanExplorer-DjGTZifP.js} +1 -1
  21. package/public/assets/PlanGraphRoute-DoIGA7_H.js +1 -0
  22. package/public/assets/{RetryWaste-D_m8ASwP.js → RetryWaste-C0znHd_w.js} +1 -1
  23. package/public/assets/{RowPagination-BLsm8I6i.js → RowPagination-2icPidMP.js} +1 -1
  24. package/public/assets/ShuffleIO-L_XSnJ_V.js +1 -0
  25. package/public/assets/{Skew-BGLJuZZR.js → Skew-w4uV5H_g.js} +1 -1
  26. package/public/assets/{SlowHost-ChdWVvGO.js → SlowHost-D5tWJ1UR.js} +1 -1
  27. package/public/assets/{SmallFiles-gTilBtjW.js → SmallFiles-CTtNiSOh.js} +1 -1
  28. package/public/assets/{SpeculationWaste-CVzgJnSj.js → SpeculationWaste-xCKwjz2K.js} +1 -1
  29. package/public/assets/{Spill-NGJGQaTB.js → Spill-BT8my24K.js} +1 -1
  30. package/public/assets/{StageFailed-sGhki53z.js → StageFailed-De3vVDbc.js} +1 -1
  31. package/public/assets/{StageFindingGroup-AyI7BMd3.js → StageFindingGroup-R4OW-g4U.js} +1 -1
  32. package/public/assets/{StageHeader-BTzGkK8k.js → StageHeader-CGeWqC-0.js} +1 -1
  33. package/public/assets/{StagePill-BlMOW_pU.js → StagePill-BwwKLcFl.js} +1 -1
  34. package/public/assets/{StageShape-DfOW93Z1.js → StageShape-D4LmPVyV.js} +1 -1
  35. package/public/assets/{StageSlowness-BMZ-K_5e.js → StageSlowness-8-z6qn4Q.js} +1 -1
  36. package/public/assets/Straggler-BIgxsowI.js +1 -0
  37. package/public/assets/{TaskFailures-DGDwUK2A.js → TaskFailures-B9lLt6Rw.js} +1 -1
  38. package/public/assets/{TinyTask-CfT8zZDr.js → TinyTask-DYXks5sT.js} +1 -1
  39. package/public/assets/{UnderBroadcast-CiOiJ0k1.js → UnderBroadcast-BWqWMpeG.js} +1 -1
  40. package/public/assets/{finding-anchor-DEV8JvgH.js → finding-anchor-CDbHtqKT.js} +1 -1
  41. package/public/assets/{inbox-_QCYkBsP.js → inbox-M_EuWNiK.js} +1 -1
  42. package/public/assets/index-pnImGbgQ.js +9 -0
  43. package/public/assets/parser-worker-C3BAO6z9.js +117 -0
  44. package/public/assets/{select-kuy1MiAo.js → select-C-MZGNpb.js} +1 -1
  45. package/public/assets/{toggle-group-CUd-KWkp.js → toggle-group-C3nz7UQ_.js} +1 -1
  46. package/public/assets/useRecentFiles-0XuhS91q.js +11 -0
  47. package/public/assets/{useSortMode-BaeAA5cw.js → useSortMode-DHlKZeWv.js} +1 -1
  48. package/public/assets/zstd-worker-C1Jq9lfb.js +1 -0
  49. package/public/docs/404.html +2 -2
  50. package/public/docs/assets/{app.APoq_UBs.js → app.DsFQcynM.js} +1 -1
  51. package/public/docs/assets/chunks/@localSearchIndexroot.BEfYag5Q.js +1 -0
  52. package/public/docs/assets/chunks/{VPLocalSearchBox.B3TPzlHq.js → VPLocalSearchBox.oa8NbaiV.js} +1 -1
  53. package/public/docs/assets/chunks/{theme.PavXO_vz.js → theme.Ca4dR9dL.js} +2 -2
  54. package/public/docs/assets/contributor-guide_architecture_board-widgets.md.9ToZXcTk.js +1 -0
  55. package/public/docs/assets/{contributor-guide_architecture_board-widgets.md.CWpj01WU.lean.js → contributor-guide_architecture_board-widgets.md.9ToZXcTk.lean.js} +1 -1
  56. package/public/docs/assets/contributor-guide_architecture_detector-contract.md.aQQfP7pj.js +1 -0
  57. package/public/docs/assets/{contributor-guide_architecture_detector-contract.md.CgzUsQ6W.lean.js → contributor-guide_architecture_detector-contract.md.aQQfP7pj.lean.js} +1 -1
  58. package/public/docs/assets/{contributor-guide_architecture_drill-down.md.BtPdlM7r.js → contributor-guide_architecture_drill-down.md.Bawx6NgP.js} +1 -1
  59. package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.sf9DU2bA.js +1 -0
  60. package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.sf9DU2bA.lean.js +1 -0
  61. package/public/docs/assets/contributor-guide_architecture_overview.md.BXQdgG5u.js +1 -0
  62. package/public/docs/assets/{contributor-guide_architecture_overview.md.CehiRmGn.lean.js → contributor-guide_architecture_overview.md.BXQdgG5u.lean.js} +1 -1
  63. package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.DYwFeyjV.js +1 -0
  64. package/public/docs/assets/{contributor-guide_architecture_worker-protocol.md.IbnfNrV3.js → contributor-guide_architecture_worker-protocol.md.P7d1vtEJ.js} +2 -2
  65. package/public/docs/assets/{contributor-guide_architecture_worker-protocol.md.IbnfNrV3.lean.js → contributor-guide_architecture_worker-protocol.md.P7d1vtEJ.lean.js} +1 -1
  66. package/public/docs/assets/{contributor-guide_development-setup.md.DvAN_9mK.js → contributor-guide_development-setup.md.DvMV1xhQ.js} +3 -3
  67. package/public/docs/assets/contributor-guide_development-setup.md.DvMV1xhQ.lean.js +1 -0
  68. package/public/docs/assets/contributor-guide_testing.md.BU6vyA-9.js +1 -0
  69. package/public/docs/assets/contributor-guide_testing.md.BU6vyA-9.lean.js +1 -0
  70. package/public/docs/assets/{tuning-reference_bottleneck-failures.md.4z5BXGJ2.js → tuning-reference_bottleneck-failures.md.CAsFf3RR.js} +1 -1
  71. package/public/docs/assets/{tuning-reference_bottleneck-gc.md.DSxzZRK7.js → tuning-reference_bottleneck-gc.md.B4T3kJYV.js} +2 -2
  72. package/public/docs/assets/{tuning-reference_bottleneck-gc.md.DSxzZRK7.lean.js → tuning-reference_bottleneck-gc.md.B4T3kJYV.lean.js} +1 -1
  73. package/public/docs/assets/tuning-reference_bottleneck-shuffle.md.DDMQVOW6.js +12 -0
  74. package/public/docs/assets/{tuning-reference_bottleneck-shuffle.md.CM-nTmIH.lean.js → tuning-reference_bottleneck-shuffle.md.DDMQVOW6.lean.js} +1 -1
  75. package/public/docs/assets/{tuning-reference_bottleneck-skew.md.BdUwiDhn.js → tuning-reference_bottleneck-skew.md.B0MzhWcX.js} +4 -4
  76. package/public/docs/assets/{tuning-reference_bottleneck-skew.md.BdUwiDhn.lean.js → tuning-reference_bottleneck-skew.md.B0MzhWcX.lean.js} +1 -1
  77. package/public/docs/assets/{tuning-reference_bottleneck-small-files.md.B8kloyx8.js → tuning-reference_bottleneck-small-files.md.DHGMTmPs.js} +2 -2
  78. package/public/docs/assets/{tuning-reference_bottleneck-small-files.md.B8kloyx8.lean.js → tuning-reference_bottleneck-small-files.md.DHGMTmPs.lean.js} +1 -1
  79. package/public/docs/assets/{tuning-reference_bottleneck-straggler.md.DY36fHN5.js → tuning-reference_bottleneck-straggler.md.DIbXdAcQ.js} +2 -2
  80. package/public/docs/assets/{tuning-reference_bottleneck-straggler.md.DY36fHN5.lean.js → tuning-reference_bottleneck-straggler.md.DIbXdAcQ.lean.js} +1 -1
  81. package/public/docs/assets/{tuning-reference_bottleneck-tiny-tasks.md.QTV7O8kU.js → tuning-reference_bottleneck-tiny-tasks.md.CB05v3zh.js} +1 -1
  82. package/public/docs/assets/tuning-reference_bottleneck-utilization.md.DqC_iGkw.js +5 -0
  83. package/public/docs/assets/tuning-reference_bottleneck-utilization.md.DqC_iGkw.lean.js +1 -0
  84. package/public/docs/assets/{tuning-reference_cluster-config.md.ZVmDGsQ3.js → tuning-reference_cluster-config.md.CD6vzYtD.js} +1 -1
  85. package/public/docs/assets/{tuning-reference_cluster-config.md.ZVmDGsQ3.lean.js → tuning-reference_cluster-config.md.CD6vzYtD.lean.js} +1 -1
  86. package/public/docs/assets/{tuning-reference_memory-model.md.DhT-n4y3.js → tuning-reference_memory-model.md.CWhvgevP.js} +1 -1
  87. package/public/docs/assets/{tuning-reference_memory-model.md.DhT-n4y3.lean.js → tuning-reference_memory-model.md.CWhvgevP.lean.js} +1 -1
  88. package/public/docs/assets/{user-guide_mcp-tools.md.Vi3RoflJ.js → user-guide_mcp-tools.md.DGvolGzt.js} +1 -1
  89. package/public/docs/assets/user-guide_understanding-findings.md.B29f0ucS.js +1 -0
  90. package/public/docs/assets/{user-guide_understanding-findings.md.DL1UDhvR.lean.js → user-guide_understanding-findings.md.B29f0ucS.lean.js} +1 -1
  91. package/public/docs/contributor-guide/architecture/board-widgets.html +5 -5
  92. package/public/docs/contributor-guide/architecture/detector-contract.html +5 -5
  93. package/public/docs/contributor-guide/architecture/drill-down.html +5 -5
  94. package/public/docs/contributor-guide/architecture/impact-estimation.html +5 -5
  95. package/public/docs/contributor-guide/architecture/index.html +3 -3
  96. package/public/docs/contributor-guide/architecture/overview.html +5 -5
  97. package/public/docs/contributor-guide/architecture/state-and-history.html +3 -3
  98. package/public/docs/contributor-guide/architecture/widget-rendering.html +5 -5
  99. package/public/docs/contributor-guide/architecture/worker-protocol.html +6 -6
  100. package/public/docs/contributor-guide/contributing.html +3 -3
  101. package/public/docs/contributor-guide/development-setup.html +5 -5
  102. package/public/docs/contributor-guide/testing.html +5 -5
  103. package/public/docs/hashmap.json +1 -1
  104. package/public/docs/index.html +3 -3
  105. package/public/docs/tuning-reference/anti-patterns.html +3 -3
  106. package/public/docs/tuning-reference/aqe.html +3 -3
  107. package/public/docs/tuning-reference/bottleneck-broadcast-sizing.html +3 -3
  108. package/public/docs/tuning-reference/bottleneck-cold-start.html +3 -3
  109. package/public/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html +3 -3
  110. package/public/docs/tuning-reference/bottleneck-failures.html +5 -5
  111. package/public/docs/tuning-reference/bottleneck-gc.html +5 -5
  112. package/public/docs/tuning-reference/bottleneck-job-failure-rate.html +3 -3
  113. package/public/docs/tuning-reference/bottleneck-memory-utilization.html +3 -3
  114. package/public/docs/tuning-reference/bottleneck-retry-waste.html +3 -3
  115. package/public/docs/tuning-reference/bottleneck-shuffle.html +6 -6
  116. package/public/docs/tuning-reference/bottleneck-skew.html +7 -7
  117. package/public/docs/tuning-reference/bottleneck-slow-host.html +3 -3
  118. package/public/docs/tuning-reference/bottleneck-small-files.html +6 -6
  119. package/public/docs/tuning-reference/bottleneck-spill.html +3 -3
  120. package/public/docs/tuning-reference/bottleneck-straggler.html +5 -5
  121. package/public/docs/tuning-reference/bottleneck-tiny-tasks.html +5 -5
  122. package/public/docs/tuning-reference/bottleneck-utilization.html +6 -6
  123. package/public/docs/tuning-reference/caching.html +3 -3
  124. package/public/docs/tuning-reference/cluster-config.html +5 -5
  125. package/public/docs/tuning-reference/config.html +3 -3
  126. package/public/docs/tuning-reference/data-formats.html +3 -3
  127. package/public/docs/tuning-reference/index.html +3 -3
  128. package/public/docs/tuning-reference/intro.html +3 -3
  129. package/public/docs/tuning-reference/joins.html +3 -3
  130. package/public/docs/tuning-reference/memory-model.html +5 -5
  131. package/public/docs/tuning-reference/metrics.html +3 -3
  132. package/public/docs/tuning-reference/partitioning.html +3 -3
  133. package/public/docs/tuning-reference/pyspark.html +3 -3
  134. package/public/docs/tuning-reference/shuffle.html +3 -3
  135. package/public/docs/tuning-reference/spark-architecture.html +3 -3
  136. package/public/docs/tuning-reference/table-formats.html +3 -3
  137. package/public/docs/user-guide/alternative-log-retrieval.html +3 -3
  138. package/public/docs/user-guide/getting-started.html +3 -3
  139. package/public/docs/user-guide/mcp-tools.html +5 -5
  140. package/public/docs/user-guide/run-comparison.html +3 -3
  141. package/public/docs/user-guide/understanding-findings.html +5 -5
  142. package/public/index.html +1 -1
  143. package/vendor-core/cli/collect-run.js +3 -2
  144. package/vendor-core/cli/native-zstd.js +351 -0
  145. package/vendor-core/detectors.js +243 -53
  146. package/vendor-core/docs-config.js +34 -8
  147. package/vendor-core/docs-content/chapters/03-memory-model.md +39 -0
  148. package/vendor-core/docs-content/chapters/11-cluster-config.md +40 -0
  149. package/vendor-core/docs-content/detection/gc.md +2 -0
  150. package/vendor-core/docs-content/detection/host.md +2 -1
  151. package/vendor-core/docs-content/detection/plan.md +3 -1
  152. package/vendor-core/docs-content/detection/shape.md +2 -1
  153. package/vendor-core/docs-content/detection/shfl.md +2 -1
  154. package/vendor-core/docs-content/detection/spill.md +1 -1
  155. package/vendor-core/docs-content/detection/strag.md +2 -1
  156. package/vendor-core/docs-content/detection/tiny.md +2 -1
  157. package/vendor-core/docs-content/tuning/failures.md +1 -1
  158. package/vendor-core/docs-content/tuning/gc.md +11 -4
  159. package/vendor-core/docs-content/tuning/shuffle.md +25 -5
  160. package/vendor-core/docs-content/tuning/skew.md +14 -6
  161. package/vendor-core/docs-content/tuning/small-files.md +12 -7
  162. package/vendor-core/docs-content/tuning/straggler.md +34 -0
  163. package/vendor-core/docs-content/tuning/tiny-tasks.md +1 -1
  164. package/vendor-core/docs-content/tuning/utilization.md +57 -7
  165. package/vendor-core/docs-content/upstream.json +4 -0
  166. package/vendor-core/event-handlers.js +321 -69
  167. package/vendor-core/event-schemas.js +8 -6
  168. package/vendor-core/evidence-report.js +3 -1
  169. package/vendor-core/impact-estimator.js +170 -34
  170. package/vendor-core/mcp-tools.js +20 -7
  171. package/vendor-core/occupancy.js +71 -2
  172. package/vendor-core/parser-worker.js +56 -24
  173. package/vendor-core/plan-summary.js +5 -1
  174. package/vendor-core/run-comparison.js +1 -15
  175. package/vendor-core/shs-fetch.js +18 -7
  176. package/vendor-core/shs-load.js +2 -1
  177. package/vendor-core/stage-quantiles.js +111 -3
  178. package/vendor-core/string-hash.js +15 -0
  179. package/vendor-core/types.js +14 -1
  180. package/vendor-core/vendor/fzstd.js +94 -18
  181. package/vendor-core/zstd-worker-client.js +180 -0
  182. package/vendor-core/zstd-worker.js +103 -0
  183. package/public/assets/CacheUtilization-lKua3J1x.js +0 -1
  184. package/public/assets/CachingOpportunity-U0g-8HPV.js +0 -1
  185. package/public/assets/ConfigAudit-ClZp5BpV.js +0 -1
  186. package/public/assets/Dashboard-Bjl-MYMm.js +0 -1
  187. package/public/assets/GcPressure-CYsmU1d7.js +0 -1
  188. package/public/assets/PartitionSizing-Cz_B0-uG.js +0 -1
  189. package/public/assets/PlanGraphRoute-Cj-GWM8W.js +0 -1
  190. package/public/assets/ShuffleIO-pyDhxdaz.js +0 -1
  191. package/public/assets/Straggler-dffcAN7t.js +0 -1
  192. package/public/assets/index-3vlFVNCg.js +0 -9
  193. package/public/assets/parser-worker-B2bc0lWe.js +0 -112
  194. package/public/assets/useRecentFiles-D0OA_EOK.js +0 -11
  195. package/public/docs/assets/chunks/@localSearchIndexroot.CvM5fuxe.js +0 -1
  196. package/public/docs/assets/contributor-guide_architecture_board-widgets.md.CWpj01WU.js +0 -1
  197. package/public/docs/assets/contributor-guide_architecture_detector-contract.md.CgzUsQ6W.js +0 -1
  198. package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.CooslVJt.js +0 -1
  199. package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.CooslVJt.lean.js +0 -1
  200. package/public/docs/assets/contributor-guide_architecture_overview.md.CehiRmGn.js +0 -1
  201. package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.R27gQrgY.js +0 -1
  202. package/public/docs/assets/contributor-guide_development-setup.md.DvAN_9mK.lean.js +0 -1
  203. package/public/docs/assets/contributor-guide_testing.md.6rIKqSyY.js +0 -1
  204. package/public/docs/assets/contributor-guide_testing.md.6rIKqSyY.lean.js +0 -1
  205. package/public/docs/assets/tuning-reference_bottleneck-shuffle.md.CM-nTmIH.js +0 -12
  206. package/public/docs/assets/tuning-reference_bottleneck-utilization.md.DTiueZC3.js +0 -5
  207. package/public/docs/assets/tuning-reference_bottleneck-utilization.md.DTiueZC3.lean.js +0 -1
  208. package/public/docs/assets/user-guide_understanding-findings.md.DL1UDhvR.js +0 -1
  209. /package/public/docs/assets/{contributor-guide_architecture_drill-down.md.BtPdlM7r.lean.js → contributor-guide_architecture_drill-down.md.Bawx6NgP.lean.js} +0 -0
  210. /package/public/docs/assets/{contributor-guide_architecture_widget-rendering.md.R27gQrgY.lean.js → contributor-guide_architecture_widget-rendering.md.DYwFeyjV.lean.js} +0 -0
  211. /package/public/docs/assets/{tuning-reference_bottleneck-failures.md.4z5BXGJ2.lean.js → tuning-reference_bottleneck-failures.md.CAsFf3RR.lean.js} +0 -0
  212. /package/public/docs/assets/{tuning-reference_bottleneck-tiny-tasks.md.QTV7O8kU.lean.js → tuning-reference_bottleneck-tiny-tasks.md.CB05v3zh.lean.js} +0 -0
  213. /package/public/docs/assets/{user-guide_mcp-tools.md.Vi3RoflJ.lean.js → user-guide_mcp-tools.md.DGvolGzt.lean.js} +0 -0
package/public/docs/tuning-reference/anti-patterns.html CHANGED
@@ -9,9 +9,9 @@
9
9
  <link rel="preload stylesheet" href="/docs/assets/style.DSixAiZE.css" as="style">
10
10
  <link rel="preload stylesheet" href="/docs/vp-icons.css" as="style">
11
11
 
12
- <script type="module" src="/docs/assets/app.APoq_UBs.js"></script>
12
+ <script type="module" src="/docs/assets/app.DsFQcynM.js"></script>
13
13
  <link rel="preload" href="/docs/assets/inter-roman-latin.Di8DUHzh.woff2" as="font" type="font/woff2" crossorigin="">
14
- <link rel="modulepreload" href="/docs/assets/chunks/theme.PavXO_vz.js">
14
+ <link rel="modulepreload" href="/docs/assets/chunks/theme.Ca4dR9dL.js">
15
15
  <link rel="modulepreload" href="/docs/assets/chunks/framework.DSg0KOwT.js">
16
16
  <link rel="modulepreload" href="/docs/assets/tuning-reference_anti-patterns.md.Df1YMIHu.lean.js">
17
17
  <link rel="icon" type="image/svg+xml" href="/docs/favicon.svg">
@@ -23,7 +23,7 @@
23
23
  </head>
24
24
  <body>
25
25
  <div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Getting Started</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/intro.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Introduction</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Optimization Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/spark-architecture.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Execution Model</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/memory-model.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Management</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/partitioning.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Partitioning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/joins.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Join Optimization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/data-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Data Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/table-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Table Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/caching.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Caching & Persistence</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/pyspark.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>PySpark Specifics</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/aqe.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Adaptive Query Execution</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/cluster-config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cluster Tuning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/anti-patterns.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Anti-Patterns</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Detector Catalog</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-skew.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Skew</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle I/O</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-spill.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory / Disk Spill</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-gc.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>GC Pressure</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-cold-start.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cold Start</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Executor Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-slow-host.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Slow Host</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-failures.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Failures</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-straggler.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Stragglers</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-retry-waste.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Retry Waste</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-tiny-tasks.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Tiny Tasks</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-job-failure-rate.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Job Failure Rate</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-memory-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Duplicate Plan Subtree</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-small-files.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Small Files</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-broadcast-sizing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Broadcast Sizing</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Reference</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/metrics.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Metrics Glossary</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Config Quick-Reference</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_tuning-reference_anti-patterns" data-v-39a288b8><div><h1 id="anti-patterns" tabindex="-1">Anti-Patterns <a class="header-anchor" href="#anti-patterns" aria-label="Permalink to &quot;Anti-Patterns {#anti-patterns}&quot;">​</a></h1><p>Practitioner checklists on Spark performance converge on a recurring handful of mistakes rather than exotic ones. Each entry below covers what the mistake is, how to detect it, why it hurts, and how to fix it.</p><ol><li><p><strong>Staying on the RDD API instead of DataFrame/Dataset</strong></p><ul><li><strong>What:</strong> Building jobs on the RDD API (or RDD-based MLlib) instead of the DataFrame/Dataset API.</li><li><strong>Detect:</strong> Code that drives its logic through <code>.map</code>/<code>.filter</code> lambdas over RDDs, or that calls into the RDD-based MLlib API.</li><li><strong>Why:</strong> RDDs are lambda-driven, so Spark cannot see inside the closures. It can&#39;t apply predicate pushdown, filter reordering, Adaptive Query Execution, or cost-based join reordering. The RDD-based MLlib API is in maintenance mode for the same reason.<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup></li><li><strong>Fix:</strong> Use the DataFrame/Dataset API so Catalyst can optimize the query plan.</li></ul></li><li><p><strong>Reading CSV/JSON without an explicit schema</strong></p><ul><li><strong>What:</strong> Reading raw CSV or JSON files without supplying an explicit schema.</li><li><strong>Detect:</strong> A read step that spends noticeable time before any real work starts, because Spark is inferring types from the data itself.</li><li><strong>Why:</strong> Schema inference forces a full scan of the data just to determine types, and CSV/JSON don&#39;t support the column pruning, predicate pushdown, or stats-based file skipping that a columnar format does.<sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup></li><li><strong>Fix:</strong> Supply an explicit schema, or prefer Parquet for analytical workloads, where column pruning, predicate pushdown, and stats-based file skipping work out of the box.<sup class="footnote-ref"><a href="#fn1" id="fnref1:2">[1:2]</a></sup></li></ul></li><li><p><strong>Compressing input files with a non-splittable codec (GZIP)</strong></p><ul><li><strong>What:</strong> Storing input data as large GZIP files.</li><li><strong>Detect:</strong> One task/executor taking far longer than the rest on a stage reading GZIP-compressed input, while other executors sit idle.</li><li><strong>Why:</strong> A GZIP file can&#39;t be split across executors, so a single node has to decompress the whole file alone.<sup class="footnote-ref"><a href="#fn1" id="fnref1:3">[1:3]</a></sup></li><li><strong>Fix:</strong> Use a splittable codec instead: Snappy, LZ4, or ZSTD.<sup class="footnote-ref"><a href="#fn1" id="fnref1:4">[1:4]</a></sup></li></ul></li><li><p><strong>Storing data as JSON instead of a binary columnar/row format</strong></p><ul><li><strong>What:</strong> Persisting datasets as JSON rather than a binary columnar or row format.</li><li><strong>Detect:</strong> Read-heavy jobs where a large, recurring share of stage time goes to parsing the same JSON structures on every read.</li><li><strong>Why:</strong> JSON has to be re-parsed from text on every single read.<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup></li><li><strong>Fix:</strong> Store data as Avro, Parquet, Thrift, or Protobuf structs in a sequence file to avoid the repeated parsing cost.<sup class="footnote-ref"><a href="#fn2" id="fnref2:1">[2:1]</a></sup></li></ul></li><li><p><strong>Not registering custom classes with Kryo</strong></p><ul><li><strong>What:</strong> Using Kryo serialization without registering the application&#39;s custom classes.</li><li><strong>Detect:</strong> Larger-than-expected serialized record sizes, and poor efficiency when using a serialized cache storage level such as <code>MEMORY_SER</code>.</li><li><strong>Why:</strong> Unregistered classes increase serialized-record size and hurt serialized cache storage levels.<sup class="footnote-ref"><a href="#fn2" id="fnref2:2">[2:2]</a></sup></li><li><strong>Fix:</strong> Register the application&#39;s custom classes with Kryo.</li></ul></li><li><p><strong>Calling <code>collect()</code> on a large DataFrame</strong></p><ul><li><p><strong>What:</strong> Materializing an entire DataFrame into driver memory with <code>collect()</code>.</p></li><li><p><strong>Detect:</strong> A driver <code>OutOfMemoryError</code>, or a job that aborts once <code>spark.driver.maxResultSize</code> (default 1 GB) is exceeded. Spark also logs a warning once a single task&#39;s serialized result passes roughly 1 MB.<sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup><sup class="footnote-ref"><a href="#fn1" id="fnref1:5">[1:5]</a></sup></p></li><li><p><strong>Why:</strong> <code>collect()</code> gathers every <code>Row</code> from every partition into a single in-memory <code>Array</code> that lives entirely on the driver, not the cluster. The driver JVM has to hold the whole result set in its own heap.<sup class="footnote-ref"><a href="#fn3" id="fnref3:1">[3:1]</a></sup><sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup><sup class="footnote-ref"><a href="#fn5" id="fnref5">[5]</a></sup><sup class="footnote-ref"><a href="#fn6" id="fnref6">[6]</a></sup></p></li><li><p><strong>Fix:</strong> Use aggregations or <code>take(n)</code> instead of collecting the full result set.</p><blockquote><p><strong>PySpark:</strong> <code>toPandas()</code> performs the same driver-side collection as <code>collect()</code> for Python users, so avoid it on large DataFrames too.<sup class="footnote-ref"><a href="#fn6" id="fnref6:1">[6:1]</a></sup></p></blockquote></li></ul></li><li><p><strong>Assuming <code>cache()</code> + <code>count()</code> guarantees the DataFrame stays fully cached</strong></p><ul><li><strong>What:</strong> Treating <code>df.cache(); df.count()</code> as a guarantee that the DataFrame remains fully persisted for later actions.</li><li><strong>Detect:</strong> Later actions on a supposedly-cached DataFrame recompute from source instead of hitting the cache. There&#39;s no built-in visibility into how much of a DataFrame is still actually cached, so this typically only surfaces as unexpectedly slow recomputation.<sup class="footnote-ref"><a href="#fn7" id="fnref7">[7]</a></sup></li><li><strong>Why:</strong> <code>count()</code> does force every partition to be computed and attempted for caching, unlike <code>take</code>/<code>limit</code>, which only touch the partitions they need.<sup class="footnote-ref"><a href="#fn5" id="fnref5:1">[5:1]</a></sup><sup class="footnote-ref"><a href="#fn7" id="fnref7:1">[7:1]</a></sup> But partitions can&#39;t be fractionally cached: if there isn&#39;t room for all of them, the ones that don&#39;t fit are silently dropped and recomputed on next access.<sup class="footnote-ref"><a href="#fn5" id="fnref5:2">[5:2]</a></sup> Cached blocks also compete with execution for the same memory pool and can be evicted under pressure without warning, and caching is tied to the <em>analyzed</em> (pre-optimization) logical plan, so a semantically identical query with a different analyzed plan bypasses the cache entirely and recomputes from source.<sup class="footnote-ref"><a href="#fn7" id="fnref7:2">[7:2]</a></sup> Losing an executor drops any cached block that wasn&#39;t stored with a replicated storage level like <code>MEMORY_AND_DISK_2</code>.<sup class="footnote-ref"><a href="#fn7" id="fnref7:3">[7:3]</a></sup></li><li><strong>Fix:</strong> Keep using <code>.cache()</code> followed by <code>.count()</code> to force materialization, but don&#39;t assume that guarantees persistence (there&#39;s no built-in way to confirm how much of the DataFrame is actually cached<sup class="footnote-ref"><a href="#fn7" id="fnref7:4">[7:4]</a></sup>), and use a replicated storage level if losing a cached partition to executor failure would be costly.</li></ul></li><li><p><strong>Row-at-a-time Python UDFs</strong></p><ul><li><strong>What:</strong> Writing Python UDFs that operate one row at a time instead of vectorized Pandas UDFs.</li><li><strong>Detect:</strong> A UDF-heavy stage where most of the time goes to serialization rather than actual computation.<sup class="footnote-ref"><a href="#fn8" id="fnref8">[8]</a></sup></li><li><strong>Why:</strong> The overhead is serialization plus per-row JVM↔Python data movement, not raw Python interpreter speed. Row-at-a-time UDFs &quot;suffer from high serialization and invocation overhead.&quot;<sup class="footnote-ref"><a href="#fn9" id="fnref9">[9]</a></sup> RDD-era Python UDFs pay a <em>double</em> serialization cost: Java/Scala objects are serialized, then re-serialized to Python via <code>cloudpickle</code> and back on every call.<sup class="footnote-ref"><a href="#fn8" id="fnref8:1">[8:1]</a></sup> A Databricks benchmark on a 10M-row DataFrame showed vectorized Pandas UDFs beating row-at-a-time UDFs &quot;across the board, ranging from 3x to over 100x.&quot;<sup class="footnote-ref"><a href="#fn9" id="fnref9:1">[9:1]</a></sup></li><li><strong>Fix:</strong> Use vectorized Pandas UDFs instead of row-at-a-time UDFs: once data is transferred via Arrow, there&#39;s no need to serialize/pickle it, since it&#39;s already in a format consumable by the Python process.<sup class="footnote-ref"><a href="#fn5" id="fnref5:3">[5:3]</a></sup></li></ul></li><li><p><strong>Leaving <code>spark.sql.shuffle.partitions</code> at its default of 200</strong></p><ul><li><strong>What:</strong> Never tuning <code>spark.sql.shuffle.partitions</code> away from its default of 200 for wide transformations (<code>join</code>, <code>groupBy</code>, aggregations).</li><li><strong>Detect:</strong> In the Spark UI Stages tab, too few partitions for the data size shows up as &quot;Spill (Memory)&quot; or &quot;Spill (Disk)&quot; entries against a stage; too many partitions for the data size shows up as a large task/stage count where each task&#39;s duration is dominated by scheduling and bookkeeping, often paired with a large number of tiny output files.<sup class="footnote-ref"><a href="#fn10" id="fnref10">[10]</a></sup></li><li><strong>Why:</strong> 200 is a fixed default regardless of whether you&#39;re joining 5MB or 5TB.<sup class="footnote-ref"><a href="#fn10" id="fnref10:1">[10:1]</a></sup><sup class="footnote-ref"><a href="#fn11" id="fnref11">[11]</a></sup> Too few partitions means each one holds more data than fits comfortably in executor memory, which increases load per executor and leads to spills once partition size exceeds available memory.<sup class="footnote-ref"><a href="#fn10" id="fnref10:2">[10:2]</a></sup> Too many partitions on a small dataset can shrink tasks to around ten rows each, so &quot;most of your CPUs will just be sitting there doing nothing.&quot;<sup class="footnote-ref"><a href="#fn10" id="fnref10:3">[10:3]</a></sup></li><li><strong>Fix:</strong> Increase <code>spark.sql.shuffle.partitions</code> if tasks are processing multiple GB each and spilling; decrease it if tasks finish in a couple of seconds and write tiny files. A rough starting target is 100–200 MB of data per task, tuned per dataset.<sup class="footnote-ref"><a href="#fn10" id="fnref10:4">[10:4]</a></sup></li></ul></li><li><p><strong>Broadcasting a table that&#39;s too large</strong></p><ul><li><strong>What:</strong> Forcing or allowing a broadcast join on a table that&#39;s too large to broadcast safely.</li><li><strong>Detect:</strong> A driver <code>OutOfMemoryError</code> during a broadcast join.</li><li><strong>Why:</strong> The small side of the join has to be collected onto the driver before it can be broadcast to every executor. That collection step is the same driver-memory operation as <code>.collect()</code>, and it&#39;s what fails, not executor-side replication or a serialization timeout: &quot;if you try to broadcast something too large, you can crash your driver node (because that collect is expensive).&quot;<sup class="footnote-ref"><a href="#fn4" id="fnref4:1">[4:1]</a></sup></li><li><strong>Fix:</strong> Use <code>spark.sql.autoBroadcastJoinThreshold</code> to control the maximum size Spark will broadcast, or increase driver memory.<sup class="footnote-ref"><a href="#fn4" id="fnref4:2">[4:2]</a></sup></li></ul></li><li><p><strong>Reaching for <code>coalesce(1)</code> before every single-file write</strong></p><ul><li><strong>What:</strong> Always using <code>coalesce(1)</code> instead of <code>repartition(1)</code> when a single output file is needed.</li><li><strong>Detect:</strong> The entire upstream stage (including filtering/transformation work that would otherwise run in parallel) collapses onto a single task/executor, leaving the rest of the cluster idle.</li><li><strong>Why:</strong> <code>coalesce</code> is a narrow transformation, so it &quot;causes the upstream partitions in the entire stage to execute with the level of parallelism assigned by coalesce,&quot; fusing all preceding work down to one task.<sup class="footnote-ref"><a href="#fn6" id="fnref6:2">[6:2]</a></sup> <code>repartition</code> triggers a full shuffle instead, which inserts an explicit shuffle boundary so the upstream stage keeps its original parallelism.<sup class="footnote-ref"><a href="#fn4" id="fnref4:3">[4:3]</a></sup><sup class="footnote-ref"><a href="#fn6" id="fnref6:3">[6:3]</a></sup></li><li><strong>Fix:</strong> Use <code>repartition(1)</code> when there is meaningful upstream computation you don&#39;t want collapsed onto a single executor; reserve <code>coalesce(1)</code> for when the upstream stage is already cheap and paying for a shuffle would be wasted cost.<sup class="footnote-ref"><a href="#fn6" id="fnref6:4">[6:4]</a></sup></li></ul></li><li><p><strong>Not verifying predicate pushdown is actually happening</strong></p><ul><li><strong>What:</strong> Assuming filters and column projections are pushed down to the data source without checking the physical plan.</li><li><strong>Detect:</strong> Run <code>.explain()</code> and look at the <code>Scan</code> node for a <code>PushedFilters</code> marker. For example, <code>*Scan JDBCRel... PushedFilters: [*In(DEST_COUNTRY_NAME, [Anguilla, Sweden])]</code>. Column pruning is verified the same way: a <code>.select()</code> on one column should show a narrowed <code>ReadSchema</code> at the scan rather than a full-table scan.<sup class="footnote-ref"><a href="#fn4" id="fnref4:4">[4:4]</a></sup></li><li><strong>Why:</strong> Spark pushes down simple filters (column equality, <code>IN</code>, <code>IS NULL</code>) to JDBC sources automatically, but anything with a computed column or cast won&#39;t push down and gets evaluated in Spark after the full read.<sup class="footnote-ref"><a href="#fn1" id="fnref1:6">[1:6]</a></sup> Some sources also only partially handle a filter and leave Spark to re-evaluate it as a safety mechanism; that&#39;s still a &quot;good&quot; pushdown since the amount of data read is reduced.<sup class="footnote-ref"><a href="#fn6" id="fnref6:5">[6:5]</a></sup></li><li><strong>Fix:</strong> Check <code>explain</code> output (paired with <code>printSchema</code>)<sup class="footnote-ref"><a href="#fn6" id="fnref6:6">[6:6]</a></sup> for <code>PushedFilters</code> after writing a filter, and rewrite filters that use casts or computed columns as simple predicates where possible so they can push down.<sup class="footnote-ref"><a href="#fn4" id="fnref4:5">[4:5]</a></sup><sup class="footnote-ref"><a href="#fn1" id="fnref1:7">[1:7]</a></sup></li></ul></li></ol><h2 id="sources" tabindex="-1">Sources <a class="header-anchor" href="#sources" aria-label="Permalink to &quot;Sources&quot;">​</a></h2><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><a href="https://luminousmen.com/post/the-apache-spark-optimization-checklist" target="_blank" rel="noreferrer">The Apache Spark Optimization Checklist</a> <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a> <a href="#fnref1:2" class="footnote-backref">↩︎</a> <a href="#fnref1:3" class="footnote-backref">↩︎</a> <a href="#fnref1:4" class="footnote-backref">↩︎</a> <a href="#fnref1:5" class="footnote-backref">↩︎</a> <a href="#fnref1:6" class="footnote-backref">↩︎</a> <a href="#fnref1:7" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><a href="https://blog.cloudera.com/how-to-tune-your-apache-spark-jobs-part-2/" target="_blank" rel="noreferrer">How to Tune Your Apache Spark Jobs (Part 2)</a> <a href="#fnref2" class="footnote-backref">↩︎</a> <a href="#fnref2:1" class="footnote-backref">↩︎</a> <a href="#fnref2:2" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><em>Advanced Analytics with PySpark</em>, Tandon, Ryza, Laserson et al., ch. 2 <a href="#fnref3" class="footnote-backref">↩︎</a> <a href="#fnref3:1" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><em>Spark: The Definitive Guide</em>, Chambers &amp; Zaharia, chs. 5, 8, 9, 18, 19 <a href="#fnref4" class="footnote-backref">↩︎</a> <a href="#fnref4:1" class="footnote-backref">↩︎</a> <a href="#fnref4:2" class="footnote-backref">↩︎</a> <a href="#fnref4:3" class="footnote-backref">↩︎</a> <a href="#fnref4:4" class="footnote-backref">↩︎</a> <a href="#fnref4:5" class="footnote-backref">↩︎</a></p></li><li id="fn5" class="footnote-item"><p><em>Learning Spark, 2nd Edition</em>, Damji, Wenig, Das &amp; Lee, chs. 3, 5, 7 <a href="#fnref5" class="footnote-backref">↩︎</a> <a href="#fnref5:1" class="footnote-backref">↩︎</a> <a href="#fnref5:2" class="footnote-backref">↩︎</a> <a href="#fnref5:3" class="footnote-backref">↩︎</a></p></li><li id="fn6" class="footnote-item"><p><em>High Performance Spark, 2nd Edition</em>, Karau, Polak &amp; Warren, chs. 5, 7 <a href="#fnref6" class="footnote-backref">↩︎</a> <a href="#fnref6:1" class="footnote-backref">↩︎</a> <a href="#fnref6:2" class="footnote-backref">↩︎</a> <a href="#fnref6:3" class="footnote-backref">↩︎</a> <a href="#fnref6:4" class="footnote-backref">↩︎</a> <a href="#fnref6:5" class="footnote-backref">↩︎</a> <a href="#fnref6:6" class="footnote-backref">↩︎</a></p></li><li id="fn7" class="footnote-item"><p><a href="https://luminousmen.com/post/explaining-the-mechanics-of-spark-caching" target="_blank" rel="noreferrer">Explaining the Mechanics of Spark Caching</a> <a href="#fnref7" class="footnote-backref">↩︎</a> <a href="#fnref7:1" class="footnote-backref">↩︎</a> <a href="#fnref7:2" class="footnote-backref">↩︎</a> <a href="#fnref7:3" class="footnote-backref">↩︎</a> <a href="#fnref7:4" class="footnote-backref">↩︎</a></p></li><li id="fn8" class="footnote-item"><p><a href="https://luminousmen.com/post/spark-tips-dataframe-api" target="_blank" rel="noreferrer">Spark Tips: DataFrame API</a> <a href="#fnref8" class="footnote-backref">↩︎</a> <a href="#fnref8:1" class="footnote-backref">↩︎</a></p></li><li id="fn9" class="footnote-item"><p><a href="https://www.databricks.com/blog/2017/10/30/introducing-vectorized-udfs-for-pyspark.html" target="_blank" rel="noreferrer">Introducing Vectorized UDFs for PySpark</a> <a href="#fnref9" class="footnote-backref">↩︎</a> <a href="#fnref9:1" class="footnote-backref">↩︎</a></p></li><li id="fn10" class="footnote-item"><p><a href="https://luminousmen.com/post/spark-partitions" target="_blank" rel="noreferrer">Spark Partitions</a> <a href="#fnref10" class="footnote-backref">↩︎</a> <a href="#fnref10:1" class="footnote-backref">↩︎</a> <a href="#fnref10:2" class="footnote-backref">↩︎</a> <a href="#fnref10:3" class="footnote-backref">↩︎</a> <a href="#fnref10:4" class="footnote-backref">↩︎</a></p></li><li id="fn11" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration — Spark</a> <a href="#fnref11" class="footnote-backref">↩︎</a></p></li></ol></section></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/tuning-reference/cluster-config.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>Cluster Tuning</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="/docs/tuning-reference/bottleneck-skew.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>Task Skew</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
26
- <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"CWpj01WU\",\"contributor-guide_architecture_detector-contract.md\":\"CgzUsQ6W\",\"contributor-guide_architecture_drill-down.md\":\"BtPdlM7r\",\"contributor-guide_architecture_impact-estimation.md\":\"CooslVJt\",\"contributor-guide_architecture_index.md\":\"3TO9ic6w\",\"contributor-guide_architecture_overview.md\":\"CehiRmGn\",\"contributor-guide_architecture_state-and-history.md\":\"C-xxn0q7\",\"contributor-guide_architecture_widget-rendering.md\":\"R27gQrgY\",\"contributor-guide_architecture_worker-protocol.md\":\"IbnfNrV3\",\"contributor-guide_contributing.md\":\"CvRsdr6J\",\"contributor-guide_development-setup.md\":\"DvAN_9mK\",\"contributor-guide_testing.md\":\"6rIKqSyY\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"Df1YMIHu\",\"tuning-reference_aqe.md\":\"BIsCtLzm\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"CEstB3Ia\",\"tuning-reference_bottleneck-cold-start.md\":\"CEuy-72y\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"CIohQDfn\",\"tuning-reference_bottleneck-failures.md\":\"4z5BXGJ2\",\"tuning-reference_bottleneck-gc.md\":\"DSxzZRK7\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"BaJl__1W\",\"tuning-reference_bottleneck-memory-utilization.md\":\"DbP-SJZc\",\"tuning-reference_bottleneck-retry-waste.md\":\"D5JMjVOt\",\"tuning-reference_bottleneck-shuffle.md\":\"CM-nTmIH\",\"tuning-reference_bottleneck-skew.md\":\"BdUwiDhn\",\"tuning-reference_bottleneck-slow-host.md\":\"BlIo6UDW\",\"tuning-reference_bottleneck-small-files.md\":\"B8kloyx8\",\"tuning-reference_bottleneck-spill.md\":\"PNH7mITt\",\"tuning-reference_bottleneck-straggler.md\":\"DY36fHN5\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"QTV7O8kU\",\"tuning-reference_bottleneck-utilization.md\":\"DTiueZC3\",\"tuning-reference_caching.md\":\"B7aQ8asB\",\"tuning-reference_cluster-config.md\":\"ZVmDGsQ3\",\"tuning-reference_config.md\":\"UvveiWG3\",\"tuning-reference_data-formats.md\":\"bjCAWH3N\",\"tuning-reference_index.md\":\"BQ_NooMV\",\"tuning-reference_intro.md\":\"CobD-lGB\",\"tuning-reference_joins.md\":\"BtKs_CuW\",\"tuning-reference_memory-model.md\":\"DhT-n4y3\",\"tuning-reference_metrics.md\":\"mLOh7Apj\",\"tuning-reference_partitioning.md\":\"q0zKF_8X\",\"tuning-reference_pyspark.md\":\"DDCfvN9t\",\"tuning-reference_shuffle.md\":\"BZZ7R4Ix\",\"tuning-reference_spark-architecture.md\":\"Dwzm5avO\",\"tuning-reference_table-formats.md\":\"D6wj-2dX\",\"user-guide_alternative-log-retrieval.md\":\"B4tPGIal\",\"user-guide_getting-started.md\":\"BJvwLEIM\",\"user-guide_mcp-tools.md\":\"Vi3RoflJ\",\"user-guide_run-comparison.md\":\"CQc1aoU8\",\"user-guide_understanding-findings.md\":\"DL1UDhvR\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Alternative log retrieval\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
26
+ <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"9ToZXcTk\",\"contributor-guide_architecture_detector-contract.md\":\"aQQfP7pj\",\"contributor-guide_architecture_drill-down.md\":\"Bawx6NgP\",\"contributor-guide_architecture_impact-estimation.md\":\"sf9DU2bA\",\"contributor-guide_architecture_index.md\":\"3TO9ic6w\",\"contributor-guide_architecture_overview.md\":\"BXQdgG5u\",\"contributor-guide_architecture_state-and-history.md\":\"C-xxn0q7\",\"contributor-guide_architecture_widget-rendering.md\":\"DYwFeyjV\",\"contributor-guide_architecture_worker-protocol.md\":\"P7d1vtEJ\",\"contributor-guide_contributing.md\":\"CvRsdr6J\",\"contributor-guide_development-setup.md\":\"DvMV1xhQ\",\"contributor-guide_testing.md\":\"BU6vyA-9\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"Df1YMIHu\",\"tuning-reference_aqe.md\":\"BIsCtLzm\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"CEstB3Ia\",\"tuning-reference_bottleneck-cold-start.md\":\"CEuy-72y\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"CIohQDfn\",\"tuning-reference_bottleneck-failures.md\":\"CAsFf3RR\",\"tuning-reference_bottleneck-gc.md\":\"B4T3kJYV\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"BaJl__1W\",\"tuning-reference_bottleneck-memory-utilization.md\":\"DbP-SJZc\",\"tuning-reference_bottleneck-retry-waste.md\":\"D5JMjVOt\",\"tuning-reference_bottleneck-shuffle.md\":\"DDMQVOW6\",\"tuning-reference_bottleneck-skew.md\":\"B0MzhWcX\",\"tuning-reference_bottleneck-slow-host.md\":\"BlIo6UDW\",\"tuning-reference_bottleneck-small-files.md\":\"DHGMTmPs\",\"tuning-reference_bottleneck-spill.md\":\"PNH7mITt\",\"tuning-reference_bottleneck-straggler.md\":\"DIbXdAcQ\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"CB05v3zh\",\"tuning-reference_bottleneck-utilization.md\":\"DqC_iGkw\",\"tuning-reference_caching.md\":\"B7aQ8asB\",\"tuning-reference_cluster-config.md\":\"CD6vzYtD\",\"tuning-reference_config.md\":\"UvveiWG3\",\"tuning-reference_data-formats.md\":\"bjCAWH3N\",\"tuning-reference_index.md\":\"BQ_NooMV\",\"tuning-reference_intro.md\":\"CobD-lGB\",\"tuning-reference_joins.md\":\"BtKs_CuW\",\"tuning-reference_memory-model.md\":\"CWhvgevP\",\"tuning-reference_metrics.md\":\"mLOh7Apj\",\"tuning-reference_partitioning.md\":\"q0zKF_8X\",\"tuning-reference_pyspark.md\":\"DDCfvN9t\",\"tuning-reference_shuffle.md\":\"BZZ7R4Ix\",\"tuning-reference_spark-architecture.md\":\"Dwzm5avO\",\"tuning-reference_table-formats.md\":\"D6wj-2dX\",\"user-guide_alternative-log-retrieval.md\":\"B4tPGIal\",\"user-guide_getting-started.md\":\"BJvwLEIM\",\"user-guide_mcp-tools.md\":\"DGvolGzt\",\"user-guide_run-comparison.md\":\"CQc1aoU8\",\"user-guide_understanding-findings.md\":\"B29f0ucS\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Alternative log retrieval\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
27
27
 
28
28
  </body>
29
29
  </html>
package/public/docs/tuning-reference/aqe.html CHANGED
@@ -9,9 +9,9 @@
9
9
  <link rel="preload stylesheet" href="/docs/assets/style.DSixAiZE.css" as="style">
10
10
  <link rel="preload stylesheet" href="/docs/vp-icons.css" as="style">
11
11
 
12
- <script type="module" src="/docs/assets/app.APoq_UBs.js"></script>
12
+ <script type="module" src="/docs/assets/app.DsFQcynM.js"></script>
13
13
  <link rel="preload" href="/docs/assets/inter-roman-latin.Di8DUHzh.woff2" as="font" type="font/woff2" crossorigin="">
14
- <link rel="modulepreload" href="/docs/assets/chunks/theme.PavXO_vz.js">
14
+ <link rel="modulepreload" href="/docs/assets/chunks/theme.Ca4dR9dL.js">
15
15
  <link rel="modulepreload" href="/docs/assets/chunks/framework.DSg0KOwT.js">
16
16
  <link rel="modulepreload" href="/docs/assets/tuning-reference_aqe.md.BIsCtLzm.lean.js">
17
17
  <link rel="icon" type="image/svg+xml" href="/docs/favicon.svg">
@@ -23,7 +23,7 @@
23
23
  </head>
24
24
  <body>
25
25
  <div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Getting Started</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/intro.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Introduction</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Optimization Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/spark-architecture.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Execution Model</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/memory-model.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Management</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/partitioning.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Partitioning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/joins.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Join Optimization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/data-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Data Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/table-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Table Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/caching.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Caching & Persistence</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/pyspark.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>PySpark Specifics</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/aqe.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Adaptive Query Execution</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/cluster-config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cluster Tuning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/anti-patterns.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Anti-Patterns</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Detector Catalog</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-skew.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Skew</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle I/O</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-spill.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory / Disk Spill</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-gc.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>GC Pressure</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-cold-start.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cold Start</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Executor Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-slow-host.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Slow Host</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-failures.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Failures</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-straggler.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Stragglers</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-retry-waste.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Retry Waste</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-tiny-tasks.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Tiny Tasks</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-job-failure-rate.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Job Failure Rate</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-memory-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Duplicate Plan Subtree</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-small-files.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Small Files</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-broadcast-sizing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Broadcast Sizing</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Reference</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/metrics.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Metrics Glossary</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Config Quick-Reference</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_tuning-reference_aqe" data-v-39a288b8><div><h1 id="aqe" tabindex="-1">Adaptive Query Execution <a class="header-anchor" href="#aqe" aria-label="Permalink to &quot;Adaptive Query Execution {#aqe}&quot;">​</a></h1><h2 id="what-it-is" tabindex="-1">What it is <a class="header-anchor" href="#what-it-is" aria-label="Permalink to &quot;What it is&quot;">​</a></h2><p>The static Catalyst optimizer plans a query once, before execution, using cost estimates derived from static statistics: row counts, min/max, NDVs, or defaults when statistics are missing.<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup> Adaptive Query Execution (AQE), shipped in Spark 3.0, instead re-optimizes the plan mid-execution using runtime statistics gathered from completed &quot;query stages&quot;: the sections of a plan bounded by shuffle or broadcast exchange materialization points.<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup> A shuffle or broadcast forces Spark to materialize its input before continuing, which makes it a natural checkpoint: once one or more leaf stages finish, AQE marks them complete, updates the logical plan with the real (not estimated) statistics, and reruns a selected set of logical and physical optimization rules (including AQE-specific rules such as partition coalescing and skew-join handling) before executing the next stages.<sup class="footnote-ref"><a href="#fn2" id="fnref2:1">[2:1]</a></sup></p><p>Shuffle statistics only become available once a stage is <em>fully</em> materialized, not incrementally as individual map tasks finish. A stage&#39;s successor can only proceed once every parallel process producing that stage&#39;s output has completed, which is exactly why materialization points are the reoptimization opportunity: it&#39;s the moment when statistics on all of a stage&#39;s partitions are known and the next stage hasn&#39;t started yet.<sup class="footnote-ref"><a href="#fn2" id="fnref2:2">[2:2]</a></sup> AQE kicks off all leaf stages (the ones with no upstream dependency) first; as each one finishes, the framework marks it complete, updates the plan, and re-optimizes before launching whichever next stages now have all their children materialized. This execute→reoptimize→execute loop repeats (once per completed stage, not just once) until the whole query finishes, so the number of reoptimizations scales with how many shuffle/broadcast boundaries the plan has.<sup class="footnote-ref"><a href="#fn2" id="fnref2:3">[2:3]</a></sup> The Spark SQL config description for <code>spark.sql.adaptive.enabled</code> frames this the same way: AQE re-optimizes &quot;the query plan in the middle of query execution, based on accurate runtime statistics.&quot;<sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup></p><img class="light-only" src="/docs/assets/aqe-loop.IwQSATHw.svg" alt="AQE runs a loop that executes leaf stages, materializes at a shuffle or broadcast boundary, collects runtime statistics, re-applies rules to coalesce partitions and split skew and promote joins to broadcast, then launches the next stages until the plan is complete."><img class="dark-only" src="/docs/assets/aqe-loop.dark.DGbaxqJE.svg" alt="AQE runs a loop that executes leaf stages, materializes at a shuffle or broadcast boundary, collects runtime statistics, re-applies rules to coalesce partitions and split skew and promote joins to broadcast, then launches the next stages until the plan is complete."><p>AQE re-optimizes three things the static optimizer cannot, because none of them are knowable before execution: the number of post-shuffle partitions (coalescing small partitions produced by wide transformations), the join strategy (converting a statically planned sort-merge join to a broadcast join once the actual materialized size of a join side is known), and skewed partitions in a shuffle join (splitting oversized partitions detected from shuffle file statistics).<sup class="footnote-ref"><a href="#fn2" id="fnref2:4">[2:4]</a></sup> High Performance Spark frames this as AQE using &quot;runtime information about the data it is processing along with the target output to go beyond static optimizations,&quot; with partitioning and join strategy singled out as the two biggest areas it impacts.<sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup> AQE also performs empty-relation propagation, replacing subqueries that turn out to be empty (impossible joins, empty unions) with a dummy empty <code>LocalRelation</code>, again something only knowable once data is materialized.<sup class="footnote-ref"><a href="#fn4" id="fnref4:1">[4:1]</a></sup></p><h2 id="how-it-s-detected" tabindex="-1">How it&#39;s detected <a class="header-anchor" href="#how-it-s-detected" aria-label="Permalink to &quot;How it&#39;s detected&quot;">​</a></h2><p>Because these are runtime decisions, none of them show up in a static <code>explain()</code> call; you have to run the query and check the Spark UI for the finalized, adapted plan.<sup class="footnote-ref"><a href="#fn4" id="fnref4:2">[4:2]</a></sup> The same caveat applies more broadly: since AQE adjusts plans at runtime, <code>explain()</code> might show one plan while the Spark UI shows a different one actually executed.<sup class="footnote-ref"><a href="#fn5" id="fnref5">[5]</a></sup></p><p>What you&#39;re looking for in that adaptive plan are the specific runtime rewrites AQE can make:</p><ul><li><strong>Partition coalescing.</strong> AQE combines <em>adjacent</em> small post-shuffle partitions into bigger ones by reading shuffle file statistics,<sup class="footnote-ref"><a href="#fn2" id="fnref2:5">[2:5]</a></sup> targeting <code>spark.sql.adaptive.advisoryPartitionSizeInBytes</code> (default 64 MB).<sup class="footnote-ref"><a href="#fn6" id="fnref6">[6]</a></sup> In a worked example, five post-shuffle partitions where three are small get coalesced into one, cutting the final-aggregation task count from five to three.<sup class="footnote-ref"><a href="#fn2" id="fnref2:6">[2:6]</a></sup></li><li><strong>Skew splitting.</strong> A partition is considered skewed if its size is larger than the median partition size times <code>spark.sql.adaptive.skewJoin.skewedPartitionFactor</code> (default 5.0) <em>and</em> larger than <code>spark.sql.adaptive.skewJoin.skewedPartitionThresholdInBytes</code> (default 256 MB).<sup class="footnote-ref"><a href="#fn7" id="fnref7">[7]</a></sup><sup class="footnote-ref"><a href="#fn8" id="fnref8">[8]</a></sup> The original design doc&#39;s worked example (table A join table B, where B&#39;s partition 0 is skewed) has the <code>OptimizeSkewedJoin</code> rule divide that partition into N smaller splits reading disjoint ranges of upstream map outputs, create N separate tasks each joining its slice of B&#39;s partition 0 against A&#39;s corresponding partition 0, then union the N results. That trades reading A&#39;s partition 0 N times against eliminating a straggler task.<sup class="footnote-ref"><a href="#fn7" id="fnref7:1">[7:1]</a></sup></li><li><strong>Join strategy promotion.</strong> When a join side&#39;s runtime-materialized size falls under the adaptive broadcast threshold, AQE converts a planned sort-merge join to a broadcast hash join. It reuses the shuffle output already written rather than re-materializing the build side. The perf-tuning guide frames the benefit as avoiding a re-sort of both join sides and reading the existing shuffle files locally instead, conditional on <code>spark.sql.adaptive.localShuffleReader.enabled</code>.<sup class="footnote-ref"><a href="#fn6" id="fnref6:1">[6:1]</a></sup> The same guide calls this conversion &quot;not as efficient as planning a broadcast hash join in the first place,&quot; consistent with reusing existing shuffle output rather than recomputing an equivalent build side from scratch.<sup class="footnote-ref"><a href="#fn6" id="fnref6:2">[6:2]</a></sup></li><li><strong>Local shuffle reads.</strong> Once that promotion happens, the side that no longer needs to be partitioned by join key can be read straight off the shuffle files each executor already has locally, instead of pulling blocks from remote executors over the network. This is what <code>spark.sql.adaptive.localShuffleReader.enabled</code> (default true since 3.0.0) does, and it applies whenever shuffle partitioning is no longer needed, such as after a sort-merge-to- broadcast conversion.<sup class="footnote-ref"><a href="#fn6" id="fnref6:3">[6:3]</a></sup><sup class="footnote-ref"><a href="#fn2" id="fnref2:7">[2:7]</a></sup></li></ul><p>Dynamic Partition Pruning is a separate mechanism worth distinguishing from AQE&#39;s loop when reading a plan: DPP inserts a <code>DynamicPruningSubquery</code>/<code>DynamicPruningExpression</code> node during logical optimization/physical planning, before execution starts, when an equi-join is on a partition column and pruning looks beneficial by static statistics.<sup class="footnote-ref"><a href="#fn9" id="fnref9">[9]</a></sup> Because DPP&#39;s calculation runs at planning time and AQE&#39;s runs later, mid-execution, off completed-stage statistics, the ordering implied is that DPP&#39;s pruning predicate is planned first, with AQE&#39;s adaptive rules applying afterward, during execution, on top of whatever DPP already pruned<sup class="footnote-ref"><a href="#fn9" id="fnref9:1">[9:1]</a></sup><sup class="footnote-ref"><a href="#fn2" id="fnref2:8">[2:8]</a></sup>, though the exact current-version integration between the two isn&#39;t detailed further in the available sources, so treat that ordering as directionally supported rather than exhaustively confirmed.</p><h2 id="why-it-matters" tabindex="-1">Why it matters <a class="header-anchor" href="#why-it-matters" aria-label="Permalink to &quot;Why it matters&quot;">​</a></h2><p>AQE exists precisely because partition sizing, join strategy, and skew are only knowable once real data has been materialized. That&#39;s the gap the static optimizer can&#39;t close on its own.<sup class="footnote-ref"><a href="#fn2" id="fnref2:9">[2:9]</a></sup><sup class="footnote-ref"><a href="#fn4" id="fnref4:3">[4:3]</a></sup> But it isn&#39;t a guaranteed win. High Performance Spark warns that &quot;while AQE generally does better than unoptimized code, there are cases, especially when targeting Iceberg tables, where AQE does more harm than good&quot;: AQE&#39;s partitioning-to-target-table matching is generally beneficial but can cause severe regressions if you&#39;ve already done deliberate work to avoid key skew via custom partitioning, since AQE&#39;s attempt to match the target table&#39;s write partitioning can undo that work, especially when there&#39;s a large amount of key skew.<sup class="footnote-ref"><a href="#fn4" id="fnref4:4">[4:4]</a></sup></p><p>A second, more general failure mode is bad input statistics feeding AQE&#39;s runtime decisions: if the source data has wrong stats (compressed JSON or Kafka streams are called out specifically), AQE can make things worse rather than better.<sup class="footnote-ref"><a href="#fn5" id="fnref5:1">[5:1]</a></sup> The same source&#39;s framing is that AQE &quot;isn&#39;t magic... it helps polish your plan; it doesn&#39;t design it for you. You still need good partitioning fundamentals&quot;. AQE can smooth over a reasonably partitioned plan but doesn&#39;t reliably rescue one built on bad upstream statistics or layout.<sup class="footnote-ref"><a href="#fn5" id="fnref5:2">[5:2]</a></sup></p><p>AQE&#39;s runtime re-optimization also doesn&#39;t reach into DataFrame caching, which is worth knowing so you don&#39;t reach for the wrong lever: <code>.cache()</code>/<code>.persist()</code> wrap the query&#39;s <em>analyzed</em> logical plan (the point after the analyzer phase but before optimization) in an <code>InMemoryRelation</code> node, and cache lookup compares the analyzed plan of the current query against what was cached, recomputing if they don&#39;t match exactly, even if the two queries would ultimately produce the same optimized physical plan.<sup class="footnote-ref"><a href="#fn10" id="fnref10">[10]</a></sup> Because AQE operates on the physical plan at runtime, well downstream of that analyzed-plan cache key, disabling AQE doesn&#39;t change whether a cached DataFrame&#39;s analyzed plan matches; it only changes how the physical/execution plan is chosen after that cache lookup already happened.<sup class="footnote-ref"><a href="#fn10" id="fnref10:1">[10:1]</a></sup></p><h2 id="how-to-fix-it" tabindex="-1">How to fix it <a class="header-anchor" href="#how-to-fix-it" aria-label="Permalink to &quot;How to fix it&quot;">​</a></h2><p>AQE&#39;s coalescing, skew-handling, and join-promotion rules only fire once <code>spark.sql.adaptive.enabled</code> is on, and each has its own knobs worth tuning rather than leaving at the defaults:</p><ul><li><code>spark.sql.adaptive.coalescePartitions.enabled</code> (default true) turns on partition coalescing; <code>spark.sql.adaptive.advisoryPartitionSizeInBytes</code> (default 64 MB) sets its target size.<sup class="footnote-ref"><a href="#fn6" id="fnref6:4">[6:4]</a></sup> <code>spark.sql.adaptive.coalescePartitions.minPartitionNum</code> bounds the minimum resulting parallelism,<sup class="footnote-ref"><a href="#fn4" id="fnref4:5">[4:5]</a></sup> and <code>spark.sql.adaptive.coalescePartitions.minPartitionSize</code> (default 1 MB, since 3.2) sets a floor on coalesced partition size for when the adaptively calculated target is too small.<sup class="footnote-ref"><a href="#fn6" id="fnref6:5">[6:5]</a></sup></li><li><code>spark.sql.adaptive.coalescePartitions.parallelismFirst</code> (default true, since 3.2) tells Spark to ignore the advisory target size entirely and instead calculate a (usually smaller) target based on cluster default parallelism, prioritizing task parallelism over hitting the 64 MB target. On a busy cluster, set it to <code>false</code> to respect the configured target size and avoid producing many small tasks.<sup class="footnote-ref"><a href="#fn6" id="fnref6:6">[6:6]</a></sup></li><li><code>spark.sql.adaptive.skewJoin.enabled</code> (default true) turns on skew-join handling for both sort-merge and shuffled hash joins.<sup class="footnote-ref"><a href="#fn8" id="fnref8:1">[8:1]</a></sup> Tune the detection thresholds via <code>spark.sql.adaptive.skewJoin.skewedPartitionFactor</code> (default 5.0) and <code>spark.sql.adaptive.skewJoin.skewedPartitionThresholdInBytes</code> (default 256 MB); set the byte threshold larger than <code>advisoryPartitionSizeInBytes</code>.<sup class="footnote-ref"><a href="#fn8" id="fnref8:2">[8:2]</a></sup> Use <code>spark.sql.adaptive.forceOptimizeSkewedJoin</code> (default false, since 3.3.0) to force the rule to fire even when it would introduce extra shuffle.<sup class="footnote-ref"><a href="#fn6" id="fnref6:7">[6:7]</a></sup></li><li><code>spark.sql.adaptive.localShuffleReader.enabled</code> (default true, since 3.0.0) keeps the local, per-mapper shuffle read enabled after a sort-merge-to-broadcast conversion.<sup class="footnote-ref"><a href="#fn6" id="fnref6:8">[6:8]</a></sup></li></ul><p>If AQE&#39;s target-table partition matching is regressing an Iceberg write that already has deliberate custom partitioning to avoid key skew, set the table&#39;s (or write-time) <code>write.distribution-mode</code> property to <code>none</code>, at the cost of a higher risk of many small files from unsorted/unhashed writers per partition.<sup class="footnote-ref"><a href="#fn4" id="fnref4:6">[4:6]</a></sup></p><h2 id="sources" tabindex="-1">Sources <a class="header-anchor" href="#sources" aria-label="Permalink to &quot;Sources&quot;">​</a></h2><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><a href="https://www.databricks.com/blog/2015/04/13/deep-dive-into-spark-sqls-catalyst-optimizer.html" target="_blank" rel="noreferrer">Deep Dive into Spark SQL&#39;s Catalyst Optimizer</a> <a href="#fnref1" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><a href="https://www.databricks.com/blog/2020/05/29/adaptive-query-execution-speeding-up-spark-sql-at-runtime.html" target="_blank" rel="noreferrer">Adaptive Query Execution: Speeding Up Spark SQL at Runtime</a> <a href="#fnref2" class="footnote-backref">↩︎</a> <a href="#fnref2:1" class="footnote-backref">↩︎</a> <a href="#fnref2:2" class="footnote-backref">↩︎</a> <a href="#fnref2:3" class="footnote-backref">↩︎</a> <a href="#fnref2:4" class="footnote-backref">↩︎</a> <a href="#fnref2:5" class="footnote-backref">↩︎</a> <a href="#fnref2:6" class="footnote-backref">↩︎</a> <a href="#fnref2:7" class="footnote-backref">↩︎</a> <a href="#fnref2:8" class="footnote-backref">↩︎</a> <a href="#fnref2:9" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><a href="https://raw.githubusercontent.com/apache/spark/v3.5.0/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala" target="_blank" rel="noreferrer">SQLConf.scala</a> <a href="#fnref3" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><em>High Performance Spark, 2nd Edition</em>, Karau, Polak &amp; Warren, ch. 5 <a href="#fnref4" class="footnote-backref">↩︎</a> <a href="#fnref4:1" class="footnote-backref">↩︎</a> <a href="#fnref4:2" class="footnote-backref">↩︎</a> <a href="#fnref4:3" class="footnote-backref">↩︎</a> <a href="#fnref4:4" class="footnote-backref">↩︎</a> <a href="#fnref4:5" class="footnote-backref">↩︎</a> <a href="#fnref4:6" class="footnote-backref">↩︎</a></p></li><li id="fn5" class="footnote-item"><p><a href="https://luminousmen.com/post/spark-partitions" target="_blank" rel="noreferrer">Spark Partitions</a> <a href="#fnref5" class="footnote-backref">↩︎</a> <a href="#fnref5:1" class="footnote-backref">↩︎</a> <a href="#fnref5:2" class="footnote-backref">↩︎</a></p></li><li id="fn6" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/sql-performance-tuning.html" target="_blank" rel="noreferrer">Performance Tuning: Spark SQL, DataFrames and Datasets Guide</a> <a href="#fnref6" class="footnote-backref">↩︎</a> <a href="#fnref6:1" class="footnote-backref">↩︎</a> <a href="#fnref6:2" class="footnote-backref">↩︎</a> <a href="#fnref6:3" class="footnote-backref">↩︎</a> <a href="#fnref6:4" class="footnote-backref">↩︎</a> <a href="#fnref6:5" class="footnote-backref">↩︎</a> <a href="#fnref6:6" class="footnote-backref">↩︎</a> <a href="#fnref6:7" class="footnote-backref">↩︎</a> <a href="#fnref6:8" class="footnote-backref">↩︎</a></p></li><li id="fn7" class="footnote-item"><p><a href="https://issues.apache.org/jira/browse/SPARK-29544" target="_blank" rel="noreferrer">SPARK-29544: Optimize Skewed Join at Runtime</a> <a href="#fnref7" class="footnote-backref">↩︎</a> <a href="#fnref7:1" class="footnote-backref">↩︎</a></p></li><li id="fn8" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration: Spark</a> <a href="#fnref8" class="footnote-backref">↩︎</a> <a href="#fnref8:1" class="footnote-backref">↩︎</a> <a href="#fnref8:2" class="footnote-backref">↩︎</a></p></li><li id="fn9" class="footnote-item"><p><a href="https://www.waitingforcode.com/apache-spark-sql/whats-new-apache-spark-3-dynamic-partition-pruning/read" target="_blank" rel="noreferrer">What&#39;s New in Apache Spark 3: Dynamic Partition Pruning</a> <a href="#fnref9" class="footnote-backref">↩︎</a> <a href="#fnref9:1" class="footnote-backref">↩︎</a></p></li><li id="fn10" class="footnote-item"><p><a href="https://luminousmen.com/post/explaining-the-mechanics-of-spark-caching" target="_blank" rel="noreferrer">Explaining the Mechanics of Spark Caching</a> <a href="#fnref10" class="footnote-backref">↩︎</a> <a href="#fnref10:1" class="footnote-backref">↩︎</a></p></li></ol></section></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/tuning-reference/pyspark.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>PySpark Specifics</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="/docs/tuning-reference/cluster-config.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>Cluster Tuning</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
26
- <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"CWpj01WU\",\"contributor-guide_architecture_detector-contract.md\":\"CgzUsQ6W\",\"contributor-guide_architecture_drill-down.md\":\"BtPdlM7r\",\"contributor-guide_architecture_impact-estimation.md\":\"CooslVJt\",\"contributor-guide_architecture_index.md\":\"3TO9ic6w\",\"contributor-guide_architecture_overview.md\":\"CehiRmGn\",\"contributor-guide_architecture_state-and-history.md\":\"C-xxn0q7\",\"contributor-guide_architecture_widget-rendering.md\":\"R27gQrgY\",\"contributor-guide_architecture_worker-protocol.md\":\"IbnfNrV3\",\"contributor-guide_contributing.md\":\"CvRsdr6J\",\"contributor-guide_development-setup.md\":\"DvAN_9mK\",\"contributor-guide_testing.md\":\"6rIKqSyY\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"Df1YMIHu\",\"tuning-reference_aqe.md\":\"BIsCtLzm\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"CEstB3Ia\",\"tuning-reference_bottleneck-cold-start.md\":\"CEuy-72y\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"CIohQDfn\",\"tuning-reference_bottleneck-failures.md\":\"4z5BXGJ2\",\"tuning-reference_bottleneck-gc.md\":\"DSxzZRK7\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"BaJl__1W\",\"tuning-reference_bottleneck-memory-utilization.md\":\"DbP-SJZc\",\"tuning-reference_bottleneck-retry-waste.md\":\"D5JMjVOt\",\"tuning-reference_bottleneck-shuffle.md\":\"CM-nTmIH\",\"tuning-reference_bottleneck-skew.md\":\"BdUwiDhn\",\"tuning-reference_bottleneck-slow-host.md\":\"BlIo6UDW\",\"tuning-reference_bottleneck-small-files.md\":\"B8kloyx8\",\"tuning-reference_bottleneck-spill.md\":\"PNH7mITt\",\"tuning-reference_bottleneck-straggler.md\":\"DY36fHN5\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"QTV7O8kU\",\"tuning-reference_bottleneck-utilization.md\":\"DTiueZC3\",\"tuning-reference_caching.md\":\"B7aQ8asB\",\"tuning-reference_cluster-config.md\":\"ZVmDGsQ3\",\"tuning-reference_config.md\":\"UvveiWG3\",\"tuning-reference_data-formats.md\":\"bjCAWH3N\",\"tuning-reference_index.md\":\"BQ_NooMV\",\"tuning-reference_intro.md\":\"CobD-lGB\",\"tuning-reference_joins.md\":\"BtKs_CuW\",\"tuning-reference_memory-model.md\":\"DhT-n4y3\",\"tuning-reference_metrics.md\":\"mLOh7Apj\",\"tuning-reference_partitioning.md\":\"q0zKF_8X\",\"tuning-reference_pyspark.md\":\"DDCfvN9t\",\"tuning-reference_shuffle.md\":\"BZZ7R4Ix\",\"tuning-reference_spark-architecture.md\":\"Dwzm5avO\",\"tuning-reference_table-formats.md\":\"D6wj-2dX\",\"user-guide_alternative-log-retrieval.md\":\"B4tPGIal\",\"user-guide_getting-started.md\":\"BJvwLEIM\",\"user-guide_mcp-tools.md\":\"Vi3RoflJ\",\"user-guide_run-comparison.md\":\"CQc1aoU8\",\"user-guide_understanding-findings.md\":\"DL1UDhvR\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Alternative log retrieval\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
26
+ <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"9ToZXcTk\",\"contributor-guide_architecture_detector-contract.md\":\"aQQfP7pj\",\"contributor-guide_architecture_drill-down.md\":\"Bawx6NgP\",\"contributor-guide_architecture_impact-estimation.md\":\"sf9DU2bA\",\"contributor-guide_architecture_index.md\":\"3TO9ic6w\",\"contributor-guide_architecture_overview.md\":\"BXQdgG5u\",\"contributor-guide_architecture_state-and-history.md\":\"C-xxn0q7\",\"contributor-guide_architecture_widget-rendering.md\":\"DYwFeyjV\",\"contributor-guide_architecture_worker-protocol.md\":\"P7d1vtEJ\",\"contributor-guide_contributing.md\":\"CvRsdr6J\",\"contributor-guide_development-setup.md\":\"DvMV1xhQ\",\"contributor-guide_testing.md\":\"BU6vyA-9\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"Df1YMIHu\",\"tuning-reference_aqe.md\":\"BIsCtLzm\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"CEstB3Ia\",\"tuning-reference_bottleneck-cold-start.md\":\"CEuy-72y\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"CIohQDfn\",\"tuning-reference_bottleneck-failures.md\":\"CAsFf3RR\",\"tuning-reference_bottleneck-gc.md\":\"B4T3kJYV\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"BaJl__1W\",\"tuning-reference_bottleneck-memory-utilization.md\":\"DbP-SJZc\",\"tuning-reference_bottleneck-retry-waste.md\":\"D5JMjVOt\",\"tuning-reference_bottleneck-shuffle.md\":\"DDMQVOW6\",\"tuning-reference_bottleneck-skew.md\":\"B0MzhWcX\",\"tuning-reference_bottleneck-slow-host.md\":\"BlIo6UDW\",\"tuning-reference_bottleneck-small-files.md\":\"DHGMTmPs\",\"tuning-reference_bottleneck-spill.md\":\"PNH7mITt\",\"tuning-reference_bottleneck-straggler.md\":\"DIbXdAcQ\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"CB05v3zh\",\"tuning-reference_bottleneck-utilization.md\":\"DqC_iGkw\",\"tuning-reference_caching.md\":\"B7aQ8asB\",\"tuning-reference_cluster-config.md\":\"CD6vzYtD\",\"tuning-reference_config.md\":\"UvveiWG3\",\"tuning-reference_data-formats.md\":\"bjCAWH3N\",\"tuning-reference_index.md\":\"BQ_NooMV\",\"tuning-reference_intro.md\":\"CobD-lGB\",\"tuning-reference_joins.md\":\"BtKs_CuW\",\"tuning-reference_memory-model.md\":\"CWhvgevP\",\"tuning-reference_metrics.md\":\"mLOh7Apj\",\"tuning-reference_partitioning.md\":\"q0zKF_8X\",\"tuning-reference_pyspark.md\":\"DDCfvN9t\",\"tuning-reference_shuffle.md\":\"BZZ7R4Ix\",\"tuning-reference_spark-architecture.md\":\"Dwzm5avO\",\"tuning-reference_table-formats.md\":\"D6wj-2dX\",\"user-guide_alternative-log-retrieval.md\":\"B4tPGIal\",\"user-guide_getting-started.md\":\"BJvwLEIM\",\"user-guide_mcp-tools.md\":\"DGvolGzt\",\"user-guide_run-comparison.md\":\"CQc1aoU8\",\"user-guide_understanding-findings.md\":\"B29f0ucS\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Alternative log retrieval\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
27
27
 
28
28
  </body>
29
29
  </html>
package/public/docs/tuning-reference/bottleneck-broadcast-sizing.html CHANGED
@@ -9,9 +9,9 @@
9
9
  <link rel="preload stylesheet" href="/docs/assets/style.DSixAiZE.css" as="style">
10
10
  <link rel="preload stylesheet" href="/docs/vp-icons.css" as="style">
11
11
 
12
- <script type="module" src="/docs/assets/app.APoq_UBs.js"></script>
12
+ <script type="module" src="/docs/assets/app.DsFQcynM.js"></script>
13
13
  <link rel="preload" href="/docs/assets/inter-roman-latin.Di8DUHzh.woff2" as="font" type="font/woff2" crossorigin="">
14
- <link rel="modulepreload" href="/docs/assets/chunks/theme.PavXO_vz.js">
14
+ <link rel="modulepreload" href="/docs/assets/chunks/theme.Ca4dR9dL.js">
15
15
  <link rel="modulepreload" href="/docs/assets/chunks/framework.DSg0KOwT.js">
16
16
  <link rel="modulepreload" href="/docs/assets/tuning-reference_bottleneck-broadcast-sizing.md.CEstB3Ia.lean.js">
17
17
  <link rel="icon" type="image/svg+xml" href="/docs/favicon.svg">
@@ -23,7 +23,7 @@
23
23
  </head>
24
24
  <body>
25
25
  <div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Getting Started</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/intro.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Introduction</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Optimization Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/spark-architecture.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Execution Model</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/memory-model.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Management</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/partitioning.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Partitioning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/joins.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Join Optimization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/data-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Data Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/table-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Table Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/caching.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Caching & Persistence</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/pyspark.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>PySpark Specifics</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/aqe.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Adaptive Query Execution</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/cluster-config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cluster Tuning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/anti-patterns.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Anti-Patterns</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Detector Catalog</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-skew.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Skew</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle I/O</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-spill.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory / Disk Spill</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-gc.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>GC Pressure</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-cold-start.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cold Start</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Executor Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-slow-host.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Slow Host</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-failures.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Failures</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-straggler.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Stragglers</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-retry-waste.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Retry Waste</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-tiny-tasks.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Tiny Tasks</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-job-failure-rate.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Job Failure Rate</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-memory-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Duplicate Plan Subtree</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-small-files.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Small Files</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-broadcast-sizing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Broadcast Sizing</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Reference</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/metrics.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Metrics Glossary</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Config Quick-Reference</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_tuning-reference_bottleneck-broadcast-sizing" data-v-39a288b8><div><h1 id="bottleneck-broadcast-sizing" tabindex="-1">Broadcast Sizing <a class="header-anchor" href="#bottleneck-broadcast-sizing" aria-label="Permalink to &quot;Broadcast Sizing {#bottleneck-broadcast-sizing}&quot;">​</a></h1><p><span class="tag">BROADCASTSIZING</span></p><h2 id="what-it-is" tabindex="-1">What it is <a class="header-anchor" href="#what-it-is" aria-label="Permalink to &quot;What it is&quot;">​</a></h2><p>A join has two ways to move data. The <a href="./joins.html">shuffle join</a> forces all-to-all traffic across the cluster; the broadcast join ships one side out to every executor and skips the <a href="./shuffle.html">shuffle</a> entirely. <code>spark.sql.autoBroadcastJoinThreshold</code> is the size gate that picks between them: it sets the largest table, in bytes, that Spark will broadcast when planning a join, and it defaults to 10 MB.<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup><sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup> Set it to <code>-1</code> and automatic broadcasting is off, so Spark always shuffles.<sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup></p><img class="light-only" src="/docs/assets/broadcast-vs-shuffle.Db4WY1XK.svg" alt="How the small side estimate against the broadcast threshold picks between broadcasting a copy to every executor for a local join and an all-to-all shuffle join, with the broadcast path&#39;s driver collect and executor memory risks noted."><img class="dark-only" src="/docs/assets/broadcast-vs-shuffle.dark.C7Bxs0mG.svg" alt="How the small side estimate against the broadcast threshold picks between broadcasting a copy to every executor for a local join and an all-to-all shuffle join, with the broadcast path&#39;s driver collect and executor memory risks noted."><p>Given a join&#39;s estimated table sizes and that gate, there are two ways the choice can go wrong:</p><ul><li><strong>underBroadcast</strong>: one side is small enough to broadcast but is being shuffled anyway. The join pays for an all-to-all exchange it could have avoided.</li><li><strong>overBroadcast</strong>: a side that is too large is being broadcast, which risks running the driver or the executors out of memory.</li></ul><h2 id="how-it-s-detected" tabindex="-1">How it&#39;s detected <a class="header-anchor" href="#how-it-s-detected" aria-label="Permalink to &quot;How it&#39;s detected&quot;">​</a></h2><p>Each join carries an estimated size for the side in question, which is compared against the broadcast threshold:</p><table tabindex="0"><thead><tr><th>Finding</th><th>Condition</th><th>What it means</th></tr></thead><tbody><tr><td><code>underBroadcast</code></td><td>small side is being shuffled instead of broadcast</td><td>a cheap broadcast join was left on the table</td></tr><tr><td><code>overBroadcast</code></td><td>an oversized side is being broadcast</td><td>the broadcast is a memory hazard</td></tr></tbody></table><h2 id="why-it-matters" tabindex="-1">Why it matters <a class="header-anchor" href="#why-it-matters" aria-label="Permalink to &quot;Why it matters&quot;">​</a></h2><p>Broadcasting the small side sends a full copy of that table to every executor. It pays off only when network bandwidth and executor memory are not the bottleneck; there, it is far cheaper than the shuffle it replaces.<sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup> That is the cost an <code>underBroadcast</code> finding is pointing at: a small dimension going through an all-to-all exchange when a copy on each executor would have run fast.</p><p><code>overBroadcast</code> is the opposite mistake, and it fails hard. Pushing a table that is too large through the broadcast path breaks in two ways: too-large task result errors and out-of-memory failures, both because the oversized table has to be collected on the driver and shipped whole to every executor.<sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup> The size gate exists for exactly this reason: a copy on every executor is harmless for a small table and a memory hazard for a large one.</p><h2 id="how-to-fix-it" tabindex="-1">How to fix it <a class="header-anchor" href="#how-to-fix-it" aria-label="Permalink to &quot;How to fix it&quot;">​</a></h2><p>For <code>underBroadcast</code>, let the small side broadcast. If its estimated size sits above the 10 MB default but still fits comfortably in executor memory, raise <code>spark.sql.autoBroadcastJoinThreshold</code> to cover it, or force the strategy with <code>broadcast(df)</code> on the small side.<sup class="footnote-ref"><a href="#fn1" id="fnref1:2">[1:2]</a></sup><sup class="footnote-ref"><a href="#fn5" id="fnref5">[5]</a></sup></p><p>For <code>overBroadcast</code>, keep the large side out of the broadcast path. Lower the threshold so the oversized table no longer qualifies, or drop the explicit <code>broadcast()</code> hint if one is forcing it, and let Spark fall back to a shuffle join.</p><h2 id="confidence" tabindex="-1">Confidence <a class="header-anchor" href="#confidence" aria-label="Permalink to &quot;Confidence&quot;">​</a></h2><p>Validated for both branches. <code>underBroadcast</code> and <code>overBroadcast</code> rest on the same well-documented mechanism: the broadcast threshold and its two failure modes are standard Spark behavior, so neither branch is experimental.</p><h2 id="limitations-false-positive-risk" tabindex="-1">Limitations / false-positive risk <a class="header-anchor" href="#limitations-false-positive-risk" aria-label="Permalink to &quot;Limitations / false-positive risk&quot;">​</a></h2><p>The comparison is only as good as the size estimate behind it. Table sizes come from statistics that may be stale or missing, so a join that looks mis-sized can be measuring the wrong number. A large broadcast can also be deliberate: an operator who has sized executor memory for it may broadcast a table well past the default on purpose.</p><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration (Spark)</a> <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a> <a href="#fnref1:2" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/sql-performance-tuning.html" target="_blank" rel="noreferrer">Performance Tuning (Spark SQL, DataFrames and Datasets Guide)</a> <a href="#fnref2" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><em>Learning Spark, 2nd Edition</em>, Damji, Wenig, Das, Lee, ch. 7 <a href="#fnref3" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><em>High Performance Spark, 2nd Edition</em>, Karau, Polak &amp; Warren, ch. 6 <a href="#fnref4" class="footnote-backref">↩︎</a></p></li><li id="fn5" class="footnote-item"><p><a href="https://luminousmen.com/post/the-apache-spark-optimization-checklist" target="_blank" rel="noreferrer">The Apache Spark Optimization Checklist</a> <a href="#fnref5" class="footnote-backref">↩︎</a></p></li></ol></section></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/tuning-reference/bottleneck-small-files.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>Small Files</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="/docs/tuning-reference/metrics.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>Metrics Glossary</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
26
- <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"CWpj01WU\",\"contributor-guide_architecture_detector-contract.md\":\"CgzUsQ6W\",\"contributor-guide_architecture_drill-down.md\":\"BtPdlM7r\",\"contributor-guide_architecture_impact-estimation.md\":\"CooslVJt\",\"contributor-guide_architecture_index.md\":\"3TO9ic6w\",\"contributor-guide_architecture_overview.md\":\"CehiRmGn\",\"contributor-guide_architecture_state-and-history.md\":\"C-xxn0q7\",\"contributor-guide_architecture_widget-rendering.md\":\"R27gQrgY\",\"contributor-guide_architecture_worker-protocol.md\":\"IbnfNrV3\",\"contributor-guide_contributing.md\":\"CvRsdr6J\",\"contributor-guide_development-setup.md\":\"DvAN_9mK\",\"contributor-guide_testing.md\":\"6rIKqSyY\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"Df1YMIHu\",\"tuning-reference_aqe.md\":\"BIsCtLzm\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"CEstB3Ia\",\"tuning-reference_bottleneck-cold-start.md\":\"CEuy-72y\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"CIohQDfn\",\"tuning-reference_bottleneck-failures.md\":\"4z5BXGJ2\",\"tuning-reference_bottleneck-gc.md\":\"DSxzZRK7\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"BaJl__1W\",\"tuning-reference_bottleneck-memory-utilization.md\":\"DbP-SJZc\",\"tuning-reference_bottleneck-retry-waste.md\":\"D5JMjVOt\",\"tuning-reference_bottleneck-shuffle.md\":\"CM-nTmIH\",\"tuning-reference_bottleneck-skew.md\":\"BdUwiDhn\",\"tuning-reference_bottleneck-slow-host.md\":\"BlIo6UDW\",\"tuning-reference_bottleneck-small-files.md\":\"B8kloyx8\",\"tuning-reference_bottleneck-spill.md\":\"PNH7mITt\",\"tuning-reference_bottleneck-straggler.md\":\"DY36fHN5\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"QTV7O8kU\",\"tuning-reference_bottleneck-utilization.md\":\"DTiueZC3\",\"tuning-reference_caching.md\":\"B7aQ8asB\",\"tuning-reference_cluster-config.md\":\"ZVmDGsQ3\",\"tuning-reference_config.md\":\"UvveiWG3\",\"tuning-reference_data-formats.md\":\"bjCAWH3N\",\"tuning-reference_index.md\":\"BQ_NooMV\",\"tuning-reference_intro.md\":\"CobD-lGB\",\"tuning-reference_joins.md\":\"BtKs_CuW\",\"tuning-reference_memory-model.md\":\"DhT-n4y3\",\"tuning-reference_metrics.md\":\"mLOh7Apj\",\"tuning-reference_partitioning.md\":\"q0zKF_8X\",\"tuning-reference_pyspark.md\":\"DDCfvN9t\",\"tuning-reference_shuffle.md\":\"BZZ7R4Ix\",\"tuning-reference_spark-architecture.md\":\"Dwzm5avO\",\"tuning-reference_table-formats.md\":\"D6wj-2dX\",\"user-guide_alternative-log-retrieval.md\":\"B4tPGIal\",\"user-guide_getting-started.md\":\"BJvwLEIM\",\"user-guide_mcp-tools.md\":\"Vi3RoflJ\",\"user-guide_run-comparison.md\":\"CQc1aoU8\",\"user-guide_understanding-findings.md\":\"DL1UDhvR\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Alternative log retrieval\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
26
+ <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"9ToZXcTk\",\"contributor-guide_architecture_detector-contract.md\":\"aQQfP7pj\",\"contributor-guide_architecture_drill-down.md\":\"Bawx6NgP\",\"contributor-guide_architecture_impact-estimation.md\":\"sf9DU2bA\",\"contributor-guide_architecture_index.md\":\"3TO9ic6w\",\"contributor-guide_architecture_overview.md\":\"BXQdgG5u\",\"contributor-guide_architecture_state-and-history.md\":\"C-xxn0q7\",\"contributor-guide_architecture_widget-rendering.md\":\"DYwFeyjV\",\"contributor-guide_architecture_worker-protocol.md\":\"P7d1vtEJ\",\"contributor-guide_contributing.md\":\"CvRsdr6J\",\"contributor-guide_development-setup.md\":\"DvMV1xhQ\",\"contributor-guide_testing.md\":\"BU6vyA-9\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"Df1YMIHu\",\"tuning-reference_aqe.md\":\"BIsCtLzm\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"CEstB3Ia\",\"tuning-reference_bottleneck-cold-start.md\":\"CEuy-72y\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"CIohQDfn\",\"tuning-reference_bottleneck-failures.md\":\"CAsFf3RR\",\"tuning-reference_bottleneck-gc.md\":\"B4T3kJYV\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"BaJl__1W\",\"tuning-reference_bottleneck-memory-utilization.md\":\"DbP-SJZc\",\"tuning-reference_bottleneck-retry-waste.md\":\"D5JMjVOt\",\"tuning-reference_bottleneck-shuffle.md\":\"DDMQVOW6\",\"tuning-reference_bottleneck-skew.md\":\"B0MzhWcX\",\"tuning-reference_bottleneck-slow-host.md\":\"BlIo6UDW\",\"tuning-reference_bottleneck-small-files.md\":\"DHGMTmPs\",\"tuning-reference_bottleneck-spill.md\":\"PNH7mITt\",\"tuning-reference_bottleneck-straggler.md\":\"DIbXdAcQ\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"CB05v3zh\",\"tuning-reference_bottleneck-utilization.md\":\"DqC_iGkw\",\"tuning-reference_caching.md\":\"B7aQ8asB\",\"tuning-reference_cluster-config.md\":\"CD6vzYtD\",\"tuning-reference_config.md\":\"UvveiWG3\",\"tuning-reference_data-formats.md\":\"bjCAWH3N\",\"tuning-reference_index.md\":\"BQ_NooMV\",\"tuning-reference_intro.md\":\"CobD-lGB\",\"tuning-reference_joins.md\":\"BtKs_CuW\",\"tuning-reference_memory-model.md\":\"CWhvgevP\",\"tuning-reference_metrics.md\":\"mLOh7Apj\",\"tuning-reference_partitioning.md\":\"q0zKF_8X\",\"tuning-reference_pyspark.md\":\"DDCfvN9t\",\"tuning-reference_shuffle.md\":\"BZZ7R4Ix\",\"tuning-reference_spark-architecture.md\":\"Dwzm5avO\",\"tuning-reference_table-formats.md\":\"D6wj-2dX\",\"user-guide_alternative-log-retrieval.md\":\"B4tPGIal\",\"user-guide_getting-started.md\":\"BJvwLEIM\",\"user-guide_mcp-tools.md\":\"DGvolGzt\",\"user-guide_run-comparison.md\":\"CQc1aoU8\",\"user-guide_understanding-findings.md\":\"B29f0ucS\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Alternative log retrieval\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
27
27
 
28
28
  </body>
29
29
  </html>