sparkforensics-server 0.4.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (257) hide show
  1. package/index.js +3 -0
  2. package/package.json +1 -1
  3. package/public/assets/AutoscalingChurn-9Zo6IMGS.js +1 -0
  4. package/public/assets/CacheUtilization-MBDfMsKC.js +1 -0
  5. package/public/assets/CachingOpportunity-Cfw3wAFa.js +1 -0
  6. package/public/assets/ColdStart-BtS12b6_.js +1 -0
  7. package/public/assets/ConfigAudit-DNPV6EQg.js +1 -0
  8. package/public/assets/CoreUsageArea-BObEeenE.js +1 -0
  9. package/public/assets/Dashboard-CC_BXrNy.js +2 -0
  10. package/public/assets/DisclosureContext-Ds0hcZFn.js +1 -0
  11. package/public/assets/DuplicatePlanSubtree-vN3zqRsi.js +1 -0
  12. package/public/assets/ExecutorUtilization-CZObSLIj.js +1 -0
  13. package/public/assets/GcPressure-ByJlLOua.js +1 -0
  14. package/public/assets/IncompleteRun-MzqV4W8y.js +1 -0
  15. package/public/assets/JobFailures-LbMvAwAb.js +1 -0
  16. package/public/assets/MemoryUtilization-DH4jYvJK.js +1 -0
  17. package/public/assets/OverBroadcast-euk6HKg4.js +1 -0
  18. package/public/assets/PartitionHint-DThaMMRT.js +1 -0
  19. package/public/assets/PartitionSizing-yIN-iZzx.js +1 -0
  20. package/public/assets/{PlanExplorer-DdmPkKO1.js → PlanExplorer-Dd9WPK4S.js} +1 -1
  21. package/public/assets/PlanGraphRoute-NV5mCXRm.js +1 -0
  22. package/public/assets/RetryWaste-CXH4D-xH.js +1 -0
  23. package/public/assets/ShuffleIO-CW7U68ph.js +1 -0
  24. package/public/assets/Skew-PL79TppX.js +1 -0
  25. package/public/assets/SlowHost-GBO73hCi.js +1 -0
  26. package/public/assets/SmallFiles-B5cUjEav.js +1 -0
  27. package/public/assets/SpeculationWaste-mxi3jgb0.js +1 -0
  28. package/public/assets/Spill-CH1bNpxL.js +1 -0
  29. package/public/assets/StageFailed-YvxC11ii.js +1 -0
  30. package/public/assets/StageFindingGroup-B1A0nf-e.js +1 -0
  31. package/public/assets/{StageHeader-BSV8ALhT.js → StageHeader-BMNmUy-s.js} +1 -1
  32. package/public/assets/StageShape-DO3mQ6NS.js +1 -0
  33. package/public/assets/StageSlowness-SwRxfAEx.js +1 -0
  34. package/public/assets/Straggler-DHLTiBY-.js +1 -0
  35. package/public/assets/TaskFailures-Bj0QxiOc.js +1 -0
  36. package/public/assets/{TinyTask-CIPRxWfT.js → TinyTask-COUQQ5bQ.js} +1 -1
  37. package/public/assets/UnderBroadcast-RAUbQ6vw.js +1 -0
  38. package/public/assets/{finding-anchor-DF1cSHO1.js → finding-anchor-Dl8mIHiU.js} +1 -1
  39. package/public/assets/{impact-sort-7z0ZcBZf.js → impact-sort-Eihua9dY.js} +1 -1
  40. package/public/assets/index-DaC8A2FS.js +13 -0
  41. package/public/assets/{parser-worker-Q5KCPdKT.js → parser-worker-CfDvPkwF.js} +5 -5
  42. package/public/assets/{select-Cy5pmbrh.js → select-DkjFHlXR.js} +1 -1
  43. package/public/assets/{usePagedRows-BjPOpUU8.js → usePagedRows-Bo-aSztx.js} +1 -1
  44. package/public/assets/useRecentFiles-Bb4-ukVz.js +11 -0
  45. package/public/assets/{useSortMode-CU0vzThu.js → useSortMode-lDedD5a3.js} +1 -1
  46. package/public/docs/404.html +4 -3
  47. package/public/docs/assets/{app.C6w47a9W.js → app.yjczoibf.js} +1 -1
  48. package/public/docs/assets/chunks/@localSearchIndexroot.r9imX8Aq.js +1 -0
  49. package/public/docs/assets/chunks/{VPLocalSearchBox.BcTNvT55.js → VPLocalSearchBox.DOBbPfix.js} +1 -1
  50. package/public/docs/assets/chunks/{theme.D-mE6j02.js → theme.DFcAl8Ws.js} +2 -2
  51. package/public/docs/assets/contributor-guide_architecture_board-widgets.md.D0Kkk_r5.js +1 -0
  52. package/public/docs/assets/contributor-guide_architecture_board-widgets.md.D0Kkk_r5.lean.js +1 -0
  53. package/public/docs/assets/contributor-guide_architecture_detector-contract.md.caA46z-u.js +1 -0
  54. package/public/docs/assets/contributor-guide_architecture_drill-down.md.CdtlKHl-.js +1 -0
  55. package/public/docs/assets/{contributor-guide_architecture_drill-down.md.CFNCE8JB.lean.js → contributor-guide_architecture_drill-down.md.CdtlKHl-.lean.js} +1 -1
  56. package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.BNp14LtL.js +1 -0
  57. package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.BNp14LtL.lean.js +1 -0
  58. package/public/docs/assets/contributor-guide_architecture_index.md.B9yZieYL.js +1 -0
  59. package/public/docs/assets/{contributor-guide_architecture_index.md.3TO9ic6w.lean.js → contributor-guide_architecture_index.md.B9yZieYL.lean.js} +1 -1
  60. package/public/docs/assets/contributor-guide_architecture_overview.md.DBGo-bwn.js +1 -0
  61. package/public/docs/assets/contributor-guide_architecture_overview.md.DBGo-bwn.lean.js +1 -0
  62. package/public/docs/assets/contributor-guide_architecture_state-and-history.md.w4lwybdh.js +1 -0
  63. package/public/docs/assets/{contributor-guide_architecture_state-and-history.md.DOTVHjTV.lean.js → contributor-guide_architecture_state-and-history.md.w4lwybdh.lean.js} +1 -1
  64. package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.CcvqxnZI.js +1 -0
  65. package/public/docs/assets/{contributor-guide_architecture_widget-rendering.md.BmpARZve.lean.js → contributor-guide_architecture_widget-rendering.md.CcvqxnZI.lean.js} +1 -1
  66. package/public/docs/assets/contributor-guide_architecture_worker-protocol.md.CKeFttKj.js +6 -0
  67. package/public/docs/assets/{contributor-guide_architecture_worker-protocol.md.21pTYhFF.lean.js → contributor-guide_architecture_worker-protocol.md.CKeFttKj.lean.js} +1 -1
  68. package/public/docs/assets/contributor-guide_contributing.md.CQMvGwFt.js +1 -0
  69. package/public/docs/assets/{contributor-guide_contributing.md.B2E9ni79.lean.js → contributor-guide_contributing.md.CQMvGwFt.lean.js} +1 -1
  70. package/public/docs/assets/contributor-guide_development-setup.md.D9bu4j_c.js +15 -0
  71. package/public/docs/assets/contributor-guide_development-setup.md.D9bu4j_c.lean.js +1 -0
  72. package/public/docs/assets/contributor-guide_testing.md.By6XGR9a.js +1 -0
  73. package/public/docs/assets/{contributor-guide_testing.md.DTwhpoN2.lean.js → contributor-guide_testing.md.By6XGR9a.lean.js} +1 -1
  74. package/public/docs/assets/{style.DSixAiZE.css → style.tmDhD3c8.css} +1 -1
  75. package/public/docs/assets/user-guide_alternative-log-retrieval.md.z-8npxNk.js +7 -0
  76. package/public/docs/assets/user-guide_alternative-log-retrieval.md.z-8npxNk.lean.js +1 -0
  77. package/public/docs/assets/user-guide_getting-started.md.BJW5oVhe.js +24 -0
  78. package/public/docs/assets/user-guide_getting-started.md.BJW5oVhe.lean.js +1 -0
  79. package/public/docs/assets/{user-guide_mcp-tools.md.dQwIvdTF.js → user-guide_mcp-tools.md.DcLiOa9t.js} +40 -23
  80. package/public/docs/assets/{user-guide_mcp-tools.md.dQwIvdTF.lean.js → user-guide_mcp-tools.md.DcLiOa9t.lean.js} +1 -1
  81. package/public/docs/assets/user-guide_run-comparison.md.qin0a4IX.js +1 -0
  82. package/public/docs/assets/{user-guide_run-comparison.md.ChLI3MK0.lean.js → user-guide_run-comparison.md.qin0a4IX.lean.js} +1 -1
  83. package/public/docs/assets/user-guide_understanding-findings.md.DzReyCXb.js +1 -0
  84. package/public/docs/contributor-guide/architecture/board-widgets.html +7 -6
  85. package/public/docs/contributor-guide/architecture/detector-contract.html +7 -6
  86. package/public/docs/contributor-guide/architecture/drill-down.html +7 -6
  87. package/public/docs/contributor-guide/architecture/impact-estimation.html +7 -6
  88. package/public/docs/contributor-guide/architecture/index.html +7 -6
  89. package/public/docs/contributor-guide/architecture/overview.html +7 -6
  90. package/public/docs/contributor-guide/architecture/state-and-history.html +7 -6
  91. package/public/docs/contributor-guide/architecture/widget-rendering.html +7 -6
  92. package/public/docs/contributor-guide/architecture/worker-protocol.html +10 -9
  93. package/public/docs/contributor-guide/contributing.html +7 -6
  94. package/public/docs/contributor-guide/development-setup.html +11 -10
  95. package/public/docs/contributor-guide/testing.html +7 -6
  96. package/public/docs/hashmap.json +1 -1
  97. package/public/docs/index.html +5 -4
  98. package/public/docs/tuning-reference/anti-patterns.html +5 -4
  99. package/public/docs/tuning-reference/aqe.html +5 -4
  100. package/public/docs/tuning-reference/bottleneck-broadcast-sizing.html +5 -4
  101. package/public/docs/tuning-reference/bottleneck-cold-start.html +5 -4
  102. package/public/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html +5 -4
  103. package/public/docs/tuning-reference/bottleneck-failures.html +5 -4
  104. package/public/docs/tuning-reference/bottleneck-gc.html +5 -4
  105. package/public/docs/tuning-reference/bottleneck-job-failure-rate.html +5 -4
  106. package/public/docs/tuning-reference/bottleneck-memory-utilization.html +5 -4
  107. package/public/docs/tuning-reference/bottleneck-retry-waste.html +5 -4
  108. package/public/docs/tuning-reference/bottleneck-shuffle.html +5 -4
  109. package/public/docs/tuning-reference/bottleneck-skew.html +5 -4
  110. package/public/docs/tuning-reference/bottleneck-slow-host.html +5 -4
  111. package/public/docs/tuning-reference/bottleneck-small-files.html +5 -4
  112. package/public/docs/tuning-reference/bottleneck-spill.html +5 -4
  113. package/public/docs/tuning-reference/bottleneck-straggler.html +5 -4
  114. package/public/docs/tuning-reference/bottleneck-tiny-tasks.html +5 -4
  115. package/public/docs/tuning-reference/bottleneck-utilization.html +5 -4
  116. package/public/docs/tuning-reference/caching.html +5 -4
  117. package/public/docs/tuning-reference/cluster-config.html +5 -4
  118. package/public/docs/tuning-reference/config.html +5 -4
  119. package/public/docs/tuning-reference/data-formats.html +5 -4
  120. package/public/docs/tuning-reference/index.html +5 -4
  121. package/public/docs/tuning-reference/intro.html +5 -4
  122. package/public/docs/tuning-reference/joins.html +5 -4
  123. package/public/docs/tuning-reference/memory-model.html +5 -4
  124. package/public/docs/tuning-reference/metrics.html +5 -4
  125. package/public/docs/tuning-reference/partitioning.html +5 -4
  126. package/public/docs/tuning-reference/pyspark.html +5 -4
  127. package/public/docs/tuning-reference/shuffle.html +5 -4
  128. package/public/docs/tuning-reference/spark-architecture.html +5 -4
  129. package/public/docs/tuning-reference/table-formats.html +5 -4
  130. package/public/docs/user-guide/alternative-log-retrieval.html +14 -7
  131. package/public/docs/user-guide/getting-started.html +27 -8
  132. package/public/docs/user-guide/mcp-tools.html +46 -28
  133. package/public/docs/user-guide/run-comparison.html +7 -6
  134. package/public/docs/user-guide/understanding-findings.html +7 -6
  135. package/public/export-template.html +44 -44
  136. package/public/index.html +2 -2
  137. package/vendor-core/allocation.js +106 -0
  138. package/vendor-core/analyzer.js +13 -13
  139. package/vendor-core/cli/budgets.js +23 -9
  140. package/vendor-core/cli/collect-run.js +11 -4
  141. package/vendor-core/cli/regression-budgets.js +83 -0
  142. package/vendor-core/comparison-verdict.js +22 -22
  143. package/vendor-core/core-source-hash.txt +1 -1
  144. package/vendor-core/detectors.js +202 -68
  145. package/vendor-core/docs-content/detection/cache.md +3 -2
  146. package/vendor-core/docs-content/detection/cfg.md +9 -8
  147. package/vendor-core/docs-content/detection/chrn.md +1 -2
  148. package/vendor-core/docs-content/detection/cold.md +4 -2
  149. package/vendor-core/docs-content/detection/fail.md +3 -2
  150. package/vendor-core/docs-content/detection/gc.md +3 -2
  151. package/vendor-core/docs-content/detection/host.md +2 -1
  152. package/vendor-core/docs-content/detection/local.md +1 -1
  153. package/vendor-core/docs-content/detection/mem.md +5 -2
  154. package/vendor-core/docs-content/detection/plan.md +2 -1
  155. package/vendor-core/docs-content/detection/sfail.md +2 -1
  156. package/vendor-core/docs-content/detection/shape.md +5 -4
  157. package/vendor-core/docs-content/detection/skew.md +3 -1
  158. package/vendor-core/docs-content/detection/slow.md +2 -2
  159. package/vendor-core/docs-content/detection/spec.md +2 -3
  160. package/vendor-core/docs-content/detection/spill.md +1 -1
  161. package/vendor-core/docs-site-config.js +1 -1
  162. package/vendor-core/effective-conf.js +107 -0
  163. package/vendor-core/efficiency-model.js +8 -6
  164. package/vendor-core/event-handlers.js +160 -47
  165. package/vendor-core/event-schemas.js +2 -0
  166. package/vendor-core/evidence-report.js +18 -10
  167. package/vendor-core/finding-generic-recommendation.js +20 -1
  168. package/vendor-core/finding-names.js +7 -0
  169. package/vendor-core/finding-presentation.js +61 -26
  170. package/vendor-core/finding-tag-help.js +1 -1
  171. package/vendor-core/finding-types.js +12 -0
  172. package/vendor-core/format-utils.js +4 -3
  173. package/vendor-core/impact-estimator.js +20 -2
  174. package/vendor-core/impact-format.js +14 -13
  175. package/vendor-core/impact-model.js +27 -5
  176. package/vendor-core/ingest.js +4 -2
  177. package/vendor-core/list-runs.js +5 -2
  178. package/vendor-core/mcp-tools.js +1 -1
  179. package/vendor-core/model-assembler.js +23 -1
  180. package/vendor-core/parser-worker.js +1 -1
  181. package/vendor-core/proxy.js +3 -1
  182. package/vendor-core/python-stage.js +25 -0
  183. package/vendor-core/recommendation-rollup.js +16 -9
  184. package/vendor-core/redact.js +51 -10
  185. package/vendor-core/remediation.js +20 -0
  186. package/vendor-core/run-comparison.js +43 -24
  187. package/vendor-core/run-interpretation.js +2 -1
  188. package/vendor-core/run-metrics.js +198 -0
  189. package/vendor-core/run-totals.js +24 -0
  190. package/vendor-core/run-verdict.js +3 -4
  191. package/vendor-core/scorecard-estimates.js +1 -0
  192. package/vendor-core/session-snapshot.js +7 -0
  193. package/vendor-core/shs-schemas.js +2 -2
  194. package/vendor-core/spark-memory.js +17 -0
  195. package/vendor-core/stage-plan-nodes.js +18 -0
  196. package/vendor-core/stage-quantiles.js +4 -0
  197. package/vendor-core/types.js +49 -1
  198. package/vendor-core/wasted-core-hours.js +10 -7
  199. package/vendor-core/write-targets.js +312 -0
  200. package/public/assets/AutoscalingChurn-CUSIvvab.js +0 -1
  201. package/public/assets/CacheUtilization-Ce0K6_X2.js +0 -1
  202. package/public/assets/CachingOpportunity-Cn-LmRA4.js +0 -1
  203. package/public/assets/ColdStart-BoiYzEyV.js +0 -1
  204. package/public/assets/ConfigAudit-BoJc6PAw.js +0 -1
  205. package/public/assets/CoreUsageArea-BFfXKu7K.js +0 -1
  206. package/public/assets/Dashboard-Bo_1jGqH.js +0 -2
  207. package/public/assets/DisclosureContext-LYzHroHO.js +0 -1
  208. package/public/assets/DuplicatePlanSubtree-BaAuGAoo.js +0 -1
  209. package/public/assets/ExecutorUtilization-OE6VMFkG.js +0 -1
  210. package/public/assets/GcPressure-AoL_ZNrV.js +0 -1
  211. package/public/assets/IncompleteRun-Bv6IzgsQ.js +0 -1
  212. package/public/assets/JobFailures-DF3U2z1f.js +0 -1
  213. package/public/assets/MemoryUtilization-B65OVZlq.js +0 -1
  214. package/public/assets/OverBroadcast-DXbJREDl.js +0 -1
  215. package/public/assets/PartitionSizing-DZLnjy0d.js +0 -1
  216. package/public/assets/PlanGraphRoute-C-BGmC6j.js +0 -1
  217. package/public/assets/RetryWaste-BbEPsdMw.js +0 -1
  218. package/public/assets/ShuffleIO-wZB94LS8.js +0 -1
  219. package/public/assets/Skew-SpQE7zeJ.js +0 -1
  220. package/public/assets/SlowHost-DwDBCTvr.js +0 -1
  221. package/public/assets/SmallFiles-yjY_6sT1.js +0 -1
  222. package/public/assets/SpeculationWaste-73PDmj0c.js +0 -1
  223. package/public/assets/Spill-DEw0Kd1i.js +0 -1
  224. package/public/assets/StageFailed-BEfXdEJc.js +0 -1
  225. package/public/assets/StageFindingGroup-Bzm2TY03.js +0 -1
  226. package/public/assets/StageShape-CzR4jHOt.js +0 -1
  227. package/public/assets/StageSlowness-vMPsnAaa.js +0 -1
  228. package/public/assets/Straggler-Cf__OGl5.js +0 -1
  229. package/public/assets/TaskFailures-BZ8kLZiO.js +0 -1
  230. package/public/assets/UnderBroadcast-CBm8ZPoS.js +0 -1
  231. package/public/assets/index-CEM9bz_x.js +0 -13
  232. package/public/assets/useRecentFiles-BrBl-AP1.js +0 -11
  233. package/public/docs/assets/chunks/@localSearchIndexroot.DbZy5EtM.js +0 -1
  234. package/public/docs/assets/contributor-guide_architecture_board-widgets.md.VTcYTH4d.js +0 -1
  235. package/public/docs/assets/contributor-guide_architecture_board-widgets.md.VTcYTH4d.lean.js +0 -1
  236. package/public/docs/assets/contributor-guide_architecture_detector-contract.md.CsqZHz_T.js +0 -1
  237. package/public/docs/assets/contributor-guide_architecture_drill-down.md.CFNCE8JB.js +0 -1
  238. package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.HT9IJGIl.js +0 -1
  239. package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.HT9IJGIl.lean.js +0 -1
  240. package/public/docs/assets/contributor-guide_architecture_index.md.3TO9ic6w.js +0 -1
  241. package/public/docs/assets/contributor-guide_architecture_overview.md.CTEqVeV4.js +0 -1
  242. package/public/docs/assets/contributor-guide_architecture_overview.md.CTEqVeV4.lean.js +0 -1
  243. package/public/docs/assets/contributor-guide_architecture_state-and-history.md.DOTVHjTV.js +0 -1
  244. package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.BmpARZve.js +0 -1
  245. package/public/docs/assets/contributor-guide_architecture_worker-protocol.md.21pTYhFF.js +0 -6
  246. package/public/docs/assets/contributor-guide_contributing.md.B2E9ni79.js +0 -1
  247. package/public/docs/assets/contributor-guide_development-setup.md.CjgZJwjK.js +0 -15
  248. package/public/docs/assets/contributor-guide_development-setup.md.CjgZJwjK.lean.js +0 -1
  249. package/public/docs/assets/contributor-guide_testing.md.DTwhpoN2.js +0 -1
  250. package/public/docs/assets/user-guide_alternative-log-retrieval.md.ZHPSbBkF.js +0 -1
  251. package/public/docs/assets/user-guide_alternative-log-retrieval.md.ZHPSbBkF.lean.js +0 -1
  252. package/public/docs/assets/user-guide_getting-started.md.CEa2ZX-l.js +0 -6
  253. package/public/docs/assets/user-guide_getting-started.md.CEa2ZX-l.lean.js +0 -1
  254. package/public/docs/assets/user-guide_run-comparison.md.ChLI3MK0.js +0 -1
  255. package/public/docs/assets/user-guide_understanding-findings.md.D1ccs60C.js +0 -1
  256. /package/public/docs/assets/{contributor-guide_architecture_detector-contract.md.CsqZHz_T.lean.js → contributor-guide_architecture_detector-contract.md.caA46z-u.lean.js} +0 -0
  257. /package/public/docs/assets/{user-guide_understanding-findings.md.D1ccs60C.lean.js → user-guide_understanding-findings.md.DzReyCXb.lean.js} +0 -0
package/public/docs/tuning-reference/shuffle.html CHANGED
@@ -6,14 +6,15 @@
6
6
  <title>Shuffle | Spark Tuning Reference</title>
7
7
  <meta name="description" content="Shuffle internals: SortShuffleManager mechanics, shuffle-avoidance paths (bucketing, Storage Partition Join), and shuffle-tuning configs.">
8
8
  <meta name="generator" content="VitePress v1.6.4">
9
- <link rel="preload stylesheet" href="/docs/assets/style.DSixAiZE.css" as="style">
9
+ <link rel="preload stylesheet" href="/docs/assets/style.tmDhD3c8.css" as="style">
10
10
  <link rel="preload stylesheet" href="/docs/vp-icons.css" as="style">
11
11
 
12
- <script type="module" src="/docs/assets/app.C6w47a9W.js"></script>
12
+ <script type="module" src="/docs/assets/app.yjczoibf.js"></script>
13
13
  <link rel="preload" href="/docs/assets/inter-roman-latin.Di8DUHzh.woff2" as="font" type="font/woff2" crossorigin="">
14
- <link rel="modulepreload" href="/docs/assets/chunks/theme.D-mE6j02.js">
14
+ <link rel="modulepreload" href="/docs/assets/chunks/theme.DFcAl8Ws.js">
15
15
  <link rel="modulepreload" href="/docs/assets/chunks/framework.DSg0KOwT.js">
16
16
  <link rel="modulepreload" href="/docs/assets/tuning-reference_shuffle.md.Iq2v1oHJ.lean.js">
17
+ <script>(()=>{const c=`${`[${["data","shuffle","product","bar"].join("-")}]`}, .VPNav, .VPLocalNav`,a=24,d=document.documentElement,l=t=>{const e=getComputedStyle(t);return e.display==="none"||t.getClientRects().length===0?0:e.position==="fixed"?t.getBoundingClientRect().bottom:e.position==="sticky"?(parseFloat(e.top)||0)+t.offsetHeight:0},u=()=>{try{return location.hash?document.getElementById(decodeURIComponent(location.hash.slice(1))):null}catch{return null}};let r=null;const s=()=>{const t=Array.from(document.querySelectorAll(c),l),e=Math.ceil(Math.max(0,...t))+a;if(e===r)return;const n=r;r=e,d.style.setProperty("--sf-anchor-offset",`${r}px`);const o=n===null?null:u();if(!o)return;const f=(parseFloat(getComputedStyle(o).paddingTop)||0)+2;Math.abs(o.getBoundingClientRect().top-n)<=f&&o.scrollIntoView()},i=()=>{const t=document.createElement("div");t.setAttribute("data-sf-anchor-offset",""),t.setAttribute("aria-hidden","true"),document.body.appendChild(t);const e=window.ResizeObserver?new ResizeObserver(s):null,n=()=>{e&&document.querySelectorAll(c).forEach(o=>e.observe(o)),s()};n(),new MutationObserver(n).observe(document.body,{childList:!0,subtree:!0}),window.addEventListener("resize",s),window.addEventListener("load",s)};document.readyState==="loading"?document.addEventListener("DOMContentLoaded",i,{once:!0}):i()})();</script>
17
18
  <link rel="icon" type="image/svg+xml" href="/docs/favicon.svg">
18
19
  <link rel="preconnect" href="https://fonts.googleapis.com">
19
20
  <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin="">
@@ -23,7 +24,7 @@
23
24
  </head>
24
25
  <body>
25
26
  <div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Getting Started</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/intro.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Introduction</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Optimization Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/spark-architecture.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Execution Model</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/memory-model.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Management</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/partitioning.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Partitioning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/joins.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Join Optimization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/data-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Data Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/table-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Table Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/caching.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Caching & Persistence</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/pyspark.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>PySpark Specifics</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/aqe.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Adaptive Query Execution</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/cluster-config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cluster Tuning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/anti-patterns.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Anti-Patterns</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Detector Catalog</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-skew.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Skew</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle I/O</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-spill.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory / Disk Spill</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-gc.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>GC Pressure</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-cold-start.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cold Start</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Executor Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-slow-host.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Slow Host</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-failures.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Failures</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-straggler.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Stragglers</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-retry-waste.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Retry Waste</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-tiny-tasks.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Tiny Tasks</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-job-failure-rate.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Job Failure Rate</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-memory-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Duplicate Plan Subtree</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-small-files.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Small Files</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-broadcast-sizing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Broadcast Sizing</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Reference</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/metrics.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Metrics Glossary</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Config Quick-Reference</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_tuning-reference_shuffle" data-v-39a288b8><div><h1 id="shuffle" tabindex="-1">Shuffle <a class="header-anchor" href="#shuffle" aria-label="Permalink to &quot;Shuffle {#shuffle}&quot;">​</a></h1><h2 id="how-the-shuffle-works" tabindex="-1">How the shuffle works <a class="header-anchor" href="#how-the-shuffle-works" aria-label="Permalink to &quot;How the shuffle works&quot;">​</a></h2><p>A shuffle is Spark&#39;s mechanism for exchanging, sorting, grouping, and merging data across executors whenever an operation needs rows that share a key to land on the same partition. At the DataFrame/SQL level, the classic wide transformations that trigger one are <code>groupBy()</code>, <code>join()</code>, <code>agg()</code>, <code>sortBy()</code>, and <code>reduceByKey()</code>-style aggregations. Any join implementation that isn&#39;t a <a href="./joins.html">broadcast join</a> (shuffle hash join, shuffle sort-merge join, or shuffle-and-replicated nested loop / Cartesian product join) also requires a shuffle<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup>. At the RDD layer underneath, the operations that can cause a shuffle are <a href="./partitioning.html">repartitioning</a> (<code>repartition</code>, <code>coalesce</code>), <code>&#39;ByKey</code> operations other than counting (<code>groupByKey</code>, <code>reduceByKey</code>), and join-family operations (<code>cogroup</code>, <code>join</code>)<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup>.</p><p>Spark can skip the shuffle outright in a few well-defined cases where it already knows the data layout. Storage Partition Join (SPJ) avoids the shuffle phase entirely when Spark can use the partitioning already reported by a compatible V2 data source (<code>spark.sql.sources.v2.bucketing.enabled</code>, default <code>true</code> since 3.3.0), generalizing bucket joins to functions registered in a <code>FunctionCatalog</code><sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup>. Classic Hive-style bucketing gets the same effect: once both sides of a join are bucketed and sorted the same way, the physical plan shows no <code>Exchange</code> operator at all<sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup>. <code>DataFrameWriter.partitionBy</code>, by contrast, does not trigger a shuffle by itself on write, though it can still leave you with a large number of small output files<sup class="footnote-ref"><a href="#fn5" id="fnref5">[5]</a></sup>.</p><p>Modern Spark ships a single shuffle manager, <code>SortShuffleManager</code> (3.5 source). Incoming records are sorted by their target partition id and written to one map output file per task; reducers then fetch contiguous regions of that file for their share of the map output, spilling sorted subsets to disk and merging them if the data doesn&#39;t fit in memory<sup class="footnote-ref"><a href="#fn6" id="fnref6">[6]</a></sup>. Internally it has two write paths: a &quot;serialized sorting&quot; path used when there&#39;s no map-side combine, the serializer supports relocation of serialized values (Kryo or Spark SQL&#39;s custom serializers), and the shuffle produces at most 16,777,216 output partitions; and a &quot;deserialized sorting&quot; path used for everything else<sup class="footnote-ref"><a href="#fn6" id="fnref6:1">[6:1]</a></sup>. Framed against the older hash-based approach (where each map task produces a separate output file per reduce task rather than one sorted file), sort-based shuffle pays a sorting cost but scales better once shuffle data gets large, which is why Spark and Flink both moved to it and pair it with an external shuffle service<sup class="footnote-ref"><a href="#fn7" id="fnref7">[7]</a></sup>.</p><img class="light-only" src="/docs/assets/shuffle-map-reduce.KuOEZVmg.svg" alt="Map tasks sorting records by target partition id into one output file each while reduce tasks fetch their partition blocks from every map output."><img class="dark-only" src="/docs/assets/shuffle-map-reduce.dark.BgQZnFSb.svg" alt="Map tasks sorting records by target partition id into one output file each while reduce tasks fetch their partition blocks from every map output."><p>Two pieces of supporting infrastructure sit alongside the core shuffle manager. The external shuffle service (<code>spark.shuffle.service.enabled</code>, default <code>false</code>) preserves the shuffle files written by executors so they can be safely removed, or so shuffle fetches continue even after an executor failure; enabling it requires standing up the service separately, since the flag alone isn&#39;t enough<sup class="footnote-ref"><a href="#fn8" id="fnref8">[8]</a></sup>. It&#39;s a long-running process on each cluster node, independent of any particular Spark application or its executors: once enabled, executors fetch shuffle files from this service instead of from each other, so shuffle state written by an executor keeps being served after that executor&#39;s own lifetime ends<sup class="footnote-ref"><a href="#fn9" id="fnref9">[9]</a></sup>. Push-based shuffle, designed under SPARK-30602 and based on LinkedIn&#39;s &quot;Magnet&quot; shuffle service, goes further: it changes the reduce side from a pull model (reducers requesting many small, randomly-ordered blocks from wherever each mapper wrote them) into a push-and-merge model, where mapper-generated blocks are pushed to remote Magnet shuffle services that opportunistically merge them into large per-partition chunks before the reduce stage starts<sup class="footnote-ref"><a href="#fn7" id="fnref7:1">[7:1]</a></sup>.</p><h2 id="reading-exchange-nodes" tabindex="-1">Reading Exchange nodes <a class="header-anchor" href="#reading-exchange-nodes" aria-label="Permalink to &quot;Reading Exchange nodes&quot;">​</a></h2><p>Whether an operation shuffles doesn&#39;t have to be guesswork: shuffles show up in the physical plan as <code>Exchange</code> nodes, and reading the plan is the most direct way to check. <code>groupBy()</code> on a DataFrame is a wide transformation and normally requires an <code>Exchange</code> so all rows for a key land on the same partition<sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup>; that node disappears only when Spark already knows the data is co-partitioned by the grouping key. The clearest documented case is a table bucketed and sorted on the join/group key, where the physical plan drops the <code>Exchange</code> because the data was already shuffled at write time<sup class="footnote-ref"><a href="#fn10" id="fnref10">[10]</a></sup>. SPJ produces the same absence of an <code>Exchange</code> for join queries against compatible V2 sources<sup class="footnote-ref"><a href="#fn3" id="fnref3:1">[3:1]</a></sup>.</p><p><code>DataFrame.repartition(key)</code> is worth watching for separately: it&#39;s a one-time, explicit shuffle, but if you cache the result, subsequent joins on that same key skip their own shuffle because the data is already partitioned that way<sup class="footnote-ref"><a href="#fn5" id="fnref5:1">[5:1]</a></sup>. Don&#39;t mistake <code>partitionBy</code> on write for the same thing: it changes the on-disk directory layout but is &quot;not the equivalent&quot; of an in-memory repartition, so <code>df.groupBy(&#39;key&#39;).sum()</code> over data written with <code>partitionBy</code> still requires an <code>Exchange</code><sup class="footnote-ref"><a href="#fn5" id="fnref5:2">[5:2]</a></sup>.</p><p>Deduplication is a case where there&#39;s no avoiding the shuffle regardless of how you write it: Spark SQL&#39;s <code>dropDuplicates</code> selects unique rows the same way RDD-level <code>distinct</code> does, and both &quot;can require a shuffle&quot;<sup class="footnote-ref"><a href="#fn11" id="fnref11">[11]</a></sup>. The only documented difference between them is key flexibility, not shuffle volume: unlike RDD <code>distinct</code>, DataFrame <code>dropDuplicates</code> can optionally drop rows based on only a subset of columns (e.g., <code>dropDuplicates(List(&quot;id&quot;))</code>) rather than requiring uniqueness across the whole row<sup class="footnote-ref"><a href="#fn11" id="fnref11:1">[11:1]</a></sup>.</p><h2 id="where-the-cost-accumulates" tabindex="-1">Where the cost accumulates <a class="header-anchor" href="#where-the-cost-accumulates" aria-label="Permalink to &quot;Where the cost accumulates&quot;">​</a></h2><p>Confirming a shuffle happened is the easy part; pricing it out is the harder one. Shuffle cost shows up in several places at once, and Spark&#39;s shuffle-tuning knobs largely exist to manage it. Inside <code>SortShuffleManager</code>, when there are fewer than <code>spark.shuffle.sort.bypassMergeThreshold</code> reduce partitions (default 200)<sup class="footnote-ref"><a href="#fn8" id="fnref8:1">[8:1]</a></sup> and no map-side aggregation is needed, Spark takes a bypass path: it writes <code>numPartitions</code> files directly per map task and concatenates them at the end, rather than merge-sorting spilled files. This avoids serializing and deserializing the data twice, which the normal merge-sort path would otherwise pay; the trade-off is having multiple files open at once, and more memory allocated to buffers<sup class="footnote-ref"><a href="#fn6" id="fnref6:2">[6:2]</a></sup>. That trade stays cheap only while the reduce-side partition count is small.</p><p>Reducers also pay a fixed memory cost for every map output fetched in parallel, since each needs a buffer to receive it; that&#39;s what <code>spark.reducer.maxSizeInFlight</code> (default 48m) caps<sup class="footnote-ref"><a href="#fn8" id="fnref8:2">[8:2]</a></sup>. And map tasks pay an I/O cost writing shuffle files: a larger <code>spark.shuffle.file.buffer</code> (default 32k) reduces the number of disk seeks and system calls needed to create those intermediate files<sup class="footnote-ref"><a href="#fn8" id="fnref8:3">[8:3]</a></sup>.</p><p>External shuffle infrastructure matters most once <a href="./cluster-config.html">dynamic allocation</a> is in play: without the external shuffle service, an executor decommissioned mid-shuffle (say, because of stragglers) would force its shuffle files to be recomputed from scratch once it&#39;s removed<sup class="footnote-ref"><a href="#fn9" id="fnref9:1">[9:1]</a></sup>. Push-based shuffle exists because, as shuffle data volume grows, the number of blocks a classic shuffle produces grows quadratically (mappers × reducers) while individual block sizes shrink to only tens of KB, which is inefficient for HDD-backed random reads<sup class="footnote-ref"><a href="#fn7" id="fnref7:2">[7:2]</a></sup>. Magnet&#39;s push-and-merge model improves disk I/O efficiency by converting many small random reads into large sequential reads of pre-merged chunks, and improves reducer data locality since merged output can be co-located with the reduce tasks that consume it; the push itself is decoupled from the mappers, so it doesn&#39;t add to map task runtime or fail map tasks if a push fails, and it&#39;s best-effort, so reducers can fetch a mix of merged and unmerged blocks<sup class="footnote-ref"><a href="#fn7" id="fnref7:3">[7:3]</a></sup>. LinkedIn&#39;s own production rollout, on a cluster running 30K+ Spark applications shuffling roughly 5 PB/day, reported a large reduction in shuffle read wait time and a corresponding drop in total executor runtime after enabling it on complex production flows<sup class="footnote-ref"><a href="#fn7" id="fnref7:4">[7:4]</a></sup>.</p><h2 id="tuning-or-avoiding-the-shuffle" tabindex="-1">Tuning or avoiding the shuffle <a class="header-anchor" href="#tuning-or-avoiding-the-shuffle" aria-label="Permalink to &quot;Tuning or avoiding the shuffle&quot;">​</a></h2><p>The levers split into two groups: avoiding a shuffle, and tuning the one you can&#39;t avoid. Where possible, avoid the shuffle rather than tune it: bucket and sort tables on the join/group key so the physical plan drops the <code>Exchange</code> entirely<sup class="footnote-ref"><a href="#fn10" id="fnref10:1">[10:1]</a></sup><sup class="footnote-ref"><a href="#fn4" id="fnref4:1">[4:1]</a></sup>, or lean on Storage Partition Join for compatible V2 sources<sup class="footnote-ref"><a href="#fn3" id="fnref3:2">[3:2]</a></sup>. If you need to reuse a partitioning scheme across several joins, <code>repartition(key)</code> once and cache the result rather than relying on <code>partitionBy</code>, which only affects the on-disk layout and doesn&#39;t exempt later aggregations from their own <code>Exchange</code><sup class="footnote-ref"><a href="#fn5" id="fnref5:3">[5:3]</a></sup>.</p><blockquote><p><strong>PySpark:</strong> <code>dropDuplicates(subset)</code> still shuffles like <code>distinct()</code>, but lets you dedupe on a subset of columns instead of the whole row; use <code>df.dropDuplicates([&quot;id&quot;])</code> when uniqueness only needs to hold on a few key columns<sup class="footnote-ref"><a href="#fn11" id="fnref11:2">[11:2]</a></sup>.</p></blockquote><p>For shuffles you can&#39;t avoid, the main tuning levers are:</p><ul><li><code>spark.shuffle.sort.bypassMergeThreshold</code> (default 200): raise or lower the reduce-partition-count cutoff for the bypass (hash-style) write path described above<sup class="footnote-ref"><a href="#fn8" id="fnref8:4">[8:4]</a></sup>.</li><li><code>spark.shuffle.compress</code> (default <code>true</code>): compresses map output files using whatever codec <code>spark.io.compression.codec</code> names (default <code>lz4</code>; <code>lzf</code>, <code>snappy</code>, and <code>zstd</code> are also available)<sup class="footnote-ref"><a href="#fn8" id="fnref8:5">[8:5]</a></sup>. <code>spark.shuffle.spill.compress</code> (also default <code>true</code>) separately governs compression of spilled shuffle data, using the same codec setting<sup class="footnote-ref"><a href="#fn8" id="fnref8:6">[8:6]</a></sup>.</li><li><code>spark.reducer.maxSizeInFlight</code> (default 48m): raising it lets more data move in flight at the cost of per-task memory; lowering it saves memory at the cost of more fetch rounds<sup class="footnote-ref"><a href="#fn8" id="fnref8:7">[8:7]</a></sup>.</li><li><code>spark.shuffle.file.buffer</code> (default 32k): Learning Spark&#39;s tuning guidance recommends bumping it to 1 MB for large jobs, trading some per-task memory for fewer disk seeks and syscalls during the shuffle-write phase<sup class="footnote-ref"><a href="#fn1" id="fnref1:2">[1:2]</a></sup>.</li><li><code>spark.shuffle.service.enabled</code> (default <code>false</code>): turn on the external shuffle service so executors can be safely removed under dynamic allocation without losing shuffle state. Dynamic allocation&#39;s documentation lists shuffle tracking (<code>spark.dynamicAllocation.shuffleTracking.enabled</code>), shuffle-block decommissioning, and a custom <code>ShuffleDataIO</code> plugin backed by reliable storage as alternatives to enabling the service outright<sup class="footnote-ref"><a href="#fn8" id="fnref8:8">[8:8]</a></sup>.</li><li>Push-based shuffle: enable it with the paired server/client flags added in Spark 3.2.0: <code>spark.shuffle.push.server.mergedShuffleFileManagerImpl</code> on the server side, and <code>spark.shuffle.push.enabled=true</code> on the client side (both disabled by default; the client flag only takes effect together with the server-side one). It&#39;s currently only supported for Spark on YARN with the external shuffle service enabled<sup class="footnote-ref"><a href="#fn8" id="fnref8:9">[8:9]</a></sup>.</li></ul><h2 id="sources" tabindex="-1">Sources <a class="header-anchor" href="#sources" aria-label="Permalink to &quot;Sources&quot;">​</a></h2><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><em>Learning Spark, 2nd Edition</em>, Damji, Wenig, Das, Lee, ch. 7: Optimizing and Tuning Spark Applications <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a> <a href="#fnref1:2" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/rdd-programming-guide.html" target="_blank" rel="noreferrer">RDD Programming Guide</a> <a href="#fnref2" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/sql-performance-tuning.html" target="_blank" rel="noreferrer">Performance Tuning: Spark SQL, DataFrames and Datasets Guide</a> <a href="#fnref3" class="footnote-backref">↩︎</a> <a href="#fnref3:1" class="footnote-backref">↩︎</a> <a href="#fnref3:2" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><a href="https://books.japila.pl/spark-sql-internals/bucketing/" target="_blank" rel="noreferrer">Bucketing (The Internals of Spark SQL)</a> <a href="#fnref4" class="footnote-backref">↩︎</a> <a href="#fnref4:1" class="footnote-backref">↩︎</a></p></li><li id="fn5" class="footnote-item"><p><a href="https://luminousmen.com/post/spark-tips-partition-tuning" target="_blank" rel="noreferrer">Spark Tips: Partition Tuning</a> <a href="#fnref5" class="footnote-backref">↩︎</a> <a href="#fnref5:1" class="footnote-backref">↩︎</a> <a href="#fnref5:2" class="footnote-backref">↩︎</a> <a href="#fnref5:3" class="footnote-backref">↩︎</a></p></li><li id="fn6" class="footnote-item"><p><a href="https://raw.githubusercontent.com/apache/spark/v3.5.0/core/src/main/scala/org/apache/spark/shuffle/sort/SortShuffleManager.scala" target="_blank" rel="noreferrer">SortShuffleManager.scala</a> <a href="#fnref6" class="footnote-backref">↩︎</a> <a href="#fnref6:1" class="footnote-backref">↩︎</a> <a href="#fnref6:2" class="footnote-backref">↩︎</a></p></li><li id="fn7" class="footnote-item"><p><a href="https://issues.apache.org/jira/browse/SPARK-30602" target="_blank" rel="noreferrer">SPARK-30602: Support push-based shuffle to improve shuffle efficiency (Magnet)</a> <a href="#fnref7" class="footnote-backref">↩︎</a> <a href="#fnref7:1" class="footnote-backref">↩︎</a> <a href="#fnref7:2" class="footnote-backref">↩︎</a> <a href="#fnref7:3" class="footnote-backref">↩︎</a> <a href="#fnref7:4" class="footnote-backref">↩︎</a></p></li><li id="fn8" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration (Spark)</a> <a href="#fnref8" class="footnote-backref">↩︎</a> <a href="#fnref8:1" class="footnote-backref">↩︎</a> <a href="#fnref8:2" class="footnote-backref">↩︎</a> <a href="#fnref8:3" class="footnote-backref">↩︎</a> <a href="#fnref8:4" class="footnote-backref">↩︎</a> <a href="#fnref8:5" class="footnote-backref">↩︎</a> <a href="#fnref8:6" class="footnote-backref">↩︎</a> <a href="#fnref8:7" class="footnote-backref">↩︎</a> <a href="#fnref8:8" class="footnote-backref">↩︎</a> <a href="#fnref8:9" class="footnote-backref">↩︎</a></p></li><li id="fn9" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/job-scheduling.html" target="_blank" rel="noreferrer">Job Scheduling</a> <a href="#fnref9" class="footnote-backref">↩︎</a> <a href="#fnref9:1" class="footnote-backref">↩︎</a></p></li><li id="fn10" class="footnote-item"><p><a href="https://www.taboola.com/engineering/bucket-the-shuffle-out-of-here/" target="_blank" rel="noreferrer">Bucket the Shuffle Out of Here (Taboola Engineering)</a> <a href="#fnref10" class="footnote-backref">↩︎</a> <a href="#fnref10:1" class="footnote-backref">↩︎</a></p></li><li id="fn11" class="footnote-item"><p><em>High Performance Spark, 2nd Edition</em>, Karau, Polak &amp; Warren, ch. 5: DataFrames, Datasets, and Spark SQL <a href="#fnref11" class="footnote-backref">↩︎</a> <a href="#fnref11:1" class="footnote-backref">↩︎</a> <a href="#fnref11:2" class="footnote-backref">↩︎</a></p></li></ol></section></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/tuning-reference/joins.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>Join Optimization</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="/docs/tuning-reference/data-formats.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>Data Formats</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
26
- <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"VTcYTH4d\",\"contributor-guide_architecture_detector-contract.md\":\"CsqZHz_T\",\"contributor-guide_architecture_drill-down.md\":\"CFNCE8JB\",\"contributor-guide_architecture_impact-estimation.md\":\"HT9IJGIl\",\"contributor-guide_architecture_index.md\":\"3TO9ic6w\",\"contributor-guide_architecture_overview.md\":\"CTEqVeV4\",\"contributor-guide_architecture_state-and-history.md\":\"DOTVHjTV\",\"contributor-guide_architecture_widget-rendering.md\":\"BmpARZve\",\"contributor-guide_architecture_worker-protocol.md\":\"21pTYhFF\",\"contributor-guide_contributing.md\":\"B2E9ni79\",\"contributor-guide_development-setup.md\":\"CjgZJwjK\",\"contributor-guide_testing.md\":\"DTwhpoN2\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"D08Uaod7\",\"tuning-reference_aqe.md\":\"C1Ae6A8u\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"6wjKuJNE\",\"tuning-reference_bottleneck-cold-start.md\":\"BVQbsuHP\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"DtaYnaiF\",\"tuning-reference_bottleneck-failures.md\":\"Z1Tdmdd3\",\"tuning-reference_bottleneck-gc.md\":\"BCnXLf1n\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"CcvQj2g9\",\"tuning-reference_bottleneck-memory-utilization.md\":\"B-C0-8h-\",\"tuning-reference_bottleneck-retry-waste.md\":\"DCnhWiqj\",\"tuning-reference_bottleneck-shuffle.md\":\"B5OMJCd3\",\"tuning-reference_bottleneck-skew.md\":\"Cvo6NQm_\",\"tuning-reference_bottleneck-slow-host.md\":\"ja5qr3wq\",\"tuning-reference_bottleneck-small-files.md\":\"Crjtb0i2\",\"tuning-reference_bottleneck-spill.md\":\"Bl8KtSRg\",\"tuning-reference_bottleneck-straggler.md\":\"B0VN8reT\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"DQCftGhu\",\"tuning-reference_bottleneck-utilization.md\":\"BXDl30f9\",\"tuning-reference_caching.md\":\"CcaiMn_7\",\"tuning-reference_cluster-config.md\":\"DbWG1yZq\",\"tuning-reference_config.md\":\"YUqSpwFB\",\"tuning-reference_data-formats.md\":\"C5DX5xL4\",\"tuning-reference_index.md\":\"ZVIpQ6Dk\",\"tuning-reference_intro.md\":\"BSSaDlqx\",\"tuning-reference_joins.md\":\"BVhNCDnE\",\"tuning-reference_memory-model.md\":\"CfjFdgbl\",\"tuning-reference_metrics.md\":\"C64VNEL4\",\"tuning-reference_partitioning.md\":\"CtLK0xsS\",\"tuning-reference_pyspark.md\":\"D-92XgF8\",\"tuning-reference_shuffle.md\":\"Iq2v1oHJ\",\"tuning-reference_spark-architecture.md\":\"_qwnE_Sc\",\"tuning-reference_table-formats.md\":\"BSYEK-eN\",\"user-guide_alternative-log-retrieval.md\":\"ZHPSbBkF\",\"user-guide_getting-started.md\":\"CEa2ZX-l\",\"user-guide_mcp-tools.md\":\"dQwIvdTF\",\"user-guide_run-comparison.md\":\"ChLI3MK0\",\"user-guide_understanding-findings.md\":\"D1ccs60C\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Alternative log retrieval\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
27
+ <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"D0Kkk_r5\",\"contributor-guide_architecture_detector-contract.md\":\"caA46z-u\",\"contributor-guide_architecture_drill-down.md\":\"CdtlKHl-\",\"contributor-guide_architecture_impact-estimation.md\":\"BNp14LtL\",\"contributor-guide_architecture_index.md\":\"B9yZieYL\",\"contributor-guide_architecture_overview.md\":\"DBGo-bwn\",\"contributor-guide_architecture_state-and-history.md\":\"w4lwybdh\",\"contributor-guide_architecture_widget-rendering.md\":\"CcvqxnZI\",\"contributor-guide_architecture_worker-protocol.md\":\"CKeFttKj\",\"contributor-guide_contributing.md\":\"CQMvGwFt\",\"contributor-guide_development-setup.md\":\"D9bu4j_c\",\"contributor-guide_testing.md\":\"By6XGR9a\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"D08Uaod7\",\"tuning-reference_aqe.md\":\"C1Ae6A8u\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"6wjKuJNE\",\"tuning-reference_bottleneck-cold-start.md\":\"BVQbsuHP\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"DtaYnaiF\",\"tuning-reference_bottleneck-failures.md\":\"Z1Tdmdd3\",\"tuning-reference_bottleneck-gc.md\":\"BCnXLf1n\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"CcvQj2g9\",\"tuning-reference_bottleneck-memory-utilization.md\":\"B-C0-8h-\",\"tuning-reference_bottleneck-retry-waste.md\":\"DCnhWiqj\",\"tuning-reference_bottleneck-shuffle.md\":\"B5OMJCd3\",\"tuning-reference_bottleneck-skew.md\":\"Cvo6NQm_\",\"tuning-reference_bottleneck-slow-host.md\":\"ja5qr3wq\",\"tuning-reference_bottleneck-small-files.md\":\"Crjtb0i2\",\"tuning-reference_bottleneck-spill.md\":\"Bl8KtSRg\",\"tuning-reference_bottleneck-straggler.md\":\"B0VN8reT\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"DQCftGhu\",\"tuning-reference_bottleneck-utilization.md\":\"BXDl30f9\",\"tuning-reference_caching.md\":\"CcaiMn_7\",\"tuning-reference_cluster-config.md\":\"DbWG1yZq\",\"tuning-reference_config.md\":\"YUqSpwFB\",\"tuning-reference_data-formats.md\":\"C5DX5xL4\",\"tuning-reference_index.md\":\"ZVIpQ6Dk\",\"tuning-reference_intro.md\":\"BSSaDlqx\",\"tuning-reference_joins.md\":\"BVhNCDnE\",\"tuning-reference_memory-model.md\":\"CfjFdgbl\",\"tuning-reference_metrics.md\":\"C64VNEL4\",\"tuning-reference_partitioning.md\":\"CtLK0xsS\",\"tuning-reference_pyspark.md\":\"D-92XgF8\",\"tuning-reference_shuffle.md\":\"Iq2v1oHJ\",\"tuning-reference_spark-architecture.md\":\"_qwnE_Sc\",\"tuning-reference_table-formats.md\":\"BSYEK-eN\",\"user-guide_alternative-log-retrieval.md\":\"z-8npxNk\",\"user-guide_getting-started.md\":\"BJW5oVhe\",\"user-guide_mcp-tools.md\":\"DcLiOa9t\",\"user-guide_run-comparison.md\":\"qin0a4IX\",\"user-guide_understanding-findings.md\":\"DzReyCXb\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Finding your event log\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":{\"selector\":\"[data-sf-anchor-offset]\",\"padding\":0},\"cleanUrls\":false}");</script>
27
28
 
28
29
  </body>
29
30
  </html>
package/public/docs/tuning-reference/spark-architecture.html CHANGED
@@ -6,14 +6,15 @@
6
6
  <title>Spark Execution Model | Spark Tuning Reference</title>
7
7
  <meta name="description" content="Spark's execution model: Driver/Executor split, DAGScheduler/TaskScheduler roles, stage boundaries at shuffles, and pipelining of narrow transformations.">
8
8
  <meta name="generator" content="VitePress v1.6.4">
9
- <link rel="preload stylesheet" href="/docs/assets/style.DSixAiZE.css" as="style">
9
+ <link rel="preload stylesheet" href="/docs/assets/style.tmDhD3c8.css" as="style">
10
10
  <link rel="preload stylesheet" href="/docs/vp-icons.css" as="style">
11
11
 
12
- <script type="module" src="/docs/assets/app.C6w47a9W.js"></script>
12
+ <script type="module" src="/docs/assets/app.yjczoibf.js"></script>
13
13
  <link rel="preload" href="/docs/assets/inter-roman-latin.Di8DUHzh.woff2" as="font" type="font/woff2" crossorigin="">
14
- <link rel="modulepreload" href="/docs/assets/chunks/theme.D-mE6j02.js">
14
+ <link rel="modulepreload" href="/docs/assets/chunks/theme.DFcAl8Ws.js">
15
15
  <link rel="modulepreload" href="/docs/assets/chunks/framework.DSg0KOwT.js">
16
16
  <link rel="modulepreload" href="/docs/assets/tuning-reference_spark-architecture.md._qwnE_Sc.lean.js">
17
+ <script>(()=>{const c=`${`[${["data","shuffle","product","bar"].join("-")}]`}, .VPNav, .VPLocalNav`,a=24,d=document.documentElement,l=t=>{const e=getComputedStyle(t);return e.display==="none"||t.getClientRects().length===0?0:e.position==="fixed"?t.getBoundingClientRect().bottom:e.position==="sticky"?(parseFloat(e.top)||0)+t.offsetHeight:0},u=()=>{try{return location.hash?document.getElementById(decodeURIComponent(location.hash.slice(1))):null}catch{return null}};let r=null;const s=()=>{const t=Array.from(document.querySelectorAll(c),l),e=Math.ceil(Math.max(0,...t))+a;if(e===r)return;const n=r;r=e,d.style.setProperty("--sf-anchor-offset",`${r}px`);const o=n===null?null:u();if(!o)return;const f=(parseFloat(getComputedStyle(o).paddingTop)||0)+2;Math.abs(o.getBoundingClientRect().top-n)<=f&&o.scrollIntoView()},i=()=>{const t=document.createElement("div");t.setAttribute("data-sf-anchor-offset",""),t.setAttribute("aria-hidden","true"),document.body.appendChild(t);const e=window.ResizeObserver?new ResizeObserver(s):null,n=()=>{e&&document.querySelectorAll(c).forEach(o=>e.observe(o)),s()};n(),new MutationObserver(n).observe(document.body,{childList:!0,subtree:!0}),window.addEventListener("resize",s),window.addEventListener("load",s)};document.readyState==="loading"?document.addEventListener("DOMContentLoaded",i,{once:!0}):i()})();</script>
17
18
  <link rel="icon" type="image/svg+xml" href="/docs/favicon.svg">
18
19
  <link rel="preconnect" href="https://fonts.googleapis.com">
19
20
  <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin="">
@@ -23,7 +24,7 @@
23
24
  </head>
24
25
  <body>
25
26
  <div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Getting Started</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/intro.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Introduction</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Optimization Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/spark-architecture.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Execution Model</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/memory-model.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Management</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/partitioning.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Partitioning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/joins.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Join Optimization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/data-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Data Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/table-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Table Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/caching.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Caching & Persistence</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/pyspark.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>PySpark Specifics</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/aqe.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Adaptive Query Execution</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/cluster-config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cluster Tuning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/anti-patterns.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Anti-Patterns</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Detector Catalog</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-skew.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Skew</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle I/O</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-spill.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory / Disk Spill</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-gc.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>GC Pressure</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-cold-start.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cold Start</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Executor Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-slow-host.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Slow Host</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-failures.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Failures</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-straggler.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Stragglers</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-retry-waste.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Retry Waste</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-tiny-tasks.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Tiny Tasks</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-job-failure-rate.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Job Failure Rate</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-memory-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Duplicate Plan Subtree</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-small-files.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Small Files</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-broadcast-sizing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Broadcast Sizing</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Reference</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/metrics.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Metrics Glossary</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Config Quick-Reference</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_tuning-reference_spark-architecture" data-v-39a288b8><div><h1 id="spark-architecture" tabindex="-1">Spark Execution Model <a class="header-anchor" href="#spark-architecture" aria-label="Permalink to &quot;Spark Execution Model {#spark-architecture}&quot;">​</a></h1><h2 id="the-driver-and-the-dag" tabindex="-1">The driver and the DAG <a class="header-anchor" href="#the-driver-and-the-dag" aria-label="Permalink to &quot;The driver and the DAG&quot;">​</a></h2><p>The Driver is the process &quot;in the driver seat&quot; of a Spark application: it controls execution and maintains all state of the cluster, including the state and tasks of the executors, and it interfaces with the cluster manager to obtain physical resources and launch executors<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup>. It runs as a single JVM process (on the submission machine in client mode, or on a dedicated cluster node in cluster mode), and if it dies, the application dies with it<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup>.</p><p>As application code executes, the Driver incrementally builds a logical DAG of RDD/DataFrame transformations. Because Spark is lazy, nothing actually computes until an action is called; only then does the Driver convert the logical DAG into a physical plan, cut it into stages at wide-dependency (<a href="./shuffle.html">shuffle</a>) boundaries, break each stage into one task per partition, and take on the scheduler&#39;s job: talking to the cluster resource manager to request executors, handing out tasks, tracking their progress, and resubmitting tasks whose executor died<sup class="footnote-ref"><a href="#fn2" id="fnref2:1">[2:1]</a></sup>.</p><p>Three components live only on the Driver:</p><ul><li>The <strong>DAGScheduler</strong> reads RDD lineage, decides stage boundaries at wide dependencies, emits per-partition TaskSets, tracks lineage so lost partitions can be recomputed, and reacts dynamically to stage completions rather than pre-scheduling the whole DAG up front; it also declares a job failed if a stage can&#39;t make progress<sup class="footnote-ref"><a href="#fn2" id="fnref2:2">[2:2]</a></sup>.</li><li>The <strong>BlockManagerMaster</strong> is the Driver-side counterpart to each executor&#39;s own BlockManager. It keeps the global map of block locations (RDD partitions, shuffle files, broadcast variables) across the whole cluster, so a task&#39;s local BlockManager knows where to fetch a remote block from<sup class="footnote-ref"><a href="#fn2" id="fnref2:3">[2:3]</a></sup>.</li><li>The <strong>SparkContext</strong> (wrapped today by SparkSession) is the entry point that lets the Driver talk to the cluster manager, request executors, create RDDs, and manage shared variables. It&#39;s instantiated once per application and lives for the application&#39;s whole lifetime<sup class="footnote-ref"><a href="#fn2" id="fnref2:4">[2:4]</a></sup>.</li></ul><p>Executors, by contrast, hold no cluster-wide state: they only run the tasks assigned to them and report back success, failure, and results<sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup>. The split even shows up at the configuration level: <code>spark.driver.host</code>/<code>spark.driver.port</code> and <code>spark.driver.blockManager.port</code> are Driver-specific listening endpoints, distinct from the equivalent executor settings<sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup>.</p><img class="light-only" src="/docs/assets/driver-executor.D5pQ7YN1.svg" alt="A Spark driver with its scheduler components dispatching tasks to executors through the cluster manager and receiving status and results back."><img class="dark-only" src="/docs/assets/driver-executor.dark.BmX9cPvh.svg" alt="A Spark driver with its scheduler components dispatching tasks to executors through the cluster manager and receiving status and results back."><p>A Spark job corresponds to one action, and each job breaks down into a series of stages: how many depends on how many shuffle operations need to happen<sup class="footnote-ref"><a href="#fn1" id="fnref1:2">[1:2]</a></sup>. A stage is a group of tasks that can execute together to compute the same operation across machines: work one executor can do without communicating with other executors or the Driver. A new stage begins whenever data has to move across the network, that is, at a shuffle<sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup>. Wide transformations, such as <code>groupByKey</code>, <code>join</code>, and <code>sortByKey</code>, create <code>ShuffleDependency</code> objects, and it&#39;s these <code>ShuffleDependency</code>s that mark the stage boundary; several narrow transformations can be grouped into the same stage<sup class="footnote-ref"><a href="#fn4" id="fnref4:1">[4:1]</a></sup>. Within a stage, the number of tasks equals the number of partitions in that stage&#39;s output RDD: one task per partition, each running the same code on a different slice of data<sup class="footnote-ref"><a href="#fn4" id="fnref4:2">[4:2]</a></sup>.</p><p>The rule is always &quot;a shuffle dependency creates the boundary,&quot; and that isn&#39;t limited to an explicit <code>.repartition()</code> or <code>.groupBy()</code> call: <code>sortByKey</code>/<code>sortBy</code> on an RDD is itself a wide transformation, not just <code>groupByKey</code>-style aggregations<sup class="footnote-ref"><a href="#fn4" id="fnref4:3">[4:3]</a></sup>. The original Spark paper defines stage boundaries generically, as the shuffle operations required for wide dependencies, or already-computed partitions that can short-circuit the computation of a parent RDD<sup class="footnote-ref"><a href="#fn5" id="fnref5">[5]</a></sup>. A wide dependency of any kind is what matters, not a specific API call.</p><p>That wide/narrow split has a formal definition behind it. Conceptually, a narrow transformation is one where each partition in the child RDD has simple, finite dependencies on partitions in the parent RDD, determinable at design time regardless of the values of the records; narrow dependencies allow pipelined, one-node execution, while wide dependencies require data from all parent partitions to be shuffled across nodes<sup class="footnote-ref"><a href="#fn5" id="fnref5:1">[5:1]</a></sup>. The formal 2012 definition is stated from the parent&#39;s side rather than the child&#39;s: a transformation is narrow if &quot;each partition of the parent RDD is used by at most one partition of the child RDD,&quot; and wide if &quot;multiple child partitions may depend on&quot; a given parent partition<sup class="footnote-ref"><a href="#fn4" id="fnref4:4">[4:4]</a></sup>. That parent-centric framing is more precise, because Spark&#39;s DAG scheduler builds the execution plan backward from the action to the input RDD, and it correctly rules out the case of one parent partition feeding multiple children under &quot;narrow&quot;<sup class="footnote-ref"><a href="#fn4" id="fnref4:5">[4:5]</a></sup>.</p><p><code>mapPartitions</code> is narrow under this definition: each child partition depends on exactly one parent partition, the same as <code>map</code> and <code>filter</code><sup class="footnote-ref"><a href="#fn4" id="fnref4:6">[4:6]</a></sup>. <code>coalesce</code> is narrow too, even though it changes the number of partitions and a child partition can depend on multiple parent partitions: the rule only requires that each parent partition be used by at most one child partition, and which parent partitions merge into which child is fixed at design time, independent of the data&#39;s values. That holds specifically when <code>coalesce</code> reduces the partition count; when it increases the count, it behaves like <code>repartition</code>, a shuffle<sup class="footnote-ref"><a href="#fn4" id="fnref4:7">[4:7]</a></sup>.</p><p>Pipelining is what makes narrow dependencies pay off. Spark performs as many steps as possible in one pass before writing data to memory or disk: any sequence of operations that feed data directly into each other, without moving data across nodes, collapses into a single stage of tasks that execute all the operations together. <code>map → filter → map</code> becomes one stage whose tasks read each record and pass it through all three operations in sequence, rather than materializing intermediate results after each step; the same collapsing happens for a DataFrame/SQL computation doing <code>select → filter → select</code><sup class="footnote-ref"><a href="#fn1" id="fnref1:3">[1:3]</a></sup>. The original paper phrases the mechanism at the scheduler level: to run an action, Spark builds stages at wide dependencies and pipelines narrow transformations inside each stage<sup class="footnote-ref"><a href="#fn5" id="fnref5:2">[5:2]</a></sup>.</p><p>Two more components take over once a job&#39;s stages exist. The <strong>DAGScheduler</strong> is Spark&#39;s high-level scheduling layer: it takes RDD dependencies, builds a DAG of stages for each job, determines where each task should run, and passes that to the TaskScheduler<sup class="footnote-ref"><a href="#fn4" id="fnref4:8">[4:8]</a></sup>. The <strong>TaskScheduler</strong> takes it from there and stops thinking in terms of RDDs, lineage, or shuffles: only machines and slots. It works with the SchedulerBackend to request executors, assigns tasks to executors while respecting data-locality preferences, and retries failed tasks, including resubmitting a task elsewhere if its executor died<sup class="footnote-ref"><a href="#fn2" id="fnref2:5">[2:5]</a></sup>. The TaskScheduler doesn&#39;t launch tasks directly; it hands them to the SchedulerBackend, the layer that actually talks to the cluster manager to request, launch, and kill executors<sup class="footnote-ref"><a href="#fn2" id="fnref2:6">[2:6]</a></sup>.</p><p>Finally, at the RDD API level <code>spark.default.parallelism</code> sets the default partition count for distributed shuffle operations like <code>reduceByKey</code> and <code>join</code>. It defaults to the largest number of partitions in the parent RDD, and for operations with no parent RDD, such as <code>parallelize</code>, the default depends on the cluster manager<sup class="footnote-ref"><a href="#fn3" id="fnref3:1">[3:1]</a></sup>. <code>spark.sql.shuffle.partitions</code> is the equivalent knob for the Spark SQL/DataFrame engine: it configures how many partitions Spark uses when shuffling data for joins or aggregations, and it defaults to 200 regardless of data size<sup class="footnote-ref"><a href="#fn6" id="fnref6">[6]</a></sup>, applying whenever a DataFrame/SQL operation triggers a shuffle: join, groupBy, distinct, orderBy, and so on<sup class="footnote-ref"><a href="#fn7" id="fnref7">[7]</a></sup>.</p><h2 id="reading-stages-and-tasks" tabindex="-1">Reading stages and tasks <a class="header-anchor" href="#reading-stages-and-tasks" aria-label="Permalink to &quot;Reading stages and tasks&quot;">​</a></h2><p>Once a job runs, stage and task counts show up directly in the Spark UI, and a worked example makes the mechanics concrete: a job that reads a range with 8 partitions produces a stage with 8 tasks; repartitioning to 6 and then 5 partitions produces stages with 6 and 5 tasks; a subsequent join shuffles into the default 200 shuffle partitions, producing a 200-task stage<sup class="footnote-ref"><a href="#fn1" id="fnref1:4">[1:4]</a></sup>. The same logic explains stage counts for RDD pipelines: the chain <code>filter → map → groupByKey → map → sortByKey → count</code> produces three stages, bounded by the <code>groupByKey</code> and <code>sortByKey</code> operations, since both are wide<sup class="footnote-ref"><a href="#fn4" id="fnref4:9">[4:9]</a></sup>.</p><img class="light-only" src="/docs/assets/dag-stages.DSz_S937.svg" alt="A three stage DAG for the pipeline filter, map, groupByKey, map, sortByKey, count, with stage boundaries at the wide groupByKey and sortByKey operations."><img class="dark-only" src="/docs/assets/dag-stages.dark.F72UzxH4.svg" alt="A three stage DAG for the pipeline filter, map, groupByKey, map, sortByKey, count, with stage boundaries at the wide groupByKey and sortByKey operations."><p>Pipelining is invisible to the application and only shows up in the Spark UI or logs, where multiple chained narrow operations appear collapsed into a single stage instead of one stage per operation<sup class="footnote-ref"><a href="#fn1" id="fnref1:5">[1:5]</a></sup>. A related signal is the &quot;skipped stage&quot; marking: because Spark always writes shuffle output to stable storage regardless of any <code>persist</code>/<code>checkpoint</code> call, if the Driver reuses an RDD that was already shuffled, Spark can skip recomputing everything up to that shuffle and read the shuffle files directly. The Spark UI shows this as a skipped stage, not as an added boundary<sup class="footnote-ref"><a href="#fn4" id="fnref4:10">[4:10]</a></sup>.</p><p>When diagnosing partition counts, checking which knob is in play matters: <code>spark.default.parallelism</code> governs the legacy RDD API&#39;s shuffle and <code>parallelize</code> operations, while <code>spark.sql.shuffle.partitions</code> governs DataFrame/Dataset/SQL shuffles<sup class="footnote-ref"><a href="#fn3" id="fnref3:2">[3:2]</a></sup>. Since Spark 3.0, <a href="./aqe.html">Adaptive Query Execution</a> (AQE) can also override the static <code>spark.sql.shuffle.partitions</code> value after the fact. It can coalesce many small post-shuffle partitions and split skewed ones at runtime, but only after the first shuffle has already happened, so it never explains the initial input partitioning<sup class="footnote-ref"><a href="#fn7" id="fnref7:1">[7:1]</a></sup>.</p><h2 id="why-the-boundary-matters" tabindex="-1">Why the boundary matters <a class="header-anchor" href="#why-the-boundary-matters" aria-label="Permalink to &quot;Why the boundary matters&quot;">​</a></h2><p>Those stage boundaries aren&#39;t just a UI detail. Because a new stage begins only at a shuffle, the boundary is exactly where Spark pays the cost of moving data across the network. Everything inside a stage runs without that cost, pipelined on a single executor<sup class="footnote-ref"><a href="#fn4" id="fnref4:11">[4:11]</a></sup>.</p><p>That&#39;s also why the wide/narrow distinction determines whether pipelining is even possible: once a shuffle is required, downstream computation can&#39;t proceed until the shuffle completes, because the records landing on each partition may change as a result of it, so narrow transformations following a wide one belong to a new stage and can&#39;t be pipelined across that boundary<sup class="footnote-ref"><a href="#fn4" id="fnref4:12">[4:12]</a></sup>.</p><p>Whether pipelining happens can also depend on partitioning state Spark already knows about. Because Spark tracks how an RDD is already partitioned, the same operation can land in different stage boundaries depending on whether the input RDD already has a known partitioner: no shuffle, and hence no new stage, is needed if one is already in place<sup class="footnote-ref"><a href="#fn4" id="fnref4:13">[4:13]</a></sup>.</p><p><a href="./caching.html">Checkpointing</a> is a separate mechanism from all of this: it breaks RDD lineage and writes data to disk so later transformations start from a fresh, trimmed plan, rather than being pipelined against the original computation<sup class="footnote-ref"><a href="#fn8" id="fnref8">[8]</a></sup>.</p><p>The choice of <a href="./partitioning.html">shuffle-partition count</a> matters for the same reason: there&#39;s no fixed formula for the right number, since it depends on data set size, number of cores, and available executor memory, and the default of 200 is called out as too high for smaller or streaming workloads, where it&#39;s better reduced toward the number of executor cores or less<sup class="footnote-ref"><a href="#fn9" id="fnref9">[9]</a></sup>. <code>coalesce</code>&#39;s narrowness carries its own tradeoff: reducing partition count without a shuffle forces all upstream partitions in that stage to run at the coalesced parallelism level, which can be undesirable<sup class="footnote-ref"><a href="#fn4" id="fnref4:14">[4:14]</a></sup>.</p><p>On the scheduling side, Spark&#39;s scheduler is fully thread-safe and supports running multiple jobs concurrently if they&#39;re submitted from separate Driver threads, common when an application serves multiple concurrent requests over the network<sup class="footnote-ref"><a href="#fn10" id="fnref10">[10]</a></sup>. By default Spark schedules jobs FIFO, which means an application running many jobs from multiple threads can have later jobs wait behind earlier ones unless a fairer policy is configured<sup class="footnote-ref"><a href="#fn4" id="fnref4:15">[4:15]</a></sup>.</p><h2 id="controlling-parallelism-and-scheduling" tabindex="-1">Controlling parallelism and scheduling <a class="header-anchor" href="#controlling-parallelism-and-scheduling" aria-label="Permalink to &quot;Controlling parallelism and scheduling&quot;">​</a></h2><p>Parallelism and scheduling are both tunable from here. When <code>coalesce</code>&#39;s single-parent-per-child constraint forces upstream parallelism lower than you want, <code>repartition</code> trades that limitation for an explicit shuffle, restoring full control over the resulting partition count<sup class="footnote-ref"><a href="#fn4" id="fnref4:16">[4:16]</a></sup>. For the SQL/DataFrame engine, don&#39;t leave <code>spark.sql.shuffle.partitions</code> at its 200 default for small or streaming workloads: reduce it toward the number of executor cores or less, since the right value depends on data size, core count, and executor memory rather than a fixed rule<sup class="footnote-ref"><a href="#fn9" id="fnref9:1">[9:1]</a></sup>. Where the data volume is unpredictable, Adaptive Query Execution (available since Spark 3.0) can take over after the fact: it coalesces many small post-shuffle partitions and splits skewed ones at runtime, though it only acts after the first shuffle has already happened and won&#39;t fix the initial input partitioning<sup class="footnote-ref"><a href="#fn7" id="fnref7:2">[7:2]</a></sup>.</p><p>For concurrent workloads, Spark also offers a fair scheduler as an alternative to the FIFO default, assigning tasks to concurrent jobs round-robin so each job gets a more even share of cluster resources<sup class="footnote-ref"><a href="#fn4" id="fnref4:17">[4:17]</a></sup>; job pools and weights for finer-grained sharing are configured via <code>spark.scheduler.mode=FAIR</code> and the <code>spark.scheduler.pool</code> local property set on the submitting thread<sup class="footnote-ref"><a href="#fn1" id="fnref1:6">[1:6]</a></sup>. Submitting jobs from separate Driver threads is what lets them run concurrently in the first place, rather than queuing behind each other<sup class="footnote-ref"><a href="#fn10" id="fnref10:1">[10:1]</a></sup>.</p><h2 id="sources" tabindex="-1">Sources <a class="header-anchor" href="#sources" aria-label="Permalink to &quot;Sources&quot;">​</a></h2><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><em>Spark: The Definitive Guide</em>, Chambers &amp; Zaharia, ch. 15–16 <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a> <a href="#fnref1:2" class="footnote-backref">↩︎</a> <a href="#fnref1:3" class="footnote-backref">↩︎</a> <a href="#fnref1:4" class="footnote-backref">↩︎</a> <a href="#fnref1:5" class="footnote-backref">↩︎</a> <a href="#fnref1:6" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><a href="https://luminousmen.com/post/spark-anatomy-of-spark-application" target="_blank" rel="noreferrer">Anatomy of Spark Application</a> <a href="#fnref2" class="footnote-backref">↩︎</a> <a href="#fnref2:1" class="footnote-backref">↩︎</a> <a href="#fnref2:2" class="footnote-backref">↩︎</a> <a href="#fnref2:3" class="footnote-backref">↩︎</a> <a href="#fnref2:4" class="footnote-backref">↩︎</a> <a href="#fnref2:5" class="footnote-backref">↩︎</a> <a href="#fnref2:6" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration — Spark</a> <a href="#fnref3" class="footnote-backref">↩︎</a> <a href="#fnref3:1" class="footnote-backref">↩︎</a> <a href="#fnref3:2" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><em>High Performance Spark, 2nd Edition</em>, Karau, Polak &amp; Warren, ch. 2, 7–8 <a href="#fnref4" class="footnote-backref">↩︎</a> <a href="#fnref4:1" class="footnote-backref">↩︎</a> <a href="#fnref4:2" class="footnote-backref">↩︎</a> <a href="#fnref4:3" class="footnote-backref">↩︎</a> <a href="#fnref4:4" class="footnote-backref">↩︎</a> <a href="#fnref4:5" class="footnote-backref">↩︎</a> <a href="#fnref4:6" class="footnote-backref">↩︎</a> <a href="#fnref4:7" class="footnote-backref">↩︎</a> <a href="#fnref4:8" class="footnote-backref">↩︎</a> <a href="#fnref4:9" class="footnote-backref">↩︎</a> <a href="#fnref4:10" class="footnote-backref">↩︎</a> <a href="#fnref4:11" class="footnote-backref">↩︎</a> <a href="#fnref4:12" class="footnote-backref">↩︎</a> <a href="#fnref4:13" class="footnote-backref">↩︎</a> <a href="#fnref4:14" class="footnote-backref">↩︎</a> <a href="#fnref4:15" class="footnote-backref">↩︎</a> <a href="#fnref4:16" class="footnote-backref">↩︎</a> <a href="#fnref4:17" class="footnote-backref">↩︎</a></p></li><li id="fn5" class="footnote-item"><p><a href="https://www.usenix.org/system/files/conference/nsdi12/nsdi12-final138.pdf" target="_blank" rel="noreferrer">Resilient Distributed Datasets: A Fault-Tolerant Abstraction for In-Memory Cluster Computing</a> <a href="#fnref5" class="footnote-backref">↩︎</a> <a href="#fnref5:1" class="footnote-backref">↩︎</a> <a href="#fnref5:2" class="footnote-backref">↩︎</a></p></li><li id="fn6" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/sql-performance-tuning.html" target="_blank" rel="noreferrer">Performance Tuning — Spark SQL, DataFrames and Datasets Guide</a> <a href="#fnref6" class="footnote-backref">↩︎</a></p></li><li id="fn7" class="footnote-item"><p><a href="https://luminousmen.com/post/spark-partitions" target="_blank" rel="noreferrer">Spark Partitions</a> <a href="#fnref7" class="footnote-backref">↩︎</a> <a href="#fnref7:1" class="footnote-backref">↩︎</a> <a href="#fnref7:2" class="footnote-backref">↩︎</a></p></li><li id="fn8" class="footnote-item"><p><a href="https://luminousmen.com/post/spark-tips-caching" target="_blank" rel="noreferrer">Spark Tips: Caching</a> <a href="#fnref8" class="footnote-backref">↩︎</a></p></li><li id="fn9" class="footnote-item"><p><em>Learning Spark, 2nd Edition</em>, Damji, Wenig, Das &amp; Lee, ch. 7 <a href="#fnref9" class="footnote-backref">↩︎</a> <a href="#fnref9:1" class="footnote-backref">↩︎</a></p></li><li id="fn10" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/job-scheduling.html" target="_blank" rel="noreferrer">Job Scheduling</a> <a href="#fnref10" class="footnote-backref">↩︎</a> <a href="#fnref10:1" class="footnote-backref">↩︎</a></p></li></ol></section></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/tuning-reference/intro.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>Introduction</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="/docs/tuning-reference/memory-model.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>Memory Management</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
26
- <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"VTcYTH4d\",\"contributor-guide_architecture_detector-contract.md\":\"CsqZHz_T\",\"contributor-guide_architecture_drill-down.md\":\"CFNCE8JB\",\"contributor-guide_architecture_impact-estimation.md\":\"HT9IJGIl\",\"contributor-guide_architecture_index.md\":\"3TO9ic6w\",\"contributor-guide_architecture_overview.md\":\"CTEqVeV4\",\"contributor-guide_architecture_state-and-history.md\":\"DOTVHjTV\",\"contributor-guide_architecture_widget-rendering.md\":\"BmpARZve\",\"contributor-guide_architecture_worker-protocol.md\":\"21pTYhFF\",\"contributor-guide_contributing.md\":\"B2E9ni79\",\"contributor-guide_development-setup.md\":\"CjgZJwjK\",\"contributor-guide_testing.md\":\"DTwhpoN2\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"D08Uaod7\",\"tuning-reference_aqe.md\":\"C1Ae6A8u\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"6wjKuJNE\",\"tuning-reference_bottleneck-cold-start.md\":\"BVQbsuHP\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"DtaYnaiF\",\"tuning-reference_bottleneck-failures.md\":\"Z1Tdmdd3\",\"tuning-reference_bottleneck-gc.md\":\"BCnXLf1n\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"CcvQj2g9\",\"tuning-reference_bottleneck-memory-utilization.md\":\"B-C0-8h-\",\"tuning-reference_bottleneck-retry-waste.md\":\"DCnhWiqj\",\"tuning-reference_bottleneck-shuffle.md\":\"B5OMJCd3\",\"tuning-reference_bottleneck-skew.md\":\"Cvo6NQm_\",\"tuning-reference_bottleneck-slow-host.md\":\"ja5qr3wq\",\"tuning-reference_bottleneck-small-files.md\":\"Crjtb0i2\",\"tuning-reference_bottleneck-spill.md\":\"Bl8KtSRg\",\"tuning-reference_bottleneck-straggler.md\":\"B0VN8reT\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"DQCftGhu\",\"tuning-reference_bottleneck-utilization.md\":\"BXDl30f9\",\"tuning-reference_caching.md\":\"CcaiMn_7\",\"tuning-reference_cluster-config.md\":\"DbWG1yZq\",\"tuning-reference_config.md\":\"YUqSpwFB\",\"tuning-reference_data-formats.md\":\"C5DX5xL4\",\"tuning-reference_index.md\":\"ZVIpQ6Dk\",\"tuning-reference_intro.md\":\"BSSaDlqx\",\"tuning-reference_joins.md\":\"BVhNCDnE\",\"tuning-reference_memory-model.md\":\"CfjFdgbl\",\"tuning-reference_metrics.md\":\"C64VNEL4\",\"tuning-reference_partitioning.md\":\"CtLK0xsS\",\"tuning-reference_pyspark.md\":\"D-92XgF8\",\"tuning-reference_shuffle.md\":\"Iq2v1oHJ\",\"tuning-reference_spark-architecture.md\":\"_qwnE_Sc\",\"tuning-reference_table-formats.md\":\"BSYEK-eN\",\"user-guide_alternative-log-retrieval.md\":\"ZHPSbBkF\",\"user-guide_getting-started.md\":\"CEa2ZX-l\",\"user-guide_mcp-tools.md\":\"dQwIvdTF\",\"user-guide_run-comparison.md\":\"ChLI3MK0\",\"user-guide_understanding-findings.md\":\"D1ccs60C\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Alternative log retrieval\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
27
+ <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"D0Kkk_r5\",\"contributor-guide_architecture_detector-contract.md\":\"caA46z-u\",\"contributor-guide_architecture_drill-down.md\":\"CdtlKHl-\",\"contributor-guide_architecture_impact-estimation.md\":\"BNp14LtL\",\"contributor-guide_architecture_index.md\":\"B9yZieYL\",\"contributor-guide_architecture_overview.md\":\"DBGo-bwn\",\"contributor-guide_architecture_state-and-history.md\":\"w4lwybdh\",\"contributor-guide_architecture_widget-rendering.md\":\"CcvqxnZI\",\"contributor-guide_architecture_worker-protocol.md\":\"CKeFttKj\",\"contributor-guide_contributing.md\":\"CQMvGwFt\",\"contributor-guide_development-setup.md\":\"D9bu4j_c\",\"contributor-guide_testing.md\":\"By6XGR9a\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"D08Uaod7\",\"tuning-reference_aqe.md\":\"C1Ae6A8u\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"6wjKuJNE\",\"tuning-reference_bottleneck-cold-start.md\":\"BVQbsuHP\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"DtaYnaiF\",\"tuning-reference_bottleneck-failures.md\":\"Z1Tdmdd3\",\"tuning-reference_bottleneck-gc.md\":\"BCnXLf1n\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"CcvQj2g9\",\"tuning-reference_bottleneck-memory-utilization.md\":\"B-C0-8h-\",\"tuning-reference_bottleneck-retry-waste.md\":\"DCnhWiqj\",\"tuning-reference_bottleneck-shuffle.md\":\"B5OMJCd3\",\"tuning-reference_bottleneck-skew.md\":\"Cvo6NQm_\",\"tuning-reference_bottleneck-slow-host.md\":\"ja5qr3wq\",\"tuning-reference_bottleneck-small-files.md\":\"Crjtb0i2\",\"tuning-reference_bottleneck-spill.md\":\"Bl8KtSRg\",\"tuning-reference_bottleneck-straggler.md\":\"B0VN8reT\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"DQCftGhu\",\"tuning-reference_bottleneck-utilization.md\":\"BXDl30f9\",\"tuning-reference_caching.md\":\"CcaiMn_7\",\"tuning-reference_cluster-config.md\":\"DbWG1yZq\",\"tuning-reference_config.md\":\"YUqSpwFB\",\"tuning-reference_data-formats.md\":\"C5DX5xL4\",\"tuning-reference_index.md\":\"ZVIpQ6Dk\",\"tuning-reference_intro.md\":\"BSSaDlqx\",\"tuning-reference_joins.md\":\"BVhNCDnE\",\"tuning-reference_memory-model.md\":\"CfjFdgbl\",\"tuning-reference_metrics.md\":\"C64VNEL4\",\"tuning-reference_partitioning.md\":\"CtLK0xsS\",\"tuning-reference_pyspark.md\":\"D-92XgF8\",\"tuning-reference_shuffle.md\":\"Iq2v1oHJ\",\"tuning-reference_spark-architecture.md\":\"_qwnE_Sc\",\"tuning-reference_table-formats.md\":\"BSYEK-eN\",\"user-guide_alternative-log-retrieval.md\":\"z-8npxNk\",\"user-guide_getting-started.md\":\"BJW5oVhe\",\"user-guide_mcp-tools.md\":\"DcLiOa9t\",\"user-guide_run-comparison.md\":\"qin0a4IX\",\"user-guide_understanding-findings.md\":\"DzReyCXb\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Finding your event log\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":{\"selector\":\"[data-sf-anchor-offset]\",\"padding\":0},\"cleanUrls\":false}");</script>
27
28
 
28
29
  </body>
29
30
  </html>