sparkforensics-server 0.6.0 → 0.8.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (410) hide show
  1. package/index.js +16 -7
  2. package/package.json +1 -1
  3. package/public/assets/AutoscalingChurn-DenWKTv-.js +1 -0
  4. package/public/assets/CacheUtilization-DLz7NdjV.js +1 -0
  5. package/public/assets/CachingOpportunity-zpQIqhzm.js +1 -0
  6. package/public/assets/ChartTheme-DAiARbLP.js +8 -0
  7. package/public/assets/ColdStart-C-9HjxWl.js +1 -0
  8. package/public/assets/ConfigAudit-D0V9tcrJ.js +1 -0
  9. package/public/assets/CoreUsageArea-yeerrljd.js +1 -0
  10. package/public/assets/Dashboard-DkUPrHKV.js +1 -0
  11. package/public/assets/DisclosureContext-DK1_iz6J.js +1 -0
  12. package/public/assets/DuplicatePlanSubtree-xDNDbCd3.js +1 -0
  13. package/public/assets/DurationHistogram-B6jsE0Ov.js +1 -0
  14. package/public/assets/EmptyState-C1uL40VU.js +1 -0
  15. package/public/assets/ExecutorUtilization-DXDZKmSc.js +1 -0
  16. package/public/assets/GcPressure-MRf7I4IG.js +1 -0
  17. package/public/assets/ImpactEstimate-Dm3xWbzV.js +1 -0
  18. package/public/assets/IncompleteRun-CaLOhMtf.js +1 -0
  19. package/public/assets/JobFailures-CzmzlvhO.js +1 -0
  20. package/public/assets/MemoryUtilization-AWUMUjCH.js +1 -0
  21. package/public/assets/NestedLoopJoin-C9sNr6Ss.js +1 -0
  22. package/public/assets/OverBroadcast-BOmRE6ca.js +1 -0
  23. package/public/assets/PartitionHint-BL2EmJfE.js +1 -0
  24. package/public/assets/PartitionSizing-BzBySvUq.js +1 -0
  25. package/public/assets/PlanExplorer-DdiQRcJU.js +1 -0
  26. package/public/assets/PlanGraphRoute-CIH9gBuR.js +1 -0
  27. package/public/assets/PlanView-B1hxzFRs.js +2 -0
  28. package/public/assets/PythonUdf-DvgCCueD.js +1 -0
  29. package/public/assets/RetryWaste-uXXu_CyM.js +1 -0
  30. package/public/assets/RowPagination-C-LzWkI5.js +1 -0
  31. package/public/assets/RowStatusCluster-Dbtmte7l.js +1 -0
  32. package/public/assets/ShuffleIO-nALwzC1_.js +1 -0
  33. package/public/assets/Skew-CHMEKgvk.js +1 -0
  34. package/public/assets/SlowHost-yyvjpHk6.js +1 -0
  35. package/public/assets/SmallFiles-y6odJ97n.js +1 -0
  36. package/public/assets/SpeculationWaste-C-SD2LOt.js +1 -0
  37. package/public/assets/Spill-DHTMmOup.js +1 -0
  38. package/public/assets/StageFailed-CQ3DxC9R.js +1 -0
  39. package/public/assets/StageFindingGroup-BgYLyj5q.js +1 -0
  40. package/public/assets/StageHeader-CIETkjVZ.js +1 -0
  41. package/public/assets/StagePill--P7PDd3s.js +1 -0
  42. package/public/assets/StageShape-BIv8vx_o.js +1 -0
  43. package/public/assets/StageSlowness-MMqzERxN.js +1 -0
  44. package/public/assets/Straggler-BgrY7AyW.js +1 -0
  45. package/public/assets/TaskFailures-BHHyKYTs.js +1 -0
  46. package/public/assets/TinyTask-DDScNhCl.js +1 -0
  47. package/public/assets/UnderBroadcast-CGIC04TI.js +1 -0
  48. package/public/assets/WidgetLeadSummary-CXFZP00C.js +1 -0
  49. package/public/assets/button-DbpTz2yk.js +1 -0
  50. package/public/assets/charts-vendor-CtTxwTeu.js +59 -0
  51. package/public/assets/composite-C9Nu2EI9.js +1 -0
  52. package/public/assets/d3-vendor-DNlD2-cz.js +1 -0
  53. package/public/assets/detector-registry-Bp8hrLua.js +2 -0
  54. package/public/assets/downsample-WmNM1qc9.js +1 -0
  55. package/public/assets/finding-anchor-Dr40LkS9.js +1 -0
  56. package/public/assets/findings-of-type-BL2u3RGZ.js +1 -0
  57. package/public/assets/impact-sort-mC5ISgX8.js +1 -0
  58. package/public/assets/inbox-KPU1Gppm.js +1 -0
  59. package/public/assets/index-CaRVPvAT.css +2 -0
  60. package/public/assets/index-fDiwfuoO.js +14 -0
  61. package/public/assets/inertValue-L9VxV0VE.js +1 -0
  62. package/public/assets/layout.worker-CLcBHxVp.js +3 -0
  63. package/public/assets/{parser-worker-CA5RxqKc.js → parser-worker-CRIOPMOW.js} +8 -8
  64. package/public/assets/plan-graph-vendor-Ce6d4HU6.js +7 -0
  65. package/public/assets/react-vendor-C6KcUD36.js +9 -0
  66. package/public/assets/select-BHqOqun5.js +1 -0
  67. package/public/assets/toggle-DPeN6aNa.js +1 -0
  68. package/public/assets/toggle-group-Kl3uyHSr.js +1 -0
  69. package/public/assets/usePagedRows-BafD6bEN.js +1 -0
  70. package/public/assets/useRecentFiles-DzJZAoOT.js +3 -0
  71. package/public/assets/useRegisterFieldControl-pQv-ZDoU.js +1 -0
  72. package/public/assets/useScrollLock-BCxn_dMp.js +1 -0
  73. package/public/assets/useSortMode-DjrAN-z4.js +1 -0
  74. package/public/docs/404.html +3 -3
  75. package/public/docs/assets/{app.CnWL6bKV.js → app.1gvzd5Zb.js} +1 -1
  76. package/public/docs/assets/chunks/@localSearchIndexroot.DS2TQP5_.js +1 -0
  77. package/public/docs/assets/chunks/{VPLocalSearchBox.C0IBKeRQ.js → VPLocalSearchBox.CNxT3s0I.js} +1 -1
  78. package/public/docs/assets/chunks/{theme.BH-2HuRf.js → theme.C5lLqUv7.js} +2 -2
  79. package/public/docs/assets/contributor-guide_architecture_board-widgets.md.CqKM_jEG.js +1 -0
  80. package/public/docs/assets/contributor-guide_architecture_board-widgets.md.CqKM_jEG.lean.js +1 -0
  81. package/public/docs/assets/{contributor-guide_architecture_board-widgets_caching-and-memory.md.CQgn1t_M.js → contributor-guide_architecture_board-widgets_caching-and-memory.md.BYDo3ClR.js} +1 -1
  82. package/public/docs/assets/contributor-guide_architecture_board-widgets_plan-advisor.md.ajUAVdbc.js +1 -0
  83. package/public/docs/assets/{contributor-guide_architecture_board-widgets_visual-system-and-templating.md.wulOueeB.js → contributor-guide_architecture_board-widgets_visual-system-and-templating.md.WqY74T0T.js} +1 -1
  84. package/public/docs/assets/contributor-guide_architecture_board-widgets_visual-system-and-templating.md.WqY74T0T.lean.js +1 -0
  85. package/public/docs/assets/contributor-guide_architecture_detector-contract.md.CHHgywoP.js +1 -0
  86. package/public/docs/assets/{contributor-guide_architecture_detector-contract.md.N4hhenrA.lean.js → contributor-guide_architecture_detector-contract.md.CHHgywoP.lean.js} +1 -1
  87. package/public/docs/assets/contributor-guide_architecture_detector-contract_plan-attribution.md.DIz3_oz9.js +1 -0
  88. package/public/docs/assets/contributor-guide_architecture_detector-contract_plan-attribution.md.DIz3_oz9.lean.js +1 -0
  89. package/public/docs/assets/contributor-guide_architecture_detector-contract_threshold-tables.md.WH4tRHPL.js +1 -0
  90. package/public/docs/assets/{contributor-guide_architecture_detector-contract_threshold-tables.md.Dp3c40rx.lean.js → contributor-guide_architecture_detector-contract_threshold-tables.md.WH4tRHPL.lean.js} +1 -1
  91. package/public/docs/assets/{contributor-guide_architecture_drill-down.md.CqZPrdo9.js → contributor-guide_architecture_drill-down.md.yB-Hz2XT.js} +1 -1
  92. package/public/docs/assets/{contributor-guide_architecture_drill-down.md.CqZPrdo9.lean.js → contributor-guide_architecture_drill-down.md.yB-Hz2XT.lean.js} +1 -1
  93. package/public/docs/assets/contributor-guide_architecture_drill-down_plan-graph.md.BQZE04sw.js +1 -0
  94. package/public/docs/assets/{contributor-guide_architecture_drill-down_plan-graph.md.DwOozIEs.lean.js → contributor-guide_architecture_drill-down_plan-graph.md.BQZE04sw.lean.js} +1 -1
  95. package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.K71YB-Rr.js +1 -0
  96. package/public/docs/assets/{contributor-guide_architecture_impact-estimation.md.DfvbqEUu.lean.js → contributor-guide_architecture_impact-estimation.md.K71YB-Rr.lean.js} +1 -1
  97. package/public/docs/assets/contributor-guide_architecture_impact-estimation_caveats-tuning-and-coverage.md.DLCAffwn.js +1 -0
  98. package/public/docs/assets/{contributor-guide_architecture_impact-estimation_caveats-tuning-and-coverage.md.Cwm1jHmn.lean.js → contributor-guide_architecture_impact-estimation_caveats-tuning-and-coverage.md.DLCAffwn.lean.js} +1 -1
  99. package/public/docs/assets/contributor-guide_architecture_impact-estimation_stage-union-rollup.md.JFrzDMwo.js +1 -0
  100. package/public/docs/assets/contributor-guide_architecture_impact-estimation_stage-union-rollup.md.JFrzDMwo.lean.js +1 -0
  101. package/public/docs/assets/contributor-guide_architecture_overview.md.nxkpOhgf.js +1 -0
  102. package/public/docs/assets/contributor-guide_architecture_state-and-history_run-comparison-internals.md.U-9KMDFK.js +1 -0
  103. package/public/docs/assets/{contributor-guide_architecture_state-and-history_run-comparison-internals.md.CZA0JLtV.lean.js → contributor-guide_architecture_state-and-history_run-comparison-internals.md.U-9KMDFK.lean.js} +1 -1
  104. package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.BrSz8Ezh.js +1 -0
  105. package/public/docs/assets/contributor-guide_architecture_widget-rendering_findings-tab.md.B7rjSF_h.js +1 -0
  106. package/public/docs/assets/contributor-guide_architecture_widget-rendering_findings-tab.md.B7rjSF_h.lean.js +1 -0
  107. package/public/docs/assets/{contributor-guide_architecture_widget-rendering_list-sort-mode.md.BuHKgH0C.js → contributor-guide_architecture_widget-rendering_list-sort-mode.md.37hcZJ7q.js} +1 -1
  108. package/public/docs/assets/{contributor-guide_architecture_widget-rendering_row-expand-contract.md.ZT89u9f1.js → contributor-guide_architecture_widget-rendering_row-expand-contract.md.mK_VmAWk.js} +1 -1
  109. package/public/docs/assets/{contributor-guide_architecture_worker-protocol.md.BAnhi2-d.js → contributor-guide_architecture_worker-protocol.md.Br6OE7RW.js} +1 -1
  110. package/public/docs/assets/contributor-guide_architecture_worker-protocol.md.Br6OE7RW.lean.js +1 -0
  111. package/public/docs/assets/contributor-guide_architecture_worker-protocol_cli-and-mcp.md.IVyaXrGe.js +1 -0
  112. package/public/docs/assets/{contributor-guide_architecture_worker-protocol_cli-and-mcp.md.BX1XnFbq.lean.js → contributor-guide_architecture_worker-protocol_cli-and-mcp.md.IVyaXrGe.lean.js} +1 -1
  113. package/public/docs/assets/{contributor-guide_architecture_worker-protocol_core-and-event-validation.md.DSVFPpWb.js → contributor-guide_architecture_worker-protocol_core-and-event-validation.md.DVSXn-bb.js} +1 -1
  114. package/public/docs/assets/{contributor-guide_architecture_worker-protocol_evidence-availability.md.CjimIDBY.js → contributor-guide_architecture_worker-protocol_evidence-availability.md.D3S5cyOW.js} +2 -2
  115. package/public/docs/assets/{contributor-guide_architecture_worker-protocol_evidence-availability.md.CjimIDBY.lean.js → contributor-guide_architecture_worker-protocol_evidence-availability.md.D3S5cyOW.lean.js} +1 -1
  116. package/public/docs/assets/contributor-guide_architecture_worker-protocol_evidence-report.md.CtmCU57I.js +1 -0
  117. package/public/docs/assets/{contributor-guide_architecture_worker-protocol_evidence-report.md.CiQl1MLb.lean.js → contributor-guide_architecture_worker-protocol_evidence-report.md.CtmCU57I.lean.js} +1 -1
  118. package/public/docs/assets/contributor-guide_contributing.md.DBW4drtZ.js +1 -0
  119. package/public/docs/assets/contributor-guide_development-setup.md.B21ZTDhn.js +15 -0
  120. package/public/docs/assets/contributor-guide_development-setup.md.B21ZTDhn.lean.js +1 -0
  121. package/public/docs/assets/contributor-guide_testing.md.DpkozNoG.js +4 -0
  122. package/public/docs/assets/{contributor-guide_testing.md.By6XGR9a.lean.js → contributor-guide_testing.md.DpkozNoG.lean.js} +1 -1
  123. package/public/docs/assets/{index.md.CHJVslga.js → index.md.lMm7SUMz.js} +1 -1
  124. package/public/docs/assets/{index.md.CHJVslga.lean.js → index.md.lMm7SUMz.lean.js} +1 -1
  125. package/public/docs/assets/{style.CyK9Ss73.css → style.Cfxqlj0y.css} +1 -1
  126. package/public/docs/assets/user-guide_getting-started_ci-and-automation.md.C0pcQoNc.js +29 -0
  127. package/public/docs/assets/{user-guide_getting-started_ci-and-automation.md.BEepbDDL.lean.js → user-guide_getting-started_ci-and-automation.md.C0pcQoNc.lean.js} +1 -1
  128. package/public/docs/assets/user-guide_getting-started_reading-the-dashboard.md.CGaxPX_1.js +1 -0
  129. package/public/docs/assets/{user-guide_getting-started_tuning-thresholds.md.B_MyjyZm.js → user-guide_getting-started_tuning-thresholds.md.BGVE-wrc.js} +2 -2
  130. package/public/docs/assets/{user-guide_getting-started_tuning-thresholds.md.B_MyjyZm.lean.js → user-guide_getting-started_tuning-thresholds.md.BGVE-wrc.lean.js} +1 -1
  131. package/public/docs/assets/{user-guide_mcp-tools.md.C65SjWUL.js → user-guide_mcp-tools.md.49r3L2YE.js} +1 -1
  132. package/public/docs/assets/{user-guide_mcp-tools_compare-runs.md.Cc_gLoix.js → user-guide_mcp-tools_compare-runs.md.BQ_kIptC.js} +11 -6
  133. package/public/docs/assets/user-guide_mcp-tools_compare-runs.md.BQ_kIptC.lean.js +1 -0
  134. package/public/docs/assets/user-guide_mcp-tools_diagnose-run.md.azCZXkH8.js +44 -0
  135. package/public/docs/assets/{user-guide_mcp-tools_diagnose-run.md.BP1vnHA5.lean.js → user-guide_mcp-tools_diagnose-run.md.azCZXkH8.lean.js} +1 -1
  136. package/public/docs/assets/{user-guide_mcp-tools_evaluate-budgets.md.BB0nDY3Z.js → user-guide_mcp-tools_evaluate-budgets.md.dpQpRQ7z.js} +2 -2
  137. package/public/docs/assets/{user-guide_mcp-tools_evaluate-budgets.md.BB0nDY3Z.lean.js → user-guide_mcp-tools_evaluate-budgets.md.dpQpRQ7z.lean.js} +1 -1
  138. package/public/docs/assets/{user-guide_mcp-tools_get-finding-evidence.md.rKyu4wts.js → user-guide_mcp-tools_get-finding-evidence.md.CH6FkNLX.js} +2 -1
  139. package/public/docs/assets/{user-guide_mcp-tools_get-finding-evidence.md.rKyu4wts.lean.js → user-guide_mcp-tools_get-finding-evidence.md.CH6FkNLX.lean.js} +1 -1
  140. package/public/docs/assets/user-guide_run-comparison.md.B_LOkdBf.js +1 -0
  141. package/public/docs/assets/user-guide_run-comparison.md.B_LOkdBf.lean.js +1 -0
  142. package/public/docs/assets/user-guide_understanding-findings.md.C4riKCKl.js +1 -0
  143. package/public/docs/assets/user-guide_understanding-findings.md.C4riKCKl.lean.js +1 -0
  144. package/public/docs/contributor-guide/architecture/board-widgets/caching-and-memory.html +6 -6
  145. package/public/docs/contributor-guide/architecture/board-widgets/plan-advisor.html +6 -6
  146. package/public/docs/contributor-guide/architecture/board-widgets/visual-system-and-templating.html +6 -6
  147. package/public/docs/contributor-guide/architecture/board-widgets.html +6 -6
  148. package/public/docs/contributor-guide/architecture/detector-contract/plan-attribution.html +6 -6
  149. package/public/docs/contributor-guide/architecture/detector-contract/threshold-tables.html +6 -6
  150. package/public/docs/contributor-guide/architecture/detector-contract.html +6 -6
  151. package/public/docs/contributor-guide/architecture/drill-down/plan-graph.html +6 -6
  152. package/public/docs/contributor-guide/architecture/drill-down.html +6 -6
  153. package/public/docs/contributor-guide/architecture/impact-estimation/caveats-tuning-and-coverage.html +6 -6
  154. package/public/docs/contributor-guide/architecture/impact-estimation/stage-union-rollup.html +6 -6
  155. package/public/docs/contributor-guide/architecture/impact-estimation.html +6 -6
  156. package/public/docs/contributor-guide/architecture/index.html +5 -5
  157. package/public/docs/contributor-guide/architecture/overview.html +6 -6
  158. package/public/docs/contributor-guide/architecture/state-and-history/history-server-intake.html +5 -5
  159. package/public/docs/contributor-guide/architecture/state-and-history/run-comparison-internals.html +6 -6
  160. package/public/docs/contributor-guide/architecture/state-and-history.html +5 -5
  161. package/public/docs/contributor-guide/architecture/widget-rendering/findings-tab.html +6 -6
  162. package/public/docs/contributor-guide/architecture/widget-rendering/investigation-routing.html +5 -5
  163. package/public/docs/contributor-guide/architecture/widget-rendering/list-sort-mode.html +6 -6
  164. package/public/docs/contributor-guide/architecture/widget-rendering/reference-section.html +5 -5
  165. package/public/docs/contributor-guide/architecture/widget-rendering/row-expand-contract.html +6 -6
  166. package/public/docs/contributor-guide/architecture/widget-rendering.html +6 -6
  167. package/public/docs/contributor-guide/architecture/worker-protocol/cli-and-mcp.html +6 -6
  168. package/public/docs/contributor-guide/architecture/worker-protocol/core-and-event-validation.html +6 -6
  169. package/public/docs/contributor-guide/architecture/worker-protocol/evidence-availability.html +7 -7
  170. package/public/docs/contributor-guide/architecture/worker-protocol/evidence-report.html +6 -6
  171. package/public/docs/contributor-guide/architecture/worker-protocol.html +6 -6
  172. package/public/docs/contributor-guide/contributing.html +6 -6
  173. package/public/docs/contributor-guide/development-setup.html +8 -8
  174. package/public/docs/contributor-guide/testing.html +9 -6
  175. package/public/docs/hashmap.json +1 -1
  176. package/public/docs/index.html +6 -6
  177. package/public/docs/logo-dark.svg +1 -0
  178. package/public/docs/logo-light.svg +1 -0
  179. package/public/docs/tuning-reference/anti-patterns.html +5 -5
  180. package/public/docs/tuning-reference/aqe.html +5 -5
  181. package/public/docs/tuning-reference/bottleneck-broadcast-sizing.html +5 -5
  182. package/public/docs/tuning-reference/bottleneck-cold-start.html +5 -5
  183. package/public/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html +5 -5
  184. package/public/docs/tuning-reference/bottleneck-failures.html +5 -5
  185. package/public/docs/tuning-reference/bottleneck-gc.html +5 -5
  186. package/public/docs/tuning-reference/bottleneck-job-failure-rate.html +5 -5
  187. package/public/docs/tuning-reference/bottleneck-memory-utilization.html +5 -5
  188. package/public/docs/tuning-reference/bottleneck-retry-waste.html +5 -5
  189. package/public/docs/tuning-reference/bottleneck-shuffle.html +5 -5
  190. package/public/docs/tuning-reference/bottleneck-skew.html +5 -5
  191. package/public/docs/tuning-reference/bottleneck-slow-host.html +5 -5
  192. package/public/docs/tuning-reference/bottleneck-small-files.html +5 -5
  193. package/public/docs/tuning-reference/bottleneck-spill.html +5 -5
  194. package/public/docs/tuning-reference/bottleneck-straggler.html +5 -5
  195. package/public/docs/tuning-reference/bottleneck-tiny-tasks.html +5 -5
  196. package/public/docs/tuning-reference/bottleneck-utilization.html +5 -5
  197. package/public/docs/tuning-reference/caching.html +5 -5
  198. package/public/docs/tuning-reference/cluster-config.html +5 -5
  199. package/public/docs/tuning-reference/config.html +5 -5
  200. package/public/docs/tuning-reference/data-formats.html +5 -5
  201. package/public/docs/tuning-reference/index.html +5 -5
  202. package/public/docs/tuning-reference/intro.html +5 -5
  203. package/public/docs/tuning-reference/joins.html +5 -5
  204. package/public/docs/tuning-reference/memory-model.html +5 -5
  205. package/public/docs/tuning-reference/metrics.html +5 -5
  206. package/public/docs/tuning-reference/partitioning.html +5 -5
  207. package/public/docs/tuning-reference/pyspark.html +5 -5
  208. package/public/docs/tuning-reference/shuffle.html +5 -5
  209. package/public/docs/tuning-reference/spark-architecture.html +5 -5
  210. package/public/docs/tuning-reference/table-formats.html +5 -5
  211. package/public/docs/user-guide/alternative-log-retrieval/managed-platforms.html +5 -5
  212. package/public/docs/user-guide/alternative-log-retrieval/ssh-bastion.html +5 -5
  213. package/public/docs/user-guide/alternative-log-retrieval.html +5 -5
  214. package/public/docs/user-guide/getting-started/ci-and-automation.html +16 -11
  215. package/public/docs/user-guide/getting-started/reading-the-dashboard.html +6 -6
  216. package/public/docs/user-guide/getting-started/regression-metric-keys.html +5 -5
  217. package/public/docs/user-guide/getting-started/tuning-thresholds.html +7 -7
  218. package/public/docs/user-guide/getting-started.html +5 -5
  219. package/public/docs/user-guide/mcp-tools/compare-runs.html +16 -11
  220. package/public/docs/user-guide/mcp-tools/diagnose-run.html +28 -7
  221. package/public/docs/user-guide/mcp-tools/evaluate-budgets.html +6 -6
  222. package/public/docs/user-guide/mcp-tools/get-finding-documentation.html +5 -5
  223. package/public/docs/user-guide/mcp-tools/get-finding-evidence.html +7 -6
  224. package/public/docs/user-guide/mcp-tools/get-reference-doc.html +5 -5
  225. package/public/docs/user-guide/mcp-tools/get-run-summary.html +5 -5
  226. package/public/docs/user-guide/mcp-tools/list-runs.html +5 -5
  227. package/public/docs/user-guide/mcp-tools.html +6 -6
  228. package/public/docs/user-guide/run-comparison/how-stages-are-matched.html +5 -5
  229. package/public/docs/user-guide/run-comparison.html +6 -6
  230. package/public/docs/user-guide/understanding-findings.html +6 -6
  231. package/public/export-template.html +45 -45
  232. package/public/index.html +8 -7
  233. package/public/sample-runs/sample-run.ndjson.gz +0 -0
  234. package/vendor-core/allocation.js +25 -23
  235. package/vendor-core/analyzer.js +85 -58
  236. package/vendor-core/api-merge-writes.js +2 -0
  237. package/vendor-core/aqe-skew.js +347 -0
  238. package/vendor-core/cli/budgets.js +6 -4
  239. package/vendor-core/cli/collect-run-pair.js +89 -0
  240. package/vendor-core/cli/collect-run-worker.js +16 -0
  241. package/vendor-core/cli/collect-run.js +1 -2
  242. package/vendor-core/cli/native-zstd.js +12 -16
  243. package/vendor-core/comparability.js +234 -0
  244. package/vendor-core/comparison-output.js +15 -7
  245. package/vendor-core/comparison-verdict.js +105 -25
  246. package/vendor-core/core-source-hash.txt +1 -1
  247. package/vendor-core/core-time-series.js +0 -1
  248. package/vendor-core/core-usage-locality.js +1 -1
  249. package/vendor-core/detectors.js +1007 -380
  250. package/vendor-core/docs-config.js +0 -4
  251. package/vendor-core/docs-content/detection/cache.md +2 -1
  252. package/vendor-core/docs-content/detection/cfg.md +19 -11
  253. package/vendor-core/docs-content/detection/cold.md +2 -1
  254. package/vendor-core/docs-content/detection/gc.md +7 -3
  255. package/vendor-core/docs-content/detection/host.md +11 -1
  256. package/vendor-core/docs-content/detection/mem.md +17 -6
  257. package/vendor-core/docs-content/detection/part.md +3 -1
  258. package/vendor-core/docs-content/detection/plan.md +55 -13
  259. package/vendor-core/docs-content/detection/shfl.md +11 -5
  260. package/vendor-core/docs-content/detection/skew.md +47 -1
  261. package/vendor-core/docs-content/detection/slow.md +4 -2
  262. package/vendor-core/docs-content/detection/spec.md +6 -1
  263. package/vendor-core/docs-content/detection/spill.md +6 -3
  264. package/vendor-core/docs-content/detection/strag.md +19 -6
  265. package/vendor-core/docs-content/detection/tiny.md +9 -3
  266. package/vendor-core/effective-conf.js +7 -10
  267. package/vendor-core/efficiency-model.js +2 -2
  268. package/vendor-core/etl-phases.js +1 -2
  269. package/vendor-core/event-handlers.js +175 -57
  270. package/vendor-core/event-schemas.js +21 -16
  271. package/vendor-core/evidence-availability.js +9 -2
  272. package/vendor-core/evidence-report.js +118 -35
  273. package/vendor-core/executor-peaks.js +41 -0
  274. package/vendor-core/export-data.js +5 -5
  275. package/vendor-core/finding-identity.js +78 -0
  276. package/vendor-core/finding-presentation.js +116 -33
  277. package/vendor-core/finding-types.js +88 -21
  278. package/vendor-core/format-utils.js +14 -15
  279. package/vendor-core/html-export.js +1 -1
  280. package/vendor-core/impact-band.js +2 -2
  281. package/vendor-core/impact-format.js +47 -19
  282. package/vendor-core/impact-model.js +26 -15
  283. package/vendor-core/job-groups.js +4 -4
  284. package/vendor-core/mcp-server-factory.js +48 -12
  285. package/vendor-core/mcp-tools.js +179 -24
  286. package/vendor-core/median.js +2 -2
  287. package/vendor-core/nested-loop-join.js +73 -0
  288. package/vendor-core/occupancy.js +1 -2
  289. package/vendor-core/parser-worker.js +1 -1
  290. package/vendor-core/plan-graph-model.js +14 -22
  291. package/vendor-core/plan-node-detail.js +2 -3
  292. package/vendor-core/plan-summary.js +24 -19
  293. package/vendor-core/plan-tree-walk.js +6 -8
  294. package/vendor-core/python-stage.js +12 -1
  295. package/vendor-core/recommendation-rollup.js +13 -12
  296. package/vendor-core/redact.js +52 -46
  297. package/vendor-core/run-aggregates.js +6 -8
  298. package/vendor-core/run-comparison.js +226 -66
  299. package/vendor-core/run-identity.js +12 -0
  300. package/vendor-core/run-interpretation.js +21 -0
  301. package/vendor-core/run-metrics.js +2 -2
  302. package/vendor-core/run-output.js +32 -10
  303. package/vendor-core/run-totals.js +12 -0
  304. package/vendor-core/run-verdict.js +76 -3
  305. package/vendor-core/scaling-sim.js +3 -4
  306. package/vendor-core/snappy-block.js +2 -2
  307. package/vendor-core/spark-conf.js +168 -0
  308. package/vendor-core/sql-stages.js +2 -2
  309. package/vendor-core/stage-alignment.js +2 -12
  310. package/vendor-core/stage-identity.js +58 -3
  311. package/vendor-core/stage-label.js +14 -0
  312. package/vendor-core/stage-plan-nodes.js +17 -3
  313. package/vendor-core/stage-quantiles.js +141 -9
  314. package/vendor-core/stage-structure.js +17 -3
  315. package/vendor-core/threshold-overrides.js +9 -3
  316. package/vendor-core/types.js +58 -21
  317. package/vendor-core/wasted-core-hours.js +0 -1
  318. package/vendor-core/write-targets.js +114 -14
  319. package/vendor-core/zstd-worker-client.js +1 -1
  320. package/vendor-core/zstd-worker.js +1 -1
  321. package/public/assets/AutoscalingChurn-CRA2Mb7F.js +0 -1
  322. package/public/assets/CacheUtilization-Bh9OY_V9.js +0 -1
  323. package/public/assets/CachingOpportunity-oCiQwmTJ.js +0 -1
  324. package/public/assets/ColdStart-BJrtjfmn.js +0 -1
  325. package/public/assets/ConfigAudit-DMvKxySP.js +0 -1
  326. package/public/assets/CoreUsageArea-REC2zsKw.js +0 -1
  327. package/public/assets/Dashboard-BHZ76JCu.js +0 -2
  328. package/public/assets/DisclosureContext-OO7X1HAx.js +0 -1
  329. package/public/assets/DuplicatePlanSubtree-BgG-L-cK.js +0 -1
  330. package/public/assets/ExecutorUtilization-D2KhSVbY.js +0 -1
  331. package/public/assets/GcPressure-D1BTE8qh.js +0 -1
  332. package/public/assets/IncompleteRun-DD40xUpy.js +0 -1
  333. package/public/assets/JobFailures-YwNzCF19.js +0 -1
  334. package/public/assets/MemoryUtilization-9b8ee_3_.js +0 -1
  335. package/public/assets/OverBroadcast-CfctS8a8.js +0 -1
  336. package/public/assets/PartitionHint-CUvbnnxy.js +0 -1
  337. package/public/assets/PartitionSizing-VG8kBgG5.js +0 -1
  338. package/public/assets/PlanExplorer-DmubTAOP.js +0 -1
  339. package/public/assets/PlanGraphRoute-DctPZjS0.js +0 -1
  340. package/public/assets/RetryWaste-CnpM8bB7.js +0 -1
  341. package/public/assets/ShuffleIO-D13vGmas.js +0 -1
  342. package/public/assets/Skew-DlagstYP.js +0 -1
  343. package/public/assets/SlowHost-cSkLRZ_e.js +0 -1
  344. package/public/assets/SmallFiles-B-kERMcq.js +0 -1
  345. package/public/assets/SpeculationWaste-DaysAzXO.js +0 -1
  346. package/public/assets/Spill-H44Gz4US.js +0 -1
  347. package/public/assets/StageFailed-DCfii9h_.js +0 -1
  348. package/public/assets/StageFindingGroup-BXJ0Mj5n.js +0 -1
  349. package/public/assets/StageHeader--oN2J-oD.js +0 -1
  350. package/public/assets/StageShape-IZ80CZiJ.js +0 -1
  351. package/public/assets/StageSlowness--BRIaBc1.js +0 -1
  352. package/public/assets/Straggler-Ctq0Df_o.js +0 -1
  353. package/public/assets/TaskFailures-Bks_gqhL.js +0 -1
  354. package/public/assets/TinyTask-BssfzGFB.js +0 -1
  355. package/public/assets/UnderBroadcast-DsScrxc-.js +0 -1
  356. package/public/assets/charts-vendor-0VMaLVLN.js +0 -59
  357. package/public/assets/finding-anchor-BYW1oz5R.js +0 -1
  358. package/public/assets/impact-sort-xzy0g-Vh.js +0 -1
  359. package/public/assets/index-DhpRGMZc.css +0 -2
  360. package/public/assets/index-DwqkFUFk.js +0 -14
  361. package/public/assets/plan-graph-vendor-BYl-AzN5.js +0 -7
  362. package/public/assets/react-vendor-DfpuPQZf.js +0 -9
  363. package/public/assets/select-C8edjYTq.js +0 -1
  364. package/public/assets/usePagedRows-CsuUFeYx.js +0 -1
  365. package/public/assets/useRecentFiles-BvkKo3JN.js +0 -11
  366. package/public/assets/useSortMode-joX71BZN.js +0 -1
  367. package/public/docs/assets/chunks/@localSearchIndexroot.tE5pz54z.js +0 -1
  368. package/public/docs/assets/contributor-guide_architecture_board-widgets.md.YQcj5EI4.js +0 -1
  369. package/public/docs/assets/contributor-guide_architecture_board-widgets.md.YQcj5EI4.lean.js +0 -1
  370. package/public/docs/assets/contributor-guide_architecture_board-widgets_plan-advisor.md.RlzvUmNm.js +0 -1
  371. package/public/docs/assets/contributor-guide_architecture_board-widgets_visual-system-and-templating.md.wulOueeB.lean.js +0 -1
  372. package/public/docs/assets/contributor-guide_architecture_detector-contract.md.N4hhenrA.js +0 -1
  373. package/public/docs/assets/contributor-guide_architecture_detector-contract_plan-attribution.md.yNex4bys.js +0 -1
  374. package/public/docs/assets/contributor-guide_architecture_detector-contract_plan-attribution.md.yNex4bys.lean.js +0 -1
  375. package/public/docs/assets/contributor-guide_architecture_detector-contract_threshold-tables.md.Dp3c40rx.js +0 -1
  376. package/public/docs/assets/contributor-guide_architecture_drill-down_plan-graph.md.DwOozIEs.js +0 -1
  377. package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.DfvbqEUu.js +0 -1
  378. package/public/docs/assets/contributor-guide_architecture_impact-estimation_caveats-tuning-and-coverage.md.Cwm1jHmn.js +0 -1
  379. package/public/docs/assets/contributor-guide_architecture_impact-estimation_stage-union-rollup.md.DdCGtAGu.js +0 -1
  380. package/public/docs/assets/contributor-guide_architecture_impact-estimation_stage-union-rollup.md.DdCGtAGu.lean.js +0 -1
  381. package/public/docs/assets/contributor-guide_architecture_overview.md.C1sSvm9i.js +0 -1
  382. package/public/docs/assets/contributor-guide_architecture_state-and-history_run-comparison-internals.md.CZA0JLtV.js +0 -1
  383. package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.BOZ1azXG.js +0 -1
  384. package/public/docs/assets/contributor-guide_architecture_widget-rendering_findings-tab.md.BQXycK6G.js +0 -1
  385. package/public/docs/assets/contributor-guide_architecture_widget-rendering_findings-tab.md.BQXycK6G.lean.js +0 -1
  386. package/public/docs/assets/contributor-guide_architecture_worker-protocol.md.BAnhi2-d.lean.js +0 -1
  387. package/public/docs/assets/contributor-guide_architecture_worker-protocol_cli-and-mcp.md.BX1XnFbq.js +0 -1
  388. package/public/docs/assets/contributor-guide_architecture_worker-protocol_evidence-report.md.CiQl1MLb.js +0 -1
  389. package/public/docs/assets/contributor-guide_contributing.md.CQMvGwFt.js +0 -1
  390. package/public/docs/assets/contributor-guide_development-setup.md.DQdzera7.js +0 -15
  391. package/public/docs/assets/contributor-guide_development-setup.md.DQdzera7.lean.js +0 -1
  392. package/public/docs/assets/contributor-guide_testing.md.By6XGR9a.js +0 -1
  393. package/public/docs/assets/user-guide_getting-started_ci-and-automation.md.BEepbDDL.js +0 -24
  394. package/public/docs/assets/user-guide_getting-started_reading-the-dashboard.md.Obf3W3__.js +0 -1
  395. package/public/docs/assets/user-guide_mcp-tools_compare-runs.md.Cc_gLoix.lean.js +0 -1
  396. package/public/docs/assets/user-guide_mcp-tools_diagnose-run.md.BP1vnHA5.js +0 -23
  397. package/public/docs/assets/user-guide_run-comparison.md.BUALkohi.js +0 -1
  398. package/public/docs/assets/user-guide_run-comparison.md.BUALkohi.lean.js +0 -1
  399. package/public/docs/assets/user-guide_understanding-findings.md.3C7KCEZO.js +0 -1
  400. package/public/docs/assets/user-guide_understanding-findings.md.3C7KCEZO.lean.js +0 -1
  401. /package/public/docs/assets/{contributor-guide_architecture_board-widgets_caching-and-memory.md.CQgn1t_M.lean.js → contributor-guide_architecture_board-widgets_caching-and-memory.md.BYDo3ClR.lean.js} +0 -0
  402. /package/public/docs/assets/{contributor-guide_architecture_board-widgets_plan-advisor.md.RlzvUmNm.lean.js → contributor-guide_architecture_board-widgets_plan-advisor.md.ajUAVdbc.lean.js} +0 -0
  403. /package/public/docs/assets/{contributor-guide_architecture_overview.md.C1sSvm9i.lean.js → contributor-guide_architecture_overview.md.nxkpOhgf.lean.js} +0 -0
  404. /package/public/docs/assets/{contributor-guide_architecture_widget-rendering.md.BOZ1azXG.lean.js → contributor-guide_architecture_widget-rendering.md.BrSz8Ezh.lean.js} +0 -0
  405. /package/public/docs/assets/{contributor-guide_architecture_widget-rendering_list-sort-mode.md.BuHKgH0C.lean.js → contributor-guide_architecture_widget-rendering_list-sort-mode.md.37hcZJ7q.lean.js} +0 -0
  406. /package/public/docs/assets/{contributor-guide_architecture_widget-rendering_row-expand-contract.md.ZT89u9f1.lean.js → contributor-guide_architecture_widget-rendering_row-expand-contract.md.mK_VmAWk.lean.js} +0 -0
  407. /package/public/docs/assets/{contributor-guide_architecture_worker-protocol_core-and-event-validation.md.DSVFPpWb.lean.js → contributor-guide_architecture_worker-protocol_core-and-event-validation.md.DVSXn-bb.lean.js} +0 -0
  408. /package/public/docs/assets/{contributor-guide_contributing.md.CQMvGwFt.lean.js → contributor-guide_contributing.md.DBW4drtZ.lean.js} +0 -0
  409. /package/public/docs/assets/{user-guide_getting-started_reading-the-dashboard.md.Obf3W3__.lean.js → user-guide_getting-started_reading-the-dashboard.md.CGaxPX_1.lean.js} +0 -0
  410. /package/public/docs/assets/{user-guide_mcp-tools.md.C65SjWUL.lean.js → user-guide_mcp-tools.md.49r3L2YE.lean.js} +0 -0
package/public/docs/tuning-reference/bottleneck-spill.html CHANGED
@@ -6,11 +6,11 @@
6
6
  <title>Memory / Disk Spill | Spark Tuning Reference</title>
7
7
  <meta name="description" content="Detecting and mitigating memory/disk spill: TaskMemoryManager pressure, the skew-vs-volume spill classification, and matching the fix to the cause.">
8
8
  <meta name="generator" content="VitePress v1.6.4">
9
- <link rel="preload stylesheet" href="/docs/assets/style.CyK9Ss73.css" as="style">
9
+ <link rel="preload stylesheet" href="/docs/assets/style.Cfxqlj0y.css" as="style">
10
10
  <link rel="preload stylesheet" href="/docs/vp-icons.css" as="style">
11
11
 
12
- <script type="module" src="/docs/assets/app.CnWL6bKV.js"></script>
13
- <link rel="modulepreload" href="/docs/assets/chunks/theme.BH-2HuRf.js">
12
+ <script type="module" src="/docs/assets/app.1gvzd5Zb.js"></script>
13
+ <link rel="modulepreload" href="/docs/assets/chunks/theme.C5lLqUv7.js">
14
14
  <link rel="modulepreload" href="/docs/assets/chunks/framework.DSg0KOwT.js">
15
15
  <link rel="modulepreload" href="/docs/assets/tuning-reference_bottleneck-spill.md.Bl8KtSRg.lean.js">
16
16
  <script>(()=>{const c=`${`[${["data","shuffle","product","bar"].join("-")}]`}, .VPNav, .VPLocalNav`,a=24,d=document.documentElement,l=t=>{const e=getComputedStyle(t);return e.display==="none"||t.getClientRects().length===0?0:e.position==="fixed"?t.getBoundingClientRect().bottom:e.position==="sticky"?(parseFloat(e.top)||0)+t.offsetHeight:0},u=()=>{try{return location.hash?document.getElementById(decodeURIComponent(location.hash.slice(1))):null}catch{return null}};let r=null;const s=()=>{const t=Array.from(document.querySelectorAll(c),l),e=Math.ceil(Math.max(0,...t))+a;if(e===r)return;const n=r;r=e,d.style.setProperty("--sf-anchor-offset",`${r}px`);const o=n===null?null:u();if(!o)return;const f=(parseFloat(getComputedStyle(o).paddingTop)||0)+2;Math.abs(o.getBoundingClientRect().top-n)<=f&&o.scrollIntoView()},i=()=>{const t=document.createElement("div");t.setAttribute("data-sf-anchor-offset",""),t.setAttribute("aria-hidden","true"),document.body.appendChild(t);const e=window.ResizeObserver?new ResizeObserver(s):null,n=()=>{e&&document.querySelectorAll(c).forEach(o=>e.observe(o)),s()};n(),new MutationObserver(n).observe(document.body,{childList:!0,subtree:!0}),window.addEventListener("resize",s),window.addEventListener("load",s)};document.readyState==="loading"?document.addEventListener("DOMContentLoaded",i,{once:!0}):i()})();</script>
@@ -19,13 +19,13 @@
19
19
  <script id="check-mac-os">document.documentElement.classList.toggle("mac",/Mac|iPhone|iPod|iPad/i.test(navigator.platform));</script>
20
20
  </head>
21
21
  <body>
22
- <div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Getting Started</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/intro.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Introduction</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Optimization Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/spark-architecture.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Execution Model</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/memory-model.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Management</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/partitioning.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Partitioning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/joins.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Join Optimization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/data-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Data Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/table-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Table Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/caching.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Caching & Persistence</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/pyspark.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>PySpark Specifics</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/aqe.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Adaptive Query Execution</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/cluster-config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cluster Tuning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/anti-patterns.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Anti-Patterns</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Detector Catalog</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-skew.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Skew</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle I/O</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-spill.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory / Disk Spill</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-gc.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>GC Pressure</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-cold-start.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cold Start</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Executor Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-slow-host.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Slow Host</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-failures.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Failures</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-straggler.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Stragglers</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-retry-waste.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Retry Waste</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-tiny-tasks.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Tiny Tasks</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-job-failure-rate.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Job Failure Rate</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-memory-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Duplicate Plan Subtree</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-small-files.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Small Files</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-broadcast-sizing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Broadcast Sizing</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Reference</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/metrics.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Metrics Glossary</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Config Quick-Reference</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_tuning-reference_bottleneck-spill" data-v-39a288b8><div><h1 id="bottleneck-spill" tabindex="-1">Memory / Disk Spill <a class="header-anchor" href="#bottleneck-spill" aria-label="Permalink to &quot;Memory / Disk Spill {#bottleneck-spill}&quot;">​</a></h1><p><span class="tag">SPILL</span></p><h2 id="what-it-is" tabindex="-1">What it is <a class="header-anchor" href="#what-it-is" aria-label="Permalink to &quot;What it is&quot;">​</a></h2><p>Spill happens when a task&#39;s share of execution memory runs out during a sort or hash aggregation. Each task gets its own <code>TaskMemoryManager</code>, which arbitrates the shared execution pool across every task running concurrently on an executor; with <code>n</code> tasks running concurrently, each is allowed to allocate somewhere between <code>1/(2n)</code> and <code>1/n</code> of total execution memory: a soft limit, not a hard stop.<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup> When a task&#39;s sort or hash-aggregation operator keeps requesting execution-memory pages it can&#39;t get, Spark doesn&#39;t fail immediately: it blocks the requesting task, triggers a spill of that task&#39;s in-memory data structure to disk, or, in the worst case, throws an <code>OutOfMemoryError</code>.<sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup></p><p>The spill path runs through Spark&#39;s map/shuffle I/O machinery: shuffle partitions created by wide transformations like <code>groupBy()</code> or <code>join()</code> spill to the executors&#39; local disks, at the location set by <code>spark.local.directory</code>.<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup> SQL physical operators apply the same idea with their own guardrails: <code>SortMergeJoinExec</code>&#39;s in-memory buffer and the cartesian-product operator&#39;s buffer both spill once they cross a configured row-count threshold, which by default is <code>spark.shuffle.spill.numElementsForceSpillThreshold</code>.<sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup> Whatever the trigger, the resulting spilled bytes are exposed on the task metrics as <code>memoryBytesSpilled</code>, visible in the Spark UI and event log.<sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup></p><h2 id="how-it-s-detected" tabindex="-1">How it&#39;s detected <a class="header-anchor" href="#how-it-s-detected" aria-label="Permalink to &quot;How it&#39;s detected&quot;">​</a></h2><p>Detection: any <code>memoryBytesSpilled &gt; 0</code> → Warning. The classification is the actionable signal.</p><p>Spill classification:</p><ul><li><code>skew</code>: ≥ 80% of tasks have zero spill; fix: address <a href="./bottleneck-skew.html">task skew</a>, not memory.</li><li><code>volume</code>: &lt; 20% of tasks have zero spill; fix: more partitions or more memory.</li><li><code>unclassified</code>: neither condition; treat as volume.</li></ul><img class="light-only" src="/docs/assets/spill-classification.BU2euYDO.svg" alt="How a nonzero memoryBytesSpilled is classified as skew, volume, or unclassified from the share of tasks with zero spill, and the fix each classification points to."><img class="dark-only" src="/docs/assets/spill-classification.dark.D7i1M40d.svg" alt="How a nonzero memoryBytesSpilled is classified as skew, volume, or unclassified from the share of tasks with zero spill, and the fix each classification points to."><h2 id="why-it-matters" tabindex="-1">Why it matters <a class="header-anchor" href="#why-it-matters" aria-label="Permalink to &quot;Why it matters&quot;">​</a></h2><p>Spill is the fallback Spark reaches for once a task can no longer get the execution memory it&#39;s asking for: rather than failing outright, it blocks the task, writes its buffered data to disk, or (if that&#39;s not enough) throws an <code>OutOfMemoryError</code>.<sup class="footnote-ref"><a href="#fn1" id="fnref1:2">[1:2]</a></sup> A task that&#39;s spilling has already given up pure in-memory speed to keep running at all, which is why the skew-vs-volume split above is the actionable part of the signal: the two failure shapes call for opposite fixes.</p><h2 id="how-to-fix-it" tabindex="-1">How to fix it <a class="header-anchor" href="#how-to-fix-it" aria-label="Permalink to &quot;How to fix it&quot;">​</a></h2><ul><li><strong>Skew</strong>: When only a few tasks spill, the problem is the shape of the data, not the size of the memory pool. <code>repartition()</code> is the tool for this: reach for it specifically to fix a lopsided partition distribution or to increase partition count<sup class="footnote-ref"><a href="#fn5" id="fnref5">[5]</a></sup>, in contrast to <code>coalesce()</code>, which never rebalances skewed data because it just stacks existing partitions together: partitions that were uneven going in are still uneven coming out.<sup class="footnote-ref"><a href="#fn6" id="fnref6">[6]</a></sup> For skewed joins, <a href="./aqe.html">Adaptive Query Execution</a> can detect and split<sup class="footnote-ref"><a href="#fn7" id="fnref7">[7]</a></sup> oversized partitions automatically: a partition counts as skewed once it&#39;s larger than <code>spark.sql.adaptive.skewJoin.skewedPartitionFactor</code> (default <code>5.0</code>) times the median partition size, and also larger than <code>spark.sql.adaptive.skewJoin.skewedPartitionThresholdInBytes</code> (default <code>256 MB</code>).<sup class="footnote-ref"><a href="#fn8" id="fnref8">[8]</a></sup><sup class="footnote-ref"><a href="#fn9" id="fnref9">[9]</a></sup> Before AQE existed, the manual equivalent was salting: adding a prefix to the skewed keys to make the same key look different, then adjusting the data distribution accordingly.<sup class="footnote-ref"><a href="#fn7" id="fnref7:1">[7:1]</a></sup> Databricks&#39; <code>/*+ SKEW(...) */</code> hint lets you name the skewed relation and specific key values directly, so the planner targets just those keys instead of relying on automatic detection.<sup class="footnote-ref"><a href="#fn10" id="fnref10">[10]</a></sup></li><li><strong>Volume</strong>: When most tasks spill, the fix is more room to work with: more partitions, more memory, or both. <code>repartition(n)</code> guarantees exactly <code>n</code> output partitions via a hash shuffle<sup class="footnote-ref"><a href="#fn11" id="fnref11">[11]</a></sup>, and since Spark&#39;s per-task startup overhead is low (unlike MapReduce&#39;s), the general bias is to err toward more partitions rather than fewer.<sup class="footnote-ref"><a href="#fn12" id="fnref12">[12]</a></sup><code>spark.sql.files.maxPartitionBytes</code> (default <code>128 MB</code>) is the equivalent lever on the input side, governing how much file data gets packed into each input partition before a shuffle even happens.<sup class="footnote-ref"><a href="#fn6" id="fnref6:1">[6:1]</a></sup> On the memory side, execution memory is a fraction of the JVM heap. <code>spark.memory.fraction</code> (default <code>0.6</code>) sizes the shared execution/storage region as <code>(JVM heap − 300 MiB) × spark.memory.fraction</code><sup class="footnote-ref"><a href="#fn13" id="fnref13">[13]</a></sup>. Because each task&#39;s slice of that region is capped by the <code>TaskMemoryManager</code> at between <code>1/(2n)</code> and <code>1/n</code> of the total,<sup class="footnote-ref"><a href="#fn1" id="fnref1:3">[1:3]</a></sup> giving executors more memory, or running fewer concurrent tasks on each one, directly raises the ceiling before spill kicks in.</li></ul><p>For the <strong>volume</strong> case, give tasks more room: defaults shown, comments say which way to move:</p><div class="language-properties vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">properties</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Create more, smaller input partitions before the shuffle (default 128m)</span></span>
22
+ <div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><!--[--><!--[--><img class="VPImage dark logo" src="/docs/logo-dark.svg" alt data-v-8426fc1a><!--]--><!--[--><img class="VPImage light logo" src="/docs/logo-light.svg" alt data-v-8426fc1a><!--]--><!--]--><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/.." target="_self" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Open app</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Getting Started</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/intro.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Introduction</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Optimization Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/spark-architecture.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Execution Model</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/memory-model.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Management</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/partitioning.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Partitioning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/joins.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Join Optimization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/data-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Data Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/table-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Table Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/caching.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Caching & Persistence</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/pyspark.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>PySpark Specifics</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/aqe.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Adaptive Query Execution</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/cluster-config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cluster Tuning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/anti-patterns.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Anti-Patterns</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Detector Catalog</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-skew.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Skew</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle I/O</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-spill.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory / Disk Spill</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-gc.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>GC Pressure</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-cold-start.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cold Start</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Executor Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-slow-host.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Slow Host</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-failures.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Failures</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-straggler.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Stragglers</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-retry-waste.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Retry Waste</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-tiny-tasks.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Tiny Tasks</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-job-failure-rate.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Job Failure Rate</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-memory-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Duplicate Plan Subtree</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-small-files.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Small Files</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-broadcast-sizing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Broadcast Sizing</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Reference</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/metrics.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Metrics Glossary</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Config Quick-Reference</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_tuning-reference_bottleneck-spill" data-v-39a288b8><div><h1 id="bottleneck-spill" tabindex="-1">Memory / Disk Spill <a class="header-anchor" href="#bottleneck-spill" aria-label="Permalink to &quot;Memory / Disk Spill {#bottleneck-spill}&quot;">​</a></h1><p><span class="tag">SPILL</span></p><h2 id="what-it-is" tabindex="-1">What it is <a class="header-anchor" href="#what-it-is" aria-label="Permalink to &quot;What it is&quot;">​</a></h2><p>Spill happens when a task&#39;s share of execution memory runs out during a sort or hash aggregation. Each task gets its own <code>TaskMemoryManager</code>, which arbitrates the shared execution pool across every task running concurrently on an executor; with <code>n</code> tasks running concurrently, each is allowed to allocate somewhere between <code>1/(2n)</code> and <code>1/n</code> of total execution memory: a soft limit, not a hard stop.<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup> When a task&#39;s sort or hash-aggregation operator keeps requesting execution-memory pages it can&#39;t get, Spark doesn&#39;t fail immediately: it blocks the requesting task, triggers a spill of that task&#39;s in-memory data structure to disk, or, in the worst case, throws an <code>OutOfMemoryError</code>.<sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup></p><p>The spill path runs through Spark&#39;s map/shuffle I/O machinery: shuffle partitions created by wide transformations like <code>groupBy()</code> or <code>join()</code> spill to the executors&#39; local disks, at the location set by <code>spark.local.directory</code>.<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup> SQL physical operators apply the same idea with their own guardrails: <code>SortMergeJoinExec</code>&#39;s in-memory buffer and the cartesian-product operator&#39;s buffer both spill once they cross a configured row-count threshold, which by default is <code>spark.shuffle.spill.numElementsForceSpillThreshold</code>.<sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup> Whatever the trigger, the resulting spilled bytes are exposed on the task metrics as <code>memoryBytesSpilled</code>, visible in the Spark UI and event log.<sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup></p><h2 id="how-it-s-detected" tabindex="-1">How it&#39;s detected <a class="header-anchor" href="#how-it-s-detected" aria-label="Permalink to &quot;How it&#39;s detected&quot;">​</a></h2><p>Detection: any <code>memoryBytesSpilled &gt; 0</code> → Warning. The classification is the actionable signal.</p><p>Spill classification:</p><ul><li><code>skew</code>: ≥ 80% of tasks have zero spill; fix: address <a href="./bottleneck-skew.html">task skew</a>, not memory.</li><li><code>volume</code>: &lt; 20% of tasks have zero spill; fix: more partitions or more memory.</li><li><code>unclassified</code>: neither condition; treat as volume.</li></ul><img class="light-only" src="/docs/assets/spill-classification.BU2euYDO.svg" alt="How a nonzero memoryBytesSpilled is classified as skew, volume, or unclassified from the share of tasks with zero spill, and the fix each classification points to."><img class="dark-only" src="/docs/assets/spill-classification.dark.D7i1M40d.svg" alt="How a nonzero memoryBytesSpilled is classified as skew, volume, or unclassified from the share of tasks with zero spill, and the fix each classification points to."><h2 id="why-it-matters" tabindex="-1">Why it matters <a class="header-anchor" href="#why-it-matters" aria-label="Permalink to &quot;Why it matters&quot;">​</a></h2><p>Spill is the fallback Spark reaches for once a task can no longer get the execution memory it&#39;s asking for: rather than failing outright, it blocks the task, writes its buffered data to disk, or (if that&#39;s not enough) throws an <code>OutOfMemoryError</code>.<sup class="footnote-ref"><a href="#fn1" id="fnref1:2">[1:2]</a></sup> A task that&#39;s spilling has already given up pure in-memory speed to keep running at all, which is why the skew-vs-volume split above is the actionable part of the signal: the two failure shapes call for opposite fixes.</p><h2 id="how-to-fix-it" tabindex="-1">How to fix it <a class="header-anchor" href="#how-to-fix-it" aria-label="Permalink to &quot;How to fix it&quot;">​</a></h2><ul><li><strong>Skew</strong>: When only a few tasks spill, the problem is the shape of the data, not the size of the memory pool. <code>repartition()</code> is the tool for this: reach for it specifically to fix a lopsided partition distribution or to increase partition count<sup class="footnote-ref"><a href="#fn5" id="fnref5">[5]</a></sup>, in contrast to <code>coalesce()</code>, which never rebalances skewed data because it just stacks existing partitions together: partitions that were uneven going in are still uneven coming out.<sup class="footnote-ref"><a href="#fn6" id="fnref6">[6]</a></sup> For skewed joins, <a href="./aqe.html">Adaptive Query Execution</a> can detect and split<sup class="footnote-ref"><a href="#fn7" id="fnref7">[7]</a></sup> oversized partitions automatically: a partition counts as skewed once it&#39;s larger than <code>spark.sql.adaptive.skewJoin.skewedPartitionFactor</code> (default <code>5.0</code>) times the median partition size, and also larger than <code>spark.sql.adaptive.skewJoin.skewedPartitionThresholdInBytes</code> (default <code>256 MB</code>).<sup class="footnote-ref"><a href="#fn8" id="fnref8">[8]</a></sup><sup class="footnote-ref"><a href="#fn9" id="fnref9">[9]</a></sup> Before AQE existed, the manual equivalent was salting: adding a prefix to the skewed keys to make the same key look different, then adjusting the data distribution accordingly.<sup class="footnote-ref"><a href="#fn7" id="fnref7:1">[7:1]</a></sup> Databricks&#39; <code>/*+ SKEW(...) */</code> hint lets you name the skewed relation and specific key values directly, so the planner targets just those keys instead of relying on automatic detection.<sup class="footnote-ref"><a href="#fn10" id="fnref10">[10]</a></sup></li><li><strong>Volume</strong>: When most tasks spill, the fix is more room to work with: more partitions, more memory, or both. <code>repartition(n)</code> guarantees exactly <code>n</code> output partitions via a hash shuffle<sup class="footnote-ref"><a href="#fn11" id="fnref11">[11]</a></sup>, and since Spark&#39;s per-task startup overhead is low (unlike MapReduce&#39;s), the general bias is to err toward more partitions rather than fewer.<sup class="footnote-ref"><a href="#fn12" id="fnref12">[12]</a></sup><code>spark.sql.files.maxPartitionBytes</code> (default <code>128 MB</code>) is the equivalent lever on the input side, governing how much file data gets packed into each input partition before a shuffle even happens.<sup class="footnote-ref"><a href="#fn6" id="fnref6:1">[6:1]</a></sup> On the memory side, execution memory is a fraction of the JVM heap. <code>spark.memory.fraction</code> (default <code>0.6</code>) sizes the shared execution/storage region as <code>(JVM heap − 300 MiB) × spark.memory.fraction</code><sup class="footnote-ref"><a href="#fn13" id="fnref13">[13]</a></sup>. Because each task&#39;s slice of that region is capped by the <code>TaskMemoryManager</code> at between <code>1/(2n)</code> and <code>1/n</code> of the total,<sup class="footnote-ref"><a href="#fn1" id="fnref1:3">[1:3]</a></sup> giving executors more memory, or running fewer concurrent tasks on each one, directly raises the ceiling before spill kicks in.</li></ul><p>For the <strong>volume</strong> case, give tasks more room: defaults shown, comments say which way to move:</p><div class="language-properties vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">properties</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Create more, smaller input partitions before the shuffle (default 128m)</span></span>
23
23
  <span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.sql.files.maxPartitionBytes</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=128m</span></span>
24
24
  <span class="line"></span>
25
25
  <span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Execution/storage share of (JVM heap - 300MiB) (default 0.6);</span></span>
26
26
  <span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># raising it grows execution memory but starves the untracked user-memory region</span></span>
27
27
  <span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.memory.fraction</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=0.6</span></span></code></pre></div><p>For the <strong>skew</strong> case (only a few tasks spill), the fix is <code>df.repartition(n)</code>, not more memory: see <a href="./partitioning.html">Partitioning</a>.</p><h2 id="confidence" tabindex="-1">Confidence <a class="header-anchor" href="#confidence" aria-label="Permalink to &quot;Confidence&quot;">​</a></h2><p>The core detection (any <code>memoryBytesSpilled &gt; 0</code>) and the skew-vs-volume classification are validated: both the skew branch (most tasks spill nothing, so the shape of the data is the problem) and the volume branch (nearly every task spills, so the pool is too small) map to a well-understood fix, and neither needs further validation. <span class="tag">EXPERIMENTAL</span> When a spill matches neither shape, it falls back to a low-confidence unclassified finding that still requires validation, because there is no confirmed cause to act on; the page defaults it to the volume remedy, but that is a guess, not a diagnosis.</p><h2 id="limitations-false-positive-risk" tabindex="-1">Limitations / false-positive risk <a class="header-anchor" href="#limitations-false-positive-risk" aria-label="Permalink to &quot;Limitations / false-positive risk&quot;">​</a></h2><p>Some spill is normal. A large aggregation or join can spill by design once its working set outgrows execution memory, and a small spill on a healthy stage rarely repays the effort of chasing it. The raw <code>memoryBytesSpilled &gt; 0</code> trigger fires on both the actionable and the routine cases, so treat a small spill as informational until the skew-vs-volume split says otherwise.</p><h2 id="related" tabindex="-1">Related <a class="header-anchor" href="#related" aria-label="Permalink to &quot;Related&quot;">​</a></h2><ul><li><strong>Why it happens:</strong> <a href="./memory-model.html">Memory Management</a>, <a href="./partitioning.html">Partitioning</a></li></ul><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><a href="https://luminousmen.com/post/dive-into-spark-memory" target="_blank" rel="noreferrer">Diving into Spark Memory Management</a> <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a> <a href="#fnref1:2" class="footnote-backref">↩︎</a> <a href="#fnref1:3" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><em>Learning Spark, 2nd Edition</em>, Damji, Wenig, Das, Lee, ch. 7 <a href="#fnref2" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><a href="https://raw.githubusercontent.com/apache/spark/v3.5.0/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala" target="_blank" rel="noreferrer">SQLConf.scala</a> <a href="#fnref3" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/monitoring.html#spark-history-server" target="_blank" rel="noreferrer">Monitoring and Instrumentation</a> <a href="#fnref4" class="footnote-backref">↩︎</a></p></li><li id="fn5" class="footnote-item"><p><em>Spark: The Definitive Guide</em>, Chambers &amp; Zaharia, ch. 19 <a href="#fnref5" class="footnote-backref">↩︎</a></p></li><li id="fn6" class="footnote-item"><p><a href="https://luminousmen.com/post/spark-partitions" target="_blank" rel="noreferrer">Spark Partitions</a> <a href="#fnref6" class="footnote-backref">↩︎</a> <a href="#fnref6:1" class="footnote-backref">↩︎</a></p></li><li id="fn7" class="footnote-item"><p><a href="https://issues.apache.org/jira/browse/SPARK-29544" target="_blank" rel="noreferrer">SPARK-29544: Optimize Skewed Join in SQL Adaptive Execution</a> <a href="#fnref7" class="footnote-backref">↩︎</a> <a href="#fnref7:1" class="footnote-backref">↩︎</a></p></li><li id="fn8" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration (Spark)</a> <a href="#fnref8" class="footnote-backref">↩︎</a></p></li><li id="fn9" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/sql-performance-tuning.html" target="_blank" rel="noreferrer">Performance Tuning (Spark SQL, DataFrames and Datasets Guide)</a> <a href="#fnref9" class="footnote-backref">↩︎</a></p></li><li id="fn10" class="footnote-item"><p><a href="https://docs.databricks.com/aws/en/archive/legacy/skew-join" target="_blank" rel="noreferrer">Skew Join Optimization</a> <a href="#fnref10" class="footnote-backref">↩︎</a></p></li><li id="fn11" class="footnote-item"><p><a href="https://raw.githubusercontent.com/apache/spark/v3.5.0/core/src/main/scala/org/apache/spark/rdd/RDD.scala" target="_blank" rel="noreferrer">RDD.scala</a> <a href="#fnref11" class="footnote-backref">↩︎</a></p></li><li id="fn12" class="footnote-item"><p><a href="https://blog.cloudera.com/how-to-tune-your-apache-spark-jobs-part-2/" target="_blank" rel="noreferrer">How to Tune Your Apache Spark Jobs (Part 2): Cloudera Engineering Blog</a> <a href="#fnref12" class="footnote-backref">↩︎</a></p></li><li id="fn13" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/tuning.html" target="_blank" rel="noreferrer">Spark Tuning Guide</a> <a href="#fnref13" class="footnote-backref">↩︎</a></p></li></ol></section></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/tuning-reference/bottleneck-shuffle.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>Shuffle I/O</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="/docs/tuning-reference/bottleneck-gc.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>GC Pressure</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
28
- <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"YQcj5EI4\",\"contributor-guide_architecture_board-widgets_caching-and-memory.md\":\"CQgn1t_M\",\"contributor-guide_architecture_board-widgets_plan-advisor.md\":\"RlzvUmNm\",\"contributor-guide_architecture_board-widgets_visual-system-and-templating.md\":\"wulOueeB\",\"contributor-guide_architecture_detector-contract.md\":\"N4hhenrA\",\"contributor-guide_architecture_detector-contract_plan-attribution.md\":\"yNex4bys\",\"contributor-guide_architecture_detector-contract_threshold-tables.md\":\"Dp3c40rx\",\"contributor-guide_architecture_drill-down.md\":\"CqZPrdo9\",\"contributor-guide_architecture_drill-down_plan-graph.md\":\"DwOozIEs\",\"contributor-guide_architecture_impact-estimation.md\":\"DfvbqEUu\",\"contributor-guide_architecture_impact-estimation_caveats-tuning-and-coverage.md\":\"Cwm1jHmn\",\"contributor-guide_architecture_impact-estimation_stage-union-rollup.md\":\"DdCGtAGu\",\"contributor-guide_architecture_index.md\":\"tymxf1Fh\",\"contributor-guide_architecture_overview.md\":\"C1sSvm9i\",\"contributor-guide_architecture_state-and-history.md\":\"DyCBdjuM\",\"contributor-guide_architecture_state-and-history_history-server-intake.md\":\"C298hrHZ\",\"contributor-guide_architecture_state-and-history_run-comparison-internals.md\":\"CZA0JLtV\",\"contributor-guide_architecture_widget-rendering.md\":\"BOZ1azXG\",\"contributor-guide_architecture_widget-rendering_findings-tab.md\":\"BQXycK6G\",\"contributor-guide_architecture_widget-rendering_investigation-routing.md\":\"BGdiyElu\",\"contributor-guide_architecture_widget-rendering_list-sort-mode.md\":\"BuHKgH0C\",\"contributor-guide_architecture_widget-rendering_reference-section.md\":\"DWqC80bN\",\"contributor-guide_architecture_widget-rendering_row-expand-contract.md\":\"ZT89u9f1\",\"contributor-guide_architecture_worker-protocol.md\":\"BAnhi2-d\",\"contributor-guide_architecture_worker-protocol_cli-and-mcp.md\":\"BX1XnFbq\",\"contributor-guide_architecture_worker-protocol_core-and-event-validation.md\":\"DSVFPpWb\",\"contributor-guide_architecture_worker-protocol_evidence-availability.md\":\"CjimIDBY\",\"contributor-guide_architecture_worker-protocol_evidence-report.md\":\"CiQl1MLb\",\"contributor-guide_contributing.md\":\"CQMvGwFt\",\"contributor-guide_development-setup.md\":\"DQdzera7\",\"contributor-guide_testing.md\":\"By6XGR9a\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"D08Uaod7\",\"tuning-reference_aqe.md\":\"C1Ae6A8u\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"6wjKuJNE\",\"tuning-reference_bottleneck-cold-start.md\":\"BVQbsuHP\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"DtaYnaiF\",\"tuning-reference_bottleneck-failures.md\":\"Z1Tdmdd3\",\"tuning-reference_bottleneck-gc.md\":\"BCnXLf1n\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"CcvQj2g9\",\"tuning-reference_bottleneck-memory-utilization.md\":\"B-C0-8h-\",\"tuning-reference_bottleneck-retry-waste.md\":\"DCnhWiqj\",\"tuning-reference_bottleneck-shuffle.md\":\"B5OMJCd3\",\"tuning-reference_bottleneck-skew.md\":\"Cvo6NQm_\",\"tuning-reference_bottleneck-slow-host.md\":\"ja5qr3wq\",\"tuning-reference_bottleneck-small-files.md\":\"Crjtb0i2\",\"tuning-reference_bottleneck-spill.md\":\"Bl8KtSRg\",\"tuning-reference_bottleneck-straggler.md\":\"B0VN8reT\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"DQCftGhu\",\"tuning-reference_bottleneck-utilization.md\":\"BXDl30f9\",\"tuning-reference_caching.md\":\"CcaiMn_7\",\"tuning-reference_cluster-config.md\":\"DbWG1yZq\",\"tuning-reference_config.md\":\"YUqSpwFB\",\"tuning-reference_data-formats.md\":\"C5DX5xL4\",\"tuning-reference_index.md\":\"ZVIpQ6Dk\",\"tuning-reference_intro.md\":\"BSSaDlqx\",\"tuning-reference_joins.md\":\"BVhNCDnE\",\"tuning-reference_memory-model.md\":\"CfjFdgbl\",\"tuning-reference_metrics.md\":\"C64VNEL4\",\"tuning-reference_partitioning.md\":\"CtLK0xsS\",\"tuning-reference_pyspark.md\":\"D-92XgF8\",\"tuning-reference_shuffle.md\":\"Iq2v1oHJ\",\"tuning-reference_spark-architecture.md\":\"_qwnE_Sc\",\"tuning-reference_table-formats.md\":\"BSYEK-eN\",\"user-guide_alternative-log-retrieval.md\":\"CJpYvHGo\",\"user-guide_alternative-log-retrieval_managed-platforms.md\":\"DvDwHNtC\",\"user-guide_alternative-log-retrieval_ssh-bastion.md\":\"Gszzlplv\",\"user-guide_getting-started.md\":\"DYHHHkQR\",\"user-guide_getting-started_ci-and-automation.md\":\"BEepbDDL\",\"user-guide_getting-started_reading-the-dashboard.md\":\"Obf3W3__\",\"user-guide_getting-started_regression-metric-keys.md\":\"B_gzr2Mw\",\"user-guide_getting-started_tuning-thresholds.md\":\"B_MyjyZm\",\"user-guide_mcp-tools.md\":\"C65SjWUL\",\"user-guide_mcp-tools_compare-runs.md\":\"Cc_gLoix\",\"user-guide_mcp-tools_diagnose-run.md\":\"BP1vnHA5\",\"user-guide_mcp-tools_evaluate-budgets.md\":\"BB0nDY3Z\",\"user-guide_mcp-tools_get-finding-documentation.md\":\"BtTTHNV6\",\"user-guide_mcp-tools_get-finding-evidence.md\":\"rKyu4wts\",\"user-guide_mcp-tools_get-reference-doc.md\":\"Yac4tH3u\",\"user-guide_mcp-tools_get-run-summary.md\":\"EhaIsele\",\"user-guide_mcp-tools_list-runs.md\":\"DpQPeCM3\",\"user-guide_run-comparison.md\":\"BUALkohi\",\"user-guide_run-comparison_how-stages-are-matched.md\":\"DSXGu2tA\",\"user-guide_understanding-findings.md\":\"3C7KCEZO\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\",\"collapsed\":true,\"items\":[{\"text\":\"Reading the dashboard\",\"link\":\"/user-guide/getting-started/reading-the-dashboard\"},{\"text\":\"CI and automation\",\"link\":\"/user-guide/getting-started/ci-and-automation\"},{\"text\":\"Tuning detector thresholds\",\"link\":\"/user-guide/getting-started/tuning-thresholds\"},{\"text\":\"Regression metric keys\",\"link\":\"/user-guide/getting-started/regression-metric-keys\"}]},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\",\"collapsed\":true,\"items\":[{\"text\":\"How stages are matched\",\"link\":\"/user-guide/run-comparison/how-stages-are-matched\"}]},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\",\"collapsed\":true,\"items\":[{\"text\":\"diagnose_run\",\"link\":\"/user-guide/mcp-tools/diagnose-run\"},{\"text\":\"get_run_summary\",\"link\":\"/user-guide/mcp-tools/get-run-summary\"},{\"text\":\"evaluate_budgets\",\"link\":\"/user-guide/mcp-tools/evaluate-budgets\"},{\"text\":\"compare_runs\",\"link\":\"/user-guide/mcp-tools/compare-runs\"},{\"text\":\"get_finding_evidence\",\"link\":\"/user-guide/mcp-tools/get-finding-evidence\"},{\"text\":\"get_finding_documentation\",\"link\":\"/user-guide/mcp-tools/get-finding-documentation\"},{\"text\":\"get_reference_doc\",\"link\":\"/user-guide/mcp-tools/get-reference-doc\"},{\"text\":\"list_runs\",\"link\":\"/user-guide/mcp-tools/list-runs\"}]},{\"text\":\"Finding your event log\",\"link\":\"/user-guide/alternative-log-retrieval\",\"collapsed\":true,\"items\":[{\"text\":\"Managed platforms\",\"link\":\"/user-guide/alternative-log-retrieval/managed-platforms\"},{\"text\":\"Behind an SSH bastion\",\"link\":\"/user-guide/alternative-log-retrieval/ssh-bastion\"}]}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\",\"collapsed\":true,\"items\":[{\"text\":\"Evidence availability\",\"link\":\"/contributor-guide/architecture/worker-protocol/evidence-availability\"},{\"text\":\"Evidence report and finding identity\",\"link\":\"/contributor-guide/architecture/worker-protocol/evidence-report\"},{\"text\":\"CLI and MCP server\",\"link\":\"/contributor-guide/architecture/worker-protocol/cli-and-mcp\"},{\"text\":\"Core and event validation\",\"link\":\"/contributor-guide/architecture/worker-protocol/core-and-event-validation\"}]},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\",\"collapsed\":true,\"items\":[{\"text\":\"Run comparison internals\",\"link\":\"/contributor-guide/architecture/state-and-history/run-comparison-internals\"},{\"text\":\"History Server intake\",\"link\":\"/contributor-guide/architecture/state-and-history/history-server-intake\"}]},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\",\"collapsed\":true,\"items\":[{\"text\":\"Plan attribution\",\"link\":\"/contributor-guide/architecture/detector-contract/plan-attribution\"},{\"text\":\"Threshold tables\",\"link\":\"/contributor-guide/architecture/detector-contract/threshold-tables\"}]},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\",\"collapsed\":true,\"items\":[{\"text\":\"Stage union rollup\",\"link\":\"/contributor-guide/architecture/impact-estimation/stage-union-rollup\"},{\"text\":\"Caveats, tuning and coverage\",\"link\":\"/contributor-guide/architecture/impact-estimation/caveats-tuning-and-coverage\"}]},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\",\"collapsed\":true,\"items\":[{\"text\":\"Findings tab\",\"link\":\"/contributor-guide/architecture/widget-rendering/findings-tab\"},{\"text\":\"List sort mode\",\"link\":\"/contributor-guide/architecture/widget-rendering/list-sort-mode\"},{\"text\":\"Row/expand contract\",\"link\":\"/contributor-guide/architecture/widget-rendering/row-expand-contract\"},{\"text\":\"ReferenceSection\",\"link\":\"/contributor-guide/architecture/widget-rendering/reference-section\"},{\"text\":\"Investigation routing\",\"link\":\"/contributor-guide/architecture/widget-rendering/investigation-routing\"}]},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\",\"collapsed\":true,\"items\":[{\"text\":\"Plan Advisor\",\"link\":\"/contributor-guide/architecture/board-widgets/plan-advisor\"},{\"text\":\"Caching and memory widgets\",\"link\":\"/contributor-guide/architecture/board-widgets/caching-and-memory\"},{\"text\":\"Visual system and templating\",\"link\":\"/contributor-guide/architecture/board-widgets/visual-system-and-templating\"}]},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\",\"collapsed\":true,\"items\":[{\"text\":\"Plan graph\",\"link\":\"/contributor-guide/architecture/drill-down/plan-graph\"}]}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":{\"selector\":\"[data-sf-anchor-offset]\",\"padding\":0},\"cleanUrls\":false}");</script>
28
+ <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"CqKM_jEG\",\"contributor-guide_architecture_board-widgets_caching-and-memory.md\":\"BYDo3ClR\",\"contributor-guide_architecture_board-widgets_plan-advisor.md\":\"ajUAVdbc\",\"contributor-guide_architecture_board-widgets_visual-system-and-templating.md\":\"WqY74T0T\",\"contributor-guide_architecture_detector-contract.md\":\"CHHgywoP\",\"contributor-guide_architecture_detector-contract_plan-attribution.md\":\"DIz3_oz9\",\"contributor-guide_architecture_detector-contract_threshold-tables.md\":\"WH4tRHPL\",\"contributor-guide_architecture_drill-down.md\":\"yB-Hz2XT\",\"contributor-guide_architecture_drill-down_plan-graph.md\":\"BQZE04sw\",\"contributor-guide_architecture_impact-estimation.md\":\"K71YB-Rr\",\"contributor-guide_architecture_impact-estimation_caveats-tuning-and-coverage.md\":\"DLCAffwn\",\"contributor-guide_architecture_impact-estimation_stage-union-rollup.md\":\"JFrzDMwo\",\"contributor-guide_architecture_index.md\":\"tymxf1Fh\",\"contributor-guide_architecture_overview.md\":\"nxkpOhgf\",\"contributor-guide_architecture_state-and-history.md\":\"DyCBdjuM\",\"contributor-guide_architecture_state-and-history_history-server-intake.md\":\"C298hrHZ\",\"contributor-guide_architecture_state-and-history_run-comparison-internals.md\":\"U-9KMDFK\",\"contributor-guide_architecture_widget-rendering.md\":\"BrSz8Ezh\",\"contributor-guide_architecture_widget-rendering_findings-tab.md\":\"B7rjSF_h\",\"contributor-guide_architecture_widget-rendering_investigation-routing.md\":\"BGdiyElu\",\"contributor-guide_architecture_widget-rendering_list-sort-mode.md\":\"37hcZJ7q\",\"contributor-guide_architecture_widget-rendering_reference-section.md\":\"DWqC80bN\",\"contributor-guide_architecture_widget-rendering_row-expand-contract.md\":\"mK_VmAWk\",\"contributor-guide_architecture_worker-protocol.md\":\"Br6OE7RW\",\"contributor-guide_architecture_worker-protocol_cli-and-mcp.md\":\"IVyaXrGe\",\"contributor-guide_architecture_worker-protocol_core-and-event-validation.md\":\"DVSXn-bb\",\"contributor-guide_architecture_worker-protocol_evidence-availability.md\":\"D3S5cyOW\",\"contributor-guide_architecture_worker-protocol_evidence-report.md\":\"CtmCU57I\",\"contributor-guide_contributing.md\":\"DBW4drtZ\",\"contributor-guide_development-setup.md\":\"B21ZTDhn\",\"contributor-guide_testing.md\":\"DpkozNoG\",\"index.md\":\"lMm7SUMz\",\"tuning-reference_anti-patterns.md\":\"D08Uaod7\",\"tuning-reference_aqe.md\":\"C1Ae6A8u\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"6wjKuJNE\",\"tuning-reference_bottleneck-cold-start.md\":\"BVQbsuHP\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"DtaYnaiF\",\"tuning-reference_bottleneck-failures.md\":\"Z1Tdmdd3\",\"tuning-reference_bottleneck-gc.md\":\"BCnXLf1n\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"CcvQj2g9\",\"tuning-reference_bottleneck-memory-utilization.md\":\"B-C0-8h-\",\"tuning-reference_bottleneck-retry-waste.md\":\"DCnhWiqj\",\"tuning-reference_bottleneck-shuffle.md\":\"B5OMJCd3\",\"tuning-reference_bottleneck-skew.md\":\"Cvo6NQm_\",\"tuning-reference_bottleneck-slow-host.md\":\"ja5qr3wq\",\"tuning-reference_bottleneck-small-files.md\":\"Crjtb0i2\",\"tuning-reference_bottleneck-spill.md\":\"Bl8KtSRg\",\"tuning-reference_bottleneck-straggler.md\":\"B0VN8reT\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"DQCftGhu\",\"tuning-reference_bottleneck-utilization.md\":\"BXDl30f9\",\"tuning-reference_caching.md\":\"CcaiMn_7\",\"tuning-reference_cluster-config.md\":\"DbWG1yZq\",\"tuning-reference_config.md\":\"YUqSpwFB\",\"tuning-reference_data-formats.md\":\"C5DX5xL4\",\"tuning-reference_index.md\":\"ZVIpQ6Dk\",\"tuning-reference_intro.md\":\"BSSaDlqx\",\"tuning-reference_joins.md\":\"BVhNCDnE\",\"tuning-reference_memory-model.md\":\"CfjFdgbl\",\"tuning-reference_metrics.md\":\"C64VNEL4\",\"tuning-reference_partitioning.md\":\"CtLK0xsS\",\"tuning-reference_pyspark.md\":\"D-92XgF8\",\"tuning-reference_shuffle.md\":\"Iq2v1oHJ\",\"tuning-reference_spark-architecture.md\":\"_qwnE_Sc\",\"tuning-reference_table-formats.md\":\"BSYEK-eN\",\"user-guide_alternative-log-retrieval.md\":\"CJpYvHGo\",\"user-guide_alternative-log-retrieval_managed-platforms.md\":\"DvDwHNtC\",\"user-guide_alternative-log-retrieval_ssh-bastion.md\":\"Gszzlplv\",\"user-guide_getting-started.md\":\"DYHHHkQR\",\"user-guide_getting-started_ci-and-automation.md\":\"C0pcQoNc\",\"user-guide_getting-started_reading-the-dashboard.md\":\"CGaxPX_1\",\"user-guide_getting-started_regression-metric-keys.md\":\"B_gzr2Mw\",\"user-guide_getting-started_tuning-thresholds.md\":\"BGVE-wrc\",\"user-guide_mcp-tools.md\":\"49r3L2YE\",\"user-guide_mcp-tools_compare-runs.md\":\"BQ_kIptC\",\"user-guide_mcp-tools_diagnose-run.md\":\"azCZXkH8\",\"user-guide_mcp-tools_evaluate-budgets.md\":\"dpQpRQ7z\",\"user-guide_mcp-tools_get-finding-documentation.md\":\"BtTTHNV6\",\"user-guide_mcp-tools_get-finding-evidence.md\":\"CH6FkNLX\",\"user-guide_mcp-tools_get-reference-doc.md\":\"Yac4tH3u\",\"user-guide_mcp-tools_get-run-summary.md\":\"EhaIsele\",\"user-guide_mcp-tools_list-runs.md\":\"DpQPeCM3\",\"user-guide_run-comparison.md\":\"B_LOkdBf\",\"user-guide_run-comparison_how-stages-are-matched.md\":\"DSXGu2tA\",\"user-guide_understanding-findings.md\":\"C4riKCKl\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":{\"light\":\"/logo-light.svg\",\"dark\":\"/logo-dark.svg\"},\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"},{\"text\":\"Open app\",\"link\":\"/..\",\"target\":\"_self\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\",\"collapsed\":true,\"items\":[{\"text\":\"Reading the dashboard\",\"link\":\"/user-guide/getting-started/reading-the-dashboard\"},{\"text\":\"CI and automation\",\"link\":\"/user-guide/getting-started/ci-and-automation\"},{\"text\":\"Tuning detector thresholds\",\"link\":\"/user-guide/getting-started/tuning-thresholds\"},{\"text\":\"Regression metric keys\",\"link\":\"/user-guide/getting-started/regression-metric-keys\"}]},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\",\"collapsed\":true,\"items\":[{\"text\":\"How stages are matched\",\"link\":\"/user-guide/run-comparison/how-stages-are-matched\"}]},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\",\"collapsed\":true,\"items\":[{\"text\":\"diagnose_run\",\"link\":\"/user-guide/mcp-tools/diagnose-run\"},{\"text\":\"get_run_summary\",\"link\":\"/user-guide/mcp-tools/get-run-summary\"},{\"text\":\"evaluate_budgets\",\"link\":\"/user-guide/mcp-tools/evaluate-budgets\"},{\"text\":\"compare_runs\",\"link\":\"/user-guide/mcp-tools/compare-runs\"},{\"text\":\"get_finding_evidence\",\"link\":\"/user-guide/mcp-tools/get-finding-evidence\"},{\"text\":\"get_finding_documentation\",\"link\":\"/user-guide/mcp-tools/get-finding-documentation\"},{\"text\":\"get_reference_doc\",\"link\":\"/user-guide/mcp-tools/get-reference-doc\"},{\"text\":\"list_runs\",\"link\":\"/user-guide/mcp-tools/list-runs\"}]},{\"text\":\"Finding your event log\",\"link\":\"/user-guide/alternative-log-retrieval\",\"collapsed\":true,\"items\":[{\"text\":\"Managed platforms\",\"link\":\"/user-guide/alternative-log-retrieval/managed-platforms\"},{\"text\":\"Behind an SSH bastion\",\"link\":\"/user-guide/alternative-log-retrieval/ssh-bastion\"}]}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\",\"collapsed\":true,\"items\":[{\"text\":\"Evidence availability\",\"link\":\"/contributor-guide/architecture/worker-protocol/evidence-availability\"},{\"text\":\"Evidence report and finding identity\",\"link\":\"/contributor-guide/architecture/worker-protocol/evidence-report\"},{\"text\":\"CLI and MCP server\",\"link\":\"/contributor-guide/architecture/worker-protocol/cli-and-mcp\"},{\"text\":\"Core and event validation\",\"link\":\"/contributor-guide/architecture/worker-protocol/core-and-event-validation\"}]},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\",\"collapsed\":true,\"items\":[{\"text\":\"Run comparison internals\",\"link\":\"/contributor-guide/architecture/state-and-history/run-comparison-internals\"},{\"text\":\"History Server intake\",\"link\":\"/contributor-guide/architecture/state-and-history/history-server-intake\"}]},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\",\"collapsed\":true,\"items\":[{\"text\":\"Plan attribution\",\"link\":\"/contributor-guide/architecture/detector-contract/plan-attribution\"},{\"text\":\"Threshold tables\",\"link\":\"/contributor-guide/architecture/detector-contract/threshold-tables\"}]},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\",\"collapsed\":true,\"items\":[{\"text\":\"Stage union rollup\",\"link\":\"/contributor-guide/architecture/impact-estimation/stage-union-rollup\"},{\"text\":\"Caveats, tuning and coverage\",\"link\":\"/contributor-guide/architecture/impact-estimation/caveats-tuning-and-coverage\"}]},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\",\"collapsed\":true,\"items\":[{\"text\":\"Findings tab\",\"link\":\"/contributor-guide/architecture/widget-rendering/findings-tab\"},{\"text\":\"List sort mode\",\"link\":\"/contributor-guide/architecture/widget-rendering/list-sort-mode\"},{\"text\":\"Row/expand contract\",\"link\":\"/contributor-guide/architecture/widget-rendering/row-expand-contract\"},{\"text\":\"ReferenceSection\",\"link\":\"/contributor-guide/architecture/widget-rendering/reference-section\"},{\"text\":\"Investigation routing\",\"link\":\"/contributor-guide/architecture/widget-rendering/investigation-routing\"}]},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\",\"collapsed\":true,\"items\":[{\"text\":\"Plan Advisor\",\"link\":\"/contributor-guide/architecture/board-widgets/plan-advisor\"},{\"text\":\"Caching and memory widgets\",\"link\":\"/contributor-guide/architecture/board-widgets/caching-and-memory\"},{\"text\":\"Visual system and templating\",\"link\":\"/contributor-guide/architecture/board-widgets/visual-system-and-templating\"}]},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\",\"collapsed\":true,\"items\":[{\"text\":\"Plan graph\",\"link\":\"/contributor-guide/architecture/drill-down/plan-graph\"}]}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":{\"selector\":\"[data-sf-anchor-offset]\",\"padding\":0},\"cleanUrls\":false}");</script>
29
29
 
30
30
  </body>
31
31
  </html>