sparkforensics-server 0.4.0 → 0.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (382) hide show
  1. package/index.js +3 -0
  2. package/package.json +3 -3
  3. package/public/assets/AutoscalingChurn-5hDGIXi4.js +1 -0
  4. package/public/assets/CacheUtilization-CT2SCnJJ.js +1 -0
  5. package/public/assets/CachingOpportunity-D_O_WkbH.js +1 -0
  6. package/public/assets/ColdStart-B38OiAOp.js +1 -0
  7. package/public/assets/ConfigAudit-BoplryCH.js +1 -0
  8. package/public/assets/CoreUsageArea-csgwNxkl.js +1 -0
  9. package/public/assets/Dashboard-BgKodJl2.js +2 -0
  10. package/public/assets/DisclosureContext-B0siMVdg.js +1 -0
  11. package/public/assets/DuplicatePlanSubtree-BqS2A2j4.js +1 -0
  12. package/public/assets/ExecutorUtilization-Dp52QGWo.js +1 -0
  13. package/public/assets/GcPressure-CT1yzR_s.js +1 -0
  14. package/public/assets/IncompleteRun-C7Qkjtjn.js +1 -0
  15. package/public/assets/JobFailures-WHl3ELfh.js +1 -0
  16. package/public/assets/MemoryUtilization-D8IUDHJI.js +1 -0
  17. package/public/assets/OverBroadcast-6oSSAQ_R.js +1 -0
  18. package/public/assets/PartitionHint-CUvbnnxy.js +1 -0
  19. package/public/assets/PartitionSizing-BwdFL-mw.js +1 -0
  20. package/public/assets/{PlanExplorer-DdmPkKO1.js → PlanExplorer-DcxmiQWL.js} +1 -1
  21. package/public/assets/PlanGraphRoute-pqnHTt4i.js +1 -0
  22. package/public/assets/RetryWaste-BTbeHkJc.js +1 -0
  23. package/public/assets/ShuffleIO-CRhrYNE0.js +1 -0
  24. package/public/assets/Skew-DSNXRG9V.js +1 -0
  25. package/public/assets/SlowHost-B7Lwu2EU.js +1 -0
  26. package/public/assets/SmallFiles-C082c2rd.js +1 -0
  27. package/public/assets/SpeculationWaste-BS2vpadr.js +1 -0
  28. package/public/assets/Spill-B4DVKHXx.js +1 -0
  29. package/public/assets/StageFailed-D3ac9Sou.js +1 -0
  30. package/public/assets/StageFindingGroup-D9naK109.js +1 -0
  31. package/public/assets/{StageHeader-BSV8ALhT.js → StageHeader-pI1nTwVf.js} +1 -1
  32. package/public/assets/StageShape-eO5le8iG.js +1 -0
  33. package/public/assets/StageSlowness-BaNyfNvL.js +1 -0
  34. package/public/assets/Straggler-D5Fnok_y.js +1 -0
  35. package/public/assets/TaskFailures-BWnC-K_G.js +1 -0
  36. package/public/assets/{TinyTask-CIPRxWfT.js → TinyTask-8_5lpq_2.js} +1 -1
  37. package/public/assets/UnderBroadcast-DJ0XSwth.js +1 -0
  38. package/public/assets/{finding-anchor-DF1cSHO1.js → finding-anchor-Cys8Z6Yn.js} +1 -1
  39. package/public/assets/{impact-sort-7z0ZcBZf.js → impact-sort-BTQndbof.js} +1 -1
  40. package/public/assets/index-DCeWge12.js +13 -0
  41. package/public/assets/parser-worker-DmeCnLWU.js +122 -0
  42. package/public/assets/{plan-graph-vendor-B0sEms9Z.js → plan-graph-vendor-BYl-AzN5.js} +1 -1
  43. package/public/assets/{react-vendor-BAKt3vny.js → react-vendor-DEdfrRTJ.js} +1 -1
  44. package/public/assets/{select-Cy5pmbrh.js → select-DIYN6D3y.js} +1 -1
  45. package/public/assets/{usePagedRows-BjPOpUU8.js → usePagedRows-BxvdFnPk.js} +1 -1
  46. package/public/assets/useRecentFiles-C3Ov47U7.js +11 -0
  47. package/public/assets/{useSortMode-CU0vzThu.js → useSortMode-B8SRfAoG.js} +1 -1
  48. package/public/docs/404.html +4 -3
  49. package/public/docs/assets/{app.C6w47a9W.js → app.D2GqAkhy.js} +1 -1
  50. package/public/docs/assets/chunks/@localSearchIndexroot.D4OdkQjy.js +1 -0
  51. package/public/docs/assets/chunks/{VPLocalSearchBox.BcTNvT55.js → VPLocalSearchBox.BS4cSmtW.js} +1 -1
  52. package/public/docs/assets/chunks/{theme.D-mE6j02.js → theme.BuxVpb0H.js} +2 -2
  53. package/public/docs/assets/contributor-guide_architecture_board-widgets.md.YQcj5EI4.js +1 -0
  54. package/public/docs/assets/contributor-guide_architecture_board-widgets.md.YQcj5EI4.lean.js +1 -0
  55. package/public/docs/assets/contributor-guide_architecture_board-widgets_caching-and-memory.md.8bH1HnZp.js +1 -0
  56. package/public/docs/assets/contributor-guide_architecture_board-widgets_caching-and-memory.md.8bH1HnZp.lean.js +1 -0
  57. package/public/docs/assets/contributor-guide_architecture_board-widgets_plan-advisor.md.RlzvUmNm.js +1 -0
  58. package/public/docs/assets/contributor-guide_architecture_board-widgets_plan-advisor.md.RlzvUmNm.lean.js +1 -0
  59. package/public/docs/assets/contributor-guide_architecture_board-widgets_visual-system-and-templating.md.DYgUpIJB.js +1 -0
  60. package/public/docs/assets/contributor-guide_architecture_board-widgets_visual-system-and-templating.md.DYgUpIJB.lean.js +1 -0
  61. package/public/docs/assets/contributor-guide_architecture_detector-contract.md.wj6Et6UD.js +1 -0
  62. package/public/docs/assets/contributor-guide_architecture_detector-contract.md.wj6Et6UD.lean.js +1 -0
  63. package/public/docs/assets/contributor-guide_architecture_detector-contract_plan-attribution.md.yNex4bys.js +1 -0
  64. package/public/docs/assets/contributor-guide_architecture_detector-contract_plan-attribution.md.yNex4bys.lean.js +1 -0
  65. package/public/docs/assets/contributor-guide_architecture_detector-contract_threshold-tables.md.fkPBXsU2.js +1 -0
  66. package/public/docs/assets/contributor-guide_architecture_detector-contract_threshold-tables.md.fkPBXsU2.lean.js +1 -0
  67. package/public/docs/assets/contributor-guide_architecture_drill-down.md.CqZPrdo9.js +1 -0
  68. package/public/docs/assets/contributor-guide_architecture_drill-down.md.CqZPrdo9.lean.js +1 -0
  69. package/public/docs/assets/contributor-guide_architecture_drill-down_plan-graph.md.DwOozIEs.js +1 -0
  70. package/public/docs/assets/contributor-guide_architecture_drill-down_plan-graph.md.DwOozIEs.lean.js +1 -0
  71. package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.CZv_ZGOp.js +1 -0
  72. package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.CZv_ZGOp.lean.js +1 -0
  73. package/public/docs/assets/contributor-guide_architecture_impact-estimation_caveats-tuning-and-coverage.md.BUucRc0A.js +1 -0
  74. package/public/docs/assets/contributor-guide_architecture_impact-estimation_caveats-tuning-and-coverage.md.BUucRc0A.lean.js +1 -0
  75. package/public/docs/assets/contributor-guide_architecture_impact-estimation_stage-union-rollup.md.DdCGtAGu.js +1 -0
  76. package/public/docs/assets/contributor-guide_architecture_impact-estimation_stage-union-rollup.md.DdCGtAGu.lean.js +1 -0
  77. package/public/docs/assets/contributor-guide_architecture_index.md.tymxf1Fh.js +1 -0
  78. package/public/docs/assets/contributor-guide_architecture_index.md.tymxf1Fh.lean.js +1 -0
  79. package/public/docs/assets/contributor-guide_architecture_overview.md.C1sSvm9i.js +1 -0
  80. package/public/docs/assets/contributor-guide_architecture_overview.md.C1sSvm9i.lean.js +1 -0
  81. package/public/docs/assets/contributor-guide_architecture_state-and-history.md.DyCBdjuM.js +1 -0
  82. package/public/docs/assets/{contributor-guide_architecture_state-and-history.md.DOTVHjTV.lean.js → contributor-guide_architecture_state-and-history.md.DyCBdjuM.lean.js} +1 -1
  83. package/public/docs/assets/contributor-guide_architecture_state-and-history_history-server-intake.md.C298hrHZ.js +1 -0
  84. package/public/docs/assets/contributor-guide_architecture_state-and-history_history-server-intake.md.C298hrHZ.lean.js +1 -0
  85. package/public/docs/assets/contributor-guide_architecture_state-and-history_run-comparison-internals.md.uuAKpY0Y.js +1 -0
  86. package/public/docs/assets/contributor-guide_architecture_state-and-history_run-comparison-internals.md.uuAKpY0Y.lean.js +1 -0
  87. package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.BOZ1azXG.js +1 -0
  88. package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.BOZ1azXG.lean.js +1 -0
  89. package/public/docs/assets/contributor-guide_architecture_widget-rendering_findings-tab.md.BQXycK6G.js +1 -0
  90. package/public/docs/assets/contributor-guide_architecture_widget-rendering_findings-tab.md.BQXycK6G.lean.js +1 -0
  91. package/public/docs/assets/contributor-guide_architecture_widget-rendering_investigation-routing.md.BGdiyElu.js +1 -0
  92. package/public/docs/assets/contributor-guide_architecture_widget-rendering_investigation-routing.md.BGdiyElu.lean.js +1 -0
  93. package/public/docs/assets/contributor-guide_architecture_widget-rendering_list-sort-mode.md.BuHKgH0C.js +1 -0
  94. package/public/docs/assets/contributor-guide_architecture_widget-rendering_list-sort-mode.md.BuHKgH0C.lean.js +1 -0
  95. package/public/docs/assets/contributor-guide_architecture_widget-rendering_reference-section.md.DWqC80bN.js +1 -0
  96. package/public/docs/assets/contributor-guide_architecture_widget-rendering_reference-section.md.DWqC80bN.lean.js +1 -0
  97. package/public/docs/assets/contributor-guide_architecture_widget-rendering_row-expand-contract.md.ZT89u9f1.js +1 -0
  98. package/public/docs/assets/contributor-guide_architecture_widget-rendering_row-expand-contract.md.ZT89u9f1.lean.js +1 -0
  99. package/public/docs/assets/contributor-guide_architecture_worker-protocol.md.BAnhi2-d.js +1 -0
  100. package/public/docs/assets/contributor-guide_architecture_worker-protocol.md.BAnhi2-d.lean.js +1 -0
  101. package/public/docs/assets/contributor-guide_architecture_worker-protocol_cli-and-mcp.md.Dk08VS7n.js +1 -0
  102. package/public/docs/assets/contributor-guide_architecture_worker-protocol_cli-and-mcp.md.Dk08VS7n.lean.js +1 -0
  103. package/public/docs/assets/contributor-guide_architecture_worker-protocol_core-and-event-validation.md.DSVFPpWb.js +1 -0
  104. package/public/docs/assets/contributor-guide_architecture_worker-protocol_core-and-event-validation.md.DSVFPpWb.lean.js +1 -0
  105. package/public/docs/assets/contributor-guide_architecture_worker-protocol_evidence-availability.md.CjimIDBY.js +6 -0
  106. package/public/docs/assets/contributor-guide_architecture_worker-protocol_evidence-availability.md.CjimIDBY.lean.js +1 -0
  107. package/public/docs/assets/contributor-guide_architecture_worker-protocol_evidence-report.md.DVGZOWvW.js +1 -0
  108. package/public/docs/assets/contributor-guide_architecture_worker-protocol_evidence-report.md.DVGZOWvW.lean.js +1 -0
  109. package/public/docs/assets/contributor-guide_contributing.md.CQMvGwFt.js +1 -0
  110. package/public/docs/assets/{contributor-guide_contributing.md.B2E9ni79.lean.js → contributor-guide_contributing.md.CQMvGwFt.lean.js} +1 -1
  111. package/public/docs/assets/contributor-guide_development-setup.md.D9bu4j_c.js +15 -0
  112. package/public/docs/assets/contributor-guide_development-setup.md.D9bu4j_c.lean.js +1 -0
  113. package/public/docs/assets/contributor-guide_testing.md.By6XGR9a.js +1 -0
  114. package/public/docs/assets/{contributor-guide_testing.md.DTwhpoN2.lean.js → contributor-guide_testing.md.By6XGR9a.lean.js} +1 -1
  115. package/public/docs/assets/{style.DSixAiZE.css → style.tmDhD3c8.css} +1 -1
  116. package/public/docs/assets/user-guide_alternative-log-retrieval.md.CJpYvHGo.js +4 -0
  117. package/public/docs/assets/user-guide_alternative-log-retrieval.md.CJpYvHGo.lean.js +1 -0
  118. package/public/docs/assets/user-guide_alternative-log-retrieval_managed-platforms.md.DvDwHNtC.js +4 -0
  119. package/public/docs/assets/user-guide_alternative-log-retrieval_managed-platforms.md.DvDwHNtC.lean.js +1 -0
  120. package/public/docs/assets/user-guide_alternative-log-retrieval_ssh-bastion.md.Gszzlplv.js +1 -0
  121. package/public/docs/assets/user-guide_alternative-log-retrieval_ssh-bastion.md.Gszzlplv.lean.js +1 -0
  122. package/public/docs/assets/user-guide_getting-started.md.DYHHHkQR.js +1 -0
  123. package/public/docs/assets/user-guide_getting-started.md.DYHHHkQR.lean.js +1 -0
  124. package/public/docs/assets/user-guide_getting-started_ci-and-automation.md.DpnfKz3x.js +22 -0
  125. package/public/docs/assets/user-guide_getting-started_ci-and-automation.md.DpnfKz3x.lean.js +1 -0
  126. package/public/docs/assets/user-guide_getting-started_reading-the-dashboard.md.Obf3W3__.js +1 -0
  127. package/public/docs/assets/user-guide_getting-started_reading-the-dashboard.md.Obf3W3__.lean.js +1 -0
  128. package/public/docs/assets/user-guide_getting-started_regression-metric-keys.md.B_gzr2Mw.js +1 -0
  129. package/public/docs/assets/user-guide_getting-started_regression-metric-keys.md.B_gzr2Mw.lean.js +1 -0
  130. package/public/docs/assets/user-guide_getting-started_tuning-thresholds.md.B_MyjyZm.js +4 -0
  131. package/public/docs/assets/user-guide_getting-started_tuning-thresholds.md.B_MyjyZm.lean.js +1 -0
  132. package/public/docs/assets/user-guide_mcp-tools.md.C65SjWUL.js +16 -0
  133. package/public/docs/assets/user-guide_mcp-tools.md.C65SjWUL.lean.js +1 -0
  134. package/public/docs/assets/user-guide_mcp-tools_compare-runs.md.CPowlb2q.js +55 -0
  135. package/public/docs/assets/user-guide_mcp-tools_compare-runs.md.CPowlb2q.lean.js +1 -0
  136. package/public/docs/assets/user-guide_mcp-tools_diagnose-run.md.BP1vnHA5.js +23 -0
  137. package/public/docs/assets/user-guide_mcp-tools_diagnose-run.md.BP1vnHA5.lean.js +1 -0
  138. package/public/docs/assets/user-guide_mcp-tools_evaluate-budgets.md.Dq5Azk8G.js +26 -0
  139. package/public/docs/assets/user-guide_mcp-tools_evaluate-budgets.md.Dq5Azk8G.lean.js +1 -0
  140. package/public/docs/assets/user-guide_mcp-tools_get-finding-documentation.md.BtTTHNV6.js +14 -0
  141. package/public/docs/assets/user-guide_mcp-tools_get-finding-documentation.md.BtTTHNV6.lean.js +1 -0
  142. package/public/docs/assets/user-guide_mcp-tools_get-finding-evidence.md.rKyu4wts.js +20 -0
  143. package/public/docs/assets/user-guide_mcp-tools_get-finding-evidence.md.rKyu4wts.lean.js +1 -0
  144. package/public/docs/assets/user-guide_mcp-tools_get-reference-doc.md.Yac4tH3u.js +5 -0
  145. package/public/docs/assets/user-guide_mcp-tools_get-reference-doc.md.Yac4tH3u.lean.js +1 -0
  146. package/public/docs/assets/user-guide_mcp-tools_get-run-summary.md.DpxZ4ZNp.js +15 -0
  147. package/public/docs/assets/user-guide_mcp-tools_get-run-summary.md.DpxZ4ZNp.lean.js +1 -0
  148. package/public/docs/assets/user-guide_mcp-tools_list-runs.md.DpQPeCM3.js +12 -0
  149. package/public/docs/assets/user-guide_mcp-tools_list-runs.md.DpQPeCM3.lean.js +1 -0
  150. package/public/docs/assets/user-guide_run-comparison.md.B6aNtDYF.js +1 -0
  151. package/public/docs/assets/user-guide_run-comparison.md.B6aNtDYF.lean.js +1 -0
  152. package/public/docs/assets/user-guide_run-comparison_how-stages-are-matched.md.XBzQ1oEU.js +1 -0
  153. package/public/docs/assets/user-guide_run-comparison_how-stages-are-matched.md.XBzQ1oEU.lean.js +1 -0
  154. package/public/docs/assets/user-guide_understanding-findings.md.Dqt2H_C1.js +1 -0
  155. package/public/docs/contributor-guide/architecture/board-widgets/caching-and-memory.html +30 -0
  156. package/public/docs/contributor-guide/architecture/board-widgets/plan-advisor.html +30 -0
  157. package/public/docs/contributor-guide/architecture/board-widgets/visual-system-and-templating.html +30 -0
  158. package/public/docs/contributor-guide/architecture/board-widgets.html +7 -6
  159. package/public/docs/contributor-guide/architecture/detector-contract/plan-attribution.html +30 -0
  160. package/public/docs/contributor-guide/architecture/detector-contract/threshold-tables.html +30 -0
  161. package/public/docs/contributor-guide/architecture/detector-contract.html +7 -6
  162. package/public/docs/contributor-guide/architecture/drill-down/plan-graph.html +30 -0
  163. package/public/docs/contributor-guide/architecture/drill-down.html +7 -6
  164. package/public/docs/contributor-guide/architecture/impact-estimation/caveats-tuning-and-coverage.html +30 -0
  165. package/public/docs/contributor-guide/architecture/impact-estimation/stage-union-rollup.html +30 -0
  166. package/public/docs/contributor-guide/architecture/impact-estimation.html +7 -6
  167. package/public/docs/contributor-guide/architecture/index.html +7 -6
  168. package/public/docs/contributor-guide/architecture/overview.html +7 -6
  169. package/public/docs/contributor-guide/architecture/state-and-history/history-server-intake.html +30 -0
  170. package/public/docs/contributor-guide/architecture/state-and-history/run-comparison-internals.html +30 -0
  171. package/public/docs/contributor-guide/architecture/state-and-history.html +7 -6
  172. package/public/docs/contributor-guide/architecture/widget-rendering/findings-tab.html +30 -0
  173. package/public/docs/contributor-guide/architecture/widget-rendering/investigation-routing.html +30 -0
  174. package/public/docs/contributor-guide/architecture/widget-rendering/list-sort-mode.html +30 -0
  175. package/public/docs/contributor-guide/architecture/widget-rendering/reference-section.html +30 -0
  176. package/public/docs/contributor-guide/architecture/widget-rendering/row-expand-contract.html +30 -0
  177. package/public/docs/contributor-guide/architecture/widget-rendering.html +7 -6
  178. package/public/docs/contributor-guide/architecture/worker-protocol/cli-and-mcp.html +30 -0
  179. package/public/docs/contributor-guide/architecture/worker-protocol/core-and-event-validation.html +30 -0
  180. package/public/docs/contributor-guide/architecture/worker-protocol/evidence-availability.html +35 -0
  181. package/public/docs/contributor-guide/architecture/worker-protocol/evidence-report.html +30 -0
  182. package/public/docs/contributor-guide/architecture/worker-protocol.html +7 -11
  183. package/public/docs/contributor-guide/contributing.html +7 -6
  184. package/public/docs/contributor-guide/development-setup.html +11 -10
  185. package/public/docs/contributor-guide/testing.html +7 -6
  186. package/public/docs/hashmap.json +1 -1
  187. package/public/docs/index.html +5 -4
  188. package/public/docs/tuning-reference/anti-patterns.html +5 -4
  189. package/public/docs/tuning-reference/aqe.html +5 -4
  190. package/public/docs/tuning-reference/bottleneck-broadcast-sizing.html +5 -4
  191. package/public/docs/tuning-reference/bottleneck-cold-start.html +5 -4
  192. package/public/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html +5 -4
  193. package/public/docs/tuning-reference/bottleneck-failures.html +5 -4
  194. package/public/docs/tuning-reference/bottleneck-gc.html +5 -4
  195. package/public/docs/tuning-reference/bottleneck-job-failure-rate.html +5 -4
  196. package/public/docs/tuning-reference/bottleneck-memory-utilization.html +5 -4
  197. package/public/docs/tuning-reference/bottleneck-retry-waste.html +5 -4
  198. package/public/docs/tuning-reference/bottleneck-shuffle.html +5 -4
  199. package/public/docs/tuning-reference/bottleneck-skew.html +5 -4
  200. package/public/docs/tuning-reference/bottleneck-slow-host.html +5 -4
  201. package/public/docs/tuning-reference/bottleneck-small-files.html +5 -4
  202. package/public/docs/tuning-reference/bottleneck-spill.html +5 -4
  203. package/public/docs/tuning-reference/bottleneck-straggler.html +5 -4
  204. package/public/docs/tuning-reference/bottleneck-tiny-tasks.html +5 -4
  205. package/public/docs/tuning-reference/bottleneck-utilization.html +5 -4
  206. package/public/docs/tuning-reference/caching.html +5 -4
  207. package/public/docs/tuning-reference/cluster-config.html +5 -4
  208. package/public/docs/tuning-reference/config.html +5 -4
  209. package/public/docs/tuning-reference/data-formats.html +5 -4
  210. package/public/docs/tuning-reference/index.html +5 -4
  211. package/public/docs/tuning-reference/intro.html +5 -4
  212. package/public/docs/tuning-reference/joins.html +5 -4
  213. package/public/docs/tuning-reference/memory-model.html +5 -4
  214. package/public/docs/tuning-reference/metrics.html +5 -4
  215. package/public/docs/tuning-reference/partitioning.html +5 -4
  216. package/public/docs/tuning-reference/pyspark.html +5 -4
  217. package/public/docs/tuning-reference/shuffle.html +5 -4
  218. package/public/docs/tuning-reference/spark-architecture.html +5 -4
  219. package/public/docs/tuning-reference/table-formats.html +5 -4
  220. package/public/docs/user-guide/alternative-log-retrieval/managed-platforms.html +33 -0
  221. package/public/docs/user-guide/alternative-log-retrieval/ssh-bastion.html +30 -0
  222. package/public/docs/user-guide/alternative-log-retrieval.html +11 -7
  223. package/public/docs/user-guide/getting-started/ci-and-automation.html +51 -0
  224. package/public/docs/user-guide/getting-started/reading-the-dashboard.html +30 -0
  225. package/public/docs/user-guide/getting-started/regression-metric-keys.html +30 -0
  226. package/public/docs/user-guide/getting-started/tuning-thresholds.html +33 -0
  227. package/public/docs/user-guide/getting-started.html +7 -11
  228. package/public/docs/user-guide/mcp-tools/compare-runs.html +84 -0
  229. package/public/docs/user-guide/mcp-tools/diagnose-run.html +52 -0
  230. package/public/docs/user-guide/mcp-tools/evaluate-budgets.html +55 -0
  231. package/public/docs/user-guide/mcp-tools/get-finding-documentation.html +43 -0
  232. package/public/docs/user-guide/mcp-tools/get-finding-evidence.html +49 -0
  233. package/public/docs/user-guide/mcp-tools/get-reference-doc.html +34 -0
  234. package/public/docs/user-guide/mcp-tools/get-run-summary.html +44 -0
  235. package/public/docs/user-guide/mcp-tools/list-runs.html +41 -0
  236. package/public/docs/user-guide/mcp-tools.html +12 -126
  237. package/public/docs/user-guide/run-comparison/how-stages-are-matched.html +30 -0
  238. package/public/docs/user-guide/run-comparison.html +7 -6
  239. package/public/docs/user-guide/understanding-findings.html +7 -6
  240. package/public/export-template.html +44 -44
  241. package/public/index.html +4 -4
  242. package/vendor-core/allocation.js +106 -0
  243. package/vendor-core/analyzer.js +13 -13
  244. package/vendor-core/api-merge-writes.js +80 -0
  245. package/vendor-core/cli/budgets.js +23 -9
  246. package/vendor-core/cli/collect-run.js +11 -4
  247. package/vendor-core/cli/regression-budgets.js +83 -0
  248. package/vendor-core/comparison-output.js +49 -0
  249. package/vendor-core/comparison-verdict.js +22 -22
  250. package/vendor-core/core-source-hash.txt +1 -1
  251. package/vendor-core/delta-commands.js +13 -0
  252. package/vendor-core/delta-log-paths.js +42 -0
  253. package/vendor-core/delta-targets.js +127 -0
  254. package/vendor-core/detectors.js +202 -68
  255. package/vendor-core/docs-content/detection/cache.md +3 -2
  256. package/vendor-core/docs-content/detection/cfg.md +9 -8
  257. package/vendor-core/docs-content/detection/chrn.md +1 -2
  258. package/vendor-core/docs-content/detection/cold.md +4 -2
  259. package/vendor-core/docs-content/detection/fail.md +3 -2
  260. package/vendor-core/docs-content/detection/gc.md +3 -2
  261. package/vendor-core/docs-content/detection/host.md +2 -1
  262. package/vendor-core/docs-content/detection/local.md +1 -1
  263. package/vendor-core/docs-content/detection/mem.md +5 -2
  264. package/vendor-core/docs-content/detection/plan.md +2 -1
  265. package/vendor-core/docs-content/detection/sfail.md +2 -1
  266. package/vendor-core/docs-content/detection/shape.md +5 -4
  267. package/vendor-core/docs-content/detection/skew.md +3 -1
  268. package/vendor-core/docs-content/detection/slow.md +2 -2
  269. package/vendor-core/docs-content/detection/spec.md +2 -3
  270. package/vendor-core/docs-content/detection/spill.md +1 -1
  271. package/vendor-core/docs-site-config.js +1 -1
  272. package/vendor-core/effective-conf.js +107 -0
  273. package/vendor-core/efficiency-model.js +8 -6
  274. package/vendor-core/event-handlers.js +255 -56
  275. package/vendor-core/event-schemas.js +5 -1
  276. package/vendor-core/evidence-report.js +31 -14
  277. package/vendor-core/finding-generic-recommendation.js +20 -1
  278. package/vendor-core/finding-names.js +7 -0
  279. package/vendor-core/finding-presentation.js +61 -26
  280. package/vendor-core/finding-tag-help.js +1 -1
  281. package/vendor-core/finding-types.js +12 -0
  282. package/vendor-core/format-utils.js +4 -3
  283. package/vendor-core/impact-estimator.js +20 -2
  284. package/vendor-core/impact-format.js +14 -13
  285. package/vendor-core/impact-model.js +27 -5
  286. package/vendor-core/ingest.js +4 -2
  287. package/vendor-core/list-runs.js +5 -2
  288. package/vendor-core/mcp-server-factory.js +13 -6
  289. package/vendor-core/mcp-tools.js +43 -34
  290. package/vendor-core/model-assembler.js +23 -1
  291. package/vendor-core/parser-worker.js +1 -1
  292. package/vendor-core/plan-graph-model.js +1 -1
  293. package/vendor-core/plan-summary.js +13 -10
  294. package/vendor-core/proxy.js +3 -1
  295. package/vendor-core/python-stage.js +25 -0
  296. package/vendor-core/recommendation-rollup.js +16 -9
  297. package/vendor-core/redact.js +51 -10
  298. package/vendor-core/remediation.js +20 -0
  299. package/vendor-core/run-comparison.js +118 -140
  300. package/vendor-core/run-interpretation.js +2 -1
  301. package/vendor-core/run-metrics.js +198 -0
  302. package/vendor-core/run-output.js +26 -0
  303. package/vendor-core/run-totals.js +24 -0
  304. package/vendor-core/run-verdict.js +3 -4
  305. package/vendor-core/scorecard-estimates.js +1 -0
  306. package/vendor-core/session-snapshot.js +7 -0
  307. package/vendor-core/shs-schemas.js +2 -2
  308. package/vendor-core/spark-memory.js +17 -0
  309. package/vendor-core/stage-alignment.js +262 -0
  310. package/vendor-core/stage-identity.js +112 -0
  311. package/vendor-core/stage-plan-nodes.js +18 -0
  312. package/vendor-core/stage-quantiles.js +4 -0
  313. package/vendor-core/types.js +54 -1
  314. package/vendor-core/wasted-core-hours.js +10 -7
  315. package/vendor-core/write-target-args.js +48 -0
  316. package/vendor-core/write-targets.js +295 -0
  317. package/public/assets/AutoscalingChurn-CUSIvvab.js +0 -1
  318. package/public/assets/CacheUtilization-Ce0K6_X2.js +0 -1
  319. package/public/assets/CachingOpportunity-Cn-LmRA4.js +0 -1
  320. package/public/assets/ColdStart-BoiYzEyV.js +0 -1
  321. package/public/assets/ConfigAudit-BoJc6PAw.js +0 -1
  322. package/public/assets/CoreUsageArea-BFfXKu7K.js +0 -1
  323. package/public/assets/Dashboard-Bo_1jGqH.js +0 -2
  324. package/public/assets/DisclosureContext-LYzHroHO.js +0 -1
  325. package/public/assets/DuplicatePlanSubtree-BaAuGAoo.js +0 -1
  326. package/public/assets/ExecutorUtilization-OE6VMFkG.js +0 -1
  327. package/public/assets/GcPressure-AoL_ZNrV.js +0 -1
  328. package/public/assets/IncompleteRun-Bv6IzgsQ.js +0 -1
  329. package/public/assets/JobFailures-DF3U2z1f.js +0 -1
  330. package/public/assets/MemoryUtilization-B65OVZlq.js +0 -1
  331. package/public/assets/OverBroadcast-DXbJREDl.js +0 -1
  332. package/public/assets/PartitionSizing-DZLnjy0d.js +0 -1
  333. package/public/assets/PlanGraphRoute-C-BGmC6j.js +0 -1
  334. package/public/assets/RetryWaste-BbEPsdMw.js +0 -1
  335. package/public/assets/ShuffleIO-wZB94LS8.js +0 -1
  336. package/public/assets/Skew-SpQE7zeJ.js +0 -1
  337. package/public/assets/SlowHost-DwDBCTvr.js +0 -1
  338. package/public/assets/SmallFiles-yjY_6sT1.js +0 -1
  339. package/public/assets/SpeculationWaste-73PDmj0c.js +0 -1
  340. package/public/assets/Spill-DEw0Kd1i.js +0 -1
  341. package/public/assets/StageFailed-BEfXdEJc.js +0 -1
  342. package/public/assets/StageFindingGroup-Bzm2TY03.js +0 -1
  343. package/public/assets/StageShape-CzR4jHOt.js +0 -1
  344. package/public/assets/StageSlowness-vMPsnAaa.js +0 -1
  345. package/public/assets/Straggler-Cf__OGl5.js +0 -1
  346. package/public/assets/TaskFailures-BZ8kLZiO.js +0 -1
  347. package/public/assets/UnderBroadcast-CBm8ZPoS.js +0 -1
  348. package/public/assets/index-CEM9bz_x.js +0 -13
  349. package/public/assets/parser-worker-Q5KCPdKT.js +0 -122
  350. package/public/assets/useRecentFiles-BrBl-AP1.js +0 -11
  351. package/public/docs/assets/chunks/@localSearchIndexroot.DbZy5EtM.js +0 -1
  352. package/public/docs/assets/contributor-guide_architecture_board-widgets.md.VTcYTH4d.js +0 -1
  353. package/public/docs/assets/contributor-guide_architecture_board-widgets.md.VTcYTH4d.lean.js +0 -1
  354. package/public/docs/assets/contributor-guide_architecture_detector-contract.md.CsqZHz_T.js +0 -1
  355. package/public/docs/assets/contributor-guide_architecture_detector-contract.md.CsqZHz_T.lean.js +0 -1
  356. package/public/docs/assets/contributor-guide_architecture_drill-down.md.CFNCE8JB.js +0 -1
  357. package/public/docs/assets/contributor-guide_architecture_drill-down.md.CFNCE8JB.lean.js +0 -1
  358. package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.HT9IJGIl.js +0 -1
  359. package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.HT9IJGIl.lean.js +0 -1
  360. package/public/docs/assets/contributor-guide_architecture_index.md.3TO9ic6w.js +0 -1
  361. package/public/docs/assets/contributor-guide_architecture_index.md.3TO9ic6w.lean.js +0 -1
  362. package/public/docs/assets/contributor-guide_architecture_overview.md.CTEqVeV4.js +0 -1
  363. package/public/docs/assets/contributor-guide_architecture_overview.md.CTEqVeV4.lean.js +0 -1
  364. package/public/docs/assets/contributor-guide_architecture_state-and-history.md.DOTVHjTV.js +0 -1
  365. package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.BmpARZve.js +0 -1
  366. package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.BmpARZve.lean.js +0 -1
  367. package/public/docs/assets/contributor-guide_architecture_worker-protocol.md.21pTYhFF.js +0 -6
  368. package/public/docs/assets/contributor-guide_architecture_worker-protocol.md.21pTYhFF.lean.js +0 -1
  369. package/public/docs/assets/contributor-guide_contributing.md.B2E9ni79.js +0 -1
  370. package/public/docs/assets/contributor-guide_development-setup.md.CjgZJwjK.js +0 -15
  371. package/public/docs/assets/contributor-guide_development-setup.md.CjgZJwjK.lean.js +0 -1
  372. package/public/docs/assets/contributor-guide_testing.md.DTwhpoN2.js +0 -1
  373. package/public/docs/assets/user-guide_alternative-log-retrieval.md.ZHPSbBkF.js +0 -1
  374. package/public/docs/assets/user-guide_alternative-log-retrieval.md.ZHPSbBkF.lean.js +0 -1
  375. package/public/docs/assets/user-guide_getting-started.md.CEa2ZX-l.js +0 -6
  376. package/public/docs/assets/user-guide_getting-started.md.CEa2ZX-l.lean.js +0 -1
  377. package/public/docs/assets/user-guide_mcp-tools.md.dQwIvdTF.js +0 -131
  378. package/public/docs/assets/user-guide_mcp-tools.md.dQwIvdTF.lean.js +0 -1
  379. package/public/docs/assets/user-guide_run-comparison.md.ChLI3MK0.js +0 -1
  380. package/public/docs/assets/user-guide_run-comparison.md.ChLI3MK0.lean.js +0 -1
  381. package/public/docs/assets/user-guide_understanding-findings.md.D1ccs60C.js +0 -1
  382. /package/public/docs/assets/{user-guide_understanding-findings.md.D1ccs60C.lean.js → user-guide_understanding-findings.md.Dqt2H_C1.lean.js} +0 -0
package/public/docs/tuning-reference/cluster-config.html CHANGED
@@ -6,14 +6,15 @@
6
6
  <title>Cluster Tuning | Spark Tuning Reference</title>
7
7
  <meta name="description" content="Cluster-level sizing: executor core/memory formulas, container memory budgeting, dynamic allocation, and locality-wait tuning.">
8
8
  <meta name="generator" content="VitePress v1.6.4">
9
- <link rel="preload stylesheet" href="/docs/assets/style.DSixAiZE.css" as="style">
9
+ <link rel="preload stylesheet" href="/docs/assets/style.tmDhD3c8.css" as="style">
10
10
  <link rel="preload stylesheet" href="/docs/vp-icons.css" as="style">
11
11
 
12
- <script type="module" src="/docs/assets/app.C6w47a9W.js"></script>
12
+ <script type="module" src="/docs/assets/app.D2GqAkhy.js"></script>
13
13
  <link rel="preload" href="/docs/assets/inter-roman-latin.Di8DUHzh.woff2" as="font" type="font/woff2" crossorigin="">
14
- <link rel="modulepreload" href="/docs/assets/chunks/theme.D-mE6j02.js">
14
+ <link rel="modulepreload" href="/docs/assets/chunks/theme.BuxVpb0H.js">
15
15
  <link rel="modulepreload" href="/docs/assets/chunks/framework.DSg0KOwT.js">
16
16
  <link rel="modulepreload" href="/docs/assets/tuning-reference_cluster-config.md.DbWG1yZq.lean.js">
17
+ <script>(()=>{const c=`${`[${["data","shuffle","product","bar"].join("-")}]`}, .VPNav, .VPLocalNav`,a=24,d=document.documentElement,l=t=>{const e=getComputedStyle(t);return e.display==="none"||t.getClientRects().length===0?0:e.position==="fixed"?t.getBoundingClientRect().bottom:e.position==="sticky"?(parseFloat(e.top)||0)+t.offsetHeight:0},u=()=>{try{return location.hash?document.getElementById(decodeURIComponent(location.hash.slice(1))):null}catch{return null}};let r=null;const s=()=>{const t=Array.from(document.querySelectorAll(c),l),e=Math.ceil(Math.max(0,...t))+a;if(e===r)return;const n=r;r=e,d.style.setProperty("--sf-anchor-offset",`${r}px`);const o=n===null?null:u();if(!o)return;const f=(parseFloat(getComputedStyle(o).paddingTop)||0)+2;Math.abs(o.getBoundingClientRect().top-n)<=f&&o.scrollIntoView()},i=()=>{const t=document.createElement("div");t.setAttribute("data-sf-anchor-offset",""),t.setAttribute("aria-hidden","true"),document.body.appendChild(t);const e=window.ResizeObserver?new ResizeObserver(s):null,n=()=>{e&&document.querySelectorAll(c).forEach(o=>e.observe(o)),s()};n(),new MutationObserver(n).observe(document.body,{childList:!0,subtree:!0}),window.addEventListener("resize",s),window.addEventListener("load",s)};document.readyState==="loading"?document.addEventListener("DOMContentLoaded",i,{once:!0}):i()})();</script>
17
18
  <link rel="icon" type="image/svg+xml" href="/docs/favicon.svg">
18
19
  <link rel="preconnect" href="https://fonts.googleapis.com">
19
20
  <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin="">
@@ -23,7 +24,7 @@
23
24
  </head>
24
25
  <body>
25
26
  <div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Getting Started</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/intro.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Introduction</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Optimization Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/spark-architecture.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Execution Model</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/memory-model.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Management</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/partitioning.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Partitioning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/joins.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Join Optimization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/data-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Data Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/table-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Table Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/caching.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Caching & Persistence</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/pyspark.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>PySpark Specifics</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/aqe.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Adaptive Query Execution</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/cluster-config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cluster Tuning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/anti-patterns.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Anti-Patterns</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Detector Catalog</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-skew.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Skew</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle I/O</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-spill.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory / Disk Spill</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-gc.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>GC Pressure</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-cold-start.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cold Start</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Executor Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-slow-host.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Slow Host</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-failures.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Failures</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-straggler.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Stragglers</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-retry-waste.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Retry Waste</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-tiny-tasks.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Tiny Tasks</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-job-failure-rate.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Job Failure Rate</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-memory-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Duplicate Plan Subtree</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-small-files.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Small Files</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-broadcast-sizing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Broadcast Sizing</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Reference</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/metrics.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Metrics Glossary</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Config Quick-Reference</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_tuning-reference_cluster-config" data-v-39a288b8><div><h1 id="cluster-config" tabindex="-1">Cluster Tuning <a class="header-anchor" href="#cluster-config" aria-label="Permalink to &quot;Cluster Tuning {#cluster-config}&quot;">​</a></h1><h2 id="sizing-executors-and-containers" tabindex="-1">Sizing executors and containers <a class="header-anchor" href="#sizing-executors-and-containers" aria-label="Permalink to &quot;Sizing executors and containers&quot;">​</a></h2><p>Cluster tuning is the set of decisions that map an application&#39;s cores, memory, and executor count onto the underlying cluster (YARN or Kubernetes) plus a handful of related knobs (data locality, dynamic allocation, per-task CPU reservation) that all interact with that sizing decision.</p><p>The starting point is executor sizing. Working through a concrete example (a 10-node cluster with 16 cores and 64GB RAM per node), the derivation runs: assign 5 cores per executor; reserve about 1 core per node for Hadoop/YARN/OS daemons, leaving 15 usable cores per node (150 total); dividing by 5 cores per executor gives 30 executors, minus 1 reserved for the YARN ApplicationMaster (29); with 3 executors per node, memory per executor is 64GB / 3 ≈ 21GB, and after subtracting about 7% for YARN memory overhead that comes out to roughly 18GB usable, a recommended shape of 29 executors × 5 cores × 18GB<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup>. Cloudera&#39;s own worked example, on a 6-node cluster with the same 16-core / 64GB-per-node hardware, lands on the same shape: <code>--num-executors 17 --executor-cores 5 --executor-memory 19G</code>, rather than one &quot;fat&quot; executor per node (<code>--num-executors 6 --executor-cores 15 --executor-memory 63G</code>)<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup>. Both sources treat the resulting task count (executor-cores × num-executors) as the single most important tuning lever, since Spark cannot compensate for too little parallelism on its own<sup class="footnote-ref"><a href="#fn2" id="fnref2:1">[2:1]</a></sup>. <code>spark.executor.cores</code> is the config that controls concurrent tasks per executor; it defaults to 1 in YARN mode, or to all available cores on the worker in standalone mode<sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup>, and <code>spark.executor.memory</code> sets the JVM heap size per executor<sup class="footnote-ref"><a href="#fn2" id="fnref2:2">[2:2]</a></sup>. Independent of the sizing formula above, the Spark tuning guide recommends targeting 2–3 tasks per CPU core across the cluster<sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup>.</p><p>On top of executor memory sits container memory. Spark computes the total container/pod allocation as <code>executorMemoryMiB + memoryOverheadMiB + memoryOffHeapMiB + pysparkMemToUseMiB</code>. On YARN this is what gets allocated for the container, on Kubernetes it becomes the pod memory limit<sup class="footnote-ref"><a href="#fn5" id="fnref5">[5]</a></sup>. <code>spark.executor.memoryOverhead</code> itself defaults to <code>max(0.1 * executorMemory, 384MB)</code><sup class="footnote-ref"><a href="#fn5" id="fnref5:1">[5:1]</a></sup><sup class="footnote-ref"><a href="#fn3" id="fnref3:1">[3:1]</a></sup>.</p><img class="light-only" src="/docs/assets/container-memory.DIO0AnIm.svg" alt="Container memory sums executor heap, memory overhead, off-heap size, and pyspark memory into the requested container size; the resource manager grants a container of that size and OOMKills the executor when its actual runtime footprint spills past that limit."><img class="dark-only" src="/docs/assets/container-memory.dark.CP-5zuCl.svg" alt="Container memory sums executor heap, memory overhead, off-heap size, and pyspark memory into the requested container size; the resource manager grants a container of that size and OOMKills the executor when its actual runtime footprint spills past that limit."><blockquote><p><strong>PySpark:</strong> <code>spark.executor.pyspark.memory</code> is only added as its own container-memory term when it&#39;s set explicitly; otherwise PySpark&#39;s memory use is folded into the general overhead budget rather than tracked separately.<sup class="footnote-ref"><a href="#fn5" id="fnref5:2">[5:2]</a></sup></p></blockquote><p>On Kubernetes specifically, <code>spark.kubernetes.executor.request.cores</code> and <code>spark.kubernetes.executor.limit.cores</code> take priority over <code>spark.executor.cores</code> for the pod&#39;s CPU request/limit sent to the Kubernetes scheduler<sup class="footnote-ref"><a href="#fn6" id="fnref6">[6]</a></sup>, while <code>spark.executor.cores</code> remains the config Spark itself uses to size the number of concurrent task slots<sup class="footnote-ref"><a href="#fn3" id="fnref3:2">[3:2]</a></sup>.</p><p>Two more knobs round out the picture. <code>spark.locality.wait</code> (default <code>3s</code>) controls how long a task waits for a data-local placement before Spark gives up and schedules it less locally, stepping through the same wait across process-local → node-local → rack-local → any; each level can be overridden independently via <code>spark.locality.wait.process</code>, <code>.node</code>, and <code>.rack</code>, all of which default to the base wait when left unset<sup class="footnote-ref"><a href="#fn3" id="fnref3:3">[3:3]</a></sup>. Dynamic allocation lets Spark add and remove executors as work changes, but it requires one of several supporting mechanisms: an external shuffle service, shuffle tracking (<code>spark.dynamicAllocation.shuffleTracking.enabled</code>, default <code>true</code> since Spark 3.0), shuffle-block decommission, or the experimental sort-IO plugin<sup class="footnote-ref"><a href="#fn3" id="fnref3:4">[3:4]</a></sup>. Removing an executor can otherwise destroy shuffle state it&#39;s holding. Finally, <code>spark.task.cpus</code> (default <code>1</code>) sets how many cores each task reserves; the number of concurrent task slots per executor is derived jointly from it and <code>spark.executor.cores</code>, roughly <code>spark.executor.cores / spark.task.cpus</code><sup class="footnote-ref"><a href="#fn3" id="fnref3:5">[3:5]</a></sup>.</p><h2 id="spotting-a-bad-sizing-decision" tabindex="-1">Spotting a bad sizing decision <a class="header-anchor" href="#spotting-a-bad-sizing-decision" aria-label="Permalink to &quot;Spotting a bad sizing decision&quot;">​</a></h2><p>Getting that sizing wrong shows up in a handful of specific symptoms. The clearest sign of an under-sized executor layout is HDFS throughput dropping under load: &quot;HDFS client has trouble with tons of concurrent threads. It was observed that HDFS achieves full write throughput with ~5 tasks per executor&quot;<sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup>. The &quot;fat executor&quot; case (one executor per node, using all 16 cores) shows the failure mode directly: &quot;with all 16 cores per executor... HDFS throughput will hurt and it&#39;ll result in excessive garbage [collection]&quot;<sup class="footnote-ref"><a href="#fn1" id="fnref1:2">[1:2]</a></sup>.</p><p>On the memory side, an executor or pod that dies with no Spark-level error is a strong signal that off-heap memory was never added into the container budget. Enabling <code>spark.memory.offHeap.enabled=true</code> with <code>spark.memory.offHeap.size=1g</code> on top of an 8G executor with default overhead (819MB) means real usage is 8192+819+1024 = 10,035MB, while the container was only granted 8192+819 = 9,011MB. The executor gets killed by YARN, or OOMKilled by the Kubernetes kubelet, with no warning from Spark itself<sup class="footnote-ref"><a href="#fn5" id="fnref5:3">[5:3]</a></sup>.</p><p>With dynamic allocation on, unexplained shuffle recomputation is a detectable symptom of executors being reclaimed mid-shuffle: &quot;In the event of stragglers... dynamic allocation may remove an executor before the shuffle completes, in which case the shuffle files written by that executor must be recomputed unnecessarily&quot;<sup class="footnote-ref"><a href="#fn6" id="fnref6:1">[6:1]</a></sup>. Jobs that abort with a serialized-result-size error are hitting the <code>spark.driver.maxResultSize</code> guardrail rather than a driver heap exhaustion: &quot;Jobs will be aborted if the total size [of serialized action results] is above this limit&quot;<sup class="footnote-ref"><a href="#fn3" id="fnref3:6">[3:6]</a></sup>. And on the resource-waste side, executors that are provisioned but never assigned work are a sign that dynamic allocation is targeting full parallelism against a workload made of many small tasks<sup class="footnote-ref"><a href="#fn3" id="fnref3:7">[3:7]</a></sup>.</p><h2 id="what-a-bad-sizing-decision-costs" tabindex="-1">What a bad sizing decision costs <a class="header-anchor" href="#what-a-bad-sizing-decision-costs" aria-label="Permalink to &quot;What a bad sizing decision costs&quot;">​</a></h2><p>Each symptom above carries a specific price tag. Task count (<code>executor-cores × num-executors</code>) is treated by both cited sources as the single most important tuning lever, since Spark cannot compensate for too little parallelism on its own<sup class="footnote-ref"><a href="#fn2" id="fnref2:3">[2:3]</a></sup>. Going the other way, cramming too many cores into one executor degrades HDFS throughput and drives up garbage collection<sup class="footnote-ref"><a href="#fn1" id="fnref1:3">[1:3]</a></sup>.</p><p>Container-memory misconfiguration matters because the failure is silent from Spark&#39;s point of view: off-heap memory that isn&#39;t accounted for in <code>spark.executor.memoryOverhead</code> causes the executor to actually use more memory than the container/pod was granted, so it gets killed externally (by YARN or the kubelet) with no Spark-level diagnostic to point at the real cause<sup class="footnote-ref"><a href="#fn5" id="fnref5:4">[5:4]</a></sup>.</p><p>Dynamic allocation&#39;s interaction with shuffle state matters because, before dynamic allocation existed, an executor exiting alongside its application meant all its state could be safely discarded; with dynamic allocation, the application keeps running after an executor is explicitly removed, so any later need for that executor&#39;s state forces a recompute<sup class="footnote-ref"><a href="#fn6" id="fnref6:2">[6:2]</a></sup>. That is exactly why Spark needs &quot;a mechanism to decommission an executor gracefully by preserving its state before removing it&quot;<sup class="footnote-ref"><a href="#fn6" id="fnref6:3">[6:3]</a></sup>. Without shuffle tracking, an external shuffle service, or shuffle-block decommission enabled, dynamic allocation either can&#39;t be turned on at all, or, if it&#39;s active regardless, an executor holding unpreserved shuffle output that gets removed forces exactly that unnecessary recompute<sup class="footnote-ref"><a href="#fn3" id="fnref3:8">[3:8]</a></sup><sup class="footnote-ref"><a href="#fn6" id="fnref6:4">[6:4]</a></sup>.</p><p>On the driver side, <code>spark.driver.maxResultSize</code> and <code>spark.driver.memory</code> are separate budgets that still interact: whether a high <code>maxResultSize</code> actually causes an out-of-memory error &quot;depends on spark.driver.memory and memory overhead of objects in JVM&quot;<sup class="footnote-ref"><a href="#fn3" id="fnref3:9">[3:9]</a></sup>. Raising one without considering the other doesn&#39;t fully protect the driver.</p><p>Finally, over-provisioning executors against a small-task workload wastes cluster resources: &quot;with small tasks this setting can waste a lot of resources due to executor allocation overhead, as some executor might not even do any work&quot;<sup class="footnote-ref"><a href="#fn3" id="fnref3:10">[3:10]</a></sup>.</p><h2 id="sizing-the-cluster-correctly" tabindex="-1">Sizing the cluster correctly <a class="header-anchor" href="#sizing-the-cluster-correctly" aria-label="Permalink to &quot;Sizing the cluster correctly&quot;">​</a></h2><p>Avoiding those costs starts from a formula, not one fat executor per node. Apply the balanced-executor formula instead; the worked examples above give shapes like 29 × 5 × 18GB for a 10-node/16-core/64GB cluster, or 17 × 5 × 19G for a 6-node cluster with the same per-node hardware<sup class="footnote-ref"><a href="#fn1" id="fnref1:4">[1:4]</a></sup><sup class="footnote-ref"><a href="#fn2" id="fnref2:4">[2:4]</a></sup>. Keep <code>--executor-cores</code> at or below roughly 5 so HDFS client concurrency stays in the range where it sustains full write throughput<sup class="footnote-ref"><a href="#fn1" id="fnref1:5">[1:5]</a></sup>.</p><p>When enabling off-heap memory, account for <code>spark.memory.offHeap.size</code> in the container/pod memory budget explicitly; <code>spark.executor.memoryOverhead</code>&#39;s default of <code>max(0.1 * executorMemory, 384MB)</code> does not include it<sup class="footnote-ref"><a href="#fn5" id="fnref5:5">[5:5]</a></sup>. If you&#39;re still referencing the older <code>spark.yarn.executor.memoryOverhead</code> name, note it was removed in Spark 3.0<sup class="footnote-ref"><a href="#fn5" id="fnref5:6">[5:6]</a></sup>.</p><p>On Kubernetes, remember that <code>spark.kubernetes.executor.request.cores</code> and <code>.limit.cores</code> govern the pod&#39;s CPU request/limit and take priority over <code>spark.executor.cores</code> for that purpose<sup class="footnote-ref"><a href="#fn6" id="fnref6:5">[6:5]</a></sup>, while <code>spark.executor.cores</code> separately governs Spark&#39;s own task-slot count<sup class="footnote-ref"><a href="#fn3" id="fnref3:11">[3:11]</a></sup>. The two need to be reasoned about together rather than assumed to be redundant.</p><p>Tune <code>spark.locality.wait</code> (and its per-level overrides) upward when tasks are long-running and locality is poor, since the default is tuned for typical workloads<sup class="footnote-ref"><a href="#fn3" id="fnref3:12">[3:12]</a></sup>; set <code>spark.locality.wait.node</code> to <code>0</code> to skip straight to rack locality when node locality isn&#39;t achievable<sup class="footnote-ref"><a href="#fn3" id="fnref3:13">[3:13]</a></sup>.</p><p>Turn on shuffle tracking (<code>spark.dynamicAllocation.shuffleTracking.enabled</code>, default <code>true</code> since Spark 3.0) or the external shuffle service so dynamic allocation can reclaim executors without losing shuffle output or forcing recomputation<sup class="footnote-ref"><a href="#fn3" id="fnref3:14">[3:14]</a></sup><sup class="footnote-ref"><a href="#fn6" id="fnref6:6">[6:6]</a></sup>. The external shuffle service additionally lets persisted RDD blocks survive executor removal when <code>spark.shuffle.service.fetch.rdd.enabled</code> is set, and executors holding cached blocks are, by default, never removed at all, tunable via <code>spark.dynamicAllocation.cachedExecutorIdleTimeout</code><sup class="footnote-ref"><a href="#fn6" id="fnref6:7">[6:7]</a></sup>.</p><p>Set <code>spark.driver.maxResultSize</code> as a guardrail on serialized action-result size, and size <code>spark.driver.memory</code> with it in mind rather than in isolation<sup class="footnote-ref"><a href="#fn3" id="fnref3:15">[3:15]</a></sup>. Use <code>spark.dynamicAllocation.executorAllocationRatio</code> (default <code>1.0</code>, added in 2.4.0) to scale down over-allocation when tasks are small; for example, a value of <code>0.5</code> halves the target executor count dynamic allocation would otherwise compute<sup class="footnote-ref"><a href="#fn3" id="fnref3:16">[3:16]</a></sup>. Raise <code>spark.task.cpus</code> above its default of <code>1</code> when a task needs more than one core; doing so proportionally reduces concurrent task slots per executor (<code>spark.executor.cores / spark.task.cpus</code>)<sup class="footnote-ref"><a href="#fn3" id="fnref3:17">[3:17]</a></sup>.</p><h2 id="bottleneck-autoscaling-churn" tabindex="-1">Autoscaling churn <a class="header-anchor" href="#bottleneck-autoscaling-churn" aria-label="Permalink to &quot;Autoscaling churn {#bottleneck-autoscaling-churn}&quot;">​</a></h2><p><span class="tag">CHRN</span> <span class="tag">EXPERIMENTAL</span></p><p>Dynamic allocation adds executors once tasks back up and frees them once they go idle<sup class="footnote-ref"><a href="#fn6" id="fnref6:8">[6:8]</a></sup>. Churn is a narrower failure mode inside that same mechanism: executors that get stood up and torn down again before they&#39;ve done much useful work, cycling through provisioning and JVM startup cost repeatedly instead of running tasks.</p><h3 id="how-it-s-detected" tabindex="-1">How it&#39;s detected <a class="header-anchor" href="#how-it-s-detected" aria-label="Permalink to &quot;How it&#39;s detected&quot;">​</a></h3><p>An executor counts as short-lived when its lifetime, from its <code>ExecutorAdded</code> event to either its <code>ExecutorRemoved</code> event or the end of the run if it was never removed, is under 2 minutes. This is evaluated once a run has added at least 5 executors, a floor that keeps a two- or three-executor job from registering on ordinary scale-down.</p><table tabindex="0"><thead><tr><th>Signal</th><th>Warning</th><th>Critical</th></tr></thead><tbody><tr><td>Share of added executors that are short-lived (&lt; 2 min lifetime)</td><td>&gt; 30%</td><td>&gt; 60%</td></tr></tbody></table><p>The 2-minute lifetime cutoff and the 30%/60% split are unvalidated heuristics rather than figures published in Spark&#39;s own documentation or benchmarks, so treat a finding here as a prompt to look at the executor timeline rather than a calibrated verdict.</p><h3 id="why-it-matters" tabindex="-1">Why it matters <a class="header-anchor" href="#why-it-matters" aria-label="Permalink to &quot;Why it matters&quot;">​</a></h3><p>Every short-lived executor pays the same provisioning and JVM-startup cost as a long-lived one, but returns little task work in exchange. A run that keeps flapping between scaling up and down spends more of its wall-clock time paying that repeated overhead than one that settles into a stable executor count.</p><h3 id="how-to-fix-it" tabindex="-1">How to fix it <a class="header-anchor" href="#how-to-fix-it" aria-label="Permalink to &quot;How to fix it&quot;">​</a></h3><ul><li>Raise <code>spark.dynamicAllocation.executorIdleTimeout</code> (default 60s<sup class="footnote-ref"><a href="#fn3" id="fnref3:18">[3:18]</a></sup>) so an executor survives a brief lull instead of being released the moment it goes idle, only to be requested again shortly after.</li><li>Widen the gap between <code>spark.dynamicAllocation.minExecutors</code> and <code>.maxExecutors</code><sup class="footnote-ref"><a href="#fn3" id="fnref3:19">[3:19]</a></sup>: bounds set too close together force Spark to repeatedly add and remove executors to track small fluctuations in the task backlog instead of settling into a stable range.</li></ul><h2 id="sources" tabindex="-1">Sources <a class="header-anchor" href="#sources" aria-label="Permalink to &quot;Sources&quot;">​</a></h2><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><a href="https://raw.githubusercontent.com/spoddutur/spark-notes/master/distribution_of_executors_cores_and_memory_for_spark_application.md" target="_blank" rel="noreferrer">Distribution of Executors, Cores and Memory for a Spark Application</a> <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a> <a href="#fnref1:2" class="footnote-backref">↩︎</a> <a href="#fnref1:3" class="footnote-backref">↩︎</a> <a href="#fnref1:4" class="footnote-backref">↩︎</a> <a href="#fnref1:5" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><a href="https://blog.cloudera.com/how-to-tune-your-apache-spark-jobs-part-2/" target="_blank" rel="noreferrer">How to Tune Your Apache Spark Jobs (Part 2)</a> <a href="#fnref2" class="footnote-backref">↩︎</a> <a href="#fnref2:1" class="footnote-backref">↩︎</a> <a href="#fnref2:2" class="footnote-backref">↩︎</a> <a href="#fnref2:3" class="footnote-backref">↩︎</a> <a href="#fnref2:4" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration — Spark</a> <a href="#fnref3" class="footnote-backref">↩︎</a> <a href="#fnref3:1" class="footnote-backref">↩︎</a> <a href="#fnref3:2" class="footnote-backref">↩︎</a> <a href="#fnref3:3" class="footnote-backref">↩︎</a> <a href="#fnref3:4" class="footnote-backref">↩︎</a> <a href="#fnref3:5" class="footnote-backref">↩︎</a> <a href="#fnref3:6" class="footnote-backref">↩︎</a> <a href="#fnref3:7" class="footnote-backref">↩︎</a> <a href="#fnref3:8" class="footnote-backref">↩︎</a> <a href="#fnref3:9" class="footnote-backref">↩︎</a> <a href="#fnref3:10" class="footnote-backref">↩︎</a> <a href="#fnref3:11" class="footnote-backref">↩︎</a> <a href="#fnref3:12" class="footnote-backref">↩︎</a> <a href="#fnref3:13" class="footnote-backref">↩︎</a> <a href="#fnref3:14" class="footnote-backref">↩︎</a> <a href="#fnref3:15" class="footnote-backref">↩︎</a> <a href="#fnref3:16" class="footnote-backref">↩︎</a> <a href="#fnref3:17" class="footnote-backref">↩︎</a> <a href="#fnref3:18" class="footnote-backref">↩︎</a> <a href="#fnref3:19" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/tuning.html" target="_blank" rel="noreferrer">Spark Tuning Guide</a> <a href="#fnref4" class="footnote-backref">↩︎</a></p></li><li id="fn5" class="footnote-item"><p><a href="https://luminousmen.com/post/dive-into-spark-memory" target="_blank" rel="noreferrer">Dive into Spark Memory</a> <a href="#fnref5" class="footnote-backref">↩︎</a> <a href="#fnref5:1" class="footnote-backref">↩︎</a> <a href="#fnref5:2" class="footnote-backref">↩︎</a> <a href="#fnref5:3" class="footnote-backref">↩︎</a> <a href="#fnref5:4" class="footnote-backref">↩︎</a> <a href="#fnref5:5" class="footnote-backref">↩︎</a> <a href="#fnref5:6" class="footnote-backref">↩︎</a></p></li><li id="fn6" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/job-scheduling.html" target="_blank" rel="noreferrer">Job Scheduling — Dynamic Resource Allocation</a> <a href="#fnref6" class="footnote-backref">↩︎</a> <a href="#fnref6:1" class="footnote-backref">↩︎</a> <a href="#fnref6:2" class="footnote-backref">↩︎</a> <a href="#fnref6:3" class="footnote-backref">↩︎</a> <a href="#fnref6:4" class="footnote-backref">↩︎</a> <a href="#fnref6:5" class="footnote-backref">↩︎</a> <a href="#fnref6:6" class="footnote-backref">↩︎</a> <a href="#fnref6:7" class="footnote-backref">↩︎</a> <a href="#fnref6:8" class="footnote-backref">↩︎</a></p></li></ol></section></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/tuning-reference/aqe.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>Adaptive Query Execution</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="/docs/tuning-reference/anti-patterns.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>Anti-Patterns</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
26
- <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"VTcYTH4d\",\"contributor-guide_architecture_detector-contract.md\":\"CsqZHz_T\",\"contributor-guide_architecture_drill-down.md\":\"CFNCE8JB\",\"contributor-guide_architecture_impact-estimation.md\":\"HT9IJGIl\",\"contributor-guide_architecture_index.md\":\"3TO9ic6w\",\"contributor-guide_architecture_overview.md\":\"CTEqVeV4\",\"contributor-guide_architecture_state-and-history.md\":\"DOTVHjTV\",\"contributor-guide_architecture_widget-rendering.md\":\"BmpARZve\",\"contributor-guide_architecture_worker-protocol.md\":\"21pTYhFF\",\"contributor-guide_contributing.md\":\"B2E9ni79\",\"contributor-guide_development-setup.md\":\"CjgZJwjK\",\"contributor-guide_testing.md\":\"DTwhpoN2\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"D08Uaod7\",\"tuning-reference_aqe.md\":\"C1Ae6A8u\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"6wjKuJNE\",\"tuning-reference_bottleneck-cold-start.md\":\"BVQbsuHP\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"DtaYnaiF\",\"tuning-reference_bottleneck-failures.md\":\"Z1Tdmdd3\",\"tuning-reference_bottleneck-gc.md\":\"BCnXLf1n\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"CcvQj2g9\",\"tuning-reference_bottleneck-memory-utilization.md\":\"B-C0-8h-\",\"tuning-reference_bottleneck-retry-waste.md\":\"DCnhWiqj\",\"tuning-reference_bottleneck-shuffle.md\":\"B5OMJCd3\",\"tuning-reference_bottleneck-skew.md\":\"Cvo6NQm_\",\"tuning-reference_bottleneck-slow-host.md\":\"ja5qr3wq\",\"tuning-reference_bottleneck-small-files.md\":\"Crjtb0i2\",\"tuning-reference_bottleneck-spill.md\":\"Bl8KtSRg\",\"tuning-reference_bottleneck-straggler.md\":\"B0VN8reT\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"DQCftGhu\",\"tuning-reference_bottleneck-utilization.md\":\"BXDl30f9\",\"tuning-reference_caching.md\":\"CcaiMn_7\",\"tuning-reference_cluster-config.md\":\"DbWG1yZq\",\"tuning-reference_config.md\":\"YUqSpwFB\",\"tuning-reference_data-formats.md\":\"C5DX5xL4\",\"tuning-reference_index.md\":\"ZVIpQ6Dk\",\"tuning-reference_intro.md\":\"BSSaDlqx\",\"tuning-reference_joins.md\":\"BVhNCDnE\",\"tuning-reference_memory-model.md\":\"CfjFdgbl\",\"tuning-reference_metrics.md\":\"C64VNEL4\",\"tuning-reference_partitioning.md\":\"CtLK0xsS\",\"tuning-reference_pyspark.md\":\"D-92XgF8\",\"tuning-reference_shuffle.md\":\"Iq2v1oHJ\",\"tuning-reference_spark-architecture.md\":\"_qwnE_Sc\",\"tuning-reference_table-formats.md\":\"BSYEK-eN\",\"user-guide_alternative-log-retrieval.md\":\"ZHPSbBkF\",\"user-guide_getting-started.md\":\"CEa2ZX-l\",\"user-guide_mcp-tools.md\":\"dQwIvdTF\",\"user-guide_run-comparison.md\":\"ChLI3MK0\",\"user-guide_understanding-findings.md\":\"D1ccs60C\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Alternative log retrieval\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
27
+ <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"YQcj5EI4\",\"contributor-guide_architecture_board-widgets_caching-and-memory.md\":\"8bH1HnZp\",\"contributor-guide_architecture_board-widgets_plan-advisor.md\":\"RlzvUmNm\",\"contributor-guide_architecture_board-widgets_visual-system-and-templating.md\":\"DYgUpIJB\",\"contributor-guide_architecture_detector-contract.md\":\"wj6Et6UD\",\"contributor-guide_architecture_detector-contract_plan-attribution.md\":\"yNex4bys\",\"contributor-guide_architecture_detector-contract_threshold-tables.md\":\"fkPBXsU2\",\"contributor-guide_architecture_drill-down.md\":\"CqZPrdo9\",\"contributor-guide_architecture_drill-down_plan-graph.md\":\"DwOozIEs\",\"contributor-guide_architecture_impact-estimation.md\":\"CZv_ZGOp\",\"contributor-guide_architecture_impact-estimation_caveats-tuning-and-coverage.md\":\"BUucRc0A\",\"contributor-guide_architecture_impact-estimation_stage-union-rollup.md\":\"DdCGtAGu\",\"contributor-guide_architecture_index.md\":\"tymxf1Fh\",\"contributor-guide_architecture_overview.md\":\"C1sSvm9i\",\"contributor-guide_architecture_state-and-history.md\":\"DyCBdjuM\",\"contributor-guide_architecture_state-and-history_history-server-intake.md\":\"C298hrHZ\",\"contributor-guide_architecture_state-and-history_run-comparison-internals.md\":\"uuAKpY0Y\",\"contributor-guide_architecture_widget-rendering.md\":\"BOZ1azXG\",\"contributor-guide_architecture_widget-rendering_findings-tab.md\":\"BQXycK6G\",\"contributor-guide_architecture_widget-rendering_investigation-routing.md\":\"BGdiyElu\",\"contributor-guide_architecture_widget-rendering_list-sort-mode.md\":\"BuHKgH0C\",\"contributor-guide_architecture_widget-rendering_reference-section.md\":\"DWqC80bN\",\"contributor-guide_architecture_widget-rendering_row-expand-contract.md\":\"ZT89u9f1\",\"contributor-guide_architecture_worker-protocol.md\":\"BAnhi2-d\",\"contributor-guide_architecture_worker-protocol_cli-and-mcp.md\":\"Dk08VS7n\",\"contributor-guide_architecture_worker-protocol_core-and-event-validation.md\":\"DSVFPpWb\",\"contributor-guide_architecture_worker-protocol_evidence-availability.md\":\"CjimIDBY\",\"contributor-guide_architecture_worker-protocol_evidence-report.md\":\"DVGZOWvW\",\"contributor-guide_contributing.md\":\"CQMvGwFt\",\"contributor-guide_development-setup.md\":\"D9bu4j_c\",\"contributor-guide_testing.md\":\"By6XGR9a\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"D08Uaod7\",\"tuning-reference_aqe.md\":\"C1Ae6A8u\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"6wjKuJNE\",\"tuning-reference_bottleneck-cold-start.md\":\"BVQbsuHP\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"DtaYnaiF\",\"tuning-reference_bottleneck-failures.md\":\"Z1Tdmdd3\",\"tuning-reference_bottleneck-gc.md\":\"BCnXLf1n\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"CcvQj2g9\",\"tuning-reference_bottleneck-memory-utilization.md\":\"B-C0-8h-\",\"tuning-reference_bottleneck-retry-waste.md\":\"DCnhWiqj\",\"tuning-reference_bottleneck-shuffle.md\":\"B5OMJCd3\",\"tuning-reference_bottleneck-skew.md\":\"Cvo6NQm_\",\"tuning-reference_bottleneck-slow-host.md\":\"ja5qr3wq\",\"tuning-reference_bottleneck-small-files.md\":\"Crjtb0i2\",\"tuning-reference_bottleneck-spill.md\":\"Bl8KtSRg\",\"tuning-reference_bottleneck-straggler.md\":\"B0VN8reT\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"DQCftGhu\",\"tuning-reference_bottleneck-utilization.md\":\"BXDl30f9\",\"tuning-reference_caching.md\":\"CcaiMn_7\",\"tuning-reference_cluster-config.md\":\"DbWG1yZq\",\"tuning-reference_config.md\":\"YUqSpwFB\",\"tuning-reference_data-formats.md\":\"C5DX5xL4\",\"tuning-reference_index.md\":\"ZVIpQ6Dk\",\"tuning-reference_intro.md\":\"BSSaDlqx\",\"tuning-reference_joins.md\":\"BVhNCDnE\",\"tuning-reference_memory-model.md\":\"CfjFdgbl\",\"tuning-reference_metrics.md\":\"C64VNEL4\",\"tuning-reference_partitioning.md\":\"CtLK0xsS\",\"tuning-reference_pyspark.md\":\"D-92XgF8\",\"tuning-reference_shuffle.md\":\"Iq2v1oHJ\",\"tuning-reference_spark-architecture.md\":\"_qwnE_Sc\",\"tuning-reference_table-formats.md\":\"BSYEK-eN\",\"user-guide_alternative-log-retrieval.md\":\"CJpYvHGo\",\"user-guide_alternative-log-retrieval_managed-platforms.md\":\"DvDwHNtC\",\"user-guide_alternative-log-retrieval_ssh-bastion.md\":\"Gszzlplv\",\"user-guide_getting-started.md\":\"DYHHHkQR\",\"user-guide_getting-started_ci-and-automation.md\":\"DpnfKz3x\",\"user-guide_getting-started_reading-the-dashboard.md\":\"Obf3W3__\",\"user-guide_getting-started_regression-metric-keys.md\":\"B_gzr2Mw\",\"user-guide_getting-started_tuning-thresholds.md\":\"B_MyjyZm\",\"user-guide_mcp-tools.md\":\"C65SjWUL\",\"user-guide_mcp-tools_compare-runs.md\":\"CPowlb2q\",\"user-guide_mcp-tools_diagnose-run.md\":\"BP1vnHA5\",\"user-guide_mcp-tools_evaluate-budgets.md\":\"Dq5Azk8G\",\"user-guide_mcp-tools_get-finding-documentation.md\":\"BtTTHNV6\",\"user-guide_mcp-tools_get-finding-evidence.md\":\"rKyu4wts\",\"user-guide_mcp-tools_get-reference-doc.md\":\"Yac4tH3u\",\"user-guide_mcp-tools_get-run-summary.md\":\"DpxZ4ZNp\",\"user-guide_mcp-tools_list-runs.md\":\"DpQPeCM3\",\"user-guide_run-comparison.md\":\"B6aNtDYF\",\"user-guide_run-comparison_how-stages-are-matched.md\":\"XBzQ1oEU\",\"user-guide_understanding-findings.md\":\"Dqt2H_C1\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\",\"collapsed\":true,\"items\":[{\"text\":\"Reading the dashboard\",\"link\":\"/user-guide/getting-started/reading-the-dashboard\"},{\"text\":\"CI and automation\",\"link\":\"/user-guide/getting-started/ci-and-automation\"},{\"text\":\"Tuning detector thresholds\",\"link\":\"/user-guide/getting-started/tuning-thresholds\"},{\"text\":\"Regression metric keys\",\"link\":\"/user-guide/getting-started/regression-metric-keys\"}]},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\",\"collapsed\":true,\"items\":[{\"text\":\"How stages are matched\",\"link\":\"/user-guide/run-comparison/how-stages-are-matched\"}]},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\",\"collapsed\":true,\"items\":[{\"text\":\"diagnose_run\",\"link\":\"/user-guide/mcp-tools/diagnose-run\"},{\"text\":\"get_run_summary\",\"link\":\"/user-guide/mcp-tools/get-run-summary\"},{\"text\":\"evaluate_budgets\",\"link\":\"/user-guide/mcp-tools/evaluate-budgets\"},{\"text\":\"compare_runs\",\"link\":\"/user-guide/mcp-tools/compare-runs\"},{\"text\":\"get_finding_evidence\",\"link\":\"/user-guide/mcp-tools/get-finding-evidence\"},{\"text\":\"get_finding_documentation\",\"link\":\"/user-guide/mcp-tools/get-finding-documentation\"},{\"text\":\"get_reference_doc\",\"link\":\"/user-guide/mcp-tools/get-reference-doc\"},{\"text\":\"list_runs\",\"link\":\"/user-guide/mcp-tools/list-runs\"}]},{\"text\":\"Finding your event log\",\"link\":\"/user-guide/alternative-log-retrieval\",\"collapsed\":true,\"items\":[{\"text\":\"Managed platforms\",\"link\":\"/user-guide/alternative-log-retrieval/managed-platforms\"},{\"text\":\"Behind an SSH bastion\",\"link\":\"/user-guide/alternative-log-retrieval/ssh-bastion\"}]}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\",\"collapsed\":true,\"items\":[{\"text\":\"Evidence availability\",\"link\":\"/contributor-guide/architecture/worker-protocol/evidence-availability\"},{\"text\":\"Evidence report and finding identity\",\"link\":\"/contributor-guide/architecture/worker-protocol/evidence-report\"},{\"text\":\"CLI and MCP server\",\"link\":\"/contributor-guide/architecture/worker-protocol/cli-and-mcp\"},{\"text\":\"Core and event validation\",\"link\":\"/contributor-guide/architecture/worker-protocol/core-and-event-validation\"}]},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\",\"collapsed\":true,\"items\":[{\"text\":\"Run comparison internals\",\"link\":\"/contributor-guide/architecture/state-and-history/run-comparison-internals\"},{\"text\":\"History Server intake\",\"link\":\"/contributor-guide/architecture/state-and-history/history-server-intake\"}]},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\",\"collapsed\":true,\"items\":[{\"text\":\"Plan attribution\",\"link\":\"/contributor-guide/architecture/detector-contract/plan-attribution\"},{\"text\":\"Threshold tables\",\"link\":\"/contributor-guide/architecture/detector-contract/threshold-tables\"}]},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\",\"collapsed\":true,\"items\":[{\"text\":\"Stage union rollup\",\"link\":\"/contributor-guide/architecture/impact-estimation/stage-union-rollup\"},{\"text\":\"Caveats, tuning and coverage\",\"link\":\"/contributor-guide/architecture/impact-estimation/caveats-tuning-and-coverage\"}]},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\",\"collapsed\":true,\"items\":[{\"text\":\"Findings tab\",\"link\":\"/contributor-guide/architecture/widget-rendering/findings-tab\"},{\"text\":\"List sort mode\",\"link\":\"/contributor-guide/architecture/widget-rendering/list-sort-mode\"},{\"text\":\"Row/expand contract\",\"link\":\"/contributor-guide/architecture/widget-rendering/row-expand-contract\"},{\"text\":\"ReferenceSection\",\"link\":\"/contributor-guide/architecture/widget-rendering/reference-section\"},{\"text\":\"Investigation routing\",\"link\":\"/contributor-guide/architecture/widget-rendering/investigation-routing\"}]},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\",\"collapsed\":true,\"items\":[{\"text\":\"Plan Advisor\",\"link\":\"/contributor-guide/architecture/board-widgets/plan-advisor\"},{\"text\":\"Caching and memory widgets\",\"link\":\"/contributor-guide/architecture/board-widgets/caching-and-memory\"},{\"text\":\"Visual system and templating\",\"link\":\"/contributor-guide/architecture/board-widgets/visual-system-and-templating\"}]},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\",\"collapsed\":true,\"items\":[{\"text\":\"Plan graph\",\"link\":\"/contributor-guide/architecture/drill-down/plan-graph\"}]}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":{\"selector\":\"[data-sf-anchor-offset]\",\"padding\":0},\"cleanUrls\":false}");</script>
27
28
 
28
29
  </body>
29
30
  </html>
package/public/docs/tuning-reference/config.html CHANGED
@@ -6,14 +6,15 @@
6
6
  <title>Spark Config Quick-Reference | Spark Tuning Reference</title>
7
7
  <meta name="description" content="Cross-reference of the Spark/PySpark config properties and JVM GC flags discussed elsewhere in the guide, with defaults and version notes.">
8
8
  <meta name="generator" content="VitePress v1.6.4">
9
- <link rel="preload stylesheet" href="/docs/assets/style.DSixAiZE.css" as="style">
9
+ <link rel="preload stylesheet" href="/docs/assets/style.tmDhD3c8.css" as="style">
10
10
  <link rel="preload stylesheet" href="/docs/vp-icons.css" as="style">
11
11
 
12
- <script type="module" src="/docs/assets/app.C6w47a9W.js"></script>
12
+ <script type="module" src="/docs/assets/app.D2GqAkhy.js"></script>
13
13
  <link rel="preload" href="/docs/assets/inter-roman-latin.Di8DUHzh.woff2" as="font" type="font/woff2" crossorigin="">
14
- <link rel="modulepreload" href="/docs/assets/chunks/theme.D-mE6j02.js">
14
+ <link rel="modulepreload" href="/docs/assets/chunks/theme.BuxVpb0H.js">
15
15
  <link rel="modulepreload" href="/docs/assets/chunks/framework.DSg0KOwT.js">
16
16
  <link rel="modulepreload" href="/docs/assets/tuning-reference_config.md.YUqSpwFB.lean.js">
17
+ <script>(()=>{const c=`${`[${["data","shuffle","product","bar"].join("-")}]`}, .VPNav, .VPLocalNav`,a=24,d=document.documentElement,l=t=>{const e=getComputedStyle(t);return e.display==="none"||t.getClientRects().length===0?0:e.position==="fixed"?t.getBoundingClientRect().bottom:e.position==="sticky"?(parseFloat(e.top)||0)+t.offsetHeight:0},u=()=>{try{return location.hash?document.getElementById(decodeURIComponent(location.hash.slice(1))):null}catch{return null}};let r=null;const s=()=>{const t=Array.from(document.querySelectorAll(c),l),e=Math.ceil(Math.max(0,...t))+a;if(e===r)return;const n=r;r=e,d.style.setProperty("--sf-anchor-offset",`${r}px`);const o=n===null?null:u();if(!o)return;const f=(parseFloat(getComputedStyle(o).paddingTop)||0)+2;Math.abs(o.getBoundingClientRect().top-n)<=f&&o.scrollIntoView()},i=()=>{const t=document.createElement("div");t.setAttribute("data-sf-anchor-offset",""),t.setAttribute("aria-hidden","true"),document.body.appendChild(t);const e=window.ResizeObserver?new ResizeObserver(s):null,n=()=>{e&&document.querySelectorAll(c).forEach(o=>e.observe(o)),s()};n(),new MutationObserver(n).observe(document.body,{childList:!0,subtree:!0}),window.addEventListener("resize",s),window.addEventListener("load",s)};document.readyState==="loading"?document.addEventListener("DOMContentLoaded",i,{once:!0}):i()})();</script>
17
18
  <link rel="icon" type="image/svg+xml" href="/docs/favicon.svg">
18
19
  <link rel="preconnect" href="https://fonts.googleapis.com">
19
20
  <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin="">
@@ -23,7 +24,7 @@
23
24
  </head>
24
25
  <body>
25
26
  <div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Getting Started</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/intro.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Introduction</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Optimization Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/spark-architecture.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Execution Model</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/memory-model.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Management</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/partitioning.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Partitioning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/joins.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Join Optimization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/data-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Data Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/table-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Table Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/caching.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Caching & Persistence</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/pyspark.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>PySpark Specifics</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/aqe.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Adaptive Query Execution</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/cluster-config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cluster Tuning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/anti-patterns.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Anti-Patterns</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Detector Catalog</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-skew.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Skew</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle I/O</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-spill.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory / Disk Spill</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-gc.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>GC Pressure</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-cold-start.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cold Start</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Executor Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-slow-host.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Slow Host</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-failures.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Failures</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-straggler.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Stragglers</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-retry-waste.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Retry Waste</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-tiny-tasks.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Tiny Tasks</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-job-failure-rate.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Job Failure Rate</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-memory-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Duplicate Plan Subtree</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-small-files.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Small Files</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-broadcast-sizing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Broadcast Sizing</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Reference</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/metrics.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Metrics Glossary</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Config Quick-Reference</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_tuning-reference_config" data-v-39a288b8><div><h1 id="config" tabindex="-1">Spark Config Quick-Reference <a class="header-anchor" href="#config" aria-label="Permalink to &quot;Spark Config Quick-Reference {#config}&quot;">​</a></h1><p>A cross-reference of the Spark and PySpark configuration properties discussed elsewhere in this guide, plus the JVM garbage-collection flags relevant to executor tuning. Defaults and version notes below are traced to Spark&#39;s own configuration docs, its SQLConf source, and the cited literature. Where a source doesn&#39;t give a hard number (a range, a ceiling, a recommended workload profile), the table leaves it out rather than guess.</p><h2 id="spark-properties" tabindex="-1">Spark properties <a class="header-anchor" href="#spark-properties" aria-label="Permalink to &quot;Spark properties&quot;">​</a></h2><table tabindex="0"><thead><tr><th>Key</th><th>Default</th><th>Recommended</th><th>Notes</th></tr></thead><tbody><tr><td><code>spark.sql.shuffle.partitions</code></td><td>200 (unchanged since Spark 1.1.0)<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup><sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup><sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup></td><td></td><td>AQE doesn&#39;t override this config value; see notes below for the effective runtime count.</td></tr><tr><td><code>spark.sql.adaptive.enabled</code></td><td><code>true</code><sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup></td><td><code>true</code></td><td></td></tr><tr><td><code>spark.sql.adaptive.coalescePartitions.enabled</code></td><td><code>true</code> (since 3.0.0)<sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup></td><td><code>true</code></td><td>Requires AQE; merges contiguous small post-shuffle partitions instead of one task per configured partition<sup class="footnote-ref"><a href="#fn1" id="fnref1:2">[1:2]</a></sup>.</td></tr><tr><td><code>spark.sql.adaptive.coalescePartitions.initialPartitionNum</code></td><td>unset → falls back to <code>spark.sql.shuffle.partitions</code> (200)<sup class="footnote-ref"><a href="#fn1" id="fnref1:3">[1:3]</a></sup></td><td></td><td>Sets the partition count entering the coalescing step.</td></tr><tr><td><code>spark.sql.adaptive.coalescePartitions.parallelismFirst</code></td><td><code>true</code> (since 3.2.0)<sup class="footnote-ref"><a href="#fn1" id="fnref1:4">[1:4]</a></sup><sup class="footnote-ref"><a href="#fn5" id="fnref5">[5]</a></sup></td><td><code>false</code> on busy clusters<sup class="footnote-ref"><a href="#fn1" id="fnref1:5">[1:5]</a></sup></td><td>When <code>true</code>, ignores <code>advisoryPartitionSizeInBytes</code> and derives a target from the cluster&#39;s default parallelism, respecting only the <code>minPartitionSize</code> floor.</td></tr><tr><td><code>spark.sql.adaptive.advisoryPartitionSizeInBytes</code></td><td>64MB (since 3.0.0)<sup class="footnote-ref"><a href="#fn1" id="fnref1:6">[1:6]</a></sup><sup class="footnote-ref"><a href="#fn4" id="fnref4:1">[4:1]</a></sup><sup class="footnote-ref"><a href="#fn5" id="fnref5:1">[5:1]</a></sup></td><td></td><td>Ignored during coalescing when <code>parallelismFirst=true</code> (the default).</td></tr><tr><td><code>spark.sql.adaptive.coalescePartitions.minPartitionSize</code></td><td>1MB (since 3.2.0)<sup class="footnote-ref"><a href="#fn5" id="fnref5:2">[5:2]</a></sup></td><td></td><td>Floor enforced when the target size is ignored: the default <code>parallelismFirst=true</code> case<sup class="footnote-ref"><a href="#fn1" id="fnref1:7">[1:7]</a></sup>.</td></tr><tr><td><code>spark.memory.fraction</code></td><td>0.6<sup class="footnote-ref"><a href="#fn6" id="fnref6">[6]</a></sup></td><td></td><td>See notes below; no hard 0–1 ceiling documented in the cited sources.</td></tr><tr><td><code>spark.memory.storageFraction</code></td><td>0.5<sup class="footnote-ref"><a href="#fn7" id="fnref7">[7]</a></sup></td><td></td><td>A fraction <em>of</em> the region sized by <code>spark.memory.fraction</code>, not an independent pool; see notes below.</td></tr><tr><td><code>spark.sql.execution.arrow.pyspark.enabled</code></td><td><code>false</code><sup class="footnote-ref"><a href="#fn8" id="fnref8">[8]</a></sup></td><td></td><td>See PySpark callout below.</td></tr><tr><td><code>spark.sql.execution.arrow.pyspark.fallback.enabled</code></td><td><code>true</code> (inherited from the deprecated <code>arrow.fallback.enabled</code>)<sup class="footnote-ref"><a href="#fn8" id="fnref8:1">[8:1]</a></sup><sup class="footnote-ref"><a href="#fn5" id="fnref5:3">[5:3]</a></sup><sup class="footnote-ref"><a href="#fn4" id="fnref4:2">[4:2]</a></sup></td><td></td><td>Falls back to the non-Arrow path automatically if a conversion error occurs, before computation runs<sup class="footnote-ref"><a href="#fn8" id="fnref8:2">[8:2]</a></sup>.</td></tr><tr><td><code>spark.dynamicAllocation.shuffleTracking.enabled</code></td><td><code>true</code> (since 3.0.0)<sup class="footnote-ref"><a href="#fn4" id="fnref4:3">[4:3]</a></sup></td><td></td><td>Lets dynamic allocation track shuffle files per executor without an external shuffle service.</td></tr><tr><td><code>spark.dynamicAllocation.shuffleTracking.timeout</code></td><td><code>infinity</code><sup class="footnote-ref"><a href="#fn4" id="fnref4:4">[4:4]</a></sup></td><td></td><td>Executors holding shuffle data wait for it to be garbage collected before release, by default; set a finite value if GC isn&#39;t keeping up.</td></tr><tr><td><code>spark.shuffle.service.enabled</code></td><td></td><td></td><td>External shuffle service; see <code>#config-shuffle-service</code> below.</td></tr><tr><td><code>spark.dynamicAllocation.minExecutors</code> / <code>maxExecutors</code></td><td><code>0</code> / <code>infinity</code><sup class="footnote-ref"><a href="#fn4" id="fnref4:5">[4:5]</a></sup></td><td></td><td>Autoscale bounds; see <code>#config-autoscale-bounds</code> below.</td></tr><tr><td><code>spark.serializer</code></td><td><code>org.apache.spark.serializer.JavaSerializer</code><sup class="footnote-ref"><a href="#fn4" id="fnref4:6">[4:6]</a></sup></td><td><code>KryoSerializer</code></td><td>See <code>#config-serializer</code> below.</td></tr><tr><td><code>spark.executor.memoryOverhead</code></td><td>greater of 10% of executor memory or 384MB<sup class="footnote-ref"><a href="#fn7" id="fnref7:1">[7:1]</a></sup></td><td></td><td>Off-heap overhead; see <code>#config-memory-overhead</code> below.</td></tr></tbody></table><h3 id="shuffle-partitions-and-aqe-coalescing" tabindex="-1">Shuffle partitions and AQE coalescing <a class="header-anchor" href="#shuffle-partitions-and-aqe-coalescing" aria-label="Permalink to &quot;Shuffle partitions and AQE coalescing&quot;">​</a></h3><p>The 200 default for <code>spark.sql.shuffle.partitions</code> hasn&#39;t moved since 1.1.0, and AQE (on by default since Spark 3.0) doesn&#39;t change that config value<sup class="footnote-ref"><a href="#fn1" id="fnref1:8">[1:8]</a></sup><sup class="footnote-ref"><a href="#fn2" id="fnref2:1">[2:1]</a></sup><sup class="footnote-ref"><a href="#fn3" id="fnref3:1">[3:1]</a></sup>. What it changes is the effective number of partitions used at runtime. With <code>coalescePartitions.enabled</code> also on by default, Spark merges small contiguous post-shuffle partitions instead of running one task per configured partition<sup class="footnote-ref"><a href="#fn1" id="fnref1:9">[1:9]</a></sup>. Because <code>parallelismFirst</code> defaults to <code>true</code> since 3.2.0, that merge target usually isn&#39;t the 64MB <code>advisoryPartitionSizeInBytes</code> value; it&#39;s derived from the cluster&#39;s default parallelism, with <code>minPartitionSize</code> (1MB) as the only enforced floor<sup class="footnote-ref"><a href="#fn1" id="fnref1:10">[1:10]</a></sup><sup class="footnote-ref"><a href="#fn5" id="fnref5:4">[5:4]</a></sup>. Databricks&#39; own writeup on AQE shows the reduce-task count actually shrinking based on measured data volume<sup class="footnote-ref"><a href="#fn9" id="fnref9">[9]</a></sup>, and Learning Spark accordingly calls the static 200 default &quot;too high for smaller or streaming workloads&quot;<sup class="footnote-ref"><a href="#fn3" id="fnref3:2">[3:2]</a></sup>. A related, distinct knob (<code>spark.sql.adaptive.coalescePartitions.minPartitionNum</code>) sets a minimum parallelism floor for data that&#39;s slow to compute despite being small, per High Performance Spark<sup class="footnote-ref"><a href="#fn10" id="fnref10">[10]</a></sup>.</p><h3 id="spark-memory-fraction-and-spark-memory-storagefraction" tabindex="-1"><code>spark.memory.fraction</code> and <code>spark.memory.storageFraction</code> <a class="header-anchor" href="#spark-memory-fraction-and-spark-memory-storagefraction" aria-label="Permalink to &quot;`spark.memory.fraction` and `spark.memory.storageFraction`&quot;">​</a></h3><p><code>spark.memory.fraction</code> sizes the unified execution/storage region (M) as a fraction of (JVM heap − 300MiB); the tuning guide frames it as a knob to fit M &quot;comfortably within the JVM&#39;s old or tenured generation&quot; rather than stating an enforced numeric range<sup class="footnote-ref"><a href="#fn6" id="fnref6:1">[6:1]</a></sup>. The cited sources document no failure threshold tied to a specific value like 0.9. What they do document is the risk of pushing the fraction high: the complement, <code>1 − spark.memory.fraction</code>, is untracked &quot;User Memory&quot; for UDFs, Python/Arrow glue, and native buffers, and starving it (which is what raising the fraction toward 0.9 does) leads to &quot;GC pressure or random OOMs,&quot; with no warning from Spark<sup class="footnote-ref"><a href="#fn7" id="fnref7:2">[7:2]</a></sup>.</p><p><code>spark.memory.storageFraction</code> isn&#39;t independent of that: it&#39;s a fraction <em>of</em> M, the same region <code>spark.memory.fraction</code> sizes<sup class="footnote-ref"><a href="#fn6" id="fnref6:2">[6:2]</a></sup>. Inside that shared pool, execution can evict storage down to the threshold set by <code>storageFraction</code>, but storage can never evict execution<sup class="footnote-ref"><a href="#fn11" id="fnref11">[11]</a></sup><sup class="footnote-ref"><a href="#fn6" id="fnref6:3">[6:3]</a></sup>. With the defaults (0.6 and 0.5), that works out to execution and storage each getting 30% of usable heap; raising <code>memory.fraction</code> scales both pools at once, while <code>storageFraction</code> only re-splits the pool that&#39;s already been carved out<sup class="footnote-ref"><a href="#fn7" id="fnref7:3">[7:3]</a></sup>.</p><blockquote><p><strong>PySpark:</strong> <code>spark.sql.execution.arrow.pyspark.enabled</code> is off by default and governs Arrow use for <code>DataFrame.toPandas()</code> and <code>SparkSession.createDataFrame()</code> from a Pandas DataFrame or NumPy array<sup class="footnote-ref"><a href="#fn8" id="fnref8:3">[8:3]</a></sup>. Its documented risk is a type-coverage gap, not silent data corruption: <code>ArrayType</code> of <code>TimestampType</code> is explicitly unsupported<sup class="footnote-ref"><a href="#fn4" id="fnref4:7">[4:7]</a></sup><sup class="footnote-ref"><a href="#fn5" id="fnref5:5">[5:5]</a></sup>, and more generally an unsupported column type raises an error rather than converting wrongly<sup class="footnote-ref"><a href="#fn8" id="fnref8:4">[8:4]</a></sup>. <code>spark.sql.execution.arrow.pyspark.fallback.enabled</code> defaults to <code>true</code> and automatically falls back to the non-Arrow path if a conversion error occurs, before any computation runs<sup class="footnote-ref"><a href="#fn8" id="fnref8:5">[8:5]</a></sup><sup class="footnote-ref"><a href="#fn5" id="fnref5:6">[5:6]</a></sup><sup class="footnote-ref"><a href="#fn4" id="fnref4:8">[4:8]</a></sup>. So the failure mode is an exception plus fallback, not a wrong result. Separately, and not specific to this Arrow config, PySpark&#39;s own type coercion has its own hazards: numeric values passed for <code>ByteType</code>/<code>ShortType</code>/<code>IntegerType</code> must fall within fixed ranges or get rejected or converted unexpectedly<sup class="footnote-ref"><a href="#fn2" id="fnref2:2">[2:2]</a></sup>.</p></blockquote><h3 id="config-shuffle-service" tabindex="-1">Shuffle service <a class="header-anchor" href="#config-shuffle-service" aria-label="Permalink to &quot;Shuffle service {#config-shuffle-service}&quot;">​</a></h3><p>During a shuffle, an executor writes its map output to local disk and then serves fetch requests for that data itself<sup class="footnote-ref"><a href="#fn12" id="fnref12">[12]</a></sup>. Dynamic allocation can reclaim an idle executor before a later stage has fetched all the shuffle blocks it holds (especially with stragglers, tasks that run much longer than their peers), and once that executor is gone, any stage that still needs its shuffle output gets a <code>FetchFailed</code> and has to recompute it<sup class="footnote-ref"><a href="#fn12" id="fnref12:1">[12:1]</a></sup>. The external shuffle service fixes this by moving shuffle-block serving out of the executor process: it&#39;s a long-running process on each node, independent of any particular application&#39;s executors, and once <code>spark.shuffle.service.enabled</code> is <code>true</code>, executors fetch shuffle blocks from it instead of from each other, so an executor&#39;s shuffle output keeps being served after that executor is reclaimed<sup class="footnote-ref"><a href="#fn12" id="fnref12:2">[12:2]</a></sup><sup class="footnote-ref"><a href="#fn4" id="fnref4:9">[4:9]</a></sup>. Setting up dynamic allocation requires enabling this service on every worker node in addition to <code>spark.dynamicAllocation.enabled</code><sup class="footnote-ref"><a href="#fn2" id="fnref2:3">[2:3]</a></sup>. As of Spark 3.0, shuffle-file tracking (<code>spark.dynamicAllocation.shuffleTracking.enabled</code>) is an alternative to the external shuffle service for the same safe-removal problem<sup class="footnote-ref"><a href="#fn4" id="fnref4:10">[4:10]</a></sup>.</p><p><strong>Limitations / false-positive risk:</strong> the service can be intentionally left off when dynamic allocation relies on shuffle-file tracking instead, so a disabled setting is not automatically a misconfiguration.</p><h3 id="config-autoscale-bounds" tabindex="-1">Autoscale bounds <a class="header-anchor" href="#config-autoscale-bounds" aria-label="Permalink to &quot;Autoscale bounds {#config-autoscale-bounds}&quot;">​</a></h3><p>By default, <code>spark.dynamicAllocation.minExecutors</code> is <code>0</code> and <code>spark.dynamicAllocation.maxExecutors</code> is <code>infinity</code><sup class="footnote-ref"><a href="#fn4" id="fnref4:11">[4:11]</a></sup>. Leaving <code>maxExecutors</code> unset therefore leaves dynamic allocation unbounded on the high end as far as Spark itself is concerned; in practice the ceiling ends up being whatever the cluster or scheduler enforces outside Spark<sup class="footnote-ref"><a href="#fn4" id="fnref4:12">[4:12]</a></sup>. <code>spark.dynamicAllocation.initialExecutors</code> defaults to <code>minExecutors</code>, unless <code>--num-executors</code> (or <code>spark.executor.instances</code>) sets a larger starting value<sup class="footnote-ref"><a href="#fn4" id="fnref4:13">[4:13]</a></sup>. Whatever executor count <code>executorAllocationRatio</code> computes to maximize parallelism, that target is still clamped by the <code>minExecutors</code>/<code>maxExecutors</code> floor and ceiling<sup class="footnote-ref"><a href="#fn4" id="fnref4:14">[4:14]</a></sup>.</p><p><strong>Limitations / false-positive risk:</strong> an unbounded <code>maxExecutors</code> is often deliberate on clusters where the scheduler enforces the real ceiling, so an unset high end is not always wrong.</p><h3 id="config-serializer" tabindex="-1">Serializer <a class="header-anchor" href="#config-serializer" aria-label="Permalink to &quot;Serializer {#config-serializer}&quot;">​</a></h3><p>The default <code>spark.serializer</code> is <code>org.apache.spark.serializer.JavaSerializer</code>, which works with any <code>Serializable</code> Java object but is &quot;quite slow&quot;; Spark&#39;s configuration reference recommends switching to <code>KryoSerializer</code> &quot;when speed is necessary&quot;<sup class="footnote-ref"><a href="#fn4" id="fnref4:15">[4:15]</a></sup>. The tuning guide is more specific: Kryo is &quot;significantly faster and more compact than Java serialization (often as much as 10x),&quot; though it doesn&#39;t support all <code>Serializable</code> types and needs its classes registered in advance for best performance<sup class="footnote-ref"><a href="#fn6" id="fnref6:4">[6:4]</a></sup>. That registration requirement is the stated reason Kryo isn&#39;t the default: &quot;The only reason Kryo is not the default is because of the custom registration requirement,&quot; and Spark recommends trying it for any network-intensive application<sup class="footnote-ref"><a href="#fn6" id="fnref6:5">[6:5]</a></sup>. Since Spark 2.0.0, Spark internally uses Kryo regardless of this setting when shuffling RDDs of simple types, arrays of simple types, or strings, and auto-registers common Scala classes via Twitter chill&#39;s <code>AllScalaRegistrar</code><sup class="footnote-ref"><a href="#fn6" id="fnref6:6">[6:6]</a></sup>. A practitioner summary puts Java serialization at &quot;2-10x slower and larger on the wire&quot; on every shuffle, with the caveat that classes left unregistered silently fall back to Java serialization unless registered via <code>spark.kryo.classesToRegister</code> or <code>spark.kryo.registrator</code><sup class="footnote-ref"><a href="#fn13" id="fnref13">[13]</a></sup>. Persisting RDDs in serialized form is another case where Kryo can be more space-efficient than Java serialization<sup class="footnote-ref"><a href="#fn10" id="fnref10:1">[10:1]</a></sup>.</p><p><strong>Limitations / false-positive risk:</strong> <code>JavaSerializer</code> may be kept on purpose when an application depends on <code>Serializable</code> types Kryo cannot handle, so a non-Kryo setting is not necessarily a mistake.</p><h3 id="config-memory-overhead" tabindex="-1">Memory overhead <a class="header-anchor" href="#config-memory-overhead" aria-label="Permalink to &quot;Memory overhead {#config-memory-overhead}&quot;">​</a></h3><p><code>spark.executor.memoryOverhead</code> covers everything an executor needs outside the JVM heap sized by <code>spark.executor.memory</code>: thread stacks, JIT buffers, metaspace, JNI, native libraries, and, if <code>spark.executor.pyspark.memory</code> isn&#39;t set separately, the memory used by PySpark&#39;s per-task Python worker processes<sup class="footnote-ref"><a href="#fn7" id="fnref7:4">[7:4]</a></sup>. Left unset, Spark defaults it to the greater of 10% of executor memory or 384MB<sup class="footnote-ref"><a href="#fn7" id="fnref7:5">[7:5]</a></sup>; the configuration reference formalizes that floor as <code>spark.executor.minMemoryOverhead</code> (default <code>384m</code>, since Spark 4.0) and the percentage as <code>spark.executor.memoryOverheadFactor</code> (default 0.10, or 0.40 for non-JVM jobs on Kubernetes, since those need more non-JVM heap space)<sup class="footnote-ref"><a href="#fn4" id="fnref4:16">[4:16]</a></sup>. The resource manager (YARN&#39;s NodeManager, or the Kubernetes kubelet) sizes the container/pod to the sum of <code>spark.executor.memoryOverhead</code>, <code>spark.executor.memory</code>, <code>spark.memory.offHeap.size</code>, and <code>spark.executor.pyspark.memory</code><sup class="footnote-ref"><a href="#fn4" id="fnref4:17">[4:17]</a></sup>. If off-heap memory or PySpark worker processes consume memory that isn&#39;t reflected in a correspondingly larger <code>memoryOverhead</code>, that extra usage still shows up in the process&#39;s actual RSS, which the resource manager does track. Once real usage exceeds the container&#39;s allocated size, the executor is killed by YARN or OOMKilled by the kubelet, with no Spark-level error, just an abrupt process death<sup class="footnote-ref"><a href="#fn7" id="fnref7:6">[7:6]</a></sup>.</p><p><strong>Limitations / false-positive risk:</strong> the default overhead is adequate for many JVM-only jobs, so a value left at the default only signals trouble on off-heap-heavy or PySpark workloads that need more non-heap room.</p><h2 id="jvm-gc-flags" tabindex="-1">JVM GC flags <a class="header-anchor" href="#jvm-gc-flags" aria-label="Permalink to &quot;JVM GC flags&quot;">​</a></h2><table tabindex="0"><thead><tr><th>Flag</th><th>Effect</th></tr></thead><tbody><tr><td><code>-XX:+UseG1GC</code></td><td>Default collector since Spark 4.0.0, which defaults to JDK 17<sup class="footnote-ref"><a href="#fn6" id="fnref6:7">[6:7]</a></sup>.</td></tr><tr><td><code>-XX:G1HeapRegionSize</code></td><td>May need raising alongside large executor heaps<sup class="footnote-ref"><a href="#fn6" id="fnref6:8">[6:8]</a></sup>.</td></tr><tr><td><code>-XX:InitiatingHeapOccupancyPercent</code></td><td>Tuned (with <code>-XX:ConcGCThreads</code> and RSet-update settings) to fix a documented ~100-second G1 full-GC pause on an 88GB executor heap<sup class="footnote-ref"><a href="#fn14" id="fnref14">[14]</a></sup>.</td></tr><tr><td><code>-XX:ConcGCThreads</code></td><td>See <code>-XX:InitiatingHeapOccupancyPercent</code> above<sup class="footnote-ref"><a href="#fn14" id="fnref14:1">[14:1]</a></sup>.</td></tr><tr><td><code>-XX:+UseZGC</code></td><td>Concurrent, low-latency collector; sub-millisecond pause target independent of heap size, from a few hundred megabytes up to 16TB<sup class="footnote-ref"><a href="#fn15" id="fnref15">[15]</a></sup>.</td></tr></tbody></table><h3 id="g1-vs-zgc" tabindex="-1">G1 vs. ZGC <a class="header-anchor" href="#g1-vs-zgc" aria-label="Permalink to &quot;G1 vs. ZGC&quot;">​</a></h3><p>G1 was designed as a CMS replacement aiming at both throughput and low latency: it partitions the heap into equal-sized regions and copies out only the live objects from collected regions rather than compacting the whole heap<sup class="footnote-ref"><a href="#fn14" id="fnref14:2">[14:2]</a></sup>. Even so, a documented 88GB-heap Spark benchmark hit &quot;unacceptable full GC,&quot; with one job pausing nearly 100 seconds under default G1 settings. Only after tuning <code>InitiatingHeapOccupancyPercent</code>, <code>ConcGCThreads</code>, and RSet-update parameters did G1 beat Parallel/CMS GC on both throughput and latency<sup class="footnote-ref"><a href="#fn14" id="fnref14:3">[14:3]</a></sup>. ZGC, per the OpenJDK project page, &quot;performs all expensive work concurrently, without stopping the execution of application threads for more than a millisecond,&quot; with pause times that stay flat as heap size scales from a few hundred megabytes up to 16TB<sup class="footnote-ref"><a href="#fn15" id="fnref15:1">[15:1]</a></sup>.</p><h2 id="sources" tabindex="-1">Sources <a class="header-anchor" href="#sources" aria-label="Permalink to &quot;Sources&quot;">​</a></h2><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/sql-performance-tuning.html" target="_blank" rel="noreferrer">Performance Tuning — Spark SQL, DataFrames and Datasets Guide</a> <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a> <a href="#fnref1:2" class="footnote-backref">↩︎</a> <a href="#fnref1:3" class="footnote-backref">↩︎</a> <a href="#fnref1:4" class="footnote-backref">↩︎</a> <a href="#fnref1:5" class="footnote-backref">↩︎</a> <a href="#fnref1:6" class="footnote-backref">↩︎</a> <a href="#fnref1:7" class="footnote-backref">↩︎</a> <a href="#fnref1:8" class="footnote-backref">↩︎</a> <a href="#fnref1:9" class="footnote-backref">↩︎</a> <a href="#fnref1:10" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><em>Spark: The Definitive Guide</em>, Chambers &amp; Zaharia, ch. 4, ch. 10 <a href="#fnref2" class="footnote-backref">↩︎</a> <a href="#fnref2:1" class="footnote-backref">↩︎</a> <a href="#fnref2:2" class="footnote-backref">↩︎</a> <a href="#fnref2:3" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><em>Learning Spark, 2nd Edition</em>, Damji, Wenig, Das &amp; Lee, ch. 7 <a href="#fnref3" class="footnote-backref">↩︎</a> <a href="#fnref3:1" class="footnote-backref">↩︎</a> <a href="#fnref3:2" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration — Spark</a> <a href="#fnref4" class="footnote-backref">↩︎</a> <a href="#fnref4:1" class="footnote-backref">↩︎</a> <a href="#fnref4:2" class="footnote-backref">↩︎</a> <a href="#fnref4:3" class="footnote-backref">↩︎</a> <a href="#fnref4:4" class="footnote-backref">↩︎</a> <a href="#fnref4:5" class="footnote-backref">↩︎</a> <a href="#fnref4:6" class="footnote-backref">↩︎</a> <a href="#fnref4:7" class="footnote-backref">↩︎</a> <a href="#fnref4:8" class="footnote-backref">↩︎</a> <a href="#fnref4:9" class="footnote-backref">↩︎</a> <a href="#fnref4:10" class="footnote-backref">↩︎</a> <a href="#fnref4:11" class="footnote-backref">↩︎</a> <a href="#fnref4:12" class="footnote-backref">↩︎</a> <a href="#fnref4:13" class="footnote-backref">↩︎</a> <a href="#fnref4:14" class="footnote-backref">↩︎</a> <a href="#fnref4:15" class="footnote-backref">↩︎</a> <a href="#fnref4:16" class="footnote-backref">↩︎</a> <a href="#fnref4:17" class="footnote-backref">↩︎</a></p></li><li id="fn5" class="footnote-item"><p><a href="https://raw.githubusercontent.com/apache/spark/v3.5.0/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala" target="_blank" rel="noreferrer">SQLConf.scala (Spark 3.5.0)</a> <a href="#fnref5" class="footnote-backref">↩︎</a> <a href="#fnref5:1" class="footnote-backref">↩︎</a> <a href="#fnref5:2" class="footnote-backref">↩︎</a> <a href="#fnref5:3" class="footnote-backref">↩︎</a> <a href="#fnref5:4" class="footnote-backref">↩︎</a> <a href="#fnref5:5" class="footnote-backref">↩︎</a> <a href="#fnref5:6" class="footnote-backref">↩︎</a></p></li><li id="fn6" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/tuning.html" target="_blank" rel="noreferrer">Tuning Spark</a> <a href="#fnref6" class="footnote-backref">↩︎</a> <a href="#fnref6:1" class="footnote-backref">↩︎</a> <a href="#fnref6:2" class="footnote-backref">↩︎</a> <a href="#fnref6:3" class="footnote-backref">↩︎</a> <a href="#fnref6:4" class="footnote-backref">↩︎</a> <a href="#fnref6:5" class="footnote-backref">↩︎</a> <a href="#fnref6:6" class="footnote-backref">↩︎</a> <a href="#fnref6:7" class="footnote-backref">↩︎</a> <a href="#fnref6:8" class="footnote-backref">↩︎</a></p></li><li id="fn7" class="footnote-item"><p><a href="https://luminousmen.com/post/dive-into-spark-memory" target="_blank" rel="noreferrer">Dive Into Spark Memory Management</a> <a href="#fnref7" class="footnote-backref">↩︎</a> <a href="#fnref7:1" class="footnote-backref">↩︎</a> <a href="#fnref7:2" class="footnote-backref">↩︎</a> <a href="#fnref7:3" class="footnote-backref">↩︎</a> <a href="#fnref7:4" class="footnote-backref">↩︎</a> <a href="#fnref7:5" class="footnote-backref">↩︎</a> <a href="#fnref7:6" class="footnote-backref">↩︎</a></p></li><li id="fn8" class="footnote-item"><p><a href="https://spark.apache.org/docs/3.5.8/api/python/user_guide/sql/arrow_pandas.html" target="_blank" rel="noreferrer">Apache Arrow in PySpark</a> <a href="#fnref8" class="footnote-backref">↩︎</a> <a href="#fnref8:1" class="footnote-backref">↩︎</a> <a href="#fnref8:2" class="footnote-backref">↩︎</a> <a href="#fnref8:3" class="footnote-backref">↩︎</a> <a href="#fnref8:4" class="footnote-backref">↩︎</a> <a href="#fnref8:5" class="footnote-backref">↩︎</a></p></li><li id="fn9" class="footnote-item"><p><a href="https://www.databricks.com/blog/2020/05/29/adaptive-query-execution-speeding-up-spark-sql-at-runtime.html" target="_blank" rel="noreferrer">Adaptive Query Execution: Speeding Up Spark SQL at Runtime</a> <a href="#fnref9" class="footnote-backref">↩︎</a></p></li><li id="fn10" class="footnote-item"><p><em>High Performance Spark, 2nd Edition</em>, Karau, Polak &amp; Warren, ch. 5 <a href="#fnref10" class="footnote-backref">↩︎</a> <a href="#fnref10:1" class="footnote-backref">↩︎</a></p></li><li id="fn11" class="footnote-item"><p><a href="https://raw.githubusercontent.com/spoddutur/spark-notes/master/task_memory_management_in_spark.md" target="_blank" rel="noreferrer">task_memory_management_in_spark.md</a> <a href="#fnref11" class="footnote-backref">↩︎</a></p></li><li id="fn12" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/job-scheduling.html" target="_blank" rel="noreferrer">Job Scheduling — Spark</a> <a href="#fnref12" class="footnote-backref">↩︎</a> <a href="#fnref12:1" class="footnote-backref">↩︎</a> <a href="#fnref12:2" class="footnote-backref">↩︎</a></p></li><li id="fn13" class="footnote-item"><p><a href="https://luminousmen.com/post/the-apache-spark-optimization-checklist" target="_blank" rel="noreferrer">The Apache Spark Optimization Checklist</a> <a href="#fnref13" class="footnote-backref">↩︎</a></p></li><li id="fn14" class="footnote-item"><p><a href="https://www.databricks.com/blog/2015/05/28/tuning-java-garbage-collection-for-spark-applications.html" target="_blank" rel="noreferrer">Tuning Java Garbage Collection for Spark Applications</a> <a href="#fnref14" class="footnote-backref">↩︎</a> <a href="#fnref14:1" class="footnote-backref">↩︎</a> <a href="#fnref14:2" class="footnote-backref">↩︎</a> <a href="#fnref14:3" class="footnote-backref">↩︎</a></p></li><li id="fn15" class="footnote-item"><p><a href="https://wiki.openjdk.org/display/zgc" target="_blank" rel="noreferrer">ZGC — The Z Garbage Collector</a> <a href="#fnref15" class="footnote-backref">↩︎</a> <a href="#fnref15:1" class="footnote-backref">↩︎</a></p></li></ol></section></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/tuning-reference/metrics.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>Metrics Glossary</span><!--]--></a></div><div class="pager" data-v-e257564d><!----></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
26
- <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"VTcYTH4d\",\"contributor-guide_architecture_detector-contract.md\":\"CsqZHz_T\",\"contributor-guide_architecture_drill-down.md\":\"CFNCE8JB\",\"contributor-guide_architecture_impact-estimation.md\":\"HT9IJGIl\",\"contributor-guide_architecture_index.md\":\"3TO9ic6w\",\"contributor-guide_architecture_overview.md\":\"CTEqVeV4\",\"contributor-guide_architecture_state-and-history.md\":\"DOTVHjTV\",\"contributor-guide_architecture_widget-rendering.md\":\"BmpARZve\",\"contributor-guide_architecture_worker-protocol.md\":\"21pTYhFF\",\"contributor-guide_contributing.md\":\"B2E9ni79\",\"contributor-guide_development-setup.md\":\"CjgZJwjK\",\"contributor-guide_testing.md\":\"DTwhpoN2\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"D08Uaod7\",\"tuning-reference_aqe.md\":\"C1Ae6A8u\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"6wjKuJNE\",\"tuning-reference_bottleneck-cold-start.md\":\"BVQbsuHP\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"DtaYnaiF\",\"tuning-reference_bottleneck-failures.md\":\"Z1Tdmdd3\",\"tuning-reference_bottleneck-gc.md\":\"BCnXLf1n\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"CcvQj2g9\",\"tuning-reference_bottleneck-memory-utilization.md\":\"B-C0-8h-\",\"tuning-reference_bottleneck-retry-waste.md\":\"DCnhWiqj\",\"tuning-reference_bottleneck-shuffle.md\":\"B5OMJCd3\",\"tuning-reference_bottleneck-skew.md\":\"Cvo6NQm_\",\"tuning-reference_bottleneck-slow-host.md\":\"ja5qr3wq\",\"tuning-reference_bottleneck-small-files.md\":\"Crjtb0i2\",\"tuning-reference_bottleneck-spill.md\":\"Bl8KtSRg\",\"tuning-reference_bottleneck-straggler.md\":\"B0VN8reT\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"DQCftGhu\",\"tuning-reference_bottleneck-utilization.md\":\"BXDl30f9\",\"tuning-reference_caching.md\":\"CcaiMn_7\",\"tuning-reference_cluster-config.md\":\"DbWG1yZq\",\"tuning-reference_config.md\":\"YUqSpwFB\",\"tuning-reference_data-formats.md\":\"C5DX5xL4\",\"tuning-reference_index.md\":\"ZVIpQ6Dk\",\"tuning-reference_intro.md\":\"BSSaDlqx\",\"tuning-reference_joins.md\":\"BVhNCDnE\",\"tuning-reference_memory-model.md\":\"CfjFdgbl\",\"tuning-reference_metrics.md\":\"C64VNEL4\",\"tuning-reference_partitioning.md\":\"CtLK0xsS\",\"tuning-reference_pyspark.md\":\"D-92XgF8\",\"tuning-reference_shuffle.md\":\"Iq2v1oHJ\",\"tuning-reference_spark-architecture.md\":\"_qwnE_Sc\",\"tuning-reference_table-formats.md\":\"BSYEK-eN\",\"user-guide_alternative-log-retrieval.md\":\"ZHPSbBkF\",\"user-guide_getting-started.md\":\"CEa2ZX-l\",\"user-guide_mcp-tools.md\":\"dQwIvdTF\",\"user-guide_run-comparison.md\":\"ChLI3MK0\",\"user-guide_understanding-findings.md\":\"D1ccs60C\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Alternative log retrieval\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
27
+ <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"YQcj5EI4\",\"contributor-guide_architecture_board-widgets_caching-and-memory.md\":\"8bH1HnZp\",\"contributor-guide_architecture_board-widgets_plan-advisor.md\":\"RlzvUmNm\",\"contributor-guide_architecture_board-widgets_visual-system-and-templating.md\":\"DYgUpIJB\",\"contributor-guide_architecture_detector-contract.md\":\"wj6Et6UD\",\"contributor-guide_architecture_detector-contract_plan-attribution.md\":\"yNex4bys\",\"contributor-guide_architecture_detector-contract_threshold-tables.md\":\"fkPBXsU2\",\"contributor-guide_architecture_drill-down.md\":\"CqZPrdo9\",\"contributor-guide_architecture_drill-down_plan-graph.md\":\"DwOozIEs\",\"contributor-guide_architecture_impact-estimation.md\":\"CZv_ZGOp\",\"contributor-guide_architecture_impact-estimation_caveats-tuning-and-coverage.md\":\"BUucRc0A\",\"contributor-guide_architecture_impact-estimation_stage-union-rollup.md\":\"DdCGtAGu\",\"contributor-guide_architecture_index.md\":\"tymxf1Fh\",\"contributor-guide_architecture_overview.md\":\"C1sSvm9i\",\"contributor-guide_architecture_state-and-history.md\":\"DyCBdjuM\",\"contributor-guide_architecture_state-and-history_history-server-intake.md\":\"C298hrHZ\",\"contributor-guide_architecture_state-and-history_run-comparison-internals.md\":\"uuAKpY0Y\",\"contributor-guide_architecture_widget-rendering.md\":\"BOZ1azXG\",\"contributor-guide_architecture_widget-rendering_findings-tab.md\":\"BQXycK6G\",\"contributor-guide_architecture_widget-rendering_investigation-routing.md\":\"BGdiyElu\",\"contributor-guide_architecture_widget-rendering_list-sort-mode.md\":\"BuHKgH0C\",\"contributor-guide_architecture_widget-rendering_reference-section.md\":\"DWqC80bN\",\"contributor-guide_architecture_widget-rendering_row-expand-contract.md\":\"ZT89u9f1\",\"contributor-guide_architecture_worker-protocol.md\":\"BAnhi2-d\",\"contributor-guide_architecture_worker-protocol_cli-and-mcp.md\":\"Dk08VS7n\",\"contributor-guide_architecture_worker-protocol_core-and-event-validation.md\":\"DSVFPpWb\",\"contributor-guide_architecture_worker-protocol_evidence-availability.md\":\"CjimIDBY\",\"contributor-guide_architecture_worker-protocol_evidence-report.md\":\"DVGZOWvW\",\"contributor-guide_contributing.md\":\"CQMvGwFt\",\"contributor-guide_development-setup.md\":\"D9bu4j_c\",\"contributor-guide_testing.md\":\"By6XGR9a\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"D08Uaod7\",\"tuning-reference_aqe.md\":\"C1Ae6A8u\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"6wjKuJNE\",\"tuning-reference_bottleneck-cold-start.md\":\"BVQbsuHP\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"DtaYnaiF\",\"tuning-reference_bottleneck-failures.md\":\"Z1Tdmdd3\",\"tuning-reference_bottleneck-gc.md\":\"BCnXLf1n\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"CcvQj2g9\",\"tuning-reference_bottleneck-memory-utilization.md\":\"B-C0-8h-\",\"tuning-reference_bottleneck-retry-waste.md\":\"DCnhWiqj\",\"tuning-reference_bottleneck-shuffle.md\":\"B5OMJCd3\",\"tuning-reference_bottleneck-skew.md\":\"Cvo6NQm_\",\"tuning-reference_bottleneck-slow-host.md\":\"ja5qr3wq\",\"tuning-reference_bottleneck-small-files.md\":\"Crjtb0i2\",\"tuning-reference_bottleneck-spill.md\":\"Bl8KtSRg\",\"tuning-reference_bottleneck-straggler.md\":\"B0VN8reT\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"DQCftGhu\",\"tuning-reference_bottleneck-utilization.md\":\"BXDl30f9\",\"tuning-reference_caching.md\":\"CcaiMn_7\",\"tuning-reference_cluster-config.md\":\"DbWG1yZq\",\"tuning-reference_config.md\":\"YUqSpwFB\",\"tuning-reference_data-formats.md\":\"C5DX5xL4\",\"tuning-reference_index.md\":\"ZVIpQ6Dk\",\"tuning-reference_intro.md\":\"BSSaDlqx\",\"tuning-reference_joins.md\":\"BVhNCDnE\",\"tuning-reference_memory-model.md\":\"CfjFdgbl\",\"tuning-reference_metrics.md\":\"C64VNEL4\",\"tuning-reference_partitioning.md\":\"CtLK0xsS\",\"tuning-reference_pyspark.md\":\"D-92XgF8\",\"tuning-reference_shuffle.md\":\"Iq2v1oHJ\",\"tuning-reference_spark-architecture.md\":\"_qwnE_Sc\",\"tuning-reference_table-formats.md\":\"BSYEK-eN\",\"user-guide_alternative-log-retrieval.md\":\"CJpYvHGo\",\"user-guide_alternative-log-retrieval_managed-platforms.md\":\"DvDwHNtC\",\"user-guide_alternative-log-retrieval_ssh-bastion.md\":\"Gszzlplv\",\"user-guide_getting-started.md\":\"DYHHHkQR\",\"user-guide_getting-started_ci-and-automation.md\":\"DpnfKz3x\",\"user-guide_getting-started_reading-the-dashboard.md\":\"Obf3W3__\",\"user-guide_getting-started_regression-metric-keys.md\":\"B_gzr2Mw\",\"user-guide_getting-started_tuning-thresholds.md\":\"B_MyjyZm\",\"user-guide_mcp-tools.md\":\"C65SjWUL\",\"user-guide_mcp-tools_compare-runs.md\":\"CPowlb2q\",\"user-guide_mcp-tools_diagnose-run.md\":\"BP1vnHA5\",\"user-guide_mcp-tools_evaluate-budgets.md\":\"Dq5Azk8G\",\"user-guide_mcp-tools_get-finding-documentation.md\":\"BtTTHNV6\",\"user-guide_mcp-tools_get-finding-evidence.md\":\"rKyu4wts\",\"user-guide_mcp-tools_get-reference-doc.md\":\"Yac4tH3u\",\"user-guide_mcp-tools_get-run-summary.md\":\"DpxZ4ZNp\",\"user-guide_mcp-tools_list-runs.md\":\"DpQPeCM3\",\"user-guide_run-comparison.md\":\"B6aNtDYF\",\"user-guide_run-comparison_how-stages-are-matched.md\":\"XBzQ1oEU\",\"user-guide_understanding-findings.md\":\"Dqt2H_C1\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\",\"collapsed\":true,\"items\":[{\"text\":\"Reading the dashboard\",\"link\":\"/user-guide/getting-started/reading-the-dashboard\"},{\"text\":\"CI and automation\",\"link\":\"/user-guide/getting-started/ci-and-automation\"},{\"text\":\"Tuning detector thresholds\",\"link\":\"/user-guide/getting-started/tuning-thresholds\"},{\"text\":\"Regression metric keys\",\"link\":\"/user-guide/getting-started/regression-metric-keys\"}]},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\",\"collapsed\":true,\"items\":[{\"text\":\"How stages are matched\",\"link\":\"/user-guide/run-comparison/how-stages-are-matched\"}]},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\",\"collapsed\":true,\"items\":[{\"text\":\"diagnose_run\",\"link\":\"/user-guide/mcp-tools/diagnose-run\"},{\"text\":\"get_run_summary\",\"link\":\"/user-guide/mcp-tools/get-run-summary\"},{\"text\":\"evaluate_budgets\",\"link\":\"/user-guide/mcp-tools/evaluate-budgets\"},{\"text\":\"compare_runs\",\"link\":\"/user-guide/mcp-tools/compare-runs\"},{\"text\":\"get_finding_evidence\",\"link\":\"/user-guide/mcp-tools/get-finding-evidence\"},{\"text\":\"get_finding_documentation\",\"link\":\"/user-guide/mcp-tools/get-finding-documentation\"},{\"text\":\"get_reference_doc\",\"link\":\"/user-guide/mcp-tools/get-reference-doc\"},{\"text\":\"list_runs\",\"link\":\"/user-guide/mcp-tools/list-runs\"}]},{\"text\":\"Finding your event log\",\"link\":\"/user-guide/alternative-log-retrieval\",\"collapsed\":true,\"items\":[{\"text\":\"Managed platforms\",\"link\":\"/user-guide/alternative-log-retrieval/managed-platforms\"},{\"text\":\"Behind an SSH bastion\",\"link\":\"/user-guide/alternative-log-retrieval/ssh-bastion\"}]}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\",\"collapsed\":true,\"items\":[{\"text\":\"Evidence availability\",\"link\":\"/contributor-guide/architecture/worker-protocol/evidence-availability\"},{\"text\":\"Evidence report and finding identity\",\"link\":\"/contributor-guide/architecture/worker-protocol/evidence-report\"},{\"text\":\"CLI and MCP server\",\"link\":\"/contributor-guide/architecture/worker-protocol/cli-and-mcp\"},{\"text\":\"Core and event validation\",\"link\":\"/contributor-guide/architecture/worker-protocol/core-and-event-validation\"}]},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\",\"collapsed\":true,\"items\":[{\"text\":\"Run comparison internals\",\"link\":\"/contributor-guide/architecture/state-and-history/run-comparison-internals\"},{\"text\":\"History Server intake\",\"link\":\"/contributor-guide/architecture/state-and-history/history-server-intake\"}]},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\",\"collapsed\":true,\"items\":[{\"text\":\"Plan attribution\",\"link\":\"/contributor-guide/architecture/detector-contract/plan-attribution\"},{\"text\":\"Threshold tables\",\"link\":\"/contributor-guide/architecture/detector-contract/threshold-tables\"}]},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\",\"collapsed\":true,\"items\":[{\"text\":\"Stage union rollup\",\"link\":\"/contributor-guide/architecture/impact-estimation/stage-union-rollup\"},{\"text\":\"Caveats, tuning and coverage\",\"link\":\"/contributor-guide/architecture/impact-estimation/caveats-tuning-and-coverage\"}]},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\",\"collapsed\":true,\"items\":[{\"text\":\"Findings tab\",\"link\":\"/contributor-guide/architecture/widget-rendering/findings-tab\"},{\"text\":\"List sort mode\",\"link\":\"/contributor-guide/architecture/widget-rendering/list-sort-mode\"},{\"text\":\"Row/expand contract\",\"link\":\"/contributor-guide/architecture/widget-rendering/row-expand-contract\"},{\"text\":\"ReferenceSection\",\"link\":\"/contributor-guide/architecture/widget-rendering/reference-section\"},{\"text\":\"Investigation routing\",\"link\":\"/contributor-guide/architecture/widget-rendering/investigation-routing\"}]},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\",\"collapsed\":true,\"items\":[{\"text\":\"Plan Advisor\",\"link\":\"/contributor-guide/architecture/board-widgets/plan-advisor\"},{\"text\":\"Caching and memory widgets\",\"link\":\"/contributor-guide/architecture/board-widgets/caching-and-memory\"},{\"text\":\"Visual system and templating\",\"link\":\"/contributor-guide/architecture/board-widgets/visual-system-and-templating\"}]},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\",\"collapsed\":true,\"items\":[{\"text\":\"Plan graph\",\"link\":\"/contributor-guide/architecture/drill-down/plan-graph\"}]}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":{\"selector\":\"[data-sf-anchor-offset]\",\"padding\":0},\"cleanUrls\":false}");</script>
27
28
 
28
29
  </body>
29
30
  </html>