sparkforensics-server 0.5.2 → 0.8.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (445) hide show
  1. package/index.js +16 -7
  2. package/package.json +1 -1
  3. package/public/assets/AutoscalingChurn-DenWKTv-.js +1 -0
  4. package/public/assets/CacheUtilization-DLz7NdjV.js +1 -0
  5. package/public/assets/CachingOpportunity-zpQIqhzm.js +1 -0
  6. package/public/assets/ChartTheme-DAiARbLP.js +8 -0
  7. package/public/assets/ColdStart-C-9HjxWl.js +1 -0
  8. package/public/assets/ConfigAudit-D0V9tcrJ.js +1 -0
  9. package/public/assets/CoreUsageArea-yeerrljd.js +1 -0
  10. package/public/assets/Dashboard-DkUPrHKV.js +1 -0
  11. package/public/assets/DisclosureContext-DK1_iz6J.js +1 -0
  12. package/public/assets/DuplicatePlanSubtree-xDNDbCd3.js +1 -0
  13. package/public/assets/DurationHistogram-B6jsE0Ov.js +1 -0
  14. package/public/assets/EmptyState-C1uL40VU.js +1 -0
  15. package/public/assets/ExecutorUtilization-DXDZKmSc.js +1 -0
  16. package/public/assets/GcPressure-MRf7I4IG.js +1 -0
  17. package/public/assets/ImpactEstimate-Dm3xWbzV.js +1 -0
  18. package/public/assets/IncompleteRun-CaLOhMtf.js +1 -0
  19. package/public/assets/JobFailures-CzmzlvhO.js +1 -0
  20. package/public/assets/MemoryUtilization-AWUMUjCH.js +1 -0
  21. package/public/assets/NestedLoopJoin-C9sNr6Ss.js +1 -0
  22. package/public/assets/OverBroadcast-BOmRE6ca.js +1 -0
  23. package/public/assets/PartitionHint-BL2EmJfE.js +1 -0
  24. package/public/assets/PartitionSizing-BzBySvUq.js +1 -0
  25. package/public/assets/PlanExplorer-DdiQRcJU.js +1 -0
  26. package/public/assets/PlanGraphRoute-CIH9gBuR.js +1 -0
  27. package/public/assets/PlanView-B1hxzFRs.js +2 -0
  28. package/public/assets/PythonUdf-DvgCCueD.js +1 -0
  29. package/public/assets/RetryWaste-uXXu_CyM.js +1 -0
  30. package/public/assets/RowPagination-C-LzWkI5.js +1 -0
  31. package/public/assets/RowStatusCluster-Dbtmte7l.js +1 -0
  32. package/public/assets/ShuffleIO-nALwzC1_.js +1 -0
  33. package/public/assets/Skew-CHMEKgvk.js +1 -0
  34. package/public/assets/SlowHost-yyvjpHk6.js +1 -0
  35. package/public/assets/SmallFiles-y6odJ97n.js +1 -0
  36. package/public/assets/SpeculationWaste-C-SD2LOt.js +1 -0
  37. package/public/assets/Spill-DHTMmOup.js +1 -0
  38. package/public/assets/StageFailed-CQ3DxC9R.js +1 -0
  39. package/public/assets/StageFindingGroup-BgYLyj5q.js +1 -0
  40. package/public/assets/StageHeader-CIETkjVZ.js +1 -0
  41. package/public/assets/StagePill--P7PDd3s.js +1 -0
  42. package/public/assets/StageShape-BIv8vx_o.js +1 -0
  43. package/public/assets/StageSlowness-MMqzERxN.js +1 -0
  44. package/public/assets/Straggler-BgrY7AyW.js +1 -0
  45. package/public/assets/TaskFailures-BHHyKYTs.js +1 -0
  46. package/public/assets/TinyTask-DDScNhCl.js +1 -0
  47. package/public/assets/UnderBroadcast-CGIC04TI.js +1 -0
  48. package/public/assets/WidgetLeadSummary-CXFZP00C.js +1 -0
  49. package/public/assets/button-DbpTz2yk.js +1 -0
  50. package/public/assets/charts-vendor-CtTxwTeu.js +59 -0
  51. package/public/assets/composite-C9Nu2EI9.js +1 -0
  52. package/public/assets/d3-vendor-DNlD2-cz.js +1 -0
  53. package/public/assets/detector-registry-Bp8hrLua.js +2 -0
  54. package/public/assets/downsample-WmNM1qc9.js +1 -0
  55. package/public/assets/finding-anchor-Dr40LkS9.js +1 -0
  56. package/public/assets/findings-of-type-BL2u3RGZ.js +1 -0
  57. package/public/assets/impact-sort-mC5ISgX8.js +1 -0
  58. package/public/assets/inbox-KPU1Gppm.js +1 -0
  59. package/public/assets/index-CaRVPvAT.css +2 -0
  60. package/public/assets/index-fDiwfuoO.js +14 -0
  61. package/public/assets/inertValue-L9VxV0VE.js +1 -0
  62. package/public/assets/instrument-sans-latin-ext-wdth-normal-C5E2Gvlv.woff2 +0 -0
  63. package/public/assets/instrument-sans-latin-wdth-normal-BVScPF0l.woff2 +0 -0
  64. package/public/assets/jetbrains-mono-latin-ext-wght-normal-DBQx-q_a.woff2 +0 -0
  65. package/public/assets/jetbrains-mono-latin-wght-normal-B9CIFXIH.woff2 +0 -0
  66. package/public/assets/layout.worker-CLcBHxVp.js +3 -0
  67. package/public/assets/{parser-worker-CA5RxqKc.js → parser-worker-CRIOPMOW.js} +8 -8
  68. package/public/assets/plan-graph-vendor-Ce6d4HU6.js +7 -0
  69. package/public/assets/react-vendor-C6KcUD36.js +9 -0
  70. package/public/assets/select-BHqOqun5.js +1 -0
  71. package/public/assets/toggle-DPeN6aNa.js +1 -0
  72. package/public/assets/toggle-group-Kl3uyHSr.js +1 -0
  73. package/public/assets/usePagedRows-BafD6bEN.js +1 -0
  74. package/public/assets/useRecentFiles-DzJZAoOT.js +3 -0
  75. package/public/assets/useRegisterFieldControl-pQv-ZDoU.js +1 -0
  76. package/public/assets/useScrollLock-BCxn_dMp.js +1 -0
  77. package/public/assets/useSortMode-DjrAN-z4.js +1 -0
  78. package/public/docs/404.html +3 -7
  79. package/public/docs/assets/{app.eYTPqTeq.js → app.1gvzd5Zb.js} +1 -1
  80. package/public/docs/assets/chunks/@localSearchIndexroot.DS2TQP5_.js +1 -0
  81. package/public/docs/assets/chunks/{VPLocalSearchBox.DsZrQKQR.js → VPLocalSearchBox.CNxT3s0I.js} +1 -1
  82. package/public/docs/assets/chunks/{theme.DsTNE2pE.js → theme.C5lLqUv7.js} +2 -2
  83. package/public/docs/assets/contributor-guide_architecture_board-widgets.md.CqKM_jEG.js +1 -0
  84. package/public/docs/assets/contributor-guide_architecture_board-widgets.md.CqKM_jEG.lean.js +1 -0
  85. package/public/docs/assets/{contributor-guide_architecture_board-widgets_caching-and-memory.md.8bH1HnZp.js → contributor-guide_architecture_board-widgets_caching-and-memory.md.BYDo3ClR.js} +1 -1
  86. package/public/docs/assets/contributor-guide_architecture_board-widgets_plan-advisor.md.ajUAVdbc.js +1 -0
  87. package/public/docs/assets/contributor-guide_architecture_board-widgets_visual-system-and-templating.md.WqY74T0T.js +1 -0
  88. package/public/docs/assets/contributor-guide_architecture_board-widgets_visual-system-and-templating.md.WqY74T0T.lean.js +1 -0
  89. package/public/docs/assets/contributor-guide_architecture_detector-contract.md.CHHgywoP.js +1 -0
  90. package/public/docs/assets/{contributor-guide_architecture_detector-contract.md.wj6Et6UD.lean.js → contributor-guide_architecture_detector-contract.md.CHHgywoP.lean.js} +1 -1
  91. package/public/docs/assets/contributor-guide_architecture_detector-contract_plan-attribution.md.DIz3_oz9.js +1 -0
  92. package/public/docs/assets/contributor-guide_architecture_detector-contract_plan-attribution.md.DIz3_oz9.lean.js +1 -0
  93. package/public/docs/assets/contributor-guide_architecture_detector-contract_threshold-tables.md.WH4tRHPL.js +1 -0
  94. package/public/docs/assets/{contributor-guide_architecture_detector-contract_threshold-tables.md.CKjWG56Y.lean.js → contributor-guide_architecture_detector-contract_threshold-tables.md.WH4tRHPL.lean.js} +1 -1
  95. package/public/docs/assets/{contributor-guide_architecture_drill-down.md.CqZPrdo9.js → contributor-guide_architecture_drill-down.md.yB-Hz2XT.js} +1 -1
  96. package/public/docs/assets/{contributor-guide_architecture_drill-down.md.CqZPrdo9.lean.js → contributor-guide_architecture_drill-down.md.yB-Hz2XT.lean.js} +1 -1
  97. package/public/docs/assets/contributor-guide_architecture_drill-down_plan-graph.md.BQZE04sw.js +1 -0
  98. package/public/docs/assets/{contributor-guide_architecture_drill-down_plan-graph.md.DwOozIEs.lean.js → contributor-guide_architecture_drill-down_plan-graph.md.BQZE04sw.lean.js} +1 -1
  99. package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.K71YB-Rr.js +1 -0
  100. package/public/docs/assets/{contributor-guide_architecture_impact-estimation.md.CZv_ZGOp.lean.js → contributor-guide_architecture_impact-estimation.md.K71YB-Rr.lean.js} +1 -1
  101. package/public/docs/assets/contributor-guide_architecture_impact-estimation_caveats-tuning-and-coverage.md.DLCAffwn.js +1 -0
  102. package/public/docs/assets/{contributor-guide_architecture_impact-estimation_caveats-tuning-and-coverage.md.BUucRc0A.lean.js → contributor-guide_architecture_impact-estimation_caveats-tuning-and-coverage.md.DLCAffwn.lean.js} +1 -1
  103. package/public/docs/assets/contributor-guide_architecture_impact-estimation_stage-union-rollup.md.JFrzDMwo.js +1 -0
  104. package/public/docs/assets/contributor-guide_architecture_impact-estimation_stage-union-rollup.md.JFrzDMwo.lean.js +1 -0
  105. package/public/docs/assets/contributor-guide_architecture_overview.md.nxkpOhgf.js +1 -0
  106. package/public/docs/assets/contributor-guide_architecture_state-and-history_run-comparison-internals.md.U-9KMDFK.js +1 -0
  107. package/public/docs/assets/{contributor-guide_architecture_state-and-history_run-comparison-internals.md.CZA0JLtV.lean.js → contributor-guide_architecture_state-and-history_run-comparison-internals.md.U-9KMDFK.lean.js} +1 -1
  108. package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.BrSz8Ezh.js +1 -0
  109. package/public/docs/assets/contributor-guide_architecture_widget-rendering_findings-tab.md.B7rjSF_h.js +1 -0
  110. package/public/docs/assets/contributor-guide_architecture_widget-rendering_findings-tab.md.B7rjSF_h.lean.js +1 -0
  111. package/public/docs/assets/{contributor-guide_architecture_widget-rendering_list-sort-mode.md.BuHKgH0C.js → contributor-guide_architecture_widget-rendering_list-sort-mode.md.37hcZJ7q.js} +1 -1
  112. package/public/docs/assets/{contributor-guide_architecture_widget-rendering_row-expand-contract.md.ZT89u9f1.js → contributor-guide_architecture_widget-rendering_row-expand-contract.md.mK_VmAWk.js} +1 -1
  113. package/public/docs/assets/{contributor-guide_architecture_worker-protocol.md.BAnhi2-d.js → contributor-guide_architecture_worker-protocol.md.Br6OE7RW.js} +1 -1
  114. package/public/docs/assets/contributor-guide_architecture_worker-protocol.md.Br6OE7RW.lean.js +1 -0
  115. package/public/docs/assets/contributor-guide_architecture_worker-protocol_cli-and-mcp.md.IVyaXrGe.js +1 -0
  116. package/public/docs/assets/{contributor-guide_architecture_worker-protocol_cli-and-mcp.md.DC1YqXBp.lean.js → contributor-guide_architecture_worker-protocol_cli-and-mcp.md.IVyaXrGe.lean.js} +1 -1
  117. package/public/docs/assets/{contributor-guide_architecture_worker-protocol_core-and-event-validation.md.DSVFPpWb.js → contributor-guide_architecture_worker-protocol_core-and-event-validation.md.DVSXn-bb.js} +1 -1
  118. package/public/docs/assets/{contributor-guide_architecture_worker-protocol_evidence-availability.md.CjimIDBY.js → contributor-guide_architecture_worker-protocol_evidence-availability.md.D3S5cyOW.js} +2 -2
  119. package/public/docs/assets/{contributor-guide_architecture_worker-protocol_evidence-availability.md.CjimIDBY.lean.js → contributor-guide_architecture_worker-protocol_evidence-availability.md.D3S5cyOW.lean.js} +1 -1
  120. package/public/docs/assets/contributor-guide_architecture_worker-protocol_evidence-report.md.CtmCU57I.js +1 -0
  121. package/public/docs/assets/{contributor-guide_architecture_worker-protocol_evidence-report.md.DVGZOWvW.lean.js → contributor-guide_architecture_worker-protocol_evidence-report.md.CtmCU57I.lean.js} +1 -1
  122. package/public/docs/assets/contributor-guide_contributing.md.DBW4drtZ.js +1 -0
  123. package/public/docs/assets/{contributor-guide_development-setup.md.D9bu4j_c.js → contributor-guide_development-setup.md.B21ZTDhn.js} +3 -3
  124. package/public/docs/assets/contributor-guide_development-setup.md.B21ZTDhn.lean.js +1 -0
  125. package/public/docs/assets/contributor-guide_testing.md.DpkozNoG.js +4 -0
  126. package/public/docs/assets/{contributor-guide_testing.md.By6XGR9a.lean.js → contributor-guide_testing.md.DpkozNoG.lean.js} +1 -1
  127. package/public/docs/assets/{index.md.CHJVslga.js → index.md.lMm7SUMz.js} +1 -1
  128. package/public/docs/assets/{index.md.CHJVslga.lean.js → index.md.lMm7SUMz.lean.js} +1 -1
  129. package/public/docs/assets/instrument-sans-latin-ext-wdth-normal.C5E2Gvlv.woff2 +0 -0
  130. package/public/docs/assets/instrument-sans-latin-wdth-normal.BVScPF0l.woff2 +0 -0
  131. package/public/docs/assets/jetbrains-mono-cyrillic-wght-normal.D73BlboJ.woff2 +0 -0
  132. package/public/docs/assets/jetbrains-mono-greek-wght-normal.Bw9x6K1M.woff2 +0 -0
  133. package/public/docs/assets/jetbrains-mono-latin-ext-wght-normal.DBQx-q_a.woff2 +0 -0
  134. package/public/docs/assets/jetbrains-mono-latin-wght-normal.B9CIFXIH.woff2 +0 -0
  135. package/public/docs/assets/jetbrains-mono-vietnamese-wght-normal.Bt-aOZkq.woff2 +0 -0
  136. package/public/docs/assets/style.Cfxqlj0y.css +1 -0
  137. package/public/docs/assets/user-guide_getting-started_ci-and-automation.md.C0pcQoNc.js +29 -0
  138. package/public/docs/assets/user-guide_getting-started_ci-and-automation.md.C0pcQoNc.lean.js +1 -0
  139. package/public/docs/assets/user-guide_getting-started_reading-the-dashboard.md.CGaxPX_1.js +1 -0
  140. package/public/docs/assets/{user-guide_getting-started_tuning-thresholds.md.B_MyjyZm.js → user-guide_getting-started_tuning-thresholds.md.BGVE-wrc.js} +2 -2
  141. package/public/docs/assets/{user-guide_getting-started_tuning-thresholds.md.B_MyjyZm.lean.js → user-guide_getting-started_tuning-thresholds.md.BGVE-wrc.lean.js} +1 -1
  142. package/public/docs/assets/{user-guide_mcp-tools.md.C65SjWUL.js → user-guide_mcp-tools.md.49r3L2YE.js} +1 -1
  143. package/public/docs/assets/{user-guide_mcp-tools_compare-runs.md.Cc_gLoix.js → user-guide_mcp-tools_compare-runs.md.BQ_kIptC.js} +11 -6
  144. package/public/docs/assets/user-guide_mcp-tools_compare-runs.md.BQ_kIptC.lean.js +1 -0
  145. package/public/docs/assets/user-guide_mcp-tools_diagnose-run.md.azCZXkH8.js +44 -0
  146. package/public/docs/assets/{user-guide_mcp-tools_diagnose-run.md.BP1vnHA5.lean.js → user-guide_mcp-tools_diagnose-run.md.azCZXkH8.lean.js} +1 -1
  147. package/public/docs/assets/{user-guide_mcp-tools_evaluate-budgets.md.BB0nDY3Z.js → user-guide_mcp-tools_evaluate-budgets.md.dpQpRQ7z.js} +2 -2
  148. package/public/docs/assets/{user-guide_mcp-tools_evaluate-budgets.md.BB0nDY3Z.lean.js → user-guide_mcp-tools_evaluate-budgets.md.dpQpRQ7z.lean.js} +1 -1
  149. package/public/docs/assets/{user-guide_mcp-tools_get-finding-evidence.md.rKyu4wts.js → user-guide_mcp-tools_get-finding-evidence.md.CH6FkNLX.js} +2 -1
  150. package/public/docs/assets/{user-guide_mcp-tools_get-finding-evidence.md.rKyu4wts.lean.js → user-guide_mcp-tools_get-finding-evidence.md.CH6FkNLX.lean.js} +1 -1
  151. package/public/docs/assets/{user-guide_mcp-tools_get-run-summary.md.DpxZ4ZNp.js → user-guide_mcp-tools_get-run-summary.md.EhaIsele.js} +1 -1
  152. package/public/docs/assets/user-guide_run-comparison.md.B_LOkdBf.js +1 -0
  153. package/public/docs/assets/user-guide_run-comparison.md.B_LOkdBf.lean.js +1 -0
  154. package/public/docs/assets/user-guide_understanding-findings.md.C4riKCKl.js +1 -0
  155. package/public/docs/assets/user-guide_understanding-findings.md.C4riKCKl.lean.js +1 -0
  156. package/public/docs/contributor-guide/architecture/board-widgets/caching-and-memory.html +6 -10
  157. package/public/docs/contributor-guide/architecture/board-widgets/plan-advisor.html +6 -10
  158. package/public/docs/contributor-guide/architecture/board-widgets/visual-system-and-templating.html +6 -10
  159. package/public/docs/contributor-guide/architecture/board-widgets.html +6 -10
  160. package/public/docs/contributor-guide/architecture/detector-contract/plan-attribution.html +6 -10
  161. package/public/docs/contributor-guide/architecture/detector-contract/threshold-tables.html +6 -10
  162. package/public/docs/contributor-guide/architecture/detector-contract.html +6 -10
  163. package/public/docs/contributor-guide/architecture/drill-down/plan-graph.html +6 -10
  164. package/public/docs/contributor-guide/architecture/drill-down.html +6 -10
  165. package/public/docs/contributor-guide/architecture/impact-estimation/caveats-tuning-and-coverage.html +6 -10
  166. package/public/docs/contributor-guide/architecture/impact-estimation/stage-union-rollup.html +6 -10
  167. package/public/docs/contributor-guide/architecture/impact-estimation.html +6 -10
  168. package/public/docs/contributor-guide/architecture/index.html +5 -9
  169. package/public/docs/contributor-guide/architecture/overview.html +6 -10
  170. package/public/docs/contributor-guide/architecture/state-and-history/history-server-intake.html +5 -9
  171. package/public/docs/contributor-guide/architecture/state-and-history/run-comparison-internals.html +6 -10
  172. package/public/docs/contributor-guide/architecture/state-and-history.html +5 -9
  173. package/public/docs/contributor-guide/architecture/widget-rendering/findings-tab.html +6 -10
  174. package/public/docs/contributor-guide/architecture/widget-rendering/investigation-routing.html +5 -9
  175. package/public/docs/contributor-guide/architecture/widget-rendering/list-sort-mode.html +6 -10
  176. package/public/docs/contributor-guide/architecture/widget-rendering/reference-section.html +5 -9
  177. package/public/docs/contributor-guide/architecture/widget-rendering/row-expand-contract.html +6 -10
  178. package/public/docs/contributor-guide/architecture/widget-rendering.html +6 -10
  179. package/public/docs/contributor-guide/architecture/worker-protocol/cli-and-mcp.html +6 -10
  180. package/public/docs/contributor-guide/architecture/worker-protocol/core-and-event-validation.html +6 -10
  181. package/public/docs/contributor-guide/architecture/worker-protocol/evidence-availability.html +7 -11
  182. package/public/docs/contributor-guide/architecture/worker-protocol/evidence-report.html +6 -10
  183. package/public/docs/contributor-guide/architecture/worker-protocol.html +6 -10
  184. package/public/docs/contributor-guide/contributing.html +6 -10
  185. package/public/docs/contributor-guide/development-setup.html +8 -12
  186. package/public/docs/contributor-guide/testing.html +9 -10
  187. package/public/docs/favicon.svg +2 -2
  188. package/public/docs/hashmap.json +1 -1
  189. package/public/docs/index.html +6 -10
  190. package/public/docs/logo-dark.svg +1 -0
  191. package/public/docs/logo-light.svg +1 -0
  192. package/public/docs/tuning-reference/anti-patterns.html +5 -9
  193. package/public/docs/tuning-reference/aqe.html +5 -9
  194. package/public/docs/tuning-reference/bottleneck-broadcast-sizing.html +5 -9
  195. package/public/docs/tuning-reference/bottleneck-cold-start.html +5 -9
  196. package/public/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html +5 -9
  197. package/public/docs/tuning-reference/bottleneck-failures.html +5 -9
  198. package/public/docs/tuning-reference/bottleneck-gc.html +5 -9
  199. package/public/docs/tuning-reference/bottleneck-job-failure-rate.html +5 -9
  200. package/public/docs/tuning-reference/bottleneck-memory-utilization.html +5 -9
  201. package/public/docs/tuning-reference/bottleneck-retry-waste.html +5 -9
  202. package/public/docs/tuning-reference/bottleneck-shuffle.html +5 -9
  203. package/public/docs/tuning-reference/bottleneck-skew.html +5 -9
  204. package/public/docs/tuning-reference/bottleneck-slow-host.html +5 -9
  205. package/public/docs/tuning-reference/bottleneck-small-files.html +5 -9
  206. package/public/docs/tuning-reference/bottleneck-spill.html +5 -9
  207. package/public/docs/tuning-reference/bottleneck-straggler.html +5 -9
  208. package/public/docs/tuning-reference/bottleneck-tiny-tasks.html +5 -9
  209. package/public/docs/tuning-reference/bottleneck-utilization.html +5 -9
  210. package/public/docs/tuning-reference/caching.html +5 -9
  211. package/public/docs/tuning-reference/cluster-config.html +5 -9
  212. package/public/docs/tuning-reference/config.html +5 -9
  213. package/public/docs/tuning-reference/data-formats.html +5 -9
  214. package/public/docs/tuning-reference/index.html +5 -9
  215. package/public/docs/tuning-reference/intro.html +5 -9
  216. package/public/docs/tuning-reference/joins.html +5 -9
  217. package/public/docs/tuning-reference/memory-model.html +5 -9
  218. package/public/docs/tuning-reference/metrics.html +5 -9
  219. package/public/docs/tuning-reference/partitioning.html +5 -9
  220. package/public/docs/tuning-reference/pyspark.html +5 -9
  221. package/public/docs/tuning-reference/shuffle.html +5 -9
  222. package/public/docs/tuning-reference/spark-architecture.html +5 -9
  223. package/public/docs/tuning-reference/table-formats.html +5 -9
  224. package/public/docs/user-guide/alternative-log-retrieval/managed-platforms.html +5 -9
  225. package/public/docs/user-guide/alternative-log-retrieval/ssh-bastion.html +5 -9
  226. package/public/docs/user-guide/alternative-log-retrieval.html +5 -9
  227. package/public/docs/user-guide/getting-started/ci-and-automation.html +17 -15
  228. package/public/docs/user-guide/getting-started/reading-the-dashboard.html +6 -10
  229. package/public/docs/user-guide/getting-started/regression-metric-keys.html +5 -9
  230. package/public/docs/user-guide/getting-started/tuning-thresholds.html +7 -11
  231. package/public/docs/user-guide/getting-started.html +5 -9
  232. package/public/docs/user-guide/mcp-tools/compare-runs.html +16 -15
  233. package/public/docs/user-guide/mcp-tools/diagnose-run.html +28 -11
  234. package/public/docs/user-guide/mcp-tools/evaluate-budgets.html +6 -10
  235. package/public/docs/user-guide/mcp-tools/get-finding-documentation.html +5 -9
  236. package/public/docs/user-guide/mcp-tools/get-finding-evidence.html +7 -10
  237. package/public/docs/user-guide/mcp-tools/get-reference-doc.html +5 -9
  238. package/public/docs/user-guide/mcp-tools/get-run-summary.html +6 -10
  239. package/public/docs/user-guide/mcp-tools/list-runs.html +5 -9
  240. package/public/docs/user-guide/mcp-tools.html +6 -10
  241. package/public/docs/user-guide/run-comparison/how-stages-are-matched.html +5 -9
  242. package/public/docs/user-guide/run-comparison.html +6 -10
  243. package/public/docs/user-guide/understanding-findings.html +6 -10
  244. package/public/export-template.html +46 -46
  245. package/public/favicon.svg +2 -2
  246. package/public/index.html +8 -10
  247. package/public/sample-runs/sample-run.ndjson.gz +0 -0
  248. package/vendor-core/allocation.js +126 -39
  249. package/vendor-core/analyzer.js +85 -58
  250. package/vendor-core/api-merge-writes.js +3 -0
  251. package/vendor-core/aqe-skew.js +347 -0
  252. package/vendor-core/cli/budgets.js +6 -4
  253. package/vendor-core/cli/collect-run-pair.js +89 -0
  254. package/vendor-core/cli/collect-run-worker.js +16 -0
  255. package/vendor-core/cli/collect-run.js +1 -2
  256. package/vendor-core/cli/native-zstd.js +12 -16
  257. package/vendor-core/comparability.js +234 -0
  258. package/vendor-core/comparison-output.js +15 -7
  259. package/vendor-core/comparison-verdict.js +105 -25
  260. package/vendor-core/core-source-hash.txt +1 -1
  261. package/vendor-core/core-time-series.js +0 -1
  262. package/vendor-core/core-usage-locality.js +1 -1
  263. package/vendor-core/delta-log-paths.js +10 -1
  264. package/vendor-core/detectors.js +1078 -416
  265. package/vendor-core/docs-config.js +0 -4
  266. package/vendor-core/docs-content/detection/cache.md +2 -1
  267. package/vendor-core/docs-content/detection/cfg.md +19 -11
  268. package/vendor-core/docs-content/detection/cold.md +2 -1
  269. package/vendor-core/docs-content/detection/gc.md +7 -3
  270. package/vendor-core/docs-content/detection/host.md +11 -1
  271. package/vendor-core/docs-content/detection/mem.md +17 -6
  272. package/vendor-core/docs-content/detection/part.md +3 -1
  273. package/vendor-core/docs-content/detection/plan.md +55 -13
  274. package/vendor-core/docs-content/detection/shfl.md +11 -5
  275. package/vendor-core/docs-content/detection/skew.md +48 -1
  276. package/vendor-core/docs-content/detection/slow.md +4 -2
  277. package/vendor-core/docs-content/detection/spec.md +6 -1
  278. package/vendor-core/docs-content/detection/spill.md +6 -3
  279. package/vendor-core/docs-content/detection/strag.md +19 -6
  280. package/vendor-core/docs-content/detection/tiny.md +9 -3
  281. package/vendor-core/docs-content/detection/util.md +6 -2
  282. package/vendor-core/effective-conf.js +7 -10
  283. package/vendor-core/efficiency-model.js +24 -16
  284. package/vendor-core/etl-phases.js +1 -2
  285. package/vendor-core/event-handlers.js +175 -57
  286. package/vendor-core/event-schemas.js +21 -16
  287. package/vendor-core/evidence-availability.js +9 -2
  288. package/vendor-core/evidence-report.js +118 -35
  289. package/vendor-core/executor-peaks.js +41 -0
  290. package/vendor-core/export-data.js +5 -5
  291. package/vendor-core/finding-identity.js +78 -0
  292. package/vendor-core/finding-presentation.js +129 -39
  293. package/vendor-core/finding-types.js +108 -30
  294. package/vendor-core/format-utils.js +14 -15
  295. package/vendor-core/html-export.js +1 -1
  296. package/vendor-core/impact-band.js +2 -2
  297. package/vendor-core/impact-format.js +47 -19
  298. package/vendor-core/impact-model.js +28 -17
  299. package/vendor-core/job-groups.js +4 -4
  300. package/vendor-core/mcp-server-factory.js +48 -12
  301. package/vendor-core/mcp-tools.js +179 -24
  302. package/vendor-core/median.js +2 -2
  303. package/vendor-core/nested-loop-join.js +73 -0
  304. package/vendor-core/occupancy.js +1 -2
  305. package/vendor-core/parser-worker.js +1 -1
  306. package/vendor-core/plan-graph-model.js +14 -22
  307. package/vendor-core/plan-node-detail.js +2 -3
  308. package/vendor-core/plan-summary.js +24 -19
  309. package/vendor-core/plan-tree-walk.js +6 -8
  310. package/vendor-core/python-stage.js +12 -1
  311. package/vendor-core/recommendation-rollup.js +13 -12
  312. package/vendor-core/redact.js +52 -46
  313. package/vendor-core/remediation.js +10 -5
  314. package/vendor-core/run-aggregates.js +6 -8
  315. package/vendor-core/run-comparison.js +226 -66
  316. package/vendor-core/run-identity.js +12 -0
  317. package/vendor-core/run-interpretation.js +21 -0
  318. package/vendor-core/run-metrics.js +2 -2
  319. package/vendor-core/run-output.js +44 -11
  320. package/vendor-core/run-totals.js +12 -0
  321. package/vendor-core/run-verdict.js +76 -3
  322. package/vendor-core/scaling-sim.js +3 -4
  323. package/vendor-core/snappy-block.js +2 -2
  324. package/vendor-core/spark-conf.js +168 -0
  325. package/vendor-core/sql-stages.js +2 -2
  326. package/vendor-core/stage-alignment.js +2 -12
  327. package/vendor-core/stage-identity.js +58 -3
  328. package/vendor-core/stage-label.js +14 -0
  329. package/vendor-core/stage-plan-nodes.js +17 -3
  330. package/vendor-core/stage-quantiles.js +141 -9
  331. package/vendor-core/stage-structure.js +17 -3
  332. package/vendor-core/threshold-overrides.js +9 -3
  333. package/vendor-core/types.js +70 -23
  334. package/vendor-core/wall-clock.js +8 -0
  335. package/vendor-core/wasted-core-hours.js +12 -12
  336. package/vendor-core/write-targets.js +146 -14
  337. package/vendor-core/zstd-worker-client.js +1 -1
  338. package/vendor-core/zstd-worker.js +1 -1
  339. package/public/assets/AutoscalingChurn-8t3U3isq.js +0 -1
  340. package/public/assets/CacheUtilization-CJs9tTum.js +0 -1
  341. package/public/assets/CachingOpportunity-D1NcHs0V.js +0 -1
  342. package/public/assets/ColdStart-CdRJ9gD8.js +0 -1
  343. package/public/assets/ConfigAudit-B5Gbe5js.js +0 -1
  344. package/public/assets/CoreUsageArea-a--NTRGY.js +0 -1
  345. package/public/assets/Dashboard-C1hNeX49.js +0 -2
  346. package/public/assets/DisclosureContext-ZpE6CkHx.js +0 -1
  347. package/public/assets/DuplicatePlanSubtree-dobXdz7E.js +0 -1
  348. package/public/assets/ExecutorUtilization-D9grwi7t.js +0 -1
  349. package/public/assets/GcPressure-8-G2ri10.js +0 -1
  350. package/public/assets/IncompleteRun-B91lky7a.js +0 -1
  351. package/public/assets/JobFailures-BsWvBMyJ.js +0 -1
  352. package/public/assets/MemoryUtilization-BM5schZI.js +0 -1
  353. package/public/assets/OverBroadcast-DdN8JIJH.js +0 -1
  354. package/public/assets/PartitionHint-CUvbnnxy.js +0 -1
  355. package/public/assets/PartitionSizing-CY5HDYtp.js +0 -1
  356. package/public/assets/PlanExplorer-DgHCDBD_.js +0 -1
  357. package/public/assets/PlanGraphRoute-DH3KJb8j.js +0 -1
  358. package/public/assets/RetryWaste-DOnE9vGF.js +0 -1
  359. package/public/assets/ShuffleIO-MMnDESkF.js +0 -1
  360. package/public/assets/Skew-BmZZJ5ms.js +0 -1
  361. package/public/assets/SlowHost-CsJzfEqo.js +0 -1
  362. package/public/assets/SmallFiles-BoK3R1Ex.js +0 -1
  363. package/public/assets/SpeculationWaste-CnVxB5AJ.js +0 -1
  364. package/public/assets/Spill-BhNicyBy.js +0 -1
  365. package/public/assets/StageFailed-mAhHU4l_.js +0 -1
  366. package/public/assets/StageFindingGroup-BEtPWcT0.js +0 -1
  367. package/public/assets/StageHeader-DJkYkrsU.js +0 -1
  368. package/public/assets/StageShape-BDxRknLp.js +0 -1
  369. package/public/assets/StageSlowness-BPRw-otN.js +0 -1
  370. package/public/assets/Straggler-CZCbnOWP.js +0 -1
  371. package/public/assets/TaskFailures-C57YcLFQ.js +0 -1
  372. package/public/assets/TinyTask-Bi84qU7n.js +0 -1
  373. package/public/assets/UnderBroadcast-DdQV7htE.js +0 -1
  374. package/public/assets/charts-vendor-0VMaLVLN.js +0 -59
  375. package/public/assets/finding-anchor-Cl6_5hfV.js +0 -1
  376. package/public/assets/impact-sort-CssZ6MT2.js +0 -1
  377. package/public/assets/index-2h_TgUAb.css +0 -2
  378. package/public/assets/index-B-03mG7K.js +0 -14
  379. package/public/assets/plan-graph-vendor-BYl-AzN5.js +0 -7
  380. package/public/assets/react-vendor-DEdfrRTJ.js +0 -9
  381. package/public/assets/select-CQjNhprm.js +0 -1
  382. package/public/assets/usePagedRows-Dyl6T8Ok.js +0 -1
  383. package/public/assets/useRecentFiles-XZEnLlal.js +0 -11
  384. package/public/assets/useSortMode-Dhvit8dt.js +0 -1
  385. package/public/docs/assets/chunks/@localSearchIndexroot.u_e5bQz4.js +0 -1
  386. package/public/docs/assets/contributor-guide_architecture_board-widgets.md.YQcj5EI4.js +0 -1
  387. package/public/docs/assets/contributor-guide_architecture_board-widgets.md.YQcj5EI4.lean.js +0 -1
  388. package/public/docs/assets/contributor-guide_architecture_board-widgets_plan-advisor.md.RlzvUmNm.js +0 -1
  389. package/public/docs/assets/contributor-guide_architecture_board-widgets_visual-system-and-templating.md.DYgUpIJB.js +0 -1
  390. package/public/docs/assets/contributor-guide_architecture_board-widgets_visual-system-and-templating.md.DYgUpIJB.lean.js +0 -1
  391. package/public/docs/assets/contributor-guide_architecture_detector-contract.md.wj6Et6UD.js +0 -1
  392. package/public/docs/assets/contributor-guide_architecture_detector-contract_plan-attribution.md.yNex4bys.js +0 -1
  393. package/public/docs/assets/contributor-guide_architecture_detector-contract_plan-attribution.md.yNex4bys.lean.js +0 -1
  394. package/public/docs/assets/contributor-guide_architecture_detector-contract_threshold-tables.md.CKjWG56Y.js +0 -1
  395. package/public/docs/assets/contributor-guide_architecture_drill-down_plan-graph.md.DwOozIEs.js +0 -1
  396. package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.CZv_ZGOp.js +0 -1
  397. package/public/docs/assets/contributor-guide_architecture_impact-estimation_caveats-tuning-and-coverage.md.BUucRc0A.js +0 -1
  398. package/public/docs/assets/contributor-guide_architecture_impact-estimation_stage-union-rollup.md.DdCGtAGu.js +0 -1
  399. package/public/docs/assets/contributor-guide_architecture_impact-estimation_stage-union-rollup.md.DdCGtAGu.lean.js +0 -1
  400. package/public/docs/assets/contributor-guide_architecture_overview.md.C1sSvm9i.js +0 -1
  401. package/public/docs/assets/contributor-guide_architecture_state-and-history_run-comparison-internals.md.CZA0JLtV.js +0 -1
  402. package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.BOZ1azXG.js +0 -1
  403. package/public/docs/assets/contributor-guide_architecture_widget-rendering_findings-tab.md.BQXycK6G.js +0 -1
  404. package/public/docs/assets/contributor-guide_architecture_widget-rendering_findings-tab.md.BQXycK6G.lean.js +0 -1
  405. package/public/docs/assets/contributor-guide_architecture_worker-protocol.md.BAnhi2-d.lean.js +0 -1
  406. package/public/docs/assets/contributor-guide_architecture_worker-protocol_cli-and-mcp.md.DC1YqXBp.js +0 -1
  407. package/public/docs/assets/contributor-guide_architecture_worker-protocol_evidence-report.md.DVGZOWvW.js +0 -1
  408. package/public/docs/assets/contributor-guide_contributing.md.CQMvGwFt.js +0 -1
  409. package/public/docs/assets/contributor-guide_development-setup.md.D9bu4j_c.lean.js +0 -1
  410. package/public/docs/assets/contributor-guide_testing.md.By6XGR9a.js +0 -1
  411. package/public/docs/assets/inter-italic-cyrillic-ext.r48I6akx.woff2 +0 -0
  412. package/public/docs/assets/inter-italic-cyrillic.By2_1cv3.woff2 +0 -0
  413. package/public/docs/assets/inter-italic-greek-ext.1u6EdAuj.woff2 +0 -0
  414. package/public/docs/assets/inter-italic-greek.DJ8dCoTZ.woff2 +0 -0
  415. package/public/docs/assets/inter-italic-latin-ext.CN1xVJS-.woff2 +0 -0
  416. package/public/docs/assets/inter-italic-latin.C2AdPX0b.woff2 +0 -0
  417. package/public/docs/assets/inter-italic-vietnamese.BSbpV94h.woff2 +0 -0
  418. package/public/docs/assets/inter-roman-cyrillic-ext.BBPuwvHQ.woff2 +0 -0
  419. package/public/docs/assets/inter-roman-cyrillic.C5lxZ8CY.woff2 +0 -0
  420. package/public/docs/assets/inter-roman-greek-ext.CqjqNYQ-.woff2 +0 -0
  421. package/public/docs/assets/inter-roman-greek.BBVDIX6e.woff2 +0 -0
  422. package/public/docs/assets/inter-roman-latin-ext.4ZJIpNVo.woff2 +0 -0
  423. package/public/docs/assets/inter-roman-latin.Di8DUHzh.woff2 +0 -0
  424. package/public/docs/assets/inter-roman-vietnamese.BjW4sHH5.woff2 +0 -0
  425. package/public/docs/assets/style.tmDhD3c8.css +0 -1
  426. package/public/docs/assets/user-guide_getting-started_ci-and-automation.md.OUQqw3mI.js +0 -23
  427. package/public/docs/assets/user-guide_getting-started_ci-and-automation.md.OUQqw3mI.lean.js +0 -1
  428. package/public/docs/assets/user-guide_getting-started_reading-the-dashboard.md.Obf3W3__.js +0 -1
  429. package/public/docs/assets/user-guide_mcp-tools_compare-runs.md.Cc_gLoix.lean.js +0 -1
  430. package/public/docs/assets/user-guide_mcp-tools_diagnose-run.md.BP1vnHA5.js +0 -23
  431. package/public/docs/assets/user-guide_run-comparison.md.BUALkohi.js +0 -1
  432. package/public/docs/assets/user-guide_run-comparison.md.BUALkohi.lean.js +0 -1
  433. package/public/docs/assets/user-guide_understanding-findings.md.C9YRruMT.js +0 -1
  434. package/public/docs/assets/user-guide_understanding-findings.md.C9YRruMT.lean.js +0 -1
  435. /package/public/docs/assets/{contributor-guide_architecture_board-widgets_caching-and-memory.md.8bH1HnZp.lean.js → contributor-guide_architecture_board-widgets_caching-and-memory.md.BYDo3ClR.lean.js} +0 -0
  436. /package/public/docs/assets/{contributor-guide_architecture_board-widgets_plan-advisor.md.RlzvUmNm.lean.js → contributor-guide_architecture_board-widgets_plan-advisor.md.ajUAVdbc.lean.js} +0 -0
  437. /package/public/docs/assets/{contributor-guide_architecture_overview.md.C1sSvm9i.lean.js → contributor-guide_architecture_overview.md.nxkpOhgf.lean.js} +0 -0
  438. /package/public/docs/assets/{contributor-guide_architecture_widget-rendering.md.BOZ1azXG.lean.js → contributor-guide_architecture_widget-rendering.md.BrSz8Ezh.lean.js} +0 -0
  439. /package/public/docs/assets/{contributor-guide_architecture_widget-rendering_list-sort-mode.md.BuHKgH0C.lean.js → contributor-guide_architecture_widget-rendering_list-sort-mode.md.37hcZJ7q.lean.js} +0 -0
  440. /package/public/docs/assets/{contributor-guide_architecture_widget-rendering_row-expand-contract.md.ZT89u9f1.lean.js → contributor-guide_architecture_widget-rendering_row-expand-contract.md.mK_VmAWk.lean.js} +0 -0
  441. /package/public/docs/assets/{contributor-guide_architecture_worker-protocol_core-and-event-validation.md.DSVFPpWb.lean.js → contributor-guide_architecture_worker-protocol_core-and-event-validation.md.DVSXn-bb.lean.js} +0 -0
  442. /package/public/docs/assets/{contributor-guide_contributing.md.CQMvGwFt.lean.js → contributor-guide_contributing.md.DBW4drtZ.lean.js} +0 -0
  443. /package/public/docs/assets/{user-guide_getting-started_reading-the-dashboard.md.Obf3W3__.lean.js → user-guide_getting-started_reading-the-dashboard.md.CGaxPX_1.lean.js} +0 -0
  444. /package/public/docs/assets/{user-guide_mcp-tools.md.C65SjWUL.lean.js → user-guide_mcp-tools.md.49r3L2YE.lean.js} +0 -0
  445. /package/public/docs/assets/{user-guide_mcp-tools_get-run-summary.md.DpxZ4ZNp.lean.js → user-guide_mcp-tools_get-run-summary.md.EhaIsele.lean.js} +0 -0
package/public/docs/tuning-reference/bottleneck-memory-utilization.html CHANGED
@@ -6,31 +6,27 @@
6
6
  <title>Memory Utilization | Spark Tuning Reference</title>
7
7
  <meta name="description" content="Detecting and mitigating held-but-idle executor memory: cores sitting idle below allocated slots, dynamic-allocation idle timeouts that reclaim an executor's memory band, and the low-confidence peak-vs-allocated waste estimate.">
8
8
  <meta name="generator" content="VitePress v1.6.4">
9
- <link rel="preload stylesheet" href="/docs/assets/style.tmDhD3c8.css" as="style">
9
+ <link rel="preload stylesheet" href="/docs/assets/style.Cfxqlj0y.css" as="style">
10
10
  <link rel="preload stylesheet" href="/docs/vp-icons.css" as="style">
11
11
 
12
- <script type="module" src="/docs/assets/app.eYTPqTeq.js"></script>
13
- <link rel="preload" href="/docs/assets/inter-roman-latin.Di8DUHzh.woff2" as="font" type="font/woff2" crossorigin="">
14
- <link rel="modulepreload" href="/docs/assets/chunks/theme.DsTNE2pE.js">
12
+ <script type="module" src="/docs/assets/app.1gvzd5Zb.js"></script>
13
+ <link rel="modulepreload" href="/docs/assets/chunks/theme.C5lLqUv7.js">
15
14
  <link rel="modulepreload" href="/docs/assets/chunks/framework.DSg0KOwT.js">
16
15
  <link rel="modulepreload" href="/docs/assets/tuning-reference_bottleneck-memory-utilization.md.B-C0-8h-.lean.js">
17
16
  <script>(()=>{const c=`${`[${["data","shuffle","product","bar"].join("-")}]`}, .VPNav, .VPLocalNav`,a=24,d=document.documentElement,l=t=>{const e=getComputedStyle(t);return e.display==="none"||t.getClientRects().length===0?0:e.position==="fixed"?t.getBoundingClientRect().bottom:e.position==="sticky"?(parseFloat(e.top)||0)+t.offsetHeight:0},u=()=>{try{return location.hash?document.getElementById(decodeURIComponent(location.hash.slice(1))):null}catch{return null}};let r=null;const s=()=>{const t=Array.from(document.querySelectorAll(c),l),e=Math.ceil(Math.max(0,...t))+a;if(e===r)return;const n=r;r=e,d.style.setProperty("--sf-anchor-offset",`${r}px`);const o=n===null?null:u();if(!o)return;const f=(parseFloat(getComputedStyle(o).paddingTop)||0)+2;Math.abs(o.getBoundingClientRect().top-n)<=f&&o.scrollIntoView()},i=()=>{const t=document.createElement("div");t.setAttribute("data-sf-anchor-offset",""),t.setAttribute("aria-hidden","true"),document.body.appendChild(t);const e=window.ResizeObserver?new ResizeObserver(s):null,n=()=>{e&&document.querySelectorAll(c).forEach(o=>e.observe(o)),s()};n(),new MutationObserver(n).observe(document.body,{childList:!0,subtree:!0}),window.addEventListener("resize",s),window.addEventListener("load",s)};document.readyState==="loading"?document.addEventListener("DOMContentLoaded",i,{once:!0}):i()})();</script>
18
17
  <link rel="icon" type="image/svg+xml" href="/docs/favicon.svg">
19
- <link rel="preconnect" href="https://fonts.googleapis.com">
20
- <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin="">
21
- <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Recursive:wght,CASL@400..700,0..1&amp;family=JetBrains+Mono:wght@400;500;600&amp;display=swap">
22
18
  <script id="check-dark-mode">(()=>{const e=localStorage.getItem("vitepress-theme-appearance")||"auto",a=window.matchMedia("(prefers-color-scheme: dark)").matches;(!e||e==="auto"?a:e==="dark")&&document.documentElement.classList.add("dark")})();</script>
23
19
  <script id="check-mac-os">document.documentElement.classList.toggle("mac",/Mac|iPhone|iPod|iPad/i.test(navigator.platform));</script>
24
20
  </head>
25
21
  <body>
26
- <div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Getting Started</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/intro.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Introduction</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Optimization Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/spark-architecture.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Execution Model</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/memory-model.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Management</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/partitioning.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Partitioning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/joins.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Join Optimization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/data-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Data Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/table-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Table Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/caching.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Caching & Persistence</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/pyspark.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>PySpark Specifics</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/aqe.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Adaptive Query Execution</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/cluster-config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cluster Tuning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/anti-patterns.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Anti-Patterns</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Detector Catalog</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-skew.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Skew</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle I/O</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-spill.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory / Disk Spill</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-gc.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>GC Pressure</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-cold-start.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cold Start</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Executor Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-slow-host.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Slow Host</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-failures.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Failures</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-straggler.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Stragglers</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-retry-waste.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Retry Waste</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-tiny-tasks.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Tiny Tasks</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-job-failure-rate.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Job Failure Rate</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-memory-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Duplicate Plan Subtree</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-small-files.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Small Files</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-broadcast-sizing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Broadcast Sizing</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Reference</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/metrics.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Metrics Glossary</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Config Quick-Reference</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_tuning-reference_bottleneck-memory-utilization" data-v-39a288b8><div><h1 id="bottleneck-memory-utilization" tabindex="-1">Memory Utilization <a class="header-anchor" href="#bottleneck-memory-utilization" aria-label="Permalink to &quot;Memory Utilization {#bottleneck-memory-utilization}&quot;">​</a></h1><p><span class="tag">MEM</span></p><h2 id="what-it-is" tabindex="-1">What it is <a class="header-anchor" href="#what-it-is" aria-label="Permalink to &quot;What it is&quot;">​</a></h2><p>An executor is a single JVM process that gets a fixed memory allocation when the application starts, and it holds that whole allocation for its entire lifetime, whether or not it has work to run<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup>. Two things can leave that memory band poorly used: cores sitting idle inside a held executor, and a held executor whose memory drains of active work but is never released. This finding covers both.</p><p>An executor&#39;s core count is its concurrency ceiling. <code>spark.executor.cores</code> sets how many tasks it can run at once, so <code>--executor-cores 5</code> caps that executor at five concurrent tasks<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup>. The scheduler turns cores into task slots from <code>spark.executor.cores</code> and <code>spark.task.cpus</code> (minimum 1), with <code>spark.task.cpus</code> defaulting to one core per task<sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup>. Cores go idle whenever there are fewer runnable tasks than slots: a stage with fewer partitions than the total slots across executors leaves slots empty, and so does the tail of a stage where a <a href="./bottleneck-straggler.html">straggler</a> or two keep running after their peers finish. Raising <code>spark.task.cpus</code> above 1 has the same effect from the other side, since each task then reserves several cores and fewer run side by side. Through all of it the JVM keeps its fixed heap<sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup>.</p><h2 id="how-it-s-detected" tabindex="-1">How it&#39;s detected <a class="header-anchor" href="#how-it-s-detected" aria-label="Permalink to &quot;How it&#39;s detected&quot;">​</a></h2><table tabindex="0"><thead><tr><th>Rule</th><th>Signal</th><th>Status</th></tr></thead><tbody><tr><td>idleCores</td><td>Task parallelism below allocated cores while the executor is held</td><td>Validated</td></tr><tr><td>memoryBand</td><td>A whole executor&#39;s memory band held with little or no active work (dynamic-allocation idle timeouts)</td><td>Validated</td></tr><tr><td>wasteModel</td><td>Peak used memory well below allocated memory (unused headroom)</td><td>Experimental</td></tr></tbody></table><h2 id="why-it-matters" tabindex="-1">Why it matters <a class="header-anchor" href="#why-it-matters" aria-label="Permalink to &quot;Why it matters&quot;">​</a></h2><p>A held executor is allocation you pay for regardless of how busy it is. On YARN that is the memory YARN grants the container; on Kubernetes it is the pod memory limit<sup class="footnote-ref"><a href="#fn1" id="fnref1:2">[1:2]</a></sup>. When cores idle or a whole executor lingers with nothing to do, that fixed band is reserved without returning work, so the cost lands whether or not tasks are running.</p><h2 id="how-to-fix-it" tabindex="-1">How to fix it <a class="header-anchor" href="#how-to-fix-it" aria-label="Permalink to &quot;How to fix it&quot;">​</a></h2><p>Keep executors from being oversized in the first place. The guidance is against packing every core of a node into one fat executor: assigning all 16 cores of a node to a single executor hurts HDFS throughput and drives excessive <a href="./bottleneck-gc.html">garbage collection</a>, so aim for a balance between tiny (one core per executor) and fat (one executor per node) sizing<sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup>. On YARN, leave cores for the OS and Hadoop daemons instead of handing 100% of a node to Spark containers<sup class="footnote-ref"><a href="#fn2" id="fnref2:1">[2:1]</a></sup>.</p><p>For the held memory band, lean on <a href="./cluster-config.html">dynamic allocation</a> to reclaim executors once the work drains. It requests executors when tasks back up and frees them when they go idle<sup class="footnote-ref"><a href="#fn2" id="fnref2:2">[2:2]</a></sup>. Executors are added in rounds once tasks have been pending for <code>spark.dynamicAllocation.schedulerBacklogTimeout</code> (default 1s), then again every <code>spark.dynamicAllocation.sustainedSchedulerBacklogTimeout</code> while the backlog holds<sup class="footnote-ref"><a href="#fn5" id="fnref5">[5]</a></sup>. On the release side, an executor is removed after it has been idle longer than <code>spark.dynamicAllocation.executorIdleTimeout</code><sup class="footnote-ref"><a href="#fn5" id="fnref5:1">[5:1]</a></sup>.</p><p>Cached data is the trap here. By default an executor holding <a href="./caching.html">cached blocks</a> is never removed, governed by <code>spark.dynamicAllocation.cachedExecutorIdleTimeout</code>, whose default is infinity<sup class="footnote-ref"><a href="#fn3" id="fnref3:1">[3:1]</a></sup>. Such an executor keeps its full memory band indefinitely with no active tasks unless you set that timeout to a finite value, or turn on <code>spark.shuffle.service.fetch.rdd.enabled</code> so executors holding only disk-persisted blocks are treated as idle after <code>spark.dynamicAllocation.executorIdleTimeout</code> and released<sup class="footnote-ref"><a href="#fn5" id="fnref5:2">[5:2]</a></sup>. To avoid holding barely-used executors at all, lower <code>spark.dynamicAllocation.executorAllocationRatio</code> (default 1.0, full parallelism) toward 0.5, since with small tasks full-parallelism allocation can request executors that never do any work<sup class="footnote-ref"><a href="#fn3" id="fnref3:2">[3:2]</a></sup>.</p><div class="language-properties vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">properties</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Balanced executor sizing, not one fat executor per node</span></span>
22
+ <div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><!--[--><!--[--><img class="VPImage dark logo" src="/docs/logo-dark.svg" alt data-v-8426fc1a><!--]--><!--[--><img class="VPImage light logo" src="/docs/logo-light.svg" alt data-v-8426fc1a><!--]--><!--]--><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/.." target="_self" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Open app</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Getting Started</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/intro.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Introduction</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Optimization Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/spark-architecture.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Execution Model</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/memory-model.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Management</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/partitioning.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Partitioning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/joins.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Join Optimization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/data-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Data Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/table-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Table Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/caching.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Caching & Persistence</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/pyspark.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>PySpark Specifics</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/aqe.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Adaptive Query Execution</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/cluster-config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cluster Tuning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/anti-patterns.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Anti-Patterns</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Detector Catalog</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-skew.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Skew</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle I/O</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-spill.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory / Disk Spill</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-gc.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>GC Pressure</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-cold-start.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cold Start</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Executor Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-slow-host.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Slow Host</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-failures.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Failures</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-straggler.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Stragglers</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-retry-waste.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Retry Waste</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-tiny-tasks.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Tiny Tasks</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-job-failure-rate.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Job Failure Rate</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-memory-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Duplicate Plan Subtree</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-small-files.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Small Files</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-broadcast-sizing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Broadcast Sizing</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Reference</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/metrics.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Metrics Glossary</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Config Quick-Reference</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_tuning-reference_bottleneck-memory-utilization" data-v-39a288b8><div><h1 id="bottleneck-memory-utilization" tabindex="-1">Memory Utilization <a class="header-anchor" href="#bottleneck-memory-utilization" aria-label="Permalink to &quot;Memory Utilization {#bottleneck-memory-utilization}&quot;">​</a></h1><p><span class="tag">MEM</span></p><h2 id="what-it-is" tabindex="-1">What it is <a class="header-anchor" href="#what-it-is" aria-label="Permalink to &quot;What it is&quot;">​</a></h2><p>An executor is a single JVM process that gets a fixed memory allocation when the application starts, and it holds that whole allocation for its entire lifetime, whether or not it has work to run<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup>. Two things can leave that memory band poorly used: cores sitting idle inside a held executor, and a held executor whose memory drains of active work but is never released. This finding covers both.</p><p>An executor&#39;s core count is its concurrency ceiling. <code>spark.executor.cores</code> sets how many tasks it can run at once, so <code>--executor-cores 5</code> caps that executor at five concurrent tasks<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup>. The scheduler turns cores into task slots from <code>spark.executor.cores</code> and <code>spark.task.cpus</code> (minimum 1), with <code>spark.task.cpus</code> defaulting to one core per task<sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup>. Cores go idle whenever there are fewer runnable tasks than slots: a stage with fewer partitions than the total slots across executors leaves slots empty, and so does the tail of a stage where a <a href="./bottleneck-straggler.html">straggler</a> or two keep running after their peers finish. Raising <code>spark.task.cpus</code> above 1 has the same effect from the other side, since each task then reserves several cores and fewer run side by side. Through all of it the JVM keeps its fixed heap<sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup>.</p><h2 id="how-it-s-detected" tabindex="-1">How it&#39;s detected <a class="header-anchor" href="#how-it-s-detected" aria-label="Permalink to &quot;How it&#39;s detected&quot;">​</a></h2><table tabindex="0"><thead><tr><th>Rule</th><th>Signal</th><th>Status</th></tr></thead><tbody><tr><td>idleCores</td><td>Task parallelism below allocated cores while the executor is held</td><td>Validated</td></tr><tr><td>memoryBand</td><td>A whole executor&#39;s memory band held with little or no active work (dynamic-allocation idle timeouts)</td><td>Validated</td></tr><tr><td>wasteModel</td><td>Peak used memory well below allocated memory (unused headroom)</td><td>Experimental</td></tr></tbody></table><h2 id="why-it-matters" tabindex="-1">Why it matters <a class="header-anchor" href="#why-it-matters" aria-label="Permalink to &quot;Why it matters&quot;">​</a></h2><p>A held executor is allocation you pay for regardless of how busy it is. On YARN that is the memory YARN grants the container; on Kubernetes it is the pod memory limit<sup class="footnote-ref"><a href="#fn1" id="fnref1:2">[1:2]</a></sup>. When cores idle or a whole executor lingers with nothing to do, that fixed band is reserved without returning work, so the cost lands whether or not tasks are running.</p><h2 id="how-to-fix-it" tabindex="-1">How to fix it <a class="header-anchor" href="#how-to-fix-it" aria-label="Permalink to &quot;How to fix it&quot;">​</a></h2><p>Keep executors from being oversized in the first place. The guidance is against packing every core of a node into one fat executor: assigning all 16 cores of a node to a single executor hurts HDFS throughput and drives excessive <a href="./bottleneck-gc.html">garbage collection</a>, so aim for a balance between tiny (one core per executor) and fat (one executor per node) sizing<sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup>. On YARN, leave cores for the OS and Hadoop daemons instead of handing 100% of a node to Spark containers<sup class="footnote-ref"><a href="#fn2" id="fnref2:1">[2:1]</a></sup>.</p><p>For the held memory band, lean on <a href="./cluster-config.html">dynamic allocation</a> to reclaim executors once the work drains. It requests executors when tasks back up and frees them when they go idle<sup class="footnote-ref"><a href="#fn2" id="fnref2:2">[2:2]</a></sup>. Executors are added in rounds once tasks have been pending for <code>spark.dynamicAllocation.schedulerBacklogTimeout</code> (default 1s), then again every <code>spark.dynamicAllocation.sustainedSchedulerBacklogTimeout</code> while the backlog holds<sup class="footnote-ref"><a href="#fn5" id="fnref5">[5]</a></sup>. On the release side, an executor is removed after it has been idle longer than <code>spark.dynamicAllocation.executorIdleTimeout</code><sup class="footnote-ref"><a href="#fn5" id="fnref5:1">[5:1]</a></sup>.</p><p>Cached data is the trap here. By default an executor holding <a href="./caching.html">cached blocks</a> is never removed, governed by <code>spark.dynamicAllocation.cachedExecutorIdleTimeout</code>, whose default is infinity<sup class="footnote-ref"><a href="#fn3" id="fnref3:1">[3:1]</a></sup>. Such an executor keeps its full memory band indefinitely with no active tasks unless you set that timeout to a finite value, or turn on <code>spark.shuffle.service.fetch.rdd.enabled</code> so executors holding only disk-persisted blocks are treated as idle after <code>spark.dynamicAllocation.executorIdleTimeout</code> and released<sup class="footnote-ref"><a href="#fn5" id="fnref5:2">[5:2]</a></sup>. To avoid holding barely-used executors at all, lower <code>spark.dynamicAllocation.executorAllocationRatio</code> (default 1.0, full parallelism) toward 0.5, since with small tasks full-parallelism allocation can request executors that never do any work<sup class="footnote-ref"><a href="#fn3" id="fnref3:2">[3:2]</a></sup>.</p><div class="language-properties vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">properties</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Balanced executor sizing, not one fat executor per node</span></span>
27
23
  <span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.executor.cores</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=5</span></span>
28
24
  <span class="line"></span>
29
25
  <span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Reclaim idle executors; give cached holders a finite timeout</span></span>
30
26
  <span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.dynamicAllocation.enabled</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=true</span></span>
31
27
  <span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.dynamicAllocation.cachedExecutorIdleTimeout</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=300s</span></span>
32
28
  <span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.dynamicAllocation.executorAllocationRatio</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=0.5</span></span></code></pre></div><h3 id="reading-the-wasted-memory-estimate-experimental" tabindex="-1">Reading the wasted-memory estimate <span class="tag">EXPERIMENTAL</span> <a class="header-anchor" href="#reading-the-wasted-memory-estimate-experimental" aria-label="Permalink to &quot;Reading the wasted-memory estimate &lt;span class=&quot;tag&quot;&gt;EXPERIMENTAL&lt;/span&gt;&quot;">​</a></h3><p>The wasteModel rule estimates unused (&quot;wasted&quot;) allocated memory from the gap between an executor&#39;s peak used memory and its allocated total. Treat it as a rough buffer heuristic, not a measurement. Peak used memory is a high-water mark rather than a ceiling: Spark records it under <code>peakMemoryMetrics.*</code>, and each figure is the maximum its pool ever reached, so peak sits at or below the allocated total by construction and the space between them is the executor&#39;s unused headroom<sup class="footnote-ref"><a href="#fn6" id="fnref6">[6]</a></sup>. Turning that headroom into a wasted number stays approximate for concrete reasons. The peak heap figure counts garbage, since <code>JVMHeapMemory</code> is the peak used heap including &quot;the amount of memory occupied by both live objects and garbage objects that have not been collected,&quot; so it overstates the live footprint<sup class="footnote-ref"><a href="#fn6" id="fnref6:1">[6:1]</a></sup>. The region boundaries also move: <code>totalOnHeapStorageMemory</code> and <code>totalOffHeapStorageMemory</code> &quot;can vary over time, depending on the MemoryManager implementation,&quot; so there is no single fixed allocated-to-storage value to subtract a peak from<sup class="footnote-ref"><a href="#fn6" id="fnref6:2">[6:2]</a></sup>. Peak metrics only reach the event log when <code>spark.eventLog.logStageExecutorMetrics</code> is true<sup class="footnote-ref"><a href="#fn6" id="fnref6:3">[6:3]</a></sup>, so without that setting there is nothing to estimate from. Use the number as a hint that an executor may be oversized, then confirm against sizing before acting on it.</p><h2 id="confidence" tabindex="-1">Confidence <a class="header-anchor" href="#confidence" aria-label="Permalink to &quot;Confidence&quot;">​</a></h2><p>The idleCores and memoryBand rules are validated: they rest on documented Spark concurrency and dynamic-allocation behavior. The wasteModel estimate is low-confidence and experimental. It is a buffer heuristic derived from peak-versus-allocated sampling, not an exact accounting of unused memory, so it should steer investigation rather than settle it.</p><h2 id="limitations-false-positive-risk" tabindex="-1">Limitations / false-positive risk <a class="header-anchor" href="#limitations-false-positive-risk" aria-label="Permalink to &quot;Limitations / false-positive risk&quot;">​</a></h2><p>Idle cores are not always waste. The tail of a stage legitimately leaves slots empty while a straggler or two finish<sup class="footnote-ref"><a href="#fn3" id="fnref3:3">[3:3]</a></sup>, so a snapshot of low parallelism can reflect a normal straggler tail rather than chronic under-utilization. The wasted-memory estimate depends on peak-memory sampling and is only as good as that sampling: the peak includes uncollected garbage, the managed-region boundaries shift under the MemoryManager, and the figures are absent entirely unless <code>spark.eventLog.logStageExecutorMetrics</code> is enabled<sup class="footnote-ref"><a href="#fn6" id="fnref6:4">[6:4]</a></sup>.</p><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><a href="https://luminousmen.com/post/dive-into-spark-memory" target="_blank" rel="noreferrer">Dive into Spark memory</a> <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a> <a href="#fnref1:2" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><a href="https://blog.cloudera.com/how-to-tune-your-apache-spark-jobs-part-2/" target="_blank" rel="noreferrer">How to Tune Your Apache Spark Jobs (Part 2)</a> <a href="#fnref2" class="footnote-backref">↩︎</a> <a href="#fnref2:1" class="footnote-backref">↩︎</a> <a href="#fnref2:2" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration — Spark</a> <a href="#fnref3" class="footnote-backref">↩︎</a> <a href="#fnref3:1" class="footnote-backref">↩︎</a> <a href="#fnref3:2" class="footnote-backref">↩︎</a> <a href="#fnref3:3" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><a href="https://raw.githubusercontent.com/spoddutur/spark-notes/master/distribution_of_executors_cores_and_memory_for_spark_application.md" target="_blank" rel="noreferrer">Distribution of executors, cores and memory for a Spark application</a> <a href="#fnref4" class="footnote-backref">↩︎</a></p></li><li id="fn5" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/job-scheduling.html" target="_blank" rel="noreferrer">Job Scheduling — Spark</a> <a href="#fnref5" class="footnote-backref">↩︎</a> <a href="#fnref5:1" class="footnote-backref">↩︎</a> <a href="#fnref5:2" class="footnote-backref">↩︎</a></p></li><li id="fn6" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/monitoring.html" target="_blank" rel="noreferrer">Monitoring — Spark</a> <a href="#fnref6" class="footnote-backref">↩︎</a> <a href="#fnref6:1" class="footnote-backref">↩︎</a> <a href="#fnref6:2" class="footnote-backref">↩︎</a> <a href="#fnref6:3" class="footnote-backref">↩︎</a> <a href="#fnref6:4" class="footnote-backref">↩︎</a></p></li></ol></section></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/tuning-reference/bottleneck-job-failure-rate.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>Job Failure Rate</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>Duplicate Plan Subtree</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
33
- <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"YQcj5EI4\",\"contributor-guide_architecture_board-widgets_caching-and-memory.md\":\"8bH1HnZp\",\"contributor-guide_architecture_board-widgets_plan-advisor.md\":\"RlzvUmNm\",\"contributor-guide_architecture_board-widgets_visual-system-and-templating.md\":\"DYgUpIJB\",\"contributor-guide_architecture_detector-contract.md\":\"wj6Et6UD\",\"contributor-guide_architecture_detector-contract_plan-attribution.md\":\"yNex4bys\",\"contributor-guide_architecture_detector-contract_threshold-tables.md\":\"CKjWG56Y\",\"contributor-guide_architecture_drill-down.md\":\"CqZPrdo9\",\"contributor-guide_architecture_drill-down_plan-graph.md\":\"DwOozIEs\",\"contributor-guide_architecture_impact-estimation.md\":\"CZv_ZGOp\",\"contributor-guide_architecture_impact-estimation_caveats-tuning-and-coverage.md\":\"BUucRc0A\",\"contributor-guide_architecture_impact-estimation_stage-union-rollup.md\":\"DdCGtAGu\",\"contributor-guide_architecture_index.md\":\"tymxf1Fh\",\"contributor-guide_architecture_overview.md\":\"C1sSvm9i\",\"contributor-guide_architecture_state-and-history.md\":\"DyCBdjuM\",\"contributor-guide_architecture_state-and-history_history-server-intake.md\":\"C298hrHZ\",\"contributor-guide_architecture_state-and-history_run-comparison-internals.md\":\"CZA0JLtV\",\"contributor-guide_architecture_widget-rendering.md\":\"BOZ1azXG\",\"contributor-guide_architecture_widget-rendering_findings-tab.md\":\"BQXycK6G\",\"contributor-guide_architecture_widget-rendering_investigation-routing.md\":\"BGdiyElu\",\"contributor-guide_architecture_widget-rendering_list-sort-mode.md\":\"BuHKgH0C\",\"contributor-guide_architecture_widget-rendering_reference-section.md\":\"DWqC80bN\",\"contributor-guide_architecture_widget-rendering_row-expand-contract.md\":\"ZT89u9f1\",\"contributor-guide_architecture_worker-protocol.md\":\"BAnhi2-d\",\"contributor-guide_architecture_worker-protocol_cli-and-mcp.md\":\"DC1YqXBp\",\"contributor-guide_architecture_worker-protocol_core-and-event-validation.md\":\"DSVFPpWb\",\"contributor-guide_architecture_worker-protocol_evidence-availability.md\":\"CjimIDBY\",\"contributor-guide_architecture_worker-protocol_evidence-report.md\":\"DVGZOWvW\",\"contributor-guide_contributing.md\":\"CQMvGwFt\",\"contributor-guide_development-setup.md\":\"D9bu4j_c\",\"contributor-guide_testing.md\":\"By6XGR9a\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"D08Uaod7\",\"tuning-reference_aqe.md\":\"C1Ae6A8u\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"6wjKuJNE\",\"tuning-reference_bottleneck-cold-start.md\":\"BVQbsuHP\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"DtaYnaiF\",\"tuning-reference_bottleneck-failures.md\":\"Z1Tdmdd3\",\"tuning-reference_bottleneck-gc.md\":\"BCnXLf1n\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"CcvQj2g9\",\"tuning-reference_bottleneck-memory-utilization.md\":\"B-C0-8h-\",\"tuning-reference_bottleneck-retry-waste.md\":\"DCnhWiqj\",\"tuning-reference_bottleneck-shuffle.md\":\"B5OMJCd3\",\"tuning-reference_bottleneck-skew.md\":\"Cvo6NQm_\",\"tuning-reference_bottleneck-slow-host.md\":\"ja5qr3wq\",\"tuning-reference_bottleneck-small-files.md\":\"Crjtb0i2\",\"tuning-reference_bottleneck-spill.md\":\"Bl8KtSRg\",\"tuning-reference_bottleneck-straggler.md\":\"B0VN8reT\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"DQCftGhu\",\"tuning-reference_bottleneck-utilization.md\":\"BXDl30f9\",\"tuning-reference_caching.md\":\"CcaiMn_7\",\"tuning-reference_cluster-config.md\":\"DbWG1yZq\",\"tuning-reference_config.md\":\"YUqSpwFB\",\"tuning-reference_data-formats.md\":\"C5DX5xL4\",\"tuning-reference_index.md\":\"ZVIpQ6Dk\",\"tuning-reference_intro.md\":\"BSSaDlqx\",\"tuning-reference_joins.md\":\"BVhNCDnE\",\"tuning-reference_memory-model.md\":\"CfjFdgbl\",\"tuning-reference_metrics.md\":\"C64VNEL4\",\"tuning-reference_partitioning.md\":\"CtLK0xsS\",\"tuning-reference_pyspark.md\":\"D-92XgF8\",\"tuning-reference_shuffle.md\":\"Iq2v1oHJ\",\"tuning-reference_spark-architecture.md\":\"_qwnE_Sc\",\"tuning-reference_table-formats.md\":\"BSYEK-eN\",\"user-guide_alternative-log-retrieval.md\":\"CJpYvHGo\",\"user-guide_alternative-log-retrieval_managed-platforms.md\":\"DvDwHNtC\",\"user-guide_alternative-log-retrieval_ssh-bastion.md\":\"Gszzlplv\",\"user-guide_getting-started.md\":\"DYHHHkQR\",\"user-guide_getting-started_ci-and-automation.md\":\"OUQqw3mI\",\"user-guide_getting-started_reading-the-dashboard.md\":\"Obf3W3__\",\"user-guide_getting-started_regression-metric-keys.md\":\"B_gzr2Mw\",\"user-guide_getting-started_tuning-thresholds.md\":\"B_MyjyZm\",\"user-guide_mcp-tools.md\":\"C65SjWUL\",\"user-guide_mcp-tools_compare-runs.md\":\"Cc_gLoix\",\"user-guide_mcp-tools_diagnose-run.md\":\"BP1vnHA5\",\"user-guide_mcp-tools_evaluate-budgets.md\":\"BB0nDY3Z\",\"user-guide_mcp-tools_get-finding-documentation.md\":\"BtTTHNV6\",\"user-guide_mcp-tools_get-finding-evidence.md\":\"rKyu4wts\",\"user-guide_mcp-tools_get-reference-doc.md\":\"Yac4tH3u\",\"user-guide_mcp-tools_get-run-summary.md\":\"DpxZ4ZNp\",\"user-guide_mcp-tools_list-runs.md\":\"DpQPeCM3\",\"user-guide_run-comparison.md\":\"BUALkohi\",\"user-guide_run-comparison_how-stages-are-matched.md\":\"DSXGu2tA\",\"user-guide_understanding-findings.md\":\"C9YRruMT\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\",\"collapsed\":true,\"items\":[{\"text\":\"Reading the dashboard\",\"link\":\"/user-guide/getting-started/reading-the-dashboard\"},{\"text\":\"CI and automation\",\"link\":\"/user-guide/getting-started/ci-and-automation\"},{\"text\":\"Tuning detector thresholds\",\"link\":\"/user-guide/getting-started/tuning-thresholds\"},{\"text\":\"Regression metric keys\",\"link\":\"/user-guide/getting-started/regression-metric-keys\"}]},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\",\"collapsed\":true,\"items\":[{\"text\":\"How stages are matched\",\"link\":\"/user-guide/run-comparison/how-stages-are-matched\"}]},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\",\"collapsed\":true,\"items\":[{\"text\":\"diagnose_run\",\"link\":\"/user-guide/mcp-tools/diagnose-run\"},{\"text\":\"get_run_summary\",\"link\":\"/user-guide/mcp-tools/get-run-summary\"},{\"text\":\"evaluate_budgets\",\"link\":\"/user-guide/mcp-tools/evaluate-budgets\"},{\"text\":\"compare_runs\",\"link\":\"/user-guide/mcp-tools/compare-runs\"},{\"text\":\"get_finding_evidence\",\"link\":\"/user-guide/mcp-tools/get-finding-evidence\"},{\"text\":\"get_finding_documentation\",\"link\":\"/user-guide/mcp-tools/get-finding-documentation\"},{\"text\":\"get_reference_doc\",\"link\":\"/user-guide/mcp-tools/get-reference-doc\"},{\"text\":\"list_runs\",\"link\":\"/user-guide/mcp-tools/list-runs\"}]},{\"text\":\"Finding your event log\",\"link\":\"/user-guide/alternative-log-retrieval\",\"collapsed\":true,\"items\":[{\"text\":\"Managed platforms\",\"link\":\"/user-guide/alternative-log-retrieval/managed-platforms\"},{\"text\":\"Behind an SSH bastion\",\"link\":\"/user-guide/alternative-log-retrieval/ssh-bastion\"}]}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\",\"collapsed\":true,\"items\":[{\"text\":\"Evidence availability\",\"link\":\"/contributor-guide/architecture/worker-protocol/evidence-availability\"},{\"text\":\"Evidence report and finding identity\",\"link\":\"/contributor-guide/architecture/worker-protocol/evidence-report\"},{\"text\":\"CLI and MCP server\",\"link\":\"/contributor-guide/architecture/worker-protocol/cli-and-mcp\"},{\"text\":\"Core and event validation\",\"link\":\"/contributor-guide/architecture/worker-protocol/core-and-event-validation\"}]},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\",\"collapsed\":true,\"items\":[{\"text\":\"Run comparison internals\",\"link\":\"/contributor-guide/architecture/state-and-history/run-comparison-internals\"},{\"text\":\"History Server intake\",\"link\":\"/contributor-guide/architecture/state-and-history/history-server-intake\"}]},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\",\"collapsed\":true,\"items\":[{\"text\":\"Plan attribution\",\"link\":\"/contributor-guide/architecture/detector-contract/plan-attribution\"},{\"text\":\"Threshold tables\",\"link\":\"/contributor-guide/architecture/detector-contract/threshold-tables\"}]},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\",\"collapsed\":true,\"items\":[{\"text\":\"Stage union rollup\",\"link\":\"/contributor-guide/architecture/impact-estimation/stage-union-rollup\"},{\"text\":\"Caveats, tuning and coverage\",\"link\":\"/contributor-guide/architecture/impact-estimation/caveats-tuning-and-coverage\"}]},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\",\"collapsed\":true,\"items\":[{\"text\":\"Findings tab\",\"link\":\"/contributor-guide/architecture/widget-rendering/findings-tab\"},{\"text\":\"List sort mode\",\"link\":\"/contributor-guide/architecture/widget-rendering/list-sort-mode\"},{\"text\":\"Row/expand contract\",\"link\":\"/contributor-guide/architecture/widget-rendering/row-expand-contract\"},{\"text\":\"ReferenceSection\",\"link\":\"/contributor-guide/architecture/widget-rendering/reference-section\"},{\"text\":\"Investigation routing\",\"link\":\"/contributor-guide/architecture/widget-rendering/investigation-routing\"}]},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\",\"collapsed\":true,\"items\":[{\"text\":\"Plan Advisor\",\"link\":\"/contributor-guide/architecture/board-widgets/plan-advisor\"},{\"text\":\"Caching and memory widgets\",\"link\":\"/contributor-guide/architecture/board-widgets/caching-and-memory\"},{\"text\":\"Visual system and templating\",\"link\":\"/contributor-guide/architecture/board-widgets/visual-system-and-templating\"}]},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\",\"collapsed\":true,\"items\":[{\"text\":\"Plan graph\",\"link\":\"/contributor-guide/architecture/drill-down/plan-graph\"}]}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":{\"selector\":\"[data-sf-anchor-offset]\",\"padding\":0},\"cleanUrls\":false}");</script>
29
+ <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"CqKM_jEG\",\"contributor-guide_architecture_board-widgets_caching-and-memory.md\":\"BYDo3ClR\",\"contributor-guide_architecture_board-widgets_plan-advisor.md\":\"ajUAVdbc\",\"contributor-guide_architecture_board-widgets_visual-system-and-templating.md\":\"WqY74T0T\",\"contributor-guide_architecture_detector-contract.md\":\"CHHgywoP\",\"contributor-guide_architecture_detector-contract_plan-attribution.md\":\"DIz3_oz9\",\"contributor-guide_architecture_detector-contract_threshold-tables.md\":\"WH4tRHPL\",\"contributor-guide_architecture_drill-down.md\":\"yB-Hz2XT\",\"contributor-guide_architecture_drill-down_plan-graph.md\":\"BQZE04sw\",\"contributor-guide_architecture_impact-estimation.md\":\"K71YB-Rr\",\"contributor-guide_architecture_impact-estimation_caveats-tuning-and-coverage.md\":\"DLCAffwn\",\"contributor-guide_architecture_impact-estimation_stage-union-rollup.md\":\"JFrzDMwo\",\"contributor-guide_architecture_index.md\":\"tymxf1Fh\",\"contributor-guide_architecture_overview.md\":\"nxkpOhgf\",\"contributor-guide_architecture_state-and-history.md\":\"DyCBdjuM\",\"contributor-guide_architecture_state-and-history_history-server-intake.md\":\"C298hrHZ\",\"contributor-guide_architecture_state-and-history_run-comparison-internals.md\":\"U-9KMDFK\",\"contributor-guide_architecture_widget-rendering.md\":\"BrSz8Ezh\",\"contributor-guide_architecture_widget-rendering_findings-tab.md\":\"B7rjSF_h\",\"contributor-guide_architecture_widget-rendering_investigation-routing.md\":\"BGdiyElu\",\"contributor-guide_architecture_widget-rendering_list-sort-mode.md\":\"37hcZJ7q\",\"contributor-guide_architecture_widget-rendering_reference-section.md\":\"DWqC80bN\",\"contributor-guide_architecture_widget-rendering_row-expand-contract.md\":\"mK_VmAWk\",\"contributor-guide_architecture_worker-protocol.md\":\"Br6OE7RW\",\"contributor-guide_architecture_worker-protocol_cli-and-mcp.md\":\"IVyaXrGe\",\"contributor-guide_architecture_worker-protocol_core-and-event-validation.md\":\"DVSXn-bb\",\"contributor-guide_architecture_worker-protocol_evidence-availability.md\":\"D3S5cyOW\",\"contributor-guide_architecture_worker-protocol_evidence-report.md\":\"CtmCU57I\",\"contributor-guide_contributing.md\":\"DBW4drtZ\",\"contributor-guide_development-setup.md\":\"B21ZTDhn\",\"contributor-guide_testing.md\":\"DpkozNoG\",\"index.md\":\"lMm7SUMz\",\"tuning-reference_anti-patterns.md\":\"D08Uaod7\",\"tuning-reference_aqe.md\":\"C1Ae6A8u\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"6wjKuJNE\",\"tuning-reference_bottleneck-cold-start.md\":\"BVQbsuHP\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"DtaYnaiF\",\"tuning-reference_bottleneck-failures.md\":\"Z1Tdmdd3\",\"tuning-reference_bottleneck-gc.md\":\"BCnXLf1n\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"CcvQj2g9\",\"tuning-reference_bottleneck-memory-utilization.md\":\"B-C0-8h-\",\"tuning-reference_bottleneck-retry-waste.md\":\"DCnhWiqj\",\"tuning-reference_bottleneck-shuffle.md\":\"B5OMJCd3\",\"tuning-reference_bottleneck-skew.md\":\"Cvo6NQm_\",\"tuning-reference_bottleneck-slow-host.md\":\"ja5qr3wq\",\"tuning-reference_bottleneck-small-files.md\":\"Crjtb0i2\",\"tuning-reference_bottleneck-spill.md\":\"Bl8KtSRg\",\"tuning-reference_bottleneck-straggler.md\":\"B0VN8reT\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"DQCftGhu\",\"tuning-reference_bottleneck-utilization.md\":\"BXDl30f9\",\"tuning-reference_caching.md\":\"CcaiMn_7\",\"tuning-reference_cluster-config.md\":\"DbWG1yZq\",\"tuning-reference_config.md\":\"YUqSpwFB\",\"tuning-reference_data-formats.md\":\"C5DX5xL4\",\"tuning-reference_index.md\":\"ZVIpQ6Dk\",\"tuning-reference_intro.md\":\"BSSaDlqx\",\"tuning-reference_joins.md\":\"BVhNCDnE\",\"tuning-reference_memory-model.md\":\"CfjFdgbl\",\"tuning-reference_metrics.md\":\"C64VNEL4\",\"tuning-reference_partitioning.md\":\"CtLK0xsS\",\"tuning-reference_pyspark.md\":\"D-92XgF8\",\"tuning-reference_shuffle.md\":\"Iq2v1oHJ\",\"tuning-reference_spark-architecture.md\":\"_qwnE_Sc\",\"tuning-reference_table-formats.md\":\"BSYEK-eN\",\"user-guide_alternative-log-retrieval.md\":\"CJpYvHGo\",\"user-guide_alternative-log-retrieval_managed-platforms.md\":\"DvDwHNtC\",\"user-guide_alternative-log-retrieval_ssh-bastion.md\":\"Gszzlplv\",\"user-guide_getting-started.md\":\"DYHHHkQR\",\"user-guide_getting-started_ci-and-automation.md\":\"C0pcQoNc\",\"user-guide_getting-started_reading-the-dashboard.md\":\"CGaxPX_1\",\"user-guide_getting-started_regression-metric-keys.md\":\"B_gzr2Mw\",\"user-guide_getting-started_tuning-thresholds.md\":\"BGVE-wrc\",\"user-guide_mcp-tools.md\":\"49r3L2YE\",\"user-guide_mcp-tools_compare-runs.md\":\"BQ_kIptC\",\"user-guide_mcp-tools_diagnose-run.md\":\"azCZXkH8\",\"user-guide_mcp-tools_evaluate-budgets.md\":\"dpQpRQ7z\",\"user-guide_mcp-tools_get-finding-documentation.md\":\"BtTTHNV6\",\"user-guide_mcp-tools_get-finding-evidence.md\":\"CH6FkNLX\",\"user-guide_mcp-tools_get-reference-doc.md\":\"Yac4tH3u\",\"user-guide_mcp-tools_get-run-summary.md\":\"EhaIsele\",\"user-guide_mcp-tools_list-runs.md\":\"DpQPeCM3\",\"user-guide_run-comparison.md\":\"B_LOkdBf\",\"user-guide_run-comparison_how-stages-are-matched.md\":\"DSXGu2tA\",\"user-guide_understanding-findings.md\":\"C4riKCKl\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":{\"light\":\"/logo-light.svg\",\"dark\":\"/logo-dark.svg\"},\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"},{\"text\":\"Open app\",\"link\":\"/..\",\"target\":\"_self\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\",\"collapsed\":true,\"items\":[{\"text\":\"Reading the dashboard\",\"link\":\"/user-guide/getting-started/reading-the-dashboard\"},{\"text\":\"CI and automation\",\"link\":\"/user-guide/getting-started/ci-and-automation\"},{\"text\":\"Tuning detector thresholds\",\"link\":\"/user-guide/getting-started/tuning-thresholds\"},{\"text\":\"Regression metric keys\",\"link\":\"/user-guide/getting-started/regression-metric-keys\"}]},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\",\"collapsed\":true,\"items\":[{\"text\":\"How stages are matched\",\"link\":\"/user-guide/run-comparison/how-stages-are-matched\"}]},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\",\"collapsed\":true,\"items\":[{\"text\":\"diagnose_run\",\"link\":\"/user-guide/mcp-tools/diagnose-run\"},{\"text\":\"get_run_summary\",\"link\":\"/user-guide/mcp-tools/get-run-summary\"},{\"text\":\"evaluate_budgets\",\"link\":\"/user-guide/mcp-tools/evaluate-budgets\"},{\"text\":\"compare_runs\",\"link\":\"/user-guide/mcp-tools/compare-runs\"},{\"text\":\"get_finding_evidence\",\"link\":\"/user-guide/mcp-tools/get-finding-evidence\"},{\"text\":\"get_finding_documentation\",\"link\":\"/user-guide/mcp-tools/get-finding-documentation\"},{\"text\":\"get_reference_doc\",\"link\":\"/user-guide/mcp-tools/get-reference-doc\"},{\"text\":\"list_runs\",\"link\":\"/user-guide/mcp-tools/list-runs\"}]},{\"text\":\"Finding your event log\",\"link\":\"/user-guide/alternative-log-retrieval\",\"collapsed\":true,\"items\":[{\"text\":\"Managed platforms\",\"link\":\"/user-guide/alternative-log-retrieval/managed-platforms\"},{\"text\":\"Behind an SSH bastion\",\"link\":\"/user-guide/alternative-log-retrieval/ssh-bastion\"}]}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\",\"collapsed\":true,\"items\":[{\"text\":\"Evidence availability\",\"link\":\"/contributor-guide/architecture/worker-protocol/evidence-availability\"},{\"text\":\"Evidence report and finding identity\",\"link\":\"/contributor-guide/architecture/worker-protocol/evidence-report\"},{\"text\":\"CLI and MCP server\",\"link\":\"/contributor-guide/architecture/worker-protocol/cli-and-mcp\"},{\"text\":\"Core and event validation\",\"link\":\"/contributor-guide/architecture/worker-protocol/core-and-event-validation\"}]},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\",\"collapsed\":true,\"items\":[{\"text\":\"Run comparison internals\",\"link\":\"/contributor-guide/architecture/state-and-history/run-comparison-internals\"},{\"text\":\"History Server intake\",\"link\":\"/contributor-guide/architecture/state-and-history/history-server-intake\"}]},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\",\"collapsed\":true,\"items\":[{\"text\":\"Plan attribution\",\"link\":\"/contributor-guide/architecture/detector-contract/plan-attribution\"},{\"text\":\"Threshold tables\",\"link\":\"/contributor-guide/architecture/detector-contract/threshold-tables\"}]},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\",\"collapsed\":true,\"items\":[{\"text\":\"Stage union rollup\",\"link\":\"/contributor-guide/architecture/impact-estimation/stage-union-rollup\"},{\"text\":\"Caveats, tuning and coverage\",\"link\":\"/contributor-guide/architecture/impact-estimation/caveats-tuning-and-coverage\"}]},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\",\"collapsed\":true,\"items\":[{\"text\":\"Findings tab\",\"link\":\"/contributor-guide/architecture/widget-rendering/findings-tab\"},{\"text\":\"List sort mode\",\"link\":\"/contributor-guide/architecture/widget-rendering/list-sort-mode\"},{\"text\":\"Row/expand contract\",\"link\":\"/contributor-guide/architecture/widget-rendering/row-expand-contract\"},{\"text\":\"ReferenceSection\",\"link\":\"/contributor-guide/architecture/widget-rendering/reference-section\"},{\"text\":\"Investigation routing\",\"link\":\"/contributor-guide/architecture/widget-rendering/investigation-routing\"}]},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\",\"collapsed\":true,\"items\":[{\"text\":\"Plan Advisor\",\"link\":\"/contributor-guide/architecture/board-widgets/plan-advisor\"},{\"text\":\"Caching and memory widgets\",\"link\":\"/contributor-guide/architecture/board-widgets/caching-and-memory\"},{\"text\":\"Visual system and templating\",\"link\":\"/contributor-guide/architecture/board-widgets/visual-system-and-templating\"}]},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\",\"collapsed\":true,\"items\":[{\"text\":\"Plan graph\",\"link\":\"/contributor-guide/architecture/drill-down/plan-graph\"}]}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":{\"selector\":\"[data-sf-anchor-offset]\",\"padding\":0},\"cleanUrls\":false}");</script>
34
30
 
35
31
  </body>
36
32
  </html>