sparkforensics-server 0.5.2 → 0.8.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (445) hide show
  1. package/index.js +16 -7
  2. package/package.json +1 -1
  3. package/public/assets/AutoscalingChurn-DenWKTv-.js +1 -0
  4. package/public/assets/CacheUtilization-DLz7NdjV.js +1 -0
  5. package/public/assets/CachingOpportunity-zpQIqhzm.js +1 -0
  6. package/public/assets/ChartTheme-DAiARbLP.js +8 -0
  7. package/public/assets/ColdStart-C-9HjxWl.js +1 -0
  8. package/public/assets/ConfigAudit-D0V9tcrJ.js +1 -0
  9. package/public/assets/CoreUsageArea-yeerrljd.js +1 -0
  10. package/public/assets/Dashboard-DkUPrHKV.js +1 -0
  11. package/public/assets/DisclosureContext-DK1_iz6J.js +1 -0
  12. package/public/assets/DuplicatePlanSubtree-xDNDbCd3.js +1 -0
  13. package/public/assets/DurationHistogram-B6jsE0Ov.js +1 -0
  14. package/public/assets/EmptyState-C1uL40VU.js +1 -0
  15. package/public/assets/ExecutorUtilization-DXDZKmSc.js +1 -0
  16. package/public/assets/GcPressure-MRf7I4IG.js +1 -0
  17. package/public/assets/ImpactEstimate-Dm3xWbzV.js +1 -0
  18. package/public/assets/IncompleteRun-CaLOhMtf.js +1 -0
  19. package/public/assets/JobFailures-CzmzlvhO.js +1 -0
  20. package/public/assets/MemoryUtilization-AWUMUjCH.js +1 -0
  21. package/public/assets/NestedLoopJoin-C9sNr6Ss.js +1 -0
  22. package/public/assets/OverBroadcast-BOmRE6ca.js +1 -0
  23. package/public/assets/PartitionHint-BL2EmJfE.js +1 -0
  24. package/public/assets/PartitionSizing-BzBySvUq.js +1 -0
  25. package/public/assets/PlanExplorer-DdiQRcJU.js +1 -0
  26. package/public/assets/PlanGraphRoute-CIH9gBuR.js +1 -0
  27. package/public/assets/PlanView-B1hxzFRs.js +2 -0
  28. package/public/assets/PythonUdf-DvgCCueD.js +1 -0
  29. package/public/assets/RetryWaste-uXXu_CyM.js +1 -0
  30. package/public/assets/RowPagination-C-LzWkI5.js +1 -0
  31. package/public/assets/RowStatusCluster-Dbtmte7l.js +1 -0
  32. package/public/assets/ShuffleIO-nALwzC1_.js +1 -0
  33. package/public/assets/Skew-CHMEKgvk.js +1 -0
  34. package/public/assets/SlowHost-yyvjpHk6.js +1 -0
  35. package/public/assets/SmallFiles-y6odJ97n.js +1 -0
  36. package/public/assets/SpeculationWaste-C-SD2LOt.js +1 -0
  37. package/public/assets/Spill-DHTMmOup.js +1 -0
  38. package/public/assets/StageFailed-CQ3DxC9R.js +1 -0
  39. package/public/assets/StageFindingGroup-BgYLyj5q.js +1 -0
  40. package/public/assets/StageHeader-CIETkjVZ.js +1 -0
  41. package/public/assets/StagePill--P7PDd3s.js +1 -0
  42. package/public/assets/StageShape-BIv8vx_o.js +1 -0
  43. package/public/assets/StageSlowness-MMqzERxN.js +1 -0
  44. package/public/assets/Straggler-BgrY7AyW.js +1 -0
  45. package/public/assets/TaskFailures-BHHyKYTs.js +1 -0
  46. package/public/assets/TinyTask-DDScNhCl.js +1 -0
  47. package/public/assets/UnderBroadcast-CGIC04TI.js +1 -0
  48. package/public/assets/WidgetLeadSummary-CXFZP00C.js +1 -0
  49. package/public/assets/button-DbpTz2yk.js +1 -0
  50. package/public/assets/charts-vendor-CtTxwTeu.js +59 -0
  51. package/public/assets/composite-C9Nu2EI9.js +1 -0
  52. package/public/assets/d3-vendor-DNlD2-cz.js +1 -0
  53. package/public/assets/detector-registry-Bp8hrLua.js +2 -0
  54. package/public/assets/downsample-WmNM1qc9.js +1 -0
  55. package/public/assets/finding-anchor-Dr40LkS9.js +1 -0
  56. package/public/assets/findings-of-type-BL2u3RGZ.js +1 -0
  57. package/public/assets/impact-sort-mC5ISgX8.js +1 -0
  58. package/public/assets/inbox-KPU1Gppm.js +1 -0
  59. package/public/assets/index-CaRVPvAT.css +2 -0
  60. package/public/assets/index-fDiwfuoO.js +14 -0
  61. package/public/assets/inertValue-L9VxV0VE.js +1 -0
  62. package/public/assets/instrument-sans-latin-ext-wdth-normal-C5E2Gvlv.woff2 +0 -0
  63. package/public/assets/instrument-sans-latin-wdth-normal-BVScPF0l.woff2 +0 -0
  64. package/public/assets/jetbrains-mono-latin-ext-wght-normal-DBQx-q_a.woff2 +0 -0
  65. package/public/assets/jetbrains-mono-latin-wght-normal-B9CIFXIH.woff2 +0 -0
  66. package/public/assets/layout.worker-CLcBHxVp.js +3 -0
  67. package/public/assets/{parser-worker-CA5RxqKc.js → parser-worker-CRIOPMOW.js} +8 -8
  68. package/public/assets/plan-graph-vendor-Ce6d4HU6.js +7 -0
  69. package/public/assets/react-vendor-C6KcUD36.js +9 -0
  70. package/public/assets/select-BHqOqun5.js +1 -0
  71. package/public/assets/toggle-DPeN6aNa.js +1 -0
  72. package/public/assets/toggle-group-Kl3uyHSr.js +1 -0
  73. package/public/assets/usePagedRows-BafD6bEN.js +1 -0
  74. package/public/assets/useRecentFiles-DzJZAoOT.js +3 -0
  75. package/public/assets/useRegisterFieldControl-pQv-ZDoU.js +1 -0
  76. package/public/assets/useScrollLock-BCxn_dMp.js +1 -0
  77. package/public/assets/useSortMode-DjrAN-z4.js +1 -0
  78. package/public/docs/404.html +3 -7
  79. package/public/docs/assets/{app.eYTPqTeq.js → app.1gvzd5Zb.js} +1 -1
  80. package/public/docs/assets/chunks/@localSearchIndexroot.DS2TQP5_.js +1 -0
  81. package/public/docs/assets/chunks/{VPLocalSearchBox.DsZrQKQR.js → VPLocalSearchBox.CNxT3s0I.js} +1 -1
  82. package/public/docs/assets/chunks/{theme.DsTNE2pE.js → theme.C5lLqUv7.js} +2 -2
  83. package/public/docs/assets/contributor-guide_architecture_board-widgets.md.CqKM_jEG.js +1 -0
  84. package/public/docs/assets/contributor-guide_architecture_board-widgets.md.CqKM_jEG.lean.js +1 -0
  85. package/public/docs/assets/{contributor-guide_architecture_board-widgets_caching-and-memory.md.8bH1HnZp.js → contributor-guide_architecture_board-widgets_caching-and-memory.md.BYDo3ClR.js} +1 -1
  86. package/public/docs/assets/contributor-guide_architecture_board-widgets_plan-advisor.md.ajUAVdbc.js +1 -0
  87. package/public/docs/assets/contributor-guide_architecture_board-widgets_visual-system-and-templating.md.WqY74T0T.js +1 -0
  88. package/public/docs/assets/contributor-guide_architecture_board-widgets_visual-system-and-templating.md.WqY74T0T.lean.js +1 -0
  89. package/public/docs/assets/contributor-guide_architecture_detector-contract.md.CHHgywoP.js +1 -0
  90. package/public/docs/assets/{contributor-guide_architecture_detector-contract.md.wj6Et6UD.lean.js → contributor-guide_architecture_detector-contract.md.CHHgywoP.lean.js} +1 -1
  91. package/public/docs/assets/contributor-guide_architecture_detector-contract_plan-attribution.md.DIz3_oz9.js +1 -0
  92. package/public/docs/assets/contributor-guide_architecture_detector-contract_plan-attribution.md.DIz3_oz9.lean.js +1 -0
  93. package/public/docs/assets/contributor-guide_architecture_detector-contract_threshold-tables.md.WH4tRHPL.js +1 -0
  94. package/public/docs/assets/{contributor-guide_architecture_detector-contract_threshold-tables.md.CKjWG56Y.lean.js → contributor-guide_architecture_detector-contract_threshold-tables.md.WH4tRHPL.lean.js} +1 -1
  95. package/public/docs/assets/{contributor-guide_architecture_drill-down.md.CqZPrdo9.js → contributor-guide_architecture_drill-down.md.yB-Hz2XT.js} +1 -1
  96. package/public/docs/assets/{contributor-guide_architecture_drill-down.md.CqZPrdo9.lean.js → contributor-guide_architecture_drill-down.md.yB-Hz2XT.lean.js} +1 -1
  97. package/public/docs/assets/contributor-guide_architecture_drill-down_plan-graph.md.BQZE04sw.js +1 -0
  98. package/public/docs/assets/{contributor-guide_architecture_drill-down_plan-graph.md.DwOozIEs.lean.js → contributor-guide_architecture_drill-down_plan-graph.md.BQZE04sw.lean.js} +1 -1
  99. package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.K71YB-Rr.js +1 -0
  100. package/public/docs/assets/{contributor-guide_architecture_impact-estimation.md.CZv_ZGOp.lean.js → contributor-guide_architecture_impact-estimation.md.K71YB-Rr.lean.js} +1 -1
  101. package/public/docs/assets/contributor-guide_architecture_impact-estimation_caveats-tuning-and-coverage.md.DLCAffwn.js +1 -0
  102. package/public/docs/assets/{contributor-guide_architecture_impact-estimation_caveats-tuning-and-coverage.md.BUucRc0A.lean.js → contributor-guide_architecture_impact-estimation_caveats-tuning-and-coverage.md.DLCAffwn.lean.js} +1 -1
  103. package/public/docs/assets/contributor-guide_architecture_impact-estimation_stage-union-rollup.md.JFrzDMwo.js +1 -0
  104. package/public/docs/assets/contributor-guide_architecture_impact-estimation_stage-union-rollup.md.JFrzDMwo.lean.js +1 -0
  105. package/public/docs/assets/contributor-guide_architecture_overview.md.nxkpOhgf.js +1 -0
  106. package/public/docs/assets/contributor-guide_architecture_state-and-history_run-comparison-internals.md.U-9KMDFK.js +1 -0
  107. package/public/docs/assets/{contributor-guide_architecture_state-and-history_run-comparison-internals.md.CZA0JLtV.lean.js → contributor-guide_architecture_state-and-history_run-comparison-internals.md.U-9KMDFK.lean.js} +1 -1
  108. package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.BrSz8Ezh.js +1 -0
  109. package/public/docs/assets/contributor-guide_architecture_widget-rendering_findings-tab.md.B7rjSF_h.js +1 -0
  110. package/public/docs/assets/contributor-guide_architecture_widget-rendering_findings-tab.md.B7rjSF_h.lean.js +1 -0
  111. package/public/docs/assets/{contributor-guide_architecture_widget-rendering_list-sort-mode.md.BuHKgH0C.js → contributor-guide_architecture_widget-rendering_list-sort-mode.md.37hcZJ7q.js} +1 -1
  112. package/public/docs/assets/{contributor-guide_architecture_widget-rendering_row-expand-contract.md.ZT89u9f1.js → contributor-guide_architecture_widget-rendering_row-expand-contract.md.mK_VmAWk.js} +1 -1
  113. package/public/docs/assets/{contributor-guide_architecture_worker-protocol.md.BAnhi2-d.js → contributor-guide_architecture_worker-protocol.md.Br6OE7RW.js} +1 -1
  114. package/public/docs/assets/contributor-guide_architecture_worker-protocol.md.Br6OE7RW.lean.js +1 -0
  115. package/public/docs/assets/contributor-guide_architecture_worker-protocol_cli-and-mcp.md.IVyaXrGe.js +1 -0
  116. package/public/docs/assets/{contributor-guide_architecture_worker-protocol_cli-and-mcp.md.DC1YqXBp.lean.js → contributor-guide_architecture_worker-protocol_cli-and-mcp.md.IVyaXrGe.lean.js} +1 -1
  117. package/public/docs/assets/{contributor-guide_architecture_worker-protocol_core-and-event-validation.md.DSVFPpWb.js → contributor-guide_architecture_worker-protocol_core-and-event-validation.md.DVSXn-bb.js} +1 -1
  118. package/public/docs/assets/{contributor-guide_architecture_worker-protocol_evidence-availability.md.CjimIDBY.js → contributor-guide_architecture_worker-protocol_evidence-availability.md.D3S5cyOW.js} +2 -2
  119. package/public/docs/assets/{contributor-guide_architecture_worker-protocol_evidence-availability.md.CjimIDBY.lean.js → contributor-guide_architecture_worker-protocol_evidence-availability.md.D3S5cyOW.lean.js} +1 -1
  120. package/public/docs/assets/contributor-guide_architecture_worker-protocol_evidence-report.md.CtmCU57I.js +1 -0
  121. package/public/docs/assets/{contributor-guide_architecture_worker-protocol_evidence-report.md.DVGZOWvW.lean.js → contributor-guide_architecture_worker-protocol_evidence-report.md.CtmCU57I.lean.js} +1 -1
  122. package/public/docs/assets/contributor-guide_contributing.md.DBW4drtZ.js +1 -0
  123. package/public/docs/assets/{contributor-guide_development-setup.md.D9bu4j_c.js → contributor-guide_development-setup.md.B21ZTDhn.js} +3 -3
  124. package/public/docs/assets/contributor-guide_development-setup.md.B21ZTDhn.lean.js +1 -0
  125. package/public/docs/assets/contributor-guide_testing.md.DpkozNoG.js +4 -0
  126. package/public/docs/assets/{contributor-guide_testing.md.By6XGR9a.lean.js → contributor-guide_testing.md.DpkozNoG.lean.js} +1 -1
  127. package/public/docs/assets/{index.md.CHJVslga.js → index.md.lMm7SUMz.js} +1 -1
  128. package/public/docs/assets/{index.md.CHJVslga.lean.js → index.md.lMm7SUMz.lean.js} +1 -1
  129. package/public/docs/assets/instrument-sans-latin-ext-wdth-normal.C5E2Gvlv.woff2 +0 -0
  130. package/public/docs/assets/instrument-sans-latin-wdth-normal.BVScPF0l.woff2 +0 -0
  131. package/public/docs/assets/jetbrains-mono-cyrillic-wght-normal.D73BlboJ.woff2 +0 -0
  132. package/public/docs/assets/jetbrains-mono-greek-wght-normal.Bw9x6K1M.woff2 +0 -0
  133. package/public/docs/assets/jetbrains-mono-latin-ext-wght-normal.DBQx-q_a.woff2 +0 -0
  134. package/public/docs/assets/jetbrains-mono-latin-wght-normal.B9CIFXIH.woff2 +0 -0
  135. package/public/docs/assets/jetbrains-mono-vietnamese-wght-normal.Bt-aOZkq.woff2 +0 -0
  136. package/public/docs/assets/style.Cfxqlj0y.css +1 -0
  137. package/public/docs/assets/user-guide_getting-started_ci-and-automation.md.C0pcQoNc.js +29 -0
  138. package/public/docs/assets/user-guide_getting-started_ci-and-automation.md.C0pcQoNc.lean.js +1 -0
  139. package/public/docs/assets/user-guide_getting-started_reading-the-dashboard.md.CGaxPX_1.js +1 -0
  140. package/public/docs/assets/{user-guide_getting-started_tuning-thresholds.md.B_MyjyZm.js → user-guide_getting-started_tuning-thresholds.md.BGVE-wrc.js} +2 -2
  141. package/public/docs/assets/{user-guide_getting-started_tuning-thresholds.md.B_MyjyZm.lean.js → user-guide_getting-started_tuning-thresholds.md.BGVE-wrc.lean.js} +1 -1
  142. package/public/docs/assets/{user-guide_mcp-tools.md.C65SjWUL.js → user-guide_mcp-tools.md.49r3L2YE.js} +1 -1
  143. package/public/docs/assets/{user-guide_mcp-tools_compare-runs.md.Cc_gLoix.js → user-guide_mcp-tools_compare-runs.md.BQ_kIptC.js} +11 -6
  144. package/public/docs/assets/user-guide_mcp-tools_compare-runs.md.BQ_kIptC.lean.js +1 -0
  145. package/public/docs/assets/user-guide_mcp-tools_diagnose-run.md.azCZXkH8.js +44 -0
  146. package/public/docs/assets/{user-guide_mcp-tools_diagnose-run.md.BP1vnHA5.lean.js → user-guide_mcp-tools_diagnose-run.md.azCZXkH8.lean.js} +1 -1
  147. package/public/docs/assets/{user-guide_mcp-tools_evaluate-budgets.md.BB0nDY3Z.js → user-guide_mcp-tools_evaluate-budgets.md.dpQpRQ7z.js} +2 -2
  148. package/public/docs/assets/{user-guide_mcp-tools_evaluate-budgets.md.BB0nDY3Z.lean.js → user-guide_mcp-tools_evaluate-budgets.md.dpQpRQ7z.lean.js} +1 -1
  149. package/public/docs/assets/{user-guide_mcp-tools_get-finding-evidence.md.rKyu4wts.js → user-guide_mcp-tools_get-finding-evidence.md.CH6FkNLX.js} +2 -1
  150. package/public/docs/assets/{user-guide_mcp-tools_get-finding-evidence.md.rKyu4wts.lean.js → user-guide_mcp-tools_get-finding-evidence.md.CH6FkNLX.lean.js} +1 -1
  151. package/public/docs/assets/{user-guide_mcp-tools_get-run-summary.md.DpxZ4ZNp.js → user-guide_mcp-tools_get-run-summary.md.EhaIsele.js} +1 -1
  152. package/public/docs/assets/user-guide_run-comparison.md.B_LOkdBf.js +1 -0
  153. package/public/docs/assets/user-guide_run-comparison.md.B_LOkdBf.lean.js +1 -0
  154. package/public/docs/assets/user-guide_understanding-findings.md.C4riKCKl.js +1 -0
  155. package/public/docs/assets/user-guide_understanding-findings.md.C4riKCKl.lean.js +1 -0
  156. package/public/docs/contributor-guide/architecture/board-widgets/caching-and-memory.html +6 -10
  157. package/public/docs/contributor-guide/architecture/board-widgets/plan-advisor.html +6 -10
  158. package/public/docs/contributor-guide/architecture/board-widgets/visual-system-and-templating.html +6 -10
  159. package/public/docs/contributor-guide/architecture/board-widgets.html +6 -10
  160. package/public/docs/contributor-guide/architecture/detector-contract/plan-attribution.html +6 -10
  161. package/public/docs/contributor-guide/architecture/detector-contract/threshold-tables.html +6 -10
  162. package/public/docs/contributor-guide/architecture/detector-contract.html +6 -10
  163. package/public/docs/contributor-guide/architecture/drill-down/plan-graph.html +6 -10
  164. package/public/docs/contributor-guide/architecture/drill-down.html +6 -10
  165. package/public/docs/contributor-guide/architecture/impact-estimation/caveats-tuning-and-coverage.html +6 -10
  166. package/public/docs/contributor-guide/architecture/impact-estimation/stage-union-rollup.html +6 -10
  167. package/public/docs/contributor-guide/architecture/impact-estimation.html +6 -10
  168. package/public/docs/contributor-guide/architecture/index.html +5 -9
  169. package/public/docs/contributor-guide/architecture/overview.html +6 -10
  170. package/public/docs/contributor-guide/architecture/state-and-history/history-server-intake.html +5 -9
  171. package/public/docs/contributor-guide/architecture/state-and-history/run-comparison-internals.html +6 -10
  172. package/public/docs/contributor-guide/architecture/state-and-history.html +5 -9
  173. package/public/docs/contributor-guide/architecture/widget-rendering/findings-tab.html +6 -10
  174. package/public/docs/contributor-guide/architecture/widget-rendering/investigation-routing.html +5 -9
  175. package/public/docs/contributor-guide/architecture/widget-rendering/list-sort-mode.html +6 -10
  176. package/public/docs/contributor-guide/architecture/widget-rendering/reference-section.html +5 -9
  177. package/public/docs/contributor-guide/architecture/widget-rendering/row-expand-contract.html +6 -10
  178. package/public/docs/contributor-guide/architecture/widget-rendering.html +6 -10
  179. package/public/docs/contributor-guide/architecture/worker-protocol/cli-and-mcp.html +6 -10
  180. package/public/docs/contributor-guide/architecture/worker-protocol/core-and-event-validation.html +6 -10
  181. package/public/docs/contributor-guide/architecture/worker-protocol/evidence-availability.html +7 -11
  182. package/public/docs/contributor-guide/architecture/worker-protocol/evidence-report.html +6 -10
  183. package/public/docs/contributor-guide/architecture/worker-protocol.html +6 -10
  184. package/public/docs/contributor-guide/contributing.html +6 -10
  185. package/public/docs/contributor-guide/development-setup.html +8 -12
  186. package/public/docs/contributor-guide/testing.html +9 -10
  187. package/public/docs/favicon.svg +2 -2
  188. package/public/docs/hashmap.json +1 -1
  189. package/public/docs/index.html +6 -10
  190. package/public/docs/logo-dark.svg +1 -0
  191. package/public/docs/logo-light.svg +1 -0
  192. package/public/docs/tuning-reference/anti-patterns.html +5 -9
  193. package/public/docs/tuning-reference/aqe.html +5 -9
  194. package/public/docs/tuning-reference/bottleneck-broadcast-sizing.html +5 -9
  195. package/public/docs/tuning-reference/bottleneck-cold-start.html +5 -9
  196. package/public/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html +5 -9
  197. package/public/docs/tuning-reference/bottleneck-failures.html +5 -9
  198. package/public/docs/tuning-reference/bottleneck-gc.html +5 -9
  199. package/public/docs/tuning-reference/bottleneck-job-failure-rate.html +5 -9
  200. package/public/docs/tuning-reference/bottleneck-memory-utilization.html +5 -9
  201. package/public/docs/tuning-reference/bottleneck-retry-waste.html +5 -9
  202. package/public/docs/tuning-reference/bottleneck-shuffle.html +5 -9
  203. package/public/docs/tuning-reference/bottleneck-skew.html +5 -9
  204. package/public/docs/tuning-reference/bottleneck-slow-host.html +5 -9
  205. package/public/docs/tuning-reference/bottleneck-small-files.html +5 -9
  206. package/public/docs/tuning-reference/bottleneck-spill.html +5 -9
  207. package/public/docs/tuning-reference/bottleneck-straggler.html +5 -9
  208. package/public/docs/tuning-reference/bottleneck-tiny-tasks.html +5 -9
  209. package/public/docs/tuning-reference/bottleneck-utilization.html +5 -9
  210. package/public/docs/tuning-reference/caching.html +5 -9
  211. package/public/docs/tuning-reference/cluster-config.html +5 -9
  212. package/public/docs/tuning-reference/config.html +5 -9
  213. package/public/docs/tuning-reference/data-formats.html +5 -9
  214. package/public/docs/tuning-reference/index.html +5 -9
  215. package/public/docs/tuning-reference/intro.html +5 -9
  216. package/public/docs/tuning-reference/joins.html +5 -9
  217. package/public/docs/tuning-reference/memory-model.html +5 -9
  218. package/public/docs/tuning-reference/metrics.html +5 -9
  219. package/public/docs/tuning-reference/partitioning.html +5 -9
  220. package/public/docs/tuning-reference/pyspark.html +5 -9
  221. package/public/docs/tuning-reference/shuffle.html +5 -9
  222. package/public/docs/tuning-reference/spark-architecture.html +5 -9
  223. package/public/docs/tuning-reference/table-formats.html +5 -9
  224. package/public/docs/user-guide/alternative-log-retrieval/managed-platforms.html +5 -9
  225. package/public/docs/user-guide/alternative-log-retrieval/ssh-bastion.html +5 -9
  226. package/public/docs/user-guide/alternative-log-retrieval.html +5 -9
  227. package/public/docs/user-guide/getting-started/ci-and-automation.html +17 -15
  228. package/public/docs/user-guide/getting-started/reading-the-dashboard.html +6 -10
  229. package/public/docs/user-guide/getting-started/regression-metric-keys.html +5 -9
  230. package/public/docs/user-guide/getting-started/tuning-thresholds.html +7 -11
  231. package/public/docs/user-guide/getting-started.html +5 -9
  232. package/public/docs/user-guide/mcp-tools/compare-runs.html +16 -15
  233. package/public/docs/user-guide/mcp-tools/diagnose-run.html +28 -11
  234. package/public/docs/user-guide/mcp-tools/evaluate-budgets.html +6 -10
  235. package/public/docs/user-guide/mcp-tools/get-finding-documentation.html +5 -9
  236. package/public/docs/user-guide/mcp-tools/get-finding-evidence.html +7 -10
  237. package/public/docs/user-guide/mcp-tools/get-reference-doc.html +5 -9
  238. package/public/docs/user-guide/mcp-tools/get-run-summary.html +6 -10
  239. package/public/docs/user-guide/mcp-tools/list-runs.html +5 -9
  240. package/public/docs/user-guide/mcp-tools.html +6 -10
  241. package/public/docs/user-guide/run-comparison/how-stages-are-matched.html +5 -9
  242. package/public/docs/user-guide/run-comparison.html +6 -10
  243. package/public/docs/user-guide/understanding-findings.html +6 -10
  244. package/public/export-template.html +46 -46
  245. package/public/favicon.svg +2 -2
  246. package/public/index.html +8 -10
  247. package/public/sample-runs/sample-run.ndjson.gz +0 -0
  248. package/vendor-core/allocation.js +126 -39
  249. package/vendor-core/analyzer.js +85 -58
  250. package/vendor-core/api-merge-writes.js +3 -0
  251. package/vendor-core/aqe-skew.js +347 -0
  252. package/vendor-core/cli/budgets.js +6 -4
  253. package/vendor-core/cli/collect-run-pair.js +89 -0
  254. package/vendor-core/cli/collect-run-worker.js +16 -0
  255. package/vendor-core/cli/collect-run.js +1 -2
  256. package/vendor-core/cli/native-zstd.js +12 -16
  257. package/vendor-core/comparability.js +234 -0
  258. package/vendor-core/comparison-output.js +15 -7
  259. package/vendor-core/comparison-verdict.js +105 -25
  260. package/vendor-core/core-source-hash.txt +1 -1
  261. package/vendor-core/core-time-series.js +0 -1
  262. package/vendor-core/core-usage-locality.js +1 -1
  263. package/vendor-core/delta-log-paths.js +10 -1
  264. package/vendor-core/detectors.js +1078 -416
  265. package/vendor-core/docs-config.js +0 -4
  266. package/vendor-core/docs-content/detection/cache.md +2 -1
  267. package/vendor-core/docs-content/detection/cfg.md +19 -11
  268. package/vendor-core/docs-content/detection/cold.md +2 -1
  269. package/vendor-core/docs-content/detection/gc.md +7 -3
  270. package/vendor-core/docs-content/detection/host.md +11 -1
  271. package/vendor-core/docs-content/detection/mem.md +17 -6
  272. package/vendor-core/docs-content/detection/part.md +3 -1
  273. package/vendor-core/docs-content/detection/plan.md +55 -13
  274. package/vendor-core/docs-content/detection/shfl.md +11 -5
  275. package/vendor-core/docs-content/detection/skew.md +48 -1
  276. package/vendor-core/docs-content/detection/slow.md +4 -2
  277. package/vendor-core/docs-content/detection/spec.md +6 -1
  278. package/vendor-core/docs-content/detection/spill.md +6 -3
  279. package/vendor-core/docs-content/detection/strag.md +19 -6
  280. package/vendor-core/docs-content/detection/tiny.md +9 -3
  281. package/vendor-core/docs-content/detection/util.md +6 -2
  282. package/vendor-core/effective-conf.js +7 -10
  283. package/vendor-core/efficiency-model.js +24 -16
  284. package/vendor-core/etl-phases.js +1 -2
  285. package/vendor-core/event-handlers.js +175 -57
  286. package/vendor-core/event-schemas.js +21 -16
  287. package/vendor-core/evidence-availability.js +9 -2
  288. package/vendor-core/evidence-report.js +118 -35
  289. package/vendor-core/executor-peaks.js +41 -0
  290. package/vendor-core/export-data.js +5 -5
  291. package/vendor-core/finding-identity.js +78 -0
  292. package/vendor-core/finding-presentation.js +129 -39
  293. package/vendor-core/finding-types.js +108 -30
  294. package/vendor-core/format-utils.js +14 -15
  295. package/vendor-core/html-export.js +1 -1
  296. package/vendor-core/impact-band.js +2 -2
  297. package/vendor-core/impact-format.js +47 -19
  298. package/vendor-core/impact-model.js +28 -17
  299. package/vendor-core/job-groups.js +4 -4
  300. package/vendor-core/mcp-server-factory.js +48 -12
  301. package/vendor-core/mcp-tools.js +179 -24
  302. package/vendor-core/median.js +2 -2
  303. package/vendor-core/nested-loop-join.js +73 -0
  304. package/vendor-core/occupancy.js +1 -2
  305. package/vendor-core/parser-worker.js +1 -1
  306. package/vendor-core/plan-graph-model.js +14 -22
  307. package/vendor-core/plan-node-detail.js +2 -3
  308. package/vendor-core/plan-summary.js +24 -19
  309. package/vendor-core/plan-tree-walk.js +6 -8
  310. package/vendor-core/python-stage.js +12 -1
  311. package/vendor-core/recommendation-rollup.js +13 -12
  312. package/vendor-core/redact.js +52 -46
  313. package/vendor-core/remediation.js +10 -5
  314. package/vendor-core/run-aggregates.js +6 -8
  315. package/vendor-core/run-comparison.js +226 -66
  316. package/vendor-core/run-identity.js +12 -0
  317. package/vendor-core/run-interpretation.js +21 -0
  318. package/vendor-core/run-metrics.js +2 -2
  319. package/vendor-core/run-output.js +44 -11
  320. package/vendor-core/run-totals.js +12 -0
  321. package/vendor-core/run-verdict.js +76 -3
  322. package/vendor-core/scaling-sim.js +3 -4
  323. package/vendor-core/snappy-block.js +2 -2
  324. package/vendor-core/spark-conf.js +168 -0
  325. package/vendor-core/sql-stages.js +2 -2
  326. package/vendor-core/stage-alignment.js +2 -12
  327. package/vendor-core/stage-identity.js +58 -3
  328. package/vendor-core/stage-label.js +14 -0
  329. package/vendor-core/stage-plan-nodes.js +17 -3
  330. package/vendor-core/stage-quantiles.js +141 -9
  331. package/vendor-core/stage-structure.js +17 -3
  332. package/vendor-core/threshold-overrides.js +9 -3
  333. package/vendor-core/types.js +70 -23
  334. package/vendor-core/wall-clock.js +8 -0
  335. package/vendor-core/wasted-core-hours.js +12 -12
  336. package/vendor-core/write-targets.js +146 -14
  337. package/vendor-core/zstd-worker-client.js +1 -1
  338. package/vendor-core/zstd-worker.js +1 -1
  339. package/public/assets/AutoscalingChurn-8t3U3isq.js +0 -1
  340. package/public/assets/CacheUtilization-CJs9tTum.js +0 -1
  341. package/public/assets/CachingOpportunity-D1NcHs0V.js +0 -1
  342. package/public/assets/ColdStart-CdRJ9gD8.js +0 -1
  343. package/public/assets/ConfigAudit-B5Gbe5js.js +0 -1
  344. package/public/assets/CoreUsageArea-a--NTRGY.js +0 -1
  345. package/public/assets/Dashboard-C1hNeX49.js +0 -2
  346. package/public/assets/DisclosureContext-ZpE6CkHx.js +0 -1
  347. package/public/assets/DuplicatePlanSubtree-dobXdz7E.js +0 -1
  348. package/public/assets/ExecutorUtilization-D9grwi7t.js +0 -1
  349. package/public/assets/GcPressure-8-G2ri10.js +0 -1
  350. package/public/assets/IncompleteRun-B91lky7a.js +0 -1
  351. package/public/assets/JobFailures-BsWvBMyJ.js +0 -1
  352. package/public/assets/MemoryUtilization-BM5schZI.js +0 -1
  353. package/public/assets/OverBroadcast-DdN8JIJH.js +0 -1
  354. package/public/assets/PartitionHint-CUvbnnxy.js +0 -1
  355. package/public/assets/PartitionSizing-CY5HDYtp.js +0 -1
  356. package/public/assets/PlanExplorer-DgHCDBD_.js +0 -1
  357. package/public/assets/PlanGraphRoute-DH3KJb8j.js +0 -1
  358. package/public/assets/RetryWaste-DOnE9vGF.js +0 -1
  359. package/public/assets/ShuffleIO-MMnDESkF.js +0 -1
  360. package/public/assets/Skew-BmZZJ5ms.js +0 -1
  361. package/public/assets/SlowHost-CsJzfEqo.js +0 -1
  362. package/public/assets/SmallFiles-BoK3R1Ex.js +0 -1
  363. package/public/assets/SpeculationWaste-CnVxB5AJ.js +0 -1
  364. package/public/assets/Spill-BhNicyBy.js +0 -1
  365. package/public/assets/StageFailed-mAhHU4l_.js +0 -1
  366. package/public/assets/StageFindingGroup-BEtPWcT0.js +0 -1
  367. package/public/assets/StageHeader-DJkYkrsU.js +0 -1
  368. package/public/assets/StageShape-BDxRknLp.js +0 -1
  369. package/public/assets/StageSlowness-BPRw-otN.js +0 -1
  370. package/public/assets/Straggler-CZCbnOWP.js +0 -1
  371. package/public/assets/TaskFailures-C57YcLFQ.js +0 -1
  372. package/public/assets/TinyTask-Bi84qU7n.js +0 -1
  373. package/public/assets/UnderBroadcast-DdQV7htE.js +0 -1
  374. package/public/assets/charts-vendor-0VMaLVLN.js +0 -59
  375. package/public/assets/finding-anchor-Cl6_5hfV.js +0 -1
  376. package/public/assets/impact-sort-CssZ6MT2.js +0 -1
  377. package/public/assets/index-2h_TgUAb.css +0 -2
  378. package/public/assets/index-B-03mG7K.js +0 -14
  379. package/public/assets/plan-graph-vendor-BYl-AzN5.js +0 -7
  380. package/public/assets/react-vendor-DEdfrRTJ.js +0 -9
  381. package/public/assets/select-CQjNhprm.js +0 -1
  382. package/public/assets/usePagedRows-Dyl6T8Ok.js +0 -1
  383. package/public/assets/useRecentFiles-XZEnLlal.js +0 -11
  384. package/public/assets/useSortMode-Dhvit8dt.js +0 -1
  385. package/public/docs/assets/chunks/@localSearchIndexroot.u_e5bQz4.js +0 -1
  386. package/public/docs/assets/contributor-guide_architecture_board-widgets.md.YQcj5EI4.js +0 -1
  387. package/public/docs/assets/contributor-guide_architecture_board-widgets.md.YQcj5EI4.lean.js +0 -1
  388. package/public/docs/assets/contributor-guide_architecture_board-widgets_plan-advisor.md.RlzvUmNm.js +0 -1
  389. package/public/docs/assets/contributor-guide_architecture_board-widgets_visual-system-and-templating.md.DYgUpIJB.js +0 -1
  390. package/public/docs/assets/contributor-guide_architecture_board-widgets_visual-system-and-templating.md.DYgUpIJB.lean.js +0 -1
  391. package/public/docs/assets/contributor-guide_architecture_detector-contract.md.wj6Et6UD.js +0 -1
  392. package/public/docs/assets/contributor-guide_architecture_detector-contract_plan-attribution.md.yNex4bys.js +0 -1
  393. package/public/docs/assets/contributor-guide_architecture_detector-contract_plan-attribution.md.yNex4bys.lean.js +0 -1
  394. package/public/docs/assets/contributor-guide_architecture_detector-contract_threshold-tables.md.CKjWG56Y.js +0 -1
  395. package/public/docs/assets/contributor-guide_architecture_drill-down_plan-graph.md.DwOozIEs.js +0 -1
  396. package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.CZv_ZGOp.js +0 -1
  397. package/public/docs/assets/contributor-guide_architecture_impact-estimation_caveats-tuning-and-coverage.md.BUucRc0A.js +0 -1
  398. package/public/docs/assets/contributor-guide_architecture_impact-estimation_stage-union-rollup.md.DdCGtAGu.js +0 -1
  399. package/public/docs/assets/contributor-guide_architecture_impact-estimation_stage-union-rollup.md.DdCGtAGu.lean.js +0 -1
  400. package/public/docs/assets/contributor-guide_architecture_overview.md.C1sSvm9i.js +0 -1
  401. package/public/docs/assets/contributor-guide_architecture_state-and-history_run-comparison-internals.md.CZA0JLtV.js +0 -1
  402. package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.BOZ1azXG.js +0 -1
  403. package/public/docs/assets/contributor-guide_architecture_widget-rendering_findings-tab.md.BQXycK6G.js +0 -1
  404. package/public/docs/assets/contributor-guide_architecture_widget-rendering_findings-tab.md.BQXycK6G.lean.js +0 -1
  405. package/public/docs/assets/contributor-guide_architecture_worker-protocol.md.BAnhi2-d.lean.js +0 -1
  406. package/public/docs/assets/contributor-guide_architecture_worker-protocol_cli-and-mcp.md.DC1YqXBp.js +0 -1
  407. package/public/docs/assets/contributor-guide_architecture_worker-protocol_evidence-report.md.DVGZOWvW.js +0 -1
  408. package/public/docs/assets/contributor-guide_contributing.md.CQMvGwFt.js +0 -1
  409. package/public/docs/assets/contributor-guide_development-setup.md.D9bu4j_c.lean.js +0 -1
  410. package/public/docs/assets/contributor-guide_testing.md.By6XGR9a.js +0 -1
  411. package/public/docs/assets/inter-italic-cyrillic-ext.r48I6akx.woff2 +0 -0
  412. package/public/docs/assets/inter-italic-cyrillic.By2_1cv3.woff2 +0 -0
  413. package/public/docs/assets/inter-italic-greek-ext.1u6EdAuj.woff2 +0 -0
  414. package/public/docs/assets/inter-italic-greek.DJ8dCoTZ.woff2 +0 -0
  415. package/public/docs/assets/inter-italic-latin-ext.CN1xVJS-.woff2 +0 -0
  416. package/public/docs/assets/inter-italic-latin.C2AdPX0b.woff2 +0 -0
  417. package/public/docs/assets/inter-italic-vietnamese.BSbpV94h.woff2 +0 -0
  418. package/public/docs/assets/inter-roman-cyrillic-ext.BBPuwvHQ.woff2 +0 -0
  419. package/public/docs/assets/inter-roman-cyrillic.C5lxZ8CY.woff2 +0 -0
  420. package/public/docs/assets/inter-roman-greek-ext.CqjqNYQ-.woff2 +0 -0
  421. package/public/docs/assets/inter-roman-greek.BBVDIX6e.woff2 +0 -0
  422. package/public/docs/assets/inter-roman-latin-ext.4ZJIpNVo.woff2 +0 -0
  423. package/public/docs/assets/inter-roman-latin.Di8DUHzh.woff2 +0 -0
  424. package/public/docs/assets/inter-roman-vietnamese.BjW4sHH5.woff2 +0 -0
  425. package/public/docs/assets/style.tmDhD3c8.css +0 -1
  426. package/public/docs/assets/user-guide_getting-started_ci-and-automation.md.OUQqw3mI.js +0 -23
  427. package/public/docs/assets/user-guide_getting-started_ci-and-automation.md.OUQqw3mI.lean.js +0 -1
  428. package/public/docs/assets/user-guide_getting-started_reading-the-dashboard.md.Obf3W3__.js +0 -1
  429. package/public/docs/assets/user-guide_mcp-tools_compare-runs.md.Cc_gLoix.lean.js +0 -1
  430. package/public/docs/assets/user-guide_mcp-tools_diagnose-run.md.BP1vnHA5.js +0 -23
  431. package/public/docs/assets/user-guide_run-comparison.md.BUALkohi.js +0 -1
  432. package/public/docs/assets/user-guide_run-comparison.md.BUALkohi.lean.js +0 -1
  433. package/public/docs/assets/user-guide_understanding-findings.md.C9YRruMT.js +0 -1
  434. package/public/docs/assets/user-guide_understanding-findings.md.C9YRruMT.lean.js +0 -1
  435. /package/public/docs/assets/{contributor-guide_architecture_board-widgets_caching-and-memory.md.8bH1HnZp.lean.js → contributor-guide_architecture_board-widgets_caching-and-memory.md.BYDo3ClR.lean.js} +0 -0
  436. /package/public/docs/assets/{contributor-guide_architecture_board-widgets_plan-advisor.md.RlzvUmNm.lean.js → contributor-guide_architecture_board-widgets_plan-advisor.md.ajUAVdbc.lean.js} +0 -0
  437. /package/public/docs/assets/{contributor-guide_architecture_overview.md.C1sSvm9i.lean.js → contributor-guide_architecture_overview.md.nxkpOhgf.lean.js} +0 -0
  438. /package/public/docs/assets/{contributor-guide_architecture_widget-rendering.md.BOZ1azXG.lean.js → contributor-guide_architecture_widget-rendering.md.BrSz8Ezh.lean.js} +0 -0
  439. /package/public/docs/assets/{contributor-guide_architecture_widget-rendering_list-sort-mode.md.BuHKgH0C.lean.js → contributor-guide_architecture_widget-rendering_list-sort-mode.md.37hcZJ7q.lean.js} +0 -0
  440. /package/public/docs/assets/{contributor-guide_architecture_widget-rendering_row-expand-contract.md.ZT89u9f1.lean.js → contributor-guide_architecture_widget-rendering_row-expand-contract.md.mK_VmAWk.lean.js} +0 -0
  441. /package/public/docs/assets/{contributor-guide_architecture_worker-protocol_core-and-event-validation.md.DSVFPpWb.lean.js → contributor-guide_architecture_worker-protocol_core-and-event-validation.md.DVSXn-bb.lean.js} +0 -0
  442. /package/public/docs/assets/{contributor-guide_contributing.md.CQMvGwFt.lean.js → contributor-guide_contributing.md.DBW4drtZ.lean.js} +0 -0
  443. /package/public/docs/assets/{user-guide_getting-started_reading-the-dashboard.md.Obf3W3__.lean.js → user-guide_getting-started_reading-the-dashboard.md.CGaxPX_1.lean.js} +0 -0
  444. /package/public/docs/assets/{user-guide_mcp-tools.md.C65SjWUL.lean.js → user-guide_mcp-tools.md.49r3L2YE.lean.js} +0 -0
  445. /package/public/docs/assets/{user-guide_mcp-tools_get-run-summary.md.DpxZ4ZNp.lean.js → user-guide_mcp-tools_get-run-summary.md.EhaIsele.lean.js} +0 -0
package/public/docs/tuning-reference/bottleneck-slow-host.html CHANGED
@@ -6,31 +6,27 @@
6
6
  <title>Slow Host | Spark Tuning Reference</title>
7
7
  <meta name="description" content="Detecting and mitigating a slow host: host-wide task duration inflation versus the cluster median, and speculative-execution or locality-wait fixes.">
8
8
  <meta name="generator" content="VitePress v1.6.4">
9
- <link rel="preload stylesheet" href="/docs/assets/style.tmDhD3c8.css" as="style">
9
+ <link rel="preload stylesheet" href="/docs/assets/style.Cfxqlj0y.css" as="style">
10
10
  <link rel="preload stylesheet" href="/docs/vp-icons.css" as="style">
11
11
 
12
- <script type="module" src="/docs/assets/app.eYTPqTeq.js"></script>
13
- <link rel="preload" href="/docs/assets/inter-roman-latin.Di8DUHzh.woff2" as="font" type="font/woff2" crossorigin="">
14
- <link rel="modulepreload" href="/docs/assets/chunks/theme.DsTNE2pE.js">
12
+ <script type="module" src="/docs/assets/app.1gvzd5Zb.js"></script>
13
+ <link rel="modulepreload" href="/docs/assets/chunks/theme.C5lLqUv7.js">
15
14
  <link rel="modulepreload" href="/docs/assets/chunks/framework.DSg0KOwT.js">
16
15
  <link rel="modulepreload" href="/docs/assets/tuning-reference_bottleneck-slow-host.md.ja5qr3wq.lean.js">
17
16
  <script>(()=>{const c=`${`[${["data","shuffle","product","bar"].join("-")}]`}, .VPNav, .VPLocalNav`,a=24,d=document.documentElement,l=t=>{const e=getComputedStyle(t);return e.display==="none"||t.getClientRects().length===0?0:e.position==="fixed"?t.getBoundingClientRect().bottom:e.position==="sticky"?(parseFloat(e.top)||0)+t.offsetHeight:0},u=()=>{try{return location.hash?document.getElementById(decodeURIComponent(location.hash.slice(1))):null}catch{return null}};let r=null;const s=()=>{const t=Array.from(document.querySelectorAll(c),l),e=Math.ceil(Math.max(0,...t))+a;if(e===r)return;const n=r;r=e,d.style.setProperty("--sf-anchor-offset",`${r}px`);const o=n===null?null:u();if(!o)return;const f=(parseFloat(getComputedStyle(o).paddingTop)||0)+2;Math.abs(o.getBoundingClientRect().top-n)<=f&&o.scrollIntoView()},i=()=>{const t=document.createElement("div");t.setAttribute("data-sf-anchor-offset",""),t.setAttribute("aria-hidden","true"),document.body.appendChild(t);const e=window.ResizeObserver?new ResizeObserver(s):null,n=()=>{e&&document.querySelectorAll(c).forEach(o=>e.observe(o)),s()};n(),new MutationObserver(n).observe(document.body,{childList:!0,subtree:!0}),window.addEventListener("resize",s),window.addEventListener("load",s)};document.readyState==="loading"?document.addEventListener("DOMContentLoaded",i,{once:!0}):i()})();</script>
18
17
  <link rel="icon" type="image/svg+xml" href="/docs/favicon.svg">
19
- <link rel="preconnect" href="https://fonts.googleapis.com">
20
- <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin="">
21
- <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Recursive:wght,CASL@400..700,0..1&amp;family=JetBrains+Mono:wght@400;500;600&amp;display=swap">
22
18
  <script id="check-dark-mode">(()=>{const e=localStorage.getItem("vitepress-theme-appearance")||"auto",a=window.matchMedia("(prefers-color-scheme: dark)").matches;(!e||e==="auto"?a:e==="dark")&&document.documentElement.classList.add("dark")})();</script>
23
19
  <script id="check-mac-os">document.documentElement.classList.toggle("mac",/Mac|iPhone|iPod|iPad/i.test(navigator.platform));</script>
24
20
  </head>
25
21
  <body>
26
- <div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Getting Started</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/intro.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Introduction</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Optimization Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/spark-architecture.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Execution Model</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/memory-model.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Management</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/partitioning.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Partitioning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/joins.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Join Optimization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/data-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Data Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/table-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Table Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/caching.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Caching & Persistence</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/pyspark.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>PySpark Specifics</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/aqe.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Adaptive Query Execution</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/cluster-config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cluster Tuning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/anti-patterns.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Anti-Patterns</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Detector Catalog</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-skew.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Skew</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle I/O</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-spill.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory / Disk Spill</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-gc.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>GC Pressure</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-cold-start.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cold Start</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Executor Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-slow-host.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Slow Host</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-failures.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Failures</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-straggler.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Stragglers</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-retry-waste.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Retry Waste</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-tiny-tasks.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Tiny Tasks</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-job-failure-rate.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Job Failure Rate</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-memory-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Duplicate Plan Subtree</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-small-files.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Small Files</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-broadcast-sizing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Broadcast Sizing</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Reference</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/metrics.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Metrics Glossary</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Config Quick-Reference</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_tuning-reference_bottleneck-slow-host" data-v-39a288b8><div><h1 id="bottleneck-slow-host" tabindex="-1">Slow Host <a class="header-anchor" href="#bottleneck-slow-host" aria-label="Permalink to &quot;Slow Host {#bottleneck-slow-host}&quot;">​</a></h1><p><span class="tag">HOST</span></p><h2 id="what-it-is" tabindex="-1">What it is <a class="header-anchor" href="#what-it-is" aria-label="Permalink to &quot;What it is&quot;">​</a></h2><p>A slow host bottleneck shows up when one machine in the cluster consistently turns in slower task times than its peers, independent of any single task&#39;s own data size. Unlike a stray <a href="./bottleneck-straggler.html">straggler task</a>, the effect is host-wide: every task Spark schedules there runs behind, which drags out the stage even when the workload itself is partitioned evenly.</p><h2 id="how-it-s-detected" tabindex="-1">How it&#39;s detected <a class="header-anchor" href="#how-it-s-detected" aria-label="Permalink to &quot;How it&#39;s detected&quot;">​</a></h2><p>Spark&#39;s event log carries the per-task detail behind this: turning on <code>spark.eventLog.enabled</code> logs the events that encode what the UI displays, persisted to storage<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup>, and that same log backs the UI&#39;s Stages tab, which drills down into individual tasks and shows per-task metrics such as duration, GC time, and shuffle bytes read<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup>.</p><p>A host reads as slow against the following synthetic threshold: host mean task duration ≥ 2× overall median AND the host holds ≥ 20% task share; the signal only applies with <code>taskCount ≥ 15</code> and <code>hosts.length ≥ 3</code>.</p><h2 id="why-it-matters" tabindex="-1">Why it matters <a class="header-anchor" href="#why-it-matters" aria-label="Permalink to &quot;Why it matters&quot;">​</a></h2><p>A host running persistently slow tasks behaves like a bottleneck baked into the cluster rather than into the workload: every task Spark places there inherits the delay, and since a stage&#39;s completion time is bounded by its slowest tasks, the rest of the cluster idles while the affected host catches up. Left alone, the same host keeps dragging down every later stage and job that lands work on it.</p><h2 id="how-to-fix-it" tabindex="-1">How to fix it <a class="header-anchor" href="#how-to-fix-it" aria-label="Permalink to &quot;How to fix it&quot;">​</a></h2><ul><li>Turn on speculative execution (<code>spark.speculation</code>, off by default) so Spark relaunches a copy of a task that&#39;s lagging far behind its peers instead of waiting on the slow host to finish it. A stage only becomes eligible once <code>spark.speculation.quantile</code> (default <code>0.9</code>) of its tasks have completed, and a task then qualifies once it runs more than <code>spark.speculation.multiplier</code> (default <code>3</code>) times the median duration, subject to a <code>spark.speculation.minTaskRuntime</code> floor (default <code>100ms</code>) so short tasks aren&#39;t speculated purely for looking slow, or, since Spark 3.4, an efficiency check (<code>spark.speculation.efficiency.enabled</code>, default <code>true</code>) that also requires the task&#39;s data-processing rate to lag the stage average<sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup>.</li><li>If the slow host also happens to hold data locality for the affected tasks, lowering <code>spark.locality.wait</code> (default <code>3s</code>), or the level-specific <code>spark.locality.wait.node</code>, <code>.rack</code>, and <code>.process</code> overrides, shortens how long Spark waits for a data-local slot before falling back to a less-local executor<sup class="footnote-ref"><a href="#fn3" id="fnref3:1">[3:1]</a></sup>.</li></ul><p>Turn on speculation and, if the slow host holds locality, shorten the wait; defaults shown:</p><div class="language-properties vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">properties</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Relaunch tasks stuck on a slow host (speculation is OFF by default)</span></span>
22
+ <div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><!--[--><!--[--><img class="VPImage dark logo" src="/docs/logo-dark.svg" alt data-v-8426fc1a><!--]--><!--[--><img class="VPImage light logo" src="/docs/logo-light.svg" alt data-v-8426fc1a><!--]--><!--]--><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/.." target="_self" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Open app</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Getting Started</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/intro.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Introduction</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Optimization Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/spark-architecture.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Execution Model</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/memory-model.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Management</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/partitioning.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Partitioning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/joins.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Join Optimization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/data-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Data Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/table-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Table Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/caching.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Caching & Persistence</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/pyspark.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>PySpark Specifics</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/aqe.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Adaptive Query Execution</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/cluster-config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cluster Tuning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/anti-patterns.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Anti-Patterns</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Detector Catalog</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-skew.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Skew</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle I/O</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-spill.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory / Disk Spill</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-gc.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>GC Pressure</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-cold-start.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cold Start</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Executor Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-slow-host.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Slow Host</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-failures.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Failures</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-straggler.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Stragglers</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-retry-waste.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Retry Waste</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-tiny-tasks.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Tiny Tasks</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-job-failure-rate.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Job Failure Rate</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-memory-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Duplicate Plan Subtree</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-small-files.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Small Files</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-broadcast-sizing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Broadcast Sizing</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Reference</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/metrics.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Metrics Glossary</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Config Quick-Reference</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_tuning-reference_bottleneck-slow-host" data-v-39a288b8><div><h1 id="bottleneck-slow-host" tabindex="-1">Slow Host <a class="header-anchor" href="#bottleneck-slow-host" aria-label="Permalink to &quot;Slow Host {#bottleneck-slow-host}&quot;">​</a></h1><p><span class="tag">HOST</span></p><h2 id="what-it-is" tabindex="-1">What it is <a class="header-anchor" href="#what-it-is" aria-label="Permalink to &quot;What it is&quot;">​</a></h2><p>A slow host bottleneck shows up when one machine in the cluster consistently turns in slower task times than its peers, independent of any single task&#39;s own data size. Unlike a stray <a href="./bottleneck-straggler.html">straggler task</a>, the effect is host-wide: every task Spark schedules there runs behind, which drags out the stage even when the workload itself is partitioned evenly.</p><h2 id="how-it-s-detected" tabindex="-1">How it&#39;s detected <a class="header-anchor" href="#how-it-s-detected" aria-label="Permalink to &quot;How it&#39;s detected&quot;">​</a></h2><p>Spark&#39;s event log carries the per-task detail behind this: turning on <code>spark.eventLog.enabled</code> logs the events that encode what the UI displays, persisted to storage<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup>, and that same log backs the UI&#39;s Stages tab, which drills down into individual tasks and shows per-task metrics such as duration, GC time, and shuffle bytes read<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup>.</p><p>A host reads as slow against the following synthetic threshold: host mean task duration ≥ 2× overall median AND the host holds ≥ 20% task share; the signal only applies with <code>taskCount ≥ 15</code> and <code>hosts.length ≥ 3</code>.</p><h2 id="why-it-matters" tabindex="-1">Why it matters <a class="header-anchor" href="#why-it-matters" aria-label="Permalink to &quot;Why it matters&quot;">​</a></h2><p>A host running persistently slow tasks behaves like a bottleneck baked into the cluster rather than into the workload: every task Spark places there inherits the delay, and since a stage&#39;s completion time is bounded by its slowest tasks, the rest of the cluster idles while the affected host catches up. Left alone, the same host keeps dragging down every later stage and job that lands work on it.</p><h2 id="how-to-fix-it" tabindex="-1">How to fix it <a class="header-anchor" href="#how-to-fix-it" aria-label="Permalink to &quot;How to fix it&quot;">​</a></h2><ul><li>Turn on speculative execution (<code>spark.speculation</code>, off by default) so Spark relaunches a copy of a task that&#39;s lagging far behind its peers instead of waiting on the slow host to finish it. A stage only becomes eligible once <code>spark.speculation.quantile</code> (default <code>0.9</code>) of its tasks have completed, and a task then qualifies once it runs more than <code>spark.speculation.multiplier</code> (default <code>3</code>) times the median duration, subject to a <code>spark.speculation.minTaskRuntime</code> floor (default <code>100ms</code>) so short tasks aren&#39;t speculated purely for looking slow, or, since Spark 3.4, an efficiency check (<code>spark.speculation.efficiency.enabled</code>, default <code>true</code>) that also requires the task&#39;s data-processing rate to lag the stage average<sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup>.</li><li>If the slow host also happens to hold data locality for the affected tasks, lowering <code>spark.locality.wait</code> (default <code>3s</code>), or the level-specific <code>spark.locality.wait.node</code>, <code>.rack</code>, and <code>.process</code> overrides, shortens how long Spark waits for a data-local slot before falling back to a less-local executor<sup class="footnote-ref"><a href="#fn3" id="fnref3:1">[3:1]</a></sup>.</li></ul><p>Turn on speculation and, if the slow host holds locality, shorten the wait; defaults shown:</p><div class="language-properties vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">properties</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Relaunch tasks stuck on a slow host (speculation is OFF by default)</span></span>
27
23
  <span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.speculation</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=true</span></span>
28
24
  <span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.speculation.quantile</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=0.9 </span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># default; fraction of tasks done before speculation starts</span></span>
29
25
  <span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.speculation.multiplier</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=3 </span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># default; a task must run &gt; 3x the median to be relaunched</span></span>
30
26
  <span class="line"></span>
31
27
  <span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># If the slow host holds data locality, wait less before falling back to another executor</span></span>
32
28
  <span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.locality.wait</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=3s </span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># default; lower to fall back sooner</span></span></code></pre></div><h2 id="confidence" tabindex="-1">Confidence <a class="header-anchor" href="#confidence" aria-label="Permalink to &quot;Confidence&quot;">​</a></h2><p>The core <code>durationShare</code> dimension is validated: it measures host-wide task-duration inflation against the cluster median, the signal that separates a genuinely slow machine from normal task-time variance. Two secondary dimensions run best-effort and stay experimental. The <code>multiDim</code> dimension <span class="tag">EXPERIMENTAL</span> folds several per-host signals into one score, but that combined heuristic has not been validated. The <code>storageMemory</code> dimension <span class="tag">EXPERIMENTAL</span> likewise reads host-level memory pressure as a contributing factor without validation, so treat both as hints rather than verdicts.</p><h2 id="limitations-false-positive-risk" tabindex="-1">Limitations / false-positive risk <a class="header-anchor" href="#limitations-false-positive-risk" aria-label="Permalink to &quot;Limitations / false-positive risk&quot;">​</a></h2><p>A host that looks slow is not always a bad node. It can simply hold data locality for the tasks scheduled there, or carry one heavy stage that happens to be pinned to it, either of which inflates its mean task time without any hardware fault. The multi-dimensional scoring that reinforces the core signal is best-effort, so a match warrants a look at what that host was actually running before you conclude the machine itself is the problem.</p><h2 id="bottleneck-stage-slowness" tabindex="-1">Stage slowness <a class="header-anchor" href="#bottleneck-stage-slowness" aria-label="Permalink to &quot;Stage slowness {#bottleneck-stage-slowness}&quot;">​</a></h2><p><span class="tag">SLOW</span> <span class="tag">EXPERIMENTAL</span></p><p>This is an experimental fallback heuristic that shares the slow-host anchor and is suppressed whenever <code>slowHost</code> fires. It answers a different question: when no single machine is dragging, why does one stage still lag the rest of the job? The cause usually traces back to how the work was divided rather than where it ran.</p><p><strong>Too few partitions caps parallelism.</strong> Wide stages (joins, groupBy, aggregations) take their partition count from <code>spark.sql.shuffle.partitions</code>, which sits at a default of 200 whether the shuffle moves 20 MB or 500 GB unless you change it<sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup>. When that count is small relative to the cluster, only a handful of tasks carry the stage while cores sit idle, so it stretches out even as better-sized neighbors finish quickly. The first move is to raise its parallelism, aiming for at least two or three tasks per CPU core on a data-heavy stage, tuned through <code>spark.default.parallelism</code> and <code>spark.sql.shuffle.partitions</code><sup class="footnote-ref"><a href="#fn5" id="fnref5">[5]</a></sup>.</p><p><strong>Large data volume per task makes each task run long.</strong> That 200-partition default does not scale with input size, so a stage handling hundreds of gigabytes hands each task a large slice: fewer tasks run at once, per-executor load climbs, and the stage often trips memory errors. Roughly 100-200 MB per task tends to work well, and tasks grinding through around 3 GB each while spilling are a sign you need more partitions<sup class="footnote-ref"><a href="#fn4" id="fnref4:1">[4:1]</a></sup>.</p><p><strong>Heavy shuffle and spill inflate the stage&#39;s cost.</strong> Shuffles move data across executors so rows with the same key land together, and during those map and <a href="./shuffle.html">shuffle</a> operations Spark writes to and reads from local-disk shuffle files, which is heavy I/O that can become a bottleneck under the default configuration<sup class="footnote-ref"><a href="#fn2" id="fnref2:1">[2:1]</a></sup>. Volume per task feeds straight into it: once a partition outgrows the memory available in an executor, Spark <a href="./bottleneck-spill.html">spills</a> part of the data to disk, and spills are about the slowest thing a job can do because of the extra disk I/O and garbage collection, so the stage finishes but runs far less efficiently than the rest<sup class="footnote-ref"><a href="#fn4" id="fnref4:2">[4:2]</a></sup>.</p><h2 id="related" tabindex="-1">Related <a class="header-anchor" href="#related" aria-label="Permalink to &quot;Related&quot;">​</a></h2><ul><li><strong>Speculation &amp; locality tuning:</strong> <a href="./cluster-config.html">Cluster Tuning</a></li></ul><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/monitoring.html#spark-history-server" target="_blank" rel="noreferrer">Monitoring and Instrumentation</a> <a href="#fnref1" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><em>Learning Spark, 2nd Edition</em>, Damji, Wenig, Das, Lee, ch. 7 <a href="#fnref2" class="footnote-backref">↩︎</a> <a href="#fnref2:1" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration (Spark)</a> <a href="#fnref3" class="footnote-backref">↩︎</a> <a href="#fnref3:1" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><a href="https://luminousmen.com/post/spark-partitions" target="_blank" rel="noreferrer">Spark Partitions</a> <a href="#fnref4" class="footnote-backref">↩︎</a> <a href="#fnref4:1" class="footnote-backref">↩︎</a> <a href="#fnref4:2" class="footnote-backref">↩︎</a></p></li><li id="fn5" class="footnote-item"><p><em>Spark: The Definitive Guide</em>, Chambers &amp; Zaharia (O&#39;Reilly, 2018) <a href="#fnref5" class="footnote-backref">↩︎</a></p></li></ol></section></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/tuning-reference/bottleneck-utilization.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>Executor Utilization</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="/docs/tuning-reference/bottleneck-failures.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>Task Failures</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
33
- <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"YQcj5EI4\",\"contributor-guide_architecture_board-widgets_caching-and-memory.md\":\"8bH1HnZp\",\"contributor-guide_architecture_board-widgets_plan-advisor.md\":\"RlzvUmNm\",\"contributor-guide_architecture_board-widgets_visual-system-and-templating.md\":\"DYgUpIJB\",\"contributor-guide_architecture_detector-contract.md\":\"wj6Et6UD\",\"contributor-guide_architecture_detector-contract_plan-attribution.md\":\"yNex4bys\",\"contributor-guide_architecture_detector-contract_threshold-tables.md\":\"CKjWG56Y\",\"contributor-guide_architecture_drill-down.md\":\"CqZPrdo9\",\"contributor-guide_architecture_drill-down_plan-graph.md\":\"DwOozIEs\",\"contributor-guide_architecture_impact-estimation.md\":\"CZv_ZGOp\",\"contributor-guide_architecture_impact-estimation_caveats-tuning-and-coverage.md\":\"BUucRc0A\",\"contributor-guide_architecture_impact-estimation_stage-union-rollup.md\":\"DdCGtAGu\",\"contributor-guide_architecture_index.md\":\"tymxf1Fh\",\"contributor-guide_architecture_overview.md\":\"C1sSvm9i\",\"contributor-guide_architecture_state-and-history.md\":\"DyCBdjuM\",\"contributor-guide_architecture_state-and-history_history-server-intake.md\":\"C298hrHZ\",\"contributor-guide_architecture_state-and-history_run-comparison-internals.md\":\"CZA0JLtV\",\"contributor-guide_architecture_widget-rendering.md\":\"BOZ1azXG\",\"contributor-guide_architecture_widget-rendering_findings-tab.md\":\"BQXycK6G\",\"contributor-guide_architecture_widget-rendering_investigation-routing.md\":\"BGdiyElu\",\"contributor-guide_architecture_widget-rendering_list-sort-mode.md\":\"BuHKgH0C\",\"contributor-guide_architecture_widget-rendering_reference-section.md\":\"DWqC80bN\",\"contributor-guide_architecture_widget-rendering_row-expand-contract.md\":\"ZT89u9f1\",\"contributor-guide_architecture_worker-protocol.md\":\"BAnhi2-d\",\"contributor-guide_architecture_worker-protocol_cli-and-mcp.md\":\"DC1YqXBp\",\"contributor-guide_architecture_worker-protocol_core-and-event-validation.md\":\"DSVFPpWb\",\"contributor-guide_architecture_worker-protocol_evidence-availability.md\":\"CjimIDBY\",\"contributor-guide_architecture_worker-protocol_evidence-report.md\":\"DVGZOWvW\",\"contributor-guide_contributing.md\":\"CQMvGwFt\",\"contributor-guide_development-setup.md\":\"D9bu4j_c\",\"contributor-guide_testing.md\":\"By6XGR9a\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"D08Uaod7\",\"tuning-reference_aqe.md\":\"C1Ae6A8u\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"6wjKuJNE\",\"tuning-reference_bottleneck-cold-start.md\":\"BVQbsuHP\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"DtaYnaiF\",\"tuning-reference_bottleneck-failures.md\":\"Z1Tdmdd3\",\"tuning-reference_bottleneck-gc.md\":\"BCnXLf1n\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"CcvQj2g9\",\"tuning-reference_bottleneck-memory-utilization.md\":\"B-C0-8h-\",\"tuning-reference_bottleneck-retry-waste.md\":\"DCnhWiqj\",\"tuning-reference_bottleneck-shuffle.md\":\"B5OMJCd3\",\"tuning-reference_bottleneck-skew.md\":\"Cvo6NQm_\",\"tuning-reference_bottleneck-slow-host.md\":\"ja5qr3wq\",\"tuning-reference_bottleneck-small-files.md\":\"Crjtb0i2\",\"tuning-reference_bottleneck-spill.md\":\"Bl8KtSRg\",\"tuning-reference_bottleneck-straggler.md\":\"B0VN8reT\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"DQCftGhu\",\"tuning-reference_bottleneck-utilization.md\":\"BXDl30f9\",\"tuning-reference_caching.md\":\"CcaiMn_7\",\"tuning-reference_cluster-config.md\":\"DbWG1yZq\",\"tuning-reference_config.md\":\"YUqSpwFB\",\"tuning-reference_data-formats.md\":\"C5DX5xL4\",\"tuning-reference_index.md\":\"ZVIpQ6Dk\",\"tuning-reference_intro.md\":\"BSSaDlqx\",\"tuning-reference_joins.md\":\"BVhNCDnE\",\"tuning-reference_memory-model.md\":\"CfjFdgbl\",\"tuning-reference_metrics.md\":\"C64VNEL4\",\"tuning-reference_partitioning.md\":\"CtLK0xsS\",\"tuning-reference_pyspark.md\":\"D-92XgF8\",\"tuning-reference_shuffle.md\":\"Iq2v1oHJ\",\"tuning-reference_spark-architecture.md\":\"_qwnE_Sc\",\"tuning-reference_table-formats.md\":\"BSYEK-eN\",\"user-guide_alternative-log-retrieval.md\":\"CJpYvHGo\",\"user-guide_alternative-log-retrieval_managed-platforms.md\":\"DvDwHNtC\",\"user-guide_alternative-log-retrieval_ssh-bastion.md\":\"Gszzlplv\",\"user-guide_getting-started.md\":\"DYHHHkQR\",\"user-guide_getting-started_ci-and-automation.md\":\"OUQqw3mI\",\"user-guide_getting-started_reading-the-dashboard.md\":\"Obf3W3__\",\"user-guide_getting-started_regression-metric-keys.md\":\"B_gzr2Mw\",\"user-guide_getting-started_tuning-thresholds.md\":\"B_MyjyZm\",\"user-guide_mcp-tools.md\":\"C65SjWUL\",\"user-guide_mcp-tools_compare-runs.md\":\"Cc_gLoix\",\"user-guide_mcp-tools_diagnose-run.md\":\"BP1vnHA5\",\"user-guide_mcp-tools_evaluate-budgets.md\":\"BB0nDY3Z\",\"user-guide_mcp-tools_get-finding-documentation.md\":\"BtTTHNV6\",\"user-guide_mcp-tools_get-finding-evidence.md\":\"rKyu4wts\",\"user-guide_mcp-tools_get-reference-doc.md\":\"Yac4tH3u\",\"user-guide_mcp-tools_get-run-summary.md\":\"DpxZ4ZNp\",\"user-guide_mcp-tools_list-runs.md\":\"DpQPeCM3\",\"user-guide_run-comparison.md\":\"BUALkohi\",\"user-guide_run-comparison_how-stages-are-matched.md\":\"DSXGu2tA\",\"user-guide_understanding-findings.md\":\"C9YRruMT\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\",\"collapsed\":true,\"items\":[{\"text\":\"Reading the dashboard\",\"link\":\"/user-guide/getting-started/reading-the-dashboard\"},{\"text\":\"CI and automation\",\"link\":\"/user-guide/getting-started/ci-and-automation\"},{\"text\":\"Tuning detector thresholds\",\"link\":\"/user-guide/getting-started/tuning-thresholds\"},{\"text\":\"Regression metric keys\",\"link\":\"/user-guide/getting-started/regression-metric-keys\"}]},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\",\"collapsed\":true,\"items\":[{\"text\":\"How stages are matched\",\"link\":\"/user-guide/run-comparison/how-stages-are-matched\"}]},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\",\"collapsed\":true,\"items\":[{\"text\":\"diagnose_run\",\"link\":\"/user-guide/mcp-tools/diagnose-run\"},{\"text\":\"get_run_summary\",\"link\":\"/user-guide/mcp-tools/get-run-summary\"},{\"text\":\"evaluate_budgets\",\"link\":\"/user-guide/mcp-tools/evaluate-budgets\"},{\"text\":\"compare_runs\",\"link\":\"/user-guide/mcp-tools/compare-runs\"},{\"text\":\"get_finding_evidence\",\"link\":\"/user-guide/mcp-tools/get-finding-evidence\"},{\"text\":\"get_finding_documentation\",\"link\":\"/user-guide/mcp-tools/get-finding-documentation\"},{\"text\":\"get_reference_doc\",\"link\":\"/user-guide/mcp-tools/get-reference-doc\"},{\"text\":\"list_runs\",\"link\":\"/user-guide/mcp-tools/list-runs\"}]},{\"text\":\"Finding your event log\",\"link\":\"/user-guide/alternative-log-retrieval\",\"collapsed\":true,\"items\":[{\"text\":\"Managed platforms\",\"link\":\"/user-guide/alternative-log-retrieval/managed-platforms\"},{\"text\":\"Behind an SSH bastion\",\"link\":\"/user-guide/alternative-log-retrieval/ssh-bastion\"}]}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\",\"collapsed\":true,\"items\":[{\"text\":\"Evidence availability\",\"link\":\"/contributor-guide/architecture/worker-protocol/evidence-availability\"},{\"text\":\"Evidence report and finding identity\",\"link\":\"/contributor-guide/architecture/worker-protocol/evidence-report\"},{\"text\":\"CLI and MCP server\",\"link\":\"/contributor-guide/architecture/worker-protocol/cli-and-mcp\"},{\"text\":\"Core and event validation\",\"link\":\"/contributor-guide/architecture/worker-protocol/core-and-event-validation\"}]},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\",\"collapsed\":true,\"items\":[{\"text\":\"Run comparison internals\",\"link\":\"/contributor-guide/architecture/state-and-history/run-comparison-internals\"},{\"text\":\"History Server intake\",\"link\":\"/contributor-guide/architecture/state-and-history/history-server-intake\"}]},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\",\"collapsed\":true,\"items\":[{\"text\":\"Plan attribution\",\"link\":\"/contributor-guide/architecture/detector-contract/plan-attribution\"},{\"text\":\"Threshold tables\",\"link\":\"/contributor-guide/architecture/detector-contract/threshold-tables\"}]},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\",\"collapsed\":true,\"items\":[{\"text\":\"Stage union rollup\",\"link\":\"/contributor-guide/architecture/impact-estimation/stage-union-rollup\"},{\"text\":\"Caveats, tuning and coverage\",\"link\":\"/contributor-guide/architecture/impact-estimation/caveats-tuning-and-coverage\"}]},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\",\"collapsed\":true,\"items\":[{\"text\":\"Findings tab\",\"link\":\"/contributor-guide/architecture/widget-rendering/findings-tab\"},{\"text\":\"List sort mode\",\"link\":\"/contributor-guide/architecture/widget-rendering/list-sort-mode\"},{\"text\":\"Row/expand contract\",\"link\":\"/contributor-guide/architecture/widget-rendering/row-expand-contract\"},{\"text\":\"ReferenceSection\",\"link\":\"/contributor-guide/architecture/widget-rendering/reference-section\"},{\"text\":\"Investigation routing\",\"link\":\"/contributor-guide/architecture/widget-rendering/investigation-routing\"}]},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\",\"collapsed\":true,\"items\":[{\"text\":\"Plan Advisor\",\"link\":\"/contributor-guide/architecture/board-widgets/plan-advisor\"},{\"text\":\"Caching and memory widgets\",\"link\":\"/contributor-guide/architecture/board-widgets/caching-and-memory\"},{\"text\":\"Visual system and templating\",\"link\":\"/contributor-guide/architecture/board-widgets/visual-system-and-templating\"}]},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\",\"collapsed\":true,\"items\":[{\"text\":\"Plan graph\",\"link\":\"/contributor-guide/architecture/drill-down/plan-graph\"}]}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":{\"selector\":\"[data-sf-anchor-offset]\",\"padding\":0},\"cleanUrls\":false}");</script>
29
+ <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"CqKM_jEG\",\"contributor-guide_architecture_board-widgets_caching-and-memory.md\":\"BYDo3ClR\",\"contributor-guide_architecture_board-widgets_plan-advisor.md\":\"ajUAVdbc\",\"contributor-guide_architecture_board-widgets_visual-system-and-templating.md\":\"WqY74T0T\",\"contributor-guide_architecture_detector-contract.md\":\"CHHgywoP\",\"contributor-guide_architecture_detector-contract_plan-attribution.md\":\"DIz3_oz9\",\"contributor-guide_architecture_detector-contract_threshold-tables.md\":\"WH4tRHPL\",\"contributor-guide_architecture_drill-down.md\":\"yB-Hz2XT\",\"contributor-guide_architecture_drill-down_plan-graph.md\":\"BQZE04sw\",\"contributor-guide_architecture_impact-estimation.md\":\"K71YB-Rr\",\"contributor-guide_architecture_impact-estimation_caveats-tuning-and-coverage.md\":\"DLCAffwn\",\"contributor-guide_architecture_impact-estimation_stage-union-rollup.md\":\"JFrzDMwo\",\"contributor-guide_architecture_index.md\":\"tymxf1Fh\",\"contributor-guide_architecture_overview.md\":\"nxkpOhgf\",\"contributor-guide_architecture_state-and-history.md\":\"DyCBdjuM\",\"contributor-guide_architecture_state-and-history_history-server-intake.md\":\"C298hrHZ\",\"contributor-guide_architecture_state-and-history_run-comparison-internals.md\":\"U-9KMDFK\",\"contributor-guide_architecture_widget-rendering.md\":\"BrSz8Ezh\",\"contributor-guide_architecture_widget-rendering_findings-tab.md\":\"B7rjSF_h\",\"contributor-guide_architecture_widget-rendering_investigation-routing.md\":\"BGdiyElu\",\"contributor-guide_architecture_widget-rendering_list-sort-mode.md\":\"37hcZJ7q\",\"contributor-guide_architecture_widget-rendering_reference-section.md\":\"DWqC80bN\",\"contributor-guide_architecture_widget-rendering_row-expand-contract.md\":\"mK_VmAWk\",\"contributor-guide_architecture_worker-protocol.md\":\"Br6OE7RW\",\"contributor-guide_architecture_worker-protocol_cli-and-mcp.md\":\"IVyaXrGe\",\"contributor-guide_architecture_worker-protocol_core-and-event-validation.md\":\"DVSXn-bb\",\"contributor-guide_architecture_worker-protocol_evidence-availability.md\":\"D3S5cyOW\",\"contributor-guide_architecture_worker-protocol_evidence-report.md\":\"CtmCU57I\",\"contributor-guide_contributing.md\":\"DBW4drtZ\",\"contributor-guide_development-setup.md\":\"B21ZTDhn\",\"contributor-guide_testing.md\":\"DpkozNoG\",\"index.md\":\"lMm7SUMz\",\"tuning-reference_anti-patterns.md\":\"D08Uaod7\",\"tuning-reference_aqe.md\":\"C1Ae6A8u\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"6wjKuJNE\",\"tuning-reference_bottleneck-cold-start.md\":\"BVQbsuHP\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"DtaYnaiF\",\"tuning-reference_bottleneck-failures.md\":\"Z1Tdmdd3\",\"tuning-reference_bottleneck-gc.md\":\"BCnXLf1n\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"CcvQj2g9\",\"tuning-reference_bottleneck-memory-utilization.md\":\"B-C0-8h-\",\"tuning-reference_bottleneck-retry-waste.md\":\"DCnhWiqj\",\"tuning-reference_bottleneck-shuffle.md\":\"B5OMJCd3\",\"tuning-reference_bottleneck-skew.md\":\"Cvo6NQm_\",\"tuning-reference_bottleneck-slow-host.md\":\"ja5qr3wq\",\"tuning-reference_bottleneck-small-files.md\":\"Crjtb0i2\",\"tuning-reference_bottleneck-spill.md\":\"Bl8KtSRg\",\"tuning-reference_bottleneck-straggler.md\":\"B0VN8reT\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"DQCftGhu\",\"tuning-reference_bottleneck-utilization.md\":\"BXDl30f9\",\"tuning-reference_caching.md\":\"CcaiMn_7\",\"tuning-reference_cluster-config.md\":\"DbWG1yZq\",\"tuning-reference_config.md\":\"YUqSpwFB\",\"tuning-reference_data-formats.md\":\"C5DX5xL4\",\"tuning-reference_index.md\":\"ZVIpQ6Dk\",\"tuning-reference_intro.md\":\"BSSaDlqx\",\"tuning-reference_joins.md\":\"BVhNCDnE\",\"tuning-reference_memory-model.md\":\"CfjFdgbl\",\"tuning-reference_metrics.md\":\"C64VNEL4\",\"tuning-reference_partitioning.md\":\"CtLK0xsS\",\"tuning-reference_pyspark.md\":\"D-92XgF8\",\"tuning-reference_shuffle.md\":\"Iq2v1oHJ\",\"tuning-reference_spark-architecture.md\":\"_qwnE_Sc\",\"tuning-reference_table-formats.md\":\"BSYEK-eN\",\"user-guide_alternative-log-retrieval.md\":\"CJpYvHGo\",\"user-guide_alternative-log-retrieval_managed-platforms.md\":\"DvDwHNtC\",\"user-guide_alternative-log-retrieval_ssh-bastion.md\":\"Gszzlplv\",\"user-guide_getting-started.md\":\"DYHHHkQR\",\"user-guide_getting-started_ci-and-automation.md\":\"C0pcQoNc\",\"user-guide_getting-started_reading-the-dashboard.md\":\"CGaxPX_1\",\"user-guide_getting-started_regression-metric-keys.md\":\"B_gzr2Mw\",\"user-guide_getting-started_tuning-thresholds.md\":\"BGVE-wrc\",\"user-guide_mcp-tools.md\":\"49r3L2YE\",\"user-guide_mcp-tools_compare-runs.md\":\"BQ_kIptC\",\"user-guide_mcp-tools_diagnose-run.md\":\"azCZXkH8\",\"user-guide_mcp-tools_evaluate-budgets.md\":\"dpQpRQ7z\",\"user-guide_mcp-tools_get-finding-documentation.md\":\"BtTTHNV6\",\"user-guide_mcp-tools_get-finding-evidence.md\":\"CH6FkNLX\",\"user-guide_mcp-tools_get-reference-doc.md\":\"Yac4tH3u\",\"user-guide_mcp-tools_get-run-summary.md\":\"EhaIsele\",\"user-guide_mcp-tools_list-runs.md\":\"DpQPeCM3\",\"user-guide_run-comparison.md\":\"B_LOkdBf\",\"user-guide_run-comparison_how-stages-are-matched.md\":\"DSXGu2tA\",\"user-guide_understanding-findings.md\":\"C4riKCKl\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":{\"light\":\"/logo-light.svg\",\"dark\":\"/logo-dark.svg\"},\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"},{\"text\":\"Open app\",\"link\":\"/..\",\"target\":\"_self\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\",\"collapsed\":true,\"items\":[{\"text\":\"Reading the dashboard\",\"link\":\"/user-guide/getting-started/reading-the-dashboard\"},{\"text\":\"CI and automation\",\"link\":\"/user-guide/getting-started/ci-and-automation\"},{\"text\":\"Tuning detector thresholds\",\"link\":\"/user-guide/getting-started/tuning-thresholds\"},{\"text\":\"Regression metric keys\",\"link\":\"/user-guide/getting-started/regression-metric-keys\"}]},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\",\"collapsed\":true,\"items\":[{\"text\":\"How stages are matched\",\"link\":\"/user-guide/run-comparison/how-stages-are-matched\"}]},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\",\"collapsed\":true,\"items\":[{\"text\":\"diagnose_run\",\"link\":\"/user-guide/mcp-tools/diagnose-run\"},{\"text\":\"get_run_summary\",\"link\":\"/user-guide/mcp-tools/get-run-summary\"},{\"text\":\"evaluate_budgets\",\"link\":\"/user-guide/mcp-tools/evaluate-budgets\"},{\"text\":\"compare_runs\",\"link\":\"/user-guide/mcp-tools/compare-runs\"},{\"text\":\"get_finding_evidence\",\"link\":\"/user-guide/mcp-tools/get-finding-evidence\"},{\"text\":\"get_finding_documentation\",\"link\":\"/user-guide/mcp-tools/get-finding-documentation\"},{\"text\":\"get_reference_doc\",\"link\":\"/user-guide/mcp-tools/get-reference-doc\"},{\"text\":\"list_runs\",\"link\":\"/user-guide/mcp-tools/list-runs\"}]},{\"text\":\"Finding your event log\",\"link\":\"/user-guide/alternative-log-retrieval\",\"collapsed\":true,\"items\":[{\"text\":\"Managed platforms\",\"link\":\"/user-guide/alternative-log-retrieval/managed-platforms\"},{\"text\":\"Behind an SSH bastion\",\"link\":\"/user-guide/alternative-log-retrieval/ssh-bastion\"}]}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\",\"collapsed\":true,\"items\":[{\"text\":\"Evidence availability\",\"link\":\"/contributor-guide/architecture/worker-protocol/evidence-availability\"},{\"text\":\"Evidence report and finding identity\",\"link\":\"/contributor-guide/architecture/worker-protocol/evidence-report\"},{\"text\":\"CLI and MCP server\",\"link\":\"/contributor-guide/architecture/worker-protocol/cli-and-mcp\"},{\"text\":\"Core and event validation\",\"link\":\"/contributor-guide/architecture/worker-protocol/core-and-event-validation\"}]},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\",\"collapsed\":true,\"items\":[{\"text\":\"Run comparison internals\",\"link\":\"/contributor-guide/architecture/state-and-history/run-comparison-internals\"},{\"text\":\"History Server intake\",\"link\":\"/contributor-guide/architecture/state-and-history/history-server-intake\"}]},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\",\"collapsed\":true,\"items\":[{\"text\":\"Plan attribution\",\"link\":\"/contributor-guide/architecture/detector-contract/plan-attribution\"},{\"text\":\"Threshold tables\",\"link\":\"/contributor-guide/architecture/detector-contract/threshold-tables\"}]},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\",\"collapsed\":true,\"items\":[{\"text\":\"Stage union rollup\",\"link\":\"/contributor-guide/architecture/impact-estimation/stage-union-rollup\"},{\"text\":\"Caveats, tuning and coverage\",\"link\":\"/contributor-guide/architecture/impact-estimation/caveats-tuning-and-coverage\"}]},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\",\"collapsed\":true,\"items\":[{\"text\":\"Findings tab\",\"link\":\"/contributor-guide/architecture/widget-rendering/findings-tab\"},{\"text\":\"List sort mode\",\"link\":\"/contributor-guide/architecture/widget-rendering/list-sort-mode\"},{\"text\":\"Row/expand contract\",\"link\":\"/contributor-guide/architecture/widget-rendering/row-expand-contract\"},{\"text\":\"ReferenceSection\",\"link\":\"/contributor-guide/architecture/widget-rendering/reference-section\"},{\"text\":\"Investigation routing\",\"link\":\"/contributor-guide/architecture/widget-rendering/investigation-routing\"}]},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\",\"collapsed\":true,\"items\":[{\"text\":\"Plan Advisor\",\"link\":\"/contributor-guide/architecture/board-widgets/plan-advisor\"},{\"text\":\"Caching and memory widgets\",\"link\":\"/contributor-guide/architecture/board-widgets/caching-and-memory\"},{\"text\":\"Visual system and templating\",\"link\":\"/contributor-guide/architecture/board-widgets/visual-system-and-templating\"}]},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\",\"collapsed\":true,\"items\":[{\"text\":\"Plan graph\",\"link\":\"/contributor-guide/architecture/drill-down/plan-graph\"}]}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":{\"selector\":\"[data-sf-anchor-offset]\",\"padding\":0},\"cleanUrls\":false}");</script>
34
30
 
35
31
  </body>
36
32
  </html>