sparkforensics-server 0.5.0 → 0.5.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (309) hide show
  1. package/package.json +3 -3
  2. package/public/assets/{AutoscalingChurn-9Zo6IMGS.js → AutoscalingChurn-8t3U3isq.js} +1 -1
  3. package/public/assets/{CacheUtilization-MBDfMsKC.js → CacheUtilization-CJs9tTum.js} +1 -1
  4. package/public/assets/CachingOpportunity-D1NcHs0V.js +1 -0
  5. package/public/assets/{ColdStart-BtS12b6_.js → ColdStart-CdRJ9gD8.js} +1 -1
  6. package/public/assets/{ConfigAudit-DNPV6EQg.js → ConfigAudit-B5Gbe5js.js} +1 -1
  7. package/public/assets/CoreUsageArea-a--NTRGY.js +1 -0
  8. package/public/assets/{Dashboard-CC_BXrNy.js → Dashboard-C1hNeX49.js} +2 -2
  9. package/public/assets/DisclosureContext-ZpE6CkHx.js +1 -0
  10. package/public/assets/{DuplicatePlanSubtree-vN3zqRsi.js → DuplicatePlanSubtree-dobXdz7E.js} +1 -1
  11. package/public/assets/{ExecutorUtilization-CZObSLIj.js → ExecutorUtilization-D9grwi7t.js} +1 -1
  12. package/public/assets/{GcPressure-ByJlLOua.js → GcPressure-8-G2ri10.js} +1 -1
  13. package/public/assets/{IncompleteRun-MzqV4W8y.js → IncompleteRun-B91lky7a.js} +1 -1
  14. package/public/assets/{JobFailures-LbMvAwAb.js → JobFailures-BsWvBMyJ.js} +1 -1
  15. package/public/assets/{MemoryUtilization-DH4jYvJK.js → MemoryUtilization-BM5schZI.js} +1 -1
  16. package/public/assets/{OverBroadcast-euk6HKg4.js → OverBroadcast-DdN8JIJH.js} +1 -1
  17. package/public/assets/{PartitionHint-DThaMMRT.js → PartitionHint-CUvbnnxy.js} +1 -1
  18. package/public/assets/{PartitionSizing-yIN-iZzx.js → PartitionSizing-CY5HDYtp.js} +1 -1
  19. package/public/assets/{PlanExplorer-Dd9WPK4S.js → PlanExplorer-DgHCDBD_.js} +1 -1
  20. package/public/assets/PlanGraphRoute-DH3KJb8j.js +1 -0
  21. package/public/assets/{RetryWaste-CXH4D-xH.js → RetryWaste-DOnE9vGF.js} +1 -1
  22. package/public/assets/{ShuffleIO-CW7U68ph.js → ShuffleIO-MMnDESkF.js} +1 -1
  23. package/public/assets/{Skew-PL79TppX.js → Skew-BmZZJ5ms.js} +1 -1
  24. package/public/assets/{SlowHost-GBO73hCi.js → SlowHost-CsJzfEqo.js} +1 -1
  25. package/public/assets/{SmallFiles-B5cUjEav.js → SmallFiles-BoK3R1Ex.js} +1 -1
  26. package/public/assets/{SpeculationWaste-mxi3jgb0.js → SpeculationWaste-CnVxB5AJ.js} +1 -1
  27. package/public/assets/Spill-BhNicyBy.js +1 -0
  28. package/public/assets/{StageFailed-YvxC11ii.js → StageFailed-mAhHU4l_.js} +1 -1
  29. package/public/assets/StageFindingGroup-BEtPWcT0.js +1 -0
  30. package/public/assets/{StageHeader-BMNmUy-s.js → StageHeader-DJkYkrsU.js} +1 -1
  31. package/public/assets/{StageShape-DO3mQ6NS.js → StageShape-BDxRknLp.js} +1 -1
  32. package/public/assets/{StageSlowness-SwRxfAEx.js → StageSlowness-BPRw-otN.js} +1 -1
  33. package/public/assets/{Straggler-DHLTiBY-.js → Straggler-CZCbnOWP.js} +1 -1
  34. package/public/assets/{TaskFailures-Bj0QxiOc.js → TaskFailures-C57YcLFQ.js} +1 -1
  35. package/public/assets/{TinyTask-COUQQ5bQ.js → TinyTask-Bi84qU7n.js} +1 -1
  36. package/public/assets/{UnderBroadcast-RAUbQ6vw.js → UnderBroadcast-DdQV7htE.js} +1 -1
  37. package/public/assets/{finding-anchor-Dl8mIHiU.js → finding-anchor-Cl6_5hfV.js} +1 -1
  38. package/public/assets/{impact-sort-Eihua9dY.js → impact-sort-CssZ6MT2.js} +1 -1
  39. package/public/assets/index-2h_TgUAb.css +2 -0
  40. package/public/assets/index-B-03mG7K.js +14 -0
  41. package/public/assets/parser-worker-CA5RxqKc.js +122 -0
  42. package/public/assets/{plan-graph-vendor-B0sEms9Z.js → plan-graph-vendor-BYl-AzN5.js} +1 -1
  43. package/public/assets/{react-vendor-BAKt3vny.js → react-vendor-DEdfrRTJ.js} +1 -1
  44. package/public/assets/{select-DkjFHlXR.js → select-CQjNhprm.js} +1 -1
  45. package/public/assets/{usePagedRows-Bo-aSztx.js → usePagedRows-Dyl6T8Ok.js} +1 -1
  46. package/public/assets/useRecentFiles-XZEnLlal.js +11 -0
  47. package/public/assets/{useSortMode-lDedD5a3.js → useSortMode-Dhvit8dt.js} +1 -1
  48. package/public/docs/404.html +2 -2
  49. package/public/docs/assets/{app.yjczoibf.js → app.eYTPqTeq.js} +1 -1
  50. package/public/docs/assets/chunks/@localSearchIndexroot.u_e5bQz4.js +1 -0
  51. package/public/docs/assets/chunks/{VPLocalSearchBox.DOBbPfix.js → VPLocalSearchBox.DsZrQKQR.js} +1 -1
  52. package/public/docs/assets/chunks/{theme.DFcAl8Ws.js → theme.DsTNE2pE.js} +2 -2
  53. package/public/docs/assets/contributor-guide_architecture_board-widgets.md.YQcj5EI4.js +1 -0
  54. package/public/docs/assets/{contributor-guide_architecture_board-widgets.md.D0Kkk_r5.lean.js → contributor-guide_architecture_board-widgets.md.YQcj5EI4.lean.js} +1 -1
  55. package/public/docs/assets/contributor-guide_architecture_board-widgets_caching-and-memory.md.8bH1HnZp.js +1 -0
  56. package/public/docs/assets/contributor-guide_architecture_board-widgets_caching-and-memory.md.8bH1HnZp.lean.js +1 -0
  57. package/public/docs/assets/contributor-guide_architecture_board-widgets_plan-advisor.md.RlzvUmNm.js +1 -0
  58. package/public/docs/assets/contributor-guide_architecture_board-widgets_plan-advisor.md.RlzvUmNm.lean.js +1 -0
  59. package/public/docs/assets/contributor-guide_architecture_board-widgets_visual-system-and-templating.md.DYgUpIJB.js +1 -0
  60. package/public/docs/assets/contributor-guide_architecture_board-widgets_visual-system-and-templating.md.DYgUpIJB.lean.js +1 -0
  61. package/public/docs/assets/contributor-guide_architecture_detector-contract.md.wj6Et6UD.js +1 -0
  62. package/public/docs/assets/contributor-guide_architecture_detector-contract.md.wj6Et6UD.lean.js +1 -0
  63. package/public/docs/assets/contributor-guide_architecture_detector-contract_plan-attribution.md.yNex4bys.js +1 -0
  64. package/public/docs/assets/contributor-guide_architecture_detector-contract_plan-attribution.md.yNex4bys.lean.js +1 -0
  65. package/public/docs/assets/contributor-guide_architecture_detector-contract_threshold-tables.md.CKjWG56Y.js +1 -0
  66. package/public/docs/assets/contributor-guide_architecture_detector-contract_threshold-tables.md.CKjWG56Y.lean.js +1 -0
  67. package/public/docs/assets/contributor-guide_architecture_drill-down.md.CqZPrdo9.js +1 -0
  68. package/public/docs/assets/contributor-guide_architecture_drill-down.md.CqZPrdo9.lean.js +1 -0
  69. package/public/docs/assets/contributor-guide_architecture_drill-down_plan-graph.md.DwOozIEs.js +1 -0
  70. package/public/docs/assets/contributor-guide_architecture_drill-down_plan-graph.md.DwOozIEs.lean.js +1 -0
  71. package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.CZv_ZGOp.js +1 -0
  72. package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.CZv_ZGOp.lean.js +1 -0
  73. package/public/docs/assets/contributor-guide_architecture_impact-estimation_caveats-tuning-and-coverage.md.BUucRc0A.js +1 -0
  74. package/public/docs/assets/contributor-guide_architecture_impact-estimation_caveats-tuning-and-coverage.md.BUucRc0A.lean.js +1 -0
  75. package/public/docs/assets/contributor-guide_architecture_impact-estimation_stage-union-rollup.md.DdCGtAGu.js +1 -0
  76. package/public/docs/assets/contributor-guide_architecture_impact-estimation_stage-union-rollup.md.DdCGtAGu.lean.js +1 -0
  77. package/public/docs/assets/contributor-guide_architecture_index.md.tymxf1Fh.js +1 -0
  78. package/public/docs/assets/{contributor-guide_architecture_index.md.B9yZieYL.lean.js → contributor-guide_architecture_index.md.tymxf1Fh.lean.js} +1 -1
  79. package/public/docs/assets/contributor-guide_architecture_overview.md.C1sSvm9i.js +1 -0
  80. package/public/docs/assets/contributor-guide_architecture_state-and-history.md.DyCBdjuM.js +1 -0
  81. package/public/docs/assets/{contributor-guide_architecture_state-and-history.md.w4lwybdh.lean.js → contributor-guide_architecture_state-and-history.md.DyCBdjuM.lean.js} +1 -1
  82. package/public/docs/assets/contributor-guide_architecture_state-and-history_history-server-intake.md.C298hrHZ.js +1 -0
  83. package/public/docs/assets/contributor-guide_architecture_state-and-history_history-server-intake.md.C298hrHZ.lean.js +1 -0
  84. package/public/docs/assets/contributor-guide_architecture_state-and-history_run-comparison-internals.md.CZA0JLtV.js +1 -0
  85. package/public/docs/assets/contributor-guide_architecture_state-and-history_run-comparison-internals.md.CZA0JLtV.lean.js +1 -0
  86. package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.BOZ1azXG.js +1 -0
  87. package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.BOZ1azXG.lean.js +1 -0
  88. package/public/docs/assets/contributor-guide_architecture_widget-rendering_findings-tab.md.BQXycK6G.js +1 -0
  89. package/public/docs/assets/contributor-guide_architecture_widget-rendering_findings-tab.md.BQXycK6G.lean.js +1 -0
  90. package/public/docs/assets/contributor-guide_architecture_widget-rendering_investigation-routing.md.BGdiyElu.js +1 -0
  91. package/public/docs/assets/contributor-guide_architecture_widget-rendering_investigation-routing.md.BGdiyElu.lean.js +1 -0
  92. package/public/docs/assets/contributor-guide_architecture_widget-rendering_list-sort-mode.md.BuHKgH0C.js +1 -0
  93. package/public/docs/assets/contributor-guide_architecture_widget-rendering_list-sort-mode.md.BuHKgH0C.lean.js +1 -0
  94. package/public/docs/assets/contributor-guide_architecture_widget-rendering_reference-section.md.DWqC80bN.js +1 -0
  95. package/public/docs/assets/contributor-guide_architecture_widget-rendering_reference-section.md.DWqC80bN.lean.js +1 -0
  96. package/public/docs/assets/contributor-guide_architecture_widget-rendering_row-expand-contract.md.ZT89u9f1.js +1 -0
  97. package/public/docs/assets/contributor-guide_architecture_widget-rendering_row-expand-contract.md.ZT89u9f1.lean.js +1 -0
  98. package/public/docs/assets/contributor-guide_architecture_worker-protocol.md.BAnhi2-d.js +1 -0
  99. package/public/docs/assets/contributor-guide_architecture_worker-protocol.md.BAnhi2-d.lean.js +1 -0
  100. package/public/docs/assets/contributor-guide_architecture_worker-protocol_cli-and-mcp.md.DC1YqXBp.js +1 -0
  101. package/public/docs/assets/contributor-guide_architecture_worker-protocol_cli-and-mcp.md.DC1YqXBp.lean.js +1 -0
  102. package/public/docs/assets/contributor-guide_architecture_worker-protocol_core-and-event-validation.md.DSVFPpWb.js +1 -0
  103. package/public/docs/assets/contributor-guide_architecture_worker-protocol_core-and-event-validation.md.DSVFPpWb.lean.js +1 -0
  104. package/public/docs/assets/contributor-guide_architecture_worker-protocol_evidence-availability.md.CjimIDBY.js +6 -0
  105. package/public/docs/assets/contributor-guide_architecture_worker-protocol_evidence-availability.md.CjimIDBY.lean.js +1 -0
  106. package/public/docs/assets/contributor-guide_architecture_worker-protocol_evidence-report.md.DVGZOWvW.js +1 -0
  107. package/public/docs/assets/contributor-guide_architecture_worker-protocol_evidence-report.md.DVGZOWvW.lean.js +1 -0
  108. package/public/docs/assets/user-guide_alternative-log-retrieval.md.CJpYvHGo.js +4 -0
  109. package/public/docs/assets/user-guide_alternative-log-retrieval.md.CJpYvHGo.lean.js +1 -0
  110. package/public/docs/assets/user-guide_alternative-log-retrieval_managed-platforms.md.DvDwHNtC.js +4 -0
  111. package/public/docs/assets/user-guide_alternative-log-retrieval_managed-platforms.md.DvDwHNtC.lean.js +1 -0
  112. package/public/docs/assets/user-guide_alternative-log-retrieval_ssh-bastion.md.Gszzlplv.js +1 -0
  113. package/public/docs/assets/user-guide_alternative-log-retrieval_ssh-bastion.md.Gszzlplv.lean.js +1 -0
  114. package/public/docs/assets/user-guide_getting-started.md.DYHHHkQR.js +1 -0
  115. package/public/docs/assets/user-guide_getting-started.md.DYHHHkQR.lean.js +1 -0
  116. package/public/docs/assets/user-guide_getting-started_ci-and-automation.md.OUQqw3mI.js +23 -0
  117. package/public/docs/assets/user-guide_getting-started_ci-and-automation.md.OUQqw3mI.lean.js +1 -0
  118. package/public/docs/assets/user-guide_getting-started_reading-the-dashboard.md.Obf3W3__.js +1 -0
  119. package/public/docs/assets/user-guide_getting-started_reading-the-dashboard.md.Obf3W3__.lean.js +1 -0
  120. package/public/docs/assets/user-guide_getting-started_regression-metric-keys.md.B_gzr2Mw.js +1 -0
  121. package/public/docs/assets/user-guide_getting-started_regression-metric-keys.md.B_gzr2Mw.lean.js +1 -0
  122. package/public/docs/assets/user-guide_getting-started_tuning-thresholds.md.B_MyjyZm.js +4 -0
  123. package/public/docs/assets/user-guide_getting-started_tuning-thresholds.md.B_MyjyZm.lean.js +1 -0
  124. package/public/docs/assets/user-guide_mcp-tools.md.C65SjWUL.js +16 -0
  125. package/public/docs/assets/user-guide_mcp-tools.md.C65SjWUL.lean.js +1 -0
  126. package/public/docs/assets/user-guide_mcp-tools_compare-runs.md.Cc_gLoix.js +56 -0
  127. package/public/docs/assets/user-guide_mcp-tools_compare-runs.md.Cc_gLoix.lean.js +1 -0
  128. package/public/docs/assets/user-guide_mcp-tools_diagnose-run.md.BP1vnHA5.js +23 -0
  129. package/public/docs/assets/user-guide_mcp-tools_diagnose-run.md.BP1vnHA5.lean.js +1 -0
  130. package/public/docs/assets/user-guide_mcp-tools_evaluate-budgets.md.BB0nDY3Z.js +26 -0
  131. package/public/docs/assets/user-guide_mcp-tools_evaluate-budgets.md.BB0nDY3Z.lean.js +1 -0
  132. package/public/docs/assets/user-guide_mcp-tools_get-finding-documentation.md.BtTTHNV6.js +14 -0
  133. package/public/docs/assets/user-guide_mcp-tools_get-finding-documentation.md.BtTTHNV6.lean.js +1 -0
  134. package/public/docs/assets/user-guide_mcp-tools_get-finding-evidence.md.rKyu4wts.js +20 -0
  135. package/public/docs/assets/user-guide_mcp-tools_get-finding-evidence.md.rKyu4wts.lean.js +1 -0
  136. package/public/docs/assets/user-guide_mcp-tools_get-reference-doc.md.Yac4tH3u.js +5 -0
  137. package/public/docs/assets/user-guide_mcp-tools_get-reference-doc.md.Yac4tH3u.lean.js +1 -0
  138. package/public/docs/assets/user-guide_mcp-tools_get-run-summary.md.DpxZ4ZNp.js +15 -0
  139. package/public/docs/assets/user-guide_mcp-tools_get-run-summary.md.DpxZ4ZNp.lean.js +1 -0
  140. package/public/docs/assets/user-guide_mcp-tools_list-runs.md.DpQPeCM3.js +12 -0
  141. package/public/docs/assets/user-guide_mcp-tools_list-runs.md.DpQPeCM3.lean.js +1 -0
  142. package/public/docs/assets/user-guide_run-comparison.md.BUALkohi.js +1 -0
  143. package/public/docs/assets/user-guide_run-comparison.md.BUALkohi.lean.js +1 -0
  144. package/public/docs/assets/user-guide_run-comparison_how-stages-are-matched.md.DSXGu2tA.js +1 -0
  145. package/public/docs/assets/user-guide_run-comparison_how-stages-are-matched.md.DSXGu2tA.lean.js +1 -0
  146. package/public/docs/assets/user-guide_understanding-findings.md.C9YRruMT.js +1 -0
  147. package/public/docs/assets/user-guide_understanding-findings.md.C9YRruMT.lean.js +1 -0
  148. package/public/docs/contributor-guide/architecture/board-widgets/caching-and-memory.html +30 -0
  149. package/public/docs/contributor-guide/architecture/board-widgets/plan-advisor.html +30 -0
  150. package/public/docs/contributor-guide/architecture/board-widgets/visual-system-and-templating.html +30 -0
  151. package/public/docs/contributor-guide/architecture/board-widgets.html +5 -5
  152. package/public/docs/contributor-guide/architecture/detector-contract/plan-attribution.html +30 -0
  153. package/public/docs/contributor-guide/architecture/detector-contract/threshold-tables.html +30 -0
  154. package/public/docs/contributor-guide/architecture/detector-contract.html +5 -5
  155. package/public/docs/contributor-guide/architecture/drill-down/plan-graph.html +30 -0
  156. package/public/docs/contributor-guide/architecture/drill-down.html +5 -5
  157. package/public/docs/contributor-guide/architecture/impact-estimation/caveats-tuning-and-coverage.html +30 -0
  158. package/public/docs/contributor-guide/architecture/impact-estimation/stage-union-rollup.html +30 -0
  159. package/public/docs/contributor-guide/architecture/impact-estimation.html +5 -5
  160. package/public/docs/contributor-guide/architecture/index.html +5 -5
  161. package/public/docs/contributor-guide/architecture/overview.html +5 -5
  162. package/public/docs/contributor-guide/architecture/state-and-history/history-server-intake.html +30 -0
  163. package/public/docs/contributor-guide/architecture/state-and-history/run-comparison-internals.html +30 -0
  164. package/public/docs/contributor-guide/architecture/state-and-history.html +5 -5
  165. package/public/docs/contributor-guide/architecture/widget-rendering/findings-tab.html +30 -0
  166. package/public/docs/contributor-guide/architecture/widget-rendering/investigation-routing.html +30 -0
  167. package/public/docs/contributor-guide/architecture/widget-rendering/list-sort-mode.html +30 -0
  168. package/public/docs/contributor-guide/architecture/widget-rendering/reference-section.html +30 -0
  169. package/public/docs/contributor-guide/architecture/widget-rendering/row-expand-contract.html +30 -0
  170. package/public/docs/contributor-guide/architecture/widget-rendering.html +5 -5
  171. package/public/docs/contributor-guide/architecture/worker-protocol/cli-and-mcp.html +30 -0
  172. package/public/docs/contributor-guide/architecture/worker-protocol/core-and-event-validation.html +30 -0
  173. package/public/docs/contributor-guide/architecture/worker-protocol/evidence-availability.html +35 -0
  174. package/public/docs/contributor-guide/architecture/worker-protocol/evidence-report.html +30 -0
  175. package/public/docs/contributor-guide/architecture/worker-protocol.html +5 -10
  176. package/public/docs/contributor-guide/contributing.html +4 -4
  177. package/public/docs/contributor-guide/development-setup.html +4 -4
  178. package/public/docs/contributor-guide/testing.html +4 -4
  179. package/public/docs/hashmap.json +1 -1
  180. package/public/docs/index.html +3 -3
  181. package/public/docs/tuning-reference/anti-patterns.html +3 -3
  182. package/public/docs/tuning-reference/aqe.html +3 -3
  183. package/public/docs/tuning-reference/bottleneck-broadcast-sizing.html +3 -3
  184. package/public/docs/tuning-reference/bottleneck-cold-start.html +3 -3
  185. package/public/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html +3 -3
  186. package/public/docs/tuning-reference/bottleneck-failures.html +3 -3
  187. package/public/docs/tuning-reference/bottleneck-gc.html +3 -3
  188. package/public/docs/tuning-reference/bottleneck-job-failure-rate.html +3 -3
  189. package/public/docs/tuning-reference/bottleneck-memory-utilization.html +3 -3
  190. package/public/docs/tuning-reference/bottleneck-retry-waste.html +3 -3
  191. package/public/docs/tuning-reference/bottleneck-shuffle.html +3 -3
  192. package/public/docs/tuning-reference/bottleneck-skew.html +3 -3
  193. package/public/docs/tuning-reference/bottleneck-slow-host.html +3 -3
  194. package/public/docs/tuning-reference/bottleneck-small-files.html +3 -3
  195. package/public/docs/tuning-reference/bottleneck-spill.html +3 -3
  196. package/public/docs/tuning-reference/bottleneck-straggler.html +3 -3
  197. package/public/docs/tuning-reference/bottleneck-tiny-tasks.html +3 -3
  198. package/public/docs/tuning-reference/bottleneck-utilization.html +3 -3
  199. package/public/docs/tuning-reference/caching.html +3 -3
  200. package/public/docs/tuning-reference/cluster-config.html +3 -3
  201. package/public/docs/tuning-reference/config.html +3 -3
  202. package/public/docs/tuning-reference/data-formats.html +3 -3
  203. package/public/docs/tuning-reference/index.html +3 -3
  204. package/public/docs/tuning-reference/intro.html +3 -3
  205. package/public/docs/tuning-reference/joins.html +3 -3
  206. package/public/docs/tuning-reference/memory-model.html +3 -3
  207. package/public/docs/tuning-reference/metrics.html +3 -3
  208. package/public/docs/tuning-reference/partitioning.html +3 -3
  209. package/public/docs/tuning-reference/pyspark.html +3 -3
  210. package/public/docs/tuning-reference/shuffle.html +3 -3
  211. package/public/docs/tuning-reference/spark-architecture.html +3 -3
  212. package/public/docs/tuning-reference/table-formats.html +3 -3
  213. package/public/docs/user-guide/alternative-log-retrieval/managed-platforms.html +33 -0
  214. package/public/docs/user-guide/alternative-log-retrieval/ssh-bastion.html +30 -0
  215. package/public/docs/user-guide/alternative-log-retrieval.html +6 -9
  216. package/public/docs/user-guide/getting-started/ci-and-automation.html +52 -0
  217. package/public/docs/user-guide/getting-started/reading-the-dashboard.html +30 -0
  218. package/public/docs/user-guide/getting-started/regression-metric-keys.html +30 -0
  219. package/public/docs/user-guide/getting-started/tuning-thresholds.html +33 -0
  220. package/public/docs/user-guide/getting-started.html +5 -28
  221. package/public/docs/user-guide/mcp-tools/compare-runs.html +85 -0
  222. package/public/docs/user-guide/mcp-tools/diagnose-run.html +52 -0
  223. package/public/docs/user-guide/mcp-tools/evaluate-budgets.html +55 -0
  224. package/public/docs/user-guide/mcp-tools/get-finding-documentation.html +43 -0
  225. package/public/docs/user-guide/mcp-tools/get-finding-evidence.html +49 -0
  226. package/public/docs/user-guide/mcp-tools/get-reference-doc.html +34 -0
  227. package/public/docs/user-guide/mcp-tools/get-run-summary.html +44 -0
  228. package/public/docs/user-guide/mcp-tools/list-runs.html +41 -0
  229. package/public/docs/user-guide/mcp-tools.html +8 -140
  230. package/public/docs/user-guide/run-comparison/how-stages-are-matched.html +30 -0
  231. package/public/docs/user-guide/run-comparison.html +5 -5
  232. package/public/docs/user-guide/understanding-findings.html +5 -5
  233. package/public/export-template.html +45 -45
  234. package/public/index.html +5 -5
  235. package/vendor-core/analyzer.js +1 -1
  236. package/vendor-core/api-merge-writes.js +80 -0
  237. package/vendor-core/cli/budgets.js +62 -2
  238. package/vendor-core/cli/regression-budgets.js +35 -0
  239. package/vendor-core/comparison-output.js +50 -0
  240. package/vendor-core/core-source-hash.txt +1 -1
  241. package/vendor-core/delta-commands.js +13 -0
  242. package/vendor-core/delta-log-paths.js +42 -0
  243. package/vendor-core/delta-targets.js +127 -0
  244. package/vendor-core/detectors.js +219 -56
  245. package/vendor-core/docs-content/detection/chrn.md +3 -1
  246. package/vendor-core/docs-content/detection/cold.md +2 -1
  247. package/vendor-core/docs-content/detection/plan.md +9 -2
  248. package/vendor-core/docs-content/detection/shfl.md +8 -2
  249. package/vendor-core/docs-content/detection/skew.md +11 -5
  250. package/vendor-core/docs-content/detection/slow.md +6 -2
  251. package/vendor-core/docs-content/detection/spill.md +4 -1
  252. package/vendor-core/docs-content/detection/strag.md +5 -3
  253. package/vendor-core/docs-content/detection/tiny.md +3 -2
  254. package/vendor-core/event-handlers.js +95 -9
  255. package/vendor-core/event-schemas.js +3 -1
  256. package/vendor-core/evidence-report.js +24 -15
  257. package/vendor-core/finding-presentation.js +39 -13
  258. package/vendor-core/finding-types.js +50 -6
  259. package/vendor-core/mcp-server-factory.js +17 -6
  260. package/vendor-core/mcp-tools.js +53 -35
  261. package/vendor-core/median.js +8 -0
  262. package/vendor-core/plan-graph-model.js +1 -1
  263. package/vendor-core/plan-summary.js +13 -10
  264. package/vendor-core/run-comparison.js +81 -120
  265. package/vendor-core/run-output.js +26 -0
  266. package/vendor-core/sequence-alignment.js +83 -0
  267. package/vendor-core/stage-alignment.js +586 -0
  268. package/vendor-core/stage-identity.js +112 -0
  269. package/vendor-core/stage-quantiles.js +3 -1
  270. package/vendor-core/stage-structure.js +83 -0
  271. package/vendor-core/types.js +5 -0
  272. package/vendor-core/write-target-args.js +48 -0
  273. package/vendor-core/write-targets.js +35 -52
  274. package/public/assets/CachingOpportunity-Cfw3wAFa.js +0 -1
  275. package/public/assets/CoreUsageArea-BObEeenE.js +0 -1
  276. package/public/assets/DisclosureContext-Ds0hcZFn.js +0 -1
  277. package/public/assets/PlanGraphRoute-NV5mCXRm.js +0 -1
  278. package/public/assets/Spill-CH1bNpxL.js +0 -1
  279. package/public/assets/StageFindingGroup-B1A0nf-e.js +0 -1
  280. package/public/assets/index-BcqTLKFp.css +0 -2
  281. package/public/assets/index-DaC8A2FS.js +0 -13
  282. package/public/assets/parser-worker-CfDvPkwF.js +0 -122
  283. package/public/assets/useRecentFiles-Bb4-ukVz.js +0 -11
  284. package/public/docs/assets/chunks/@localSearchIndexroot.r9imX8Aq.js +0 -1
  285. package/public/docs/assets/contributor-guide_architecture_board-widgets.md.D0Kkk_r5.js +0 -1
  286. package/public/docs/assets/contributor-guide_architecture_detector-contract.md.caA46z-u.js +0 -1
  287. package/public/docs/assets/contributor-guide_architecture_detector-contract.md.caA46z-u.lean.js +0 -1
  288. package/public/docs/assets/contributor-guide_architecture_drill-down.md.CdtlKHl-.js +0 -1
  289. package/public/docs/assets/contributor-guide_architecture_drill-down.md.CdtlKHl-.lean.js +0 -1
  290. package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.BNp14LtL.js +0 -1
  291. package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.BNp14LtL.lean.js +0 -1
  292. package/public/docs/assets/contributor-guide_architecture_index.md.B9yZieYL.js +0 -1
  293. package/public/docs/assets/contributor-guide_architecture_overview.md.DBGo-bwn.js +0 -1
  294. package/public/docs/assets/contributor-guide_architecture_state-and-history.md.w4lwybdh.js +0 -1
  295. package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.CcvqxnZI.js +0 -1
  296. package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.CcvqxnZI.lean.js +0 -1
  297. package/public/docs/assets/contributor-guide_architecture_worker-protocol.md.CKeFttKj.js +0 -6
  298. package/public/docs/assets/contributor-guide_architecture_worker-protocol.md.CKeFttKj.lean.js +0 -1
  299. package/public/docs/assets/user-guide_alternative-log-retrieval.md.z-8npxNk.js +0 -7
  300. package/public/docs/assets/user-guide_alternative-log-retrieval.md.z-8npxNk.lean.js +0 -1
  301. package/public/docs/assets/user-guide_getting-started.md.BJW5oVhe.js +0 -24
  302. package/public/docs/assets/user-guide_getting-started.md.BJW5oVhe.lean.js +0 -1
  303. package/public/docs/assets/user-guide_mcp-tools.md.DcLiOa9t.js +0 -148
  304. package/public/docs/assets/user-guide_mcp-tools.md.DcLiOa9t.lean.js +0 -1
  305. package/public/docs/assets/user-guide_run-comparison.md.qin0a4IX.js +0 -1
  306. package/public/docs/assets/user-guide_run-comparison.md.qin0a4IX.lean.js +0 -1
  307. package/public/docs/assets/user-guide_understanding-findings.md.DzReyCXb.js +0 -1
  308. package/public/docs/assets/user-guide_understanding-findings.md.DzReyCXb.lean.js +0 -1
  309. /package/public/docs/assets/{contributor-guide_architecture_overview.md.DBGo-bwn.lean.js → contributor-guide_architecture_overview.md.C1sSvm9i.lean.js} +0 -0
package/public/docs/tuning-reference/data-formats.html CHANGED
@@ -9,9 +9,9 @@
9
9
  <link rel="preload stylesheet" href="/docs/assets/style.tmDhD3c8.css" as="style">
10
10
  <link rel="preload stylesheet" href="/docs/vp-icons.css" as="style">
11
11
 
12
- <script type="module" src="/docs/assets/app.yjczoibf.js"></script>
12
+ <script type="module" src="/docs/assets/app.eYTPqTeq.js"></script>
13
13
  <link rel="preload" href="/docs/assets/inter-roman-latin.Di8DUHzh.woff2" as="font" type="font/woff2" crossorigin="">
14
- <link rel="modulepreload" href="/docs/assets/chunks/theme.DFcAl8Ws.js">
14
+ <link rel="modulepreload" href="/docs/assets/chunks/theme.DsTNE2pE.js">
15
15
  <link rel="modulepreload" href="/docs/assets/chunks/framework.DSg0KOwT.js">
16
16
  <link rel="modulepreload" href="/docs/assets/tuning-reference_data-formats.md.C5DX5xL4.lean.js">
17
17
  <script>(()=>{const c=`${`[${["data","shuffle","product","bar"].join("-")}]`}, .VPNav, .VPLocalNav`,a=24,d=document.documentElement,l=t=>{const e=getComputedStyle(t);return e.display==="none"||t.getClientRects().length===0?0:e.position==="fixed"?t.getBoundingClientRect().bottom:e.position==="sticky"?(parseFloat(e.top)||0)+t.offsetHeight:0},u=()=>{try{return location.hash?document.getElementById(decodeURIComponent(location.hash.slice(1))):null}catch{return null}};let r=null;const s=()=>{const t=Array.from(document.querySelectorAll(c),l),e=Math.ceil(Math.max(0,...t))+a;if(e===r)return;const n=r;r=e,d.style.setProperty("--sf-anchor-offset",`${r}px`);const o=n===null?null:u();if(!o)return;const f=(parseFloat(getComputedStyle(o).paddingTop)||0)+2;Math.abs(o.getBoundingClientRect().top-n)<=f&&o.scrollIntoView()},i=()=>{const t=document.createElement("div");t.setAttribute("data-sf-anchor-offset",""),t.setAttribute("aria-hidden","true"),document.body.appendChild(t);const e=window.ResizeObserver?new ResizeObserver(s):null,n=()=>{e&&document.querySelectorAll(c).forEach(o=>e.observe(o)),s()};n(),new MutationObserver(n).observe(document.body,{childList:!0,subtree:!0}),window.addEventListener("resize",s),window.addEventListener("load",s)};document.readyState==="loading"?document.addEventListener("DOMContentLoaded",i,{once:!0}):i()})();</script>
@@ -24,7 +24,7 @@
24
24
  </head>
25
25
  <body>
26
26
  <div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Getting Started</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/intro.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Introduction</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Optimization Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/spark-architecture.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Execution Model</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/memory-model.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Management</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/partitioning.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Partitioning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/joins.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Join Optimization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/data-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Data Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/table-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Table Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/caching.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Caching & Persistence</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/pyspark.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>PySpark Specifics</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/aqe.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Adaptive Query Execution</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/cluster-config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cluster Tuning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/anti-patterns.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Anti-Patterns</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Detector Catalog</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-skew.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Skew</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle I/O</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-spill.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory / Disk Spill</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-gc.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>GC Pressure</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-cold-start.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cold Start</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Executor Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-slow-host.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Slow Host</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-failures.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Failures</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-straggler.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Stragglers</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-retry-waste.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Retry Waste</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-tiny-tasks.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Tiny Tasks</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-job-failure-rate.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Job Failure Rate</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-memory-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Duplicate Plan Subtree</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-small-files.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Small Files</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-broadcast-sizing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Broadcast Sizing</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Reference</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/metrics.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Metrics Glossary</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Config Quick-Reference</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_tuning-reference_data-formats" data-v-39a288b8><div><h1 id="data-formats" tabindex="-1">Data Formats <a class="header-anchor" href="#data-formats" aria-label="Permalink to &quot;Data Formats {#data-formats}&quot;">​</a></h1><h2 id="how-parquet-and-orc-lay-out-data" tabindex="-1">How Parquet and ORC lay out data <a class="header-anchor" href="#how-parquet-and-orc-lay-out-data" aria-label="Permalink to &quot;How Parquet and ORC lay out data&quot;">​</a></h2><p>Parquet and ORC are both self-describing, columnar file formats for storing Spark&#39;s structured data<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup>. They&#39;re closer than they are different: <em>Spark: The Definitive Guide</em> frames the choice as &quot;for the most part, they&#39;re quite similar; the fundamental difference is that Parquet is further optimized for use with Spark, whereas ORC is further optimized for Hive&quot;<sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup>, and notes that ORC &quot;has no options for reading in data because Spark understands the file format quite well&quot;<sup class="footnote-ref"><a href="#fn1" id="fnref1:2">[1:2]</a></sup>: Spark treats it as a format it understands natively rather than one with extra read-side knobs.</p><p>Both formats group rows into chunks (row groups in Parquet, stripes in ORC), each holding every column&#39;s data for that slice, with a footer or file-level metadata section recording where each chunk lives on disk. Parquet&#39;s <code>FileMetaData</code> footer records the offset and size of every row group and column chunk, and &quot;readers are expected to first read the file metadata to find all the column chunks they are interested in. The column chunks should then be read sequentially&quot;<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup>. Because compression is applied per column chunk rather than as one continuous stream over the whole file, a reader can seek straight to a row group&#39;s footer-recorded offset and decode just that chunk, which is exactly why Parquet files compressed with Snappy stay splittable even though the Snappy stream format itself provides no split points: it has &quot;no entropy encoder backend nor framing layer -- the latter is assumed to be handled by other parts of the system&quot;<sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup>.</p><img class="light-only" src="/docs/assets/columnar-layout.PghGeOEA.svg" alt="A columnar file&#39;s footer holds row-group offsets and per-column min/max statistics that let a reader skip row groups whose stats exclude the predicate."><img class="dark-only" src="/docs/assets/columnar-layout.dark.BVNlz0ff.svg" alt="A columnar file&#39;s footer holds row-group offsets and per-column min/max statistics that let a reader skip row groups whose stats exclude the predicate."><p>Both formats also carry the statistics that later drive predicate pushdown. Parquet records min/max values per column chunk, plus an optional page-level <code>ColumnIndex</code> that lets a reader binary-search ordered columns for matching pages<sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup>, and optional Bloom filters for columns whose cardinality is too high for a dictionary to be practical<sup class="footnote-ref"><a href="#fn5" id="fnref5">[5]</a></sup>. ORC&#39;s <code>ColumnStatistics</code> protobuf records row count and, for most primitive types, min/max (plus sum for numeric types); from Hive 1.1.0 it also records a <code>hasNull</code> flag used specifically by ORC&#39;s predicate pushdown to answer <code>IS NULL</code> queries<sup class="footnote-ref"><a href="#fn6" id="fnref6">[6]</a></sup>. ORC layers this at two granularities: a <code>RowIndexEntry</code> per row group (10,000 rows by default), kept at the front of each stripe so it&#39;s read only when pushdown or seeking is actually needed<sup class="footnote-ref"><a href="#fn7" id="fnref7">[7]</a></sup>, and file-level <code>StripeStatistics</code> that let whole stripes be skipped by predicate pushdown<sup class="footnote-ref"><a href="#fn8" id="fnref8">[8]</a></sup>. ORC also supports Bloom filters (Hive 1.2.0+), but only evaluates them against row groups that already passed the min/max row-index check first: a second-stage filter, not an independent one<sup class="footnote-ref"><a href="#fn7" id="fnref7:1">[7:1]</a></sup>.</p><p>On compression: Parquet defaults to Snappy, while ORC&#39;s default has been Zstd since Spark 2.3.0<sup class="footnote-ref"><a href="#fn9" id="fnref9">[9]</a></sup>. Zstandard&#39;s own manual describes it as &quot;a fast lossless compression algorithm, targeting real-time compression scenarios at zlib-level and better compression ratios,&quot; offering regular levels 1–22 plus negative levels that trade ratio for speed<sup class="footnote-ref"><a href="#fn10" id="fnref10">[10]</a></sup>. Spark exposes the level directly through <code>spark.io.compression.zstd.level</code> (default <code>1</code>)<sup class="footnote-ref"><a href="#fn9" id="fnref9:1">[9:1]</a></sup>, along with <code>spark.io.compression.zstd.workers</code> for parallel compression threads (default <code>0</code>, since 4.0.0) and <code>spark.io.compression.zstd.bufferSize</code> (default 32k)<sup class="footnote-ref"><a href="#fn9" id="fnref9:2">[9:2]</a></sup>.</p><p>Parquet has one more structural knob: <code>parquet.writer.version</code>. Version 2 changes the on-disk page-header encoding, and the spec is explicit that this is a forward-<em>incompatible</em> change: &quot;a reader that only understands <code>DataPageHeader</code> cannot parse <code>DataPageHeaderV2</code> pages&quot;<sup class="footnote-ref"><a href="#fn11" id="fnref11">[11]</a></sup>. The spec also flags that the file&#39;s own version marker &quot;has historically been used inconsistently: writers populate 1 or 2 without a consistent relationship to the features actually used&quot;<sup class="footnote-ref"><a href="#fn11" id="fnref11:1">[11:1]</a></sup>, so the marker itself isn&#39;t a reliable way to tell which page format a file actually contains.</p><h2 id="what-shows-up-at-read-time" tabindex="-1">What shows up at read time <a class="header-anchor" href="#what-shows-up-at-read-time" aria-label="Permalink to &quot;What shows up at read time&quot;">​</a></h2><p>A file&#39;s layout shapes the job the moment Spark reads it. Spark&#39;s file readers derive partition counts from file metadata rather than from a fixed rule: &quot;for structured formats like Parquet, ORC, or Avro, Spark actually reads the metadata (footers, row groups, that kind of thing) and tries to slice the file in a way that makes sense. Often you&#39;ll see one partition per row group, though Spark may merge or split depending on file sizes and configs&quot;<sup class="footnote-ref"><a href="#fn12" id="fnref12">[12]</a></sup>. A concrete case: a single 30 GB Parquet file with 300 row groups gets split into exactly 300 partitions<sup class="footnote-ref"><a href="#fn12" id="fnref12:1">[12:1]</a></sup>. So a surprising partition count for a given file is usually explained by its row-group layout, separately from <code>spark.sql.files.maxPartitionBytes</code> (128 MB by default) and its <code>maxPartitionNum</code>/<code>minPartitionNum</code> companions, which govern splitting for file-based sources generally<sup class="footnote-ref"><a href="#fn13" id="fnref13">[13]</a></sup>.</p><p>The <a href="./bottleneck-small-files.html">small-files problem</a> shows up as metadata overhead rather than raw I/O cost: &quot;when you&#39;re writing lots of small files, there&#39;s a significant metadata overhead that you incur managing all of those files. Spark especially does not do well with small files&quot;<sup class="footnote-ref"><a href="#fn1" id="fnref1:3">[1:3]</a></sup>. At the other extreme, oversized or misaligned row groups show up as lost locality: the Parquet spec&#39;s rationale for matching block size to row-group size is that &quot;since an entire row group might need to be read, we want it to completely fit on one HDFS block&quot;<sup class="footnote-ref"><a href="#fn14" id="fnref14">[14]</a></sup>. A row group straddling a block boundary can&#39;t get that benefit.</p><p>Raw (non-Parquet) gzip or zip source files reveal themselves as a single-executor bottleneck at read time, visible as one task doing dramatically more work than the rest of its stage: Spark &quot;needs to download the whole file on one executor, unpack it on just one core, and then redistribute the partitions to the cluster nodes&quot;<sup class="footnote-ref"><a href="#fn15" id="fnref15">[15]</a></sup>.</p><p>Version and schema mismatches surface as read failures rather than silent corruption. A file written with <code>parquet.writer.version=2</code> can&#39;t be parsed by an older Spark version, or a non-Spark engine such as Hive or Impala, if that reader only implements the original <code>DataPageHeader</code><sup class="footnote-ref"><a href="#fn11" id="fnref11:2">[11:2]</a></sup>. <em>Spark: The Definitive Guide</em> frames this as a general risk: &quot;you can still encounter problems if you&#39;re working with incompatible Parquet files. Be careful when you write out Parquet files with different versions of Spark (especially older ones) because this can cause significant headache&quot;<sup class="footnote-ref"><a href="#fn1" id="fnref1:4">[1:4]</a></sup>.</p><h2 id="what-the-layout-buys-you" tabindex="-1">What the layout buys you <a class="header-anchor" href="#what-the-layout-buys-you" aria-label="Permalink to &quot;What the layout buys you&quot;">​</a></h2><p>Those read-time symptoms trace back to specific tradeoffs in the layout. Splittability is what decides whether a file can be processed in parallel at all. A non-splittable raw compressed source file forces Spark onto a single core for the whole download-and-unpack step before it can redistribute anything: &quot;as you can imagine, this becomes a huge bottleneck in your distributed processing&quot;<sup class="footnote-ref"><a href="#fn15" id="fnref15:1">[15:1]</a></sup>. Parquet sidesteps this at the container level: because its footer records row-group and column-chunk offsets independently of whichever codec compressed the bytes inside them, a task can seek and decode a row group on its own, so even a non-splittable codec like Snappy doesn&#39;t cost the file its parallelism<sup class="footnote-ref"><a href="#fn2" id="fnref2:1">[2:1]</a></sup><sup class="footnote-ref"><a href="#fn3" id="fnref3:1">[3:1]</a></sup>.</p><p>Row-group and block-size alignment governs the same kind of locality at a coarser grain. The spec recommends large row groups, 512 MB–1 GB, &quot;because larger row groups allow for larger column chunks which makes it possible to do larger sequential IO,&quot; at the cost of more write-side buffering, paired with a block size sized the same way; its own worked example is &quot;1GB row groups, 1GB HDFS block size, 1 HDFS block per HDFS file&quot;<sup class="footnote-ref"><a href="#fn14" id="fnref14:1">[14:1]</a></sup>.</p><p>The statistics both formats carry are what predicate pushdown runs against. Spark plans queries directly off &quot;statistics that Spark reads directly from the underlying data source, like the counts and min/max values in the metadata of Parquet files&quot;<sup class="footnote-ref"><a href="#fn16" id="fnref16">[16]</a></sup>, gated by <code>spark.sql.parquet.filterPushdown</code> (default <code>true</code> since Spark 1.2.0) and, for ORC, <code>spark.sql.orc.filterPushdown</code> (default <code>true</code> since 1.4.0)<sup class="footnote-ref"><a href="#fn17" id="fnref17">[17]</a></sup><sup class="footnote-ref"><a href="#fn18" id="fnref18">[18]</a></sup>. A more aggressive option, <code>spark.sql.parquet.aggregatePushdown</code> (default <code>false</code>, since 3.3.0), pushes <code>MIN</code>, <code>MAX</code>, and <code>COUNT</code> down to Parquet&#39;s footer statistics directly, and throws if the needed statistic is missing from a file&#39;s footer<sup class="footnote-ref"><a href="#fn17" id="fnref17:1">[17:1]</a></sup>. Skipping a row group or stripe this way means its bytes are never read off disk, which beats any amount of post-read filtering.</p><p>Small files cost more in metadata management than their data volume would suggest, and Spark &quot;especially does not do well&quot; with them<sup class="footnote-ref"><a href="#fn1" id="fnref1:5">[1:5]</a></sup>. On the scheduling side, Spark&#39;s tuning guide notes it &quot;can efficiently support tasks as short as 200 ms&quot; because it reuses one executor JVM across many tasks and has low task-launch cost, while separately recommending &quot;2-3 tasks per CPU core in your cluster&quot;<sup class="footnote-ref"><a href="#fn19" id="fnref19">[19]</a></sup>, which implies <a href="./bottleneck-tiny-tasks.html">scheduling overhead</a> only becomes proportionally significant once tasks (and the files behind them) shrink below roughly that 200 ms floor.</p><p>Compression choice trades CPU against I/O and storage. Once I/O stops being the bottleneck, paying a codec&#39;s decompression cost is a net loss: &quot;uncompressed files are clearly outperforming compressed files. This is because uncompressed files are I/O bound, and compressed files are CPU bound, but I/O is good enough here&quot;<sup class="footnote-ref"><a href="#fn15" id="fnref15:2">[15:2]</a></sup>. Bzip2 illustrates the opposite failure mode: it&#39;s splittable, but compresses so aggressively that &quot;you get very few partitions and therefore they can be poorly distributed&quot;<sup class="footnote-ref"><a href="#fn15" id="fnref15:3">[15:3]</a></sup>.</p><p>Writer-version and schema changes matter because they fail at read time, for whoever reads the data next (not at write time, for whoever wrote it). A <code>parquet.writer.version=2</code> file is simply unreadable by any engine that only understands the original page header<sup class="footnote-ref"><a href="#fn11" id="fnref11:3">[11:3]</a></sup>. Schema drift has its own correctness angle: for <a href="./table-formats.html">Delta Lake</a> tables, adding a column via <code>mergeSchema</code> causes existing rows, when read back, to have that new column&#39;s value read as <code>NULL</code><sup class="footnote-ref"><a href="#fn20" id="fnref20">[20]</a></sup>, a defined outcome, but one that changes what a downstream query sees for rows written before the schema changed.</p><h2 id="choosing-and-tuning-the-format" tabindex="-1">Choosing and tuning the format <a class="header-anchor" href="#choosing-and-tuning-the-format" aria-label="Permalink to &quot;Choosing and tuning the format&quot;">​</a></h2><p>A few defaults cover most cases despite those tradeoffs. Default to Parquet. Reach for ORC specifically when the same data also has to serve Hive consumers or existing Hive ORC tables, where ORC is the better-optimized target<sup class="footnote-ref"><a href="#fn1" id="fnref1:6">[1:6]</a></sup>.</p><p>Size row groups and the underlying filesystem block together, not independently. The Parquet spec&#39;s own recommended setup is 1 GB row groups paired with a 1 GB HDFS block size, one block per file<sup class="footnote-ref"><a href="#fn14" id="fnref14:2">[14:2]</a></sup>. Leave <code>spark.sql.files.maxPartitionBytes</code> (128 MB default) and its <code>maxPartitionNum</code>/<code>minPartitionNum</code> companions for the general file-splitting case, but expect row-group boundaries (not these settings) to be what actually decides partition count for row-group-oriented formats<sup class="footnote-ref"><a href="#fn13" id="fnref13:1">[13:1]</a></sup><sup class="footnote-ref"><a href="#fn12" id="fnref12:2">[12:2]</a></sup>.</p><p>Leave predicate pushdown on: <code>spark.sql.parquet.filterPushdown</code> and <code>spark.sql.orc.filterPushdown</code> both default to <code>true</code> already<sup class="footnote-ref"><a href="#fn17" id="fnref17:2">[17:2]</a></sup><sup class="footnote-ref"><a href="#fn18" id="fnref18:1">[18:1]</a></sup>, so the main action is not disabling them, plus considering <code>spark.sql.parquet.aggregatePushdown</code> when a workload is dominated by <code>MIN</code>/<code>MAX</code>/<code>COUNT</code> over Parquet sources with complete footer statistics<sup class="footnote-ref"><a href="#fn17" id="fnref17:3">[17:3]</a></sup>.</p><p>Cap output file size directly instead of letting the small-files problem accumulate: <code>maxRecordsPerFile</code>, introduced in Spark 2.2, targets an optimum file size by capping the number of records written per file, e.g. <code>df.write.option(&quot;maxRecordsPerFile&quot;, 5000)</code><sup class="footnote-ref"><a href="#fn1" id="fnref1:7">[1:7]</a></sup>.</p><p>Pick a compression codec based on the actual bottleneck. Parquet&#39;s default, Snappy, favors speed; ORC has already defaulted to Zstd since Spark 2.3.0<sup class="footnote-ref"><a href="#fn9" id="fnref9:3">[9:3]</a></sup>, and at least one optimization checklist recommends overriding Parquet&#39;s default to Zstd as well<sup class="footnote-ref"><a href="#fn21" id="fnref21">[21]</a></sup>. Zstd&#39;s level is tunable through <code>spark.io.compression.zstd.level</code>: higher levels buy better compression &quot;at the expense of more CPU and memory&quot;<sup class="footnote-ref"><a href="#fn9" id="fnref9:4">[9:4]</a></sup>. Avoid feeding Spark large raw <code>.gz</code>/<code>.zip</code> source files directly (unpack them before loading), since the non-splittability lives in the raw container format rather than in gzip itself; the same <em>Definitive Guide</em> that warns about raw gzip files elsewhere still recommends Parquet with gzip compression once gzip is wrapped inside Parquet&#39;s row-group container<sup class="footnote-ref"><a href="#fn1" id="fnref1:8">[1:8]</a></sup>.</p><p>Treat <code>spark.sql.parquet.mergeSchema</code> (default <code>false</code>) as opt-in rather than default-on: enabling it &quot;merges schemas collected from all data files,&quot; instead of trusting a single summary file or a random file&#39;s schema<sup class="footnote-ref"><a href="#fn22" id="fnref22">[22]</a></sup>, useful for evolving schemas, but it means every file in the dataset gets scanned for its schema. For Delta tables specifically, remember that <code>mergeSchema</code>-added columns read back as <code>NULL</code> on pre-existing rows<sup class="footnote-ref"><a href="#fn20" id="fnref20:1">[20:1]</a></sup> before relying on it for a backfill.</p><p>Don&#39;t flip <code>parquet.writer.version</code> to <code>2</code> without confirming every downstream reader of that data (an older Spark version, Hive, Impala, or anything else touching the files) actually supports <code>DataPageHeaderV2</code> first; it&#39;s a forward-incompatible page-format change, not an additive one<sup class="footnote-ref"><a href="#fn11" id="fnref11:4">[11:4]</a></sup><sup class="footnote-ref"><a href="#fn1" id="fnref1:9">[1:9]</a></sup>.</p><h2 id="sources" tabindex="-1">Sources <a class="header-anchor" href="#sources" aria-label="Permalink to &quot;Sources&quot;">​</a></h2><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><em>Spark: The Definitive Guide</em>, Chambers &amp; Zaharia, ch. 9 <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a> <a href="#fnref1:2" class="footnote-backref">↩︎</a> <a href="#fnref1:3" class="footnote-backref">↩︎</a> <a href="#fnref1:4" class="footnote-backref">↩︎</a> <a href="#fnref1:5" class="footnote-backref">↩︎</a> <a href="#fnref1:6" class="footnote-backref">↩︎</a> <a href="#fnref1:7" class="footnote-backref">↩︎</a> <a href="#fnref1:8" class="footnote-backref">↩︎</a> <a href="#fnref1:9" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><a href="https://parquet.apache.org/_print/docs/file-format/" target="_blank" rel="noreferrer">Parquet File Format: File Metadata and Row Groups</a> <a href="#fnref2" class="footnote-backref">↩︎</a> <a href="#fnref2:1" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><a href="https://github.com/google/snappy/blob/main/format_description.txt" target="_blank" rel="noreferrer">Snappy Compressed Format Description</a> <a href="#fnref3" class="footnote-backref">↩︎</a> <a href="#fnref3:1" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><a href="https://parquet.apache.org/_print/docs/file-format/" target="_blank" rel="noreferrer">Parquet File Format: Column Index</a> <a href="#fnref4" class="footnote-backref">↩︎</a></p></li><li id="fn5" class="footnote-item"><p><a href="https://parquet.apache.org/docs/file-format/bloomfilter/" target="_blank" rel="noreferrer">Parquet File Format: Bloom Filter</a> <a href="#fnref5" class="footnote-backref">↩︎</a></p></li><li id="fn6" class="footnote-item"><p><a href="https://orc.apache.org/specification/ORCv1/" target="_blank" rel="noreferrer">ORC Specification v1: Column Statistics</a> <a href="#fnref6" class="footnote-backref">↩︎</a></p></li><li id="fn7" class="footnote-item"><p><a href="https://orc.apache.org/specification/ORCv1/" target="_blank" rel="noreferrer">ORC Specification v1: Row Index and Bloom Filters</a> <a href="#fnref7" class="footnote-backref">↩︎</a> <a href="#fnref7:1" class="footnote-backref">↩︎</a></p></li><li id="fn8" class="footnote-item"><p><a href="https://orc.apache.org/specification/ORCv1/" target="_blank" rel="noreferrer">ORC Specification v1: Stripe Statistics</a> <a href="#fnref8" class="footnote-backref">↩︎</a></p></li><li id="fn9" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration: Spark</a> <a href="#fnref9" class="footnote-backref">↩︎</a> <a href="#fnref9:1" class="footnote-backref">↩︎</a> <a href="#fnref9:2" class="footnote-backref">↩︎</a> <a href="#fnref9:3" class="footnote-backref">↩︎</a> <a href="#fnref9:4" class="footnote-backref">↩︎</a></p></li><li id="fn10" class="footnote-item"><p><a href="https://facebook.github.io/zstd/zstd_manual.html" target="_blank" rel="noreferrer">Zstandard Manual</a> <a href="#fnref10" class="footnote-backref">↩︎</a></p></li><li id="fn11" class="footnote-item"><p><a href="https://parquet.apache.org/_print/docs/file-format/" target="_blank" rel="noreferrer">Parquet File Format: Data Pages (V1/V2)</a> <a href="#fnref11" class="footnote-backref">↩︎</a> <a href="#fnref11:1" class="footnote-backref">↩︎</a> <a href="#fnref11:2" class="footnote-backref">↩︎</a> <a href="#fnref11:3" class="footnote-backref">↩︎</a> <a href="#fnref11:4" class="footnote-backref">↩︎</a></p></li><li id="fn12" class="footnote-item"><p><a href="https://luminousmen.com/post/spark-partitions" target="_blank" rel="noreferrer">How Spark Determines Partitions for a File</a> <a href="#fnref12" class="footnote-backref">↩︎</a> <a href="#fnref12:1" class="footnote-backref">↩︎</a> <a href="#fnref12:2" class="footnote-backref">↩︎</a></p></li><li id="fn13" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/sql-performance-tuning.html" target="_blank" rel="noreferrer">Performance Tuning: Spark SQL, DataFrames and Datasets Guide</a> <a href="#fnref13" class="footnote-backref">↩︎</a> <a href="#fnref13:1" class="footnote-backref">↩︎</a></p></li><li id="fn14" class="footnote-item"><p><a href="https://parquet.apache.org/_print/docs/file-format/" target="_blank" rel="noreferrer">Parquet File Format: Row Group Size</a> <a href="#fnref14" class="footnote-backref">↩︎</a> <a href="#fnref14:1" class="footnote-backref">↩︎</a> <a href="#fnref14:2" class="footnote-backref">↩︎</a></p></li><li id="fn15" class="footnote-item"><p><a href="https://luminousmen.com/post/spark-tips-dont-collect-data-on-driver" target="_blank" rel="noreferrer">Spark Tips: Don&#39;t Collect Data on Driver</a> <a href="#fnref15" class="footnote-backref">↩︎</a> <a href="#fnref15:1" class="footnote-backref">↩︎</a> <a href="#fnref15:2" class="footnote-backref">↩︎</a> <a href="#fnref15:3" class="footnote-backref">↩︎</a></p></li><li id="fn16" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/sql-performance-tuning.html" target="_blank" rel="noreferrer">Performance Tuning: Spark SQL, DataFrames and Datasets Guide</a> <a href="#fnref16" class="footnote-backref">↩︎</a></p></li><li id="fn17" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration: Spark</a> <a href="#fnref17" class="footnote-backref">↩︎</a> <a href="#fnref17:1" class="footnote-backref">↩︎</a> <a href="#fnref17:2" class="footnote-backref">↩︎</a> <a href="#fnref17:3" class="footnote-backref">↩︎</a></p></li><li id="fn18" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration: Spark</a> <a href="#fnref18" class="footnote-backref">↩︎</a> <a href="#fnref18:1" class="footnote-backref">↩︎</a></p></li><li id="fn19" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/tuning.html" target="_blank" rel="noreferrer">Spark Tuning Guide: Level of Parallelism</a> <a href="#fnref19" class="footnote-backref">↩︎</a></p></li><li id="fn20" class="footnote-item"><p><em>Learning Spark, 2nd Edition</em>, Damji, Wenig, Das &amp; Lee, ch. 9 <a href="#fnref20" class="footnote-backref">↩︎</a> <a href="#fnref20:1" class="footnote-backref">↩︎</a></p></li><li id="fn21" class="footnote-item"><p><a href="https://luminousmen.com/post/the-apache-spark-optimization-checklist" target="_blank" rel="noreferrer">The Apache Spark Optimization Checklist</a> <a href="#fnref21" class="footnote-backref">↩︎</a></p></li><li id="fn22" class="footnote-item"><p><a href="https://raw.githubusercontent.com/apache/spark/v3.5.0/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala" target="_blank" rel="noreferrer">SQLConf.scala</a> <a href="#fnref22" class="footnote-backref">↩︎</a></p></li></ol></section></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/tuning-reference/shuffle.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>Shuffle</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="/docs/tuning-reference/table-formats.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>Table Formats</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
27
- <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"D0Kkk_r5\",\"contributor-guide_architecture_detector-contract.md\":\"caA46z-u\",\"contributor-guide_architecture_drill-down.md\":\"CdtlKHl-\",\"contributor-guide_architecture_impact-estimation.md\":\"BNp14LtL\",\"contributor-guide_architecture_index.md\":\"B9yZieYL\",\"contributor-guide_architecture_overview.md\":\"DBGo-bwn\",\"contributor-guide_architecture_state-and-history.md\":\"w4lwybdh\",\"contributor-guide_architecture_widget-rendering.md\":\"CcvqxnZI\",\"contributor-guide_architecture_worker-protocol.md\":\"CKeFttKj\",\"contributor-guide_contributing.md\":\"CQMvGwFt\",\"contributor-guide_development-setup.md\":\"D9bu4j_c\",\"contributor-guide_testing.md\":\"By6XGR9a\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"D08Uaod7\",\"tuning-reference_aqe.md\":\"C1Ae6A8u\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"6wjKuJNE\",\"tuning-reference_bottleneck-cold-start.md\":\"BVQbsuHP\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"DtaYnaiF\",\"tuning-reference_bottleneck-failures.md\":\"Z1Tdmdd3\",\"tuning-reference_bottleneck-gc.md\":\"BCnXLf1n\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"CcvQj2g9\",\"tuning-reference_bottleneck-memory-utilization.md\":\"B-C0-8h-\",\"tuning-reference_bottleneck-retry-waste.md\":\"DCnhWiqj\",\"tuning-reference_bottleneck-shuffle.md\":\"B5OMJCd3\",\"tuning-reference_bottleneck-skew.md\":\"Cvo6NQm_\",\"tuning-reference_bottleneck-slow-host.md\":\"ja5qr3wq\",\"tuning-reference_bottleneck-small-files.md\":\"Crjtb0i2\",\"tuning-reference_bottleneck-spill.md\":\"Bl8KtSRg\",\"tuning-reference_bottleneck-straggler.md\":\"B0VN8reT\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"DQCftGhu\",\"tuning-reference_bottleneck-utilization.md\":\"BXDl30f9\",\"tuning-reference_caching.md\":\"CcaiMn_7\",\"tuning-reference_cluster-config.md\":\"DbWG1yZq\",\"tuning-reference_config.md\":\"YUqSpwFB\",\"tuning-reference_data-formats.md\":\"C5DX5xL4\",\"tuning-reference_index.md\":\"ZVIpQ6Dk\",\"tuning-reference_intro.md\":\"BSSaDlqx\",\"tuning-reference_joins.md\":\"BVhNCDnE\",\"tuning-reference_memory-model.md\":\"CfjFdgbl\",\"tuning-reference_metrics.md\":\"C64VNEL4\",\"tuning-reference_partitioning.md\":\"CtLK0xsS\",\"tuning-reference_pyspark.md\":\"D-92XgF8\",\"tuning-reference_shuffle.md\":\"Iq2v1oHJ\",\"tuning-reference_spark-architecture.md\":\"_qwnE_Sc\",\"tuning-reference_table-formats.md\":\"BSYEK-eN\",\"user-guide_alternative-log-retrieval.md\":\"z-8npxNk\",\"user-guide_getting-started.md\":\"BJW5oVhe\",\"user-guide_mcp-tools.md\":\"DcLiOa9t\",\"user-guide_run-comparison.md\":\"qin0a4IX\",\"user-guide_understanding-findings.md\":\"DzReyCXb\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Finding your event log\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":{\"selector\":\"[data-sf-anchor-offset]\",\"padding\":0},\"cleanUrls\":false}");</script>
27
+ <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"YQcj5EI4\",\"contributor-guide_architecture_board-widgets_caching-and-memory.md\":\"8bH1HnZp\",\"contributor-guide_architecture_board-widgets_plan-advisor.md\":\"RlzvUmNm\",\"contributor-guide_architecture_board-widgets_visual-system-and-templating.md\":\"DYgUpIJB\",\"contributor-guide_architecture_detector-contract.md\":\"wj6Et6UD\",\"contributor-guide_architecture_detector-contract_plan-attribution.md\":\"yNex4bys\",\"contributor-guide_architecture_detector-contract_threshold-tables.md\":\"CKjWG56Y\",\"contributor-guide_architecture_drill-down.md\":\"CqZPrdo9\",\"contributor-guide_architecture_drill-down_plan-graph.md\":\"DwOozIEs\",\"contributor-guide_architecture_impact-estimation.md\":\"CZv_ZGOp\",\"contributor-guide_architecture_impact-estimation_caveats-tuning-and-coverage.md\":\"BUucRc0A\",\"contributor-guide_architecture_impact-estimation_stage-union-rollup.md\":\"DdCGtAGu\",\"contributor-guide_architecture_index.md\":\"tymxf1Fh\",\"contributor-guide_architecture_overview.md\":\"C1sSvm9i\",\"contributor-guide_architecture_state-and-history.md\":\"DyCBdjuM\",\"contributor-guide_architecture_state-and-history_history-server-intake.md\":\"C298hrHZ\",\"contributor-guide_architecture_state-and-history_run-comparison-internals.md\":\"CZA0JLtV\",\"contributor-guide_architecture_widget-rendering.md\":\"BOZ1azXG\",\"contributor-guide_architecture_widget-rendering_findings-tab.md\":\"BQXycK6G\",\"contributor-guide_architecture_widget-rendering_investigation-routing.md\":\"BGdiyElu\",\"contributor-guide_architecture_widget-rendering_list-sort-mode.md\":\"BuHKgH0C\",\"contributor-guide_architecture_widget-rendering_reference-section.md\":\"DWqC80bN\",\"contributor-guide_architecture_widget-rendering_row-expand-contract.md\":\"ZT89u9f1\",\"contributor-guide_architecture_worker-protocol.md\":\"BAnhi2-d\",\"contributor-guide_architecture_worker-protocol_cli-and-mcp.md\":\"DC1YqXBp\",\"contributor-guide_architecture_worker-protocol_core-and-event-validation.md\":\"DSVFPpWb\",\"contributor-guide_architecture_worker-protocol_evidence-availability.md\":\"CjimIDBY\",\"contributor-guide_architecture_worker-protocol_evidence-report.md\":\"DVGZOWvW\",\"contributor-guide_contributing.md\":\"CQMvGwFt\",\"contributor-guide_development-setup.md\":\"D9bu4j_c\",\"contributor-guide_testing.md\":\"By6XGR9a\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"D08Uaod7\",\"tuning-reference_aqe.md\":\"C1Ae6A8u\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"6wjKuJNE\",\"tuning-reference_bottleneck-cold-start.md\":\"BVQbsuHP\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"DtaYnaiF\",\"tuning-reference_bottleneck-failures.md\":\"Z1Tdmdd3\",\"tuning-reference_bottleneck-gc.md\":\"BCnXLf1n\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"CcvQj2g9\",\"tuning-reference_bottleneck-memory-utilization.md\":\"B-C0-8h-\",\"tuning-reference_bottleneck-retry-waste.md\":\"DCnhWiqj\",\"tuning-reference_bottleneck-shuffle.md\":\"B5OMJCd3\",\"tuning-reference_bottleneck-skew.md\":\"Cvo6NQm_\",\"tuning-reference_bottleneck-slow-host.md\":\"ja5qr3wq\",\"tuning-reference_bottleneck-small-files.md\":\"Crjtb0i2\",\"tuning-reference_bottleneck-spill.md\":\"Bl8KtSRg\",\"tuning-reference_bottleneck-straggler.md\":\"B0VN8reT\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"DQCftGhu\",\"tuning-reference_bottleneck-utilization.md\":\"BXDl30f9\",\"tuning-reference_caching.md\":\"CcaiMn_7\",\"tuning-reference_cluster-config.md\":\"DbWG1yZq\",\"tuning-reference_config.md\":\"YUqSpwFB\",\"tuning-reference_data-formats.md\":\"C5DX5xL4\",\"tuning-reference_index.md\":\"ZVIpQ6Dk\",\"tuning-reference_intro.md\":\"BSSaDlqx\",\"tuning-reference_joins.md\":\"BVhNCDnE\",\"tuning-reference_memory-model.md\":\"CfjFdgbl\",\"tuning-reference_metrics.md\":\"C64VNEL4\",\"tuning-reference_partitioning.md\":\"CtLK0xsS\",\"tuning-reference_pyspark.md\":\"D-92XgF8\",\"tuning-reference_shuffle.md\":\"Iq2v1oHJ\",\"tuning-reference_spark-architecture.md\":\"_qwnE_Sc\",\"tuning-reference_table-formats.md\":\"BSYEK-eN\",\"user-guide_alternative-log-retrieval.md\":\"CJpYvHGo\",\"user-guide_alternative-log-retrieval_managed-platforms.md\":\"DvDwHNtC\",\"user-guide_alternative-log-retrieval_ssh-bastion.md\":\"Gszzlplv\",\"user-guide_getting-started.md\":\"DYHHHkQR\",\"user-guide_getting-started_ci-and-automation.md\":\"OUQqw3mI\",\"user-guide_getting-started_reading-the-dashboard.md\":\"Obf3W3__\",\"user-guide_getting-started_regression-metric-keys.md\":\"B_gzr2Mw\",\"user-guide_getting-started_tuning-thresholds.md\":\"B_MyjyZm\",\"user-guide_mcp-tools.md\":\"C65SjWUL\",\"user-guide_mcp-tools_compare-runs.md\":\"Cc_gLoix\",\"user-guide_mcp-tools_diagnose-run.md\":\"BP1vnHA5\",\"user-guide_mcp-tools_evaluate-budgets.md\":\"BB0nDY3Z\",\"user-guide_mcp-tools_get-finding-documentation.md\":\"BtTTHNV6\",\"user-guide_mcp-tools_get-finding-evidence.md\":\"rKyu4wts\",\"user-guide_mcp-tools_get-reference-doc.md\":\"Yac4tH3u\",\"user-guide_mcp-tools_get-run-summary.md\":\"DpxZ4ZNp\",\"user-guide_mcp-tools_list-runs.md\":\"DpQPeCM3\",\"user-guide_run-comparison.md\":\"BUALkohi\",\"user-guide_run-comparison_how-stages-are-matched.md\":\"DSXGu2tA\",\"user-guide_understanding-findings.md\":\"C9YRruMT\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\",\"collapsed\":true,\"items\":[{\"text\":\"Reading the dashboard\",\"link\":\"/user-guide/getting-started/reading-the-dashboard\"},{\"text\":\"CI and automation\",\"link\":\"/user-guide/getting-started/ci-and-automation\"},{\"text\":\"Tuning detector thresholds\",\"link\":\"/user-guide/getting-started/tuning-thresholds\"},{\"text\":\"Regression metric keys\",\"link\":\"/user-guide/getting-started/regression-metric-keys\"}]},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\",\"collapsed\":true,\"items\":[{\"text\":\"How stages are matched\",\"link\":\"/user-guide/run-comparison/how-stages-are-matched\"}]},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\",\"collapsed\":true,\"items\":[{\"text\":\"diagnose_run\",\"link\":\"/user-guide/mcp-tools/diagnose-run\"},{\"text\":\"get_run_summary\",\"link\":\"/user-guide/mcp-tools/get-run-summary\"},{\"text\":\"evaluate_budgets\",\"link\":\"/user-guide/mcp-tools/evaluate-budgets\"},{\"text\":\"compare_runs\",\"link\":\"/user-guide/mcp-tools/compare-runs\"},{\"text\":\"get_finding_evidence\",\"link\":\"/user-guide/mcp-tools/get-finding-evidence\"},{\"text\":\"get_finding_documentation\",\"link\":\"/user-guide/mcp-tools/get-finding-documentation\"},{\"text\":\"get_reference_doc\",\"link\":\"/user-guide/mcp-tools/get-reference-doc\"},{\"text\":\"list_runs\",\"link\":\"/user-guide/mcp-tools/list-runs\"}]},{\"text\":\"Finding your event log\",\"link\":\"/user-guide/alternative-log-retrieval\",\"collapsed\":true,\"items\":[{\"text\":\"Managed platforms\",\"link\":\"/user-guide/alternative-log-retrieval/managed-platforms\"},{\"text\":\"Behind an SSH bastion\",\"link\":\"/user-guide/alternative-log-retrieval/ssh-bastion\"}]}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\",\"collapsed\":true,\"items\":[{\"text\":\"Evidence availability\",\"link\":\"/contributor-guide/architecture/worker-protocol/evidence-availability\"},{\"text\":\"Evidence report and finding identity\",\"link\":\"/contributor-guide/architecture/worker-protocol/evidence-report\"},{\"text\":\"CLI and MCP server\",\"link\":\"/contributor-guide/architecture/worker-protocol/cli-and-mcp\"},{\"text\":\"Core and event validation\",\"link\":\"/contributor-guide/architecture/worker-protocol/core-and-event-validation\"}]},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\",\"collapsed\":true,\"items\":[{\"text\":\"Run comparison internals\",\"link\":\"/contributor-guide/architecture/state-and-history/run-comparison-internals\"},{\"text\":\"History Server intake\",\"link\":\"/contributor-guide/architecture/state-and-history/history-server-intake\"}]},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\",\"collapsed\":true,\"items\":[{\"text\":\"Plan attribution\",\"link\":\"/contributor-guide/architecture/detector-contract/plan-attribution\"},{\"text\":\"Threshold tables\",\"link\":\"/contributor-guide/architecture/detector-contract/threshold-tables\"}]},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\",\"collapsed\":true,\"items\":[{\"text\":\"Stage union rollup\",\"link\":\"/contributor-guide/architecture/impact-estimation/stage-union-rollup\"},{\"text\":\"Caveats, tuning and coverage\",\"link\":\"/contributor-guide/architecture/impact-estimation/caveats-tuning-and-coverage\"}]},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\",\"collapsed\":true,\"items\":[{\"text\":\"Findings tab\",\"link\":\"/contributor-guide/architecture/widget-rendering/findings-tab\"},{\"text\":\"List sort mode\",\"link\":\"/contributor-guide/architecture/widget-rendering/list-sort-mode\"},{\"text\":\"Row/expand contract\",\"link\":\"/contributor-guide/architecture/widget-rendering/row-expand-contract\"},{\"text\":\"ReferenceSection\",\"link\":\"/contributor-guide/architecture/widget-rendering/reference-section\"},{\"text\":\"Investigation routing\",\"link\":\"/contributor-guide/architecture/widget-rendering/investigation-routing\"}]},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\",\"collapsed\":true,\"items\":[{\"text\":\"Plan Advisor\",\"link\":\"/contributor-guide/architecture/board-widgets/plan-advisor\"},{\"text\":\"Caching and memory widgets\",\"link\":\"/contributor-guide/architecture/board-widgets/caching-and-memory\"},{\"text\":\"Visual system and templating\",\"link\":\"/contributor-guide/architecture/board-widgets/visual-system-and-templating\"}]},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\",\"collapsed\":true,\"items\":[{\"text\":\"Plan graph\",\"link\":\"/contributor-guide/architecture/drill-down/plan-graph\"}]}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":{\"selector\":\"[data-sf-anchor-offset]\",\"padding\":0},\"cleanUrls\":false}");</script>
28
28
 
29
29
  </body>
30
30
  </html>
package/public/docs/tuning-reference/index.html CHANGED
@@ -9,9 +9,9 @@
9
9
  <link rel="preload stylesheet" href="/docs/assets/style.tmDhD3c8.css" as="style">
10
10
  <link rel="preload stylesheet" href="/docs/vp-icons.css" as="style">
11
11
 
12
- <script type="module" src="/docs/assets/app.yjczoibf.js"></script>
12
+ <script type="module" src="/docs/assets/app.eYTPqTeq.js"></script>
13
13
  <link rel="preload" href="/docs/assets/inter-roman-latin.Di8DUHzh.woff2" as="font" type="font/woff2" crossorigin="">
14
- <link rel="modulepreload" href="/docs/assets/chunks/theme.DFcAl8Ws.js">
14
+ <link rel="modulepreload" href="/docs/assets/chunks/theme.DsTNE2pE.js">
15
15
  <link rel="modulepreload" href="/docs/assets/chunks/framework.DSg0KOwT.js">
16
16
  <link rel="modulepreload" href="/docs/assets/tuning-reference_index.md.ZVIpQ6Dk.lean.js">
17
17
  <script>(()=>{const c=`${`[${["data","shuffle","product","bar"].join("-")}]`}, .VPNav, .VPLocalNav`,a=24,d=document.documentElement,l=t=>{const e=getComputedStyle(t);return e.display==="none"||t.getClientRects().length===0?0:e.position==="fixed"?t.getBoundingClientRect().bottom:e.position==="sticky"?(parseFloat(e.top)||0)+t.offsetHeight:0},u=()=>{try{return location.hash?document.getElementById(decodeURIComponent(location.hash.slice(1))):null}catch{return null}};let r=null;const s=()=>{const t=Array.from(document.querySelectorAll(c),l),e=Math.ceil(Math.max(0,...t))+a;if(e===r)return;const n=r;r=e,d.style.setProperty("--sf-anchor-offset",`${r}px`);const o=n===null?null:u();if(!o)return;const f=(parseFloat(getComputedStyle(o).paddingTop)||0)+2;Math.abs(o.getBoundingClientRect().top-n)<=f&&o.scrollIntoView()},i=()=>{const t=document.createElement("div");t.setAttribute("data-sf-anchor-offset",""),t.setAttribute("aria-hidden","true"),document.body.appendChild(t);const e=window.ResizeObserver?new ResizeObserver(s):null,n=()=>{e&&document.querySelectorAll(c).forEach(o=>e.observe(o)),s()};n(),new MutationObserver(n).observe(document.body,{childList:!0,subtree:!0}),window.addEventListener("resize",s),window.addEventListener("load",s)};document.readyState==="loading"?document.addEventListener("DOMContentLoaded",i,{once:!0}):i()})();</script>
@@ -24,7 +24,7 @@
24
24
  </head>
25
25
  <body>
26
26
  <div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink active" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav empty fixed" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><!----><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><!----><div class="VPContent" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPPage" data-v-1428d186><!--[--><!--]--><div style="position:relative;"><div><main id="main-content" class="tuning-landing" data-v-5d65fdb9><section class="hero" aria-labelledby="page-title" data-v-5d65fdb9><div class="container hero-grid" data-v-5d65fdb9><div class="hero-copy" data-v-5d65fdb9><p class="eyebrow" data-v-5d65fdb9>Practical mechanics, stable references</p><h1 id="page-title" data-v-5d65fdb9>Spark tuning reference</h1><p class="hero-summary" data-v-5d65fdb9> Move from a job symptom to the Spark mechanics and settings that can change its outcome without treating a tuning guess as evidence. </p><div class="hero-actions" data-v-5d65fdb9><a class="button button-primary" href="/docs/tuning-reference/intro" data-v-5d65fdb9>Browse the full reference <span class="arrow" aria-hidden="true" data-v-5d65fdb9>→</span></a><a class="button button-secondary" href="#symptoms" data-v-5d65fdb9>Choose a symptom</a><form class="known-term" data-v-5d65fdb9><label for="known-term-input" data-v-5d65fdb9>Know the term or config key?</label><div class="known-term-row" data-v-5d65fdb9><input id="known-term-input" value="" name="term" type="search" list="known-term-options" placeholder="e.g. spark.sql.shuffle.partitions" autocomplete="off" data-v-5d65fdb9><button class="button button-secondary" type="submit" data-v-5d65fdb9>Jump to topic</button></div><small data-v-5d65fdb9>Known routes: memory, shuffle, joins, AQE, and config. Other terms open the configuration reference.</small><datalist id="known-term-options" data-v-5d65fdb9><option value="memory" data-v-5d65fdb9></option><option value="shuffle" data-v-5d65fdb9></option><option value="joins" data-v-5d65fdb9></option><option value="AQE" data-v-5d65fdb9></option><option value="config" data-v-5d65fdb9></option><option value="spark.memory.fraction" data-v-5d65fdb9></option><option value="spark.sql.shuffle.partitions" data-v-5d65fdb9></option><option value="spark.sql.adaptive.enabled" data-v-5d65fdb9></option><option value="spark.sql.autoBroadcastJoinThreshold" data-v-5d65fdb9></option></datalist></form></div></div><aside class="signal-board" aria-label="Example symptom paths" data-v-5d65fdb9><div class="board-label" data-v-5d65fdb9>evidence → mechanic → lever</div><ul class="signal-list" data-v-5d65fdb9><li data-v-5d65fdb9><a class="signal-link" href="/docs/tuning-reference/memory-model" data-v-5d65fdb9><span class="signal-marker" aria-hidden="true" data-v-5d65fdb9></span><span class="signal-title" data-v-5d65fdb9>Executor memory pressure</span><span class="signal-destination" data-v-5d65fdb9>#memory-model</span></a></li><li data-v-5d65fdb9><a class="signal-link" href="/docs/tuning-reference/shuffle" data-v-5d65fdb9><span class="signal-marker" aria-hidden="true" data-v-5d65fdb9></span><span class="signal-title" data-v-5d65fdb9>Slow, wide stages</span><span class="signal-destination" data-v-5d65fdb9>#shuffle</span></a></li><li data-v-5d65fdb9><a class="signal-link" href="/docs/tuning-reference/joins" data-v-5d65fdb9><span class="signal-marker" aria-hidden="true" data-v-5d65fdb9></span><span class="signal-title" data-v-5d65fdb9>Join strategy drift</span><span class="signal-destination" data-v-5d65fdb9>#joins</span></a></li></ul></aside></div></section><section class="section" id="symptoms" aria-labelledby="symptoms-title" data-v-5d65fdb9><div class="container" data-v-5d65fdb9><div class="section-heading" data-v-5d65fdb9><h2 id="symptoms-title" data-v-5d65fdb9>Symptoms, mapped to the right lever</h2><p data-v-5d65fdb9>Each path stays on the canonical reference document, so existing links and saved anchors continue to work.</p><p class="decision-note" data-v-5d65fdb9><strong data-v-5d65fdb9>How to use this:</strong> SparkForensics reads your Spark event log and surfaces evidence; this reference explains the mechanics and settings behind it. Diagnose the symptom, change one informed lever, then validate against the next run&#39;s evidence. Confirm behavior and defaults against your deployed Spark version and distribution. </p></div><div class="symptom-grid" data-v-5d65fdb9><article class="symptom" data-v-5d65fdb9><div data-v-5d65fdb9><h3 data-v-5d65fdb9>Spill or GC pressure</h3><p data-v-5d65fdb9>Trace executor memory, object lifetime, and the trade-offs behind changing memory settings.</p></div><a href="/docs/tuning-reference/memory-model" data-v-5d65fdb9>Open memory model <span aria-hidden="true" data-v-5d65fdb9>→</span></a></article><article class="symptom" data-v-5d65fdb9><div data-v-5d65fdb9><h3 data-v-5d65fdb9>Shuffle-heavy stages</h3><p data-v-5d65fdb9>Understand write/read costs, partition size, and why moving data becomes the bottleneck.</p></div><a href="/docs/tuning-reference/shuffle" data-v-5d65fdb9>Open shuffle guide <span aria-hidden="true" data-v-5d65fdb9>→</span></a></article><article class="symptom" data-v-5d65fdb9><div data-v-5d65fdb9><h3 data-v-5d65fdb9>Slow or skewed joins</h3><p data-v-5d65fdb9>Compare join mechanics before changing broadcast thresholds or reshaping a plan.</p></div><a href="/docs/tuning-reference/joins" data-v-5d65fdb9>Open join optimization <span aria-hidden="true" data-v-5d65fdb9>→</span></a></article><article class="symptom" data-v-5d65fdb9><div data-v-5d65fdb9><h3 data-v-5d65fdb9>Plan adapts poorly</h3><p data-v-5d65fdb9>Use AQE&#39;s runtime choices as an inspectable mechanism, not a checkbox to enable blindly.</p></div><a href="/docs/tuning-reference/aqe" data-v-5d65fdb9>Open AQE guide <span aria-hidden="true" data-v-5d65fdb9>→</span></a></article><article class="symptom" data-v-5d65fdb9><div data-v-5d65fdb9><h3 data-v-5d65fdb9>Need the exact setting</h3><p data-v-5d65fdb9>Check a setting&#39;s scope, default, and interaction before applying it to a cluster.</p></div><a href="/docs/tuning-reference/config" data-v-5d65fdb9>Open config reference <span aria-hidden="true" data-v-5d65fdb9>→</span></a></article></div></div></section><section class="section workflow" aria-labelledby="workflow-title" data-v-5d65fdb9><div class="container workflow-grid" data-v-5d65fdb9><div class="workflow-copy" data-v-5d65fdb9><h2 id="workflow-title" data-v-5d65fdb9>Let SparkForensics surface the evidence. Use this reference to reason about it.</h2><p data-v-5d65fdb9>The two tools stay deliberately separate: SparkForensics reads the event log; this static reference explains the mechanics and settings behind what you find.</p></div><ol class="steps" data-v-5d65fdb9><li data-v-5d65fdb9><span class="step-label" data-v-5d65fdb9>Observe</span><div data-v-5d65fdb9><strong data-v-5d65fdb9>Read the stage or executor behavior</strong><span data-v-5d65fdb9>Start with the event log, metrics, and plan details Spark already produces.</span></div></li><li data-v-5d65fdb9><span class="step-label" data-v-5d65fdb9>Explain</span><div data-v-5d65fdb9><strong data-v-5d65fdb9>Name the mechanism at work</strong><span data-v-5d65fdb9>Use the stable reference anchors to connect symptoms to memory, shuffle, joins, AQE, or configuration.</span></div></li><li data-v-5d65fdb9><span class="step-label" data-v-5d65fdb9>Test</span><div data-v-5d65fdb9><strong data-v-5d65fdb9>Change one informed lever</strong><span data-v-5d65fdb9>Make a measured change, then compare the evidence again rather than carrying a tuning superstition forward.</span></div></li></ol></div></section></main></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
27
- <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"D0Kkk_r5\",\"contributor-guide_architecture_detector-contract.md\":\"caA46z-u\",\"contributor-guide_architecture_drill-down.md\":\"CdtlKHl-\",\"contributor-guide_architecture_impact-estimation.md\":\"BNp14LtL\",\"contributor-guide_architecture_index.md\":\"B9yZieYL\",\"contributor-guide_architecture_overview.md\":\"DBGo-bwn\",\"contributor-guide_architecture_state-and-history.md\":\"w4lwybdh\",\"contributor-guide_architecture_widget-rendering.md\":\"CcvqxnZI\",\"contributor-guide_architecture_worker-protocol.md\":\"CKeFttKj\",\"contributor-guide_contributing.md\":\"CQMvGwFt\",\"contributor-guide_development-setup.md\":\"D9bu4j_c\",\"contributor-guide_testing.md\":\"By6XGR9a\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"D08Uaod7\",\"tuning-reference_aqe.md\":\"C1Ae6A8u\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"6wjKuJNE\",\"tuning-reference_bottleneck-cold-start.md\":\"BVQbsuHP\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"DtaYnaiF\",\"tuning-reference_bottleneck-failures.md\":\"Z1Tdmdd3\",\"tuning-reference_bottleneck-gc.md\":\"BCnXLf1n\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"CcvQj2g9\",\"tuning-reference_bottleneck-memory-utilization.md\":\"B-C0-8h-\",\"tuning-reference_bottleneck-retry-waste.md\":\"DCnhWiqj\",\"tuning-reference_bottleneck-shuffle.md\":\"B5OMJCd3\",\"tuning-reference_bottleneck-skew.md\":\"Cvo6NQm_\",\"tuning-reference_bottleneck-slow-host.md\":\"ja5qr3wq\",\"tuning-reference_bottleneck-small-files.md\":\"Crjtb0i2\",\"tuning-reference_bottleneck-spill.md\":\"Bl8KtSRg\",\"tuning-reference_bottleneck-straggler.md\":\"B0VN8reT\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"DQCftGhu\",\"tuning-reference_bottleneck-utilization.md\":\"BXDl30f9\",\"tuning-reference_caching.md\":\"CcaiMn_7\",\"tuning-reference_cluster-config.md\":\"DbWG1yZq\",\"tuning-reference_config.md\":\"YUqSpwFB\",\"tuning-reference_data-formats.md\":\"C5DX5xL4\",\"tuning-reference_index.md\":\"ZVIpQ6Dk\",\"tuning-reference_intro.md\":\"BSSaDlqx\",\"tuning-reference_joins.md\":\"BVhNCDnE\",\"tuning-reference_memory-model.md\":\"CfjFdgbl\",\"tuning-reference_metrics.md\":\"C64VNEL4\",\"tuning-reference_partitioning.md\":\"CtLK0xsS\",\"tuning-reference_pyspark.md\":\"D-92XgF8\",\"tuning-reference_shuffle.md\":\"Iq2v1oHJ\",\"tuning-reference_spark-architecture.md\":\"_qwnE_Sc\",\"tuning-reference_table-formats.md\":\"BSYEK-eN\",\"user-guide_alternative-log-retrieval.md\":\"z-8npxNk\",\"user-guide_getting-started.md\":\"BJW5oVhe\",\"user-guide_mcp-tools.md\":\"DcLiOa9t\",\"user-guide_run-comparison.md\":\"qin0a4IX\",\"user-guide_understanding-findings.md\":\"DzReyCXb\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Finding your event log\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":{\"selector\":\"[data-sf-anchor-offset]\",\"padding\":0},\"cleanUrls\":false}");</script>
27
+ <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"YQcj5EI4\",\"contributor-guide_architecture_board-widgets_caching-and-memory.md\":\"8bH1HnZp\",\"contributor-guide_architecture_board-widgets_plan-advisor.md\":\"RlzvUmNm\",\"contributor-guide_architecture_board-widgets_visual-system-and-templating.md\":\"DYgUpIJB\",\"contributor-guide_architecture_detector-contract.md\":\"wj6Et6UD\",\"contributor-guide_architecture_detector-contract_plan-attribution.md\":\"yNex4bys\",\"contributor-guide_architecture_detector-contract_threshold-tables.md\":\"CKjWG56Y\",\"contributor-guide_architecture_drill-down.md\":\"CqZPrdo9\",\"contributor-guide_architecture_drill-down_plan-graph.md\":\"DwOozIEs\",\"contributor-guide_architecture_impact-estimation.md\":\"CZv_ZGOp\",\"contributor-guide_architecture_impact-estimation_caveats-tuning-and-coverage.md\":\"BUucRc0A\",\"contributor-guide_architecture_impact-estimation_stage-union-rollup.md\":\"DdCGtAGu\",\"contributor-guide_architecture_index.md\":\"tymxf1Fh\",\"contributor-guide_architecture_overview.md\":\"C1sSvm9i\",\"contributor-guide_architecture_state-and-history.md\":\"DyCBdjuM\",\"contributor-guide_architecture_state-and-history_history-server-intake.md\":\"C298hrHZ\",\"contributor-guide_architecture_state-and-history_run-comparison-internals.md\":\"CZA0JLtV\",\"contributor-guide_architecture_widget-rendering.md\":\"BOZ1azXG\",\"contributor-guide_architecture_widget-rendering_findings-tab.md\":\"BQXycK6G\",\"contributor-guide_architecture_widget-rendering_investigation-routing.md\":\"BGdiyElu\",\"contributor-guide_architecture_widget-rendering_list-sort-mode.md\":\"BuHKgH0C\",\"contributor-guide_architecture_widget-rendering_reference-section.md\":\"DWqC80bN\",\"contributor-guide_architecture_widget-rendering_row-expand-contract.md\":\"ZT89u9f1\",\"contributor-guide_architecture_worker-protocol.md\":\"BAnhi2-d\",\"contributor-guide_architecture_worker-protocol_cli-and-mcp.md\":\"DC1YqXBp\",\"contributor-guide_architecture_worker-protocol_core-and-event-validation.md\":\"DSVFPpWb\",\"contributor-guide_architecture_worker-protocol_evidence-availability.md\":\"CjimIDBY\",\"contributor-guide_architecture_worker-protocol_evidence-report.md\":\"DVGZOWvW\",\"contributor-guide_contributing.md\":\"CQMvGwFt\",\"contributor-guide_development-setup.md\":\"D9bu4j_c\",\"contributor-guide_testing.md\":\"By6XGR9a\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"D08Uaod7\",\"tuning-reference_aqe.md\":\"C1Ae6A8u\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"6wjKuJNE\",\"tuning-reference_bottleneck-cold-start.md\":\"BVQbsuHP\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"DtaYnaiF\",\"tuning-reference_bottleneck-failures.md\":\"Z1Tdmdd3\",\"tuning-reference_bottleneck-gc.md\":\"BCnXLf1n\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"CcvQj2g9\",\"tuning-reference_bottleneck-memory-utilization.md\":\"B-C0-8h-\",\"tuning-reference_bottleneck-retry-waste.md\":\"DCnhWiqj\",\"tuning-reference_bottleneck-shuffle.md\":\"B5OMJCd3\",\"tuning-reference_bottleneck-skew.md\":\"Cvo6NQm_\",\"tuning-reference_bottleneck-slow-host.md\":\"ja5qr3wq\",\"tuning-reference_bottleneck-small-files.md\":\"Crjtb0i2\",\"tuning-reference_bottleneck-spill.md\":\"Bl8KtSRg\",\"tuning-reference_bottleneck-straggler.md\":\"B0VN8reT\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"DQCftGhu\",\"tuning-reference_bottleneck-utilization.md\":\"BXDl30f9\",\"tuning-reference_caching.md\":\"CcaiMn_7\",\"tuning-reference_cluster-config.md\":\"DbWG1yZq\",\"tuning-reference_config.md\":\"YUqSpwFB\",\"tuning-reference_data-formats.md\":\"C5DX5xL4\",\"tuning-reference_index.md\":\"ZVIpQ6Dk\",\"tuning-reference_intro.md\":\"BSSaDlqx\",\"tuning-reference_joins.md\":\"BVhNCDnE\",\"tuning-reference_memory-model.md\":\"CfjFdgbl\",\"tuning-reference_metrics.md\":\"C64VNEL4\",\"tuning-reference_partitioning.md\":\"CtLK0xsS\",\"tuning-reference_pyspark.md\":\"D-92XgF8\",\"tuning-reference_shuffle.md\":\"Iq2v1oHJ\",\"tuning-reference_spark-architecture.md\":\"_qwnE_Sc\",\"tuning-reference_table-formats.md\":\"BSYEK-eN\",\"user-guide_alternative-log-retrieval.md\":\"CJpYvHGo\",\"user-guide_alternative-log-retrieval_managed-platforms.md\":\"DvDwHNtC\",\"user-guide_alternative-log-retrieval_ssh-bastion.md\":\"Gszzlplv\",\"user-guide_getting-started.md\":\"DYHHHkQR\",\"user-guide_getting-started_ci-and-automation.md\":\"OUQqw3mI\",\"user-guide_getting-started_reading-the-dashboard.md\":\"Obf3W3__\",\"user-guide_getting-started_regression-metric-keys.md\":\"B_gzr2Mw\",\"user-guide_getting-started_tuning-thresholds.md\":\"B_MyjyZm\",\"user-guide_mcp-tools.md\":\"C65SjWUL\",\"user-guide_mcp-tools_compare-runs.md\":\"Cc_gLoix\",\"user-guide_mcp-tools_diagnose-run.md\":\"BP1vnHA5\",\"user-guide_mcp-tools_evaluate-budgets.md\":\"BB0nDY3Z\",\"user-guide_mcp-tools_get-finding-documentation.md\":\"BtTTHNV6\",\"user-guide_mcp-tools_get-finding-evidence.md\":\"rKyu4wts\",\"user-guide_mcp-tools_get-reference-doc.md\":\"Yac4tH3u\",\"user-guide_mcp-tools_get-run-summary.md\":\"DpxZ4ZNp\",\"user-guide_mcp-tools_list-runs.md\":\"DpQPeCM3\",\"user-guide_run-comparison.md\":\"BUALkohi\",\"user-guide_run-comparison_how-stages-are-matched.md\":\"DSXGu2tA\",\"user-guide_understanding-findings.md\":\"C9YRruMT\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\",\"collapsed\":true,\"items\":[{\"text\":\"Reading the dashboard\",\"link\":\"/user-guide/getting-started/reading-the-dashboard\"},{\"text\":\"CI and automation\",\"link\":\"/user-guide/getting-started/ci-and-automation\"},{\"text\":\"Tuning detector thresholds\",\"link\":\"/user-guide/getting-started/tuning-thresholds\"},{\"text\":\"Regression metric keys\",\"link\":\"/user-guide/getting-started/regression-metric-keys\"}]},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\",\"collapsed\":true,\"items\":[{\"text\":\"How stages are matched\",\"link\":\"/user-guide/run-comparison/how-stages-are-matched\"}]},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\",\"collapsed\":true,\"items\":[{\"text\":\"diagnose_run\",\"link\":\"/user-guide/mcp-tools/diagnose-run\"},{\"text\":\"get_run_summary\",\"link\":\"/user-guide/mcp-tools/get-run-summary\"},{\"text\":\"evaluate_budgets\",\"link\":\"/user-guide/mcp-tools/evaluate-budgets\"},{\"text\":\"compare_runs\",\"link\":\"/user-guide/mcp-tools/compare-runs\"},{\"text\":\"get_finding_evidence\",\"link\":\"/user-guide/mcp-tools/get-finding-evidence\"},{\"text\":\"get_finding_documentation\",\"link\":\"/user-guide/mcp-tools/get-finding-documentation\"},{\"text\":\"get_reference_doc\",\"link\":\"/user-guide/mcp-tools/get-reference-doc\"},{\"text\":\"list_runs\",\"link\":\"/user-guide/mcp-tools/list-runs\"}]},{\"text\":\"Finding your event log\",\"link\":\"/user-guide/alternative-log-retrieval\",\"collapsed\":true,\"items\":[{\"text\":\"Managed platforms\",\"link\":\"/user-guide/alternative-log-retrieval/managed-platforms\"},{\"text\":\"Behind an SSH bastion\",\"link\":\"/user-guide/alternative-log-retrieval/ssh-bastion\"}]}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\",\"collapsed\":true,\"items\":[{\"text\":\"Evidence availability\",\"link\":\"/contributor-guide/architecture/worker-protocol/evidence-availability\"},{\"text\":\"Evidence report and finding identity\",\"link\":\"/contributor-guide/architecture/worker-protocol/evidence-report\"},{\"text\":\"CLI and MCP server\",\"link\":\"/contributor-guide/architecture/worker-protocol/cli-and-mcp\"},{\"text\":\"Core and event validation\",\"link\":\"/contributor-guide/architecture/worker-protocol/core-and-event-validation\"}]},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\",\"collapsed\":true,\"items\":[{\"text\":\"Run comparison internals\",\"link\":\"/contributor-guide/architecture/state-and-history/run-comparison-internals\"},{\"text\":\"History Server intake\",\"link\":\"/contributor-guide/architecture/state-and-history/history-server-intake\"}]},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\",\"collapsed\":true,\"items\":[{\"text\":\"Plan attribution\",\"link\":\"/contributor-guide/architecture/detector-contract/plan-attribution\"},{\"text\":\"Threshold tables\",\"link\":\"/contributor-guide/architecture/detector-contract/threshold-tables\"}]},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\",\"collapsed\":true,\"items\":[{\"text\":\"Stage union rollup\",\"link\":\"/contributor-guide/architecture/impact-estimation/stage-union-rollup\"},{\"text\":\"Caveats, tuning and coverage\",\"link\":\"/contributor-guide/architecture/impact-estimation/caveats-tuning-and-coverage\"}]},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\",\"collapsed\":true,\"items\":[{\"text\":\"Findings tab\",\"link\":\"/contributor-guide/architecture/widget-rendering/findings-tab\"},{\"text\":\"List sort mode\",\"link\":\"/contributor-guide/architecture/widget-rendering/list-sort-mode\"},{\"text\":\"Row/expand contract\",\"link\":\"/contributor-guide/architecture/widget-rendering/row-expand-contract\"},{\"text\":\"ReferenceSection\",\"link\":\"/contributor-guide/architecture/widget-rendering/reference-section\"},{\"text\":\"Investigation routing\",\"link\":\"/contributor-guide/architecture/widget-rendering/investigation-routing\"}]},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\",\"collapsed\":true,\"items\":[{\"text\":\"Plan Advisor\",\"link\":\"/contributor-guide/architecture/board-widgets/plan-advisor\"},{\"text\":\"Caching and memory widgets\",\"link\":\"/contributor-guide/architecture/board-widgets/caching-and-memory\"},{\"text\":\"Visual system and templating\",\"link\":\"/contributor-guide/architecture/board-widgets/visual-system-and-templating\"}]},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\",\"collapsed\":true,\"items\":[{\"text\":\"Plan graph\",\"link\":\"/contributor-guide/architecture/drill-down/plan-graph\"}]}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":{\"selector\":\"[data-sf-anchor-offset]\",\"padding\":0},\"cleanUrls\":false}");</script>
28
28
 
29
29
  </body>
30
30
  </html>
package/public/docs/tuning-reference/intro.html CHANGED
@@ -9,9 +9,9 @@
9
9
  <link rel="preload stylesheet" href="/docs/assets/style.tmDhD3c8.css" as="style">
10
10
  <link rel="preload stylesheet" href="/docs/vp-icons.css" as="style">
11
11
 
12
- <script type="module" src="/docs/assets/app.yjczoibf.js"></script>
12
+ <script type="module" src="/docs/assets/app.eYTPqTeq.js"></script>
13
13
  <link rel="preload" href="/docs/assets/inter-roman-latin.Di8DUHzh.woff2" as="font" type="font/woff2" crossorigin="">
14
- <link rel="modulepreload" href="/docs/assets/chunks/theme.DFcAl8Ws.js">
14
+ <link rel="modulepreload" href="/docs/assets/chunks/theme.DsTNE2pE.js">
15
15
  <link rel="modulepreload" href="/docs/assets/chunks/framework.DSg0KOwT.js">
16
16
  <link rel="modulepreload" href="/docs/assets/tuning-reference_intro.md.BSSaDlqx.lean.js">
17
17
  <script>(()=>{const c=`${`[${["data","shuffle","product","bar"].join("-")}]`}, .VPNav, .VPLocalNav`,a=24,d=document.documentElement,l=t=>{const e=getComputedStyle(t);return e.display==="none"||t.getClientRects().length===0?0:e.position==="fixed"?t.getBoundingClientRect().bottom:e.position==="sticky"?(parseFloat(e.top)||0)+t.offsetHeight:0},u=()=>{try{return location.hash?document.getElementById(decodeURIComponent(location.hash.slice(1))):null}catch{return null}};let r=null;const s=()=>{const t=Array.from(document.querySelectorAll(c),l),e=Math.ceil(Math.max(0,...t))+a;if(e===r)return;const n=r;r=e,d.style.setProperty("--sf-anchor-offset",`${r}px`);const o=n===null?null:u();if(!o)return;const f=(parseFloat(getComputedStyle(o).paddingTop)||0)+2;Math.abs(o.getBoundingClientRect().top-n)<=f&&o.scrollIntoView()},i=()=>{const t=document.createElement("div");t.setAttribute("data-sf-anchor-offset",""),t.setAttribute("aria-hidden","true"),document.body.appendChild(t);const e=window.ResizeObserver?new ResizeObserver(s):null,n=()=>{e&&document.querySelectorAll(c).forEach(o=>e.observe(o)),s()};n(),new MutationObserver(n).observe(document.body,{childList:!0,subtree:!0}),window.addEventListener("resize",s),window.addEventListener("load",s)};document.readyState==="loading"?document.addEventListener("DOMContentLoaded",i,{once:!0}):i()})();</script>
@@ -24,7 +24,7 @@
24
24
  </head>
25
25
  <body>
26
26
  <div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Getting Started</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/intro.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Introduction</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Optimization Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/spark-architecture.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Execution Model</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/memory-model.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Management</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/partitioning.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Partitioning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/joins.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Join Optimization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/data-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Data Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/table-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Table Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/caching.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Caching & Persistence</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/pyspark.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>PySpark Specifics</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/aqe.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Adaptive Query Execution</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/cluster-config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cluster Tuning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/anti-patterns.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Anti-Patterns</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Detector Catalog</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-skew.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Skew</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle I/O</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-spill.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory / Disk Spill</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-gc.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>GC Pressure</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-cold-start.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cold Start</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Executor Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-slow-host.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Slow Host</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-failures.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Failures</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-straggler.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Stragglers</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-retry-waste.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Retry Waste</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-tiny-tasks.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Tiny Tasks</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-job-failure-rate.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Job Failure Rate</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-memory-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Duplicate Plan Subtree</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-small-files.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Small Files</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-broadcast-sizing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Broadcast Sizing</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Reference</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/metrics.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Metrics Glossary</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Config Quick-Reference</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_tuning-reference_intro" data-v-39a288b8><div><h1 id="intro" tabindex="-1">Introduction <a class="header-anchor" href="#intro" aria-label="Permalink to &quot;Introduction {#intro}&quot;">​</a></h1><p>This guide is a hands-on reference for optimizing Spark and PySpark jobs. It covers <a href="./spark-architecture.html">Spark internals</a>, <a href="./memory-model.html">memory management</a>, <a href="./joins.html">joins</a>, <a href="./shuffle.html">shuffle</a>, <a href="./data-formats.html">data formats</a>, <a href="./pyspark.html">PySpark-specific patterns</a>, <a href="./aqe.html">Adaptive Query Execution</a> (AQE), and <a href="./cluster-config.html">cluster tuning</a>, followed by a bottleneck diagnostic reference.</p><p>Read it top to bottom for a full grounding in Spark performance, or jump straight to the <strong>Bottleneck Reference</strong> section below when you already know which symptom you&#39;re chasing.</p><h2 id="severity-dots" tabindex="-1">Severity dots <a class="header-anchor" href="#severity-dots" aria-label="Permalink to &quot;Severity dots&quot;">​</a></h2><p>Each finding is marked with a severity dot:</p><ul><li><span class="severity-dot info"></span> <strong>Info</strong>: worth knowing, not yet a problem</li><li><span class="severity-dot warning"></span> <strong>Warning</strong>: likely hurting job performance</li><li><span class="severity-dot critical"></span> <strong>Critical</strong>: actively bottlenecking the job</li></ul><p>Each bottleneck section below documents the exact thresholds behind these dots.</p><h2 id="tag-system" tabindex="-1">Tag system <a class="header-anchor" href="#tag-system" aria-label="Permalink to &quot;Tag system&quot;">​</a></h2><p>Every bottleneck has a short tag used throughout this reference:</p><p><span class="tag">SKEW</span> <span class="tag">SHFL</span> <span class="tag">SPILL</span><span class="tag">GC</span> <span class="tag">COLD</span> <span class="tag">UTIL</span><span class="tag">HOST</span> <span class="tag">FAIL</span> <span class="tag">STRAG</span><span class="tag">RETRY</span> <span class="tag">TINY</span> <span class="tag">FAIL-RATE</span></p><p>See the <a href="./bottleneck-skew.html">Bottleneck Reference</a> for what each one means, how it&#39;s detected, and how to fix it.</p></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><!----></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="/docs/tuning-reference/spark-architecture.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>Spark Execution Model</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
27
- <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"D0Kkk_r5\",\"contributor-guide_architecture_detector-contract.md\":\"caA46z-u\",\"contributor-guide_architecture_drill-down.md\":\"CdtlKHl-\",\"contributor-guide_architecture_impact-estimation.md\":\"BNp14LtL\",\"contributor-guide_architecture_index.md\":\"B9yZieYL\",\"contributor-guide_architecture_overview.md\":\"DBGo-bwn\",\"contributor-guide_architecture_state-and-history.md\":\"w4lwybdh\",\"contributor-guide_architecture_widget-rendering.md\":\"CcvqxnZI\",\"contributor-guide_architecture_worker-protocol.md\":\"CKeFttKj\",\"contributor-guide_contributing.md\":\"CQMvGwFt\",\"contributor-guide_development-setup.md\":\"D9bu4j_c\",\"contributor-guide_testing.md\":\"By6XGR9a\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"D08Uaod7\",\"tuning-reference_aqe.md\":\"C1Ae6A8u\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"6wjKuJNE\",\"tuning-reference_bottleneck-cold-start.md\":\"BVQbsuHP\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"DtaYnaiF\",\"tuning-reference_bottleneck-failures.md\":\"Z1Tdmdd3\",\"tuning-reference_bottleneck-gc.md\":\"BCnXLf1n\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"CcvQj2g9\",\"tuning-reference_bottleneck-memory-utilization.md\":\"B-C0-8h-\",\"tuning-reference_bottleneck-retry-waste.md\":\"DCnhWiqj\",\"tuning-reference_bottleneck-shuffle.md\":\"B5OMJCd3\",\"tuning-reference_bottleneck-skew.md\":\"Cvo6NQm_\",\"tuning-reference_bottleneck-slow-host.md\":\"ja5qr3wq\",\"tuning-reference_bottleneck-small-files.md\":\"Crjtb0i2\",\"tuning-reference_bottleneck-spill.md\":\"Bl8KtSRg\",\"tuning-reference_bottleneck-straggler.md\":\"B0VN8reT\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"DQCftGhu\",\"tuning-reference_bottleneck-utilization.md\":\"BXDl30f9\",\"tuning-reference_caching.md\":\"CcaiMn_7\",\"tuning-reference_cluster-config.md\":\"DbWG1yZq\",\"tuning-reference_config.md\":\"YUqSpwFB\",\"tuning-reference_data-formats.md\":\"C5DX5xL4\",\"tuning-reference_index.md\":\"ZVIpQ6Dk\",\"tuning-reference_intro.md\":\"BSSaDlqx\",\"tuning-reference_joins.md\":\"BVhNCDnE\",\"tuning-reference_memory-model.md\":\"CfjFdgbl\",\"tuning-reference_metrics.md\":\"C64VNEL4\",\"tuning-reference_partitioning.md\":\"CtLK0xsS\",\"tuning-reference_pyspark.md\":\"D-92XgF8\",\"tuning-reference_shuffle.md\":\"Iq2v1oHJ\",\"tuning-reference_spark-architecture.md\":\"_qwnE_Sc\",\"tuning-reference_table-formats.md\":\"BSYEK-eN\",\"user-guide_alternative-log-retrieval.md\":\"z-8npxNk\",\"user-guide_getting-started.md\":\"BJW5oVhe\",\"user-guide_mcp-tools.md\":\"DcLiOa9t\",\"user-guide_run-comparison.md\":\"qin0a4IX\",\"user-guide_understanding-findings.md\":\"DzReyCXb\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Finding your event log\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":{\"selector\":\"[data-sf-anchor-offset]\",\"padding\":0},\"cleanUrls\":false}");</script>
27
+ <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"YQcj5EI4\",\"contributor-guide_architecture_board-widgets_caching-and-memory.md\":\"8bH1HnZp\",\"contributor-guide_architecture_board-widgets_plan-advisor.md\":\"RlzvUmNm\",\"contributor-guide_architecture_board-widgets_visual-system-and-templating.md\":\"DYgUpIJB\",\"contributor-guide_architecture_detector-contract.md\":\"wj6Et6UD\",\"contributor-guide_architecture_detector-contract_plan-attribution.md\":\"yNex4bys\",\"contributor-guide_architecture_detector-contract_threshold-tables.md\":\"CKjWG56Y\",\"contributor-guide_architecture_drill-down.md\":\"CqZPrdo9\",\"contributor-guide_architecture_drill-down_plan-graph.md\":\"DwOozIEs\",\"contributor-guide_architecture_impact-estimation.md\":\"CZv_ZGOp\",\"contributor-guide_architecture_impact-estimation_caveats-tuning-and-coverage.md\":\"BUucRc0A\",\"contributor-guide_architecture_impact-estimation_stage-union-rollup.md\":\"DdCGtAGu\",\"contributor-guide_architecture_index.md\":\"tymxf1Fh\",\"contributor-guide_architecture_overview.md\":\"C1sSvm9i\",\"contributor-guide_architecture_state-and-history.md\":\"DyCBdjuM\",\"contributor-guide_architecture_state-and-history_history-server-intake.md\":\"C298hrHZ\",\"contributor-guide_architecture_state-and-history_run-comparison-internals.md\":\"CZA0JLtV\",\"contributor-guide_architecture_widget-rendering.md\":\"BOZ1azXG\",\"contributor-guide_architecture_widget-rendering_findings-tab.md\":\"BQXycK6G\",\"contributor-guide_architecture_widget-rendering_investigation-routing.md\":\"BGdiyElu\",\"contributor-guide_architecture_widget-rendering_list-sort-mode.md\":\"BuHKgH0C\",\"contributor-guide_architecture_widget-rendering_reference-section.md\":\"DWqC80bN\",\"contributor-guide_architecture_widget-rendering_row-expand-contract.md\":\"ZT89u9f1\",\"contributor-guide_architecture_worker-protocol.md\":\"BAnhi2-d\",\"contributor-guide_architecture_worker-protocol_cli-and-mcp.md\":\"DC1YqXBp\",\"contributor-guide_architecture_worker-protocol_core-and-event-validation.md\":\"DSVFPpWb\",\"contributor-guide_architecture_worker-protocol_evidence-availability.md\":\"CjimIDBY\",\"contributor-guide_architecture_worker-protocol_evidence-report.md\":\"DVGZOWvW\",\"contributor-guide_contributing.md\":\"CQMvGwFt\",\"contributor-guide_development-setup.md\":\"D9bu4j_c\",\"contributor-guide_testing.md\":\"By6XGR9a\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"D08Uaod7\",\"tuning-reference_aqe.md\":\"C1Ae6A8u\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"6wjKuJNE\",\"tuning-reference_bottleneck-cold-start.md\":\"BVQbsuHP\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"DtaYnaiF\",\"tuning-reference_bottleneck-failures.md\":\"Z1Tdmdd3\",\"tuning-reference_bottleneck-gc.md\":\"BCnXLf1n\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"CcvQj2g9\",\"tuning-reference_bottleneck-memory-utilization.md\":\"B-C0-8h-\",\"tuning-reference_bottleneck-retry-waste.md\":\"DCnhWiqj\",\"tuning-reference_bottleneck-shuffle.md\":\"B5OMJCd3\",\"tuning-reference_bottleneck-skew.md\":\"Cvo6NQm_\",\"tuning-reference_bottleneck-slow-host.md\":\"ja5qr3wq\",\"tuning-reference_bottleneck-small-files.md\":\"Crjtb0i2\",\"tuning-reference_bottleneck-spill.md\":\"Bl8KtSRg\",\"tuning-reference_bottleneck-straggler.md\":\"B0VN8reT\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"DQCftGhu\",\"tuning-reference_bottleneck-utilization.md\":\"BXDl30f9\",\"tuning-reference_caching.md\":\"CcaiMn_7\",\"tuning-reference_cluster-config.md\":\"DbWG1yZq\",\"tuning-reference_config.md\":\"YUqSpwFB\",\"tuning-reference_data-formats.md\":\"C5DX5xL4\",\"tuning-reference_index.md\":\"ZVIpQ6Dk\",\"tuning-reference_intro.md\":\"BSSaDlqx\",\"tuning-reference_joins.md\":\"BVhNCDnE\",\"tuning-reference_memory-model.md\":\"CfjFdgbl\",\"tuning-reference_metrics.md\":\"C64VNEL4\",\"tuning-reference_partitioning.md\":\"CtLK0xsS\",\"tuning-reference_pyspark.md\":\"D-92XgF8\",\"tuning-reference_shuffle.md\":\"Iq2v1oHJ\",\"tuning-reference_spark-architecture.md\":\"_qwnE_Sc\",\"tuning-reference_table-formats.md\":\"BSYEK-eN\",\"user-guide_alternative-log-retrieval.md\":\"CJpYvHGo\",\"user-guide_alternative-log-retrieval_managed-platforms.md\":\"DvDwHNtC\",\"user-guide_alternative-log-retrieval_ssh-bastion.md\":\"Gszzlplv\",\"user-guide_getting-started.md\":\"DYHHHkQR\",\"user-guide_getting-started_ci-and-automation.md\":\"OUQqw3mI\",\"user-guide_getting-started_reading-the-dashboard.md\":\"Obf3W3__\",\"user-guide_getting-started_regression-metric-keys.md\":\"B_gzr2Mw\",\"user-guide_getting-started_tuning-thresholds.md\":\"B_MyjyZm\",\"user-guide_mcp-tools.md\":\"C65SjWUL\",\"user-guide_mcp-tools_compare-runs.md\":\"Cc_gLoix\",\"user-guide_mcp-tools_diagnose-run.md\":\"BP1vnHA5\",\"user-guide_mcp-tools_evaluate-budgets.md\":\"BB0nDY3Z\",\"user-guide_mcp-tools_get-finding-documentation.md\":\"BtTTHNV6\",\"user-guide_mcp-tools_get-finding-evidence.md\":\"rKyu4wts\",\"user-guide_mcp-tools_get-reference-doc.md\":\"Yac4tH3u\",\"user-guide_mcp-tools_get-run-summary.md\":\"DpxZ4ZNp\",\"user-guide_mcp-tools_list-runs.md\":\"DpQPeCM3\",\"user-guide_run-comparison.md\":\"BUALkohi\",\"user-guide_run-comparison_how-stages-are-matched.md\":\"DSXGu2tA\",\"user-guide_understanding-findings.md\":\"C9YRruMT\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\",\"collapsed\":true,\"items\":[{\"text\":\"Reading the dashboard\",\"link\":\"/user-guide/getting-started/reading-the-dashboard\"},{\"text\":\"CI and automation\",\"link\":\"/user-guide/getting-started/ci-and-automation\"},{\"text\":\"Tuning detector thresholds\",\"link\":\"/user-guide/getting-started/tuning-thresholds\"},{\"text\":\"Regression metric keys\",\"link\":\"/user-guide/getting-started/regression-metric-keys\"}]},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\",\"collapsed\":true,\"items\":[{\"text\":\"How stages are matched\",\"link\":\"/user-guide/run-comparison/how-stages-are-matched\"}]},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\",\"collapsed\":true,\"items\":[{\"text\":\"diagnose_run\",\"link\":\"/user-guide/mcp-tools/diagnose-run\"},{\"text\":\"get_run_summary\",\"link\":\"/user-guide/mcp-tools/get-run-summary\"},{\"text\":\"evaluate_budgets\",\"link\":\"/user-guide/mcp-tools/evaluate-budgets\"},{\"text\":\"compare_runs\",\"link\":\"/user-guide/mcp-tools/compare-runs\"},{\"text\":\"get_finding_evidence\",\"link\":\"/user-guide/mcp-tools/get-finding-evidence\"},{\"text\":\"get_finding_documentation\",\"link\":\"/user-guide/mcp-tools/get-finding-documentation\"},{\"text\":\"get_reference_doc\",\"link\":\"/user-guide/mcp-tools/get-reference-doc\"},{\"text\":\"list_runs\",\"link\":\"/user-guide/mcp-tools/list-runs\"}]},{\"text\":\"Finding your event log\",\"link\":\"/user-guide/alternative-log-retrieval\",\"collapsed\":true,\"items\":[{\"text\":\"Managed platforms\",\"link\":\"/user-guide/alternative-log-retrieval/managed-platforms\"},{\"text\":\"Behind an SSH bastion\",\"link\":\"/user-guide/alternative-log-retrieval/ssh-bastion\"}]}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\",\"collapsed\":true,\"items\":[{\"text\":\"Evidence availability\",\"link\":\"/contributor-guide/architecture/worker-protocol/evidence-availability\"},{\"text\":\"Evidence report and finding identity\",\"link\":\"/contributor-guide/architecture/worker-protocol/evidence-report\"},{\"text\":\"CLI and MCP server\",\"link\":\"/contributor-guide/architecture/worker-protocol/cli-and-mcp\"},{\"text\":\"Core and event validation\",\"link\":\"/contributor-guide/architecture/worker-protocol/core-and-event-validation\"}]},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\",\"collapsed\":true,\"items\":[{\"text\":\"Run comparison internals\",\"link\":\"/contributor-guide/architecture/state-and-history/run-comparison-internals\"},{\"text\":\"History Server intake\",\"link\":\"/contributor-guide/architecture/state-and-history/history-server-intake\"}]},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\",\"collapsed\":true,\"items\":[{\"text\":\"Plan attribution\",\"link\":\"/contributor-guide/architecture/detector-contract/plan-attribution\"},{\"text\":\"Threshold tables\",\"link\":\"/contributor-guide/architecture/detector-contract/threshold-tables\"}]},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\",\"collapsed\":true,\"items\":[{\"text\":\"Stage union rollup\",\"link\":\"/contributor-guide/architecture/impact-estimation/stage-union-rollup\"},{\"text\":\"Caveats, tuning and coverage\",\"link\":\"/contributor-guide/architecture/impact-estimation/caveats-tuning-and-coverage\"}]},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\",\"collapsed\":true,\"items\":[{\"text\":\"Findings tab\",\"link\":\"/contributor-guide/architecture/widget-rendering/findings-tab\"},{\"text\":\"List sort mode\",\"link\":\"/contributor-guide/architecture/widget-rendering/list-sort-mode\"},{\"text\":\"Row/expand contract\",\"link\":\"/contributor-guide/architecture/widget-rendering/row-expand-contract\"},{\"text\":\"ReferenceSection\",\"link\":\"/contributor-guide/architecture/widget-rendering/reference-section\"},{\"text\":\"Investigation routing\",\"link\":\"/contributor-guide/architecture/widget-rendering/investigation-routing\"}]},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\",\"collapsed\":true,\"items\":[{\"text\":\"Plan Advisor\",\"link\":\"/contributor-guide/architecture/board-widgets/plan-advisor\"},{\"text\":\"Caching and memory widgets\",\"link\":\"/contributor-guide/architecture/board-widgets/caching-and-memory\"},{\"text\":\"Visual system and templating\",\"link\":\"/contributor-guide/architecture/board-widgets/visual-system-and-templating\"}]},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\",\"collapsed\":true,\"items\":[{\"text\":\"Plan graph\",\"link\":\"/contributor-guide/architecture/drill-down/plan-graph\"}]}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":{\"selector\":\"[data-sf-anchor-offset]\",\"padding\":0},\"cleanUrls\":false}");</script>
28
28
 
29
29
  </body>
30
30
  </html>