sparkforensics-server 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (440) hide show
  1. package/README.md +6 -0
  2. package/index.js +40 -49
  3. package/lib/static-files.js +10 -6
  4. package/package.json +20 -1
  5. package/public/assets/AutoscalingChurn-CFJkiqxW.js +1 -0
  6. package/public/assets/CacheUtilization-kTKZBzDA.js +1 -0
  7. package/public/assets/CachingOpportunity-CU70EWcF.js +1 -0
  8. package/public/assets/ColdStart-hOI-WOvm.js +1 -0
  9. package/public/assets/ConfigAudit-B4oAksZY.js +1 -0
  10. package/public/assets/CoreUsageArea-D-dR3SQk.js +1 -0
  11. package/public/assets/Dashboard-MDhohsq4.js +4 -0
  12. package/public/assets/DuplicatePlanSubtree-54RKsfI7.js +1 -0
  13. package/public/assets/DurationHistogram-CviQyPNM.js +1 -0
  14. package/public/assets/EmptyState-v2ehFncc.js +1 -0
  15. package/public/assets/ExecutorUtilization-vHyFHgL6.js +1 -0
  16. package/public/assets/GcPressure-DINuhcno.js +1 -0
  17. package/public/assets/ImpactEstimate-C6GzFOY9.js +1 -0
  18. package/public/assets/IncompleteRun-a1kuCnU6.js +1 -0
  19. package/public/assets/JobFailures-CYwPfUhR.js +1 -0
  20. package/public/assets/MemoryUtilization-Dlfj0e_f.js +1 -0
  21. package/public/assets/OverBroadcast-BMu7Y8l9.js +1 -0
  22. package/public/assets/PartitionSizing-CvMGfSVB.js +1 -0
  23. package/public/assets/PlanExplorer-xYFme_Rx.js +1 -0
  24. package/public/assets/PlanGraphRoute-a3OGg6ET.js +1 -0
  25. package/public/assets/RetryWaste-JKRjOZ-J.js +1 -0
  26. package/public/assets/RowPagination-DypWbnic.js +1 -0
  27. package/public/assets/ShuffleIO-Mi8HiVu-.js +1 -0
  28. package/public/assets/Skew-C3iOScTC.js +1 -0
  29. package/public/assets/SlowHost-DKBg18v-.js +1 -0
  30. package/public/assets/SmallFiles-CCCB6AIu.js +1 -0
  31. package/public/assets/SpeculationWaste-DUNupw0_.js +1 -0
  32. package/public/assets/Spill-BlLdatik.js +1 -0
  33. package/public/assets/StageFailed-qtY7C7Yo.js +1 -0
  34. package/public/assets/StageFindingGroup-CoVdEZYs.js +1 -0
  35. package/public/assets/StageHeader-uDBLHgdg.js +1 -0
  36. package/public/assets/StagePill-CYCMdlIx.js +1 -0
  37. package/public/assets/StageShape-zs0mAjQJ.js +1 -0
  38. package/public/assets/StageSlowness-C7vg0u5Q.js +1 -0
  39. package/public/assets/Straggler-4BRGAQP4.js +1 -0
  40. package/public/assets/TaskFailures-CSjO23Kg.js +1 -0
  41. package/public/assets/TinyTask-6Zuovcn5.js +1 -0
  42. package/public/assets/UnderBroadcast-CePjo0ZV.js +1 -0
  43. package/public/assets/WidgetLeadSummary-DRNRVvn7.js +1 -0
  44. package/public/assets/{charts-vendor-NMqKV0H4.js → charts-vendor-Cf1-wzeH.js} +2 -2
  45. package/public/assets/downsample-D8XmJk6i.js +1 -0
  46. package/public/assets/finding-anchor-4TmpXd7j.js +1 -0
  47. package/public/assets/impact-sort-Cq6KOLqh.js +1 -0
  48. package/public/assets/inbox-DBFovbUX.js +1 -0
  49. package/public/assets/index-CHjV2-jB.js +9 -0
  50. package/public/assets/index-DMEtaP4u.css +1 -0
  51. package/public/assets/parser-worker-BMPPE37w.js +64 -0
  52. package/public/assets/plan-graph-vendor-B96a-e4N.js +7 -0
  53. package/public/assets/react-vendor-DA6opS-q.js +9 -0
  54. package/public/assets/select-D5fK-za0.js +1 -0
  55. package/public/assets/toggle-group-lbIXRG0W.js +1 -0
  56. package/public/assets/usePagedRows-CFoZrPr4.js +1 -0
  57. package/public/assets/useRecentFiles-BPGQ6XII.js +13 -0
  58. package/public/assets/useSortMode-BOPpfLhJ.js +1 -0
  59. package/public/docs/404.html +3 -3
  60. package/public/docs/assets/{app.Bxy4_HVA.js → app.BgzsaTS3.js} +1 -1
  61. package/public/docs/assets/aqe-loop.IwQSATHw.svg +1 -0
  62. package/public/docs/assets/aqe-loop.dark.DGbaxqJE.svg +1 -0
  63. package/public/docs/assets/broadcast-vs-shuffle.Db4WY1XK.svg +1 -0
  64. package/public/docs/assets/broadcast-vs-shuffle.dark.C7Bxs0mG.svg +1 -0
  65. package/public/docs/assets/cache-lifecycle.dark.B-hS7AgU.svg +1 -0
  66. package/public/docs/assets/cache-lifecycle.rEOVYQNU.svg +1 -0
  67. package/public/docs/assets/chunks/@localSearchIndexroot.PQ5JepCp.js +1 -0
  68. package/public/docs/assets/chunks/{VPLocalSearchBox.L5jNOIR3.js → VPLocalSearchBox.DOq3t-On.js} +1 -1
  69. package/public/docs/assets/chunks/duplicate-plan-subtree.dark.Cdp70QhV.js +1 -0
  70. package/public/docs/assets/chunks/{framework.CNDaxUod.js → framework.DSg0KOwT.js} +1 -1
  71. package/public/docs/assets/chunks/retry-escalation-ladder.dark.DHipdJgZ.js +1 -0
  72. package/public/docs/assets/chunks/theme.BP14rGBx.js +2 -0
  73. package/public/docs/assets/cold-start-timeline.DxC_Sc7w.svg +1 -0
  74. package/public/docs/assets/cold-start-timeline.dark.CZ17YcAG.svg +1 -0
  75. package/public/docs/assets/columnar-layout.PghGeOEA.svg +1 -0
  76. package/public/docs/assets/columnar-layout.dark.BVNlz0ff.svg +1 -0
  77. package/public/docs/assets/container-memory.DIO0AnIm.svg +1 -0
  78. package/public/docs/assets/container-memory.dark.CP-5zuCl.svg +1 -0
  79. package/public/docs/assets/contributor-guide_architecture_board-widgets.md.B-OsL91z.js +1 -0
  80. package/public/docs/assets/contributor-guide_architecture_board-widgets.md.B-OsL91z.lean.js +1 -0
  81. package/public/docs/assets/contributor-guide_architecture_detector-contract.md.BOeH4d1J.js +1 -0
  82. package/public/docs/assets/contributor-guide_architecture_detector-contract.md.BOeH4d1J.lean.js +1 -0
  83. package/public/docs/assets/contributor-guide_architecture_drill-down.md.BtPdlM7r.js +1 -0
  84. package/public/docs/assets/contributor-guide_architecture_drill-down.md.BtPdlM7r.lean.js +1 -0
  85. package/public/docs/assets/{contributor-guide_architecture_impact-estimation.md.CzlXOnCq.js → contributor-guide_architecture_impact-estimation.md.DYCDPgkh.js} +1 -1
  86. package/public/docs/assets/{contributor-guide_architecture_impact-estimation.md.CzlXOnCq.lean.js → contributor-guide_architecture_impact-estimation.md.DYCDPgkh.lean.js} +1 -1
  87. package/public/docs/assets/{contributor-guide_architecture_index.md.94lNbF-4.js → contributor-guide_architecture_index.md.3TO9ic6w.js} +1 -1
  88. package/public/docs/assets/{contributor-guide_architecture_index.md.94lNbF-4.lean.js → contributor-guide_architecture_index.md.3TO9ic6w.lean.js} +1 -1
  89. package/public/docs/assets/{contributor-guide_architecture_overview.md.mQFlN66o.js → contributor-guide_architecture_overview.md.CehiRmGn.js} +1 -1
  90. package/public/docs/assets/{contributor-guide_architecture_overview.md.mQFlN66o.lean.js → contributor-guide_architecture_overview.md.CehiRmGn.lean.js} +1 -1
  91. package/public/docs/assets/contributor-guide_architecture_state-and-history.md.m3S3UdMk.js +1 -0
  92. package/public/docs/assets/contributor-guide_architecture_state-and-history.md.m3S3UdMk.lean.js +1 -0
  93. package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.DbqPf2OT.js +1 -0
  94. package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.DbqPf2OT.lean.js +1 -0
  95. package/public/docs/assets/{contributor-guide_architecture_worker-protocol.md.BaS_L10g.js → contributor-guide_architecture_worker-protocol.md.B93qJ_tT.js} +2 -2
  96. package/public/docs/assets/{contributor-guide_architecture_worker-protocol.md.BaS_L10g.lean.js → contributor-guide_architecture_worker-protocol.md.B93qJ_tT.lean.js} +1 -1
  97. package/public/docs/assets/contributor-guide_contributing.md.CvRsdr6J.js +1 -0
  98. package/public/docs/assets/contributor-guide_contributing.md.CvRsdr6J.lean.js +1 -0
  99. package/public/docs/assets/{contributor-guide_development-setup.md.CpUMVtIK.js → contributor-guide_development-setup.md.DvAN_9mK.js} +1 -1
  100. package/public/docs/assets/{contributor-guide_development-setup.md.CpUMVtIK.lean.js → contributor-guide_development-setup.md.DvAN_9mK.lean.js} +1 -1
  101. package/public/docs/assets/{contributor-guide_testing.md.C0q-BJfl.js → contributor-guide_testing.md.6rIKqSyY.js} +1 -1
  102. package/public/docs/assets/{contributor-guide_testing.md.C0q-BJfl.lean.js → contributor-guide_testing.md.6rIKqSyY.lean.js} +1 -1
  103. package/public/docs/assets/dag-stages.DSz_S937.svg +1 -0
  104. package/public/docs/assets/dag-stages.dark.F72UzxH4.svg +1 -0
  105. package/public/docs/assets/driver-executor.D5pQ7YN1.svg +1 -0
  106. package/public/docs/assets/driver-executor.dark.BmX9cPvh.svg +1 -0
  107. package/public/docs/assets/duplicate-plan-subtree.B4cvN6fj.svg +1 -0
  108. package/public/docs/assets/duplicate-plan-subtree.dark.Dw8wS0Ag.svg +1 -0
  109. package/public/docs/assets/{index.md.CETnDZzR.js → index.md.CHJVslga.js} +1 -1
  110. package/public/docs/assets/{index.md.CETnDZzR.lean.js → index.md.CHJVslga.lean.js} +1 -1
  111. package/public/docs/assets/join-strategy.C_FvrCEo.svg +1 -0
  112. package/public/docs/assets/join-strategy.dark.ChMLnNII.svg +1 -0
  113. package/public/docs/assets/memory-borrowing.BqQRJg0u.svg +1 -0
  114. package/public/docs/assets/memory-borrowing.dark.Yhh20O9C.svg +1 -0
  115. package/public/docs/assets/memory-regions.XHvO7jHG.svg +1 -0
  116. package/public/docs/assets/memory-regions.dark.D4TP9_08.svg +1 -0
  117. package/public/docs/assets/repartition-vs-coalesce.BovLRrpj.svg +1 -0
  118. package/public/docs/assets/repartition-vs-coalesce.dark.BhAczKZQ.svg +1 -0
  119. package/public/docs/assets/retry-escalation-ladder.DyTKJJmZ.svg +1 -0
  120. package/public/docs/assets/retry-escalation-ladder.dark.BdsabtU3.svg +1 -0
  121. package/public/docs/assets/shuffle-map-reduce.KuOEZVmg.svg +1 -0
  122. package/public/docs/assets/shuffle-map-reduce.dark.BgQZnFSb.svg +1 -0
  123. package/public/docs/assets/spill-classification.BU2euYDO.svg +1 -0
  124. package/public/docs/assets/spill-classification.dark.D7i1M40d.svg +1 -0
  125. package/public/docs/assets/{style.D2mY0HUs.css → style.DXOMCXxn.css} +1 -1
  126. package/public/docs/assets/tuning-reference_anti-patterns.md.Df1YMIHu.js +1 -0
  127. package/public/docs/assets/tuning-reference_anti-patterns.md.Df1YMIHu.lean.js +1 -0
  128. package/public/docs/assets/tuning-reference_aqe.md.BIsCtLzm.js +1 -0
  129. package/public/docs/assets/tuning-reference_aqe.md.BIsCtLzm.lean.js +1 -0
  130. package/public/docs/assets/tuning-reference_bottleneck-broadcast-sizing.md.CEstB3Ia.js +1 -0
  131. package/public/docs/assets/tuning-reference_bottleneck-broadcast-sizing.md.CEstB3Ia.lean.js +1 -0
  132. package/public/docs/assets/tuning-reference_bottleneck-cold-start.md.CEuy-72y.js +7 -0
  133. package/public/docs/assets/tuning-reference_bottleneck-cold-start.md.CEuy-72y.lean.js +1 -0
  134. package/public/docs/assets/tuning-reference_bottleneck-duplicate-plan-subtree.md.CIohQDfn.js +1 -0
  135. package/public/docs/assets/tuning-reference_bottleneck-duplicate-plan-subtree.md.CIohQDfn.lean.js +1 -0
  136. package/public/docs/assets/tuning-reference_bottleneck-failures.md.4z5BXGJ2.js +6 -0
  137. package/public/docs/assets/tuning-reference_bottleneck-failures.md.4z5BXGJ2.lean.js +1 -0
  138. package/public/docs/assets/tuning-reference_bottleneck-gc.md.DSxzZRK7.js +6 -0
  139. package/public/docs/assets/tuning-reference_bottleneck-gc.md.DSxzZRK7.lean.js +1 -0
  140. package/public/docs/assets/tuning-reference_bottleneck-job-failure-rate.md.BaJl__1W.js +8 -0
  141. package/public/docs/assets/tuning-reference_bottleneck-job-failure-rate.md.BaJl__1W.lean.js +1 -0
  142. package/public/docs/assets/tuning-reference_bottleneck-memory-utilization.md.DbP-SJZc.js +7 -0
  143. package/public/docs/assets/tuning-reference_bottleneck-memory-utilization.md.DbP-SJZc.lean.js +1 -0
  144. package/public/docs/assets/tuning-reference_bottleneck-retry-waste.md.D5JMjVOt.js +1 -0
  145. package/public/docs/assets/tuning-reference_bottleneck-retry-waste.md.D5JMjVOt.lean.js +1 -0
  146. package/public/docs/assets/tuning-reference_bottleneck-shuffle.md.CM-nTmIH.js +12 -0
  147. package/public/docs/assets/tuning-reference_bottleneck-shuffle.md.CM-nTmIH.lean.js +1 -0
  148. package/public/docs/assets/tuning-reference_bottleneck-skew.md.BdUwiDhn.js +14 -0
  149. package/public/docs/assets/tuning-reference_bottleneck-skew.md.BdUwiDhn.lean.js +1 -0
  150. package/public/docs/assets/tuning-reference_bottleneck-slow-host.md.BlIo6UDW.js +7 -0
  151. package/public/docs/assets/tuning-reference_bottleneck-slow-host.md.BlIo6UDW.lean.js +1 -0
  152. package/public/docs/assets/tuning-reference_bottleneck-small-files.md.B8kloyx8.js +5 -0
  153. package/public/docs/assets/tuning-reference_bottleneck-small-files.md.B8kloyx8.lean.js +1 -0
  154. package/public/docs/assets/tuning-reference_bottleneck-spill.md.PNH7mITt.js +6 -0
  155. package/public/docs/assets/tuning-reference_bottleneck-spill.md.PNH7mITt.lean.js +1 -0
  156. package/public/docs/assets/tuning-reference_bottleneck-straggler.md.DY36fHN5.js +7 -0
  157. package/public/docs/assets/tuning-reference_bottleneck-straggler.md.DY36fHN5.lean.js +1 -0
  158. package/public/docs/assets/tuning-reference_bottleneck-tiny-tasks.md.QTV7O8kU.js +8 -0
  159. package/public/docs/assets/tuning-reference_bottleneck-tiny-tasks.md.QTV7O8kU.lean.js +1 -0
  160. package/public/docs/assets/tuning-reference_bottleneck-utilization.md.DTiueZC3.js +5 -0
  161. package/public/docs/assets/tuning-reference_bottleneck-utilization.md.DTiueZC3.lean.js +1 -0
  162. package/public/docs/assets/tuning-reference_caching.md.B7aQ8asB.js +1 -0
  163. package/public/docs/assets/tuning-reference_caching.md.B7aQ8asB.lean.js +1 -0
  164. package/public/docs/assets/tuning-reference_cluster-config.md.ZVmDGsQ3.js +1 -0
  165. package/public/docs/assets/tuning-reference_cluster-config.md.ZVmDGsQ3.lean.js +1 -0
  166. package/public/docs/assets/tuning-reference_config.md.UvveiWG3.js +1 -0
  167. package/public/docs/assets/tuning-reference_config.md.UvveiWG3.lean.js +1 -0
  168. package/public/docs/assets/tuning-reference_data-formats.md.bjCAWH3N.js +1 -0
  169. package/public/docs/assets/tuning-reference_data-formats.md.bjCAWH3N.lean.js +1 -0
  170. package/public/docs/assets/tuning-reference_index.md.BQ_NooMV.js +1 -0
  171. package/public/docs/assets/tuning-reference_index.md.BQ_NooMV.lean.js +1 -0
  172. package/public/docs/assets/tuning-reference_intro.md.CobD-lGB.js +1 -0
  173. package/public/docs/assets/tuning-reference_intro.md.CobD-lGB.lean.js +1 -0
  174. package/public/docs/assets/tuning-reference_joins.md.BtKs_CuW.js +1 -0
  175. package/public/docs/assets/tuning-reference_joins.md.BtKs_CuW.lean.js +1 -0
  176. package/public/docs/assets/tuning-reference_memory-model.md.DhT-n4y3.js +1 -0
  177. package/public/docs/assets/tuning-reference_memory-model.md.DhT-n4y3.lean.js +1 -0
  178. package/public/docs/assets/tuning-reference_metrics.md.mLOh7Apj.js +1 -0
  179. package/public/docs/assets/tuning-reference_metrics.md.mLOh7Apj.lean.js +1 -0
  180. package/public/docs/assets/tuning-reference_partitioning.md.q0zKF_8X.js +1 -0
  181. package/public/docs/assets/tuning-reference_partitioning.md.q0zKF_8X.lean.js +1 -0
  182. package/public/docs/assets/tuning-reference_pyspark.md.DDCfvN9t.js +6 -0
  183. package/public/docs/assets/tuning-reference_pyspark.md.DDCfvN9t.lean.js +1 -0
  184. package/public/docs/assets/tuning-reference_shuffle.md.BZZ7R4Ix.js +1 -0
  185. package/public/docs/assets/tuning-reference_shuffle.md.BZZ7R4Ix.lean.js +1 -0
  186. package/public/docs/assets/tuning-reference_spark-architecture.md.Dwzm5avO.js +1 -0
  187. package/public/docs/assets/tuning-reference_spark-architecture.md.Dwzm5avO.lean.js +1 -0
  188. package/public/docs/assets/tuning-reference_table-formats.md.D6wj-2dX.js +1 -0
  189. package/public/docs/assets/tuning-reference_table-formats.md.D6wj-2dX.lean.js +1 -0
  190. package/public/docs/assets/udf-execution-models.BUFDICuG.svg +1 -0
  191. package/public/docs/assets/udf-execution-models.dark.YTNS6GDq.svg +1 -0
  192. package/public/docs/assets/user-guide_alternative-log-retrieval.md.sU3KGarf.js +1 -0
  193. package/public/docs/assets/user-guide_alternative-log-retrieval.md.sU3KGarf.lean.js +1 -0
  194. package/public/docs/assets/user-guide_getting-started.md.DtEM37MK.js +3 -0
  195. package/public/docs/assets/user-guide_getting-started.md.DtEM37MK.lean.js +1 -0
  196. package/public/docs/assets/{user-guide_mcp-tools.md.DwYI1d6C.js → user-guide_mcp-tools.md.C8MiIu7F.js} +35 -5
  197. package/public/docs/assets/{user-guide_mcp-tools.md.DwYI1d6C.lean.js → user-guide_mcp-tools.md.C8MiIu7F.lean.js} +1 -1
  198. package/public/docs/assets/{user-guide_run-comparison.md.5-oaOfTu.js → user-guide_run-comparison.md.S0TWWmLY.js} +1 -1
  199. package/public/docs/assets/{user-guide_run-comparison.md.5-oaOfTu.lean.js → user-guide_run-comparison.md.S0TWWmLY.lean.js} +1 -1
  200. package/public/docs/assets/user-guide_understanding-findings.md.D0R_Y-R2.js +1 -0
  201. package/public/docs/assets/{user-guide_understanding-findings.md.CK2WZ0Tf.lean.js → user-guide_understanding-findings.md.D0R_Y-R2.lean.js} +1 -1
  202. package/public/docs/contributor-guide/architecture/board-widgets.html +7 -7
  203. package/public/docs/contributor-guide/architecture/detector-contract.html +7 -7
  204. package/public/docs/contributor-guide/architecture/drill-down.html +7 -7
  205. package/public/docs/contributor-guide/architecture/impact-estimation.html +7 -7
  206. package/public/docs/contributor-guide/architecture/index.html +7 -7
  207. package/public/docs/contributor-guide/architecture/overview.html +7 -7
  208. package/public/docs/contributor-guide/architecture/state-and-history.html +7 -7
  209. package/public/docs/contributor-guide/architecture/widget-rendering.html +7 -7
  210. package/public/docs/contributor-guide/architecture/worker-protocol.html +8 -8
  211. package/public/docs/contributor-guide/contributing.html +7 -7
  212. package/public/docs/contributor-guide/development-setup.html +7 -7
  213. package/public/docs/contributor-guide/testing.html +7 -7
  214. package/public/docs/hashmap.json +1 -1
  215. package/public/docs/index.html +7 -7
  216. package/public/docs/tuning-reference/anti-patterns.html +29 -0
  217. package/public/docs/tuning-reference/aqe.html +29 -0
  218. package/public/docs/tuning-reference/bottleneck-broadcast-sizing.html +29 -0
  219. package/public/docs/tuning-reference/bottleneck-cold-start.html +35 -0
  220. package/public/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html +30 -0
  221. package/public/docs/tuning-reference/bottleneck-failures.html +35 -0
  222. package/public/docs/tuning-reference/bottleneck-gc.html +34 -0
  223. package/public/docs/tuning-reference/bottleneck-job-failure-rate.html +37 -0
  224. package/public/docs/tuning-reference/bottleneck-memory-utilization.html +35 -0
  225. package/public/docs/tuning-reference/bottleneck-retry-waste.html +30 -0
  226. package/public/docs/tuning-reference/bottleneck-shuffle.html +40 -0
  227. package/public/docs/tuning-reference/bottleneck-skew.html +42 -0
  228. package/public/docs/tuning-reference/bottleneck-slow-host.html +35 -0
  229. package/public/docs/tuning-reference/bottleneck-small-files.html +33 -0
  230. package/public/docs/tuning-reference/bottleneck-spill.html +34 -0
  231. package/public/docs/tuning-reference/bottleneck-straggler.html +35 -0
  232. package/public/docs/tuning-reference/bottleneck-tiny-tasks.html +36 -0
  233. package/public/docs/tuning-reference/bottleneck-utilization.html +34 -0
  234. package/public/docs/tuning-reference/caching.html +29 -0
  235. package/public/docs/tuning-reference/cluster-config.html +29 -0
  236. package/public/docs/tuning-reference/config.html +29 -0
  237. package/public/docs/tuning-reference/data-formats.html +29 -0
  238. package/public/docs/tuning-reference/index.html +29 -0
  239. package/public/docs/tuning-reference/intro.html +29 -0
  240. package/public/docs/tuning-reference/joins.html +29 -0
  241. package/public/docs/tuning-reference/memory-model.html +29 -0
  242. package/public/docs/tuning-reference/metrics.html +29 -0
  243. package/public/docs/tuning-reference/partitioning.html +29 -0
  244. package/public/docs/tuning-reference/pyspark.html +34 -0
  245. package/public/docs/tuning-reference/shuffle.html +29 -0
  246. package/public/docs/tuning-reference/spark-architecture.html +29 -0
  247. package/public/docs/tuning-reference/table-formats.html +29 -0
  248. package/public/docs/user-guide/alternative-log-retrieval.html +29 -0
  249. package/public/docs/user-guide/getting-started.html +8 -8
  250. package/public/docs/user-guide/mcp-tools.html +41 -11
  251. package/public/docs/user-guide/run-comparison.html +7 -7
  252. package/public/docs/user-guide/understanding-findings.html +7 -7
  253. package/public/index.html +6 -6
  254. package/vendor-core/analyzer.js +74 -74
  255. package/vendor-core/cli/budgets.js +13 -27
  256. package/vendor-core/cli/collect-run.js +43 -19
  257. package/vendor-core/core-count.js +25 -27
  258. package/vendor-core/core-locality-ratio.js +4 -11
  259. package/vendor-core/core-time-series.js +6 -12
  260. package/vendor-core/core-usage-locality.js +3 -4
  261. package/vendor-core/detectors.js +256 -375
  262. package/vendor-core/docs-config.js +69 -21
  263. package/vendor-core/docs-content/chapters/01-intro.md +32 -0
  264. package/vendor-core/docs-content/chapters/02-spark-architecture.md +76 -0
  265. package/vendor-core/docs-content/chapters/03-memory-model.md +73 -0
  266. package/vendor-core/docs-content/chapters/04-partitioning.md +65 -0
  267. package/vendor-core/docs-content/chapters/05-joins.md +62 -0
  268. package/vendor-core/docs-content/chapters/06-shuffle.md +59 -0
  269. package/vendor-core/docs-content/chapters/07-data-formats.md +81 -0
  270. package/vendor-core/docs-content/chapters/07b-table-formats.md +56 -0
  271. package/vendor-core/docs-content/chapters/08-caching.md +58 -0
  272. package/vendor-core/docs-content/chapters/09-pyspark.md +78 -0
  273. package/vendor-core/docs-content/chapters/10-aqe.md +167 -0
  274. package/vendor-core/docs-content/chapters/11-cluster-config.md +170 -0
  275. package/vendor-core/docs-content/chapters/12-anti-patterns.md +171 -0
  276. package/vendor-core/docs-content/chapters/14-metrics.md +87 -0
  277. package/vendor-core/docs-content/chapters/15-config.md +93 -0
  278. package/vendor-core/docs-content/chapters/nav-index.json +370 -0
  279. package/vendor-core/docs-content/detection/cache.md +6 -0
  280. package/vendor-core/docs-content/detection/cfg.md +15 -0
  281. package/vendor-core/docs-content/detection/chrn.md +7 -0
  282. package/vendor-core/docs-content/detection/cold.md +4 -0
  283. package/vendor-core/docs-content/detection/cstor.md +4 -0
  284. package/vendor-core/docs-content/detection/fail.md +5 -0
  285. package/vendor-core/docs-content/detection/gc.md +4 -0
  286. package/vendor-core/docs-content/detection/host.md +5 -0
  287. package/vendor-core/docs-content/detection/incmp.md +6 -0
  288. package/vendor-core/docs-content/detection/jobs.md +4 -0
  289. package/vendor-core/docs-content/detection/local.md +7 -0
  290. package/vendor-core/docs-content/detection/mem.md +10 -0
  291. package/vendor-core/docs-content/detection/part.md +5 -0
  292. package/vendor-core/docs-content/detection/plan.md +14 -0
  293. package/vendor-core/docs-content/detection/retry.md +4 -0
  294. package/vendor-core/docs-content/detection/sfail.md +5 -0
  295. package/vendor-core/docs-content/detection/shape.md +5 -0
  296. package/vendor-core/docs-content/detection/shfl.md +4 -0
  297. package/vendor-core/docs-content/detection/skew.md +6 -0
  298. package/vendor-core/docs-content/detection/slow.md +6 -0
  299. package/vendor-core/docs-content/detection/spec.md +7 -0
  300. package/vendor-core/docs-content/detection/spill.md +7 -0
  301. package/vendor-core/docs-content/detection/strag.md +5 -0
  302. package/vendor-core/docs-content/detection/tiny.md +4 -0
  303. package/vendor-core/docs-content/detection/util.md +4 -0
  304. package/vendor-core/docs-content/diagrams/aqe-loop.dark.svg +1 -0
  305. package/vendor-core/docs-content/diagrams/aqe-loop.svg +1 -0
  306. package/vendor-core/docs-content/diagrams/broadcast-vs-shuffle.dark.svg +1 -0
  307. package/vendor-core/docs-content/diagrams/broadcast-vs-shuffle.svg +1 -0
  308. package/vendor-core/docs-content/diagrams/cache-lifecycle.dark.svg +1 -0
  309. package/vendor-core/docs-content/diagrams/cache-lifecycle.svg +1 -0
  310. package/vendor-core/docs-content/diagrams/cold-start-timeline.dark.svg +1 -0
  311. package/vendor-core/docs-content/diagrams/cold-start-timeline.svg +1 -0
  312. package/vendor-core/docs-content/diagrams/columnar-layout.dark.svg +1 -0
  313. package/vendor-core/docs-content/diagrams/columnar-layout.svg +1 -0
  314. package/vendor-core/docs-content/diagrams/container-memory.dark.svg +1 -0
  315. package/vendor-core/docs-content/diagrams/container-memory.svg +1 -0
  316. package/vendor-core/docs-content/diagrams/dag-stages.dark.svg +1 -0
  317. package/vendor-core/docs-content/diagrams/dag-stages.svg +1 -0
  318. package/vendor-core/docs-content/diagrams/driver-executor.dark.svg +1 -0
  319. package/vendor-core/docs-content/diagrams/driver-executor.svg +1 -0
  320. package/vendor-core/docs-content/diagrams/duplicate-plan-subtree.dark.svg +1 -0
  321. package/vendor-core/docs-content/diagrams/duplicate-plan-subtree.svg +1 -0
  322. package/vendor-core/docs-content/diagrams/join-strategy.dark.svg +1 -0
  323. package/vendor-core/docs-content/diagrams/join-strategy.svg +1 -0
  324. package/vendor-core/docs-content/diagrams/memory-borrowing.dark.svg +1 -0
  325. package/vendor-core/docs-content/diagrams/memory-borrowing.svg +1 -0
  326. package/vendor-core/docs-content/diagrams/memory-regions.dark.svg +1 -0
  327. package/vendor-core/docs-content/diagrams/memory-regions.svg +1 -0
  328. package/vendor-core/docs-content/diagrams/repartition-vs-coalesce.dark.svg +1 -0
  329. package/vendor-core/docs-content/diagrams/repartition-vs-coalesce.svg +1 -0
  330. package/vendor-core/docs-content/diagrams/retry-escalation-ladder.dark.svg +1 -0
  331. package/vendor-core/docs-content/diagrams/retry-escalation-ladder.svg +1 -0
  332. package/vendor-core/docs-content/diagrams/shuffle-map-reduce.dark.svg +1 -0
  333. package/vendor-core/docs-content/diagrams/shuffle-map-reduce.svg +1 -0
  334. package/vendor-core/docs-content/diagrams/spill-classification.dark.svg +1 -0
  335. package/vendor-core/docs-content/diagrams/spill-classification.svg +1 -0
  336. package/vendor-core/docs-content/diagrams/udf-execution-models.dark.svg +1 -0
  337. package/vendor-core/docs-content/diagrams/udf-execution-models.svg +1 -0
  338. package/vendor-core/docs-content/tuning/broadcast-sizing.md +78 -0
  339. package/vendor-core/docs-content/tuning/cold-start.md +81 -0
  340. package/vendor-core/docs-content/tuning/duplicate-plan-subtree.md +45 -0
  341. package/vendor-core/docs-content/tuning/failures.md +124 -0
  342. package/vendor-core/docs-content/tuning/gc.md +110 -0
  343. package/vendor-core/docs-content/tuning/job-failure-rate.md +101 -0
  344. package/vendor-core/docs-content/tuning/memory-utilization.md +58 -0
  345. package/vendor-core/docs-content/tuning/retry-waste.md +90 -0
  346. package/vendor-core/docs-content/tuning/shuffle.md +154 -0
  347. package/vendor-core/docs-content/tuning/skew.md +123 -0
  348. package/vendor-core/docs-content/tuning/slow-host.md +117 -0
  349. package/vendor-core/docs-content/tuning/small-files.md +99 -0
  350. package/vendor-core/docs-content/tuning/spill.md +114 -0
  351. package/vendor-core/docs-content/tuning/straggler.md +103 -0
  352. package/vendor-core/docs-content/tuning/tiny-tasks.md +94 -0
  353. package/vendor-core/docs-content/tuning/utilization.md +90 -0
  354. package/vendor-core/docs-site-config.js +10 -17
  355. package/vendor-core/efficiency-model.js +7 -13
  356. package/vendor-core/etl-phases.js +3 -5
  357. package/vendor-core/event-handlers.js +232 -134
  358. package/vendor-core/event-schemas.js +48 -114
  359. package/vendor-core/evidence-availability.js +5 -10
  360. package/vendor-core/evidence-report.js +72 -122
  361. package/vendor-core/export-data.js +48 -0
  362. package/vendor-core/finding-action-label.js +4 -10
  363. package/vendor-core/finding-filter-predicate.js +3 -7
  364. package/vendor-core/finding-generic-recommendation.js +112 -0
  365. package/vendor-core/finding-names.js +51 -0
  366. package/vendor-core/format-utils.js +112 -38
  367. package/vendor-core/impact-band.js +18 -24
  368. package/vendor-core/impact-estimator.js +38 -74
  369. package/vendor-core/ingest.js +7 -13
  370. package/vendor-core/job-groups.js +3 -6
  371. package/vendor-core/list-runs.js +278 -0
  372. package/vendor-core/load-vendored.js +6 -12
  373. package/vendor-core/log-header-peek.js +81 -0
  374. package/vendor-core/lz4-block.js +4 -6
  375. package/vendor-core/mcp-server-factory.js +38 -8
  376. package/vendor-core/mcp-tools.js +105 -76
  377. package/vendor-core/model-assembler.js +8 -16
  378. package/vendor-core/occupancy.js +5 -9
  379. package/vendor-core/parser-worker.js +18 -27
  380. package/vendor-core/plan-dot.js +2 -5
  381. package/vendor-core/plan-duration-attribution.js +78 -29
  382. package/vendor-core/plan-graph-model.js +126 -69
  383. package/vendor-core/plan-node-detail.js +31 -17
  384. package/vendor-core/plan-summary.js +19 -8
  385. package/vendor-core/recommendation-rollup.js +35 -39
  386. package/vendor-core/redact.js +72 -16
  387. package/vendor-core/rolling-log-reassembly.js +4 -6
  388. package/vendor-core/run-comparison.js +65 -70
  389. package/vendor-core/scaling-sim.js +5 -7
  390. package/vendor-core/session-snapshot.js +1 -1
  391. package/vendor-core/shs-fetch.js +4 -6
  392. package/vendor-core/shs-load.js +9 -13
  393. package/vendor-core/shs-request.js +1 -1
  394. package/vendor-core/stage-quantiles.js +14 -0
  395. package/vendor-core/types.js +78 -18
  396. package/vendor-core/wasted-core-hours.js +7 -12
  397. package/public/assets/AutoscalingChurn-CP5OjN6v.js +0 -1
  398. package/public/assets/CacheUtilization-BjyawXL6.js +0 -1
  399. package/public/assets/CachingOpportunity-DV2ABG2G.js +0 -1
  400. package/public/assets/ConfigAudit-DMiBhAoG.js +0 -1
  401. package/public/assets/CoreUsageArea-CGYIgPnt.js +0 -1
  402. package/public/assets/Dashboard-D1_IrWRs.js +0 -12
  403. package/public/assets/ExecutorTimeline-DkaAGqQT.js +0 -1
  404. package/public/assets/Failures-Crngoicu.js +0 -1
  405. package/public/assets/GcPressure-CFXm4BnA.js +0 -1
  406. package/public/assets/IncompleteRun-BWEJTb13.js +0 -1
  407. package/public/assets/JobFailures-BmjxCA-h.js +0 -1
  408. package/public/assets/MemoryUtilization-Bu2uQJbd.js +0 -1
  409. package/public/assets/PlanExplorer-DKUwF0uz.js +0 -1
  410. package/public/assets/PlanGraphRoute-BjPSugIG.js +0 -1
  411. package/public/assets/ShuffleIO--Rur3neu.js +0 -1
  412. package/public/assets/Spill-DIMpt8c2.js +0 -1
  413. package/public/assets/TaskSkew-BDr1DbeI.js +0 -1
  414. package/public/assets/index-4id52G-A.js +0 -9
  415. package/public/assets/index-D9kwVYIO.css +0 -1
  416. package/public/assets/parser-worker-00O9WEQC.js +0 -65
  417. package/public/assets/plan-graph-vendor-vZQHG2EJ.js +0 -7
  418. package/public/assets/react-vendor-u4pyvC5Y.js +0 -9
  419. package/public/assets/useRecentFiles-46iBU9zJ.js +0 -5
  420. package/public/docs/assets/chunks/@localSearchIndexroot.DclgKtlT.js +0 -1
  421. package/public/docs/assets/chunks/theme.CoT6y5bf.js +0 -2
  422. package/public/docs/assets/contributor-guide_architecture_board-widgets.md.BtjoE_1k.js +0 -1
  423. package/public/docs/assets/contributor-guide_architecture_board-widgets.md.BtjoE_1k.lean.js +0 -1
  424. package/public/docs/assets/contributor-guide_architecture_detector-contract.md.DQyCHQNM.js +0 -1
  425. package/public/docs/assets/contributor-guide_architecture_detector-contract.md.DQyCHQNM.lean.js +0 -1
  426. package/public/docs/assets/contributor-guide_architecture_drill-down.md.BKfyDQck.js +0 -1
  427. package/public/docs/assets/contributor-guide_architecture_drill-down.md.BKfyDQck.lean.js +0 -1
  428. package/public/docs/assets/contributor-guide_architecture_state-and-history.md.DaqttKgQ.js +0 -1
  429. package/public/docs/assets/contributor-guide_architecture_state-and-history.md.DaqttKgQ.lean.js +0 -1
  430. package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.Bk6Iyj9S.js +0 -1
  431. package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.Bk6Iyj9S.lean.js +0 -1
  432. package/public/docs/assets/contributor-guide_contributing.md.C-lKs70I.js +0 -1
  433. package/public/docs/assets/contributor-guide_contributing.md.C-lKs70I.lean.js +0 -1
  434. package/public/docs/assets/user-guide_getting-started.md.BJdD7a7_.js +0 -3
  435. package/public/docs/assets/user-guide_getting-started.md.BJdD7a7_.lean.js +0 -1
  436. package/public/docs/assets/user-guide_understanding-findings.md.CK2WZ0Tf.js +0 -1
  437. package/public/vendor/spark-doc/anchors.json +0 -205
  438. package/public/vendor/spark-doc/index.html +0 -4202
  439. package/public/vendor/spark-doc/landing.html +0 -418
  440. package/public/vendor/spark-doc/meta.html +0 -970
package/public/docs/tuning-reference/metrics.html ADDED
@@ -0,0 +1,29 @@
1
+ <!DOCTYPE html>
2
+ <html lang="en-US" dir="ltr">
3
+ <head>
4
+ <meta charset="utf-8">
5
+ <meta name="viewport" content="width=device-width,initial-scale=1">
6
+ <title>Metrics Glossary | SparkForensics</title>
7
+ <meta name="description" content="Docs for using and contributing to SparkForensics">
8
+ <meta name="generator" content="VitePress v1.6.4">
9
+ <link rel="preload stylesheet" href="/docs/assets/style.DXOMCXxn.css" as="style">
10
+ <link rel="preload stylesheet" href="/docs/vp-icons.css" as="style">
11
+
12
+ <script type="module" src="/docs/assets/app.BgzsaTS3.js"></script>
13
+ <link rel="preload" href="/docs/assets/inter-roman-latin.Di8DUHzh.woff2" as="font" type="font/woff2" crossorigin="">
14
+ <link rel="modulepreload" href="/docs/assets/chunks/theme.BP14rGBx.js">
15
+ <link rel="modulepreload" href="/docs/assets/chunks/framework.DSg0KOwT.js">
16
+ <link rel="modulepreload" href="/docs/assets/tuning-reference_metrics.md.mLOh7Apj.lean.js">
17
+ <link rel="icon" type="image/svg+xml" href="/docs/favicon.svg">
18
+ <link rel="preconnect" href="https://fonts.googleapis.com">
19
+ <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin="">
20
+ <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Recursive:wght,CASL@400..700,0..1&amp;family=JetBrains+Mono:wght@400;500;600&amp;display=swap">
21
+ <script id="check-dark-mode">(()=>{const e=localStorage.getItem("vitepress-theme-appearance")||"auto",a=window.matchMedia("(prefers-color-scheme: dark)").matches;(!e||e==="auto"?a:e==="dark")&&document.documentElement.classList.add("dark")})();</script>
22
+ <script id="check-mac-os">document.documentElement.classList.toggle("mac",/Mac|iPhone|iPod|iPad/i.test(navigator.platform));</script>
23
+ </head>
24
+ <body>
25
+ <div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Getting Started</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/intro.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Introduction</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Optimization Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/spark-architecture.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Execution Model</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/memory-model.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Management</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/partitioning.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Partitioning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/joins.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Join Optimization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/data-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Data Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/table-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Table Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/caching.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Caching & Persistence</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/pyspark.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>PySpark Specifics</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/aqe.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Adaptive Query Execution</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/cluster-config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cluster Tuning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/anti-patterns.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Anti-Patterns</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Detector Catalog</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-skew.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Skew</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle I/O</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-spill.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory / Disk Spill</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-gc.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>GC Pressure</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-cold-start.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cold Start</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Executor Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-slow-host.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Slow Host</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-failures.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Failures</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-straggler.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Stragglers</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-retry-waste.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Retry Waste</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-tiny-tasks.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Tiny Tasks</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-job-failure-rate.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Job Failure Rate</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-memory-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Duplicate Plan Subtree</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-small-files.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Small Files</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-broadcast-sizing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Broadcast Sizing</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Reference</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/metrics.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Metrics Glossary</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Config Quick-Reference</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_tuning-reference_metrics" data-v-39a288b8><div><h1 id="metrics" tabindex="-1">Metrics Glossary <a class="header-anchor" href="#metrics" aria-label="Permalink to &quot;Metrics Glossary {#metrics}&quot;">​</a></h1><p>Reference for every metric surfaced elsewhere in this guide: what it measures, where Spark records it in the event log, and what a problematic value looks like.</p><h2 id="metric-task-duration" tabindex="-1">Task duration (P50/P95/max) <a class="header-anchor" href="#metric-task-duration" aria-label="Permalink to &quot;Task duration (P50/P95/max) {#metric-task-duration}&quot;">​</a></h2><p>Per-task wall-clock duration, aggregated across a stage&#39;s tasks into percentiles (P50, P95, max) to surface skew and stragglers. It is computed from <code>SparkListenerTaskEnd</code> events rather than read off a single field: each task end carries its own duration, and the percentiles are derived by aggregating those values across the stage.</p><p>A problematic value looks like a small fraction of tasks running far past the rest: a stage is classified as a straggler when more than 5% of tasks run at least 4x the median duration (with at least 10 tasks in the stage), or when speculative tasks were fired.</p><h2 id="metric-shuffle-read-bytes" tabindex="-1">Shuffle read bytes <a class="header-anchor" href="#metric-shuffle-read-bytes" aria-label="Permalink to &quot;Shuffle read bytes {#metric-shuffle-read-bytes}&quot;">​</a></h2><p>The volume of shuffle data a task reads from remote executors, recorded in <code>taskMetrics.shuffleReadMetrics.remoteBytesRead</code>.</p><h2 id="metric-shuffle-write-bytes" tabindex="-1">Shuffle write bytes <a class="header-anchor" href="#metric-shuffle-write-bytes" aria-label="Permalink to &quot;Shuffle write bytes {#metric-shuffle-write-bytes}&quot;">​</a></h2><p>The size of the shuffle output a task writes, recorded in <code>taskMetrics.shuffleWriteMetrics.bytesWritten</code>. Spark&#39;s own metric description calls it simply the &quot;Number of bytes written in shuffle operations,&quot; without stating compression state on its own<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup>.</p><p>The sort-based shuffle writer fills in the mechanics: incoming records are serialized as soon as they reach the shuffle writer and buffered in serialized form while sorting<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup>. When the spill compression codec supports concatenating compressed data, the final merge step concatenates the already-compressed spill partitions directly into the output file, using <code>transferTo</code> rather than decompressing and recompressing<sup class="footnote-ref"><a href="#fn2" id="fnref2:1">[2:1]</a></sup>. So <code>bytesWritten</code> counts compressed, post-serialization bytes, and in this common fast-merge path the final written output is built directly from data that had already been spilled to disk during sorting, rather than written fresh at merge time.</p><h2 id="metric-memory-bytes-spilled" tabindex="-1">Memory bytes spilled <a class="header-anchor" href="#metric-memory-bytes-spilled" aria-label="Permalink to &quot;Memory bytes spilled {#metric-memory-bytes-spilled}&quot;">​</a></h2><p>Bytes a task spilled from in-memory structures, recorded in <code>taskMetrics.memoryBytesSpilled</code>. Any value above zero is a spill warning, with the skew-vs-volume classification (based on what fraction of a stage&#39;s tasks show zero spill) as the actionable signal.</p><h2 id="metric-disk-bytes-spilled" tabindex="-1">Disk bytes spilled <a class="header-anchor" href="#metric-disk-bytes-spilled" aria-label="Permalink to &quot;Disk bytes spilled {#metric-disk-bytes-spilled}&quot;">​</a></h2><p>The on-disk counterpart to memory spill: bytes a task spilled to disk, recorded in <code>taskMetrics.diskBytesSpilled</code>. It feeds the same spill classification as memory bytes spilled.</p><h2 id="metric-jvm-gc-time" tabindex="-1">JVM GC time <a class="header-anchor" href="#metric-jvm-gc-time" aria-label="Permalink to &quot;JVM GC time {#metric-jvm-gc-time}&quot;">​</a></h2><p>Elapsed time the JVM spent in garbage collection while a task executed, recorded in <code>taskMetrics.jvmGCTime</code> and expressed in milliseconds<sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup>. The value is cumulative across the task&#39;s full execution window: the sum of every GC pause that occurred during that task&#39;s run, not just the most recent one<sup class="footnote-ref"><a href="#fn1" id="fnref1:2">[1:2]</a></sup>. This matches how it&#39;s serialized: as a single scalar long, consistent with an accumulator rather than a per-GC-event log entry<sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup>.</p><h2 id="metric-gcpct" tabindex="-1">gcPct <a class="header-anchor" href="#metric-gcpct" aria-label="Permalink to &quot;gcPct {#metric-gcpct}&quot;">​</a></h2><p>A synthetic ratio, <code>jvmGCTime / executorRunTime</code>, not a raw Spark field. It drives GC-bottleneck classification: above 10% is a warning, above 20% is critical.</p><h2 id="metric-executor-run-time" tabindex="-1">Executor run time <a class="header-anchor" href="#metric-executor-run-time" aria-label="Permalink to &quot;Executor run time {#metric-executor-run-time}&quot;">​</a></h2><p>Elapsed time the executor spent running a task, recorded in <code>taskMetrics.executorRunTime</code> and expressed in milliseconds<sup class="footnote-ref"><a href="#fn1" id="fnref1:3">[1:3]</a></sup>. <code>TaskMetrics</code> (and therefore <code>executorRunTime</code>) is serialized as an optional part of the <code>SparkListenerTaskEnd</code> payload, not guaranteed on every task end<sup class="footnote-ref"><a href="#fn3" id="fnref3:1">[3:1]</a></sup>. In practice it is available for failed tasks that got far enough to actually run, such as <code>ExceptionFailure</code> or <code>TaskKilled</code>. The event-log deserialization code even falls back to reading accumulator updates out of the embedded <code>TaskMetrics</code> for old, Spark-1.x-era logs, which only makes sense if the metrics object is normally populated for that failure type<sup class="footnote-ref"><a href="#fn3" id="fnref3:2">[3:2]</a></sup>. It can be absent for reasons like <code>Resubmitted</code>, where the task attempt never completed on that executor<sup class="footnote-ref"><a href="#fn3" id="fnref3:3">[3:3]</a></sup>.</p><h2 id="metric-fetch-wait-time-ratio" tabindex="-1">Fetch wait time ratio <a class="header-anchor" href="#metric-fetch-wait-time-ratio" aria-label="Permalink to &quot;Fetch wait time ratio {#metric-fetch-wait-time-ratio}&quot;">​</a></h2><p>A synthetic ratio, <code>fetchWaitTime / taskDuration</code>, not a raw Spark field: the time a task spent blocked waiting on remote shuffle blocks, relative to its total duration.</p><h2 id="metric-input-bytes" tabindex="-1">Input bytes <a class="header-anchor" href="#metric-input-bytes" aria-label="Permalink to &quot;Input bytes {#metric-input-bytes}&quot;">​</a></h2><p>Bytes a task read as input, recorded in <code>taskMetrics.inputMetrics.bytesRead</code>.</p><h2 id="metric-output-bytes" tabindex="-1">Output bytes <a class="header-anchor" href="#metric-output-bytes" aria-label="Permalink to &quot;Output bytes {#metric-output-bytes}&quot;">​</a></h2><p>Bytes a task wrote as output, recorded in <code>taskMetrics.outputMetrics.bytesWritten</code>.</p><h2 id="metric-io-ratio" tabindex="-1">I/O ratio <a class="header-anchor" href="#metric-io-ratio" aria-label="Permalink to &quot;I/O ratio {#metric-io-ratio}&quot;">​</a></h2><p>A synthetic ratio, <code>outputBytes / inputBytes</code>, not a raw Spark field.</p><h2 id="metric-peak-execution-memory" tabindex="-1">Peak execution memory <a class="header-anchor" href="#metric-peak-execution-memory" aria-label="Permalink to &quot;Peak execution memory {#metric-peak-execution-memory}&quot;">​</a></h2><p>Peak memory recorded in <code>taskMetrics.peakExecutionMemory</code>. This is a task-level accumulator, distinct from the separate executor-level <code>peakMemoryMetrics.OnHeapExecutionMemory</code> and <code>.OffHeapExecutionMemory</code> gauges, which report the on-heap and off-heap execution pools as two separate numbers at the executor level rather than as a single per-task figure<sup class="footnote-ref"><a href="#fn1" id="fnref1:4">[1:4]</a></sup>.</p><h2 id="metric-failed-tasks" tabindex="-1">Failed tasks / failure rate <a class="header-anchor" href="#metric-failed-tasks" aria-label="Permalink to &quot;Failed tasks / failure rate {#metric-failed-tasks}&quot;">​</a></h2><p>Tasks whose <code>SparkListenerTaskEnd</code> reason is not <code>Success</code>. The <code>Reason</code> field holds the formatted class name of whichever <code>TaskEndReason</code> was assigned to that task end<sup class="footnote-ref"><a href="#fn3" id="fnref3:4">[3:4]</a></sup>, and the canonical set of reasons includes <code>FetchFailed</code>, <code>ExceptionFailure</code>, <code>TaskResultLost</code>, <code>TaskKilled</code>, <code>TaskCommitDenied</code>, <code>ExecutorLostFailure</code>, and <code>UnknownReason</code><sup class="footnote-ref"><a href="#fn3" id="fnref3:5">[3:5]</a></sup>. Because <code>TaskMetrics</code> is only an optional part of the task-end payload, it can be absent for some of these reasons: for example <code>Resubmitted</code>, where the task attempt never actually completed on that executor<sup class="footnote-ref"><a href="#fn3" id="fnref3:6">[3:6]</a></sup>.</p><h2 id="metric-speculative-tasks" tabindex="-1">Speculative tasks / straggler count <a class="header-anchor" href="#metric-speculative-tasks" aria-label="Permalink to &quot;Speculative tasks / straggler count {#metric-speculative-tasks}&quot;">​</a></h2><p>Tasks launched as speculative retries of a slow-running task, recorded via <code>SparkListenerTaskStart</code> where <code>speculative = true</code>. Any speculative task firing (or more than 5% of a stage&#39;s tasks running at least 4x the median duration, with at least 10 tasks in the stage) is a straggler signal.</p><h2 id="metric-executor-count" tabindex="-1">Executor count (added/removed/concurrent) <a class="header-anchor" href="#metric-executor-count" aria-label="Permalink to &quot;Executor count (added/removed/concurrent) {#metric-executor-count}&quot;">​</a></h2><p>The number of executors added, removed, or concurrently running, recorded via <code>SparkListenerExecutorAdded</code> / <code>SparkListenerExecutorRemoved</code> events.</p><h2 id="metric-stage-duration" tabindex="-1">Stage wall-clock duration <a class="header-anchor" href="#metric-stage-duration" aria-label="Permalink to &quot;Stage wall-clock duration {#metric-stage-duration}&quot;">​</a></h2><p>A stage&#39;s total elapsed time, recorded from <code>SparkListenerStageCompleted</code>: <code>completionTime − submissionTime</code>.</p><h2 id="metric-first-stage-submitted-at" tabindex="-1">firstStageSubmittedAt <a class="header-anchor" href="#metric-first-stage-submitted-at" aria-label="Permalink to &quot;firstStageSubmittedAt {#metric-first-stage-submitted-at}&quot;">​</a></h2><p>A synthetic field: the timestamp of the first <code>SparkListenerStageSubmitted</code> event in the event log. The gap between this timestamp and the application&#39;s start time drives cold-start classification: more than 30 seconds is a warning.</p><h2 id="sources" tabindex="-1">Sources <a class="header-anchor" href="#sources" aria-label="Permalink to &quot;Sources&quot;">​</a></h2><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/monitoring.html#spark-history-server" target="_blank" rel="noreferrer">Monitoring and Instrumentation</a> <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a> <a href="#fnref1:2" class="footnote-backref">↩︎</a> <a href="#fnref1:3" class="footnote-backref">↩︎</a> <a href="#fnref1:4" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><a href="https://raw.githubusercontent.com/apache/spark/v3.5.0/core/src/main/scala/org/apache/spark/shuffle/sort/SortShuffleManager.scala" target="_blank" rel="noreferrer">SortShuffleManager.scala</a> <a href="#fnref2" class="footnote-backref">↩︎</a> <a href="#fnref2:1" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><a href="https://raw.githubusercontent.com/apache/spark/v3.5.0/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala" target="_blank" rel="noreferrer">JsonProtocol.scala</a> <a href="#fnref3" class="footnote-backref">↩︎</a> <a href="#fnref3:1" class="footnote-backref">↩︎</a> <a href="#fnref3:2" class="footnote-backref">↩︎</a> <a href="#fnref3:3" class="footnote-backref">↩︎</a> <a href="#fnref3:4" class="footnote-backref">↩︎</a> <a href="#fnref3:5" class="footnote-backref">↩︎</a> <a href="#fnref3:6" class="footnote-backref">↩︎</a></p></li></ol></section></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/tuning-reference/bottleneck-broadcast-sizing.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>Broadcast Sizing</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="/docs/tuning-reference/config.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>Spark Config Quick-Reference</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
26
+ <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"B-OsL91z\",\"contributor-guide_architecture_detector-contract.md\":\"BOeH4d1J\",\"contributor-guide_architecture_drill-down.md\":\"BtPdlM7r\",\"contributor-guide_architecture_impact-estimation.md\":\"DYCDPgkh\",\"contributor-guide_architecture_index.md\":\"3TO9ic6w\",\"contributor-guide_architecture_overview.md\":\"CehiRmGn\",\"contributor-guide_architecture_state-and-history.md\":\"m3S3UdMk\",\"contributor-guide_architecture_widget-rendering.md\":\"DbqPf2OT\",\"contributor-guide_architecture_worker-protocol.md\":\"B93qJ_tT\",\"contributor-guide_contributing.md\":\"CvRsdr6J\",\"contributor-guide_development-setup.md\":\"DvAN_9mK\",\"contributor-guide_testing.md\":\"6rIKqSyY\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"Df1YMIHu\",\"tuning-reference_aqe.md\":\"BIsCtLzm\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"CEstB3Ia\",\"tuning-reference_bottleneck-cold-start.md\":\"CEuy-72y\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"CIohQDfn\",\"tuning-reference_bottleneck-failures.md\":\"4z5BXGJ2\",\"tuning-reference_bottleneck-gc.md\":\"DSxzZRK7\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"BaJl__1W\",\"tuning-reference_bottleneck-memory-utilization.md\":\"DbP-SJZc\",\"tuning-reference_bottleneck-retry-waste.md\":\"D5JMjVOt\",\"tuning-reference_bottleneck-shuffle.md\":\"CM-nTmIH\",\"tuning-reference_bottleneck-skew.md\":\"BdUwiDhn\",\"tuning-reference_bottleneck-slow-host.md\":\"BlIo6UDW\",\"tuning-reference_bottleneck-small-files.md\":\"B8kloyx8\",\"tuning-reference_bottleneck-spill.md\":\"PNH7mITt\",\"tuning-reference_bottleneck-straggler.md\":\"DY36fHN5\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"QTV7O8kU\",\"tuning-reference_bottleneck-utilization.md\":\"DTiueZC3\",\"tuning-reference_caching.md\":\"B7aQ8asB\",\"tuning-reference_cluster-config.md\":\"ZVmDGsQ3\",\"tuning-reference_config.md\":\"UvveiWG3\",\"tuning-reference_data-formats.md\":\"bjCAWH3N\",\"tuning-reference_index.md\":\"BQ_NooMV\",\"tuning-reference_intro.md\":\"CobD-lGB\",\"tuning-reference_joins.md\":\"BtKs_CuW\",\"tuning-reference_memory-model.md\":\"DhT-n4y3\",\"tuning-reference_metrics.md\":\"mLOh7Apj\",\"tuning-reference_partitioning.md\":\"q0zKF_8X\",\"tuning-reference_pyspark.md\":\"DDCfvN9t\",\"tuning-reference_shuffle.md\":\"BZZ7R4Ix\",\"tuning-reference_spark-architecture.md\":\"Dwzm5avO\",\"tuning-reference_table-formats.md\":\"D6wj-2dX\",\"user-guide_alternative-log-retrieval.md\":\"sU3KGarf\",\"user-guide_getting-started.md\":\"DtEM37MK\",\"user-guide_mcp-tools.md\":\"C8MiIu7F\",\"user-guide_run-comparison.md\":\"S0TWWmLY\",\"user-guide_understanding-findings.md\":\"D0R_Y-R2\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Alternative log retrieval\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
27
+
28
+ </body>
29
+ </html>
package/public/docs/tuning-reference/partitioning.html ADDED
@@ -0,0 +1,29 @@
1
+ <!DOCTYPE html>
2
+ <html lang="en-US" dir="ltr">
3
+ <head>
4
+ <meta charset="utf-8">
5
+ <meta name="viewport" content="width=device-width,initial-scale=1">
6
+ <title>Partitioning | SparkForensics</title>
7
+ <meta name="description" content="Docs for using and contributing to SparkForensics">
8
+ <meta name="generator" content="VitePress v1.6.4">
9
+ <link rel="preload stylesheet" href="/docs/assets/style.DXOMCXxn.css" as="style">
10
+ <link rel="preload stylesheet" href="/docs/vp-icons.css" as="style">
11
+
12
+ <script type="module" src="/docs/assets/app.BgzsaTS3.js"></script>
13
+ <link rel="preload" href="/docs/assets/inter-roman-latin.Di8DUHzh.woff2" as="font" type="font/woff2" crossorigin="">
14
+ <link rel="modulepreload" href="/docs/assets/chunks/theme.BP14rGBx.js">
15
+ <link rel="modulepreload" href="/docs/assets/chunks/framework.DSg0KOwT.js">
16
+ <link rel="modulepreload" href="/docs/assets/tuning-reference_partitioning.md.q0zKF_8X.lean.js">
17
+ <link rel="icon" type="image/svg+xml" href="/docs/favicon.svg">
18
+ <link rel="preconnect" href="https://fonts.googleapis.com">
19
+ <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin="">
20
+ <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Recursive:wght,CASL@400..700,0..1&amp;family=JetBrains+Mono:wght@400;500;600&amp;display=swap">
21
+ <script id="check-dark-mode">(()=>{const e=localStorage.getItem("vitepress-theme-appearance")||"auto",a=window.matchMedia("(prefers-color-scheme: dark)").matches;(!e||e==="auto"?a:e==="dark")&&document.documentElement.classList.add("dark")})();</script>
22
+ <script id="check-mac-os">document.documentElement.classList.toggle("mac",/Mac|iPhone|iPod|iPad/i.test(navigator.platform));</script>
23
+ </head>
24
+ <body>
25
+ <div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Getting Started</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/intro.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Introduction</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Optimization Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/spark-architecture.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Execution Model</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/memory-model.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Management</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/partitioning.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Partitioning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/joins.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Join Optimization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/data-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Data Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/table-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Table Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/caching.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Caching & Persistence</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/pyspark.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>PySpark Specifics</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/aqe.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Adaptive Query Execution</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/cluster-config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cluster Tuning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/anti-patterns.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Anti-Patterns</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Detector Catalog</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-skew.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Skew</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle I/O</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-spill.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory / Disk Spill</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-gc.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>GC Pressure</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-cold-start.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cold Start</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Executor Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-slow-host.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Slow Host</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-failures.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Failures</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-straggler.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Stragglers</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-retry-waste.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Retry Waste</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-tiny-tasks.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Tiny Tasks</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-job-failure-rate.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Job Failure Rate</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-memory-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Duplicate Plan Subtree</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-small-files.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Small Files</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-broadcast-sizing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Broadcast Sizing</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Reference</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/metrics.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Metrics Glossary</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Config Quick-Reference</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_tuning-reference_partitioning" data-v-39a288b8><div><h1 id="partitioning" tabindex="-1">Partitioning <a class="header-anchor" href="#partitioning" aria-label="Permalink to &quot;Partitioning {#partitioning}&quot;">​</a></h1><h2 id="how-partitions-get-sized-and-shuffled" tabindex="-1">How partitions get sized and shuffled <a class="header-anchor" href="#how-partitions-get-sized-and-shuffled" aria-label="Permalink to &quot;How partitions get sized and shuffled&quot;">​</a></h2><p>Every partition Spark creates maps to exactly one task and one thread, which is why partition count and size drive so much of a job&#39;s performance<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup>. The <a href="./shuffle.html">shuffle</a> side of that is governed by a single default: <code>spark.sql.shuffle.partitions</code>, which defaults to 200 and applies to every <code>join()</code>, <code>groupBy()</code>, and aggregation regardless of how much data is actually moving<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup><sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup>.</p><p>Two operations reshape that layout, and they are not interchangeable. <code>repartition(numPartitions)</code> &quot;return[s] a new RDD that has exactly numPartitions partitions&quot;<sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup>, and it earns that guarantee by always running a full hash shuffle: records are first spread across a temporary key space starting from a randomized position, seeded per-partition via <code>XORShiftRandom</code>, so upstream data ends up distributed evenly rather than clustered by its original layout. That feeds into a <code>ShuffledRDD</code> keyed by a <code>HashPartitioner(numPartitions)</code>, then a <code>CoalescedRDD</code> lands it on exactly the requested count<sup class="footnote-ref"><a href="#fn4" id="fnref4:1">[4:1]</a></sup>. High Performance Spark puts the same mechanics more plainly: &quot;repartition shuffles the RDD with a hash partitioner and the given number of partitions&quot;<sup class="footnote-ref"><a href="#fn5" id="fnref5">[5]</a></sup>. That holds whether the new count is bigger or smaller than the current one. Repartition always performs the full shuffle just described, unlike coalesce<sup class="footnote-ref"><a href="#fn6" id="fnref6">[6]</a></sup>.</p><p><code>coalesce</code>, left at its default, does not shuffle at all: it&#39;s a narrow transformation where each output partition is simply the union of a fixed set of parent partitions decided at plan time, not routed by data values, so the stage&#39;s task count just drops to the coalesced number<sup class="footnote-ref"><a href="#fn5" id="fnref5:1">[5:1]</a></sup>.</p><p>Neither <code>repartition</code> nor <code>coalesce</code> leaves behind a &quot;known partitioner&quot; the way <code>partitionBy</code> does<sup class="footnote-ref"><a href="#fn5" id="fnref5:2">[5:2]</a></sup>. That distinction is what Spark&#39;s join planner checks before it will skip a shuffle: it only does so when both sides already carry partitioner objects it can prove equal: matching partition counts for a <code>HashPartitioner</code>, matching range bounds for a <code>RangePartitioner</code><sup class="footnote-ref"><a href="#fn5" id="fnref5:3">[5:3]</a></sup>. Its default shuffle hash join path makes the same point from the other side: it partitions the second dataset using the same partitioner as the first specifically so matching keys land together, which only lets a later shuffle be skipped when a shared, recognized partitioner is already in place beforehand<sup class="footnote-ref"><a href="#fn5" id="fnref5:4">[5:4]</a></sup>.</p><p>There&#39;s also a hard ceiling on shuffle output, though it isn&#39;t something to design around: Spark&#39;s sort-based shuffle manager caps output partitions in serialized mode at <code>PackedRecordPointer.MAXIMUM_PARTITION_ID + 1</code>, roughly 16.8 million. The source code itself calls this &quot;an extreme defensive programming measure,&quot; since no real shuffle comes remotely close to it<sup class="footnote-ref"><a href="#fn7" id="fnref7">[7]</a></sup>.</p><h2 id="spotting-a-bad-layout" tabindex="-1">Spotting a bad layout <a class="header-anchor" href="#spotting-a-bad-layout" aria-label="Permalink to &quot;Spotting a bad layout&quot;">​</a></h2><p>That layout choice shows up as overhead in both directions once it&#39;s wrong: partitions that are too small flood the cluster with per-task scheduling overhead, while partitions that are too large create memory pressure and straggler tasks<sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup>. Learning Spark 2nd Edition frames the healthy target from the parallelism side rather than an absolute number: at least as many partitions as there are cores across the executors, so no core sits idle; more partitions than cores is fine as long as it doesn&#39;t drift into the small-partition overhead regime above<sup class="footnote-ref"><a href="#fn3" id="fnref3:1">[3:1]</a></sup>.</p><p>A heavy filter is an easy-to-miss cause of imbalance: Spark doesn&#39;t shrink partition count when rows are filtered out, so 2,000 partitions holding 5% of the original rows just become 2,000 mostly-empty partitions<sup class="footnote-ref"><a href="#fn1" id="fnref1:2">[1:2]</a></sup>.</p><p>To size shuffle partitions from a job&#39;s actual behavior rather than a guess, Cloudera&#39;s tuning guide takes a stage that already ran, computes the ratio between its Shuffle Spill (Memory) and Shuffle Spill (Disk) metrics, and multiplies total shuffle write by that ratio to estimate in-memory shuffle size, then rounds the resulting partition count up rather than down<sup class="footnote-ref"><a href="#fn8" id="fnref8">[8]</a></sup>.</p><p><a href="./bottleneck-skew.html">Skew</a> specifically has documented, numeric detection thresholds under <a href="./aqe.html">Adaptive Query Execution</a>: a partition counts as skewed if it&#39;s larger than <code>spark.sql.adaptive.skewJoin.skewedPartitionFactor</code> (default 5.0) times the median partition size, and also larger than <code>spark.sql.adaptive.skewJoin.skewedPartitionThresholdInBytes</code> (default 256 MB)<sup class="footnote-ref"><a href="#fn2" id="fnref2:1">[2:1]</a></sup><sup class="footnote-ref"><a href="#fn9" id="fnref9">[9]</a></sup>.</p><p>On the input side, <code>spark.sql.files.maxPartitionBytes</code> (defaulting to 128 MB) governs the target chunk size when Spark splits splittable file-based sources (CSV, JSON, line-delimited text) into input partitions, based on file size<sup class="footnote-ref"><a href="#fn1" id="fnref1:3">[1:3]</a></sup>.</p><h2 id="where-a-bad-layout-costs-you" tabindex="-1">Where a bad layout costs you <a class="header-anchor" href="#where-a-bad-layout-costs-you" aria-label="Permalink to &quot;Where a bad layout costs you&quot;">​</a></h2><p>Getting the partition count wrong costs more than the immediate slowdown suggests. Cloudera&#39;s tuning guide argues it&#39;s safer to over-provision partitions than to under-provision them, because Spark, unlike MapReduce, has low per-task startup overhead: &quot;when in doubt, it&#39;s almost always better to err on the side of a larger number of tasks&quot;<sup class="footnote-ref"><a href="#fn8" id="fnref8:1">[8:1]</a></sup>.</p><p>Getting <code>coalesce</code> wrong costs more than the coalesce step itself: because it&#39;s narrow, it forces the <em>entire</em> upstream stage to run at the reduced parallelism, not just the final step<sup class="footnote-ref"><a href="#fn5" id="fnref5:5">[5:5]</a></sup>. Pushed too far (<code>coalesce(1)</code>), it kills parallelism outright, because coalesce doesn&#39;t rebalance data, it just stacks existing partitions together, so partitions that were uneven going in are still uneven coming out<sup class="footnote-ref"><a href="#fn1" id="fnref1:4">[1:4]</a></sup>.</p><p>Once a shuffle stage has run, its output file count is locked in: you can&#39;t change it after the fact without inserting a stage barrier, such as writing to temporary storage or calling <code>localCheckpoint()</code>, between the shuffle and the write<sup class="footnote-ref"><a href="#fn10" id="fnref10">[10]</a></sup>.</p><p>Join alignment has its own gotcha: calling <code>repartition(n, col)</code> with the same column and count on two separate DataFrames produces data that&#39;s plausibly laid out the same way, but it doesn&#39;t leave a trackable partitioner object behind. The planner has no recorded partitioner to compare, so it has no basis for treating the two sides as co-partitioned and skipping the shuffle at join time, even though the call sites look identical<sup class="footnote-ref"><a href="#fn5" id="fnref5:6">[5:6]</a></sup>.</p><h2 id="fixing-the-layout" tabindex="-1">Fixing the layout <a class="header-anchor" href="#fixing-the-layout" aria-label="Permalink to &quot;Fixing the layout&quot;">​</a></h2><p>Fixing those costs starts with accepting there&#39;s no formula to look up. There&#39;s no single formula for the right <code>spark.sql.shuffle.partitions</code> value: it depends on data set size, core count, and executor memory, and comes down to trial and error<sup class="footnote-ref"><a href="#fn3" id="fnref3:2">[3:2]</a></sup>. As a starting point, Learning Spark notes the default of 200 is usually too high for small or streaming workloads, where it should be pulled down toward the executor core count<sup class="footnote-ref"><a href="#fn3" id="fnref3:3">[3:3]</a></sup>.</p><p>For the repartition-vs-coalesce decision itself: reach for <code>coalesce</code> first whenever you&#39;re only reducing partition count, since it merges partitions already on the same node without a shuffle<sup class="footnote-ref"><a href="#fn6" id="fnref6:1">[6:1]</a></sup>. Reach for <code>repartition</code> when you actually need the shuffle: increasing partition count, fixing a lopsided distribution, or preparing a DataFrame ahead of a join or a <code>cache()</code> call, where even parallelism is worth more than the shuffle cost<sup class="footnote-ref"><a href="#fn6" id="fnref6:2">[6:2]</a></sup>. High Performance Spark reduces this to a readability rule: use <code>repartition</code> when you want a shuffle, <code>coalesce</code> when you don&#39;t, rather than leaning on coalesce&#39;s shuffle toggle to blur the line<sup class="footnote-ref"><a href="#fn5" id="fnref5:7">[5:7]</a></sup>. There is a middle option, <code>coalesce(n, shuffle=True)</code>, which behaves more like repartition, paying for a shuffle but getting real rebalancing on the way down<sup class="footnote-ref"><a href="#fn1" id="fnref1:5">[1:5]</a></sup>. <code>coalesce()</code> itself belongs at the tail of a pipeline, right before a write, purely to cut down <a href="./bottleneck-small-files.html">output file count</a><sup class="footnote-ref"><a href="#fn1" id="fnref1:6">[1:6]</a></sup>; after a heavy filter has left partitions mostly empty, following up with <code>repartition(100)</code> (or similar) buys back real parallelism<sup class="footnote-ref"><a href="#fn1" id="fnref1:7">[1:7]</a></sup>. At the SQL layer, both directions have dedicated hints for controlling output partitioning directly: <code>/*+ REPARTITION(n) */</code>, <code>/*+ REPARTITION(cols) */</code>, <code>/*+ REPARTITION_BY_RANGE(cols) */</code>, and <code>/*+ COALESCE(n) */</code><sup class="footnote-ref"><a href="#fn9" id="fnref9:1">[9:1]</a></sup>.</p><img class="light-only" src="/docs/assets/repartition-vs-coalesce.BovLRrpj.svg" alt="A decision flowchart for choosing repartition versus coalesce based on whether a shuffle and a rebalance are needed."><img class="dark-only" src="/docs/assets/repartition-vs-coalesce.dark.BhAczKZQ.svg" alt="A decision flowchart for choosing repartition versus coalesce based on whether a shuffle and a rebalance are needed."><p>Separate from the DataFrame <code>.coalesce()</code> call, Adaptive Query Execution has its own runtime coalescing behavior: <code>spark.sql.adaptive.coalescePartitions.enabled</code> (default true) merges small, contiguous post-shuffle partitions toward a target size at runtime, correcting over-partitioning without a manual <code>.coalesce()</code> call<sup class="footnote-ref"><a href="#fn9" id="fnref9:2">[9:2]</a></sup><sup class="footnote-ref"><a href="#fn1" id="fnref1:8">[1:8]</a></sup>.</p><p>For skewed joins specifically, salting (adding a prefix to skewed keys so the same key is treated as several different keys, then adjusting the data distribution accordingly) was one of three manual approaches used before adaptive execution existed, alongside raising <code>spark.sql.shuffle.partitions</code> and raising the broadcast hash join threshold to push a sort-merge join toward a broadcast hash join instead. All three carry &quot;lots of limitations&quot; and require manual processing, which is exactly the gap AQE&#39;s skew-join optimization was built to close<sup class="footnote-ref"><a href="#fn11" id="fnref11">[11]</a></sup>. Where automatic detection isn&#39;t precise enough, Databricks&#39; <code>/*+ SKEW(...) */</code> hint names the skewed relation and column(s), and optionally the specific skewed key values, letting the planner target just those keys directly instead of relying on automatic detection<sup class="footnote-ref"><a href="#fn12" id="fnref12">[12]</a></sup>.</p><p>For the join-alignment gotcha above, the mechanism that does reliably guarantee a shuffle can be skipped is storage-partitioned joins: both tables are physically bucketed identically at the catalog level, for example <a href="./table-formats.html">Iceberg</a> tables created with matching <code>PARTITIONED BY (bucket(...))</code> clauses. When Spark recognizes both sides report the same partitioning through <code>SupportsReportPartitioning</code>, it can drop the Exchange (shuffle) node entirely, or shuffle only one side. A plain DataFrame-level <code>repartition(col)</code> doesn&#39;t offer that guarantee, because it isn&#39;t backed by catalog-level partitioning metadata<sup class="footnote-ref"><a href="#fn9" id="fnref9:3">[9:3]</a></sup><sup class="footnote-ref"><a href="#fn2" id="fnref2:2">[2:2]</a></sup>.</p><h2 id="sources" tabindex="-1">Sources <a class="header-anchor" href="#sources" aria-label="Permalink to &quot;Sources&quot;">​</a></h2><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><a href="https://luminousmen.com/post/spark-partitions" target="_blank" rel="noreferrer">Spark Partitions</a> <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a> <a href="#fnref1:2" class="footnote-backref">↩︎</a> <a href="#fnref1:3" class="footnote-backref">↩︎</a> <a href="#fnref1:4" class="footnote-backref">↩︎</a> <a href="#fnref1:5" class="footnote-backref">↩︎</a> <a href="#fnref1:6" class="footnote-backref">↩︎</a> <a href="#fnref1:7" class="footnote-backref">↩︎</a> <a href="#fnref1:8" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration — Spark</a> <a href="#fnref2" class="footnote-backref">↩︎</a> <a href="#fnref2:1" class="footnote-backref">↩︎</a> <a href="#fnref2:2" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><em>Learning Spark, 2nd Edition</em>, Damji, Wenig, Das &amp; Lee, ch. 7 <a href="#fnref3" class="footnote-backref">↩︎</a> <a href="#fnref3:1" class="footnote-backref">↩︎</a> <a href="#fnref3:2" class="footnote-backref">↩︎</a> <a href="#fnref3:3" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><a href="https://raw.githubusercontent.com/apache/spark/v3.5.0/core/src/main/scala/org/apache/spark/rdd/RDD.scala" target="_blank" rel="noreferrer">RDD.scala</a> <a href="#fnref4" class="footnote-backref">↩︎</a> <a href="#fnref4:1" class="footnote-backref">↩︎</a></p></li><li id="fn5" class="footnote-item"><p><em>High Performance Spark, 2nd Edition</em>, Karau, Polak &amp; Warren, ch. 8 <a href="#fnref5" class="footnote-backref">↩︎</a> <a href="#fnref5:1" class="footnote-backref">↩︎</a> <a href="#fnref5:2" class="footnote-backref">↩︎</a> <a href="#fnref5:3" class="footnote-backref">↩︎</a> <a href="#fnref5:4" class="footnote-backref">↩︎</a> <a href="#fnref5:5" class="footnote-backref">↩︎</a> <a href="#fnref5:6" class="footnote-backref">↩︎</a> <a href="#fnref5:7" class="footnote-backref">↩︎</a></p></li><li id="fn6" class="footnote-item"><p><em>Spark: The Definitive Guide</em>, Chambers &amp; Zaharia, ch. 19 <a href="#fnref6" class="footnote-backref">↩︎</a> <a href="#fnref6:1" class="footnote-backref">↩︎</a> <a href="#fnref6:2" class="footnote-backref">↩︎</a></p></li><li id="fn7" class="footnote-item"><p><a href="https://raw.githubusercontent.com/apache/spark/v3.5.0/core/src/main/scala/org/apache/spark/shuffle/sort/SortShuffleManager.scala" target="_blank" rel="noreferrer">SortShuffleManager.scala</a> <a href="#fnref7" class="footnote-backref">↩︎</a></p></li><li id="fn8" class="footnote-item"><p><a href="https://blog.cloudera.com/how-to-tune-your-apache-spark-jobs-part-2/" target="_blank" rel="noreferrer">How to Tune Your Apache Spark Jobs (Part 2)</a> <a href="#fnref8" class="footnote-backref">↩︎</a> <a href="#fnref8:1" class="footnote-backref">↩︎</a></p></li><li id="fn9" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/sql-performance-tuning.html" target="_blank" rel="noreferrer">Performance Tuning — Spark SQL, DataFrames and Datasets Guide</a> <a href="#fnref9" class="footnote-backref">↩︎</a> <a href="#fnref9:1" class="footnote-backref">↩︎</a> <a href="#fnref9:2" class="footnote-backref">↩︎</a> <a href="#fnref9:3" class="footnote-backref">↩︎</a></p></li><li id="fn10" class="footnote-item"><p><a href="https://luminousmen.com/post/spark-tips-partition-tuning" target="_blank" rel="noreferrer">Spark Tips: Partition Tuning</a> <a href="#fnref10" class="footnote-backref">↩︎</a></p></li><li id="fn11" class="footnote-item"><p><a href="https://issues.apache.org/jira/browse/SPARK-29544" target="_blank" rel="noreferrer">SPARK-29544 — Optimize skewed join at runtime</a> <a href="#fnref11" class="footnote-backref">↩︎</a></p></li><li id="fn12" class="footnote-item"><p><a href="https://docs.databricks.com/aws/en/archive/legacy/skew-join" target="_blank" rel="noreferrer">Skew Join (legacy)</a> <a href="#fnref12" class="footnote-backref">↩︎</a></p></li></ol></section></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/tuning-reference/memory-model.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>Memory Management</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="/docs/tuning-reference/joins.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>Join Optimization</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
26
+ <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"B-OsL91z\",\"contributor-guide_architecture_detector-contract.md\":\"BOeH4d1J\",\"contributor-guide_architecture_drill-down.md\":\"BtPdlM7r\",\"contributor-guide_architecture_impact-estimation.md\":\"DYCDPgkh\",\"contributor-guide_architecture_index.md\":\"3TO9ic6w\",\"contributor-guide_architecture_overview.md\":\"CehiRmGn\",\"contributor-guide_architecture_state-and-history.md\":\"m3S3UdMk\",\"contributor-guide_architecture_widget-rendering.md\":\"DbqPf2OT\",\"contributor-guide_architecture_worker-protocol.md\":\"B93qJ_tT\",\"contributor-guide_contributing.md\":\"CvRsdr6J\",\"contributor-guide_development-setup.md\":\"DvAN_9mK\",\"contributor-guide_testing.md\":\"6rIKqSyY\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"Df1YMIHu\",\"tuning-reference_aqe.md\":\"BIsCtLzm\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"CEstB3Ia\",\"tuning-reference_bottleneck-cold-start.md\":\"CEuy-72y\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"CIohQDfn\",\"tuning-reference_bottleneck-failures.md\":\"4z5BXGJ2\",\"tuning-reference_bottleneck-gc.md\":\"DSxzZRK7\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"BaJl__1W\",\"tuning-reference_bottleneck-memory-utilization.md\":\"DbP-SJZc\",\"tuning-reference_bottleneck-retry-waste.md\":\"D5JMjVOt\",\"tuning-reference_bottleneck-shuffle.md\":\"CM-nTmIH\",\"tuning-reference_bottleneck-skew.md\":\"BdUwiDhn\",\"tuning-reference_bottleneck-slow-host.md\":\"BlIo6UDW\",\"tuning-reference_bottleneck-small-files.md\":\"B8kloyx8\",\"tuning-reference_bottleneck-spill.md\":\"PNH7mITt\",\"tuning-reference_bottleneck-straggler.md\":\"DY36fHN5\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"QTV7O8kU\",\"tuning-reference_bottleneck-utilization.md\":\"DTiueZC3\",\"tuning-reference_caching.md\":\"B7aQ8asB\",\"tuning-reference_cluster-config.md\":\"ZVmDGsQ3\",\"tuning-reference_config.md\":\"UvveiWG3\",\"tuning-reference_data-formats.md\":\"bjCAWH3N\",\"tuning-reference_index.md\":\"BQ_NooMV\",\"tuning-reference_intro.md\":\"CobD-lGB\",\"tuning-reference_joins.md\":\"BtKs_CuW\",\"tuning-reference_memory-model.md\":\"DhT-n4y3\",\"tuning-reference_metrics.md\":\"mLOh7Apj\",\"tuning-reference_partitioning.md\":\"q0zKF_8X\",\"tuning-reference_pyspark.md\":\"DDCfvN9t\",\"tuning-reference_shuffle.md\":\"BZZ7R4Ix\",\"tuning-reference_spark-architecture.md\":\"Dwzm5avO\",\"tuning-reference_table-formats.md\":\"D6wj-2dX\",\"user-guide_alternative-log-retrieval.md\":\"sU3KGarf\",\"user-guide_getting-started.md\":\"DtEM37MK\",\"user-guide_mcp-tools.md\":\"C8MiIu7F\",\"user-guide_run-comparison.md\":\"S0TWWmLY\",\"user-guide_understanding-findings.md\":\"D0R_Y-R2\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Alternative log retrieval\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
27
+
28
+ </body>
29
+ </html>
package/public/docs/tuning-reference/pyspark.html ADDED
@@ -0,0 +1,34 @@
1
+ <!DOCTYPE html>
2
+ <html lang="en-US" dir="ltr">
3
+ <head>
4
+ <meta charset="utf-8">
5
+ <meta name="viewport" content="width=device-width,initial-scale=1">
6
+ <title>PySpark Specifics | SparkForensics</title>
7
+ <meta name="description" content="Docs for using and contributing to SparkForensics">
8
+ <meta name="generator" content="VitePress v1.6.4">
9
+ <link rel="preload stylesheet" href="/docs/assets/style.DXOMCXxn.css" as="style">
10
+ <link rel="preload stylesheet" href="/docs/vp-icons.css" as="style">
11
+
12
+ <script type="module" src="/docs/assets/app.BgzsaTS3.js"></script>
13
+ <link rel="preload" href="/docs/assets/inter-roman-latin.Di8DUHzh.woff2" as="font" type="font/woff2" crossorigin="">
14
+ <link rel="modulepreload" href="/docs/assets/chunks/theme.BP14rGBx.js">
15
+ <link rel="modulepreload" href="/docs/assets/chunks/framework.DSg0KOwT.js">
16
+ <link rel="modulepreload" href="/docs/assets/tuning-reference_pyspark.md.DDCfvN9t.lean.js">
17
+ <link rel="icon" type="image/svg+xml" href="/docs/favicon.svg">
18
+ <link rel="preconnect" href="https://fonts.googleapis.com">
19
+ <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin="">
20
+ <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Recursive:wght,CASL@400..700,0..1&amp;family=JetBrains+Mono:wght@400;500;600&amp;display=swap">
21
+ <script id="check-dark-mode">(()=>{const e=localStorage.getItem("vitepress-theme-appearance")||"auto",a=window.matchMedia("(prefers-color-scheme: dark)").matches;(!e||e==="auto"?a:e==="dark")&&document.documentElement.classList.add("dark")})();</script>
22
+ <script id="check-mac-os">document.documentElement.classList.toggle("mac",/Mac|iPhone|iPod|iPad/i.test(navigator.platform));</script>
23
+ </head>
24
+ <body>
25
+ <div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Getting Started</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/intro.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Introduction</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Optimization Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/spark-architecture.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Execution Model</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/memory-model.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Management</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/partitioning.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Partitioning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/joins.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Join Optimization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/data-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Data Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/table-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Table Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/caching.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Caching & Persistence</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/pyspark.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>PySpark Specifics</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/aqe.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Adaptive Query Execution</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/cluster-config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cluster Tuning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/anti-patterns.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Anti-Patterns</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Detector Catalog</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-skew.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Skew</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle I/O</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-spill.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory / Disk Spill</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-gc.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>GC Pressure</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-cold-start.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cold Start</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Executor Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-slow-host.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Slow Host</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-failures.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Failures</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-straggler.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Stragglers</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-retry-waste.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Retry Waste</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-tiny-tasks.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Tiny Tasks</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-job-failure-rate.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Job Failure Rate</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-memory-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Duplicate Plan Subtree</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-small-files.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Small Files</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-broadcast-sizing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Broadcast Sizing</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Reference</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/metrics.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Metrics Glossary</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Config Quick-Reference</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_tuning-reference_pyspark" data-v-39a288b8><div><h1 id="pyspark" tabindex="-1">PySpark Specifics <a class="header-anchor" href="#pyspark" aria-label="Permalink to &quot;PySpark Specifics {#pyspark}&quot;">​</a></h1><h2 id="crossing-the-jvm-python-boundary" tabindex="-1">Crossing the JVM/Python boundary <a class="header-anchor" href="#crossing-the-jvm-python-boundary" aria-label="Permalink to &quot;Crossing the JVM/Python boundary&quot;">​</a></h2><p>A plain PySpark UDF runs one row at a time in a separate Python process, and getting each row there costs something real: &quot;PySpark UDFs required data movement between the JVM and Python, which was quite expensive,&quot; using pickle to serialize the data across that boundary<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup>. <em>Spark: The Definitive Guide</em> breaks the cost down further. Starting the extra Python process is one expense, but &quot;the real cost is in serializing the data to Python,&quot; and once the data is over there the JVM can no longer manage that worker&#39;s memory, so JVM and Python end up competing for the same machine&#39;s RAM and the worker can fail under pressure<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup>. That&#39;s the underlying reason the same source recommends writing UDFs in Scala or Java and calling them from Python when that&#39;s practical<sup class="footnote-ref"><a href="#fn2" id="fnref2:1">[2:1]</a></sup>.</p><p>Since Spark 3.4 the row-at-a-time path can itself be Arrow-optimized without giving up row semantics: set <code>useArrow=True</code> on <code>udf()</code>, or flip the session-wide <code>spark.sql.execution.pythonUDF.arrow.enabled</code> (default <code>false</code>), and a regular Python UDF becomes an &quot;Arrow Python UDF&quot; that still runs row by row but moves data with Arrow instead of pickle<sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup><sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup>. The session config only takes effect when <code>useArrow</code> is left unset on the UDF itself<sup class="footnote-ref"><a href="#fn3" id="fnref3:1">[3:1]</a></sup>.</p><img class="light-only" src="/docs/assets/udf-execution-models.BUFDICuG.svg" alt="Across the JVM to Python boundary a plain Python UDF pickles row-at-a-time, an Arrow-optimized UDF uses Arrow transfer with row semantics, and a pandas UDF passes whole Arrow batches with no per-row handoff."><img class="dark-only" src="/docs/assets/udf-execution-models.dark.YTNS6GDq.svg" alt="Across the JVM to Python boundary a plain Python UDF pickles row-at-a-time, an Arrow-optimized UDF uses Arrow transfer with row semantics, and a pandas UDF passes whole Arrow batches with no per-row handoff."><p>The pandas UDF (vectorized UDF, introduced in Spark 2.3) goes further and drops the per-row JVM↔Python handoff entirely: it hands the Python worker whole Arrow batches, operated on as pandas Series or DataFrames, so there&#39;s nothing to pickle row by row<sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup>. Three shapes cover most cases. <strong>SCALAR</strong> (<code>pandas.Series, ... -&gt; pandas.Series</code>) is the direct vectorized swap-in for a row-at-a-time scalar UDF (computing <code>v + 1</code>, or <code>cubed(x)</code>); PySpark calls the function once per Arrow batch and concatenates the results back into a column<sup class="footnote-ref"><a href="#fn5" id="fnref5">[5]</a></sup><sup class="footnote-ref"><a href="#fn1" id="fnref1:2">[1:2]</a></sup>. <strong>SCALAR_ITER</strong> (<code>Iterator[Series] -&gt; Iterator[Series]</code>) works the same way internally, but takes and yields an iterator instead of a single Series, which lets a function prefetch across batches<sup class="footnote-ref"><a href="#fn3" id="fnref3:2">[3:2]</a></sup>. <strong>MAP_ITER</strong>, exposed as <code>DataFrame.mapInPandas()</code> rather than as a <code>pandas_udf</code> type, maps an iterator of whole <code>pandas.DataFrame</code> partitions to another iterator of <code>pandas.DataFrame</code>s and, unlike the other two, can change the row count<sup class="footnote-ref"><a href="#fn3" id="fnref3:3">[3:3]</a></sup>. A related grouped-map API, <code>DataFrame.groupBy().applyInPandas()</code>, splits the DataFrame into groups and runs a <code>pandas.DataFrame -&gt; pandas.DataFrame</code> function per group: split, apply, combine<sup class="footnote-ref"><a href="#fn5" id="fnref5:1">[5:1]</a></sup><sup class="footnote-ref"><a href="#fn3" id="fnref3:4">[3:4]</a></sup>.</p><p><code>spark.sql.execution.arrow.pyspark.enabled</code> isn&#39;t limited to pandas UDFs, either: it also governs Arrow-based columnar transfer for <code>DataFrame.toPandas()</code> and for <code>SparkSession.createDataFrame()</code> when given a pandas DataFrame or NumPy ndarray<sup class="footnote-ref"><a href="#fn4" id="fnref4:1">[4:1]</a></sup><sup class="footnote-ref"><a href="#fn3" id="fnref3:5">[3:5]</a></sup>. A companion flag, <code>spark.sql.execution.arrow.pyspark.fallback.enabled</code>, silently drops back to the non-Arrow path if an error occurs before computation starts<sup class="footnote-ref"><a href="#fn3" id="fnref3:6">[3:6]</a></sup><sup class="footnote-ref"><a href="#fn4" id="fnref4:2">[4:2]</a></sup>.</p><p>Two worker-level configs round out the picture. <code>spark.python.worker.reuse</code> (default <code>true</code>) keeps a fixed pool of Python worker processes alive across tasks instead of forking a fresh one each time, which also means a large broadcast variable doesn&#39;t have to cross the JVM↔Python boundary again for every task<sup class="footnote-ref"><a href="#fn4" id="fnref4:3">[4:3]</a></sup>. <code>spark.python.worker.memory</code> (default <code>512m</code>) is a per-worker, Spark-managed accounting threshold for in-worker aggregation buffering (not an OS-enforced cap), and Spark <a href="./bottleneck-spill.html">spills to disk</a> once it&#39;s exceeded<sup class="footnote-ref"><a href="#fn4" id="fnref4:4">[4:4]</a></sup>.</p><h2 id="measuring-the-gap" tabindex="-1">Measuring the gap <a class="header-anchor" href="#measuring-the-gap" aria-label="Permalink to &quot;Measuring the gap&quot;">​</a></h2><p>The clearest measurement here is workload-level. Databricks&#39; introductory pandas-UDF post ran three operations (Plus One, Cumulative Probability, Subtract Mean) over a 10M-row, two-column DataFrame on a single-node Databricks Community Edition cluster, and found pandas UDFs &quot;perform much better than row-at-a-time UDFs across the board, ranging from 3x to over 100x&quot;<sup class="footnote-ref"><a href="#fn5" id="fnref5:2">[5:2]</a></sup>. If a job&#39;s Python UDFs are a suspected bottleneck, expect an order-of-magnitude gap between a row-at-a-time UDF and its pandas-UDF equivalent, not a marginal one.</p><p>The comparisons in the corpus consistently favor <code>pyspark.sql.functions</code> and vectorized code over row-at-a-time UDFs. For the &quot;plus one&quot; example, &quot;built-in column operators can perform much faster in this scenario,&quot; and the pandas UDF equivalent is &quot;much faster than the row-at-a-time version&quot; because it&#39;s vectorized over the Series<sup class="footnote-ref"><a href="#fn5" id="fnref5:3">[5:3]</a></sup>. A Python UDF that could instead be expressed with built-in column functions is worth flagging on its own.</p><p>On the memory side, the two failure modes look different and are worth telling apart. A worker that spills is showing up in Spark&#39;s own accounting: it exceeded <code>spark.python.worker.memory</code> during aggregation and wrote to disk, which is expected behavior, not a crash<sup class="footnote-ref"><a href="#fn4" id="fnref4:5">[4:5]</a></sup>. A worker that&#39;s actually killed is a container-level event: if total container memory (JVM heap plus off-heap plus the Python process) exceeds what YARN or Kubernetes allocated, &quot;Kubernetes won&#39;t hesitate&quot; and the process is &quot;OOMKilled&quot;<sup class="footnote-ref"><a href="#fn6" id="fnref6">[6]</a></sup>. That boundary is governed by <a href="./memory-model.html">executor overhead</a> and <code>spark.executor.pyspark.memory</code> settings, not by <code>worker.memory</code>.</p><h2 id="what-the-gap-costs" tabindex="-1">What the gap costs <a class="header-anchor" href="#what-the-gap-costs" aria-label="Permalink to &quot;What the gap costs&quot;">​</a></h2><p>Those numbers point to a structural cost, not a tuning quirk. The JVM↔Python boundary is where a plain Python UDF pays twice: once to start the separate process, and again (the larger cost) to serialize every row across it<sup class="footnote-ref"><a href="#fn2" id="fnref2:2">[2:2]</a></sup>. Because the JVM can&#39;t manage memory inside the Python process once data has crossed over, the two runtimes end up competing for the same machine&#39;s memory, and the Python worker can fail under that pressure<sup class="footnote-ref"><a href="#fn2" id="fnref2:3">[2:3]</a></sup>. That&#39;s a correctness risk as well as a performance one, and it&#39;s why <em>Spark: The Definitive Guide</em> recommends Scala/Java UDFs called from Python over native Python UDFs wherever that&#39;s practical<sup class="footnote-ref"><a href="#fn2" id="fnref2:4">[2:4]</a></sup>.</p><p>The magnitude backs this up: Databricks measured pandas UDFs beating row-at-a-time UDFs by 3x to over 100x depending on the operation<sup class="footnote-ref"><a href="#fn5" id="fnref5:4">[5:4]</a></sup>. Skipping per-row pickling (by moving to Arrow-based row UDFs or, further, to pandas UDFs operating on whole batches) isn&#39;t a marginal tuning knob here; it changes which order of magnitude a job runs at.</p><p>The two memory configs matter for different reasons, too. <code>spark.python.worker.memory</code> only controls when Spark chooses to spill aggregation state to disk; tuning it trades disk I/O for headroom, it doesn&#39;t prevent a crash<sup class="footnote-ref"><a href="#fn4" id="fnref4:6">[4:6]</a></sup>. Actual OOM kills happen at the container boundary, and <code>spark.executor.pyspark.memory</code> (which leans on Python&#39;s <code>resource</code> module and so doesn&#39;t cap memory on macOS and doesn&#39;t exist at all on Windows) is the config that&#39;s actually in that path<sup class="footnote-ref"><a href="#fn4" id="fnref4:7">[4:7]</a></sup>. Conflating the two means tuning the wrong knob when a Python worker gets OOMKilled.</p><h2 id="closing-the-gap" tabindex="-1">Closing the gap <a class="header-anchor" href="#closing-the-gap" aria-label="Permalink to &quot;Closing the gap&quot;">​</a></h2><p>Closing that gap means avoiding the boundary crossing altogether, or crossing it as cheaply as possible. Where the logic allows it, prefer a built-in <code>pyspark.sql.functions</code> expression or a Scala/Java UDF called from Python over a plain row-at-a-time Python UDF<sup class="footnote-ref"><a href="#fn2" id="fnref2:5">[2:5]</a></sup><sup class="footnote-ref"><a href="#fn5" id="fnref5:5">[5:5]</a></sup>; this is the change with the largest documented payoff, 3x to over 100x<sup class="footnote-ref"><a href="#fn5" id="fnref5:6">[5:6]</a></sup>.</p><p>Where a Python UDF is unavoidable, cut the row-at-a-time cost first by turning on Arrow for it:</p><div class="language-python vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">python</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">@udf</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70;">returnType</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">=</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&#39;int&#39;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70;">useArrow</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">=</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">True</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">) </span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># An Arrow Python UDF</span></span>
26
+ <span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">def</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;"> arrow_slen</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(s):</span></span>
27
+ <span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> return</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> len</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(s)</span></span></code></pre></div><p>or set the session-wide equivalent so existing UDFs pick it up without code changes.</p><blockquote><p><strong>PySpark:</strong> <code>spark.conf.set(&quot;spark.sql.execution.pythonUDF.arrow.enabled&quot;, &quot;true&quot;)</code> turns plain UDFs into Arrow-backed ones, as long as <code>useArrow</code> isn&#39;t explicitly set on the UDF itself<sup class="footnote-ref"><a href="#fn3" id="fnref3:7">[3:7]</a></sup><sup class="footnote-ref"><a href="#fn4" id="fnref4:8">[4:8]</a></sup>.</p></blockquote><p>Beyond that, reach for the vectorized APIs instead of a scalar UDF:</p><ul><li>Use a <strong>SCALAR</strong> pandas UDF (<code>pandas.Series, ... -&gt; pandas.Series</code>) as the default vectorized replacement for a row-at-a-time scalar UDF<sup class="footnote-ref"><a href="#fn5" id="fnref5:7">[5:7]</a></sup><sup class="footnote-ref"><a href="#fn1" id="fnref1:3">[1:3]</a></sup>.</li><li>Use <strong>SCALAR_ITER</strong> (<code>Iterator[Series] -&gt; Iterator[Series]</code>) when the function needs expensive one-time setup. The documented pattern initializes state once, then loops over the batch iterator reusing it, instead of re-initializing per batch<sup class="footnote-ref"><a href="#fn3" id="fnref3:8">[3:8]</a></sup>:</li></ul><div class="language-python vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">python</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">def</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;"> apply_with_state</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(iterator):</span></span>
28
+ <span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> state </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">=</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> very_expensive_initialization()</span></span>
29
+ <span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> for</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> batch </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">in</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> iterator:</span></span>
30
+ <span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> yield</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> calculate_with_state(batch, state)</span></span></code></pre></div><ul><li>Use <code>DataFrame.mapInPandas()</code> when the transform needs to change row count (filtering, expansion, deduplication) rather than map one-to-one<sup class="footnote-ref"><a href="#fn3" id="fnref3:9">[3:9]</a></sup>.</li><li>Use <code>DataFrame.groupBy().applyInPandas()</code> for per-group logic that needs the whole group as state, such as subtracting a group mean or fitting a per-group regression<sup class="footnote-ref"><a href="#fn5" id="fnref5:8">[5:8]</a></sup><sup class="footnote-ref"><a href="#fn3" id="fnref3:10">[3:10]</a></sup>. Size groups with care: a full group loads into memory before the function runs, and <code>maxRecordsPerBatch</code> doesn&#39;t apply to groups, so a skewed group risks OOM<sup class="footnote-ref"><a href="#fn3" id="fnref3:11">[3:11]</a></sup>.</li></ul><p>For pandas/NumPy conversion at the driver, turn on Arrow explicitly rather than relying on defaults.</p><blockquote><p><strong>PySpark:</strong> with <code>spark.sql.execution.arrow.pyspark.enabled</code> set to <code>&quot;true&quot;</code>, <code>spark.createDataFrame(pdf)</code> and <code>df.select(&quot;*&quot;).toPandas()</code> convert through Arrow and return the same result as the non-Arrow path<sup class="footnote-ref"><a href="#fn4" id="fnref4:9">[4:9]</a></sup><sup class="footnote-ref"><a href="#fn3" id="fnref3:12">[3:12]</a></sup>. <code>spark.sql.execution.arrow.pyspark.fallback.enabled</code> keeps a safety net by falling back silently on pre-computation errors<sup class="footnote-ref"><a href="#fn3" id="fnref3:13">[3:13]</a></sup><sup class="footnote-ref"><a href="#fn4" id="fnref4:10">[4:10]</a></sup>.</p></blockquote><p>Leave <code>spark.python.worker.reuse</code> at its default (<code>true</code>) unless there&#39;s a specific reason not to; it keeps a fixed pool of Python workers alive so a large broadcast variable isn&#39;t re-shipped to Python for every task<sup class="footnote-ref"><a href="#fn4" id="fnref4:11">[4:11]</a></sup>. If a job spills at the Python-worker level, <code>spark.python.worker.memory</code> is the knob for that; if it&#39;s getting OOMKilled at the container level, look at executor overhead and <code>spark.executor.pyspark.memory</code> instead, keeping in mind the latter&#39;s <code>resource</code>-module limitations on macOS and its absence on Windows<sup class="footnote-ref"><a href="#fn4" id="fnref4:12">[4:12]</a></sup><sup class="footnote-ref"><a href="#fn6" id="fnref6:1">[6:1]</a></sup>.</p><h2 id="sources" tabindex="-1">Sources <a class="header-anchor" href="#sources" aria-label="Permalink to &quot;Sources&quot;">​</a></h2><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><em>Learning Spark, 2nd Edition</em>, Damji, Wenig, Das &amp; Lee, ch. 5 <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a> <a href="#fnref1:2" class="footnote-backref">↩︎</a> <a href="#fnref1:3" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><em>Spark: The Definitive Guide</em>, Chambers &amp; Zaharia, ch. 6 <a href="#fnref2" class="footnote-backref">↩︎</a> <a href="#fnref2:1" class="footnote-backref">↩︎</a> <a href="#fnref2:2" class="footnote-backref">↩︎</a> <a href="#fnref2:3" class="footnote-backref">↩︎</a> <a href="#fnref2:4" class="footnote-backref">↩︎</a> <a href="#fnref2:5" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><a href="https://spark.apache.org/docs/3.5.8/api/python/user_guide/sql/arrow_pandas.html" target="_blank" rel="noreferrer">Apache Arrow in PySpark</a> <a href="#fnref3" class="footnote-backref">↩︎</a> <a href="#fnref3:1" class="footnote-backref">↩︎</a> <a href="#fnref3:2" class="footnote-backref">↩︎</a> <a href="#fnref3:3" class="footnote-backref">↩︎</a> <a href="#fnref3:4" class="footnote-backref">↩︎</a> <a href="#fnref3:5" class="footnote-backref">↩︎</a> <a href="#fnref3:6" class="footnote-backref">↩︎</a> <a href="#fnref3:7" class="footnote-backref">↩︎</a> <a href="#fnref3:8" class="footnote-backref">↩︎</a> <a href="#fnref3:9" class="footnote-backref">↩︎</a> <a href="#fnref3:10" class="footnote-backref">↩︎</a> <a href="#fnref3:11" class="footnote-backref">↩︎</a> <a href="#fnref3:12" class="footnote-backref">↩︎</a> <a href="#fnref3:13" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration — Spark</a> <a href="#fnref4" class="footnote-backref">↩︎</a> <a href="#fnref4:1" class="footnote-backref">↩︎</a> <a href="#fnref4:2" class="footnote-backref">↩︎</a> <a href="#fnref4:3" class="footnote-backref">↩︎</a> <a href="#fnref4:4" class="footnote-backref">↩︎</a> <a href="#fnref4:5" class="footnote-backref">↩︎</a> <a href="#fnref4:6" class="footnote-backref">↩︎</a> <a href="#fnref4:7" class="footnote-backref">↩︎</a> <a href="#fnref4:8" class="footnote-backref">↩︎</a> <a href="#fnref4:9" class="footnote-backref">↩︎</a> <a href="#fnref4:10" class="footnote-backref">↩︎</a> <a href="#fnref4:11" class="footnote-backref">↩︎</a> <a href="#fnref4:12" class="footnote-backref">↩︎</a></p></li><li id="fn5" class="footnote-item"><p><a href="https://www.databricks.com/blog/2017/10/30/introducing-vectorized-udfs-for-pyspark.html" target="_blank" rel="noreferrer">Introducing Pandas UDF for PySpark</a> <a href="#fnref5" class="footnote-backref">↩︎</a> <a href="#fnref5:1" class="footnote-backref">↩︎</a> <a href="#fnref5:2" class="footnote-backref">↩︎</a> <a href="#fnref5:3" class="footnote-backref">↩︎</a> <a href="#fnref5:4" class="footnote-backref">↩︎</a> <a href="#fnref5:5" class="footnote-backref">↩︎</a> <a href="#fnref5:6" class="footnote-backref">↩︎</a> <a href="#fnref5:7" class="footnote-backref">↩︎</a> <a href="#fnref5:8" class="footnote-backref">↩︎</a></p></li><li id="fn6" class="footnote-item"><p><a href="https://luminousmen.com/post/dive-into-spark-memory" target="_blank" rel="noreferrer">Dive into Spark memory management</a> <a href="#fnref6" class="footnote-backref">↩︎</a> <a href="#fnref6:1" class="footnote-backref">↩︎</a></p></li></ol></section></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/tuning-reference/caching.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>Caching & Persistence</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="/docs/tuning-reference/aqe.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>Adaptive Query Execution</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
31
+ <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"B-OsL91z\",\"contributor-guide_architecture_detector-contract.md\":\"BOeH4d1J\",\"contributor-guide_architecture_drill-down.md\":\"BtPdlM7r\",\"contributor-guide_architecture_impact-estimation.md\":\"DYCDPgkh\",\"contributor-guide_architecture_index.md\":\"3TO9ic6w\",\"contributor-guide_architecture_overview.md\":\"CehiRmGn\",\"contributor-guide_architecture_state-and-history.md\":\"m3S3UdMk\",\"contributor-guide_architecture_widget-rendering.md\":\"DbqPf2OT\",\"contributor-guide_architecture_worker-protocol.md\":\"B93qJ_tT\",\"contributor-guide_contributing.md\":\"CvRsdr6J\",\"contributor-guide_development-setup.md\":\"DvAN_9mK\",\"contributor-guide_testing.md\":\"6rIKqSyY\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"Df1YMIHu\",\"tuning-reference_aqe.md\":\"BIsCtLzm\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"CEstB3Ia\",\"tuning-reference_bottleneck-cold-start.md\":\"CEuy-72y\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"CIohQDfn\",\"tuning-reference_bottleneck-failures.md\":\"4z5BXGJ2\",\"tuning-reference_bottleneck-gc.md\":\"DSxzZRK7\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"BaJl__1W\",\"tuning-reference_bottleneck-memory-utilization.md\":\"DbP-SJZc\",\"tuning-reference_bottleneck-retry-waste.md\":\"D5JMjVOt\",\"tuning-reference_bottleneck-shuffle.md\":\"CM-nTmIH\",\"tuning-reference_bottleneck-skew.md\":\"BdUwiDhn\",\"tuning-reference_bottleneck-slow-host.md\":\"BlIo6UDW\",\"tuning-reference_bottleneck-small-files.md\":\"B8kloyx8\",\"tuning-reference_bottleneck-spill.md\":\"PNH7mITt\",\"tuning-reference_bottleneck-straggler.md\":\"DY36fHN5\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"QTV7O8kU\",\"tuning-reference_bottleneck-utilization.md\":\"DTiueZC3\",\"tuning-reference_caching.md\":\"B7aQ8asB\",\"tuning-reference_cluster-config.md\":\"ZVmDGsQ3\",\"tuning-reference_config.md\":\"UvveiWG3\",\"tuning-reference_data-formats.md\":\"bjCAWH3N\",\"tuning-reference_index.md\":\"BQ_NooMV\",\"tuning-reference_intro.md\":\"CobD-lGB\",\"tuning-reference_joins.md\":\"BtKs_CuW\",\"tuning-reference_memory-model.md\":\"DhT-n4y3\",\"tuning-reference_metrics.md\":\"mLOh7Apj\",\"tuning-reference_partitioning.md\":\"q0zKF_8X\",\"tuning-reference_pyspark.md\":\"DDCfvN9t\",\"tuning-reference_shuffle.md\":\"BZZ7R4Ix\",\"tuning-reference_spark-architecture.md\":\"Dwzm5avO\",\"tuning-reference_table-formats.md\":\"D6wj-2dX\",\"user-guide_alternative-log-retrieval.md\":\"sU3KGarf\",\"user-guide_getting-started.md\":\"DtEM37MK\",\"user-guide_mcp-tools.md\":\"C8MiIu7F\",\"user-guide_run-comparison.md\":\"S0TWWmLY\",\"user-guide_understanding-findings.md\":\"D0R_Y-R2\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Alternative log retrieval\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
32
+
33
+ </body>
34
+ </html>