sparkforensics-server 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (440) hide show
  1. package/README.md +6 -0
  2. package/index.js +40 -49
  3. package/lib/static-files.js +10 -6
  4. package/package.json +20 -1
  5. package/public/assets/AutoscalingChurn-CFJkiqxW.js +1 -0
  6. package/public/assets/CacheUtilization-kTKZBzDA.js +1 -0
  7. package/public/assets/CachingOpportunity-CU70EWcF.js +1 -0
  8. package/public/assets/ColdStart-hOI-WOvm.js +1 -0
  9. package/public/assets/ConfigAudit-B4oAksZY.js +1 -0
  10. package/public/assets/CoreUsageArea-D-dR3SQk.js +1 -0
  11. package/public/assets/Dashboard-MDhohsq4.js +4 -0
  12. package/public/assets/DuplicatePlanSubtree-54RKsfI7.js +1 -0
  13. package/public/assets/DurationHistogram-CviQyPNM.js +1 -0
  14. package/public/assets/EmptyState-v2ehFncc.js +1 -0
  15. package/public/assets/ExecutorUtilization-vHyFHgL6.js +1 -0
  16. package/public/assets/GcPressure-DINuhcno.js +1 -0
  17. package/public/assets/ImpactEstimate-C6GzFOY9.js +1 -0
  18. package/public/assets/IncompleteRun-a1kuCnU6.js +1 -0
  19. package/public/assets/JobFailures-CYwPfUhR.js +1 -0
  20. package/public/assets/MemoryUtilization-Dlfj0e_f.js +1 -0
  21. package/public/assets/OverBroadcast-BMu7Y8l9.js +1 -0
  22. package/public/assets/PartitionSizing-CvMGfSVB.js +1 -0
  23. package/public/assets/PlanExplorer-xYFme_Rx.js +1 -0
  24. package/public/assets/PlanGraphRoute-a3OGg6ET.js +1 -0
  25. package/public/assets/RetryWaste-JKRjOZ-J.js +1 -0
  26. package/public/assets/RowPagination-DypWbnic.js +1 -0
  27. package/public/assets/ShuffleIO-Mi8HiVu-.js +1 -0
  28. package/public/assets/Skew-C3iOScTC.js +1 -0
  29. package/public/assets/SlowHost-DKBg18v-.js +1 -0
  30. package/public/assets/SmallFiles-CCCB6AIu.js +1 -0
  31. package/public/assets/SpeculationWaste-DUNupw0_.js +1 -0
  32. package/public/assets/Spill-BlLdatik.js +1 -0
  33. package/public/assets/StageFailed-qtY7C7Yo.js +1 -0
  34. package/public/assets/StageFindingGroup-CoVdEZYs.js +1 -0
  35. package/public/assets/StageHeader-uDBLHgdg.js +1 -0
  36. package/public/assets/StagePill-CYCMdlIx.js +1 -0
  37. package/public/assets/StageShape-zs0mAjQJ.js +1 -0
  38. package/public/assets/StageSlowness-C7vg0u5Q.js +1 -0
  39. package/public/assets/Straggler-4BRGAQP4.js +1 -0
  40. package/public/assets/TaskFailures-CSjO23Kg.js +1 -0
  41. package/public/assets/TinyTask-6Zuovcn5.js +1 -0
  42. package/public/assets/UnderBroadcast-CePjo0ZV.js +1 -0
  43. package/public/assets/WidgetLeadSummary-DRNRVvn7.js +1 -0
  44. package/public/assets/{charts-vendor-NMqKV0H4.js → charts-vendor-Cf1-wzeH.js} +2 -2
  45. package/public/assets/downsample-D8XmJk6i.js +1 -0
  46. package/public/assets/finding-anchor-4TmpXd7j.js +1 -0
  47. package/public/assets/impact-sort-Cq6KOLqh.js +1 -0
  48. package/public/assets/inbox-DBFovbUX.js +1 -0
  49. package/public/assets/index-CHjV2-jB.js +9 -0
  50. package/public/assets/index-DMEtaP4u.css +1 -0
  51. package/public/assets/parser-worker-BMPPE37w.js +64 -0
  52. package/public/assets/plan-graph-vendor-B96a-e4N.js +7 -0
  53. package/public/assets/react-vendor-DA6opS-q.js +9 -0
  54. package/public/assets/select-D5fK-za0.js +1 -0
  55. package/public/assets/toggle-group-lbIXRG0W.js +1 -0
  56. package/public/assets/usePagedRows-CFoZrPr4.js +1 -0
  57. package/public/assets/useRecentFiles-BPGQ6XII.js +13 -0
  58. package/public/assets/useSortMode-BOPpfLhJ.js +1 -0
  59. package/public/docs/404.html +3 -3
  60. package/public/docs/assets/{app.Bxy4_HVA.js → app.BgzsaTS3.js} +1 -1
  61. package/public/docs/assets/aqe-loop.IwQSATHw.svg +1 -0
  62. package/public/docs/assets/aqe-loop.dark.DGbaxqJE.svg +1 -0
  63. package/public/docs/assets/broadcast-vs-shuffle.Db4WY1XK.svg +1 -0
  64. package/public/docs/assets/broadcast-vs-shuffle.dark.C7Bxs0mG.svg +1 -0
  65. package/public/docs/assets/cache-lifecycle.dark.B-hS7AgU.svg +1 -0
  66. package/public/docs/assets/cache-lifecycle.rEOVYQNU.svg +1 -0
  67. package/public/docs/assets/chunks/@localSearchIndexroot.PQ5JepCp.js +1 -0
  68. package/public/docs/assets/chunks/{VPLocalSearchBox.L5jNOIR3.js → VPLocalSearchBox.DOq3t-On.js} +1 -1
  69. package/public/docs/assets/chunks/duplicate-plan-subtree.dark.Cdp70QhV.js +1 -0
  70. package/public/docs/assets/chunks/{framework.CNDaxUod.js → framework.DSg0KOwT.js} +1 -1
  71. package/public/docs/assets/chunks/retry-escalation-ladder.dark.DHipdJgZ.js +1 -0
  72. package/public/docs/assets/chunks/theme.BP14rGBx.js +2 -0
  73. package/public/docs/assets/cold-start-timeline.DxC_Sc7w.svg +1 -0
  74. package/public/docs/assets/cold-start-timeline.dark.CZ17YcAG.svg +1 -0
  75. package/public/docs/assets/columnar-layout.PghGeOEA.svg +1 -0
  76. package/public/docs/assets/columnar-layout.dark.BVNlz0ff.svg +1 -0
  77. package/public/docs/assets/container-memory.DIO0AnIm.svg +1 -0
  78. package/public/docs/assets/container-memory.dark.CP-5zuCl.svg +1 -0
  79. package/public/docs/assets/contributor-guide_architecture_board-widgets.md.B-OsL91z.js +1 -0
  80. package/public/docs/assets/contributor-guide_architecture_board-widgets.md.B-OsL91z.lean.js +1 -0
  81. package/public/docs/assets/contributor-guide_architecture_detector-contract.md.BOeH4d1J.js +1 -0
  82. package/public/docs/assets/contributor-guide_architecture_detector-contract.md.BOeH4d1J.lean.js +1 -0
  83. package/public/docs/assets/contributor-guide_architecture_drill-down.md.BtPdlM7r.js +1 -0
  84. package/public/docs/assets/contributor-guide_architecture_drill-down.md.BtPdlM7r.lean.js +1 -0
  85. package/public/docs/assets/{contributor-guide_architecture_impact-estimation.md.CzlXOnCq.js → contributor-guide_architecture_impact-estimation.md.DYCDPgkh.js} +1 -1
  86. package/public/docs/assets/{contributor-guide_architecture_impact-estimation.md.CzlXOnCq.lean.js → contributor-guide_architecture_impact-estimation.md.DYCDPgkh.lean.js} +1 -1
  87. package/public/docs/assets/{contributor-guide_architecture_index.md.94lNbF-4.js → contributor-guide_architecture_index.md.3TO9ic6w.js} +1 -1
  88. package/public/docs/assets/{contributor-guide_architecture_index.md.94lNbF-4.lean.js → contributor-guide_architecture_index.md.3TO9ic6w.lean.js} +1 -1
  89. package/public/docs/assets/{contributor-guide_architecture_overview.md.mQFlN66o.js → contributor-guide_architecture_overview.md.CehiRmGn.js} +1 -1
  90. package/public/docs/assets/{contributor-guide_architecture_overview.md.mQFlN66o.lean.js → contributor-guide_architecture_overview.md.CehiRmGn.lean.js} +1 -1
  91. package/public/docs/assets/contributor-guide_architecture_state-and-history.md.m3S3UdMk.js +1 -0
  92. package/public/docs/assets/contributor-guide_architecture_state-and-history.md.m3S3UdMk.lean.js +1 -0
  93. package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.DbqPf2OT.js +1 -0
  94. package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.DbqPf2OT.lean.js +1 -0
  95. package/public/docs/assets/{contributor-guide_architecture_worker-protocol.md.BaS_L10g.js → contributor-guide_architecture_worker-protocol.md.B93qJ_tT.js} +2 -2
  96. package/public/docs/assets/{contributor-guide_architecture_worker-protocol.md.BaS_L10g.lean.js → contributor-guide_architecture_worker-protocol.md.B93qJ_tT.lean.js} +1 -1
  97. package/public/docs/assets/contributor-guide_contributing.md.CvRsdr6J.js +1 -0
  98. package/public/docs/assets/contributor-guide_contributing.md.CvRsdr6J.lean.js +1 -0
  99. package/public/docs/assets/{contributor-guide_development-setup.md.CpUMVtIK.js → contributor-guide_development-setup.md.DvAN_9mK.js} +1 -1
  100. package/public/docs/assets/{contributor-guide_development-setup.md.CpUMVtIK.lean.js → contributor-guide_development-setup.md.DvAN_9mK.lean.js} +1 -1
  101. package/public/docs/assets/{contributor-guide_testing.md.C0q-BJfl.js → contributor-guide_testing.md.6rIKqSyY.js} +1 -1
  102. package/public/docs/assets/{contributor-guide_testing.md.C0q-BJfl.lean.js → contributor-guide_testing.md.6rIKqSyY.lean.js} +1 -1
  103. package/public/docs/assets/dag-stages.DSz_S937.svg +1 -0
  104. package/public/docs/assets/dag-stages.dark.F72UzxH4.svg +1 -0
  105. package/public/docs/assets/driver-executor.D5pQ7YN1.svg +1 -0
  106. package/public/docs/assets/driver-executor.dark.BmX9cPvh.svg +1 -0
  107. package/public/docs/assets/duplicate-plan-subtree.B4cvN6fj.svg +1 -0
  108. package/public/docs/assets/duplicate-plan-subtree.dark.Dw8wS0Ag.svg +1 -0
  109. package/public/docs/assets/{index.md.CETnDZzR.js → index.md.CHJVslga.js} +1 -1
  110. package/public/docs/assets/{index.md.CETnDZzR.lean.js → index.md.CHJVslga.lean.js} +1 -1
  111. package/public/docs/assets/join-strategy.C_FvrCEo.svg +1 -0
  112. package/public/docs/assets/join-strategy.dark.ChMLnNII.svg +1 -0
  113. package/public/docs/assets/memory-borrowing.BqQRJg0u.svg +1 -0
  114. package/public/docs/assets/memory-borrowing.dark.Yhh20O9C.svg +1 -0
  115. package/public/docs/assets/memory-regions.XHvO7jHG.svg +1 -0
  116. package/public/docs/assets/memory-regions.dark.D4TP9_08.svg +1 -0
  117. package/public/docs/assets/repartition-vs-coalesce.BovLRrpj.svg +1 -0
  118. package/public/docs/assets/repartition-vs-coalesce.dark.BhAczKZQ.svg +1 -0
  119. package/public/docs/assets/retry-escalation-ladder.DyTKJJmZ.svg +1 -0
  120. package/public/docs/assets/retry-escalation-ladder.dark.BdsabtU3.svg +1 -0
  121. package/public/docs/assets/shuffle-map-reduce.KuOEZVmg.svg +1 -0
  122. package/public/docs/assets/shuffle-map-reduce.dark.BgQZnFSb.svg +1 -0
  123. package/public/docs/assets/spill-classification.BU2euYDO.svg +1 -0
  124. package/public/docs/assets/spill-classification.dark.D7i1M40d.svg +1 -0
  125. package/public/docs/assets/{style.D2mY0HUs.css → style.DXOMCXxn.css} +1 -1
  126. package/public/docs/assets/tuning-reference_anti-patterns.md.Df1YMIHu.js +1 -0
  127. package/public/docs/assets/tuning-reference_anti-patterns.md.Df1YMIHu.lean.js +1 -0
  128. package/public/docs/assets/tuning-reference_aqe.md.BIsCtLzm.js +1 -0
  129. package/public/docs/assets/tuning-reference_aqe.md.BIsCtLzm.lean.js +1 -0
  130. package/public/docs/assets/tuning-reference_bottleneck-broadcast-sizing.md.CEstB3Ia.js +1 -0
  131. package/public/docs/assets/tuning-reference_bottleneck-broadcast-sizing.md.CEstB3Ia.lean.js +1 -0
  132. package/public/docs/assets/tuning-reference_bottleneck-cold-start.md.CEuy-72y.js +7 -0
  133. package/public/docs/assets/tuning-reference_bottleneck-cold-start.md.CEuy-72y.lean.js +1 -0
  134. package/public/docs/assets/tuning-reference_bottleneck-duplicate-plan-subtree.md.CIohQDfn.js +1 -0
  135. package/public/docs/assets/tuning-reference_bottleneck-duplicate-plan-subtree.md.CIohQDfn.lean.js +1 -0
  136. package/public/docs/assets/tuning-reference_bottleneck-failures.md.4z5BXGJ2.js +6 -0
  137. package/public/docs/assets/tuning-reference_bottleneck-failures.md.4z5BXGJ2.lean.js +1 -0
  138. package/public/docs/assets/tuning-reference_bottleneck-gc.md.DSxzZRK7.js +6 -0
  139. package/public/docs/assets/tuning-reference_bottleneck-gc.md.DSxzZRK7.lean.js +1 -0
  140. package/public/docs/assets/tuning-reference_bottleneck-job-failure-rate.md.BaJl__1W.js +8 -0
  141. package/public/docs/assets/tuning-reference_bottleneck-job-failure-rate.md.BaJl__1W.lean.js +1 -0
  142. package/public/docs/assets/tuning-reference_bottleneck-memory-utilization.md.DbP-SJZc.js +7 -0
  143. package/public/docs/assets/tuning-reference_bottleneck-memory-utilization.md.DbP-SJZc.lean.js +1 -0
  144. package/public/docs/assets/tuning-reference_bottleneck-retry-waste.md.D5JMjVOt.js +1 -0
  145. package/public/docs/assets/tuning-reference_bottleneck-retry-waste.md.D5JMjVOt.lean.js +1 -0
  146. package/public/docs/assets/tuning-reference_bottleneck-shuffle.md.CM-nTmIH.js +12 -0
  147. package/public/docs/assets/tuning-reference_bottleneck-shuffle.md.CM-nTmIH.lean.js +1 -0
  148. package/public/docs/assets/tuning-reference_bottleneck-skew.md.BdUwiDhn.js +14 -0
  149. package/public/docs/assets/tuning-reference_bottleneck-skew.md.BdUwiDhn.lean.js +1 -0
  150. package/public/docs/assets/tuning-reference_bottleneck-slow-host.md.BlIo6UDW.js +7 -0
  151. package/public/docs/assets/tuning-reference_bottleneck-slow-host.md.BlIo6UDW.lean.js +1 -0
  152. package/public/docs/assets/tuning-reference_bottleneck-small-files.md.B8kloyx8.js +5 -0
  153. package/public/docs/assets/tuning-reference_bottleneck-small-files.md.B8kloyx8.lean.js +1 -0
  154. package/public/docs/assets/tuning-reference_bottleneck-spill.md.PNH7mITt.js +6 -0
  155. package/public/docs/assets/tuning-reference_bottleneck-spill.md.PNH7mITt.lean.js +1 -0
  156. package/public/docs/assets/tuning-reference_bottleneck-straggler.md.DY36fHN5.js +7 -0
  157. package/public/docs/assets/tuning-reference_bottleneck-straggler.md.DY36fHN5.lean.js +1 -0
  158. package/public/docs/assets/tuning-reference_bottleneck-tiny-tasks.md.QTV7O8kU.js +8 -0
  159. package/public/docs/assets/tuning-reference_bottleneck-tiny-tasks.md.QTV7O8kU.lean.js +1 -0
  160. package/public/docs/assets/tuning-reference_bottleneck-utilization.md.DTiueZC3.js +5 -0
  161. package/public/docs/assets/tuning-reference_bottleneck-utilization.md.DTiueZC3.lean.js +1 -0
  162. package/public/docs/assets/tuning-reference_caching.md.B7aQ8asB.js +1 -0
  163. package/public/docs/assets/tuning-reference_caching.md.B7aQ8asB.lean.js +1 -0
  164. package/public/docs/assets/tuning-reference_cluster-config.md.ZVmDGsQ3.js +1 -0
  165. package/public/docs/assets/tuning-reference_cluster-config.md.ZVmDGsQ3.lean.js +1 -0
  166. package/public/docs/assets/tuning-reference_config.md.UvveiWG3.js +1 -0
  167. package/public/docs/assets/tuning-reference_config.md.UvveiWG3.lean.js +1 -0
  168. package/public/docs/assets/tuning-reference_data-formats.md.bjCAWH3N.js +1 -0
  169. package/public/docs/assets/tuning-reference_data-formats.md.bjCAWH3N.lean.js +1 -0
  170. package/public/docs/assets/tuning-reference_index.md.BQ_NooMV.js +1 -0
  171. package/public/docs/assets/tuning-reference_index.md.BQ_NooMV.lean.js +1 -0
  172. package/public/docs/assets/tuning-reference_intro.md.CobD-lGB.js +1 -0
  173. package/public/docs/assets/tuning-reference_intro.md.CobD-lGB.lean.js +1 -0
  174. package/public/docs/assets/tuning-reference_joins.md.BtKs_CuW.js +1 -0
  175. package/public/docs/assets/tuning-reference_joins.md.BtKs_CuW.lean.js +1 -0
  176. package/public/docs/assets/tuning-reference_memory-model.md.DhT-n4y3.js +1 -0
  177. package/public/docs/assets/tuning-reference_memory-model.md.DhT-n4y3.lean.js +1 -0
  178. package/public/docs/assets/tuning-reference_metrics.md.mLOh7Apj.js +1 -0
  179. package/public/docs/assets/tuning-reference_metrics.md.mLOh7Apj.lean.js +1 -0
  180. package/public/docs/assets/tuning-reference_partitioning.md.q0zKF_8X.js +1 -0
  181. package/public/docs/assets/tuning-reference_partitioning.md.q0zKF_8X.lean.js +1 -0
  182. package/public/docs/assets/tuning-reference_pyspark.md.DDCfvN9t.js +6 -0
  183. package/public/docs/assets/tuning-reference_pyspark.md.DDCfvN9t.lean.js +1 -0
  184. package/public/docs/assets/tuning-reference_shuffle.md.BZZ7R4Ix.js +1 -0
  185. package/public/docs/assets/tuning-reference_shuffle.md.BZZ7R4Ix.lean.js +1 -0
  186. package/public/docs/assets/tuning-reference_spark-architecture.md.Dwzm5avO.js +1 -0
  187. package/public/docs/assets/tuning-reference_spark-architecture.md.Dwzm5avO.lean.js +1 -0
  188. package/public/docs/assets/tuning-reference_table-formats.md.D6wj-2dX.js +1 -0
  189. package/public/docs/assets/tuning-reference_table-formats.md.D6wj-2dX.lean.js +1 -0
  190. package/public/docs/assets/udf-execution-models.BUFDICuG.svg +1 -0
  191. package/public/docs/assets/udf-execution-models.dark.YTNS6GDq.svg +1 -0
  192. package/public/docs/assets/user-guide_alternative-log-retrieval.md.sU3KGarf.js +1 -0
  193. package/public/docs/assets/user-guide_alternative-log-retrieval.md.sU3KGarf.lean.js +1 -0
  194. package/public/docs/assets/user-guide_getting-started.md.DtEM37MK.js +3 -0
  195. package/public/docs/assets/user-guide_getting-started.md.DtEM37MK.lean.js +1 -0
  196. package/public/docs/assets/{user-guide_mcp-tools.md.DwYI1d6C.js → user-guide_mcp-tools.md.C8MiIu7F.js} +35 -5
  197. package/public/docs/assets/{user-guide_mcp-tools.md.DwYI1d6C.lean.js → user-guide_mcp-tools.md.C8MiIu7F.lean.js} +1 -1
  198. package/public/docs/assets/{user-guide_run-comparison.md.5-oaOfTu.js → user-guide_run-comparison.md.S0TWWmLY.js} +1 -1
  199. package/public/docs/assets/{user-guide_run-comparison.md.5-oaOfTu.lean.js → user-guide_run-comparison.md.S0TWWmLY.lean.js} +1 -1
  200. package/public/docs/assets/user-guide_understanding-findings.md.D0R_Y-R2.js +1 -0
  201. package/public/docs/assets/{user-guide_understanding-findings.md.CK2WZ0Tf.lean.js → user-guide_understanding-findings.md.D0R_Y-R2.lean.js} +1 -1
  202. package/public/docs/contributor-guide/architecture/board-widgets.html +7 -7
  203. package/public/docs/contributor-guide/architecture/detector-contract.html +7 -7
  204. package/public/docs/contributor-guide/architecture/drill-down.html +7 -7
  205. package/public/docs/contributor-guide/architecture/impact-estimation.html +7 -7
  206. package/public/docs/contributor-guide/architecture/index.html +7 -7
  207. package/public/docs/contributor-guide/architecture/overview.html +7 -7
  208. package/public/docs/contributor-guide/architecture/state-and-history.html +7 -7
  209. package/public/docs/contributor-guide/architecture/widget-rendering.html +7 -7
  210. package/public/docs/contributor-guide/architecture/worker-protocol.html +8 -8
  211. package/public/docs/contributor-guide/contributing.html +7 -7
  212. package/public/docs/contributor-guide/development-setup.html +7 -7
  213. package/public/docs/contributor-guide/testing.html +7 -7
  214. package/public/docs/hashmap.json +1 -1
  215. package/public/docs/index.html +7 -7
  216. package/public/docs/tuning-reference/anti-patterns.html +29 -0
  217. package/public/docs/tuning-reference/aqe.html +29 -0
  218. package/public/docs/tuning-reference/bottleneck-broadcast-sizing.html +29 -0
  219. package/public/docs/tuning-reference/bottleneck-cold-start.html +35 -0
  220. package/public/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html +30 -0
  221. package/public/docs/tuning-reference/bottleneck-failures.html +35 -0
  222. package/public/docs/tuning-reference/bottleneck-gc.html +34 -0
  223. package/public/docs/tuning-reference/bottleneck-job-failure-rate.html +37 -0
  224. package/public/docs/tuning-reference/bottleneck-memory-utilization.html +35 -0
  225. package/public/docs/tuning-reference/bottleneck-retry-waste.html +30 -0
  226. package/public/docs/tuning-reference/bottleneck-shuffle.html +40 -0
  227. package/public/docs/tuning-reference/bottleneck-skew.html +42 -0
  228. package/public/docs/tuning-reference/bottleneck-slow-host.html +35 -0
  229. package/public/docs/tuning-reference/bottleneck-small-files.html +33 -0
  230. package/public/docs/tuning-reference/bottleneck-spill.html +34 -0
  231. package/public/docs/tuning-reference/bottleneck-straggler.html +35 -0
  232. package/public/docs/tuning-reference/bottleneck-tiny-tasks.html +36 -0
  233. package/public/docs/tuning-reference/bottleneck-utilization.html +34 -0
  234. package/public/docs/tuning-reference/caching.html +29 -0
  235. package/public/docs/tuning-reference/cluster-config.html +29 -0
  236. package/public/docs/tuning-reference/config.html +29 -0
  237. package/public/docs/tuning-reference/data-formats.html +29 -0
  238. package/public/docs/tuning-reference/index.html +29 -0
  239. package/public/docs/tuning-reference/intro.html +29 -0
  240. package/public/docs/tuning-reference/joins.html +29 -0
  241. package/public/docs/tuning-reference/memory-model.html +29 -0
  242. package/public/docs/tuning-reference/metrics.html +29 -0
  243. package/public/docs/tuning-reference/partitioning.html +29 -0
  244. package/public/docs/tuning-reference/pyspark.html +34 -0
  245. package/public/docs/tuning-reference/shuffle.html +29 -0
  246. package/public/docs/tuning-reference/spark-architecture.html +29 -0
  247. package/public/docs/tuning-reference/table-formats.html +29 -0
  248. package/public/docs/user-guide/alternative-log-retrieval.html +29 -0
  249. package/public/docs/user-guide/getting-started.html +8 -8
  250. package/public/docs/user-guide/mcp-tools.html +41 -11
  251. package/public/docs/user-guide/run-comparison.html +7 -7
  252. package/public/docs/user-guide/understanding-findings.html +7 -7
  253. package/public/index.html +6 -6
  254. package/vendor-core/analyzer.js +74 -74
  255. package/vendor-core/cli/budgets.js +13 -27
  256. package/vendor-core/cli/collect-run.js +43 -19
  257. package/vendor-core/core-count.js +25 -27
  258. package/vendor-core/core-locality-ratio.js +4 -11
  259. package/vendor-core/core-time-series.js +6 -12
  260. package/vendor-core/core-usage-locality.js +3 -4
  261. package/vendor-core/detectors.js +256 -375
  262. package/vendor-core/docs-config.js +69 -21
  263. package/vendor-core/docs-content/chapters/01-intro.md +32 -0
  264. package/vendor-core/docs-content/chapters/02-spark-architecture.md +76 -0
  265. package/vendor-core/docs-content/chapters/03-memory-model.md +73 -0
  266. package/vendor-core/docs-content/chapters/04-partitioning.md +65 -0
  267. package/vendor-core/docs-content/chapters/05-joins.md +62 -0
  268. package/vendor-core/docs-content/chapters/06-shuffle.md +59 -0
  269. package/vendor-core/docs-content/chapters/07-data-formats.md +81 -0
  270. package/vendor-core/docs-content/chapters/07b-table-formats.md +56 -0
  271. package/vendor-core/docs-content/chapters/08-caching.md +58 -0
  272. package/vendor-core/docs-content/chapters/09-pyspark.md +78 -0
  273. package/vendor-core/docs-content/chapters/10-aqe.md +167 -0
  274. package/vendor-core/docs-content/chapters/11-cluster-config.md +170 -0
  275. package/vendor-core/docs-content/chapters/12-anti-patterns.md +171 -0
  276. package/vendor-core/docs-content/chapters/14-metrics.md +87 -0
  277. package/vendor-core/docs-content/chapters/15-config.md +93 -0
  278. package/vendor-core/docs-content/chapters/nav-index.json +370 -0
  279. package/vendor-core/docs-content/detection/cache.md +6 -0
  280. package/vendor-core/docs-content/detection/cfg.md +15 -0
  281. package/vendor-core/docs-content/detection/chrn.md +7 -0
  282. package/vendor-core/docs-content/detection/cold.md +4 -0
  283. package/vendor-core/docs-content/detection/cstor.md +4 -0
  284. package/vendor-core/docs-content/detection/fail.md +5 -0
  285. package/vendor-core/docs-content/detection/gc.md +4 -0
  286. package/vendor-core/docs-content/detection/host.md +5 -0
  287. package/vendor-core/docs-content/detection/incmp.md +6 -0
  288. package/vendor-core/docs-content/detection/jobs.md +4 -0
  289. package/vendor-core/docs-content/detection/local.md +7 -0
  290. package/vendor-core/docs-content/detection/mem.md +10 -0
  291. package/vendor-core/docs-content/detection/part.md +5 -0
  292. package/vendor-core/docs-content/detection/plan.md +14 -0
  293. package/vendor-core/docs-content/detection/retry.md +4 -0
  294. package/vendor-core/docs-content/detection/sfail.md +5 -0
  295. package/vendor-core/docs-content/detection/shape.md +5 -0
  296. package/vendor-core/docs-content/detection/shfl.md +4 -0
  297. package/vendor-core/docs-content/detection/skew.md +6 -0
  298. package/vendor-core/docs-content/detection/slow.md +6 -0
  299. package/vendor-core/docs-content/detection/spec.md +7 -0
  300. package/vendor-core/docs-content/detection/spill.md +7 -0
  301. package/vendor-core/docs-content/detection/strag.md +5 -0
  302. package/vendor-core/docs-content/detection/tiny.md +4 -0
  303. package/vendor-core/docs-content/detection/util.md +4 -0
  304. package/vendor-core/docs-content/diagrams/aqe-loop.dark.svg +1 -0
  305. package/vendor-core/docs-content/diagrams/aqe-loop.svg +1 -0
  306. package/vendor-core/docs-content/diagrams/broadcast-vs-shuffle.dark.svg +1 -0
  307. package/vendor-core/docs-content/diagrams/broadcast-vs-shuffle.svg +1 -0
  308. package/vendor-core/docs-content/diagrams/cache-lifecycle.dark.svg +1 -0
  309. package/vendor-core/docs-content/diagrams/cache-lifecycle.svg +1 -0
  310. package/vendor-core/docs-content/diagrams/cold-start-timeline.dark.svg +1 -0
  311. package/vendor-core/docs-content/diagrams/cold-start-timeline.svg +1 -0
  312. package/vendor-core/docs-content/diagrams/columnar-layout.dark.svg +1 -0
  313. package/vendor-core/docs-content/diagrams/columnar-layout.svg +1 -0
  314. package/vendor-core/docs-content/diagrams/container-memory.dark.svg +1 -0
  315. package/vendor-core/docs-content/diagrams/container-memory.svg +1 -0
  316. package/vendor-core/docs-content/diagrams/dag-stages.dark.svg +1 -0
  317. package/vendor-core/docs-content/diagrams/dag-stages.svg +1 -0
  318. package/vendor-core/docs-content/diagrams/driver-executor.dark.svg +1 -0
  319. package/vendor-core/docs-content/diagrams/driver-executor.svg +1 -0
  320. package/vendor-core/docs-content/diagrams/duplicate-plan-subtree.dark.svg +1 -0
  321. package/vendor-core/docs-content/diagrams/duplicate-plan-subtree.svg +1 -0
  322. package/vendor-core/docs-content/diagrams/join-strategy.dark.svg +1 -0
  323. package/vendor-core/docs-content/diagrams/join-strategy.svg +1 -0
  324. package/vendor-core/docs-content/diagrams/memory-borrowing.dark.svg +1 -0
  325. package/vendor-core/docs-content/diagrams/memory-borrowing.svg +1 -0
  326. package/vendor-core/docs-content/diagrams/memory-regions.dark.svg +1 -0
  327. package/vendor-core/docs-content/diagrams/memory-regions.svg +1 -0
  328. package/vendor-core/docs-content/diagrams/repartition-vs-coalesce.dark.svg +1 -0
  329. package/vendor-core/docs-content/diagrams/repartition-vs-coalesce.svg +1 -0
  330. package/vendor-core/docs-content/diagrams/retry-escalation-ladder.dark.svg +1 -0
  331. package/vendor-core/docs-content/diagrams/retry-escalation-ladder.svg +1 -0
  332. package/vendor-core/docs-content/diagrams/shuffle-map-reduce.dark.svg +1 -0
  333. package/vendor-core/docs-content/diagrams/shuffle-map-reduce.svg +1 -0
  334. package/vendor-core/docs-content/diagrams/spill-classification.dark.svg +1 -0
  335. package/vendor-core/docs-content/diagrams/spill-classification.svg +1 -0
  336. package/vendor-core/docs-content/diagrams/udf-execution-models.dark.svg +1 -0
  337. package/vendor-core/docs-content/diagrams/udf-execution-models.svg +1 -0
  338. package/vendor-core/docs-content/tuning/broadcast-sizing.md +78 -0
  339. package/vendor-core/docs-content/tuning/cold-start.md +81 -0
  340. package/vendor-core/docs-content/tuning/duplicate-plan-subtree.md +45 -0
  341. package/vendor-core/docs-content/tuning/failures.md +124 -0
  342. package/vendor-core/docs-content/tuning/gc.md +110 -0
  343. package/vendor-core/docs-content/tuning/job-failure-rate.md +101 -0
  344. package/vendor-core/docs-content/tuning/memory-utilization.md +58 -0
  345. package/vendor-core/docs-content/tuning/retry-waste.md +90 -0
  346. package/vendor-core/docs-content/tuning/shuffle.md +154 -0
  347. package/vendor-core/docs-content/tuning/skew.md +123 -0
  348. package/vendor-core/docs-content/tuning/slow-host.md +117 -0
  349. package/vendor-core/docs-content/tuning/small-files.md +99 -0
  350. package/vendor-core/docs-content/tuning/spill.md +114 -0
  351. package/vendor-core/docs-content/tuning/straggler.md +103 -0
  352. package/vendor-core/docs-content/tuning/tiny-tasks.md +94 -0
  353. package/vendor-core/docs-content/tuning/utilization.md +90 -0
  354. package/vendor-core/docs-site-config.js +10 -17
  355. package/vendor-core/efficiency-model.js +7 -13
  356. package/vendor-core/etl-phases.js +3 -5
  357. package/vendor-core/event-handlers.js +232 -134
  358. package/vendor-core/event-schemas.js +48 -114
  359. package/vendor-core/evidence-availability.js +5 -10
  360. package/vendor-core/evidence-report.js +72 -122
  361. package/vendor-core/export-data.js +48 -0
  362. package/vendor-core/finding-action-label.js +4 -10
  363. package/vendor-core/finding-filter-predicate.js +3 -7
  364. package/vendor-core/finding-generic-recommendation.js +112 -0
  365. package/vendor-core/finding-names.js +51 -0
  366. package/vendor-core/format-utils.js +112 -38
  367. package/vendor-core/impact-band.js +18 -24
  368. package/vendor-core/impact-estimator.js +38 -74
  369. package/vendor-core/ingest.js +7 -13
  370. package/vendor-core/job-groups.js +3 -6
  371. package/vendor-core/list-runs.js +278 -0
  372. package/vendor-core/load-vendored.js +6 -12
  373. package/vendor-core/log-header-peek.js +81 -0
  374. package/vendor-core/lz4-block.js +4 -6
  375. package/vendor-core/mcp-server-factory.js +38 -8
  376. package/vendor-core/mcp-tools.js +105 -76
  377. package/vendor-core/model-assembler.js +8 -16
  378. package/vendor-core/occupancy.js +5 -9
  379. package/vendor-core/parser-worker.js +18 -27
  380. package/vendor-core/plan-dot.js +2 -5
  381. package/vendor-core/plan-duration-attribution.js +78 -29
  382. package/vendor-core/plan-graph-model.js +126 -69
  383. package/vendor-core/plan-node-detail.js +31 -17
  384. package/vendor-core/plan-summary.js +19 -8
  385. package/vendor-core/recommendation-rollup.js +35 -39
  386. package/vendor-core/redact.js +72 -16
  387. package/vendor-core/rolling-log-reassembly.js +4 -6
  388. package/vendor-core/run-comparison.js +65 -70
  389. package/vendor-core/scaling-sim.js +5 -7
  390. package/vendor-core/session-snapshot.js +1 -1
  391. package/vendor-core/shs-fetch.js +4 -6
  392. package/vendor-core/shs-load.js +9 -13
  393. package/vendor-core/shs-request.js +1 -1
  394. package/vendor-core/stage-quantiles.js +14 -0
  395. package/vendor-core/types.js +78 -18
  396. package/vendor-core/wasted-core-hours.js +7 -12
  397. package/public/assets/AutoscalingChurn-CP5OjN6v.js +0 -1
  398. package/public/assets/CacheUtilization-BjyawXL6.js +0 -1
  399. package/public/assets/CachingOpportunity-DV2ABG2G.js +0 -1
  400. package/public/assets/ConfigAudit-DMiBhAoG.js +0 -1
  401. package/public/assets/CoreUsageArea-CGYIgPnt.js +0 -1
  402. package/public/assets/Dashboard-D1_IrWRs.js +0 -12
  403. package/public/assets/ExecutorTimeline-DkaAGqQT.js +0 -1
  404. package/public/assets/Failures-Crngoicu.js +0 -1
  405. package/public/assets/GcPressure-CFXm4BnA.js +0 -1
  406. package/public/assets/IncompleteRun-BWEJTb13.js +0 -1
  407. package/public/assets/JobFailures-BmjxCA-h.js +0 -1
  408. package/public/assets/MemoryUtilization-Bu2uQJbd.js +0 -1
  409. package/public/assets/PlanExplorer-DKUwF0uz.js +0 -1
  410. package/public/assets/PlanGraphRoute-BjPSugIG.js +0 -1
  411. package/public/assets/ShuffleIO--Rur3neu.js +0 -1
  412. package/public/assets/Spill-DIMpt8c2.js +0 -1
  413. package/public/assets/TaskSkew-BDr1DbeI.js +0 -1
  414. package/public/assets/index-4id52G-A.js +0 -9
  415. package/public/assets/index-D9kwVYIO.css +0 -1
  416. package/public/assets/parser-worker-00O9WEQC.js +0 -65
  417. package/public/assets/plan-graph-vendor-vZQHG2EJ.js +0 -7
  418. package/public/assets/react-vendor-u4pyvC5Y.js +0 -9
  419. package/public/assets/useRecentFiles-46iBU9zJ.js +0 -5
  420. package/public/docs/assets/chunks/@localSearchIndexroot.DclgKtlT.js +0 -1
  421. package/public/docs/assets/chunks/theme.CoT6y5bf.js +0 -2
  422. package/public/docs/assets/contributor-guide_architecture_board-widgets.md.BtjoE_1k.js +0 -1
  423. package/public/docs/assets/contributor-guide_architecture_board-widgets.md.BtjoE_1k.lean.js +0 -1
  424. package/public/docs/assets/contributor-guide_architecture_detector-contract.md.DQyCHQNM.js +0 -1
  425. package/public/docs/assets/contributor-guide_architecture_detector-contract.md.DQyCHQNM.lean.js +0 -1
  426. package/public/docs/assets/contributor-guide_architecture_drill-down.md.BKfyDQck.js +0 -1
  427. package/public/docs/assets/contributor-guide_architecture_drill-down.md.BKfyDQck.lean.js +0 -1
  428. package/public/docs/assets/contributor-guide_architecture_state-and-history.md.DaqttKgQ.js +0 -1
  429. package/public/docs/assets/contributor-guide_architecture_state-and-history.md.DaqttKgQ.lean.js +0 -1
  430. package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.Bk6Iyj9S.js +0 -1
  431. package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.Bk6Iyj9S.lean.js +0 -1
  432. package/public/docs/assets/contributor-guide_contributing.md.C-lKs70I.js +0 -1
  433. package/public/docs/assets/contributor-guide_contributing.md.C-lKs70I.lean.js +0 -1
  434. package/public/docs/assets/user-guide_getting-started.md.BJdD7a7_.js +0 -3
  435. package/public/docs/assets/user-guide_getting-started.md.BJdD7a7_.lean.js +0 -1
  436. package/public/docs/assets/user-guide_understanding-findings.md.CK2WZ0Tf.js +0 -1
  437. package/public/vendor/spark-doc/anchors.json +0 -205
  438. package/public/vendor/spark-doc/index.html +0 -4202
  439. package/public/vendor/spark-doc/landing.html +0 -418
  440. package/public/vendor/spark-doc/meta.html +0 -970
package/public/docs/tuning-reference/bottleneck-straggler.html ADDED
@@ -0,0 +1,35 @@
1
+ <!DOCTYPE html>
2
+ <html lang="en-US" dir="ltr">
3
+ <head>
4
+ <meta charset="utf-8">
5
+ <meta name="viewport" content="width=device-width,initial-scale=1">
6
+ <title>Stragglers | SparkForensics</title>
7
+ <meta name="description" content="Docs for using and contributing to SparkForensics">
8
+ <meta name="generator" content="VitePress v1.6.4">
9
+ <link rel="preload stylesheet" href="/docs/assets/style.DXOMCXxn.css" as="style">
10
+ <link rel="preload stylesheet" href="/docs/vp-icons.css" as="style">
11
+
12
+ <script type="module" src="/docs/assets/app.BgzsaTS3.js"></script>
13
+ <link rel="preload" href="/docs/assets/inter-roman-latin.Di8DUHzh.woff2" as="font" type="font/woff2" crossorigin="">
14
+ <link rel="modulepreload" href="/docs/assets/chunks/theme.BP14rGBx.js">
15
+ <link rel="modulepreload" href="/docs/assets/chunks/framework.DSg0KOwT.js">
16
+ <link rel="modulepreload" href="/docs/assets/tuning-reference_bottleneck-straggler.md.DY36fHN5.lean.js">
17
+ <link rel="icon" type="image/svg+xml" href="/docs/favicon.svg">
18
+ <link rel="preconnect" href="https://fonts.googleapis.com">
19
+ <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin="">
20
+ <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Recursive:wght,CASL@400..700,0..1&amp;family=JetBrains+Mono:wght@400;500;600&amp;display=swap">
21
+ <script id="check-dark-mode">(()=>{const e=localStorage.getItem("vitepress-theme-appearance")||"auto",a=window.matchMedia("(prefers-color-scheme: dark)").matches;(!e||e==="auto"?a:e==="dark")&&document.documentElement.classList.add("dark")})();</script>
22
+ <script id="check-mac-os">document.documentElement.classList.toggle("mac",/Mac|iPhone|iPod|iPad/i.test(navigator.platform));</script>
23
+ </head>
24
+ <body>
25
+ <div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Getting Started</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/intro.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Introduction</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Optimization Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/spark-architecture.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Execution Model</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/memory-model.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Management</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/partitioning.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Partitioning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/joins.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Join Optimization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/data-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Data Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/table-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Table Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/caching.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Caching & Persistence</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/pyspark.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>PySpark Specifics</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/aqe.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Adaptive Query Execution</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/cluster-config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cluster Tuning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/anti-patterns.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Anti-Patterns</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Detector Catalog</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-skew.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Skew</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle I/O</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-spill.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory / Disk Spill</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-gc.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>GC Pressure</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-cold-start.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cold Start</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Executor Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-slow-host.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Slow Host</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-failures.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Failures</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-straggler.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Stragglers</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-retry-waste.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Retry Waste</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-tiny-tasks.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Tiny Tasks</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-job-failure-rate.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Job Failure Rate</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-memory-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Duplicate Plan Subtree</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-small-files.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Small Files</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-broadcast-sizing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Broadcast Sizing</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Reference</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/metrics.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Metrics Glossary</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Config Quick-Reference</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_tuning-reference_bottleneck-straggler" data-v-39a288b8><div><h1 id="bottleneck-straggler" tabindex="-1">Stragglers <a class="header-anchor" href="#bottleneck-straggler" aria-label="Permalink to &quot;Stragglers {#bottleneck-straggler}&quot;">​</a></h1><p><span class="tag">STRAG</span></p><h2 id="what-it-is" tabindex="-1">What it is <a class="header-anchor" href="#what-it-is" aria-label="Permalink to &quot;What it is&quot;">​</a></h2><p>A straggler is one task (or a small handful of tasks) that runs far longer than the rest of the tasks in its stage, even when the stage is otherwise healthy. Because a stage only completes once its last task finishes, a single straggler holds up the whole stage, and every other executor sits idle waiting for it to catch up.</p><h2 id="how-it-s-detected" tabindex="-1">How it&#39;s detected <a class="header-anchor" href="#how-it-s-detected" aria-label="Permalink to &quot;How it&#39;s detected&quot;">​</a></h2><p>A task counts as a straggler when speculative tasks fired for it, or when more than 5% of the tasks in a stage run at least 4× the stage&#39;s median task duration; this rule only applies once a stage has at least 10 tasks.</p><p>&quot;Duration&quot; here is <code>executorRunTime</code>: elapsed wall-clock time, not CPU time, and it already includes any time the task spent blocked fetching shuffle data<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup>. That matters when tracking down why a task is slow: a long <code>executorRunTime</code> doesn&#39;t necessarily mean more compute happened. A <a href="./bottleneck-gc.html">garbage-collection pause</a> is counted inside it rather than added on top (<code>jvmGCTime</code> is a subset of <code>executorRunTime</code>, not additive<sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup>), and a task waiting on a remote shuffle block it needs next shows up in <code>shuffleReadMetrics.fetchWaitTime</code>, which only counts genuine blocking time, not blocks being prefetched in the background<sup class="footnote-ref"><a href="#fn1" id="fnref1:2">[1:2]</a></sup>.</p><p>The &quot;speculative tasks fired&quot; half of the rule is Spark&#39;s own detector: once <code>spark.speculation.quantile</code> (default <code>0.9</code>) of a stage&#39;s tasks finish, Spark compares each remaining task&#39;s duration against <code>spark.speculation.multiplier</code> (default <code>3</code>) times the median of the tasks that already finished, subject to a <code>spark.speculation.minTaskRuntime</code> floor (default <code>100ms</code>) so short tasks aren&#39;t flagged just for being slower than a tiny median<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup>. Speculation itself is off by default (<code>spark.speculation</code> defaults to <code>false</code>), so this half of the rule only fires on stages where it&#39;s been turned on<sup class="footnote-ref"><a href="#fn2" id="fnref2:1">[2:1]</a></sup>.</p><p>Watch for overlap with <a href="./bottleneck-skew.html">task skew</a>: an unevenly distributed key sends one partition far more data than its peers, and that partition&#39;s task will trip this same duration threshold even though the underlying cause is data volume, not a slow host or a GC pause. Check the skew signals before assuming the latter.</p><h2 id="why-it-matters" tabindex="-1">Why it matters <a class="header-anchor" href="#why-it-matters" aria-label="Permalink to &quot;Why it matters&quot;">​</a></h2><p>A straggler wastes cluster capacity the same way a skewed stage does: every executor other than the one running the slow task finishes early and idles, while total job time still tracks the single slowest task. Because a straggler&#39;s duration can be inflated by a GC pause<sup class="footnote-ref"><a href="#fn1" id="fnref1:3">[1:3]</a></sup> or a slow <a href="./shuffle.html">shuffle</a> fetch<sup class="footnote-ref"><a href="#fn1" id="fnref1:4">[1:4]</a></sup> rather than genuinely more work, it&#39;s worth checking those angles (and whether the real cause is skew rather than anything task-local) before assuming a hardware explanation.</p><h2 id="how-to-fix-it" tabindex="-1">How to fix it <a class="header-anchor" href="#how-to-fix-it" aria-label="Permalink to &quot;How to fix it&quot;">​</a></h2><ul><li>Enable speculative execution: <code>spark.speculation</code> is <code>false</code> by default, so nothing reruns a slow task automatically until it&#39;s turned on<sup class="footnote-ref"><a href="#fn2" id="fnref2:2">[2:2]</a></sup>.</li><li>Tune the trigger: <code>spark.speculation.quantile</code> (default <code>0.9</code>) sets how much of the stage must finish before speculation kicks in, and <code>spark.speculation.multiplier</code> (default <code>3</code>) sets how many times slower than the median a task must be. For stages with very few tasks, <code>spark.speculation.task.duration.threshold</code> (available since 3.0.0) gives an absolute-duration trigger instead of relying on the median<sup class="footnote-ref"><a href="#fn2" id="fnref2:3">[2:3]</a></sup>.</li><li>Since Spark 3.4, <code>spark.speculation.efficiency.enabled</code> (default <code>true</code>) adds a filter so a task is only speculated if its data-process rate is below the stage average (times <code>spark.speculation.efficiency.processRateMultiplier</code>, default <code>0.75</code>) or its duration exceeds <code>spark.speculation.efficiency.longRunTaskFactor</code> (default <code>2</code>) times the same time threshold. This avoids wasting a duplicate task slot on work that&#39;s simply doing more, not running slower<sup class="footnote-ref"><a href="#fn2" id="fnref2:4">[2:4]</a></sup>.</li><li>If the cause is really a skewed key, treat it as <a href="./bottleneck-skew.html">task skew</a> instead: AQE&#39;s skew-join optimization automatically splits a partition once it&#39;s larger than <code>spark.sql.adaptive.skewJoin.skewedPartitionFactor</code> (default <code>5.0</code>) times the median partition size and above <code>spark.sql.adaptive.skewJoin.skewedPartitionThresholdInBytes</code> (default <code>256 MB</code>)<sup class="footnote-ref"><a href="#fn2" id="fnref2:5">[2:5]</a></sup><sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup>. Manual salting (appending a random prefix to the skewed key so it spreads across more partitions) is the pre-AQE fallback, though the design doc behind AQE&#39;s skew handling calls salting and other manual approaches limited compared to the automatic option<sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup>.</li></ul><blockquote><p><strong>PySpark:</strong> speculation settings can be set on the session directly, no <code>spark-submit</code> flag needed: <code>spark.conf.set(&quot;spark.speculation&quot;, &quot;true&quot;)</code>, then the quantile/multiplier equivalents the same way.</p></blockquote><p>Speculation config: defaults shown, plus the absolute-duration trigger for small stages:</p><div class="language-properties vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">properties</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Speculatively relaunch straggler tasks (OFF by default)</span></span>
26
+ <span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.speculation</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=true</span></span>
27
+ <span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.speculation.quantile</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=0.9 </span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># default; portion of tasks finished before speculation begins</span></span>
28
+ <span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.speculation.multiplier</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=3 </span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># default; multiple of median duration that marks a task slow</span></span>
29
+ <span class="line"></span>
30
+ <span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Absolute-duration trigger for stages with very few tasks (since Spark 3.0)</span></span>
31
+ <span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.speculation.task.duration.threshold</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=10s </span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># 10s is an example</span></span></code></pre></div><h2 id="confidence" tabindex="-1">Confidence <a class="header-anchor" href="#confidence" aria-label="Permalink to &quot;Confidence&quot;">​</a></h2><p>Validated.</p><h2 id="limitations-false-positive-risk" tabindex="-1">Limitations / false-positive risk <a class="header-anchor" href="#limitations-false-positive-risk" aria-label="Permalink to &quot;Limitations / false-positive risk&quot;">​</a></h2><p>The 4x-median rule flags a slow task, but slow is not the same as broken. The same threshold trips on a skewed key that simply has more data to process, or on a task that spent its time in a GC pause rather than doing extra work, so a flagged task is not automatically a slow host. Small stages make this worse: with only a handful of tasks the median is unstable, and one moderately slow task can look like a straggler against a median computed from too few peers.</p><h2 id="related" tabindex="-1">Related <a class="header-anchor" href="#related" aria-label="Permalink to &quot;Related&quot;">​</a></h2><ul><li><strong>When the real cause is a skewed key:</strong> <a href="./partitioning.html">Partitioning</a>, <a href="./aqe.html">Adaptive Query Execution</a></li></ul><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/monitoring.html#spark-history-server" target="_blank" rel="noreferrer">Monitoring and Instrumentation</a> <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a> <a href="#fnref1:2" class="footnote-backref">↩︎</a> <a href="#fnref1:3" class="footnote-backref">↩︎</a> <a href="#fnref1:4" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration (Spark)</a> <a href="#fnref2" class="footnote-backref">↩︎</a> <a href="#fnref2:1" class="footnote-backref">↩︎</a> <a href="#fnref2:2" class="footnote-backref">↩︎</a> <a href="#fnref2:3" class="footnote-backref">↩︎</a> <a href="#fnref2:4" class="footnote-backref">↩︎</a> <a href="#fnref2:5" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/sql-performance-tuning.html#optimizing-skew-join" target="_blank" rel="noreferrer">Optimizing Skew Join (Spark SQL, DataFrames and Datasets Guide)</a> <a href="#fnref3" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><a href="https://issues.apache.org/jira/browse/SPARK-29544" target="_blank" rel="noreferrer">SPARK-29544: Optimize Skewed Join at Runtime with New Adaptive Execution</a> <a href="#fnref4" class="footnote-backref">↩︎</a></p></li></ol></section></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/tuning-reference/bottleneck-failures.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>Task Failures</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="/docs/tuning-reference/bottleneck-retry-waste.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>Retry Waste</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
32
+ <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"B-OsL91z\",\"contributor-guide_architecture_detector-contract.md\":\"BOeH4d1J\",\"contributor-guide_architecture_drill-down.md\":\"BtPdlM7r\",\"contributor-guide_architecture_impact-estimation.md\":\"DYCDPgkh\",\"contributor-guide_architecture_index.md\":\"3TO9ic6w\",\"contributor-guide_architecture_overview.md\":\"CehiRmGn\",\"contributor-guide_architecture_state-and-history.md\":\"m3S3UdMk\",\"contributor-guide_architecture_widget-rendering.md\":\"DbqPf2OT\",\"contributor-guide_architecture_worker-protocol.md\":\"B93qJ_tT\",\"contributor-guide_contributing.md\":\"CvRsdr6J\",\"contributor-guide_development-setup.md\":\"DvAN_9mK\",\"contributor-guide_testing.md\":\"6rIKqSyY\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"Df1YMIHu\",\"tuning-reference_aqe.md\":\"BIsCtLzm\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"CEstB3Ia\",\"tuning-reference_bottleneck-cold-start.md\":\"CEuy-72y\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"CIohQDfn\",\"tuning-reference_bottleneck-failures.md\":\"4z5BXGJ2\",\"tuning-reference_bottleneck-gc.md\":\"DSxzZRK7\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"BaJl__1W\",\"tuning-reference_bottleneck-memory-utilization.md\":\"DbP-SJZc\",\"tuning-reference_bottleneck-retry-waste.md\":\"D5JMjVOt\",\"tuning-reference_bottleneck-shuffle.md\":\"CM-nTmIH\",\"tuning-reference_bottleneck-skew.md\":\"BdUwiDhn\",\"tuning-reference_bottleneck-slow-host.md\":\"BlIo6UDW\",\"tuning-reference_bottleneck-small-files.md\":\"B8kloyx8\",\"tuning-reference_bottleneck-spill.md\":\"PNH7mITt\",\"tuning-reference_bottleneck-straggler.md\":\"DY36fHN5\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"QTV7O8kU\",\"tuning-reference_bottleneck-utilization.md\":\"DTiueZC3\",\"tuning-reference_caching.md\":\"B7aQ8asB\",\"tuning-reference_cluster-config.md\":\"ZVmDGsQ3\",\"tuning-reference_config.md\":\"UvveiWG3\",\"tuning-reference_data-formats.md\":\"bjCAWH3N\",\"tuning-reference_index.md\":\"BQ_NooMV\",\"tuning-reference_intro.md\":\"CobD-lGB\",\"tuning-reference_joins.md\":\"BtKs_CuW\",\"tuning-reference_memory-model.md\":\"DhT-n4y3\",\"tuning-reference_metrics.md\":\"mLOh7Apj\",\"tuning-reference_partitioning.md\":\"q0zKF_8X\",\"tuning-reference_pyspark.md\":\"DDCfvN9t\",\"tuning-reference_shuffle.md\":\"BZZ7R4Ix\",\"tuning-reference_spark-architecture.md\":\"Dwzm5avO\",\"tuning-reference_table-formats.md\":\"D6wj-2dX\",\"user-guide_alternative-log-retrieval.md\":\"sU3KGarf\",\"user-guide_getting-started.md\":\"DtEM37MK\",\"user-guide_mcp-tools.md\":\"C8MiIu7F\",\"user-guide_run-comparison.md\":\"S0TWWmLY\",\"user-guide_understanding-findings.md\":\"D0R_Y-R2\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Alternative log retrieval\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
33
+
34
+ </body>
35
+ </html>
package/public/docs/tuning-reference/bottleneck-tiny-tasks.html ADDED
@@ -0,0 +1,36 @@
1
+ <!DOCTYPE html>
2
+ <html lang="en-US" dir="ltr">
3
+ <head>
4
+ <meta charset="utf-8">
5
+ <meta name="viewport" content="width=device-width,initial-scale=1">
6
+ <title>Tiny Tasks | SparkForensics</title>
7
+ <meta name="description" content="Docs for using and contributing to SparkForensics">
8
+ <meta name="generator" content="VitePress v1.6.4">
9
+ <link rel="preload stylesheet" href="/docs/assets/style.DXOMCXxn.css" as="style">
10
+ <link rel="preload stylesheet" href="/docs/vp-icons.css" as="style">
11
+
12
+ <script type="module" src="/docs/assets/app.BgzsaTS3.js"></script>
13
+ <link rel="preload" href="/docs/assets/inter-roman-latin.Di8DUHzh.woff2" as="font" type="font/woff2" crossorigin="">
14
+ <link rel="modulepreload" href="/docs/assets/chunks/theme.BP14rGBx.js">
15
+ <link rel="modulepreload" href="/docs/assets/chunks/framework.DSg0KOwT.js">
16
+ <link rel="modulepreload" href="/docs/assets/tuning-reference_bottleneck-tiny-tasks.md.QTV7O8kU.lean.js">
17
+ <link rel="icon" type="image/svg+xml" href="/docs/favicon.svg">
18
+ <link rel="preconnect" href="https://fonts.googleapis.com">
19
+ <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin="">
20
+ <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Recursive:wght,CASL@400..700,0..1&amp;family=JetBrains+Mono:wght@400;500;600&amp;display=swap">
21
+ <script id="check-dark-mode">(()=>{const e=localStorage.getItem("vitepress-theme-appearance")||"auto",a=window.matchMedia("(prefers-color-scheme: dark)").matches;(!e||e==="auto"?a:e==="dark")&&document.documentElement.classList.add("dark")})();</script>
22
+ <script id="check-mac-os">document.documentElement.classList.toggle("mac",/Mac|iPhone|iPod|iPad/i.test(navigator.platform));</script>
23
+ </head>
24
+ <body>
25
+ <div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Getting Started</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/intro.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Introduction</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Optimization Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/spark-architecture.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Execution Model</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/memory-model.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Management</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/partitioning.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Partitioning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/joins.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Join Optimization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/data-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Data Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/table-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Table Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/caching.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Caching & Persistence</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/pyspark.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>PySpark Specifics</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/aqe.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Adaptive Query Execution</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/cluster-config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cluster Tuning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/anti-patterns.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Anti-Patterns</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Detector Catalog</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-skew.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Skew</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle I/O</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-spill.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory / Disk Spill</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-gc.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>GC Pressure</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-cold-start.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cold Start</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Executor Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-slow-host.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Slow Host</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-failures.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Failures</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-straggler.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Stragglers</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-retry-waste.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Retry Waste</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-tiny-tasks.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Tiny Tasks</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-job-failure-rate.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Job Failure Rate</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-memory-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Duplicate Plan Subtree</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-small-files.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Small Files</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-broadcast-sizing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Broadcast Sizing</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Reference</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/metrics.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Metrics Glossary</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Config Quick-Reference</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_tuning-reference_bottleneck-tiny-tasks" data-v-39a288b8><div><h1 id="bottleneck-tiny-tasks" tabindex="-1">Tiny Tasks <a class="header-anchor" href="#bottleneck-tiny-tasks" aria-label="Permalink to &quot;Tiny Tasks {#bottleneck-tiny-tasks}&quot;">​</a></h1><p><span class="tag">TINY</span></p><h2 id="what-it-is" tabindex="-1">What it is <a class="header-anchor" href="#what-it-is" aria-label="Permalink to &quot;What it is&quot;">​</a></h2><p>Every task pays a fixed placement and serialization cost before it does any real work. Spark schedules around data locality rather than moving data to code: &quot;Spark builds its scheduling around this general principle&quot; that shipping serialized code is cheaper than shipping data, preferring <code>PROCESS_LOCAL</code> locality down to <code>ANY</code> and waiting a configurable timeout for a busy CPU to free up before shipping data to a farther executor<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup>. On top of that placement cost, every task also pays a serialization cost for its closure and, if not using Kryo, for its data: Java&#39;s default <code>ObjectOutputStream</code>-based serialization &quot;is flexible but often quite slow, and leads to large serialized formats,&quot; while Kryo is &quot;significantly faster and more compact than Java serialization (often as much as 10x)&quot;<sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup>. None of that overhead is large per task: Spark&#39;s own guidance leans toward more, smaller tasks rather than fewer, larger ones, unlike MapReduce: &quot;it&#39;s almost always better to err on the side of a larger number of tasks (and thus partitions)... The difference stems from the fact that MapReduce has a high startup overhead for tasks, while Spark does not&quot;<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup>. But once partitions get small enough, that per-task overhead stops being negligible and starts to dominate.</p><h2 id="how-it-s-detected" tabindex="-1">How it&#39;s detected <a class="header-anchor" href="#how-it-s-detected" aria-label="Permalink to &quot;How it&#39;s detected&quot;">​</a></h2><p>The failure mode is partitions so small that scheduling and serialization overhead outweighs the actual work. With Spark&#39;s default of 200 shuffle partitions applied to a dataset of only a few megabytes, &quot;those 200 partitions will each get like ten rows. Tasks become microscopic. Most of your CPUs will just be sitting there doing nothing, wasting cluster hours&quot;<sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup>. The same small-partition penalty shows up on the <a href="./bottleneck-shuffle.html">shuffle-read side</a>: production measurements found &quot;the average shuffle block size is only 10s of KBs, which leads to delayed shuffle data fetch,&quot; and the shuffle reduce stages with the largest fetch delays (over 30 seconds per task) were consistently the ones with small block sizes<sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup>. A high task count paired with very short median task duration, and/or very small shuffle block sizes on the read side, are the signals to look for.</p><h2 id="why-it-matters" tabindex="-1">Why it matters <a class="header-anchor" href="#why-it-matters" aria-label="Permalink to &quot;Why it matters&quot;">​</a></h2><p>Tiny tasks waste cluster capacity even though no single task is a straggler the way <a href="./bottleneck-skew.html">skewed</a> or <a href="./bottleneck-straggler.html">straggler</a> tasks are: the cost here is spread evenly across thousands of tasks, each individually cheap but collectively adding up in scheduling and serialization overhead, while cores that could be doing other useful work sit mostly idle between task launches<sup class="footnote-ref"><a href="#fn3" id="fnref3:1">[3:1]</a></sup>.</p><h2 id="how-to-fix-it" tabindex="-1">How to fix it <a class="header-anchor" href="#how-to-fix-it" aria-label="Permalink to &quot;How to fix it&quot;">​</a></h2><ul><li><code>coalesce()</code> merges existing partitions without a shuffle (&quot;no data movement, no shuffle&quot;) and is typically used right before a write, e.g. <code>df.coalesce(100).write.parquet(...)</code>, to collapse many tiny output files into a manageable number cheaply<sup class="footnote-ref"><a href="#fn3" id="fnref3:2">[3:2]</a></sup>. Because it only stacks partitions together rather than redistributing data, it doesn&#39;t fix an uneven underlying distribution (uneven input partitions stay uneven, just grouped) and pushing it too far (<code>coalesce(1)</code>) kills parallelism by funneling all work onto one executor while the rest sit idle<sup class="footnote-ref"><a href="#fn3" id="fnref3:3">[3:3]</a></sup>.</li><li><code>repartition()</code> performs a full reshuffle, giving control and rebalancing that Spark won&#39;t do on its own: with <code>df.repartition(200)</code>, &quot;you&#39;re paying for predictability&quot;<sup class="footnote-ref"><a href="#fn3" id="fnref3:4">[3:4]</a></sup>. Use it when the partitioning itself needs to be fixed or rebalanced, not merely reduced in count, accepting the shuffle cost to get there.</li><li>There&#39;s a middle option, <code>coalesce(N, shuffle=True)</code>, which &quot;acts more like a repartition, but leaning toward reduction... not free (you pay for the shuffle cost) but you get better distribution and fewer partitions&quot;<sup class="footnote-ref"><a href="#fn3" id="fnref3:5">[3:5]</a></sup>.</li><li>Rule of thumb: reach for <code>coalesce()</code> when only the partition count needs to shrink and the existing distribution is already reasonably even (e.g. collapsing output files); reach for <code>repartition()</code> when the distribution itself is the problem.</li></ul><blockquote><p><strong>PySpark:</strong> both are one-line calls on a DataFrame: <code>df.coalesce(100)</code> for a shuffle-free merge, <code>df.repartition(200)</code> for a full reshuffle, or <code>df.coalesce(100, shuffle=True)</code> for the shuffled middle option.</p></blockquote><p>The three sizing calls, side by side:</p><div class="language-python vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">python</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Collapse many tiny output files without a shuffle (use when distribution is already even)</span></span>
26
+ <span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">df.coalesce(</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">100</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">).write.parquet(path)</span></span>
27
+ <span class="line"></span>
28
+ <span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Full reshuffle to a target count, use when the distribution itself needs rebalancing</span></span>
29
+ <span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">df </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">=</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> df.repartition(</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">200</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">)</span></span>
30
+ <span class="line"></span>
31
+ <span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Middle ground: reduce partition count but still rebalance (pays a shuffle)</span></span>
32
+ <span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">df </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">=</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> df.coalesce(</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">100</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70;">shuffle</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">=</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">True</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">)</span></span></code></pre></div><h2 id="limitations-false-positive-risk" tabindex="-1">Limitations / false-positive risk <a class="header-anchor" href="#limitations-false-positive-risk" aria-label="Permalink to &quot;Limitations / false-positive risk&quot;">​</a></h2><p>Very short tasks are not always a problem. A tiny final stage can be perfectly fine, and one that just writes a small result set doesn&#39;t need to be resized. The scheduling-overhead penalty only bites when tiny tasks dominate the stage, so treat a handful of short tasks as noise rather than a finding.</p><h2 id="related" tabindex="-1">Related <a class="header-anchor" href="#related" aria-label="Permalink to &quot;Related&quot;">​</a></h2><ul><li><strong>Partition sizing:</strong> <a href="./partitioning.html">Partitioning</a></li></ul><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/tuning.html" target="_blank" rel="noreferrer">Tuning (Spark)</a> <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><a href="https://blog.cloudera.com/how-to-tune-your-apache-spark-jobs-part-2/" target="_blank" rel="noreferrer">How to Tune Your Apache Spark Jobs (Part 2): Cloudera</a> <a href="#fnref2" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><a href="https://luminousmen.com/post/spark-partitions" target="_blank" rel="noreferrer">Spark Partitions</a> <a href="#fnref3" class="footnote-backref">↩︎</a> <a href="#fnref3:1" class="footnote-backref">↩︎</a> <a href="#fnref3:2" class="footnote-backref">↩︎</a> <a href="#fnref3:3" class="footnote-backref">↩︎</a> <a href="#fnref3:4" class="footnote-backref">↩︎</a> <a href="#fnref3:5" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><a href="https://issues.apache.org/jira/browse/SPARK-30602" target="_blank" rel="noreferrer">SPARK-30602</a> <a href="#fnref4" class="footnote-backref">↩︎</a></p></li></ol></section></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/tuning-reference/bottleneck-retry-waste.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>Retry Waste</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="/docs/tuning-reference/bottleneck-job-failure-rate.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>Job Failure Rate</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
33
+ <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"B-OsL91z\",\"contributor-guide_architecture_detector-contract.md\":\"BOeH4d1J\",\"contributor-guide_architecture_drill-down.md\":\"BtPdlM7r\",\"contributor-guide_architecture_impact-estimation.md\":\"DYCDPgkh\",\"contributor-guide_architecture_index.md\":\"3TO9ic6w\",\"contributor-guide_architecture_overview.md\":\"CehiRmGn\",\"contributor-guide_architecture_state-and-history.md\":\"m3S3UdMk\",\"contributor-guide_architecture_widget-rendering.md\":\"DbqPf2OT\",\"contributor-guide_architecture_worker-protocol.md\":\"B93qJ_tT\",\"contributor-guide_contributing.md\":\"CvRsdr6J\",\"contributor-guide_development-setup.md\":\"DvAN_9mK\",\"contributor-guide_testing.md\":\"6rIKqSyY\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"Df1YMIHu\",\"tuning-reference_aqe.md\":\"BIsCtLzm\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"CEstB3Ia\",\"tuning-reference_bottleneck-cold-start.md\":\"CEuy-72y\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"CIohQDfn\",\"tuning-reference_bottleneck-failures.md\":\"4z5BXGJ2\",\"tuning-reference_bottleneck-gc.md\":\"DSxzZRK7\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"BaJl__1W\",\"tuning-reference_bottleneck-memory-utilization.md\":\"DbP-SJZc\",\"tuning-reference_bottleneck-retry-waste.md\":\"D5JMjVOt\",\"tuning-reference_bottleneck-shuffle.md\":\"CM-nTmIH\",\"tuning-reference_bottleneck-skew.md\":\"BdUwiDhn\",\"tuning-reference_bottleneck-slow-host.md\":\"BlIo6UDW\",\"tuning-reference_bottleneck-small-files.md\":\"B8kloyx8\",\"tuning-reference_bottleneck-spill.md\":\"PNH7mITt\",\"tuning-reference_bottleneck-straggler.md\":\"DY36fHN5\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"QTV7O8kU\",\"tuning-reference_bottleneck-utilization.md\":\"DTiueZC3\",\"tuning-reference_caching.md\":\"B7aQ8asB\",\"tuning-reference_cluster-config.md\":\"ZVmDGsQ3\",\"tuning-reference_config.md\":\"UvveiWG3\",\"tuning-reference_data-formats.md\":\"bjCAWH3N\",\"tuning-reference_index.md\":\"BQ_NooMV\",\"tuning-reference_intro.md\":\"CobD-lGB\",\"tuning-reference_joins.md\":\"BtKs_CuW\",\"tuning-reference_memory-model.md\":\"DhT-n4y3\",\"tuning-reference_metrics.md\":\"mLOh7Apj\",\"tuning-reference_partitioning.md\":\"q0zKF_8X\",\"tuning-reference_pyspark.md\":\"DDCfvN9t\",\"tuning-reference_shuffle.md\":\"BZZ7R4Ix\",\"tuning-reference_spark-architecture.md\":\"Dwzm5avO\",\"tuning-reference_table-formats.md\":\"D6wj-2dX\",\"user-guide_alternative-log-retrieval.md\":\"sU3KGarf\",\"user-guide_getting-started.md\":\"DtEM37MK\",\"user-guide_mcp-tools.md\":\"C8MiIu7F\",\"user-guide_run-comparison.md\":\"S0TWWmLY\",\"user-guide_understanding-findings.md\":\"D0R_Y-R2\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Alternative log retrieval\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
34
+
35
+ </body>
36
+ </html>
package/public/docs/tuning-reference/bottleneck-utilization.html ADDED
@@ -0,0 +1,34 @@
1
+ <!DOCTYPE html>
2
+ <html lang="en-US" dir="ltr">
3
+ <head>
4
+ <meta charset="utf-8">
5
+ <meta name="viewport" content="width=device-width,initial-scale=1">
6
+ <title>Executor Utilization | SparkForensics</title>
7
+ <meta name="description" content="Docs for using and contributing to SparkForensics">
8
+ <meta name="generator" content="VitePress v1.6.4">
9
+ <link rel="preload stylesheet" href="/docs/assets/style.DXOMCXxn.css" as="style">
10
+ <link rel="preload stylesheet" href="/docs/vp-icons.css" as="style">
11
+
12
+ <script type="module" src="/docs/assets/app.BgzsaTS3.js"></script>
13
+ <link rel="preload" href="/docs/assets/inter-roman-latin.Di8DUHzh.woff2" as="font" type="font/woff2" crossorigin="">
14
+ <link rel="modulepreload" href="/docs/assets/chunks/theme.BP14rGBx.js">
15
+ <link rel="modulepreload" href="/docs/assets/chunks/framework.DSg0KOwT.js">
16
+ <link rel="modulepreload" href="/docs/assets/chunks/duplicate-plan-subtree.dark.Cdp70QhV.js">
17
+ <link rel="modulepreload" href="/docs/assets/tuning-reference_bottleneck-utilization.md.DTiueZC3.lean.js">
18
+ <link rel="icon" type="image/svg+xml" href="/docs/favicon.svg">
19
+ <link rel="preconnect" href="https://fonts.googleapis.com">
20
+ <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin="">
21
+ <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Recursive:wght,CASL@400..700,0..1&amp;family=JetBrains+Mono:wght@400;500;600&amp;display=swap">
22
+ <script id="check-dark-mode">(()=>{const e=localStorage.getItem("vitepress-theme-appearance")||"auto",a=window.matchMedia("(prefers-color-scheme: dark)").matches;(!e||e==="auto"?a:e==="dark")&&document.documentElement.classList.add("dark")})();</script>
23
+ <script id="check-mac-os">document.documentElement.classList.toggle("mac",/Mac|iPhone|iPod|iPad/i.test(navigator.platform));</script>
24
+ </head>
25
+ <body>
26
+ <div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Getting Started</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/intro.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Introduction</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Optimization Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/spark-architecture.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Execution Model</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/memory-model.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Management</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/partitioning.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Partitioning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/joins.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Join Optimization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/data-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Data Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/table-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Table Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/caching.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Caching & Persistence</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/pyspark.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>PySpark Specifics</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/aqe.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Adaptive Query Execution</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/cluster-config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cluster Tuning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/anti-patterns.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Anti-Patterns</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Detector Catalog</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-skew.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Skew</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle I/O</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-spill.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory / Disk Spill</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-gc.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>GC Pressure</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-cold-start.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cold Start</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Executor Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-slow-host.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Slow Host</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-failures.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Failures</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-straggler.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Stragglers</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-retry-waste.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Retry Waste</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-tiny-tasks.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Tiny Tasks</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-job-failure-rate.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Job Failure Rate</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-memory-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Duplicate Plan Subtree</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-small-files.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Small Files</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-broadcast-sizing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Broadcast Sizing</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Reference</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/metrics.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Metrics Glossary</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Config Quick-Reference</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_tuning-reference_bottleneck-utilization" data-v-39a288b8><div><h1 id="bottleneck-utilization" tabindex="-1">Executor Utilization <a class="header-anchor" href="#bottleneck-utilization" aria-label="Permalink to &quot;Executor Utilization {#bottleneck-utilization}&quot;">​</a></h1><p><span class="tag">UTIL</span></p><h2 id="what-it-is" tabindex="-1">What it is <a class="header-anchor" href="#what-it-is" aria-label="Permalink to &quot;What it is&quot;">​</a></h2><p>Executor utilization measures how much of the cluster&#39;s allocated executor capacity a job actually keeps busy. When the average number of active executors trails the peak number allocated, the cluster is holding compute (cores and memory) that isn&#39;t running any tasks.</p><h2 id="how-it-s-detected" tabindex="-1">How it&#39;s detected <a class="header-anchor" href="#how-it-s-detected" aria-label="Permalink to &quot;How it&#39;s detected&quot;">​</a></h2><p>The signal is the ratio of average active executors to the peak active executor count observed over the job&#39;s lifetime:</p><table tabindex="0"><thead><tr><th>avg active executors / peak</th><th>Level</th></tr></thead><tbody><tr><td>&lt; 60%</td><td>Info</td></tr><tr><td>&lt; 40%</td><td>Warning</td></tr><tr><td>&lt; 20%</td><td>Critical</td></tr></tbody></table><h2 id="why-it-matters" tabindex="-1">Why it matters <a class="header-anchor" href="#why-it-matters" aria-label="Permalink to &quot;Why it matters&quot;">​</a></h2><p>Idle executors are allocation you&#39;re paying for without getting work done. One common cause is too little task parallelism to occupy the allocated cores: the guidance is to keep at least as many partitions as there are cores across the executors, so no core sits idle<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup>. Parallelism can also be lost by accident rather than by under-partitioning upfront: because <code>coalesce</code> is a narrow transformation, reducing partition count with it forces the <em>entire</em> upstream stage down to the reduced parallelism, not just the coalesce step, trading a shuffle for lost concurrency<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup>. Separately, under dynamic allocation, a workload with many small tasks can end up over-provisioned: by default it requests enough executors to maximize parallelism for the task count, and with small tasks that can mean some executors &quot;might not even do any work,&quot; wasting resources on allocation overhead<sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup>.</p><h2 id="how-to-fix-it" tabindex="-1">How to fix it <a class="header-anchor" href="#how-to-fix-it" aria-label="Permalink to &quot;How to fix it&quot;">​</a></h2><ul><li>Reach for <code>repartition</code> (not <code>coalesce</code>) when you need to raise partition count or rebalance data: <code>coalesce</code> only avoids a shuffle when shrinking partition count, and doing so also drags the entire upstream stage down to the reduced parallelism<sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup><sup class="footnote-ref"><a href="#fn2" id="fnref2:1">[2:1]</a></sup>.</li><li>Size partitions so the task count is at least the core count available across executors, to avoid leaving cores idle<sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup>.</li><li>Under dynamic allocation, lower <code>spark.dynamicAllocation.executorAllocationRatio</code> below its default of <code>1.0</code> to scale back the number of executors requested for workloads made up of many small tasks<sup class="footnote-ref"><a href="#fn3" id="fnref3:1">[3:1]</a></sup>.</li><li>Dynamic allocation requires shuffle tracking, the external shuffle service, or shuffle-block decommissioning to be enabled as a precondition<sup class="footnote-ref"><a href="#fn3" id="fnref3:2">[3:2]</a></sup>; without one of them, an executor removed mid-shuffle takes its unfetched shuffle files with it, forcing a recompute<sup class="footnote-ref"><a href="#fn5" id="fnref5">[5]</a></sup>, which undercuts using dynamic allocation to shed idle executors in the first place. <code>spark.dynamicAllocation.shuffleTracking.enabled</code> defaults to <code>true</code> since Spark 3.0, satisfying that precondition without needing a separate external shuffle service<sup class="footnote-ref"><a href="#fn3" id="fnref3:3">[3:3]</a></sup>.</li></ul><p>Restore parallelism after a heavy filter, and rein in over-provisioning for many-small-task jobs:</p><div class="language-python vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">python</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># repartition (not coalesce) rebalances and can raise partition count; aim &gt;= total executor cores</span></span>
27
+ <span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">df </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">=</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> df.filter(heavy_predicate).repartition(spark.sparkContext.defaultParallelism)</span></span>
28
+ <span class="line"></span>
29
+ <span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Scale back executors requested for many-small-task workloads (default ratio 1.0)</span></span>
30
+ <span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">spark.conf.set(</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;spark.dynamicAllocation.executorAllocationRatio&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;0.5&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">) </span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># 0.5 is an example</span></span></code></pre></div><h2 id="limitations-false-positive-risk" tabindex="-1">Limitations / false-positive risk <a class="header-anchor" href="#limitations-false-positive-risk" aria-label="Permalink to &quot;Limitations / false-positive risk&quot;">​</a></h2><p>A low average-to-peak ratio isn&#39;t always waste. A bursty or I/O-bound job legitimately holds executors while tasks wait on external systems rather than burning cores, and the ratio is sensitive to short stages, where a brief spike in allocation skews the average without meaning the cluster was genuinely idle.</p><h2 id="caching-opportunity" tabindex="-1">Caching opportunity <a class="header-anchor" href="#caching-opportunity" aria-label="Permalink to &quot;Caching opportunity&quot;">​</a></h2><p><span class="tag">CACHE</span></p><p>When the same DataFrame, RDD, or input is scanned more than once, low utilization can trace back to repeated recomputation rather than idle cores. Spark keeps nothing between actions: transformations only build a DAG, and once an action finishes its intermediate results are discarded<sup class="footnote-ref"><a href="#fn6" id="fnref6">[6]</a></sup>. Call a second action on the same logic and Spark re-runs the whole DAG from the source, which can mean re-reading a terabyte from S3, re-reading Kafka, or repeating expensive decompression<sup class="footnote-ref"><a href="#fn6" id="fnref6:1">[6:1]</a></sup>. Fork that logic into two pipeline branches and you sign up to recompute everything twice<sup class="footnote-ref"><a href="#fn6" id="fnref6:2">[6:2]</a></sup>.</p><img class="light-only" src="/docs/assets/duplicate-plan-subtree.B4cvN6fj.svg" alt="How two branches that repeat the same scan and operators each recompute it, until a shared cached or reused node lets both read one materialized result."><img class="dark-only" src="/docs/assets/duplicate-plan-subtree.dark.Dw8wS0Ag.svg" alt="How two branches that repeat the same scan and operators each recompute it, until a shared cached or reused node lets both read one materialized result."><p><a href="./caching.html"><code>cache()</code> and <code>persist()</code></a> are how you stop that. Persisting materializes the RDD (usually in memory on the executors) so it can be reused within the job, while Spark keeps its lineage to recompute any lost partition<sup class="footnote-ref"><a href="#fn7" id="fnref7">[7]</a></sup>. For a dataset you read repeatedly, this is one of the most useful optimizations available: it parks a DataFrame, table, or RDD in temporary storage across the executors and makes later reads fast<sup class="footnote-ref"><a href="#fn8" id="fnref8">[8]</a></sup>. Materialization is lazy and per-block, so only an action that touches every partition (<code>count()</code> or a full write) caches all of it; a subset scan like <code>take(10)</code> leaves a partial cache<sup class="footnote-ref"><a href="#fn6" id="fnref6:3">[6:3]</a></sup>.</p><p><code>cache()</code> is shorthand; <code>persist()</code> takes a <code>StorageLevel</code> and lets you pick the tradeoff:</p><table tabindex="0"><thead><tr><th>Level</th><th>Tradeoff</th></tr></thead><tbody><tr><td><code>MEMORY_ONLY</code></td><td>Fastest reads (deserialized JVM objects), but partitions that don&#39;t fit are recomputed on the fly each time<sup class="footnote-ref"><a href="#fn8" id="fnref8:1">[8:1]</a></sup>.</td></tr><tr><td><code>MEMORY_AND_DISK</code></td><td>Spills the overflow to disk instead of recomputing; Spark&#39;s default caching strategy and fine for most pipelines<sup class="footnote-ref"><a href="#fn9" id="fnref9">[9]</a></sup>. For DataFrames, <code>.cache()</code> maps to this<sup class="footnote-ref"><a href="#fn6" id="fnref6:4">[6:4]</a></sup>.</td></tr><tr><td>Serialized (<code>_SER</code>)</td><td>Byte arrays instead of raw objects: smaller footprint, slower reads. Reach for it when memory is tight<sup class="footnote-ref"><a href="#fn9" id="fnref9:1">[9:1]</a></sup>.</td></tr><tr><td>Replicated (<code>_2</code>)</td><td>A second copy for fast fault recovery instead of waiting on recomputation, at twice the space<sup class="footnote-ref"><a href="#fn10" id="fnref10">[10]</a></sup><sup class="footnote-ref"><a href="#fn9" id="fnref9:2">[9:2]</a></sup>.</td></tr></tbody></table><p>Whether to spill to disk hinges on recomputation cost: reading a block back from disk only beats recomputing it when the work that produced the data is expensive or filters out a large fraction, so the RDD guide says don&#39;t enable disk otherwise<sup class="footnote-ref"><a href="#fn10" id="fnref10:1">[10:1]</a></sup>.</p><p>Caching is not free, and several cases don&#39;t warrant it:</p><ul><li><strong>Single use.</strong> Caching adds serialization, deserialization, and storage cost, so caching data you read once only slows you down<sup class="footnote-ref"><a href="#fn8" id="fnref8:2">[8:2]</a></sup>.</li><li><strong>Data larger than storage memory,</strong> or a cheap transformation that isn&#39;t reused often regardless of size<sup class="footnote-ref"><a href="#fn11" id="fnref11">[11]</a></sup>.</li><li><strong>Memory pressure.</strong> Cache memory is memory taken from processing, and under the default spill strategy cached data can land on slower disk, so caching can cost more than just re-reading the source<sup class="footnote-ref"><a href="#fn9" id="fnref9:3">[9:3]</a></sup>.</li><li><strong>Lost optimizer freedom.</strong> Once a dataset is cached, Catalyst works on the in-memory copy and can no longer push filters down to the source<sup class="footnote-ref"><a href="#fn9" id="fnref9:4">[9:4]</a></sup>.</li></ul><p>A persisted RDD you&#39;ve stopped using still occupies memory until the app ends or eviction forces it out, so call <code>unpersist()</code> to reclaim it deliberately, which matters most on shared clusters<sup class="footnote-ref"><a href="#fn9" id="fnref9:5">[9:5]</a></sup>.</p><h2 id="related" tabindex="-1">Related <a class="header-anchor" href="#related" aria-label="Permalink to &quot;Related&quot;">​</a></h2><ul><li><strong>Task parallelism:</strong> <a href="./partitioning.html">Partitioning</a></li><li><strong>Executor sizing:</strong> <a href="./cluster-config.html">Cluster Tuning</a></li></ul><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><em>Learning Spark, 2nd Edition</em>, Damji, Wenig, Das &amp; Lee, ch. 7 <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><em>High Performance Spark, 2nd Edition</em>, Karau, Polak &amp; Warren, ch. 8 <a href="#fnref2" class="footnote-backref">↩︎</a> <a href="#fnref2:1" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration — Spark</a> <a href="#fnref3" class="footnote-backref">↩︎</a> <a href="#fnref3:1" class="footnote-backref">↩︎</a> <a href="#fnref3:2" class="footnote-backref">↩︎</a> <a href="#fnref3:3" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><em>Spark: The Definitive Guide</em>, Chambers &amp; Zaharia, ch. 19 <a href="#fnref4" class="footnote-backref">↩︎</a></p></li><li id="fn5" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/job-scheduling.html" target="_blank" rel="noreferrer">Job Scheduling — Dynamic Resource Allocation</a> <a href="#fnref5" class="footnote-backref">↩︎</a></p></li><li id="fn6" class="footnote-item"><p><a href="https://luminousmen.com/post/explaining-the-mechanics-of-spark-caching" target="_blank" rel="noreferrer">Explaining the Mechanics of Spark Caching</a> <a href="#fnref6" class="footnote-backref">↩︎</a> <a href="#fnref6:1" class="footnote-backref">↩︎</a> <a href="#fnref6:2" class="footnote-backref">↩︎</a> <a href="#fnref6:3" class="footnote-backref">↩︎</a> <a href="#fnref6:4" class="footnote-backref">↩︎</a></p></li><li id="fn7" class="footnote-item"><p><em>High Performance Spark, 2nd Edition</em>, Karau, Polak &amp; Warren, ch. 2 <a href="#fnref7" class="footnote-backref">↩︎</a></p></li><li id="fn8" class="footnote-item"><p><em>Spark: The Definitive Guide</em>, Chambers &amp; Zaharia, ch. 19 <a href="#fnref8" class="footnote-backref">↩︎</a> <a href="#fnref8:1" class="footnote-backref">↩︎</a> <a href="#fnref8:2" class="footnote-backref">↩︎</a></p></li><li id="fn9" class="footnote-item"><p><a href="https://luminousmen.com/post/spark-tips-caching" target="_blank" rel="noreferrer">Spark Tips: Caching</a> <a href="#fnref9" class="footnote-backref">↩︎</a> <a href="#fnref9:1" class="footnote-backref">↩︎</a> <a href="#fnref9:2" class="footnote-backref">↩︎</a> <a href="#fnref9:3" class="footnote-backref">↩︎</a> <a href="#fnref9:4" class="footnote-backref">↩︎</a> <a href="#fnref9:5" class="footnote-backref">↩︎</a></p></li><li id="fn10" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/rdd-programming-guide.html" target="_blank" rel="noreferrer">RDD Programming Guide — Spark</a> <a href="#fnref10" class="footnote-backref">↩︎</a> <a href="#fnref10:1" class="footnote-backref">↩︎</a></p></li><li id="fn11" class="footnote-item"><p><em>Learning Spark, 2nd Edition</em>, Damji, Wenig, Das &amp; Lee, ch. 7 <a href="#fnref11" class="footnote-backref">↩︎</a></p></li></ol></section></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/tuning-reference/bottleneck-cold-start.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>Cold Start</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="/docs/tuning-reference/bottleneck-slow-host.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>Slow Host</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
31
+ <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"B-OsL91z\",\"contributor-guide_architecture_detector-contract.md\":\"BOeH4d1J\",\"contributor-guide_architecture_drill-down.md\":\"BtPdlM7r\",\"contributor-guide_architecture_impact-estimation.md\":\"DYCDPgkh\",\"contributor-guide_architecture_index.md\":\"3TO9ic6w\",\"contributor-guide_architecture_overview.md\":\"CehiRmGn\",\"contributor-guide_architecture_state-and-history.md\":\"m3S3UdMk\",\"contributor-guide_architecture_widget-rendering.md\":\"DbqPf2OT\",\"contributor-guide_architecture_worker-protocol.md\":\"B93qJ_tT\",\"contributor-guide_contributing.md\":\"CvRsdr6J\",\"contributor-guide_development-setup.md\":\"DvAN_9mK\",\"contributor-guide_testing.md\":\"6rIKqSyY\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"Df1YMIHu\",\"tuning-reference_aqe.md\":\"BIsCtLzm\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"CEstB3Ia\",\"tuning-reference_bottleneck-cold-start.md\":\"CEuy-72y\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"CIohQDfn\",\"tuning-reference_bottleneck-failures.md\":\"4z5BXGJ2\",\"tuning-reference_bottleneck-gc.md\":\"DSxzZRK7\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"BaJl__1W\",\"tuning-reference_bottleneck-memory-utilization.md\":\"DbP-SJZc\",\"tuning-reference_bottleneck-retry-waste.md\":\"D5JMjVOt\",\"tuning-reference_bottleneck-shuffle.md\":\"CM-nTmIH\",\"tuning-reference_bottleneck-skew.md\":\"BdUwiDhn\",\"tuning-reference_bottleneck-slow-host.md\":\"BlIo6UDW\",\"tuning-reference_bottleneck-small-files.md\":\"B8kloyx8\",\"tuning-reference_bottleneck-spill.md\":\"PNH7mITt\",\"tuning-reference_bottleneck-straggler.md\":\"DY36fHN5\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"QTV7O8kU\",\"tuning-reference_bottleneck-utilization.md\":\"DTiueZC3\",\"tuning-reference_caching.md\":\"B7aQ8asB\",\"tuning-reference_cluster-config.md\":\"ZVmDGsQ3\",\"tuning-reference_config.md\":\"UvveiWG3\",\"tuning-reference_data-formats.md\":\"bjCAWH3N\",\"tuning-reference_index.md\":\"BQ_NooMV\",\"tuning-reference_intro.md\":\"CobD-lGB\",\"tuning-reference_joins.md\":\"BtKs_CuW\",\"tuning-reference_memory-model.md\":\"DhT-n4y3\",\"tuning-reference_metrics.md\":\"mLOh7Apj\",\"tuning-reference_partitioning.md\":\"q0zKF_8X\",\"tuning-reference_pyspark.md\":\"DDCfvN9t\",\"tuning-reference_shuffle.md\":\"BZZ7R4Ix\",\"tuning-reference_spark-architecture.md\":\"Dwzm5avO\",\"tuning-reference_table-formats.md\":\"D6wj-2dX\",\"user-guide_alternative-log-retrieval.md\":\"sU3KGarf\",\"user-guide_getting-started.md\":\"DtEM37MK\",\"user-guide_mcp-tools.md\":\"C8MiIu7F\",\"user-guide_run-comparison.md\":\"S0TWWmLY\",\"user-guide_understanding-findings.md\":\"D0R_Y-R2\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Alternative log retrieval\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
32
+
33
+ </body>
34
+ </html>