sparkforensics-server 0.1.0 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (448) hide show
  1. package/README.md +6 -0
  2. package/index.js +41 -50
  3. package/lib/static-files.js +15 -10
  4. package/package.json +23 -3
  5. package/public/assets/AutoscalingChurn-B96FKx97.js +1 -0
  6. package/public/assets/CacheUtilization-lKua3J1x.js +1 -0
  7. package/public/assets/CachingOpportunity-U0g-8HPV.js +1 -0
  8. package/public/assets/ColdStart-w8QYTNKo.js +1 -0
  9. package/public/assets/ConfigAudit-ClZp5BpV.js +1 -0
  10. package/public/assets/CoreUsageArea-o3kqQwyi.js +1 -0
  11. package/public/assets/Dashboard-Bjl-MYMm.js +1 -0
  12. package/public/assets/DuplicatePlanSubtree-Cm_UU3lY.js +1 -0
  13. package/public/assets/DurationHistogram-ltF-7M0A.js +1 -0
  14. package/public/assets/EmptyState-BCA3xioa.js +1 -0
  15. package/public/assets/ExecutorUtilization-Cn69i0uq.js +1 -0
  16. package/public/assets/GcPressure-CYsmU1d7.js +1 -0
  17. package/public/assets/ImpactEstimate-BGt-4WX5.js +1 -0
  18. package/public/assets/IncompleteRun-s6uQ8OLf.js +1 -0
  19. package/public/assets/JobFailures-DJ2dDvPA.js +1 -0
  20. package/public/assets/MemoryUtilization-C9G7P94X.js +1 -0
  21. package/public/assets/OverBroadcast-DYauoYWQ.js +1 -0
  22. package/public/assets/PartitionSizing-Cz_B0-uG.js +1 -0
  23. package/public/assets/PlanExplorer-_2oB5ouy.js +1 -0
  24. package/public/assets/PlanGraphRoute-Cj-GWM8W.js +1 -0
  25. package/public/assets/RetryWaste-D_m8ASwP.js +1 -0
  26. package/public/assets/RowPagination-BLsm8I6i.js +1 -0
  27. package/public/assets/ShuffleIO-pyDhxdaz.js +1 -0
  28. package/public/assets/Skew-BGLJuZZR.js +1 -0
  29. package/public/assets/SlowHost-ChdWVvGO.js +1 -0
  30. package/public/assets/SmallFiles-gTilBtjW.js +1 -0
  31. package/public/assets/SpeculationWaste-CVzgJnSj.js +1 -0
  32. package/public/assets/Spill-NGJGQaTB.js +1 -0
  33. package/public/assets/StageFailed-sGhki53z.js +1 -0
  34. package/public/assets/StageFindingGroup-AyI7BMd3.js +1 -0
  35. package/public/assets/StageHeader-BTzGkK8k.js +1 -0
  36. package/public/assets/StagePill-BlMOW_pU.js +1 -0
  37. package/public/assets/StageShape-DfOW93Z1.js +1 -0
  38. package/public/assets/StageSlowness-BMZ-K_5e.js +1 -0
  39. package/public/assets/Straggler-dffcAN7t.js +1 -0
  40. package/public/assets/TaskFailures-DGDwUK2A.js +1 -0
  41. package/public/assets/TinyTask-CfT8zZDr.js +1 -0
  42. package/public/assets/UnderBroadcast-CiOiJ0k1.js +1 -0
  43. package/public/assets/WidgetLeadSummary-BdUUi7L5.js +1 -0
  44. package/public/assets/charts-vendor-0VMaLVLN.js +59 -0
  45. package/public/assets/downsample-WmNM1qc9.js +1 -0
  46. package/public/assets/finding-anchor-DEV8JvgH.js +1 -0
  47. package/public/assets/impact-sort-DxaUheWV.js +1 -0
  48. package/public/assets/inbox-_QCYkBsP.js +1 -0
  49. package/public/assets/index-3vlFVNCg.js +9 -0
  50. package/public/assets/index-CjVklAdW.css +2 -0
  51. package/public/assets/parser-worker-B2bc0lWe.js +112 -0
  52. package/public/assets/plan-graph-vendor-B0sEms9Z.js +7 -0
  53. package/public/assets/plan-graph-vendor-DLioOiRN.css +1 -0
  54. package/public/assets/react-vendor-D2TLlSzI.js +9 -0
  55. package/public/assets/rolldown-runtime-hePW80VL.js +1 -0
  56. package/public/assets/select-kuy1MiAo.js +1 -0
  57. package/public/assets/toggle-group-CUd-KWkp.js +1 -0
  58. package/public/assets/usePagedRows-jNUWgVTD.js +1 -0
  59. package/public/assets/useRecentFiles-D0OA_EOK.js +11 -0
  60. package/public/assets/useSortMode-BaeAA5cw.js +1 -0
  61. package/public/docs/404.html +3 -3
  62. package/public/docs/assets/{app.Bxy4_HVA.js → app.APoq_UBs.js} +1 -1
  63. package/public/docs/assets/aqe-loop.IwQSATHw.svg +1 -0
  64. package/public/docs/assets/aqe-loop.dark.DGbaxqJE.svg +1 -0
  65. package/public/docs/assets/broadcast-vs-shuffle.Db4WY1XK.svg +1 -0
  66. package/public/docs/assets/broadcast-vs-shuffle.dark.C7Bxs0mG.svg +1 -0
  67. package/public/docs/assets/cache-lifecycle.dark.B-hS7AgU.svg +1 -0
  68. package/public/docs/assets/cache-lifecycle.rEOVYQNU.svg +1 -0
  69. package/public/docs/assets/chunks/@localSearchIndexroot.CvM5fuxe.js +1 -0
  70. package/public/docs/assets/chunks/{VPLocalSearchBox.L5jNOIR3.js → VPLocalSearchBox.B3TPzlHq.js} +1 -1
  71. package/public/docs/assets/chunks/duplicate-plan-subtree.dark.Cdp70QhV.js +1 -0
  72. package/public/docs/assets/chunks/{framework.CNDaxUod.js → framework.DSg0KOwT.js} +1 -1
  73. package/public/docs/assets/chunks/retry-escalation-ladder.dark.DHipdJgZ.js +1 -0
  74. package/public/docs/assets/chunks/theme.PavXO_vz.js +2 -0
  75. package/public/docs/assets/cold-start-timeline.DxC_Sc7w.svg +1 -0
  76. package/public/docs/assets/cold-start-timeline.dark.CZ17YcAG.svg +1 -0
  77. package/public/docs/assets/columnar-layout.PghGeOEA.svg +1 -0
  78. package/public/docs/assets/columnar-layout.dark.BVNlz0ff.svg +1 -0
  79. package/public/docs/assets/container-memory.DIO0AnIm.svg +1 -0
  80. package/public/docs/assets/container-memory.dark.CP-5zuCl.svg +1 -0
  81. package/public/docs/assets/contributor-guide_architecture_board-widgets.md.CWpj01WU.js +1 -0
  82. package/public/docs/assets/contributor-guide_architecture_board-widgets.md.CWpj01WU.lean.js +1 -0
  83. package/public/docs/assets/contributor-guide_architecture_detector-contract.md.CgzUsQ6W.js +1 -0
  84. package/public/docs/assets/contributor-guide_architecture_detector-contract.md.CgzUsQ6W.lean.js +1 -0
  85. package/public/docs/assets/contributor-guide_architecture_drill-down.md.BtPdlM7r.js +1 -0
  86. package/public/docs/assets/contributor-guide_architecture_drill-down.md.BtPdlM7r.lean.js +1 -0
  87. package/public/docs/assets/{contributor-guide_architecture_impact-estimation.md.CzlXOnCq.js → contributor-guide_architecture_impact-estimation.md.CooslVJt.js} +1 -1
  88. package/public/docs/assets/{contributor-guide_architecture_impact-estimation.md.CzlXOnCq.lean.js → contributor-guide_architecture_impact-estimation.md.CooslVJt.lean.js} +1 -1
  89. package/public/docs/assets/{contributor-guide_architecture_index.md.94lNbF-4.js → contributor-guide_architecture_index.md.3TO9ic6w.js} +1 -1
  90. package/public/docs/assets/{contributor-guide_architecture_index.md.94lNbF-4.lean.js → contributor-guide_architecture_index.md.3TO9ic6w.lean.js} +1 -1
  91. package/public/docs/assets/{contributor-guide_architecture_overview.md.mQFlN66o.js → contributor-guide_architecture_overview.md.CehiRmGn.js} +1 -1
  92. package/public/docs/assets/{contributor-guide_architecture_overview.md.mQFlN66o.lean.js → contributor-guide_architecture_overview.md.CehiRmGn.lean.js} +1 -1
  93. package/public/docs/assets/contributor-guide_architecture_state-and-history.md.C-xxn0q7.js +1 -0
  94. package/public/docs/assets/contributor-guide_architecture_state-and-history.md.C-xxn0q7.lean.js +1 -0
  95. package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.R27gQrgY.js +1 -0
  96. package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.R27gQrgY.lean.js +1 -0
  97. package/public/docs/assets/{contributor-guide_architecture_worker-protocol.md.BaS_L10g.js → contributor-guide_architecture_worker-protocol.md.IbnfNrV3.js} +2 -2
  98. package/public/docs/assets/{contributor-guide_architecture_worker-protocol.md.BaS_L10g.lean.js → contributor-guide_architecture_worker-protocol.md.IbnfNrV3.lean.js} +1 -1
  99. package/public/docs/assets/contributor-guide_contributing.md.CvRsdr6J.js +1 -0
  100. package/public/docs/assets/contributor-guide_contributing.md.CvRsdr6J.lean.js +1 -0
  101. package/public/docs/assets/{contributor-guide_development-setup.md.CpUMVtIK.js → contributor-guide_development-setup.md.DvAN_9mK.js} +1 -1
  102. package/public/docs/assets/{contributor-guide_development-setup.md.CpUMVtIK.lean.js → contributor-guide_development-setup.md.DvAN_9mK.lean.js} +1 -1
  103. package/public/docs/assets/{contributor-guide_testing.md.C0q-BJfl.js → contributor-guide_testing.md.6rIKqSyY.js} +1 -1
  104. package/public/docs/assets/{contributor-guide_testing.md.C0q-BJfl.lean.js → contributor-guide_testing.md.6rIKqSyY.lean.js} +1 -1
  105. package/public/docs/assets/dag-stages.DSz_S937.svg +1 -0
  106. package/public/docs/assets/dag-stages.dark.F72UzxH4.svg +1 -0
  107. package/public/docs/assets/driver-executor.D5pQ7YN1.svg +1 -0
  108. package/public/docs/assets/driver-executor.dark.BmX9cPvh.svg +1 -0
  109. package/public/docs/assets/duplicate-plan-subtree.B4cvN6fj.svg +1 -0
  110. package/public/docs/assets/duplicate-plan-subtree.dark.Dw8wS0Ag.svg +1 -0
  111. package/public/docs/assets/{index.md.CETnDZzR.js → index.md.CHJVslga.js} +1 -1
  112. package/public/docs/assets/{index.md.CETnDZzR.lean.js → index.md.CHJVslga.lean.js} +1 -1
  113. package/public/docs/assets/join-strategy.C_FvrCEo.svg +1 -0
  114. package/public/docs/assets/join-strategy.dark.ChMLnNII.svg +1 -0
  115. package/public/docs/assets/memory-borrowing.BqQRJg0u.svg +1 -0
  116. package/public/docs/assets/memory-borrowing.dark.Yhh20O9C.svg +1 -0
  117. package/public/docs/assets/memory-regions.XHvO7jHG.svg +1 -0
  118. package/public/docs/assets/memory-regions.dark.D4TP9_08.svg +1 -0
  119. package/public/docs/assets/repartition-vs-coalesce.BovLRrpj.svg +1 -0
  120. package/public/docs/assets/repartition-vs-coalesce.dark.BhAczKZQ.svg +1 -0
  121. package/public/docs/assets/retry-escalation-ladder.DyTKJJmZ.svg +1 -0
  122. package/public/docs/assets/retry-escalation-ladder.dark.BdsabtU3.svg +1 -0
  123. package/public/docs/assets/shuffle-map-reduce.KuOEZVmg.svg +1 -0
  124. package/public/docs/assets/shuffle-map-reduce.dark.BgQZnFSb.svg +1 -0
  125. package/public/docs/assets/spill-classification.BU2euYDO.svg +1 -0
  126. package/public/docs/assets/spill-classification.dark.D7i1M40d.svg +1 -0
  127. package/public/docs/assets/{style.D2mY0HUs.css → style.DSixAiZE.css} +1 -1
  128. package/public/docs/assets/tuning-reference_anti-patterns.md.Df1YMIHu.js +1 -0
  129. package/public/docs/assets/tuning-reference_anti-patterns.md.Df1YMIHu.lean.js +1 -0
  130. package/public/docs/assets/tuning-reference_aqe.md.BIsCtLzm.js +1 -0
  131. package/public/docs/assets/tuning-reference_aqe.md.BIsCtLzm.lean.js +1 -0
  132. package/public/docs/assets/tuning-reference_bottleneck-broadcast-sizing.md.CEstB3Ia.js +1 -0
  133. package/public/docs/assets/tuning-reference_bottleneck-broadcast-sizing.md.CEstB3Ia.lean.js +1 -0
  134. package/public/docs/assets/tuning-reference_bottleneck-cold-start.md.CEuy-72y.js +7 -0
  135. package/public/docs/assets/tuning-reference_bottleneck-cold-start.md.CEuy-72y.lean.js +1 -0
  136. package/public/docs/assets/tuning-reference_bottleneck-duplicate-plan-subtree.md.CIohQDfn.js +1 -0
  137. package/public/docs/assets/tuning-reference_bottleneck-duplicate-plan-subtree.md.CIohQDfn.lean.js +1 -0
  138. package/public/docs/assets/tuning-reference_bottleneck-failures.md.4z5BXGJ2.js +6 -0
  139. package/public/docs/assets/tuning-reference_bottleneck-failures.md.4z5BXGJ2.lean.js +1 -0
  140. package/public/docs/assets/tuning-reference_bottleneck-gc.md.DSxzZRK7.js +6 -0
  141. package/public/docs/assets/tuning-reference_bottleneck-gc.md.DSxzZRK7.lean.js +1 -0
  142. package/public/docs/assets/tuning-reference_bottleneck-job-failure-rate.md.BaJl__1W.js +8 -0
  143. package/public/docs/assets/tuning-reference_bottleneck-job-failure-rate.md.BaJl__1W.lean.js +1 -0
  144. package/public/docs/assets/tuning-reference_bottleneck-memory-utilization.md.DbP-SJZc.js +7 -0
  145. package/public/docs/assets/tuning-reference_bottleneck-memory-utilization.md.DbP-SJZc.lean.js +1 -0
  146. package/public/docs/assets/tuning-reference_bottleneck-retry-waste.md.D5JMjVOt.js +1 -0
  147. package/public/docs/assets/tuning-reference_bottleneck-retry-waste.md.D5JMjVOt.lean.js +1 -0
  148. package/public/docs/assets/tuning-reference_bottleneck-shuffle.md.CM-nTmIH.js +12 -0
  149. package/public/docs/assets/tuning-reference_bottleneck-shuffle.md.CM-nTmIH.lean.js +1 -0
  150. package/public/docs/assets/tuning-reference_bottleneck-skew.md.BdUwiDhn.js +14 -0
  151. package/public/docs/assets/tuning-reference_bottleneck-skew.md.BdUwiDhn.lean.js +1 -0
  152. package/public/docs/assets/tuning-reference_bottleneck-slow-host.md.BlIo6UDW.js +7 -0
  153. package/public/docs/assets/tuning-reference_bottleneck-slow-host.md.BlIo6UDW.lean.js +1 -0
  154. package/public/docs/assets/tuning-reference_bottleneck-small-files.md.B8kloyx8.js +5 -0
  155. package/public/docs/assets/tuning-reference_bottleneck-small-files.md.B8kloyx8.lean.js +1 -0
  156. package/public/docs/assets/tuning-reference_bottleneck-spill.md.PNH7mITt.js +6 -0
  157. package/public/docs/assets/tuning-reference_bottleneck-spill.md.PNH7mITt.lean.js +1 -0
  158. package/public/docs/assets/tuning-reference_bottleneck-straggler.md.DY36fHN5.js +7 -0
  159. package/public/docs/assets/tuning-reference_bottleneck-straggler.md.DY36fHN5.lean.js +1 -0
  160. package/public/docs/assets/tuning-reference_bottleneck-tiny-tasks.md.QTV7O8kU.js +8 -0
  161. package/public/docs/assets/tuning-reference_bottleneck-tiny-tasks.md.QTV7O8kU.lean.js +1 -0
  162. package/public/docs/assets/tuning-reference_bottleneck-utilization.md.DTiueZC3.js +5 -0
  163. package/public/docs/assets/tuning-reference_bottleneck-utilization.md.DTiueZC3.lean.js +1 -0
  164. package/public/docs/assets/tuning-reference_caching.md.B7aQ8asB.js +1 -0
  165. package/public/docs/assets/tuning-reference_caching.md.B7aQ8asB.lean.js +1 -0
  166. package/public/docs/assets/tuning-reference_cluster-config.md.ZVmDGsQ3.js +1 -0
  167. package/public/docs/assets/tuning-reference_cluster-config.md.ZVmDGsQ3.lean.js +1 -0
  168. package/public/docs/assets/tuning-reference_config.md.UvveiWG3.js +1 -0
  169. package/public/docs/assets/tuning-reference_config.md.UvveiWG3.lean.js +1 -0
  170. package/public/docs/assets/tuning-reference_data-formats.md.bjCAWH3N.js +1 -0
  171. package/public/docs/assets/tuning-reference_data-formats.md.bjCAWH3N.lean.js +1 -0
  172. package/public/docs/assets/tuning-reference_index.md.BQ_NooMV.js +1 -0
  173. package/public/docs/assets/tuning-reference_index.md.BQ_NooMV.lean.js +1 -0
  174. package/public/docs/assets/tuning-reference_intro.md.CobD-lGB.js +1 -0
  175. package/public/docs/assets/tuning-reference_intro.md.CobD-lGB.lean.js +1 -0
  176. package/public/docs/assets/tuning-reference_joins.md.BtKs_CuW.js +1 -0
  177. package/public/docs/assets/tuning-reference_joins.md.BtKs_CuW.lean.js +1 -0
  178. package/public/docs/assets/tuning-reference_memory-model.md.DhT-n4y3.js +1 -0
  179. package/public/docs/assets/tuning-reference_memory-model.md.DhT-n4y3.lean.js +1 -0
  180. package/public/docs/assets/tuning-reference_metrics.md.mLOh7Apj.js +1 -0
  181. package/public/docs/assets/tuning-reference_metrics.md.mLOh7Apj.lean.js +1 -0
  182. package/public/docs/assets/tuning-reference_partitioning.md.q0zKF_8X.js +1 -0
  183. package/public/docs/assets/tuning-reference_partitioning.md.q0zKF_8X.lean.js +1 -0
  184. package/public/docs/assets/tuning-reference_pyspark.md.DDCfvN9t.js +6 -0
  185. package/public/docs/assets/tuning-reference_pyspark.md.DDCfvN9t.lean.js +1 -0
  186. package/public/docs/assets/tuning-reference_shuffle.md.BZZ7R4Ix.js +1 -0
  187. package/public/docs/assets/tuning-reference_shuffle.md.BZZ7R4Ix.lean.js +1 -0
  188. package/public/docs/assets/tuning-reference_spark-architecture.md.Dwzm5avO.js +1 -0
  189. package/public/docs/assets/tuning-reference_spark-architecture.md.Dwzm5avO.lean.js +1 -0
  190. package/public/docs/assets/tuning-reference_table-formats.md.D6wj-2dX.js +1 -0
  191. package/public/docs/assets/tuning-reference_table-formats.md.D6wj-2dX.lean.js +1 -0
  192. package/public/docs/assets/udf-execution-models.BUFDICuG.svg +1 -0
  193. package/public/docs/assets/udf-execution-models.dark.YTNS6GDq.svg +1 -0
  194. package/public/docs/assets/user-guide_alternative-log-retrieval.md.B4tPGIal.js +1 -0
  195. package/public/docs/assets/user-guide_alternative-log-retrieval.md.B4tPGIal.lean.js +1 -0
  196. package/public/docs/assets/user-guide_getting-started.md.BJvwLEIM.js +3 -0
  197. package/public/docs/assets/user-guide_getting-started.md.BJvwLEIM.lean.js +1 -0
  198. package/public/docs/assets/{user-guide_mcp-tools.md.DwYI1d6C.js → user-guide_mcp-tools.md.Vi3RoflJ.js} +35 -5
  199. package/public/docs/assets/{user-guide_mcp-tools.md.DwYI1d6C.lean.js → user-guide_mcp-tools.md.Vi3RoflJ.lean.js} +1 -1
  200. package/public/docs/assets/user-guide_run-comparison.md.CQc1aoU8.js +1 -0
  201. package/public/docs/assets/user-guide_run-comparison.md.CQc1aoU8.lean.js +1 -0
  202. package/public/docs/assets/user-guide_understanding-findings.md.DL1UDhvR.js +1 -0
  203. package/public/docs/assets/user-guide_understanding-findings.md.DL1UDhvR.lean.js +1 -0
  204. package/public/docs/contributor-guide/architecture/board-widgets.html +7 -7
  205. package/public/docs/contributor-guide/architecture/detector-contract.html +7 -7
  206. package/public/docs/contributor-guide/architecture/drill-down.html +7 -7
  207. package/public/docs/contributor-guide/architecture/impact-estimation.html +7 -7
  208. package/public/docs/contributor-guide/architecture/index.html +7 -7
  209. package/public/docs/contributor-guide/architecture/overview.html +7 -7
  210. package/public/docs/contributor-guide/architecture/state-and-history.html +7 -7
  211. package/public/docs/contributor-guide/architecture/widget-rendering.html +7 -7
  212. package/public/docs/contributor-guide/architecture/worker-protocol.html +8 -8
  213. package/public/docs/contributor-guide/contributing.html +7 -7
  214. package/public/docs/contributor-guide/development-setup.html +7 -7
  215. package/public/docs/contributor-guide/testing.html +7 -7
  216. package/public/docs/hashmap.json +1 -1
  217. package/public/docs/index.html +7 -7
  218. package/public/docs/tuning-reference/anti-patterns.html +29 -0
  219. package/public/docs/tuning-reference/aqe.html +29 -0
  220. package/public/docs/tuning-reference/bottleneck-broadcast-sizing.html +29 -0
  221. package/public/docs/tuning-reference/bottleneck-cold-start.html +35 -0
  222. package/public/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html +30 -0
  223. package/public/docs/tuning-reference/bottleneck-failures.html +35 -0
  224. package/public/docs/tuning-reference/bottleneck-gc.html +34 -0
  225. package/public/docs/tuning-reference/bottleneck-job-failure-rate.html +37 -0
  226. package/public/docs/tuning-reference/bottleneck-memory-utilization.html +35 -0
  227. package/public/docs/tuning-reference/bottleneck-retry-waste.html +30 -0
  228. package/public/docs/tuning-reference/bottleneck-shuffle.html +40 -0
  229. package/public/docs/tuning-reference/bottleneck-skew.html +42 -0
  230. package/public/docs/tuning-reference/bottleneck-slow-host.html +35 -0
  231. package/public/docs/tuning-reference/bottleneck-small-files.html +33 -0
  232. package/public/docs/tuning-reference/bottleneck-spill.html +34 -0
  233. package/public/docs/tuning-reference/bottleneck-straggler.html +35 -0
  234. package/public/docs/tuning-reference/bottleneck-tiny-tasks.html +36 -0
  235. package/public/docs/tuning-reference/bottleneck-utilization.html +34 -0
  236. package/public/docs/tuning-reference/caching.html +29 -0
  237. package/public/docs/tuning-reference/cluster-config.html +29 -0
  238. package/public/docs/tuning-reference/config.html +29 -0
  239. package/public/docs/tuning-reference/data-formats.html +29 -0
  240. package/public/docs/tuning-reference/index.html +29 -0
  241. package/public/docs/tuning-reference/intro.html +29 -0
  242. package/public/docs/tuning-reference/joins.html +29 -0
  243. package/public/docs/tuning-reference/memory-model.html +29 -0
  244. package/public/docs/tuning-reference/metrics.html +29 -0
  245. package/public/docs/tuning-reference/partitioning.html +29 -0
  246. package/public/docs/tuning-reference/pyspark.html +34 -0
  247. package/public/docs/tuning-reference/shuffle.html +29 -0
  248. package/public/docs/tuning-reference/spark-architecture.html +29 -0
  249. package/public/docs/tuning-reference/table-formats.html +29 -0
  250. package/public/docs/user-guide/alternative-log-retrieval.html +29 -0
  251. package/public/docs/user-guide/getting-started.html +8 -8
  252. package/public/docs/user-guide/mcp-tools.html +41 -11
  253. package/public/docs/user-guide/run-comparison.html +7 -7
  254. package/public/docs/user-guide/understanding-findings.html +7 -7
  255. package/public/index.html +8 -7
  256. package/public/sample-runs/sample-run.ndjson.gz +0 -0
  257. package/vendor-core/analyzer.js +74 -74
  258. package/vendor-core/cli/budgets.js +13 -27
  259. package/vendor-core/cli/collect-run.js +43 -19
  260. package/vendor-core/core-count.js +25 -27
  261. package/vendor-core/core-locality-ratio.js +4 -11
  262. package/vendor-core/core-time-series.js +6 -12
  263. package/vendor-core/core-usage-locality.js +3 -4
  264. package/vendor-core/detectors.js +395 -389
  265. package/vendor-core/docs-config.js +69 -21
  266. package/vendor-core/docs-content/chapters/01-intro.md +32 -0
  267. package/vendor-core/docs-content/chapters/02-spark-architecture.md +76 -0
  268. package/vendor-core/docs-content/chapters/03-memory-model.md +73 -0
  269. package/vendor-core/docs-content/chapters/04-partitioning.md +65 -0
  270. package/vendor-core/docs-content/chapters/05-joins.md +62 -0
  271. package/vendor-core/docs-content/chapters/06-shuffle.md +59 -0
  272. package/vendor-core/docs-content/chapters/07-data-formats.md +81 -0
  273. package/vendor-core/docs-content/chapters/07b-table-formats.md +56 -0
  274. package/vendor-core/docs-content/chapters/08-caching.md +58 -0
  275. package/vendor-core/docs-content/chapters/09-pyspark.md +78 -0
  276. package/vendor-core/docs-content/chapters/10-aqe.md +167 -0
  277. package/vendor-core/docs-content/chapters/11-cluster-config.md +170 -0
  278. package/vendor-core/docs-content/chapters/12-anti-patterns.md +171 -0
  279. package/vendor-core/docs-content/chapters/14-metrics.md +87 -0
  280. package/vendor-core/docs-content/chapters/15-config.md +93 -0
  281. package/vendor-core/docs-content/chapters/nav-index.json +370 -0
  282. package/vendor-core/docs-content/detection/cache.md +7 -0
  283. package/vendor-core/docs-content/detection/cfg.md +15 -0
  284. package/vendor-core/docs-content/detection/chrn.md +9 -0
  285. package/vendor-core/docs-content/detection/cold.md +4 -0
  286. package/vendor-core/docs-content/detection/cstor.md +4 -0
  287. package/vendor-core/docs-content/detection/fail.md +5 -0
  288. package/vendor-core/docs-content/detection/gc.md +4 -0
  289. package/vendor-core/docs-content/detection/host.md +5 -0
  290. package/vendor-core/docs-content/detection/incmp.md +6 -0
  291. package/vendor-core/docs-content/detection/jobs.md +4 -0
  292. package/vendor-core/docs-content/detection/local.md +6 -0
  293. package/vendor-core/docs-content/detection/mem.md +10 -0
  294. package/vendor-core/docs-content/detection/part.md +5 -0
  295. package/vendor-core/docs-content/detection/plan.md +14 -0
  296. package/vendor-core/docs-content/detection/retry.md +4 -0
  297. package/vendor-core/docs-content/detection/sfail.md +5 -0
  298. package/vendor-core/docs-content/detection/shape.md +5 -0
  299. package/vendor-core/docs-content/detection/shfl.md +4 -0
  300. package/vendor-core/docs-content/detection/skew.md +6 -0
  301. package/vendor-core/docs-content/detection/slow.md +6 -0
  302. package/vendor-core/docs-content/detection/spec.md +8 -0
  303. package/vendor-core/docs-content/detection/spill.md +7 -0
  304. package/vendor-core/docs-content/detection/strag.md +5 -0
  305. package/vendor-core/docs-content/detection/tiny.md +4 -0
  306. package/vendor-core/docs-content/detection/util.md +4 -0
  307. package/vendor-core/docs-content/diagrams/aqe-loop.dark.svg +1 -0
  308. package/vendor-core/docs-content/diagrams/aqe-loop.svg +1 -0
  309. package/vendor-core/docs-content/diagrams/broadcast-vs-shuffle.dark.svg +1 -0
  310. package/vendor-core/docs-content/diagrams/broadcast-vs-shuffle.svg +1 -0
  311. package/vendor-core/docs-content/diagrams/cache-lifecycle.dark.svg +1 -0
  312. package/vendor-core/docs-content/diagrams/cache-lifecycle.svg +1 -0
  313. package/vendor-core/docs-content/diagrams/cold-start-timeline.dark.svg +1 -0
  314. package/vendor-core/docs-content/diagrams/cold-start-timeline.svg +1 -0
  315. package/vendor-core/docs-content/diagrams/columnar-layout.dark.svg +1 -0
  316. package/vendor-core/docs-content/diagrams/columnar-layout.svg +1 -0
  317. package/vendor-core/docs-content/diagrams/container-memory.dark.svg +1 -0
  318. package/vendor-core/docs-content/diagrams/container-memory.svg +1 -0
  319. package/vendor-core/docs-content/diagrams/dag-stages.dark.svg +1 -0
  320. package/vendor-core/docs-content/diagrams/dag-stages.svg +1 -0
  321. package/vendor-core/docs-content/diagrams/driver-executor.dark.svg +1 -0
  322. package/vendor-core/docs-content/diagrams/driver-executor.svg +1 -0
  323. package/vendor-core/docs-content/diagrams/duplicate-plan-subtree.dark.svg +1 -0
  324. package/vendor-core/docs-content/diagrams/duplicate-plan-subtree.svg +1 -0
  325. package/vendor-core/docs-content/diagrams/join-strategy.dark.svg +1 -0
  326. package/vendor-core/docs-content/diagrams/join-strategy.svg +1 -0
  327. package/vendor-core/docs-content/diagrams/memory-borrowing.dark.svg +1 -0
  328. package/vendor-core/docs-content/diagrams/memory-borrowing.svg +1 -0
  329. package/vendor-core/docs-content/diagrams/memory-regions.dark.svg +1 -0
  330. package/vendor-core/docs-content/diagrams/memory-regions.svg +1 -0
  331. package/vendor-core/docs-content/diagrams/repartition-vs-coalesce.dark.svg +1 -0
  332. package/vendor-core/docs-content/diagrams/repartition-vs-coalesce.svg +1 -0
  333. package/vendor-core/docs-content/diagrams/retry-escalation-ladder.dark.svg +1 -0
  334. package/vendor-core/docs-content/diagrams/retry-escalation-ladder.svg +1 -0
  335. package/vendor-core/docs-content/diagrams/shuffle-map-reduce.dark.svg +1 -0
  336. package/vendor-core/docs-content/diagrams/shuffle-map-reduce.svg +1 -0
  337. package/vendor-core/docs-content/diagrams/spill-classification.dark.svg +1 -0
  338. package/vendor-core/docs-content/diagrams/spill-classification.svg +1 -0
  339. package/vendor-core/docs-content/diagrams/udf-execution-models.dark.svg +1 -0
  340. package/vendor-core/docs-content/diagrams/udf-execution-models.svg +1 -0
  341. package/vendor-core/docs-content/tuning/broadcast-sizing.md +78 -0
  342. package/vendor-core/docs-content/tuning/cold-start.md +81 -0
  343. package/vendor-core/docs-content/tuning/duplicate-plan-subtree.md +45 -0
  344. package/vendor-core/docs-content/tuning/failures.md +124 -0
  345. package/vendor-core/docs-content/tuning/gc.md +110 -0
  346. package/vendor-core/docs-content/tuning/job-failure-rate.md +101 -0
  347. package/vendor-core/docs-content/tuning/memory-utilization.md +58 -0
  348. package/vendor-core/docs-content/tuning/retry-waste.md +90 -0
  349. package/vendor-core/docs-content/tuning/shuffle.md +154 -0
  350. package/vendor-core/docs-content/tuning/skew.md +123 -0
  351. package/vendor-core/docs-content/tuning/slow-host.md +117 -0
  352. package/vendor-core/docs-content/tuning/small-files.md +99 -0
  353. package/vendor-core/docs-content/tuning/spill.md +114 -0
  354. package/vendor-core/docs-content/tuning/straggler.md +103 -0
  355. package/vendor-core/docs-content/tuning/tiny-tasks.md +94 -0
  356. package/vendor-core/docs-content/tuning/utilization.md +90 -0
  357. package/vendor-core/docs-site-config.js +10 -17
  358. package/vendor-core/efficiency-model.js +7 -13
  359. package/vendor-core/etl-phases.js +3 -5
  360. package/vendor-core/event-handlers.js +232 -134
  361. package/vendor-core/event-schemas.js +48 -114
  362. package/vendor-core/evidence-availability.js +5 -10
  363. package/vendor-core/evidence-report.js +73 -123
  364. package/vendor-core/export-data.js +48 -0
  365. package/vendor-core/finding-action-label.js +4 -10
  366. package/vendor-core/finding-filter-predicate.js +3 -7
  367. package/vendor-core/finding-generic-recommendation.js +112 -0
  368. package/vendor-core/finding-names.js +51 -0
  369. package/vendor-core/format-utils.js +112 -38
  370. package/vendor-core/impact-band.js +18 -24
  371. package/vendor-core/impact-estimator.js +38 -74
  372. package/vendor-core/ingest.js +7 -13
  373. package/vendor-core/job-groups.js +3 -6
  374. package/vendor-core/list-runs.js +278 -0
  375. package/vendor-core/load-vendored.js +6 -12
  376. package/vendor-core/log-header-peek.js +81 -0
  377. package/vendor-core/lz4-block.js +4 -6
  378. package/vendor-core/mcp-server-factory.js +38 -8
  379. package/vendor-core/mcp-tools.js +105 -76
  380. package/vendor-core/model-assembler.js +8 -16
  381. package/vendor-core/occupancy.js +5 -9
  382. package/vendor-core/parser-worker.js +20 -29
  383. package/vendor-core/plan-dot.js +2 -5
  384. package/vendor-core/plan-duration-attribution.js +78 -29
  385. package/vendor-core/plan-graph-model.js +126 -69
  386. package/vendor-core/plan-node-detail.js +31 -17
  387. package/vendor-core/plan-summary.js +19 -8
  388. package/vendor-core/recommendation-rollup.js +35 -39
  389. package/vendor-core/redact.js +72 -16
  390. package/vendor-core/rolling-log-reassembly.js +4 -6
  391. package/vendor-core/run-comparison.js +86 -72
  392. package/vendor-core/scaling-sim.js +5 -7
  393. package/vendor-core/session-snapshot.js +1 -1
  394. package/vendor-core/shs-fetch.js +4 -6
  395. package/vendor-core/shs-load.js +9 -13
  396. package/vendor-core/shs-request.js +1 -1
  397. package/vendor-core/stage-quantiles.js +14 -0
  398. package/vendor-core/types.js +78 -18
  399. package/vendor-core/wasted-core-hours.js +7 -12
  400. package/public/assets/AutoscalingChurn-CP5OjN6v.js +0 -1
  401. package/public/assets/CacheUtilization-BjyawXL6.js +0 -1
  402. package/public/assets/CachingOpportunity-DV2ABG2G.js +0 -1
  403. package/public/assets/ConfigAudit-DMiBhAoG.js +0 -1
  404. package/public/assets/CoreUsageArea-CGYIgPnt.js +0 -1
  405. package/public/assets/Dashboard-D1_IrWRs.js +0 -12
  406. package/public/assets/ExecutorTimeline-DkaAGqQT.js +0 -1
  407. package/public/assets/Failures-Crngoicu.js +0 -1
  408. package/public/assets/GcPressure-CFXm4BnA.js +0 -1
  409. package/public/assets/IncompleteRun-BWEJTb13.js +0 -1
  410. package/public/assets/JobFailures-BmjxCA-h.js +0 -1
  411. package/public/assets/MemoryUtilization-Bu2uQJbd.js +0 -1
  412. package/public/assets/PlanExplorer-DKUwF0uz.js +0 -1
  413. package/public/assets/PlanGraphRoute-BjPSugIG.js +0 -1
  414. package/public/assets/ShuffleIO--Rur3neu.js +0 -1
  415. package/public/assets/Spill-DIMpt8c2.js +0 -1
  416. package/public/assets/TaskSkew-BDr1DbeI.js +0 -1
  417. package/public/assets/charts-vendor-NMqKV0H4.js +0 -59
  418. package/public/assets/index-4id52G-A.js +0 -9
  419. package/public/assets/index-D9kwVYIO.css +0 -1
  420. package/public/assets/parser-worker-00O9WEQC.js +0 -65
  421. package/public/assets/plan-graph-vendor-BnuhLJ6X.css +0 -1
  422. package/public/assets/plan-graph-vendor-vZQHG2EJ.js +0 -7
  423. package/public/assets/react-vendor-u4pyvC5Y.js +0 -9
  424. package/public/assets/useRecentFiles-46iBU9zJ.js +0 -5
  425. package/public/docs/assets/chunks/@localSearchIndexroot.DclgKtlT.js +0 -1
  426. package/public/docs/assets/chunks/theme.CoT6y5bf.js +0 -2
  427. package/public/docs/assets/contributor-guide_architecture_board-widgets.md.BtjoE_1k.js +0 -1
  428. package/public/docs/assets/contributor-guide_architecture_board-widgets.md.BtjoE_1k.lean.js +0 -1
  429. package/public/docs/assets/contributor-guide_architecture_detector-contract.md.DQyCHQNM.js +0 -1
  430. package/public/docs/assets/contributor-guide_architecture_detector-contract.md.DQyCHQNM.lean.js +0 -1
  431. package/public/docs/assets/contributor-guide_architecture_drill-down.md.BKfyDQck.js +0 -1
  432. package/public/docs/assets/contributor-guide_architecture_drill-down.md.BKfyDQck.lean.js +0 -1
  433. package/public/docs/assets/contributor-guide_architecture_state-and-history.md.DaqttKgQ.js +0 -1
  434. package/public/docs/assets/contributor-guide_architecture_state-and-history.md.DaqttKgQ.lean.js +0 -1
  435. package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.Bk6Iyj9S.js +0 -1
  436. package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.Bk6Iyj9S.lean.js +0 -1
  437. package/public/docs/assets/contributor-guide_contributing.md.C-lKs70I.js +0 -1
  438. package/public/docs/assets/contributor-guide_contributing.md.C-lKs70I.lean.js +0 -1
  439. package/public/docs/assets/user-guide_getting-started.md.BJdD7a7_.js +0 -3
  440. package/public/docs/assets/user-guide_getting-started.md.BJdD7a7_.lean.js +0 -1
  441. package/public/docs/assets/user-guide_run-comparison.md.5-oaOfTu.js +0 -1
  442. package/public/docs/assets/user-guide_run-comparison.md.5-oaOfTu.lean.js +0 -1
  443. package/public/docs/assets/user-guide_understanding-findings.md.CK2WZ0Tf.js +0 -1
  444. package/public/docs/assets/user-guide_understanding-findings.md.CK2WZ0Tf.lean.js +0 -1
  445. package/public/vendor/spark-doc/anchors.json +0 -205
  446. package/public/vendor/spark-doc/index.html +0 -4202
  447. package/public/vendor/spark-doc/landing.html +0 -418
  448. package/public/vendor/spark-doc/meta.html +0 -970
package/public/docs/tuning-reference/bottleneck-cold-start.html ADDED
@@ -0,0 +1,35 @@
1
+ <!DOCTYPE html>
2
+ <html lang="en-US" dir="ltr">
3
+ <head>
4
+ <meta charset="utf-8">
5
+ <meta name="viewport" content="width=device-width,initial-scale=1">
6
+ <title>Cold Start | SparkForensics</title>
7
+ <meta name="description" content="Docs for using and contributing to SparkForensics">
8
+ <meta name="generator" content="VitePress v1.6.4">
9
+ <link rel="preload stylesheet" href="/docs/assets/style.DSixAiZE.css" as="style">
10
+ <link rel="preload stylesheet" href="/docs/vp-icons.css" as="style">
11
+
12
+ <script type="module" src="/docs/assets/app.APoq_UBs.js"></script>
13
+ <link rel="preload" href="/docs/assets/inter-roman-latin.Di8DUHzh.woff2" as="font" type="font/woff2" crossorigin="">
14
+ <link rel="modulepreload" href="/docs/assets/chunks/theme.PavXO_vz.js">
15
+ <link rel="modulepreload" href="/docs/assets/chunks/framework.DSg0KOwT.js">
16
+ <link rel="modulepreload" href="/docs/assets/tuning-reference_bottleneck-cold-start.md.CEuy-72y.lean.js">
17
+ <link rel="icon" type="image/svg+xml" href="/docs/favicon.svg">
18
+ <link rel="preconnect" href="https://fonts.googleapis.com">
19
+ <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin="">
20
+ <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Recursive:wght,CASL@400..700,0..1&amp;family=JetBrains+Mono:wght@400;500;600&amp;display=swap">
21
+ <script id="check-dark-mode">(()=>{const e=localStorage.getItem("vitepress-theme-appearance")||"auto",a=window.matchMedia("(prefers-color-scheme: dark)").matches;(!e||e==="auto"?a:e==="dark")&&document.documentElement.classList.add("dark")})();</script>
22
+ <script id="check-mac-os">document.documentElement.classList.toggle("mac",/Mac|iPhone|iPod|iPad/i.test(navigator.platform));</script>
23
+ </head>
24
+ <body>
25
+ <div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Getting Started</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/intro.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Introduction</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Optimization Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/spark-architecture.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Execution Model</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/memory-model.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Management</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/partitioning.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Partitioning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/joins.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Join Optimization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/data-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Data Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/table-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Table Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/caching.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Caching & Persistence</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/pyspark.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>PySpark Specifics</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/aqe.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Adaptive Query Execution</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/cluster-config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cluster Tuning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/anti-patterns.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Anti-Patterns</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Detector Catalog</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-skew.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Skew</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle I/O</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-spill.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory / Disk Spill</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-gc.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>GC Pressure</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-cold-start.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cold Start</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Executor Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-slow-host.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Slow Host</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-failures.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Failures</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-straggler.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Stragglers</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-retry-waste.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Retry Waste</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-tiny-tasks.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Tiny Tasks</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-job-failure-rate.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Job Failure Rate</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-memory-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Duplicate Plan Subtree</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-small-files.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Small Files</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-broadcast-sizing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Broadcast Sizing</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Reference</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/metrics.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Metrics Glossary</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Config Quick-Reference</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_tuning-reference_bottleneck-cold-start" data-v-39a288b8><div><h1 id="bottleneck-cold-start" tabindex="-1">Cold Start <a class="header-anchor" href="#bottleneck-cold-start" aria-label="Permalink to &quot;Cold Start {#bottleneck-cold-start}&quot;">​</a></h1><p><span class="tag">COLD</span></p><h2 id="what-it-is" tabindex="-1">What it is <a class="header-anchor" href="#what-it-is" aria-label="Permalink to &quot;What it is&quot;">​</a></h2><p>Cold start is the gap between when a Spark application starts and when it actually begins doing work: the delay before the <a href="./spark-architecture.html">DAGScheduler</a> hands off the first stage for execution. In the event log, that moment is marked by <code>SparkListenerStageSubmitted</code>, whose payload is a direct serialization of a <code>StageInfo</code>: the event name plus the <code>stageInfo</code> and any properties, with no task-level data attached<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup>. That shape matches what the event records. The DAGScheduler doesn&#39;t pre-schedule the whole DAG upfront (it reacts to stage completions, unlocking child stages one at a time), and once it has carved the DAG into stages and handed a <code>TaskSet</code> for the next stage to the TaskScheduler, only then does the TaskScheduler assign individual tasks to executors<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup>. <code>SparkListenerStageSubmitted</code> therefore captures the moment the DAGScheduler creates/submits that <code>TaskSet</code>, before any task has actually been placed on an executor.</p><h2 id="how-it-s-detected" tabindex="-1">How it&#39;s detected <a class="header-anchor" href="#how-it-s-detected" aria-label="Permalink to &quot;How it&#39;s detected&quot;">​</a></h2><p><code>firstStageSubmittedAt − app.startTime</code> &gt; 30 s → Warning. <code>firstStageSubmittedAt</code> is a synthetic field: the timestamp of the first <code>SparkListenerStageSubmitted</code> event in the event log.</p><img class="light-only" src="/docs/assets/cold-start-timeline.DxC_Sc7w.svg" alt="A timeline from app.startTime through the gap where executors are acquired and no tasks run to firstStageSubmittedAt, with that gap marked as the cold-start delay."><img class="dark-only" src="/docs/assets/cold-start-timeline.dark.CZ17YcAG.svg" alt="A timeline from app.startTime through the gap where executors are acquired and no tasks run to firstStageSubmittedAt, with that gap marked as the cold-start delay."><h2 id="why-it-matters" tabindex="-1">Why it matters <a class="header-anchor" href="#why-it-matters" aria-label="Permalink to &quot;Why it matters&quot;">​</a></h2><p>Because the event carries only stage-level metadata and fires ahead of any executor task placement<sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup><sup class="footnote-ref"><a href="#fn2" id="fnref2:1">[2:1]</a></sup>, a large gap here reflects time spent before the DAGScheduler could submit work at all, not time spent inside tasks. It&#39;s a signal about scheduling/startup latency, distinct from the per-task metrics that describe work once tasks are actually running on executors.</p><h2 id="how-to-fix-it" tabindex="-1">How to fix it <a class="header-anchor" href="#how-to-fix-it" aria-label="Permalink to &quot;How to fix it&quot;">​</a></h2><p>Dynamic-allocation ramp-up isn&#39;t directly tied to <code>firstStageSubmittedAt</code> timing, but the configuration below governs how many executors an application has available at launch, and how reliably that number can grow:</p><ul><li>Size executors deliberately instead of defaulting to one large executor per node. For a 10-node, 16-core/64GB cluster targeting ~5 cores per executor, the standard derivation reserves 1 core per node for daemons (15 usable cores/node, 150 total), divides by 5 cores/executor (30), and subtracts 1 for the YARN ApplicationMaster: 29 executors × 5 cores × ~18GB<sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup>. Cloudera&#39;s equivalent 6-node walkthrough lands on the same shape: 17 executors × 5 cores × 19G, rather than one fat executor per node<sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup>.</li><li>If dynamic allocation is enabled, its ability to add executors at all depends on one of a few supporting mechanisms being turned on. One such mechanism is <code>spark.dynamicAllocation.shuffleTracking.enabled</code> (default <code>true</code> since Spark 3.0), which &quot;enables shuffle file tracking for executors, which allows dynamic allocation without the need for an external shuffle service,&quot; and &quot;will try to keep alive executors that are storing shuffle data for active jobs&quot;<sup class="footnote-ref"><a href="#fn5" id="fnref5">[5]</a></sup>. Without shuffle tracking or an external shuffle service configured, dynamic allocation cannot be enabled at all<sup class="footnote-ref"><a href="#fn5" id="fnref5:1">[5:1]</a></sup>.</li></ul><p>Deliberate executor sizing plus the dynamic-allocation precondition (values from the 16-core/64GB-node derivation above):</p><div class="language-properties vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">properties</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># ~5 cores per executor instead of one fat executor per node</span></span>
26
+ <span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.executor.cores</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=5</span></span>
27
+ <span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.executor.memory</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=18g</span></span>
28
+ <span class="line"></span>
29
+ <span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Dynamic allocation, with its required shuffle-tracking precondition (default true since Spark 3.0)</span></span>
30
+ <span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.dynamicAllocation.enabled</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=true</span></span>
31
+ <span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.dynamicAllocation.shuffleTracking.enabled</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=true</span></span></code></pre></div><h2 id="limitations-false-positive-risk" tabindex="-1">Limitations / false-positive risk <a class="header-anchor" href="#limitations-false-positive-risk" aria-label="Permalink to &quot;Limitations / false-positive risk&quot;">​</a></h2><p>Some startup latency is unavoidable: a cluster still has to provision before it can run anything. The signal measures the gap before the first stage is submitted, and that gap can legitimately vary by cluster manager, so a warning here doesn&#39;t always mean something is wrong.</p><h2 id="related" tabindex="-1">Related <a class="header-anchor" href="#related" aria-label="Permalink to &quot;Related&quot;">​</a></h2><ul><li><strong>Executor sizing &amp; dynamic allocation:</strong> <a href="./cluster-config.html">Cluster Tuning</a></li></ul><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><a href="https://raw.githubusercontent.com/apache/spark/v3.5.0/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala" target="_blank" rel="noreferrer">JsonProtocol.scala</a> <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><a href="https://luminousmen.com/post/spark-anatomy-of-spark-application" target="_blank" rel="noreferrer">Anatomy of a Spark Application</a> <a href="#fnref2" class="footnote-backref">↩︎</a> <a href="#fnref2:1" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><a href="https://raw.githubusercontent.com/spoddutur/spark-notes/master/distribution_of_executors_cores_and_memory_for_spark_application.md" target="_blank" rel="noreferrer">Distribution of Executors, Cores and Memory for a Spark Application</a> <a href="#fnref3" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><a href="https://blog.cloudera.com/how-to-tune-your-apache-spark-jobs-part-2/" target="_blank" rel="noreferrer">How to Tune Your Apache Spark Jobs (Part 2)</a> <a href="#fnref4" class="footnote-backref">↩︎</a></p></li><li id="fn5" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration (Spark)</a> <a href="#fnref5" class="footnote-backref">↩︎</a> <a href="#fnref5:1" class="footnote-backref">↩︎</a></p></li></ol></section></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/tuning-reference/bottleneck-gc.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>GC Pressure</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="/docs/tuning-reference/bottleneck-utilization.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>Executor Utilization</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
32
+ <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"CWpj01WU\",\"contributor-guide_architecture_detector-contract.md\":\"CgzUsQ6W\",\"contributor-guide_architecture_drill-down.md\":\"BtPdlM7r\",\"contributor-guide_architecture_impact-estimation.md\":\"CooslVJt\",\"contributor-guide_architecture_index.md\":\"3TO9ic6w\",\"contributor-guide_architecture_overview.md\":\"CehiRmGn\",\"contributor-guide_architecture_state-and-history.md\":\"C-xxn0q7\",\"contributor-guide_architecture_widget-rendering.md\":\"R27gQrgY\",\"contributor-guide_architecture_worker-protocol.md\":\"IbnfNrV3\",\"contributor-guide_contributing.md\":\"CvRsdr6J\",\"contributor-guide_development-setup.md\":\"DvAN_9mK\",\"contributor-guide_testing.md\":\"6rIKqSyY\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"Df1YMIHu\",\"tuning-reference_aqe.md\":\"BIsCtLzm\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"CEstB3Ia\",\"tuning-reference_bottleneck-cold-start.md\":\"CEuy-72y\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"CIohQDfn\",\"tuning-reference_bottleneck-failures.md\":\"4z5BXGJ2\",\"tuning-reference_bottleneck-gc.md\":\"DSxzZRK7\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"BaJl__1W\",\"tuning-reference_bottleneck-memory-utilization.md\":\"DbP-SJZc\",\"tuning-reference_bottleneck-retry-waste.md\":\"D5JMjVOt\",\"tuning-reference_bottleneck-shuffle.md\":\"CM-nTmIH\",\"tuning-reference_bottleneck-skew.md\":\"BdUwiDhn\",\"tuning-reference_bottleneck-slow-host.md\":\"BlIo6UDW\",\"tuning-reference_bottleneck-small-files.md\":\"B8kloyx8\",\"tuning-reference_bottleneck-spill.md\":\"PNH7mITt\",\"tuning-reference_bottleneck-straggler.md\":\"DY36fHN5\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"QTV7O8kU\",\"tuning-reference_bottleneck-utilization.md\":\"DTiueZC3\",\"tuning-reference_caching.md\":\"B7aQ8asB\",\"tuning-reference_cluster-config.md\":\"ZVmDGsQ3\",\"tuning-reference_config.md\":\"UvveiWG3\",\"tuning-reference_data-formats.md\":\"bjCAWH3N\",\"tuning-reference_index.md\":\"BQ_NooMV\",\"tuning-reference_intro.md\":\"CobD-lGB\",\"tuning-reference_joins.md\":\"BtKs_CuW\",\"tuning-reference_memory-model.md\":\"DhT-n4y3\",\"tuning-reference_metrics.md\":\"mLOh7Apj\",\"tuning-reference_partitioning.md\":\"q0zKF_8X\",\"tuning-reference_pyspark.md\":\"DDCfvN9t\",\"tuning-reference_shuffle.md\":\"BZZ7R4Ix\",\"tuning-reference_spark-architecture.md\":\"Dwzm5avO\",\"tuning-reference_table-formats.md\":\"D6wj-2dX\",\"user-guide_alternative-log-retrieval.md\":\"B4tPGIal\",\"user-guide_getting-started.md\":\"BJvwLEIM\",\"user-guide_mcp-tools.md\":\"Vi3RoflJ\",\"user-guide_run-comparison.md\":\"CQc1aoU8\",\"user-guide_understanding-findings.md\":\"DL1UDhvR\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Alternative log retrieval\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
33
+
34
+ </body>
35
+ </html>
package/public/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html ADDED
@@ -0,0 +1,30 @@
1
+ <!DOCTYPE html>
2
+ <html lang="en-US" dir="ltr">
3
+ <head>
4
+ <meta charset="utf-8">
5
+ <meta name="viewport" content="width=device-width,initial-scale=1">
6
+ <title>Duplicate Plan Subtree | SparkForensics</title>
7
+ <meta name="description" content="Docs for using and contributing to SparkForensics">
8
+ <meta name="generator" content="VitePress v1.6.4">
9
+ <link rel="preload stylesheet" href="/docs/assets/style.DSixAiZE.css" as="style">
10
+ <link rel="preload stylesheet" href="/docs/vp-icons.css" as="style">
11
+
12
+ <script type="module" src="/docs/assets/app.APoq_UBs.js"></script>
13
+ <link rel="preload" href="/docs/assets/inter-roman-latin.Di8DUHzh.woff2" as="font" type="font/woff2" crossorigin="">
14
+ <link rel="modulepreload" href="/docs/assets/chunks/theme.PavXO_vz.js">
15
+ <link rel="modulepreload" href="/docs/assets/chunks/framework.DSg0KOwT.js">
16
+ <link rel="modulepreload" href="/docs/assets/chunks/duplicate-plan-subtree.dark.Cdp70QhV.js">
17
+ <link rel="modulepreload" href="/docs/assets/tuning-reference_bottleneck-duplicate-plan-subtree.md.CIohQDfn.lean.js">
18
+ <link rel="icon" type="image/svg+xml" href="/docs/favicon.svg">
19
+ <link rel="preconnect" href="https://fonts.googleapis.com">
20
+ <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin="">
21
+ <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Recursive:wght,CASL@400..700,0..1&amp;family=JetBrains+Mono:wght@400;500;600&amp;display=swap">
22
+ <script id="check-dark-mode">(()=>{const e=localStorage.getItem("vitepress-theme-appearance")||"auto",a=window.matchMedia("(prefers-color-scheme: dark)").matches;(!e||e==="auto"?a:e==="dark")&&document.documentElement.classList.add("dark")})();</script>
23
+ <script id="check-mac-os">document.documentElement.classList.toggle("mac",/Mac|iPhone|iPod|iPad/i.test(navigator.platform));</script>
24
+ </head>
25
+ <body>
26
+ <div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Getting Started</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/intro.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Introduction</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Optimization Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/spark-architecture.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Execution Model</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/memory-model.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Management</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/partitioning.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Partitioning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/joins.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Join Optimization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/data-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Data Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/table-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Table Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/caching.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Caching & Persistence</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/pyspark.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>PySpark Specifics</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/aqe.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Adaptive Query Execution</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/cluster-config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cluster Tuning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/anti-patterns.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Anti-Patterns</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Detector Catalog</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-skew.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Skew</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle I/O</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-spill.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory / Disk Spill</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-gc.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>GC Pressure</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-cold-start.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cold Start</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Executor Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-slow-host.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Slow Host</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-failures.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Failures</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-straggler.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Stragglers</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-retry-waste.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Retry Waste</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-tiny-tasks.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Tiny Tasks</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-job-failure-rate.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Job Failure Rate</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-memory-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Duplicate Plan Subtree</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-small-files.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Small Files</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-broadcast-sizing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Broadcast Sizing</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Reference</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/metrics.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Metrics Glossary</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Config Quick-Reference</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_tuning-reference_bottleneck-duplicate-plan-subtree" data-v-39a288b8><div><h1 id="bottleneck-duplicate-plan-subtree" tabindex="-1">Duplicate Plan Subtree <a class="header-anchor" href="#bottleneck-duplicate-plan-subtree" aria-label="Permalink to &quot;Duplicate Plan Subtree {#bottleneck-duplicate-plan-subtree}&quot;">​</a></h1><p><span class="tag">DUPPLAN</span></p><h2 id="what-it-is" tabindex="-1">What it is <a class="header-anchor" href="#what-it-is" aria-label="Permalink to &quot;What it is&quot;">​</a></h2><p>The physical plan is the form Spark actually runs: after optimizing the logical plan, the engine produces a plan that &quot;specifies how the logical plan will execute on the cluster,&quot; compiling the query into a series of RDDs and transformations<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup>. A duplicated subtree is the same source read, together with the operators stacked above it, appearing in more than one branch of that plan. When the same logic sits in two different branches of a pipeline, you are &quot;basically signing up to recompute everything from scratch multiple times,&quot; because every action on an uncached DataFrame starts back at the source and rebuilds all the intermediate steps<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup>.</p><h2 id="how-it-s-detected" tabindex="-1">How it&#39;s detected <a class="header-anchor" href="#how-it-s-detected" aria-label="Permalink to &quot;How it&#39;s detected&quot;">​</a></h2><p>Exchanges are the giveaway. A <a href="./shuffle.html">shuffle</a> or broadcast is an exchange, and these are the points where Spark&#39;s pipelining stops: &quot;Spark operators are often pipelined and executed in parallel processes. However, a shuffle or broadcast exchange breaks this pipeline. We call them materialization points and use the term &#39;query stages&#39; to denote subsections bounded by these materialization points&quot;<sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup>. The signal is the same source-plus-operators subtree feeding more than one branch across those materialization boundaries, which is the shape that gets recomputed unless something intervenes.</p><img class="light-only" src="/docs/assets/duplicate-plan-subtree.B4cvN6fj.svg" alt="Before and after: the same scan and operators feed two branches and run twice, then a single shared node from exchange reuse or a cache feeds both consumers."><img class="dark-only" src="/docs/assets/duplicate-plan-subtree.dark.Dw8wS0Ag.svg" alt="Before and after: the same scan and operators feed two branches and run twice, then a single shared node from exchange reuse or a cache feeds both consumers."><table tabindex="0"><thead><tr><th>Signal</th><th>What it points to</th></tr></thead><tbody><tr><td>Identical scan + operators in two or more branches</td><td>Same subtree recomputed per branch</td></tr><tr><td>A materialization point (shuffle/broadcast) below the repeat</td><td>The pipeline breaks and the branch restarts from source</td></tr></tbody></table><h2 id="why-it-matters" tabindex="-1">Why it matters <a class="header-anchor" href="#why-it-matters" aria-label="Permalink to &quot;Why it matters&quot;">​</a></h2><p>Left alone, a repeated subtree is repeated work. Each branch re-reads the source and re-runs everything above it, so a subtree that shows up twice is roughly paid for twice<sup class="footnote-ref"><a href="#fn2" id="fnref2:1">[2:1]</a></sup>. That cost lands on exactly the expensive parts of a plan (scans and shuffles) rather than on cheap row-level operators.</p><h2 id="how-to-fix-it" tabindex="-1">How to fix it <a class="header-anchor" href="#how-to-fix-it" aria-label="Permalink to &quot;How to fix it&quot;">​</a></h2><p>Two mechanisms let a plan reuse an identical subtree instead of rebuilding it.</p><ul><li>Exchange reuse leans on the fact that a shuffle already writes its output to disk. Spark &quot;always executes shuffles by first having the &#39;source&#39; tasks ... write shuffle files to their local disks,&quot; so &quot;running a new job over data that&#39;s already been shuffled does not rerun the &#39;source&#39; side of the shuffle. Because the shuffle files were already written to disk earlier, Spark knows that it can use them to run the later stages of the job, and it need not redo the earlier ones. In the Spark UI and logs, you will see the pre-shuffle stages marked as &#39;skipped&#39;&quot;<sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup>. When the same materialized exchange feeds two consumers, the second reads those existing shuffle files instead of recomputing the subtree beneath it.</li><li><code>cache()</code> / <code>persist()</code> truncate recomputation by materializing the result and keying future lookups on it. Persisting &quot;means materializing an RDD (usually by storing it in memory on the executors) for reuse during the current job&quot;<sup class="footnote-ref"><a href="#fn2" id="fnref2:2">[2:2]</a></sup>, and in the Structured API the lookup is plan-keyed: &quot;caching is done based on the physical plan. This means that we effectively store the physical plan as our key ... and perform a lookup prior to the execution of a Structured job&quot;<sup class="footnote-ref"><a href="#fn1" id="fnref1:2">[1:2]</a></sup>. On a hit, Spark returns the stored data rather than rebuilding intermediate steps from the source<sup class="footnote-ref"><a href="#fn2" id="fnref2:3">[2:3]</a></sup>.</li></ul><blockquote><p><strong>PySpark:</strong> if the repeated subtree is a DataFrame you reuse across branches, <code>df.cache()</code> (or <code>df.persist()</code>) before the branches split lets both read the materialized result instead of recomputing it.</p></blockquote><h2 id="confidence" tabindex="-1">Confidence <a class="header-anchor" href="#confidence" aria-label="Permalink to &quot;Confidence&quot;">​</a></h2><p>Medium. Inferring a duplicated subtree from the static plan is a medium-confidence signal that requires runtime validation: whether a repeated subtree actually turns into avoided work depends on runtime facts the plan never shows, such as cache hits, retained shuffle files, and eviction<sup class="footnote-ref"><a href="#fn2" id="fnref2:4">[2:4]</a></sup>.</p><h2 id="limitations-false-positive-risk" tabindex="-1">Limitations / false-positive risk <a class="header-anchor" href="#limitations-false-positive-risk" aria-label="Permalink to &quot;Limitations / false-positive risk&quot;">​</a></h2><p>Two subtrees that look identical in the printed plan are not always the same recomputed work. The cache &quot;is tied to the analyzed logical plan, not the final optimized one,&quot; so semantically identical queries with different analyzed plans miss the cache and recompute even though &quot;the optimizer will eventually produce the same physical plan&quot;<sup class="footnote-ref"><a href="#fn2" id="fnref2:5">[2:5]</a></sup>. <a href="./caching.html">Caching</a> is also lazy and often partial: it happens &quot;only as they are accessed,&quot; &quot;the only blocks that get cached are the ones Spark was forced to touch,&quot; and Spark gives &quot;no built-in visibility into how much of your DataFrame is actually cached, or how much has already been evicted&quot;<sup class="footnote-ref"><a href="#fn2" id="fnref2:6">[2:6]</a></sup>. Plan-level inference can therefore over-count, flagging repetition that either never materializes or is only partly reused.</p><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><a href="https://www.oreilly.com/library/view/spark-the-definitive/9781491912201/" target="_blank" rel="noreferrer">Spark: The Definitive Guide</a>, Chambers &amp; Zaharia, chs. 4, 15, 19 <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a> <a href="#fnref1:2" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><a href="https://luminousmen.com/post/explaining-the-mechanics-of-spark-caching" target="_blank" rel="noreferrer">Explaining the mechanics of Spark caching</a> <a href="#fnref2" class="footnote-backref">↩︎</a> <a href="#fnref2:1" class="footnote-backref">↩︎</a> <a href="#fnref2:2" class="footnote-backref">↩︎</a> <a href="#fnref2:3" class="footnote-backref">↩︎</a> <a href="#fnref2:4" class="footnote-backref">↩︎</a> <a href="#fnref2:5" class="footnote-backref">↩︎</a> <a href="#fnref2:6" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><a href="https://www.databricks.com/blog/2020/05/29/adaptive-query-execution-speeding-up-spark-sql-at-runtime.html" target="_blank" rel="noreferrer">Adaptive Query Execution: Speeding Up Spark SQL at Runtime</a> <a href="#fnref3" class="footnote-backref">↩︎</a></p></li></ol></section></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/tuning-reference/bottleneck-memory-utilization.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>Memory Utilization</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="/docs/tuning-reference/bottleneck-small-files.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>Small Files</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
27
+ <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"CWpj01WU\",\"contributor-guide_architecture_detector-contract.md\":\"CgzUsQ6W\",\"contributor-guide_architecture_drill-down.md\":\"BtPdlM7r\",\"contributor-guide_architecture_impact-estimation.md\":\"CooslVJt\",\"contributor-guide_architecture_index.md\":\"3TO9ic6w\",\"contributor-guide_architecture_overview.md\":\"CehiRmGn\",\"contributor-guide_architecture_state-and-history.md\":\"C-xxn0q7\",\"contributor-guide_architecture_widget-rendering.md\":\"R27gQrgY\",\"contributor-guide_architecture_worker-protocol.md\":\"IbnfNrV3\",\"contributor-guide_contributing.md\":\"CvRsdr6J\",\"contributor-guide_development-setup.md\":\"DvAN_9mK\",\"contributor-guide_testing.md\":\"6rIKqSyY\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"Df1YMIHu\",\"tuning-reference_aqe.md\":\"BIsCtLzm\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"CEstB3Ia\",\"tuning-reference_bottleneck-cold-start.md\":\"CEuy-72y\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"CIohQDfn\",\"tuning-reference_bottleneck-failures.md\":\"4z5BXGJ2\",\"tuning-reference_bottleneck-gc.md\":\"DSxzZRK7\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"BaJl__1W\",\"tuning-reference_bottleneck-memory-utilization.md\":\"DbP-SJZc\",\"tuning-reference_bottleneck-retry-waste.md\":\"D5JMjVOt\",\"tuning-reference_bottleneck-shuffle.md\":\"CM-nTmIH\",\"tuning-reference_bottleneck-skew.md\":\"BdUwiDhn\",\"tuning-reference_bottleneck-slow-host.md\":\"BlIo6UDW\",\"tuning-reference_bottleneck-small-files.md\":\"B8kloyx8\",\"tuning-reference_bottleneck-spill.md\":\"PNH7mITt\",\"tuning-reference_bottleneck-straggler.md\":\"DY36fHN5\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"QTV7O8kU\",\"tuning-reference_bottleneck-utilization.md\":\"DTiueZC3\",\"tuning-reference_caching.md\":\"B7aQ8asB\",\"tuning-reference_cluster-config.md\":\"ZVmDGsQ3\",\"tuning-reference_config.md\":\"UvveiWG3\",\"tuning-reference_data-formats.md\":\"bjCAWH3N\",\"tuning-reference_index.md\":\"BQ_NooMV\",\"tuning-reference_intro.md\":\"CobD-lGB\",\"tuning-reference_joins.md\":\"BtKs_CuW\",\"tuning-reference_memory-model.md\":\"DhT-n4y3\",\"tuning-reference_metrics.md\":\"mLOh7Apj\",\"tuning-reference_partitioning.md\":\"q0zKF_8X\",\"tuning-reference_pyspark.md\":\"DDCfvN9t\",\"tuning-reference_shuffle.md\":\"BZZ7R4Ix\",\"tuning-reference_spark-architecture.md\":\"Dwzm5avO\",\"tuning-reference_table-formats.md\":\"D6wj-2dX\",\"user-guide_alternative-log-retrieval.md\":\"B4tPGIal\",\"user-guide_getting-started.md\":\"BJvwLEIM\",\"user-guide_mcp-tools.md\":\"Vi3RoflJ\",\"user-guide_run-comparison.md\":\"CQc1aoU8\",\"user-guide_understanding-findings.md\":\"DL1UDhvR\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Alternative log retrieval\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
28
+
29
+ </body>
30
+ </html>
package/public/docs/tuning-reference/bottleneck-failures.html ADDED
@@ -0,0 +1,35 @@
1
+ <!DOCTYPE html>
2
+ <html lang="en-US" dir="ltr">
3
+ <head>
4
+ <meta charset="utf-8">
5
+ <meta name="viewport" content="width=device-width,initial-scale=1">
6
+ <title>Task Failures | SparkForensics</title>
7
+ <meta name="description" content="Docs for using and contributing to SparkForensics">
8
+ <meta name="generator" content="VitePress v1.6.4">
9
+ <link rel="preload stylesheet" href="/docs/assets/style.DSixAiZE.css" as="style">
10
+ <link rel="preload stylesheet" href="/docs/vp-icons.css" as="style">
11
+
12
+ <script type="module" src="/docs/assets/app.APoq_UBs.js"></script>
13
+ <link rel="preload" href="/docs/assets/inter-roman-latin.Di8DUHzh.woff2" as="font" type="font/woff2" crossorigin="">
14
+ <link rel="modulepreload" href="/docs/assets/chunks/theme.PavXO_vz.js">
15
+ <link rel="modulepreload" href="/docs/assets/chunks/framework.DSg0KOwT.js">
16
+ <link rel="modulepreload" href="/docs/assets/chunks/retry-escalation-ladder.dark.DHipdJgZ.js">
17
+ <link rel="modulepreload" href="/docs/assets/tuning-reference_bottleneck-failures.md.4z5BXGJ2.lean.js">
18
+ <link rel="icon" type="image/svg+xml" href="/docs/favicon.svg">
19
+ <link rel="preconnect" href="https://fonts.googleapis.com">
20
+ <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin="">
21
+ <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Recursive:wght,CASL@400..700,0..1&amp;family=JetBrains+Mono:wght@400;500;600&amp;display=swap">
22
+ <script id="check-dark-mode">(()=>{const e=localStorage.getItem("vitepress-theme-appearance")||"auto",a=window.matchMedia("(prefers-color-scheme: dark)").matches;(!e||e==="auto"?a:e==="dark")&&document.documentElement.classList.add("dark")})();</script>
23
+ <script id="check-mac-os">document.documentElement.classList.toggle("mac",/Mac|iPhone|iPod|iPad/i.test(navigator.platform));</script>
24
+ </head>
25
+ <body>
26
+ <div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Getting Started</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/intro.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Introduction</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Optimization Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/spark-architecture.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Execution Model</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/memory-model.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Management</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/partitioning.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Partitioning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/joins.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Join Optimization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/data-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Data Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/table-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Table Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/caching.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Caching & Persistence</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/pyspark.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>PySpark Specifics</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/aqe.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Adaptive Query Execution</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/cluster-config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cluster Tuning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/anti-patterns.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Anti-Patterns</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Detector Catalog</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-skew.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Skew</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle I/O</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-spill.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory / Disk Spill</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-gc.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>GC Pressure</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-cold-start.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cold Start</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Executor Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-slow-host.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Slow Host</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-failures.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Failures</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-straggler.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Stragglers</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-retry-waste.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Retry Waste</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-tiny-tasks.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Tiny Tasks</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-job-failure-rate.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Job Failure Rate</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-memory-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Duplicate Plan Subtree</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-small-files.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Small Files</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-broadcast-sizing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Broadcast Sizing</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Reference</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/metrics.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Metrics Glossary</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Config Quick-Reference</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_tuning-reference_bottleneck-failures" data-v-39a288b8><div><h1 id="bottleneck-failures" tabindex="-1">Task Failures <a class="header-anchor" href="#bottleneck-failures" aria-label="Permalink to &quot;Task Failures {#bottleneck-failures}&quot;">​</a></h1><p><span class="tag">FAIL</span></p><h2 id="what-it-is" tabindex="-1">What it is <a class="header-anchor" href="#what-it-is" aria-label="Permalink to &quot;What it is&quot;">​</a></h2><p>A task failure is any task end whose <code>Reason</code> field is something other than success. Spark&#39;s event log serializes this as the formatted class name of whichever <code>TaskEndReason</code> instance was assigned to that task attempt<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup>, and the canonical set of reasons is <code>Success</code>, <code>FetchFailed</code>, <code>ExceptionFailure</code>, <code>TaskResultLost</code>, <code>TaskKilled</code>, <code>TaskCommitDenied</code>, <code>ExecutorLostFailure</code>, <code>Resubmitted</code>, and <code>UnknownReason</code><sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup>. Each carries its own extra detail: a <code>FetchFailed</code> records a reduce ID and message; an <code>ExceptionFailure</code> carries the exception&#39;s class name, description, and stack trace, plus accumulator updates (falling back to reading them out of the task&#39;s metrics object for logs written by Spark 1.x); a <code>TaskKilled</code> records a kill reason; a <code>TaskCommitDenied</code> records the job ID, partition ID, and attempt number; an <code>ExecutorLostFailure</code> records whether the loss was caused by the application, the executor ID, and a loss reason; <code>UnknownReason</code> carries no extra fields<sup class="footnote-ref"><a href="#fn1" id="fnref1:2">[1:2]</a></sup>.</p><h2 id="how-it-s-detected" tabindex="-1">How it&#39;s detected <a class="header-anchor" href="#how-it-s-detected" aria-label="Permalink to &quot;How it&#39;s detected&quot;">​</a></h2><table tabindex="0"><thead><tr><th>failed tasks (share)</th><th>Level</th></tr></thead><tbody><tr><td>&gt; 5%</td><td>Warning</td></tr><tr><td>&gt; 20%</td><td>Critical</td></tr></tbody></table><p>The task-metrics object attached to a <code>SparkListenerTaskEnd</code> event is optional in the schema, so it isn&#39;t guaranteed to be present for every failure. It tends to be populated for reasons where the attempt actually ran far enough to produce metrics (an <code>ExceptionFailure</code> or <code>TaskKilled</code>, for instance) but can be absent for a <code>Resubmitted</code> attempt that never completed on its executor at all<sup class="footnote-ref"><a href="#fn1" id="fnref1:3">[1:3]</a></sup>. Detection has to key off the <code>Reason</code> field itself rather than assume metrics will always be there to inspect.</p><h2 id="why-it-matters" tabindex="-1">Why it matters <a class="header-anchor" href="#why-it-matters" aria-label="Permalink to &quot;Why it matters&quot;">​</a></h2><p>Some reasons point at something worse than a one-off hiccup. An <code>ExecutorLostFailure</code> can be the knock-on effect of a container or pod killed for exceeding its memory grant: enabling off-heap memory without also raising <code>spark.executor.memoryOverhead</code> (or setting <code>spark.executor.pyspark.memory</code> explicitly) lets real usage exceed what YARN or Kubernetes allocated, and the executor is killed with a generic container-memory error rather than a Spark-level exception<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup>. Losing an executor mid-shuffle can also cascade into failures elsewhere: if dynamic allocation removes an executor before a shuffle it wrote data for has completed, that shuffle output is gone, and downstream tasks reading it fail and force an unnecessary recompute<sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup>.</p><h2 id="how-to-fix-it" tabindex="-1">How to fix it <a class="header-anchor" href="#how-to-fix-it" aria-label="Permalink to &quot;How to fix it&quot;">​</a></h2><ul><li>Read the <code>Reason</code> field before treating all failures the same: an <code>ExceptionFailure</code> points at application code, and an <code>ExecutorLostFailure</code> usually points at memory sizing or infrastructure.</li><li>For memory-driven <code>ExecutorLostFailure</code>s, size <code>spark.executor.memoryOverhead</code> (which defaults to <code>max(384 MB, 10% of executor memory)</code>) to actually cover off-heap and PySpark memory, since neither is folded into that default budget automatically<sup class="footnote-ref"><a href="#fn2" id="fnref2:1">[2:1]</a></sup>.</li><li>For failures caused by dynamic allocation reclaiming an executor mid-shuffle, enable the external <a href="./shuffle.html">shuffle service</a> or <code>spark.dynamicAllocation.shuffleTracking.enabled</code> so shuffle output can outlive the executor that wrote it, instead of being lost and recomputed<sup class="footnote-ref"><a href="#fn3" id="fnref3:1">[3:1]</a></sup>.</li></ul><p>Config-only levers for memory-driven <code>ExecutorLostFailure</code>s and mid-shuffle executor loss:</p><div class="language-properties vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">properties</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Cover off-heap + PySpark memory the default overhead budget does NOT include.</span></span>
27
+ <span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Default is max(384m, 10% of executor memory); 2g is an example, size to real off-heap/PySpark use.</span></span>
28
+ <span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.executor.memoryOverhead</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=2g</span></span>
29
+ <span class="line"></span>
30
+ <span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Keep shuffle output alive when dynamic allocation reclaims an executor mid-shuffle</span></span>
31
+ <span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.dynamicAllocation.shuffleTracking.enabled</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=true</span></span></code></pre></div><h2 id="failed-stage" tabindex="-1">Failed stage <a class="header-anchor" href="#failed-stage" aria-label="Permalink to &quot;Failed stage&quot;">​</a></h2><p><span class="tag">SFAIL</span></p><p>A single task failing and a whole stage failing are handled by different parts of the scheduler. Task retries sit in the TaskScheduler; the stage-level verdict belongs to the <a href="./spark-architecture.html">DAGScheduler</a>, which decides whether the job as a whole lives or dies. When a stage fails, it is not abandoned on the spot: the DAGScheduler resubmits it as a fresh attempt, and the job only fails once the stage cannot make progress after repeated attempts<sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup>. So the difference between &quot;retried and succeeds&quot; and &quot;aborts the job&quot; is whether a later attempt finishes before the attempt budget runs out.</p><img class="light-only" src="/docs/assets/retry-escalation-ladder.DyTKJJmZ.svg" alt="The layered budgets a repeated failure climbs through, from task retries to stage resubmission to the executor and application ceilings that abort the job."><img class="dark-only" src="/docs/assets/retry-escalation-ladder.dark.BdsabtU3.svg" alt="The layered budgets a repeated failure climbs through, from task retries to stage resubmission to the executor and application ceilings that abort the job."><p>That budget is <code>spark.stage.maxConsecutiveAttempts</code>, default 4: the number of consecutive stage attempts allowed before the stage is aborted<sup class="footnote-ref"><a href="#fn5" id="fnref5">[5]</a></sup>. Fail, resubmit, and complete within those attempts and the job stays alive; keep failing until the limit is spent and the job aborts.</p><p>A <code>FetchFailed</code> is the classic trigger. It counts against that stage attempt limit, which is why <code>spark.stage.ignoreDecommissionFetchFailure</code> (default <code>true</code> since 3.4.0) exists: it keeps a fetch failure caused by an executor being decommissioned from counting toward <code>spark.stage.maxConsecutiveAttempts</code>, which by implication means ordinary fetch failures otherwise do count toward the abort budget<sup class="footnote-ref"><a href="#fn5" id="fnref5:1">[5:1]</a></sup>.</p><p>Executor exclusion is the other stage-level lever. The <code>spark.excludeOnFailure.*</code> family is off by default (<code>spark.excludeOnFailure.enabled</code> is <code>false</code>) and tracks how many tasks must fail on one executor within a stage before that executor is excluded for the stage; it can also kill executors excluded on a fetch failure<sup class="footnote-ref"><a href="#fn5" id="fnref5:2">[5:2]</a></sup>. If exclusion leaves a TaskSet with no schedulable executor left, <code>spark.scheduler.excludeOnFailure.unschedulableTaskSetTimeout</code> (default 120s) caps how long Spark waits to acquire a new executor before aborting that TaskSet<sup class="footnote-ref"><a href="#fn5" id="fnref5:3">[5:3]</a></sup>.</p><h2 id="confidence" tabindex="-1">Confidence <a class="header-anchor" href="#confidence" aria-label="Permalink to &quot;Confidence&quot;">​</a></h2><p>The failed-task-share thresholds and the stage-failure behaviour both map to documented Spark scheduler mechanics; no branch here is experimental.</p><h2 id="limitations-false-positive-risk" tabindex="-1">Limitations / false-positive risk <a class="header-anchor" href="#limitations-false-positive-risk" aria-label="Permalink to &quot;Limitations / false-positive risk&quot;">​</a></h2><p>A high failed-task share is a blunt signal. It can be dominated by one systemic cause, such as a single lost executor whose tasks all fail at once, rather than many independent failures pointing at a code or data problem. Transient failures that a later attempt retries and succeeds are also not always harmful: task retries reset their count on success, so a job can carry a nonzero failure share and still finish correctly. Read the <code>Reason</code> distribution before treating the share as a defect.</p><h2 id="related" tabindex="-1">Related <a class="header-anchor" href="#related" aria-label="Permalink to &quot;Related&quot;">​</a></h2><ul><li><strong>Executor sizing:</strong> <a href="./cluster-config.html">Cluster Tuning</a></li><li><strong>Memory overhead &amp; off-heap:</strong> <a href="./memory-model.html">Memory Management</a></li></ul><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><a href="https://raw.githubusercontent.com/apache/spark/v3.5.0/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala" target="_blank" rel="noreferrer">JsonProtocol.scala</a> <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a> <a href="#fnref1:2" class="footnote-backref">↩︎</a> <a href="#fnref1:3" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><a href="https://luminousmen.com/post/dive-into-spark-memory" target="_blank" rel="noreferrer">Dive Into Spark Memory</a> <a href="#fnref2" class="footnote-backref">↩︎</a> <a href="#fnref2:1" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/job-scheduling.html" target="_blank" rel="noreferrer">Job Scheduling (Apache Spark Documentation)</a> <a href="#fnref3" class="footnote-backref">↩︎</a> <a href="#fnref3:1" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><a href="https://luminousmen.com/post/spark-anatomy-of-spark-application" target="_blank" rel="noreferrer">Anatomy of a Spark Application</a> <a href="#fnref4" class="footnote-backref">↩︎</a></p></li><li id="fn5" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Spark Configuration (Apache Spark Documentation)</a> <a href="#fnref5" class="footnote-backref">↩︎</a> <a href="#fnref5:1" class="footnote-backref">↩︎</a> <a href="#fnref5:2" class="footnote-backref">↩︎</a> <a href="#fnref5:3" class="footnote-backref">↩︎</a></p></li></ol></section></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/tuning-reference/bottleneck-slow-host.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>Slow Host</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="/docs/tuning-reference/bottleneck-straggler.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>Stragglers</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
32
+ <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"CWpj01WU\",\"contributor-guide_architecture_detector-contract.md\":\"CgzUsQ6W\",\"contributor-guide_architecture_drill-down.md\":\"BtPdlM7r\",\"contributor-guide_architecture_impact-estimation.md\":\"CooslVJt\",\"contributor-guide_architecture_index.md\":\"3TO9ic6w\",\"contributor-guide_architecture_overview.md\":\"CehiRmGn\",\"contributor-guide_architecture_state-and-history.md\":\"C-xxn0q7\",\"contributor-guide_architecture_widget-rendering.md\":\"R27gQrgY\",\"contributor-guide_architecture_worker-protocol.md\":\"IbnfNrV3\",\"contributor-guide_contributing.md\":\"CvRsdr6J\",\"contributor-guide_development-setup.md\":\"DvAN_9mK\",\"contributor-guide_testing.md\":\"6rIKqSyY\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"Df1YMIHu\",\"tuning-reference_aqe.md\":\"BIsCtLzm\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"CEstB3Ia\",\"tuning-reference_bottleneck-cold-start.md\":\"CEuy-72y\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"CIohQDfn\",\"tuning-reference_bottleneck-failures.md\":\"4z5BXGJ2\",\"tuning-reference_bottleneck-gc.md\":\"DSxzZRK7\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"BaJl__1W\",\"tuning-reference_bottleneck-memory-utilization.md\":\"DbP-SJZc\",\"tuning-reference_bottleneck-retry-waste.md\":\"D5JMjVOt\",\"tuning-reference_bottleneck-shuffle.md\":\"CM-nTmIH\",\"tuning-reference_bottleneck-skew.md\":\"BdUwiDhn\",\"tuning-reference_bottleneck-slow-host.md\":\"BlIo6UDW\",\"tuning-reference_bottleneck-small-files.md\":\"B8kloyx8\",\"tuning-reference_bottleneck-spill.md\":\"PNH7mITt\",\"tuning-reference_bottleneck-straggler.md\":\"DY36fHN5\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"QTV7O8kU\",\"tuning-reference_bottleneck-utilization.md\":\"DTiueZC3\",\"tuning-reference_caching.md\":\"B7aQ8asB\",\"tuning-reference_cluster-config.md\":\"ZVmDGsQ3\",\"tuning-reference_config.md\":\"UvveiWG3\",\"tuning-reference_data-formats.md\":\"bjCAWH3N\",\"tuning-reference_index.md\":\"BQ_NooMV\",\"tuning-reference_intro.md\":\"CobD-lGB\",\"tuning-reference_joins.md\":\"BtKs_CuW\",\"tuning-reference_memory-model.md\":\"DhT-n4y3\",\"tuning-reference_metrics.md\":\"mLOh7Apj\",\"tuning-reference_partitioning.md\":\"q0zKF_8X\",\"tuning-reference_pyspark.md\":\"DDCfvN9t\",\"tuning-reference_shuffle.md\":\"BZZ7R4Ix\",\"tuning-reference_spark-architecture.md\":\"Dwzm5avO\",\"tuning-reference_table-formats.md\":\"D6wj-2dX\",\"user-guide_alternative-log-retrieval.md\":\"B4tPGIal\",\"user-guide_getting-started.md\":\"BJvwLEIM\",\"user-guide_mcp-tools.md\":\"Vi3RoflJ\",\"user-guide_run-comparison.md\":\"CQc1aoU8\",\"user-guide_understanding-findings.md\":\"DL1UDhvR\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Alternative log retrieval\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
33
+
34
+ </body>
35
+ </html>