sparkforensics-server 0.1.0 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (448) hide show
  1. package/README.md +6 -0
  2. package/index.js +41 -50
  3. package/lib/static-files.js +15 -10
  4. package/package.json +23 -3
  5. package/public/assets/AutoscalingChurn-B96FKx97.js +1 -0
  6. package/public/assets/CacheUtilization-lKua3J1x.js +1 -0
  7. package/public/assets/CachingOpportunity-U0g-8HPV.js +1 -0
  8. package/public/assets/ColdStart-w8QYTNKo.js +1 -0
  9. package/public/assets/ConfigAudit-ClZp5BpV.js +1 -0
  10. package/public/assets/CoreUsageArea-o3kqQwyi.js +1 -0
  11. package/public/assets/Dashboard-Bjl-MYMm.js +1 -0
  12. package/public/assets/DuplicatePlanSubtree-Cm_UU3lY.js +1 -0
  13. package/public/assets/DurationHistogram-ltF-7M0A.js +1 -0
  14. package/public/assets/EmptyState-BCA3xioa.js +1 -0
  15. package/public/assets/ExecutorUtilization-Cn69i0uq.js +1 -0
  16. package/public/assets/GcPressure-CYsmU1d7.js +1 -0
  17. package/public/assets/ImpactEstimate-BGt-4WX5.js +1 -0
  18. package/public/assets/IncompleteRun-s6uQ8OLf.js +1 -0
  19. package/public/assets/JobFailures-DJ2dDvPA.js +1 -0
  20. package/public/assets/MemoryUtilization-C9G7P94X.js +1 -0
  21. package/public/assets/OverBroadcast-DYauoYWQ.js +1 -0
  22. package/public/assets/PartitionSizing-Cz_B0-uG.js +1 -0
  23. package/public/assets/PlanExplorer-_2oB5ouy.js +1 -0
  24. package/public/assets/PlanGraphRoute-Cj-GWM8W.js +1 -0
  25. package/public/assets/RetryWaste-D_m8ASwP.js +1 -0
  26. package/public/assets/RowPagination-BLsm8I6i.js +1 -0
  27. package/public/assets/ShuffleIO-pyDhxdaz.js +1 -0
  28. package/public/assets/Skew-BGLJuZZR.js +1 -0
  29. package/public/assets/SlowHost-ChdWVvGO.js +1 -0
  30. package/public/assets/SmallFiles-gTilBtjW.js +1 -0
  31. package/public/assets/SpeculationWaste-CVzgJnSj.js +1 -0
  32. package/public/assets/Spill-NGJGQaTB.js +1 -0
  33. package/public/assets/StageFailed-sGhki53z.js +1 -0
  34. package/public/assets/StageFindingGroup-AyI7BMd3.js +1 -0
  35. package/public/assets/StageHeader-BTzGkK8k.js +1 -0
  36. package/public/assets/StagePill-BlMOW_pU.js +1 -0
  37. package/public/assets/StageShape-DfOW93Z1.js +1 -0
  38. package/public/assets/StageSlowness-BMZ-K_5e.js +1 -0
  39. package/public/assets/Straggler-dffcAN7t.js +1 -0
  40. package/public/assets/TaskFailures-DGDwUK2A.js +1 -0
  41. package/public/assets/TinyTask-CfT8zZDr.js +1 -0
  42. package/public/assets/UnderBroadcast-CiOiJ0k1.js +1 -0
  43. package/public/assets/WidgetLeadSummary-BdUUi7L5.js +1 -0
  44. package/public/assets/charts-vendor-0VMaLVLN.js +59 -0
  45. package/public/assets/downsample-WmNM1qc9.js +1 -0
  46. package/public/assets/finding-anchor-DEV8JvgH.js +1 -0
  47. package/public/assets/impact-sort-DxaUheWV.js +1 -0
  48. package/public/assets/inbox-_QCYkBsP.js +1 -0
  49. package/public/assets/index-3vlFVNCg.js +9 -0
  50. package/public/assets/index-CjVklAdW.css +2 -0
  51. package/public/assets/parser-worker-B2bc0lWe.js +112 -0
  52. package/public/assets/plan-graph-vendor-B0sEms9Z.js +7 -0
  53. package/public/assets/plan-graph-vendor-DLioOiRN.css +1 -0
  54. package/public/assets/react-vendor-D2TLlSzI.js +9 -0
  55. package/public/assets/rolldown-runtime-hePW80VL.js +1 -0
  56. package/public/assets/select-kuy1MiAo.js +1 -0
  57. package/public/assets/toggle-group-CUd-KWkp.js +1 -0
  58. package/public/assets/usePagedRows-jNUWgVTD.js +1 -0
  59. package/public/assets/useRecentFiles-D0OA_EOK.js +11 -0
  60. package/public/assets/useSortMode-BaeAA5cw.js +1 -0
  61. package/public/docs/404.html +3 -3
  62. package/public/docs/assets/{app.Bxy4_HVA.js → app.APoq_UBs.js} +1 -1
  63. package/public/docs/assets/aqe-loop.IwQSATHw.svg +1 -0
  64. package/public/docs/assets/aqe-loop.dark.DGbaxqJE.svg +1 -0
  65. package/public/docs/assets/broadcast-vs-shuffle.Db4WY1XK.svg +1 -0
  66. package/public/docs/assets/broadcast-vs-shuffle.dark.C7Bxs0mG.svg +1 -0
  67. package/public/docs/assets/cache-lifecycle.dark.B-hS7AgU.svg +1 -0
  68. package/public/docs/assets/cache-lifecycle.rEOVYQNU.svg +1 -0
  69. package/public/docs/assets/chunks/@localSearchIndexroot.CvM5fuxe.js +1 -0
  70. package/public/docs/assets/chunks/{VPLocalSearchBox.L5jNOIR3.js → VPLocalSearchBox.B3TPzlHq.js} +1 -1
  71. package/public/docs/assets/chunks/duplicate-plan-subtree.dark.Cdp70QhV.js +1 -0
  72. package/public/docs/assets/chunks/{framework.CNDaxUod.js → framework.DSg0KOwT.js} +1 -1
  73. package/public/docs/assets/chunks/retry-escalation-ladder.dark.DHipdJgZ.js +1 -0
  74. package/public/docs/assets/chunks/theme.PavXO_vz.js +2 -0
  75. package/public/docs/assets/cold-start-timeline.DxC_Sc7w.svg +1 -0
  76. package/public/docs/assets/cold-start-timeline.dark.CZ17YcAG.svg +1 -0
  77. package/public/docs/assets/columnar-layout.PghGeOEA.svg +1 -0
  78. package/public/docs/assets/columnar-layout.dark.BVNlz0ff.svg +1 -0
  79. package/public/docs/assets/container-memory.DIO0AnIm.svg +1 -0
  80. package/public/docs/assets/container-memory.dark.CP-5zuCl.svg +1 -0
  81. package/public/docs/assets/contributor-guide_architecture_board-widgets.md.CWpj01WU.js +1 -0
  82. package/public/docs/assets/contributor-guide_architecture_board-widgets.md.CWpj01WU.lean.js +1 -0
  83. package/public/docs/assets/contributor-guide_architecture_detector-contract.md.CgzUsQ6W.js +1 -0
  84. package/public/docs/assets/contributor-guide_architecture_detector-contract.md.CgzUsQ6W.lean.js +1 -0
  85. package/public/docs/assets/contributor-guide_architecture_drill-down.md.BtPdlM7r.js +1 -0
  86. package/public/docs/assets/contributor-guide_architecture_drill-down.md.BtPdlM7r.lean.js +1 -0
  87. package/public/docs/assets/{contributor-guide_architecture_impact-estimation.md.CzlXOnCq.js → contributor-guide_architecture_impact-estimation.md.CooslVJt.js} +1 -1
  88. package/public/docs/assets/{contributor-guide_architecture_impact-estimation.md.CzlXOnCq.lean.js → contributor-guide_architecture_impact-estimation.md.CooslVJt.lean.js} +1 -1
  89. package/public/docs/assets/{contributor-guide_architecture_index.md.94lNbF-4.js → contributor-guide_architecture_index.md.3TO9ic6w.js} +1 -1
  90. package/public/docs/assets/{contributor-guide_architecture_index.md.94lNbF-4.lean.js → contributor-guide_architecture_index.md.3TO9ic6w.lean.js} +1 -1
  91. package/public/docs/assets/{contributor-guide_architecture_overview.md.mQFlN66o.js → contributor-guide_architecture_overview.md.CehiRmGn.js} +1 -1
  92. package/public/docs/assets/{contributor-guide_architecture_overview.md.mQFlN66o.lean.js → contributor-guide_architecture_overview.md.CehiRmGn.lean.js} +1 -1
  93. package/public/docs/assets/contributor-guide_architecture_state-and-history.md.C-xxn0q7.js +1 -0
  94. package/public/docs/assets/contributor-guide_architecture_state-and-history.md.C-xxn0q7.lean.js +1 -0
  95. package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.R27gQrgY.js +1 -0
  96. package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.R27gQrgY.lean.js +1 -0
  97. package/public/docs/assets/{contributor-guide_architecture_worker-protocol.md.BaS_L10g.js → contributor-guide_architecture_worker-protocol.md.IbnfNrV3.js} +2 -2
  98. package/public/docs/assets/{contributor-guide_architecture_worker-protocol.md.BaS_L10g.lean.js → contributor-guide_architecture_worker-protocol.md.IbnfNrV3.lean.js} +1 -1
  99. package/public/docs/assets/contributor-guide_contributing.md.CvRsdr6J.js +1 -0
  100. package/public/docs/assets/contributor-guide_contributing.md.CvRsdr6J.lean.js +1 -0
  101. package/public/docs/assets/{contributor-guide_development-setup.md.CpUMVtIK.js → contributor-guide_development-setup.md.DvAN_9mK.js} +1 -1
  102. package/public/docs/assets/{contributor-guide_development-setup.md.CpUMVtIK.lean.js → contributor-guide_development-setup.md.DvAN_9mK.lean.js} +1 -1
  103. package/public/docs/assets/{contributor-guide_testing.md.C0q-BJfl.js → contributor-guide_testing.md.6rIKqSyY.js} +1 -1
  104. package/public/docs/assets/{contributor-guide_testing.md.C0q-BJfl.lean.js → contributor-guide_testing.md.6rIKqSyY.lean.js} +1 -1
  105. package/public/docs/assets/dag-stages.DSz_S937.svg +1 -0
  106. package/public/docs/assets/dag-stages.dark.F72UzxH4.svg +1 -0
  107. package/public/docs/assets/driver-executor.D5pQ7YN1.svg +1 -0
  108. package/public/docs/assets/driver-executor.dark.BmX9cPvh.svg +1 -0
  109. package/public/docs/assets/duplicate-plan-subtree.B4cvN6fj.svg +1 -0
  110. package/public/docs/assets/duplicate-plan-subtree.dark.Dw8wS0Ag.svg +1 -0
  111. package/public/docs/assets/{index.md.CETnDZzR.js → index.md.CHJVslga.js} +1 -1
  112. package/public/docs/assets/{index.md.CETnDZzR.lean.js → index.md.CHJVslga.lean.js} +1 -1
  113. package/public/docs/assets/join-strategy.C_FvrCEo.svg +1 -0
  114. package/public/docs/assets/join-strategy.dark.ChMLnNII.svg +1 -0
  115. package/public/docs/assets/memory-borrowing.BqQRJg0u.svg +1 -0
  116. package/public/docs/assets/memory-borrowing.dark.Yhh20O9C.svg +1 -0
  117. package/public/docs/assets/memory-regions.XHvO7jHG.svg +1 -0
  118. package/public/docs/assets/memory-regions.dark.D4TP9_08.svg +1 -0
  119. package/public/docs/assets/repartition-vs-coalesce.BovLRrpj.svg +1 -0
  120. package/public/docs/assets/repartition-vs-coalesce.dark.BhAczKZQ.svg +1 -0
  121. package/public/docs/assets/retry-escalation-ladder.DyTKJJmZ.svg +1 -0
  122. package/public/docs/assets/retry-escalation-ladder.dark.BdsabtU3.svg +1 -0
  123. package/public/docs/assets/shuffle-map-reduce.KuOEZVmg.svg +1 -0
  124. package/public/docs/assets/shuffle-map-reduce.dark.BgQZnFSb.svg +1 -0
  125. package/public/docs/assets/spill-classification.BU2euYDO.svg +1 -0
  126. package/public/docs/assets/spill-classification.dark.D7i1M40d.svg +1 -0
  127. package/public/docs/assets/{style.D2mY0HUs.css → style.DSixAiZE.css} +1 -1
  128. package/public/docs/assets/tuning-reference_anti-patterns.md.Df1YMIHu.js +1 -0
  129. package/public/docs/assets/tuning-reference_anti-patterns.md.Df1YMIHu.lean.js +1 -0
  130. package/public/docs/assets/tuning-reference_aqe.md.BIsCtLzm.js +1 -0
  131. package/public/docs/assets/tuning-reference_aqe.md.BIsCtLzm.lean.js +1 -0
  132. package/public/docs/assets/tuning-reference_bottleneck-broadcast-sizing.md.CEstB3Ia.js +1 -0
  133. package/public/docs/assets/tuning-reference_bottleneck-broadcast-sizing.md.CEstB3Ia.lean.js +1 -0
  134. package/public/docs/assets/tuning-reference_bottleneck-cold-start.md.CEuy-72y.js +7 -0
  135. package/public/docs/assets/tuning-reference_bottleneck-cold-start.md.CEuy-72y.lean.js +1 -0
  136. package/public/docs/assets/tuning-reference_bottleneck-duplicate-plan-subtree.md.CIohQDfn.js +1 -0
  137. package/public/docs/assets/tuning-reference_bottleneck-duplicate-plan-subtree.md.CIohQDfn.lean.js +1 -0
  138. package/public/docs/assets/tuning-reference_bottleneck-failures.md.4z5BXGJ2.js +6 -0
  139. package/public/docs/assets/tuning-reference_bottleneck-failures.md.4z5BXGJ2.lean.js +1 -0
  140. package/public/docs/assets/tuning-reference_bottleneck-gc.md.DSxzZRK7.js +6 -0
  141. package/public/docs/assets/tuning-reference_bottleneck-gc.md.DSxzZRK7.lean.js +1 -0
  142. package/public/docs/assets/tuning-reference_bottleneck-job-failure-rate.md.BaJl__1W.js +8 -0
  143. package/public/docs/assets/tuning-reference_bottleneck-job-failure-rate.md.BaJl__1W.lean.js +1 -0
  144. package/public/docs/assets/tuning-reference_bottleneck-memory-utilization.md.DbP-SJZc.js +7 -0
  145. package/public/docs/assets/tuning-reference_bottleneck-memory-utilization.md.DbP-SJZc.lean.js +1 -0
  146. package/public/docs/assets/tuning-reference_bottleneck-retry-waste.md.D5JMjVOt.js +1 -0
  147. package/public/docs/assets/tuning-reference_bottleneck-retry-waste.md.D5JMjVOt.lean.js +1 -0
  148. package/public/docs/assets/tuning-reference_bottleneck-shuffle.md.CM-nTmIH.js +12 -0
  149. package/public/docs/assets/tuning-reference_bottleneck-shuffle.md.CM-nTmIH.lean.js +1 -0
  150. package/public/docs/assets/tuning-reference_bottleneck-skew.md.BdUwiDhn.js +14 -0
  151. package/public/docs/assets/tuning-reference_bottleneck-skew.md.BdUwiDhn.lean.js +1 -0
  152. package/public/docs/assets/tuning-reference_bottleneck-slow-host.md.BlIo6UDW.js +7 -0
  153. package/public/docs/assets/tuning-reference_bottleneck-slow-host.md.BlIo6UDW.lean.js +1 -0
  154. package/public/docs/assets/tuning-reference_bottleneck-small-files.md.B8kloyx8.js +5 -0
  155. package/public/docs/assets/tuning-reference_bottleneck-small-files.md.B8kloyx8.lean.js +1 -0
  156. package/public/docs/assets/tuning-reference_bottleneck-spill.md.PNH7mITt.js +6 -0
  157. package/public/docs/assets/tuning-reference_bottleneck-spill.md.PNH7mITt.lean.js +1 -0
  158. package/public/docs/assets/tuning-reference_bottleneck-straggler.md.DY36fHN5.js +7 -0
  159. package/public/docs/assets/tuning-reference_bottleneck-straggler.md.DY36fHN5.lean.js +1 -0
  160. package/public/docs/assets/tuning-reference_bottleneck-tiny-tasks.md.QTV7O8kU.js +8 -0
  161. package/public/docs/assets/tuning-reference_bottleneck-tiny-tasks.md.QTV7O8kU.lean.js +1 -0
  162. package/public/docs/assets/tuning-reference_bottleneck-utilization.md.DTiueZC3.js +5 -0
  163. package/public/docs/assets/tuning-reference_bottleneck-utilization.md.DTiueZC3.lean.js +1 -0
  164. package/public/docs/assets/tuning-reference_caching.md.B7aQ8asB.js +1 -0
  165. package/public/docs/assets/tuning-reference_caching.md.B7aQ8asB.lean.js +1 -0
  166. package/public/docs/assets/tuning-reference_cluster-config.md.ZVmDGsQ3.js +1 -0
  167. package/public/docs/assets/tuning-reference_cluster-config.md.ZVmDGsQ3.lean.js +1 -0
  168. package/public/docs/assets/tuning-reference_config.md.UvveiWG3.js +1 -0
  169. package/public/docs/assets/tuning-reference_config.md.UvveiWG3.lean.js +1 -0
  170. package/public/docs/assets/tuning-reference_data-formats.md.bjCAWH3N.js +1 -0
  171. package/public/docs/assets/tuning-reference_data-formats.md.bjCAWH3N.lean.js +1 -0
  172. package/public/docs/assets/tuning-reference_index.md.BQ_NooMV.js +1 -0
  173. package/public/docs/assets/tuning-reference_index.md.BQ_NooMV.lean.js +1 -0
  174. package/public/docs/assets/tuning-reference_intro.md.CobD-lGB.js +1 -0
  175. package/public/docs/assets/tuning-reference_intro.md.CobD-lGB.lean.js +1 -0
  176. package/public/docs/assets/tuning-reference_joins.md.BtKs_CuW.js +1 -0
  177. package/public/docs/assets/tuning-reference_joins.md.BtKs_CuW.lean.js +1 -0
  178. package/public/docs/assets/tuning-reference_memory-model.md.DhT-n4y3.js +1 -0
  179. package/public/docs/assets/tuning-reference_memory-model.md.DhT-n4y3.lean.js +1 -0
  180. package/public/docs/assets/tuning-reference_metrics.md.mLOh7Apj.js +1 -0
  181. package/public/docs/assets/tuning-reference_metrics.md.mLOh7Apj.lean.js +1 -0
  182. package/public/docs/assets/tuning-reference_partitioning.md.q0zKF_8X.js +1 -0
  183. package/public/docs/assets/tuning-reference_partitioning.md.q0zKF_8X.lean.js +1 -0
  184. package/public/docs/assets/tuning-reference_pyspark.md.DDCfvN9t.js +6 -0
  185. package/public/docs/assets/tuning-reference_pyspark.md.DDCfvN9t.lean.js +1 -0
  186. package/public/docs/assets/tuning-reference_shuffle.md.BZZ7R4Ix.js +1 -0
  187. package/public/docs/assets/tuning-reference_shuffle.md.BZZ7R4Ix.lean.js +1 -0
  188. package/public/docs/assets/tuning-reference_spark-architecture.md.Dwzm5avO.js +1 -0
  189. package/public/docs/assets/tuning-reference_spark-architecture.md.Dwzm5avO.lean.js +1 -0
  190. package/public/docs/assets/tuning-reference_table-formats.md.D6wj-2dX.js +1 -0
  191. package/public/docs/assets/tuning-reference_table-formats.md.D6wj-2dX.lean.js +1 -0
  192. package/public/docs/assets/udf-execution-models.BUFDICuG.svg +1 -0
  193. package/public/docs/assets/udf-execution-models.dark.YTNS6GDq.svg +1 -0
  194. package/public/docs/assets/user-guide_alternative-log-retrieval.md.B4tPGIal.js +1 -0
  195. package/public/docs/assets/user-guide_alternative-log-retrieval.md.B4tPGIal.lean.js +1 -0
  196. package/public/docs/assets/user-guide_getting-started.md.BJvwLEIM.js +3 -0
  197. package/public/docs/assets/user-guide_getting-started.md.BJvwLEIM.lean.js +1 -0
  198. package/public/docs/assets/{user-guide_mcp-tools.md.DwYI1d6C.js → user-guide_mcp-tools.md.Vi3RoflJ.js} +35 -5
  199. package/public/docs/assets/{user-guide_mcp-tools.md.DwYI1d6C.lean.js → user-guide_mcp-tools.md.Vi3RoflJ.lean.js} +1 -1
  200. package/public/docs/assets/user-guide_run-comparison.md.CQc1aoU8.js +1 -0
  201. package/public/docs/assets/user-guide_run-comparison.md.CQc1aoU8.lean.js +1 -0
  202. package/public/docs/assets/user-guide_understanding-findings.md.DL1UDhvR.js +1 -0
  203. package/public/docs/assets/user-guide_understanding-findings.md.DL1UDhvR.lean.js +1 -0
  204. package/public/docs/contributor-guide/architecture/board-widgets.html +7 -7
  205. package/public/docs/contributor-guide/architecture/detector-contract.html +7 -7
  206. package/public/docs/contributor-guide/architecture/drill-down.html +7 -7
  207. package/public/docs/contributor-guide/architecture/impact-estimation.html +7 -7
  208. package/public/docs/contributor-guide/architecture/index.html +7 -7
  209. package/public/docs/contributor-guide/architecture/overview.html +7 -7
  210. package/public/docs/contributor-guide/architecture/state-and-history.html +7 -7
  211. package/public/docs/contributor-guide/architecture/widget-rendering.html +7 -7
  212. package/public/docs/contributor-guide/architecture/worker-protocol.html +8 -8
  213. package/public/docs/contributor-guide/contributing.html +7 -7
  214. package/public/docs/contributor-guide/development-setup.html +7 -7
  215. package/public/docs/contributor-guide/testing.html +7 -7
  216. package/public/docs/hashmap.json +1 -1
  217. package/public/docs/index.html +7 -7
  218. package/public/docs/tuning-reference/anti-patterns.html +29 -0
  219. package/public/docs/tuning-reference/aqe.html +29 -0
  220. package/public/docs/tuning-reference/bottleneck-broadcast-sizing.html +29 -0
  221. package/public/docs/tuning-reference/bottleneck-cold-start.html +35 -0
  222. package/public/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html +30 -0
  223. package/public/docs/tuning-reference/bottleneck-failures.html +35 -0
  224. package/public/docs/tuning-reference/bottleneck-gc.html +34 -0
  225. package/public/docs/tuning-reference/bottleneck-job-failure-rate.html +37 -0
  226. package/public/docs/tuning-reference/bottleneck-memory-utilization.html +35 -0
  227. package/public/docs/tuning-reference/bottleneck-retry-waste.html +30 -0
  228. package/public/docs/tuning-reference/bottleneck-shuffle.html +40 -0
  229. package/public/docs/tuning-reference/bottleneck-skew.html +42 -0
  230. package/public/docs/tuning-reference/bottleneck-slow-host.html +35 -0
  231. package/public/docs/tuning-reference/bottleneck-small-files.html +33 -0
  232. package/public/docs/tuning-reference/bottleneck-spill.html +34 -0
  233. package/public/docs/tuning-reference/bottleneck-straggler.html +35 -0
  234. package/public/docs/tuning-reference/bottleneck-tiny-tasks.html +36 -0
  235. package/public/docs/tuning-reference/bottleneck-utilization.html +34 -0
  236. package/public/docs/tuning-reference/caching.html +29 -0
  237. package/public/docs/tuning-reference/cluster-config.html +29 -0
  238. package/public/docs/tuning-reference/config.html +29 -0
  239. package/public/docs/tuning-reference/data-formats.html +29 -0
  240. package/public/docs/tuning-reference/index.html +29 -0
  241. package/public/docs/tuning-reference/intro.html +29 -0
  242. package/public/docs/tuning-reference/joins.html +29 -0
  243. package/public/docs/tuning-reference/memory-model.html +29 -0
  244. package/public/docs/tuning-reference/metrics.html +29 -0
  245. package/public/docs/tuning-reference/partitioning.html +29 -0
  246. package/public/docs/tuning-reference/pyspark.html +34 -0
  247. package/public/docs/tuning-reference/shuffle.html +29 -0
  248. package/public/docs/tuning-reference/spark-architecture.html +29 -0
  249. package/public/docs/tuning-reference/table-formats.html +29 -0
  250. package/public/docs/user-guide/alternative-log-retrieval.html +29 -0
  251. package/public/docs/user-guide/getting-started.html +8 -8
  252. package/public/docs/user-guide/mcp-tools.html +41 -11
  253. package/public/docs/user-guide/run-comparison.html +7 -7
  254. package/public/docs/user-guide/understanding-findings.html +7 -7
  255. package/public/index.html +8 -7
  256. package/public/sample-runs/sample-run.ndjson.gz +0 -0
  257. package/vendor-core/analyzer.js +74 -74
  258. package/vendor-core/cli/budgets.js +13 -27
  259. package/vendor-core/cli/collect-run.js +43 -19
  260. package/vendor-core/core-count.js +25 -27
  261. package/vendor-core/core-locality-ratio.js +4 -11
  262. package/vendor-core/core-time-series.js +6 -12
  263. package/vendor-core/core-usage-locality.js +3 -4
  264. package/vendor-core/detectors.js +395 -389
  265. package/vendor-core/docs-config.js +69 -21
  266. package/vendor-core/docs-content/chapters/01-intro.md +32 -0
  267. package/vendor-core/docs-content/chapters/02-spark-architecture.md +76 -0
  268. package/vendor-core/docs-content/chapters/03-memory-model.md +73 -0
  269. package/vendor-core/docs-content/chapters/04-partitioning.md +65 -0
  270. package/vendor-core/docs-content/chapters/05-joins.md +62 -0
  271. package/vendor-core/docs-content/chapters/06-shuffle.md +59 -0
  272. package/vendor-core/docs-content/chapters/07-data-formats.md +81 -0
  273. package/vendor-core/docs-content/chapters/07b-table-formats.md +56 -0
  274. package/vendor-core/docs-content/chapters/08-caching.md +58 -0
  275. package/vendor-core/docs-content/chapters/09-pyspark.md +78 -0
  276. package/vendor-core/docs-content/chapters/10-aqe.md +167 -0
  277. package/vendor-core/docs-content/chapters/11-cluster-config.md +170 -0
  278. package/vendor-core/docs-content/chapters/12-anti-patterns.md +171 -0
  279. package/vendor-core/docs-content/chapters/14-metrics.md +87 -0
  280. package/vendor-core/docs-content/chapters/15-config.md +93 -0
  281. package/vendor-core/docs-content/chapters/nav-index.json +370 -0
  282. package/vendor-core/docs-content/detection/cache.md +7 -0
  283. package/vendor-core/docs-content/detection/cfg.md +15 -0
  284. package/vendor-core/docs-content/detection/chrn.md +9 -0
  285. package/vendor-core/docs-content/detection/cold.md +4 -0
  286. package/vendor-core/docs-content/detection/cstor.md +4 -0
  287. package/vendor-core/docs-content/detection/fail.md +5 -0
  288. package/vendor-core/docs-content/detection/gc.md +4 -0
  289. package/vendor-core/docs-content/detection/host.md +5 -0
  290. package/vendor-core/docs-content/detection/incmp.md +6 -0
  291. package/vendor-core/docs-content/detection/jobs.md +4 -0
  292. package/vendor-core/docs-content/detection/local.md +6 -0
  293. package/vendor-core/docs-content/detection/mem.md +10 -0
  294. package/vendor-core/docs-content/detection/part.md +5 -0
  295. package/vendor-core/docs-content/detection/plan.md +14 -0
  296. package/vendor-core/docs-content/detection/retry.md +4 -0
  297. package/vendor-core/docs-content/detection/sfail.md +5 -0
  298. package/vendor-core/docs-content/detection/shape.md +5 -0
  299. package/vendor-core/docs-content/detection/shfl.md +4 -0
  300. package/vendor-core/docs-content/detection/skew.md +6 -0
  301. package/vendor-core/docs-content/detection/slow.md +6 -0
  302. package/vendor-core/docs-content/detection/spec.md +8 -0
  303. package/vendor-core/docs-content/detection/spill.md +7 -0
  304. package/vendor-core/docs-content/detection/strag.md +5 -0
  305. package/vendor-core/docs-content/detection/tiny.md +4 -0
  306. package/vendor-core/docs-content/detection/util.md +4 -0
  307. package/vendor-core/docs-content/diagrams/aqe-loop.dark.svg +1 -0
  308. package/vendor-core/docs-content/diagrams/aqe-loop.svg +1 -0
  309. package/vendor-core/docs-content/diagrams/broadcast-vs-shuffle.dark.svg +1 -0
  310. package/vendor-core/docs-content/diagrams/broadcast-vs-shuffle.svg +1 -0
  311. package/vendor-core/docs-content/diagrams/cache-lifecycle.dark.svg +1 -0
  312. package/vendor-core/docs-content/diagrams/cache-lifecycle.svg +1 -0
  313. package/vendor-core/docs-content/diagrams/cold-start-timeline.dark.svg +1 -0
  314. package/vendor-core/docs-content/diagrams/cold-start-timeline.svg +1 -0
  315. package/vendor-core/docs-content/diagrams/columnar-layout.dark.svg +1 -0
  316. package/vendor-core/docs-content/diagrams/columnar-layout.svg +1 -0
  317. package/vendor-core/docs-content/diagrams/container-memory.dark.svg +1 -0
  318. package/vendor-core/docs-content/diagrams/container-memory.svg +1 -0
  319. package/vendor-core/docs-content/diagrams/dag-stages.dark.svg +1 -0
  320. package/vendor-core/docs-content/diagrams/dag-stages.svg +1 -0
  321. package/vendor-core/docs-content/diagrams/driver-executor.dark.svg +1 -0
  322. package/vendor-core/docs-content/diagrams/driver-executor.svg +1 -0
  323. package/vendor-core/docs-content/diagrams/duplicate-plan-subtree.dark.svg +1 -0
  324. package/vendor-core/docs-content/diagrams/duplicate-plan-subtree.svg +1 -0
  325. package/vendor-core/docs-content/diagrams/join-strategy.dark.svg +1 -0
  326. package/vendor-core/docs-content/diagrams/join-strategy.svg +1 -0
  327. package/vendor-core/docs-content/diagrams/memory-borrowing.dark.svg +1 -0
  328. package/vendor-core/docs-content/diagrams/memory-borrowing.svg +1 -0
  329. package/vendor-core/docs-content/diagrams/memory-regions.dark.svg +1 -0
  330. package/vendor-core/docs-content/diagrams/memory-regions.svg +1 -0
  331. package/vendor-core/docs-content/diagrams/repartition-vs-coalesce.dark.svg +1 -0
  332. package/vendor-core/docs-content/diagrams/repartition-vs-coalesce.svg +1 -0
  333. package/vendor-core/docs-content/diagrams/retry-escalation-ladder.dark.svg +1 -0
  334. package/vendor-core/docs-content/diagrams/retry-escalation-ladder.svg +1 -0
  335. package/vendor-core/docs-content/diagrams/shuffle-map-reduce.dark.svg +1 -0
  336. package/vendor-core/docs-content/diagrams/shuffle-map-reduce.svg +1 -0
  337. package/vendor-core/docs-content/diagrams/spill-classification.dark.svg +1 -0
  338. package/vendor-core/docs-content/diagrams/spill-classification.svg +1 -0
  339. package/vendor-core/docs-content/diagrams/udf-execution-models.dark.svg +1 -0
  340. package/vendor-core/docs-content/diagrams/udf-execution-models.svg +1 -0
  341. package/vendor-core/docs-content/tuning/broadcast-sizing.md +78 -0
  342. package/vendor-core/docs-content/tuning/cold-start.md +81 -0
  343. package/vendor-core/docs-content/tuning/duplicate-plan-subtree.md +45 -0
  344. package/vendor-core/docs-content/tuning/failures.md +124 -0
  345. package/vendor-core/docs-content/tuning/gc.md +110 -0
  346. package/vendor-core/docs-content/tuning/job-failure-rate.md +101 -0
  347. package/vendor-core/docs-content/tuning/memory-utilization.md +58 -0
  348. package/vendor-core/docs-content/tuning/retry-waste.md +90 -0
  349. package/vendor-core/docs-content/tuning/shuffle.md +154 -0
  350. package/vendor-core/docs-content/tuning/skew.md +123 -0
  351. package/vendor-core/docs-content/tuning/slow-host.md +117 -0
  352. package/vendor-core/docs-content/tuning/small-files.md +99 -0
  353. package/vendor-core/docs-content/tuning/spill.md +114 -0
  354. package/vendor-core/docs-content/tuning/straggler.md +103 -0
  355. package/vendor-core/docs-content/tuning/tiny-tasks.md +94 -0
  356. package/vendor-core/docs-content/tuning/utilization.md +90 -0
  357. package/vendor-core/docs-site-config.js +10 -17
  358. package/vendor-core/efficiency-model.js +7 -13
  359. package/vendor-core/etl-phases.js +3 -5
  360. package/vendor-core/event-handlers.js +232 -134
  361. package/vendor-core/event-schemas.js +48 -114
  362. package/vendor-core/evidence-availability.js +5 -10
  363. package/vendor-core/evidence-report.js +73 -123
  364. package/vendor-core/export-data.js +48 -0
  365. package/vendor-core/finding-action-label.js +4 -10
  366. package/vendor-core/finding-filter-predicate.js +3 -7
  367. package/vendor-core/finding-generic-recommendation.js +112 -0
  368. package/vendor-core/finding-names.js +51 -0
  369. package/vendor-core/format-utils.js +112 -38
  370. package/vendor-core/impact-band.js +18 -24
  371. package/vendor-core/impact-estimator.js +38 -74
  372. package/vendor-core/ingest.js +7 -13
  373. package/vendor-core/job-groups.js +3 -6
  374. package/vendor-core/list-runs.js +278 -0
  375. package/vendor-core/load-vendored.js +6 -12
  376. package/vendor-core/log-header-peek.js +81 -0
  377. package/vendor-core/lz4-block.js +4 -6
  378. package/vendor-core/mcp-server-factory.js +38 -8
  379. package/vendor-core/mcp-tools.js +105 -76
  380. package/vendor-core/model-assembler.js +8 -16
  381. package/vendor-core/occupancy.js +5 -9
  382. package/vendor-core/parser-worker.js +20 -29
  383. package/vendor-core/plan-dot.js +2 -5
  384. package/vendor-core/plan-duration-attribution.js +78 -29
  385. package/vendor-core/plan-graph-model.js +126 -69
  386. package/vendor-core/plan-node-detail.js +31 -17
  387. package/vendor-core/plan-summary.js +19 -8
  388. package/vendor-core/recommendation-rollup.js +35 -39
  389. package/vendor-core/redact.js +72 -16
  390. package/vendor-core/rolling-log-reassembly.js +4 -6
  391. package/vendor-core/run-comparison.js +86 -72
  392. package/vendor-core/scaling-sim.js +5 -7
  393. package/vendor-core/session-snapshot.js +1 -1
  394. package/vendor-core/shs-fetch.js +4 -6
  395. package/vendor-core/shs-load.js +9 -13
  396. package/vendor-core/shs-request.js +1 -1
  397. package/vendor-core/stage-quantiles.js +14 -0
  398. package/vendor-core/types.js +78 -18
  399. package/vendor-core/wasted-core-hours.js +7 -12
  400. package/public/assets/AutoscalingChurn-CP5OjN6v.js +0 -1
  401. package/public/assets/CacheUtilization-BjyawXL6.js +0 -1
  402. package/public/assets/CachingOpportunity-DV2ABG2G.js +0 -1
  403. package/public/assets/ConfigAudit-DMiBhAoG.js +0 -1
  404. package/public/assets/CoreUsageArea-CGYIgPnt.js +0 -1
  405. package/public/assets/Dashboard-D1_IrWRs.js +0 -12
  406. package/public/assets/ExecutorTimeline-DkaAGqQT.js +0 -1
  407. package/public/assets/Failures-Crngoicu.js +0 -1
  408. package/public/assets/GcPressure-CFXm4BnA.js +0 -1
  409. package/public/assets/IncompleteRun-BWEJTb13.js +0 -1
  410. package/public/assets/JobFailures-BmjxCA-h.js +0 -1
  411. package/public/assets/MemoryUtilization-Bu2uQJbd.js +0 -1
  412. package/public/assets/PlanExplorer-DKUwF0uz.js +0 -1
  413. package/public/assets/PlanGraphRoute-BjPSugIG.js +0 -1
  414. package/public/assets/ShuffleIO--Rur3neu.js +0 -1
  415. package/public/assets/Spill-DIMpt8c2.js +0 -1
  416. package/public/assets/TaskSkew-BDr1DbeI.js +0 -1
  417. package/public/assets/charts-vendor-NMqKV0H4.js +0 -59
  418. package/public/assets/index-4id52G-A.js +0 -9
  419. package/public/assets/index-D9kwVYIO.css +0 -1
  420. package/public/assets/parser-worker-00O9WEQC.js +0 -65
  421. package/public/assets/plan-graph-vendor-BnuhLJ6X.css +0 -1
  422. package/public/assets/plan-graph-vendor-vZQHG2EJ.js +0 -7
  423. package/public/assets/react-vendor-u4pyvC5Y.js +0 -9
  424. package/public/assets/useRecentFiles-46iBU9zJ.js +0 -5
  425. package/public/docs/assets/chunks/@localSearchIndexroot.DclgKtlT.js +0 -1
  426. package/public/docs/assets/chunks/theme.CoT6y5bf.js +0 -2
  427. package/public/docs/assets/contributor-guide_architecture_board-widgets.md.BtjoE_1k.js +0 -1
  428. package/public/docs/assets/contributor-guide_architecture_board-widgets.md.BtjoE_1k.lean.js +0 -1
  429. package/public/docs/assets/contributor-guide_architecture_detector-contract.md.DQyCHQNM.js +0 -1
  430. package/public/docs/assets/contributor-guide_architecture_detector-contract.md.DQyCHQNM.lean.js +0 -1
  431. package/public/docs/assets/contributor-guide_architecture_drill-down.md.BKfyDQck.js +0 -1
  432. package/public/docs/assets/contributor-guide_architecture_drill-down.md.BKfyDQck.lean.js +0 -1
  433. package/public/docs/assets/contributor-guide_architecture_state-and-history.md.DaqttKgQ.js +0 -1
  434. package/public/docs/assets/contributor-guide_architecture_state-and-history.md.DaqttKgQ.lean.js +0 -1
  435. package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.Bk6Iyj9S.js +0 -1
  436. package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.Bk6Iyj9S.lean.js +0 -1
  437. package/public/docs/assets/contributor-guide_contributing.md.C-lKs70I.js +0 -1
  438. package/public/docs/assets/contributor-guide_contributing.md.C-lKs70I.lean.js +0 -1
  439. package/public/docs/assets/user-guide_getting-started.md.BJdD7a7_.js +0 -3
  440. package/public/docs/assets/user-guide_getting-started.md.BJdD7a7_.lean.js +0 -1
  441. package/public/docs/assets/user-guide_run-comparison.md.5-oaOfTu.js +0 -1
  442. package/public/docs/assets/user-guide_run-comparison.md.5-oaOfTu.lean.js +0 -1
  443. package/public/docs/assets/user-guide_understanding-findings.md.CK2WZ0Tf.js +0 -1
  444. package/public/docs/assets/user-guide_understanding-findings.md.CK2WZ0Tf.lean.js +0 -1
  445. package/public/vendor/spark-doc/anchors.json +0 -205
  446. package/public/vendor/spark-doc/index.html +0 -4202
  447. package/public/vendor/spark-doc/landing.html +0 -418
  448. package/public/vendor/spark-doc/meta.html +0 -970
package/public/docs/user-guide/run-comparison.html CHANGED
@@ -6,14 +6,14 @@
6
6
  <title>Run comparison mode | SparkForensics</title>
7
7
  <meta name="description" content="Docs for using and contributing to SparkForensics">
8
8
  <meta name="generator" content="VitePress v1.6.4">
9
- <link rel="preload stylesheet" href="/docs/assets/style.D2mY0HUs.css" as="style">
9
+ <link rel="preload stylesheet" href="/docs/assets/style.DSixAiZE.css" as="style">
10
10
  <link rel="preload stylesheet" href="/docs/vp-icons.css" as="style">
11
11
 
12
- <script type="module" src="/docs/assets/app.Bxy4_HVA.js"></script>
12
+ <script type="module" src="/docs/assets/app.APoq_UBs.js"></script>
13
13
  <link rel="preload" href="/docs/assets/inter-roman-latin.Di8DUHzh.woff2" as="font" type="font/woff2" crossorigin="">
14
- <link rel="modulepreload" href="/docs/assets/chunks/theme.CoT6y5bf.js">
15
- <link rel="modulepreload" href="/docs/assets/chunks/framework.CNDaxUod.js">
16
- <link rel="modulepreload" href="/docs/assets/user-guide_run-comparison.md.5-oaOfTu.lean.js">
14
+ <link rel="modulepreload" href="/docs/assets/chunks/theme.PavXO_vz.js">
15
+ <link rel="modulepreload" href="/docs/assets/chunks/framework.DSg0KOwT.js">
16
+ <link rel="modulepreload" href="/docs/assets/user-guide_run-comparison.md.CQc1aoU8.lean.js">
17
17
  <link rel="icon" type="image/svg+xml" href="/docs/favicon.svg">
18
18
  <link rel="preconnect" href="https://fonts.googleapis.com">
19
19
  <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin="">
@@ -22,8 +22,8 @@
22
22
  <script id="check-mac-os">document.documentElement.classList.toggle("mac",/Mac|iPhone|iPod|iPad/i.test(navigator.platform));</script>
23
23
  </head>
24
24
  <body>
25
- <div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>User Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/getting-started.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Getting started</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/understanding-findings.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Understanding findings</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/run-comparison.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Run comparison mode</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/mcp-tools.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>MCP tools reference</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_user-guide_run-comparison" data-v-39a288b8><div><h1 id="run-comparison-mode" tabindex="-1">Run comparison mode <a class="header-anchor" href="#run-comparison-mode" aria-label="Permalink to &quot;Run comparison mode&quot;">​</a></h1><p>Put two runs side by side to see whether a tuning change helped.</p><h2 id="starting-a-comparison" tabindex="-1">Starting a comparison <a class="header-anchor" href="#starting-a-comparison" aria-label="Permalink to &quot;Starting a comparison&quot;">​</a></h2><p>From the landing page, click <strong>Compare two runs</strong>. Two slots appear, <strong>Run A</strong> and <strong>Run B</strong>.</p><ol><li>Load a file into each slot the same way you&#39;d load a single run (drag and drop, or <strong>Choose file</strong>).</li><li>Click <strong>Compare</strong>.</li></ol><p>The runs parse one after the other through the same background worker. When both finish, the comparison view opens.</p><h2 id="reading-the-comparison" tabindex="-1">Reading the comparison <a class="header-anchor" href="#reading-the-comparison" aria-label="Permalink to &quot;Reading the comparison&quot;">​</a></h2><p>The comparison page shows what changed between the two runs: findings that appeared or disappeared, plus metric deltas for duration, spill, GC time and so on.</p><p>For either run&#39;s full dashboard, click <strong>View run A dashboard</strong> or <strong>View run B dashboard</strong>. <strong>← Back to comparison</strong> takes you back.</p></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/user-guide/understanding-findings.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>Understanding findings</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="/docs/user-guide/mcp-tools.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>MCP tools reference</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
26
- <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"BtjoE_1k\",\"contributor-guide_architecture_detector-contract.md\":\"DQyCHQNM\",\"contributor-guide_architecture_drill-down.md\":\"BKfyDQck\",\"contributor-guide_architecture_impact-estimation.md\":\"CzlXOnCq\",\"contributor-guide_architecture_index.md\":\"94lNbF-4\",\"contributor-guide_architecture_overview.md\":\"mQFlN66o\",\"contributor-guide_architecture_state-and-history.md\":\"DaqttKgQ\",\"contributor-guide_architecture_widget-rendering.md\":\"Bk6Iyj9S\",\"contributor-guide_architecture_worker-protocol.md\":\"BaS_L10g\",\"contributor-guide_contributing.md\":\"C-lKs70I\",\"contributor-guide_development-setup.md\":\"CpUMVtIK\",\"contributor-guide_testing.md\":\"C0q-BJfl\",\"index.md\":\"CETnDZzR\",\"user-guide_getting-started.md\":\"BJdD7a7_\",\"user-guide_mcp-tools.md\":\"DwYI1d6C\",\"user-guide_run-comparison.md\":\"5-oaOfTu\",\"user-guide_understanding-findings.md\":\"CK2WZ0Tf\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"}],\"sidebar\":{\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
25
+ <div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>User Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/getting-started.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Getting started</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/understanding-findings.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Understanding findings</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/run-comparison.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Run comparison mode</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/mcp-tools.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>MCP tools reference</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/alternative-log-retrieval.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Alternative log retrieval</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_user-guide_run-comparison" data-v-39a288b8><div><h1 id="run-comparison-mode" tabindex="-1">Run comparison mode <a class="header-anchor" href="#run-comparison-mode" aria-label="Permalink to &quot;Run comparison mode&quot;">​</a></h1><p>Put two runs side by side to see whether a tuning change helped.</p><h2 id="starting-a-comparison" tabindex="-1">Starting a comparison <a class="header-anchor" href="#starting-a-comparison" aria-label="Permalink to &quot;Starting a comparison&quot;">​</a></h2><p>From the landing page, click <strong>Compare two runs</strong>. Two slots appear: <strong>Run A (baseline)</strong> and <strong>Run B (candidate)</strong>.</p><ol><li>Load a file into each slot the same way you&#39;d load a single run (drag and drop, or <strong>Choose file</strong>).</li><li>Click <strong>Compare</strong>.</li></ol><p>The runs parse one after the other through the same background worker. When both finish, the comparison view opens.</p><h2 id="reading-the-comparison" tabindex="-1">Reading the comparison <a class="header-anchor" href="#reading-the-comparison" aria-label="Permalink to &quot;Reading the comparison&quot;">​</a></h2><p>The <strong>Metrics</strong> table covers the whole run: wall-clock duration, shuffle spill, task skew, failed-task rate, disk spill, GC time, input/output bytes, executor run-time, and task/executor counts, baseline against candidate with the delta. <strong>Findings by category</strong> lists which finding types appeared or disappeared between the two runs, with the affected stages for each.</p><p>Stage-level detail depends on matching a stage in the baseline to its counterpart in the candidate, and SparkForensics only does that automatically when a stage&#39;s identity (its position in the SQL plan) resolves to exactly one match on both sides; AQE&#39;s runtime replanning makes a plain stage-ID match unreliable. The <strong>Per-stage task skew</strong> table covers only stages matched this way, and the page states what percentage of stages that was. If matching is uncertain (for example, the two runs came from different application names), a warning banner says so; the metric deltas above it still hold; they don&#39;t depend on stage matching.</p><p>For everything else, matching is manual: the <strong>Pinned per-stage deltas</strong> widget lets you pick one stage from the baseline and one from the candidate yourself and pin the pair, then shows their duration, executor run-time, GC time, memory/disk spill, input/output bytes, task count, and failed tasks side by side. Pin as many pairs as you want to check.</p><p>For either run&#39;s full dashboard, click <strong>View run A dashboard</strong> or <strong>View run B dashboard</strong>. <strong>← Back to comparison</strong> takes you back.</p><h2 id="comparing-without-the-dashboard" tabindex="-1">Comparing without the dashboard <a class="header-anchor" href="#comparing-without-the-dashboard" aria-label="Permalink to &quot;Comparing without the dashboard&quot;">​</a></h2><p>The same comparison runs headlessly. The CLI&#39;s <code>--baseline</code> flag adds a comparison section to its output and can gate a build on it (<code>--max-regression-pct</code>, <code>--fail-on-introduced</code>; see <a href="./getting-started.html#ci-and-automation">Getting started</a>). The MCP server&#39;s <code>compare_runs</code> and <code>evaluate_budgets</code> tools do the same for an AI assistant; see <a href="./mcp-tools.html">MCP tools reference</a>.</p></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/user-guide/understanding-findings.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>Understanding findings</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="/docs/user-guide/mcp-tools.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>MCP tools reference</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
26
+ <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"CWpj01WU\",\"contributor-guide_architecture_detector-contract.md\":\"CgzUsQ6W\",\"contributor-guide_architecture_drill-down.md\":\"BtPdlM7r\",\"contributor-guide_architecture_impact-estimation.md\":\"CooslVJt\",\"contributor-guide_architecture_index.md\":\"3TO9ic6w\",\"contributor-guide_architecture_overview.md\":\"CehiRmGn\",\"contributor-guide_architecture_state-and-history.md\":\"C-xxn0q7\",\"contributor-guide_architecture_widget-rendering.md\":\"R27gQrgY\",\"contributor-guide_architecture_worker-protocol.md\":\"IbnfNrV3\",\"contributor-guide_contributing.md\":\"CvRsdr6J\",\"contributor-guide_development-setup.md\":\"DvAN_9mK\",\"contributor-guide_testing.md\":\"6rIKqSyY\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"Df1YMIHu\",\"tuning-reference_aqe.md\":\"BIsCtLzm\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"CEstB3Ia\",\"tuning-reference_bottleneck-cold-start.md\":\"CEuy-72y\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"CIohQDfn\",\"tuning-reference_bottleneck-failures.md\":\"4z5BXGJ2\",\"tuning-reference_bottleneck-gc.md\":\"DSxzZRK7\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"BaJl__1W\",\"tuning-reference_bottleneck-memory-utilization.md\":\"DbP-SJZc\",\"tuning-reference_bottleneck-retry-waste.md\":\"D5JMjVOt\",\"tuning-reference_bottleneck-shuffle.md\":\"CM-nTmIH\",\"tuning-reference_bottleneck-skew.md\":\"BdUwiDhn\",\"tuning-reference_bottleneck-slow-host.md\":\"BlIo6UDW\",\"tuning-reference_bottleneck-small-files.md\":\"B8kloyx8\",\"tuning-reference_bottleneck-spill.md\":\"PNH7mITt\",\"tuning-reference_bottleneck-straggler.md\":\"DY36fHN5\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"QTV7O8kU\",\"tuning-reference_bottleneck-utilization.md\":\"DTiueZC3\",\"tuning-reference_caching.md\":\"B7aQ8asB\",\"tuning-reference_cluster-config.md\":\"ZVmDGsQ3\",\"tuning-reference_config.md\":\"UvveiWG3\",\"tuning-reference_data-formats.md\":\"bjCAWH3N\",\"tuning-reference_index.md\":\"BQ_NooMV\",\"tuning-reference_intro.md\":\"CobD-lGB\",\"tuning-reference_joins.md\":\"BtKs_CuW\",\"tuning-reference_memory-model.md\":\"DhT-n4y3\",\"tuning-reference_metrics.md\":\"mLOh7Apj\",\"tuning-reference_partitioning.md\":\"q0zKF_8X\",\"tuning-reference_pyspark.md\":\"DDCfvN9t\",\"tuning-reference_shuffle.md\":\"BZZ7R4Ix\",\"tuning-reference_spark-architecture.md\":\"Dwzm5avO\",\"tuning-reference_table-formats.md\":\"D6wj-2dX\",\"user-guide_alternative-log-retrieval.md\":\"B4tPGIal\",\"user-guide_getting-started.md\":\"BJvwLEIM\",\"user-guide_mcp-tools.md\":\"Vi3RoflJ\",\"user-guide_run-comparison.md\":\"CQc1aoU8\",\"user-guide_understanding-findings.md\":\"DL1UDhvR\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Alternative log retrieval\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
27
27
 
28
28
  </body>
29
29
  </html>
package/public/docs/user-guide/understanding-findings.html CHANGED
@@ -6,14 +6,14 @@
6
6
  <title>Understanding findings | SparkForensics</title>
7
7
  <meta name="description" content="Docs for using and contributing to SparkForensics">
8
8
  <meta name="generator" content="VitePress v1.6.4">
9
- <link rel="preload stylesheet" href="/docs/assets/style.D2mY0HUs.css" as="style">
9
+ <link rel="preload stylesheet" href="/docs/assets/style.DSixAiZE.css" as="style">
10
10
  <link rel="preload stylesheet" href="/docs/vp-icons.css" as="style">
11
11
 
12
- <script type="module" src="/docs/assets/app.Bxy4_HVA.js"></script>
12
+ <script type="module" src="/docs/assets/app.APoq_UBs.js"></script>
13
13
  <link rel="preload" href="/docs/assets/inter-roman-latin.Di8DUHzh.woff2" as="font" type="font/woff2" crossorigin="">
14
- <link rel="modulepreload" href="/docs/assets/chunks/theme.CoT6y5bf.js">
15
- <link rel="modulepreload" href="/docs/assets/chunks/framework.CNDaxUod.js">
16
- <link rel="modulepreload" href="/docs/assets/user-guide_understanding-findings.md.CK2WZ0Tf.lean.js">
14
+ <link rel="modulepreload" href="/docs/assets/chunks/theme.PavXO_vz.js">
15
+ <link rel="modulepreload" href="/docs/assets/chunks/framework.DSg0KOwT.js">
16
+ <link rel="modulepreload" href="/docs/assets/user-guide_understanding-findings.md.DL1UDhvR.lean.js">
17
17
  <link rel="icon" type="image/svg+xml" href="/docs/favicon.svg">
18
18
  <link rel="preconnect" href="https://fonts.googleapis.com">
19
19
  <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin="">
@@ -22,8 +22,8 @@
22
22
  <script id="check-mac-os">document.documentElement.classList.toggle("mac",/Mac|iPhone|iPod|iPad/i.test(navigator.platform));</script>
23
23
  </head>
24
24
  <body>
25
- <div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>User Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/getting-started.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Getting started</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/understanding-findings.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Understanding findings</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/run-comparison.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Run comparison mode</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/mcp-tools.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>MCP tools reference</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_user-guide_understanding-findings" data-v-39a288b8><div><h1 id="understanding-findings" tabindex="-1">Understanding findings <a class="header-anchor" href="#understanding-findings" aria-label="Permalink to &quot;Understanding findings&quot;">​</a></h1><p>Every flagged problem carries a short ALL-CAPS tag. This page has one entry per tag: what it means, and what to do about it.</p><p>A few tags share their in-app &quot;Reference panel&quot; background reading with another tag, because the underlying Spark-tuning material overlaps: <code>SFAIL</code> with <code>FAIL</code>, <code>PART</code> with <code>SHFL</code>, <code>SPEC</code> with <code>STRAG</code>, and <code>CACHE</code>/<code>LOCAL</code> with <code>UTIL</code>.</p><h2 id="per-stage" tabindex="-1">Per-stage <a class="header-anchor" href="#per-stage" aria-label="Permalink to &quot;Per-stage&quot;">​</a></h2><h3 id="skew" tabindex="-1"><code>SKEW</code>: Task skew <a class="header-anchor" href="#skew" aria-label="Permalink to &quot;`SKEW`: Task skew {#skew}&quot;">​</a></h3><p>A small number of tasks take much longer than their peers in the same stage. For join-driven skew, enable AQE skew-join handling (<code>spark.sql.adaptive.skewJoin.enabled</code>); otherwise salt the key or repartition on a better key.</p><h3 id="shfl" tabindex="-1"><code>SHFL</code>: Shuffle I/O <a class="header-anchor" href="#shfl" aria-label="Permalink to &quot;`SHFL`: Shuffle I/O {#shfl}&quot;">​</a></h3><p>Tasks move a large amount of intermediate data between stages. Raise <code>spark.sql.shuffle.partitions</code>, or add a broadcast join.</p><h3 id="spill" tabindex="-1"><code>SPILL</code>: Memory and disk spill <a class="header-anchor" href="#spill" aria-label="Permalink to &quot;`SPILL`: Memory and disk spill {#spill}&quot;">​</a></h3><p>Tasks are writing data out of memory, which slows execution. Two spill patterns get flagged differently: skew spill, where a few heavy tasks spill while most don&#39;t (rebalance partitioning), and volume spill, where most tasks spill because the data genuinely exceeds available memory (add partitions).</p><h3 id="gc" tabindex="-1"><code>GC</code>: Garbage collection pressure <a class="header-anchor" href="#gc" aria-label="Permalink to &quot;`GC`: Garbage collection pressure {#gc}&quot;">​</a></h3><p>Tasks spend an unusually large share of time reclaiming memory. Reduce object creation: use primitive types, avoid UDFs, or raise executor memory.</p><h3 id="fail" tabindex="-1"><code>FAIL</code>: Failed tasks <a class="header-anchor" href="#fail" aria-label="Permalink to &quot;`FAIL`: Failed tasks {#fail}&quot;">​</a></h3><p>Tasks fail often enough to affect the stage. Failed tasks point to executor instability or data-driven errors: check driver logs for the dominant failure reason.</p><h3 id="sfail" tabindex="-1"><code>SFAIL</code>: Failed stage <a class="header-anchor" href="#sfail" aria-label="Permalink to &quot;`SFAIL`: Failed stage {#sfail}&quot;">​</a></h3><p>A stage attempt failed outright rather than losing individual tasks within it. Inspect the driver log for the failure reason and the job that triggered it.</p><h3 id="strag" tabindex="-1"><code>STRAG</code>: Straggler tasks <a class="header-anchor" href="#strag" aria-label="Permalink to &quot;`STRAG`: Straggler tasks {#strag}&quot;">​</a></h3><p>A few tasks run much slower than the rest of their stage. Likely causes: data skew that AQE&#39;s skew-join handling would catch, or data-locality issues.</p><h3 id="spec" tabindex="-1"><code>SPEC</code>: Speculation waste <a class="header-anchor" href="#spec" aria-label="Permalink to &quot;`SPEC`: Speculation waste {#spec}&quot;">​</a></h3><p>Speculative task attempts used a lot of executor time without confirming a genuine straggler. Self-flagged low-confidence: a design spike, not yet validated against real-world runs. If task durations are just naturally variable rather than genuine stragglers, tune <code>spark.speculation.multiplier</code>/<code>spark.speculation.quantile</code>.</p><h3 id="retry" tabindex="-1"><code>RETRY</code>: Retry waste <a class="header-anchor" href="#retry" aria-label="Permalink to &quot;`RETRY`: Retry waste {#retry}&quot;">​</a></h3><p>Repeated task attempts ate into execution time even though the stage completed. Investigate executor loss or fetch failures.</p><h3 id="tiny" tabindex="-1"><code>TINY</code>: Tiny tasks <a class="header-anchor" href="#tiny" aria-label="Permalink to &quot;`TINY`: Tiny tasks {#tiny}&quot;">​</a></h3><p>Many very short tasks add scheduling overhead out of proportion to the work each one does. Repartition to fewer, larger tasks.</p><h3 id="part" tabindex="-1"><code>PART</code>: Partition sizing <a class="header-anchor" href="#part" aria-label="Permalink to &quot;`PART`: Partition sizing {#part}&quot;">​</a></h3><p>Shuffle partitions are too large, too uneven, or too few for the work. A single shuffle partition over 5 GB, for example, will OOM or spill heavily: repartition to break it up before the stage runs.</p><h3 id="slow" tabindex="-1"><code>SLOW</code>: Stage slowness <a class="header-anchor" href="#slow" aria-label="Permalink to &quot;`SLOW`: Stage slowness {#slow}&quot;">​</a></h3><p>A stage is slow overall (high average executor runtime) without a more specific cause getting flagged. Profile its query plan and check for wide shuffles or expensive UDFs.</p><h3 id="shape" tabindex="-1"><code>SHAPE</code>: Stage shape <a class="header-anchor" href="#shape" aria-label="Permalink to &quot;`SHAPE`: Stage shape {#shape}&quot;">​</a></h3><p>The stage has an inefficient task count, output shape, or task-to-stage balance: for example, one straggler task taking a large fraction of the stage&#39;s wall-clock time.</p><h3 id="host" tabindex="-1"><code>HOST</code>: Slow host <a class="header-anchor" href="#host" aria-label="Permalink to &quot;`HOST`: Slow host {#host}&quot;">​</a></h3><p>One executor is much slower than its peers. Check that executor&#39;s logs: possible bad node, disk pressure, or NUMA misalignment.</p><h2 id="app-level" tabindex="-1">App-level <a class="header-anchor" href="#app-level" aria-label="Permalink to &quot;App-level&quot;">​</a></h2><h3 id="cold" tabindex="-1"><code>COLD</code>: Executor cold start <a class="header-anchor" href="#cold" aria-label="Permalink to &quot;`COLD`: Executor cold start {#cold}&quot;">​</a></h3><p>New executors take time to become available for work. Pre-warm the cluster, or use dynamic allocation.</p><h3 id="util" tabindex="-1"><code>UTIL</code>: Low utilization <a class="header-anchor" href="#util" aria-label="Permalink to &quot;`UTIL`: Low utilization {#util}&quot;">​</a></h3><p>Allocated executors sit idle for a large share of the application run. Consider a smaller cluster, or enable dynamic allocation.</p><h3 id="mem" tabindex="-1"><code>MEM</code>: Memory utilization <a class="header-anchor" href="#mem" aria-label="Permalink to &quot;`MEM`: Memory utilization {#mem}&quot;">​</a></h3><p>Executor memory or core capacity may be over- or under-provisioned. Some detail here needs <code>spark.eventLog.logStageExecutorMetrics=true</code> on the run being analyzed; without it, per-executor memory usage can&#39;t be broken down. Review <code>spark.executor.memory</code> and executor count if allocated memory sat largely idle over the run. That idle-memory variant is self-flagged low-confidence: it estimates waste from allocated-versus-used memory-time against an unverified 1.5x buffer ported from Dr. Elephant. Check it against the Spark UI before resizing anything.</p><h3 id="cache" tabindex="-1"><code>CACHE</code>: Caching opportunity <a class="header-anchor" href="#cache" aria-label="Permalink to &quot;`CACHE`: Caching opportunity {#cache}&quot;">​</a></h3><p>A reusable dataset (re-read via the same SQL relation more than once) may be worth persisting between stages. Self-flagged low-confidence: reuse is only inferred, from plan-scan identity across SQL executions, so confirm the reads really do hit the same data before you cache anything.</p><h3 id="cstor" tabindex="-1"><code>CSTOR</code>: Cache storage <a class="header-anchor" href="#cstor" aria-label="Permalink to &quot;`CSTOR`: Cache storage {#cstor}&quot;">​</a></h3><p>A persisted dataset is not fully cached in memory, or is spilling to disk. Raise executor memory, or shrink the cached dataset.</p><h3 id="local" tabindex="-1"><code>LOCAL</code>: Core usage locality <a class="header-anchor" href="#local" aria-label="Permalink to &quot;`LOCAL`: Core usage locality {#local}&quot;">​</a></h3><p>Tasks run without process- or node-local data placement more often than expected. Check <code>spark.locality.wait</code> settings and executor/data colocation. Self-flagged low-confidence: the non-local-ratio thresholds are unvalidated design-spike values, and no external tool publishes an equivalent metric to calibrate them against.</p><h3 id="chrn" tabindex="-1"><code>CHRN</code>: Autoscaling churn <a class="header-anchor" href="#chrn" aria-label="Permalink to &quot;`CHRN`: Autoscaling churn {#chrn}&quot;">​</a></h3><p>Executors are stood up and torn down again before they can do useful work: re-provisioning churn rather than normal scale-down. Raise <code>spark.dynamicAllocation.executorIdleTimeout</code>, or widen the <code>minExecutors</code>/<code>maxExecutors</code> bounds to reduce flapping. Self-flagged low-confidence: a design spike, not yet validated against real-world runs.</p><h3 id="jobs" tabindex="-1"><code>JOBS</code>: Job failure rate <a class="header-anchor" href="#jobs" aria-label="Permalink to &quot;`JOBS`: Job failure rate {#jobs}&quot;">​</a></h3><p>A large share of completed jobs did not succeed. Inspect the driver log for the failed job(s) and the stage failures that triggered them.</p><h3 id="incmp" tabindex="-1"><code>INCMP</code>: Incomplete run <a class="header-anchor" href="#incmp" aria-label="Permalink to &quot;`INCMP`: Incomplete run {#incmp}&quot;">​</a></h3><p>This event log never recorded an <code>ApplicationEnd</code> event: capture stopped before the run finished (an in-flight job, a rotated-away log, or a cut-short capture). Every other finding and metric on the board reflects only what was captured up to that point, not the full run.</p><h2 id="configuration-scope" tabindex="-1">Configuration scope <a class="header-anchor" href="#configuration-scope" aria-label="Permalink to &quot;Configuration scope&quot;">​</a></h2><h3 id="cfg" tabindex="-1"><code>CFG</code>: Configuration audit <a class="header-anchor" href="#cfg" aria-label="Permalink to &quot;`CFG`: Configuration audit {#cfg}&quot;">​</a></h3><p>Flags configuration settings that may cause reliability or efficiency problems, independent of any one stage&#39;s behavior. Four properties are audited today:</p><ul><li><code>spark.shuffle.service.enabled</code>: flagged when dynamic allocation is on but the external shuffle service is off, since shuffle data won&#39;t survive executor removal.</li><li><code>spark.dynamicAllocation.maxExecutors</code>: flagged for inverted bounds or a missing upper bound.</li><li><code>spark.serializer</code>: flagged when still on the default Java serializer; <code>org.apache.spark.serializer.KryoSerializer</code> is faster and produces smaller buffers.</li><li><code>spark.executor.memoryOverhead</code>: flagged when set below a safe floor.</li></ul><h2 id="sql-scope" tabindex="-1">SQL scope <a class="header-anchor" href="#sql-scope" aria-label="Permalink to &quot;SQL scope&quot;">​</a></h2><h3 id="plan" tabindex="-1"><code>PLAN</code>: Plan advisor <a class="header-anchor" href="#plan" aria-label="Permalink to &quot;`PLAN`: Plan advisor {#plan}&quot;">​</a></h3><p>Flags patterns in the SQL execution plan worth reviewing. Four checks share this tag:</p><ul><li>Duplicate plan subtree: the same subtree recomputed more than once in the plan.</li><li>Small files: reading an excessive number of small files.</li><li>Under-broadcast: the smaller side of a Sort Merge Join looks well under the broadcast threshold; consider a <code>broadcast()</code> hint or raising <code>spark.sql.autoBroadcastJoinThreshold</code>.</li><li>Over-broadcast: a broadcast exceeds the 1 GB threshold; check for a misapplied broadcast hint or a misconfigured <code>spark.sql.autoBroadcastJoinThreshold</code>.</li></ul></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/user-guide/getting-started.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>Getting started</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="/docs/user-guide/run-comparison.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>Run comparison mode</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
26
- <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"BtjoE_1k\",\"contributor-guide_architecture_detector-contract.md\":\"DQyCHQNM\",\"contributor-guide_architecture_drill-down.md\":\"BKfyDQck\",\"contributor-guide_architecture_impact-estimation.md\":\"CzlXOnCq\",\"contributor-guide_architecture_index.md\":\"94lNbF-4\",\"contributor-guide_architecture_overview.md\":\"mQFlN66o\",\"contributor-guide_architecture_state-and-history.md\":\"DaqttKgQ\",\"contributor-guide_architecture_widget-rendering.md\":\"Bk6Iyj9S\",\"contributor-guide_architecture_worker-protocol.md\":\"BaS_L10g\",\"contributor-guide_contributing.md\":\"C-lKs70I\",\"contributor-guide_development-setup.md\":\"CpUMVtIK\",\"contributor-guide_testing.md\":\"C0q-BJfl\",\"index.md\":\"CETnDZzR\",\"user-guide_getting-started.md\":\"BJdD7a7_\",\"user-guide_mcp-tools.md\":\"DwYI1d6C\",\"user-guide_run-comparison.md\":\"5-oaOfTu\",\"user-guide_understanding-findings.md\":\"CK2WZ0Tf\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"}],\"sidebar\":{\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
25
+ <div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>User Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/getting-started.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Getting started</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/understanding-findings.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Understanding findings</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/run-comparison.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Run comparison mode</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/mcp-tools.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>MCP tools reference</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/alternative-log-retrieval.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Alternative log retrieval</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_user-guide_understanding-findings" data-v-39a288b8><div><h1 id="understanding-findings" tabindex="-1">Understanding findings <a class="header-anchor" href="#understanding-findings" aria-label="Permalink to &quot;Understanding findings&quot;">​</a></h1><p>Every flagged problem carries a short ALL-CAPS tag. This page has one entry per tag: what it means, and what to do about it.</p><p>Turn on <a href="./getting-started.html#advanced-view">Advanced view</a> in the dashboard to see, per finding, the confidence level and a link to the same background reading collected here, opened in an in-app &quot;Reference&quot; panel next to the board. A few tags share that reading with another tag, because the underlying Spark-tuning material overlaps: <code>SFAIL</code> with <code>FAIL</code>, <code>PART</code> with <code>SHFL</code>, <code>SPEC</code> with <code>STRAG</code>, and <code>CACHE</code>/<code>LOCAL</code> with <code>UTIL</code>.</p><h2 id="per-stage" tabindex="-1">Per-stage <a class="header-anchor" href="#per-stage" aria-label="Permalink to &quot;Per-stage&quot;">​</a></h2><h3 id="skew" tabindex="-1"><code>SKEW</code>: Task skew <a class="header-anchor" href="#skew" aria-label="Permalink to &quot;`SKEW`: Task skew {#skew}&quot;">​</a></h3><p>A small number of tasks take much longer than their peers in the same stage. For join-driven skew, enable AQE skew-join handling (<code>spark.sql.adaptive.skewJoin.enabled</code>); otherwise salt the key or repartition on a better key.</p><h3 id="shfl" tabindex="-1"><code>SHFL</code>: Shuffle I/O <a class="header-anchor" href="#shfl" aria-label="Permalink to &quot;`SHFL`: Shuffle I/O {#shfl}&quot;">​</a></h3><p>Tasks move a large amount of intermediate data between stages. Raise <code>spark.sql.shuffle.partitions</code>, or add a broadcast join.</p><h3 id="spill" tabindex="-1"><code>SPILL</code>: Memory and disk spill <a class="header-anchor" href="#spill" aria-label="Permalink to &quot;`SPILL`: Memory and disk spill {#spill}&quot;">​</a></h3><p>Tasks are writing data out of memory, which slows execution. Two spill patterns get flagged differently: skew spill, where a few heavy tasks spill while most don&#39;t (rebalance partitioning), and volume spill, where most tasks spill because the data genuinely exceeds available memory (add partitions).</p><h3 id="gc" tabindex="-1"><code>GC</code>: Garbage collection pressure <a class="header-anchor" href="#gc" aria-label="Permalink to &quot;`GC`: Garbage collection pressure {#gc}&quot;">​</a></h3><p>Tasks spend an unusually large share of time reclaiming memory. Reduce object creation: use primitive types, avoid UDFs, or raise executor memory.</p><h3 id="fail" tabindex="-1"><code>FAIL</code>: Failed tasks <a class="header-anchor" href="#fail" aria-label="Permalink to &quot;`FAIL`: Failed tasks {#fail}&quot;">​</a></h3><p>Tasks fail often enough to affect the stage. Failed tasks point to executor instability or data-driven errors: check driver logs for the dominant failure reason.</p><h3 id="sfail" tabindex="-1"><code>SFAIL</code>: Failed stage <a class="header-anchor" href="#sfail" aria-label="Permalink to &quot;`SFAIL`: Failed stage {#sfail}&quot;">​</a></h3><p>A stage attempt failed outright rather than losing individual tasks within it. Inspect the driver log for the failure reason and the job that triggered it.</p><h3 id="strag" tabindex="-1"><code>STRAG</code>: Straggler tasks <a class="header-anchor" href="#strag" aria-label="Permalink to &quot;`STRAG`: Straggler tasks {#strag}&quot;">​</a></h3><p>A few tasks run much slower than the rest of their stage. Rule out a GC pause or a slow shuffle fetch before assuming a hardware issue; if a skewed key is the real cause, that&#39;s a candidate for AQE&#39;s skew-join handling.</p><h3 id="spec" tabindex="-1"><code>SPEC</code>: Speculation waste <a class="header-anchor" href="#spec" aria-label="Permalink to &quot;`SPEC`: Speculation waste {#spec}&quot;">​</a></h3><p>Speculative task attempts used a lot of executor time without confirming a genuine straggler. Self-flags a confidence that scales with how far the wasted time sits past the threshold: these thresholds are still a design spike, not yet validated against real-world runs. If task durations are just naturally variable rather than genuine stragglers, tune <code>spark.speculation.multiplier</code>/<code>spark.speculation.quantile</code>.</p><h3 id="retry" tabindex="-1"><code>RETRY</code>: Retry waste <a class="header-anchor" href="#retry" aria-label="Permalink to &quot;`RETRY`: Retry waste {#retry}&quot;">​</a></h3><p>Repeated task attempts ate into execution time even though the stage completed. Investigate executor loss or fetch failures.</p><h3 id="tiny" tabindex="-1"><code>TINY</code>: Tiny tasks <a class="header-anchor" href="#tiny" aria-label="Permalink to &quot;`TINY`: Tiny tasks {#tiny}&quot;">​</a></h3><p>Many very short tasks add scheduling overhead out of proportion to the work each one does. Repartition to fewer, larger tasks.</p><h3 id="part" tabindex="-1"><code>PART</code>: Partition sizing <a class="header-anchor" href="#part" aria-label="Permalink to &quot;`PART`: Partition sizing {#part}&quot;">​</a></h3><p>Shuffle partitions are too large, too uneven, or too few for the work. A single shuffle partition over 5 GB, for example, will OOM or spill heavily: repartition to break it up before the stage runs.</p><h3 id="slow" tabindex="-1"><code>SLOW</code>: Stage slowness <a class="header-anchor" href="#slow" aria-label="Permalink to &quot;`SLOW`: Stage slowness {#slow}&quot;">​</a></h3><p>A stage ran long overall without a more specific cause getting flagged. Often a partition-count problem: raise parallelism via <code>spark.sql.shuffle.partitions</code> or <code>spark.default.parallelism</code>, or check for a large per-task data volume driving heavy shuffle and spill.</p><h3 id="shape" tabindex="-1"><code>SHAPE</code>: Stage shape <a class="header-anchor" href="#shape" aria-label="Permalink to &quot;`SHAPE`: Stage shape {#shape}&quot;">​</a></h3><p>The stage has an inefficient task count, output shape, or task-to-stage balance: for example, one straggler task taking a large fraction of the stage&#39;s wall-clock time.</p><h3 id="host" tabindex="-1"><code>HOST</code>: Slow host <a class="header-anchor" href="#host" aria-label="Permalink to &quot;`HOST`: Slow host {#host}&quot;">​</a></h3><p>One executor is much slower than its peers. It may just hold data locality for its tasks or carry one heavy stage, rather than a hardware fault. Enable <code>spark.speculation</code> to relaunch a lagging task automatically.</p><h2 id="app-level" tabindex="-1">App-level <a class="header-anchor" href="#app-level" aria-label="Permalink to &quot;App-level&quot;">​</a></h2><h3 id="cold" tabindex="-1"><code>COLD</code>: Executor cold start <a class="header-anchor" href="#cold" aria-label="Permalink to &quot;`COLD`: Executor cold start {#cold}&quot;">​</a></h3><p>New executors take time to become available for work. Pre-warm the cluster, or use dynamic allocation.</p><h3 id="util" tabindex="-1"><code>UTIL</code>: Low utilization <a class="header-anchor" href="#util" aria-label="Permalink to &quot;`UTIL`: Low utilization {#util}&quot;">​</a></h3><p>Allocated executors sit idle for a large share of the application run. Consider a smaller cluster, or enable dynamic allocation.</p><h3 id="mem" tabindex="-1"><code>MEM</code>: Memory utilization <a class="header-anchor" href="#mem" aria-label="Permalink to &quot;`MEM`: Memory utilization {#mem}&quot;">​</a></h3><p>Executor memory or core capacity may be over- or under-provisioned. Some detail here needs <code>spark.eventLog.logStageExecutorMetrics=true</code> on the run being analyzed; without it, per-executor memory usage can&#39;t be broken down. Review <code>spark.executor.memory</code> and executor count if allocated memory sat largely idle over the run. That idle-memory variant self-flags a confidence that scales with how far the estimated waste sits past a 1.5x buffer: it estimates waste from allocated-versus-used memory-time. Check it against the Spark UI before resizing anything.</p><h3 id="cache" tabindex="-1"><code>CACHE</code>: Caching opportunity <a class="header-anchor" href="#cache" aria-label="Permalink to &quot;`CACHE`: Caching opportunity {#cache}&quot;">​</a></h3><p>A reusable dataset (re-read via the same SQL relation more than once) may be worth persisting between stages. Self-flags a confidence that scales with how many executions reuse the same relation: reuse is only inferred, from plan-scan identity across SQL executions, so confirm the reads really do hit the same data before you cache anything.</p><h3 id="cstor" tabindex="-1"><code>CSTOR</code>: Cache storage <a class="header-anchor" href="#cstor" aria-label="Permalink to &quot;`CSTOR`: Cache storage {#cstor}&quot;">​</a></h3><p>A persisted dataset is not fully cached in memory, or is spilling to disk. Raise executor memory, or shrink the cached dataset.</p><h3 id="local" tabindex="-1"><code>LOCAL</code>: Core usage locality <a class="header-anchor" href="#local" aria-label="Permalink to &quot;`LOCAL`: Core usage locality {#local}&quot;">​</a></h3><p>Tasks run without process- or node-local data placement more often than expected. Check <code>spark.locality.wait</code> settings and executor/data colocation. Self-flags a confidence that scales with the non-local ratio and sample size: the thresholds are our own noise floor for this metric.</p><h3 id="chrn" tabindex="-1"><code>CHRN</code>: Autoscaling churn <a class="header-anchor" href="#chrn" aria-label="Permalink to &quot;`CHRN`: Autoscaling churn {#chrn}&quot;">​</a></h3><p>Executors are stood up and torn down again before they can do useful work: re-provisioning churn rather than normal scale-down. Raise <code>spark.dynamicAllocation.executorIdleTimeout</code>, or widen the <code>minExecutors</code>/<code>maxExecutors</code> bounds to reduce flapping. Self-flags a confidence that scales with how far the short-lived-executor share sits past the threshold: these thresholds are still a design spike, not yet validated against real-world runs.</p><h3 id="jobs" tabindex="-1"><code>JOBS</code>: Job failure rate <a class="header-anchor" href="#jobs" aria-label="Permalink to &quot;`JOBS`: Job failure rate {#jobs}&quot;">​</a></h3><p>A large share of completed jobs did not succeed. Inspect the driver log for the failed job(s) and the stage failures that triggered them.</p><h3 id="incmp" tabindex="-1"><code>INCMP</code>: Incomplete run <a class="header-anchor" href="#incmp" aria-label="Permalink to &quot;`INCMP`: Incomplete run {#incmp}&quot;">​</a></h3><p>This event log never recorded an <code>ApplicationEnd</code> event: capture stopped before the run finished (an in-flight job, a rotated-away log, or a cut-short capture). Every other finding and metric on the board reflects only what was captured up to that point, not the full run.</p><h2 id="configuration-scope" tabindex="-1">Configuration scope <a class="header-anchor" href="#configuration-scope" aria-label="Permalink to &quot;Configuration scope&quot;">​</a></h2><h3 id="cfg" tabindex="-1"><code>CFG</code>: Configuration audit <a class="header-anchor" href="#cfg" aria-label="Permalink to &quot;`CFG`: Configuration audit {#cfg}&quot;">​</a></h3><p>Flags configuration settings that may cause reliability or efficiency problems, independent of any one stage&#39;s behavior. Four properties are audited today:</p><ul><li><code>spark.shuffle.service.enabled</code>: flagged when dynamic allocation is on but the external shuffle service is off, since shuffle data won&#39;t survive executor removal.</li><li><code>spark.dynamicAllocation.maxExecutors</code>: flagged for inverted bounds or a missing upper bound.</li><li><code>spark.serializer</code>: flagged when still on the default Java serializer; <code>org.apache.spark.serializer.KryoSerializer</code> is faster and produces smaller buffers.</li><li><code>spark.executor.memoryOverhead</code>: flagged when set below a safe floor.</li></ul><h2 id="sql-scope" tabindex="-1">SQL scope <a class="header-anchor" href="#sql-scope" aria-label="Permalink to &quot;SQL scope&quot;">​</a></h2><h3 id="plan" tabindex="-1"><code>PLAN</code>: Plan advisor <a class="header-anchor" href="#plan" aria-label="Permalink to &quot;`PLAN`: Plan advisor {#plan}&quot;">​</a></h3><p>Flags patterns in the SQL execution plan worth reviewing. Four checks share this tag:</p><ul><li>Duplicate plan subtree: the same subtree recomputed more than once in the plan.</li><li>Small files: reading an excessive number of small files.</li><li>Under-broadcast: the smaller side of a Sort Merge Join looks well under the broadcast threshold; consider a <code>broadcast()</code> hint or raising <code>spark.sql.autoBroadcastJoinThreshold</code>.</li><li>Over-broadcast: a broadcast exceeds the 1 GB threshold; check for a misapplied broadcast hint or a misconfigured <code>spark.sql.autoBroadcastJoinThreshold</code>.</li></ul></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/user-guide/getting-started.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>Getting started</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="/docs/user-guide/run-comparison.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>Run comparison mode</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
26
+ <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"CWpj01WU\",\"contributor-guide_architecture_detector-contract.md\":\"CgzUsQ6W\",\"contributor-guide_architecture_drill-down.md\":\"BtPdlM7r\",\"contributor-guide_architecture_impact-estimation.md\":\"CooslVJt\",\"contributor-guide_architecture_index.md\":\"3TO9ic6w\",\"contributor-guide_architecture_overview.md\":\"CehiRmGn\",\"contributor-guide_architecture_state-and-history.md\":\"C-xxn0q7\",\"contributor-guide_architecture_widget-rendering.md\":\"R27gQrgY\",\"contributor-guide_architecture_worker-protocol.md\":\"IbnfNrV3\",\"contributor-guide_contributing.md\":\"CvRsdr6J\",\"contributor-guide_development-setup.md\":\"DvAN_9mK\",\"contributor-guide_testing.md\":\"6rIKqSyY\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"Df1YMIHu\",\"tuning-reference_aqe.md\":\"BIsCtLzm\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"CEstB3Ia\",\"tuning-reference_bottleneck-cold-start.md\":\"CEuy-72y\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"CIohQDfn\",\"tuning-reference_bottleneck-failures.md\":\"4z5BXGJ2\",\"tuning-reference_bottleneck-gc.md\":\"DSxzZRK7\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"BaJl__1W\",\"tuning-reference_bottleneck-memory-utilization.md\":\"DbP-SJZc\",\"tuning-reference_bottleneck-retry-waste.md\":\"D5JMjVOt\",\"tuning-reference_bottleneck-shuffle.md\":\"CM-nTmIH\",\"tuning-reference_bottleneck-skew.md\":\"BdUwiDhn\",\"tuning-reference_bottleneck-slow-host.md\":\"BlIo6UDW\",\"tuning-reference_bottleneck-small-files.md\":\"B8kloyx8\",\"tuning-reference_bottleneck-spill.md\":\"PNH7mITt\",\"tuning-reference_bottleneck-straggler.md\":\"DY36fHN5\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"QTV7O8kU\",\"tuning-reference_bottleneck-utilization.md\":\"DTiueZC3\",\"tuning-reference_caching.md\":\"B7aQ8asB\",\"tuning-reference_cluster-config.md\":\"ZVmDGsQ3\",\"tuning-reference_config.md\":\"UvveiWG3\",\"tuning-reference_data-formats.md\":\"bjCAWH3N\",\"tuning-reference_index.md\":\"BQ_NooMV\",\"tuning-reference_intro.md\":\"CobD-lGB\",\"tuning-reference_joins.md\":\"BtKs_CuW\",\"tuning-reference_memory-model.md\":\"DhT-n4y3\",\"tuning-reference_metrics.md\":\"mLOh7Apj\",\"tuning-reference_partitioning.md\":\"q0zKF_8X\",\"tuning-reference_pyspark.md\":\"DDCfvN9t\",\"tuning-reference_shuffle.md\":\"BZZ7R4Ix\",\"tuning-reference_spark-architecture.md\":\"Dwzm5avO\",\"tuning-reference_table-formats.md\":\"D6wj-2dX\",\"user-guide_alternative-log-retrieval.md\":\"B4tPGIal\",\"user-guide_getting-started.md\":\"BJvwLEIM\",\"user-guide_mcp-tools.md\":\"Vi3RoflJ\",\"user-guide_run-comparison.md\":\"CQc1aoU8\",\"user-guide_understanding-findings.md\":\"DL1UDhvR\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Alternative log retrieval\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
27
27
 
28
28
  </body>
29
29
  </html>
package/public/index.html CHANGED
@@ -4,7 +4,7 @@
4
4
  <meta charset="UTF-8" />
5
5
  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
6
6
  <title>SparkForensics</title>
7
- <meta name="description" content="SparkForensics is a zero-backend browser app that parses Spark History Server event logs to surface stage-level bottlenecks — skew, spill, GC, shuffle, cold start — without uploading cluster data anywhere.">
7
+ <meta name="description" content="SparkForensics is a zero-backend browser app that parses Spark History Server event logs to surface stage-level bottlenecks (skew, spill, GC, shuffle, cold start) without uploading cluster data anywhere.">
8
8
  <link rel="icon" type="image/svg+xml" href="./favicon.svg">
9
9
  <meta property="og:type" content="website">
10
10
  <meta property="og:url" content="https://shuffle-works.github.io/sparkforensics/">
@@ -23,12 +23,13 @@
23
23
  document.documentElement.setAttribute('data-theme', 'light');
24
24
  } catch (_) {}
25
25
  </script>
26
- <script type="module" crossorigin src="./assets/index-4id52G-A.js"></script>
27
- <link rel="modulepreload" crossorigin href="./assets/react-vendor-u4pyvC5Y.js">
28
- <link rel="modulepreload" crossorigin href="./assets/plan-graph-vendor-vZQHG2EJ.js">
29
- <link rel="modulepreload" crossorigin href="./assets/charts-vendor-NMqKV0H4.js">
30
- <link rel="stylesheet" crossorigin href="./assets/plan-graph-vendor-BnuhLJ6X.css">
31
- <link rel="stylesheet" crossorigin href="./assets/index-D9kwVYIO.css">
26
+ <script type="module" crossorigin src="./assets/index-3vlFVNCg.js"></script>
27
+ <link rel="modulepreload" crossorigin href="./assets/rolldown-runtime-hePW80VL.js">
28
+ <link rel="modulepreload" crossorigin href="./assets/charts-vendor-0VMaLVLN.js">
29
+ <link rel="modulepreload" crossorigin href="./assets/plan-graph-vendor-B0sEms9Z.js">
30
+ <link rel="modulepreload" crossorigin href="./assets/react-vendor-D2TLlSzI.js">
31
+ <link rel="stylesheet" crossorigin href="./assets/plan-graph-vendor-DLioOiRN.css">
32
+ <link rel="stylesheet" crossorigin href="./assets/index-CjVklAdW.css">
32
33
  </head>
33
34
  <body>
34
35
  <div id="root"></div>
package/vendor-core/analyzer.js CHANGED
@@ -9,9 +9,7 @@ import { IMPACT_BAND_ORDER } from './format-utils.js';
9
9
 
10
10
 
11
11
 
12
- // FNV-1a 32-bit: a small, dependency-free stable string hash. Used to derive
13
- // a deterministic finding `id` at the single choke point below so the same
14
- // logical finding keeps the same id across runs (no timestamps, no randomness).
12
+ // FNV-1a 32-bit stable string hash: deterministic finding id across runs, no timestamps/randomness.
15
13
  function fnv1a(str ) {
16
14
  let h = 0x811c9dc5;
17
15
  for (let i = 0; i < str.length; i++) {
@@ -22,36 +20,19 @@ function fnv1a(str ) {
22
20
  }
23
21
 
24
22
  export function findingId(f ) {
25
- // Location key mirrors the report's stable identity: stage, else SQL
26
- // execution, else the audited config property.
23
+ // Location key: stage, else SQL execution, else audited config property.
27
24
  const locKey = f.stageId ?? f.executionId ?? f.property ?? '';
28
- // Discriminators for detectors that intentionally emit multiple findings on
29
- // the same location+metric; without them these siblings hash to one id:
30
- // stage-scope: slowHost (host), memoryUtilization (executorId + dimension),
31
- // partitionSizing (rule);
32
- // app-scope: cacheUtilization (rddId + variant, stageId is always null),
33
- // cachingOpportunity (stageId/executionId both null too: `relation`/
34
- // `format` distinguish leaf findings, `operator`+`relation` distinguish
35
- // composite findings, `executionIds` as a last resort when everything
36
- // else matches);
37
- // SQL plan-advisor: smallFiles (direction/nodeName),
38
- // duplicatePlanSubtree (rootName/subtreeSize/groupIndex: two unrelated
39
- // duplicate-subtree groups can share rootName+subtreeSize, e.g. the same
40
- // "BroadcastExchange over a Project/Filter/Scan" shape repeated once per
41
- // dimension table; groupIndex (the group's deterministic position within
42
- // one execution's plan) is the actual uniqueness guarantee, since the
43
- // informational `sampleRelation` field can be null or coincide for two
44
- // groups),
45
- // broadcastSizing (per node/side, distinguished by value + largerSideBytes).
46
- // The metric `value` folds in the per-node magnitude that broadcastSizing
47
- // siblings carry no other field for; it is deterministic for a given log, so
48
- // the id stays stable across re-parses (no timestamp/randomness).
49
- // memoryUtilization's `rule` (heapNearCapacity/heapOverProvisioned) is excluded here: an
50
- // executor falls in exactly one heap band, so `executorId` alone already guarantees
51
- // uniqueness and folding `rule` in too would just add gratuitous id-rotation risk if the
52
- // band logic ever changes. partitionSizing's `rule` (maxPartitionTooBig/
53
- // shufflePartitionSkew/lowShuffleParallelism) IS load-bearing here: a stage can emit more
54
- // than one of those rules at once, sharing the same stageId+metric.
25
+ // Discriminators for detectors that emit multiple findings on the same
26
+ // location+metric; without them these siblings hash to one id:
27
+ // slowHost (host), memoryUtilization (executorId+dimension), partitionSizing (rule);
28
+ // cacheUtilization (rddId+variant), cachingOpportunity (relation/format for leaf,
29
+ // operator+relation for composite, executionIds as last resort);
30
+ // smallFiles (direction/nodeName), duplicatePlanSubtree (groupIndex is the real
31
+ // uniqueness guarantee: rootName+subtreeSize can collide across groups),
32
+ // broadcastSizing (value+largerSideBytes per node/side).
33
+ // memoryUtilization excludes `rule`: one heap band per executor, so executorId is already
34
+ // unique and folding rule in risks id churn if band logic changes. partitionSizing keeps
35
+ // `rule`: a stage can emit several rules at once sharing stageId+metric.
55
36
  const rule = f.type === 'memoryUtilization' ? undefined : f.rule;
56
37
  const disc = [
57
38
  f.host, f.executorId, rule, f.variant, f.dimension,
@@ -62,6 +43,35 @@ export function findingId(f ) {
62
43
  return fnv1a(`${f.type}|${locKey}|${f.metric ?? ''}|${f.value ?? ''}|${disc}`);
63
44
  }
64
45
 
46
+ // skew's max/median branch (stage.taskCount below minTasksForP95) and straggler are both driven
47
+ // by the identical (taskDurationMax - taskDurationP50) delta on the same stage: the same
48
+ // dominant outlier task reported by two detectors, each independently clipped (see "Overlap
49
+ // caveat: skew / straggler" in impact-estimation.md). skew's P95/median branch samples a
50
+ // different task and stays independent. Flags both sides via validationRequired (rather than
51
+ // suppressing either) so neither finding's own diagnostic value is lost; the flag rides the same
52
+ // confidence-caveat UI a reader already sees before trusting either finding's magnitude.
53
+ function overlapNote(otherType ) {
54
+ return `This overlaps with the ${otherType} finding on this stage: both are driven by the same dominant outlier task, so don't add their recoverable-time figures together.`;
55
+ }
56
+
57
+ function flagSkewStragglerOverlap(findings ) {
58
+ const maxMedianSkewStages = new Set(
59
+ findings.filter((f) => f.type === 'skew' && f.metric === 'max/median' && f.stageId != null).map((f) => f.stageId),
60
+ );
61
+ if (maxMedianSkewStages.size === 0) return;
62
+ const stragglerStages = new Set(
63
+ findings.filter((f) => f.type === 'straggler' && f.stageId != null).map((f) => f.stageId),
64
+ );
65
+ const overlapStages = new Set([...maxMedianSkewStages].filter((id) => stragglerStages.has(id)));
66
+ if (overlapStages.size === 0) return;
67
+ for (const f of findings) {
68
+ if (f.stageId == null || !overlapStages.has(f.stageId)) continue;
69
+ const note = f.type === 'skew' ? overlapNote('straggler') : f.type === 'straggler' ? overlapNote('skew') : null;
70
+ if (!note) continue;
71
+ f.validationRequired = f.validationRequired ? `${f.validationRequired} ${note}` : note;
72
+ }
73
+ }
74
+
65
75
  function push(out , entry , result ) {
66
76
  if (!result) return;
67
77
  const detectorVersion = entry.version ?? 1;
@@ -70,28 +80,15 @@ function push(out , entry , result
70
80
  if (entry.suppressWhen && entry.suppressWhen(f, out)) continue;
71
81
  const stamped = { ...f, docAnchor: f.docAnchor ?? entry.docAnchor, detectorVersion };
72
82
  const id = findingId(stamped);
73
- // Guardrail against any detector (this one or a future one) emitting two
74
- // structurally-identical findings for what should be one logical
75
- // occurrence: same id => same finding, keep only the first. Added after a
76
- // real-log bug where duplicatePlanSubtree emitted two distinct findings
77
- // sharing one id (see findingId's groupIndex discriminator above and
78
- // tests/detectors-plan.test.js's duplicatePlanSubtree regression tests);
79
- // this guard's correctness depends on findingId's discriminator list
80
- // actually being unique per distinct finding, not on this line itself.
83
+ // Dedup guard: same id => same finding, keep first. Correctness depends on
84
+ // findingId's discriminators being unique per distinct finding, not on this line.
81
85
  if (out.some((existing) => existing.id === id)) continue;
82
86
  out.push({ ...stamped, id });
83
87
  }
84
88
  }
85
89
 
86
- // `app` is widened to `SparkAppInfo | null` rather than the plan's literal
87
- // non-nullable `SparkAppInfo`: real callers (evidence-report.ts's
88
- // `buildJson`, useIngest.ts's `runDone`/`snapshotParsedRun`) pass
89
- // `AppModel.app`, which is genuinely `SparkAppInfo | null` at the type
90
- // level even though by the time `analyze()` runs in practice a full parse
91
- // has completed and `app` is always populated (the same reasoning
92
- // detectors.ts's `DetectorCtx.app` comment gives for treating its own,
93
- // separate `app` field as non-nullable). Widened to match the real caller
94
- // type rather than forcing a cast at every call site.
90
+ // `app` widened to `SparkAppInfo | null` to match real callers (AppModel.app is
91
+ // nullable at the type level); every detector below tolerates a null app.
95
92
  export function analyze(
96
93
  app ,
97
94
  stages ,
@@ -101,29 +98,18 @@ export function analyze(
101
98
  sql = new Map(),
102
99
  runAggregates = null,
103
100
  ) {
104
- // `app ?? {}`: every detector below tolerates a null `app` (malformed logs
105
- // with no ApplicationStart), so this call must too; computePeakConcurrentCores
106
- // just falls back to the executor-derived core sum when `resources` is absent.
107
- // computePeakConcurrentCores (not computeTotalCores) here specifically: the
108
- // ceiling this feeds (src/occupancy.ts's computeCeiling) needs a genuine
109
- // concurrent-capacity bound, and computeTotalCores's cumulative sum over
110
- // every addition overstates that under dynamic allocation/executor
111
- // replacement (a churned-through executor's cores were never actually
112
- // concurrent with its replacement's).
113
- // `executorsAdded`/`executorsRemoved` casts: computePeakConcurrentCores only
114
- // reads `executorId`/`timestamp`/`totalCores`, all present on
115
- // ExecutorAddedEvent/ExecutorRemovedEvent, but `totalCores` isn't on
116
- // ExecutorRemovedEvent, so the `ExecutorEvent` union as a whole is a
117
- // structural mismatch against its parameter shapes.
101
+ // `app ?? {}`: detectors tolerate a null app (malformed logs), so this must too.
102
+ // computePeakConcurrentCores (not computeTotalCores): the occupancy ceiling needs a
103
+ // concurrent-capacity bound; a cumulative sum overstates it under dynamic allocation.
104
+ // Casts: the function reads only executorId/timestamp/totalCores, but totalCores isn't
105
+ // on ExecutorRemovedEvent, so the ExecutorEvent union mismatches its parameter shapes.
118
106
  const totalCores = computePeakConcurrentCores(
119
107
  app ?? {},
120
108
  executorsAdded ,
121
109
  executorsRemoved ,
122
110
  );
123
- // Computed once, up front, so detectors can gate impact band on the same
124
- // occupancy-clipped waste figure estimateImpact() below displays as that
125
- // finding's savings (src/detectors.ts's `clippedWasteMs`), not a raw
126
- // pre-clip delta the two passes would otherwise disagree on.
111
+ // Computed once so detectors gate impact band on the same occupancy-clipped waste
112
+ // estimateImpact displays as savings, not a raw pre-clip delta the two passes would disagree on.
127
113
  const occupancy = computeOccupancy(stages , totalCores);
128
114
  const ctx = {
129
115
  app, stages, executorsAdded, executorsRemoved, jobs, sql, runAggregates, occupancy,
@@ -148,20 +134,34 @@ export function analyze(
148
134
  }
149
135
  estimateImpact(out, stages, totalCores);
150
136
  deriveImpactBand(out, app);
151
- // Ascending IMPACT_BAND_ORDER (critical 0 → info 2) puts the worst band
152
- // first, matching the old descending 3/2/1 rank this replaced; `sort` is
153
- // stable, so findings sharing a band keep their DETECTORS declaration order.
137
+ flagSkewStragglerOverlap(out);
138
+ // Ascending IMPACT_BAND_ORDER (critical 0 -> info 2) puts the worst band first;
139
+ // stable sort keeps DETECTORS declaration order within a band.
154
140
  out.sort((a, b) => IMPACT_BAND_ORDER[a.impactBand] - IMPACT_BAND_ORDER[b.impactBand]);
155
141
  return out;
156
142
  }
157
143
 
158
- export function auditConfig(app ) {
144
+ // Memoizes auditConfig by `app` identity (like evidence-report.ts's jsonCache) so config detectors
145
+ // don't re-run when both the export and the report path audit the same app. WeakMap can't key on
146
+ // `null`, so that case skips the cache. Returns a fresh copy each call so a caller's in-place
147
+ // mutation (e.g. `.sort()`) can't corrupt the cached array.
148
+ const auditConfigCache = new WeakMap ();
149
+
150
+ function computeAuditConfig(app ) {
159
151
  const out = [];
160
152
  for (const d of DETECTORS) if (d.scope === 'config') push(out, d, d.detect({ app }));
161
- // configAudit's own impact-estimator case (src/impact-estimator.ts) needs neither
162
- // `stages` nor `totalCores`: it's unconditionally `costOnly('none')`. An empty stages
163
- // map is enough for parity with the same finding type produced via analyze().
153
+ // configAudit's impact case is unconditionally costOnly('none'): needs no stages/totalCores,
154
+ // an empty stages map gives parity with analyze().
164
155
  estimateImpact(out, new Map());
165
156
  deriveImpactBand(out, app);
166
157
  return out.map((f) => ({ ...f, stageId: f.stageId ?? null }));
167
158
  }
159
+
160
+ export function auditConfig(app ) {
161
+ if (app === null) return computeAuditConfig(app);
162
+ const cached = auditConfigCache.get(app);
163
+ if (cached) return cached.slice();
164
+ const result = computeAuditConfig(app);
165
+ auditConfigCache.set(app, result);
166
+ return result.slice();
167
+ }