sparkforensics-server 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (382) hide show
  1. package/index.js +5 -4
  2. package/package.json +1 -1
  3. package/public/assets/AutoscalingChurn-CUSIvvab.js +1 -0
  4. package/public/assets/CacheUtilization-Ce0K6_X2.js +1 -0
  5. package/public/assets/CachingOpportunity-Cn-LmRA4.js +1 -0
  6. package/public/assets/ColdStart-BoiYzEyV.js +1 -0
  7. package/public/assets/ConfigAudit-BoJc6PAw.js +1 -0
  8. package/public/assets/CoreUsageArea-BFfXKu7K.js +1 -0
  9. package/public/assets/Dashboard-Bo_1jGqH.js +2 -0
  10. package/public/assets/DisclosureContext-LYzHroHO.js +1 -0
  11. package/public/assets/DuplicatePlanSubtree-BaAuGAoo.js +1 -0
  12. package/public/assets/ExecutorUtilization-OE6VMFkG.js +1 -0
  13. package/public/assets/GcPressure-AoL_ZNrV.js +1 -0
  14. package/public/assets/IncompleteRun-Bv6IzgsQ.js +1 -0
  15. package/public/assets/JobFailures-DF3U2z1f.js +1 -0
  16. package/public/assets/MemoryUtilization-B65OVZlq.js +1 -0
  17. package/public/assets/OverBroadcast-DXbJREDl.js +1 -0
  18. package/public/assets/PartitionSizing-DZLnjy0d.js +1 -0
  19. package/public/assets/PlanExplorer-DdmPkKO1.js +1 -0
  20. package/public/assets/PlanGraphRoute-C-BGmC6j.js +1 -0
  21. package/public/assets/RetryWaste-BbEPsdMw.js +1 -0
  22. package/public/assets/ShuffleIO-wZB94LS8.js +1 -0
  23. package/public/assets/Skew-SpQE7zeJ.js +1 -0
  24. package/public/assets/SlowHost-DwDBCTvr.js +1 -0
  25. package/public/assets/SmallFiles-yjY_6sT1.js +1 -0
  26. package/public/assets/SpeculationWaste-73PDmj0c.js +1 -0
  27. package/public/assets/Spill-DEw0Kd1i.js +1 -0
  28. package/public/assets/StageFailed-BEfXdEJc.js +1 -0
  29. package/public/assets/StageFindingGroup-Bzm2TY03.js +1 -0
  30. package/public/assets/StageHeader-BSV8ALhT.js +1 -0
  31. package/public/assets/StageShape-CzR4jHOt.js +1 -0
  32. package/public/assets/StageSlowness-vMPsnAaa.js +1 -0
  33. package/public/assets/Straggler-Cf__OGl5.js +1 -0
  34. package/public/assets/TaskFailures-BZ8kLZiO.js +1 -0
  35. package/public/assets/{TinyTask-Be-pvTjJ.js → TinyTask-CIPRxWfT.js} +1 -1
  36. package/public/assets/UnderBroadcast-CBm8ZPoS.js +1 -0
  37. package/public/assets/{finding-anchor-vVODA_VC.js → finding-anchor-DF1cSHO1.js} +1 -1
  38. package/public/assets/impact-sort-7z0ZcBZf.js +1 -0
  39. package/public/assets/index-BcqTLKFp.css +2 -0
  40. package/public/assets/index-CEM9bz_x.js +13 -0
  41. package/public/assets/{parser-worker-jAP0kWvS.js → parser-worker-Q5KCPdKT.js} +9 -9
  42. package/public/assets/react-vendor-BAKt3vny.js +9 -0
  43. package/public/assets/select-Cy5pmbrh.js +1 -0
  44. package/public/assets/usePagedRows-BjPOpUU8.js +1 -0
  45. package/public/assets/useRecentFiles-BrBl-AP1.js +11 -0
  46. package/public/assets/useSortMode-CU0vzThu.js +1 -0
  47. package/public/docs/404.html +2 -2
  48. package/public/docs/assets/{app.C5E-wy6e.js → app.C6w47a9W.js} +1 -1
  49. package/public/docs/assets/chunks/@localSearchIndexroot.DbZy5EtM.js +1 -0
  50. package/public/docs/assets/chunks/{VPLocalSearchBox.DtyygpbL.js → VPLocalSearchBox.BcTNvT55.js} +1 -1
  51. package/public/docs/assets/chunks/{theme.DQS9-w_Z.js → theme.D-mE6j02.js} +2 -2
  52. package/public/docs/assets/{contributor-guide_architecture_board-widgets.md.9ToZXcTk.js → contributor-guide_architecture_board-widgets.md.VTcYTH4d.js} +1 -1
  53. package/public/docs/assets/contributor-guide_architecture_detector-contract.md.CsqZHz_T.js +1 -0
  54. package/public/docs/assets/{contributor-guide_architecture_detector-contract.md.CsCytCho.lean.js → contributor-guide_architecture_detector-contract.md.CsqZHz_T.lean.js} +1 -1
  55. package/public/docs/assets/contributor-guide_architecture_drill-down.md.CFNCE8JB.js +1 -0
  56. package/public/docs/assets/{contributor-guide_architecture_drill-down.md.Bawx6NgP.lean.js → contributor-guide_architecture_drill-down.md.CFNCE8JB.lean.js} +1 -1
  57. package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.HT9IJGIl.js +1 -0
  58. package/public/docs/assets/{contributor-guide_architecture_impact-estimation.md.sf9DU2bA.lean.js → contributor-guide_architecture_impact-estimation.md.HT9IJGIl.lean.js} +1 -1
  59. package/public/docs/assets/contributor-guide_architecture_overview.md.CTEqVeV4.js +1 -0
  60. package/public/docs/assets/contributor-guide_architecture_overview.md.CTEqVeV4.lean.js +1 -0
  61. package/public/docs/assets/contributor-guide_architecture_state-and-history.md.DOTVHjTV.js +1 -0
  62. package/public/docs/assets/{contributor-guide_architecture_state-and-history.md.B8Wc_ZTg.lean.js → contributor-guide_architecture_state-and-history.md.DOTVHjTV.lean.js} +1 -1
  63. package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.BmpARZve.js +1 -0
  64. package/public/docs/assets/{contributor-guide_architecture_widget-rendering.md.DYwFeyjV.lean.js → contributor-guide_architecture_widget-rendering.md.BmpARZve.lean.js} +1 -1
  65. package/public/docs/assets/contributor-guide_architecture_worker-protocol.md.21pTYhFF.js +6 -0
  66. package/public/docs/assets/contributor-guide_architecture_worker-protocol.md.21pTYhFF.lean.js +1 -0
  67. package/public/docs/assets/contributor-guide_development-setup.md.CjgZJwjK.js +15 -0
  68. package/public/docs/assets/contributor-guide_development-setup.md.CjgZJwjK.lean.js +1 -0
  69. package/public/docs/assets/contributor-guide_testing.md.DTwhpoN2.js +1 -0
  70. package/public/docs/assets/{contributor-guide_testing.md.Cnn6gz1C.lean.js → contributor-guide_testing.md.DTwhpoN2.lean.js} +1 -1
  71. package/public/docs/assets/tuning-reference_anti-patterns.md.D08Uaod7.js +1 -0
  72. package/public/docs/assets/tuning-reference_anti-patterns.md.D08Uaod7.lean.js +1 -0
  73. package/public/docs/assets/tuning-reference_aqe.md.C1Ae6A8u.js +1 -0
  74. package/public/docs/assets/tuning-reference_aqe.md.C1Ae6A8u.lean.js +1 -0
  75. package/public/docs/assets/tuning-reference_bottleneck-broadcast-sizing.md.6wjKuJNE.js +1 -0
  76. package/public/docs/assets/tuning-reference_bottleneck-broadcast-sizing.md.6wjKuJNE.lean.js +1 -0
  77. package/public/docs/assets/tuning-reference_bottleneck-cold-start.md.BVQbsuHP.js +7 -0
  78. package/public/docs/assets/tuning-reference_bottleneck-cold-start.md.BVQbsuHP.lean.js +1 -0
  79. package/public/docs/assets/tuning-reference_bottleneck-duplicate-plan-subtree.md.DtaYnaiF.js +1 -0
  80. package/public/docs/assets/tuning-reference_bottleneck-duplicate-plan-subtree.md.DtaYnaiF.lean.js +1 -0
  81. package/public/docs/assets/{tuning-reference_bottleneck-failures.md.CAsFf3RR.js → tuning-reference_bottleneck-failures.md.Z1Tdmdd3.js} +2 -2
  82. package/public/docs/assets/tuning-reference_bottleneck-failures.md.Z1Tdmdd3.lean.js +1 -0
  83. package/public/docs/assets/tuning-reference_bottleneck-gc.md.BCnXLf1n.js +6 -0
  84. package/public/docs/assets/tuning-reference_bottleneck-gc.md.BCnXLf1n.lean.js +1 -0
  85. package/public/docs/assets/tuning-reference_bottleneck-job-failure-rate.md.CcvQj2g9.js +8 -0
  86. package/public/docs/assets/tuning-reference_bottleneck-job-failure-rate.md.CcvQj2g9.lean.js +1 -0
  87. package/public/docs/assets/tuning-reference_bottleneck-memory-utilization.md.B-C0-8h-.js +7 -0
  88. package/public/docs/assets/tuning-reference_bottleneck-memory-utilization.md.B-C0-8h-.lean.js +1 -0
  89. package/public/docs/assets/tuning-reference_bottleneck-retry-waste.md.DCnhWiqj.js +1 -0
  90. package/public/docs/assets/tuning-reference_bottleneck-retry-waste.md.DCnhWiqj.lean.js +1 -0
  91. package/public/docs/assets/{tuning-reference_bottleneck-shuffle.md.DDMQVOW6.js → tuning-reference_bottleneck-shuffle.md.B5OMJCd3.js} +2 -2
  92. package/public/docs/assets/tuning-reference_bottleneck-shuffle.md.B5OMJCd3.lean.js +1 -0
  93. package/public/docs/assets/{tuning-reference_bottleneck-skew.md.B0MzhWcX.js → tuning-reference_bottleneck-skew.md.Cvo6NQm_.js} +1 -1
  94. package/public/docs/assets/tuning-reference_bottleneck-skew.md.Cvo6NQm_.lean.js +1 -0
  95. package/public/docs/assets/{tuning-reference_bottleneck-slow-host.md.BlIo6UDW.js → tuning-reference_bottleneck-slow-host.md.ja5qr3wq.js} +1 -1
  96. package/public/docs/assets/tuning-reference_bottleneck-slow-host.md.ja5qr3wq.lean.js +1 -0
  97. package/public/docs/assets/tuning-reference_bottleneck-small-files.md.Crjtb0i2.js +5 -0
  98. package/public/docs/assets/tuning-reference_bottleneck-small-files.md.Crjtb0i2.lean.js +1 -0
  99. package/public/docs/assets/tuning-reference_bottleneck-spill.md.Bl8KtSRg.js +6 -0
  100. package/public/docs/assets/tuning-reference_bottleneck-spill.md.Bl8KtSRg.lean.js +1 -0
  101. package/public/docs/assets/tuning-reference_bottleneck-straggler.md.B0VN8reT.js +7 -0
  102. package/public/docs/assets/tuning-reference_bottleneck-straggler.md.B0VN8reT.lean.js +1 -0
  103. package/public/docs/assets/tuning-reference_bottleneck-tiny-tasks.md.DQCftGhu.js +8 -0
  104. package/public/docs/assets/tuning-reference_bottleneck-tiny-tasks.md.DQCftGhu.lean.js +1 -0
  105. package/public/docs/assets/{tuning-reference_bottleneck-utilization.md.DqC_iGkw.js → tuning-reference_bottleneck-utilization.md.BXDl30f9.js} +1 -1
  106. package/public/docs/assets/tuning-reference_bottleneck-utilization.md.BXDl30f9.lean.js +1 -0
  107. package/public/docs/assets/tuning-reference_caching.md.CcaiMn_7.js +1 -0
  108. package/public/docs/assets/tuning-reference_caching.md.CcaiMn_7.lean.js +1 -0
  109. package/public/docs/assets/tuning-reference_cluster-config.md.DbWG1yZq.js +1 -0
  110. package/public/docs/assets/tuning-reference_cluster-config.md.DbWG1yZq.lean.js +1 -0
  111. package/public/docs/assets/tuning-reference_config.md.YUqSpwFB.js +1 -0
  112. package/public/docs/assets/tuning-reference_config.md.YUqSpwFB.lean.js +1 -0
  113. package/public/docs/assets/tuning-reference_data-formats.md.C5DX5xL4.js +1 -0
  114. package/public/docs/assets/tuning-reference_data-formats.md.C5DX5xL4.lean.js +1 -0
  115. package/public/docs/assets/tuning-reference_index.md.ZVIpQ6Dk.js +1 -0
  116. package/public/docs/assets/tuning-reference_index.md.ZVIpQ6Dk.lean.js +1 -0
  117. package/public/docs/assets/tuning-reference_intro.md.BSSaDlqx.js +1 -0
  118. package/public/docs/assets/tuning-reference_intro.md.BSSaDlqx.lean.js +1 -0
  119. package/public/docs/assets/tuning-reference_joins.md.BVhNCDnE.js +1 -0
  120. package/public/docs/assets/tuning-reference_joins.md.BVhNCDnE.lean.js +1 -0
  121. package/public/docs/assets/tuning-reference_memory-model.md.CfjFdgbl.js +1 -0
  122. package/public/docs/assets/tuning-reference_memory-model.md.CfjFdgbl.lean.js +1 -0
  123. package/public/docs/assets/tuning-reference_metrics.md.C64VNEL4.js +1 -0
  124. package/public/docs/assets/tuning-reference_metrics.md.C64VNEL4.lean.js +1 -0
  125. package/public/docs/assets/tuning-reference_partitioning.md.CtLK0xsS.js +1 -0
  126. package/public/docs/assets/tuning-reference_partitioning.md.CtLK0xsS.lean.js +1 -0
  127. package/public/docs/assets/tuning-reference_pyspark.md.D-92XgF8.js +6 -0
  128. package/public/docs/assets/tuning-reference_pyspark.md.D-92XgF8.lean.js +1 -0
  129. package/public/docs/assets/tuning-reference_shuffle.md.Iq2v1oHJ.js +1 -0
  130. package/public/docs/assets/tuning-reference_shuffle.md.Iq2v1oHJ.lean.js +1 -0
  131. package/public/docs/assets/tuning-reference_spark-architecture.md._qwnE_Sc.js +1 -0
  132. package/public/docs/assets/tuning-reference_spark-architecture.md._qwnE_Sc.lean.js +1 -0
  133. package/public/docs/assets/tuning-reference_table-formats.md.BSYEK-eN.js +1 -0
  134. package/public/docs/assets/tuning-reference_table-formats.md.BSYEK-eN.lean.js +1 -0
  135. package/public/docs/assets/user-guide_getting-started.md.CEa2ZX-l.js +6 -0
  136. package/public/docs/assets/user-guide_getting-started.md.CEa2ZX-l.lean.js +1 -0
  137. package/public/docs/assets/{user-guide_mcp-tools.md.DrphooJv.js → user-guide_mcp-tools.md.dQwIvdTF.js} +13 -7
  138. package/public/docs/assets/user-guide_mcp-tools.md.dQwIvdTF.lean.js +1 -0
  139. package/public/docs/assets/user-guide_run-comparison.md.ChLI3MK0.js +1 -0
  140. package/public/docs/assets/user-guide_run-comparison.md.ChLI3MK0.lean.js +1 -0
  141. package/public/docs/assets/{user-guide_understanding-findings.md.YBBLOwWi.js → user-guide_understanding-findings.md.D1ccs60C.js} +1 -1
  142. package/public/docs/assets/{user-guide_understanding-findings.md.YBBLOwWi.lean.js → user-guide_understanding-findings.md.D1ccs60C.lean.js} +1 -1
  143. package/public/docs/contributor-guide/architecture/board-widgets.html +5 -5
  144. package/public/docs/contributor-guide/architecture/detector-contract.html +5 -5
  145. package/public/docs/contributor-guide/architecture/drill-down.html +5 -5
  146. package/public/docs/contributor-guide/architecture/impact-estimation.html +5 -5
  147. package/public/docs/contributor-guide/architecture/index.html +3 -3
  148. package/public/docs/contributor-guide/architecture/overview.html +5 -5
  149. package/public/docs/contributor-guide/architecture/state-and-history.html +5 -5
  150. package/public/docs/contributor-guide/architecture/widget-rendering.html +5 -5
  151. package/public/docs/contributor-guide/architecture/worker-protocol.html +6 -6
  152. package/public/docs/contributor-guide/contributing.html +3 -3
  153. package/public/docs/contributor-guide/development-setup.html +7 -7
  154. package/public/docs/contributor-guide/testing.html +5 -5
  155. package/public/docs/hashmap.json +1 -1
  156. package/public/docs/index.html +3 -3
  157. package/public/docs/tuning-reference/anti-patterns.html +6 -6
  158. package/public/docs/tuning-reference/aqe.html +6 -6
  159. package/public/docs/tuning-reference/bottleneck-broadcast-sizing.html +6 -6
  160. package/public/docs/tuning-reference/bottleneck-cold-start.html +6 -6
  161. package/public/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html +6 -6
  162. package/public/docs/tuning-reference/bottleneck-failures.html +6 -6
  163. package/public/docs/tuning-reference/bottleneck-gc.html +6 -6
  164. package/public/docs/tuning-reference/bottleneck-job-failure-rate.html +6 -6
  165. package/public/docs/tuning-reference/bottleneck-memory-utilization.html +6 -6
  166. package/public/docs/tuning-reference/bottleneck-retry-waste.html +6 -6
  167. package/public/docs/tuning-reference/bottleneck-shuffle.html +6 -6
  168. package/public/docs/tuning-reference/bottleneck-skew.html +6 -6
  169. package/public/docs/tuning-reference/bottleneck-slow-host.html +6 -6
  170. package/public/docs/tuning-reference/bottleneck-small-files.html +6 -6
  171. package/public/docs/tuning-reference/bottleneck-spill.html +6 -6
  172. package/public/docs/tuning-reference/bottleneck-straggler.html +6 -6
  173. package/public/docs/tuning-reference/bottleneck-tiny-tasks.html +6 -6
  174. package/public/docs/tuning-reference/bottleneck-utilization.html +6 -6
  175. package/public/docs/tuning-reference/caching.html +6 -6
  176. package/public/docs/tuning-reference/cluster-config.html +6 -6
  177. package/public/docs/tuning-reference/config.html +6 -6
  178. package/public/docs/tuning-reference/data-formats.html +6 -6
  179. package/public/docs/tuning-reference/index.html +6 -6
  180. package/public/docs/tuning-reference/intro.html +6 -6
  181. package/public/docs/tuning-reference/joins.html +6 -6
  182. package/public/docs/tuning-reference/memory-model.html +6 -6
  183. package/public/docs/tuning-reference/metrics.html +6 -6
  184. package/public/docs/tuning-reference/partitioning.html +6 -6
  185. package/public/docs/tuning-reference/pyspark.html +6 -6
  186. package/public/docs/tuning-reference/shuffle.html +6 -6
  187. package/public/docs/tuning-reference/spark-architecture.html +6 -6
  188. package/public/docs/tuning-reference/table-formats.html +6 -6
  189. package/public/docs/user-guide/alternative-log-retrieval.html +3 -3
  190. package/public/docs/user-guide/getting-started.html +9 -6
  191. package/public/docs/user-guide/mcp-tools.html +15 -9
  192. package/public/docs/user-guide/run-comparison.html +5 -5
  193. package/public/docs/user-guide/understanding-findings.html +5 -5
  194. package/public/export-template.html +111 -0
  195. package/public/index.html +4 -3
  196. package/vendor-core/analyzer.js +156 -48
  197. package/vendor-core/check-coverage.js +88 -0
  198. package/vendor-core/cli/budgets.js +31 -18
  199. package/vendor-core/cli/collect-run.js +76 -31
  200. package/vendor-core/cli/threshold-config.js +28 -0
  201. package/vendor-core/comparison-verdict.js +177 -0
  202. package/vendor-core/core-source-hash.txt +1 -0
  203. package/vendor-core/core-usage-locality.js +56 -2
  204. package/vendor-core/detector-docs.js +58 -0
  205. package/vendor-core/detectors.js +918 -458
  206. package/vendor-core/docs-config.js +0 -36
  207. package/vendor-core/docs-content/chapters/nav-index.json +31 -0
  208. package/vendor-core/docs-content/detection/cstor.md +9 -0
  209. package/vendor-core/docs-site-config.js +3 -0
  210. package/vendor-core/event-handlers.js +170 -6
  211. package/vendor-core/event-schemas.js +21 -0
  212. package/vendor-core/evidence-report.js +421 -112
  213. package/vendor-core/export-data.js +79 -6
  214. package/vendor-core/finding-action-label.js +9 -88
  215. package/vendor-core/finding-filter-predicate.js +9 -0
  216. package/vendor-core/finding-generic-recommendation.js +6 -104
  217. package/vendor-core/finding-names.js +21 -45
  218. package/vendor-core/finding-presentation.js +333 -0
  219. package/vendor-core/finding-tag-help.js +110 -0
  220. package/vendor-core/finding-types.js +361 -0
  221. package/vendor-core/findings-of-type.js +11 -0
  222. package/vendor-core/format-utils.js +92 -27
  223. package/vendor-core/html-export.js +51 -0
  224. package/vendor-core/impact-band.js +21 -8
  225. package/vendor-core/impact-estimator.js +8 -521
  226. package/vendor-core/impact-format.js +114 -0
  227. package/vendor-core/impact-model.js +175 -0
  228. package/vendor-core/ingest.js +2 -0
  229. package/vendor-core/intervals.js +13 -0
  230. package/vendor-core/list-runs.js +2 -3
  231. package/vendor-core/load-vendored.js +70 -5
  232. package/vendor-core/mcp-server-factory.js +14 -10
  233. package/vendor-core/mcp-tools.js +105 -45
  234. package/vendor-core/model-assembler.js +12 -0
  235. package/vendor-core/occupancy.js +1 -1
  236. package/vendor-core/parser-worker.js +1 -1
  237. package/vendor-core/plan-graph-model.js +3 -2
  238. package/vendor-core/plan-node-detail.js +1 -1
  239. package/vendor-core/recommendation-rollup.js +63 -3
  240. package/vendor-core/redact.js +45 -27
  241. package/vendor-core/run-comparison.js +32 -7
  242. package/vendor-core/run-interpretation.js +290 -0
  243. package/vendor-core/run-outcome.js +74 -0
  244. package/vendor-core/run-payload.js +17 -0
  245. package/vendor-core/run-shape.js +40 -0
  246. package/vendor-core/run-verdict.js +353 -0
  247. package/vendor-core/scaling-sim.js +4 -5
  248. package/vendor-core/scorecard-estimates.js +62 -0
  249. package/vendor-core/sql-stages.js +11 -0
  250. package/vendor-core/stage-quantiles.js +2 -0
  251. package/vendor-core/threshold-overrides.js +160 -0
  252. package/vendor-core/threshold-summary.js +11 -33
  253. package/vendor-core/types.js +6 -42
  254. package/vendor-core/wall-clock.js +1 -12
  255. package/vendor-core/wasted-core-hours.js +2 -2
  256. package/public/assets/AutoscalingChurn-lxZHWIwq.js +0 -1
  257. package/public/assets/CacheUtilization-DuAJ-Lve.js +0 -1
  258. package/public/assets/CachingOpportunity-DC1t7o2_.js +0 -1
  259. package/public/assets/ColdStart-BOYXeiUZ.js +0 -1
  260. package/public/assets/ConfigAudit-MCPrJomC.js +0 -1
  261. package/public/assets/CoreUsageArea-ChNuqGA0.js +0 -1
  262. package/public/assets/Dashboard-VuyUGvZx.js +0 -1
  263. package/public/assets/DuplicatePlanSubtree-JZut9xwq.js +0 -1
  264. package/public/assets/DurationHistogram-_7C1w0l4.js +0 -1
  265. package/public/assets/EmptyState-D_ulfIIi.js +0 -1
  266. package/public/assets/ExecutorUtilization-B08E76jT.js +0 -1
  267. package/public/assets/GcPressure-BZMi2fTx.js +0 -1
  268. package/public/assets/ImpactEstimate-36WxuPdV.js +0 -1
  269. package/public/assets/IncompleteRun-t8JU7HRt.js +0 -1
  270. package/public/assets/JobFailures-CguB1bec.js +0 -1
  271. package/public/assets/MemoryUtilization-DDk8lz9-.js +0 -1
  272. package/public/assets/OverBroadcast-BHXLt6s7.js +0 -1
  273. package/public/assets/PartitionSizing-COj3uwV-.js +0 -1
  274. package/public/assets/PlanExplorer-CnCdEY_i.js +0 -1
  275. package/public/assets/PlanGraphRoute-BesZOW1n.js +0 -1
  276. package/public/assets/RetryWaste-CcU4ctAn.js +0 -1
  277. package/public/assets/RowPagination-D2aIcfGv.js +0 -1
  278. package/public/assets/ShuffleIO-NPHkWbg1.js +0 -1
  279. package/public/assets/Skew-CzztP-C4.js +0 -1
  280. package/public/assets/SlowHost-Dh2KHypN.js +0 -1
  281. package/public/assets/SmallFiles-CgYSuZQw.js +0 -1
  282. package/public/assets/SpeculationWaste-C8J5z1WS.js +0 -1
  283. package/public/assets/Spill-DwTQPZ_L.js +0 -1
  284. package/public/assets/StageFailed-aFeIEh3p.js +0 -1
  285. package/public/assets/StageFindingGroup-DQmAE3uk.js +0 -1
  286. package/public/assets/StageHeader-D6GBbcZp.js +0 -1
  287. package/public/assets/StagePill-CSKyM8Kd.js +0 -1
  288. package/public/assets/StageShape-J6U4cUcM.js +0 -1
  289. package/public/assets/StageSlowness-C5F2v_ZO.js +0 -1
  290. package/public/assets/Straggler-DcP3dFGA.js +0 -1
  291. package/public/assets/TaskFailures-C-vIEheo.js +0 -1
  292. package/public/assets/UnderBroadcast-DfjYGhoK.js +0 -1
  293. package/public/assets/WidgetLeadSummary-BdUUi7L5.js +0 -1
  294. package/public/assets/downsample-WmNM1qc9.js +0 -1
  295. package/public/assets/impact-sort-DxaUheWV.js +0 -1
  296. package/public/assets/inbox-LtbS_kBS.js +0 -1
  297. package/public/assets/index-27MbqTFr.js +0 -11
  298. package/public/assets/index-CWjjbtyo.css +0 -2
  299. package/public/assets/react-vendor-D2TLlSzI.js +0 -9
  300. package/public/assets/select-CC2Lyrmi.js +0 -1
  301. package/public/assets/toggle-group-lmgO21t9.js +0 -1
  302. package/public/assets/usePagedRows-jNUWgVTD.js +0 -1
  303. package/public/assets/useRecentFiles-qH5Ap4xm.js +0 -12
  304. package/public/assets/useSortMode-CvPY99UV.js +0 -1
  305. package/public/docs/assets/chunks/@localSearchIndexroot.CJBoRv1b.js +0 -1
  306. package/public/docs/assets/contributor-guide_architecture_detector-contract.md.CsCytCho.js +0 -1
  307. package/public/docs/assets/contributor-guide_architecture_drill-down.md.Bawx6NgP.js +0 -1
  308. package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.sf9DU2bA.js +0 -1
  309. package/public/docs/assets/contributor-guide_architecture_overview.md.rhfxoGsI.js +0 -1
  310. package/public/docs/assets/contributor-guide_architecture_overview.md.rhfxoGsI.lean.js +0 -1
  311. package/public/docs/assets/contributor-guide_architecture_state-and-history.md.B8Wc_ZTg.js +0 -1
  312. package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.DYwFeyjV.js +0 -1
  313. package/public/docs/assets/contributor-guide_architecture_worker-protocol.md.CBJkwH24.js +0 -6
  314. package/public/docs/assets/contributor-guide_architecture_worker-protocol.md.CBJkwH24.lean.js +0 -1
  315. package/public/docs/assets/contributor-guide_development-setup.md.s85j2l8b.js +0 -15
  316. package/public/docs/assets/contributor-guide_development-setup.md.s85j2l8b.lean.js +0 -1
  317. package/public/docs/assets/contributor-guide_testing.md.Cnn6gz1C.js +0 -1
  318. package/public/docs/assets/tuning-reference_anti-patterns.md.Df1YMIHu.js +0 -1
  319. package/public/docs/assets/tuning-reference_anti-patterns.md.Df1YMIHu.lean.js +0 -1
  320. package/public/docs/assets/tuning-reference_aqe.md.BIsCtLzm.js +0 -1
  321. package/public/docs/assets/tuning-reference_aqe.md.BIsCtLzm.lean.js +0 -1
  322. package/public/docs/assets/tuning-reference_bottleneck-broadcast-sizing.md.CEstB3Ia.js +0 -1
  323. package/public/docs/assets/tuning-reference_bottleneck-broadcast-sizing.md.CEstB3Ia.lean.js +0 -1
  324. package/public/docs/assets/tuning-reference_bottleneck-cold-start.md.CEuy-72y.js +0 -7
  325. package/public/docs/assets/tuning-reference_bottleneck-cold-start.md.CEuy-72y.lean.js +0 -1
  326. package/public/docs/assets/tuning-reference_bottleneck-duplicate-plan-subtree.md.CIohQDfn.js +0 -1
  327. package/public/docs/assets/tuning-reference_bottleneck-duplicate-plan-subtree.md.CIohQDfn.lean.js +0 -1
  328. package/public/docs/assets/tuning-reference_bottleneck-failures.md.CAsFf3RR.lean.js +0 -1
  329. package/public/docs/assets/tuning-reference_bottleneck-gc.md.B4T3kJYV.js +0 -6
  330. package/public/docs/assets/tuning-reference_bottleneck-gc.md.B4T3kJYV.lean.js +0 -1
  331. package/public/docs/assets/tuning-reference_bottleneck-job-failure-rate.md.BaJl__1W.js +0 -8
  332. package/public/docs/assets/tuning-reference_bottleneck-job-failure-rate.md.BaJl__1W.lean.js +0 -1
  333. package/public/docs/assets/tuning-reference_bottleneck-memory-utilization.md.DbP-SJZc.js +0 -7
  334. package/public/docs/assets/tuning-reference_bottleneck-memory-utilization.md.DbP-SJZc.lean.js +0 -1
  335. package/public/docs/assets/tuning-reference_bottleneck-retry-waste.md.D5JMjVOt.js +0 -1
  336. package/public/docs/assets/tuning-reference_bottleneck-retry-waste.md.D5JMjVOt.lean.js +0 -1
  337. package/public/docs/assets/tuning-reference_bottleneck-shuffle.md.DDMQVOW6.lean.js +0 -1
  338. package/public/docs/assets/tuning-reference_bottleneck-skew.md.B0MzhWcX.lean.js +0 -1
  339. package/public/docs/assets/tuning-reference_bottleneck-slow-host.md.BlIo6UDW.lean.js +0 -1
  340. package/public/docs/assets/tuning-reference_bottleneck-small-files.md.DHGMTmPs.js +0 -5
  341. package/public/docs/assets/tuning-reference_bottleneck-small-files.md.DHGMTmPs.lean.js +0 -1
  342. package/public/docs/assets/tuning-reference_bottleneck-spill.md.PNH7mITt.js +0 -6
  343. package/public/docs/assets/tuning-reference_bottleneck-spill.md.PNH7mITt.lean.js +0 -1
  344. package/public/docs/assets/tuning-reference_bottleneck-straggler.md.DIbXdAcQ.js +0 -7
  345. package/public/docs/assets/tuning-reference_bottleneck-straggler.md.DIbXdAcQ.lean.js +0 -1
  346. package/public/docs/assets/tuning-reference_bottleneck-tiny-tasks.md.CB05v3zh.js +0 -8
  347. package/public/docs/assets/tuning-reference_bottleneck-tiny-tasks.md.CB05v3zh.lean.js +0 -1
  348. package/public/docs/assets/tuning-reference_bottleneck-utilization.md.DqC_iGkw.lean.js +0 -1
  349. package/public/docs/assets/tuning-reference_caching.md.B7aQ8asB.js +0 -1
  350. package/public/docs/assets/tuning-reference_caching.md.B7aQ8asB.lean.js +0 -1
  351. package/public/docs/assets/tuning-reference_cluster-config.md.CD6vzYtD.js +0 -1
  352. package/public/docs/assets/tuning-reference_cluster-config.md.CD6vzYtD.lean.js +0 -1
  353. package/public/docs/assets/tuning-reference_config.md.UvveiWG3.js +0 -1
  354. package/public/docs/assets/tuning-reference_config.md.UvveiWG3.lean.js +0 -1
  355. package/public/docs/assets/tuning-reference_data-formats.md.bjCAWH3N.js +0 -1
  356. package/public/docs/assets/tuning-reference_data-formats.md.bjCAWH3N.lean.js +0 -1
  357. package/public/docs/assets/tuning-reference_index.md.BQ_NooMV.js +0 -1
  358. package/public/docs/assets/tuning-reference_index.md.BQ_NooMV.lean.js +0 -1
  359. package/public/docs/assets/tuning-reference_intro.md.CobD-lGB.js +0 -1
  360. package/public/docs/assets/tuning-reference_intro.md.CobD-lGB.lean.js +0 -1
  361. package/public/docs/assets/tuning-reference_joins.md.BtKs_CuW.js +0 -1
  362. package/public/docs/assets/tuning-reference_joins.md.BtKs_CuW.lean.js +0 -1
  363. package/public/docs/assets/tuning-reference_memory-model.md.CWhvgevP.js +0 -1
  364. package/public/docs/assets/tuning-reference_memory-model.md.CWhvgevP.lean.js +0 -1
  365. package/public/docs/assets/tuning-reference_metrics.md.mLOh7Apj.js +0 -1
  366. package/public/docs/assets/tuning-reference_metrics.md.mLOh7Apj.lean.js +0 -1
  367. package/public/docs/assets/tuning-reference_partitioning.md.q0zKF_8X.js +0 -1
  368. package/public/docs/assets/tuning-reference_partitioning.md.q0zKF_8X.lean.js +0 -1
  369. package/public/docs/assets/tuning-reference_pyspark.md.DDCfvN9t.js +0 -6
  370. package/public/docs/assets/tuning-reference_pyspark.md.DDCfvN9t.lean.js +0 -1
  371. package/public/docs/assets/tuning-reference_shuffle.md.BZZ7R4Ix.js +0 -1
  372. package/public/docs/assets/tuning-reference_shuffle.md.BZZ7R4Ix.lean.js +0 -1
  373. package/public/docs/assets/tuning-reference_spark-architecture.md.Dwzm5avO.js +0 -1
  374. package/public/docs/assets/tuning-reference_spark-architecture.md.Dwzm5avO.lean.js +0 -1
  375. package/public/docs/assets/tuning-reference_table-formats.md.D6wj-2dX.js +0 -1
  376. package/public/docs/assets/tuning-reference_table-formats.md.D6wj-2dX.lean.js +0 -1
  377. package/public/docs/assets/user-guide_getting-started.md.Cpe0U-eO.js +0 -3
  378. package/public/docs/assets/user-guide_getting-started.md.Cpe0U-eO.lean.js +0 -1
  379. package/public/docs/assets/user-guide_mcp-tools.md.DrphooJv.lean.js +0 -1
  380. package/public/docs/assets/user-guide_run-comparison.md.CQc1aoU8.js +0 -1
  381. package/public/docs/assets/user-guide_run-comparison.md.CQc1aoU8.lean.js +0 -1
  382. /package/public/docs/assets/{contributor-guide_architecture_board-widgets.md.9ToZXcTk.lean.js → contributor-guide_architecture_board-widgets.md.VTcYTH4d.lean.js} +0 -0
package/public/docs/contributor-guide/architecture/detector-contract.html CHANGED
@@ -9,11 +9,11 @@
9
9
  <link rel="preload stylesheet" href="/docs/assets/style.DSixAiZE.css" as="style">
10
10
  <link rel="preload stylesheet" href="/docs/vp-icons.css" as="style">
11
11
 
12
- <script type="module" src="/docs/assets/app.C5E-wy6e.js"></script>
12
+ <script type="module" src="/docs/assets/app.C6w47a9W.js"></script>
13
13
  <link rel="preload" href="/docs/assets/inter-roman-latin.Di8DUHzh.woff2" as="font" type="font/woff2" crossorigin="">
14
- <link rel="modulepreload" href="/docs/assets/chunks/theme.DQS9-w_Z.js">
14
+ <link rel="modulepreload" href="/docs/assets/chunks/theme.D-mE6j02.js">
15
15
  <link rel="modulepreload" href="/docs/assets/chunks/framework.DSg0KOwT.js">
16
- <link rel="modulepreload" href="/docs/assets/contributor-guide_architecture_detector-contract.md.CsCytCho.lean.js">
16
+ <link rel="modulepreload" href="/docs/assets/contributor-guide_architecture_detector-contract.md.CsqZHz_T.lean.js">
17
17
  <link rel="icon" type="image/svg+xml" href="/docs/favicon.svg">
18
18
  <link rel="preconnect" href="https://fonts.googleapis.com">
19
19
  <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin="">
@@ -22,8 +22,8 @@
22
22
  <script id="check-mac-os">document.documentElement.classList.toggle("mac",/Mac|iPhone|iPod|iPad/i.test(navigator.platform));</script>
23
23
  </head>
24
24
  <body>
25
- <div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Contributor Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/development-setup.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Development setup</p><!--]--></a><!----></div><!----></div><section class="VPSidebarItem level-1 collapsible collapsed is-link has-active" data-v-b3fd67f8><div class="item" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/" data-v-b3fd67f8><!--[--><h3 class="text" data-v-b3fd67f8>Architecture</h3><!--]--></a><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-b3fd67f8><span class="vpi-chevron-right caret-icon" data-v-b3fd67f8></span></div></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/overview.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Overview</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/worker-protocol.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Worker protocol</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/state-and-history.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>State & history intake</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/detector-contract.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Detector contract</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/impact-estimation.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Impact estimation</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/widget-rendering.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Widget rendering</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/board-widgets.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Board widgets</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/drill-down.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Drill-down</p><!--]--></a><!----></div><!----></div><!--]--></div></section><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/testing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Testing & verification</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/contributing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Contributing</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_contributor-guide_architecture_detector-contract" data-v-39a288b8><div><h1 id="detector-contract" tabindex="-1">Detector contract <a class="header-anchor" href="#detector-contract" aria-label="Permalink to &quot;Detector contract&quot;">​</a></h1><p><code>packages/core/src/detectors.ts</code> is the single source of Spark-optimization logic: one declarative <code>DETECTORS</code> entry per pattern, each carrying <code>type</code>, <code>scope</code> (<code>stage</code> / <code>app</code> / <code>config</code> / <code>sql</code>), <code>order</code>, <code>fixEffort</code>, a <code>thresholds</code> object, impactBand/copy, a <code>docAnchor</code>, and a co-located <code>detect()</code> method. Both consumers are thin loops over that array:</p><ul><li><p><code>packages/core/src/analyzer.ts</code>: <code>analyze()</code> runs every entry regardless of scope, skipping only <code>inScorecard:false</code> ones; <code>auditConfig()</code> separately runs the <code>scope:&#39;config&#39;</code> entries. The four <code>configAudit</code> entries stay out of the bottleneck catalog because each sets <code>inScorecard:false</code>, not because of <code>scope:&#39;config&#39;</code>: a future config-scope detector without that flag would run through <code>analyze()</code> too. Each finding is stamped with its entry&#39;s <code>docAnchor</code>.</p></li><li><p><code>src/view/detector-registry.tsx</code>: a <code>REGISTRY: Record&lt;findingType, {component, region}&gt;</code> replaces <code>dashboard-renderer.js</code>&#39;s <code>render:</code> bindings, one entry per emitted finding type. <code>orderedWidgets()</code> walks <code>DETECTORS</code> ascending by <code>order</code>, then sorts <code>action</code>-region components before <code>reference</code>-region ones. Every <code>finding.type</code> maps to its own component now (the 2026-09 widget/finding-type 1:1 mapping redesign split six components that used to multiplex several types each: <code>TaskSkew</code> into <code>Skew</code>/<code>StageShape</code>/<code>TinyTask</code>; <code>ShuffleIO</code> narrowed to <code>shuffle</code> only, plus a new <code>PartitionSizing</code>; <code>Failures</code> into <code>StageFailed</code>/ <code>TaskFailures</code>/<code>RetryWaste</code>; <code>ExecutorTimeline</code> into <code>SlowHost</code>/ <code>StageSlowness</code>/<code>Straggler</code>/<code>SpeculationWaste</code>/<code>ColdStart</code> (its non-finding-driven executor-count chart moved to <code>ExecutorCountChart</code>, a <code>ReferenceSection</code> tile, not a <code>REGISTRY</code> entry); <code>MemoryUtilization</code> narrowed to <code>memoryUtilization</code> only, plus a new <code>ExecutorUtilization</code> for <code>utilization</code>; <code>PlanFindings</code> into <code>DuplicatePlanSubtree</code>/<code>SmallFiles</code>/ <code>UnderBroadcast</code>/<code>OverBroadcast</code>, dropping the dead <code>broadcastSizing</code> key entirely). No two <code>REGISTRY</code> entries share a <code>component</code> value any more.</p><p>Each widget component receives the full catalog and self-gates when it has nothing to show, rendering <code>null</code> or a muted &quot;no issue&quot; card for the always-visible ones. <code>orderedWidgets()</code> itself has no empty/non-empty branching, since it iterates the static <code>DETECTORS</code> import, not the runtime <code>catalog</code>.</p></li></ul><p>Thresholds live only in each entry&#39;s <code>thresholds</code>; see <a href="#bottleneck-thresholds-spec-§4">Bottleneck thresholds</a>.</p><h2 id="confidence-disclosure" tabindex="-1">Confidence disclosure <a class="header-anchor" href="#confidence-disclosure" aria-label="Permalink to &quot;Confidence disclosure&quot;">​</a></h2><p>A <code>Detector</code> entry (or the <code>Finding</code> it returns) may carry <code>confidence: &#39;low&#39; | &#39;medium&#39; | &#39;high&#39;</code> plus a <code>validationRequired</code> string. <code>RowStatusCluster</code> (<code>src/view/RowStatusCluster.tsx</code>) is the one place that renders it, gated to Advanced density: a plain &quot;&lt;confidence&gt; confidence&quot; badge whose tooltip carries the full <code>validationRequired</code> text. A finding with no <code>confidence</code> field renders identically to a fully-validated one, so every detector whose thresholds are our own unvalidated noise floor (marked <code>NOT SOURCED</code> in a code comment) should set both fields, not just the ones that happen to already have <code>RowStatusCluster</code> wired into their widget. <code>skew</code>, <code>straggler</code>, and <code>gc</code> set <code>confidence</code> for exactly this reason: their runtime-floor thresholds carry the same kind of unvalidated-noise-floor caveat <code>coreLocality</code>, <code>autoscalingChurn</code>, and <code>memoryUtilization</code>&#39;s <code>wasteModel</code> variant already disclose. None of these hardcode a single confidence value: each scales <code>&#39;low&#39; | &#39;medium&#39; | &#39;high&#39;</code> off how far the finding sits past its own detector&#39;s threshold, via a small named helper placed just above the <code>DETECTORS</code> array (e.g. <code>skewConfidence</code>, <code>coreLocalityConfidence</code>, <code>cachingReuseConfidence</code>) rather than an inline literal.</p><h2 id="the-fixeffort-field" tabindex="-1">The <code>fixEffort</code> field <a class="header-anchor" href="#the-fixeffort-field" aria-label="Permalink to &quot;The `fixEffort` field&quot;">​</a></h2><p>Each <code>Detector</code> entry also carries <code>fixEffort: &#39;config&#39; | &#39;code&#39; | &#39;rearchitect&#39;</code>, alongside <code>order</code> and <code>thresholds</code>: a rough estimate of how much work resolving the finding takes.</p><p>No view currently reads it. The quadrant impact/effort bucketing this field was meant to feed (<code>bucketFinding</code>/<code>effortTier</code>/<code>computeImpactMagnitude</code> in a since-deleted <code>src/quadrant-bucket.ts</code>, gated behind a <code>FIX_EFFORT_MAPPING_REVIEWED</code> flag that never flipped to <code>true</code>) was removed as dead code in the recommendations-consolidation redesign: <code>FixTheseFirst</code> (<code>src/view/widgets/FixTheseFirst.tsx</code>) ranks purely by impact magnitude. See <a href="./widget-rendering.html#widget-rendering-order-fixed-spec-§5">Widget rendering order</a> for how it ranks findings today.</p><p>Two shared helpers back multiple detectors and reports. <code>packages/core/src/plan-tree-walk.ts</code>&#39;s <code>walkPlanTree(root, visit, {dedupe})</code> is the iterative pre-order plan-tree traversal used by <code>detectors.ts</code> and every <code>plan-*.ts</code> module (<code>plan-summary.ts</code>, <code>plan-duration-attribution.ts</code>, <code>plan-node-detail.ts</code>, <code>plan-dot.ts</code>). <code>packages/core/src/core-count.ts</code>&#39;s <code>computeTotalCores(app, executorsAdded)</code> is the shared core-count logic used by <code>efficiency-model.ts</code>, <code>scaling-sim.ts</code>, and <code>wasted-core-hours.ts</code>. <code>detectors.ts</code>&#39;s own <code>utilization</code> and <code>memoryUtilization</code> entries use the same file&#39;s <code>computePeakConcurrentCores</code>/<code>computePeakConcurrentExecutorCount</code> instead: <code>computeTotalCores</code> sums every <code>ExecutorAdded</code> event with no regard for overlap, so under executor churn (spot preemption, <code>dynamicAllocation</code> replacement) it double-counts a churned executor&#39;s capacity against its replacement&#39;s; the peak-concurrent sweeps don&#39;t.</p><h2 id="cross-detector-suppression" tabindex="-1">Cross-detector suppression <a class="header-anchor" href="#cross-detector-suppression" aria-label="Permalink to &quot;Cross-detector suppression&quot;">​</a></h2><p>An entry may declare an optional <code>suppressWhen(finding, out)</code> method. <code>analyzer.ts</code>&#39;s <code>push()</code>, the single choke point every finding passes through, calls it per-finding, after the null guard and before the push, and drops the finding silently when it returns <code>true</code>. <code>out</code> is the findings accumulated so far. Since <code>analyze()</code>&#39;s loop is detector-outer / stage-inner, every finding from a detector declared earlier in <code>DETECTORS</code> is already in <code>out</code> by the time a later detector runs, for every stage. That makes the pattern purely declaration-order-driven: the suppressing detector must be declared earlier in the <code>DETECTORS</code> array than the suppressed one.</p><p><code>stageSlowness</code> uses this to defer to <code>slowHost</code>. It is spliced immediately after the <code>slowHost</code> entry regardless of its <code>order</code> field (<code>order</code> only controls render sequencing, not evaluation order), and <code>tests/analyzer.test.js</code>&#39;s &quot;detector contract&quot; suite asserts the array-index ordering so a future reorder can&#39;t silently break the suppression. The mechanism is deliberately minimal: a same-array, predicate-in-<code>push()</code> filter, not a general dependency graph. <code>auditConfig()</code>&#39;s own <code>push()</code> call is unaffected, since <code>scope:&#39;config&#39;</code> entries declare no <code>suppressWhen</code>.</p><h2 id="per-operator-duration-attribution" tabindex="-1">Per-operator duration attribution <a class="header-anchor" href="#per-operator-duration-attribution" aria-label="Permalink to &quot;Per-operator duration attribution&quot;">​</a></h2><p><code>packages/core/src/plan-duration-attribution.ts</code> (entry <code>attributeStageDurationToPlan(planTree, stagesById, sqlExec)</code>) approximates how a SQL execution&#39;s stage wall-time splits across plan operators, returning a <code>Map&lt;planNode, milliseconds&gt;</code>. It cuts the plan tree at Exchange boundaries into connected components: since the Exchange write/read split (<code>resolvePlanTree</code> in <code>event-handlers.ts</code> always synthesizes a <code>read</code> node wrapping a <code>write</code> node for every raw <code>Exchange</code>/<code>BroadcastExchange</code>), the cut is keyed off <code>PlanNode.exchangeRole === &#39;read&#39;</code> on the parent, not a name regex: the write half starts the new component, the read half stays in its parent&#39;s. A node with no <code>exchangeRole</code> at all (for example <code>ReusedExchange</code>, which is never split) never starts a new component on its own, unlike the old name-based regex, which matched any Exchange-family name regardless of split state. Each component receives a stable pre-order identity and separate parent/depth/traversal metadata; the identity itself does not encode its count of Exchange ancestors. It zips components deepest-first by that explicit depth against submission-ordered stage IDs, then apportions each matched stage&#39;s wall-time across that component&#39;s nodes by timing-metric weight, falling back to an even split when no node carries a timing metric.</p><p>This is best-effort inference, not measurement. Spark&#39;s event model exposes no ground truth for per-operator time within a stage; the Exchange-boundary segmentation and deepest-component-to-earliest-stage zip are heuristics. Treat the per-operator numbers as directional hints, never as authoritative timings, and do not build hard thresholds or findings on top of them.</p><h2 id="stage-id-attribution-for-plan-advisor-findings" tabindex="-1">Stage-ID attribution for Plan Advisor findings <a class="header-anchor" href="#stage-id-attribution-for-plan-advisor-findings" aria-label="Permalink to &quot;Stage-ID attribution for Plan Advisor findings&quot;">​</a></h2><p>The Plan Advisor detectors (<code>duplicatePlanSubtree</code>, <code>smallFiles</code>, <code>broadcastSizing</code> in <code>packages/core/src/detectors.ts</code>) each attribute their finding to a narrowed <code>stageIds</code> set rather than the whole SQL execution: <code>PlanNode.stageIds</code> is resolved once per plan tree at parse time by unioning, per node, every metric&#39;s accumulator ID against a <code>taskAccumStages: Map&lt;accumulatorId, Set&lt;stageId&gt;&gt;</code> built while parsing <code>TaskEnd</code> events, then clipping the result to the execution&#39;s own stage set. An accumulator ID occasionally points to a <em>different</em> execution&#39;s stages, e.g. a <code>ReusedSubquery</code> computed once and reused verbatim, and the clip prevents misattributing that other execution&#39;s work. Each detector unions its implicated node(s)&#39; <code>stageIds</code> and falls back to the execution-wide set only when no implicated node has any coverage; a finding never partially blends a narrowed set with the execution-wide one. When an execution has no stage universe at all (no jobs ever recorded against it, which is true for 42% of real-log SQL executions with a plan tree, typically job-less/driver-only executions), the clip drops every candidate stage ID instead of passing them through: every node in that execution&#39;s tree ends up with no <code>stageIds</code> anywhere, same &quot;coverage is partial&quot; framing as below. (An earlier version of this clip treated &quot;no stage universe&quot; as &quot;no clip,&quot; which let a foreign accumulator ID collision, e.g. the <code>ReusedSubquery</code> case above, leak another execution&#39;s stages into a job-less execution&#39;s nodes; the clip is now unconditional on <code>executionStageIds</code> being present.)</p><p>Coverage is partial by Spark&#39;s own design: whole-stage-codegen wrapper nodes (<code>InputAdapter</code>, and other purely structural passthrough markers) carry no accumulators at all, and <code>BroadcastExchangeExec</code>&#39;s own metrics are computed entirely on the driver and never appear on any <code>TaskEnd</code> (real Spark behavior). Since the Exchange write/read split, those driver-computed metrics live specifically on the synthesized <em>write</em> half (<code>exchangeRole: &#39;write&#39;</code>); the <em>read</em> half always carries <code>metrics: []</code>. The write half&#39;s immediate child, which does carry executor-side metrics, is unioned in instead, see <code>overBroadcast</code>&#39;s wiring. A <code>TaskEnd</code> arriving after its stage has already been finalized is also silently excluded from <code>taskAccumStages</code>, consistent with the parser&#39;s existing out-of-order tolerance elsewhere.</p><p><code>planTree</code> itself is kept current against Spark&#39;s adaptive query execution (AQE) re-plans: <code>SparkListenerSQLAdaptiveExecutionUpdate</code> events overwrite the execution&#39;s <code>sparkPlanInfo</code> last-write-wins, so accumulator-ID evidence is matched against the plan that actually ran rather than a stale pre-AQE snapshot. Since a superseded plan is never read, <code>dispatchLine</code> holds an open execution&#39;s latest update as unparsed text and parses only that one, when <code>SQLExecutionEnd</code> arrives or at parse completion (<code>deferAdaptiveUpdate</code>, <code>event-handlers.ts</code>). It recognizes an update from the flat first and last pieces <code>buildChunkDecoder</code> reports for a line joined across slices (<code>JoinedLine</code>), so a superseded update is never copied into one flat string either. The raw <code>sparkPlanInfo</code> stays worker-side and is released once <code>SQLExecutionEnd</code> resolves it into <code>planTree</code>; <code>physicalPlanDescription</code> (Spark&#39;s text rendering of the plan, which nothing reads) is emptied before <code>JSON.parse</code> and never retained (<code>stripPlanDescription</code>, <code>event-handlers.ts</code>). <code>buildChunkDecoder</code> decodes each decompressed chunk in slices of at most 512 KiB, and drops the bytes of a value that crosses a slice boundary without decoding them.</p><p>No eviction/pruning is added to <code>taskAccumStages</code>, a deliberate choice, not an oversight: measured on real logs, it holds roughly 1,050 keys per compressed MB (9,850 keys on an 11.6 MB fixture, about 29,000 keys on a 28.1 MB fixture). Extrapolated to a 240MB+ log, the scale this tool targets (see <code>CLAUDE.md</code>), that is roughly 250,000 keys, around 45 MB of heap for an equivalent synthetic <code>Map&lt;number, Set&lt;number&gt;&gt;</code>. This heap estimate is still small relative to this tool&#39;s other in-memory state. It is higher, though, than the fixture-only measurements taken when this mechanism was built suggested.</p><p>Per-execution pruning (e.g. dropping a <code>taskAccumStages</code> entry once its stage finalizes or its owning SQL execution resolves, mirroring how <code>accumState</code> is cleared in <code>endSqlExecution</code>) is deliberately not done either: unlike <code>accumState</code>, <code>taskAccumStages</code> is one global, un-scoped map read by every execution&#39;s <code>resolvePlanTree</code> call, and the <code>ReusedSubquery</code> case above depends on a stage recorded under one execution still being visible when a later execution resolves. Pruning on any single execution&#39;s lifecycle would break that cross-execution lookup. What is bounded is the growth from a single pathological event: <code>TaskEndEventSchema</code>&#39;s <code>Accumulables</code> array is capped at <code>MAX_ACCUMULABLES_PER_TASK</code> (10,000, <code>event-schemas.ts</code>), well above any real plan&#39;s per-task metric count, so a single crafted <code>TaskEnd</code> can&#39;t grow the map past that per-event bound; a <code>TaskEnd</code> exceeding it fails schema validation and the line is skipped (counted in <code>skippedLines</code>) like any other malformed event. Each accumulable is checked for its numeric <code>ID</code> only: <code>Update</code>/<code>Value</code> are unread, and Spark writes some as JSON arrays (<code>internal.metrics.updatedBlockStatuses</code>), which a stricter check would reject along with the whole task. When every entry has Spark&#39;s flat <code>{&quot;ID&quot;:n,...}</code> form, <code>parseTaskEnd</code> (<code>event-handlers.ts</code>) reads the IDs with a string scan and cuts the array out before <code>JSON.parse</code>; any other shape is parsed whole.</p><h2 id="bottleneck-thresholds-spec-§4" tabindex="-1">Bottleneck thresholds (spec §4) <a class="header-anchor" href="#bottleneck-thresholds-spec-§4" aria-label="Permalink to &quot;Bottleneck thresholds (spec §4)&quot;">​</a></h2><p>Every change to <code>packages/core/src/detectors.ts</code> should reference this table.</p><p>Every finding&#39;s <code>impactBand</code> comes from one of two places. For any finding whose <code>impactEstimate</code> carries a <code>wallClock</code> estimate (the common case for most rules below), <code>analyzer.ts</code> calls <code>deriveImpactBand()</code> (<code>packages/core/src/impact-band.ts</code>) immediately after <code>estimateImpact()</code>, which sets <code>.impactBand</code> purely from <code>wallClock.high</code> as a fraction of the app&#39;s total duration (<code>&gt;= 2%</code> critical, <code>&gt;= 0.5%</code> warning, else info: the same <code>floorPctWarn</code>/<code>floorPctCrit</code> values <code>skew</code>/<code>straggler</code> use for their own thresholds below). For those rules, the table below documents their firing gate plus their fixed fallback constant, which surfaces only when this run&#39;s finding of that type didn&#39;t get a wallClock estimate (a stage excluded from the occupancy sweep). For rules whose finding type never gets a wallClock estimate (<code>resourceOnly</code>/<code>informational</code> basis, e.g. <code>configAudit</code>, or a rule that keeps its own ratio-tiered classification per the design&#39;s Decision 2, e.g. <code>failures</code>), the full threshold table below is the real, displayed classification: <code>detectors.ts</code> sets <code>impactBand</code> directly and nothing overwrites it. <code>partitionSizing</code>&#39;s <code>maxPartitionTooBig</code> rule is a third case: it does carry a <code>wallClock</code> estimate but is explicitly exempted in <code>deriveImpactBand()</code> because it&#39;s a hardcoded-critical OOM/crash-risk safety signal, not a time-recovery one, so <code>detectors.ts</code>&#39;s own classification stands regardless of how small that estimate is relative to the run.</p><h3 id="fixed-fallback-only-usually-wallclock-derived-instead" tabindex="-1">Fixed fallback only (usually wallClock-derived instead) <a class="header-anchor" href="#fixed-fallback-only-usually-wallclock-derived-instead" aria-label="Permalink to &quot;Fixed fallback only (usually wallClock-derived instead)&quot;">​</a></h3><p>These rules&#39; <em>band</em> tiers were deleted from <code>packages/core/src/detectors.ts</code> (they were always overwritten by <code>deriveImpactBand</code> whenever a wallClock estimate was available); the constant in the last column is only a floor-case fallback. Their <em>firing</em> gate is untouched and still lives in each entry&#39;s <code>thresholds</code> object: it decides whether the rule reports anything, so it stays documented here in full.</p><table tabindex="0"><thead><tr><th>Rule</th><th>Fires when</th><th>Fallback</th></tr></thead><tbody><tr><td>Task skew</td><td><code>taskDurationP95 / taskDurationP50 &gt; 3×</code> (<code>taskDurationMax / P50</code> for stages under <code>minTasksForP95</code> = 20 tasks), <strong>and</strong> the occupancy-clipped tail recovery (<code>tailRecoveryMs</code>: <code>finalizeStage</code>&#39;s task-level replay of the stage with every task over 4× P50 capped at P50) is ≥ <code>floorPctWarn</code> = 0.5% of app runtime. The clip floors the claim at the longest task the fix leaves, not the current one, and at the core work the fix leaves over every core (see impact-estimation.md&#39;s occupancy section); <code>straggler</code>&#39;s floors use the same clip</td><td><code>warning</code></td></tr><tr><td>Shuffle read</td><td><code>shuffleReadBytes &gt; minBytes</code> = 50 MiB, on a stage that isn&#39;t shorter than <code>stageFloorPct</code> = 0.5% of the run (a zero-length stage or an unknown run duration passes; the claim is clipped to the stage, so a shorter one graded <code>info</code>: 182 of 284 on the 14 real logs, the shuffle itself still there)</td><td><code>info</code></td></tr><tr><td>Partition sizing: skew</td><td><code>shuffleReadMax &gt; 5×</code> <code>shuffleReadP50</code> <strong>and</strong> <code>shuffleReadMax &gt; 256 MiB</code></td><td><code>warning</code></td></tr><tr><td>Partition sizing: low parallelism</td><td><code>shuffleReadBytes ≥ 1 GiB</code> <strong>and</strong> <code>taskCount ≤ 7</code></td><td><code>warning</code></td></tr><tr><td>Partition sizing: oversized partition</td><td><code>shuffleReadMax ≥ 5 GiB</code></td><td><code>critical</code></td></tr><tr><td>GC</td><td><code>executorRunTime ≥ minRunTimeMs</code> = 10 s <strong>and</strong> <code>gcPct &gt; 10%</code></td><td><code>warning</code></td></tr><tr><td>GC (low / cost)</td><td><code>executorRunTime ≥ 10 s</code> <strong>and</strong> <code>gcPct &lt; lowInfoPct100</code> = 5% (checked only when the GC row above did not fire) <strong>and</strong> the stage lasts ≥ <code>lowInfoFloorPct</code> = 0.5% of the run (passes when the run&#39;s duration is unknown or the stage has zero length, such as one with no completion time). Gets no wall-clock estimate (an over-provisioning signal), so this band always stands. The floor dropped 464 of 685 low-GC notes on the 14 real logs (none on the corpus): low GC is still true on those stages, but a memory-sizing note from a stage that barely ran adds nothing</td><td><code>info</code></td></tr><tr><td>Spill (magnitude v2)</td><td>any non-zero <code>memoryBytesSpilled</code>, on a stage that isn&#39;t shorter than <code>stageFloorPct</code> = 0.5% of the run (as for shuffle read: 12 of 38 on the 14 real logs, all <code>info</code>). The magnitude sub-table below classifies <em>how much</em>, but does not gate firing</td><td><code>warning</code></td></tr><tr><td>Cold start</td><td><code>firstExecutorAddedAt − firstStageSubmittedAt &gt; gapSeconds</code> = 30 s (no finding without executor-added events): the time a runnable stage waited for its first executor. An executor added before the first stage and still alive at submission means no wait and no finding; one removed at or before submission is ignored, so the gap runs to the next executor added after it. It used to be <code>firstStageSubmittedAt − app.startTime</code>, the driver&#39;s own startup, which read 36-47 s on all 9 real logs that fired whatever the executors did, and flagged 6 corpus logs whose first executor was up 44-313 s before the first stage (5 of them) or arrived 4.4 s after it</td><td><code>warning</code></td></tr><tr><td>Slow host: mean-duration ratio</td><td>stage has ≥ <code>minHosts</code> = 3 hosts (or executors) and ≥ <code>minTasks</code> = 15 tasks, and lasts ≥ <code>stageFloorPct</code> = 0.5% of the run (passes when the run&#39;s duration is unknown or the stage has zero length, which gets no estimate and keeps its fallback band; every slow-host finding on a shorter stage graded <code>info</code>, 324 of 452 on the 14 real logs, the imbalance itself still true; this gate covers every slowHost row); then per host: mean task duration / overall median ≥ <code>ratioWarn</code> = 2.0× <strong>and</strong> host task-share ≥ <code>minShare</code> = 20% <strong>and</strong> host mean ≥ <code>floorMs</code> = 1000 ms (absolute-magnitude floor, rules out sub-second noise)</td><td><code>warning</code></td></tr><tr><td>Slow host: duration-share</td><td>same stage gate as the row above; then per host: ≥ <code>shareWarn</code> = 75% of the stage&#39;s total task-duration <strong>and</strong> ≥ <code>taskShareWarn</code> = 50% of its task count</td><td><code>warning</code></td></tr><tr><td>Stage slowness: absolute fallback, suppressed when <code>slowHost</code> already fired</td><td>stage wall-clock duration ≥ <code>infoMin</code> = 15 min. The band then comes from the partitioning-headroom estimate (see impact-estimation.md), not the duration</td><td><code>info</code></td></tr><tr><td>Straggler / speculative-execution</td><td><code>taskCount ≥ minTasks</code> = 10, <strong>and</strong> the stage lasts ≥ <code>floorPctWarn</code> = 0.5% of the run (passes when the run&#39;s duration is unknown or the stage has zero length, such as one with no completion time; a shorter stage&#39;s tail can&#39;t cost more than its own duration, so every finding there graded <code>info</code>: 671 of 753 on the 14 real logs, 3 on the corpus, the slow tail itself still true), <strong>and</strong> either any speculative task ran, <strong>or</strong> straggler share &gt; <code>shareWarn</code> = 5%, <strong>or</strong> straggler share &gt; <code>shareWarnAtFloor</code> = 2.5% with the occupancy-clipped tail recovery (<code>tailRecoveryMs</code>, as for skew) already ≥ <code>floorPctWarn</code> (0.5% of app runtime). The lower gate is scored against a task-level replay of every stage on 14 real logs (recoverable = list-scheduling replay with each task over 4× P50 capped at P50; positive = ≥ 0.5% of app runtime): it found 3 stages whose stragglers gated them for 10-48s at 2.7-4% of their tasks, for 1 borderline miss, lifting skew-or-straggler recall from 0.86 to 0.92 at precision 0.91 → 0.89. Admitting every 2.5% share instead would add 86 findings below the floor. The same sweep kept skew&#39;s <code>ratioWarn</code> = 3 (2.5 added false positives, 4 lost true ones) and the 0.5% floor (1% halved skew recall). <code>warnPct</code>/<code>critPct</code> (10%/20% speculative share) and <code>floorPctWarn</code>/<code>floorPctCrit</code> (0.5%/2% of app runtime) no longer set the band; they rank the straggler-vs-speculative tiers that pick which <em>metric</em> the finding reports</td><td><code>info</code></td></tr><tr><td>Speculation waste (new)</td><td><code>speculationWastedAttempts ≥ minWasted</code> = 5 <strong>and</strong> <code>speculationWasteMs ≥ minWasteMs</code> = 60 s</td><td><code>warning</code></td></tr><tr><td>Retry waste</td><td><code>wastedAttempts ≥ minWasted</code> = 3 <strong>and</strong> <code>retryWasteMs ≥ minWasteMs</code> = 30 s, on a stage that still completed</td><td><code>warning</code></td></tr><tr><td>Tiny tasks</td><td><code>taskCount ≥ minTasks</code> = 100 <strong>and</strong> <code>taskDurationP50 ≤ maxP50</code> = 500 ms <strong>and</strong> <code>taskDurationP95 ≤ maxP95</code> = 1000 ms <strong>and</strong> the stage lasts ≥ <code>stageFloorPct</code> = 0.5% of the run (passes when the run&#39;s duration is unknown or the stage has zero length; coalescing can&#39;t save more than the stage&#39;s own duration, so every finding on a shorter stage graded <code>info</code>: 132 of 151 on the 14 real logs, the tasks still tiny). Checked (2026-09-23) against each stage&#39;s measured per-task overhead (task wall time minus executor run time, the estimate&#39;s own input) on 14 real logs and the corpus: of 228 stages with 100+ tasks outside the P50/P95 gate, none would save 0.5% of its run by coalescing (median overhead 0.2-3% of task time); of 170 inside it, 25 would, and all 25 grade above <code>info</code></td><td><code>info</code></td></tr><tr><td>Duplicate plan subtree</td><td>a subtree of ≥ <code>minSubtreeSize</code> = 3 nodes whose shape fingerprint repeats ≥ <code>minOccurrences</code> = 2× in the plan, unless its linked stages together lasted less than <code>stageFloorPct</code> = 0.5% of the run (a repeat with no linked stage time, or an unknown run duration, is kept; the claim counts at most each stage&#39;s own task-active time, so such a repeat graded <code>info</code>: 340 of 545 on the 14 real logs, the repeat still in the plan). <code>occurrencesIdentical</code> records whether the repeats also agree node-for-node on normalized detail, ignoring AQE query-stage numbers; when they don&#39;t (same shape over another table, filter or projection: 270 of 546 groups on the 14 real logs) the finding is <code>info</code> with confidence <code>low</code> and no time claim. <code>stageShares</code> gives, per stage, the repeated operators&#39; share of the stage&#39;s operators (WholeStageCodegen wrappers and Exchange write halves not counted); with no attributed stage the finding is <code>info</code></td><td><code>warning</code></td></tr><tr><td>Small files read/write</td><td>per read/write side: file count &gt; <code>minFiles</code> = 100 <strong>and</strong> average file size &lt; <code>maxAvgFileSizeMB</code> = 3 MiB</td><td><code>warning</code></td></tr><tr><td>Broadcast sizing: missed</td><td>a 2-child <code>SortMergeJoin</code> whose smaller side is &lt; 10 MiB (unconditional), or &lt; 100 MiB with the larger side &gt; 10 GiB, or &lt; 1 GiB with larger &gt; 300 GiB, or &lt; 5 GiB with larger &gt; 1 TiB (<code>broadcastTiers</code> × <code>comparisonTiers</code>)</td><td><code>info</code></td></tr><tr><td>Broadcast sizing: oversized</td><td>a <code>BroadcastExchange</code> node whose <code>data size</code> metric &gt; <code>overBroadcastBytes</code> = 1 GiB</td><td><code>warning</code></td></tr></tbody></table><h4 id="spill-magnitude-tiers" tabindex="-1">Spill magnitude tiers <a class="header-anchor" href="#spill-magnitude-tiers" aria-label="Permalink to &quot;Spill magnitude tiers&quot;">​</a></h4><p>The spill row&#39;s band is a fixed <code>warning</code> fallback, but <code>computeSpillMagnitude</code> (<code>packages/core/src/detectors.ts</code>) still runs on every spill finding and sets its <code>spillMagnitude</code> field, which the Spill widget displays. Its tiers, in evaluation order (first match wins, <code>null</code> when nothing matches):</p><table tabindex="0"><thead><tr><th>Condition</th><th>Magnitude</th></tr></thead><tbody><tr><td>Single-task stage: <code>spillDiskMax ≥ singleTaskDiskGiB</code> = 1 GiB <strong>or</strong> <code>spillMemMax ≥ singleTaskMemGiB</code> = 4 GiB</td><td><code>severe</code></td></tr><tr><td>Multi-task stage: <code>spillDiskMax ≥ highDiskGiB</code> = 1 GiB, <strong>or</strong> <code>spillDiskMax / taskCount ≥ highTaskDiskMB</code> = 512 MiB (per-task proxy), <strong>or</strong> <code>spillMemMax ≥ highMemGiB</code> = 4 GiB</td><td><code>high</code></td></tr><tr><td>Multi-task stage: <code>spillDiskMax ≥ medDiskMB</code> = 256 MiB <strong>or</strong> <code>spillMemMax ≥ medMemGiB</code> = 1 GiB</td><td><code>medium</code></td></tr><tr><td>Skew (<code>taskCount ≥ skewMinTasks</code> = 10): <code>spillDiskMax / spillDiskP50 &gt; skewRatio</code> = 5× <strong>and</strong> <code>spillDiskMax ≥ skewDiskFloorMB</code> = 128 MiB</td><td><code>high</code></td></tr><tr><td>Skew (<code>taskCount ≥ 10</code>): <code>spillMemMax / spillMemP50 &gt; 5×</code> <strong>and</strong> <code>spillMemMax ≥ skewMemFloorMB</code> = 256 MiB</td><td><code>medium</code></td></tr></tbody></table><p>Disk spill is weighted worse than memory spill by design: the memory thresholds sit well above their disk counterparts at every tier.</p><h3 id="full-threshold-table-never-wallclock-derived" tabindex="-1">Full threshold table (never wallClock-derived) <a class="header-anchor" href="#full-threshold-table-never-wallclock-derived" aria-label="Permalink to &quot;Full threshold table (never wallClock-derived)&quot;">​</a></h3><table tabindex="0"><thead><tr><th>Rule</th><th>Warning</th><th>Critical</th></tr></thead><tbody><tr><td>Stage shape: PRatio</td><td><code>taskCount / totalCores &lt; 0.5</code> (info, under-parallelized), on a stage lasting ≥ <code>lowParallelismFloorPct</code> = 0.5% of the run (passes when the run&#39;s duration is unknown or the stage has zero length): parallelizing can&#39;t save more than the stage&#39;s duration, and on the 14 real logs 2839 of 3005 firings were below it</td><td>none</td></tr><tr><td>Stage shape: OIRatio</td><td><code>outputBytes / inputBytes &gt; 10×</code> (info, data explosion)</td><td>none</td></tr><tr><td>Stage shape: TaskStageSkew</td><td><code>taskDurationMax / stageDuration &gt; 3×</code> (info)</td><td>none</td></tr><tr><td>Failed tasks</td><td>failure rate &gt; 5% (min 10 tasks)</td><td>&gt; 20%</td></tr><tr><td>Stage failed outright</td><td>none</td><td>any <code>stageFailureReason</code> present</td></tr><tr><td>Slow host: multi-dimensional</td><td>max/median ratio across taskTime/inputBytes/shuffleBytes/storageMemory ≥ 1.33× (info); each dimension&#39;s sample must also clear an absolute floor (1000 ms for taskTime, 64 MiB for the byte dimensions). A stage lasting under <code>stageFloorPct</code> = 0.5% of the run is skipped (see the mean-duration row): a byte dimension there, which has no time estimate, used to keep its ratio tier (66 of 90 warning/critical on the 14 real logs) until 6298149 graded it <code>info</code></td><td>≥ 3.16× warning, ≥ 10× critical</td></tr><tr><td>Utilization</td><td>avg active executors / peak &lt; 60% (info)</td><td>none</td></tr><tr><td>Autoscaling churn: short-lived executors (design spike, unvalidated thresholds)</td><td>&gt; 30% of executors alive under 2 min (min 5 executors)</td><td>&gt; 60%</td></tr><tr><td>Job failure rate</td><td>≥ 30% (≥ 10% info)</td><td>≥ 50%</td></tr><tr><td>Idle cores</td><td>busy-core-time / (peak cores × wall-clock) idle &gt; 50% (warning)</td><td>none</td></tr><tr><td>Memory band</td><td>peak heap / allocated &gt; 95% too-small (warning); &lt; 70% over-provisioned (info)</td><td>none</td></tr><tr><td>Caching opportunity</td><td>same input relation (or join/union subtree) scanned by ≥ <code>minExecutions</code> = 2 SQL executions in one run</td><td>none (single tier, info)</td></tr><tr><td>Cache utilization: partial caching (this repo)</td><td><code>numCachedPartitions / numPartitions &lt; 0.90</code> (info)</td><td><code>&lt; 0.50</code> (warning)</td></tr><tr><td>Cache utilization: disk spillover (this repo)</td><td><code>diskSize / (memorySize + diskSize) &gt; 0.15</code> (info), <code>MEMORY_AND_DISK*</code> only</td><td><code>&gt; 0.40</code> (warning)</td></tr></tbody></table><p>Spill classification: ≥80% tasks with zero spill → <code>skew</code>; &lt;20% zero → <code>volume</code>; else <code>unclassified</code>. The classification badge is always shown in both compact and expanded spill widget states. This is independent of the magnitude tiers above: classification says <em>what kind</em> of spill, magnitude says <em>how much</em>.</p><h4 id="evidence-fields-stagefailed-retrywaste" tabindex="-1">Evidence fields (<code>stageFailed</code> / <code>retryWaste</code>) <a class="header-anchor" href="#evidence-fields-stagefailed-retrywaste" aria-label="Permalink to &quot;Evidence fields (`stageFailed` / `retryWaste`)&quot;">​</a></h4><p>Neither entry&#39;s <code>detect()</code> used to put anything beyond a scalar <code>metric</code>/ <code>value</code> on its finding. Both now also attach:</p><ul><li><code>numTasks</code>: <code>stage.taskCount</code> at detection time.</li><li><code>memoryBytesSpilled</code>: <code>stage.memoryBytesSpilled</code> at detection time.</li><li><code>stageFailed</code> only: <code>failedTaskDetails</code>: up to 20 <code>FailedTaskSample</code> records (<code>taskId</code>, <code>attemptNumber</code>, <code>host</code>, <code>executorId</code>, <code>reason</code>, <code>peakExecMem</code>, <code>memSpilled</code>, <code>shuffleWrite</code>) for tasks still marked failed when the stage was finalized (<code>finalizeStage</code>, <code>stage-quantiles.ts</code>).</li><li><code>retryWaste</code> only: <code>retriedTaskDetails</code>: up to 20 <code>FailedTaskSample</code> records for attempts discarded by the retry-dedup logic in <code>accumulateTask</code> (<code>event-handlers.ts</code>): captured at the moment they&#39;d otherwise be thrown away, since by finalize time only the winning attempt survives.</li></ul><p>Both sample arrays are capped at 20 entries, filled in first-encountered order (finalize order for <code>failedTaskDetails</code>, discard order for <code>retriedTaskDetails</code>), not spread across distinct hosts/executors: a stage with failures clustered on one bad host could fill the cap before a more informative failure elsewhere in the stage is ever sampled.</p><h4 id="evidence-fields-failures" tabindex="-1">Evidence fields (<code>failures</code>) <a class="header-anchor" href="#evidence-fields-failures" aria-label="Permalink to &quot;Evidence fields (`failures`)&quot;">​</a></h4><ul><li><code>dominantReason</code>: the most frequent end-reason tag (<code>ExceptionFailure</code>, <code>ExecutorLostFailure</code>, ...) among tasks still failed at finalize.</li><li><code>dominantError</code>: the error behind that tag, from its largest failure group: the exception class, or <code>&lt;tag&gt;: &lt;loss reason&gt;</code>; falls back to the tag. The recommendation names it and never embeds a message.</li><li><code>failureGroups</code>: up to 5 <code>TaskFailureGroup</code>s (<code>reason</code>, <code>className</code>, <code>message</code>, <code>lossReason</code>, <code>stackExcerpt</code>, <code>count</code>), most frequent first: one group per distinct tag, class, message and loss reason, one excerpt each.</li><li><code>otherFailedTasks</code>: failed tasks that no shown group covers.</li></ul><p><code>packages/core/src/task-failure.ts</code> reads the details from <code>Task End Reason</code> (<code>Class Name</code>, <code>Description</code>, <code>Full Stack Trace</code>, <code>Loss Reason</code>, FetchFailed <code>Message</code>, <code>Kill Reason</code>) and bounds them at ingest: a message or loss reason to its first line (for a Python traceback, its final error line), 300 characters; an excerpt to the header, 8 frames, a Python traceback&#39;s error line and the last <code>Caused by:</code> line, 2000 characters in all. While parsing, a stage keeps at most 50 distinct failures (<code>StageRecord.failureDetails</code>, freed at finalize); later ones count only toward the tag. Redaction (<code>redact.ts</code>) replaces every <code>failureGroups[].message</code> and strips message text from <code>stackExcerpt</code>.</p></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/contributor-guide/architecture/state-and-history.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>State & history intake</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="/docs/contributor-guide/architecture/impact-estimation.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>Impact estimation</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
26
- <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"9ToZXcTk\",\"contributor-guide_architecture_detector-contract.md\":\"CsCytCho\",\"contributor-guide_architecture_drill-down.md\":\"Bawx6NgP\",\"contributor-guide_architecture_impact-estimation.md\":\"sf9DU2bA\",\"contributor-guide_architecture_index.md\":\"3TO9ic6w\",\"contributor-guide_architecture_overview.md\":\"rhfxoGsI\",\"contributor-guide_architecture_state-and-history.md\":\"B8Wc_ZTg\",\"contributor-guide_architecture_widget-rendering.md\":\"DYwFeyjV\",\"contributor-guide_architecture_worker-protocol.md\":\"CBJkwH24\",\"contributor-guide_contributing.md\":\"B2E9ni79\",\"contributor-guide_development-setup.md\":\"s85j2l8b\",\"contributor-guide_testing.md\":\"Cnn6gz1C\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"Df1YMIHu\",\"tuning-reference_aqe.md\":\"BIsCtLzm\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"CEstB3Ia\",\"tuning-reference_bottleneck-cold-start.md\":\"CEuy-72y\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"CIohQDfn\",\"tuning-reference_bottleneck-failures.md\":\"CAsFf3RR\",\"tuning-reference_bottleneck-gc.md\":\"B4T3kJYV\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"BaJl__1W\",\"tuning-reference_bottleneck-memory-utilization.md\":\"DbP-SJZc\",\"tuning-reference_bottleneck-retry-waste.md\":\"D5JMjVOt\",\"tuning-reference_bottleneck-shuffle.md\":\"DDMQVOW6\",\"tuning-reference_bottleneck-skew.md\":\"B0MzhWcX\",\"tuning-reference_bottleneck-slow-host.md\":\"BlIo6UDW\",\"tuning-reference_bottleneck-small-files.md\":\"DHGMTmPs\",\"tuning-reference_bottleneck-spill.md\":\"PNH7mITt\",\"tuning-reference_bottleneck-straggler.md\":\"DIbXdAcQ\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"CB05v3zh\",\"tuning-reference_bottleneck-utilization.md\":\"DqC_iGkw\",\"tuning-reference_caching.md\":\"B7aQ8asB\",\"tuning-reference_cluster-config.md\":\"CD6vzYtD\",\"tuning-reference_config.md\":\"UvveiWG3\",\"tuning-reference_data-formats.md\":\"bjCAWH3N\",\"tuning-reference_index.md\":\"BQ_NooMV\",\"tuning-reference_intro.md\":\"CobD-lGB\",\"tuning-reference_joins.md\":\"BtKs_CuW\",\"tuning-reference_memory-model.md\":\"CWhvgevP\",\"tuning-reference_metrics.md\":\"mLOh7Apj\",\"tuning-reference_partitioning.md\":\"q0zKF_8X\",\"tuning-reference_pyspark.md\":\"DDCfvN9t\",\"tuning-reference_shuffle.md\":\"BZZ7R4Ix\",\"tuning-reference_spark-architecture.md\":\"Dwzm5avO\",\"tuning-reference_table-formats.md\":\"D6wj-2dX\",\"user-guide_alternative-log-retrieval.md\":\"ZHPSbBkF\",\"user-guide_getting-started.md\":\"Cpe0U-eO\",\"user-guide_mcp-tools.md\":\"DrphooJv\",\"user-guide_run-comparison.md\":\"CQc1aoU8\",\"user-guide_understanding-findings.md\":\"YBBLOwWi\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Alternative log retrieval\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
25
+ <div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Contributor Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/development-setup.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Development setup</p><!--]--></a><!----></div><!----></div><section class="VPSidebarItem level-1 collapsible collapsed is-link has-active" data-v-b3fd67f8><div class="item" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/" data-v-b3fd67f8><!--[--><h3 class="text" data-v-b3fd67f8>Architecture</h3><!--]--></a><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-b3fd67f8><span class="vpi-chevron-right caret-icon" data-v-b3fd67f8></span></div></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/overview.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Overview</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/worker-protocol.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Worker protocol</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/state-and-history.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>State & history intake</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/detector-contract.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Detector contract</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/impact-estimation.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Impact estimation</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/widget-rendering.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Widget rendering</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/board-widgets.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Board widgets</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/drill-down.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Drill-down</p><!--]--></a><!----></div><!----></div><!--]--></div></section><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/testing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Testing & verification</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/contributing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Contributing</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_contributor-guide_architecture_detector-contract" data-v-39a288b8><div><h1 id="detector-contract" tabindex="-1">Detector contract <a class="header-anchor" href="#detector-contract" aria-label="Permalink to &quot;Detector contract&quot;">​</a></h1><p><code>packages/core/src/detectors.ts</code> is the single source of Spark-optimization logic: one declarative <code>DETECTORS</code> entry per pattern, each carrying <code>type</code>, <code>scope</code> (<code>stage</code> / <code>app</code> / <code>config</code> / <code>sql</code>), <code>order</code>, <code>fixEffort</code>, a <code>thresholds</code> object, impactBand/copy, a <code>docAnchor</code>, an <code>emits</code> list, and co-located <code>detect()</code> and <code>estimate()</code> functions.</p><p>Each entry is built by the helper for its scope: <code>defineStageDetector</code>, <code>defineSqlDetector</code>, <code>defineAppDetector</code> or <code>defineConfigDetector</code>. The helper sets <code>scope</code>, infers the thresholds type from the entry&#39;s own <code>thresholds</code> literal, and types <code>detect</code> as a function property, so its parameters are checked strictly rather than bivariantly:</p><table tabindex="0"><thead><tr><th>Scope</th><th><code>detect</code> signature</th></tr></thead><tbody><tr><td><code>stage</code></td><td><code>detect(stage, ctx, thresholds)</code></td></tr><tr><td><code>sql</code></td><td><code>detect(sqlExec, ctx, thresholds)</code></td></tr><tr><td><code>app</code></td><td><code>detect(ctx, thresholds)</code></td></tr><tr><td><code>config</code></td><td><code>detect(target, thresholds)</code>, where <code>target</code> is <code>{ app }</code></td></tr></tbody></table><p><code>ctx</code> (<code>DetectorCtx</code>) is required, and its <code>app</code> is nullable as on <code>AppModel.app</code>, so a detector that reads the app without a guard, reads a threshold its entry doesn&#39;t declare, or expects another scope&#39;s target fails to compile. <code>detect</code> never reads <code>this</code>. The helper freezes the entry&#39;s <code>thresholds</code> and adds <code>withThresholds(overrides?)</code>, which returns <code>detect</code> with the thresholds bound: the entry&#39;s own, or the caller&#39;s overrides merged over them (see <a href="#tuning-thresholds">Tuning thresholds</a>). Runners such as <code>analyze()</code> call only that.</p><p><code>estimate(finding, ctx)</code> prices one of the entry&#39;s own findings: <code>finding</code> is typed as the <code>Finding</code> member of a type the entry <code>emits</code>, and <code>ctx</code> (<code>EstimateCtx</code>: <code>stages</code>, <code>occupancy</code>, <code>totalCores</code>) is the one occupancy sweep <code>analyze()</code> builds per run. It returns an <code>ImpactEstimate</code> or null (see <a href="./impact-estimation.html">Impact estimation</a>). Every entry must declare one; an entry with no waste model passes <code>noWasteModel</code>. <code>estimateImpact()</code> runs it after suppression, keyed by emitted type. The same <code>EstimateCtx</code> reaches <code>detect()</code> as <code>ctx.impact</code>, so a runtime floor can gate on the estimate the finding will display: <code>skew</code> and <code>straggler</code> build their tail claim once (<code>skewTailClaim</code>/<code>stragglerTailClaim</code>) and both their floor and their <code>estimate()</code> read it through <code>tailClaimImpact</code>. Shared model constants and helpers live in <code>packages/core/src/impact-model.ts</code>.</p><p><code>DETECTORS</code> is declared <code>as const satisfies readonly Detector[]</code>, so each entry keeps its literal <code>type</code> and <code>emits</code>. Two unions derive from it: <code>DetectorType</code> (every entry&#39;s own <code>type</code>) and <code>FindingType</code> (every type an entry&#39;s <code>emits</code> lists, the finding types that actually appear on findings). <code>emits</code> is <code>[type]</code> for every entry except <code>broadcastSizing</code>, whose one plan walk emits <code>overBroadcast</code> and <code>underBroadcast</code> and never its own name. Every per-type lookup keys on the emitted <code>FindingType</code>, never on <code>DetectorType</code>: code that needs the emitted types of an entry reads its <code>emits</code> list. Code that iterates entries generically, such as <code>analyze()</code>, reads them through the <code>Detector</code> type, a union over scopes with the thresholds type erased, <code>detect</code> left off and <code>estimate</code> taking any <code>Finding</code>, so switching on <code>scope</code> narrows the bound function <code>withThresholds()</code> returns.</p><p>How a finding type is presented is registered once, in <code>FINDING_PRESENTATION</code> (<code>packages/core/src/finding-presentation.ts</code>), typed <code>{ [T in FindingType]: FindingPresentation&lt;T&gt; }</code> so the compiler requires exactly one row per emitted type. A row holds the type&#39;s <code>name</code>, board <code>tag</code>, <code>actionLabel(finding)</code>, <code>genericRecommendation(finding)</code> and <code>thresholdSummary(thresholds)</code>, each finding argument typed as that type&#39;s <code>Finding</code> member and <code>thresholds</code> as the emitting entry&#39;s own <code>thresholds</code>. <code>FINDING_NAMES</code>, <code>TYPE_TAG_MAP</code>, <code>getThresholdSummary</code>, <code>findingActionLabel</code> and <code>coreFindingGenericRecommendation</code> all read it. The table sits beside <code>DETECTORS</code> rather than on its entries because the HTML export renders names, tags and labels but may not reach <code>detectors.ts</code> (see <a href="./state-and-history.html#run-interpretation">Run interpretation</a>); its import from <code>detectors.ts</code> is type-only. A detector&#39;s <code>scope</code> and <code>order</code> reach renderers through <code>detectorInfoByType()</code> (<code>detector-docs.ts</code>), keyed by emitted type, which the run interpretation ships; the Alerts clean-check grouping reads <code>detectors[type].scope</code>.</p><p>Each finding type has its own shape in <code>packages/core/src/finding-types.ts</code>: <code>Finding</code> is a union discriminated on <code>type</code>, and a compile-time check in <code>detectors.ts</code> fails when its members and <code>FindingType</code> differ. Each member splits into a <code>&lt;Type&gt;Evidence</code> interface, the fields the evidence report publishes (listed again in <code>EVIDENCE_KEYS</code> in <code>evidence-report.ts</code>, checked both ways), and fields declared only on <code>&lt;Type&gt;Finding</code>, which other core modules read but the report never publishes. <code>value</code> is always a magnitude; a text-valued finding (<code>stageFailed</code>, <code>configAudit</code>, <code>incompleteRun</code>) sets <code>valueText</code> instead. So a new detector type needs an <code>emits</code> entry, an <code>estimate()</code> on that entry that covers it, a <code>finding-types.ts</code> member, an <code>EVIDENCE_KEYS</code> entry, an <code>ID_DISCRIMINATORS</code> entry in <code>analyzer.ts</code>, a <code>FINDING_PRESENTATION</code> row and a view <code>REGISTRY</code> entry. The compiler reports each one that is missing. The view narrows with <code>findingsOfType(catalog, type)</code> (<code>packages/core/src/findings-of-type.ts</code>) rather than re-declaring a finding&#39;s fields.</p><p>Both consumers are thin loops over that array:</p><ul><li><p><code>packages/core/src/analyzer.ts</code>: <code>analyze()</code> runs every entry regardless of scope, skipping only <code>inScorecard:false</code> ones, then applies <code>suppressedBy</code> (below); <code>auditConfig()</code> separately runs the <code>scope:&#39;config&#39;</code> entries. The four <code>configAudit</code> entries stay out of the bottleneck catalog because each sets <code>inScorecard:false</code>, not because of <code>scope:&#39;config&#39;</code>: a future config-scope detector without that flag would run through <code>analyze()</code> too. Each finding is stamped with its entry&#39;s <code>docAnchor</code>.</p></li><li><p><code>src/view/detector-registry.tsx</code>: a <code>REGISTRY: Record&lt;findingType, {component, region, widgetId, routeable}&gt;</code>, view-only concerns, replaces <code>dashboard-renderer.js</code>&#39;s <code>render:</code> bindings, one entry per emitted finding type. <code>orderedWidgets()</code> walks <code>DETECTORS</code> ascending by <code>order</code>, then sorts <code>action</code>-region components before <code>reference</code>-region ones. Every <code>finding.type</code> maps to its own component now (the 2026-09 widget/finding-type 1:1 mapping redesign split six components that used to multiplex several types each: <code>TaskSkew</code> into <code>Skew</code>/<code>StageShape</code>/<code>TinyTask</code>; <code>ShuffleIO</code> narrowed to <code>shuffle</code> only, plus a new <code>PartitionSizing</code>; <code>Failures</code> into <code>StageFailed</code>/ <code>TaskFailures</code>/<code>RetryWaste</code>; <code>ExecutorTimeline</code> into <code>SlowHost</code>/ <code>StageSlowness</code>/<code>Straggler</code>/<code>SpeculationWaste</code>/<code>ColdStart</code> (its non-finding-driven executor-count chart moved to <code>ExecutorCountChart</code>, a <code>ReferenceSection</code> tile, not a <code>REGISTRY</code> entry); <code>MemoryUtilization</code> narrowed to <code>memoryUtilization</code> only, plus a new <code>ExecutorUtilization</code> for <code>utilization</code>; <code>PlanFindings</code> into <code>DuplicatePlanSubtree</code>/<code>SmallFiles</code>/ <code>UnderBroadcast</code>/<code>OverBroadcast</code>, dropping the dead <code>broadcastSizing</code> key entirely). No two <code>REGISTRY</code> entries share a <code>component</code> value any more.</p><p>Each widget component receives the full catalog and self-gates when it has nothing to show, rendering <code>null</code> or a muted &quot;no issue&quot; card for the always-visible ones. <code>orderedWidgets()</code> itself has no empty/non-empty branching, since it iterates the static <code>DETECTORS</code> import, not the runtime <code>catalog</code>.</p></li></ul><p>Default thresholds live only in each entry&#39;s <code>thresholds</code>; see <a href="#bottleneck-thresholds-spec-§4">Bottleneck thresholds</a>. Only the CLI and the MCP server can override them, per run: see <a href="#tuning-thresholds">Tuning thresholds</a>.</p><h2 id="confidence-disclosure" tabindex="-1">Confidence disclosure <a class="header-anchor" href="#confidence-disclosure" aria-label="Permalink to &quot;Confidence disclosure&quot;">​</a></h2><p>A <code>Finding</code> may carry <code>confidence: &#39;low&#39; | &#39;medium&#39; | &#39;high&#39;</code> plus a <code>validationRequired</code> string. <code>RowStatusCluster</code> (<code>src/view/RowStatusCluster.tsx</code>) is the one place that renders it, gated to Advanced density: a plain &quot;&lt;confidence&gt; confidence&quot; badge whose tooltip carries the full <code>validationRequired</code> text. A finding with no <code>confidence</code> field renders identically to a fully-validated one, so every detector whose thresholds are our own unvalidated noise floor (marked <code>NOT SOURCED</code> in a code comment) should set both fields, not just the ones that happen to already have <code>RowStatusCluster</code> wired into their widget. <code>skew</code>, <code>straggler</code>, and <code>gc</code> set <code>confidence</code> for exactly this reason: their runtime-floor thresholds carry the same kind of unvalidated-noise-floor caveat <code>coreLocality</code>, <code>autoscalingChurn</code>, and <code>memoryUtilization</code>&#39;s <code>wasteModel</code> variant already disclose. None of these hardcode a single confidence value: each scales <code>&#39;low&#39; | &#39;medium&#39; | &#39;high&#39;</code> off how far the finding sits past its own detector&#39;s threshold, via a small named helper placed just above the <code>DETECTORS</code> array (e.g. <code>skewConfidence</code>, <code>coreLocalityConfidence</code>, <code>cachingReuseConfidence</code>) rather than an inline literal.</p><h2 id="the-fixeffort-field" tabindex="-1">The <code>fixEffort</code> field <a class="header-anchor" href="#the-fixeffort-field" aria-label="Permalink to &quot;The `fixEffort` field&quot;">​</a></h2><p>Each <code>Detector</code> entry also carries <code>fixEffort: &#39;config&#39; | &#39;code&#39; | &#39;rearchitect&#39;</code>, alongside <code>order</code> and <code>thresholds</code>: a rough estimate of how much work resolving the finding takes.</p><p>No view currently reads it. The quadrant impact/effort bucketing this field was meant to feed (<code>bucketFinding</code>/<code>effortTier</code>/<code>computeImpactMagnitude</code> in a since-deleted <code>src/quadrant-bucket.ts</code>, gated behind a <code>FIX_EFFORT_MAPPING_REVIEWED</code> flag that never flipped to <code>true</code>) was removed as dead code in the recommendations-consolidation redesign: <code>FixTheseFirst</code> (<code>src/view/widgets/FixTheseFirst.tsx</code>) ranks purely by impact magnitude. See <a href="./widget-rendering.html#widget-rendering-order-fixed-spec-§5">Widget rendering order</a> for how it ranks findings today.</p><p>Two shared helpers back multiple detectors and reports. <code>packages/core/src/plan-tree-walk.ts</code>&#39;s <code>walkPlanTree(root, visit, {dedupe})</code> is the iterative pre-order plan-tree traversal used by <code>detectors.ts</code> and every <code>plan-*.ts</code> module (<code>plan-summary.ts</code>, <code>plan-duration-attribution.ts</code>, <code>plan-node-detail.ts</code>, <code>plan-dot.ts</code>). <code>packages/core/src/core-count.ts</code>&#39;s <code>computeTotalCores(app, executorsAdded)</code> is the shared core-count logic used by <code>efficiency-model.ts</code>, <code>scaling-sim.ts</code>, and <code>wasted-core-hours.ts</code>. <code>detectors.ts</code>&#39;s own <code>utilization</code> and <code>memoryUtilization</code> entries use the same file&#39;s <code>computePeakConcurrentCores</code>/<code>computePeakConcurrentExecutorCount</code> instead: <code>computeTotalCores</code> sums every <code>ExecutorAdded</code> event with no regard for overlap, so under executor churn (spot preemption, <code>dynamicAllocation</code> replacement) it double-counts a churned executor&#39;s capacity against its replacement&#39;s; the peak-concurrent sweeps don&#39;t.</p><h2 id="cross-detector-suppression" tabindex="-1">Cross-detector suppression <a class="header-anchor" href="#cross-detector-suppression" aria-label="Permalink to &quot;Cross-detector suppression&quot;">​</a></h2><p>An entry may name another entry&#39;s <code>type</code> in <code>suppressedBy</code>. Once every detector has run, <code>analyze()</code>&#39;s <code>applySuppression()</code> drops each of that entry&#39;s findings on a stage where the named detector emitted a finding. It reads the unsuppressed findings, so neither the two entries&#39; declaration order nor the order suppressions apply in changes the result, and <code>order</code> stays a display field only. A compile-time check in <code>detectors.ts</code> (<code>SuppressorsAreDetectors</code>) fails when <code>suppressedBy</code> names no entry.</p><p><code>stageSlowness</code> sets <code>suppressedBy: &#39;slowHost&#39;</code>: a stage <code>slowHost</code> already explains needs no generic &quot;this stage is slow&quot; finding. Suppression follows what <code>slowHost</code> actually emitted, so a run whose <code>slowHost</code> thresholds are tuned so it can&#39;t fire gets its <code>stageSlowness</code> findings back. The mechanism is deliberately minimal (same-stage, one named suppressor per entry), not a general dependency graph. <code>auditConfig()</code> doesn&#39;t apply it, since no <code>scope:&#39;config&#39;</code> entry sets <code>suppressedBy</code>.</p><h2 id="tuning-thresholds" tabindex="-1">Tuning thresholds <a class="header-anchor" href="#tuning-thresholds" aria-label="Permalink to &quot;Tuning thresholds&quot;">​</a></h2><p><code>analyze()</code>&#39;s eighth argument is <code>{ thresholds?: ThresholdOverrides }</code>: per-entry overrides keyed by entry <code>type</code>, each a partial of that entry&#39;s own <code>thresholds</code>. Omitted, every entry runs its defaults; the dashboard never passes it. The CLI&#39;s and the MCP server&#39;s <code>--thresholds &lt;file&gt;</code> read a JSON file of that shape (<code>packages/core/src/cli/threshold-config.ts</code>) and validate it with <code>parseThresholdOverrides()</code> (<code>packages/core/src/threshold-overrides.ts</code>), which refuses an unknown detector or threshold, a negative or non-numeric value, a tier table of a different length or out of ascending order, and any <code>configAudit</code> override: those checks compare against Spark&#39;s own defaults, so there is nothing to tune. A file that can&#39;t be read or parsed refuses the run the same way. The <a href="./../../user-guide/getting-started.html#tuning-detector-thresholds">user guide</a> documents the file.</p><p>A finding from an entry whose overrides move a threshold off its default carries <code>tunedThresholds</code> (<code>{ &lt;name&gt;: { value, default } }</code>), and once its estimate is attached the analyzer appends a caveat to its <code>validationRequired</code> naming the tuned values. When the finding has an estimate figure (wall-clock or raw waste), the caveat adds that impact estimates are calibrated against the default thresholds (see <a href="./impact-estimation.html">Impact estimation</a>), so its estimate is unvalidated; an informational finding gets the label alone. An override equal to the default labels nothing. Tuning a <code>suppressedBy</code> target changes which of the suppressed entry&#39;s findings survive, so those findings carry the suppressor&#39;s tuned thresholds too, named <code>&lt;suppressor&gt;.&lt;name&gt;</code> (e.g. <code>slowHost.minHosts</code> on <code>stageSlowness</code>). Only that one link is followed. The evidence report repeats the label on the finding row, the clean check, the <code>detectors</code> catalog row (whose <code>thresholds</code> are then the effective ones) and in <code>summary.tunedThresholds</code>; see <a href="./worker-protocol.html#portable-evidence-report-v1">Portable evidence report</a>.</p><p>A tuned <code>floorPctWarn</code>/<code>floorPctCrit</code> on <code>skew</code> or <code>straggler</code> also grades that entry&#39;s own findings in <code>deriveImpactBand</code> (<code>impact-band.ts</code>); every other finding keeps the run-wide default floors. <code>--max-skew</code> recomputes the ratio with the run&#39;s effective <code>minTasksForP95</code>, so the budget measures the same ratio the skew finding reports. Caveat text that names a threshold (<code>gc</code>, <code>skew</code>, <code>straggler</code>, <code>memoryUtilization</code>, <code>coreLocality</code>) and the <code>broadcastSizing</code> over-broadcast recommendation state the value the detector ran with. The HTML export still renders the default-threshold analysis.</p><h2 id="per-operator-duration-attribution" tabindex="-1">Per-operator duration attribution <a class="header-anchor" href="#per-operator-duration-attribution" aria-label="Permalink to &quot;Per-operator duration attribution&quot;">​</a></h2><p><code>packages/core/src/plan-duration-attribution.ts</code> (entry <code>attributeStageDurationToPlan(planTree, stagesById, sqlExec)</code>) approximates how a SQL execution&#39;s stage wall-time splits across plan operators, returning a <code>Map&lt;planNode, milliseconds&gt;</code>. It cuts the plan tree at Exchange boundaries into connected components: since the Exchange write/read split (<code>resolvePlanTree</code> in <code>event-handlers.ts</code> always synthesizes a <code>read</code> node wrapping a <code>write</code> node for every raw <code>Exchange</code>/<code>BroadcastExchange</code>), the cut is keyed off <code>PlanNode.exchangeRole === &#39;read&#39;</code> on the parent, not a name regex: the write half starts the new component, the read half stays in its parent&#39;s. A node with no <code>exchangeRole</code> at all (for example <code>ReusedExchange</code>, which is never split) never starts a new component on its own, unlike the old name-based regex, which matched any Exchange-family name regardless of split state. Each component receives a stable pre-order identity and separate parent/depth/traversal metadata; the identity itself does not encode its count of Exchange ancestors. It zips components deepest-first by that explicit depth against submission-ordered stage IDs, then apportions each matched stage&#39;s wall-time across that component&#39;s nodes by timing-metric weight, falling back to an even split when no node carries a timing metric.</p><p>This is best-effort inference, not measurement. Spark&#39;s event model exposes no ground truth for per-operator time within a stage; the Exchange-boundary segmentation and deepest-component-to-earliest-stage zip are heuristics. Treat the per-operator numbers as directional hints, never as authoritative timings, and do not build hard thresholds or findings on top of them.</p><h2 id="stage-id-attribution-for-plan-advisor-findings" tabindex="-1">Stage-ID attribution for Plan Advisor findings <a class="header-anchor" href="#stage-id-attribution-for-plan-advisor-findings" aria-label="Permalink to &quot;Stage-ID attribution for Plan Advisor findings&quot;">​</a></h2><p>The Plan Advisor detectors (<code>duplicatePlanSubtree</code>, <code>smallFiles</code>, <code>broadcastSizing</code> in <code>packages/core/src/detectors.ts</code>) each attribute their finding to a narrowed <code>stageIds</code> set rather than the whole SQL execution: <code>PlanNode.stageIds</code> is resolved once per plan tree at parse time by unioning, per node, every metric&#39;s accumulator ID against a <code>taskAccumStages: Map&lt;accumulatorId, Set&lt;stageId&gt;&gt;</code> built while parsing <code>TaskEnd</code> events, then clipping the result to the execution&#39;s own stage set. An accumulator ID occasionally points to a <em>different</em> execution&#39;s stages, e.g. a <code>ReusedSubquery</code> computed once and reused verbatim, and the clip prevents misattributing that other execution&#39;s work. Each detector unions its implicated node(s)&#39; <code>stageIds</code> and falls back to the execution-wide set only when no implicated node has any coverage; a finding never partially blends a narrowed set with the execution-wide one. When an execution has no stage universe at all (no jobs ever recorded against it, which is true for 42% of real-log SQL executions with a plan tree, typically job-less/driver-only executions), the clip drops every candidate stage ID instead of passing them through: every node in that execution&#39;s tree ends up with no <code>stageIds</code> anywhere, same &quot;coverage is partial&quot; framing as below. (An earlier version of this clip treated &quot;no stage universe&quot; as &quot;no clip,&quot; which let a foreign accumulator ID collision, e.g. the <code>ReusedSubquery</code> case above, leak another execution&#39;s stages into a job-less execution&#39;s nodes; the clip is now unconditional on <code>executionStageIds</code> being present.)</p><p>Coverage is partial by Spark&#39;s own design: whole-stage-codegen wrapper nodes (<code>InputAdapter</code>, and other purely structural passthrough markers) carry no accumulators at all, and <code>BroadcastExchangeExec</code>&#39;s own metrics are computed entirely on the driver and never appear on any <code>TaskEnd</code> (real Spark behavior). Since the Exchange write/read split, those driver-computed metrics live specifically on the synthesized <em>write</em> half (<code>exchangeRole: &#39;write&#39;</code>); the <em>read</em> half always carries <code>metrics: []</code>. The write half&#39;s immediate child, which does carry executor-side metrics, is unioned in instead, see <code>overBroadcast</code>&#39;s wiring. A <code>TaskEnd</code> arriving after its stage has already been finalized is also silently excluded from <code>taskAccumStages</code>, consistent with the parser&#39;s existing out-of-order tolerance elsewhere.</p><p><code>planTree</code> itself is kept current against Spark&#39;s adaptive query execution (AQE) re-plans: <code>SparkListenerSQLAdaptiveExecutionUpdate</code> events overwrite the execution&#39;s <code>sparkPlanInfo</code> last-write-wins, so accumulator-ID evidence is matched against the plan that actually ran rather than a stale pre-AQE snapshot. Since a superseded plan is never read, <code>dispatchLine</code> holds an open execution&#39;s latest update as unparsed text and parses only that one, when <code>SQLExecutionEnd</code> arrives or at parse completion (<code>deferAdaptiveUpdate</code>, <code>event-handlers.ts</code>). It recognizes an update from the flat first and last pieces <code>buildChunkDecoder</code> reports for a line joined across slices (<code>JoinedLine</code>), so a superseded update is never copied into one flat string either. The raw <code>sparkPlanInfo</code> stays worker-side and is released once <code>SQLExecutionEnd</code> resolves it into <code>planTree</code>; <code>physicalPlanDescription</code> (Spark&#39;s text rendering of the plan, which nothing reads) is emptied before <code>JSON.parse</code> and never retained (<code>stripPlanDescription</code>, <code>event-handlers.ts</code>). <code>buildChunkDecoder</code> decodes each decompressed chunk in slices of at most 512 KiB, and drops the bytes of a value that crosses a slice boundary without decoding them.</p><p>No eviction/pruning is added to <code>taskAccumStages</code>, a deliberate choice, not an oversight: measured on real logs, it holds roughly 1,050 keys per compressed MB (9,850 keys on an 11.6 MB fixture, about 29,000 keys on a 28.1 MB fixture). Extrapolated to a 240MB+ log, the scale this tool targets (see <code>CLAUDE.md</code>), that is roughly 250,000 keys, around 45 MB of heap for an equivalent synthetic <code>Map&lt;number, Set&lt;number&gt;&gt;</code>. This heap estimate is still small relative to this tool&#39;s other in-memory state. It is higher, though, than the fixture-only measurements taken when this mechanism was built suggested.</p><p>Per-execution pruning (e.g. dropping a <code>taskAccumStages</code> entry once its stage finalizes or its owning SQL execution resolves, mirroring how <code>accumState</code> is cleared in <code>endSqlExecution</code>) is deliberately not done either: unlike <code>accumState</code>, <code>taskAccumStages</code> is one global, un-scoped map read by every execution&#39;s <code>resolvePlanTree</code> call, and the <code>ReusedSubquery</code> case above depends on a stage recorded under one execution still being visible when a later execution resolves. Pruning on any single execution&#39;s lifecycle would break that cross-execution lookup. What is bounded is the growth from a single pathological event: <code>TaskEndEventSchema</code>&#39;s <code>Accumulables</code> array is capped at <code>MAX_ACCUMULABLES_PER_TASK</code> (10,000, <code>event-schemas.ts</code>), well above any real plan&#39;s per-task metric count, so a single crafted <code>TaskEnd</code> can&#39;t grow the map past that per-event bound; a <code>TaskEnd</code> exceeding it fails schema validation and the line is skipped (counted in <code>skippedLines</code>) like any other malformed event. Each accumulable is checked for its numeric <code>ID</code> only: <code>Update</code>/<code>Value</code> are unread, and Spark writes some as JSON arrays (<code>internal.metrics.updatedBlockStatuses</code>), which a stricter check would reject along with the whole task. When every entry has Spark&#39;s flat <code>{&quot;ID&quot;:n,...}</code> form, <code>parseTaskEnd</code> (<code>event-handlers.ts</code>) reads the IDs with a string scan and cuts the array out before <code>JSON.parse</code>; any other shape is parsed whole.</p><h2 id="bottleneck-thresholds-spec-§4" tabindex="-1">Bottleneck thresholds (spec §4) <a class="header-anchor" href="#bottleneck-thresholds-spec-§4" aria-label="Permalink to &quot;Bottleneck thresholds (spec §4)&quot;">​</a></h2><p>Every change to <code>packages/core/src/detectors.ts</code> should reference this table.</p><p>Every finding&#39;s <code>impactBand</code> comes from one of two places. For any finding whose <code>impactEstimate</code> carries a <code>wallClock</code> estimate (the common case for most rules below), <code>analyzer.ts</code> calls <code>deriveImpactBand()</code> (<code>packages/core/src/impact-band.ts</code>) immediately after <code>estimateImpact()</code>, which sets <code>.impactBand</code> purely from <code>wallClock.high</code> as a fraction of the app&#39;s total duration (<code>&gt;= 2%</code> critical, <code>&gt;= 0.5%</code> warning, else info: <code>IMPACT_FLOOR_PCT_CRIT</code>/ <code>IMPACT_FLOOR_PCT_WARN</code>, which <code>skew</code>&#39;s <code>floorPctWarn</code> and <code>straggler</code>&#39;s <code>floorPctWarn</code>/<code>floorPctCrit</code> default to, so a tail those gates admit grades at least warning; a tuned <code>skew</code> or <code>straggler</code> floor also grades that entry&#39;s own findings, see <a href="#tuning-thresholds">Tuning thresholds</a>). For those rules, the table below documents their firing gate plus their fixed fallback constant, which surfaces only when this run&#39;s finding of that type didn&#39;t get a wallClock estimate (a stage excluded from the occupancy sweep). For rules whose finding type never gets a wallClock estimate (<code>resourceOnly</code>/<code>informational</code> basis, e.g. <code>configAudit</code>, or a rule that keeps its own ratio-tiered classification per the design&#39;s Decision 2, e.g. <code>failures</code>), the full threshold table below is the real, displayed classification: <code>detectors.ts</code> sets <code>impactBand</code> directly and nothing overwrites it. <code>partitionSizing</code>&#39;s <code>maxPartitionTooBig</code> rule is a third case: it does carry a <code>wallClock</code> estimate but is explicitly exempted in <code>deriveImpactBand()</code> because it&#39;s a hardcoded-critical OOM/crash-risk safety signal, not a time-recovery one, so <code>detectors.ts</code>&#39;s own classification stands regardless of how small that estimate is relative to the run.</p><h3 id="fixed-fallback-only-usually-wallclock-derived-instead" tabindex="-1">Fixed fallback only (usually wallClock-derived instead) <a class="header-anchor" href="#fixed-fallback-only-usually-wallclock-derived-instead" aria-label="Permalink to &quot;Fixed fallback only (usually wallClock-derived instead)&quot;">​</a></h3><p>These rules&#39; <em>band</em> tiers were deleted from <code>packages/core/src/detectors.ts</code> (they were always overwritten by <code>deriveImpactBand</code> whenever a wallClock estimate was available); the constant in the last column is only a floor-case fallback. Their <em>firing</em> gate is untouched and still lives in each entry&#39;s <code>thresholds</code> object: it decides whether the rule reports anything, so it stays documented here in full.</p><table tabindex="0"><thead><tr><th>Rule</th><th>Fires when</th><th>Fallback</th></tr></thead><tbody><tr><td>Task skew</td><td><code>taskDurationP95 / taskDurationP50 &gt; 3×</code> (<code>taskDurationMax / P50</code> for stages under <code>minTasksForP95</code> = 20 tasks), <strong>and</strong> the occupancy-clipped tail recovery (<code>tailRecoveryMs</code>: <code>finalizeStage</code>&#39;s task-level replay of the stage with every task over 4× P50 capped at P50) is ≥ <code>floorPctWarn</code> = 0.5% of app runtime. The clip floors the claim at the longest task the fix leaves, not the current one, and at the core work the fix leaves over every core (see impact-estimation.md&#39;s occupancy section); <code>straggler</code>&#39;s floors use the same clip</td><td><code>warning</code></td></tr><tr><td>Shuffle read</td><td><code>shuffleReadBytes &gt; minBytes</code> = 50 MiB, on a stage that isn&#39;t shorter than <code>stageFloorPct</code> = 0.5% of the run (a zero-length stage or an unknown run duration passes; the claim is clipped to the stage, so a shorter one graded <code>info</code>: 182 of 284 on the 14 real logs, the shuffle itself still there)</td><td><code>info</code></td></tr><tr><td>Partition sizing: skew</td><td><code>shuffleReadMax &gt; 5×</code> <code>shuffleReadP50</code> <strong>and</strong> <code>shuffleReadMax &gt; 256 MiB</code></td><td><code>warning</code></td></tr><tr><td>Partition sizing: low parallelism</td><td><code>shuffleReadBytes ≥ 1 GiB</code> <strong>and</strong> <code>taskCount ≤ 7</code></td><td><code>warning</code></td></tr><tr><td>Partition sizing: oversized partition</td><td><code>shuffleReadMax ≥ 5 GiB</code></td><td><code>critical</code></td></tr><tr><td>GC</td><td><code>executorRunTime ≥ minRunTimeMs</code> = 10 s <strong>and</strong> <code>gcPct &gt; 10%</code></td><td><code>warning</code></td></tr><tr><td>GC (low / cost)</td><td><code>executorRunTime ≥ 10 s</code> <strong>and</strong> <code>gcPct &lt; lowInfoPct100</code> = 5% (checked only when the GC row above did not fire) <strong>and</strong> the stage lasts ≥ <code>lowInfoFloorPct</code> = 0.5% of the run (passes when the run&#39;s duration is unknown or the stage has zero length, such as one with no completion time). Gets no wall-clock estimate (an over-provisioning signal), so this band always stands. The floor dropped 464 of 685 low-GC notes on the 14 real logs (none on the corpus): low GC is still true on those stages, but a memory-sizing note from a stage that barely ran adds nothing</td><td><code>info</code></td></tr><tr><td>Spill (magnitude v2)</td><td>any non-zero <code>memoryBytesSpilled</code>, on a stage that isn&#39;t shorter than <code>stageFloorPct</code> = 0.5% of the run (as for shuffle read: 12 of 38 on the 14 real logs, all <code>info</code>). The magnitude sub-table below classifies <em>how much</em>, but does not gate firing</td><td><code>warning</code></td></tr><tr><td>Cold start</td><td><code>firstExecutorAddedAt − firstStageSubmittedAt &gt; gapSeconds</code> = 30 s (no finding without executor-added events): the time a runnable stage waited for its first executor. An executor added before the first stage and still alive at submission means no wait and no finding; one removed at or before submission is ignored, so the gap runs to the next executor added after it. It used to be <code>firstStageSubmittedAt − app.startTime</code>, the driver&#39;s own startup, which read 36-47 s on all 9 real logs that fired whatever the executors did, and flagged 6 corpus logs whose first executor was up 44-313 s before the first stage (5 of them) or arrived 4.4 s after it</td><td><code>warning</code></td></tr><tr><td>Slow host: mean-duration ratio</td><td>stage has ≥ <code>minHosts</code> = 3 hosts (or executors) and ≥ <code>minTasks</code> = 15 tasks, and lasts ≥ <code>stageFloorPct</code> = 0.5% of the run (passes when the run&#39;s duration is unknown or the stage has zero length, which gets no estimate and keeps its fallback band; every slow-host finding on a shorter stage graded <code>info</code>, 324 of 452 on the 14 real logs, the imbalance itself still true; this gate covers every slowHost row); then per host: mean task duration / overall median ≥ <code>ratioWarn</code> = 2.0× <strong>and</strong> host task-share ≥ <code>minShare</code> = 20% <strong>and</strong> host mean ≥ <code>floorMs</code> = 1000 ms (absolute-magnitude floor, rules out sub-second noise)</td><td><code>warning</code></td></tr><tr><td>Slow host: duration-share</td><td>same stage gate as the row above; then per host: ≥ <code>shareWarn</code> = 75% of the stage&#39;s total task-duration <strong>and</strong> ≥ <code>taskShareWarn</code> = 50% of its task count</td><td><code>warning</code></td></tr><tr><td>Stage slowness: absolute fallback, suppressed when <code>slowHost</code> already fired</td><td>stage wall-clock duration ≥ <code>infoMin</code> = 15 min. The band then comes from the partitioning-headroom estimate (see impact-estimation.md), not the duration</td><td><code>info</code></td></tr><tr><td>Straggler / speculative-execution</td><td><code>taskCount ≥ minTasks</code> = 10, <strong>and</strong> the stage lasts ≥ <code>floorPctWarn</code> = 0.5% of the run (passes when the run&#39;s duration is unknown or the stage has zero length, such as one with no completion time; a shorter stage&#39;s tail can&#39;t cost more than its own duration, so every finding there graded <code>info</code>: 671 of 753 on the 14 real logs, 3 on the corpus, the slow tail itself still true), <strong>and</strong> either any speculative task ran, <strong>or</strong> straggler share &gt; <code>shareWarn</code> = 5%, <strong>or</strong> straggler share &gt; <code>shareWarnAtFloor</code> = 2.5% with the occupancy-clipped tail recovery (<code>tailRecoveryMs</code>, as for skew) already ≥ <code>floorPctWarn</code> (0.5% of app runtime). The lower gate is scored against a task-level replay of every stage on 14 real logs (recoverable = list-scheduling replay with each task over 4× P50 capped at P50; positive = ≥ 0.5% of app runtime): it found 3 stages whose stragglers gated them for 10-48s at 2.7-4% of their tasks, for 1 borderline miss, lifting skew-or-straggler recall from 0.86 to 0.92 at precision 0.91 → 0.89. Admitting every 2.5% share instead would add 86 findings below the floor. The same sweep kept skew&#39;s <code>ratioWarn</code> = 3 (2.5 added false positives, 4 lost true ones) and the 0.5% floor (1% halved skew recall). <code>warnPct</code>/<code>critPct</code> (10%/20% speculative share) and <code>floorPctWarn</code>/<code>floorPctCrit</code> (0.5%/2% of app runtime) no longer set the band; they rank the straggler-vs-speculative tiers that pick which <em>metric</em> the finding reports</td><td><code>info</code></td></tr><tr><td>Speculation waste (new)</td><td><code>speculationWastedAttempts ≥ minWasted</code> = 5 <strong>and</strong> <code>speculationWasteMs ≥ minWasteMs</code> = 60 s</td><td><code>warning</code></td></tr><tr><td>Retry waste</td><td><code>wastedAttempts ≥ minWasted</code> = 3 <strong>and</strong> <code>retryWasteMs ≥ minWasteMs</code> = 30 s, on a stage that still completed</td><td><code>warning</code></td></tr><tr><td>Tiny tasks</td><td><code>taskCount ≥ minTasks</code> = 100 <strong>and</strong> <code>taskDurationP50 ≤ maxP50</code> = 500 ms <strong>and</strong> <code>taskDurationP95 ≤ maxP95</code> = 1000 ms <strong>and</strong> the stage lasts ≥ <code>stageFloorPct</code> = 0.5% of the run (passes when the run&#39;s duration is unknown or the stage has zero length; coalescing can&#39;t save more than the stage&#39;s own duration, so every finding on a shorter stage graded <code>info</code>: 132 of 151 on the 14 real logs, the tasks still tiny). Checked (2026-09-23) against each stage&#39;s measured per-task overhead (task wall time minus executor run time, the estimate&#39;s own input) on 14 real logs and the corpus: of 228 stages with 100+ tasks outside the P50/P95 gate, none would save 0.5% of its run by coalescing (median overhead 0.2-3% of task time); of 170 inside it, 25 would, and all 25 grade above <code>info</code></td><td><code>info</code></td></tr><tr><td>Duplicate plan subtree</td><td>a subtree of ≥ <code>minSubtreeSize</code> = 3 nodes whose shape fingerprint repeats ≥ <code>minOccurrences</code> = 2× in the plan, unless its linked stages together lasted less than <code>stageFloorPct</code> = 0.5% of the run (a repeat with no linked stage time, or an unknown run duration, is kept; the claim counts at most each stage&#39;s own task-active time, so such a repeat graded <code>info</code>: 340 of 545 on the 14 real logs, the repeat still in the plan). <code>occurrencesIdentical</code> records whether the repeats also agree node-for-node on normalized detail, ignoring AQE query-stage numbers; when they don&#39;t (same shape over another table, filter or projection: 270 of 546 groups on the 14 real logs) the finding is <code>info</code> with confidence <code>low</code> and no time claim. <code>stageShares</code> gives, per stage, the repeated operators&#39; share of the stage&#39;s operators (WholeStageCodegen wrappers and Exchange write halves not counted); with no attributed stage the finding is <code>info</code></td><td><code>warning</code></td></tr><tr><td>Small files read/write</td><td>per read/write side: file count &gt; <code>minFiles</code> = 100 <strong>and</strong> average file size &lt; <code>maxAvgFileSizeMB</code> = 3 MiB</td><td><code>warning</code></td></tr><tr><td>Broadcast sizing: missed</td><td>a 2-child <code>SortMergeJoin</code> whose smaller side is &lt; 10 MiB (unconditional), or &lt; 100 MiB with the larger side &gt; 10 GiB, or &lt; 1 GiB with larger &gt; 300 GiB, or &lt; 5 GiB with larger &gt; 1 TiB (<code>broadcastTiers</code> × <code>comparisonTiers</code>)</td><td><code>info</code></td></tr><tr><td>Broadcast sizing: oversized</td><td>a <code>BroadcastExchange</code> node whose <code>data size</code> metric &gt; <code>overBroadcastBytes</code> = 1 GiB</td><td><code>warning</code></td></tr></tbody></table><h4 id="spill-magnitude-tiers" tabindex="-1">Spill magnitude tiers <a class="header-anchor" href="#spill-magnitude-tiers" aria-label="Permalink to &quot;Spill magnitude tiers&quot;">​</a></h4><p>The spill row&#39;s band is a fixed <code>warning</code> fallback, but <code>computeSpillMagnitude</code> (<code>packages/core/src/detectors.ts</code>) still runs on every spill finding and sets its <code>spillMagnitude</code> field, which the Spill widget displays. Its tiers, in evaluation order (first match wins, <code>null</code> when nothing matches):</p><table tabindex="0"><thead><tr><th>Condition</th><th>Magnitude</th></tr></thead><tbody><tr><td>Single-task stage: <code>spillDiskMax ≥ singleTaskDiskGiB</code> = 1 GiB <strong>or</strong> <code>spillMemMax ≥ singleTaskMemGiB</code> = 4 GiB</td><td><code>severe</code></td></tr><tr><td>Multi-task stage: <code>spillDiskMax ≥ highDiskGiB</code> = 1 GiB, <strong>or</strong> <code>spillDiskMax / taskCount ≥ highTaskDiskMB</code> = 512 MiB (per-task proxy), <strong>or</strong> <code>spillMemMax ≥ highMemGiB</code> = 4 GiB</td><td><code>high</code></td></tr><tr><td>Multi-task stage: <code>spillDiskMax ≥ medDiskMB</code> = 256 MiB <strong>or</strong> <code>spillMemMax ≥ medMemGiB</code> = 1 GiB</td><td><code>medium</code></td></tr><tr><td>Skew (<code>taskCount ≥ skewMinTasks</code> = 10): <code>spillDiskMax / spillDiskP50 &gt; skewRatio</code> = 5× <strong>and</strong> <code>spillDiskMax ≥ skewDiskFloorMB</code> = 128 MiB</td><td><code>high</code></td></tr><tr><td>Skew (<code>taskCount ≥ 10</code>): <code>spillMemMax / spillMemP50 &gt; 5×</code> <strong>and</strong> <code>spillMemMax ≥ skewMemFloorMB</code> = 256 MiB</td><td><code>medium</code></td></tr></tbody></table><p>Disk spill is weighted worse than memory spill by design: the memory thresholds sit well above their disk counterparts at every tier.</p><h3 id="full-threshold-table-never-wallclock-derived" tabindex="-1">Full threshold table (never wallClock-derived) <a class="header-anchor" href="#full-threshold-table-never-wallclock-derived" aria-label="Permalink to &quot;Full threshold table (never wallClock-derived)&quot;">​</a></h3><table tabindex="0"><thead><tr><th>Rule</th><th>Warning</th><th>Critical</th></tr></thead><tbody><tr><td>Stage shape: PRatio</td><td><code>taskCount / totalCores &lt; 0.5</code> (info, under-parallelized), on a stage lasting ≥ <code>lowParallelismFloorPct</code> = 0.5% of the run (passes when the run&#39;s duration is unknown or the stage has zero length): parallelizing can&#39;t save more than the stage&#39;s duration, and on the 14 real logs 2839 of 3005 firings were below it</td><td>none</td></tr><tr><td>Stage shape: OIRatio</td><td><code>outputBytes / inputBytes &gt; 10×</code> (info, data explosion)</td><td>none</td></tr><tr><td>Stage shape: TaskStageSkew</td><td><code>taskDurationMax / stageDuration &gt; 3×</code> (info)</td><td>none</td></tr><tr><td>Failed tasks</td><td>failure rate &gt; 5% (min 10 tasks)</td><td>&gt; 20%</td></tr><tr><td>Stage failed outright</td><td>none</td><td>any <code>stageFailureReason</code> present</td></tr><tr><td>Slow host: multi-dimensional</td><td>max/median ratio across taskTime/inputBytes/shuffleBytes/storageMemory ≥ 1.33× (info); each dimension&#39;s sample must also clear an absolute floor (1000 ms for taskTime, 64 MiB for the byte dimensions). A stage lasting under <code>stageFloorPct</code> = 0.5% of the run is skipped (see the mean-duration row): a byte dimension there, which has no time estimate, used to keep its ratio tier (66 of 90 warning/critical on the 14 real logs) until 6298149 graded it <code>info</code></td><td>≥ 3.16× warning, ≥ 10× critical</td></tr><tr><td>Utilization</td><td>avg active executors / peak &lt; 60% (info)</td><td>none</td></tr><tr><td>Autoscaling churn: short-lived executors (design spike, unvalidated thresholds)</td><td>&gt; 30% of executors alive under 2 min (min 5 executors)</td><td>&gt; 60%</td></tr><tr><td>Job failure rate</td><td>≥ 30% (≥ 10% info)</td><td>≥ 50%</td></tr><tr><td>Idle cores</td><td>busy-core-time / (peak cores × wall-clock) idle &gt; 50% (warning)</td><td>none</td></tr><tr><td>Memory band</td><td>peak heap / allocated &gt; 95% too-small (warning); &lt; 70% over-provisioned (info)</td><td>none</td></tr><tr><td>Caching opportunity</td><td>same input relation (or join/union subtree) scanned by ≥ <code>minExecutions</code> = 2 SQL executions in one run</td><td>none (single tier, info)</td></tr><tr><td>Cache utilization: partial caching (this repo)</td><td><code>numCachedPartitions / numPartitions &lt; 0.90</code> (info)</td><td><code>&lt; 0.50</code> (warning)</td></tr><tr><td>Cache utilization: disk spillover (this repo)</td><td><code>diskSize / (memorySize + diskSize) &gt; 0.15</code> (info), <code>MEMORY_AND_DISK*</code> only</td><td><code>&gt; 0.40</code> (warning)</td></tr><tr><td>Cache utilization: storage unobserved</td><td>persisted RDDs, but no <code>SparkListenerBlockUpdated</code> for any <code>rdd_*</code> block and every RDD Info figure 0 (<code>spark.eventLog.logBlockUpdates.enabled</code> off on Spark 2.3+): a missing-evidence caveat, not a threshold</td><td>none (single tier, info)</td></tr></tbody></table><p>The RDD Info cache figures on stage events (<code>Number of Cached Partitions</code>, <code>Memory Size</code>, <code>Disk Size</code>) are always 0 since Spark 2.3; Spark 1.x fills them only on <code>StageCompleted</code>. Real cache evidence is <code>SparkListenerBlockUpdated</code>, written only with <code>spark.eventLog.logBlockUpdates.enabled=true</code> (<code>recordBlockUpdate</code> in <code>event-handlers.ts</code>); the corpus <code>cache-memory-only</code>/<code>cache-memory-and-disk</code> logs carry it. Count a block&#39;s sizes only where its storage level says it lives: a drop to disk still reports the dropped bytes as <code>Memory Size</code>.</p><p>Spill classification: ≥80% tasks with zero spill → <code>skew</code>; &lt;20% zero → <code>volume</code>; else <code>unclassified</code>. The classification badge is always shown in both compact and expanded spill widget states. This is independent of the magnitude tiers above: classification says <em>what kind</em> of spill, magnitude says <em>how much</em>.</p><h4 id="evidence-fields-stagefailed-retrywaste" tabindex="-1">Evidence fields (<code>stageFailed</code> / <code>retryWaste</code>) <a class="header-anchor" href="#evidence-fields-stagefailed-retrywaste" aria-label="Permalink to &quot;Evidence fields (`stageFailed` / `retryWaste`)&quot;">​</a></h4><p>Neither entry&#39;s <code>detect()</code> used to put anything beyond a scalar <code>metric</code>/ <code>value</code> on its finding. Both now also attach:</p><ul><li><code>numTasks</code>: <code>stage.taskCount</code> at detection time.</li><li><code>memoryBytesSpilled</code>: <code>stage.memoryBytesSpilled</code> at detection time.</li><li><code>stageFailed</code> only: <code>failedTaskDetails</code>: up to 20 <code>FailedTaskSample</code> records (<code>taskId</code>, <code>attemptNumber</code>, <code>host</code>, <code>executorId</code>, <code>reason</code>, <code>peakExecMem</code>, <code>memSpilled</code>, <code>shuffleWrite</code>) for tasks still marked failed when the stage was finalized (<code>finalizeStage</code>, <code>stage-quantiles.ts</code>).</li><li><code>retryWaste</code> only: <code>retriedTaskDetails</code>: up to 20 <code>FailedTaskSample</code> records for attempts discarded by the retry-dedup logic in <code>accumulateTask</code> (<code>event-handlers.ts</code>): captured at the moment they&#39;d otherwise be thrown away, since by finalize time only the winning attempt survives.</li></ul><p>Both sample arrays are capped at 20 entries, filled in first-encountered order (finalize order for <code>failedTaskDetails</code>, discard order for <code>retriedTaskDetails</code>), not spread across distinct hosts/executors: a stage with failures clustered on one bad host could fill the cap before a more informative failure elsewhere in the stage is ever sampled.</p><h4 id="evidence-fields-failures" tabindex="-1">Evidence fields (<code>failures</code>) <a class="header-anchor" href="#evidence-fields-failures" aria-label="Permalink to &quot;Evidence fields (`failures`)&quot;">​</a></h4><ul><li><code>dominantReason</code>: the most frequent end-reason tag (<code>ExceptionFailure</code>, <code>ExecutorLostFailure</code>, ...) among tasks still failed at finalize.</li><li><code>dominantError</code>: the error behind that tag, from its largest failure group: the exception class, or <code>&lt;tag&gt;: &lt;loss reason&gt;</code>; falls back to the tag. The recommendation names it and never embeds a message.</li><li><code>failureGroups</code>: up to 5 <code>TaskFailureGroup</code>s (<code>reason</code>, <code>className</code>, <code>message</code>, <code>lossReason</code>, <code>stackExcerpt</code>, <code>count</code>), most frequent first: one group per distinct tag, class, message and loss reason, one excerpt each.</li><li><code>otherFailedTasks</code>: failed tasks that no shown group covers.</li></ul><p><code>packages/core/src/task-failure.ts</code> reads the details from <code>Task End Reason</code> (<code>Class Name</code>, <code>Description</code>, <code>Full Stack Trace</code>, <code>Loss Reason</code>, FetchFailed <code>Message</code>, <code>Kill Reason</code>) and bounds them at ingest: a message or loss reason to its first line (for a Python traceback, its final error line), 300 characters; an excerpt to the header, 8 frames, a Python traceback&#39;s error line and the last <code>Caused by:</code> line, 2000 characters in all. While parsing, a stage keeps at most 50 distinct failures (<code>StageRecord.failureDetails</code>, freed at finalize); later ones count only toward the tag. Redaction (<code>redact.ts</code>) replaces every <code>failureGroups[].message</code> and strips message text from <code>stackExcerpt</code>.</p></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/contributor-guide/architecture/state-and-history.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>State & history intake</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="/docs/contributor-guide/architecture/impact-estimation.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>Impact estimation</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
26
+ <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"VTcYTH4d\",\"contributor-guide_architecture_detector-contract.md\":\"CsqZHz_T\",\"contributor-guide_architecture_drill-down.md\":\"CFNCE8JB\",\"contributor-guide_architecture_impact-estimation.md\":\"HT9IJGIl\",\"contributor-guide_architecture_index.md\":\"3TO9ic6w\",\"contributor-guide_architecture_overview.md\":\"CTEqVeV4\",\"contributor-guide_architecture_state-and-history.md\":\"DOTVHjTV\",\"contributor-guide_architecture_widget-rendering.md\":\"BmpARZve\",\"contributor-guide_architecture_worker-protocol.md\":\"21pTYhFF\",\"contributor-guide_contributing.md\":\"B2E9ni79\",\"contributor-guide_development-setup.md\":\"CjgZJwjK\",\"contributor-guide_testing.md\":\"DTwhpoN2\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"D08Uaod7\",\"tuning-reference_aqe.md\":\"C1Ae6A8u\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"6wjKuJNE\",\"tuning-reference_bottleneck-cold-start.md\":\"BVQbsuHP\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"DtaYnaiF\",\"tuning-reference_bottleneck-failures.md\":\"Z1Tdmdd3\",\"tuning-reference_bottleneck-gc.md\":\"BCnXLf1n\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"CcvQj2g9\",\"tuning-reference_bottleneck-memory-utilization.md\":\"B-C0-8h-\",\"tuning-reference_bottleneck-retry-waste.md\":\"DCnhWiqj\",\"tuning-reference_bottleneck-shuffle.md\":\"B5OMJCd3\",\"tuning-reference_bottleneck-skew.md\":\"Cvo6NQm_\",\"tuning-reference_bottleneck-slow-host.md\":\"ja5qr3wq\",\"tuning-reference_bottleneck-small-files.md\":\"Crjtb0i2\",\"tuning-reference_bottleneck-spill.md\":\"Bl8KtSRg\",\"tuning-reference_bottleneck-straggler.md\":\"B0VN8reT\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"DQCftGhu\",\"tuning-reference_bottleneck-utilization.md\":\"BXDl30f9\",\"tuning-reference_caching.md\":\"CcaiMn_7\",\"tuning-reference_cluster-config.md\":\"DbWG1yZq\",\"tuning-reference_config.md\":\"YUqSpwFB\",\"tuning-reference_data-formats.md\":\"C5DX5xL4\",\"tuning-reference_index.md\":\"ZVIpQ6Dk\",\"tuning-reference_intro.md\":\"BSSaDlqx\",\"tuning-reference_joins.md\":\"BVhNCDnE\",\"tuning-reference_memory-model.md\":\"CfjFdgbl\",\"tuning-reference_metrics.md\":\"C64VNEL4\",\"tuning-reference_partitioning.md\":\"CtLK0xsS\",\"tuning-reference_pyspark.md\":\"D-92XgF8\",\"tuning-reference_shuffle.md\":\"Iq2v1oHJ\",\"tuning-reference_spark-architecture.md\":\"_qwnE_Sc\",\"tuning-reference_table-formats.md\":\"BSYEK-eN\",\"user-guide_alternative-log-retrieval.md\":\"ZHPSbBkF\",\"user-guide_getting-started.md\":\"CEa2ZX-l\",\"user-guide_mcp-tools.md\":\"dQwIvdTF\",\"user-guide_run-comparison.md\":\"ChLI3MK0\",\"user-guide_understanding-findings.md\":\"D1ccs60C\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Alternative log retrieval\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
27
27
 
28
28
  </body>
29
29
  </html>