sparkforensics-server 0.4.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (257) hide show
  1. package/index.js +3 -0
  2. package/package.json +1 -1
  3. package/public/assets/AutoscalingChurn-9Zo6IMGS.js +1 -0
  4. package/public/assets/CacheUtilization-MBDfMsKC.js +1 -0
  5. package/public/assets/CachingOpportunity-Cfw3wAFa.js +1 -0
  6. package/public/assets/ColdStart-BtS12b6_.js +1 -0
  7. package/public/assets/ConfigAudit-DNPV6EQg.js +1 -0
  8. package/public/assets/CoreUsageArea-BObEeenE.js +1 -0
  9. package/public/assets/Dashboard-CC_BXrNy.js +2 -0
  10. package/public/assets/DisclosureContext-Ds0hcZFn.js +1 -0
  11. package/public/assets/DuplicatePlanSubtree-vN3zqRsi.js +1 -0
  12. package/public/assets/ExecutorUtilization-CZObSLIj.js +1 -0
  13. package/public/assets/GcPressure-ByJlLOua.js +1 -0
  14. package/public/assets/IncompleteRun-MzqV4W8y.js +1 -0
  15. package/public/assets/JobFailures-LbMvAwAb.js +1 -0
  16. package/public/assets/MemoryUtilization-DH4jYvJK.js +1 -0
  17. package/public/assets/OverBroadcast-euk6HKg4.js +1 -0
  18. package/public/assets/PartitionHint-DThaMMRT.js +1 -0
  19. package/public/assets/PartitionSizing-yIN-iZzx.js +1 -0
  20. package/public/assets/{PlanExplorer-DdmPkKO1.js → PlanExplorer-Dd9WPK4S.js} +1 -1
  21. package/public/assets/PlanGraphRoute-NV5mCXRm.js +1 -0
  22. package/public/assets/RetryWaste-CXH4D-xH.js +1 -0
  23. package/public/assets/ShuffleIO-CW7U68ph.js +1 -0
  24. package/public/assets/Skew-PL79TppX.js +1 -0
  25. package/public/assets/SlowHost-GBO73hCi.js +1 -0
  26. package/public/assets/SmallFiles-B5cUjEav.js +1 -0
  27. package/public/assets/SpeculationWaste-mxi3jgb0.js +1 -0
  28. package/public/assets/Spill-CH1bNpxL.js +1 -0
  29. package/public/assets/StageFailed-YvxC11ii.js +1 -0
  30. package/public/assets/StageFindingGroup-B1A0nf-e.js +1 -0
  31. package/public/assets/{StageHeader-BSV8ALhT.js → StageHeader-BMNmUy-s.js} +1 -1
  32. package/public/assets/StageShape-DO3mQ6NS.js +1 -0
  33. package/public/assets/StageSlowness-SwRxfAEx.js +1 -0
  34. package/public/assets/Straggler-DHLTiBY-.js +1 -0
  35. package/public/assets/TaskFailures-Bj0QxiOc.js +1 -0
  36. package/public/assets/{TinyTask-CIPRxWfT.js → TinyTask-COUQQ5bQ.js} +1 -1
  37. package/public/assets/UnderBroadcast-RAUbQ6vw.js +1 -0
  38. package/public/assets/{finding-anchor-DF1cSHO1.js → finding-anchor-Dl8mIHiU.js} +1 -1
  39. package/public/assets/{impact-sort-7z0ZcBZf.js → impact-sort-Eihua9dY.js} +1 -1
  40. package/public/assets/index-DaC8A2FS.js +13 -0
  41. package/public/assets/{parser-worker-Q5KCPdKT.js → parser-worker-CfDvPkwF.js} +5 -5
  42. package/public/assets/{select-Cy5pmbrh.js → select-DkjFHlXR.js} +1 -1
  43. package/public/assets/{usePagedRows-BjPOpUU8.js → usePagedRows-Bo-aSztx.js} +1 -1
  44. package/public/assets/useRecentFiles-Bb4-ukVz.js +11 -0
  45. package/public/assets/{useSortMode-CU0vzThu.js → useSortMode-lDedD5a3.js} +1 -1
  46. package/public/docs/404.html +4 -3
  47. package/public/docs/assets/{app.C6w47a9W.js → app.yjczoibf.js} +1 -1
  48. package/public/docs/assets/chunks/@localSearchIndexroot.r9imX8Aq.js +1 -0
  49. package/public/docs/assets/chunks/{VPLocalSearchBox.BcTNvT55.js → VPLocalSearchBox.DOBbPfix.js} +1 -1
  50. package/public/docs/assets/chunks/{theme.D-mE6j02.js → theme.DFcAl8Ws.js} +2 -2
  51. package/public/docs/assets/contributor-guide_architecture_board-widgets.md.D0Kkk_r5.js +1 -0
  52. package/public/docs/assets/contributor-guide_architecture_board-widgets.md.D0Kkk_r5.lean.js +1 -0
  53. package/public/docs/assets/contributor-guide_architecture_detector-contract.md.caA46z-u.js +1 -0
  54. package/public/docs/assets/contributor-guide_architecture_drill-down.md.CdtlKHl-.js +1 -0
  55. package/public/docs/assets/{contributor-guide_architecture_drill-down.md.CFNCE8JB.lean.js → contributor-guide_architecture_drill-down.md.CdtlKHl-.lean.js} +1 -1
  56. package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.BNp14LtL.js +1 -0
  57. package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.BNp14LtL.lean.js +1 -0
  58. package/public/docs/assets/contributor-guide_architecture_index.md.B9yZieYL.js +1 -0
  59. package/public/docs/assets/{contributor-guide_architecture_index.md.3TO9ic6w.lean.js → contributor-guide_architecture_index.md.B9yZieYL.lean.js} +1 -1
  60. package/public/docs/assets/contributor-guide_architecture_overview.md.DBGo-bwn.js +1 -0
  61. package/public/docs/assets/contributor-guide_architecture_overview.md.DBGo-bwn.lean.js +1 -0
  62. package/public/docs/assets/contributor-guide_architecture_state-and-history.md.w4lwybdh.js +1 -0
  63. package/public/docs/assets/{contributor-guide_architecture_state-and-history.md.DOTVHjTV.lean.js → contributor-guide_architecture_state-and-history.md.w4lwybdh.lean.js} +1 -1
  64. package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.CcvqxnZI.js +1 -0
  65. package/public/docs/assets/{contributor-guide_architecture_widget-rendering.md.BmpARZve.lean.js → contributor-guide_architecture_widget-rendering.md.CcvqxnZI.lean.js} +1 -1
  66. package/public/docs/assets/contributor-guide_architecture_worker-protocol.md.CKeFttKj.js +6 -0
  67. package/public/docs/assets/{contributor-guide_architecture_worker-protocol.md.21pTYhFF.lean.js → contributor-guide_architecture_worker-protocol.md.CKeFttKj.lean.js} +1 -1
  68. package/public/docs/assets/contributor-guide_contributing.md.CQMvGwFt.js +1 -0
  69. package/public/docs/assets/{contributor-guide_contributing.md.B2E9ni79.lean.js → contributor-guide_contributing.md.CQMvGwFt.lean.js} +1 -1
  70. package/public/docs/assets/contributor-guide_development-setup.md.D9bu4j_c.js +15 -0
  71. package/public/docs/assets/contributor-guide_development-setup.md.D9bu4j_c.lean.js +1 -0
  72. package/public/docs/assets/contributor-guide_testing.md.By6XGR9a.js +1 -0
  73. package/public/docs/assets/{contributor-guide_testing.md.DTwhpoN2.lean.js → contributor-guide_testing.md.By6XGR9a.lean.js} +1 -1
  74. package/public/docs/assets/{style.DSixAiZE.css → style.tmDhD3c8.css} +1 -1
  75. package/public/docs/assets/user-guide_alternative-log-retrieval.md.z-8npxNk.js +7 -0
  76. package/public/docs/assets/user-guide_alternative-log-retrieval.md.z-8npxNk.lean.js +1 -0
  77. package/public/docs/assets/user-guide_getting-started.md.BJW5oVhe.js +24 -0
  78. package/public/docs/assets/user-guide_getting-started.md.BJW5oVhe.lean.js +1 -0
  79. package/public/docs/assets/{user-guide_mcp-tools.md.dQwIvdTF.js → user-guide_mcp-tools.md.DcLiOa9t.js} +40 -23
  80. package/public/docs/assets/{user-guide_mcp-tools.md.dQwIvdTF.lean.js → user-guide_mcp-tools.md.DcLiOa9t.lean.js} +1 -1
  81. package/public/docs/assets/user-guide_run-comparison.md.qin0a4IX.js +1 -0
  82. package/public/docs/assets/{user-guide_run-comparison.md.ChLI3MK0.lean.js → user-guide_run-comparison.md.qin0a4IX.lean.js} +1 -1
  83. package/public/docs/assets/user-guide_understanding-findings.md.DzReyCXb.js +1 -0
  84. package/public/docs/contributor-guide/architecture/board-widgets.html +7 -6
  85. package/public/docs/contributor-guide/architecture/detector-contract.html +7 -6
  86. package/public/docs/contributor-guide/architecture/drill-down.html +7 -6
  87. package/public/docs/contributor-guide/architecture/impact-estimation.html +7 -6
  88. package/public/docs/contributor-guide/architecture/index.html +7 -6
  89. package/public/docs/contributor-guide/architecture/overview.html +7 -6
  90. package/public/docs/contributor-guide/architecture/state-and-history.html +7 -6
  91. package/public/docs/contributor-guide/architecture/widget-rendering.html +7 -6
  92. package/public/docs/contributor-guide/architecture/worker-protocol.html +10 -9
  93. package/public/docs/contributor-guide/contributing.html +7 -6
  94. package/public/docs/contributor-guide/development-setup.html +11 -10
  95. package/public/docs/contributor-guide/testing.html +7 -6
  96. package/public/docs/hashmap.json +1 -1
  97. package/public/docs/index.html +5 -4
  98. package/public/docs/tuning-reference/anti-patterns.html +5 -4
  99. package/public/docs/tuning-reference/aqe.html +5 -4
  100. package/public/docs/tuning-reference/bottleneck-broadcast-sizing.html +5 -4
  101. package/public/docs/tuning-reference/bottleneck-cold-start.html +5 -4
  102. package/public/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html +5 -4
  103. package/public/docs/tuning-reference/bottleneck-failures.html +5 -4
  104. package/public/docs/tuning-reference/bottleneck-gc.html +5 -4
  105. package/public/docs/tuning-reference/bottleneck-job-failure-rate.html +5 -4
  106. package/public/docs/tuning-reference/bottleneck-memory-utilization.html +5 -4
  107. package/public/docs/tuning-reference/bottleneck-retry-waste.html +5 -4
  108. package/public/docs/tuning-reference/bottleneck-shuffle.html +5 -4
  109. package/public/docs/tuning-reference/bottleneck-skew.html +5 -4
  110. package/public/docs/tuning-reference/bottleneck-slow-host.html +5 -4
  111. package/public/docs/tuning-reference/bottleneck-small-files.html +5 -4
  112. package/public/docs/tuning-reference/bottleneck-spill.html +5 -4
  113. package/public/docs/tuning-reference/bottleneck-straggler.html +5 -4
  114. package/public/docs/tuning-reference/bottleneck-tiny-tasks.html +5 -4
  115. package/public/docs/tuning-reference/bottleneck-utilization.html +5 -4
  116. package/public/docs/tuning-reference/caching.html +5 -4
  117. package/public/docs/tuning-reference/cluster-config.html +5 -4
  118. package/public/docs/tuning-reference/config.html +5 -4
  119. package/public/docs/tuning-reference/data-formats.html +5 -4
  120. package/public/docs/tuning-reference/index.html +5 -4
  121. package/public/docs/tuning-reference/intro.html +5 -4
  122. package/public/docs/tuning-reference/joins.html +5 -4
  123. package/public/docs/tuning-reference/memory-model.html +5 -4
  124. package/public/docs/tuning-reference/metrics.html +5 -4
  125. package/public/docs/tuning-reference/partitioning.html +5 -4
  126. package/public/docs/tuning-reference/pyspark.html +5 -4
  127. package/public/docs/tuning-reference/shuffle.html +5 -4
  128. package/public/docs/tuning-reference/spark-architecture.html +5 -4
  129. package/public/docs/tuning-reference/table-formats.html +5 -4
  130. package/public/docs/user-guide/alternative-log-retrieval.html +14 -7
  131. package/public/docs/user-guide/getting-started.html +27 -8
  132. package/public/docs/user-guide/mcp-tools.html +46 -28
  133. package/public/docs/user-guide/run-comparison.html +7 -6
  134. package/public/docs/user-guide/understanding-findings.html +7 -6
  135. package/public/export-template.html +44 -44
  136. package/public/index.html +2 -2
  137. package/vendor-core/allocation.js +106 -0
  138. package/vendor-core/analyzer.js +13 -13
  139. package/vendor-core/cli/budgets.js +23 -9
  140. package/vendor-core/cli/collect-run.js +11 -4
  141. package/vendor-core/cli/regression-budgets.js +83 -0
  142. package/vendor-core/comparison-verdict.js +22 -22
  143. package/vendor-core/core-source-hash.txt +1 -1
  144. package/vendor-core/detectors.js +202 -68
  145. package/vendor-core/docs-content/detection/cache.md +3 -2
  146. package/vendor-core/docs-content/detection/cfg.md +9 -8
  147. package/vendor-core/docs-content/detection/chrn.md +1 -2
  148. package/vendor-core/docs-content/detection/cold.md +4 -2
  149. package/vendor-core/docs-content/detection/fail.md +3 -2
  150. package/vendor-core/docs-content/detection/gc.md +3 -2
  151. package/vendor-core/docs-content/detection/host.md +2 -1
  152. package/vendor-core/docs-content/detection/local.md +1 -1
  153. package/vendor-core/docs-content/detection/mem.md +5 -2
  154. package/vendor-core/docs-content/detection/plan.md +2 -1
  155. package/vendor-core/docs-content/detection/sfail.md +2 -1
  156. package/vendor-core/docs-content/detection/shape.md +5 -4
  157. package/vendor-core/docs-content/detection/skew.md +3 -1
  158. package/vendor-core/docs-content/detection/slow.md +2 -2
  159. package/vendor-core/docs-content/detection/spec.md +2 -3
  160. package/vendor-core/docs-content/detection/spill.md +1 -1
  161. package/vendor-core/docs-site-config.js +1 -1
  162. package/vendor-core/effective-conf.js +107 -0
  163. package/vendor-core/efficiency-model.js +8 -6
  164. package/vendor-core/event-handlers.js +160 -47
  165. package/vendor-core/event-schemas.js +2 -0
  166. package/vendor-core/evidence-report.js +18 -10
  167. package/vendor-core/finding-generic-recommendation.js +20 -1
  168. package/vendor-core/finding-names.js +7 -0
  169. package/vendor-core/finding-presentation.js +61 -26
  170. package/vendor-core/finding-tag-help.js +1 -1
  171. package/vendor-core/finding-types.js +12 -0
  172. package/vendor-core/format-utils.js +4 -3
  173. package/vendor-core/impact-estimator.js +20 -2
  174. package/vendor-core/impact-format.js +14 -13
  175. package/vendor-core/impact-model.js +27 -5
  176. package/vendor-core/ingest.js +4 -2
  177. package/vendor-core/list-runs.js +5 -2
  178. package/vendor-core/mcp-tools.js +1 -1
  179. package/vendor-core/model-assembler.js +23 -1
  180. package/vendor-core/parser-worker.js +1 -1
  181. package/vendor-core/proxy.js +3 -1
  182. package/vendor-core/python-stage.js +25 -0
  183. package/vendor-core/recommendation-rollup.js +16 -9
  184. package/vendor-core/redact.js +51 -10
  185. package/vendor-core/remediation.js +20 -0
  186. package/vendor-core/run-comparison.js +43 -24
  187. package/vendor-core/run-interpretation.js +2 -1
  188. package/vendor-core/run-metrics.js +198 -0
  189. package/vendor-core/run-totals.js +24 -0
  190. package/vendor-core/run-verdict.js +3 -4
  191. package/vendor-core/scorecard-estimates.js +1 -0
  192. package/vendor-core/session-snapshot.js +7 -0
  193. package/vendor-core/shs-schemas.js +2 -2
  194. package/vendor-core/spark-memory.js +17 -0
  195. package/vendor-core/stage-plan-nodes.js +18 -0
  196. package/vendor-core/stage-quantiles.js +4 -0
  197. package/vendor-core/types.js +49 -1
  198. package/vendor-core/wasted-core-hours.js +10 -7
  199. package/vendor-core/write-targets.js +312 -0
  200. package/public/assets/AutoscalingChurn-CUSIvvab.js +0 -1
  201. package/public/assets/CacheUtilization-Ce0K6_X2.js +0 -1
  202. package/public/assets/CachingOpportunity-Cn-LmRA4.js +0 -1
  203. package/public/assets/ColdStart-BoiYzEyV.js +0 -1
  204. package/public/assets/ConfigAudit-BoJc6PAw.js +0 -1
  205. package/public/assets/CoreUsageArea-BFfXKu7K.js +0 -1
  206. package/public/assets/Dashboard-Bo_1jGqH.js +0 -2
  207. package/public/assets/DisclosureContext-LYzHroHO.js +0 -1
  208. package/public/assets/DuplicatePlanSubtree-BaAuGAoo.js +0 -1
  209. package/public/assets/ExecutorUtilization-OE6VMFkG.js +0 -1
  210. package/public/assets/GcPressure-AoL_ZNrV.js +0 -1
  211. package/public/assets/IncompleteRun-Bv6IzgsQ.js +0 -1
  212. package/public/assets/JobFailures-DF3U2z1f.js +0 -1
  213. package/public/assets/MemoryUtilization-B65OVZlq.js +0 -1
  214. package/public/assets/OverBroadcast-DXbJREDl.js +0 -1
  215. package/public/assets/PartitionSizing-DZLnjy0d.js +0 -1
  216. package/public/assets/PlanGraphRoute-C-BGmC6j.js +0 -1
  217. package/public/assets/RetryWaste-BbEPsdMw.js +0 -1
  218. package/public/assets/ShuffleIO-wZB94LS8.js +0 -1
  219. package/public/assets/Skew-SpQE7zeJ.js +0 -1
  220. package/public/assets/SlowHost-DwDBCTvr.js +0 -1
  221. package/public/assets/SmallFiles-yjY_6sT1.js +0 -1
  222. package/public/assets/SpeculationWaste-73PDmj0c.js +0 -1
  223. package/public/assets/Spill-DEw0Kd1i.js +0 -1
  224. package/public/assets/StageFailed-BEfXdEJc.js +0 -1
  225. package/public/assets/StageFindingGroup-Bzm2TY03.js +0 -1
  226. package/public/assets/StageShape-CzR4jHOt.js +0 -1
  227. package/public/assets/StageSlowness-vMPsnAaa.js +0 -1
  228. package/public/assets/Straggler-Cf__OGl5.js +0 -1
  229. package/public/assets/TaskFailures-BZ8kLZiO.js +0 -1
  230. package/public/assets/UnderBroadcast-CBm8ZPoS.js +0 -1
  231. package/public/assets/index-CEM9bz_x.js +0 -13
  232. package/public/assets/useRecentFiles-BrBl-AP1.js +0 -11
  233. package/public/docs/assets/chunks/@localSearchIndexroot.DbZy5EtM.js +0 -1
  234. package/public/docs/assets/contributor-guide_architecture_board-widgets.md.VTcYTH4d.js +0 -1
  235. package/public/docs/assets/contributor-guide_architecture_board-widgets.md.VTcYTH4d.lean.js +0 -1
  236. package/public/docs/assets/contributor-guide_architecture_detector-contract.md.CsqZHz_T.js +0 -1
  237. package/public/docs/assets/contributor-guide_architecture_drill-down.md.CFNCE8JB.js +0 -1
  238. package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.HT9IJGIl.js +0 -1
  239. package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.HT9IJGIl.lean.js +0 -1
  240. package/public/docs/assets/contributor-guide_architecture_index.md.3TO9ic6w.js +0 -1
  241. package/public/docs/assets/contributor-guide_architecture_overview.md.CTEqVeV4.js +0 -1
  242. package/public/docs/assets/contributor-guide_architecture_overview.md.CTEqVeV4.lean.js +0 -1
  243. package/public/docs/assets/contributor-guide_architecture_state-and-history.md.DOTVHjTV.js +0 -1
  244. package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.BmpARZve.js +0 -1
  245. package/public/docs/assets/contributor-guide_architecture_worker-protocol.md.21pTYhFF.js +0 -6
  246. package/public/docs/assets/contributor-guide_contributing.md.B2E9ni79.js +0 -1
  247. package/public/docs/assets/contributor-guide_development-setup.md.CjgZJwjK.js +0 -15
  248. package/public/docs/assets/contributor-guide_development-setup.md.CjgZJwjK.lean.js +0 -1
  249. package/public/docs/assets/contributor-guide_testing.md.DTwhpoN2.js +0 -1
  250. package/public/docs/assets/user-guide_alternative-log-retrieval.md.ZHPSbBkF.js +0 -1
  251. package/public/docs/assets/user-guide_alternative-log-retrieval.md.ZHPSbBkF.lean.js +0 -1
  252. package/public/docs/assets/user-guide_getting-started.md.CEa2ZX-l.js +0 -6
  253. package/public/docs/assets/user-guide_getting-started.md.CEa2ZX-l.lean.js +0 -1
  254. package/public/docs/assets/user-guide_run-comparison.md.ChLI3MK0.js +0 -1
  255. package/public/docs/assets/user-guide_understanding-findings.md.D1ccs60C.js +0 -1
  256. /package/public/docs/assets/{contributor-guide_architecture_detector-contract.md.CsqZHz_T.lean.js → contributor-guide_architecture_detector-contract.md.caA46z-u.lean.js} +0 -0
  257. /package/public/docs/assets/{user-guide_understanding-findings.md.D1ccs60C.lean.js → user-guide_understanding-findings.md.DzReyCXb.lean.js} +0 -0
package/public/docs/contributor-guide/architecture/board-widgets.html CHANGED
@@ -6,14 +6,15 @@
6
6
  <title>Board widgets | SparkForensics</title>
7
7
  <meta name="description" content="Docs for using and contributing to SparkForensics">
8
8
  <meta name="generator" content="VitePress v1.6.4">
9
- <link rel="preload stylesheet" href="/docs/assets/style.DSixAiZE.css" as="style">
9
+ <link rel="preload stylesheet" href="/docs/assets/style.tmDhD3c8.css" as="style">
10
10
  <link rel="preload stylesheet" href="/docs/vp-icons.css" as="style">
11
11
 
12
- <script type="module" src="/docs/assets/app.C6w47a9W.js"></script>
12
+ <script type="module" src="/docs/assets/app.yjczoibf.js"></script>
13
13
  <link rel="preload" href="/docs/assets/inter-roman-latin.Di8DUHzh.woff2" as="font" type="font/woff2" crossorigin="">
14
- <link rel="modulepreload" href="/docs/assets/chunks/theme.D-mE6j02.js">
14
+ <link rel="modulepreload" href="/docs/assets/chunks/theme.DFcAl8Ws.js">
15
15
  <link rel="modulepreload" href="/docs/assets/chunks/framework.DSg0KOwT.js">
16
- <link rel="modulepreload" href="/docs/assets/contributor-guide_architecture_board-widgets.md.VTcYTH4d.lean.js">
16
+ <link rel="modulepreload" href="/docs/assets/contributor-guide_architecture_board-widgets.md.D0Kkk_r5.lean.js">
17
+ <script>(()=>{const c=`${`[${["data","shuffle","product","bar"].join("-")}]`}, .VPNav, .VPLocalNav`,a=24,d=document.documentElement,l=t=>{const e=getComputedStyle(t);return e.display==="none"||t.getClientRects().length===0?0:e.position==="fixed"?t.getBoundingClientRect().bottom:e.position==="sticky"?(parseFloat(e.top)||0)+t.offsetHeight:0},u=()=>{try{return location.hash?document.getElementById(decodeURIComponent(location.hash.slice(1))):null}catch{return null}};let r=null;const s=()=>{const t=Array.from(document.querySelectorAll(c),l),e=Math.ceil(Math.max(0,...t))+a;if(e===r)return;const n=r;r=e,d.style.setProperty("--sf-anchor-offset",`${r}px`);const o=n===null?null:u();if(!o)return;const f=(parseFloat(getComputedStyle(o).paddingTop)||0)+2;Math.abs(o.getBoundingClientRect().top-n)<=f&&o.scrollIntoView()},i=()=>{const t=document.createElement("div");t.setAttribute("data-sf-anchor-offset",""),t.setAttribute("aria-hidden","true"),document.body.appendChild(t);const e=window.ResizeObserver?new ResizeObserver(s):null,n=()=>{e&&document.querySelectorAll(c).forEach(o=>e.observe(o)),s()};n(),new MutationObserver(n).observe(document.body,{childList:!0,subtree:!0}),window.addEventListener("resize",s),window.addEventListener("load",s)};document.readyState==="loading"?document.addEventListener("DOMContentLoaded",i,{once:!0}):i()})();</script>
17
18
  <link rel="icon" type="image/svg+xml" href="/docs/favicon.svg">
18
19
  <link rel="preconnect" href="https://fonts.googleapis.com">
19
20
  <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin="">
@@ -22,8 +23,8 @@
22
23
  <script id="check-mac-os">document.documentElement.classList.toggle("mac",/Mac|iPhone|iPod|iPad/i.test(navigator.platform));</script>
23
24
  </head>
24
25
  <body>
25
- <div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Contributor Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/development-setup.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Development setup</p><!--]--></a><!----></div><!----></div><section class="VPSidebarItem level-1 collapsible collapsed is-link has-active" data-v-b3fd67f8><div class="item" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/" data-v-b3fd67f8><!--[--><h3 class="text" data-v-b3fd67f8>Architecture</h3><!--]--></a><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-b3fd67f8><span class="vpi-chevron-right caret-icon" data-v-b3fd67f8></span></div></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/overview.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Overview</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/worker-protocol.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Worker protocol</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/state-and-history.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>State & history intake</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/detector-contract.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Detector contract</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/impact-estimation.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Impact estimation</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/widget-rendering.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Widget rendering</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/board-widgets.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Board widgets</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/drill-down.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Drill-down</p><!--]--></a><!----></div><!----></div><!--]--></div></section><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/testing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Testing & verification</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/contributing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Contributing</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_contributor-guide_architecture_board-widgets" data-v-39a288b8><div><h1 id="board-widgets" tabindex="-1">Board widgets <a class="header-anchor" href="#board-widgets" aria-label="Permalink to &quot;Board widgets&quot;">​</a></h1><h2 id="board-widgets-beyond-the-fixed-six" tabindex="-1">Beyond the fixed six <a class="header-anchor" href="#board-widgets-beyond-the-fixed-six" aria-label="Permalink to &quot;Beyond the fixed six {#board-widgets-beyond-the-fixed-six}&quot;">​</a></h2><p>(Components below live in <code>src/view/widgets/</code>, one file per widget name, e.g. <code>JobFailures.tsx</code>, <code>MemoryUtilization.tsx</code>.)</p><p>Six extra cards render in the Findings tab&#39;s active grid alongside the fixed spec §5 six: Incomplete Run, Job Failures, Caching Opportunities, Config Audit, Plan Advisor, and Autoscaling Churn, all <code>region: &#39;action&#39;</code> in <code>detector-registry.tsx</code>:</p><ul><li><p><strong>Incomplete Run</strong> (tag <code>INCMP</code>, <code>IncompleteRun.tsx</code>): app-level &quot;the capture never finished&quot; caveat (DETECTORS entry <code>incompleteRun</code>). Fires whenever <code>app.startTime</code> was observed but <code>app.endTime</code> was not, i.e. no <code>SparkListenerApplicationEnd</code> in the log: an in-flight job, a rotated-away log, or a capture cut short. Order 5, ahead of every other detector, so its card is first among the Findings tab&#39;s <code>action</code>-region active widgets when present. The Findings tab&#39;s recommendation rollup (rendered by <code>FixTheseFirst.tsx</code>) excludes <code>incompleteRun</code> from that list outright (see <a href="./widget-rendering.html#widget-rendering-order-fixed-spec-§5">Widget rendering order</a>): it&#39;s a pipeline-completeness caveat, not an addressable fix, so it never competes with other findings for a ranked slot there. Self-gates to <code>null</code> (no card in the DOM) once the run completed normally. Unrelated to <code>evidence-availability.ts</code>&#39;s own <code>trustworthy</code> gate (see <a href="./worker-protocol.html#evidence-availability-contract-v1">Evidence-availability contract</a>): that ledger only downgrades <em>absence</em> conclusions for individual evidence categories, never becomes a <code>DETECTORS</code> finding itself, and this card does not read it. No <code>docAnchor</code> is set: this is a tool-specific signal with no upstream <code>spark-tuning-reference</code> section.</p></li><li><p><strong>Job Failures</strong> (tag <code>JOBS</code>): app-level job-failure-rate rollup (DETECTORS entry <code>jobFailureRate</code> in <code>packages/core/src/detectors.ts</code>), computed from <code>app.jobs</code> (<code>SparkListenerJobEnd</code> results): ≥10% info, ≥30% warning, ≥50% critical. Complements the per-stage, task-level Failed Tasks card (tag <code>FAIL</code>).</p></li><li><p><strong>Caching Opportunities</strong> (tag <code>CACHE</code>, <code>CachingOpportunity.tsx</code>): app-level SQL relation-reuse detector (DETECTORS entry <code>cachingOpportunity</code>), computed from <code>ctx.sql</code>. It walks each execution&#39;s <code>planTree</code> and keys every scan by its stable pre-AQE identity via <code>scanRelationId</code> (<code>plan-summary.ts</code>): <code>parquet:&lt;db.table&gt;</code>, <code>delta:&lt;db.table&gt;</code>, <code>jdbc:&lt;schema.table&gt;</code>. That dedupes relations within one execution (self-joins count once) and flags any relation scanned by <code>&gt;= minExecutions</code> (2) distinct executions. Relation identity comes from the catalog-qualified scan name (nodeName, e.g. <code>Scan parquet spark_catalog.db.t</code>), not the <code>Location:</code> path, since Spark truncates that path at ~100 chars and points it at <code>_delta_log</code> for Delta tables; internal Delta-log metadata scans are dropped. Read bytes come from the scan&#39;s <code>size of files read</code> metric (<code>0</code>/unknown for JDBC). Findings carry <code>executionReuse</code> (<code>value</code> = distinct-execution count), <code>relation</code>, <code>format</code>, <code>executionIds</code>, <code>totalReadBytes</code>, and a size-aware <code>recommendation</code> (<code>confidence</code> scaled <code>low</code>/<code>medium</code>/<code>high</code> via <code>cachingReuseConfidence</code> off reuse-execution count, plus <code>validationRequired</code>). Renders nothing when clean (no card in the DOM). One row per relation: name + format badge, reuse count, <code>Data read</code> (<code>formatBytes</code>, em-dash when unknown), recommendation, sorted by <code>totalReadBytes</code> then reuse count descending. Rows reveal 6 initially, then up to 30 more per click. Pure-RDD-API apps (no SQL executions) produce no finding: a deliberate trade-off replacing the former RDD-lineage heuristic, which surfaced only internal query-engine RDDs on DataFrame/SQL workloads.</p><p>It also detects composite reuse. When the same join/union subtree (not just a leaf scan) recurs across <code>&gt;= minExecutions</code> distinct SQL executions, the detector emits one <code>variant:&#39;composite&#39;</code> finding recommending caching the derived join/union result instead of two independent leaf-relation rows, and suppresses the leaf findings for relations fully covered by it. A relation reused beyond the composite&#39;s executions keeps a residual leaf finding for just the uncovered executions. Composite identity is structural: an anchor plan-shape fingerprint (<code>findCompositeCandidates</code>/<code>computePlanShapes</code>&#39;s <code>opts.includeDetail</code> path, <code>detectors.ts</code>) folds the join/union node&#39;s own normalized <code>detail</code> (join type, columns, literals, with expr ids, <code>plan_id=</code>, codegen-stage numbers, and AQE&#39;s BuildLeft/BuildRight stripped, and commutative equality operands canonicalized) plus its children&#39;s detail-free shapes. Descendant nodes never contribute detail text, only structural shape, so a shared join reused across differently-filtered pre-join scans still matches (a deliberate trade-off) while a different join <em>condition</em> on the same tables does not collide. Nested composites (an inner join reused both standalone and inside an outer join) dedupe one level at a time: an inner composite fully covered by a qualifying outer composite&#39;s execution set is suppressed entirely, and a superset recomputes a residual finding over just its extra executions. Reuse via pure projection/aggregation with no join/union underneath stays leaf-level, not modeled as composite. <code>variant:&#39;composite&#39;</code> findings carry <code>operator</code> (<code>&#39;join&#39;</code>|<code>&#39;union&#39;</code>), <code>relations</code> (leaf relations under the composite, for display), and <code>format:&#39;derived&#39;</code> (a sentinel: composites have no scan storage format). They render a <code>JOIN</code>/<code>UNION</code> operator badge (styled distinct from the format badges) plus a confidence marker (<code>confidence</code> scaled via <code>cachingReuseConfidence</code>, <code>validationRequired</code>) in <code>CachingOpportunity.tsx</code>.</p></li><li><p><strong>Autoscaling Churn</strong> (tag <code>CHRN</code>, <code>AutoscalingChurn.tsx</code>): app-level short-lived-executor detector (DETECTORS entry <code>autoscalingChurn</code> in <code>packages/core/src/detectors.ts</code>; reuses the same <code>executorsAdded</code>/<code>executorsRemoved</code> matching logic as <code>utilization</code>). For each added executor, finds its matching removal event (falling back to <code>app.endTime</code> for an executor still alive when the log ends) and flags it short-lived if its lifetime is under 2 minutes (<code>thresholds.shortLivedMs</code>). Warns above 30% short-lived, escalates to critical above 60% (<code>confidence</code> scales <code>low</code>/<code>medium</code>/<code>high</code> via <code>autoscalingChurnConfidence</code>, off how far the short-lived share sits past <code>warningPct</code>/<code>criticalPct</code>: these thresholds are still an unvalidated design-spike estimate, not yet checked against real autoscaling-heavy logs). Returns no finding below 5 total executors (noise floor) or when <code>app.endTime</code> is missing (truncated/still-running log). The widget itself is unchanged apart from a finding-driven verdict banner (impact dot + <code>CHRN</code> tag + recommendation) above its existing add/remove chart. The chart&#39;s muted scale-down bar coloring is untouched. Its <code>docAnchor</code> is <code>#bottleneck-autoscaling-churn</code>, a section of the cluster-config chapter (<code>docs-content/chapters/11-cluster-config.md</code>), not a bottleneck page of its own: <code>pageForAnchor</code> maps it there, so the <code>CHRN</code> pill opens <code>cluster-config.html#bottleneck-autoscaling-churn</code>.</p></li><li><p><strong>Config Audit</strong> (tag <code>CFG</code>): static Spark-config sanity findings derived from <code>app.config</code>/<code>app.resources</code> (parsed from <code>SparkListenerEnvironmentUpdate</code>): dynamic-allocation vs. shuffle-service mismatch, inverted/missing autoscaling bounds, non-Kryo serializer, low executor <code>memoryOverhead</code>. Computed outside the runtime bottleneck catalog: its findings live in the separate <code>configFindings</code> stream, not <code>catalog</code>. But <code>FixTheseFirst</code>/<code>Alerts</code> both merge <code>catalog</code> and <code>configFindings</code> (neither reads <code>region</code> any more), so a Config Audit finding is still eligible for a row in the Findings tab&#39;s recommendation rollup and still gets its own card in the active grid, alongside genuine bottleneck-catalog findings.</p></li><li><p><strong>Plan Advisor</strong> (tag <code>PLAN</code>): SQL-plan-level findings computed from <code>appModel.sql</code>&#39;s resolved <code>planTree</code> (DETECTORS entries <code>duplicatePlanSubtree</code>, <code>smallFiles</code>, <code>broadcastSizing</code> in <code>packages/core/src/detectors.ts</code>): repeated plan subtrees (≥3 nodes, ≥2 occurrences, critical if the repeated root is an <code>Exchange</code>), small-files read/write (&gt;100 files averaging ❤️ MiB), and broadcast-join sizing in both directions (missed- broadcast info finding, over-broadcast warning at &gt;1 GB). Each of the four emitted types now renders as its own card (<code>DuplicatePlanSubtree.tsx</code>, <code>SmallFiles.tsx</code>, <code>UnderBroadcast.tsx</code>, <code>OverBroadcast.tsx</code>; the 2026-09 widget/finding-type 1:1 mapping redesign split what used to be one shared <code>PlanFindings.tsx</code> card): all four still share the <code>PLAN</code> tag and the same <code>--plan-aggregate</code> badge tint (<code>src/view/plan-finding-shared.ts</code>). Their <code>docAnchor</code>s (<code>#bottleneck-duplicate-plan-subtree</code>, <code>#bottleneck-small-files</code>, <code>#bottleneck-broadcast-sizing</code>) each resolve to their own page in the vendored tuning reference.</p></li></ul><p><code>detector-registry.tsx</code>&#39;s <code>REGISTRY</code> still carries <code>region: &#39;reference&#39;</code> on four entries, each its own component now: <code>memoryUtilization</code> (<code>MemoryUtilization</code>), <code>utilization</code> (<code>ExecutorUtilization</code>), <code>cacheUtilization</code> (<code>CacheUtilization</code>), and <code>coreLocality</code> (<code>CoreUsageArea</code>). <code>broadcastSizing</code> is gone from <code>REGISTRY</code> entirely (the 2026-09 widget/finding-type 1:1 mapping redesign dropped it): it never backed a real <code>Finding</code> (the <code>broadcastSizing</code> <code>DETECTORS</code> entry only ever emits <code>underBroadcast</code>/<code>overBroadcast</code>, both <code>region: &#39;action&#39;</code>, each now its own Plan Advisor card), so there is no key left for it to occupy or a clean-check line for it to render. Autoscaling Churn, the other executor-provisioning-lifecycle detector alongside <code>utilization</code>/<code>memoryUtilization</code>, was deliberately given <code>action</code> rather than <code>reference</code> (see &quot;Beyond the fixed six&quot; above).</p><p><code>region</code> decides one thing (see <a href="./widget-rendering.html#widget-rendering-order-fixed-spec-§5">Widget rendering order</a>): whether a widget always mounts. <code>isAlwaysMountedType()</code> flags exactly one of the four <code>reference</code>-region types: Core Usage by Locality. That one mounts unconditionally from <code>appModel</code> at the head of the Full app report tab&#39;s reference grid, regardless of finding state. Memory Utilization, Executor Utilization, and Cache Storage are <code>reference</code> too, but all three are excluded by product decision, not a component-sharing constraint (<code>ALWAYS_MOUNTED_EXCEPTIONS</code> in <code>src/view/detector-registry.tsx</code> carries <code>cacheUtilization</code>, <code>memoryUtilization</code>, and <code>utilization</code>): a clean run on any of them isn&#39;t evidence worth surfacing unconditionally, so each collapses to an ordinary <code>CleanCheckRow</code> like any other action-region type on a clean run. Every other <code>REGISTRY</code> widget still renders unconditionally as either an active card or a clean-check line, and <code>region</code> still sets <code>orderedWidgets()</code>&#39;s sort order within the active grid (<code>action</code> components first, <code>reference</code> ones after). The Full app report tab reads no other <code>REGISTRY</code> entry. So Core Usage by Locality, the one widget still tagged <code>region: &#39;reference&#39;</code> and exempt from <code>ALWAYS_MOUNTED_EXCEPTIONS</code>, renders in the Full app report beside the other run-wide reference views, and a route to a <code>coreLocality</code> finding switches to that tab (Memory Utilization, Executor Utilization, and Cache Storage all render through the ordinary active/clean paths instead):</p><ul><li><strong>Memory Utilization</strong> (tag <code>MEM</code>): app-level card combining three sub-findings from the <code>memoryUtilization</code> DETECTORS entry (<code>packages/core/src/detectors.ts</code>): idle-cores rate (busy-core-time from the worker&#39;s run-aggregates sweep vs. peak-cores × wall-clock), per-executor memory bands (peak heap vs. allocated, gated on <code>spark.eventLog.logStageExecutorMetrics</code>: a distinct <code>dataUnavailable</code> finding renders when that config was off), and an unverified memory-waste model (<code>confidence</code> scales <code>low</code>/<code>medium</code>/<code>high</code> via <code>memoryWasteConfidence</code>, off how far the wasted/used ratio sits past the 1.5× buffer). The driver-memory half of the original spec is dropped: the worker only extracts <em>allocated</em> <code>spark.driver.memory</code>, never a driver actual-usage metric, so there is nothing to band against. The separate <code>utilization</code> DETECTORS entry (an <code>avgUtilization</code> info finding: active-executor-time fraction below 60%) has its own card, <strong>Executor Utilization</strong> (tag <code>UTIL</code>, <code>ExecutorUtilization.tsx</code>): a <code>reference</code>-region widget in its own right that renders only with an active finding, split out of this same combined widget in the 2026-09 widget/finding-type 1:1 mapping redesign (it used to render inline here; the former <code>ExecutorTimeline.tsx</code> never owned this type, despite the tag&#39;s letters).</li><li><strong>Cache Storage</strong> (tag <code>CSTOR</code>): app-level card driven by the <code>cacheUtilization</code> DETECTORS entry (<code>packages/core/src/detectors.ts</code>), evaluating two per-RDD proxies over <code>ctx.app.rddInfo</code> since Spark event logs carry no runtime block-access/read-count data. <code>rddInfo</code>&#39;s cache figures come from <code>SparkListenerBlockUpdated</code> (<code>recordBlockUpdate</code> in <code>event-handlers.ts</code>, only written with <code>spark.eventLog.logBlockUpdates.enabled=true</code>): each RDD&#39;s peak count of resident partitions, with the memory/disk bytes at the latest moment that peak held, so an <code>unpersist()</code> before the log ends doesn&#39;t erase it. A block&#39;s bytes count only where its storage level says it lives (as in Spark&#39;s <code>AppStatusListener</code>): a drop from memory to disk still reports the dropped bytes as <code>Memory Size</code>. A removed executor&#39;s blocks are dropped with it (Spark logs no update for them), and once an RDD has block updates a later stage&#39;s RDD Info can&#39;t reset its storage level to <code>NONE</code> after an <code>unpersist()</code>. The corpus <code>cache-memory-only</code> and <code>cache-memory-and-disk</code> logs exercise both rules. Without block updates they fall back to <code>SparkListenerStageSubmitted</code>&#39;s RDD Info (<code>storageSource</code> records which), which is always 0 since Spark 2.3; Spark 1.x fills it only on <code>StageCompleted</code>, which isn&#39;t read. The two proxies are partial caching (<code>numCachedPartitions / numPartitions &lt; 0.90</code>, <code>&lt; 0.50</code> for the warning tier) and disk spillover for <code>MEMORY_AND_DISK*</code> RDDs (<code>diskSize / (memorySize + diskSize) &gt; 0.15</code>, <code>&gt; 0.40</code> for the warning tier; <code>DISK_ONLY</code> RDDs are never flagged). <code>confidence</code> scales <code>low</code>/<code>medium</code>/<code>high</code> via <code>cacheSampleConfidence(rdd.numPartitions)</code>, because the ratio is a storage snapshot, not a runtime read-count, and more partitions average that snapshot noise into a more stable ratio. When persisted RDDs have no storage evidence at all (no block updates, block-update logging not enabled in the app config, a recorded Spark version of 2.3 or later, and every RDD Info figure 0), the detector emits one <code>storageUnobserved</code> caveat (<code>dataUnavailable: true</code>, <code>info</code>) naming <code>spark.eventLog.logBlockUpdates.enabled</code>. Unlike <code>memoryUtilization</code>&#39;s caveat it counts for <code>isRealFinding</code>, so the card still mounts (with the caveat and no table) and Cache Storage never lands in Clean checks for a run that couldn&#39;t be checked; <code>isEligible</code> keeps it out of Fix these first. The existing RDD table (ported from the legacy <code>src/widgets/cache-utilization.js</code> canvas widget) still renders unconditionally; flagged rows get an inline <code>CSTOR</code> tag next to the RDD name, and every flagged RDD&#39;s recommendation renders below the table, worst-first.</li><li><strong>Core Usage by Locality</strong> (tag <code>LOCAL</code>, <code>CoreUsageArea.tsx</code>): app-level non-local-task-ratio threshold (DETECTORS entry <code>coreLocality</code>; the other half of a wasted-cores-ratio check, the idle-core half already covered by Memory Utilization&#39;s <code>idleCores</code> variant above). Sums <code>RACK_LOCAL</code> + <code>ANY</code> task counts against total task count across every stage&#39;s <code>stage.localityStats</code> (pure reducer <code>computeCoreLocalityRatio</code>, <code>packages/core/src/core-locality-ratio.ts</code>). <code>NO_PREF</code> stays in the denominator only, since it&#39;s what shuffle-read stages legitimately report with no locality problem. Below 50 total tasks or below a 15% non-local ratio: no finding; 15%-35%: warning; &gt;= 35%: critical, still unvalidated design-spike thresholds (<code>confidence</code> scales <code>low</code>/<code>medium</code>/<code>high</code> via <code>coreLocalityConfidence</code>, off whichever is weaker of the non-local ratio and the sampled task count, the same evidence-strength convention as the memory-waste model above). The widget&#39;s always-rendered stacked-area chart (<code>packages/core/src/core-usage-locality.ts</code>) is unaffected. The finding only adds a threshold/impact-band section above it, a per-stage non-local breakdown below it (shown whenever any non-local tasks exist at all, independent of whether the aggregate crossed threshold), and a one-line cross-reference to Memory Utilization when its <code>idleCores</code> finding also fired (idle-core ratio itself is not duplicated here).</li></ul><p>Documented ALL-CAPS tag vocabulary: <code>SKEW</code>, <code>SHFL</code>, <code>SPILL</code>, <code>GC</code>, <code>COLD</code>, <code>UTIL</code>, <code>MEM</code> (Memory Utilization), <code>CSTOR</code> (Cache Storage), <code>LOCAL</code> (Core Usage by Locality), plus <code>FAIL</code> (Failed Tasks), <code>JOBS</code> (Job Failures), <code>CFG</code> (Config Audit), <code>PLAN</code> (Plan Advisor), <code>SFAIL</code> (stage failed outright), <code>PART</code> (partition sizing), <code>SLOW</code> (stage overall slowness), <code>SHAPE</code> (stage shape smells), <code>CACHE</code> (caching opportunity) and <code>CHRN</code> (autoscaling churn).</p><p>ETL Phase Attribution (<code>packages/core/src/etl-phases.ts</code> + <code>EtlPhases.tsx</code>), What-If Executor Scaling (<code>packages/core/src/scaling-sim.ts</code> + <code>ScalingSim.tsx</code>, design spike: makespan predictions are unvalidated and carry a Model Error indicator), and Compute Efficiency (<code>packages/core/src/efficiency-model.ts</code> + <code>EfficiencyModel.tsx</code>, design spike: driver-vs-executor waste split, two theoretical floors, and the §6 right-sizing copy) are main-thread report modules, not <code>DETECTORS</code> entries: descriptive lenses with no impact-band threshold, rendered unconditionally into the Full app report tab rather than participating in the bottleneck catalog. <code>packages/core/src/job-groups.ts</code> (<code>checkConcurrentJobGroups</code>) is likewise a report helper: it flags when concurrent SQL-execution job groups make wall-clock-based estimates unreliable, and both the scaling simulator and the efficiency model gate their precision on it, showing a caveat banner rather than suppressing the estimate.</p><h2 id="confidence-metadata" tabindex="-1">Confidence metadata <a class="header-anchor" href="#confidence-metadata" aria-label="Permalink to &quot;Confidence metadata&quot;">​</a></h2><p>Best-effort (non-deterministic) findings carry a <code>confidence</code> + <code>validationRequired</code> marker, rendered inline by each consuming widget (e.g. <code>Spill.tsx</code>, <code>DuplicatePlanSubtree.tsx</code>, <code>SmallFiles.tsx</code>, <code>UnderBroadcast.tsx</code>, <code>OverBroadcast.tsx</code>, <code>EfficiencyModel.tsx</code>) when <code>confidence</code> is present and not <code>&#39;high&#39;</code>. There is no shared helper: each widget renders its own muted &quot;N confidence: verify&quot; text, with the <code>title</code> attribute carrying <code>validationRequired</code> as a tooltip. Spill classification is <code>medium</code> when classified (skew/volume) and <code>low</code> when unclassified; plan-summary warnings are <code>low</code>. Deterministic detectors stay unmarked, treated as high confidence.</p><h2 id="visual-system" tabindex="-1">Visual system <a class="header-anchor" href="#visual-system" aria-label="Permalink to &quot;Visual system&quot;">​</a></h2><p>Tailwind CSS v4 + shadcn/ui (Base UI primitives). The design tokens (colors, including the telemetry-console dark palette) are defined as CSS variables in an <code>@theme inline</code> block and consumed via Tailwind utility classes; there is no more hand-authored BEM CSS. Theme polarity is unchanged from the legacy app: dark = bare <code>:root</code> (no attribute), light = <code>:root[data-theme=&quot;light&quot;]</code>, toggled by <code>src/theme/ThemeProvider.tsx</code> and persisted to <code>localStorage</code>. <code>index.html</code>&#39;s inline FOUC-prevention script still runs before React mounts. Mono is still the typeface for numerics (<code>--font-mono</code>), applied via a <code>.num</code>-equivalent Tailwind utility per call site rather than one global class. Charts are Recharts (<code>src/view/charts/ChartTheme.tsx</code>&#39;s <code>CHART_COLORS</code>, read from the same CSS tokens) instead of Chart.js. React re-renders on theme toggle, so chart colors update live with the theme, unlike the old Chart.js canvases, which were built once per parse. <code>ChartFrame</code> can also expose the underlying rows through a toggleable accessible table and copy them to the clipboard as TSV.</p><h2 id="templating-xss-safe" tabindex="-1">Templating (XSS-safe) <a class="header-anchor" href="#templating-xss-safe" aria-label="Permalink to &quot;Templating (XSS-safe)&quot;">​</a></h2><p>JSX auto-escapes every interpolated value by default. The hand-rolled auto-escaping <code> html</code> `` tagged template (<code>src/widgets/utils.js</code>) and its <code>no-raw-innerhtml</code> guard test are gone, and no <code>.innerHTML</code> assignment is left anywhere in the view layer.</p><p>One Base UI-specific gotcha carried no equivalent in the legacy app: <code>WidgetCard.tsx</code> passes <code>aria-expanded={String(open) as &#39;true&#39; | &#39;false&#39;}</code> rather than the raw boolean, because Base UI&#39;s <code>Collapsible.Trigger</code> otherwise overrides a boolean <code>aria-expanded</code> prop with its own internal state via prop merging. The explicit <code>String()</code> cast keeps the rendered attribute in sync with this app&#39;s own <code>open</code> state.</p></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/contributor-guide/architecture/widget-rendering.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>Widget rendering</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="/docs/contributor-guide/architecture/drill-down.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>Drill-down</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
26
- <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"VTcYTH4d\",\"contributor-guide_architecture_detector-contract.md\":\"CsqZHz_T\",\"contributor-guide_architecture_drill-down.md\":\"CFNCE8JB\",\"contributor-guide_architecture_impact-estimation.md\":\"HT9IJGIl\",\"contributor-guide_architecture_index.md\":\"3TO9ic6w\",\"contributor-guide_architecture_overview.md\":\"CTEqVeV4\",\"contributor-guide_architecture_state-and-history.md\":\"DOTVHjTV\",\"contributor-guide_architecture_widget-rendering.md\":\"BmpARZve\",\"contributor-guide_architecture_worker-protocol.md\":\"21pTYhFF\",\"contributor-guide_contributing.md\":\"B2E9ni79\",\"contributor-guide_development-setup.md\":\"CjgZJwjK\",\"contributor-guide_testing.md\":\"DTwhpoN2\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"D08Uaod7\",\"tuning-reference_aqe.md\":\"C1Ae6A8u\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"6wjKuJNE\",\"tuning-reference_bottleneck-cold-start.md\":\"BVQbsuHP\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"DtaYnaiF\",\"tuning-reference_bottleneck-failures.md\":\"Z1Tdmdd3\",\"tuning-reference_bottleneck-gc.md\":\"BCnXLf1n\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"CcvQj2g9\",\"tuning-reference_bottleneck-memory-utilization.md\":\"B-C0-8h-\",\"tuning-reference_bottleneck-retry-waste.md\":\"DCnhWiqj\",\"tuning-reference_bottleneck-shuffle.md\":\"B5OMJCd3\",\"tuning-reference_bottleneck-skew.md\":\"Cvo6NQm_\",\"tuning-reference_bottleneck-slow-host.md\":\"ja5qr3wq\",\"tuning-reference_bottleneck-small-files.md\":\"Crjtb0i2\",\"tuning-reference_bottleneck-spill.md\":\"Bl8KtSRg\",\"tuning-reference_bottleneck-straggler.md\":\"B0VN8reT\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"DQCftGhu\",\"tuning-reference_bottleneck-utilization.md\":\"BXDl30f9\",\"tuning-reference_caching.md\":\"CcaiMn_7\",\"tuning-reference_cluster-config.md\":\"DbWG1yZq\",\"tuning-reference_config.md\":\"YUqSpwFB\",\"tuning-reference_data-formats.md\":\"C5DX5xL4\",\"tuning-reference_index.md\":\"ZVIpQ6Dk\",\"tuning-reference_intro.md\":\"BSSaDlqx\",\"tuning-reference_joins.md\":\"BVhNCDnE\",\"tuning-reference_memory-model.md\":\"CfjFdgbl\",\"tuning-reference_metrics.md\":\"C64VNEL4\",\"tuning-reference_partitioning.md\":\"CtLK0xsS\",\"tuning-reference_pyspark.md\":\"D-92XgF8\",\"tuning-reference_shuffle.md\":\"Iq2v1oHJ\",\"tuning-reference_spark-architecture.md\":\"_qwnE_Sc\",\"tuning-reference_table-formats.md\":\"BSYEK-eN\",\"user-guide_alternative-log-retrieval.md\":\"ZHPSbBkF\",\"user-guide_getting-started.md\":\"CEa2ZX-l\",\"user-guide_mcp-tools.md\":\"dQwIvdTF\",\"user-guide_run-comparison.md\":\"ChLI3MK0\",\"user-guide_understanding-findings.md\":\"D1ccs60C\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Alternative log retrieval\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
26
+ <div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Contributor Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/development-setup.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Development setup</p><!--]--></a><!----></div><!----></div><section class="VPSidebarItem level-1 collapsible collapsed is-link has-active" data-v-b3fd67f8><div class="item" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/" data-v-b3fd67f8><!--[--><h3 class="text" data-v-b3fd67f8>Architecture</h3><!--]--></a><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-b3fd67f8><span class="vpi-chevron-right caret-icon" data-v-b3fd67f8></span></div></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/overview.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Overview</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/worker-protocol.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Worker protocol</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/state-and-history.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>State & history intake</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/detector-contract.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Detector contract</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/impact-estimation.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Impact estimation</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/widget-rendering.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Widget rendering</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/board-widgets.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Board widgets</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/drill-down.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Drill-down</p><!--]--></a><!----></div><!----></div><!--]--></div></section><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/testing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Testing & verification</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/contributing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Contributing</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_contributor-guide_architecture_board-widgets" data-v-39a288b8><div><h1 id="board-widgets" tabindex="-1">Board widgets <a class="header-anchor" href="#board-widgets" aria-label="Permalink to &quot;Board widgets&quot;">​</a></h1><h2 id="board-widgets-app-and-plan-widgets" tabindex="-1">App- and plan-level widgets <a class="header-anchor" href="#board-widgets-app-and-plan-widgets" aria-label="Permalink to &quot;App- and plan-level widgets {#board-widgets-app-and-plan-widgets}&quot;">​</a></h2><p>(Components below live in <code>src/view/widgets/</code>, one file per widget name, e.g. <code>JobFailures.tsx</code>, <code>MemoryUtilization.tsx</code>.)</p><p>Beyond the stage-level cards, nine app- and plan-level cards render in the Findings tab, all <code>region: &#39;action&#39;</code> in <code>detector-registry.tsx</code>: Incomplete Run, Job Failures, Caching Opportunities, Autoscaling Churn, Config Audit, and the four Plan Advisor cards (Redundant Plan Subtree, Excessive Small Files, Missed Broadcast Join, Oversized Broadcast Join). <code>REGISTRY</code> maps every finding type to its own component: 24 <code>action</code> and 4 <code>reference</code> entries.</p><ul><li><p><strong>Incomplete Run</strong> (tag <code>INCMP</code>, <code>IncompleteRun.tsx</code>): app-level &quot;the capture never finished&quot; caveat (DETECTORS entry <code>incompleteRun</code>). Fires whenever <code>app.startTime</code> was observed but <code>app.endTime</code> was not, i.e. no <code>SparkListenerApplicationEnd</code> in the log: an in-flight job, a rotated-away log, or a capture cut short. Order 5, the lowest of any detector, so within the Warning band (its fixed impact band) its card comes first. The Findings tab&#39;s recommendation rollup (rendered by <code>FixTheseFirst.tsx</code>) excludes <code>incompleteRun</code> from that list outright (see <a href="./widget-rendering.html#widget-rendering-order">Widget rendering order</a>): it&#39;s a pipeline-completeness caveat, not an addressable fix, so it never competes with other findings for a ranked slot there. Self-gates to <code>null</code> (no card in the DOM) once the run completed normally. Unrelated to <code>evidence-availability.ts</code>&#39;s own <code>trustworthy</code> gate (see <a href="./worker-protocol.html#evidence-availability-contract">Evidence-availability contract</a>): that ledger only downgrades <em>absence</em> conclusions for individual evidence categories, never becomes a <code>DETECTORS</code> finding itself, and this card does not read it. No <code>docAnchor</code> is set: this is a tool-specific signal with no upstream <code>spark-tuning-reference</code> section.</p></li><li><p><strong>Job Failures</strong> (tag <code>JOBS</code>): app-level job-failure-rate rollup (DETECTORS entry <code>jobFailureRate</code> in <code>packages/core/src/detectors.ts</code>), computed from the run&#39;s <code>jobs</code> map (<code>ctx.jobs</code>, <code>SparkListenerJobEnd</code> results): ≥10% info, ≥30% warning, ≥50% critical. Complements the per-stage, task-level Failed Tasks card (tag <code>FAIL</code>).</p></li><li><p><strong>Caching Opportunities</strong> (tag <code>CACHE</code>, <code>CachingOpportunity.tsx</code>): app-level SQL relation-reuse detector (DETECTORS entry <code>cachingOpportunity</code>), computed from <code>ctx.sql</code>. It walks each execution&#39;s <code>planTree</code> and keys every scan by its stable pre-AQE identity via <code>scanRelationId</code> (<code>plan-summary.ts</code>): <code>&lt;parquet|orc|csv|json&gt;:&lt;db.table&gt;</code>, <code>delta:&lt;db.table&gt;</code>, <code>jdbc:&lt;schema.table&gt;</code>. That dedupes relations within one execution (self-joins count once) and flags any relation scanned by <code>&gt;= minExecutions</code> (2) distinct executions. Relation identity comes from the catalog-qualified scan name (nodeName, e.g. <code>Scan parquet spark_catalog.db.t</code>), not the <code>Location:</code> path, since Spark truncates that path at ~100 chars and points it at <code>_delta_log</code> for Delta tables; internal Delta-log metadata scans are dropped. Read bytes come from the scan&#39;s <code>size of files read</code> metric (<code>0</code>/unknown for JDBC). Findings carry <code>executionReuse</code> (<code>value</code> = distinct-execution count), <code>relation</code>, <code>format</code>, <code>executionIds</code>, <code>totalReadBytes</code>, and a size-aware <code>recommendation</code> (<code>confidence</code> scaled <code>low</code>/<code>medium</code>/<code>high</code> via <code>cachingReuseConfidence</code> off reuse-execution count, plus <code>validationRequired</code>). Renders nothing when clean (no card in the DOM). One row per relation: name + format badge, reuse count, <code>Data read</code> (<code>formatBytes</code>, em-dash when unknown), sorted by <code>totalReadBytes</code> then reuse count descending; the card states the fix once (<code>fixFor</code>), not per row. Rows are paged <code>VISIBLE_LIMIT</code> (6) at a time (<code>usePagedRows</code> + <code>RowPagination</code>), and a route to a row jumps to its page. Pure-RDD-API apps (no SQL executions) produce no finding: a deliberate trade-off, since an RDD-lineage heuristic surfaces only internal query-engine RDDs on DataFrame/SQL workloads.</p><p>It also detects composite reuse. When the same join/union subtree (not just a leaf scan) recurs across <code>&gt;= minExecutions</code> distinct SQL executions, the detector emits one <code>variant:&#39;composite&#39;</code> finding recommending caching the derived join/union result instead of two independent leaf-relation rows, and suppresses the leaf findings for relations fully covered by it. A relation reused beyond the composite&#39;s executions keeps a residual leaf finding for just the uncovered executions. Composite identity is structural: an anchor plan-shape fingerprint built by <code>findCompositeCandidates</code> (<code>packages/core/src/detectors.ts</code>; the same shape <code>computePlanShapes</code> produces with <code>opts.includeDetail</code>, computed inline in one bottom-up pass) folds the join/union node&#39;s own normalized <code>detail</code> (join type, columns, literals, with expr ids, <code>plan_id=</code>, codegen-stage numbers, and AQE&#39;s BuildLeft/BuildRight stripped, and commutative equality operands canonicalized) plus its children&#39;s detail-free shapes. Descendant nodes never contribute detail text, only structural shape, so a shared join reused across differently-filtered pre-join scans still matches (a deliberate trade-off) while a different join <em>condition</em> on the same tables does not collide. Nested composites (an inner join reused both standalone and inside an outer join) dedupe one level at a time: an inner composite fully covered by a qualifying outer composite&#39;s execution set is suppressed entirely, and a superset recomputes a residual finding over just its extra executions. Reuse via pure projection/aggregation with no join/union underneath stays leaf-level, not modeled as composite. <code>variant:&#39;composite&#39;</code> findings carry <code>operator</code> (<code>&#39;join&#39;</code>|<code>&#39;union&#39;</code>), <code>relations</code> (leaf relations under the composite, for display), and <code>format:&#39;derived&#39;</code> (a sentinel: composites have no scan storage format). They render a <code>JOIN</code>/<code>UNION</code> operator badge (styled distinct from the format badges) plus a confidence marker (<code>confidence</code> scaled via <code>cachingReuseConfidence</code>, <code>validationRequired</code>) in <code>CachingOpportunity.tsx</code>.</p></li><li><p><strong>Autoscaling Churn</strong> (tag <code>CHRN</code>, <code>AutoscalingChurn.tsx</code>): app-level short-lived-executor detector (DETECTORS entry <code>autoscalingChurn</code> in <code>packages/core/src/detectors.ts</code>; reuses the same <code>executorsAdded</code>/<code>executorsRemoved</code> matching logic as <code>utilization</code>). For each added executor, finds its matching removal event (falling back to <code>app.endTime</code> for an executor still alive when the log ends) and flags it short-lived if its lifetime is under 2 minutes (<code>thresholds.shortLivedMs</code>). Warns above 30% short-lived, escalates to critical above 60% (<code>confidence</code> scales <code>low</code>/<code>medium</code>/<code>high</code> via <code>autoscalingChurnConfidence</code>, off how far the short-lived share sits past <code>warningPct</code>/<code>criticalPct</code>: these thresholds are unvalidated estimates, not checked against real autoscaling-heavy logs). Returns no finding below 5 total executors (noise floor) or when <code>app.endTime</code> is missing (truncated/still-running log). The widget shows a finding-driven verdict banner (impact dot + <code>CHRN</code> tag + recommendation) above its add/remove chart, whose scale-down bars are muted. Its <code>docAnchor</code> is <code>#bottleneck-autoscaling-churn</code>, a section of the cluster-config chapter (<code>docs-content/chapters/11-cluster-config.md</code>), not a bottleneck page of its own: <code>pageForAnchor</code> maps it there, so the <code>CHRN</code> pill opens <code>cluster-config.html#bottleneck-autoscaling-churn</code>.</p></li><li><p><strong>Config Audit</strong> (tag <code>CFG</code>): static Spark-config sanity findings derived from <code>app.config</code>/<code>app.resources</code> (parsed from <code>SparkListenerEnvironmentUpdate</code>): dynamic-allocation vs. shuffle-service mismatch, inverted/missing autoscaling bounds, non-Kryo serializer, low executor <code>memoryOverhead</code>. Computed outside the runtime bottleneck catalog: its findings live in the separate <code>configFindings</code> stream, not <code>catalog</code>. But <code>FixTheseFirst</code>/<code>Alerts</code> both merge <code>catalog</code> and <code>configFindings</code> (neither reads <code>region</code>), so a Config Audit finding is eligible for a row in the Findings tab&#39;s recommendation rollup and gets its own card in the active grid, alongside genuine bottleneck-catalog findings.</p></li><li><p><strong>Plan Advisor</strong> (tag <code>PLAN</code>): SQL-plan-level findings computed from <code>appModel.sql</code>&#39;s resolved <code>planTree</code> (DETECTORS entries <code>duplicatePlanSubtree</code>, <code>smallFiles</code>, <code>broadcastSizing</code> in <code>packages/core/src/detectors.ts</code>): repeated plan subtrees (≥3 nodes, ≥2 occurrences; dropped when their linked stages take under 0.5% of the run; banded from the recovered wall-clock like other findings, with a <code>warning</code>/<code>info</code> fallback; an <code>Exchange</code> root only changes the recommendation to a possible missed exchange reuse), small-files read/write (&gt;100 files averaging ❤️ MiB), and broadcast-join sizing in both directions (missed- broadcast info finding, over-broadcast warning at &gt;1 GB). Each of the four emitted types renders as its own card (<code>DuplicatePlanSubtree.tsx</code>, <code>SmallFiles.tsx</code>, <code>UnderBroadcast.tsx</code>, <code>OverBroadcast.tsx</code>): all four share the <code>PLAN</code> tag and the same <code>--plan-aggregate</code> badge tint (<code>src/view/plan-finding-shared.ts</code>). Their <code>docAnchor</code>s (<code>#bottleneck-duplicate-plan-subtree</code>, <code>#bottleneck-small-files</code>, <code>#bottleneck-broadcast-sizing</code>) each resolve to their own page in the vendored tuning reference.</p></li></ul><p><code>detector-registry.tsx</code>&#39;s <code>REGISTRY</code> carries <code>region: &#39;reference&#39;</code> on four entries, each its own component: <code>memoryUtilization</code> (<code>MemoryUtilization</code>), <code>utilization</code> (<code>ExecutorUtilization</code>), <code>cacheUtilization</code> (<code>CacheUtilization</code>), and <code>coreLocality</code> (<code>CoreUsageArea</code>). <code>broadcastSizing</code> has no <code>REGISTRY</code> entry: it never backs a real <code>Finding</code> (the <code>broadcastSizing</code> <code>DETECTORS</code> entry only emits <code>underBroadcast</code>/<code>overBroadcast</code>, both <code>region: &#39;action&#39;</code>, each its own Plan Advisor card), so there is no key for it to occupy or clean-check line for it to render. Autoscaling Churn, the other executor-provisioning-lifecycle detector alongside <code>utilization</code>/<code>memoryUtilization</code>, is <code>action</code>, not <code>reference</code> (see &quot;App- and plan-level widgets&quot; above).</p><p><code>region</code> decides one thing (see <a href="./widget-rendering.html#widget-rendering-order">Widget rendering order</a>): whether a widget always mounts. <code>isAlwaysMountedType()</code> flags exactly one of the four <code>reference</code>-region types: Core Usage by Locality. That one mounts unconditionally from <code>appModel</code> at the head of the Full app report tab&#39;s reference grid, regardless of finding state. Memory Utilization, Executor Utilization, and Cache Storage are <code>reference</code> too, but all three are excluded by product decision, not a component-sharing constraint (<code>ALWAYS_MOUNTED_EXCEPTIONS</code> in <code>src/view/detector-registry.tsx</code> carries <code>cacheUtilization</code>, <code>memoryUtilization</code>, and <code>utilization</code>): a clean run on any of them isn&#39;t evidence worth surfacing unconditionally, so each collapses to an ordinary <code>CleanCheckRow</code> like any other action-region type on a clean run. Every other <code>REGISTRY</code> widget renders unconditionally as either an active card or a clean-check line, and <code>region</code>, then detector order, breaks ties between cards of the same impact band (<code>computeActiveWidgets</code> ranks by worst impact band first; <code>ImpactBoard</code> groups cards Critical, Warning, Info). The Full app report tab reads no other <code>REGISTRY</code> entry. So Core Usage by Locality, the one widget tagged <code>region: &#39;reference&#39;</code> and exempt from <code>ALWAYS_MOUNTED_EXCEPTIONS</code>, renders in the Full app report beside the other run-wide reference views, and a route to a <code>coreLocality</code> finding switches to that tab (Memory Utilization, Executor Utilization, and Cache Storage all render through the ordinary active/clean paths instead):</p><ul><li><strong>Memory Utilization</strong> (tag <code>MEM</code>): app-level card combining three sub-findings from the <code>memoryUtilization</code> DETECTORS entry (<code>packages/core/src/detectors.ts</code>): idle-cores rate (busy-core-time from the worker&#39;s run-aggregates sweep vs. peak-cores × wall-clock), per-executor memory bands (peak heap vs. allocated, gated on <code>spark.eventLog.logStageExecutorMetrics</code>: a distinct <code>dataUnavailable</code> finding renders when that config was off), and an unverified memory-waste model (<code>confidence</code> scales <code>low</code>/<code>medium</code>/<code>high</code> via <code>memoryWasteConfidence</code>, off how far the wasted/used ratio sits past the 1.5× buffer). There is no driver-memory band: the worker only extracts <em>allocated</em> <code>spark.driver.memory</code>, never a driver actual-usage metric, so there is nothing to band against. The separate <code>utilization</code> DETECTORS entry (an <code>avgUtilization</code> info finding: active-executor-time fraction below 60%) has its own card, <strong>Executor Utilization</strong> (tag <code>UTIL</code>, <code>ExecutorUtilization.tsx</code>): a <code>reference</code>-region widget in its own right that renders only with an active finding.</li><li><strong>Cache Storage</strong> (tag <code>CSTOR</code>): app-level card driven by the <code>cacheUtilization</code> DETECTORS entry (<code>packages/core/src/detectors.ts</code>), evaluating two per-RDD proxies over <code>ctx.app.rddInfo</code> since Spark event logs carry no runtime block-access/read-count data. <code>rddInfo</code>&#39;s cache figures come from <code>SparkListenerBlockUpdated</code> (<code>recordBlockUpdate</code> in <code>event-handlers.ts</code>, only written with <code>spark.eventLog.logBlockUpdates.enabled=true</code>): each RDD&#39;s peak count of resident partitions, with the memory/disk bytes at the latest moment that peak held, so an <code>unpersist()</code> before the log ends doesn&#39;t erase it. A block&#39;s bytes count only where its storage level says it lives (as in Spark&#39;s <code>AppStatusListener</code>): a drop from memory to disk still reports the dropped bytes as <code>Memory Size</code>. A removed executor&#39;s blocks are dropped with it (Spark logs no update for them), and once an RDD has block updates a later stage&#39;s RDD Info can&#39;t reset its storage level to <code>NONE</code> after an <code>unpersist()</code>. The corpus <code>cache-memory-only</code> and <code>cache-memory-and-disk</code> logs exercise both rules. Without block updates they fall back to <code>SparkListenerStageSubmitted</code>&#39;s RDD Info (<code>storageSource</code> records which), which is always 0 since Spark 2.3; Spark 1.x fills it only on <code>StageCompleted</code>, which isn&#39;t read. The two proxies are partial caching (<code>numCachedPartitions / numPartitions &lt; 0.90</code>, <code>&lt; 0.50</code> for the warning tier) and disk spillover for <code>MEMORY_AND_DISK*</code> RDDs (<code>diskSize / (memorySize + diskSize) &gt; 0.15</code>, <code>&gt; 0.40</code> for the warning tier; <code>DISK_ONLY</code> RDDs are never flagged). <code>confidence</code> scales <code>low</code>/<code>medium</code>/<code>high</code> via <code>cacheSampleConfidence(rdd.numPartitions)</code>, because the ratio is a storage snapshot, not a runtime read-count, and more partitions average that snapshot noise into a more stable ratio. When persisted RDDs have no storage evidence at all (no block updates, block-update logging not enabled in the app config, a recorded Spark version of 2.3 or later, and every RDD Info figure 0), the detector emits one <code>storageUnobserved</code> caveat (<code>dataUnavailable: true</code>, <code>info</code>) naming <code>spark.eventLog.logBlockUpdates.enabled</code>. Unlike <code>memoryUtilization</code>&#39;s caveat it counts for <code>isRealFinding</code>, so the card still mounts (with the caveat and no table) and Cache Storage never lands in Clean checks for a run that couldn&#39;t be checked; <code>isEligible</code> keeps it out of Fix these first. The RDD table renders unconditionally; flagged rows get an inline <code>CSTOR</code> tag next to the RDD name, and every flagged RDD&#39;s recommendation renders below the table, worst-first.</li><li><strong>Core Usage by Locality</strong> (tag <code>LOCAL</code>, <code>CoreUsageArea.tsx</code>): app-level non-local-task-ratio threshold (DETECTORS entry <code>coreLocality</code>; the other half of a wasted-cores-ratio check, the idle-core half already covered by Memory Utilization&#39;s <code>idleCores</code> variant above). Sums <code>RACK_LOCAL</code> + <code>ANY</code> task counts against total task count across every stage&#39;s <code>stage.localityStats</code> (pure reducer <code>computeCoreLocalityRatio</code>, <code>packages/core/src/core-locality-ratio.ts</code>). <code>NO_PREF</code> stays in the denominator only, since it&#39;s what shuffle-read stages legitimately report with no locality problem. Below 50 total tasks or below a 15% non-local ratio: no finding; 15%-35%: warning; &gt;= 35%: critical; these thresholds are unvalidated estimates (<code>confidence</code> scales <code>low</code>/<code>medium</code>/<code>high</code> via <code>coreLocalityConfidence</code>, off whichever is weaker of the non-local ratio and the sampled task count, the same evidence-strength convention as the memory-waste model above). The widget&#39;s always-rendered stacked-area chart (<code>packages/core/src/core-usage-locality.ts</code>) does not depend on the finding. The finding adds a threshold/impact-band section above it, a per-stage non-local breakdown below it (shown whenever any non-local tasks exist at all, independent of whether the aggregate crossed threshold), and a one-line cross-reference to Memory Utilization when its <code>idleCores</code> finding also fired (idle-core ratio itself is not duplicated here).</li></ul><p>Documented ALL-CAPS tag vocabulary: <code>SKEW</code>, <code>SHFL</code>, <code>SPILL</code>, <code>GC</code>, <code>COLD</code>, <code>UTIL</code>, <code>MEM</code> (Memory Utilization), <code>CSTOR</code> (Cache Storage), <code>LOCAL</code> (Core Usage by Locality), plus <code>FAIL</code> (Failed Tasks), <code>JOBS</code> (Job Failures), <code>CFG</code> (Config Audit), <code>PLAN</code> (Plan Advisor), <code>SFAIL</code> (stage failed outright), <code>PART</code> (partition sizing), <code>SLOW</code> (stage overall slowness), <code>SHAPE</code> (stage shape smells), <code>CACHE</code> (caching opportunity), <code>CHRN</code> (autoscaling churn), <code>TINY</code> (tiny tasks), <code>RETRY</code> (retry waste), <code>HOST</code> (slow executor host), <code>STRAG</code> (straggling task), <code>SPEC</code> (speculation waste) and <code>INCMP</code> (incomplete run); <code>packages/core/test/tag-vocabulary.test.js</code> checks the list.</p><p>ETL Phase Attribution (<code>packages/core/src/etl-phases.ts</code> + <code>EtlPhases.tsx</code>), What-If Executor Scaling (<code>packages/core/src/scaling-sim.ts</code> + <code>ScalingSim.tsx</code>, makespan predictions are unvalidated and carry a Model Error indicator), and Compute Efficiency (<code>packages/core/src/efficiency-model.ts</code> + <code>EfficiencyModel.tsx</code>, available vs. used core-hours with <code>wasted-core-hours.ts</code>&#39;s top stages by task core-time, the driver-vs-executor waste split, two theoretical floors, and the right-sizing copy) are main-thread report modules, not <code>DETECTORS</code> entries: descriptive lenses with no impact-band threshold, rendered unconditionally into the Full app report tab rather than participating in the bottleneck catalog. <code>packages/core/src/job-groups.ts</code> (<code>checkConcurrentJobGroups</code>) is likewise a report helper: it flags when concurrent SQL-execution job groups make wall-clock-based estimates unreliable. It feeds the run interpretation&#39;s <code>wallClockReliable</code>; the scaling simulator and the efficiency model read that flag and show a caveat banner rather than suppressing the estimate. The same Full app report grid also holds Evidence Availability and the Core Usage Histogram.</p><h2 id="confidence-metadata" tabindex="-1">Confidence metadata <a class="header-anchor" href="#confidence-metadata" aria-label="Permalink to &quot;Confidence metadata&quot;">​</a></h2><p>Best-effort (non-deterministic) findings carry a <code>confidence</code> + <code>validationRequired</code> marker, rendered by the shared <code>RowStatusCluster</code> (<code>src/view/RowStatusCluster.tsx</code>) when <code>confidence</code> is present and not <code>&#39;high&#39;</code>: a muted &quot;<code>&lt;level&gt;</code> confidence&quot; pill whose accessible tooltip (<code>useAccessibleTooltip</code>: <code>title</code> + <code>aria-describedby</code>, keyboard-focusable) carries <code>validationRequired</code>. Consumers include <code>Spill.tsx</code>, <code>DuplicatePlanSubtree.tsx</code>, <code>SmallFiles.tsx</code>, <code>UnderBroadcast.tsx</code>, <code>OverBroadcast.tsx</code>, <code>CachingOpportunity.tsx</code>, <code>PlanView.tsx</code> and <code>EfficiencyModel.tsx</code>. Spill classification is <code>medium</code> when classified (skew/volume) and <code>low</code> when unclassified; plan-summary warnings are <code>low</code>. Deterministic detectors stay unmarked, treated as high confidence.</p><h2 id="visual-system" tabindex="-1">Visual system <a class="header-anchor" href="#visual-system" aria-label="Permalink to &quot;Visual system&quot;">​</a></h2><p>Tailwind CSS v4 + shadcn/ui (Base UI primitives). The design tokens (colors, including the telemetry-console dark palette) are defined as CSS variables in an <code>@theme inline</code> block and consumed via Tailwind utility classes; there is no hand-authored BEM CSS. Theme polarity: dark = bare <code>:root</code> (no attribute), light = <code>:root[data-theme=&quot;light&quot;]</code>, toggled by <code>src/theme/ThemeProvider.tsx</code> and persisted to <code>localStorage</code>. <code>index.html</code>&#39;s inline FOUC-prevention script runs before React mounts. Mono is the typeface for numerics (<code>--font-mono</code>), applied via a <code>.num</code>-equivalent Tailwind utility per call site rather than one global class. Charts are Recharts (<code>src/view/charts/ChartTheme.tsx</code>&#39;s <code>CHART_COLORS</code>, read from the same CSS tokens). React re-renders on theme toggle, so chart colors update live with the theme. <code>ChartFrame</code> can also expose the underlying rows through a toggleable accessible table and copy them to the clipboard as TSV.</p><h2 id="templating-xss-safe" tabindex="-1">Templating (XSS-safe) <a class="header-anchor" href="#templating-xss-safe" aria-label="Permalink to &quot;Templating (XSS-safe)&quot;">​</a></h2><p>JSX auto-escapes every interpolated value by default. No <code>.innerHTML</code> assignment exists anywhere in the view layer (the one <code>dangerouslySetInnerHTML</code>, shadcn&#39;s <code>ChartStyle</code> in <code>src/components/ui/chart.tsx</code>, only writes CSS variables from the static chart config).</p><p>One Base UI-specific gotcha: <code>WidgetCard.tsx</code> passes <code>aria-expanded={String(open) as &#39;true&#39; | &#39;false&#39;}</code> rather than the raw boolean, because Base UI&#39;s <code>Collapsible.Trigger</code> otherwise overrides a boolean <code>aria-expanded</code> prop with its own internal state via prop merging. The explicit <code>String()</code> cast keeps the rendered attribute in sync with this app&#39;s own <code>open</code> state.</p></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/contributor-guide/architecture/widget-rendering.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>Widget rendering</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="/docs/contributor-guide/architecture/drill-down.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>Drill-down</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
27
+ <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"D0Kkk_r5\",\"contributor-guide_architecture_detector-contract.md\":\"caA46z-u\",\"contributor-guide_architecture_drill-down.md\":\"CdtlKHl-\",\"contributor-guide_architecture_impact-estimation.md\":\"BNp14LtL\",\"contributor-guide_architecture_index.md\":\"B9yZieYL\",\"contributor-guide_architecture_overview.md\":\"DBGo-bwn\",\"contributor-guide_architecture_state-and-history.md\":\"w4lwybdh\",\"contributor-guide_architecture_widget-rendering.md\":\"CcvqxnZI\",\"contributor-guide_architecture_worker-protocol.md\":\"CKeFttKj\",\"contributor-guide_contributing.md\":\"CQMvGwFt\",\"contributor-guide_development-setup.md\":\"D9bu4j_c\",\"contributor-guide_testing.md\":\"By6XGR9a\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"D08Uaod7\",\"tuning-reference_aqe.md\":\"C1Ae6A8u\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"6wjKuJNE\",\"tuning-reference_bottleneck-cold-start.md\":\"BVQbsuHP\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"DtaYnaiF\",\"tuning-reference_bottleneck-failures.md\":\"Z1Tdmdd3\",\"tuning-reference_bottleneck-gc.md\":\"BCnXLf1n\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"CcvQj2g9\",\"tuning-reference_bottleneck-memory-utilization.md\":\"B-C0-8h-\",\"tuning-reference_bottleneck-retry-waste.md\":\"DCnhWiqj\",\"tuning-reference_bottleneck-shuffle.md\":\"B5OMJCd3\",\"tuning-reference_bottleneck-skew.md\":\"Cvo6NQm_\",\"tuning-reference_bottleneck-slow-host.md\":\"ja5qr3wq\",\"tuning-reference_bottleneck-small-files.md\":\"Crjtb0i2\",\"tuning-reference_bottleneck-spill.md\":\"Bl8KtSRg\",\"tuning-reference_bottleneck-straggler.md\":\"B0VN8reT\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"DQCftGhu\",\"tuning-reference_bottleneck-utilization.md\":\"BXDl30f9\",\"tuning-reference_caching.md\":\"CcaiMn_7\",\"tuning-reference_cluster-config.md\":\"DbWG1yZq\",\"tuning-reference_config.md\":\"YUqSpwFB\",\"tuning-reference_data-formats.md\":\"C5DX5xL4\",\"tuning-reference_index.md\":\"ZVIpQ6Dk\",\"tuning-reference_intro.md\":\"BSSaDlqx\",\"tuning-reference_joins.md\":\"BVhNCDnE\",\"tuning-reference_memory-model.md\":\"CfjFdgbl\",\"tuning-reference_metrics.md\":\"C64VNEL4\",\"tuning-reference_partitioning.md\":\"CtLK0xsS\",\"tuning-reference_pyspark.md\":\"D-92XgF8\",\"tuning-reference_shuffle.md\":\"Iq2v1oHJ\",\"tuning-reference_spark-architecture.md\":\"_qwnE_Sc\",\"tuning-reference_table-formats.md\":\"BSYEK-eN\",\"user-guide_alternative-log-retrieval.md\":\"z-8npxNk\",\"user-guide_getting-started.md\":\"BJW5oVhe\",\"user-guide_mcp-tools.md\":\"DcLiOa9t\",\"user-guide_run-comparison.md\":\"qin0a4IX\",\"user-guide_understanding-findings.md\":\"DzReyCXb\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Finding your event log\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":{\"selector\":\"[data-sf-anchor-offset]\",\"padding\":0},\"cleanUrls\":false}");</script>
27
28
 
28
29
  </body>
29
30
  </html>