sparkforensics-server 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (382) hide show
  1. package/index.js +5 -4
  2. package/package.json +1 -1
  3. package/public/assets/AutoscalingChurn-CUSIvvab.js +1 -0
  4. package/public/assets/CacheUtilization-Ce0K6_X2.js +1 -0
  5. package/public/assets/CachingOpportunity-Cn-LmRA4.js +1 -0
  6. package/public/assets/ColdStart-BoiYzEyV.js +1 -0
  7. package/public/assets/ConfigAudit-BoJc6PAw.js +1 -0
  8. package/public/assets/CoreUsageArea-BFfXKu7K.js +1 -0
  9. package/public/assets/Dashboard-Bo_1jGqH.js +2 -0
  10. package/public/assets/DisclosureContext-LYzHroHO.js +1 -0
  11. package/public/assets/DuplicatePlanSubtree-BaAuGAoo.js +1 -0
  12. package/public/assets/ExecutorUtilization-OE6VMFkG.js +1 -0
  13. package/public/assets/GcPressure-AoL_ZNrV.js +1 -0
  14. package/public/assets/IncompleteRun-Bv6IzgsQ.js +1 -0
  15. package/public/assets/JobFailures-DF3U2z1f.js +1 -0
  16. package/public/assets/MemoryUtilization-B65OVZlq.js +1 -0
  17. package/public/assets/OverBroadcast-DXbJREDl.js +1 -0
  18. package/public/assets/PartitionSizing-DZLnjy0d.js +1 -0
  19. package/public/assets/PlanExplorer-DdmPkKO1.js +1 -0
  20. package/public/assets/PlanGraphRoute-C-BGmC6j.js +1 -0
  21. package/public/assets/RetryWaste-BbEPsdMw.js +1 -0
  22. package/public/assets/ShuffleIO-wZB94LS8.js +1 -0
  23. package/public/assets/Skew-SpQE7zeJ.js +1 -0
  24. package/public/assets/SlowHost-DwDBCTvr.js +1 -0
  25. package/public/assets/SmallFiles-yjY_6sT1.js +1 -0
  26. package/public/assets/SpeculationWaste-73PDmj0c.js +1 -0
  27. package/public/assets/Spill-DEw0Kd1i.js +1 -0
  28. package/public/assets/StageFailed-BEfXdEJc.js +1 -0
  29. package/public/assets/StageFindingGroup-Bzm2TY03.js +1 -0
  30. package/public/assets/StageHeader-BSV8ALhT.js +1 -0
  31. package/public/assets/StageShape-CzR4jHOt.js +1 -0
  32. package/public/assets/StageSlowness-vMPsnAaa.js +1 -0
  33. package/public/assets/Straggler-Cf__OGl5.js +1 -0
  34. package/public/assets/TaskFailures-BZ8kLZiO.js +1 -0
  35. package/public/assets/{TinyTask-Be-pvTjJ.js → TinyTask-CIPRxWfT.js} +1 -1
  36. package/public/assets/UnderBroadcast-CBm8ZPoS.js +1 -0
  37. package/public/assets/{finding-anchor-vVODA_VC.js → finding-anchor-DF1cSHO1.js} +1 -1
  38. package/public/assets/impact-sort-7z0ZcBZf.js +1 -0
  39. package/public/assets/index-BcqTLKFp.css +2 -0
  40. package/public/assets/index-CEM9bz_x.js +13 -0
  41. package/public/assets/{parser-worker-jAP0kWvS.js → parser-worker-Q5KCPdKT.js} +9 -9
  42. package/public/assets/react-vendor-BAKt3vny.js +9 -0
  43. package/public/assets/select-Cy5pmbrh.js +1 -0
  44. package/public/assets/usePagedRows-BjPOpUU8.js +1 -0
  45. package/public/assets/useRecentFiles-BrBl-AP1.js +11 -0
  46. package/public/assets/useSortMode-CU0vzThu.js +1 -0
  47. package/public/docs/404.html +2 -2
  48. package/public/docs/assets/{app.C5E-wy6e.js → app.C6w47a9W.js} +1 -1
  49. package/public/docs/assets/chunks/@localSearchIndexroot.DbZy5EtM.js +1 -0
  50. package/public/docs/assets/chunks/{VPLocalSearchBox.DtyygpbL.js → VPLocalSearchBox.BcTNvT55.js} +1 -1
  51. package/public/docs/assets/chunks/{theme.DQS9-w_Z.js → theme.D-mE6j02.js} +2 -2
  52. package/public/docs/assets/{contributor-guide_architecture_board-widgets.md.9ToZXcTk.js → contributor-guide_architecture_board-widgets.md.VTcYTH4d.js} +1 -1
  53. package/public/docs/assets/contributor-guide_architecture_detector-contract.md.CsqZHz_T.js +1 -0
  54. package/public/docs/assets/{contributor-guide_architecture_detector-contract.md.CsCytCho.lean.js → contributor-guide_architecture_detector-contract.md.CsqZHz_T.lean.js} +1 -1
  55. package/public/docs/assets/contributor-guide_architecture_drill-down.md.CFNCE8JB.js +1 -0
  56. package/public/docs/assets/{contributor-guide_architecture_drill-down.md.Bawx6NgP.lean.js → contributor-guide_architecture_drill-down.md.CFNCE8JB.lean.js} +1 -1
  57. package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.HT9IJGIl.js +1 -0
  58. package/public/docs/assets/{contributor-guide_architecture_impact-estimation.md.sf9DU2bA.lean.js → contributor-guide_architecture_impact-estimation.md.HT9IJGIl.lean.js} +1 -1
  59. package/public/docs/assets/contributor-guide_architecture_overview.md.CTEqVeV4.js +1 -0
  60. package/public/docs/assets/contributor-guide_architecture_overview.md.CTEqVeV4.lean.js +1 -0
  61. package/public/docs/assets/contributor-guide_architecture_state-and-history.md.DOTVHjTV.js +1 -0
  62. package/public/docs/assets/{contributor-guide_architecture_state-and-history.md.B8Wc_ZTg.lean.js → contributor-guide_architecture_state-and-history.md.DOTVHjTV.lean.js} +1 -1
  63. package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.BmpARZve.js +1 -0
  64. package/public/docs/assets/{contributor-guide_architecture_widget-rendering.md.DYwFeyjV.lean.js → contributor-guide_architecture_widget-rendering.md.BmpARZve.lean.js} +1 -1
  65. package/public/docs/assets/contributor-guide_architecture_worker-protocol.md.21pTYhFF.js +6 -0
  66. package/public/docs/assets/contributor-guide_architecture_worker-protocol.md.21pTYhFF.lean.js +1 -0
  67. package/public/docs/assets/contributor-guide_development-setup.md.CjgZJwjK.js +15 -0
  68. package/public/docs/assets/contributor-guide_development-setup.md.CjgZJwjK.lean.js +1 -0
  69. package/public/docs/assets/contributor-guide_testing.md.DTwhpoN2.js +1 -0
  70. package/public/docs/assets/{contributor-guide_testing.md.Cnn6gz1C.lean.js → contributor-guide_testing.md.DTwhpoN2.lean.js} +1 -1
  71. package/public/docs/assets/tuning-reference_anti-patterns.md.D08Uaod7.js +1 -0
  72. package/public/docs/assets/tuning-reference_anti-patterns.md.D08Uaod7.lean.js +1 -0
  73. package/public/docs/assets/tuning-reference_aqe.md.C1Ae6A8u.js +1 -0
  74. package/public/docs/assets/tuning-reference_aqe.md.C1Ae6A8u.lean.js +1 -0
  75. package/public/docs/assets/tuning-reference_bottleneck-broadcast-sizing.md.6wjKuJNE.js +1 -0
  76. package/public/docs/assets/tuning-reference_bottleneck-broadcast-sizing.md.6wjKuJNE.lean.js +1 -0
  77. package/public/docs/assets/tuning-reference_bottleneck-cold-start.md.BVQbsuHP.js +7 -0
  78. package/public/docs/assets/tuning-reference_bottleneck-cold-start.md.BVQbsuHP.lean.js +1 -0
  79. package/public/docs/assets/tuning-reference_bottleneck-duplicate-plan-subtree.md.DtaYnaiF.js +1 -0
  80. package/public/docs/assets/tuning-reference_bottleneck-duplicate-plan-subtree.md.DtaYnaiF.lean.js +1 -0
  81. package/public/docs/assets/{tuning-reference_bottleneck-failures.md.CAsFf3RR.js → tuning-reference_bottleneck-failures.md.Z1Tdmdd3.js} +2 -2
  82. package/public/docs/assets/tuning-reference_bottleneck-failures.md.Z1Tdmdd3.lean.js +1 -0
  83. package/public/docs/assets/tuning-reference_bottleneck-gc.md.BCnXLf1n.js +6 -0
  84. package/public/docs/assets/tuning-reference_bottleneck-gc.md.BCnXLf1n.lean.js +1 -0
  85. package/public/docs/assets/tuning-reference_bottleneck-job-failure-rate.md.CcvQj2g9.js +8 -0
  86. package/public/docs/assets/tuning-reference_bottleneck-job-failure-rate.md.CcvQj2g9.lean.js +1 -0
  87. package/public/docs/assets/tuning-reference_bottleneck-memory-utilization.md.B-C0-8h-.js +7 -0
  88. package/public/docs/assets/tuning-reference_bottleneck-memory-utilization.md.B-C0-8h-.lean.js +1 -0
  89. package/public/docs/assets/tuning-reference_bottleneck-retry-waste.md.DCnhWiqj.js +1 -0
  90. package/public/docs/assets/tuning-reference_bottleneck-retry-waste.md.DCnhWiqj.lean.js +1 -0
  91. package/public/docs/assets/{tuning-reference_bottleneck-shuffle.md.DDMQVOW6.js → tuning-reference_bottleneck-shuffle.md.B5OMJCd3.js} +2 -2
  92. package/public/docs/assets/tuning-reference_bottleneck-shuffle.md.B5OMJCd3.lean.js +1 -0
  93. package/public/docs/assets/{tuning-reference_bottleneck-skew.md.B0MzhWcX.js → tuning-reference_bottleneck-skew.md.Cvo6NQm_.js} +1 -1
  94. package/public/docs/assets/tuning-reference_bottleneck-skew.md.Cvo6NQm_.lean.js +1 -0
  95. package/public/docs/assets/{tuning-reference_bottleneck-slow-host.md.BlIo6UDW.js → tuning-reference_bottleneck-slow-host.md.ja5qr3wq.js} +1 -1
  96. package/public/docs/assets/tuning-reference_bottleneck-slow-host.md.ja5qr3wq.lean.js +1 -0
  97. package/public/docs/assets/tuning-reference_bottleneck-small-files.md.Crjtb0i2.js +5 -0
  98. package/public/docs/assets/tuning-reference_bottleneck-small-files.md.Crjtb0i2.lean.js +1 -0
  99. package/public/docs/assets/tuning-reference_bottleneck-spill.md.Bl8KtSRg.js +6 -0
  100. package/public/docs/assets/tuning-reference_bottleneck-spill.md.Bl8KtSRg.lean.js +1 -0
  101. package/public/docs/assets/tuning-reference_bottleneck-straggler.md.B0VN8reT.js +7 -0
  102. package/public/docs/assets/tuning-reference_bottleneck-straggler.md.B0VN8reT.lean.js +1 -0
  103. package/public/docs/assets/tuning-reference_bottleneck-tiny-tasks.md.DQCftGhu.js +8 -0
  104. package/public/docs/assets/tuning-reference_bottleneck-tiny-tasks.md.DQCftGhu.lean.js +1 -0
  105. package/public/docs/assets/{tuning-reference_bottleneck-utilization.md.DqC_iGkw.js → tuning-reference_bottleneck-utilization.md.BXDl30f9.js} +1 -1
  106. package/public/docs/assets/tuning-reference_bottleneck-utilization.md.BXDl30f9.lean.js +1 -0
  107. package/public/docs/assets/tuning-reference_caching.md.CcaiMn_7.js +1 -0
  108. package/public/docs/assets/tuning-reference_caching.md.CcaiMn_7.lean.js +1 -0
  109. package/public/docs/assets/tuning-reference_cluster-config.md.DbWG1yZq.js +1 -0
  110. package/public/docs/assets/tuning-reference_cluster-config.md.DbWG1yZq.lean.js +1 -0
  111. package/public/docs/assets/tuning-reference_config.md.YUqSpwFB.js +1 -0
  112. package/public/docs/assets/tuning-reference_config.md.YUqSpwFB.lean.js +1 -0
  113. package/public/docs/assets/tuning-reference_data-formats.md.C5DX5xL4.js +1 -0
  114. package/public/docs/assets/tuning-reference_data-formats.md.C5DX5xL4.lean.js +1 -0
  115. package/public/docs/assets/tuning-reference_index.md.ZVIpQ6Dk.js +1 -0
  116. package/public/docs/assets/tuning-reference_index.md.ZVIpQ6Dk.lean.js +1 -0
  117. package/public/docs/assets/tuning-reference_intro.md.BSSaDlqx.js +1 -0
  118. package/public/docs/assets/tuning-reference_intro.md.BSSaDlqx.lean.js +1 -0
  119. package/public/docs/assets/tuning-reference_joins.md.BVhNCDnE.js +1 -0
  120. package/public/docs/assets/tuning-reference_joins.md.BVhNCDnE.lean.js +1 -0
  121. package/public/docs/assets/tuning-reference_memory-model.md.CfjFdgbl.js +1 -0
  122. package/public/docs/assets/tuning-reference_memory-model.md.CfjFdgbl.lean.js +1 -0
  123. package/public/docs/assets/tuning-reference_metrics.md.C64VNEL4.js +1 -0
  124. package/public/docs/assets/tuning-reference_metrics.md.C64VNEL4.lean.js +1 -0
  125. package/public/docs/assets/tuning-reference_partitioning.md.CtLK0xsS.js +1 -0
  126. package/public/docs/assets/tuning-reference_partitioning.md.CtLK0xsS.lean.js +1 -0
  127. package/public/docs/assets/tuning-reference_pyspark.md.D-92XgF8.js +6 -0
  128. package/public/docs/assets/tuning-reference_pyspark.md.D-92XgF8.lean.js +1 -0
  129. package/public/docs/assets/tuning-reference_shuffle.md.Iq2v1oHJ.js +1 -0
  130. package/public/docs/assets/tuning-reference_shuffle.md.Iq2v1oHJ.lean.js +1 -0
  131. package/public/docs/assets/tuning-reference_spark-architecture.md._qwnE_Sc.js +1 -0
  132. package/public/docs/assets/tuning-reference_spark-architecture.md._qwnE_Sc.lean.js +1 -0
  133. package/public/docs/assets/tuning-reference_table-formats.md.BSYEK-eN.js +1 -0
  134. package/public/docs/assets/tuning-reference_table-formats.md.BSYEK-eN.lean.js +1 -0
  135. package/public/docs/assets/user-guide_getting-started.md.CEa2ZX-l.js +6 -0
  136. package/public/docs/assets/user-guide_getting-started.md.CEa2ZX-l.lean.js +1 -0
  137. package/public/docs/assets/{user-guide_mcp-tools.md.DrphooJv.js → user-guide_mcp-tools.md.dQwIvdTF.js} +13 -7
  138. package/public/docs/assets/user-guide_mcp-tools.md.dQwIvdTF.lean.js +1 -0
  139. package/public/docs/assets/user-guide_run-comparison.md.ChLI3MK0.js +1 -0
  140. package/public/docs/assets/user-guide_run-comparison.md.ChLI3MK0.lean.js +1 -0
  141. package/public/docs/assets/{user-guide_understanding-findings.md.YBBLOwWi.js → user-guide_understanding-findings.md.D1ccs60C.js} +1 -1
  142. package/public/docs/assets/{user-guide_understanding-findings.md.YBBLOwWi.lean.js → user-guide_understanding-findings.md.D1ccs60C.lean.js} +1 -1
  143. package/public/docs/contributor-guide/architecture/board-widgets.html +5 -5
  144. package/public/docs/contributor-guide/architecture/detector-contract.html +5 -5
  145. package/public/docs/contributor-guide/architecture/drill-down.html +5 -5
  146. package/public/docs/contributor-guide/architecture/impact-estimation.html +5 -5
  147. package/public/docs/contributor-guide/architecture/index.html +3 -3
  148. package/public/docs/contributor-guide/architecture/overview.html +5 -5
  149. package/public/docs/contributor-guide/architecture/state-and-history.html +5 -5
  150. package/public/docs/contributor-guide/architecture/widget-rendering.html +5 -5
  151. package/public/docs/contributor-guide/architecture/worker-protocol.html +6 -6
  152. package/public/docs/contributor-guide/contributing.html +3 -3
  153. package/public/docs/contributor-guide/development-setup.html +7 -7
  154. package/public/docs/contributor-guide/testing.html +5 -5
  155. package/public/docs/hashmap.json +1 -1
  156. package/public/docs/index.html +3 -3
  157. package/public/docs/tuning-reference/anti-patterns.html +6 -6
  158. package/public/docs/tuning-reference/aqe.html +6 -6
  159. package/public/docs/tuning-reference/bottleneck-broadcast-sizing.html +6 -6
  160. package/public/docs/tuning-reference/bottleneck-cold-start.html +6 -6
  161. package/public/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html +6 -6
  162. package/public/docs/tuning-reference/bottleneck-failures.html +6 -6
  163. package/public/docs/tuning-reference/bottleneck-gc.html +6 -6
  164. package/public/docs/tuning-reference/bottleneck-job-failure-rate.html +6 -6
  165. package/public/docs/tuning-reference/bottleneck-memory-utilization.html +6 -6
  166. package/public/docs/tuning-reference/bottleneck-retry-waste.html +6 -6
  167. package/public/docs/tuning-reference/bottleneck-shuffle.html +6 -6
  168. package/public/docs/tuning-reference/bottleneck-skew.html +6 -6
  169. package/public/docs/tuning-reference/bottleneck-slow-host.html +6 -6
  170. package/public/docs/tuning-reference/bottleneck-small-files.html +6 -6
  171. package/public/docs/tuning-reference/bottleneck-spill.html +6 -6
  172. package/public/docs/tuning-reference/bottleneck-straggler.html +6 -6
  173. package/public/docs/tuning-reference/bottleneck-tiny-tasks.html +6 -6
  174. package/public/docs/tuning-reference/bottleneck-utilization.html +6 -6
  175. package/public/docs/tuning-reference/caching.html +6 -6
  176. package/public/docs/tuning-reference/cluster-config.html +6 -6
  177. package/public/docs/tuning-reference/config.html +6 -6
  178. package/public/docs/tuning-reference/data-formats.html +6 -6
  179. package/public/docs/tuning-reference/index.html +6 -6
  180. package/public/docs/tuning-reference/intro.html +6 -6
  181. package/public/docs/tuning-reference/joins.html +6 -6
  182. package/public/docs/tuning-reference/memory-model.html +6 -6
  183. package/public/docs/tuning-reference/metrics.html +6 -6
  184. package/public/docs/tuning-reference/partitioning.html +6 -6
  185. package/public/docs/tuning-reference/pyspark.html +6 -6
  186. package/public/docs/tuning-reference/shuffle.html +6 -6
  187. package/public/docs/tuning-reference/spark-architecture.html +6 -6
  188. package/public/docs/tuning-reference/table-formats.html +6 -6
  189. package/public/docs/user-guide/alternative-log-retrieval.html +3 -3
  190. package/public/docs/user-guide/getting-started.html +9 -6
  191. package/public/docs/user-guide/mcp-tools.html +15 -9
  192. package/public/docs/user-guide/run-comparison.html +5 -5
  193. package/public/docs/user-guide/understanding-findings.html +5 -5
  194. package/public/export-template.html +111 -0
  195. package/public/index.html +4 -3
  196. package/vendor-core/analyzer.js +156 -48
  197. package/vendor-core/check-coverage.js +88 -0
  198. package/vendor-core/cli/budgets.js +31 -18
  199. package/vendor-core/cli/collect-run.js +76 -31
  200. package/vendor-core/cli/threshold-config.js +28 -0
  201. package/vendor-core/comparison-verdict.js +177 -0
  202. package/vendor-core/core-source-hash.txt +1 -0
  203. package/vendor-core/core-usage-locality.js +56 -2
  204. package/vendor-core/detector-docs.js +58 -0
  205. package/vendor-core/detectors.js +918 -458
  206. package/vendor-core/docs-config.js +0 -36
  207. package/vendor-core/docs-content/chapters/nav-index.json +31 -0
  208. package/vendor-core/docs-content/detection/cstor.md +9 -0
  209. package/vendor-core/docs-site-config.js +3 -0
  210. package/vendor-core/event-handlers.js +170 -6
  211. package/vendor-core/event-schemas.js +21 -0
  212. package/vendor-core/evidence-report.js +421 -112
  213. package/vendor-core/export-data.js +79 -6
  214. package/vendor-core/finding-action-label.js +9 -88
  215. package/vendor-core/finding-filter-predicate.js +9 -0
  216. package/vendor-core/finding-generic-recommendation.js +6 -104
  217. package/vendor-core/finding-names.js +21 -45
  218. package/vendor-core/finding-presentation.js +333 -0
  219. package/vendor-core/finding-tag-help.js +110 -0
  220. package/vendor-core/finding-types.js +361 -0
  221. package/vendor-core/findings-of-type.js +11 -0
  222. package/vendor-core/format-utils.js +92 -27
  223. package/vendor-core/html-export.js +51 -0
  224. package/vendor-core/impact-band.js +21 -8
  225. package/vendor-core/impact-estimator.js +8 -521
  226. package/vendor-core/impact-format.js +114 -0
  227. package/vendor-core/impact-model.js +175 -0
  228. package/vendor-core/ingest.js +2 -0
  229. package/vendor-core/intervals.js +13 -0
  230. package/vendor-core/list-runs.js +2 -3
  231. package/vendor-core/load-vendored.js +70 -5
  232. package/vendor-core/mcp-server-factory.js +14 -10
  233. package/vendor-core/mcp-tools.js +105 -45
  234. package/vendor-core/model-assembler.js +12 -0
  235. package/vendor-core/occupancy.js +1 -1
  236. package/vendor-core/parser-worker.js +1 -1
  237. package/vendor-core/plan-graph-model.js +3 -2
  238. package/vendor-core/plan-node-detail.js +1 -1
  239. package/vendor-core/recommendation-rollup.js +63 -3
  240. package/vendor-core/redact.js +45 -27
  241. package/vendor-core/run-comparison.js +32 -7
  242. package/vendor-core/run-interpretation.js +290 -0
  243. package/vendor-core/run-outcome.js +74 -0
  244. package/vendor-core/run-payload.js +17 -0
  245. package/vendor-core/run-shape.js +40 -0
  246. package/vendor-core/run-verdict.js +353 -0
  247. package/vendor-core/scaling-sim.js +4 -5
  248. package/vendor-core/scorecard-estimates.js +62 -0
  249. package/vendor-core/sql-stages.js +11 -0
  250. package/vendor-core/stage-quantiles.js +2 -0
  251. package/vendor-core/threshold-overrides.js +160 -0
  252. package/vendor-core/threshold-summary.js +11 -33
  253. package/vendor-core/types.js +6 -42
  254. package/vendor-core/wall-clock.js +1 -12
  255. package/vendor-core/wasted-core-hours.js +2 -2
  256. package/public/assets/AutoscalingChurn-lxZHWIwq.js +0 -1
  257. package/public/assets/CacheUtilization-DuAJ-Lve.js +0 -1
  258. package/public/assets/CachingOpportunity-DC1t7o2_.js +0 -1
  259. package/public/assets/ColdStart-BOYXeiUZ.js +0 -1
  260. package/public/assets/ConfigAudit-MCPrJomC.js +0 -1
  261. package/public/assets/CoreUsageArea-ChNuqGA0.js +0 -1
  262. package/public/assets/Dashboard-VuyUGvZx.js +0 -1
  263. package/public/assets/DuplicatePlanSubtree-JZut9xwq.js +0 -1
  264. package/public/assets/DurationHistogram-_7C1w0l4.js +0 -1
  265. package/public/assets/EmptyState-D_ulfIIi.js +0 -1
  266. package/public/assets/ExecutorUtilization-B08E76jT.js +0 -1
  267. package/public/assets/GcPressure-BZMi2fTx.js +0 -1
  268. package/public/assets/ImpactEstimate-36WxuPdV.js +0 -1
  269. package/public/assets/IncompleteRun-t8JU7HRt.js +0 -1
  270. package/public/assets/JobFailures-CguB1bec.js +0 -1
  271. package/public/assets/MemoryUtilization-DDk8lz9-.js +0 -1
  272. package/public/assets/OverBroadcast-BHXLt6s7.js +0 -1
  273. package/public/assets/PartitionSizing-COj3uwV-.js +0 -1
  274. package/public/assets/PlanExplorer-CnCdEY_i.js +0 -1
  275. package/public/assets/PlanGraphRoute-BesZOW1n.js +0 -1
  276. package/public/assets/RetryWaste-CcU4ctAn.js +0 -1
  277. package/public/assets/RowPagination-D2aIcfGv.js +0 -1
  278. package/public/assets/ShuffleIO-NPHkWbg1.js +0 -1
  279. package/public/assets/Skew-CzztP-C4.js +0 -1
  280. package/public/assets/SlowHost-Dh2KHypN.js +0 -1
  281. package/public/assets/SmallFiles-CgYSuZQw.js +0 -1
  282. package/public/assets/SpeculationWaste-C8J5z1WS.js +0 -1
  283. package/public/assets/Spill-DwTQPZ_L.js +0 -1
  284. package/public/assets/StageFailed-aFeIEh3p.js +0 -1
  285. package/public/assets/StageFindingGroup-DQmAE3uk.js +0 -1
  286. package/public/assets/StageHeader-D6GBbcZp.js +0 -1
  287. package/public/assets/StagePill-CSKyM8Kd.js +0 -1
  288. package/public/assets/StageShape-J6U4cUcM.js +0 -1
  289. package/public/assets/StageSlowness-C5F2v_ZO.js +0 -1
  290. package/public/assets/Straggler-DcP3dFGA.js +0 -1
  291. package/public/assets/TaskFailures-C-vIEheo.js +0 -1
  292. package/public/assets/UnderBroadcast-DfjYGhoK.js +0 -1
  293. package/public/assets/WidgetLeadSummary-BdUUi7L5.js +0 -1
  294. package/public/assets/downsample-WmNM1qc9.js +0 -1
  295. package/public/assets/impact-sort-DxaUheWV.js +0 -1
  296. package/public/assets/inbox-LtbS_kBS.js +0 -1
  297. package/public/assets/index-27MbqTFr.js +0 -11
  298. package/public/assets/index-CWjjbtyo.css +0 -2
  299. package/public/assets/react-vendor-D2TLlSzI.js +0 -9
  300. package/public/assets/select-CC2Lyrmi.js +0 -1
  301. package/public/assets/toggle-group-lmgO21t9.js +0 -1
  302. package/public/assets/usePagedRows-jNUWgVTD.js +0 -1
  303. package/public/assets/useRecentFiles-qH5Ap4xm.js +0 -12
  304. package/public/assets/useSortMode-CvPY99UV.js +0 -1
  305. package/public/docs/assets/chunks/@localSearchIndexroot.CJBoRv1b.js +0 -1
  306. package/public/docs/assets/contributor-guide_architecture_detector-contract.md.CsCytCho.js +0 -1
  307. package/public/docs/assets/contributor-guide_architecture_drill-down.md.Bawx6NgP.js +0 -1
  308. package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.sf9DU2bA.js +0 -1
  309. package/public/docs/assets/contributor-guide_architecture_overview.md.rhfxoGsI.js +0 -1
  310. package/public/docs/assets/contributor-guide_architecture_overview.md.rhfxoGsI.lean.js +0 -1
  311. package/public/docs/assets/contributor-guide_architecture_state-and-history.md.B8Wc_ZTg.js +0 -1
  312. package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.DYwFeyjV.js +0 -1
  313. package/public/docs/assets/contributor-guide_architecture_worker-protocol.md.CBJkwH24.js +0 -6
  314. package/public/docs/assets/contributor-guide_architecture_worker-protocol.md.CBJkwH24.lean.js +0 -1
  315. package/public/docs/assets/contributor-guide_development-setup.md.s85j2l8b.js +0 -15
  316. package/public/docs/assets/contributor-guide_development-setup.md.s85j2l8b.lean.js +0 -1
  317. package/public/docs/assets/contributor-guide_testing.md.Cnn6gz1C.js +0 -1
  318. package/public/docs/assets/tuning-reference_anti-patterns.md.Df1YMIHu.js +0 -1
  319. package/public/docs/assets/tuning-reference_anti-patterns.md.Df1YMIHu.lean.js +0 -1
  320. package/public/docs/assets/tuning-reference_aqe.md.BIsCtLzm.js +0 -1
  321. package/public/docs/assets/tuning-reference_aqe.md.BIsCtLzm.lean.js +0 -1
  322. package/public/docs/assets/tuning-reference_bottleneck-broadcast-sizing.md.CEstB3Ia.js +0 -1
  323. package/public/docs/assets/tuning-reference_bottleneck-broadcast-sizing.md.CEstB3Ia.lean.js +0 -1
  324. package/public/docs/assets/tuning-reference_bottleneck-cold-start.md.CEuy-72y.js +0 -7
  325. package/public/docs/assets/tuning-reference_bottleneck-cold-start.md.CEuy-72y.lean.js +0 -1
  326. package/public/docs/assets/tuning-reference_bottleneck-duplicate-plan-subtree.md.CIohQDfn.js +0 -1
  327. package/public/docs/assets/tuning-reference_bottleneck-duplicate-plan-subtree.md.CIohQDfn.lean.js +0 -1
  328. package/public/docs/assets/tuning-reference_bottleneck-failures.md.CAsFf3RR.lean.js +0 -1
  329. package/public/docs/assets/tuning-reference_bottleneck-gc.md.B4T3kJYV.js +0 -6
  330. package/public/docs/assets/tuning-reference_bottleneck-gc.md.B4T3kJYV.lean.js +0 -1
  331. package/public/docs/assets/tuning-reference_bottleneck-job-failure-rate.md.BaJl__1W.js +0 -8
  332. package/public/docs/assets/tuning-reference_bottleneck-job-failure-rate.md.BaJl__1W.lean.js +0 -1
  333. package/public/docs/assets/tuning-reference_bottleneck-memory-utilization.md.DbP-SJZc.js +0 -7
  334. package/public/docs/assets/tuning-reference_bottleneck-memory-utilization.md.DbP-SJZc.lean.js +0 -1
  335. package/public/docs/assets/tuning-reference_bottleneck-retry-waste.md.D5JMjVOt.js +0 -1
  336. package/public/docs/assets/tuning-reference_bottleneck-retry-waste.md.D5JMjVOt.lean.js +0 -1
  337. package/public/docs/assets/tuning-reference_bottleneck-shuffle.md.DDMQVOW6.lean.js +0 -1
  338. package/public/docs/assets/tuning-reference_bottleneck-skew.md.B0MzhWcX.lean.js +0 -1
  339. package/public/docs/assets/tuning-reference_bottleneck-slow-host.md.BlIo6UDW.lean.js +0 -1
  340. package/public/docs/assets/tuning-reference_bottleneck-small-files.md.DHGMTmPs.js +0 -5
  341. package/public/docs/assets/tuning-reference_bottleneck-small-files.md.DHGMTmPs.lean.js +0 -1
  342. package/public/docs/assets/tuning-reference_bottleneck-spill.md.PNH7mITt.js +0 -6
  343. package/public/docs/assets/tuning-reference_bottleneck-spill.md.PNH7mITt.lean.js +0 -1
  344. package/public/docs/assets/tuning-reference_bottleneck-straggler.md.DIbXdAcQ.js +0 -7
  345. package/public/docs/assets/tuning-reference_bottleneck-straggler.md.DIbXdAcQ.lean.js +0 -1
  346. package/public/docs/assets/tuning-reference_bottleneck-tiny-tasks.md.CB05v3zh.js +0 -8
  347. package/public/docs/assets/tuning-reference_bottleneck-tiny-tasks.md.CB05v3zh.lean.js +0 -1
  348. package/public/docs/assets/tuning-reference_bottleneck-utilization.md.DqC_iGkw.lean.js +0 -1
  349. package/public/docs/assets/tuning-reference_caching.md.B7aQ8asB.js +0 -1
  350. package/public/docs/assets/tuning-reference_caching.md.B7aQ8asB.lean.js +0 -1
  351. package/public/docs/assets/tuning-reference_cluster-config.md.CD6vzYtD.js +0 -1
  352. package/public/docs/assets/tuning-reference_cluster-config.md.CD6vzYtD.lean.js +0 -1
  353. package/public/docs/assets/tuning-reference_config.md.UvveiWG3.js +0 -1
  354. package/public/docs/assets/tuning-reference_config.md.UvveiWG3.lean.js +0 -1
  355. package/public/docs/assets/tuning-reference_data-formats.md.bjCAWH3N.js +0 -1
  356. package/public/docs/assets/tuning-reference_data-formats.md.bjCAWH3N.lean.js +0 -1
  357. package/public/docs/assets/tuning-reference_index.md.BQ_NooMV.js +0 -1
  358. package/public/docs/assets/tuning-reference_index.md.BQ_NooMV.lean.js +0 -1
  359. package/public/docs/assets/tuning-reference_intro.md.CobD-lGB.js +0 -1
  360. package/public/docs/assets/tuning-reference_intro.md.CobD-lGB.lean.js +0 -1
  361. package/public/docs/assets/tuning-reference_joins.md.BtKs_CuW.js +0 -1
  362. package/public/docs/assets/tuning-reference_joins.md.BtKs_CuW.lean.js +0 -1
  363. package/public/docs/assets/tuning-reference_memory-model.md.CWhvgevP.js +0 -1
  364. package/public/docs/assets/tuning-reference_memory-model.md.CWhvgevP.lean.js +0 -1
  365. package/public/docs/assets/tuning-reference_metrics.md.mLOh7Apj.js +0 -1
  366. package/public/docs/assets/tuning-reference_metrics.md.mLOh7Apj.lean.js +0 -1
  367. package/public/docs/assets/tuning-reference_partitioning.md.q0zKF_8X.js +0 -1
  368. package/public/docs/assets/tuning-reference_partitioning.md.q0zKF_8X.lean.js +0 -1
  369. package/public/docs/assets/tuning-reference_pyspark.md.DDCfvN9t.js +0 -6
  370. package/public/docs/assets/tuning-reference_pyspark.md.DDCfvN9t.lean.js +0 -1
  371. package/public/docs/assets/tuning-reference_shuffle.md.BZZ7R4Ix.js +0 -1
  372. package/public/docs/assets/tuning-reference_shuffle.md.BZZ7R4Ix.lean.js +0 -1
  373. package/public/docs/assets/tuning-reference_spark-architecture.md.Dwzm5avO.js +0 -1
  374. package/public/docs/assets/tuning-reference_spark-architecture.md.Dwzm5avO.lean.js +0 -1
  375. package/public/docs/assets/tuning-reference_table-formats.md.D6wj-2dX.js +0 -1
  376. package/public/docs/assets/tuning-reference_table-formats.md.D6wj-2dX.lean.js +0 -1
  377. package/public/docs/assets/user-guide_getting-started.md.Cpe0U-eO.js +0 -3
  378. package/public/docs/assets/user-guide_getting-started.md.Cpe0U-eO.lean.js +0 -1
  379. package/public/docs/assets/user-guide_mcp-tools.md.DrphooJv.lean.js +0 -1
  380. package/public/docs/assets/user-guide_run-comparison.md.CQc1aoU8.js +0 -1
  381. package/public/docs/assets/user-guide_run-comparison.md.CQc1aoU8.lean.js +0 -1
  382. /package/public/docs/assets/{contributor-guide_architecture_board-widgets.md.9ToZXcTk.lean.js → contributor-guide_architecture_board-widgets.md.VTcYTH4d.lean.js} +0 -0
package/public/docs/user-guide/getting-started.html CHANGED
@@ -9,11 +9,11 @@
9
9
  <link rel="preload stylesheet" href="/docs/assets/style.DSixAiZE.css" as="style">
10
10
  <link rel="preload stylesheet" href="/docs/vp-icons.css" as="style">
11
11
 
12
- <script type="module" src="/docs/assets/app.C5E-wy6e.js"></script>
12
+ <script type="module" src="/docs/assets/app.C6w47a9W.js"></script>
13
13
  <link rel="preload" href="/docs/assets/inter-roman-latin.Di8DUHzh.woff2" as="font" type="font/woff2" crossorigin="">
14
- <link rel="modulepreload" href="/docs/assets/chunks/theme.DQS9-w_Z.js">
14
+ <link rel="modulepreload" href="/docs/assets/chunks/theme.D-mE6j02.js">
15
15
  <link rel="modulepreload" href="/docs/assets/chunks/framework.DSg0KOwT.js">
16
- <link rel="modulepreload" href="/docs/assets/user-guide_getting-started.md.Cpe0U-eO.lean.js">
16
+ <link rel="modulepreload" href="/docs/assets/user-guide_getting-started.md.CEa2ZX-l.lean.js">
17
17
  <link rel="icon" type="image/svg+xml" href="/docs/favicon.svg">
18
18
  <link rel="preconnect" href="https://fonts.googleapis.com">
19
19
  <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin="">
@@ -22,10 +22,13 @@
22
22
  <script id="check-mac-os">document.documentElement.classList.toggle("mac",/Mac|iPhone|iPod|iPad/i.test(navigator.platform));</script>
23
23
  </head>
24
24
  <body>
25
- <div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink active" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>User Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/getting-started.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Getting started</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/understanding-findings.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Understanding findings</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/run-comparison.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Run comparison mode</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/mcp-tools.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>MCP tools reference</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/alternative-log-retrieval.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Alternative log retrieval</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_user-guide_getting-started" data-v-39a288b8><div><h1 id="getting-started" tabindex="-1">Getting started <a class="header-anchor" href="#getting-started" aria-label="Permalink to &quot;Getting started&quot;">​</a></h1><p>SparkForensics reads a Spark History Server event log and turns it into a dashboard of flagged bottlenecks. No account needed.</p><h2 id="local-server-mode" tabindex="-1">Run it locally <a class="header-anchor" href="#local-server-mode" aria-label="Permalink to &quot;Run it locally {#local-server-mode}&quot;">​</a></h2><p>The recommended way to run SparkForensics on your machine:</p><div class="language-sh vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">npx</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> sparkforensics-server</span></span></code></pre></div><p>Open <code>http://127.0.0.1:4173</code> and drop a log in. This is <strong>local-server mode</strong>: it also fetches runs from a Spark History Server on your behalf. Port and environment-variable overrides, and what changes for a static deploy, are in the <a href="https://github.com/shuffle-works/sparkforensics#deploy-modes" target="_blank" rel="noreferrer">project README</a>.</p><p>The <a href="https://shuffle-works.github.io/sparkforensics/" target="_blank" rel="noreferrer">hosted demo</a> runs the same dashboard but can&#39;t fetch from a History Server. To work on SparkForensics itself, see <a href="./../contributor-guide/development-setup.html">Development setup</a>.</p><h2 id="load-a-run" tabindex="-1">Load a run <a class="header-anchor" href="#load-a-run" aria-label="Permalink to &quot;Load a run&quot;">​</a></h2><p>Drop a log file onto the landing page, or click <strong>Choose file</strong> to pick one. Parsing runs in a background worker, off the browser&#39;s main thread, so a multi-hundred-megabyte event stream doesn&#39;t freeze the tab.</p><p>No log of your own yet? Click <strong>Try a sample run</strong> on the landing page to load a bundled example run and see a populated dashboard right away.</p><p>The app takes a newline-delimited JSON event log (one JSON event per line, the format Spark writes to <code>spark.eventLog.dir</code>), either plain or gzip/Zstandard/LZ4/Snappy-compressed. It also takes the zip a Spark History Server hands back, from the Spark UI&#39;s download link or from <code>GET /api/v1/applications/&lt;appId&gt;/logs</code>, as-is: drop the <code>.zip</code> and the app unwraps the log inside it, or reassembles the parts of a rolling log. The zip must hold one attempt: for an application that ran more than once, download <code>/api/v1/applications/&lt;appId&gt;/&lt;attemptId&gt;/logs</code> instead.</p><p>Click <strong>Other sources</strong> on the landing page for two more ways in:</p><ul><li><strong>Choose rolling-log folder</strong>, for an <code>eventlog_v2_*</code> rolling directory. Drop the folder or point the picker at it; the app reassembles its parts in order before parsing.</li><li><strong>Fetch from Spark History Server</strong>, to pull an application straight from a reachable History Server. This needs <strong>local-server mode</strong> (see <a href="#local-server-mode">Run it locally</a>) and an application ID in one of the forms Spark itself uses: <code>application_&lt;timestamp&gt;_&lt;id&gt;</code>, <code>local-&lt;timestamp&gt;</code>, <code>app-&lt;id&gt;</code>, <code>spark-&lt;id&gt;</code>, or <code>driver-&lt;id&gt;</code>. If the fetch fails, the panel names the problem (server unreachable, application not found, local server not running, and so on) and suggests what to try next.</li></ul><p>Can&#39;t reach the History Server directly (it&#39;s only reachable through an SSH bastion)? See <a href="./alternative-log-retrieval.html">Alternative ways to get the logs</a>.</p><p>Files you have already loaded stay listed under <strong>Recent files</strong> on the landing page.</p><h2 id="reading-the-dashboard" tabindex="-1">Reading the dashboard <a class="header-anchor" href="#reading-the-dashboard" aria-label="Permalink to &quot;Reading the dashboard&quot;">​</a></h2><p>When parsing finishes, the dashboard shows a board of widgets, each in its own card with a title. Every widget that flags a problem uses the same convention: a colored impact dot (critical / warning / info) and an ALL-CAPS tag for the bottleneck category (<code>SKEW</code>, <code>SPILL</code>, <code>GC</code>, and so on: see <a href="./understanding-findings.html">Understanding findings</a>). Widgets list every affected stage, not just the worst one. The dot&#39;s color tracks how much run time the finding could save you rather than how unusual the metric looks, so a small-looking anomaly with a big payoff can outrank a dramatic one that would barely move your run time.</p><p>A run scorecard (wall-clock, efficiency, wastage) always shows at the top. Below it, two tabs split the rest of the board:</p><ol><li><strong>Findings</strong>: every flagged finding and its detail widget, grouped by impact band (Critical, Warning, Info). Within a band, a recommendation row for a bottleneck type collapses into a summary row when it fires more than once; clicking the summary row expands its full list, and clicking any single row or widget jumps straight to that finding. Below the impact-band groups, memory and core-usage utilization always show, even on a clean run. Widgets that found nothing fold away into a &quot;Clean checks&quot; disclosure. This is the tab you land on.</li><li><strong>Full app report</strong>: the wall-clock and executor timelines, the stage table, and the reference-only cards.</li></ol><p>Click a finding&#39;s documentation link (or the topbar&#39;s <strong>Docs</strong> button) to open the reference material in a slide-in panel beside the dashboard: the dashboard stays visible and interactive, so you can check a metric against the reference without losing your place.</p><h3 id="advanced-view" tabindex="-1">Advanced view <a class="header-anchor" href="#advanced-view" aria-label="Permalink to &quot;Advanced view&quot;">​</a></h3><p>The topbar has an <strong>Advanced view</strong> toggle. It&#39;s off by default, which keeps each widget to the finding itself and what to do about it. Turn it on to also show confidence levels, supporting evidence, and documentation links for each finding, plus a few extra table columns. Your choice is remembered across runs.</p><h3 id="the-rest-of-the-topbar" tabindex="-1">The rest of the topbar <a class="header-anchor" href="#the-rest-of-the-topbar" aria-label="Permalink to &quot;The rest of the topbar&quot;">​</a></h3><p>Once a run is loaded, the topbar also carries a few more controls. <strong>New analysis</strong> goes back to the landing page to load another run. <strong>Plan graph</strong> opens an interactive node-and-edge view of the run&#39;s SQL execution plan, filterable down to I/O operators (scan, exchange), a broader &quot;basic&quot; set, or every operator. <strong>Export evidence</strong> downloads the current run&#39;s findings as a portable Markdown or JSON report, the same shape the CLI and MCP tools produce; turn on <strong>Redact identifiers</strong> first if the report is headed outside the environment that produced it, since that pseudonymizes the app id and any host/IP tokens. <strong>Keyboard shortcuts</strong> (press <code>?</code> from anywhere) lists every shortcut. Rounding it out: a <strong>Docs</strong> link and a theme toggle.</p><h2 id="compare-two-runs" tabindex="-1">Compare two runs <a class="header-anchor" href="#compare-two-runs" aria-label="Permalink to &quot;Compare two runs&quot;">​</a></h2><p>To compare a baseline run against a candidate, say to check whether a tuning change helped, see <a href="./run-comparison.html">Run comparison mode</a>.</p><h2 id="ci-and-automation" tabindex="-1">CI and automation <a class="header-anchor" href="#ci-and-automation" aria-label="Permalink to &quot;CI and automation&quot;">​</a></h2><p>In a pipeline or a script, use the <code>analyze</code> CLI instead of the browser dashboard. It parses the same event logs and exits non-zero when a run crosses a threshold you set, so it can gate a build:</p><div class="language-sh vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">npx</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> -p</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> sparkforensics-cli</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> sparkforensics-analyze</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> &lt;</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">file</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">|</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">dir</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">&gt; [--max-runtime </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">ms]</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> [--max-skew </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">ratio]</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \</span></span>
25
+ <div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink active" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>User Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/getting-started.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Getting started</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/understanding-findings.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Understanding findings</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/run-comparison.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Run comparison mode</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/mcp-tools.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>MCP tools reference</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/alternative-log-retrieval.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Alternative log retrieval</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_user-guide_getting-started" data-v-39a288b8><div><h1 id="getting-started" tabindex="-1">Getting started <a class="header-anchor" href="#getting-started" aria-label="Permalink to &quot;Getting started&quot;">​</a></h1><p>SparkForensics reads a Spark History Server event log and turns it into a dashboard of flagged bottlenecks. No account needed.</p><h2 id="local-server-mode" tabindex="-1">Run it locally <a class="header-anchor" href="#local-server-mode" aria-label="Permalink to &quot;Run it locally {#local-server-mode}&quot;">​</a></h2><p>The recommended way to run SparkForensics on your machine:</p><div class="language-sh vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">npx</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> sparkforensics-server</span></span></code></pre></div><p>Open <code>http://127.0.0.1:4173</code> and drop a log in. This is <strong>local-server mode</strong>: it also fetches runs from a Spark History Server on your behalf. Port and environment-variable overrides, and what changes for a static deploy, are in the <a href="https://github.com/shuffle-works/sparkforensics#deploy-modes" target="_blank" rel="noreferrer">project README</a>.</p><p>The <a href="https://shuffle-works.github.io/sparkforensics/" target="_blank" rel="noreferrer">hosted demo</a> runs the same dashboard but can&#39;t fetch from a History Server. To work on SparkForensics itself, see <a href="./../contributor-guide/development-setup.html">Development setup</a>.</p><h2 id="load-a-run" tabindex="-1">Load a run <a class="header-anchor" href="#load-a-run" aria-label="Permalink to &quot;Load a run&quot;">​</a></h2><p>Drop a log file onto the landing page, or click <strong>Choose file</strong> to pick one. Parsing runs in a background worker, off the browser&#39;s main thread, so a multi-hundred-megabyte event stream doesn&#39;t freeze the tab. While it runs, the progress screen shows how far along it is and roughly how long is left; <strong>Cancel</strong> returns to the landing page.</p><p>No log of your own yet? Click <strong>Try a sample run</strong> on the landing page to load a bundled example run and see a populated dashboard right away. A line above its verdict says it is the sample, with <strong>Load my event log</strong> and a link to where to find one, so you can switch to your own run from there. <strong>Where do I find my event log?</strong>, under the buttons, is a short guide to turning event logging on and downloading a log from a History Server.</p><p>The app takes a newline-delimited JSON event log (one JSON event per line, the format Spark writes to <code>spark.eventLog.dir</code>), either plain or gzip/Zstandard/LZ4/Snappy-compressed. It also takes the zip a Spark History Server hands back, from the Spark UI&#39;s download link or from <code>GET /api/v1/applications/&lt;appId&gt;/logs</code>, as-is: drop the <code>.zip</code> and the app unwraps the log inside it, or reassembles the parts of a rolling log. The zip must hold one attempt: for an application that ran more than once, download <code>/api/v1/applications/&lt;appId&gt;/&lt;attemptId&gt;/logs</code> instead.</p><p>Click <strong>Other sources</strong> on the landing page for two more ways in:</p><ul><li><strong>Choose rolling-log folder</strong>, for an <code>eventlog_v2_*</code> rolling directory. Drop the folder or point the picker at it; the app reassembles its parts in order before parsing.</li><li><strong>Fetch from Spark History Server</strong>, to pull an application straight from a reachable History Server. This needs <strong>local-server mode</strong> (see <a href="#local-server-mode">Run it locally</a>) and an application ID in one of the forms Spark itself uses: <code>application_&lt;timestamp&gt;_&lt;id&gt;</code>, <code>local-&lt;timestamp&gt;</code>, <code>app-&lt;id&gt;</code>, <code>spark-&lt;id&gt;</code>, or <code>driver-&lt;id&gt;</code>. If the fetch fails, the panel names the problem (server unreachable, application not found, local server not running, and so on) and suggests what to try next.</li></ul><p>Can&#39;t reach the History Server directly (it&#39;s only reachable through an SSH bastion)? See <a href="./alternative-log-retrieval.html">Alternative ways to get the logs</a>.</p><p>Files you have already loaded stay listed under <strong>Recent files</strong> on the landing page.</p><h2 id="reading-the-dashboard" tabindex="-1">Reading the dashboard <a class="header-anchor" href="#reading-the-dashboard" aria-label="Permalink to &quot;Reading the dashboard&quot;">​</a></h2><p>When parsing finishes, the dashboard shows a board of widgets, each in its own card with a title. Every widget that flags a problem uses the same convention: a colored impact dot (critical / warning / info) and an ALL-CAPS tag for the bottleneck category (<code>SKEW</code>, <code>SPILL</code>, <code>GC</code>, and so on: see <a href="./understanding-findings.html">Understanding findings</a>). Widgets list every affected stage, not just the worst one. The dot&#39;s color tracks how much run time the finding could save you rather than how unusual the metric looks, so a small-looking anomaly with a big payoff can outrank a dramatic one that would barely move your run time.</p><p>The board opens with a verdict: one line saying where to start, a short summary of what was found, and up to three numbered next steps. Each step explains in plain language what is happening, says what to try, and has a <strong>Show evidence</strong> button that jumps to the finding&#39;s detail widget. Findings on the same stage are folded into one step, because they usually share a cause and their savings overlap rather than add up. Each savings figure says what it counts: a time such as &quot;58.6s of run time&quot; is how much sooner the run could finish, while a resource figure such as &quot;3.0 GB-h of unused executor memory&quot; or &quot;0.7 core-h of core time&quot; is cluster time a fix would free up, which cuts cost but may not shorten the run. <strong>Copy next steps</strong> copies the whole plan as a plain checklist (run, verdict, and numbered steps with their stage and savings) to paste into a ticket or a message. Steps follow the same savings ranking as the rest of the board. When much of the run&#39;s executor capacity sat idle, the summary says so, without moving cluster size ahead of a bigger fix.</p><p>The verdict also says how the run ended. When every job succeeded it says so. When a job failed, the title says the run failed (or how many of its jobs did), quotes the first line of the reason Spark recorded, and puts the failure first in the next steps, ahead of any speed-up, since a job has to finish before its speed matters.</p><p>A run is called clean only when the log had everything its checks need. When something was missing, the verdict title says some checks could not run, and the Findings tab&#39;s <strong>Clean checks</strong> list says which ones and, where Spark has one, the setting to turn on for the next run (for example <code>spark.eventLog.logStageExecutorMetrics=true</code> for per-executor memory).</p><p>A run scorecard sits under the verdict: <strong>Wall-clock</strong> (total run time), <strong>Efficiency</strong> (the share of that time with a stage running; higher is better; <strong>Not measured</strong> when no stage in the log recorded an end) and <strong>Unused core time</strong> (driver idle plus executor slack across the whole run, so it can run higher than the idle capacity a verdict step reports; lower is better). A collapsed <strong>New to Spark tuning?</strong> primer in the verdict explains stages, tasks, executors, shuffle and how to read savings. Stage labels such as <strong>Stage 7</strong> open that stage&#39;s details: how long it ran and what share of the run that was, then each of its findings with what is happening, what to try and a <strong>Show evidence</strong> button, followed by its task, locality, I/O and plan sections. Below it, two tabs split the rest of the board:</p><ol><li><strong>Findings</strong>: every flagged finding and its detail widget, grouped by impact band (Critical, Warning, Info). Within a band, a recommendation row for a bottleneck type collapses into a summary row when it fires more than once; clicking the summary row expands its full list, and clicking any single row or widget jumps straight to that finding. Each band leads with its recommendation rows; its detail widgets (the charts and per-stage numbers behind them) sit under <strong>Show the evidence</strong>, and <strong>Show evidence</strong> on any finding opens them for you. Advanced view shows the widgets without that step. Widgets that found nothing fold away into a &quot;Clean checks&quot; disclosure. A check the log lacked the data for is listed there under <strong>Not checked on this log</strong>, not as a pass, with the reason and the setting to turn on: for example every per-stage check when no stage finished, or core usage, memory and executor churn when the log has no end-of-run record. This is the tab you land on.</li><li><strong>Full app report</strong>: the wall-clock and executor timelines, the stage table, and the reference-only cards, led by core usage by locality, which shows even on a clean run.</li></ol><p>Click a finding&#39;s documentation link (or the topbar&#39;s <strong>Docs</strong> button) to open the reference material in a slide-in panel beside the dashboard: the dashboard stays visible and interactive, so you can check a metric against the reference without losing your place. Esc closes the panel, even while you are reading or scrolling inside it.</p><h3 id="advanced-view" tabindex="-1">Advanced view <a class="header-anchor" href="#advanced-view" aria-label="Permalink to &quot;Advanced view&quot;">​</a></h3><p>The topbar has an <strong>Advanced view</strong> toggle. It&#39;s off by default, which keeps each widget to the finding itself and what to do about it. Turn it on to also show confidence levels, supporting evidence, and documentation links for each finding, plus a few extra table columns and the finding filter bar (impact, type, stage). A filter that is already active, for example from a shared link, keeps the filter bar visible either way. In the verdict, each step also says how its savings figure was estimated (measured or modeled, and whether the stage ran alone or shared the cluster, which makes the figure a range from a floor to an optimistic high), shows any confidence marker, and a line states how the steps are ordered. The scorecard switches from plain captions to the raw run and idle-time breakdown, and the newcomer primer is hidden. Advanced view also turns on single-key triage shortcuts: <code>j</code> and <code>k</code> step through the verdict&#39;s steps and then every finding row, <code>Enter</code> shows the focused finding&#39;s evidence, or expands a grouped finding, <code>f</code> jumps to the filters, and <code>1</code> and <code>2</code> switch between <strong>Findings</strong> and <strong>Full app report</strong>. They stay off in the default view, so they never surprise a first-time visitor or a screen-reader user. Your choice is remembered across runs.</p><h3 id="the-rest-of-the-topbar" tabindex="-1">The rest of the topbar <a class="header-anchor" href="#the-rest-of-the-topbar" aria-label="Permalink to &quot;The rest of the topbar&quot;">​</a></h3><p>Once a run is loaded, the topbar also carries a few more controls. The count chip (&quot;4 critical&quot;) counts the same findings the verdict ranks; click it to jump to that band of the Findings list (a board filter hiding the band is cleared, with a notice saying so). It reads <strong>No findings</strong> only when the verdict calls the run clean, <strong>Not fully checked</strong> when the log lacked evidence for some checks, and <strong>Run failed</strong> when a job failed. For keyboard users, the first Tab stop is <strong>Skip to the verdict</strong>. <strong>New analysis</strong> goes back to the landing page to load another run. <strong>Compare with another run</strong> keeps this run as the baseline and asks only for the other one (see <a href="./run-comparison.html">Run comparison mode</a>). <strong>Plan graph</strong> opens an interactive node-and-edge view of the run&#39;s SQL execution plan, filterable down to I/O operators (scan, exchange), a broader &quot;basic&quot; set, or every operator. <strong>Export evidence</strong> downloads the current run&#39;s findings as a portable Markdown or JSON report, the same shape the CLI and MCP tools produce, or as <strong>Download HTML dashboard</strong>: one <code>.html</code> file holding this dashboard for the run, which opens in any browser with no server, like the CLI&#39;s <code>--export-html</code> folder. Exported dashboards carry no docs links: finding tags and &quot;learn more&quot; references show as plain text, so look them up in these docs yourself. An exported dashboard shows the conclusions worked out when it was exported, and its footer names the tool, core version and build that produced it; it has no <strong>Export evidence</strong> menu of its own. A file written by an incompatible release opens to a message saying so; export the run again with your current release. Turn on <strong>Redact identifiers</strong> first if the export is headed outside the environment that produced it, since that pseudonymizes the app id and any host/IP tokens in all three formats. <strong>Keyboard shortcuts</strong> (press <code>?</code> from anywhere) lists every shortcut. Rounding it out: a <strong>Docs</strong> link and a theme toggle.</p><h2 id="compare-two-runs" tabindex="-1">Compare two runs <a class="header-anchor" href="#compare-two-runs" aria-label="Permalink to &quot;Compare two runs&quot;">​</a></h2><p>To compare a baseline run against a candidate, say to check whether a tuning change helped, see <a href="./run-comparison.html">Run comparison mode</a>.</p><h2 id="ci-and-automation" tabindex="-1">CI and automation <a class="header-anchor" href="#ci-and-automation" aria-label="Permalink to &quot;CI and automation&quot;">​</a></h2><p>In a pipeline or a script, use the <code>analyze</code> CLI instead of the browser dashboard. It parses the same event logs and exits non-zero when a run crosses a threshold you set, so it can gate a build:</p><div class="language-sh vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">npx</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> -p</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> sparkforensics-cli</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> sparkforensics-analyze</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> &lt;</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">file</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">|</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">dir</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">&gt; [--max-runtime </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">ms]</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> [--max-skew </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">ratio]</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \</span></span>
26
26
  <span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> [--max-spill </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">gb]</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> [--max-failed-task-rate </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">pct]</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> [--min-efficiency </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">pct]</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \</span></span>
27
- <span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> [--out </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">path]</span></span></code></pre></div><p>The command ships in the <code>sparkforensics-cli</code> package. To install it once: <code>npm i -g sparkforensics-cli</code>, then run <code>sparkforensics-analyze</code> directly.</p><p>Point it at a single event-log file, or at one run&#39;s <code>eventlog_v2_*</code> rolling-log directory; any other directory is rejected. Output goes to stdout; pass <code>--out &lt;path&gt;</code> to write it to that file instead.</p><p>Pass <code>--export-html &lt;dir&gt;</code> to write a self-contained HTML dashboard for the run into <code>&lt;dir&gt;</code> (which must not already exist or must be empty). Open <code>&lt;dir&gt;/index.html</code> directly in a browser over <code>file://</code>, with no server, to get the same interactive dashboard offline. This makes it easy to archive or share a run. <code>--redact</code> applies to the exported report too.</p><p>The CLI also supports fetching a run directly from a reachable Spark History Server (<code>--shs-base-url</code>/<code>--app-id</code>/<code>--attempt-id</code>) instead of a local file, comparing a candidate run against a baseline with regression gating (<code>--baseline</code>/<code>--max-regression-pct</code>/<code>--regression-metric</code>/ <code>--fail-on-introduced</code>), redacting the app id and any host/IP tokens before sharing output (<code>--redact</code>), and narrowing the findings to certain impact bands, types, or a stage (<code>--impact</code>/<code>--type</code>/<code>--stage</code>). Run it with <code>--help</code> for the full flag list.</p><h3 id="regression-metric-keys" tabindex="-1">Regression metric keys <a class="header-anchor" href="#regression-metric-keys" aria-label="Permalink to &quot;Regression metric keys&quot;">​</a></h3><p><code>--regression-metric</code> (and the MCP <code>evaluate_budgets</code> tool&#39;s <code>regressionMetric</code>) takes one of these keys; the default is <code>wallClock</code>:</p><ul><li><code>wallClock</code>: wall-clock duration</li><li><code>executorRunTime</code>: summed executor run time</li><li><code>shuffleSpill</code>: shuffle spill</li><li><code>diskSpill</code>: disk spill</li><li><code>gcTime</code>: JVM GC time</li><li><code>taskSkew</code>: p95 task skew</li><li><code>failedTaskRate</code>: failed-task rate</li></ul><p>Four more keys, <code>inputBytes</code>, <code>outputBytes</code>, <code>taskCount</code> and <code>executorsAdded</code>, measure workload volume rather than performance. They have no better or worse direction, so a regression budget on one of them reports <code>inconclusive</code> whenever the value changes, and passes when it doesn&#39;t.</p><p>Same caveat as above: if the History Server is only reachable through an SSH bastion, <code>--shs-base-url</code> can&#39;t reach it either: see <a href="./alternative-log-retrieval.html">Alternative ways to get the logs</a>.</p><p>Running in Airflow instead of a plain CI pipeline? See <a href="https://github.com/shuffle-works/sparkforensics-operator" target="_blank" rel="noreferrer">sparkforensics-operator</a>, an Airflow operator that wraps the CLI and acts on the result after each Spark job, so you don&#39;t have to wire up the call yourself.</p><p>Want an AI assistant to diagnose a run directly, without the dashboard or a CI gate? See <a href="./mcp-tools.html">MCP tools reference</a>.</p></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><!----></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="/docs/user-guide/understanding-findings.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>Understanding findings</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
28
- <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"9ToZXcTk\",\"contributor-guide_architecture_detector-contract.md\":\"CsCytCho\",\"contributor-guide_architecture_drill-down.md\":\"Bawx6NgP\",\"contributor-guide_architecture_impact-estimation.md\":\"sf9DU2bA\",\"contributor-guide_architecture_index.md\":\"3TO9ic6w\",\"contributor-guide_architecture_overview.md\":\"rhfxoGsI\",\"contributor-guide_architecture_state-and-history.md\":\"B8Wc_ZTg\",\"contributor-guide_architecture_widget-rendering.md\":\"DYwFeyjV\",\"contributor-guide_architecture_worker-protocol.md\":\"CBJkwH24\",\"contributor-guide_contributing.md\":\"B2E9ni79\",\"contributor-guide_development-setup.md\":\"s85j2l8b\",\"contributor-guide_testing.md\":\"Cnn6gz1C\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"Df1YMIHu\",\"tuning-reference_aqe.md\":\"BIsCtLzm\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"CEstB3Ia\",\"tuning-reference_bottleneck-cold-start.md\":\"CEuy-72y\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"CIohQDfn\",\"tuning-reference_bottleneck-failures.md\":\"CAsFf3RR\",\"tuning-reference_bottleneck-gc.md\":\"B4T3kJYV\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"BaJl__1W\",\"tuning-reference_bottleneck-memory-utilization.md\":\"DbP-SJZc\",\"tuning-reference_bottleneck-retry-waste.md\":\"D5JMjVOt\",\"tuning-reference_bottleneck-shuffle.md\":\"DDMQVOW6\",\"tuning-reference_bottleneck-skew.md\":\"B0MzhWcX\",\"tuning-reference_bottleneck-slow-host.md\":\"BlIo6UDW\",\"tuning-reference_bottleneck-small-files.md\":\"DHGMTmPs\",\"tuning-reference_bottleneck-spill.md\":\"PNH7mITt\",\"tuning-reference_bottleneck-straggler.md\":\"DIbXdAcQ\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"CB05v3zh\",\"tuning-reference_bottleneck-utilization.md\":\"DqC_iGkw\",\"tuning-reference_caching.md\":\"B7aQ8asB\",\"tuning-reference_cluster-config.md\":\"CD6vzYtD\",\"tuning-reference_config.md\":\"UvveiWG3\",\"tuning-reference_data-formats.md\":\"bjCAWH3N\",\"tuning-reference_index.md\":\"BQ_NooMV\",\"tuning-reference_intro.md\":\"CobD-lGB\",\"tuning-reference_joins.md\":\"BtKs_CuW\",\"tuning-reference_memory-model.md\":\"CWhvgevP\",\"tuning-reference_metrics.md\":\"mLOh7Apj\",\"tuning-reference_partitioning.md\":\"q0zKF_8X\",\"tuning-reference_pyspark.md\":\"DDCfvN9t\",\"tuning-reference_shuffle.md\":\"BZZ7R4Ix\",\"tuning-reference_spark-architecture.md\":\"Dwzm5avO\",\"tuning-reference_table-formats.md\":\"D6wj-2dX\",\"user-guide_alternative-log-retrieval.md\":\"ZHPSbBkF\",\"user-guide_getting-started.md\":\"Cpe0U-eO\",\"user-guide_mcp-tools.md\":\"DrphooJv\",\"user-guide_run-comparison.md\":\"CQc1aoU8\",\"user-guide_understanding-findings.md\":\"YBBLOwWi\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Alternative log retrieval\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
27
+ <span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> [--out </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">path]</span></span></code></pre></div><p><code>--min-efficiency</code> checks busy core time, the share of executor core time that ran tasks (100 minus the dashboard&#39;s Unused core time). It is not the dashboard&#39;s Efficiency tile, which is the share of wall-clock with a stage running, so the two can differ widely on the same run.</p><p>The command ships in the <code>sparkforensics-cli</code> package. To install it once: <code>npm i -g sparkforensics-cli</code>, then run <code>sparkforensics-analyze</code> directly.</p><p>Point it at a single event-log file, or at one run&#39;s <code>eventlog_v2_*</code> rolling-log directory; any other directory is rejected. Output goes to stdout; pass <code>--out &lt;path&gt;</code> to write it to that file instead.</p><p>Pass <code>--export-html &lt;dir&gt;</code> to write a self-contained HTML dashboard for the run into <code>&lt;dir&gt;</code> (which must not already exist or must be empty). Open <code>&lt;dir&gt;/index.html</code> directly in a browser over <code>file://</code>, with no server, to get the same interactive dashboard offline, without the docs links. This makes it easy to archive or share a run. <code>--redact</code> applies to the exported report too.</p><p>The CLI also supports fetching a run directly from a reachable Spark History Server (<code>--shs-base-url</code>/<code>--app-id</code>/<code>--attempt-id</code>) instead of a local file, comparing a candidate run against a baseline with regression gating (<code>--baseline</code>/<code>--max-regression-pct</code>/<code>--regression-metric</code>/ <code>--fail-on-introduced</code>), redacting the app id and any host/IP tokens before sharing output (<code>--redact</code>), and narrowing the findings to certain impact bands, types, or a stage (<code>--impact</code>/<code>--type</code>/<code>--stage</code>), and tuning detector thresholds from a file (<code>--thresholds</code>, below). Run it with <code>--help</code> for the full flag list.</p><h3 id="tuning-detector-thresholds" tabindex="-1">Tuning detector thresholds <a class="header-anchor" href="#tuning-detector-thresholds" aria-label="Permalink to &quot;Tuning detector thresholds&quot;">​</a></h3><p>Every check fires at a fixed default threshold. When your normal workload trips one on purpose, such as a join you skew deliberately, pass <code>--thresholds &lt;file&gt;</code> to run that check with your own value. The file is JSON, keyed by detector, then by threshold name:</p><div class="language-json vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">json</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">{</span></span>
28
+ <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> &quot;skew&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: { </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;ratioWarn&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">6</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> },</span></span>
29
+ <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> &quot;shuffle&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: { </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;minBytes&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">1073741824</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> }</span></span>
30
+ <span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">}</span></span></code></pre></div><p>The names, units and defaults are the <code>detectors</code> catalog in the CLI&#39;s JSON report (<code>thresholds</code> on each row): bytes are raw byte counts, times are milliseconds, and <code>*Pct</code>/<code>*Rate</code> values are fractions (<code>0.05</code> is 5%). A tier list such as <code>slowHost.ratioTiers</code> takes the same number of values, in ascending order. The <code>configAudit</code> checks can&#39;t be tuned: they compare your Spark settings against Spark&#39;s own defaults.</p><p>The CLI refuses to run, with exit code 2 and a message naming the problem, when the file can&#39;t be read, isn&#39;t valid JSON, or names an unknown detector or threshold, or a value of the wrong shape. It never falls back to the defaults silently.</p><p>A tuned run says so wherever it reports:</p><ul><li>Each finding from a tuned detector carries <code>tunedThresholds</code> (each overridden threshold&#39;s <code>value</code> and <code>default</code>), and its <code>validationRequired</code> text names them and, when the finding has an impact estimate, says that estimate is unvalidated. The estimates are calibrated against the default thresholds, so they were never checked for a finding your override lets through. Tuning <code>slowHost</code> also labels <code>stageSlowness</code> findings (as <code>slowHost.&lt;name&gt;</code>), because a slow host hides a stage&#39;s slowness finding, so the override decides which of those you see.</li><li>A clean check measured against a tuned threshold carries <code>tunedThresholds</code> too, and its <code>thresholdSummary</code> states the tuned value.</li><li><code>summary.tunedThresholds</code> lists every tuned detector, and each tuned row of the <code>detectors</code> catalog shows the thresholds the run used.</li><li>The Markdown report adds a <code>Tuned thresholds</code> line to its header and a <code>tuned thresholds</code> line to each affected finding.</li></ul><p>An override equal to the default changes nothing and is not labeled. <code>--baseline</code> runs the baseline with the same overrides, so the comparison compares like with like. The budget flags are separate gates computed from the run&#39;s own figures; the one that recomputes a detector&#39;s figure, <code>--max-skew</code>, uses the file&#39;s <code>skew.minTasksForP95</code>, so it measures the same ratio the skew finding reports. <code>--export-html</code> writes the dashboard with the default thresholds, because the dashboard never tunes, and prints a note to stderr saying so. The browser dashboard has no tuning.</p><h3 id="regression-metric-keys" tabindex="-1">Regression metric keys <a class="header-anchor" href="#regression-metric-keys" aria-label="Permalink to &quot;Regression metric keys&quot;">​</a></h3><p><code>--regression-metric</code> (and the MCP <code>evaluate_budgets</code> tool&#39;s <code>regressionMetric</code>) takes one of these keys; the default is <code>wallClock</code>:</p><ul><li><code>wallClock</code>: wall-clock duration</li><li><code>executorRunTime</code>: summed executor run time</li><li><code>shuffleSpill</code>: shuffle spill</li><li><code>diskSpill</code>: disk spill</li><li><code>gcTime</code>: JVM GC time</li><li><code>taskSkew</code>: p95 task skew</li><li><code>failedTaskRate</code>: failed-task rate</li></ul><p>Four more keys, <code>inputBytes</code>, <code>outputBytes</code>, <code>taskCount</code> and <code>executorsAdded</code>, measure workload volume rather than performance. They have no better or worse direction, so a regression budget on one of them reports <code>inconclusive</code> whenever the value changes, and passes when it doesn&#39;t.</p><p>Same caveat as above: if the History Server is only reachable through an SSH bastion, <code>--shs-base-url</code> can&#39;t reach it either: see <a href="./alternative-log-retrieval.html">Alternative ways to get the logs</a>.</p><p>Running in Airflow instead of a plain CI pipeline? See <a href="https://github.com/shuffle-works/sparkforensics-operator" target="_blank" rel="noreferrer">sparkforensics-operator</a>, an Airflow operator that wraps the CLI and acts on the result after each Spark job, so you don&#39;t have to wire up the call yourself.</p><p>Want an AI assistant to diagnose a run directly, without the dashboard or a CI gate? See <a href="./mcp-tools.html">MCP tools reference</a>.</p></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><!----></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="/docs/user-guide/understanding-findings.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>Understanding findings</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
31
+ <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"VTcYTH4d\",\"contributor-guide_architecture_detector-contract.md\":\"CsqZHz_T\",\"contributor-guide_architecture_drill-down.md\":\"CFNCE8JB\",\"contributor-guide_architecture_impact-estimation.md\":\"HT9IJGIl\",\"contributor-guide_architecture_index.md\":\"3TO9ic6w\",\"contributor-guide_architecture_overview.md\":\"CTEqVeV4\",\"contributor-guide_architecture_state-and-history.md\":\"DOTVHjTV\",\"contributor-guide_architecture_widget-rendering.md\":\"BmpARZve\",\"contributor-guide_architecture_worker-protocol.md\":\"21pTYhFF\",\"contributor-guide_contributing.md\":\"B2E9ni79\",\"contributor-guide_development-setup.md\":\"CjgZJwjK\",\"contributor-guide_testing.md\":\"DTwhpoN2\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"D08Uaod7\",\"tuning-reference_aqe.md\":\"C1Ae6A8u\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"6wjKuJNE\",\"tuning-reference_bottleneck-cold-start.md\":\"BVQbsuHP\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"DtaYnaiF\",\"tuning-reference_bottleneck-failures.md\":\"Z1Tdmdd3\",\"tuning-reference_bottleneck-gc.md\":\"BCnXLf1n\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"CcvQj2g9\",\"tuning-reference_bottleneck-memory-utilization.md\":\"B-C0-8h-\",\"tuning-reference_bottleneck-retry-waste.md\":\"DCnhWiqj\",\"tuning-reference_bottleneck-shuffle.md\":\"B5OMJCd3\",\"tuning-reference_bottleneck-skew.md\":\"Cvo6NQm_\",\"tuning-reference_bottleneck-slow-host.md\":\"ja5qr3wq\",\"tuning-reference_bottleneck-small-files.md\":\"Crjtb0i2\",\"tuning-reference_bottleneck-spill.md\":\"Bl8KtSRg\",\"tuning-reference_bottleneck-straggler.md\":\"B0VN8reT\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"DQCftGhu\",\"tuning-reference_bottleneck-utilization.md\":\"BXDl30f9\",\"tuning-reference_caching.md\":\"CcaiMn_7\",\"tuning-reference_cluster-config.md\":\"DbWG1yZq\",\"tuning-reference_config.md\":\"YUqSpwFB\",\"tuning-reference_data-formats.md\":\"C5DX5xL4\",\"tuning-reference_index.md\":\"ZVIpQ6Dk\",\"tuning-reference_intro.md\":\"BSSaDlqx\",\"tuning-reference_joins.md\":\"BVhNCDnE\",\"tuning-reference_memory-model.md\":\"CfjFdgbl\",\"tuning-reference_metrics.md\":\"C64VNEL4\",\"tuning-reference_partitioning.md\":\"CtLK0xsS\",\"tuning-reference_pyspark.md\":\"D-92XgF8\",\"tuning-reference_shuffle.md\":\"Iq2v1oHJ\",\"tuning-reference_spark-architecture.md\":\"_qwnE_Sc\",\"tuning-reference_table-formats.md\":\"BSYEK-eN\",\"user-guide_alternative-log-retrieval.md\":\"ZHPSbBkF\",\"user-guide_getting-started.md\":\"CEa2ZX-l\",\"user-guide_mcp-tools.md\":\"dQwIvdTF\",\"user-guide_run-comparison.md\":\"ChLI3MK0\",\"user-guide_understanding-findings.md\":\"D1ccs60C\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Alternative log retrieval\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
29
32
 
30
33
  </body>
31
34
  </html>
package/public/docs/user-guide/mcp-tools.html CHANGED
@@ -9,11 +9,11 @@
9
9
  <link rel="preload stylesheet" href="/docs/assets/style.DSixAiZE.css" as="style">
10
10
  <link rel="preload stylesheet" href="/docs/vp-icons.css" as="style">
11
11
 
12
- <script type="module" src="/docs/assets/app.C5E-wy6e.js"></script>
12
+ <script type="module" src="/docs/assets/app.C6w47a9W.js"></script>
13
13
  <link rel="preload" href="/docs/assets/inter-roman-latin.Di8DUHzh.woff2" as="font" type="font/woff2" crossorigin="">
14
- <link rel="modulepreload" href="/docs/assets/chunks/theme.DQS9-w_Z.js">
14
+ <link rel="modulepreload" href="/docs/assets/chunks/theme.D-mE6j02.js">
15
15
  <link rel="modulepreload" href="/docs/assets/chunks/framework.DSg0KOwT.js">
16
- <link rel="modulepreload" href="/docs/assets/user-guide_mcp-tools.md.DrphooJv.lean.js">
16
+ <link rel="modulepreload" href="/docs/assets/user-guide_mcp-tools.md.dQwIvdTF.lean.js">
17
17
  <link rel="icon" type="image/svg+xml" href="/docs/favicon.svg">
18
18
  <link rel="preconnect" href="https://fonts.googleapis.com">
19
19
  <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin="">
@@ -29,7 +29,7 @@
29
29
  <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> &quot;args&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;sparkforensics-mcp&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">]</span></span>
30
30
  <span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> }</span></span>
31
31
  <span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> }</span></span>
32
- <span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">}</span></span></code></pre></div><h2 id="diagnose-run" tabindex="-1"><code>diagnose_run</code> <a class="header-anchor" href="#diagnose-run" aria-label="Permalink to &quot;`diagnose_run`&quot;">​</a></h2><p>Diagnose a Spark run: thresholded findings with remediation text.</p><p>Parameters (all optional: provide either a <code>source</code> to load a fresh run, or a <code>runId</code> for one already loaded in this session):</p><ul><li><code>source</code>: <code>{ path: string }</code> or <code>{ shsBaseUrl: string, appId: string, attemptId?: string }</code></li><li><code>runId</code>: <code>string</code></li><li><code>redact</code>: <code>boolean</code> (default <code>false</code>), pseudonymizes the app id and any host/IP tokens in the response</li><li><code>include</code>: array of <code>&quot;summary&quot; | &quot;evidenceAvailability&quot; | &quot;detectors&quot;</code> (default omitted, i.e. none). Each requested value adds one extra top-level field to the response, on top of the default <code>findings</code>/<code>recommendations</code>/ <code>cleanChecks</code>/<code>runComplete</code>: <ul><li><code>summary</code>: app id/name/Spark version, stage/job/SQL-execution counts, and a finding count broken down by impact band</li><li><code>evidenceAvailability</code>: which event types the log actually contained, so you can tell &quot;this check came back clean&quot; apart from &quot;this check couldn&#39;t run because the log is missing data&quot;</li><li><code>detectors</code>: the full catalog of checks this tool can run: type, version, scope (stage/sql/app/config), current thresholds, and a doc-page anchor This is opt-in because <code>detectors</code> in particular is a large, mostly-static catalog that would bloat a routine &quot;what&#39;s wrong with this run&quot; call.</li></ul></li><li><code>impactBand</code>: array of impact bands (e.g. <code>[&quot;critical&quot;, &quot;warning&quot;]</code>), narrows the <code>findings</code> array to only these bands</li><li><code>type</code>: array of finding <code>type</code> values, narrows <code>findings</code> to only these types</li><li><code>stageId</code>: number, narrows <code>findings</code> to only findings on this stage</li><li><code>format</code>: <code>&quot;json&quot; | &quot;md&quot;</code> (default <code>&quot;json&quot;</code>), switches <code>content[0].text</code> to a rendered Markdown report instead of JSON. <code>structuredContent</code> always stays JSON-shaped, regardless of <code>format</code>.</li></ul><p><code>impactBand</code>/<code>type</code>/<code>stageId</code> only filter the <code>findings</code> array: <code>recommendations</code>, <code>cleanChecks</code>, and the finding counts in <code>summary</code> (when requested via <code>include</code>) always stay computed from the full, unfiltered set, so a narrow filter never hides that other checks passed or other fixes exist.</p><p>A <code>runId</code> expires. Loaded runs are cached in memory, capped at 8 (LRU-evicted) and expired after 15 minutes (override with <code>SPARKFORENSICS_MCP_CACHE_CAP</code> and <code>SPARKFORENSICS_MCP_CACHE_TTL_MS</code>). A <code>runId</code> from an earlier <code>diagnose_run</code> call may not resolve when you pass it to <code>get_finding_evidence</code>, <code>compare_runs</code>, or <code>evaluate_budgets</code>; re-load the run with <code>source</code> if you get <code>run-not-found</code>.</p><p>Example call:</p><div class="language-json vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">json</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">{ </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;name&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;diagnose_run&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;arguments&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: { </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;source&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: { </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;path&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;app-20260101.zstd&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> } } }</span></span></code></pre></div><p>Example response (an event log with no <code>ApplicationEnd</code> event):</p><div class="language-json vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">json</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">{</span></span>
32
+ <span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">}</span></span></code></pre></div><p>To tune detector thresholds, start the server with <code>--thresholds &lt;file&gt;</code> (add <code>&quot;--thresholds&quot;, &quot;/absolute/path/thresholds.json&quot;</code> to <code>args</code>). The file format is the CLI&#39;s: see <a href="./getting-started.html#tuning-detector-thresholds">Tuning detector thresholds</a>. The overrides apply to every tool for the life of the server; a client can&#39;t change them per call. An unreadable or invalid file stops the server from starting, with the problem on stderr. On a tuned server, <code>diagnose_run</code>, <code>compare_runs</code> and <code>evaluate_budgets</code> add a top-level <code>tunedThresholds</code>, and each finding and clean check from a tuned detector carries its own <code>tunedThresholds</code>, as in the CLI report. The Markdown output of <code>diagnose_run</code> and <code>compare_runs</code> names the tuned thresholds too.</p><h2 id="diagnose-run" tabindex="-1"><code>diagnose_run</code> <a class="header-anchor" href="#diagnose-run" aria-label="Permalink to &quot;`diagnose_run`&quot;">​</a></h2><p>Diagnose a Spark run: thresholded findings with remediation text, led by the same verdict the dashboard opens with. <code>verdict</code> gives a title (&quot;Start with Stage 3&quot;, &quot;1 of 3 jobs failed in this run&quot;), summary sentences, the first three places to look in the dashboard&#39;s order (each with its action, what to try, the potential savings and what that figure counts, and the other finding types flagged at the same place), how many more places the full list holds, and <code>copyText</code>, the dashboard&#39;s &quot;Copy next steps&quot; checklist.</p><p>Parameters (all optional: provide either a <code>source</code> to load a fresh run, or a <code>runId</code> for one already loaded in this session):</p><ul><li><code>source</code>: <code>{ path: string }</code> or <code>{ shsBaseUrl: string, appId: string, attemptId?: string }</code></li><li><code>runId</code>: <code>string</code></li><li><code>redact</code>: <code>boolean</code> (default <code>false</code>), pseudonymizes the app id and any host/IP tokens in the response</li><li><code>include</code>: array of <code>&quot;summary&quot; | &quot;evidenceAvailability&quot; | &quot;detectors&quot;</code> (default omitted, i.e. none). Each requested value adds one extra top-level field to the response, on top of the default <code>verdict</code>/<code>findings</code>/<code>recommendations</code>/ <code>cleanChecks</code>/<code>notRunChecks</code>/<code>runComplete</code>: <ul><li><code>summary</code>: app id/name/Spark version, stage/job/SQL-execution counts, a finding count broken down by impact band, the same counts without evidence caveats and the incomplete-run row (<code>actionableFindingCount</code>, <code>actionableImpactBandCounts</code>, what the dashboard counts), and <code>clean</code></li><li><code>evidenceAvailability</code>: which event types the log actually contained, so you can tell &quot;this check came back clean&quot; apart from &quot;this check couldn&#39;t run because the log is missing data&quot;</li><li><code>detectors</code>: the full catalog of checks this tool can run: type, version, scope (stage/sql/app/config), current thresholds, and a doc-page anchor This is opt-in because <code>detectors</code> in particular is a large, mostly-static catalog that would bloat a routine &quot;what&#39;s wrong with this run&quot; call.</li></ul></li><li><code>impactBand</code>: array of impact bands (e.g. <code>[&quot;critical&quot;, &quot;warning&quot;]</code>), narrows the <code>findings</code> array to only these bands</li><li><code>type</code>: array of finding <code>type</code> values, narrows <code>findings</code> to only these types</li><li><code>stageId</code>: number, narrows <code>findings</code> to only findings on this stage, including a SQL plan finding whose only stage it is (the dashboard&#39;s stage details rule)</li><li><code>format</code>: <code>&quot;json&quot; | &quot;md&quot;</code> (default <code>&quot;json&quot;</code>), switches <code>content[0].text</code> to a rendered Markdown report instead of JSON. <code>structuredContent</code> always stays JSON-shaped, regardless of <code>format</code>.</li></ul><p><code>impactBand</code>/<code>type</code>/<code>stageId</code> only filter the <code>findings</code> array: <code>recommendations</code>, <code>cleanChecks</code>, <code>notRunChecks</code>, and the finding counts in <code>summary</code> (when requested via <code>include</code>) always stay computed from the full, unfiltered set, so a narrow filter never hides that other checks passed or other fixes exist.</p><p>A <code>runId</code> expires. Loaded runs are cached in memory, capped at 8 (LRU-evicted) and expired after 15 minutes (override with <code>SPARKFORENSICS_MCP_CACHE_CAP</code> and <code>SPARKFORENSICS_MCP_CACHE_TTL_MS</code>). A <code>runId</code> from an earlier <code>diagnose_run</code> call may not resolve when you pass it to <code>get_finding_evidence</code>, <code>compare_runs</code>, or <code>evaluate_budgets</code>; re-load the run with <code>source</code> if you get <code>run-not-found</code>.</p><p>Example call:</p><div class="language-json vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">json</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">{ </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;name&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;diagnose_run&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;arguments&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: { </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;source&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: { </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;path&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;app-20260101.zstd&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> } } }</span></span></code></pre></div><p>Example response (an event log with no <code>ApplicationEnd</code> event):</p><div class="language-json vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">json</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">{</span></span>
33
33
  <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> &quot;runId&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;b8b2c1a4-...&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
34
34
  <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> &quot;runComplete&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">false</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
35
35
  <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> &quot;findings&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [</span></span>
@@ -45,7 +45,7 @@
45
45
  <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> &quot;evidence&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: {}</span></span>
46
46
  <span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> }</span></span>
47
47
  <span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> ]</span></span>
48
- <span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">}</span></span></code></pre></div><p>The <code>evidence</code> shape is detector-specific. The example above is <code>incompleteRun</code>&#39;s, which is empty; other detectors attach the metrics that drove the finding.</p><h2 id="get-run-summary" tabindex="-1"><code>get_run_summary</code> <a class="header-anchor" href="#get-run-summary" aria-label="Permalink to &quot;`get_run_summary`&quot;">​</a></h2><p>App/stage/job/sql counts and duration for a run: no findings.</p><p>Parameters: same as <code>diagnose_run</code> (<code>source</code>, <code>runId</code>, <code>redact</code>, all optional).</p><p>Example call:</p><div class="language-json vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">json</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">{ </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;name&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;get_run_summary&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;arguments&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: { </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;source&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: { </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;path&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;app-20260101.zstd&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> } } }</span></span></code></pre></div><p>Example response:</p><div class="language-json vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">json</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">{</span></span>
48
+ <span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">}</span></span></code></pre></div><p>The <code>evidence</code> shape is detector-specific. The example above is <code>incompleteRun</code>&#39;s, which is empty; other detectors attach the metrics that drove the finding.</p><h2 id="get-run-summary" tabindex="-1"><code>get_run_summary</code> <a class="header-anchor" href="#get-run-summary" aria-label="Permalink to &quot;`get_run_summary`&quot;">​</a></h2><p>App/stage/job/sql counts, duration, and how the run ended: no findings. <code>failedJobs</code>/<code>totalJobs</code> count only jobs with an end record, and <code>failureReason</code> is the first line of Spark&#39;s own recorded reason when a job failed, the same line the dashboard verdict quotes (<code>failureReasonStageId</code> is the stage it came from, or <code>null</code> when there is no reason or it came from a job&#39;s exception). <code>runShape</code> carries the dashboard&#39;s run-shape figures: <code>wallClockMs</code>, <code>efficiencyPct</code> (the share of the run with a stage running, the Scorecard&#39;s Efficiency), <code>unusedCoreTimePct</code> (driver idle plus executor slack as a share of available core time; <code>minEfficiencyPct</code> checks 100 minus this), <code>etlPhasesMs</code> (<code>extract</code>/<code>transform</code>/<code>load</code> summed stage time, so a phase can exceed the run) and <code>peakBusyCores</code> (busy cores at the peak of Core Usage by Locality). Each is <code>null</code> where the dashboard shows &quot;Not measured&quot; or &quot;Unavailable&quot;.</p><p>Parameters: same as <code>diagnose_run</code> (<code>source</code>, <code>runId</code>, <code>redact</code>, all optional).</p><p>Example call:</p><div class="language-json vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">json</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">{ </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;name&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;get_run_summary&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;arguments&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: { </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;source&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: { </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;path&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;app-20260101.zstd&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> } } }</span></span></code></pre></div><p>Example response:</p><div class="language-json vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">json</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">{</span></span>
49
49
  <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> &quot;runId&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;3f9c2b7e-...&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
50
50
  <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> &quot;app&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: { </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;id&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;app-1&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;name&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;t&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;sparkVersion&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">null</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> },</span></span>
51
51
  <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> &quot;stageCount&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">0</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
@@ -53,8 +53,13 @@
53
53
  <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> &quot;sqlExecutionCount&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">0</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
54
54
  <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> &quot;executorCount&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: { </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;added&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">0</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;removed&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">0</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> },</span></span>
55
55
  <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> &quot;durationMs&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">100</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
56
- <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> &quot;runComplete&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">true</span></span>
57
- <span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">}</span></span></code></pre></div><h2 id="evaluate-budgets" tabindex="-1"><code>evaluate_budgets</code> <a class="header-anchor" href="#evaluate-budgets" aria-label="Permalink to &quot;`evaluate_budgets`&quot;">​</a></h2><p>Evaluate a run against pass/fail thresholds: the MCP equivalent of the CLI&#39;s <code>--max-runtime</code>/<code>--max-spill</code>/etc. budget flags, thin-wrapped over the same <code>evaluateBudgets()</code> the <code>sparkforensics-analyze</code> CLI uses.</p><p>Parameters (all optional):</p><ul><li><code>source</code> / <code>runId</code>: identify the run to evaluate (same shape as <code>diagnose_run</code>)</li><li><code>maxRuntimeMs</code>, <code>maxSpillGb</code>, <code>maxSkewRatio</code>, <code>maxFailedTaskRatePct</code>, <code>minEfficiencyPct</code>: single-run budgets, each evaluated only if provided</li><li><code>runIdB</code> / <code>sourceB</code>: a second run to compare the first against, so regression budgets can be evaluated. Same <code>runId</code>/<code>source</code> shape, resolved the same way. Omit both to skip regression budgets.</li><li><code>maxRegressionPct</code>, <code>regressionMetric</code>: fail if <code>regressionMetric</code> (default <code>wallClock</code>; see <a href="./getting-started.html#regression-metric-keys">Regression metric keys</a> for the full list) regressed by more than <code>maxRegressionPct</code>% between the first run (baseline) and the second run (candidate). Requires <code>runIdB</code>/<code>sourceB</code>.</li><li><code>failOnIntroduced</code>: fail if the second run introduces any finding in the given impact band (<code>&quot;all&quot;</code> or one of the impact band names). Requires <code>runIdB</code>/<code>sourceB</code>.</li></ul><p>A budget whose required evidence is missing (e.g. no <code>runIdB</code>/<code>sourceB</code> for a regression budget, or a run with no trustworthy task-level evidence) reports <code>inconclusive</code>, not a false pass.</p><p>Example call:</p><div class="language-json vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">json</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">{</span></span>
56
+ <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> &quot;runComplete&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">true</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
57
+ <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> &quot;failedJobs&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">0</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
58
+ <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> &quot;totalJobs&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">0</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
59
+ <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> &quot;failureReason&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">null</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
60
+ <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> &quot;failureReasonStageId&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">null</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
61
+ <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> &quot;runShape&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: { </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;wallClockMs&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">100</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;efficiencyPct&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">0</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;unusedCoreTimePct&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">null</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;etlPhasesMs&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">null</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;peakBusyCores&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">null</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> }</span></span>
62
+ <span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">}</span></span></code></pre></div><h2 id="evaluate-budgets" tabindex="-1"><code>evaluate_budgets</code> <a class="header-anchor" href="#evaluate-budgets" aria-label="Permalink to &quot;`evaluate_budgets`&quot;">​</a></h2><p>Evaluate a run against pass/fail thresholds: the MCP equivalent of the CLI&#39;s <code>--max-runtime</code>/<code>--max-spill</code>/etc. budget flags, thin-wrapped over the same <code>evaluateBudgets()</code> the <code>sparkforensics-analyze</code> CLI uses.</p><p>Parameters (all optional):</p><ul><li><code>source</code> / <code>runId</code>: identify the run to evaluate (same shape as <code>diagnose_run</code>). When <code>runIdB</code>/<code>sourceB</code> is also given, this run is the regression baseline instead.</li><li><code>maxRuntimeMs</code>, <code>maxSpillGb</code>, <code>maxSkewRatio</code>, <code>maxFailedTaskRatePct</code>, <code>minEfficiencyPct</code>: absolute budgets, each evaluated only if provided. With two runs they apply to the candidate (<code>runIdB</code>/<code>sourceB</code>), the same as the CLI&#39;s <code>--baseline</code> mode. <code>minEfficiencyPct</code> measures busy core time, the share of executor core time that ran tasks (100 minus the dashboard&#39;s Unused core time), not the dashboard&#39;s Efficiency tile.</li><li><code>runIdB</code> / <code>sourceB</code>: a candidate run to compare against the first, so regression budgets can be evaluated. Same <code>runId</code>/<code>source</code> shape, resolved the same way. Omit both to skip regression budgets.</li><li><code>maxRegressionPct</code>, <code>regressionMetric</code>: fail if <code>regressionMetric</code> (default <code>wallClock</code>; see <a href="./getting-started.html#regression-metric-keys">Regression metric keys</a> for the full list) regressed by more than <code>maxRegressionPct</code>% between the first run (baseline) and the second run (candidate). Requires <code>runIdB</code>/<code>sourceB</code>.</li><li><code>failOnIntroduced</code>: fail if the second run introduces any finding in the given impact band (<code>&quot;all&quot;</code> or one of the impact band names). Requires <code>runIdB</code>/<code>sourceB</code>.</li></ul><p>A budget whose required evidence is missing (e.g. no <code>runIdB</code>/<code>sourceB</code> for a regression budget, or a run with no trustworthy task-level evidence) reports <code>inconclusive</code>, not a false pass. Independent of which budgets you pass, an evaluated run with no ApplicationEnd event adds a <code>run-complete</code> result with status <code>inconclusive</code>, the same check that makes the CLI exit <code>3</code>. With two runs, the check applies to the candidate.</p><p>Example call:</p><div class="language-json vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">json</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">{</span></span>
58
63
  <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> &quot;name&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;evaluate_budgets&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
59
64
  <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> &quot;arguments&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: {</span></span>
60
65
  <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> &quot;source&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: { </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;path&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;baseline.zstd&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> },</span></span>
@@ -78,7 +83,7 @@
78
83
  <span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> ],</span></span>
79
84
  <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> &quot;violated&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">true</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
80
85
  <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> &quot;inconclusive&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">false</span></span>
81
- <span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">}</span></span></code></pre></div><h2 id="compare-runs" tabindex="-1"><code>compare_runs</code> <a class="header-anchor" href="#compare-runs" aria-label="Permalink to &quot;`compare_runs`&quot;">​</a></h2><p>Compare two runs: categorized findings delta and metric deltas.</p><p>Parameters:</p><ul><li><code>runIdA</code> / <code>sourceA</code>: identify run A (same <code>source</code> shape as above)</li><li><code>runIdB</code> / <code>sourceB</code>: identify run B</li><li><code>redact</code>: <code>boolean</code> (default <code>false</code>), pseudonymizes any app id or host/IP tokens embedded in free text (stage names and similar) in the response: see the note at the top of this page</li><li><code>format</code>: <code>&quot;json&quot; | &quot;md&quot;</code> (default <code>&quot;json&quot;</code>), switches <code>content[0].text</code> to a rendered Markdown report instead of JSON. <code>structuredContent</code> always stays JSON-shaped, regardless of <code>format</code>.</li></ul><p>Each side takes either a <code>runId</code> or a <code>source</code>, and you can mix them: a cached run ID for the baseline, a fresh file for the candidate.</p><p>Example call:</p><div class="language-json vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">json</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">{</span></span>
86
+ <span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">}</span></span></code></pre></div><h2 id="compare-runs" tabindex="-1"><code>compare_runs</code> <a class="header-anchor" href="#compare-runs" aria-label="Permalink to &quot;`compare_runs`&quot;">​</a></h2><p>Compare two runs: the comparison page&#39;s verdict, categorized findings delta and metric deltas. <code>verdict</code> is the headline the dashboard&#39;s comparison page opens with (run A is <code>runIdA</code>/<code>sourceA</code>, run B is <code>runIdB</code>/<code>sourceB</code>): a <code>title</code> such as &quot;Run B finished 9.9s faster than run A (37%)&quot;, or a failed-job headline when either run had jobs fail, a <code>tone</code> (<code>better</code>, <code>worse</code>, <code>same</code> or <code>unknown</code>), and <code>sentences</code> naming which cost metrics and finding categories moved each way.</p><p>Parameters:</p><ul><li><code>runIdA</code> / <code>sourceA</code>: identify run A (same <code>source</code> shape as above)</li><li><code>runIdB</code> / <code>sourceB</code>: identify run B</li><li><code>redact</code>: <code>boolean</code> (default <code>false</code>), pseudonymizes any app id or host/IP tokens embedded in free text (stage names and similar) in the response: see the note at the top of this page</li><li><code>format</code>: <code>&quot;json&quot; | &quot;md&quot;</code> (default <code>&quot;json&quot;</code>), switches <code>content[0].text</code> to a rendered Markdown report instead of JSON. <code>structuredContent</code> always stays JSON-shaped, regardless of <code>format</code>.</li></ul><p>Each side takes either a <code>runId</code> or a <code>source</code>, and you can mix them: a cached run ID for the baseline, a fresh file for the candidate.</p><p>Example call:</p><div class="language-json vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">json</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">{</span></span>
82
87
  <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> &quot;name&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;compare_runs&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
83
88
  <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> &quot;arguments&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: {</span></span>
84
89
  <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> &quot;sourceA&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: { </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;path&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;baseline.zstd&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> },</span></span>
@@ -87,6 +92,7 @@
87
92
  <span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">}</span></span></code></pre></div><p>Example response:</p><div class="language-json vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">json</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">{</span></span>
88
93
  <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> &quot;runIdA&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;aaaa1111-...&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
89
94
  <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> &quot;runIdB&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;bbbb2222-...&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
95
+ <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> &quot;verdict&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: { </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;title&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;Run B finished 1.0s faster than run A (50%)&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;tone&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;better&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;sentences&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [] },</span></span>
90
96
  <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> &quot;findingsDelta&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: { </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;introduced&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [], </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;resolved&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [] },</span></span>
91
97
  <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> &quot;metricDeltas&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [</span></span>
92
98
  <span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> {</span></span>
@@ -147,7 +153,7 @@
147
153
  <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> &quot;content&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [{ </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;type&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;text&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;text&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;&lt;error message&gt;&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> }],</span></span>
148
154
  <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> &quot;structuredContent&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: { </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;code&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;&lt;error-code&gt;&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> }</span></span>
149
155
  <span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">}</span></span></code></pre></div><p>The codes:</p><ul><li><code>run-not-found</code>: no cached run for the <code>runId</code> you passed.</li><li><code>finding-not-found</code>: that <code>findingId</code> isn&#39;t on that run.</li><li><code>invalid-date-filter</code>: <code>list_runs</code>&#39;s <code>minDate</code> or <code>maxDate</code> isn&#39;t a parseable date.</li><li><code>invalid-type</code>: that finding <code>type</code> isn&#39;t one <code>get_finding_documentation</code> recognizes.</li><li><code>invalid-anchor</code>: that <code>anchor</code> doesn&#39;t resolve to a known <code>get_reference_doc</code> page.</li><li><code>invalid-event-log</code>: the file doesn&#39;t exist, or the event log (or History Server archive) couldn&#39;t be decoded.</li><li><code>application-not-found</code>: the History Server returned a 404 for that <code>appId</code>/<code>attemptId</code>.</li><li><code>upstream-unreachable</code>: the History Server response stalled mid-body. Override the idle timeout with <code>SPARKFORENSICS_SHS_TIMEOUT_MS</code>. If the server is unreachable entirely (an SSH-only cluster), see <a href="./alternative-log-retrieval.html">Alternative ways to get the logs</a>.</li><li><code>archive-too-large</code>: the History Server archive blew the byte cap. Override it with <code>SPARKFORENSICS_MAX_ARCHIVE_BYTES</code>.</li><li><code>directory-not-found</code>: <code>list_runs</code>&#39;s <code>dir</code> doesn&#39;t exist or isn&#39;t readable.</li><li><code>invalid-shs-base-url</code>: <code>list_runs</code>&#39;s <code>shsBaseUrl</code> isn&#39;t an absolute HTTP(S) URL without credentials, query, or fragment.</li><li><code>access-or-upstream-failure</code>: the fallback code. Bad parameters, a failed History Server fetch, or any error that carries no more specific code.</li></ul></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/user-guide/run-comparison.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>Run comparison mode</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="/docs/user-guide/alternative-log-retrieval.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>Alternative log retrieval</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
150
- <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"9ToZXcTk\",\"contributor-guide_architecture_detector-contract.md\":\"CsCytCho\",\"contributor-guide_architecture_drill-down.md\":\"Bawx6NgP\",\"contributor-guide_architecture_impact-estimation.md\":\"sf9DU2bA\",\"contributor-guide_architecture_index.md\":\"3TO9ic6w\",\"contributor-guide_architecture_overview.md\":\"rhfxoGsI\",\"contributor-guide_architecture_state-and-history.md\":\"B8Wc_ZTg\",\"contributor-guide_architecture_widget-rendering.md\":\"DYwFeyjV\",\"contributor-guide_architecture_worker-protocol.md\":\"CBJkwH24\",\"contributor-guide_contributing.md\":\"B2E9ni79\",\"contributor-guide_development-setup.md\":\"s85j2l8b\",\"contributor-guide_testing.md\":\"Cnn6gz1C\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"Df1YMIHu\",\"tuning-reference_aqe.md\":\"BIsCtLzm\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"CEstB3Ia\",\"tuning-reference_bottleneck-cold-start.md\":\"CEuy-72y\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"CIohQDfn\",\"tuning-reference_bottleneck-failures.md\":\"CAsFf3RR\",\"tuning-reference_bottleneck-gc.md\":\"B4T3kJYV\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"BaJl__1W\",\"tuning-reference_bottleneck-memory-utilization.md\":\"DbP-SJZc\",\"tuning-reference_bottleneck-retry-waste.md\":\"D5JMjVOt\",\"tuning-reference_bottleneck-shuffle.md\":\"DDMQVOW6\",\"tuning-reference_bottleneck-skew.md\":\"B0MzhWcX\",\"tuning-reference_bottleneck-slow-host.md\":\"BlIo6UDW\",\"tuning-reference_bottleneck-small-files.md\":\"DHGMTmPs\",\"tuning-reference_bottleneck-spill.md\":\"PNH7mITt\",\"tuning-reference_bottleneck-straggler.md\":\"DIbXdAcQ\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"CB05v3zh\",\"tuning-reference_bottleneck-utilization.md\":\"DqC_iGkw\",\"tuning-reference_caching.md\":\"B7aQ8asB\",\"tuning-reference_cluster-config.md\":\"CD6vzYtD\",\"tuning-reference_config.md\":\"UvveiWG3\",\"tuning-reference_data-formats.md\":\"bjCAWH3N\",\"tuning-reference_index.md\":\"BQ_NooMV\",\"tuning-reference_intro.md\":\"CobD-lGB\",\"tuning-reference_joins.md\":\"BtKs_CuW\",\"tuning-reference_memory-model.md\":\"CWhvgevP\",\"tuning-reference_metrics.md\":\"mLOh7Apj\",\"tuning-reference_partitioning.md\":\"q0zKF_8X\",\"tuning-reference_pyspark.md\":\"DDCfvN9t\",\"tuning-reference_shuffle.md\":\"BZZ7R4Ix\",\"tuning-reference_spark-architecture.md\":\"Dwzm5avO\",\"tuning-reference_table-formats.md\":\"D6wj-2dX\",\"user-guide_alternative-log-retrieval.md\":\"ZHPSbBkF\",\"user-guide_getting-started.md\":\"Cpe0U-eO\",\"user-guide_mcp-tools.md\":\"DrphooJv\",\"user-guide_run-comparison.md\":\"CQc1aoU8\",\"user-guide_understanding-findings.md\":\"YBBLOwWi\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Alternative log retrieval\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
156
+ <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"VTcYTH4d\",\"contributor-guide_architecture_detector-contract.md\":\"CsqZHz_T\",\"contributor-guide_architecture_drill-down.md\":\"CFNCE8JB\",\"contributor-guide_architecture_impact-estimation.md\":\"HT9IJGIl\",\"contributor-guide_architecture_index.md\":\"3TO9ic6w\",\"contributor-guide_architecture_overview.md\":\"CTEqVeV4\",\"contributor-guide_architecture_state-and-history.md\":\"DOTVHjTV\",\"contributor-guide_architecture_widget-rendering.md\":\"BmpARZve\",\"contributor-guide_architecture_worker-protocol.md\":\"21pTYhFF\",\"contributor-guide_contributing.md\":\"B2E9ni79\",\"contributor-guide_development-setup.md\":\"CjgZJwjK\",\"contributor-guide_testing.md\":\"DTwhpoN2\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"D08Uaod7\",\"tuning-reference_aqe.md\":\"C1Ae6A8u\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"6wjKuJNE\",\"tuning-reference_bottleneck-cold-start.md\":\"BVQbsuHP\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"DtaYnaiF\",\"tuning-reference_bottleneck-failures.md\":\"Z1Tdmdd3\",\"tuning-reference_bottleneck-gc.md\":\"BCnXLf1n\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"CcvQj2g9\",\"tuning-reference_bottleneck-memory-utilization.md\":\"B-C0-8h-\",\"tuning-reference_bottleneck-retry-waste.md\":\"DCnhWiqj\",\"tuning-reference_bottleneck-shuffle.md\":\"B5OMJCd3\",\"tuning-reference_bottleneck-skew.md\":\"Cvo6NQm_\",\"tuning-reference_bottleneck-slow-host.md\":\"ja5qr3wq\",\"tuning-reference_bottleneck-small-files.md\":\"Crjtb0i2\",\"tuning-reference_bottleneck-spill.md\":\"Bl8KtSRg\",\"tuning-reference_bottleneck-straggler.md\":\"B0VN8reT\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"DQCftGhu\",\"tuning-reference_bottleneck-utilization.md\":\"BXDl30f9\",\"tuning-reference_caching.md\":\"CcaiMn_7\",\"tuning-reference_cluster-config.md\":\"DbWG1yZq\",\"tuning-reference_config.md\":\"YUqSpwFB\",\"tuning-reference_data-formats.md\":\"C5DX5xL4\",\"tuning-reference_index.md\":\"ZVIpQ6Dk\",\"tuning-reference_intro.md\":\"BSSaDlqx\",\"tuning-reference_joins.md\":\"BVhNCDnE\",\"tuning-reference_memory-model.md\":\"CfjFdgbl\",\"tuning-reference_metrics.md\":\"C64VNEL4\",\"tuning-reference_partitioning.md\":\"CtLK0xsS\",\"tuning-reference_pyspark.md\":\"D-92XgF8\",\"tuning-reference_shuffle.md\":\"Iq2v1oHJ\",\"tuning-reference_spark-architecture.md\":\"_qwnE_Sc\",\"tuning-reference_table-formats.md\":\"BSYEK-eN\",\"user-guide_alternative-log-retrieval.md\":\"ZHPSbBkF\",\"user-guide_getting-started.md\":\"CEa2ZX-l\",\"user-guide_mcp-tools.md\":\"dQwIvdTF\",\"user-guide_run-comparison.md\":\"ChLI3MK0\",\"user-guide_understanding-findings.md\":\"D1ccs60C\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Alternative log retrieval\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
151
157
 
152
158
  </body>
153
159
  </html>