sparkforensics-server 0.2.2 → 0.2.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (213) hide show
  1. package/package.json +1 -1
  2. package/public/assets/{AutoscalingChurn-B96FKx97.js → AutoscalingChurn-BOiUUfuY.js} +1 -1
  3. package/public/assets/CacheUtilization-DwX7lisd.js +1 -0
  4. package/public/assets/CachingOpportunity-DUE3GKpx.js +1 -0
  5. package/public/assets/{ColdStart-w8QYTNKo.js → ColdStart-ZHxPrJmI.js} +1 -1
  6. package/public/assets/ConfigAudit-BkVnq2XE.js +1 -0
  7. package/public/assets/{CoreUsageArea-o3kqQwyi.js → CoreUsageArea-DobrSXk2.js} +1 -1
  8. package/public/assets/Dashboard-y3_9iQJo.js +1 -0
  9. package/public/assets/{DuplicatePlanSubtree-Cm_UU3lY.js → DuplicatePlanSubtree-BSQZFeNq.js} +1 -1
  10. package/public/assets/{DurationHistogram-ltF-7M0A.js → DurationHistogram-DlQv4ZCB.js} +1 -1
  11. package/public/assets/{EmptyState-BCA3xioa.js → EmptyState-DqnSxTjJ.js} +1 -1
  12. package/public/assets/{ExecutorUtilization-Cn69i0uq.js → ExecutorUtilization-CuQUxaCd.js} +1 -1
  13. package/public/assets/GcPressure-CLT5hkMJ.js +1 -0
  14. package/public/assets/{ImpactEstimate-BGt-4WX5.js → ImpactEstimate-Df_laG-n.js} +1 -1
  15. package/public/assets/{IncompleteRun-s6uQ8OLf.js → IncompleteRun-CpIkS4XZ.js} +1 -1
  16. package/public/assets/{JobFailures-DJ2dDvPA.js → JobFailures-DKbvYc4J.js} +1 -1
  17. package/public/assets/{MemoryUtilization-C9G7P94X.js → MemoryUtilization-C5kHmXIb.js} +1 -1
  18. package/public/assets/{OverBroadcast-DYauoYWQ.js → OverBroadcast-C5pF66aq.js} +1 -1
  19. package/public/assets/PartitionSizing-BvEJQ5M1.js +1 -0
  20. package/public/assets/{PlanExplorer-_2oB5ouy.js → PlanExplorer-DjGTZifP.js} +1 -1
  21. package/public/assets/PlanGraphRoute-DoIGA7_H.js +1 -0
  22. package/public/assets/{RetryWaste-D_m8ASwP.js → RetryWaste-C0znHd_w.js} +1 -1
  23. package/public/assets/{RowPagination-BLsm8I6i.js → RowPagination-2icPidMP.js} +1 -1
  24. package/public/assets/ShuffleIO-L_XSnJ_V.js +1 -0
  25. package/public/assets/{Skew-BGLJuZZR.js → Skew-w4uV5H_g.js} +1 -1
  26. package/public/assets/{SlowHost-ChdWVvGO.js → SlowHost-D5tWJ1UR.js} +1 -1
  27. package/public/assets/{SmallFiles-gTilBtjW.js → SmallFiles-CTtNiSOh.js} +1 -1
  28. package/public/assets/{SpeculationWaste-CVzgJnSj.js → SpeculationWaste-xCKwjz2K.js} +1 -1
  29. package/public/assets/{Spill-NGJGQaTB.js → Spill-BT8my24K.js} +1 -1
  30. package/public/assets/{StageFailed-sGhki53z.js → StageFailed-De3vVDbc.js} +1 -1
  31. package/public/assets/{StageFindingGroup-AyI7BMd3.js → StageFindingGroup-R4OW-g4U.js} +1 -1
  32. package/public/assets/{StageHeader-BTzGkK8k.js → StageHeader-CGeWqC-0.js} +1 -1
  33. package/public/assets/{StagePill-BlMOW_pU.js → StagePill-BwwKLcFl.js} +1 -1
  34. package/public/assets/{StageShape-DfOW93Z1.js → StageShape-D4LmPVyV.js} +1 -1
  35. package/public/assets/{StageSlowness-BMZ-K_5e.js → StageSlowness-8-z6qn4Q.js} +1 -1
  36. package/public/assets/Straggler-BIgxsowI.js +1 -0
  37. package/public/assets/{TaskFailures-DGDwUK2A.js → TaskFailures-B9lLt6Rw.js} +1 -1
  38. package/public/assets/{TinyTask-CfT8zZDr.js → TinyTask-DYXks5sT.js} +1 -1
  39. package/public/assets/{UnderBroadcast-CiOiJ0k1.js → UnderBroadcast-BWqWMpeG.js} +1 -1
  40. package/public/assets/{finding-anchor-DEV8JvgH.js → finding-anchor-CDbHtqKT.js} +1 -1
  41. package/public/assets/{inbox-_QCYkBsP.js → inbox-M_EuWNiK.js} +1 -1
  42. package/public/assets/index-pnImGbgQ.js +9 -0
  43. package/public/assets/parser-worker-C3BAO6z9.js +117 -0
  44. package/public/assets/{select-kuy1MiAo.js → select-C-MZGNpb.js} +1 -1
  45. package/public/assets/{toggle-group-CUd-KWkp.js → toggle-group-C3nz7UQ_.js} +1 -1
  46. package/public/assets/useRecentFiles-0XuhS91q.js +11 -0
  47. package/public/assets/{useSortMode-BaeAA5cw.js → useSortMode-DHlKZeWv.js} +1 -1
  48. package/public/assets/zstd-worker-C1Jq9lfb.js +1 -0
  49. package/public/docs/404.html +2 -2
  50. package/public/docs/assets/{app.APoq_UBs.js → app.DsFQcynM.js} +1 -1
  51. package/public/docs/assets/chunks/@localSearchIndexroot.BEfYag5Q.js +1 -0
  52. package/public/docs/assets/chunks/{VPLocalSearchBox.B3TPzlHq.js → VPLocalSearchBox.oa8NbaiV.js} +1 -1
  53. package/public/docs/assets/chunks/{theme.PavXO_vz.js → theme.Ca4dR9dL.js} +2 -2
  54. package/public/docs/assets/contributor-guide_architecture_board-widgets.md.9ToZXcTk.js +1 -0
  55. package/public/docs/assets/{contributor-guide_architecture_board-widgets.md.CWpj01WU.lean.js → contributor-guide_architecture_board-widgets.md.9ToZXcTk.lean.js} +1 -1
  56. package/public/docs/assets/contributor-guide_architecture_detector-contract.md.aQQfP7pj.js +1 -0
  57. package/public/docs/assets/{contributor-guide_architecture_detector-contract.md.CgzUsQ6W.lean.js → contributor-guide_architecture_detector-contract.md.aQQfP7pj.lean.js} +1 -1
  58. package/public/docs/assets/{contributor-guide_architecture_drill-down.md.BtPdlM7r.js → contributor-guide_architecture_drill-down.md.Bawx6NgP.js} +1 -1
  59. package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.sf9DU2bA.js +1 -0
  60. package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.sf9DU2bA.lean.js +1 -0
  61. package/public/docs/assets/contributor-guide_architecture_overview.md.BXQdgG5u.js +1 -0
  62. package/public/docs/assets/{contributor-guide_architecture_overview.md.CehiRmGn.lean.js → contributor-guide_architecture_overview.md.BXQdgG5u.lean.js} +1 -1
  63. package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.DYwFeyjV.js +1 -0
  64. package/public/docs/assets/{contributor-guide_architecture_worker-protocol.md.IbnfNrV3.js → contributor-guide_architecture_worker-protocol.md.P7d1vtEJ.js} +2 -2
  65. package/public/docs/assets/{contributor-guide_architecture_worker-protocol.md.IbnfNrV3.lean.js → contributor-guide_architecture_worker-protocol.md.P7d1vtEJ.lean.js} +1 -1
  66. package/public/docs/assets/{contributor-guide_development-setup.md.DvAN_9mK.js → contributor-guide_development-setup.md.DvMV1xhQ.js} +3 -3
  67. package/public/docs/assets/contributor-guide_development-setup.md.DvMV1xhQ.lean.js +1 -0
  68. package/public/docs/assets/contributor-guide_testing.md.BU6vyA-9.js +1 -0
  69. package/public/docs/assets/contributor-guide_testing.md.BU6vyA-9.lean.js +1 -0
  70. package/public/docs/assets/{tuning-reference_bottleneck-failures.md.4z5BXGJ2.js → tuning-reference_bottleneck-failures.md.CAsFf3RR.js} +1 -1
  71. package/public/docs/assets/{tuning-reference_bottleneck-gc.md.DSxzZRK7.js → tuning-reference_bottleneck-gc.md.B4T3kJYV.js} +2 -2
  72. package/public/docs/assets/{tuning-reference_bottleneck-gc.md.DSxzZRK7.lean.js → tuning-reference_bottleneck-gc.md.B4T3kJYV.lean.js} +1 -1
  73. package/public/docs/assets/tuning-reference_bottleneck-shuffle.md.DDMQVOW6.js +12 -0
  74. package/public/docs/assets/{tuning-reference_bottleneck-shuffle.md.CM-nTmIH.lean.js → tuning-reference_bottleneck-shuffle.md.DDMQVOW6.lean.js} +1 -1
  75. package/public/docs/assets/{tuning-reference_bottleneck-skew.md.BdUwiDhn.js → tuning-reference_bottleneck-skew.md.B0MzhWcX.js} +4 -4
  76. package/public/docs/assets/{tuning-reference_bottleneck-skew.md.BdUwiDhn.lean.js → tuning-reference_bottleneck-skew.md.B0MzhWcX.lean.js} +1 -1
  77. package/public/docs/assets/{tuning-reference_bottleneck-small-files.md.B8kloyx8.js → tuning-reference_bottleneck-small-files.md.DHGMTmPs.js} +2 -2
  78. package/public/docs/assets/{tuning-reference_bottleneck-small-files.md.B8kloyx8.lean.js → tuning-reference_bottleneck-small-files.md.DHGMTmPs.lean.js} +1 -1
  79. package/public/docs/assets/{tuning-reference_bottleneck-straggler.md.DY36fHN5.js → tuning-reference_bottleneck-straggler.md.DIbXdAcQ.js} +2 -2
  80. package/public/docs/assets/{tuning-reference_bottleneck-straggler.md.DY36fHN5.lean.js → tuning-reference_bottleneck-straggler.md.DIbXdAcQ.lean.js} +1 -1
  81. package/public/docs/assets/{tuning-reference_bottleneck-tiny-tasks.md.QTV7O8kU.js → tuning-reference_bottleneck-tiny-tasks.md.CB05v3zh.js} +1 -1
  82. package/public/docs/assets/tuning-reference_bottleneck-utilization.md.DqC_iGkw.js +5 -0
  83. package/public/docs/assets/tuning-reference_bottleneck-utilization.md.DqC_iGkw.lean.js +1 -0
  84. package/public/docs/assets/{tuning-reference_cluster-config.md.ZVmDGsQ3.js → tuning-reference_cluster-config.md.CD6vzYtD.js} +1 -1
  85. package/public/docs/assets/{tuning-reference_cluster-config.md.ZVmDGsQ3.lean.js → tuning-reference_cluster-config.md.CD6vzYtD.lean.js} +1 -1
  86. package/public/docs/assets/{tuning-reference_memory-model.md.DhT-n4y3.js → tuning-reference_memory-model.md.CWhvgevP.js} +1 -1
  87. package/public/docs/assets/{tuning-reference_memory-model.md.DhT-n4y3.lean.js → tuning-reference_memory-model.md.CWhvgevP.lean.js} +1 -1
  88. package/public/docs/assets/{user-guide_mcp-tools.md.Vi3RoflJ.js → user-guide_mcp-tools.md.DGvolGzt.js} +1 -1
  89. package/public/docs/assets/user-guide_understanding-findings.md.B29f0ucS.js +1 -0
  90. package/public/docs/assets/{user-guide_understanding-findings.md.DL1UDhvR.lean.js → user-guide_understanding-findings.md.B29f0ucS.lean.js} +1 -1
  91. package/public/docs/contributor-guide/architecture/board-widgets.html +5 -5
  92. package/public/docs/contributor-guide/architecture/detector-contract.html +5 -5
  93. package/public/docs/contributor-guide/architecture/drill-down.html +5 -5
  94. package/public/docs/contributor-guide/architecture/impact-estimation.html +5 -5
  95. package/public/docs/contributor-guide/architecture/index.html +3 -3
  96. package/public/docs/contributor-guide/architecture/overview.html +5 -5
  97. package/public/docs/contributor-guide/architecture/state-and-history.html +3 -3
  98. package/public/docs/contributor-guide/architecture/widget-rendering.html +5 -5
  99. package/public/docs/contributor-guide/architecture/worker-protocol.html +6 -6
  100. package/public/docs/contributor-guide/contributing.html +3 -3
  101. package/public/docs/contributor-guide/development-setup.html +5 -5
  102. package/public/docs/contributor-guide/testing.html +5 -5
  103. package/public/docs/hashmap.json +1 -1
  104. package/public/docs/index.html +3 -3
  105. package/public/docs/tuning-reference/anti-patterns.html +3 -3
  106. package/public/docs/tuning-reference/aqe.html +3 -3
  107. package/public/docs/tuning-reference/bottleneck-broadcast-sizing.html +3 -3
  108. package/public/docs/tuning-reference/bottleneck-cold-start.html +3 -3
  109. package/public/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html +3 -3
  110. package/public/docs/tuning-reference/bottleneck-failures.html +5 -5
  111. package/public/docs/tuning-reference/bottleneck-gc.html +5 -5
  112. package/public/docs/tuning-reference/bottleneck-job-failure-rate.html +3 -3
  113. package/public/docs/tuning-reference/bottleneck-memory-utilization.html +3 -3
  114. package/public/docs/tuning-reference/bottleneck-retry-waste.html +3 -3
  115. package/public/docs/tuning-reference/bottleneck-shuffle.html +6 -6
  116. package/public/docs/tuning-reference/bottleneck-skew.html +7 -7
  117. package/public/docs/tuning-reference/bottleneck-slow-host.html +3 -3
  118. package/public/docs/tuning-reference/bottleneck-small-files.html +6 -6
  119. package/public/docs/tuning-reference/bottleneck-spill.html +3 -3
  120. package/public/docs/tuning-reference/bottleneck-straggler.html +5 -5
  121. package/public/docs/tuning-reference/bottleneck-tiny-tasks.html +5 -5
  122. package/public/docs/tuning-reference/bottleneck-utilization.html +6 -6
  123. package/public/docs/tuning-reference/caching.html +3 -3
  124. package/public/docs/tuning-reference/cluster-config.html +5 -5
  125. package/public/docs/tuning-reference/config.html +3 -3
  126. package/public/docs/tuning-reference/data-formats.html +3 -3
  127. package/public/docs/tuning-reference/index.html +3 -3
  128. package/public/docs/tuning-reference/intro.html +3 -3
  129. package/public/docs/tuning-reference/joins.html +3 -3
  130. package/public/docs/tuning-reference/memory-model.html +5 -5
  131. package/public/docs/tuning-reference/metrics.html +3 -3
  132. package/public/docs/tuning-reference/partitioning.html +3 -3
  133. package/public/docs/tuning-reference/pyspark.html +3 -3
  134. package/public/docs/tuning-reference/shuffle.html +3 -3
  135. package/public/docs/tuning-reference/spark-architecture.html +3 -3
  136. package/public/docs/tuning-reference/table-formats.html +3 -3
  137. package/public/docs/user-guide/alternative-log-retrieval.html +3 -3
  138. package/public/docs/user-guide/getting-started.html +3 -3
  139. package/public/docs/user-guide/mcp-tools.html +5 -5
  140. package/public/docs/user-guide/run-comparison.html +3 -3
  141. package/public/docs/user-guide/understanding-findings.html +5 -5
  142. package/public/index.html +1 -1
  143. package/vendor-core/cli/collect-run.js +3 -2
  144. package/vendor-core/cli/native-zstd.js +351 -0
  145. package/vendor-core/detectors.js +243 -53
  146. package/vendor-core/docs-config.js +34 -8
  147. package/vendor-core/docs-content/chapters/03-memory-model.md +39 -0
  148. package/vendor-core/docs-content/chapters/11-cluster-config.md +40 -0
  149. package/vendor-core/docs-content/detection/gc.md +2 -0
  150. package/vendor-core/docs-content/detection/host.md +2 -1
  151. package/vendor-core/docs-content/detection/plan.md +3 -1
  152. package/vendor-core/docs-content/detection/shape.md +2 -1
  153. package/vendor-core/docs-content/detection/shfl.md +2 -1
  154. package/vendor-core/docs-content/detection/spill.md +1 -1
  155. package/vendor-core/docs-content/detection/strag.md +2 -1
  156. package/vendor-core/docs-content/detection/tiny.md +2 -1
  157. package/vendor-core/docs-content/tuning/failures.md +1 -1
  158. package/vendor-core/docs-content/tuning/gc.md +11 -4
  159. package/vendor-core/docs-content/tuning/shuffle.md +25 -5
  160. package/vendor-core/docs-content/tuning/skew.md +14 -6
  161. package/vendor-core/docs-content/tuning/small-files.md +12 -7
  162. package/vendor-core/docs-content/tuning/straggler.md +34 -0
  163. package/vendor-core/docs-content/tuning/tiny-tasks.md +1 -1
  164. package/vendor-core/docs-content/tuning/utilization.md +57 -7
  165. package/vendor-core/docs-content/upstream.json +4 -0
  166. package/vendor-core/event-handlers.js +321 -69
  167. package/vendor-core/event-schemas.js +8 -6
  168. package/vendor-core/evidence-report.js +3 -1
  169. package/vendor-core/impact-estimator.js +170 -34
  170. package/vendor-core/mcp-tools.js +20 -7
  171. package/vendor-core/occupancy.js +71 -2
  172. package/vendor-core/parser-worker.js +56 -24
  173. package/vendor-core/plan-summary.js +5 -1
  174. package/vendor-core/run-comparison.js +1 -15
  175. package/vendor-core/shs-fetch.js +18 -7
  176. package/vendor-core/shs-load.js +2 -1
  177. package/vendor-core/stage-quantiles.js +111 -3
  178. package/vendor-core/string-hash.js +15 -0
  179. package/vendor-core/types.js +14 -1
  180. package/vendor-core/vendor/fzstd.js +94 -18
  181. package/vendor-core/zstd-worker-client.js +180 -0
  182. package/vendor-core/zstd-worker.js +103 -0
  183. package/public/assets/CacheUtilization-lKua3J1x.js +0 -1
  184. package/public/assets/CachingOpportunity-U0g-8HPV.js +0 -1
  185. package/public/assets/ConfigAudit-ClZp5BpV.js +0 -1
  186. package/public/assets/Dashboard-Bjl-MYMm.js +0 -1
  187. package/public/assets/GcPressure-CYsmU1d7.js +0 -1
  188. package/public/assets/PartitionSizing-Cz_B0-uG.js +0 -1
  189. package/public/assets/PlanGraphRoute-Cj-GWM8W.js +0 -1
  190. package/public/assets/ShuffleIO-pyDhxdaz.js +0 -1
  191. package/public/assets/Straggler-dffcAN7t.js +0 -1
  192. package/public/assets/index-3vlFVNCg.js +0 -9
  193. package/public/assets/parser-worker-B2bc0lWe.js +0 -112
  194. package/public/assets/useRecentFiles-D0OA_EOK.js +0 -11
  195. package/public/docs/assets/chunks/@localSearchIndexroot.CvM5fuxe.js +0 -1
  196. package/public/docs/assets/contributor-guide_architecture_board-widgets.md.CWpj01WU.js +0 -1
  197. package/public/docs/assets/contributor-guide_architecture_detector-contract.md.CgzUsQ6W.js +0 -1
  198. package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.CooslVJt.js +0 -1
  199. package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.CooslVJt.lean.js +0 -1
  200. package/public/docs/assets/contributor-guide_architecture_overview.md.CehiRmGn.js +0 -1
  201. package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.R27gQrgY.js +0 -1
  202. package/public/docs/assets/contributor-guide_development-setup.md.DvAN_9mK.lean.js +0 -1
  203. package/public/docs/assets/contributor-guide_testing.md.6rIKqSyY.js +0 -1
  204. package/public/docs/assets/contributor-guide_testing.md.6rIKqSyY.lean.js +0 -1
  205. package/public/docs/assets/tuning-reference_bottleneck-shuffle.md.CM-nTmIH.js +0 -12
  206. package/public/docs/assets/tuning-reference_bottleneck-utilization.md.DTiueZC3.js +0 -5
  207. package/public/docs/assets/tuning-reference_bottleneck-utilization.md.DTiueZC3.lean.js +0 -1
  208. package/public/docs/assets/user-guide_understanding-findings.md.DL1UDhvR.js +0 -1
  209. /package/public/docs/assets/{contributor-guide_architecture_drill-down.md.BtPdlM7r.lean.js → contributor-guide_architecture_drill-down.md.Bawx6NgP.lean.js} +0 -0
  210. /package/public/docs/assets/{contributor-guide_architecture_widget-rendering.md.R27gQrgY.lean.js → contributor-guide_architecture_widget-rendering.md.DYwFeyjV.lean.js} +0 -0
  211. /package/public/docs/assets/{tuning-reference_bottleneck-failures.md.4z5BXGJ2.lean.js → tuning-reference_bottleneck-failures.md.CAsFf3RR.lean.js} +0 -0
  212. /package/public/docs/assets/{tuning-reference_bottleneck-tiny-tasks.md.QTV7O8kU.lean.js → tuning-reference_bottleneck-tiny-tasks.md.CB05v3zh.lean.js} +0 -0
  213. /package/public/docs/assets/{user-guide_mcp-tools.md.Vi3RoflJ.lean.js → user-guide_mcp-tools.md.DGvolGzt.lean.js} +0 -0
package/public/docs/contributor-guide/architecture/overview.html CHANGED
@@ -9,11 +9,11 @@
9
9
  <link rel="preload stylesheet" href="/docs/assets/style.DSixAiZE.css" as="style">
10
10
  <link rel="preload stylesheet" href="/docs/vp-icons.css" as="style">
11
11
 
12
- <script type="module" src="/docs/assets/app.APoq_UBs.js"></script>
12
+ <script type="module" src="/docs/assets/app.DsFQcynM.js"></script>
13
13
  <link rel="preload" href="/docs/assets/inter-roman-latin.Di8DUHzh.woff2" as="font" type="font/woff2" crossorigin="">
14
- <link rel="modulepreload" href="/docs/assets/chunks/theme.PavXO_vz.js">
14
+ <link rel="modulepreload" href="/docs/assets/chunks/theme.Ca4dR9dL.js">
15
15
  <link rel="modulepreload" href="/docs/assets/chunks/framework.DSg0KOwT.js">
16
- <link rel="modulepreload" href="/docs/assets/contributor-guide_architecture_overview.md.CehiRmGn.lean.js">
16
+ <link rel="modulepreload" href="/docs/assets/contributor-guide_architecture_overview.md.BXQdgG5u.lean.js">
17
17
  <link rel="icon" type="image/svg+xml" href="/docs/favicon.svg">
18
18
  <link rel="preconnect" href="https://fonts.googleapis.com">
19
19
  <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin="">
@@ -22,8 +22,8 @@
22
22
  <script id="check-mac-os">document.documentElement.classList.toggle("mac",/Mac|iPhone|iPod|iPad/i.test(navigator.platform));</script>
23
23
  </head>
24
24
  <body>
25
- <div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Contributor Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/development-setup.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Development setup</p><!--]--></a><!----></div><!----></div><section class="VPSidebarItem level-1 collapsible collapsed is-link has-active" data-v-b3fd67f8><div class="item" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/" data-v-b3fd67f8><!--[--><h3 class="text" data-v-b3fd67f8>Architecture</h3><!--]--></a><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-b3fd67f8><span class="vpi-chevron-right caret-icon" data-v-b3fd67f8></span></div></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/overview.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Overview</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/worker-protocol.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Worker protocol</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/state-and-history.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>State & history intake</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/detector-contract.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Detector contract</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/impact-estimation.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Impact estimation</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/widget-rendering.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Widget rendering</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/board-widgets.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Board widgets</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/drill-down.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Drill-down</p><!--]--></a><!----></div><!----></div><!--]--></div></section><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/testing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Testing & verification</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/contributing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Contributing</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_contributor-guide_architecture_overview" data-v-39a288b8><div><h1 id="architecture-overview" tabindex="-1">Architecture overview <a class="header-anchor" href="#architecture-overview" aria-label="Permalink to &quot;Architecture overview&quot;">​</a></h1><h2 id="core-invariant" tabindex="-1">Core invariant <a class="header-anchor" href="#core-invariant" aria-label="Permalink to &quot;Core invariant&quot;">​</a></h2><p>Task data must never live on the main thread. A Spark job can emit millions of <code>SparkListenerTaskEnd</code> events (240 MB+ of metrics); parsing them on the UI thread freezes Chrome. The design below enforces that in both deploy modes: the static zero-backend app, and the optional local-server mode in <code>server/</code>, which also proxies Spark History Server fetches to sidestep CORS.</p><h2 id="two-actors" tabindex="-1">Two actors <a class="header-anchor" href="#two-actors" aria-label="Permalink to &quot;Two actors&quot;">​</a></h2><h3 id="main-thread" tabindex="-1">Main thread <a class="header-anchor" href="#main-thread" aria-label="Permalink to &quot;Main thread&quot;">​</a></h3><p>Owns the DOM (React) and a summary <code>AppModel</code> (jobs, stages, sql, executors, no raw tasks), held in a Zustand store (<code>src/store/store.ts</code>).</p><p><code>src/store/useIngest.ts</code> is the composition root. It owns the <code>IngestClient</code> lifecycle and wires <code>model-assembler.ts</code>&#39;s <code>createModelCallbacks</code> (worker-message → <code>AppModel</code>, unchanged from before the view migration) into store updates: <code>onProgress</code> → <code>parse</code>; <code>onDone</code> → derives <code>evidenceAvailability</code> from the final normalized model plus <code>done.skippedLines</code>, runs <code>analyzer.ts</code>, sets <code>catalog</code>, and prefetches flagged-stage task data.</p><p><code>src/App.tsx</code> routes on store <code>status</code>: idle/error → <code>DropZone</code>, parsing → a live progress readout, ready → <code>Dashboard</code>. <code>src/view/Dashboard.tsx</code> and <code>src/view/detector-registry.tsx</code> replace <code>dashboard-renderer.js</code>&#39;s widget board (see <a href="./widget-rendering.html">Widget rendering</a>).</p><h3 id="worker-web-worker-core-js" tabindex="-1">Worker (Web Worker, core JS) <a class="header-anchor" href="#worker-web-worker-core-js" aria-label="Permalink to &quot;Worker (Web Worker, core JS)&quot;">​</a></h3><p>Owns the file and <code>taskStore: Map&lt;stageId, Float64Array&gt;</code>, keyed by 8-field stride <code>[duration, gcTime, memSpilled, diskSpilled, shuffleRead, shuffleWrite, launchTime, finishTime]</code>. Four modules:</p><ul><li><code>src/stage-quantiles.ts</code>, a pure leaf, exporting <code>finalizeStage</code>, <code>computeFieldQuantiles</code>, <code>computeDurationQuantiles</code>, <code>classifySpill</code>, <code>FIELDS</code>, and <code>TASK_FIELD_NAMES</code> for the task-field packed-array layout.</li><li><code>src/event-handlers.ts</code>, holding <code>processEvent</code> and the per-event handler functions, <code>accumulateTask</code>, <code>createState</code>, <code>dispatchLine</code>, <code>buildChunkDecoder</code>, <code>emitParseCompletion</code>, <code>collectStageExecutorMetrics</code>.</li><li><code>src/shs-fetch.ts</code>, the SHS zip-fetch/decompress path: <code>runParseFromUrl</code>, <code>naturalCompare</code>, <code>reassembleRollingEntries</code>, <code>sniffCodec</code>.</li><li><code>src/parser-worker.ts</code>, a thin entrypoint (<code>streamFile</code>, <code>runParse</code>, <code>runParseFiles</code>, the <code>isWorker</code>/<code>self.onmessage</code> bus) that barrel-re-exports the other three. It is the only piece needing the File/Blob streaming API.</li></ul><h2 id="streaming" tabindex="-1">Streaming <a class="header-anchor" href="#streaming" aria-label="Permalink to &quot;Streaming&quot;">​</a></h2><p>Worker reads the <code>File</code> in 4 MB chunks via <code>file.slice(...).arrayBuffer()</code>, <code>TextDecoder({ stream: true })</code>, splits on <code>\n</code>, JSON-parses, dispatches per event type. Quantiles (P50/P95/max) and spill classification (<code>skew | volume | unclassified</code>) are computed at <code>SparkListenerStageCompleted</code> time, before posting <code>StageAggregate</code> to main.</p><p>When <code>spark.eventLog.logStageExecutorMetrics=true</code> (default <code>false</code>), <code>SparkListenerStageExecutorMetrics</code> events populate <code>stage.executorMetrics: Map&lt;execId, {...}&gt;</code> with the 23 raw peak-memory/GC fields verbatim (camelCased), consumed by the <code>memoryUtilization</code> detector&#39;s per-executor memory bands (see <a href="./board-widgets.html">Memory Utilization</a>).</p><p>These events can arrive <em>after</em> <code>SparkListenerStageCompleted</code> for the same stage, so the per-stage <code>StageAggregate</code> message posted at completion time never carries them. Instead, <code>collectStageExecutorMetrics(state)</code> walks every stage once more just before <code>done</code>, and the worker re-posts a single <code>stageExecutorMetrics</code> message (<code>Map&lt;stageId, Map&lt;execId, metrics&gt;&gt;</code>, empty when the log has no such data). Main-thread consumers get every stage&#39;s metrics without a per-stage race.</p><p>Compressed logs are inflated inline. <code>sniffCodec</code> reads the leading magic bytes: gzip (<code>1f 8b</code>), Zstandard (<code>28 b5 2f fd</code>, Spark&#39;s <code>spark.io.compression.codec=zstd</code>), Spark&#39;s custom <code>LZ4Block</code> framing, or Spark&#39;s Snappy framing (<code>org.xerial.snappy</code>&#39;s <code>\x82SNAPPY\0</code> header, <code>spark.io.compression.codec=snappy</code>), falling back to the filename suffix. Both the dropped-file path and the SHS-fetch path stream block-by-block (fflate <code>Gunzip</code> / fzstd <code>Decompress</code> / the LZ4Block decoder / the Snappy block decoder) to keep one decompressed chunk live at a time.</p><p>The SHS-fetch path does buffer the downloaded zip whole for <code>unzipSync</code>. But one-shotting the decompression of a single entry (fzstd&#39;s <code>decompress()</code>, fflate&#39;s <code>gunzipSync()</code>) would allocate that entry&#39;s full declared content size as one ArrayBuffer, which fails outright for a multi-GB event log. So <code>decodeEntry(name, raw, onChunk)</code> picks the codec&#39;s streaming decoder per entry and invokes <code>onChunk</code> per decompressed piece. Decompressors are vendored under <code>src/vendor/</code> (<code>fflate.js</code>, <code>fzstd.js</code>) plus <code>src/lz4-block.ts</code> and <code>src/snappy-block.ts</code>.</p><p>Rolling <code>eventlog_v2_*</code> directories (Spark&#39;s multi-file event-log format, <code>events_&lt;index&gt;_&lt;appId&gt;(.codec)</code> files plus a zero-byte <code>appstatus_*</code> completion marker and, periodically, one <code>*.compact</code> merge file) are reassembled into parse order by <code>reassembleRollingEntries(names)</code>: drop the marker, drop every non-compact <code>events_*</code> file at or below the most recent <code>.compact</code> file&#39;s index, sort the rest numerically. Both the SHS-zip fetch path (<code>runParseFromUrl</code>) and the local folder-drop path (<code>runParseFiles</code>) call this one function.</p><p><code>runParseFiles</code> streams each file in the resulting order through the same chunked codec dispatch as <code>runParse</code>, with a fresh decompressor per file, since codecs never span a roll boundary. The NDJSON line-decoder is the exception: it stays alive across all files, because nothing guarantees a roll boundary lands on a line boundary.</p></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/contributor-guide/architecture/" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>Architecture</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="/docs/contributor-guide/architecture/worker-protocol.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>Worker protocol</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
26
- <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"CWpj01WU\",\"contributor-guide_architecture_detector-contract.md\":\"CgzUsQ6W\",\"contributor-guide_architecture_drill-down.md\":\"BtPdlM7r\",\"contributor-guide_architecture_impact-estimation.md\":\"CooslVJt\",\"contributor-guide_architecture_index.md\":\"3TO9ic6w\",\"contributor-guide_architecture_overview.md\":\"CehiRmGn\",\"contributor-guide_architecture_state-and-history.md\":\"C-xxn0q7\",\"contributor-guide_architecture_widget-rendering.md\":\"R27gQrgY\",\"contributor-guide_architecture_worker-protocol.md\":\"IbnfNrV3\",\"contributor-guide_contributing.md\":\"CvRsdr6J\",\"contributor-guide_development-setup.md\":\"DvAN_9mK\",\"contributor-guide_testing.md\":\"6rIKqSyY\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"Df1YMIHu\",\"tuning-reference_aqe.md\":\"BIsCtLzm\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"CEstB3Ia\",\"tuning-reference_bottleneck-cold-start.md\":\"CEuy-72y\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"CIohQDfn\",\"tuning-reference_bottleneck-failures.md\":\"4z5BXGJ2\",\"tuning-reference_bottleneck-gc.md\":\"DSxzZRK7\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"BaJl__1W\",\"tuning-reference_bottleneck-memory-utilization.md\":\"DbP-SJZc\",\"tuning-reference_bottleneck-retry-waste.md\":\"D5JMjVOt\",\"tuning-reference_bottleneck-shuffle.md\":\"CM-nTmIH\",\"tuning-reference_bottleneck-skew.md\":\"BdUwiDhn\",\"tuning-reference_bottleneck-slow-host.md\":\"BlIo6UDW\",\"tuning-reference_bottleneck-small-files.md\":\"B8kloyx8\",\"tuning-reference_bottleneck-spill.md\":\"PNH7mITt\",\"tuning-reference_bottleneck-straggler.md\":\"DY36fHN5\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"QTV7O8kU\",\"tuning-reference_bottleneck-utilization.md\":\"DTiueZC3\",\"tuning-reference_caching.md\":\"B7aQ8asB\",\"tuning-reference_cluster-config.md\":\"ZVmDGsQ3\",\"tuning-reference_config.md\":\"UvveiWG3\",\"tuning-reference_data-formats.md\":\"bjCAWH3N\",\"tuning-reference_index.md\":\"BQ_NooMV\",\"tuning-reference_intro.md\":\"CobD-lGB\",\"tuning-reference_joins.md\":\"BtKs_CuW\",\"tuning-reference_memory-model.md\":\"DhT-n4y3\",\"tuning-reference_metrics.md\":\"mLOh7Apj\",\"tuning-reference_partitioning.md\":\"q0zKF_8X\",\"tuning-reference_pyspark.md\":\"DDCfvN9t\",\"tuning-reference_shuffle.md\":\"BZZ7R4Ix\",\"tuning-reference_spark-architecture.md\":\"Dwzm5avO\",\"tuning-reference_table-formats.md\":\"D6wj-2dX\",\"user-guide_alternative-log-retrieval.md\":\"B4tPGIal\",\"user-guide_getting-started.md\":\"BJvwLEIM\",\"user-guide_mcp-tools.md\":\"Vi3RoflJ\",\"user-guide_run-comparison.md\":\"CQc1aoU8\",\"user-guide_understanding-findings.md\":\"DL1UDhvR\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Alternative log retrieval\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
25
+ <div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Contributor Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/development-setup.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Development setup</p><!--]--></a><!----></div><!----></div><section class="VPSidebarItem level-1 collapsible collapsed is-link has-active" data-v-b3fd67f8><div class="item" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/" data-v-b3fd67f8><!--[--><h3 class="text" data-v-b3fd67f8>Architecture</h3><!--]--></a><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-b3fd67f8><span class="vpi-chevron-right caret-icon" data-v-b3fd67f8></span></div></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/overview.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Overview</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/worker-protocol.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Worker protocol</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/state-and-history.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>State & history intake</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/detector-contract.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Detector contract</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/impact-estimation.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Impact estimation</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/widget-rendering.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Widget rendering</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/board-widgets.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Board widgets</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/drill-down.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Drill-down</p><!--]--></a><!----></div><!----></div><!--]--></div></section><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/testing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Testing & verification</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/contributing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Contributing</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_contributor-guide_architecture_overview" data-v-39a288b8><div><h1 id="architecture-overview" tabindex="-1">Architecture overview <a class="header-anchor" href="#architecture-overview" aria-label="Permalink to &quot;Architecture overview&quot;">​</a></h1><h2 id="core-invariant" tabindex="-1">Core invariant <a class="header-anchor" href="#core-invariant" aria-label="Permalink to &quot;Core invariant&quot;">​</a></h2><p>Task data must never live on the main thread. A Spark job can emit millions of <code>SparkListenerTaskEnd</code> events (240 MB+ of metrics); parsing them on the UI thread freezes Chrome. The design below enforces that in both deploy modes: the static zero-backend app, and the optional local-server mode in <code>server/</code>, which also proxies Spark History Server fetches to sidestep CORS.</p><h2 id="two-actors" tabindex="-1">Two actors <a class="header-anchor" href="#two-actors" aria-label="Permalink to &quot;Two actors&quot;">​</a></h2><h3 id="main-thread" tabindex="-1">Main thread <a class="header-anchor" href="#main-thread" aria-label="Permalink to &quot;Main thread&quot;">​</a></h3><p>Owns the DOM (React) and a summary <code>AppModel</code> (jobs, stages, sql, executors, no raw tasks), held in a Zustand store (<code>src/store/store.ts</code>).</p><p><code>src/store/useIngest.ts</code> is the composition root. It owns the <code>IngestClient</code> lifecycle and wires <code>model-assembler.ts</code>&#39;s <code>createModelCallbacks</code> (worker-message → <code>AppModel</code>, unchanged from before the view migration) into store updates: <code>onProgress</code> → <code>parse</code>; <code>onDone</code> → derives <code>evidenceAvailability</code> from the final normalized model plus <code>done.skippedLines</code>, runs <code>analyzer.ts</code>, sets <code>catalog</code>, and prefetches flagged-stage task data.</p><p><code>src/App.tsx</code> routes on store <code>status</code>: idle/error → <code>DropZone</code>, parsing → a live progress readout, ready → <code>Dashboard</code>. <code>src/view/Dashboard.tsx</code> and <code>src/view/detector-registry.tsx</code> replace <code>dashboard-renderer.js</code>&#39;s widget board (see <a href="./widget-rendering.html">Widget rendering</a>).</p><h3 id="worker-web-worker-core-js" tabindex="-1">Worker (Web Worker, core JS) <a class="header-anchor" href="#worker-web-worker-core-js" aria-label="Permalink to &quot;Worker (Web Worker, core JS)&quot;">​</a></h3><p>Owns the file and <code>taskStore: Map&lt;stageId, Float64Array&gt;</code>, keyed by 8-field stride <code>[duration, gcTime, memSpilled, diskSpilled, shuffleRead, shuffleWrite, launchTime, finishTime]</code>. Four modules:</p><ul><li><code>src/stage-quantiles.ts</code>, a pure leaf, exporting <code>finalizeStage</code>, <code>computeFieldQuantiles</code>, <code>computeDurationQuantiles</code>, <code>classifySpill</code>, <code>FIELDS</code>, and <code>TASK_FIELD_NAMES</code> for the task-field packed-array layout.</li><li><code>src/event-handlers.ts</code>, holding <code>processEvent</code> and the per-event handler functions, <code>accumulateTask</code>, <code>createState</code>, <code>dispatchLine</code>, <code>buildChunkDecoder</code>, <code>emitParseCompletion</code>, <code>collectStageExecutorMetrics</code>.</li><li><code>src/shs-fetch.ts</code>, the SHS zip-fetch/decompress path: <code>runParseFromUrl</code>, <code>naturalCompare</code>, <code>reassembleRollingEntries</code>, <code>sniffCodec</code>.</li><li><code>src/parser-worker.ts</code>, a thin entrypoint (<code>streamFile</code>, <code>runParse</code>, <code>runParseFiles</code>, the <code>isWorker</code>/<code>self.onmessage</code> bus) that barrel-re-exports the other three. It is the only piece needing the File/Blob streaming API.</li></ul><p>Dropped zstd files also run a nested decompress worker, <code>src/zstd-worker.ts</code>, driven from the parse worker by <code>src/zstd-worker-client.ts</code> (see <a href="./worker-protocol.html#decompress-worker">Decompress worker</a>).</p><h2 id="streaming" tabindex="-1">Streaming <a class="header-anchor" href="#streaming" aria-label="Permalink to &quot;Streaming&quot;">​</a></h2><p>Worker reads the <code>File</code> in 4 MB chunks via <code>file.slice(...).arrayBuffer()</code>, <code>TextDecoder({ stream: true })</code>, splits on <code>\n</code>, JSON-parses, dispatches per event type. Quantiles (P50/P95/max) and spill classification (<code>skew | volume | unclassified</code>) are computed at <code>SparkListenerStageCompleted</code> time, before posting <code>StageAggregate</code> to main.</p><p>When <code>spark.eventLog.logStageExecutorMetrics=true</code> (default <code>false</code>), <code>SparkListenerStageExecutorMetrics</code> events populate <code>stage.executorMetrics: Map&lt;execId, {...}&gt;</code> with the 23 raw peak-memory/GC fields verbatim (camelCased), consumed by the <code>memoryUtilization</code> detector&#39;s per-executor memory bands (see <a href="./board-widgets.html">Memory Utilization</a>).</p><p>These events can arrive <em>after</em> <code>SparkListenerStageCompleted</code> for the same stage, so the per-stage <code>StageAggregate</code> message posted at completion time never carries them. Instead, <code>collectStageExecutorMetrics(state)</code> walks every stage once more just before <code>done</code>, and the worker re-posts a single <code>stageExecutorMetrics</code> message (<code>Map&lt;stageId, Map&lt;execId, metrics&gt;&gt;</code>, empty when the log has no such data). Main-thread consumers get every stage&#39;s metrics without a per-stage race.</p><p>Compressed logs are inflated inline. <code>sniffCodec</code> reads the leading magic bytes: gzip (<code>1f 8b</code>), Zstandard (<code>28 b5 2f fd</code>, Spark&#39;s <code>spark.io.compression.codec=zstd</code>), Spark&#39;s custom <code>LZ4Block</code> framing, or Spark&#39;s Snappy framing (<code>org.xerial.snappy</code>&#39;s <code>\x82SNAPPY\0</code> header, <code>spark.io.compression.codec=snappy</code>), falling back to the filename suffix. Both the dropped-file path and the SHS-fetch path stream block-by-block (fflate <code>Gunzip</code> / fzstd <code>Decompress</code> / the LZ4Block decoder / the Snappy block decoder) to keep one decompressed chunk live at a time. In the browser, a dropped zstd file decompresses in a second worker while the parse worker parses earlier output, with a three-slice window bounding what is queued between them (see <a href="./worker-protocol.html#decompress-worker">Decompress worker</a>). The Node CLI/MCP paths (<code>collectRun</code> for local files, <code>shs-load.ts</code> for SHS archives) swap fzstd for Node&#39;s native zlib zstd where the running Node has it (22.15+/23.8+), through the <code>zstdDecoder</code> option of <code>runParse</code>/<code>decodeShsArchive</code>: <code>packages/core/src/cli/native-zstd.ts</code> decompresses one frame at a time, since Node&#39;s own decoders stop after a stream&#39;s first frame and Spark writes thousands of small ones. A frame declaring more than 64 MiB of content, one still incomplete after 64 MiB compressed, a malformed one, or a truncated tail goes to fzstd instead. Spark&#39;s frames declare no content size, so each one&#39;s output is capped at 64 MiB as it decodes: past that, the inline decoder hands the frame to fzstd and the threaded one streams it, pausing while 64 MiB wait to be parsed. On the real logs this made parsing 42% faster. For local files (<code>nodeParseCodecs</code>, <code>createThreadedZstdDecoder</code>), frames of 64 KB or more compressed decompress on libuv&#39;s threadpool, up to 4 at a time, while the main thread parses earlier output; <code>streamFile</code> awaits each <code>push</code>, and chunks still arrive in stream order. That took another 24% off the largest real log (4.2s to 3.2s) for 139 MB more peak RSS. <code>decodeShsArchive</code> decodes each archive entry in one synchronous call, so SHS archives keep the inline decoder (<code>nodeArchiveCodecs</code>).</p><p>The SHS-fetch path does buffer the downloaded zip whole for <code>unzipSync</code>. But one-shotting the decompression of a single entry (fzstd&#39;s <code>decompress()</code>, fflate&#39;s <code>gunzipSync()</code>) would allocate that entry&#39;s full declared content size as one ArrayBuffer, which fails outright for a multi-GB event log. So <code>decodeEntry(name, raw, onChunk)</code> picks the codec&#39;s streaming decoder per entry and invokes <code>onChunk</code> per decompressed piece. Decompressors are vendored under <code>src/vendor/</code> (<code>fflate.js</code>, <code>fzstd.js</code>) plus <code>src/lz4-block.ts</code> and <code>src/snappy-block.ts</code>.</p><p>Rolling <code>eventlog_v2_*</code> directories (Spark&#39;s multi-file event-log format, <code>events_&lt;index&gt;_&lt;appId&gt;(.codec)</code> files plus a zero-byte <code>appstatus_*</code> completion marker and, periodically, one <code>*.compact</code> merge file) are reassembled into parse order by <code>reassembleRollingEntries(names)</code>: drop the marker, drop every non-compact <code>events_*</code> file at or below the most recent <code>.compact</code> file&#39;s index, sort the rest numerically. Both the SHS-zip fetch path (<code>runParseFromUrl</code>) and the local folder-drop path (<code>runParseFiles</code>) call this one function.</p><p><code>runParseFiles</code> streams each file in the resulting order through the same chunked codec dispatch as <code>runParse</code>, with a fresh decompressor per file, since codecs never span a roll boundary. The NDJSON line-decoder is the exception: it stays alive across all files, because nothing guarantees a roll boundary lands on a line boundary.</p></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/contributor-guide/architecture/" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>Architecture</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="/docs/contributor-guide/architecture/worker-protocol.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>Worker protocol</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
26
+ <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"9ToZXcTk\",\"contributor-guide_architecture_detector-contract.md\":\"aQQfP7pj\",\"contributor-guide_architecture_drill-down.md\":\"Bawx6NgP\",\"contributor-guide_architecture_impact-estimation.md\":\"sf9DU2bA\",\"contributor-guide_architecture_index.md\":\"3TO9ic6w\",\"contributor-guide_architecture_overview.md\":\"BXQdgG5u\",\"contributor-guide_architecture_state-and-history.md\":\"C-xxn0q7\",\"contributor-guide_architecture_widget-rendering.md\":\"DYwFeyjV\",\"contributor-guide_architecture_worker-protocol.md\":\"P7d1vtEJ\",\"contributor-guide_contributing.md\":\"CvRsdr6J\",\"contributor-guide_development-setup.md\":\"DvMV1xhQ\",\"contributor-guide_testing.md\":\"BU6vyA-9\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"Df1YMIHu\",\"tuning-reference_aqe.md\":\"BIsCtLzm\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"CEstB3Ia\",\"tuning-reference_bottleneck-cold-start.md\":\"CEuy-72y\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"CIohQDfn\",\"tuning-reference_bottleneck-failures.md\":\"CAsFf3RR\",\"tuning-reference_bottleneck-gc.md\":\"B4T3kJYV\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"BaJl__1W\",\"tuning-reference_bottleneck-memory-utilization.md\":\"DbP-SJZc\",\"tuning-reference_bottleneck-retry-waste.md\":\"D5JMjVOt\",\"tuning-reference_bottleneck-shuffle.md\":\"DDMQVOW6\",\"tuning-reference_bottleneck-skew.md\":\"B0MzhWcX\",\"tuning-reference_bottleneck-slow-host.md\":\"BlIo6UDW\",\"tuning-reference_bottleneck-small-files.md\":\"DHGMTmPs\",\"tuning-reference_bottleneck-spill.md\":\"PNH7mITt\",\"tuning-reference_bottleneck-straggler.md\":\"DIbXdAcQ\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"CB05v3zh\",\"tuning-reference_bottleneck-utilization.md\":\"DqC_iGkw\",\"tuning-reference_caching.md\":\"B7aQ8asB\",\"tuning-reference_cluster-config.md\":\"CD6vzYtD\",\"tuning-reference_config.md\":\"UvveiWG3\",\"tuning-reference_data-formats.md\":\"bjCAWH3N\",\"tuning-reference_index.md\":\"BQ_NooMV\",\"tuning-reference_intro.md\":\"CobD-lGB\",\"tuning-reference_joins.md\":\"BtKs_CuW\",\"tuning-reference_memory-model.md\":\"CWhvgevP\",\"tuning-reference_metrics.md\":\"mLOh7Apj\",\"tuning-reference_partitioning.md\":\"q0zKF_8X\",\"tuning-reference_pyspark.md\":\"DDCfvN9t\",\"tuning-reference_shuffle.md\":\"BZZ7R4Ix\",\"tuning-reference_spark-architecture.md\":\"Dwzm5avO\",\"tuning-reference_table-formats.md\":\"D6wj-2dX\",\"user-guide_alternative-log-retrieval.md\":\"B4tPGIal\",\"user-guide_getting-started.md\":\"BJvwLEIM\",\"user-guide_mcp-tools.md\":\"DGvolGzt\",\"user-guide_run-comparison.md\":\"CQc1aoU8\",\"user-guide_understanding-findings.md\":\"B29f0ucS\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Alternative log retrieval\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
27
27
 
28
28
  </body>
29
29
  </html>
package/public/docs/contributor-guide/architecture/state-and-history.html CHANGED
@@ -9,9 +9,9 @@
9
9
  <link rel="preload stylesheet" href="/docs/assets/style.DSixAiZE.css" as="style">
10
10
  <link rel="preload stylesheet" href="/docs/vp-icons.css" as="style">
11
11
 
12
- <script type="module" src="/docs/assets/app.APoq_UBs.js"></script>
12
+ <script type="module" src="/docs/assets/app.DsFQcynM.js"></script>
13
13
  <link rel="preload" href="/docs/assets/inter-roman-latin.Di8DUHzh.woff2" as="font" type="font/woff2" crossorigin="">
14
- <link rel="modulepreload" href="/docs/assets/chunks/theme.PavXO_vz.js">
14
+ <link rel="modulepreload" href="/docs/assets/chunks/theme.Ca4dR9dL.js">
15
15
  <link rel="modulepreload" href="/docs/assets/chunks/framework.DSg0KOwT.js">
16
16
  <link rel="modulepreload" href="/docs/assets/contributor-guide_architecture_state-and-history.md.C-xxn0q7.lean.js">
17
17
  <link rel="icon" type="image/svg+xml" href="/docs/favicon.svg">
@@ -23,7 +23,7 @@
23
23
  </head>
24
24
  <body>
25
25
  <div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Contributor Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/development-setup.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Development setup</p><!--]--></a><!----></div><!----></div><section class="VPSidebarItem level-1 collapsible collapsed is-link has-active" data-v-b3fd67f8><div class="item" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/" data-v-b3fd67f8><!--[--><h3 class="text" data-v-b3fd67f8>Architecture</h3><!--]--></a><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-b3fd67f8><span class="vpi-chevron-right caret-icon" data-v-b3fd67f8></span></div></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/overview.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Overview</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/worker-protocol.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Worker protocol</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/state-and-history.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>State & history intake</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/detector-contract.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Detector contract</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/impact-estimation.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Impact estimation</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/widget-rendering.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Widget rendering</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/board-widgets.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Board widgets</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/drill-down.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Drill-down</p><!--]--></a><!----></div><!----></div><!--]--></div></section><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/testing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Testing & verification</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/contributing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Contributing</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_contributor-guide_architecture_state-and-history" data-v-39a288b8><div><h1 id="state-and-history-server-intake" tabindex="-1">State and History Server intake <a class="header-anchor" href="#state-and-history-server-intake" aria-label="Permalink to &quot;State and History Server intake&quot;">​</a></h1><h2 id="state-model" tabindex="-1">State model <a class="header-anchor" href="#state-model" aria-label="Permalink to &quot;State model&quot;">​</a></h2><p><code>src/store/store.ts</code> is a single Zustand store (<code>createStore</code> from <code>zustand/vanilla</code>, wrapped by a <code>useStore</code> hook). It holds all shared state; no component keeps a <code>useState</code> of its own for anything shared: <code>appModel: AppModel</code>, <code>catalog: Finding[]</code>, <code>activeFileId</code>, <code>sessionCache</code> (in-memory snapshot cache for instant file-switching, see <code>session-snapshot.ts</code>), <code>taskDataCache</code>, <code>parse: {pct, lines, etaMs}</code>, <code>status: &#39;idle&#39;|&#39;parsing&#39;|&#39;ready&#39;|&#39;error&#39;</code>, <code>errorMessage</code>, <code>theme</code>, <code>skippedLines</code> (malformed-JSON-line count from the parser&#39;s <code>done</code> payload).</p><p><code>src/store/useIngest.ts</code> is the only writer during a parse. It builds <code>createModelCallbacks</code>&#39; <code>onProgress</code>/<code>onDone</code>/<code>onError</code> handlers to call the store&#39;s setters directly (<code>setParse</code>, <code>setCatalog</code>, <code>setStatus</code>, <code>setSkippedLines</code>, ...). Components never talk to the worker: they use <code>useIngest()</code>&#39;s returned actions (<code>startLoad</code>/<code>startLoadFolder</code>/<code>startLoadFromUrl</code>/<code>pickRecent</code>/<code>getTaskData</code>/ <code>resetToDropZone</code>) and the store&#39;s read state.</p><p><code>resetModel()</code> empties <code>appModel</code>/<code>catalog</code>/<code>taskDataCache</code>/<code>skippedLines</code> on every new parse or reset-to-drop-zone, and bumps <code>modelResetCount</code>. That counter has no setter of its own; only <code>PlanGraphRoute.tsx</code>&#39;s <code>store.subscribe</code> reads it, to evict the plan-graph model memo cache (see <a href="./drill-down.html#plan-graph-view">Plan graph view</a>).</p><h3 id="finding-filter-state" tabindex="-1">Finding filter state <a class="header-anchor" href="#finding-filter-state" aria-label="Permalink to &quot;Finding filter state&quot;">​</a></h3><p>The board-wide finding filter (impact band, raw <code>finding.type</code>, stage) lives outside the Zustand store, in <code>FindingFilterContext</code> (<code>src/view/FindingFilterContext.tsx</code>): a <code>createContext</code>+<code>useState</code><code>FilterSelection</code> (<code>src/view/finding-filter.ts</code>, three <code>Set</code>s) that every widget filters <code>catalog</code> through via <code>filterFindings</code>. It is seeded from the URL&#39;s <code>impact</code>/<code>type</code>/<code>stage</code> query params on mount and nowhere else, so a reload with no params gives the unfiltered board. Every change writes those params back with <code>history.replaceState</code>, never <code>push</code>, so filtering doesn&#39;t grow Back history. A <code>popstate</code> listener re-seeds the selection from the URL, so Back and forward re-apply filters.</p><p>Switching files resets the selection to empty, keyed on the stable file id rather than <code>catalog</code>, so a same-file catalog refresh keeps the active filter. A page reload re-reads the address bar, so deep-linked filtered URLs still restore. Filters are seeded only from the URL and never persisted anywhere else (no localStorage, no restore-as-default): a filtered view silently becoming the default on reload would risk hiding findings from a user who didn&#39;t realize a filter was still active, so a plain reload with no filter params is always the unfiltered board.</p><h3 id="recent-files-vs-session-cache" tabindex="-1">Recent files vs. session cache <a class="header-anchor" href="#recent-files-vs-session-cache" aria-label="Permalink to &quot;Recent files vs. session cache&quot;">​</a></h3><p>Two mechanisms cover reopening a file, at different lifetimes.</p><p><code>sessionCache</code> (in the Zustand store, see <a href="#state-model">State model</a>) is in-memory and per-session: it makes switching between files already loaded in the current tab instant, and it is gone on reload.</p><p>Recent files (<code>src/recent-files.ts</code>, consumed by <code>src/view/useRecentFiles.ts</code>) is IndexedDB-backed and cross-session. It persists each file&#39;s <code>FileSystemFileHandle</code> plus light metadata (name, size, <code>lastModified</code>, app name, issue count, <code>lastOpenedAt</code>), capped at 10 entries with the oldest evicted past the cap, so a file can be reopened after a full browser restart, pending the browser re-granting permission on the handle. Picking a recent entry re-parses from the handle; no parsed model is ever persisted.</p><h2 id="run-comparison" tabindex="-1">Run comparison <a class="header-anchor" href="#run-comparison" aria-label="Permalink to &quot;Run comparison&quot;">​</a></h2><p><code>src/run-comparison.ts</code> is the whole A/B engine. The entry point <code>compareRuns(baseline, candidate)</code> takes two <code>{ label, snapshot }</code> run records (each <code>snapshot</code> a normalized model: <code>app</code>, <code>stages</code>, <code>sql</code>, <code>catalog</code>, <code>executors</code>) and returns one plain object the view renders. It runs on already-parsed snapshots, with no worker involved.</p><ul><li><code>stageIdentity(stage, snapshot)</code> is a run-independent key: <code>normalizeStageName</code> (lowercased, digit-runs and long hex ids collapsed to <code>#</code>) joined with the stage&#39;s SQL-execution plan identity. That identity is scoped to only the plan nodes this stage&#39;s tasks were attributed to (<code>node.stageIds</code>), not the whole tree, so two stages sharing one SQL execution (e.g. a self-join&#39;s two Exchange stages) don&#39;t collapse onto one identity; it falls back to a bottom-up structural fingerprint of the whole resolved <code>planTree</code> (<code>planTreeIdentity</code>, <code>normalizeDetail</code>-normalized: the same normalizer <code>cachingOpportunity</code> uses in <code>src/detectors.ts</code>) when a stage has no such attribution. <code>matchStages(baseSnap, candSnap)</code> indexes each run by that identity and pairs identities that map to exactly one stage on both sides. An identity colliding equally on both sides (same count) is also paired, positionally by sorted stage id: exact when comparing a run against itself (every stage matches itself), a best-effort guess otherwise (two unrelated same-named stages with no SQL/attribution could get cross-paired). Collisions are still recorded in <code>collisionIdentities</code> even when resolved this way; a differing count leaves them there unpaired. <code>coverage</code> reports the matched fraction.</li><li><code>metricDeltas</code> computes whole-run aggregate deltas (wall-clock, spill, task skew p95, failed-task rate, GC, I/O bytes, executor count, ...) as plain sums over all stages, deliberately not gated on stage matching, since matching is unreliable on real logs. Each metric carries a <code>direction</code> (improvement/regression/unchanged) and an <code>unavailableReason</code> when a side lacks the field.</li><li><code>findingsDelta</code> tallies each run&#39;s <code>catalog</code> by <code>(rule × impact band)</code> and reports <code>introduced</code> vs <code>resolved</code> categories: a count diff, also matching-free.</li><li>Only the per-stage skew deltas (<code>stageSkewDeltas</code>) and the pinned-stage panel consume the <code>matchStages</code> pairs, so low match coverage degrades those two surfaces without invalidating the aggregate deltas.</li></ul><p><code>compareRuns</code> also flags <code>confidence: &#39;low&#39;</code> when the two app names differ, or independently when matched stage coverage falls below 0.5 (<code>LOW_COVERAGE_THRESHOLD</code>; either condition alone is enough, both are weak signals, not a hard gate). The view lives in <code>src/view/RunComparison.tsx</code> (the comparison page), <code>CompareLanding.tsx</code> (the two-slot Run A / Run B intake off the landing), and <code>PinnedStageDeltas.tsx</code> (the manual per-stage pinning panel fed by <code>baseStages</code>/<code>candStages</code>).</p><h2 id="history-server-intake-and-recovery" tabindex="-1">History Server intake and recovery <a class="header-anchor" href="#history-server-intake-and-recovery" aria-label="Permalink to &quot;History Server intake and recovery&quot;">​</a></h2><p><code>DropZone</code> keeps the History Server disclosure, Base URL, Application ID, optional Attempt ID, validation/touched state, recoverable SHS error, and local-server reachability in mounted React state rather than Zustand. The local browser-first path is still the default: <strong>Choose file</strong> loads a single event log, while <strong>Choose rolling-log folder</strong> accepts only an <code>eventlog_v2_*</code> directory and directs a rejected folder back to the file picker.</p><p>The three text fields also mirror to <code>window.localStorage</code> (<code>shuffle-works-shs-base-url</code>/<code>-app-id</code>/<code>-attempt-id</code>), read back as each <code>useState</code>&#39;s initializer, so a returning visitor&#39;s values survive a reload; storage access is wrapped in try/catch and silently ignored when unavailable, matching <code>store.ts</code>&#39;s <code>initialTheme</code>/<code>initialWidgetDensity</code> pattern. Both this disclosure&#39;s toggle and the <strong>Other sources</strong> toggle show a chevron (<code>ChevronDownIcon</code>/<code>ChevronUpIcon</code>) that flips with <code>aria-expanded</code>, so the open/closed state has a visual signal beyond the attribute.</p><p>On mount (skipped in <code>compact</code> mode), <code>DropZone</code> probes reachability with an empty, short-timeout <code>fetch(&#39;/shs-proxy&#39;)</code>: a 400 means <code>validateShsRequest</code> (<code>packages/core/src/proxy.js</code>) rejected the empty request synchronously, which only happens when a local server is actually routing that path, so it flips <code>shsReachable</code> to <code>true</code>. A network error, a 404 (static deploy, no such route), or a probe still in flight all leave <code>shsReachable</code> at its default <code>false</code>, so nothing changes on screen after paint unless the server is confirmed present. When <code>shsReachable</code> is <code>true</code>, the landing page shows a neutral callout above the <strong>Other sources</strong> disclosure pointing the user at it; the disclosure itself doesn&#39;t move or auto-expand.</p><p>The collapsed <strong>Fetch from Spark History Server</strong> disclosure requires local-server mode, a reachable History Server, and a supported base application ID: <code>application_&lt;timestamp&gt;_&lt;id&gt;</code>, <code>local-&lt;timestamp&gt;</code>, or <code>app-&lt;identifier&gt;</code>. <code>server/lib/shs-request.js</code> trims and validates the three request fields, accepts only absolute credential-free <code>http:</code>/<code>https:</code> base URLs without a query or fragment, preserves a reverse-proxy path prefix, and canonicalizes the base URL to one trailing slash. The optional attempt is a separate path-safe identifier; neither identifier can contain a path separator. The shared helper builds the encoded <code>/shs-proxy</code> request and the encoded <code>api/v1/applications/&lt;app&gt;[/&lt;attempt&gt;]/logs</code> upstream path from that normalized object only.</p><p>The optional Node server is loopback-only: a narrow CORS proxy, not a general or hosted proxy. It validates the same request contract, sends no credentials, follows no upstream redirects, and returns only stable safe error codes. It never forwards upstream response text, status details, locations, or credentials to the browser.</p><p>Routing preserves the recovery boundary: local file and folder failures use the existing page-level error route. A typed SHS failure instead resets the model to idle and returns to the still-mounted, expanded History Server disclosure, which retains its values and shows safe recovery guidance along with local file intake. During an SHS parse, the mounted intake shows progress in place; successful completion follows the normal dashboard route.</p></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/contributor-guide/architecture/worker-protocol.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>Worker protocol</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="/docs/contributor-guide/architecture/detector-contract.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>Detector contract</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
26
- <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"CWpj01WU\",\"contributor-guide_architecture_detector-contract.md\":\"CgzUsQ6W\",\"contributor-guide_architecture_drill-down.md\":\"BtPdlM7r\",\"contributor-guide_architecture_impact-estimation.md\":\"CooslVJt\",\"contributor-guide_architecture_index.md\":\"3TO9ic6w\",\"contributor-guide_architecture_overview.md\":\"CehiRmGn\",\"contributor-guide_architecture_state-and-history.md\":\"C-xxn0q7\",\"contributor-guide_architecture_widget-rendering.md\":\"R27gQrgY\",\"contributor-guide_architecture_worker-protocol.md\":\"IbnfNrV3\",\"contributor-guide_contributing.md\":\"CvRsdr6J\",\"contributor-guide_development-setup.md\":\"DvAN_9mK\",\"contributor-guide_testing.md\":\"6rIKqSyY\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"Df1YMIHu\",\"tuning-reference_aqe.md\":\"BIsCtLzm\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"CEstB3Ia\",\"tuning-reference_bottleneck-cold-start.md\":\"CEuy-72y\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"CIohQDfn\",\"tuning-reference_bottleneck-failures.md\":\"4z5BXGJ2\",\"tuning-reference_bottleneck-gc.md\":\"DSxzZRK7\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"BaJl__1W\",\"tuning-reference_bottleneck-memory-utilization.md\":\"DbP-SJZc\",\"tuning-reference_bottleneck-retry-waste.md\":\"D5JMjVOt\",\"tuning-reference_bottleneck-shuffle.md\":\"CM-nTmIH\",\"tuning-reference_bottleneck-skew.md\":\"BdUwiDhn\",\"tuning-reference_bottleneck-slow-host.md\":\"BlIo6UDW\",\"tuning-reference_bottleneck-small-files.md\":\"B8kloyx8\",\"tuning-reference_bottleneck-spill.md\":\"PNH7mITt\",\"tuning-reference_bottleneck-straggler.md\":\"DY36fHN5\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"QTV7O8kU\",\"tuning-reference_bottleneck-utilization.md\":\"DTiueZC3\",\"tuning-reference_caching.md\":\"B7aQ8asB\",\"tuning-reference_cluster-config.md\":\"ZVmDGsQ3\",\"tuning-reference_config.md\":\"UvveiWG3\",\"tuning-reference_data-formats.md\":\"bjCAWH3N\",\"tuning-reference_index.md\":\"BQ_NooMV\",\"tuning-reference_intro.md\":\"CobD-lGB\",\"tuning-reference_joins.md\":\"BtKs_CuW\",\"tuning-reference_memory-model.md\":\"DhT-n4y3\",\"tuning-reference_metrics.md\":\"mLOh7Apj\",\"tuning-reference_partitioning.md\":\"q0zKF_8X\",\"tuning-reference_pyspark.md\":\"DDCfvN9t\",\"tuning-reference_shuffle.md\":\"BZZ7R4Ix\",\"tuning-reference_spark-architecture.md\":\"Dwzm5avO\",\"tuning-reference_table-formats.md\":\"D6wj-2dX\",\"user-guide_alternative-log-retrieval.md\":\"B4tPGIal\",\"user-guide_getting-started.md\":\"BJvwLEIM\",\"user-guide_mcp-tools.md\":\"Vi3RoflJ\",\"user-guide_run-comparison.md\":\"CQc1aoU8\",\"user-guide_understanding-findings.md\":\"DL1UDhvR\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Alternative log retrieval\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
26
+ <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"9ToZXcTk\",\"contributor-guide_architecture_detector-contract.md\":\"aQQfP7pj\",\"contributor-guide_architecture_drill-down.md\":\"Bawx6NgP\",\"contributor-guide_architecture_impact-estimation.md\":\"sf9DU2bA\",\"contributor-guide_architecture_index.md\":\"3TO9ic6w\",\"contributor-guide_architecture_overview.md\":\"BXQdgG5u\",\"contributor-guide_architecture_state-and-history.md\":\"C-xxn0q7\",\"contributor-guide_architecture_widget-rendering.md\":\"DYwFeyjV\",\"contributor-guide_architecture_worker-protocol.md\":\"P7d1vtEJ\",\"contributor-guide_contributing.md\":\"CvRsdr6J\",\"contributor-guide_development-setup.md\":\"DvMV1xhQ\",\"contributor-guide_testing.md\":\"BU6vyA-9\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"Df1YMIHu\",\"tuning-reference_aqe.md\":\"BIsCtLzm\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"CEstB3Ia\",\"tuning-reference_bottleneck-cold-start.md\":\"CEuy-72y\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"CIohQDfn\",\"tuning-reference_bottleneck-failures.md\":\"CAsFf3RR\",\"tuning-reference_bottleneck-gc.md\":\"B4T3kJYV\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"BaJl__1W\",\"tuning-reference_bottleneck-memory-utilization.md\":\"DbP-SJZc\",\"tuning-reference_bottleneck-retry-waste.md\":\"D5JMjVOt\",\"tuning-reference_bottleneck-shuffle.md\":\"DDMQVOW6\",\"tuning-reference_bottleneck-skew.md\":\"B0MzhWcX\",\"tuning-reference_bottleneck-slow-host.md\":\"BlIo6UDW\",\"tuning-reference_bottleneck-small-files.md\":\"DHGMTmPs\",\"tuning-reference_bottleneck-spill.md\":\"PNH7mITt\",\"tuning-reference_bottleneck-straggler.md\":\"DIbXdAcQ\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"CB05v3zh\",\"tuning-reference_bottleneck-utilization.md\":\"DqC_iGkw\",\"tuning-reference_caching.md\":\"B7aQ8asB\",\"tuning-reference_cluster-config.md\":\"CD6vzYtD\",\"tuning-reference_config.md\":\"UvveiWG3\",\"tuning-reference_data-formats.md\":\"bjCAWH3N\",\"tuning-reference_index.md\":\"BQ_NooMV\",\"tuning-reference_intro.md\":\"CobD-lGB\",\"tuning-reference_joins.md\":\"BtKs_CuW\",\"tuning-reference_memory-model.md\":\"CWhvgevP\",\"tuning-reference_metrics.md\":\"mLOh7Apj\",\"tuning-reference_partitioning.md\":\"q0zKF_8X\",\"tuning-reference_pyspark.md\":\"DDCfvN9t\",\"tuning-reference_shuffle.md\":\"BZZ7R4Ix\",\"tuning-reference_spark-architecture.md\":\"Dwzm5avO\",\"tuning-reference_table-formats.md\":\"D6wj-2dX\",\"user-guide_alternative-log-retrieval.md\":\"B4tPGIal\",\"user-guide_getting-started.md\":\"BJvwLEIM\",\"user-guide_mcp-tools.md\":\"DGvolGzt\",\"user-guide_run-comparison.md\":\"CQc1aoU8\",\"user-guide_understanding-findings.md\":\"B29f0ucS\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Alternative log retrieval\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
27
27
 
28
28
  </body>
29
29
  </html>