sparkforensics-server 0.2.2 → 0.2.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (213) hide show
  1. package/package.json +1 -1
  2. package/public/assets/{AutoscalingChurn-B96FKx97.js → AutoscalingChurn-BOiUUfuY.js} +1 -1
  3. package/public/assets/CacheUtilization-DwX7lisd.js +1 -0
  4. package/public/assets/CachingOpportunity-DUE3GKpx.js +1 -0
  5. package/public/assets/{ColdStart-w8QYTNKo.js → ColdStart-ZHxPrJmI.js} +1 -1
  6. package/public/assets/ConfigAudit-BkVnq2XE.js +1 -0
  7. package/public/assets/{CoreUsageArea-o3kqQwyi.js → CoreUsageArea-DobrSXk2.js} +1 -1
  8. package/public/assets/Dashboard-y3_9iQJo.js +1 -0
  9. package/public/assets/{DuplicatePlanSubtree-Cm_UU3lY.js → DuplicatePlanSubtree-BSQZFeNq.js} +1 -1
  10. package/public/assets/{DurationHistogram-ltF-7M0A.js → DurationHistogram-DlQv4ZCB.js} +1 -1
  11. package/public/assets/{EmptyState-BCA3xioa.js → EmptyState-DqnSxTjJ.js} +1 -1
  12. package/public/assets/{ExecutorUtilization-Cn69i0uq.js → ExecutorUtilization-CuQUxaCd.js} +1 -1
  13. package/public/assets/GcPressure-CLT5hkMJ.js +1 -0
  14. package/public/assets/{ImpactEstimate-BGt-4WX5.js → ImpactEstimate-Df_laG-n.js} +1 -1
  15. package/public/assets/{IncompleteRun-s6uQ8OLf.js → IncompleteRun-CpIkS4XZ.js} +1 -1
  16. package/public/assets/{JobFailures-DJ2dDvPA.js → JobFailures-DKbvYc4J.js} +1 -1
  17. package/public/assets/{MemoryUtilization-C9G7P94X.js → MemoryUtilization-C5kHmXIb.js} +1 -1
  18. package/public/assets/{OverBroadcast-DYauoYWQ.js → OverBroadcast-C5pF66aq.js} +1 -1
  19. package/public/assets/PartitionSizing-BvEJQ5M1.js +1 -0
  20. package/public/assets/{PlanExplorer-_2oB5ouy.js → PlanExplorer-DjGTZifP.js} +1 -1
  21. package/public/assets/PlanGraphRoute-DoIGA7_H.js +1 -0
  22. package/public/assets/{RetryWaste-D_m8ASwP.js → RetryWaste-C0znHd_w.js} +1 -1
  23. package/public/assets/{RowPagination-BLsm8I6i.js → RowPagination-2icPidMP.js} +1 -1
  24. package/public/assets/ShuffleIO-L_XSnJ_V.js +1 -0
  25. package/public/assets/{Skew-BGLJuZZR.js → Skew-w4uV5H_g.js} +1 -1
  26. package/public/assets/{SlowHost-ChdWVvGO.js → SlowHost-D5tWJ1UR.js} +1 -1
  27. package/public/assets/{SmallFiles-gTilBtjW.js → SmallFiles-CTtNiSOh.js} +1 -1
  28. package/public/assets/{SpeculationWaste-CVzgJnSj.js → SpeculationWaste-xCKwjz2K.js} +1 -1
  29. package/public/assets/{Spill-NGJGQaTB.js → Spill-BT8my24K.js} +1 -1
  30. package/public/assets/{StageFailed-sGhki53z.js → StageFailed-De3vVDbc.js} +1 -1
  31. package/public/assets/{StageFindingGroup-AyI7BMd3.js → StageFindingGroup-R4OW-g4U.js} +1 -1
  32. package/public/assets/{StageHeader-BTzGkK8k.js → StageHeader-CGeWqC-0.js} +1 -1
  33. package/public/assets/{StagePill-BlMOW_pU.js → StagePill-BwwKLcFl.js} +1 -1
  34. package/public/assets/{StageShape-DfOW93Z1.js → StageShape-D4LmPVyV.js} +1 -1
  35. package/public/assets/{StageSlowness-BMZ-K_5e.js → StageSlowness-8-z6qn4Q.js} +1 -1
  36. package/public/assets/Straggler-BIgxsowI.js +1 -0
  37. package/public/assets/{TaskFailures-DGDwUK2A.js → TaskFailures-B9lLt6Rw.js} +1 -1
  38. package/public/assets/{TinyTask-CfT8zZDr.js → TinyTask-DYXks5sT.js} +1 -1
  39. package/public/assets/{UnderBroadcast-CiOiJ0k1.js → UnderBroadcast-BWqWMpeG.js} +1 -1
  40. package/public/assets/{finding-anchor-DEV8JvgH.js → finding-anchor-CDbHtqKT.js} +1 -1
  41. package/public/assets/{inbox-_QCYkBsP.js → inbox-M_EuWNiK.js} +1 -1
  42. package/public/assets/index-pnImGbgQ.js +9 -0
  43. package/public/assets/parser-worker-C3BAO6z9.js +117 -0
  44. package/public/assets/{select-kuy1MiAo.js → select-C-MZGNpb.js} +1 -1
  45. package/public/assets/{toggle-group-CUd-KWkp.js → toggle-group-C3nz7UQ_.js} +1 -1
  46. package/public/assets/useRecentFiles-0XuhS91q.js +11 -0
  47. package/public/assets/{useSortMode-BaeAA5cw.js → useSortMode-DHlKZeWv.js} +1 -1
  48. package/public/assets/zstd-worker-C1Jq9lfb.js +1 -0
  49. package/public/docs/404.html +2 -2
  50. package/public/docs/assets/{app.APoq_UBs.js → app.DsFQcynM.js} +1 -1
  51. package/public/docs/assets/chunks/@localSearchIndexroot.BEfYag5Q.js +1 -0
  52. package/public/docs/assets/chunks/{VPLocalSearchBox.B3TPzlHq.js → VPLocalSearchBox.oa8NbaiV.js} +1 -1
  53. package/public/docs/assets/chunks/{theme.PavXO_vz.js → theme.Ca4dR9dL.js} +2 -2
  54. package/public/docs/assets/contributor-guide_architecture_board-widgets.md.9ToZXcTk.js +1 -0
  55. package/public/docs/assets/{contributor-guide_architecture_board-widgets.md.CWpj01WU.lean.js → contributor-guide_architecture_board-widgets.md.9ToZXcTk.lean.js} +1 -1
  56. package/public/docs/assets/contributor-guide_architecture_detector-contract.md.aQQfP7pj.js +1 -0
  57. package/public/docs/assets/{contributor-guide_architecture_detector-contract.md.CgzUsQ6W.lean.js → contributor-guide_architecture_detector-contract.md.aQQfP7pj.lean.js} +1 -1
  58. package/public/docs/assets/{contributor-guide_architecture_drill-down.md.BtPdlM7r.js → contributor-guide_architecture_drill-down.md.Bawx6NgP.js} +1 -1
  59. package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.sf9DU2bA.js +1 -0
  60. package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.sf9DU2bA.lean.js +1 -0
  61. package/public/docs/assets/contributor-guide_architecture_overview.md.BXQdgG5u.js +1 -0
  62. package/public/docs/assets/{contributor-guide_architecture_overview.md.CehiRmGn.lean.js → contributor-guide_architecture_overview.md.BXQdgG5u.lean.js} +1 -1
  63. package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.DYwFeyjV.js +1 -0
  64. package/public/docs/assets/{contributor-guide_architecture_worker-protocol.md.IbnfNrV3.js → contributor-guide_architecture_worker-protocol.md.P7d1vtEJ.js} +2 -2
  65. package/public/docs/assets/{contributor-guide_architecture_worker-protocol.md.IbnfNrV3.lean.js → contributor-guide_architecture_worker-protocol.md.P7d1vtEJ.lean.js} +1 -1
  66. package/public/docs/assets/{contributor-guide_development-setup.md.DvAN_9mK.js → contributor-guide_development-setup.md.DvMV1xhQ.js} +3 -3
  67. package/public/docs/assets/contributor-guide_development-setup.md.DvMV1xhQ.lean.js +1 -0
  68. package/public/docs/assets/contributor-guide_testing.md.BU6vyA-9.js +1 -0
  69. package/public/docs/assets/contributor-guide_testing.md.BU6vyA-9.lean.js +1 -0
  70. package/public/docs/assets/{tuning-reference_bottleneck-failures.md.4z5BXGJ2.js → tuning-reference_bottleneck-failures.md.CAsFf3RR.js} +1 -1
  71. package/public/docs/assets/{tuning-reference_bottleneck-gc.md.DSxzZRK7.js → tuning-reference_bottleneck-gc.md.B4T3kJYV.js} +2 -2
  72. package/public/docs/assets/{tuning-reference_bottleneck-gc.md.DSxzZRK7.lean.js → tuning-reference_bottleneck-gc.md.B4T3kJYV.lean.js} +1 -1
  73. package/public/docs/assets/tuning-reference_bottleneck-shuffle.md.DDMQVOW6.js +12 -0
  74. package/public/docs/assets/{tuning-reference_bottleneck-shuffle.md.CM-nTmIH.lean.js → tuning-reference_bottleneck-shuffle.md.DDMQVOW6.lean.js} +1 -1
  75. package/public/docs/assets/{tuning-reference_bottleneck-skew.md.BdUwiDhn.js → tuning-reference_bottleneck-skew.md.B0MzhWcX.js} +4 -4
  76. package/public/docs/assets/{tuning-reference_bottleneck-skew.md.BdUwiDhn.lean.js → tuning-reference_bottleneck-skew.md.B0MzhWcX.lean.js} +1 -1
  77. package/public/docs/assets/{tuning-reference_bottleneck-small-files.md.B8kloyx8.js → tuning-reference_bottleneck-small-files.md.DHGMTmPs.js} +2 -2
  78. package/public/docs/assets/{tuning-reference_bottleneck-small-files.md.B8kloyx8.lean.js → tuning-reference_bottleneck-small-files.md.DHGMTmPs.lean.js} +1 -1
  79. package/public/docs/assets/{tuning-reference_bottleneck-straggler.md.DY36fHN5.js → tuning-reference_bottleneck-straggler.md.DIbXdAcQ.js} +2 -2
  80. package/public/docs/assets/{tuning-reference_bottleneck-straggler.md.DY36fHN5.lean.js → tuning-reference_bottleneck-straggler.md.DIbXdAcQ.lean.js} +1 -1
  81. package/public/docs/assets/{tuning-reference_bottleneck-tiny-tasks.md.QTV7O8kU.js → tuning-reference_bottleneck-tiny-tasks.md.CB05v3zh.js} +1 -1
  82. package/public/docs/assets/tuning-reference_bottleneck-utilization.md.DqC_iGkw.js +5 -0
  83. package/public/docs/assets/tuning-reference_bottleneck-utilization.md.DqC_iGkw.lean.js +1 -0
  84. package/public/docs/assets/{tuning-reference_cluster-config.md.ZVmDGsQ3.js → tuning-reference_cluster-config.md.CD6vzYtD.js} +1 -1
  85. package/public/docs/assets/{tuning-reference_cluster-config.md.ZVmDGsQ3.lean.js → tuning-reference_cluster-config.md.CD6vzYtD.lean.js} +1 -1
  86. package/public/docs/assets/{tuning-reference_memory-model.md.DhT-n4y3.js → tuning-reference_memory-model.md.CWhvgevP.js} +1 -1
  87. package/public/docs/assets/{tuning-reference_memory-model.md.DhT-n4y3.lean.js → tuning-reference_memory-model.md.CWhvgevP.lean.js} +1 -1
  88. package/public/docs/assets/{user-guide_mcp-tools.md.Vi3RoflJ.js → user-guide_mcp-tools.md.DGvolGzt.js} +1 -1
  89. package/public/docs/assets/user-guide_understanding-findings.md.B29f0ucS.js +1 -0
  90. package/public/docs/assets/{user-guide_understanding-findings.md.DL1UDhvR.lean.js → user-guide_understanding-findings.md.B29f0ucS.lean.js} +1 -1
  91. package/public/docs/contributor-guide/architecture/board-widgets.html +5 -5
  92. package/public/docs/contributor-guide/architecture/detector-contract.html +5 -5
  93. package/public/docs/contributor-guide/architecture/drill-down.html +5 -5
  94. package/public/docs/contributor-guide/architecture/impact-estimation.html +5 -5
  95. package/public/docs/contributor-guide/architecture/index.html +3 -3
  96. package/public/docs/contributor-guide/architecture/overview.html +5 -5
  97. package/public/docs/contributor-guide/architecture/state-and-history.html +3 -3
  98. package/public/docs/contributor-guide/architecture/widget-rendering.html +5 -5
  99. package/public/docs/contributor-guide/architecture/worker-protocol.html +6 -6
  100. package/public/docs/contributor-guide/contributing.html +3 -3
  101. package/public/docs/contributor-guide/development-setup.html +5 -5
  102. package/public/docs/contributor-guide/testing.html +5 -5
  103. package/public/docs/hashmap.json +1 -1
  104. package/public/docs/index.html +3 -3
  105. package/public/docs/tuning-reference/anti-patterns.html +3 -3
  106. package/public/docs/tuning-reference/aqe.html +3 -3
  107. package/public/docs/tuning-reference/bottleneck-broadcast-sizing.html +3 -3
  108. package/public/docs/tuning-reference/bottleneck-cold-start.html +3 -3
  109. package/public/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html +3 -3
  110. package/public/docs/tuning-reference/bottleneck-failures.html +5 -5
  111. package/public/docs/tuning-reference/bottleneck-gc.html +5 -5
  112. package/public/docs/tuning-reference/bottleneck-job-failure-rate.html +3 -3
  113. package/public/docs/tuning-reference/bottleneck-memory-utilization.html +3 -3
  114. package/public/docs/tuning-reference/bottleneck-retry-waste.html +3 -3
  115. package/public/docs/tuning-reference/bottleneck-shuffle.html +6 -6
  116. package/public/docs/tuning-reference/bottleneck-skew.html +7 -7
  117. package/public/docs/tuning-reference/bottleneck-slow-host.html +3 -3
  118. package/public/docs/tuning-reference/bottleneck-small-files.html +6 -6
  119. package/public/docs/tuning-reference/bottleneck-spill.html +3 -3
  120. package/public/docs/tuning-reference/bottleneck-straggler.html +5 -5
  121. package/public/docs/tuning-reference/bottleneck-tiny-tasks.html +5 -5
  122. package/public/docs/tuning-reference/bottleneck-utilization.html +6 -6
  123. package/public/docs/tuning-reference/caching.html +3 -3
  124. package/public/docs/tuning-reference/cluster-config.html +5 -5
  125. package/public/docs/tuning-reference/config.html +3 -3
  126. package/public/docs/tuning-reference/data-formats.html +3 -3
  127. package/public/docs/tuning-reference/index.html +3 -3
  128. package/public/docs/tuning-reference/intro.html +3 -3
  129. package/public/docs/tuning-reference/joins.html +3 -3
  130. package/public/docs/tuning-reference/memory-model.html +5 -5
  131. package/public/docs/tuning-reference/metrics.html +3 -3
  132. package/public/docs/tuning-reference/partitioning.html +3 -3
  133. package/public/docs/tuning-reference/pyspark.html +3 -3
  134. package/public/docs/tuning-reference/shuffle.html +3 -3
  135. package/public/docs/tuning-reference/spark-architecture.html +3 -3
  136. package/public/docs/tuning-reference/table-formats.html +3 -3
  137. package/public/docs/user-guide/alternative-log-retrieval.html +3 -3
  138. package/public/docs/user-guide/getting-started.html +3 -3
  139. package/public/docs/user-guide/mcp-tools.html +5 -5
  140. package/public/docs/user-guide/run-comparison.html +3 -3
  141. package/public/docs/user-guide/understanding-findings.html +5 -5
  142. package/public/index.html +1 -1
  143. package/vendor-core/cli/collect-run.js +3 -2
  144. package/vendor-core/cli/native-zstd.js +351 -0
  145. package/vendor-core/detectors.js +243 -53
  146. package/vendor-core/docs-config.js +34 -8
  147. package/vendor-core/docs-content/chapters/03-memory-model.md +39 -0
  148. package/vendor-core/docs-content/chapters/11-cluster-config.md +40 -0
  149. package/vendor-core/docs-content/detection/gc.md +2 -0
  150. package/vendor-core/docs-content/detection/host.md +2 -1
  151. package/vendor-core/docs-content/detection/plan.md +3 -1
  152. package/vendor-core/docs-content/detection/shape.md +2 -1
  153. package/vendor-core/docs-content/detection/shfl.md +2 -1
  154. package/vendor-core/docs-content/detection/spill.md +1 -1
  155. package/vendor-core/docs-content/detection/strag.md +2 -1
  156. package/vendor-core/docs-content/detection/tiny.md +2 -1
  157. package/vendor-core/docs-content/tuning/failures.md +1 -1
  158. package/vendor-core/docs-content/tuning/gc.md +11 -4
  159. package/vendor-core/docs-content/tuning/shuffle.md +25 -5
  160. package/vendor-core/docs-content/tuning/skew.md +14 -6
  161. package/vendor-core/docs-content/tuning/small-files.md +12 -7
  162. package/vendor-core/docs-content/tuning/straggler.md +34 -0
  163. package/vendor-core/docs-content/tuning/tiny-tasks.md +1 -1
  164. package/vendor-core/docs-content/tuning/utilization.md +57 -7
  165. package/vendor-core/docs-content/upstream.json +4 -0
  166. package/vendor-core/event-handlers.js +321 -69
  167. package/vendor-core/event-schemas.js +8 -6
  168. package/vendor-core/evidence-report.js +3 -1
  169. package/vendor-core/impact-estimator.js +170 -34
  170. package/vendor-core/mcp-tools.js +20 -7
  171. package/vendor-core/occupancy.js +71 -2
  172. package/vendor-core/parser-worker.js +56 -24
  173. package/vendor-core/plan-summary.js +5 -1
  174. package/vendor-core/run-comparison.js +1 -15
  175. package/vendor-core/shs-fetch.js +18 -7
  176. package/vendor-core/shs-load.js +2 -1
  177. package/vendor-core/stage-quantiles.js +111 -3
  178. package/vendor-core/string-hash.js +15 -0
  179. package/vendor-core/types.js +14 -1
  180. package/vendor-core/vendor/fzstd.js +94 -18
  181. package/vendor-core/zstd-worker-client.js +180 -0
  182. package/vendor-core/zstd-worker.js +103 -0
  183. package/public/assets/CacheUtilization-lKua3J1x.js +0 -1
  184. package/public/assets/CachingOpportunity-U0g-8HPV.js +0 -1
  185. package/public/assets/ConfigAudit-ClZp5BpV.js +0 -1
  186. package/public/assets/Dashboard-Bjl-MYMm.js +0 -1
  187. package/public/assets/GcPressure-CYsmU1d7.js +0 -1
  188. package/public/assets/PartitionSizing-Cz_B0-uG.js +0 -1
  189. package/public/assets/PlanGraphRoute-Cj-GWM8W.js +0 -1
  190. package/public/assets/ShuffleIO-pyDhxdaz.js +0 -1
  191. package/public/assets/Straggler-dffcAN7t.js +0 -1
  192. package/public/assets/index-3vlFVNCg.js +0 -9
  193. package/public/assets/parser-worker-B2bc0lWe.js +0 -112
  194. package/public/assets/useRecentFiles-D0OA_EOK.js +0 -11
  195. package/public/docs/assets/chunks/@localSearchIndexroot.CvM5fuxe.js +0 -1
  196. package/public/docs/assets/contributor-guide_architecture_board-widgets.md.CWpj01WU.js +0 -1
  197. package/public/docs/assets/contributor-guide_architecture_detector-contract.md.CgzUsQ6W.js +0 -1
  198. package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.CooslVJt.js +0 -1
  199. package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.CooslVJt.lean.js +0 -1
  200. package/public/docs/assets/contributor-guide_architecture_overview.md.CehiRmGn.js +0 -1
  201. package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.R27gQrgY.js +0 -1
  202. package/public/docs/assets/contributor-guide_development-setup.md.DvAN_9mK.lean.js +0 -1
  203. package/public/docs/assets/contributor-guide_testing.md.6rIKqSyY.js +0 -1
  204. package/public/docs/assets/contributor-guide_testing.md.6rIKqSyY.lean.js +0 -1
  205. package/public/docs/assets/tuning-reference_bottleneck-shuffle.md.CM-nTmIH.js +0 -12
  206. package/public/docs/assets/tuning-reference_bottleneck-utilization.md.DTiueZC3.js +0 -5
  207. package/public/docs/assets/tuning-reference_bottleneck-utilization.md.DTiueZC3.lean.js +0 -1
  208. package/public/docs/assets/user-guide_understanding-findings.md.DL1UDhvR.js +0 -1
  209. /package/public/docs/assets/{contributor-guide_architecture_drill-down.md.BtPdlM7r.lean.js → contributor-guide_architecture_drill-down.md.Bawx6NgP.lean.js} +0 -0
  210. /package/public/docs/assets/{contributor-guide_architecture_widget-rendering.md.R27gQrgY.lean.js → contributor-guide_architecture_widget-rendering.md.DYwFeyjV.lean.js} +0 -0
  211. /package/public/docs/assets/{tuning-reference_bottleneck-failures.md.4z5BXGJ2.lean.js → tuning-reference_bottleneck-failures.md.CAsFf3RR.lean.js} +0 -0
  212. /package/public/docs/assets/{tuning-reference_bottleneck-tiny-tasks.md.QTV7O8kU.lean.js → tuning-reference_bottleneck-tiny-tasks.md.CB05v3zh.lean.js} +0 -0
  213. /package/public/docs/assets/{user-guide_mcp-tools.md.Vi3RoflJ.lean.js → user-guide_mcp-tools.md.DGvolGzt.lean.js} +0 -0
package/public/docs/user-guide/understanding-findings.html CHANGED
@@ -9,11 +9,11 @@
9
9
  <link rel="preload stylesheet" href="/docs/assets/style.DSixAiZE.css" as="style">
10
10
  <link rel="preload stylesheet" href="/docs/vp-icons.css" as="style">
11
11
 
12
- <script type="module" src="/docs/assets/app.APoq_UBs.js"></script>
12
+ <script type="module" src="/docs/assets/app.DsFQcynM.js"></script>
13
13
  <link rel="preload" href="/docs/assets/inter-roman-latin.Di8DUHzh.woff2" as="font" type="font/woff2" crossorigin="">
14
- <link rel="modulepreload" href="/docs/assets/chunks/theme.PavXO_vz.js">
14
+ <link rel="modulepreload" href="/docs/assets/chunks/theme.Ca4dR9dL.js">
15
15
  <link rel="modulepreload" href="/docs/assets/chunks/framework.DSg0KOwT.js">
16
- <link rel="modulepreload" href="/docs/assets/user-guide_understanding-findings.md.DL1UDhvR.lean.js">
16
+ <link rel="modulepreload" href="/docs/assets/user-guide_understanding-findings.md.B29f0ucS.lean.js">
17
17
  <link rel="icon" type="image/svg+xml" href="/docs/favicon.svg">
18
18
  <link rel="preconnect" href="https://fonts.googleapis.com">
19
19
  <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin="">
@@ -22,8 +22,8 @@
22
22
  <script id="check-mac-os">document.documentElement.classList.toggle("mac",/Mac|iPhone|iPod|iPad/i.test(navigator.platform));</script>
23
23
  </head>
24
24
  <body>
25
- <div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>User Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/getting-started.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Getting started</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/understanding-findings.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Understanding findings</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/run-comparison.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Run comparison mode</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/mcp-tools.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>MCP tools reference</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/alternative-log-retrieval.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Alternative log retrieval</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_user-guide_understanding-findings" data-v-39a288b8><div><h1 id="understanding-findings" tabindex="-1">Understanding findings <a class="header-anchor" href="#understanding-findings" aria-label="Permalink to &quot;Understanding findings&quot;">​</a></h1><p>Every flagged problem carries a short ALL-CAPS tag. This page has one entry per tag: what it means, and what to do about it.</p><p>Turn on <a href="./getting-started.html#advanced-view">Advanced view</a> in the dashboard to see, per finding, the confidence level and a link to the same background reading collected here, opened in an in-app &quot;Reference&quot; panel next to the board. A few tags share that reading with another tag, because the underlying Spark-tuning material overlaps: <code>SFAIL</code> with <code>FAIL</code>, <code>PART</code> with <code>SHFL</code>, <code>SPEC</code> with <code>STRAG</code>, and <code>CACHE</code>/<code>LOCAL</code> with <code>UTIL</code>.</p><h2 id="per-stage" tabindex="-1">Per-stage <a class="header-anchor" href="#per-stage" aria-label="Permalink to &quot;Per-stage&quot;">​</a></h2><h3 id="skew" tabindex="-1"><code>SKEW</code>: Task skew <a class="header-anchor" href="#skew" aria-label="Permalink to &quot;`SKEW`: Task skew {#skew}&quot;">​</a></h3><p>A small number of tasks take much longer than their peers in the same stage. For join-driven skew, enable AQE skew-join handling (<code>spark.sql.adaptive.skewJoin.enabled</code>); otherwise salt the key or repartition on a better key.</p><h3 id="shfl" tabindex="-1"><code>SHFL</code>: Shuffle I/O <a class="header-anchor" href="#shfl" aria-label="Permalink to &quot;`SHFL`: Shuffle I/O {#shfl}&quot;">​</a></h3><p>Tasks move a large amount of intermediate data between stages. Raise <code>spark.sql.shuffle.partitions</code>, or add a broadcast join.</p><h3 id="spill" tabindex="-1"><code>SPILL</code>: Memory and disk spill <a class="header-anchor" href="#spill" aria-label="Permalink to &quot;`SPILL`: Memory and disk spill {#spill}&quot;">​</a></h3><p>Tasks are writing data out of memory, which slows execution. Two spill patterns get flagged differently: skew spill, where a few heavy tasks spill while most don&#39;t (rebalance partitioning), and volume spill, where most tasks spill because the data genuinely exceeds available memory (add partitions).</p><h3 id="gc" tabindex="-1"><code>GC</code>: Garbage collection pressure <a class="header-anchor" href="#gc" aria-label="Permalink to &quot;`GC`: Garbage collection pressure {#gc}&quot;">​</a></h3><p>Tasks spend an unusually large share of time reclaiming memory. Reduce object creation: use primitive types, avoid UDFs, or raise executor memory.</p><h3 id="fail" tabindex="-1"><code>FAIL</code>: Failed tasks <a class="header-anchor" href="#fail" aria-label="Permalink to &quot;`FAIL`: Failed tasks {#fail}&quot;">​</a></h3><p>Tasks fail often enough to affect the stage. Failed tasks point to executor instability or data-driven errors: check driver logs for the dominant failure reason.</p><h3 id="sfail" tabindex="-1"><code>SFAIL</code>: Failed stage <a class="header-anchor" href="#sfail" aria-label="Permalink to &quot;`SFAIL`: Failed stage {#sfail}&quot;">​</a></h3><p>A stage attempt failed outright rather than losing individual tasks within it. Inspect the driver log for the failure reason and the job that triggered it.</p><h3 id="strag" tabindex="-1"><code>STRAG</code>: Straggler tasks <a class="header-anchor" href="#strag" aria-label="Permalink to &quot;`STRAG`: Straggler tasks {#strag}&quot;">​</a></h3><p>A few tasks run much slower than the rest of their stage. Rule out a GC pause or a slow shuffle fetch before assuming a hardware issue; if a skewed key is the real cause, that&#39;s a candidate for AQE&#39;s skew-join handling.</p><h3 id="spec" tabindex="-1"><code>SPEC</code>: Speculation waste <a class="header-anchor" href="#spec" aria-label="Permalink to &quot;`SPEC`: Speculation waste {#spec}&quot;">​</a></h3><p>Speculative task attempts used a lot of executor time without confirming a genuine straggler. Self-flags a confidence that scales with how far the wasted time sits past the threshold: these thresholds are still a design spike, not yet validated against real-world runs. If task durations are just naturally variable rather than genuine stragglers, tune <code>spark.speculation.multiplier</code>/<code>spark.speculation.quantile</code>.</p><h3 id="retry" tabindex="-1"><code>RETRY</code>: Retry waste <a class="header-anchor" href="#retry" aria-label="Permalink to &quot;`RETRY`: Retry waste {#retry}&quot;">​</a></h3><p>Repeated task attempts ate into execution time even though the stage completed. Investigate executor loss or fetch failures.</p><h3 id="tiny" tabindex="-1"><code>TINY</code>: Tiny tasks <a class="header-anchor" href="#tiny" aria-label="Permalink to &quot;`TINY`: Tiny tasks {#tiny}&quot;">​</a></h3><p>Many very short tasks add scheduling overhead out of proportion to the work each one does. Repartition to fewer, larger tasks.</p><h3 id="part" tabindex="-1"><code>PART</code>: Partition sizing <a class="header-anchor" href="#part" aria-label="Permalink to &quot;`PART`: Partition sizing {#part}&quot;">​</a></h3><p>Shuffle partitions are too large, too uneven, or too few for the work. A single shuffle partition over 5 GB, for example, will OOM or spill heavily: repartition to break it up before the stage runs.</p><h3 id="slow" tabindex="-1"><code>SLOW</code>: Stage slowness <a class="header-anchor" href="#slow" aria-label="Permalink to &quot;`SLOW`: Stage slowness {#slow}&quot;">​</a></h3><p>A stage ran long overall without a more specific cause getting flagged. Often a partition-count problem: raise parallelism via <code>spark.sql.shuffle.partitions</code> or <code>spark.default.parallelism</code>, or check for a large per-task data volume driving heavy shuffle and spill.</p><h3 id="shape" tabindex="-1"><code>SHAPE</code>: Stage shape <a class="header-anchor" href="#shape" aria-label="Permalink to &quot;`SHAPE`: Stage shape {#shape}&quot;">​</a></h3><p>The stage has an inefficient task count, output shape, or task-to-stage balance: for example, one straggler task taking a large fraction of the stage&#39;s wall-clock time.</p><h3 id="host" tabindex="-1"><code>HOST</code>: Slow host <a class="header-anchor" href="#host" aria-label="Permalink to &quot;`HOST`: Slow host {#host}&quot;">​</a></h3><p>One executor is much slower than its peers. It may just hold data locality for its tasks or carry one heavy stage, rather than a hardware fault. Enable <code>spark.speculation</code> to relaunch a lagging task automatically.</p><h2 id="app-level" tabindex="-1">App-level <a class="header-anchor" href="#app-level" aria-label="Permalink to &quot;App-level&quot;">​</a></h2><h3 id="cold" tabindex="-1"><code>COLD</code>: Executor cold start <a class="header-anchor" href="#cold" aria-label="Permalink to &quot;`COLD`: Executor cold start {#cold}&quot;">​</a></h3><p>New executors take time to become available for work. Pre-warm the cluster, or use dynamic allocation.</p><h3 id="util" tabindex="-1"><code>UTIL</code>: Low utilization <a class="header-anchor" href="#util" aria-label="Permalink to &quot;`UTIL`: Low utilization {#util}&quot;">​</a></h3><p>Allocated executors sit idle for a large share of the application run. Consider a smaller cluster, or enable dynamic allocation.</p><h3 id="mem" tabindex="-1"><code>MEM</code>: Memory utilization <a class="header-anchor" href="#mem" aria-label="Permalink to &quot;`MEM`: Memory utilization {#mem}&quot;">​</a></h3><p>Executor memory or core capacity may be over- or under-provisioned. Some detail here needs <code>spark.eventLog.logStageExecutorMetrics=true</code> on the run being analyzed; without it, per-executor memory usage can&#39;t be broken down. Review <code>spark.executor.memory</code> and executor count if allocated memory sat largely idle over the run. That idle-memory variant self-flags a confidence that scales with how far the estimated waste sits past a 1.5x buffer: it estimates waste from allocated-versus-used memory-time. Check it against the Spark UI before resizing anything.</p><h3 id="cache" tabindex="-1"><code>CACHE</code>: Caching opportunity <a class="header-anchor" href="#cache" aria-label="Permalink to &quot;`CACHE`: Caching opportunity {#cache}&quot;">​</a></h3><p>A reusable dataset (re-read via the same SQL relation more than once) may be worth persisting between stages. Self-flags a confidence that scales with how many executions reuse the same relation: reuse is only inferred, from plan-scan identity across SQL executions, so confirm the reads really do hit the same data before you cache anything.</p><h3 id="cstor" tabindex="-1"><code>CSTOR</code>: Cache storage <a class="header-anchor" href="#cstor" aria-label="Permalink to &quot;`CSTOR`: Cache storage {#cstor}&quot;">​</a></h3><p>A persisted dataset is not fully cached in memory, or is spilling to disk. Raise executor memory, or shrink the cached dataset.</p><h3 id="local" tabindex="-1"><code>LOCAL</code>: Core usage locality <a class="header-anchor" href="#local" aria-label="Permalink to &quot;`LOCAL`: Core usage locality {#local}&quot;">​</a></h3><p>Tasks run without process- or node-local data placement more often than expected. Check <code>spark.locality.wait</code> settings and executor/data colocation. Self-flags a confidence that scales with the non-local ratio and sample size: the thresholds are our own noise floor for this metric.</p><h3 id="chrn" tabindex="-1"><code>CHRN</code>: Autoscaling churn <a class="header-anchor" href="#chrn" aria-label="Permalink to &quot;`CHRN`: Autoscaling churn {#chrn}&quot;">​</a></h3><p>Executors are stood up and torn down again before they can do useful work: re-provisioning churn rather than normal scale-down. Raise <code>spark.dynamicAllocation.executorIdleTimeout</code>, or widen the <code>minExecutors</code>/<code>maxExecutors</code> bounds to reduce flapping. Self-flags a confidence that scales with how far the short-lived-executor share sits past the threshold: these thresholds are still a design spike, not yet validated against real-world runs.</p><h3 id="jobs" tabindex="-1"><code>JOBS</code>: Job failure rate <a class="header-anchor" href="#jobs" aria-label="Permalink to &quot;`JOBS`: Job failure rate {#jobs}&quot;">​</a></h3><p>A large share of completed jobs did not succeed. Inspect the driver log for the failed job(s) and the stage failures that triggered them.</p><h3 id="incmp" tabindex="-1"><code>INCMP</code>: Incomplete run <a class="header-anchor" href="#incmp" aria-label="Permalink to &quot;`INCMP`: Incomplete run {#incmp}&quot;">​</a></h3><p>This event log never recorded an <code>ApplicationEnd</code> event: capture stopped before the run finished (an in-flight job, a rotated-away log, or a cut-short capture). Every other finding and metric on the board reflects only what was captured up to that point, not the full run.</p><h2 id="configuration-scope" tabindex="-1">Configuration scope <a class="header-anchor" href="#configuration-scope" aria-label="Permalink to &quot;Configuration scope&quot;">​</a></h2><h3 id="cfg" tabindex="-1"><code>CFG</code>: Configuration audit <a class="header-anchor" href="#cfg" aria-label="Permalink to &quot;`CFG`: Configuration audit {#cfg}&quot;">​</a></h3><p>Flags configuration settings that may cause reliability or efficiency problems, independent of any one stage&#39;s behavior. Four properties are audited today:</p><ul><li><code>spark.shuffle.service.enabled</code>: flagged when dynamic allocation is on but the external shuffle service is off, since shuffle data won&#39;t survive executor removal.</li><li><code>spark.dynamicAllocation.maxExecutors</code>: flagged for inverted bounds or a missing upper bound.</li><li><code>spark.serializer</code>: flagged when still on the default Java serializer; <code>org.apache.spark.serializer.KryoSerializer</code> is faster and produces smaller buffers.</li><li><code>spark.executor.memoryOverhead</code>: flagged when set below a safe floor.</li></ul><h2 id="sql-scope" tabindex="-1">SQL scope <a class="header-anchor" href="#sql-scope" aria-label="Permalink to &quot;SQL scope&quot;">​</a></h2><h3 id="plan" tabindex="-1"><code>PLAN</code>: Plan advisor <a class="header-anchor" href="#plan" aria-label="Permalink to &quot;`PLAN`: Plan advisor {#plan}&quot;">​</a></h3><p>Flags patterns in the SQL execution plan worth reviewing. Four checks share this tag:</p><ul><li>Duplicate plan subtree: the same subtree recomputed more than once in the plan.</li><li>Small files: reading an excessive number of small files.</li><li>Under-broadcast: the smaller side of a Sort Merge Join looks well under the broadcast threshold; consider a <code>broadcast()</code> hint or raising <code>spark.sql.autoBroadcastJoinThreshold</code>.</li><li>Over-broadcast: a broadcast exceeds the 1 GB threshold; check for a misapplied broadcast hint or a misconfigured <code>spark.sql.autoBroadcastJoinThreshold</code>.</li></ul></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/user-guide/getting-started.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>Getting started</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="/docs/user-guide/run-comparison.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>Run comparison mode</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
26
- <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"CWpj01WU\",\"contributor-guide_architecture_detector-contract.md\":\"CgzUsQ6W\",\"contributor-guide_architecture_drill-down.md\":\"BtPdlM7r\",\"contributor-guide_architecture_impact-estimation.md\":\"CooslVJt\",\"contributor-guide_architecture_index.md\":\"3TO9ic6w\",\"contributor-guide_architecture_overview.md\":\"CehiRmGn\",\"contributor-guide_architecture_state-and-history.md\":\"C-xxn0q7\",\"contributor-guide_architecture_widget-rendering.md\":\"R27gQrgY\",\"contributor-guide_architecture_worker-protocol.md\":\"IbnfNrV3\",\"contributor-guide_contributing.md\":\"CvRsdr6J\",\"contributor-guide_development-setup.md\":\"DvAN_9mK\",\"contributor-guide_testing.md\":\"6rIKqSyY\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"Df1YMIHu\",\"tuning-reference_aqe.md\":\"BIsCtLzm\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"CEstB3Ia\",\"tuning-reference_bottleneck-cold-start.md\":\"CEuy-72y\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"CIohQDfn\",\"tuning-reference_bottleneck-failures.md\":\"4z5BXGJ2\",\"tuning-reference_bottleneck-gc.md\":\"DSxzZRK7\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"BaJl__1W\",\"tuning-reference_bottleneck-memory-utilization.md\":\"DbP-SJZc\",\"tuning-reference_bottleneck-retry-waste.md\":\"D5JMjVOt\",\"tuning-reference_bottleneck-shuffle.md\":\"CM-nTmIH\",\"tuning-reference_bottleneck-skew.md\":\"BdUwiDhn\",\"tuning-reference_bottleneck-slow-host.md\":\"BlIo6UDW\",\"tuning-reference_bottleneck-small-files.md\":\"B8kloyx8\",\"tuning-reference_bottleneck-spill.md\":\"PNH7mITt\",\"tuning-reference_bottleneck-straggler.md\":\"DY36fHN5\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"QTV7O8kU\",\"tuning-reference_bottleneck-utilization.md\":\"DTiueZC3\",\"tuning-reference_caching.md\":\"B7aQ8asB\",\"tuning-reference_cluster-config.md\":\"ZVmDGsQ3\",\"tuning-reference_config.md\":\"UvveiWG3\",\"tuning-reference_data-formats.md\":\"bjCAWH3N\",\"tuning-reference_index.md\":\"BQ_NooMV\",\"tuning-reference_intro.md\":\"CobD-lGB\",\"tuning-reference_joins.md\":\"BtKs_CuW\",\"tuning-reference_memory-model.md\":\"DhT-n4y3\",\"tuning-reference_metrics.md\":\"mLOh7Apj\",\"tuning-reference_partitioning.md\":\"q0zKF_8X\",\"tuning-reference_pyspark.md\":\"DDCfvN9t\",\"tuning-reference_shuffle.md\":\"BZZ7R4Ix\",\"tuning-reference_spark-architecture.md\":\"Dwzm5avO\",\"tuning-reference_table-formats.md\":\"D6wj-2dX\",\"user-guide_alternative-log-retrieval.md\":\"B4tPGIal\",\"user-guide_getting-started.md\":\"BJvwLEIM\",\"user-guide_mcp-tools.md\":\"Vi3RoflJ\",\"user-guide_run-comparison.md\":\"CQc1aoU8\",\"user-guide_understanding-findings.md\":\"DL1UDhvR\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Alternative log retrieval\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
25
+ <div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>User Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/getting-started.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Getting started</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/understanding-findings.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Understanding findings</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/run-comparison.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Run comparison mode</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/mcp-tools.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>MCP tools reference</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/alternative-log-retrieval.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Alternative log retrieval</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_user-guide_understanding-findings" data-v-39a288b8><div><h1 id="understanding-findings" tabindex="-1">Understanding findings <a class="header-anchor" href="#understanding-findings" aria-label="Permalink to &quot;Understanding findings&quot;">​</a></h1><p>Every flagged problem carries a short ALL-CAPS tag. This page has one entry per tag: what it means, and what to do about it.</p><p>Turn on <a href="./getting-started.html#advanced-view">Advanced view</a> in the dashboard to see, per finding, the confidence level and a link to the same background reading collected here, opened in an in-app &quot;Reference&quot; panel next to the board. A few tags share that reading with another tag, because the underlying Spark-tuning material overlaps: <code>SFAIL</code> with <code>FAIL</code>, <code>PART</code> with <code>SHFL</code>, <code>SPEC</code> with <code>STRAG</code>, and <code>CACHE</code>/<code>LOCAL</code> with <code>UTIL</code>.</p><h2 id="per-stage" tabindex="-1">Per-stage <a class="header-anchor" href="#per-stage" aria-label="Permalink to &quot;Per-stage&quot;">​</a></h2><h3 id="skew" tabindex="-1"><code>SKEW</code>: Task skew <a class="header-anchor" href="#skew" aria-label="Permalink to &quot;`SKEW`: Task skew {#skew}&quot;">​</a></h3><p>A small number of tasks take much longer than their peers in the same stage. For join-driven skew, enable AQE skew-join handling (<code>spark.sql.adaptive.skewJoin.enabled</code>); otherwise salt the key or repartition on a better key.</p><h3 id="shfl" tabindex="-1"><code>SHFL</code>: Shuffle I/O <a class="header-anchor" href="#shfl" aria-label="Permalink to &quot;`SHFL`: Shuffle I/O {#shfl}&quot;">​</a></h3><p>Tasks move a large amount of intermediate data between stages. Raise <code>spark.sql.shuffle.partitions</code>, or add a broadcast join. Only flagged on stages that take at least 0.5% of the run.</p><h3 id="spill" tabindex="-1"><code>SPILL</code>: Memory and disk spill <a class="header-anchor" href="#spill" aria-label="Permalink to &quot;`SPILL`: Memory and disk spill {#spill}&quot;">​</a></h3><p>Tasks are writing data out of memory, which slows execution. Two spill patterns get flagged differently: skew spill, where a few heavy tasks spill while most don&#39;t (rebalance partitioning), and volume spill, where most tasks spill because the data genuinely exceeds available memory (add partitions). Only flagged on stages that take at least 0.5% of the run.</p><h3 id="gc" tabindex="-1"><code>GC</code>: Garbage collection pressure <a class="header-anchor" href="#gc" aria-label="Permalink to &quot;`GC`: Garbage collection pressure {#gc}&quot;">​</a></h3><p>Tasks spend an unusually large share of time reclaiming memory. Reduce object creation: use primitive types, avoid UDFs, or raise executor memory. A stage with very little GC gets an informational note that executor memory may be over-provisioned, only on stages that take at least 0.5% of the run.</p><h3 id="fail" tabindex="-1"><code>FAIL</code>: Failed tasks <a class="header-anchor" href="#fail" aria-label="Permalink to &quot;`FAIL`: Failed tasks {#fail}&quot;">​</a></h3><p>Tasks fail often enough to affect the stage. Failed tasks point to executor instability or data-driven errors: check driver logs for the dominant failure reason.</p><h3 id="sfail" tabindex="-1"><code>SFAIL</code>: Failed stage <a class="header-anchor" href="#sfail" aria-label="Permalink to &quot;`SFAIL`: Failed stage {#sfail}&quot;">​</a></h3><p>A stage attempt failed outright rather than losing individual tasks within it. Inspect the driver log for the failure reason and the job that triggered it.</p><h3 id="strag" tabindex="-1"><code>STRAG</code>: Straggler tasks <a class="header-anchor" href="#strag" aria-label="Permalink to &quot;`STRAG`: Straggler tasks {#strag}&quot;">​</a></h3><p>A few tasks run much slower than the rest of their stage. Rule out a GC pause or a slow shuffle fetch before assuming a hardware issue; if a skewed key is the real cause, that&#39;s a candidate for AQE&#39;s skew-join handling. Only flagged on stages that take at least 0.5% of the run.</p><h3 id="spec" tabindex="-1"><code>SPEC</code>: Speculation waste <a class="header-anchor" href="#spec" aria-label="Permalink to &quot;`SPEC`: Speculation waste {#spec}&quot;">​</a></h3><p>Speculative task attempts used a lot of executor time without confirming a genuine straggler. Self-flags a confidence that scales with how far the wasted time sits past the threshold: these thresholds are still a design spike, not yet validated against real-world runs. If task durations are just naturally variable rather than genuine stragglers, tune <code>spark.speculation.multiplier</code>/<code>spark.speculation.quantile</code>.</p><h3 id="retry" tabindex="-1"><code>RETRY</code>: Retry waste <a class="header-anchor" href="#retry" aria-label="Permalink to &quot;`RETRY`: Retry waste {#retry}&quot;">​</a></h3><p>Repeated task attempts ate into execution time even though the stage completed. Investigate executor loss or fetch failures.</p><h3 id="tiny" tabindex="-1"><code>TINY</code>: Tiny tasks <a class="header-anchor" href="#tiny" aria-label="Permalink to &quot;`TINY`: Tiny tasks {#tiny}&quot;">​</a></h3><p>Many very short tasks add scheduling overhead out of proportion to the work each one does. Repartition to fewer, larger tasks. Only flagged on stages that take at least 0.5% of the run.</p><h3 id="part" tabindex="-1"><code>PART</code>: Partition sizing <a class="header-anchor" href="#part" aria-label="Permalink to &quot;`PART`: Partition sizing {#part}&quot;">​</a></h3><p>Shuffle partitions are too large, too uneven, or too few for the work. A single shuffle partition over 5 GB, for example, will OOM or spill heavily: repartition to break it up before the stage runs.</p><h3 id="slow" tabindex="-1"><code>SLOW</code>: Stage slowness <a class="header-anchor" href="#slow" aria-label="Permalink to &quot;`SLOW`: Stage slowness {#slow}&quot;">​</a></h3><p>A stage ran long overall without a more specific cause getting flagged. Often a partition-count problem: raise parallelism via <code>spark.sql.shuffle.partitions</code> or <code>spark.default.parallelism</code>, or check for a large per-task data volume driving heavy shuffle and spill.</p><h3 id="shape" tabindex="-1"><code>SHAPE</code>: Stage shape <a class="header-anchor" href="#shape" aria-label="Permalink to &quot;`SHAPE`: Stage shape {#shape}&quot;">​</a></h3><p>The stage has an inefficient task count, output shape, or task-to-stage balance: for example, one straggler task taking a large fraction of the stage&#39;s wall-clock time. A too-low task count is only flagged on stages that take at least 0.5% of the run.</p><h3 id="host" tabindex="-1"><code>HOST</code>: Slow host <a class="header-anchor" href="#host" aria-label="Permalink to &quot;`HOST`: Slow host {#host}&quot;">​</a></h3><p>One executor is much slower than its peers. It may just hold data locality for its tasks or carry one heavy stage, rather than a hardware fault. Enable <code>spark.speculation</code> to relaunch a lagging task automatically. Only flagged on stages that take at least 0.5% of the run.</p><h2 id="app-level" tabindex="-1">App-level <a class="header-anchor" href="#app-level" aria-label="Permalink to &quot;App-level&quot;">​</a></h2><h3 id="cold" tabindex="-1"><code>COLD</code>: Executor cold start <a class="header-anchor" href="#cold" aria-label="Permalink to &quot;`COLD`: Executor cold start {#cold}&quot;">​</a></h3><p>New executors take time to become available for work. Pre-warm the cluster, or use dynamic allocation.</p><h3 id="util" tabindex="-1"><code>UTIL</code>: Low utilization <a class="header-anchor" href="#util" aria-label="Permalink to &quot;`UTIL`: Low utilization {#util}&quot;">​</a></h3><p>Allocated executors sit idle for a large share of the application run. Consider a smaller cluster, or enable dynamic allocation.</p><h3 id="mem" tabindex="-1"><code>MEM</code>: Memory utilization <a class="header-anchor" href="#mem" aria-label="Permalink to &quot;`MEM`: Memory utilization {#mem}&quot;">​</a></h3><p>Executor memory or core capacity may be over- or under-provisioned. Some detail here needs <code>spark.eventLog.logStageExecutorMetrics=true</code> on the run being analyzed; without it, per-executor memory usage can&#39;t be broken down. Review <code>spark.executor.memory</code> and executor count if allocated memory sat largely idle over the run. That idle-memory variant self-flags a confidence that scales with how far the estimated waste sits past a 1.5x buffer: it estimates waste from allocated-versus-used memory-time. Check it against the Spark UI before resizing anything.</p><h3 id="cache" tabindex="-1"><code>CACHE</code>: Caching opportunity <a class="header-anchor" href="#cache" aria-label="Permalink to &quot;`CACHE`: Caching opportunity {#cache}&quot;">​</a></h3><p>A reusable dataset (re-read via the same SQL relation more than once) may be worth persisting between stages. Self-flags a confidence that scales with how many executions reuse the same relation: reuse is only inferred, from plan-scan identity across SQL executions, so confirm the reads really do hit the same data before you cache anything.</p><h3 id="cstor" tabindex="-1"><code>CSTOR</code>: Cache storage <a class="header-anchor" href="#cstor" aria-label="Permalink to &quot;`CSTOR`: Cache storage {#cstor}&quot;">​</a></h3><p>A persisted dataset is not fully cached in memory, or is spilling to disk. Raise executor memory, or shrink the cached dataset.</p><h3 id="local" tabindex="-1"><code>LOCAL</code>: Core usage locality <a class="header-anchor" href="#local" aria-label="Permalink to &quot;`LOCAL`: Core usage locality {#local}&quot;">​</a></h3><p>Tasks run without process- or node-local data placement more often than expected. Check <code>spark.locality.wait</code> settings and executor/data colocation. Self-flags a confidence that scales with the non-local ratio and sample size: the thresholds are our own noise floor for this metric.</p><h3 id="chrn" tabindex="-1"><code>CHRN</code>: Autoscaling churn <a class="header-anchor" href="#chrn" aria-label="Permalink to &quot;`CHRN`: Autoscaling churn {#chrn}&quot;">​</a></h3><p>Executors are stood up and torn down again before they can do useful work: re-provisioning churn rather than normal scale-down. Raise <code>spark.dynamicAllocation.executorIdleTimeout</code>, or widen the <code>minExecutors</code>/<code>maxExecutors</code> bounds to reduce flapping. Self-flags a confidence that scales with how far the short-lived-executor share sits past the threshold: these thresholds are still a design spike, not yet validated against real-world runs.</p><h3 id="jobs" tabindex="-1"><code>JOBS</code>: Job failure rate <a class="header-anchor" href="#jobs" aria-label="Permalink to &quot;`JOBS`: Job failure rate {#jobs}&quot;">​</a></h3><p>A large share of completed jobs did not succeed. Inspect the driver log for the failed job(s) and the stage failures that triggered them.</p><h3 id="incmp" tabindex="-1"><code>INCMP</code>: Incomplete run <a class="header-anchor" href="#incmp" aria-label="Permalink to &quot;`INCMP`: Incomplete run {#incmp}&quot;">​</a></h3><p>This event log never recorded an <code>ApplicationEnd</code> event: capture stopped before the run finished (an in-flight job, a rotated-away log, or a cut-short capture). Every other finding and metric on the board reflects only what was captured up to that point, not the full run.</p><h2 id="configuration-scope" tabindex="-1">Configuration scope <a class="header-anchor" href="#configuration-scope" aria-label="Permalink to &quot;Configuration scope&quot;">​</a></h2><h3 id="cfg" tabindex="-1"><code>CFG</code>: Configuration audit <a class="header-anchor" href="#cfg" aria-label="Permalink to &quot;`CFG`: Configuration audit {#cfg}&quot;">​</a></h3><p>Flags configuration settings that may cause reliability or efficiency problems, independent of any one stage&#39;s behavior. Four properties are audited today:</p><ul><li><code>spark.shuffle.service.enabled</code>: flagged when dynamic allocation is on but the external shuffle service is off, since shuffle data won&#39;t survive executor removal.</li><li><code>spark.dynamicAllocation.maxExecutors</code>: flagged for inverted bounds or a missing upper bound.</li><li><code>spark.serializer</code>: flagged when still on the default Java serializer; <code>org.apache.spark.serializer.KryoSerializer</code> is faster and produces smaller buffers.</li><li><code>spark.executor.memoryOverhead</code>: flagged when set below a safe floor.</li></ul><h2 id="sql-scope" tabindex="-1">SQL scope <a class="header-anchor" href="#sql-scope" aria-label="Permalink to &quot;SQL scope&quot;">​</a></h2><h3 id="plan" tabindex="-1"><code>PLAN</code>: Plan advisor <a class="header-anchor" href="#plan" aria-label="Permalink to &quot;`PLAN`: Plan advisor {#plan}&quot;">​</a></h3><p>Flags patterns in the SQL execution plan worth reviewing. Four checks share this tag:</p><ul><li>Duplicate plan subtree: the same subtree recomputed more than once in the plan. When the repeats have the same shape but different filters, columns or tables, the finding stays informational and claims no time. Only flagged when the repeat&#39;s stages take at least 0.5% of the run.</li><li>Small files: reading an excessive number of small files.</li><li>Under-broadcast: the smaller side of a Sort Merge Join looks well under the broadcast threshold; consider a <code>broadcast()</code> hint or raising <code>spark.sql.autoBroadcastJoinThreshold</code>.</li><li>Over-broadcast: a broadcast exceeds the 1 GB threshold; check for a misapplied broadcast hint or a misconfigured <code>spark.sql.autoBroadcastJoinThreshold</code>.</li></ul></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/user-guide/getting-started.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>Getting started</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="/docs/user-guide/run-comparison.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>Run comparison mode</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
26
+ <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"9ToZXcTk\",\"contributor-guide_architecture_detector-contract.md\":\"aQQfP7pj\",\"contributor-guide_architecture_drill-down.md\":\"Bawx6NgP\",\"contributor-guide_architecture_impact-estimation.md\":\"sf9DU2bA\",\"contributor-guide_architecture_index.md\":\"3TO9ic6w\",\"contributor-guide_architecture_overview.md\":\"BXQdgG5u\",\"contributor-guide_architecture_state-and-history.md\":\"C-xxn0q7\",\"contributor-guide_architecture_widget-rendering.md\":\"DYwFeyjV\",\"contributor-guide_architecture_worker-protocol.md\":\"P7d1vtEJ\",\"contributor-guide_contributing.md\":\"CvRsdr6J\",\"contributor-guide_development-setup.md\":\"DvMV1xhQ\",\"contributor-guide_testing.md\":\"BU6vyA-9\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"Df1YMIHu\",\"tuning-reference_aqe.md\":\"BIsCtLzm\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"CEstB3Ia\",\"tuning-reference_bottleneck-cold-start.md\":\"CEuy-72y\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"CIohQDfn\",\"tuning-reference_bottleneck-failures.md\":\"CAsFf3RR\",\"tuning-reference_bottleneck-gc.md\":\"B4T3kJYV\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"BaJl__1W\",\"tuning-reference_bottleneck-memory-utilization.md\":\"DbP-SJZc\",\"tuning-reference_bottleneck-retry-waste.md\":\"D5JMjVOt\",\"tuning-reference_bottleneck-shuffle.md\":\"DDMQVOW6\",\"tuning-reference_bottleneck-skew.md\":\"B0MzhWcX\",\"tuning-reference_bottleneck-slow-host.md\":\"BlIo6UDW\",\"tuning-reference_bottleneck-small-files.md\":\"DHGMTmPs\",\"tuning-reference_bottleneck-spill.md\":\"PNH7mITt\",\"tuning-reference_bottleneck-straggler.md\":\"DIbXdAcQ\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"CB05v3zh\",\"tuning-reference_bottleneck-utilization.md\":\"DqC_iGkw\",\"tuning-reference_caching.md\":\"B7aQ8asB\",\"tuning-reference_cluster-config.md\":\"CD6vzYtD\",\"tuning-reference_config.md\":\"UvveiWG3\",\"tuning-reference_data-formats.md\":\"bjCAWH3N\",\"tuning-reference_index.md\":\"BQ_NooMV\",\"tuning-reference_intro.md\":\"CobD-lGB\",\"tuning-reference_joins.md\":\"BtKs_CuW\",\"tuning-reference_memory-model.md\":\"CWhvgevP\",\"tuning-reference_metrics.md\":\"mLOh7Apj\",\"tuning-reference_partitioning.md\":\"q0zKF_8X\",\"tuning-reference_pyspark.md\":\"DDCfvN9t\",\"tuning-reference_shuffle.md\":\"BZZ7R4Ix\",\"tuning-reference_spark-architecture.md\":\"Dwzm5avO\",\"tuning-reference_table-formats.md\":\"D6wj-2dX\",\"user-guide_alternative-log-retrieval.md\":\"B4tPGIal\",\"user-guide_getting-started.md\":\"BJvwLEIM\",\"user-guide_mcp-tools.md\":\"DGvolGzt\",\"user-guide_run-comparison.md\":\"CQc1aoU8\",\"user-guide_understanding-findings.md\":\"B29f0ucS\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Alternative log retrieval\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
27
27
 
28
28
  </body>
29
29
  </html>
package/public/index.html CHANGED
@@ -23,7 +23,7 @@
23
23
  document.documentElement.setAttribute('data-theme', 'light');
24
24
  } catch (_) {}
25
25
  </script>
26
- <script type="module" crossorigin src="./assets/index-3vlFVNCg.js"></script>
26
+ <script type="module" crossorigin src="./assets/index-pnImGbgQ.js"></script>
27
27
  <link rel="modulepreload" crossorigin href="./assets/rolldown-runtime-hePW80VL.js">
28
28
  <link rel="modulepreload" crossorigin href="./assets/charts-vendor-0VMaLVLN.js">
29
29
  <link rel="modulepreload" crossorigin href="./assets/plan-graph-vendor-B0sEms9Z.js">
package/vendor-core/cli/collect-run.js CHANGED
@@ -1,6 +1,7 @@
1
1
  import { readFileSync, readdirSync, statSync, openSync, readSync, closeSync } from 'node:fs';
2
2
  import { join, basename } from 'node:path';
3
3
  import { createState, runParse, runParseFiles, reassembleRollingEntries } from '../parser-worker.js';
4
+ import { nodeParseCodecs } from './native-zstd.js';
4
5
  import { createModelCallbacks } from '../model-assembler.js';
5
6
  import { routeMessage, } from '../ingest.js';
6
7
 
@@ -123,9 +124,9 @@ export async function collectRun(inputPath )
123
124
  const files = ordered.map((name) => nodeFileFromPath(join(inputPath, name)));
124
125
  // .catch(reject), not void: a throw past the parser's guards would otherwise leave
125
126
  // this Promise pending forever, surfacing only as an unhandled rejection.
126
- runParseFiles(files, state, { emit }).catch(reject);
127
+ runParseFiles(files, state, { emit, ...nodeParseCodecs }).catch(reject);
127
128
  } else {
128
- runParse(nodeFileFromPath(inputPath), state, { emit }).catch(reject);
129
+ runParse(nodeFileFromPath(inputPath), state, { emit, ...nodeParseCodecs }).catch(reject);
129
130
  }
130
131
  }, (msg) => new Error((msg ).message));
131
132
  }
package/vendor-core/cli/native-zstd.js ADDED
@@ -0,0 +1,351 @@
1
+ // Node-only zstd decoding for the CLI/MCP parse path: Node's native zlib zstd, one frame at a
2
+ // time, with the vendored fzstd (what the browser uses) as the fallback.
3
+ //
4
+ // Spark writes an event log as thousands of small zstd frames (one per flush: 10217 frames for
5
+ // the largest real log's 3.5 GB). Node's zstdDecompressSync and createZstdDecompress both stop
6
+ // after the first frame (the stream then fails with "Unknown frame descriptor"), so this walks
7
+ // frame boundaries itself and decompresses each complete frame natively: 3-5x faster than fzstd
8
+ // on the real logs (5.6s -> 1.3-1.8s on the largest).
9
+ import * as zlib from 'node:zlib';
10
+ import { Decompress as ZstdDecompress } from '../vendor/fzstd.js';
11
+
12
+
13
+
14
+
15
+ // zstdDecompressSync materializes a whole frame, so a frame declaring more content than this, or
16
+ // still incomplete after this many compressed bytes, is streamed through fzstd instead: a log
17
+ // compressed as a single frame (the zstd CLI's default) must never be held whole in memory. A
18
+ // frame that declares no size (every frame Spark writes, and `zstd < log > log.zst`) is decoded
19
+ // with its output capped at this: past it, it streams too (decompressBounded).
20
+ const MAX_NATIVE_FRAME_BYTES = 64 * 1024 * 1024;
21
+
22
+ const ZSTD_MAGIC = 0xfd2fb528;
23
+ const SKIPPABLE_MAGIC = 0x184d2a50;
24
+
25
+ const INCOMPLETE = -1;
26
+ const NOT_NATIVE = -2;
27
+
28
+ function readUint32LE(buf , at ) {
29
+ return (buf[at] | (buf[at + 1] << 8) | (buf[at + 2] << 16) | (buf[at + 3] << 24)) >>> 0;
30
+ }
31
+
32
+ // Where a walk of an incomplete frame stopped, relative to the frame's first byte: `next` is the
33
+ // next block header (or the checksum once `blocksDone`), -1 while the frame header is incomplete.
34
+
35
+
36
+ const freshWalk = () => ({ next: -1, checksum: false, blocksDone: false });
37
+
38
+ // End offset of the zstd or skippable frame starting at `off` (RFC 8878), INCOMPLETE when `buf`
39
+ // ends before it does, or NOT_NATIVE for a frame this path won't decode itself: oversized, or
40
+ // not a well-formed frame (fzstd then reports the corruption exactly as the browser would). An
41
+ // INCOMPLETE walk leaves its place in `walk`, so the next call on more of the frame resumes there.
42
+ function frameEnd(buf , off , maxFrameBytes , walk = freshWalk()) {
43
+ let p ;
44
+ let hasChecksum ;
45
+ if (walk.next >= 0) {
46
+ p = off + walk.next;
47
+ hasChecksum = walk.checksum;
48
+ } else {
49
+ if (off + 4 > buf.length) return INCOMPLETE;
50
+ const magic = readUint32LE(buf, off);
51
+ if ((magic & 0xfffffff0) === SKIPPABLE_MAGIC) {
52
+ if (off + 8 > buf.length) return INCOMPLETE;
53
+ const end = off + 8 + readUint32LE(buf, off + 4);
54
+ return end <= buf.length ? end : INCOMPLETE;
55
+ }
56
+ if (magic !== ZSTD_MAGIC) return NOT_NATIVE;
57
+ p = off + 4;
58
+ if (p >= buf.length) return INCOMPLETE;
59
+ const descriptor = buf[p++];
60
+ const contentSizeFlag = descriptor >> 6;
61
+ const singleSegment = (descriptor >> 5) & 1;
62
+ hasChecksum = ((descriptor >> 2) & 1) === 1;
63
+ p += (singleSegment ? 0 : 1) + [0, 1, 2, 4][descriptor & 3];
64
+ const contentSizeBytes = contentSizeFlag === 0 ? singleSegment : [0, 2, 4, 8][contentSizeFlag];
65
+ if (p + contentSizeBytes > buf.length) return INCOMPLETE;
66
+ const contentSize =
67
+ contentSizeBytes === 1 ? buf[p]
68
+ : contentSizeBytes === 2 ? (buf[p] | (buf[p + 1] << 8)) + 256
69
+ : contentSizeBytes === 4 ? readUint32LE(buf, p)
70
+ : contentSizeBytes === 8 ? readUint32LE(buf, p) + readUint32LE(buf, p + 4) * 2 ** 32
71
+ : 0;
72
+ if (contentSize > maxFrameBytes) return NOT_NATIVE;
73
+ p += contentSizeBytes;
74
+ walk.checksum = hasChecksum;
75
+ }
76
+ while (!walk.blocksDone) {
77
+ walk.next = p - off;
78
+ if (p + 3 > buf.length) return INCOMPLETE;
79
+ const header = buf[p] | (buf[p + 1] << 8) | (buf[p + 2] << 16);
80
+ p += 3;
81
+ const blockType = (header >> 1) & 3;
82
+ if (blockType === 3) return NOT_NATIVE; // reserved: corrupt data
83
+ p += blockType === 1 ? 1 : header >> 3; // an RLE block stores its one repeated byte
84
+ if (p > buf.length) return INCOMPLETE;
85
+ if (header & 1) { // last block
86
+ walk.blocksDone = true;
87
+ walk.next = p - off;
88
+ }
89
+ }
90
+ p += hasChecksum ? 4 : 0;
91
+ return p <= buf.length ? p : INCOMPLETE;
92
+ }
93
+
94
+ export const nativeZstdAvailable =
95
+ typeof zlib.zstdDecompressSync === 'function' && typeof zlib.createZstdDecompress === 'function';
96
+
97
+ // One step of FrameSplitter.split: a whole frame (`data` false for a skippable one), or the bytes
98
+ // from a frame this path won't decode natively to the end of the push, which fzstd then takes.
99
+
100
+
101
+ // Splits a zstd byte stream into whole frames as it arrives, for both decoders below. A frame
102
+ // falls back to fzstd when frameEnd says NOT_NATIVE, when it is still incomplete after
103
+ // `maxFrameBytes` compressed bytes, or when a final push ends inside it. A frame that arrives
104
+ // over many pushes is gathered into one buffer that grows by doubling, and its walk resumes where
105
+ // the last push left it: copying the gathered bytes into a new buffer and walking the frame from
106
+ // its start on every push cost about 4 GB of copying near the 64 MB limit.
107
+ class FrameSplitter {
108
+ // The incomplete frame's bytes so far, gathered[0, gatheredLength); null when none is pending.
109
+ gathered = null;
110
+ gatheredLength = 0;
111
+ walk = freshWalk();
112
+ maxFrameBytes ;
113
+ // The current push's bytes after the last frame split() yielded, for rest().
114
+ current = new Uint8Array(0);
115
+ restAt = 0;
116
+
117
+ constructor(maxFrameBytes ) {
118
+ this.maxFrameBytes = maxFrameBytes;
119
+ }
120
+
121
+ gather(bytes ) {
122
+ const length = this.gatheredLength + bytes.length;
123
+ if (this.gathered === null || length > this.gathered.length) {
124
+ const grown = new Uint8Array(Math.max(length, (this.gathered?.length ?? 0) * 2));
125
+ if (this.gathered !== null) grown.set(this.gathered.subarray(0, this.gatheredLength));
126
+ this.gathered = grown;
127
+ }
128
+ this.gathered.set(bytes, this.gatheredLength);
129
+ this.gatheredLength = length;
130
+ return this.gathered.subarray(0, length);
131
+ }
132
+
133
+ // Frames and fallback bytes yielded from a gathered buffer stay valid: a completed buffer is
134
+ // dropped, never reused.
135
+ *split(chunk , final ) {
136
+ let off = 0;
137
+ if (this.gathered !== null) {
138
+ const held = this.gather(chunk);
139
+ const end = frameEnd(held, 0, this.maxFrameBytes, this.walk);
140
+ if (end === INCOMPLETE && held.length <= this.maxFrameBytes && !final) return;
141
+ this.gathered = null;
142
+ this.gatheredLength = 0;
143
+ if (end === NOT_NATIVE || end === INCOMPLETE) {
144
+ yield { fallback: held };
145
+ return;
146
+ }
147
+ off = chunk.length - (held.length - end); // the rest of the chunk follows that frame
148
+ this.current = chunk;
149
+ this.restAt = off;
150
+ yield { frame: held.subarray(0, end), data: readUint32LE(held, 0) === ZSTD_MAGIC };
151
+ }
152
+ while (off < chunk.length) {
153
+ this.walk = freshWalk();
154
+ const end = frameEnd(chunk, off, this.maxFrameBytes, this.walk);
155
+ if (end === NOT_NATIVE || (end === INCOMPLETE && chunk.length - off > this.maxFrameBytes)) {
156
+ yield { fallback: chunk.subarray(off) };
157
+ return;
158
+ }
159
+ if (end === INCOMPLETE) break;
160
+ this.current = chunk;
161
+ this.restAt = end;
162
+ yield { frame: chunk.subarray(off, end), data: readUint32LE(chunk, off) === ZSTD_MAGIC };
163
+ off = end;
164
+ }
165
+ if (off < chunk.length) {
166
+ if (final) yield { fallback: chunk.subarray(off) };
167
+ else this.gather(chunk.subarray(off));
168
+ }
169
+ }
170
+
171
+ // The current push's bytes after the frame split() last yielded, for a caller that stops there
172
+ // and hands the rest of the stream to fzstd. The splitter is not used after this.
173
+ rest() {
174
+ return this.current.subarray(this.restAt);
175
+ }
176
+ }
177
+
178
+ // A frame's output, or null when it would pass `maxBytes`: without a declared content size
179
+ // nothing else bounds it, and zstdDecompressSync would build the whole thing as one buffer.
180
+ function decompressBounded(frame , maxBytes ) {
181
+ try {
182
+ return zlib.zstdDecompressSync(frame, { maxOutputLength: maxBytes });
183
+ } catch (err) {
184
+ if ((err ).code === 'ERR_BUFFER_TOO_LARGE') return null;
185
+ throw err;
186
+ }
187
+ }
188
+
189
+ // Same push(chunk, final) contract as fzstd's Decompress. Once a frame falls back, fzstd takes
190
+ // the rest of the stream from that frame's first byte; a truncated last frame goes to fzstd too,
191
+ // so it fails with fzstd's own "unexpected EOF", as it always has. `maxFrameBytes` is for tests.
192
+ export function createNativeZstdDecoder(
193
+ onChunk ,
194
+ { maxFrameBytes = MAX_NATIVE_FRAME_BYTES } = {},
195
+ ) {
196
+ const splitter = new FrameSplitter(maxFrameBytes);
197
+ let fallback = null;
198
+ const toFallback = (bytes , final ) => {
199
+ fallback ??= new (ZstdDecompress )(onChunk);
200
+ fallback.push(bytes, final);
201
+ };
202
+ return {
203
+ push(chunk , final = false) {
204
+ if (fallback) { fallback.push(chunk, final); return; }
205
+ for (const step of splitter.split(chunk, final)) {
206
+ if (step.fallback) { toFallback(step.fallback, final); return; }
207
+ if (!step.data) continue;
208
+ const output = decompressBounded(step.frame, maxFrameBytes);
209
+ if (output) { onChunk(output); continue; }
210
+ // Past the bound: fzstd streams this frame block by block, then the rest of the stream.
211
+ toFallback(step.frame, false);
212
+ toFallback(splitter.rest(), final);
213
+ return;
214
+ }
215
+ },
216
+ };
217
+ }
218
+
219
+ // Frames at least this big (compressed) are decompressed on libuv's threadpool, up to
220
+ // MAX_FRAMES_IN_FLIGHT at a time, while the main thread parses the output of earlier ones.
221
+ // Smaller frames cost less to decompress inline than to hand off (p50 is 5.8 KB of output on the
222
+ // largest real log; 16-64 KB thresholds measured the same, 256 KB lost most of the gain). On that
223
+ // log 4 in flight (the pool's default size) parsed in 3.1s at 711 MB peak RSS, 8 in 3.2s at 886 MB
224
+ // and 2 in 3.5s at 607 MB. The pool's output chunk size barely mattered (64 KB to 1 MB).
225
+ const THREADED_MIN_FRAME_BYTES = 64 * 1024;
226
+ const MAX_FRAMES_IN_FLIGHT = 4;
227
+ const THREADED_CHUNK_BYTES = 256 * 1024;
228
+ // Small frames that arrive while an earlier frame is in flight wait in the queue behind it. With
229
+ // no cap, a first version held most of the largest real log's output (RSS 0.6 -> 4.1 GB).
230
+ const MAX_QUEUED_FRAMES = 64;
231
+
232
+ // One frame decompressing on the threadpool. Node's zstd stream ends after one frame, so each
233
+ // frame gets its own. Its output chunks are kept as they are (zstdDecompressSync would copy them
234
+ // into one buffer on the main thread) until deliver() hands them over, in its turn. Up to
235
+ // `maxBufferedBytes` of them wait; then the stream pauses until deliver() drains them, so a frame
236
+ // without a declared size is never held whole.
237
+ class OffThreadFrame {
238
+ chunks = [];
239
+ buffered = 0;
240
+ ended = false;
241
+ error = null;
242
+ wake = null;
243
+ stream ;
244
+
245
+ constructor(frame , maxBufferedBytes ) {
246
+ this.stream = zlib.createZstdDecompress({ chunkSize: THREADED_CHUNK_BYTES });
247
+ this.stream.on('data', (chunk ) => {
248
+ this.chunks.push(chunk);
249
+ this.buffered += chunk.length;
250
+ if (this.buffered >= maxBufferedBytes) this.stream.pause();
251
+ this.signal();
252
+ });
253
+ this.stream.on('end', () => { this.ended = true; this.signal(); });
254
+ // Kept until deliver() reaches it: rethrown there, never an unhandled error before then.
255
+ this.stream.on('error', (err ) => { this.error = err; this.ended = true; this.signal(); });
256
+ this.stream.end(frame);
257
+ }
258
+
259
+ signal() {
260
+ const wake = this.wake;
261
+ this.wake = null;
262
+ wake?.();
263
+ }
264
+
265
+ async deliver(onChunk ) {
266
+ for (;;) {
267
+ const ready = this.chunks;
268
+ this.chunks = [];
269
+ this.buffered = 0;
270
+ for (const chunk of ready) onChunk(chunk);
271
+ if (this.error !== null) throw this.error;
272
+ if (this.ended) return;
273
+ const more = new Promise ((resolve) => { this.wake = resolve; });
274
+ this.stream.resume();
275
+ await more;
276
+ }
277
+ }
278
+ }
279
+
280
+ // createNativeZstdDecoder's contract, with large frames decompressed in parallel off the main
281
+ // thread: push() resolves once its complete frames are queued (or delivered, when the queue is
282
+ // full), and a final push resolves after every chunk has reached onChunk, in stream order. Each
283
+ // push must be awaited before the next. It keeps references to pushed bytes until their frames
284
+ // are decoded, so callers must not reuse a pushed buffer. Fallback to fzstd, for the same frames
285
+ // as createNativeZstdDecoder, happens only after every queued frame is delivered; a failed frame
286
+ // rejects the push that reaches it. `threadedMinFrameBytes` is for tests. Measured on the 14 real
287
+ // logs: parse 9.4s -> 8.1s; logs with few large frames (12 of 6455 on a 28 MB one) gain nothing.
288
+ export function createThreadedZstdDecoder(
289
+ onChunk ,
290
+ { maxFrameBytes = MAX_NATIVE_FRAME_BYTES, threadedMinFrameBytes = THREADED_MIN_FRAME_BYTES }
291
+ = {},
292
+ ) {
293
+ const splitter = new FrameSplitter(maxFrameBytes);
294
+ let fallback = null;
295
+ // Frames not yet delivered, oldest first: off-thread output, or a small frame still compressed,
296
+ // decompressed inline only when its turn comes so its output is fresh in cache for the parse.
297
+
298
+ const queued = [];
299
+ let threadedQueued = 0;
300
+ // A small frame, inline; one whose output passes maxFrameBytes streams like an off-thread one.
301
+ const deliverInline = async (frame ) => {
302
+ const output = decompressBounded(frame, maxFrameBytes);
303
+ if (output) onChunk(output);
304
+ else await new OffThreadFrame(frame, maxFrameBytes).deliver(onChunk);
305
+ };
306
+ const deliverOldest = async () => {
307
+ const entry = queued.shift() ;
308
+ if (entry.frame) { await deliverInline(entry.frame); return; }
309
+ threadedQueued--;
310
+ await entry.output .deliver(onChunk);
311
+ };
312
+ const enqueue = async (entry ) => {
313
+ queued.push(entry);
314
+ if (entry.output) threadedQueued++;
315
+ while (threadedQueued >= MAX_FRAMES_IN_FLIGHT || queued.length >= MAX_QUEUED_FRAMES) await deliverOldest();
316
+ };
317
+ const deliverAll = async () => {
318
+ while (queued.length > 0) await deliverOldest();
319
+ };
320
+ const toFallback = async (bytes , final ) => {
321
+ await deliverAll();
322
+ fallback ??= new (ZstdDecompress )(onChunk);
323
+ fallback.push(bytes, final);
324
+ };
325
+ return {
326
+ async push(chunk , final = false) {
327
+ if (fallback) { fallback.push(chunk, final); return; }
328
+ for (const step of splitter.split(chunk, final)) {
329
+ if (step.fallback) { await toFallback(step.fallback, final); return; }
330
+ if (!step.data) continue;
331
+ const frame = step.frame;
332
+ if (frame.length >= threadedMinFrameBytes) {
333
+ await enqueue({ output: new OffThreadFrame(frame, maxFrameBytes) });
334
+ } else if (queued.length === 0) {
335
+ await deliverInline(frame);
336
+ } else {
337
+ await enqueue({ frame });
338
+ }
339
+ }
340
+ if (final) await deliverAll();
341
+ },
342
+ };
343
+ }
344
+
345
+ // Node's native zstd where this Node has it (22.15+/23.8+); older Nodes keep the vendored fzstd.
346
+ // runParse/runParseFiles (collectRun) take the threaded decoder. decodeShsArchive decodes each
347
+ // archive entry in one synchronous call, so the SHS archive loader takes the inline one.
348
+ export const nodeParseCodecs =
349
+ nativeZstdAvailable ? { zstdDecoder: createThreadedZstdDecoder } : {};
350
+ export const nodeArchiveCodecs =
351
+ nativeZstdAvailable ? { zstdDecoder: createNativeZstdDecoder } : {};