sparkforensics-server 0.2.0 → 0.2.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (288) hide show
  1. package/index.js +1 -1
  2. package/lib/static-files.js +5 -4
  3. package/package.json +4 -3
  4. package/public/assets/AutoscalingChurn-Bf5TN-s_.js +1 -0
  5. package/public/assets/CacheUtilization-DRf_FNrK.js +1 -0
  6. package/public/assets/CachingOpportunity-BTJzIhtI.js +1 -0
  7. package/public/assets/ColdStart-DnFgnD3n.js +1 -0
  8. package/public/assets/ConfigAudit-B5FsOdmu.js +1 -0
  9. package/public/assets/CoreUsageArea-CTksBPuq.js +1 -0
  10. package/public/assets/Dashboard-BFMsMuc_.js +1 -0
  11. package/public/assets/DuplicatePlanSubtree-BCt-Guuh.js +1 -0
  12. package/public/assets/DurationHistogram-xRcnND4k.js +1 -0
  13. package/public/assets/EmptyState-CnTrSjtU.js +1 -0
  14. package/public/assets/ExecutorUtilization-CP21XlUq.js +1 -0
  15. package/public/assets/GcPressure-BoEJsLjb.js +1 -0
  16. package/public/assets/ImpactEstimate-BHcqJkcZ.js +1 -0
  17. package/public/assets/IncompleteRun-b9FWrOHn.js +1 -0
  18. package/public/assets/JobFailures-C5FkTq3O.js +1 -0
  19. package/public/assets/MemoryUtilization-D2W4AB_f.js +1 -0
  20. package/public/assets/OverBroadcast-Btu3xorf.js +1 -0
  21. package/public/assets/PartitionSizing-yZWhLN0e.js +1 -0
  22. package/public/assets/PlanExplorer-DSk1ivS4.js +1 -0
  23. package/public/assets/PlanGraphRoute-C3iOzUdB.js +1 -0
  24. package/public/assets/RetryWaste-B4436kr1.js +1 -0
  25. package/public/assets/RowPagination-oeWB8gBm.js +1 -0
  26. package/public/assets/ShuffleIO-DUiv8EJe.js +1 -0
  27. package/public/assets/Skew-D2lB1oQc.js +1 -0
  28. package/public/assets/SlowHost-Qb_8aa-W.js +1 -0
  29. package/public/assets/SmallFiles-u6suTgfj.js +1 -0
  30. package/public/assets/SpeculationWaste-vUCniiFP.js +1 -0
  31. package/public/assets/Spill-BskXFX2e.js +1 -0
  32. package/public/assets/StageFailed-D9b83Jvy.js +1 -0
  33. package/public/assets/StageFindingGroup-DagqHOUz.js +1 -0
  34. package/public/assets/StageHeader-DibMSunO.js +1 -0
  35. package/public/assets/StagePill-Dt6V9yVw.js +1 -0
  36. package/public/assets/StageShape-CfdrDUZa.js +1 -0
  37. package/public/assets/StageSlowness-BoGLIYKR.js +1 -0
  38. package/public/assets/Straggler-DU1WSU-m.js +1 -0
  39. package/public/assets/TaskFailures-DkY6zBNJ.js +1 -0
  40. package/public/assets/TinyTask-Lsnt5mJF.js +1 -0
  41. package/public/assets/UnderBroadcast-DxHc0lwy.js +1 -0
  42. package/public/assets/WidgetLeadSummary-BdUUi7L5.js +1 -0
  43. package/public/assets/charts-vendor-0VMaLVLN.js +59 -0
  44. package/public/assets/downsample-WmNM1qc9.js +1 -0
  45. package/public/assets/finding-anchor-Bl1hRK5E.js +1 -0
  46. package/public/assets/impact-sort-DxaUheWV.js +1 -0
  47. package/public/assets/inbox-DHIFHqcn.js +1 -0
  48. package/public/assets/index-CjVklAdW.css +2 -0
  49. package/public/assets/index-U654cmB8.js +9 -0
  50. package/public/assets/parser-worker-5a9aUx3o.js +115 -0
  51. package/public/assets/plan-graph-vendor-B0sEms9Z.js +7 -0
  52. package/public/assets/plan-graph-vendor-DLioOiRN.css +1 -0
  53. package/public/assets/react-vendor-D2TLlSzI.js +9 -0
  54. package/public/assets/rolldown-runtime-hePW80VL.js +1 -0
  55. package/public/assets/select-CtKaJ5Hx.js +1 -0
  56. package/public/assets/toggle-group-DB2VfFIx.js +1 -0
  57. package/public/assets/usePagedRows-jNUWgVTD.js +1 -0
  58. package/public/assets/useRecentFiles-BiYdWs8J.js +11 -0
  59. package/public/assets/useSortMode-BhPky1lW.js +1 -0
  60. package/public/docs/404.html +3 -3
  61. package/public/docs/assets/{app.BgzsaTS3.js → app.BczRxXPr.js} +1 -1
  62. package/public/docs/assets/chunks/@localSearchIndexroot.c2HfrwDg.js +1 -0
  63. package/public/docs/assets/chunks/{VPLocalSearchBox.DOq3t-On.js → VPLocalSearchBox.DXvkTVnC.js} +1 -1
  64. package/public/docs/assets/chunks/theme.ByUdwrLG.js +2 -0
  65. package/public/docs/assets/contributor-guide_architecture_board-widgets.md.9ToZXcTk.js +1 -0
  66. package/public/docs/assets/contributor-guide_architecture_board-widgets.md.9ToZXcTk.lean.js +1 -0
  67. package/public/docs/assets/contributor-guide_architecture_detector-contract.md.B1vUqtry.js +1 -0
  68. package/public/docs/assets/{contributor-guide_architecture_detector-contract.md.BOeH4d1J.lean.js → contributor-guide_architecture_detector-contract.md.B1vUqtry.lean.js} +1 -1
  69. package/public/docs/assets/{contributor-guide_architecture_drill-down.md.BtPdlM7r.js → contributor-guide_architecture_drill-down.md.Bawx6NgP.js} +1 -1
  70. package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.DHVq1UPt.js +1 -0
  71. package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.DHVq1UPt.lean.js +1 -0
  72. package/public/docs/assets/{contributor-guide_architecture_overview.md.CehiRmGn.js → contributor-guide_architecture_overview.md.DkXdJ9jE.js} +1 -1
  73. package/public/docs/assets/contributor-guide_architecture_overview.md.DkXdJ9jE.lean.js +1 -0
  74. package/public/docs/assets/contributor-guide_architecture_state-and-history.md.C-xxn0q7.js +1 -0
  75. package/public/docs/assets/contributor-guide_architecture_state-and-history.md.C-xxn0q7.lean.js +1 -0
  76. package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.DYwFeyjV.js +1 -0
  77. package/public/docs/assets/contributor-guide_architecture_worker-protocol.md.C0SovZX8.js +6 -0
  78. package/public/docs/assets/{contributor-guide_architecture_worker-protocol.md.B93qJ_tT.lean.js → contributor-guide_architecture_worker-protocol.md.C0SovZX8.lean.js} +1 -1
  79. package/public/docs/assets/{contributor-guide_development-setup.md.DvAN_9mK.js → contributor-guide_development-setup.md.DvMV1xhQ.js} +3 -3
  80. package/public/docs/assets/contributor-guide_development-setup.md.DvMV1xhQ.lean.js +1 -0
  81. package/public/docs/assets/{contributor-guide_testing.md.6rIKqSyY.js → contributor-guide_testing.md.BohuQWvl.js} +1 -1
  82. package/public/docs/assets/{contributor-guide_testing.md.6rIKqSyY.lean.js → contributor-guide_testing.md.BohuQWvl.lean.js} +1 -1
  83. package/public/docs/assets/{style.DXOMCXxn.css → style.DSixAiZE.css} +1 -1
  84. package/public/docs/assets/{tuning-reference_bottleneck-failures.md.4z5BXGJ2.js → tuning-reference_bottleneck-failures.md.CAsFf3RR.js} +1 -1
  85. package/public/docs/assets/{tuning-reference_bottleneck-gc.md.DSxzZRK7.js → tuning-reference_bottleneck-gc.md.B4T3kJYV.js} +2 -2
  86. package/public/docs/assets/{tuning-reference_bottleneck-gc.md.DSxzZRK7.lean.js → tuning-reference_bottleneck-gc.md.B4T3kJYV.lean.js} +1 -1
  87. package/public/docs/assets/tuning-reference_bottleneck-shuffle.md.DDMQVOW6.js +12 -0
  88. package/public/docs/assets/{tuning-reference_bottleneck-shuffle.md.CM-nTmIH.lean.js → tuning-reference_bottleneck-shuffle.md.DDMQVOW6.lean.js} +1 -1
  89. package/public/docs/assets/{tuning-reference_bottleneck-skew.md.BdUwiDhn.js → tuning-reference_bottleneck-skew.md.B0MzhWcX.js} +4 -4
  90. package/public/docs/assets/{tuning-reference_bottleneck-skew.md.BdUwiDhn.lean.js → tuning-reference_bottleneck-skew.md.B0MzhWcX.lean.js} +1 -1
  91. package/public/docs/assets/{tuning-reference_bottleneck-small-files.md.B8kloyx8.js → tuning-reference_bottleneck-small-files.md.DHGMTmPs.js} +2 -2
  92. package/public/docs/assets/{tuning-reference_bottleneck-small-files.md.B8kloyx8.lean.js → tuning-reference_bottleneck-small-files.md.DHGMTmPs.lean.js} +1 -1
  93. package/public/docs/assets/{tuning-reference_bottleneck-straggler.md.DY36fHN5.js → tuning-reference_bottleneck-straggler.md.DIbXdAcQ.js} +2 -2
  94. package/public/docs/assets/{tuning-reference_bottleneck-straggler.md.DY36fHN5.lean.js → tuning-reference_bottleneck-straggler.md.DIbXdAcQ.lean.js} +1 -1
  95. package/public/docs/assets/{tuning-reference_bottleneck-tiny-tasks.md.QTV7O8kU.js → tuning-reference_bottleneck-tiny-tasks.md.CB05v3zh.js} +1 -1
  96. package/public/docs/assets/tuning-reference_bottleneck-utilization.md.DqC_iGkw.js +5 -0
  97. package/public/docs/assets/tuning-reference_bottleneck-utilization.md.DqC_iGkw.lean.js +1 -0
  98. package/public/docs/assets/{tuning-reference_cluster-config.md.ZVmDGsQ3.js → tuning-reference_cluster-config.md.CD6vzYtD.js} +1 -1
  99. package/public/docs/assets/{tuning-reference_cluster-config.md.ZVmDGsQ3.lean.js → tuning-reference_cluster-config.md.CD6vzYtD.lean.js} +1 -1
  100. package/public/docs/assets/{tuning-reference_memory-model.md.DhT-n4y3.js → tuning-reference_memory-model.md.CWhvgevP.js} +1 -1
  101. package/public/docs/assets/{tuning-reference_memory-model.md.DhT-n4y3.lean.js → tuning-reference_memory-model.md.CWhvgevP.lean.js} +1 -1
  102. package/public/docs/assets/{user-guide_alternative-log-retrieval.md.sU3KGarf.js → user-guide_alternative-log-retrieval.md.B4tPGIal.js} +1 -1
  103. package/public/docs/assets/{user-guide_alternative-log-retrieval.md.sU3KGarf.lean.js → user-guide_alternative-log-retrieval.md.B4tPGIal.lean.js} +1 -1
  104. package/public/docs/assets/user-guide_getting-started.md.BJvwLEIM.js +3 -0
  105. package/public/docs/assets/user-guide_getting-started.md.BJvwLEIM.lean.js +1 -0
  106. package/public/docs/assets/{user-guide_mcp-tools.md.C8MiIu7F.js → user-guide_mcp-tools.md.DGvolGzt.js} +4 -4
  107. package/public/docs/assets/{user-guide_mcp-tools.md.C8MiIu7F.lean.js → user-guide_mcp-tools.md.DGvolGzt.lean.js} +1 -1
  108. package/public/docs/assets/user-guide_run-comparison.md.CQc1aoU8.js +1 -0
  109. package/public/docs/assets/user-guide_run-comparison.md.CQc1aoU8.lean.js +1 -0
  110. package/public/docs/assets/user-guide_understanding-findings.md.B29f0ucS.js +1 -0
  111. package/public/docs/assets/user-guide_understanding-findings.md.B29f0ucS.lean.js +1 -0
  112. package/public/docs/contributor-guide/architecture/board-widgets.html +6 -6
  113. package/public/docs/contributor-guide/architecture/detector-contract.html +6 -6
  114. package/public/docs/contributor-guide/architecture/drill-down.html +6 -6
  115. package/public/docs/contributor-guide/architecture/impact-estimation.html +6 -6
  116. package/public/docs/contributor-guide/architecture/index.html +4 -4
  117. package/public/docs/contributor-guide/architecture/overview.html +6 -6
  118. package/public/docs/contributor-guide/architecture/state-and-history.html +6 -6
  119. package/public/docs/contributor-guide/architecture/widget-rendering.html +6 -6
  120. package/public/docs/contributor-guide/architecture/worker-protocol.html +7 -7
  121. package/public/docs/contributor-guide/contributing.html +4 -4
  122. package/public/docs/contributor-guide/development-setup.html +6 -6
  123. package/public/docs/contributor-guide/testing.html +6 -6
  124. package/public/docs/hashmap.json +1 -1
  125. package/public/docs/index.html +4 -4
  126. package/public/docs/tuning-reference/anti-patterns.html +4 -4
  127. package/public/docs/tuning-reference/aqe.html +4 -4
  128. package/public/docs/tuning-reference/bottleneck-broadcast-sizing.html +4 -4
  129. package/public/docs/tuning-reference/bottleneck-cold-start.html +4 -4
  130. package/public/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html +4 -4
  131. package/public/docs/tuning-reference/bottleneck-failures.html +6 -6
  132. package/public/docs/tuning-reference/bottleneck-gc.html +6 -6
  133. package/public/docs/tuning-reference/bottleneck-job-failure-rate.html +4 -4
  134. package/public/docs/tuning-reference/bottleneck-memory-utilization.html +4 -4
  135. package/public/docs/tuning-reference/bottleneck-retry-waste.html +4 -4
  136. package/public/docs/tuning-reference/bottleneck-shuffle.html +7 -7
  137. package/public/docs/tuning-reference/bottleneck-skew.html +8 -8
  138. package/public/docs/tuning-reference/bottleneck-slow-host.html +4 -4
  139. package/public/docs/tuning-reference/bottleneck-small-files.html +7 -7
  140. package/public/docs/tuning-reference/bottleneck-spill.html +4 -4
  141. package/public/docs/tuning-reference/bottleneck-straggler.html +6 -6
  142. package/public/docs/tuning-reference/bottleneck-tiny-tasks.html +6 -6
  143. package/public/docs/tuning-reference/bottleneck-utilization.html +7 -7
  144. package/public/docs/tuning-reference/caching.html +4 -4
  145. package/public/docs/tuning-reference/cluster-config.html +6 -6
  146. package/public/docs/tuning-reference/config.html +4 -4
  147. package/public/docs/tuning-reference/data-formats.html +4 -4
  148. package/public/docs/tuning-reference/index.html +4 -4
  149. package/public/docs/tuning-reference/intro.html +4 -4
  150. package/public/docs/tuning-reference/joins.html +4 -4
  151. package/public/docs/tuning-reference/memory-model.html +6 -6
  152. package/public/docs/tuning-reference/metrics.html +4 -4
  153. package/public/docs/tuning-reference/partitioning.html +4 -4
  154. package/public/docs/tuning-reference/pyspark.html +4 -4
  155. package/public/docs/tuning-reference/shuffle.html +4 -4
  156. package/public/docs/tuning-reference/spark-architecture.html +4 -4
  157. package/public/docs/tuning-reference/table-formats.html +4 -4
  158. package/public/docs/user-guide/alternative-log-retrieval.html +6 -6
  159. package/public/docs/user-guide/getting-started.html +7 -7
  160. package/public/docs/user-guide/mcp-tools.html +9 -9
  161. package/public/docs/user-guide/run-comparison.html +6 -6
  162. package/public/docs/user-guide/understanding-findings.html +6 -6
  163. package/public/index.html +7 -6
  164. package/public/sample-runs/sample-run.ndjson.gz +0 -0
  165. package/vendor-core/cli/collect-run.js +3 -2
  166. package/vendor-core/cli/native-zstd.js +351 -0
  167. package/vendor-core/detectors.js +390 -75
  168. package/vendor-core/docs-config.js +34 -8
  169. package/vendor-core/docs-content/chapters/03-memory-model.md +39 -0
  170. package/vendor-core/docs-content/chapters/11-cluster-config.md +40 -0
  171. package/vendor-core/docs-content/detection/cache.md +4 -3
  172. package/vendor-core/docs-content/detection/chrn.md +4 -2
  173. package/vendor-core/docs-content/detection/gc.md +2 -0
  174. package/vendor-core/docs-content/detection/host.md +2 -1
  175. package/vendor-core/docs-content/detection/local.md +2 -3
  176. package/vendor-core/docs-content/detection/mem.md +3 -3
  177. package/vendor-core/docs-content/detection/plan.md +3 -1
  178. package/vendor-core/docs-content/detection/shape.md +2 -1
  179. package/vendor-core/docs-content/detection/shfl.md +2 -1
  180. package/vendor-core/docs-content/detection/spec.md +4 -3
  181. package/vendor-core/docs-content/detection/spill.md +1 -1
  182. package/vendor-core/docs-content/detection/strag.md +2 -1
  183. package/vendor-core/docs-content/detection/tiny.md +2 -1
  184. package/vendor-core/docs-content/tuning/failures.md +1 -1
  185. package/vendor-core/docs-content/tuning/gc.md +11 -4
  186. package/vendor-core/docs-content/tuning/shuffle.md +25 -5
  187. package/vendor-core/docs-content/tuning/skew.md +14 -6
  188. package/vendor-core/docs-content/tuning/small-files.md +12 -7
  189. package/vendor-core/docs-content/tuning/straggler.md +34 -0
  190. package/vendor-core/docs-content/tuning/tiny-tasks.md +1 -1
  191. package/vendor-core/docs-content/tuning/utilization.md +57 -7
  192. package/vendor-core/docs-content/upstream.json +4 -0
  193. package/vendor-core/event-handlers.js +321 -69
  194. package/vendor-core/event-schemas.js +8 -6
  195. package/vendor-core/evidence-report.js +4 -2
  196. package/vendor-core/impact-estimator.js +150 -34
  197. package/vendor-core/mcp-tools.js +20 -7
  198. package/vendor-core/occupancy.js +70 -2
  199. package/vendor-core/parser-worker.js +30 -14
  200. package/vendor-core/plan-summary.js +4 -0
  201. package/vendor-core/run-comparison.js +22 -17
  202. package/vendor-core/shs-fetch.js +18 -7
  203. package/vendor-core/shs-load.js +2 -1
  204. package/vendor-core/stage-quantiles.js +59 -3
  205. package/vendor-core/string-hash.js +15 -0
  206. package/vendor-core/types.js +9 -1
  207. package/vendor-core/vendor/fzstd.js +94 -18
  208. package/public/assets/AutoscalingChurn-CFJkiqxW.js +0 -1
  209. package/public/assets/CacheUtilization-kTKZBzDA.js +0 -1
  210. package/public/assets/CachingOpportunity-CU70EWcF.js +0 -1
  211. package/public/assets/ColdStart-hOI-WOvm.js +0 -1
  212. package/public/assets/ConfigAudit-B4oAksZY.js +0 -1
  213. package/public/assets/CoreUsageArea-D-dR3SQk.js +0 -1
  214. package/public/assets/Dashboard-MDhohsq4.js +0 -4
  215. package/public/assets/DuplicatePlanSubtree-54RKsfI7.js +0 -1
  216. package/public/assets/DurationHistogram-CviQyPNM.js +0 -1
  217. package/public/assets/EmptyState-v2ehFncc.js +0 -1
  218. package/public/assets/ExecutorUtilization-vHyFHgL6.js +0 -1
  219. package/public/assets/GcPressure-DINuhcno.js +0 -1
  220. package/public/assets/ImpactEstimate-C6GzFOY9.js +0 -1
  221. package/public/assets/IncompleteRun-a1kuCnU6.js +0 -1
  222. package/public/assets/JobFailures-CYwPfUhR.js +0 -1
  223. package/public/assets/MemoryUtilization-Dlfj0e_f.js +0 -1
  224. package/public/assets/OverBroadcast-BMu7Y8l9.js +0 -1
  225. package/public/assets/PartitionSizing-CvMGfSVB.js +0 -1
  226. package/public/assets/PlanExplorer-xYFme_Rx.js +0 -1
  227. package/public/assets/PlanGraphRoute-a3OGg6ET.js +0 -1
  228. package/public/assets/RetryWaste-JKRjOZ-J.js +0 -1
  229. package/public/assets/RowPagination-DypWbnic.js +0 -1
  230. package/public/assets/ShuffleIO-Mi8HiVu-.js +0 -1
  231. package/public/assets/Skew-C3iOScTC.js +0 -1
  232. package/public/assets/SlowHost-DKBg18v-.js +0 -1
  233. package/public/assets/SmallFiles-CCCB6AIu.js +0 -1
  234. package/public/assets/SpeculationWaste-DUNupw0_.js +0 -1
  235. package/public/assets/Spill-BlLdatik.js +0 -1
  236. package/public/assets/StageFailed-qtY7C7Yo.js +0 -1
  237. package/public/assets/StageFindingGroup-CoVdEZYs.js +0 -1
  238. package/public/assets/StageHeader-uDBLHgdg.js +0 -1
  239. package/public/assets/StagePill-CYCMdlIx.js +0 -1
  240. package/public/assets/StageShape-zs0mAjQJ.js +0 -1
  241. package/public/assets/StageSlowness-C7vg0u5Q.js +0 -1
  242. package/public/assets/Straggler-4BRGAQP4.js +0 -1
  243. package/public/assets/TaskFailures-CSjO23Kg.js +0 -1
  244. package/public/assets/TinyTask-6Zuovcn5.js +0 -1
  245. package/public/assets/UnderBroadcast-CePjo0ZV.js +0 -1
  246. package/public/assets/WidgetLeadSummary-DRNRVvn7.js +0 -1
  247. package/public/assets/charts-vendor-Cf1-wzeH.js +0 -59
  248. package/public/assets/downsample-D8XmJk6i.js +0 -1
  249. package/public/assets/finding-anchor-4TmpXd7j.js +0 -1
  250. package/public/assets/impact-sort-Cq6KOLqh.js +0 -1
  251. package/public/assets/inbox-DBFovbUX.js +0 -1
  252. package/public/assets/index-CHjV2-jB.js +0 -9
  253. package/public/assets/index-DMEtaP4u.css +0 -1
  254. package/public/assets/parser-worker-BMPPE37w.js +0 -64
  255. package/public/assets/plan-graph-vendor-B96a-e4N.js +0 -7
  256. package/public/assets/plan-graph-vendor-BnuhLJ6X.css +0 -1
  257. package/public/assets/react-vendor-DA6opS-q.js +0 -9
  258. package/public/assets/select-D5fK-za0.js +0 -1
  259. package/public/assets/toggle-group-lbIXRG0W.js +0 -1
  260. package/public/assets/usePagedRows-CFoZrPr4.js +0 -1
  261. package/public/assets/useRecentFiles-BPGQ6XII.js +0 -13
  262. package/public/assets/useSortMode-BOPpfLhJ.js +0 -1
  263. package/public/docs/assets/chunks/@localSearchIndexroot.PQ5JepCp.js +0 -1
  264. package/public/docs/assets/chunks/theme.BP14rGBx.js +0 -2
  265. package/public/docs/assets/contributor-guide_architecture_board-widgets.md.B-OsL91z.js +0 -1
  266. package/public/docs/assets/contributor-guide_architecture_board-widgets.md.B-OsL91z.lean.js +0 -1
  267. package/public/docs/assets/contributor-guide_architecture_detector-contract.md.BOeH4d1J.js +0 -1
  268. package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.DYCDPgkh.js +0 -1
  269. package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.DYCDPgkh.lean.js +0 -1
  270. package/public/docs/assets/contributor-guide_architecture_overview.md.CehiRmGn.lean.js +0 -1
  271. package/public/docs/assets/contributor-guide_architecture_state-and-history.md.m3S3UdMk.js +0 -1
  272. package/public/docs/assets/contributor-guide_architecture_state-and-history.md.m3S3UdMk.lean.js +0 -1
  273. package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.DbqPf2OT.js +0 -1
  274. package/public/docs/assets/contributor-guide_architecture_worker-protocol.md.B93qJ_tT.js +0 -6
  275. package/public/docs/assets/contributor-guide_development-setup.md.DvAN_9mK.lean.js +0 -1
  276. package/public/docs/assets/tuning-reference_bottleneck-shuffle.md.CM-nTmIH.js +0 -12
  277. package/public/docs/assets/tuning-reference_bottleneck-utilization.md.DTiueZC3.js +0 -5
  278. package/public/docs/assets/tuning-reference_bottleneck-utilization.md.DTiueZC3.lean.js +0 -1
  279. package/public/docs/assets/user-guide_getting-started.md.DtEM37MK.js +0 -3
  280. package/public/docs/assets/user-guide_getting-started.md.DtEM37MK.lean.js +0 -1
  281. package/public/docs/assets/user-guide_run-comparison.md.S0TWWmLY.js +0 -1
  282. package/public/docs/assets/user-guide_run-comparison.md.S0TWWmLY.lean.js +0 -1
  283. package/public/docs/assets/user-guide_understanding-findings.md.D0R_Y-R2.js +0 -1
  284. package/public/docs/assets/user-guide_understanding-findings.md.D0R_Y-R2.lean.js +0 -1
  285. /package/public/docs/assets/{contributor-guide_architecture_drill-down.md.BtPdlM7r.lean.js → contributor-guide_architecture_drill-down.md.Bawx6NgP.lean.js} +0 -0
  286. /package/public/docs/assets/{contributor-guide_architecture_widget-rendering.md.DbqPf2OT.lean.js → contributor-guide_architecture_widget-rendering.md.DYwFeyjV.lean.js} +0 -0
  287. /package/public/docs/assets/{tuning-reference_bottleneck-failures.md.4z5BXGJ2.lean.js → tuning-reference_bottleneck-failures.md.CAsFf3RR.lean.js} +0 -0
  288. /package/public/docs/assets/{tuning-reference_bottleneck-tiny-tasks.md.QTV7O8kU.lean.js → tuning-reference_bottleneck-tiny-tasks.md.CB05v3zh.lean.js} +0 -0
package/public/docs/tuning-reference/memory-model.html CHANGED
@@ -6,14 +6,14 @@
6
6
  <title>Memory Management | SparkForensics</title>
7
7
  <meta name="description" content="Docs for using and contributing to SparkForensics">
8
8
  <meta name="generator" content="VitePress v1.6.4">
9
- <link rel="preload stylesheet" href="/docs/assets/style.DXOMCXxn.css" as="style">
9
+ <link rel="preload stylesheet" href="/docs/assets/style.DSixAiZE.css" as="style">
10
10
  <link rel="preload stylesheet" href="/docs/vp-icons.css" as="style">
11
11
 
12
- <script type="module" src="/docs/assets/app.BgzsaTS3.js"></script>
12
+ <script type="module" src="/docs/assets/app.BczRxXPr.js"></script>
13
13
  <link rel="preload" href="/docs/assets/inter-roman-latin.Di8DUHzh.woff2" as="font" type="font/woff2" crossorigin="">
14
- <link rel="modulepreload" href="/docs/assets/chunks/theme.BP14rGBx.js">
14
+ <link rel="modulepreload" href="/docs/assets/chunks/theme.ByUdwrLG.js">
15
15
  <link rel="modulepreload" href="/docs/assets/chunks/framework.DSg0KOwT.js">
16
- <link rel="modulepreload" href="/docs/assets/tuning-reference_memory-model.md.DhT-n4y3.lean.js">
16
+ <link rel="modulepreload" href="/docs/assets/tuning-reference_memory-model.md.CWhvgevP.lean.js">
17
17
  <link rel="icon" type="image/svg+xml" href="/docs/favicon.svg">
18
18
  <link rel="preconnect" href="https://fonts.googleapis.com">
19
19
  <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin="">
@@ -22,8 +22,8 @@
22
22
  <script id="check-mac-os">document.documentElement.classList.toggle("mac",/Mac|iPhone|iPod|iPad/i.test(navigator.platform));</script>
23
23
  </head>
24
24
  <body>
25
- <div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Getting Started</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/intro.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Introduction</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Optimization Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/spark-architecture.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Execution Model</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/memory-model.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Management</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/partitioning.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Partitioning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/joins.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Join Optimization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/data-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Data Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/table-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Table Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/caching.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Caching & Persistence</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/pyspark.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>PySpark Specifics</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/aqe.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Adaptive Query Execution</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/cluster-config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cluster Tuning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/anti-patterns.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Anti-Patterns</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Detector Catalog</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-skew.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Skew</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle I/O</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-spill.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory / Disk Spill</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-gc.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>GC Pressure</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-cold-start.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cold Start</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Executor Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-slow-host.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Slow Host</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-failures.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Failures</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-straggler.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Stragglers</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-retry-waste.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Retry Waste</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-tiny-tasks.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Tiny Tasks</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-job-failure-rate.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Job Failure Rate</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-memory-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Duplicate Plan Subtree</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-small-files.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Small Files</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-broadcast-sizing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Broadcast Sizing</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Reference</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/metrics.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Metrics Glossary</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Config Quick-Reference</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_tuning-reference_memory-model" data-v-39a288b8><div><h1 id="memory-model" tabindex="-1">Memory Management <a class="header-anchor" href="#memory-model" aria-label="Permalink to &quot;Memory Management {#memory-model}&quot;">​</a></h1><h2 id="the-unified-memory-model" tabindex="-1">The unified memory model <a class="header-anchor" href="#the-unified-memory-model" aria-label="Permalink to &quot;The unified memory model&quot;">​</a></h2><p>Since Spark 1.6, the <code>UnifiedMemoryManager</code> has replaced the earlier static split between execution and storage memory with a single shared region, referred to as M<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup><sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup>. M is carved out of the JVM heap after first setting aside 300 MiB of reserved system memory: <code>M = (JVM heap − 300 MiB) × spark.memory.fraction</code>, with <code>spark.memory.fraction</code> defaulting to 0.6<sup class="footnote-ref"><a href="#fn2" id="fnref2:1">[2:1]</a></sup><sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup>. Within M, <code>spark.memory.storageFraction</code> (default 0.5) marks off a subregion reserved for cached blocks that are immune to eviction; with defaults, this works out to roughly a 30%/30% on-heap split between execution and storage out of total executor memory<sup class="footnote-ref"><a href="#fn3" id="fnref3:1">[3:1]</a></sup>.</p><img class="light-only" src="/docs/assets/memory-regions.XHvO7jHG.svg" alt="The executor memory layout showing reserved memory, the unified region split into execution and storage, and user memory inside the JVM heap, with the off-heap pool and memory overhead outside it."><img class="dark-only" src="/docs/assets/memory-regions.dark.D4TP9_08.svg" alt="The executor memory layout showing reserved memory, the unified region split into execution and storage, and user memory inside the JVM heap, with the off-heap pool and memory overhead outside it."><p>Execution and storage borrow from each other dynamically rather than sitting behind a hard wall: when execution memory is unused, storage can acquire all of it, and vice versa<sup class="footnote-ref"><a href="#fn2" id="fnref2:2">[2:2]</a></sup><sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup>. The two are not equal peers, though. Execution always has priority, taking memory immediately and evicting cached storage blocks if necessary<sup class="footnote-ref"><a href="#fn3" id="fnref3:2">[3:2]</a></sup>.</p><p>The reserved 300 MB is a flat constant (<code>RESERVED_SYSTEM_MEMORY_BYTES</code> in the Spark source), not a formula, and it&#39;s hardcoded: there is no supported production configuration to change it. The only override is the internal <code>spark.testing.reservedMemory</code> property, which exists for Spark&#39;s own test suite rather than production tuning<sup class="footnote-ref"><a href="#fn3" id="fnref3:3">[3:3]</a></sup>. <code>spark.memory.fraction</code> is applied against heap minus that reserved 300 MB, not against total heap. The tuning guide is explicit that it &quot;expresses the size of M as a fraction of the (JVM heap space − 300MiB)&quot;<sup class="footnote-ref"><a href="#fn2" id="fnref2:3">[2:3]</a></sup>.</p><p>Off-heap memory extends the same model outside the JVM heap. When <code>spark.memory.offHeap.enabled=true</code>, Spark allocates raw off-heap buffers via <code>sun.misc.Unsafe</code> (routed through <code>jdk.internal.misc.Unsafe</code> on JDK 17+, which is why some builds need <code>--add-opens</code> flags), and that off-heap region is split into its own execution and storage pools, following the same borrowing rules as on-heap memory: execution has priority, storage gets evicted first<sup class="footnote-ref"><a href="#fn3" id="fnref3:4">[3:4]</a></sup>. Enabling it requires <code>spark.memory.offHeap.size</code> to be set to a positive value; leaving it enabled with the default size of <code>0</code> is a documented-invalid combination, not a supported way to run with off-heap &quot;on but empty&quot;<sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup>. The setting also &quot;has no impact on heap memory usage&quot;. Turning on off-heap memory does not shrink the JVM <code>-Xmx</code> for you, so the on-heap size has to be reduced manually to keep total footprint constant<sup class="footnote-ref"><a href="#fn4" id="fnref4:1">[4:1]</a></sup>. This off-heap execution memory underlies Project Tungsten: its compact binary row encoding, its explicit-memory-managed hash map for aggregations, and its cache-aware sort/join algorithms are all built to run against off-heap, GC-invisible memory<sup class="footnote-ref"><a href="#fn5" id="fnref5">[5]</a></sup><sup class="footnote-ref"><a href="#fn6" id="fnref6">[6]</a></sup>.</p><p>Executor memory overhead is a separate pool layered on top of M. <code>spark.executor.memoryOverhead</code> defaults to <code>executorMemory × spark.executor.memoryOverheadFactor</code>, floored at <code>spark.executor.minMemoryOverhead</code> (384 MiB)<sup class="footnote-ref"><a href="#fn4" id="fnref4:2">[4:2]</a></sup>. <code>spark.executor.memoryOverheadFactor</code> itself defaults to 0.10 for ordinary JVM executors, but Spark bumps that default to 0.40 specifically for Kubernetes non-JVM jobs, since those workloads tend to need more non-JVM heap space<sup class="footnote-ref"><a href="#fn4" id="fnref4:3">[4:3]</a></sup>. The legacy, YARN-only <code>spark.yarn.executor.memoryOverhead</code> property was removed in Spark 3.0 in favor of the cluster-manager-agnostic <code>spark.executor.memoryOverhead</code><sup class="footnote-ref"><a href="#fn3" id="fnref3:5">[3:5]</a></sup>.</p><blockquote><p><strong>PySpark:</strong> <code>spark.python.worker.memory</code> (default <code>512m</code>) is a soft spill threshold for a Python worker&#39;s own aggregation buffering, not a JVM-enforced cap. A PySpark worker runs as a separate OS process outside the JVM heap, so the JVM cannot police it directly<sup class="footnote-ref"><a href="#fn3" id="fnref3:6">[3:6]</a></sup>. The only setting that actually tries to bound PySpark memory per executor is <code>spark.executor.pyspark.memory</code>, and even that depends on Python&#39;s <code>resource</code> module, which isn&#39;t supported on Windows and doesn&#39;t actually limit anything on macOS<sup class="footnote-ref"><a href="#fn4" id="fnref4:4">[4:4]</a></sup>. Left unset, PySpark memory usage is folded into <code>spark.executor.memoryOverhead</code> by default<sup class="footnote-ref"><a href="#fn4" id="fnref4:5">[4:5]</a></sup>.</p></blockquote><h2 id="watching-eviction-and-spill" tabindex="-1">Watching eviction and spill <a class="header-anchor" href="#watching-eviction-and-spill" aria-label="Permalink to &quot;Watching eviction and spill&quot;">​</a></h2><p>That borrowing leaves a trace once eviction actually happens. Storage eviction under memory pressure follows an LRU (least recently used) policy<sup class="footnote-ref"><a href="#fn1" id="fnref1:2">[1:2]</a></sup><sup class="footnote-ref"><a href="#fn7" id="fnref7">[7]</a></sup>. The granularity is the block: the <code>BlockManager</code> tracks cached data as per-partition blocks (one block per RDD/DataFrame partition) and LRU operates at that level rather than evicting an entire RDD or cached table in one shot<sup class="footnote-ref"><a href="#fn8" id="fnref8">[8]</a></sup><sup class="footnote-ref"><a href="#fn9" id="fnref9">[9]</a></sup>. In practice this tends to evict the oldest partitions first, those materialized in the earliest job or stage, though lazy evaluation makes it hard to predict exactly which partitions will go ahead of time<sup class="footnote-ref"><a href="#fn9" id="fnref9:1">[9:1]</a></sup>.</p><p>What happens to an evicted block is visible in its downstream effect and depends on its <code>StorageLevel</code>. For <code>MEMORY_ONLY</code>, the block is simply dropped and recomputed from the RDD&#39;s lineage the next time it&#39;s needed. For a disk-backed level like <code>MEMORY_AND_DISK</code>, the evicted block is written to disk and read back from there instead of being recomputed<sup class="footnote-ref"><a href="#fn9" id="fnref9:2">[9:2]</a></sup><sup class="footnote-ref"><a href="#fn7" id="fnref7:1">[7:1]</a></sup>. Blocks can also be evicted well before you ever try to reuse them: <code>.cache()</code> does not guarantee a block stays resident, especially on busy clusters or long pipelines<sup class="footnote-ref"><a href="#fn7" id="fnref7:2">[7:2]</a></sup>.</p><p>On the execution side, each task gets its own <code>TaskMemoryManager</code>, which enforces a soft per-task cap: with <code>n</code> tasks running concurrently, each task is allowed to allocate somewhere between <code>1/(2n)</code> and <code>1/n</code> of total execution memory, and the first task to arrive on an idle executor typically grabs more than its later-arriving neighbors<sup class="footnote-ref"><a href="#fn3" id="fnref3:7">[3:7]</a></sup>.</p><p>When a sort or hash-aggregation operator keeps requesting execution-memory pages and can&#39;t get more, Spark doesn&#39;t fail immediately. It blocks the requesting task, spills that task&#39;s in-memory data structure to disk, or, in the worst case, throws an <code>OutOfMemoryError</code><sup class="footnote-ref"><a href="#fn3" id="fnref3:8">[3:8]</a></sup>. The spill path runs through Spark&#39;s map/shuffle I/O machinery: shuffle partitions created by wide transformations like <code>groupBy()</code> or <code>join()</code> <a href="./bottleneck-spill.html">spill</a> to the executors&#39; local disks at the location set by <code>spark.local.directory</code><sup class="footnote-ref"><a href="#fn6" id="fnref6:1">[6:1]</a></sup>. SQL physical operators apply the same idea with their own row-count guardrails: <code>sortMergeJoinExec</code>&#39;s in-memory buffer and the cartesian-product operator&#39;s buffer both spill once they cross a configured row threshold, which by default is set to the value of <code>spark.shuffle.spill.numElementsForceSpillThreshold</code><sup class="footnote-ref"><a href="#fn10" id="fnref10">[10]</a></sup>.</p><p>Whatever the trigger, the resulting spilled bytes are exposed on the task metrics as <code>memoryBytesSpilled</code>, visible in the Spark UI and event log. This is the concrete signal to watch for execution memory pressure<sup class="footnote-ref"><a href="#fn11" id="fnref11">[11]</a></sup>.</p><p>Memory-overhead misconfiguration shows up differently: it manifests as the container or pod being OOMKilled with a vague container-memory error rather than a Spark-level exception, since the enforcement happens at the YARN NodeManager or Kubernetes kubelet/cgroup layer, not inside the JVM<sup class="footnote-ref"><a href="#fn3" id="fnref3:9">[3:9]</a></sup>. Under the old default overhead factor of 0.10, non-JVM Kubernetes jobs commonly failed with &quot;Memory Overhead Exceeded&quot; errors, which is exactly why Spark bumped the Kubernetes non-JVM default to 0.40<sup class="footnote-ref"><a href="#fn4" id="fnref4:6">[4:6]</a></sup>.</p><h2 id="what-the-borrowing-costs-you" tabindex="-1">What the borrowing costs you <a class="header-anchor" href="#what-the-borrowing-costs-you" aria-label="Permalink to &quot;What the borrowing costs you&quot;">​</a></h2><p>None of this is free for whoever&#39;s counting on cached data staying put. Because execution always wins the tug-of-war over storage, <a href="./caching.html">caching</a> is never a guarantee; it&#39;s a best effort. As luminousmen-memory-management puts it: &quot;If Execution needs memory, it takes it. If Storage is using that space (cached RDDs, broadcasts), Spark starts evicting blocks. If Execution is idle, Storage can grow into that space, until Execution comes back.&quot;<sup class="footnote-ref"><a href="#fn3" id="fnref3:10">[3:10]</a></sup> That growth-then-shrink-back behavior is dynamic borrowing, not storage evicting execution; eviction is strictly one-directional. As spark-notes-task-memory-management summarizes the agreement between the two regions: &quot;keep acquiring execution memory and evict storage as you need more execution memory&quot;<sup class="footnote-ref"><a href="#fn1" id="fnref1:3">[1:3]</a></sup>, never the reverse. The same asymmetric rule carries over when off-heap memory is enabled: execution still has priority and storage still gets evicted first<sup class="footnote-ref"><a href="#fn3" id="fnref3:11">[3:11]</a></sup>. Practically, this means a cached DataFrame can silently lose blocks under memory pressure, forcing an expensive lineage recompute (for <code>MEMORY_ONLY</code>) or an extra disk round-trip (for <code>MEMORY_AND_DISK</code>) the next time it&#39;s touched.</p><img class="light-only" src="/docs/assets/memory-borrowing.BqQRJg0u.svg" alt="How storage memory borrows idle execution space while execution reclaims its own space by evicting storage in one direction."><img class="dark-only" src="/docs/assets/memory-borrowing.dark.Yhh20O9C.svg" alt="How storage memory borrows idle execution space while execution reclaims its own space by evicting storage in one direction."><p>The <code>TaskMemoryManager</code>&#39;s soft per-task cap explains why spill behavior is workload-shape-dependent rather than a fixed threshold: with more tasks packed onto an executor, each one&#39;s guaranteed share of execution memory shrinks toward <code>1/(2n)</code>, making spills more likely under high task concurrency even when total execution memory hasn&#39;t changed<sup class="footnote-ref"><a href="#fn3" id="fnref3:12">[3:12]</a></sup>.</p><p>Off-heap memory&#39;s payoff is specifically about <a href="./bottleneck-gc.html">garbage collection</a>: because off-heap buffers sit outside the JVM heap, they are invisible to the garbage collector, so fewer and smaller live objects need to be tracked, scanned, and copied, which reduces both the frequency and duration of GC pauses<sup class="footnote-ref"><a href="#fn3" id="fnref3:13">[3:13]</a></sup>. Project Tungsten&#39;s off-heap hash map for aggregations was benchmarked at over 1 million operations per second in a single thread, with &quot;almost no performance degradation as memory utilization increases,&quot; unlike the JVM default <code>java.util.HashMap</code>, which eventually thrashes on GC<sup class="footnote-ref"><a href="#fn5" id="fnref5:1">[5:1]</a></sup><sup class="footnote-ref"><a href="#fn6" id="fnref6:2">[6:2]</a></sup>. The tradeoff is that off-heap memory removes the GC safety net along with the GC overhead: there&#39;s no garbage collector cleaning up if something goes wrong<sup class="footnote-ref"><a href="#fn3" id="fnref3:14">[3:14]</a></sup>.</p><p>Memory overhead matters because off-heap memory and (if unset) PySpark memory are not automatically folded into the overhead calculation the way heap memory is. Enabling <code>spark.memory.offHeap.size</code> without also raising <code>spark.executor.memoryOverhead</code> (or explicitly setting <code>spark.executor.pyspark.memory</code>) can push the executor&#39;s real footprint past what the cluster manager granted, and the process gets killed with a vague container-memory error instead of a Spark-level exception<sup class="footnote-ref"><a href="#fn3" id="fnref3:15">[3:15]</a></sup>. Worked example: with <code>--executor-memory=8G</code>, the default 10% overhead gives <code>max(0.1 × 8192 MB, 384 MB) = 819 MB</code>, so the total memory requested from the cluster manager is <code>8192 + 819 = 9011 MB</code><sup class="footnote-ref"><a href="#fn3" id="fnref3:16">[3:16]</a></sup>, useful to keep in mind when <a href="./cluster-config.html">sizing containers or pods</a> against a cluster&#39;s available capacity.</p><h2 id="tuning-the-memory-pools" tabindex="-1">Tuning the memory pools <a class="header-anchor" href="#tuning-the-memory-pools" aria-label="Permalink to &quot;Tuning the memory pools&quot;">​</a></h2><p>Most of this is adjustable, within limits. Tune <code>spark.memory.fraction</code> and <code>spark.memory.storageFraction</code> only if your workload&#39;s execution/storage balance genuinely needs to shift away from the ~30/30 default. But don&#39;t try to reclaim the 300 MB reserved region; it&#39;s a fixed, non-tunable constant in production<sup class="footnote-ref"><a href="#fn3" id="fnref3:17">[3:17]</a></sup>.</p><p>If losing cached partitions to eviction is costly, prefer a disk-backed <code>StorageLevel</code> such as <code>MEMORY_AND_DISK</code> over <code>MEMORY_ONLY</code>: an evicted block gets written to disk and read back rather than triggering a full lineage recompute<sup class="footnote-ref"><a href="#fn9" id="fnref9:3">[9:3]</a></sup><sup class="footnote-ref"><a href="#fn7" id="fnref7:3">[7:3]</a></sup>. Keep in mind <code>.cache()</code> alone is not a residency guarantee even with this choice<sup class="footnote-ref"><a href="#fn7" id="fnref7:4">[7:4]</a></sup>.</p><p>Watch <code>memoryBytesSpilled</code> in the Spark UI or history server to catch execution-memory pressure early<sup class="footnote-ref"><a href="#fn11" id="fnref11:1">[11:1]</a></sup>. If sort or hash-aggregation spills are heavy, consider giving the executor more memory or reducing the number of concurrently running tasks per executor, since the <code>TaskMemoryManager</code>&#39;s per-task guarantee shrinks as task concurrency <code>n</code> grows<sup class="footnote-ref"><a href="#fn3" id="fnref3:18">[3:18]</a></sup>. For SQL joins and cartesian products, <code>spark.shuffle.spill.numElementsForceSpillThreshold</code> governs when <code>sortMergeJoinExec</code> and the cartesian-product operator spill their buffers<sup class="footnote-ref"><a href="#fn10" id="fnref10:1">[10:1]</a></sup>.</p><p>Set <code>spark.executor.memoryOverhead</code> explicitly rather than relying on the default, especially on Kubernetes for non-JVM workloads (where the default factor is already bumped to 0.40) or whenever off-heap memory or heavy PySpark usage is in play: those aren&#39;t automatically folded into the overhead calculation, so an unadjusted overhead can lead to an OOMKilled container instead of a clean Spark-level error<sup class="footnote-ref"><a href="#fn3" id="fnref3:19">[3:19]</a></sup><sup class="footnote-ref"><a href="#fn4" id="fnref4:7">[4:7]</a></sup>. Use <code>spark.executor.memoryOverhead</code>, not the removed <code>spark.yarn.executor.memoryOverhead</code><sup class="footnote-ref"><a href="#fn3" id="fnref3:20">[3:20]</a></sup>. For capacity planning, remember the total requested from the cluster manager is executor memory plus overhead, e.g., <code>8192 + 819 = 9011 MB</code> for an 8 GB executor at the default 10% factor<sup class="footnote-ref"><a href="#fn3" id="fnref3:21">[3:21]</a></sup>.</p><p>When enabling off-heap memory, always pair <code>spark.memory.offHeap.enabled=true</code> with an explicit positive <code>spark.memory.offHeap.size</code> (never leave it enabled at the default size of <code>0</code><sup class="footnote-ref"><a href="#fn4" id="fnref4:8">[4:8]</a></sup>) and manually reduce the JVM <code>-Xmx</code> to compensate, since enabling off-heap memory does not shrink the heap for you<sup class="footnote-ref"><a href="#fn4" id="fnref4:9">[4:9]</a></sup>.</p><blockquote><p><strong>PySpark:</strong> if you need PySpark&#39;s own memory bounded rather than folded silently into the overhead, set <code>spark.executor.pyspark.memory</code> explicitly, though its enforcement relies on Python&#39;s <code>resource</code> module and won&#39;t work on Windows and won&#39;t actually limit anything on macOS<sup class="footnote-ref"><a href="#fn4" id="fnref4:10">[4:10]</a></sup>.</p></blockquote><h2 id="sources" tabindex="-1">Sources <a class="header-anchor" href="#sources" aria-label="Permalink to &quot;Sources&quot;">​</a></h2><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><a href="https://raw.githubusercontent.com/spoddutur/spark-notes/master/task_memory_management_in_spark.md" target="_blank" rel="noreferrer">Task Memory Management in Spark</a> <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a> <a href="#fnref1:2" class="footnote-backref">↩︎</a> <a href="#fnref1:3" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/tuning.html" target="_blank" rel="noreferrer">Tuning Spark</a> <a href="#fnref2" class="footnote-backref">↩︎</a> <a href="#fnref2:1" class="footnote-backref">↩︎</a> <a href="#fnref2:2" class="footnote-backref">↩︎</a> <a href="#fnref2:3" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><a href="https://luminousmen.com/post/dive-into-spark-memory" target="_blank" rel="noreferrer">Dive into Spark Memory</a> <a href="#fnref3" class="footnote-backref">↩︎</a> <a href="#fnref3:1" class="footnote-backref">↩︎</a> <a href="#fnref3:2" class="footnote-backref">↩︎</a> <a href="#fnref3:3" class="footnote-backref">↩︎</a> <a href="#fnref3:4" class="footnote-backref">↩︎</a> <a href="#fnref3:5" class="footnote-backref">↩︎</a> <a href="#fnref3:6" class="footnote-backref">↩︎</a> <a href="#fnref3:7" class="footnote-backref">↩︎</a> <a href="#fnref3:8" class="footnote-backref">↩︎</a> <a href="#fnref3:9" class="footnote-backref">↩︎</a> <a href="#fnref3:10" class="footnote-backref">↩︎</a> <a href="#fnref3:11" class="footnote-backref">↩︎</a> <a href="#fnref3:12" class="footnote-backref">↩︎</a> <a href="#fnref3:13" class="footnote-backref">↩︎</a> <a href="#fnref3:14" class="footnote-backref">↩︎</a> <a href="#fnref3:15" class="footnote-backref">↩︎</a> <a href="#fnref3:16" class="footnote-backref">↩︎</a> <a href="#fnref3:17" class="footnote-backref">↩︎</a> <a href="#fnref3:18" class="footnote-backref">↩︎</a> <a href="#fnref3:19" class="footnote-backref">↩︎</a> <a href="#fnref3:20" class="footnote-backref">↩︎</a> <a href="#fnref3:21" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration — Spark</a> <a href="#fnref4" class="footnote-backref">↩︎</a> <a href="#fnref4:1" class="footnote-backref">↩︎</a> <a href="#fnref4:2" class="footnote-backref">↩︎</a> <a href="#fnref4:3" class="footnote-backref">↩︎</a> <a href="#fnref4:4" class="footnote-backref">↩︎</a> <a href="#fnref4:5" class="footnote-backref">↩︎</a> <a href="#fnref4:6" class="footnote-backref">↩︎</a> <a href="#fnref4:7" class="footnote-backref">↩︎</a> <a href="#fnref4:8" class="footnote-backref">↩︎</a> <a href="#fnref4:9" class="footnote-backref">↩︎</a> <a href="#fnref4:10" class="footnote-backref">↩︎</a></p></li><li id="fn5" class="footnote-item"><p><a href="https://www.databricks.com/blog/2015/04/28/project-tungsten-bringing-spark-closer-to-bare-metal.html" target="_blank" rel="noreferrer">Project Tungsten: Bringing Spark Closer to Bare Metal</a> <a href="#fnref5" class="footnote-backref">↩︎</a> <a href="#fnref5:1" class="footnote-backref">↩︎</a></p></li><li id="fn6" class="footnote-item"><p><em>Learning Spark, 2nd Edition</em>, Damji, Wenig, Das, Lee, ch. 6–7 <a href="#fnref6" class="footnote-backref">↩︎</a> <a href="#fnref6:1" class="footnote-backref">↩︎</a> <a href="#fnref6:2" class="footnote-backref">↩︎</a></p></li><li id="fn7" class="footnote-item"><p><a href="https://luminousmen.com/post/explaining-the-mechanics-of-spark-caching" target="_blank" rel="noreferrer">Explaining the Mechanics of Spark Caching</a> <a href="#fnref7" class="footnote-backref">↩︎</a> <a href="#fnref7:1" class="footnote-backref">↩︎</a> <a href="#fnref7:2" class="footnote-backref">↩︎</a> <a href="#fnref7:3" class="footnote-backref">↩︎</a> <a href="#fnref7:4" class="footnote-backref">↩︎</a></p></li><li id="fn8" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/rdd-programming-guide.html" target="_blank" rel="noreferrer">RDD Programming Guide</a> <a href="#fnref8" class="footnote-backref">↩︎</a></p></li><li id="fn9" class="footnote-item"><p><em>High Performance Spark, 2nd Edition</em>, Karau, Polak &amp; Warren, ch. 7 <a href="#fnref9" class="footnote-backref">↩︎</a> <a href="#fnref9:1" class="footnote-backref">↩︎</a> <a href="#fnref9:2" class="footnote-backref">↩︎</a> <a href="#fnref9:3" class="footnote-backref">↩︎</a></p></li><li id="fn10" class="footnote-item"><p><a href="https://raw.githubusercontent.com/apache/spark/v3.5.0/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala" target="_blank" rel="noreferrer">SQLConf.scala</a> <a href="#fnref10" class="footnote-backref">↩︎</a> <a href="#fnref10:1" class="footnote-backref">↩︎</a></p></li><li id="fn11" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/monitoring.html#spark-history-server" target="_blank" rel="noreferrer">Monitoring and Instrumentation</a> <a href="#fnref11" class="footnote-backref">↩︎</a> <a href="#fnref11:1" class="footnote-backref">↩︎</a></p></li></ol></section></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/tuning-reference/spark-architecture.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>Spark Execution Model</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="/docs/tuning-reference/partitioning.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>Partitioning</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
26
- <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"B-OsL91z\",\"contributor-guide_architecture_detector-contract.md\":\"BOeH4d1J\",\"contributor-guide_architecture_drill-down.md\":\"BtPdlM7r\",\"contributor-guide_architecture_impact-estimation.md\":\"DYCDPgkh\",\"contributor-guide_architecture_index.md\":\"3TO9ic6w\",\"contributor-guide_architecture_overview.md\":\"CehiRmGn\",\"contributor-guide_architecture_state-and-history.md\":\"m3S3UdMk\",\"contributor-guide_architecture_widget-rendering.md\":\"DbqPf2OT\",\"contributor-guide_architecture_worker-protocol.md\":\"B93qJ_tT\",\"contributor-guide_contributing.md\":\"CvRsdr6J\",\"contributor-guide_development-setup.md\":\"DvAN_9mK\",\"contributor-guide_testing.md\":\"6rIKqSyY\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"Df1YMIHu\",\"tuning-reference_aqe.md\":\"BIsCtLzm\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"CEstB3Ia\",\"tuning-reference_bottleneck-cold-start.md\":\"CEuy-72y\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"CIohQDfn\",\"tuning-reference_bottleneck-failures.md\":\"4z5BXGJ2\",\"tuning-reference_bottleneck-gc.md\":\"DSxzZRK7\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"BaJl__1W\",\"tuning-reference_bottleneck-memory-utilization.md\":\"DbP-SJZc\",\"tuning-reference_bottleneck-retry-waste.md\":\"D5JMjVOt\",\"tuning-reference_bottleneck-shuffle.md\":\"CM-nTmIH\",\"tuning-reference_bottleneck-skew.md\":\"BdUwiDhn\",\"tuning-reference_bottleneck-slow-host.md\":\"BlIo6UDW\",\"tuning-reference_bottleneck-small-files.md\":\"B8kloyx8\",\"tuning-reference_bottleneck-spill.md\":\"PNH7mITt\",\"tuning-reference_bottleneck-straggler.md\":\"DY36fHN5\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"QTV7O8kU\",\"tuning-reference_bottleneck-utilization.md\":\"DTiueZC3\",\"tuning-reference_caching.md\":\"B7aQ8asB\",\"tuning-reference_cluster-config.md\":\"ZVmDGsQ3\",\"tuning-reference_config.md\":\"UvveiWG3\",\"tuning-reference_data-formats.md\":\"bjCAWH3N\",\"tuning-reference_index.md\":\"BQ_NooMV\",\"tuning-reference_intro.md\":\"CobD-lGB\",\"tuning-reference_joins.md\":\"BtKs_CuW\",\"tuning-reference_memory-model.md\":\"DhT-n4y3\",\"tuning-reference_metrics.md\":\"mLOh7Apj\",\"tuning-reference_partitioning.md\":\"q0zKF_8X\",\"tuning-reference_pyspark.md\":\"DDCfvN9t\",\"tuning-reference_shuffle.md\":\"BZZ7R4Ix\",\"tuning-reference_spark-architecture.md\":\"Dwzm5avO\",\"tuning-reference_table-formats.md\":\"D6wj-2dX\",\"user-guide_alternative-log-retrieval.md\":\"sU3KGarf\",\"user-guide_getting-started.md\":\"DtEM37MK\",\"user-guide_mcp-tools.md\":\"C8MiIu7F\",\"user-guide_run-comparison.md\":\"S0TWWmLY\",\"user-guide_understanding-findings.md\":\"D0R_Y-R2\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Alternative log retrieval\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
25
+ <div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Getting Started</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/intro.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Introduction</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Optimization Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/spark-architecture.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Execution Model</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/memory-model.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Management</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/partitioning.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Partitioning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/joins.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Join Optimization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/data-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Data Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/table-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Table Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/caching.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Caching & Persistence</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/pyspark.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>PySpark Specifics</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/aqe.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Adaptive Query Execution</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/cluster-config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cluster Tuning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/anti-patterns.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Anti-Patterns</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Detector Catalog</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-skew.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Skew</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle I/O</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-spill.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory / Disk Spill</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-gc.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>GC Pressure</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-cold-start.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cold Start</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Executor Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-slow-host.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Slow Host</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-failures.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Failures</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-straggler.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Stragglers</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-retry-waste.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Retry Waste</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-tiny-tasks.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Tiny Tasks</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-job-failure-rate.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Job Failure Rate</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-memory-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Duplicate Plan Subtree</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-small-files.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Small Files</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-broadcast-sizing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Broadcast Sizing</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Reference</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/metrics.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Metrics Glossary</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Config Quick-Reference</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_tuning-reference_memory-model" data-v-39a288b8><div><h1 id="memory-model" tabindex="-1">Memory Management <a class="header-anchor" href="#memory-model" aria-label="Permalink to &quot;Memory Management {#memory-model}&quot;">​</a></h1><h2 id="the-unified-memory-model" tabindex="-1">The unified memory model <a class="header-anchor" href="#the-unified-memory-model" aria-label="Permalink to &quot;The unified memory model&quot;">​</a></h2><p>Since Spark 1.6, the <code>UnifiedMemoryManager</code> has replaced the earlier static split between execution and storage memory with a single shared region, referred to as M<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup><sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup>. M is carved out of the JVM heap after first setting aside 300 MiB of reserved system memory: <code>M = (JVM heap − 300 MiB) × spark.memory.fraction</code>, with <code>spark.memory.fraction</code> defaulting to 0.6<sup class="footnote-ref"><a href="#fn2" id="fnref2:1">[2:1]</a></sup><sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup>. Within M, <code>spark.memory.storageFraction</code> (default 0.5) marks off a subregion reserved for cached blocks that are immune to eviction; with defaults, this works out to roughly a 30%/30% on-heap split between execution and storage out of total executor memory<sup class="footnote-ref"><a href="#fn3" id="fnref3:1">[3:1]</a></sup>.</p><img class="light-only" src="/docs/assets/memory-regions.XHvO7jHG.svg" alt="The executor memory layout showing reserved memory, the unified region split into execution and storage, and user memory inside the JVM heap, with the off-heap pool and memory overhead outside it."><img class="dark-only" src="/docs/assets/memory-regions.dark.D4TP9_08.svg" alt="The executor memory layout showing reserved memory, the unified region split into execution and storage, and user memory inside the JVM heap, with the off-heap pool and memory overhead outside it."><p>Execution and storage borrow from each other dynamically rather than sitting behind a hard wall: when execution memory is unused, storage can acquire all of it, and vice versa<sup class="footnote-ref"><a href="#fn2" id="fnref2:2">[2:2]</a></sup><sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup>. The two are not equal peers, though. Execution always has priority, taking memory immediately and evicting cached storage blocks if necessary<sup class="footnote-ref"><a href="#fn3" id="fnref3:2">[3:2]</a></sup>.</p><p>The reserved 300 MB is a flat constant (<code>RESERVED_SYSTEM_MEMORY_BYTES</code> in the Spark source), not a formula, and it&#39;s hardcoded: there is no supported production configuration to change it. The only override is the internal <code>spark.testing.reservedMemory</code> property, which exists for Spark&#39;s own test suite rather than production tuning<sup class="footnote-ref"><a href="#fn3" id="fnref3:3">[3:3]</a></sup>. <code>spark.memory.fraction</code> is applied against heap minus that reserved 300 MB, not against total heap. The tuning guide is explicit that it &quot;expresses the size of M as a fraction of the (JVM heap space − 300MiB)&quot;<sup class="footnote-ref"><a href="#fn2" id="fnref2:3">[2:3]</a></sup>.</p><p>Off-heap memory extends the same model outside the JVM heap. When <code>spark.memory.offHeap.enabled=true</code>, Spark allocates raw off-heap buffers via <code>sun.misc.Unsafe</code> (routed through <code>jdk.internal.misc.Unsafe</code> on JDK 17+, which is why some builds need <code>--add-opens</code> flags), and that off-heap region is split into its own execution and storage pools, following the same borrowing rules as on-heap memory: execution has priority, storage gets evicted first<sup class="footnote-ref"><a href="#fn3" id="fnref3:4">[3:4]</a></sup>. Enabling it requires <code>spark.memory.offHeap.size</code> to be set to a positive value; leaving it enabled with the default size of <code>0</code> is a documented-invalid combination, not a supported way to run with off-heap &quot;on but empty&quot;<sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup>. The setting also &quot;has no impact on heap memory usage&quot;. Turning on off-heap memory does not shrink the JVM <code>-Xmx</code> for you, so the on-heap size has to be reduced manually to keep total footprint constant<sup class="footnote-ref"><a href="#fn4" id="fnref4:1">[4:1]</a></sup>. This off-heap execution memory underlies Project Tungsten: its compact binary row encoding, its explicit-memory-managed hash map for aggregations, and its cache-aware sort/join algorithms are all built to run against off-heap, GC-invisible memory<sup class="footnote-ref"><a href="#fn5" id="fnref5">[5]</a></sup><sup class="footnote-ref"><a href="#fn6" id="fnref6">[6]</a></sup>.</p><p>Executor memory overhead is a separate pool layered on top of M. <code>spark.executor.memoryOverhead</code> defaults to <code>executorMemory × spark.executor.memoryOverheadFactor</code>, floored at <code>spark.executor.minMemoryOverhead</code> (384 MiB)<sup class="footnote-ref"><a href="#fn4" id="fnref4:2">[4:2]</a></sup>. <code>spark.executor.memoryOverheadFactor</code> itself defaults to 0.10 for ordinary JVM executors, but Spark bumps that default to 0.40 specifically for Kubernetes non-JVM jobs, since those workloads tend to need more non-JVM heap space<sup class="footnote-ref"><a href="#fn4" id="fnref4:3">[4:3]</a></sup>. The legacy, YARN-only <code>spark.yarn.executor.memoryOverhead</code> property was removed in Spark 3.0 in favor of the cluster-manager-agnostic <code>spark.executor.memoryOverhead</code><sup class="footnote-ref"><a href="#fn3" id="fnref3:5">[3:5]</a></sup>.</p><blockquote><p><strong>PySpark:</strong> <code>spark.python.worker.memory</code> (default <code>512m</code>) is a soft spill threshold for a Python worker&#39;s own aggregation buffering, not a JVM-enforced cap. A PySpark worker runs as a separate OS process outside the JVM heap, so the JVM cannot police it directly<sup class="footnote-ref"><a href="#fn3" id="fnref3:6">[3:6]</a></sup>. The only setting that actually tries to bound PySpark memory per executor is <code>spark.executor.pyspark.memory</code>, and even that depends on Python&#39;s <code>resource</code> module, which isn&#39;t supported on Windows and doesn&#39;t actually limit anything on macOS<sup class="footnote-ref"><a href="#fn4" id="fnref4:4">[4:4]</a></sup>. Left unset, PySpark memory usage is folded into <code>spark.executor.memoryOverhead</code> by default<sup class="footnote-ref"><a href="#fn4" id="fnref4:5">[4:5]</a></sup>.</p></blockquote><h2 id="watching-eviction-and-spill" tabindex="-1">Watching eviction and spill <a class="header-anchor" href="#watching-eviction-and-spill" aria-label="Permalink to &quot;Watching eviction and spill&quot;">​</a></h2><p>That borrowing leaves a trace once eviction actually happens. Storage eviction under memory pressure follows an LRU (least recently used) policy<sup class="footnote-ref"><a href="#fn1" id="fnref1:2">[1:2]</a></sup><sup class="footnote-ref"><a href="#fn7" id="fnref7">[7]</a></sup>. The granularity is the block: the <code>BlockManager</code> tracks cached data as per-partition blocks (one block per RDD/DataFrame partition) and LRU operates at that level rather than evicting an entire RDD or cached table in one shot<sup class="footnote-ref"><a href="#fn8" id="fnref8">[8]</a></sup><sup class="footnote-ref"><a href="#fn9" id="fnref9">[9]</a></sup>. In practice this tends to evict the oldest partitions first, those materialized in the earliest job or stage, though lazy evaluation makes it hard to predict exactly which partitions will go ahead of time<sup class="footnote-ref"><a href="#fn9" id="fnref9:1">[9:1]</a></sup>.</p><p>What happens to an evicted block is visible in its downstream effect and depends on its <code>StorageLevel</code>. For <code>MEMORY_ONLY</code>, the block is simply dropped and recomputed from the RDD&#39;s lineage the next time it&#39;s needed. For a disk-backed level like <code>MEMORY_AND_DISK</code>, the evicted block is written to disk and read back from there instead of being recomputed<sup class="footnote-ref"><a href="#fn9" id="fnref9:2">[9:2]</a></sup><sup class="footnote-ref"><a href="#fn7" id="fnref7:1">[7:1]</a></sup>. Blocks can also be evicted well before you ever try to reuse them: <code>.cache()</code> does not guarantee a block stays resident, especially on busy clusters or long pipelines<sup class="footnote-ref"><a href="#fn7" id="fnref7:2">[7:2]</a></sup>.</p><p>On the execution side, each task gets its own <code>TaskMemoryManager</code>, which enforces a soft per-task cap: with <code>n</code> tasks running concurrently, each task is allowed to allocate somewhere between <code>1/(2n)</code> and <code>1/n</code> of total execution memory, and the first task to arrive on an idle executor typically grabs more than its later-arriving neighbors<sup class="footnote-ref"><a href="#fn3" id="fnref3:7">[3:7]</a></sup>.</p><p>When a sort or hash-aggregation operator keeps requesting execution-memory pages and can&#39;t get more, Spark doesn&#39;t fail immediately. It blocks the requesting task, spills that task&#39;s in-memory data structure to disk, or, in the worst case, throws an <code>OutOfMemoryError</code><sup class="footnote-ref"><a href="#fn3" id="fnref3:8">[3:8]</a></sup>. The spill path runs through Spark&#39;s map/shuffle I/O machinery: shuffle partitions created by wide transformations like <code>groupBy()</code> or <code>join()</code> <a href="./bottleneck-spill.html">spill</a> to the executors&#39; local disks at the location set by <code>spark.local.directory</code><sup class="footnote-ref"><a href="#fn6" id="fnref6:1">[6:1]</a></sup>. SQL physical operators apply the same idea with their own row-count guardrails: <code>sortMergeJoinExec</code>&#39;s in-memory buffer and the cartesian-product operator&#39;s buffer both spill once they cross a configured row threshold, which by default is set to the value of <code>spark.shuffle.spill.numElementsForceSpillThreshold</code><sup class="footnote-ref"><a href="#fn10" id="fnref10">[10]</a></sup>.</p><p>Whatever the trigger, the resulting spilled bytes are exposed on the task metrics as <code>memoryBytesSpilled</code>, visible in the Spark UI and event log. This is the concrete signal to watch for execution memory pressure<sup class="footnote-ref"><a href="#fn11" id="fnref11">[11]</a></sup>.</p><p>Memory-overhead misconfiguration shows up differently: it manifests as the container or pod being OOMKilled with a vague container-memory error rather than a Spark-level exception, since the enforcement happens at the YARN NodeManager or Kubernetes kubelet/cgroup layer, not inside the JVM<sup class="footnote-ref"><a href="#fn3" id="fnref3:9">[3:9]</a></sup>. Under the old default overhead factor of 0.10, non-JVM Kubernetes jobs commonly failed with &quot;Memory Overhead Exceeded&quot; errors, which is exactly why Spark bumped the Kubernetes non-JVM default to 0.40<sup class="footnote-ref"><a href="#fn4" id="fnref4:6">[4:6]</a></sup>.</p><h2 id="what-the-borrowing-costs-you" tabindex="-1">What the borrowing costs you <a class="header-anchor" href="#what-the-borrowing-costs-you" aria-label="Permalink to &quot;What the borrowing costs you&quot;">​</a></h2><p>None of this is free for whoever&#39;s counting on cached data staying put. Because execution always wins the tug-of-war over storage, <a href="./caching.html">caching</a> is never a guarantee; it&#39;s a best effort. As luminousmen-memory-management puts it: &quot;If Execution needs memory, it takes it. If Storage is using that space (cached RDDs, broadcasts), Spark starts evicting blocks. If Execution is idle, Storage can grow into that space, until Execution comes back.&quot;<sup class="footnote-ref"><a href="#fn3" id="fnref3:10">[3:10]</a></sup> That growth-then-shrink-back behavior is dynamic borrowing, not storage evicting execution; eviction is strictly one-directional. As spark-notes-task-memory-management summarizes the agreement between the two regions: &quot;keep acquiring execution memory and evict storage as you need more execution memory&quot;<sup class="footnote-ref"><a href="#fn1" id="fnref1:3">[1:3]</a></sup>, never the reverse. The same asymmetric rule carries over when off-heap memory is enabled: execution still has priority and storage still gets evicted first<sup class="footnote-ref"><a href="#fn3" id="fnref3:11">[3:11]</a></sup>. Practically, this means a cached DataFrame can silently lose blocks under memory pressure, forcing an expensive lineage recompute (for <code>MEMORY_ONLY</code>) or an extra disk round-trip (for <code>MEMORY_AND_DISK</code>) the next time it&#39;s touched.</p><img class="light-only" src="/docs/assets/memory-borrowing.BqQRJg0u.svg" alt="How storage memory borrows idle execution space while execution reclaims its own space by evicting storage in one direction."><img class="dark-only" src="/docs/assets/memory-borrowing.dark.Yhh20O9C.svg" alt="How storage memory borrows idle execution space while execution reclaims its own space by evicting storage in one direction."><p>The <code>TaskMemoryManager</code>&#39;s soft per-task cap explains why spill behavior is workload-shape-dependent rather than a fixed threshold: with more tasks packed onto an executor, each one&#39;s guaranteed share of execution memory shrinks toward <code>1/(2n)</code>, making spills more likely under high task concurrency even when total execution memory hasn&#39;t changed<sup class="footnote-ref"><a href="#fn3" id="fnref3:12">[3:12]</a></sup>.</p><p>Off-heap memory&#39;s payoff is specifically about <a href="./bottleneck-gc.html">garbage collection</a>: because off-heap buffers sit outside the JVM heap, they are invisible to the garbage collector, so fewer and smaller live objects need to be tracked, scanned, and copied, which reduces both the frequency and duration of GC pauses<sup class="footnote-ref"><a href="#fn3" id="fnref3:13">[3:13]</a></sup>. Project Tungsten&#39;s off-heap hash map for aggregations was benchmarked at over 1 million operations per second in a single thread, with &quot;almost no performance degradation as memory utilization increases,&quot; unlike the JVM default <code>java.util.HashMap</code>, which eventually thrashes on GC<sup class="footnote-ref"><a href="#fn5" id="fnref5:1">[5:1]</a></sup><sup class="footnote-ref"><a href="#fn6" id="fnref6:2">[6:2]</a></sup>. The tradeoff is that off-heap memory removes the GC safety net along with the GC overhead: there&#39;s no garbage collector cleaning up if something goes wrong<sup class="footnote-ref"><a href="#fn3" id="fnref3:14">[3:14]</a></sup>.</p><p>Memory overhead matters because off-heap memory and (if unset) PySpark memory are not automatically folded into the overhead calculation the way heap memory is. Enabling <code>spark.memory.offHeap.size</code> without also raising <code>spark.executor.memoryOverhead</code> (or explicitly setting <code>spark.executor.pyspark.memory</code>) can push the executor&#39;s real footprint past what the cluster manager granted, and the process gets killed with a vague container-memory error instead of a Spark-level exception<sup class="footnote-ref"><a href="#fn3" id="fnref3:15">[3:15]</a></sup>. Worked example: with <code>--executor-memory=8G</code>, the default 10% overhead gives <code>max(0.1 × 8192 MB, 384 MB) = 819 MB</code>, so the total memory requested from the cluster manager is <code>8192 + 819 = 9011 MB</code><sup class="footnote-ref"><a href="#fn3" id="fnref3:16">[3:16]</a></sup>, useful to keep in mind when <a href="./cluster-config.html">sizing containers or pods</a> against a cluster&#39;s available capacity.</p><h2 id="tuning-the-memory-pools" tabindex="-1">Tuning the memory pools <a class="header-anchor" href="#tuning-the-memory-pools" aria-label="Permalink to &quot;Tuning the memory pools&quot;">​</a></h2><p>Most of this is adjustable, within limits. Tune <code>spark.memory.fraction</code> and <code>spark.memory.storageFraction</code> only if your workload&#39;s execution/storage balance genuinely needs to shift away from the ~30/30 default. But don&#39;t try to reclaim the 300 MB reserved region; it&#39;s a fixed, non-tunable constant in production<sup class="footnote-ref"><a href="#fn3" id="fnref3:17">[3:17]</a></sup>.</p><p>If losing cached partitions to eviction is costly, prefer a disk-backed <code>StorageLevel</code> such as <code>MEMORY_AND_DISK</code> over <code>MEMORY_ONLY</code>: an evicted block gets written to disk and read back rather than triggering a full lineage recompute<sup class="footnote-ref"><a href="#fn9" id="fnref9:3">[9:3]</a></sup><sup class="footnote-ref"><a href="#fn7" id="fnref7:3">[7:3]</a></sup>. Keep in mind <code>.cache()</code> alone is not a residency guarantee even with this choice<sup class="footnote-ref"><a href="#fn7" id="fnref7:4">[7:4]</a></sup>.</p><p>Watch <code>memoryBytesSpilled</code> in the Spark UI or history server to catch execution-memory pressure early<sup class="footnote-ref"><a href="#fn11" id="fnref11:1">[11:1]</a></sup>. If sort or hash-aggregation spills are heavy, consider giving the executor more memory or reducing the number of concurrently running tasks per executor, since the <code>TaskMemoryManager</code>&#39;s per-task guarantee shrinks as task concurrency <code>n</code> grows<sup class="footnote-ref"><a href="#fn3" id="fnref3:18">[3:18]</a></sup>. For SQL joins and cartesian products, <code>spark.shuffle.spill.numElementsForceSpillThreshold</code> governs when <code>sortMergeJoinExec</code> and the cartesian-product operator spill their buffers<sup class="footnote-ref"><a href="#fn10" id="fnref10:1">[10:1]</a></sup>.</p><p>Set <code>spark.executor.memoryOverhead</code> explicitly rather than relying on the default, especially on Kubernetes for non-JVM workloads (where the default factor is already bumped to 0.40) or whenever off-heap memory or heavy PySpark usage is in play: those aren&#39;t automatically folded into the overhead calculation, so an unadjusted overhead can lead to an OOMKilled container instead of a clean Spark-level error<sup class="footnote-ref"><a href="#fn3" id="fnref3:19">[3:19]</a></sup><sup class="footnote-ref"><a href="#fn4" id="fnref4:7">[4:7]</a></sup>. Use <code>spark.executor.memoryOverhead</code>, not the removed <code>spark.yarn.executor.memoryOverhead</code><sup class="footnote-ref"><a href="#fn3" id="fnref3:20">[3:20]</a></sup>. For capacity planning, remember the total requested from the cluster manager is executor memory plus overhead, e.g., <code>8192 + 819 = 9011 MB</code> for an 8 GB executor at the default 10% factor<sup class="footnote-ref"><a href="#fn3" id="fnref3:21">[3:21]</a></sup>.</p><p>When enabling off-heap memory, always pair <code>spark.memory.offHeap.enabled=true</code> with an explicit positive <code>spark.memory.offHeap.size</code> (never leave it enabled at the default size of <code>0</code><sup class="footnote-ref"><a href="#fn4" id="fnref4:8">[4:8]</a></sup>) and manually reduce the JVM <code>-Xmx</code> to compensate, since enabling off-heap memory does not shrink the heap for you<sup class="footnote-ref"><a href="#fn4" id="fnref4:9">[4:9]</a></sup>.</p><blockquote><p><strong>PySpark:</strong> if you need PySpark&#39;s own memory bounded rather than folded silently into the overhead, set <code>spark.executor.pyspark.memory</code> explicitly, though its enforcement relies on Python&#39;s <code>resource</code> module and won&#39;t work on Windows and won&#39;t actually limit anything on macOS<sup class="footnote-ref"><a href="#fn4" id="fnref4:10">[4:10]</a></sup>.</p></blockquote><h2 id="bottleneck-cache-utilization" tabindex="-1">Cache utilization <a class="header-anchor" href="#bottleneck-cache-utilization" aria-label="Permalink to &quot;Cache utilization {#bottleneck-cache-utilization}&quot;">​</a></h2><p><span class="tag">CSTOR</span></p><p>Spark&#39;s event log carries no block-access or hit-rate events, so cache utilization is read from periodic per-RDD storage snapshots instead: how much of a persisted RDD is actually resident where it was asked to be.</p><h3 id="how-it-s-detected" tabindex="-1">How it&#39;s detected <a class="header-anchor" href="#how-it-s-detected" aria-label="Permalink to &quot;How it&#39;s detected&quot;">​</a></h3><p>A persisted RDD, one whose storage level requests memory and/or disk with at least one partition actually cached, is evaluated on two independent measures and can register on both at once:</p><table tabindex="0"><thead><tr><th>Signal</th><th>Info</th><th>Warning</th></tr></thead><tbody><tr><td>Cached ratio (cached partitions ÷ total partitions)</td><td>&lt; 90%</td><td>&lt; 50%</td></tr><tr><td>Disk ratio (disk bytes ÷ (memory bytes + disk bytes)), <code>MEMORY_AND_DISK*</code> levels only</td><td>&gt; 15%</td><td>&gt; 40%</td></tr></tbody></table><p>Both ratios come from a point-in-time storage snapshot taken at stage-submission events, not a runtime block-access count, so confirm against the Spark UI&#39;s Storage tab before acting. Confidence scales with the RDD&#39;s partition count: below 10 partitions the ratio is noisy enough to call low confidence, at 50 or more it&#39;s high.</p><h3 id="why-it-matters" tabindex="-1">Why it matters <a class="header-anchor" href="#why-it-matters" aria-label="Permalink to &quot;Why it matters&quot;">​</a></h3><p>A partially cached RDD still pays the recomputation cost for its evicted share on every later read, undercutting the reason it was cached in the first place. Disk spillover under a <code>MEMORY_AND_DISK*</code> level keeps that data around instead of forcing a recompute, but a disk read is still far slower than serving it out of memory.</p><h3 id="how-to-fix-it" tabindex="-1">How to fix it <a class="header-anchor" href="#how-to-fix-it" aria-label="Permalink to &quot;How to fix it&quot;">​</a></h3><ul><li>Increase executor memory, or reduce the size of the dataset being cached, so more of it fits in the storage region without eviction.</li><li>If the RDD is only cached for the occasional narrow scan, weigh whether persisting it at all is worth the partial-cache overhead, versus recomputing the slice you actually need.</li></ul><h2 id="sources" tabindex="-1">Sources <a class="header-anchor" href="#sources" aria-label="Permalink to &quot;Sources&quot;">​</a></h2><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><a href="https://raw.githubusercontent.com/spoddutur/spark-notes/master/task_memory_management_in_spark.md" target="_blank" rel="noreferrer">Task Memory Management in Spark</a> <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a> <a href="#fnref1:2" class="footnote-backref">↩︎</a> <a href="#fnref1:3" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/tuning.html" target="_blank" rel="noreferrer">Tuning Spark</a> <a href="#fnref2" class="footnote-backref">↩︎</a> <a href="#fnref2:1" class="footnote-backref">↩︎</a> <a href="#fnref2:2" class="footnote-backref">↩︎</a> <a href="#fnref2:3" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><a href="https://luminousmen.com/post/dive-into-spark-memory" target="_blank" rel="noreferrer">Dive into Spark Memory</a> <a href="#fnref3" class="footnote-backref">↩︎</a> <a href="#fnref3:1" class="footnote-backref">↩︎</a> <a href="#fnref3:2" class="footnote-backref">↩︎</a> <a href="#fnref3:3" class="footnote-backref">↩︎</a> <a href="#fnref3:4" class="footnote-backref">↩︎</a> <a href="#fnref3:5" class="footnote-backref">↩︎</a> <a href="#fnref3:6" class="footnote-backref">↩︎</a> <a href="#fnref3:7" class="footnote-backref">↩︎</a> <a href="#fnref3:8" class="footnote-backref">↩︎</a> <a href="#fnref3:9" class="footnote-backref">↩︎</a> <a href="#fnref3:10" class="footnote-backref">↩︎</a> <a href="#fnref3:11" class="footnote-backref">↩︎</a> <a href="#fnref3:12" class="footnote-backref">↩︎</a> <a href="#fnref3:13" class="footnote-backref">↩︎</a> <a href="#fnref3:14" class="footnote-backref">↩︎</a> <a href="#fnref3:15" class="footnote-backref">↩︎</a> <a href="#fnref3:16" class="footnote-backref">↩︎</a> <a href="#fnref3:17" class="footnote-backref">↩︎</a> <a href="#fnref3:18" class="footnote-backref">↩︎</a> <a href="#fnref3:19" class="footnote-backref">↩︎</a> <a href="#fnref3:20" class="footnote-backref">↩︎</a> <a href="#fnref3:21" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration — Spark</a> <a href="#fnref4" class="footnote-backref">↩︎</a> <a href="#fnref4:1" class="footnote-backref">↩︎</a> <a href="#fnref4:2" class="footnote-backref">↩︎</a> <a href="#fnref4:3" class="footnote-backref">↩︎</a> <a href="#fnref4:4" class="footnote-backref">↩︎</a> <a href="#fnref4:5" class="footnote-backref">↩︎</a> <a href="#fnref4:6" class="footnote-backref">↩︎</a> <a href="#fnref4:7" class="footnote-backref">↩︎</a> <a href="#fnref4:8" class="footnote-backref">↩︎</a> <a href="#fnref4:9" class="footnote-backref">↩︎</a> <a href="#fnref4:10" class="footnote-backref">↩︎</a></p></li><li id="fn5" class="footnote-item"><p><a href="https://www.databricks.com/blog/2015/04/28/project-tungsten-bringing-spark-closer-to-bare-metal.html" target="_blank" rel="noreferrer">Project Tungsten: Bringing Spark Closer to Bare Metal</a> <a href="#fnref5" class="footnote-backref">↩︎</a> <a href="#fnref5:1" class="footnote-backref">↩︎</a></p></li><li id="fn6" class="footnote-item"><p><em>Learning Spark, 2nd Edition</em>, Damji, Wenig, Das, Lee, ch. 6–7 <a href="#fnref6" class="footnote-backref">↩︎</a> <a href="#fnref6:1" class="footnote-backref">↩︎</a> <a href="#fnref6:2" class="footnote-backref">↩︎</a></p></li><li id="fn7" class="footnote-item"><p><a href="https://luminousmen.com/post/explaining-the-mechanics-of-spark-caching" target="_blank" rel="noreferrer">Explaining the Mechanics of Spark Caching</a> <a href="#fnref7" class="footnote-backref">↩︎</a> <a href="#fnref7:1" class="footnote-backref">↩︎</a> <a href="#fnref7:2" class="footnote-backref">↩︎</a> <a href="#fnref7:3" class="footnote-backref">↩︎</a> <a href="#fnref7:4" class="footnote-backref">↩︎</a></p></li><li id="fn8" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/rdd-programming-guide.html" target="_blank" rel="noreferrer">RDD Programming Guide</a> <a href="#fnref8" class="footnote-backref">↩︎</a></p></li><li id="fn9" class="footnote-item"><p><em>High Performance Spark, 2nd Edition</em>, Karau, Polak &amp; Warren, ch. 7 <a href="#fnref9" class="footnote-backref">↩︎</a> <a href="#fnref9:1" class="footnote-backref">↩︎</a> <a href="#fnref9:2" class="footnote-backref">↩︎</a> <a href="#fnref9:3" class="footnote-backref">↩︎</a></p></li><li id="fn10" class="footnote-item"><p><a href="https://raw.githubusercontent.com/apache/spark/v3.5.0/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala" target="_blank" rel="noreferrer">SQLConf.scala</a> <a href="#fnref10" class="footnote-backref">↩︎</a> <a href="#fnref10:1" class="footnote-backref">↩︎</a></p></li><li id="fn11" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/monitoring.html#spark-history-server" target="_blank" rel="noreferrer">Monitoring and Instrumentation</a> <a href="#fnref11" class="footnote-backref">↩︎</a> <a href="#fnref11:1" class="footnote-backref">↩︎</a></p></li></ol></section></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/tuning-reference/spark-architecture.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>Spark Execution Model</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="/docs/tuning-reference/partitioning.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>Partitioning</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
26
+ <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"9ToZXcTk\",\"contributor-guide_architecture_detector-contract.md\":\"B1vUqtry\",\"contributor-guide_architecture_drill-down.md\":\"Bawx6NgP\",\"contributor-guide_architecture_impact-estimation.md\":\"DHVq1UPt\",\"contributor-guide_architecture_index.md\":\"3TO9ic6w\",\"contributor-guide_architecture_overview.md\":\"DkXdJ9jE\",\"contributor-guide_architecture_state-and-history.md\":\"C-xxn0q7\",\"contributor-guide_architecture_widget-rendering.md\":\"DYwFeyjV\",\"contributor-guide_architecture_worker-protocol.md\":\"C0SovZX8\",\"contributor-guide_contributing.md\":\"CvRsdr6J\",\"contributor-guide_development-setup.md\":\"DvMV1xhQ\",\"contributor-guide_testing.md\":\"BohuQWvl\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"Df1YMIHu\",\"tuning-reference_aqe.md\":\"BIsCtLzm\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"CEstB3Ia\",\"tuning-reference_bottleneck-cold-start.md\":\"CEuy-72y\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"CIohQDfn\",\"tuning-reference_bottleneck-failures.md\":\"CAsFf3RR\",\"tuning-reference_bottleneck-gc.md\":\"B4T3kJYV\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"BaJl__1W\",\"tuning-reference_bottleneck-memory-utilization.md\":\"DbP-SJZc\",\"tuning-reference_bottleneck-retry-waste.md\":\"D5JMjVOt\",\"tuning-reference_bottleneck-shuffle.md\":\"DDMQVOW6\",\"tuning-reference_bottleneck-skew.md\":\"B0MzhWcX\",\"tuning-reference_bottleneck-slow-host.md\":\"BlIo6UDW\",\"tuning-reference_bottleneck-small-files.md\":\"DHGMTmPs\",\"tuning-reference_bottleneck-spill.md\":\"PNH7mITt\",\"tuning-reference_bottleneck-straggler.md\":\"DIbXdAcQ\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"CB05v3zh\",\"tuning-reference_bottleneck-utilization.md\":\"DqC_iGkw\",\"tuning-reference_caching.md\":\"B7aQ8asB\",\"tuning-reference_cluster-config.md\":\"CD6vzYtD\",\"tuning-reference_config.md\":\"UvveiWG3\",\"tuning-reference_data-formats.md\":\"bjCAWH3N\",\"tuning-reference_index.md\":\"BQ_NooMV\",\"tuning-reference_intro.md\":\"CobD-lGB\",\"tuning-reference_joins.md\":\"BtKs_CuW\",\"tuning-reference_memory-model.md\":\"CWhvgevP\",\"tuning-reference_metrics.md\":\"mLOh7Apj\",\"tuning-reference_partitioning.md\":\"q0zKF_8X\",\"tuning-reference_pyspark.md\":\"DDCfvN9t\",\"tuning-reference_shuffle.md\":\"BZZ7R4Ix\",\"tuning-reference_spark-architecture.md\":\"Dwzm5avO\",\"tuning-reference_table-formats.md\":\"D6wj-2dX\",\"user-guide_alternative-log-retrieval.md\":\"B4tPGIal\",\"user-guide_getting-started.md\":\"BJvwLEIM\",\"user-guide_mcp-tools.md\":\"DGvolGzt\",\"user-guide_run-comparison.md\":\"CQc1aoU8\",\"user-guide_understanding-findings.md\":\"B29f0ucS\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Alternative log retrieval\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
27
27
 
28
28
  </body>
29
29
  </html>
package/public/docs/tuning-reference/metrics.html CHANGED
@@ -6,12 +6,12 @@
6
6
  <title>Metrics Glossary | SparkForensics</title>
7
7
  <meta name="description" content="Docs for using and contributing to SparkForensics">
8
8
  <meta name="generator" content="VitePress v1.6.4">
9
- <link rel="preload stylesheet" href="/docs/assets/style.DXOMCXxn.css" as="style">
9
+ <link rel="preload stylesheet" href="/docs/assets/style.DSixAiZE.css" as="style">
10
10
  <link rel="preload stylesheet" href="/docs/vp-icons.css" as="style">
11
11
 
12
- <script type="module" src="/docs/assets/app.BgzsaTS3.js"></script>
12
+ <script type="module" src="/docs/assets/app.BczRxXPr.js"></script>
13
13
  <link rel="preload" href="/docs/assets/inter-roman-latin.Di8DUHzh.woff2" as="font" type="font/woff2" crossorigin="">
14
- <link rel="modulepreload" href="/docs/assets/chunks/theme.BP14rGBx.js">
14
+ <link rel="modulepreload" href="/docs/assets/chunks/theme.ByUdwrLG.js">
15
15
  <link rel="modulepreload" href="/docs/assets/chunks/framework.DSg0KOwT.js">
16
16
  <link rel="modulepreload" href="/docs/assets/tuning-reference_metrics.md.mLOh7Apj.lean.js">
17
17
  <link rel="icon" type="image/svg+xml" href="/docs/favicon.svg">
@@ -23,7 +23,7 @@
23
23
  </head>
24
24
  <body>
25
25
  <div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Getting Started</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/intro.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Introduction</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Optimization Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/spark-architecture.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Execution Model</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/memory-model.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Management</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/partitioning.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Partitioning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/joins.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Join Optimization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/data-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Data Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/table-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Table Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/caching.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Caching & Persistence</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/pyspark.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>PySpark Specifics</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/aqe.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Adaptive Query Execution</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/cluster-config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cluster Tuning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/anti-patterns.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Anti-Patterns</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Detector Catalog</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-skew.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Skew</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle I/O</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-spill.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory / Disk Spill</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-gc.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>GC Pressure</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-cold-start.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cold Start</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Executor Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-slow-host.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Slow Host</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-failures.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Failures</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-straggler.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Stragglers</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-retry-waste.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Retry Waste</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-tiny-tasks.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Tiny Tasks</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-job-failure-rate.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Job Failure Rate</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-memory-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Duplicate Plan Subtree</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-small-files.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Small Files</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-broadcast-sizing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Broadcast Sizing</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Reference</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/metrics.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Metrics Glossary</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Config Quick-Reference</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_tuning-reference_metrics" data-v-39a288b8><div><h1 id="metrics" tabindex="-1">Metrics Glossary <a class="header-anchor" href="#metrics" aria-label="Permalink to &quot;Metrics Glossary {#metrics}&quot;">​</a></h1><p>Reference for every metric surfaced elsewhere in this guide: what it measures, where Spark records it in the event log, and what a problematic value looks like.</p><h2 id="metric-task-duration" tabindex="-1">Task duration (P50/P95/max) <a class="header-anchor" href="#metric-task-duration" aria-label="Permalink to &quot;Task duration (P50/P95/max) {#metric-task-duration}&quot;">​</a></h2><p>Per-task wall-clock duration, aggregated across a stage&#39;s tasks into percentiles (P50, P95, max) to surface skew and stragglers. It is computed from <code>SparkListenerTaskEnd</code> events rather than read off a single field: each task end carries its own duration, and the percentiles are derived by aggregating those values across the stage.</p><p>A problematic value looks like a small fraction of tasks running far past the rest: a stage is classified as a straggler when more than 5% of tasks run at least 4x the median duration (with at least 10 tasks in the stage), or when speculative tasks were fired.</p><h2 id="metric-shuffle-read-bytes" tabindex="-1">Shuffle read bytes <a class="header-anchor" href="#metric-shuffle-read-bytes" aria-label="Permalink to &quot;Shuffle read bytes {#metric-shuffle-read-bytes}&quot;">​</a></h2><p>The volume of shuffle data a task reads from remote executors, recorded in <code>taskMetrics.shuffleReadMetrics.remoteBytesRead</code>.</p><h2 id="metric-shuffle-write-bytes" tabindex="-1">Shuffle write bytes <a class="header-anchor" href="#metric-shuffle-write-bytes" aria-label="Permalink to &quot;Shuffle write bytes {#metric-shuffle-write-bytes}&quot;">​</a></h2><p>The size of the shuffle output a task writes, recorded in <code>taskMetrics.shuffleWriteMetrics.bytesWritten</code>. Spark&#39;s own metric description calls it simply the &quot;Number of bytes written in shuffle operations,&quot; without stating compression state on its own<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup>.</p><p>The sort-based shuffle writer fills in the mechanics: incoming records are serialized as soon as they reach the shuffle writer and buffered in serialized form while sorting<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup>. When the spill compression codec supports concatenating compressed data, the final merge step concatenates the already-compressed spill partitions directly into the output file, using <code>transferTo</code> rather than decompressing and recompressing<sup class="footnote-ref"><a href="#fn2" id="fnref2:1">[2:1]</a></sup>. So <code>bytesWritten</code> counts compressed, post-serialization bytes, and in this common fast-merge path the final written output is built directly from data that had already been spilled to disk during sorting, rather than written fresh at merge time.</p><h2 id="metric-memory-bytes-spilled" tabindex="-1">Memory bytes spilled <a class="header-anchor" href="#metric-memory-bytes-spilled" aria-label="Permalink to &quot;Memory bytes spilled {#metric-memory-bytes-spilled}&quot;">​</a></h2><p>Bytes a task spilled from in-memory structures, recorded in <code>taskMetrics.memoryBytesSpilled</code>. Any value above zero is a spill warning, with the skew-vs-volume classification (based on what fraction of a stage&#39;s tasks show zero spill) as the actionable signal.</p><h2 id="metric-disk-bytes-spilled" tabindex="-1">Disk bytes spilled <a class="header-anchor" href="#metric-disk-bytes-spilled" aria-label="Permalink to &quot;Disk bytes spilled {#metric-disk-bytes-spilled}&quot;">​</a></h2><p>The on-disk counterpart to memory spill: bytes a task spilled to disk, recorded in <code>taskMetrics.diskBytesSpilled</code>. It feeds the same spill classification as memory bytes spilled.</p><h2 id="metric-jvm-gc-time" tabindex="-1">JVM GC time <a class="header-anchor" href="#metric-jvm-gc-time" aria-label="Permalink to &quot;JVM GC time {#metric-jvm-gc-time}&quot;">​</a></h2><p>Elapsed time the JVM spent in garbage collection while a task executed, recorded in <code>taskMetrics.jvmGCTime</code> and expressed in milliseconds<sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup>. The value is cumulative across the task&#39;s full execution window: the sum of every GC pause that occurred during that task&#39;s run, not just the most recent one<sup class="footnote-ref"><a href="#fn1" id="fnref1:2">[1:2]</a></sup>. This matches how it&#39;s serialized: as a single scalar long, consistent with an accumulator rather than a per-GC-event log entry<sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup>.</p><h2 id="metric-gcpct" tabindex="-1">gcPct <a class="header-anchor" href="#metric-gcpct" aria-label="Permalink to &quot;gcPct {#metric-gcpct}&quot;">​</a></h2><p>A synthetic ratio, <code>jvmGCTime / executorRunTime</code>, not a raw Spark field. It drives GC-bottleneck classification: above 10% is a warning, above 20% is critical.</p><h2 id="metric-executor-run-time" tabindex="-1">Executor run time <a class="header-anchor" href="#metric-executor-run-time" aria-label="Permalink to &quot;Executor run time {#metric-executor-run-time}&quot;">​</a></h2><p>Elapsed time the executor spent running a task, recorded in <code>taskMetrics.executorRunTime</code> and expressed in milliseconds<sup class="footnote-ref"><a href="#fn1" id="fnref1:3">[1:3]</a></sup>. <code>TaskMetrics</code> (and therefore <code>executorRunTime</code>) is serialized as an optional part of the <code>SparkListenerTaskEnd</code> payload, not guaranteed on every task end<sup class="footnote-ref"><a href="#fn3" id="fnref3:1">[3:1]</a></sup>. In practice it is available for failed tasks that got far enough to actually run, such as <code>ExceptionFailure</code> or <code>TaskKilled</code>. The event-log deserialization code even falls back to reading accumulator updates out of the embedded <code>TaskMetrics</code> for old, Spark-1.x-era logs, which only makes sense if the metrics object is normally populated for that failure type<sup class="footnote-ref"><a href="#fn3" id="fnref3:2">[3:2]</a></sup>. It can be absent for reasons like <code>Resubmitted</code>, where the task attempt never completed on that executor<sup class="footnote-ref"><a href="#fn3" id="fnref3:3">[3:3]</a></sup>.</p><h2 id="metric-fetch-wait-time-ratio" tabindex="-1">Fetch wait time ratio <a class="header-anchor" href="#metric-fetch-wait-time-ratio" aria-label="Permalink to &quot;Fetch wait time ratio {#metric-fetch-wait-time-ratio}&quot;">​</a></h2><p>A synthetic ratio, <code>fetchWaitTime / taskDuration</code>, not a raw Spark field: the time a task spent blocked waiting on remote shuffle blocks, relative to its total duration.</p><h2 id="metric-input-bytes" tabindex="-1">Input bytes <a class="header-anchor" href="#metric-input-bytes" aria-label="Permalink to &quot;Input bytes {#metric-input-bytes}&quot;">​</a></h2><p>Bytes a task read as input, recorded in <code>taskMetrics.inputMetrics.bytesRead</code>.</p><h2 id="metric-output-bytes" tabindex="-1">Output bytes <a class="header-anchor" href="#metric-output-bytes" aria-label="Permalink to &quot;Output bytes {#metric-output-bytes}&quot;">​</a></h2><p>Bytes a task wrote as output, recorded in <code>taskMetrics.outputMetrics.bytesWritten</code>.</p><h2 id="metric-io-ratio" tabindex="-1">I/O ratio <a class="header-anchor" href="#metric-io-ratio" aria-label="Permalink to &quot;I/O ratio {#metric-io-ratio}&quot;">​</a></h2><p>A synthetic ratio, <code>outputBytes / inputBytes</code>, not a raw Spark field.</p><h2 id="metric-peak-execution-memory" tabindex="-1">Peak execution memory <a class="header-anchor" href="#metric-peak-execution-memory" aria-label="Permalink to &quot;Peak execution memory {#metric-peak-execution-memory}&quot;">​</a></h2><p>Peak memory recorded in <code>taskMetrics.peakExecutionMemory</code>. This is a task-level accumulator, distinct from the separate executor-level <code>peakMemoryMetrics.OnHeapExecutionMemory</code> and <code>.OffHeapExecutionMemory</code> gauges, which report the on-heap and off-heap execution pools as two separate numbers at the executor level rather than as a single per-task figure<sup class="footnote-ref"><a href="#fn1" id="fnref1:4">[1:4]</a></sup>.</p><h2 id="metric-failed-tasks" tabindex="-1">Failed tasks / failure rate <a class="header-anchor" href="#metric-failed-tasks" aria-label="Permalink to &quot;Failed tasks / failure rate {#metric-failed-tasks}&quot;">​</a></h2><p>Tasks whose <code>SparkListenerTaskEnd</code> reason is not <code>Success</code>. The <code>Reason</code> field holds the formatted class name of whichever <code>TaskEndReason</code> was assigned to that task end<sup class="footnote-ref"><a href="#fn3" id="fnref3:4">[3:4]</a></sup>, and the canonical set of reasons includes <code>FetchFailed</code>, <code>ExceptionFailure</code>, <code>TaskResultLost</code>, <code>TaskKilled</code>, <code>TaskCommitDenied</code>, <code>ExecutorLostFailure</code>, and <code>UnknownReason</code><sup class="footnote-ref"><a href="#fn3" id="fnref3:5">[3:5]</a></sup>. Because <code>TaskMetrics</code> is only an optional part of the task-end payload, it can be absent for some of these reasons: for example <code>Resubmitted</code>, where the task attempt never actually completed on that executor<sup class="footnote-ref"><a href="#fn3" id="fnref3:6">[3:6]</a></sup>.</p><h2 id="metric-speculative-tasks" tabindex="-1">Speculative tasks / straggler count <a class="header-anchor" href="#metric-speculative-tasks" aria-label="Permalink to &quot;Speculative tasks / straggler count {#metric-speculative-tasks}&quot;">​</a></h2><p>Tasks launched as speculative retries of a slow-running task, recorded via <code>SparkListenerTaskStart</code> where <code>speculative = true</code>. Any speculative task firing (or more than 5% of a stage&#39;s tasks running at least 4x the median duration, with at least 10 tasks in the stage) is a straggler signal.</p><h2 id="metric-executor-count" tabindex="-1">Executor count (added/removed/concurrent) <a class="header-anchor" href="#metric-executor-count" aria-label="Permalink to &quot;Executor count (added/removed/concurrent) {#metric-executor-count}&quot;">​</a></h2><p>The number of executors added, removed, or concurrently running, recorded via <code>SparkListenerExecutorAdded</code> / <code>SparkListenerExecutorRemoved</code> events.</p><h2 id="metric-stage-duration" tabindex="-1">Stage wall-clock duration <a class="header-anchor" href="#metric-stage-duration" aria-label="Permalink to &quot;Stage wall-clock duration {#metric-stage-duration}&quot;">​</a></h2><p>A stage&#39;s total elapsed time, recorded from <code>SparkListenerStageCompleted</code>: <code>completionTime − submissionTime</code>.</p><h2 id="metric-first-stage-submitted-at" tabindex="-1">firstStageSubmittedAt <a class="header-anchor" href="#metric-first-stage-submitted-at" aria-label="Permalink to &quot;firstStageSubmittedAt {#metric-first-stage-submitted-at}&quot;">​</a></h2><p>A synthetic field: the timestamp of the first <code>SparkListenerStageSubmitted</code> event in the event log. The gap between this timestamp and the application&#39;s start time drives cold-start classification: more than 30 seconds is a warning.</p><h2 id="sources" tabindex="-1">Sources <a class="header-anchor" href="#sources" aria-label="Permalink to &quot;Sources&quot;">​</a></h2><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/monitoring.html#spark-history-server" target="_blank" rel="noreferrer">Monitoring and Instrumentation</a> <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a> <a href="#fnref1:2" class="footnote-backref">↩︎</a> <a href="#fnref1:3" class="footnote-backref">↩︎</a> <a href="#fnref1:4" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><a href="https://raw.githubusercontent.com/apache/spark/v3.5.0/core/src/main/scala/org/apache/spark/shuffle/sort/SortShuffleManager.scala" target="_blank" rel="noreferrer">SortShuffleManager.scala</a> <a href="#fnref2" class="footnote-backref">↩︎</a> <a href="#fnref2:1" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><a href="https://raw.githubusercontent.com/apache/spark/v3.5.0/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala" target="_blank" rel="noreferrer">JsonProtocol.scala</a> <a href="#fnref3" class="footnote-backref">↩︎</a> <a href="#fnref3:1" class="footnote-backref">↩︎</a> <a href="#fnref3:2" class="footnote-backref">↩︎</a> <a href="#fnref3:3" class="footnote-backref">↩︎</a> <a href="#fnref3:4" class="footnote-backref">↩︎</a> <a href="#fnref3:5" class="footnote-backref">↩︎</a> <a href="#fnref3:6" class="footnote-backref">↩︎</a></p></li></ol></section></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/tuning-reference/bottleneck-broadcast-sizing.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>Broadcast Sizing</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="/docs/tuning-reference/config.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>Spark Config Quick-Reference</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
26
- <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"B-OsL91z\",\"contributor-guide_architecture_detector-contract.md\":\"BOeH4d1J\",\"contributor-guide_architecture_drill-down.md\":\"BtPdlM7r\",\"contributor-guide_architecture_impact-estimation.md\":\"DYCDPgkh\",\"contributor-guide_architecture_index.md\":\"3TO9ic6w\",\"contributor-guide_architecture_overview.md\":\"CehiRmGn\",\"contributor-guide_architecture_state-and-history.md\":\"m3S3UdMk\",\"contributor-guide_architecture_widget-rendering.md\":\"DbqPf2OT\",\"contributor-guide_architecture_worker-protocol.md\":\"B93qJ_tT\",\"contributor-guide_contributing.md\":\"CvRsdr6J\",\"contributor-guide_development-setup.md\":\"DvAN_9mK\",\"contributor-guide_testing.md\":\"6rIKqSyY\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"Df1YMIHu\",\"tuning-reference_aqe.md\":\"BIsCtLzm\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"CEstB3Ia\",\"tuning-reference_bottleneck-cold-start.md\":\"CEuy-72y\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"CIohQDfn\",\"tuning-reference_bottleneck-failures.md\":\"4z5BXGJ2\",\"tuning-reference_bottleneck-gc.md\":\"DSxzZRK7\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"BaJl__1W\",\"tuning-reference_bottleneck-memory-utilization.md\":\"DbP-SJZc\",\"tuning-reference_bottleneck-retry-waste.md\":\"D5JMjVOt\",\"tuning-reference_bottleneck-shuffle.md\":\"CM-nTmIH\",\"tuning-reference_bottleneck-skew.md\":\"BdUwiDhn\",\"tuning-reference_bottleneck-slow-host.md\":\"BlIo6UDW\",\"tuning-reference_bottleneck-small-files.md\":\"B8kloyx8\",\"tuning-reference_bottleneck-spill.md\":\"PNH7mITt\",\"tuning-reference_bottleneck-straggler.md\":\"DY36fHN5\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"QTV7O8kU\",\"tuning-reference_bottleneck-utilization.md\":\"DTiueZC3\",\"tuning-reference_caching.md\":\"B7aQ8asB\",\"tuning-reference_cluster-config.md\":\"ZVmDGsQ3\",\"tuning-reference_config.md\":\"UvveiWG3\",\"tuning-reference_data-formats.md\":\"bjCAWH3N\",\"tuning-reference_index.md\":\"BQ_NooMV\",\"tuning-reference_intro.md\":\"CobD-lGB\",\"tuning-reference_joins.md\":\"BtKs_CuW\",\"tuning-reference_memory-model.md\":\"DhT-n4y3\",\"tuning-reference_metrics.md\":\"mLOh7Apj\",\"tuning-reference_partitioning.md\":\"q0zKF_8X\",\"tuning-reference_pyspark.md\":\"DDCfvN9t\",\"tuning-reference_shuffle.md\":\"BZZ7R4Ix\",\"tuning-reference_spark-architecture.md\":\"Dwzm5avO\",\"tuning-reference_table-formats.md\":\"D6wj-2dX\",\"user-guide_alternative-log-retrieval.md\":\"sU3KGarf\",\"user-guide_getting-started.md\":\"DtEM37MK\",\"user-guide_mcp-tools.md\":\"C8MiIu7F\",\"user-guide_run-comparison.md\":\"S0TWWmLY\",\"user-guide_understanding-findings.md\":\"D0R_Y-R2\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Alternative log retrieval\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
26
+ <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"9ToZXcTk\",\"contributor-guide_architecture_detector-contract.md\":\"B1vUqtry\",\"contributor-guide_architecture_drill-down.md\":\"Bawx6NgP\",\"contributor-guide_architecture_impact-estimation.md\":\"DHVq1UPt\",\"contributor-guide_architecture_index.md\":\"3TO9ic6w\",\"contributor-guide_architecture_overview.md\":\"DkXdJ9jE\",\"contributor-guide_architecture_state-and-history.md\":\"C-xxn0q7\",\"contributor-guide_architecture_widget-rendering.md\":\"DYwFeyjV\",\"contributor-guide_architecture_worker-protocol.md\":\"C0SovZX8\",\"contributor-guide_contributing.md\":\"CvRsdr6J\",\"contributor-guide_development-setup.md\":\"DvMV1xhQ\",\"contributor-guide_testing.md\":\"BohuQWvl\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"Df1YMIHu\",\"tuning-reference_aqe.md\":\"BIsCtLzm\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"CEstB3Ia\",\"tuning-reference_bottleneck-cold-start.md\":\"CEuy-72y\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"CIohQDfn\",\"tuning-reference_bottleneck-failures.md\":\"CAsFf3RR\",\"tuning-reference_bottleneck-gc.md\":\"B4T3kJYV\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"BaJl__1W\",\"tuning-reference_bottleneck-memory-utilization.md\":\"DbP-SJZc\",\"tuning-reference_bottleneck-retry-waste.md\":\"D5JMjVOt\",\"tuning-reference_bottleneck-shuffle.md\":\"DDMQVOW6\",\"tuning-reference_bottleneck-skew.md\":\"B0MzhWcX\",\"tuning-reference_bottleneck-slow-host.md\":\"BlIo6UDW\",\"tuning-reference_bottleneck-small-files.md\":\"DHGMTmPs\",\"tuning-reference_bottleneck-spill.md\":\"PNH7mITt\",\"tuning-reference_bottleneck-straggler.md\":\"DIbXdAcQ\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"CB05v3zh\",\"tuning-reference_bottleneck-utilization.md\":\"DqC_iGkw\",\"tuning-reference_caching.md\":\"B7aQ8asB\",\"tuning-reference_cluster-config.md\":\"CD6vzYtD\",\"tuning-reference_config.md\":\"UvveiWG3\",\"tuning-reference_data-formats.md\":\"bjCAWH3N\",\"tuning-reference_index.md\":\"BQ_NooMV\",\"tuning-reference_intro.md\":\"CobD-lGB\",\"tuning-reference_joins.md\":\"BtKs_CuW\",\"tuning-reference_memory-model.md\":\"CWhvgevP\",\"tuning-reference_metrics.md\":\"mLOh7Apj\",\"tuning-reference_partitioning.md\":\"q0zKF_8X\",\"tuning-reference_pyspark.md\":\"DDCfvN9t\",\"tuning-reference_shuffle.md\":\"BZZ7R4Ix\",\"tuning-reference_spark-architecture.md\":\"Dwzm5avO\",\"tuning-reference_table-formats.md\":\"D6wj-2dX\",\"user-guide_alternative-log-retrieval.md\":\"B4tPGIal\",\"user-guide_getting-started.md\":\"BJvwLEIM\",\"user-guide_mcp-tools.md\":\"DGvolGzt\",\"user-guide_run-comparison.md\":\"CQc1aoU8\",\"user-guide_understanding-findings.md\":\"B29f0ucS\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Alternative log retrieval\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
27
27
 
28
28
  </body>
29
29
  </html>