sparkforensics-server 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (382) hide show
  1. package/index.js +5 -4
  2. package/package.json +1 -1
  3. package/public/assets/AutoscalingChurn-CUSIvvab.js +1 -0
  4. package/public/assets/CacheUtilization-Ce0K6_X2.js +1 -0
  5. package/public/assets/CachingOpportunity-Cn-LmRA4.js +1 -0
  6. package/public/assets/ColdStart-BoiYzEyV.js +1 -0
  7. package/public/assets/ConfigAudit-BoJc6PAw.js +1 -0
  8. package/public/assets/CoreUsageArea-BFfXKu7K.js +1 -0
  9. package/public/assets/Dashboard-Bo_1jGqH.js +2 -0
  10. package/public/assets/DisclosureContext-LYzHroHO.js +1 -0
  11. package/public/assets/DuplicatePlanSubtree-BaAuGAoo.js +1 -0
  12. package/public/assets/ExecutorUtilization-OE6VMFkG.js +1 -0
  13. package/public/assets/GcPressure-AoL_ZNrV.js +1 -0
  14. package/public/assets/IncompleteRun-Bv6IzgsQ.js +1 -0
  15. package/public/assets/JobFailures-DF3U2z1f.js +1 -0
  16. package/public/assets/MemoryUtilization-B65OVZlq.js +1 -0
  17. package/public/assets/OverBroadcast-DXbJREDl.js +1 -0
  18. package/public/assets/PartitionSizing-DZLnjy0d.js +1 -0
  19. package/public/assets/PlanExplorer-DdmPkKO1.js +1 -0
  20. package/public/assets/PlanGraphRoute-C-BGmC6j.js +1 -0
  21. package/public/assets/RetryWaste-BbEPsdMw.js +1 -0
  22. package/public/assets/ShuffleIO-wZB94LS8.js +1 -0
  23. package/public/assets/Skew-SpQE7zeJ.js +1 -0
  24. package/public/assets/SlowHost-DwDBCTvr.js +1 -0
  25. package/public/assets/SmallFiles-yjY_6sT1.js +1 -0
  26. package/public/assets/SpeculationWaste-73PDmj0c.js +1 -0
  27. package/public/assets/Spill-DEw0Kd1i.js +1 -0
  28. package/public/assets/StageFailed-BEfXdEJc.js +1 -0
  29. package/public/assets/StageFindingGroup-Bzm2TY03.js +1 -0
  30. package/public/assets/StageHeader-BSV8ALhT.js +1 -0
  31. package/public/assets/StageShape-CzR4jHOt.js +1 -0
  32. package/public/assets/StageSlowness-vMPsnAaa.js +1 -0
  33. package/public/assets/Straggler-Cf__OGl5.js +1 -0
  34. package/public/assets/TaskFailures-BZ8kLZiO.js +1 -0
  35. package/public/assets/{TinyTask-Be-pvTjJ.js → TinyTask-CIPRxWfT.js} +1 -1
  36. package/public/assets/UnderBroadcast-CBm8ZPoS.js +1 -0
  37. package/public/assets/{finding-anchor-vVODA_VC.js → finding-anchor-DF1cSHO1.js} +1 -1
  38. package/public/assets/impact-sort-7z0ZcBZf.js +1 -0
  39. package/public/assets/index-BcqTLKFp.css +2 -0
  40. package/public/assets/index-CEM9bz_x.js +13 -0
  41. package/public/assets/{parser-worker-jAP0kWvS.js → parser-worker-Q5KCPdKT.js} +9 -9
  42. package/public/assets/react-vendor-BAKt3vny.js +9 -0
  43. package/public/assets/select-Cy5pmbrh.js +1 -0
  44. package/public/assets/usePagedRows-BjPOpUU8.js +1 -0
  45. package/public/assets/useRecentFiles-BrBl-AP1.js +11 -0
  46. package/public/assets/useSortMode-CU0vzThu.js +1 -0
  47. package/public/docs/404.html +2 -2
  48. package/public/docs/assets/{app.C5E-wy6e.js → app.C6w47a9W.js} +1 -1
  49. package/public/docs/assets/chunks/@localSearchIndexroot.DbZy5EtM.js +1 -0
  50. package/public/docs/assets/chunks/{VPLocalSearchBox.DtyygpbL.js → VPLocalSearchBox.BcTNvT55.js} +1 -1
  51. package/public/docs/assets/chunks/{theme.DQS9-w_Z.js → theme.D-mE6j02.js} +2 -2
  52. package/public/docs/assets/{contributor-guide_architecture_board-widgets.md.9ToZXcTk.js → contributor-guide_architecture_board-widgets.md.VTcYTH4d.js} +1 -1
  53. package/public/docs/assets/contributor-guide_architecture_detector-contract.md.CsqZHz_T.js +1 -0
  54. package/public/docs/assets/{contributor-guide_architecture_detector-contract.md.CsCytCho.lean.js → contributor-guide_architecture_detector-contract.md.CsqZHz_T.lean.js} +1 -1
  55. package/public/docs/assets/contributor-guide_architecture_drill-down.md.CFNCE8JB.js +1 -0
  56. package/public/docs/assets/{contributor-guide_architecture_drill-down.md.Bawx6NgP.lean.js → contributor-guide_architecture_drill-down.md.CFNCE8JB.lean.js} +1 -1
  57. package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.HT9IJGIl.js +1 -0
  58. package/public/docs/assets/{contributor-guide_architecture_impact-estimation.md.sf9DU2bA.lean.js → contributor-guide_architecture_impact-estimation.md.HT9IJGIl.lean.js} +1 -1
  59. package/public/docs/assets/contributor-guide_architecture_overview.md.CTEqVeV4.js +1 -0
  60. package/public/docs/assets/contributor-guide_architecture_overview.md.CTEqVeV4.lean.js +1 -0
  61. package/public/docs/assets/contributor-guide_architecture_state-and-history.md.DOTVHjTV.js +1 -0
  62. package/public/docs/assets/{contributor-guide_architecture_state-and-history.md.B8Wc_ZTg.lean.js → contributor-guide_architecture_state-and-history.md.DOTVHjTV.lean.js} +1 -1
  63. package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.BmpARZve.js +1 -0
  64. package/public/docs/assets/{contributor-guide_architecture_widget-rendering.md.DYwFeyjV.lean.js → contributor-guide_architecture_widget-rendering.md.BmpARZve.lean.js} +1 -1
  65. package/public/docs/assets/contributor-guide_architecture_worker-protocol.md.21pTYhFF.js +6 -0
  66. package/public/docs/assets/contributor-guide_architecture_worker-protocol.md.21pTYhFF.lean.js +1 -0
  67. package/public/docs/assets/contributor-guide_development-setup.md.CjgZJwjK.js +15 -0
  68. package/public/docs/assets/contributor-guide_development-setup.md.CjgZJwjK.lean.js +1 -0
  69. package/public/docs/assets/contributor-guide_testing.md.DTwhpoN2.js +1 -0
  70. package/public/docs/assets/{contributor-guide_testing.md.Cnn6gz1C.lean.js → contributor-guide_testing.md.DTwhpoN2.lean.js} +1 -1
  71. package/public/docs/assets/tuning-reference_anti-patterns.md.D08Uaod7.js +1 -0
  72. package/public/docs/assets/tuning-reference_anti-patterns.md.D08Uaod7.lean.js +1 -0
  73. package/public/docs/assets/tuning-reference_aqe.md.C1Ae6A8u.js +1 -0
  74. package/public/docs/assets/tuning-reference_aqe.md.C1Ae6A8u.lean.js +1 -0
  75. package/public/docs/assets/tuning-reference_bottleneck-broadcast-sizing.md.6wjKuJNE.js +1 -0
  76. package/public/docs/assets/tuning-reference_bottleneck-broadcast-sizing.md.6wjKuJNE.lean.js +1 -0
  77. package/public/docs/assets/tuning-reference_bottleneck-cold-start.md.BVQbsuHP.js +7 -0
  78. package/public/docs/assets/tuning-reference_bottleneck-cold-start.md.BVQbsuHP.lean.js +1 -0
  79. package/public/docs/assets/tuning-reference_bottleneck-duplicate-plan-subtree.md.DtaYnaiF.js +1 -0
  80. package/public/docs/assets/tuning-reference_bottleneck-duplicate-plan-subtree.md.DtaYnaiF.lean.js +1 -0
  81. package/public/docs/assets/{tuning-reference_bottleneck-failures.md.CAsFf3RR.js → tuning-reference_bottleneck-failures.md.Z1Tdmdd3.js} +2 -2
  82. package/public/docs/assets/tuning-reference_bottleneck-failures.md.Z1Tdmdd3.lean.js +1 -0
  83. package/public/docs/assets/tuning-reference_bottleneck-gc.md.BCnXLf1n.js +6 -0
  84. package/public/docs/assets/tuning-reference_bottleneck-gc.md.BCnXLf1n.lean.js +1 -0
  85. package/public/docs/assets/tuning-reference_bottleneck-job-failure-rate.md.CcvQj2g9.js +8 -0
  86. package/public/docs/assets/tuning-reference_bottleneck-job-failure-rate.md.CcvQj2g9.lean.js +1 -0
  87. package/public/docs/assets/tuning-reference_bottleneck-memory-utilization.md.B-C0-8h-.js +7 -0
  88. package/public/docs/assets/tuning-reference_bottleneck-memory-utilization.md.B-C0-8h-.lean.js +1 -0
  89. package/public/docs/assets/tuning-reference_bottleneck-retry-waste.md.DCnhWiqj.js +1 -0
  90. package/public/docs/assets/tuning-reference_bottleneck-retry-waste.md.DCnhWiqj.lean.js +1 -0
  91. package/public/docs/assets/{tuning-reference_bottleneck-shuffle.md.DDMQVOW6.js → tuning-reference_bottleneck-shuffle.md.B5OMJCd3.js} +2 -2
  92. package/public/docs/assets/tuning-reference_bottleneck-shuffle.md.B5OMJCd3.lean.js +1 -0
  93. package/public/docs/assets/{tuning-reference_bottleneck-skew.md.B0MzhWcX.js → tuning-reference_bottleneck-skew.md.Cvo6NQm_.js} +1 -1
  94. package/public/docs/assets/tuning-reference_bottleneck-skew.md.Cvo6NQm_.lean.js +1 -0
  95. package/public/docs/assets/{tuning-reference_bottleneck-slow-host.md.BlIo6UDW.js → tuning-reference_bottleneck-slow-host.md.ja5qr3wq.js} +1 -1
  96. package/public/docs/assets/tuning-reference_bottleneck-slow-host.md.ja5qr3wq.lean.js +1 -0
  97. package/public/docs/assets/tuning-reference_bottleneck-small-files.md.Crjtb0i2.js +5 -0
  98. package/public/docs/assets/tuning-reference_bottleneck-small-files.md.Crjtb0i2.lean.js +1 -0
  99. package/public/docs/assets/tuning-reference_bottleneck-spill.md.Bl8KtSRg.js +6 -0
  100. package/public/docs/assets/tuning-reference_bottleneck-spill.md.Bl8KtSRg.lean.js +1 -0
  101. package/public/docs/assets/tuning-reference_bottleneck-straggler.md.B0VN8reT.js +7 -0
  102. package/public/docs/assets/tuning-reference_bottleneck-straggler.md.B0VN8reT.lean.js +1 -0
  103. package/public/docs/assets/tuning-reference_bottleneck-tiny-tasks.md.DQCftGhu.js +8 -0
  104. package/public/docs/assets/tuning-reference_bottleneck-tiny-tasks.md.DQCftGhu.lean.js +1 -0
  105. package/public/docs/assets/{tuning-reference_bottleneck-utilization.md.DqC_iGkw.js → tuning-reference_bottleneck-utilization.md.BXDl30f9.js} +1 -1
  106. package/public/docs/assets/tuning-reference_bottleneck-utilization.md.BXDl30f9.lean.js +1 -0
  107. package/public/docs/assets/tuning-reference_caching.md.CcaiMn_7.js +1 -0
  108. package/public/docs/assets/tuning-reference_caching.md.CcaiMn_7.lean.js +1 -0
  109. package/public/docs/assets/tuning-reference_cluster-config.md.DbWG1yZq.js +1 -0
  110. package/public/docs/assets/tuning-reference_cluster-config.md.DbWG1yZq.lean.js +1 -0
  111. package/public/docs/assets/tuning-reference_config.md.YUqSpwFB.js +1 -0
  112. package/public/docs/assets/tuning-reference_config.md.YUqSpwFB.lean.js +1 -0
  113. package/public/docs/assets/tuning-reference_data-formats.md.C5DX5xL4.js +1 -0
  114. package/public/docs/assets/tuning-reference_data-formats.md.C5DX5xL4.lean.js +1 -0
  115. package/public/docs/assets/tuning-reference_index.md.ZVIpQ6Dk.js +1 -0
  116. package/public/docs/assets/tuning-reference_index.md.ZVIpQ6Dk.lean.js +1 -0
  117. package/public/docs/assets/tuning-reference_intro.md.BSSaDlqx.js +1 -0
  118. package/public/docs/assets/tuning-reference_intro.md.BSSaDlqx.lean.js +1 -0
  119. package/public/docs/assets/tuning-reference_joins.md.BVhNCDnE.js +1 -0
  120. package/public/docs/assets/tuning-reference_joins.md.BVhNCDnE.lean.js +1 -0
  121. package/public/docs/assets/tuning-reference_memory-model.md.CfjFdgbl.js +1 -0
  122. package/public/docs/assets/tuning-reference_memory-model.md.CfjFdgbl.lean.js +1 -0
  123. package/public/docs/assets/tuning-reference_metrics.md.C64VNEL4.js +1 -0
  124. package/public/docs/assets/tuning-reference_metrics.md.C64VNEL4.lean.js +1 -0
  125. package/public/docs/assets/tuning-reference_partitioning.md.CtLK0xsS.js +1 -0
  126. package/public/docs/assets/tuning-reference_partitioning.md.CtLK0xsS.lean.js +1 -0
  127. package/public/docs/assets/tuning-reference_pyspark.md.D-92XgF8.js +6 -0
  128. package/public/docs/assets/tuning-reference_pyspark.md.D-92XgF8.lean.js +1 -0
  129. package/public/docs/assets/tuning-reference_shuffle.md.Iq2v1oHJ.js +1 -0
  130. package/public/docs/assets/tuning-reference_shuffle.md.Iq2v1oHJ.lean.js +1 -0
  131. package/public/docs/assets/tuning-reference_spark-architecture.md._qwnE_Sc.js +1 -0
  132. package/public/docs/assets/tuning-reference_spark-architecture.md._qwnE_Sc.lean.js +1 -0
  133. package/public/docs/assets/tuning-reference_table-formats.md.BSYEK-eN.js +1 -0
  134. package/public/docs/assets/tuning-reference_table-formats.md.BSYEK-eN.lean.js +1 -0
  135. package/public/docs/assets/user-guide_getting-started.md.CEa2ZX-l.js +6 -0
  136. package/public/docs/assets/user-guide_getting-started.md.CEa2ZX-l.lean.js +1 -0
  137. package/public/docs/assets/{user-guide_mcp-tools.md.DrphooJv.js → user-guide_mcp-tools.md.dQwIvdTF.js} +13 -7
  138. package/public/docs/assets/user-guide_mcp-tools.md.dQwIvdTF.lean.js +1 -0
  139. package/public/docs/assets/user-guide_run-comparison.md.ChLI3MK0.js +1 -0
  140. package/public/docs/assets/user-guide_run-comparison.md.ChLI3MK0.lean.js +1 -0
  141. package/public/docs/assets/{user-guide_understanding-findings.md.YBBLOwWi.js → user-guide_understanding-findings.md.D1ccs60C.js} +1 -1
  142. package/public/docs/assets/{user-guide_understanding-findings.md.YBBLOwWi.lean.js → user-guide_understanding-findings.md.D1ccs60C.lean.js} +1 -1
  143. package/public/docs/contributor-guide/architecture/board-widgets.html +5 -5
  144. package/public/docs/contributor-guide/architecture/detector-contract.html +5 -5
  145. package/public/docs/contributor-guide/architecture/drill-down.html +5 -5
  146. package/public/docs/contributor-guide/architecture/impact-estimation.html +5 -5
  147. package/public/docs/contributor-guide/architecture/index.html +3 -3
  148. package/public/docs/contributor-guide/architecture/overview.html +5 -5
  149. package/public/docs/contributor-guide/architecture/state-and-history.html +5 -5
  150. package/public/docs/contributor-guide/architecture/widget-rendering.html +5 -5
  151. package/public/docs/contributor-guide/architecture/worker-protocol.html +6 -6
  152. package/public/docs/contributor-guide/contributing.html +3 -3
  153. package/public/docs/contributor-guide/development-setup.html +7 -7
  154. package/public/docs/contributor-guide/testing.html +5 -5
  155. package/public/docs/hashmap.json +1 -1
  156. package/public/docs/index.html +3 -3
  157. package/public/docs/tuning-reference/anti-patterns.html +6 -6
  158. package/public/docs/tuning-reference/aqe.html +6 -6
  159. package/public/docs/tuning-reference/bottleneck-broadcast-sizing.html +6 -6
  160. package/public/docs/tuning-reference/bottleneck-cold-start.html +6 -6
  161. package/public/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html +6 -6
  162. package/public/docs/tuning-reference/bottleneck-failures.html +6 -6
  163. package/public/docs/tuning-reference/bottleneck-gc.html +6 -6
  164. package/public/docs/tuning-reference/bottleneck-job-failure-rate.html +6 -6
  165. package/public/docs/tuning-reference/bottleneck-memory-utilization.html +6 -6
  166. package/public/docs/tuning-reference/bottleneck-retry-waste.html +6 -6
  167. package/public/docs/tuning-reference/bottleneck-shuffle.html +6 -6
  168. package/public/docs/tuning-reference/bottleneck-skew.html +6 -6
  169. package/public/docs/tuning-reference/bottleneck-slow-host.html +6 -6
  170. package/public/docs/tuning-reference/bottleneck-small-files.html +6 -6
  171. package/public/docs/tuning-reference/bottleneck-spill.html +6 -6
  172. package/public/docs/tuning-reference/bottleneck-straggler.html +6 -6
  173. package/public/docs/tuning-reference/bottleneck-tiny-tasks.html +6 -6
  174. package/public/docs/tuning-reference/bottleneck-utilization.html +6 -6
  175. package/public/docs/tuning-reference/caching.html +6 -6
  176. package/public/docs/tuning-reference/cluster-config.html +6 -6
  177. package/public/docs/tuning-reference/config.html +6 -6
  178. package/public/docs/tuning-reference/data-formats.html +6 -6
  179. package/public/docs/tuning-reference/index.html +6 -6
  180. package/public/docs/tuning-reference/intro.html +6 -6
  181. package/public/docs/tuning-reference/joins.html +6 -6
  182. package/public/docs/tuning-reference/memory-model.html +6 -6
  183. package/public/docs/tuning-reference/metrics.html +6 -6
  184. package/public/docs/tuning-reference/partitioning.html +6 -6
  185. package/public/docs/tuning-reference/pyspark.html +6 -6
  186. package/public/docs/tuning-reference/shuffle.html +6 -6
  187. package/public/docs/tuning-reference/spark-architecture.html +6 -6
  188. package/public/docs/tuning-reference/table-formats.html +6 -6
  189. package/public/docs/user-guide/alternative-log-retrieval.html +3 -3
  190. package/public/docs/user-guide/getting-started.html +9 -6
  191. package/public/docs/user-guide/mcp-tools.html +15 -9
  192. package/public/docs/user-guide/run-comparison.html +5 -5
  193. package/public/docs/user-guide/understanding-findings.html +5 -5
  194. package/public/export-template.html +111 -0
  195. package/public/index.html +4 -3
  196. package/vendor-core/analyzer.js +156 -48
  197. package/vendor-core/check-coverage.js +88 -0
  198. package/vendor-core/cli/budgets.js +31 -18
  199. package/vendor-core/cli/collect-run.js +76 -31
  200. package/vendor-core/cli/threshold-config.js +28 -0
  201. package/vendor-core/comparison-verdict.js +177 -0
  202. package/vendor-core/core-source-hash.txt +1 -0
  203. package/vendor-core/core-usage-locality.js +56 -2
  204. package/vendor-core/detector-docs.js +58 -0
  205. package/vendor-core/detectors.js +918 -458
  206. package/vendor-core/docs-config.js +0 -36
  207. package/vendor-core/docs-content/chapters/nav-index.json +31 -0
  208. package/vendor-core/docs-content/detection/cstor.md +9 -0
  209. package/vendor-core/docs-site-config.js +3 -0
  210. package/vendor-core/event-handlers.js +170 -6
  211. package/vendor-core/event-schemas.js +21 -0
  212. package/vendor-core/evidence-report.js +421 -112
  213. package/vendor-core/export-data.js +79 -6
  214. package/vendor-core/finding-action-label.js +9 -88
  215. package/vendor-core/finding-filter-predicate.js +9 -0
  216. package/vendor-core/finding-generic-recommendation.js +6 -104
  217. package/vendor-core/finding-names.js +21 -45
  218. package/vendor-core/finding-presentation.js +333 -0
  219. package/vendor-core/finding-tag-help.js +110 -0
  220. package/vendor-core/finding-types.js +361 -0
  221. package/vendor-core/findings-of-type.js +11 -0
  222. package/vendor-core/format-utils.js +92 -27
  223. package/vendor-core/html-export.js +51 -0
  224. package/vendor-core/impact-band.js +21 -8
  225. package/vendor-core/impact-estimator.js +8 -521
  226. package/vendor-core/impact-format.js +114 -0
  227. package/vendor-core/impact-model.js +175 -0
  228. package/vendor-core/ingest.js +2 -0
  229. package/vendor-core/intervals.js +13 -0
  230. package/vendor-core/list-runs.js +2 -3
  231. package/vendor-core/load-vendored.js +70 -5
  232. package/vendor-core/mcp-server-factory.js +14 -10
  233. package/vendor-core/mcp-tools.js +105 -45
  234. package/vendor-core/model-assembler.js +12 -0
  235. package/vendor-core/occupancy.js +1 -1
  236. package/vendor-core/parser-worker.js +1 -1
  237. package/vendor-core/plan-graph-model.js +3 -2
  238. package/vendor-core/plan-node-detail.js +1 -1
  239. package/vendor-core/recommendation-rollup.js +63 -3
  240. package/vendor-core/redact.js +45 -27
  241. package/vendor-core/run-comparison.js +32 -7
  242. package/vendor-core/run-interpretation.js +290 -0
  243. package/vendor-core/run-outcome.js +74 -0
  244. package/vendor-core/run-payload.js +17 -0
  245. package/vendor-core/run-shape.js +40 -0
  246. package/vendor-core/run-verdict.js +353 -0
  247. package/vendor-core/scaling-sim.js +4 -5
  248. package/vendor-core/scorecard-estimates.js +62 -0
  249. package/vendor-core/sql-stages.js +11 -0
  250. package/vendor-core/stage-quantiles.js +2 -0
  251. package/vendor-core/threshold-overrides.js +160 -0
  252. package/vendor-core/threshold-summary.js +11 -33
  253. package/vendor-core/types.js +6 -42
  254. package/vendor-core/wall-clock.js +1 -12
  255. package/vendor-core/wasted-core-hours.js +2 -2
  256. package/public/assets/AutoscalingChurn-lxZHWIwq.js +0 -1
  257. package/public/assets/CacheUtilization-DuAJ-Lve.js +0 -1
  258. package/public/assets/CachingOpportunity-DC1t7o2_.js +0 -1
  259. package/public/assets/ColdStart-BOYXeiUZ.js +0 -1
  260. package/public/assets/ConfigAudit-MCPrJomC.js +0 -1
  261. package/public/assets/CoreUsageArea-ChNuqGA0.js +0 -1
  262. package/public/assets/Dashboard-VuyUGvZx.js +0 -1
  263. package/public/assets/DuplicatePlanSubtree-JZut9xwq.js +0 -1
  264. package/public/assets/DurationHistogram-_7C1w0l4.js +0 -1
  265. package/public/assets/EmptyState-D_ulfIIi.js +0 -1
  266. package/public/assets/ExecutorUtilization-B08E76jT.js +0 -1
  267. package/public/assets/GcPressure-BZMi2fTx.js +0 -1
  268. package/public/assets/ImpactEstimate-36WxuPdV.js +0 -1
  269. package/public/assets/IncompleteRun-t8JU7HRt.js +0 -1
  270. package/public/assets/JobFailures-CguB1bec.js +0 -1
  271. package/public/assets/MemoryUtilization-DDk8lz9-.js +0 -1
  272. package/public/assets/OverBroadcast-BHXLt6s7.js +0 -1
  273. package/public/assets/PartitionSizing-COj3uwV-.js +0 -1
  274. package/public/assets/PlanExplorer-CnCdEY_i.js +0 -1
  275. package/public/assets/PlanGraphRoute-BesZOW1n.js +0 -1
  276. package/public/assets/RetryWaste-CcU4ctAn.js +0 -1
  277. package/public/assets/RowPagination-D2aIcfGv.js +0 -1
  278. package/public/assets/ShuffleIO-NPHkWbg1.js +0 -1
  279. package/public/assets/Skew-CzztP-C4.js +0 -1
  280. package/public/assets/SlowHost-Dh2KHypN.js +0 -1
  281. package/public/assets/SmallFiles-CgYSuZQw.js +0 -1
  282. package/public/assets/SpeculationWaste-C8J5z1WS.js +0 -1
  283. package/public/assets/Spill-DwTQPZ_L.js +0 -1
  284. package/public/assets/StageFailed-aFeIEh3p.js +0 -1
  285. package/public/assets/StageFindingGroup-DQmAE3uk.js +0 -1
  286. package/public/assets/StageHeader-D6GBbcZp.js +0 -1
  287. package/public/assets/StagePill-CSKyM8Kd.js +0 -1
  288. package/public/assets/StageShape-J6U4cUcM.js +0 -1
  289. package/public/assets/StageSlowness-C5F2v_ZO.js +0 -1
  290. package/public/assets/Straggler-DcP3dFGA.js +0 -1
  291. package/public/assets/TaskFailures-C-vIEheo.js +0 -1
  292. package/public/assets/UnderBroadcast-DfjYGhoK.js +0 -1
  293. package/public/assets/WidgetLeadSummary-BdUUi7L5.js +0 -1
  294. package/public/assets/downsample-WmNM1qc9.js +0 -1
  295. package/public/assets/impact-sort-DxaUheWV.js +0 -1
  296. package/public/assets/inbox-LtbS_kBS.js +0 -1
  297. package/public/assets/index-27MbqTFr.js +0 -11
  298. package/public/assets/index-CWjjbtyo.css +0 -2
  299. package/public/assets/react-vendor-D2TLlSzI.js +0 -9
  300. package/public/assets/select-CC2Lyrmi.js +0 -1
  301. package/public/assets/toggle-group-lmgO21t9.js +0 -1
  302. package/public/assets/usePagedRows-jNUWgVTD.js +0 -1
  303. package/public/assets/useRecentFiles-qH5Ap4xm.js +0 -12
  304. package/public/assets/useSortMode-CvPY99UV.js +0 -1
  305. package/public/docs/assets/chunks/@localSearchIndexroot.CJBoRv1b.js +0 -1
  306. package/public/docs/assets/contributor-guide_architecture_detector-contract.md.CsCytCho.js +0 -1
  307. package/public/docs/assets/contributor-guide_architecture_drill-down.md.Bawx6NgP.js +0 -1
  308. package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.sf9DU2bA.js +0 -1
  309. package/public/docs/assets/contributor-guide_architecture_overview.md.rhfxoGsI.js +0 -1
  310. package/public/docs/assets/contributor-guide_architecture_overview.md.rhfxoGsI.lean.js +0 -1
  311. package/public/docs/assets/contributor-guide_architecture_state-and-history.md.B8Wc_ZTg.js +0 -1
  312. package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.DYwFeyjV.js +0 -1
  313. package/public/docs/assets/contributor-guide_architecture_worker-protocol.md.CBJkwH24.js +0 -6
  314. package/public/docs/assets/contributor-guide_architecture_worker-protocol.md.CBJkwH24.lean.js +0 -1
  315. package/public/docs/assets/contributor-guide_development-setup.md.s85j2l8b.js +0 -15
  316. package/public/docs/assets/contributor-guide_development-setup.md.s85j2l8b.lean.js +0 -1
  317. package/public/docs/assets/contributor-guide_testing.md.Cnn6gz1C.js +0 -1
  318. package/public/docs/assets/tuning-reference_anti-patterns.md.Df1YMIHu.js +0 -1
  319. package/public/docs/assets/tuning-reference_anti-patterns.md.Df1YMIHu.lean.js +0 -1
  320. package/public/docs/assets/tuning-reference_aqe.md.BIsCtLzm.js +0 -1
  321. package/public/docs/assets/tuning-reference_aqe.md.BIsCtLzm.lean.js +0 -1
  322. package/public/docs/assets/tuning-reference_bottleneck-broadcast-sizing.md.CEstB3Ia.js +0 -1
  323. package/public/docs/assets/tuning-reference_bottleneck-broadcast-sizing.md.CEstB3Ia.lean.js +0 -1
  324. package/public/docs/assets/tuning-reference_bottleneck-cold-start.md.CEuy-72y.js +0 -7
  325. package/public/docs/assets/tuning-reference_bottleneck-cold-start.md.CEuy-72y.lean.js +0 -1
  326. package/public/docs/assets/tuning-reference_bottleneck-duplicate-plan-subtree.md.CIohQDfn.js +0 -1
  327. package/public/docs/assets/tuning-reference_bottleneck-duplicate-plan-subtree.md.CIohQDfn.lean.js +0 -1
  328. package/public/docs/assets/tuning-reference_bottleneck-failures.md.CAsFf3RR.lean.js +0 -1
  329. package/public/docs/assets/tuning-reference_bottleneck-gc.md.B4T3kJYV.js +0 -6
  330. package/public/docs/assets/tuning-reference_bottleneck-gc.md.B4T3kJYV.lean.js +0 -1
  331. package/public/docs/assets/tuning-reference_bottleneck-job-failure-rate.md.BaJl__1W.js +0 -8
  332. package/public/docs/assets/tuning-reference_bottleneck-job-failure-rate.md.BaJl__1W.lean.js +0 -1
  333. package/public/docs/assets/tuning-reference_bottleneck-memory-utilization.md.DbP-SJZc.js +0 -7
  334. package/public/docs/assets/tuning-reference_bottleneck-memory-utilization.md.DbP-SJZc.lean.js +0 -1
  335. package/public/docs/assets/tuning-reference_bottleneck-retry-waste.md.D5JMjVOt.js +0 -1
  336. package/public/docs/assets/tuning-reference_bottleneck-retry-waste.md.D5JMjVOt.lean.js +0 -1
  337. package/public/docs/assets/tuning-reference_bottleneck-shuffle.md.DDMQVOW6.lean.js +0 -1
  338. package/public/docs/assets/tuning-reference_bottleneck-skew.md.B0MzhWcX.lean.js +0 -1
  339. package/public/docs/assets/tuning-reference_bottleneck-slow-host.md.BlIo6UDW.lean.js +0 -1
  340. package/public/docs/assets/tuning-reference_bottleneck-small-files.md.DHGMTmPs.js +0 -5
  341. package/public/docs/assets/tuning-reference_bottleneck-small-files.md.DHGMTmPs.lean.js +0 -1
  342. package/public/docs/assets/tuning-reference_bottleneck-spill.md.PNH7mITt.js +0 -6
  343. package/public/docs/assets/tuning-reference_bottleneck-spill.md.PNH7mITt.lean.js +0 -1
  344. package/public/docs/assets/tuning-reference_bottleneck-straggler.md.DIbXdAcQ.js +0 -7
  345. package/public/docs/assets/tuning-reference_bottleneck-straggler.md.DIbXdAcQ.lean.js +0 -1
  346. package/public/docs/assets/tuning-reference_bottleneck-tiny-tasks.md.CB05v3zh.js +0 -8
  347. package/public/docs/assets/tuning-reference_bottleneck-tiny-tasks.md.CB05v3zh.lean.js +0 -1
  348. package/public/docs/assets/tuning-reference_bottleneck-utilization.md.DqC_iGkw.lean.js +0 -1
  349. package/public/docs/assets/tuning-reference_caching.md.B7aQ8asB.js +0 -1
  350. package/public/docs/assets/tuning-reference_caching.md.B7aQ8asB.lean.js +0 -1
  351. package/public/docs/assets/tuning-reference_cluster-config.md.CD6vzYtD.js +0 -1
  352. package/public/docs/assets/tuning-reference_cluster-config.md.CD6vzYtD.lean.js +0 -1
  353. package/public/docs/assets/tuning-reference_config.md.UvveiWG3.js +0 -1
  354. package/public/docs/assets/tuning-reference_config.md.UvveiWG3.lean.js +0 -1
  355. package/public/docs/assets/tuning-reference_data-formats.md.bjCAWH3N.js +0 -1
  356. package/public/docs/assets/tuning-reference_data-formats.md.bjCAWH3N.lean.js +0 -1
  357. package/public/docs/assets/tuning-reference_index.md.BQ_NooMV.js +0 -1
  358. package/public/docs/assets/tuning-reference_index.md.BQ_NooMV.lean.js +0 -1
  359. package/public/docs/assets/tuning-reference_intro.md.CobD-lGB.js +0 -1
  360. package/public/docs/assets/tuning-reference_intro.md.CobD-lGB.lean.js +0 -1
  361. package/public/docs/assets/tuning-reference_joins.md.BtKs_CuW.js +0 -1
  362. package/public/docs/assets/tuning-reference_joins.md.BtKs_CuW.lean.js +0 -1
  363. package/public/docs/assets/tuning-reference_memory-model.md.CWhvgevP.js +0 -1
  364. package/public/docs/assets/tuning-reference_memory-model.md.CWhvgevP.lean.js +0 -1
  365. package/public/docs/assets/tuning-reference_metrics.md.mLOh7Apj.js +0 -1
  366. package/public/docs/assets/tuning-reference_metrics.md.mLOh7Apj.lean.js +0 -1
  367. package/public/docs/assets/tuning-reference_partitioning.md.q0zKF_8X.js +0 -1
  368. package/public/docs/assets/tuning-reference_partitioning.md.q0zKF_8X.lean.js +0 -1
  369. package/public/docs/assets/tuning-reference_pyspark.md.DDCfvN9t.js +0 -6
  370. package/public/docs/assets/tuning-reference_pyspark.md.DDCfvN9t.lean.js +0 -1
  371. package/public/docs/assets/tuning-reference_shuffle.md.BZZ7R4Ix.js +0 -1
  372. package/public/docs/assets/tuning-reference_shuffle.md.BZZ7R4Ix.lean.js +0 -1
  373. package/public/docs/assets/tuning-reference_spark-architecture.md.Dwzm5avO.js +0 -1
  374. package/public/docs/assets/tuning-reference_spark-architecture.md.Dwzm5avO.lean.js +0 -1
  375. package/public/docs/assets/tuning-reference_table-formats.md.D6wj-2dX.js +0 -1
  376. package/public/docs/assets/tuning-reference_table-formats.md.D6wj-2dX.lean.js +0 -1
  377. package/public/docs/assets/user-guide_getting-started.md.Cpe0U-eO.js +0 -3
  378. package/public/docs/assets/user-guide_getting-started.md.Cpe0U-eO.lean.js +0 -1
  379. package/public/docs/assets/user-guide_mcp-tools.md.DrphooJv.lean.js +0 -1
  380. package/public/docs/assets/user-guide_run-comparison.md.CQc1aoU8.js +0 -1
  381. package/public/docs/assets/user-guide_run-comparison.md.CQc1aoU8.lean.js +0 -1
  382. /package/public/docs/assets/{contributor-guide_architecture_board-widgets.md.9ToZXcTk.lean.js → contributor-guide_architecture_board-widgets.md.VTcYTH4d.lean.js} +0 -0
package/public/docs/tuning-reference/table-formats.html CHANGED
@@ -3,17 +3,17 @@
3
3
  <head>
4
4
  <meta charset="utf-8">
5
5
  <meta name="viewport" content="width=device-width,initial-scale=1">
6
- <title>Table Formats | SparkForensics</title>
7
- <meta name="description" content="Docs for using and contributing to SparkForensics">
6
+ <title>Table Formats | Spark Tuning Reference</title>
7
+ <meta name="description" content="Lakehouse table-format tuning across Delta Lake, Iceberg, and Hudi: file sizing, compaction/OPTIMIZE, clustering (Z-order/liquid/sort), metadata and manifest overhead, snapshot/version expiry, and deletion vectors vs. merge-on-read/copy-on-write.">
8
8
  <meta name="generator" content="VitePress v1.6.4">
9
9
  <link rel="preload stylesheet" href="/docs/assets/style.DSixAiZE.css" as="style">
10
10
  <link rel="preload stylesheet" href="/docs/vp-icons.css" as="style">
11
11
 
12
- <script type="module" src="/docs/assets/app.C5E-wy6e.js"></script>
12
+ <script type="module" src="/docs/assets/app.C6w47a9W.js"></script>
13
13
  <link rel="preload" href="/docs/assets/inter-roman-latin.Di8DUHzh.woff2" as="font" type="font/woff2" crossorigin="">
14
- <link rel="modulepreload" href="/docs/assets/chunks/theme.DQS9-w_Z.js">
14
+ <link rel="modulepreload" href="/docs/assets/chunks/theme.D-mE6j02.js">
15
15
  <link rel="modulepreload" href="/docs/assets/chunks/framework.DSg0KOwT.js">
16
- <link rel="modulepreload" href="/docs/assets/tuning-reference_table-formats.md.D6wj-2dX.lean.js">
16
+ <link rel="modulepreload" href="/docs/assets/tuning-reference_table-formats.md.BSYEK-eN.lean.js">
17
17
  <link rel="icon" type="image/svg+xml" href="/docs/favicon.svg">
18
18
  <link rel="preconnect" href="https://fonts.googleapis.com">
19
19
  <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin="">
@@ -23,7 +23,7 @@
23
23
  </head>
24
24
  <body>
25
25
  <div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Getting Started</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/intro.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Introduction</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Optimization Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/spark-architecture.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Execution Model</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/memory-model.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Management</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/partitioning.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Partitioning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/joins.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Join Optimization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/data-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Data Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/table-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Table Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/caching.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Caching & Persistence</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/pyspark.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>PySpark Specifics</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/aqe.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Adaptive Query Execution</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/cluster-config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cluster Tuning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/anti-patterns.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Anti-Patterns</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Detector Catalog</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-skew.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Skew</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle I/O</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-spill.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory / Disk Spill</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-gc.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>GC Pressure</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-cold-start.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cold Start</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Executor Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-slow-host.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Slow Host</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-failures.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Failures</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-straggler.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Stragglers</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-retry-waste.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Retry Waste</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-tiny-tasks.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Tiny Tasks</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-job-failure-rate.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Job Failure Rate</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-memory-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Duplicate Plan Subtree</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-small-files.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Small Files</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-broadcast-sizing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Broadcast Sizing</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Reference</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/metrics.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Metrics Glossary</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Config Quick-Reference</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_tuning-reference_table-formats" data-v-39a288b8><div><h1 id="table-formats" tabindex="-1">Table Formats <a class="header-anchor" href="#table-formats" aria-label="Permalink to &quot;Table Formats {#table-formats}&quot;">​</a></h1><h2 id="the-knobs-these-formats-share" tabindex="-1">The knobs these formats share <a class="header-anchor" href="#the-knobs-these-formats-share" aria-label="Permalink to &quot;The knobs these formats share&quot;">​</a></h2><p><a href="./data-formats.html">Parquet and ORC</a> stop at the file. A lakehouse <em>table format</em> wraps a directory of Parquet files with a transactional metadata layer that tracks which files belong to the table right now, so engines get ACID commits, time travel, schema evolution, and row-level updates over plain columnar files. The three in wide use are Delta Lake, Apache Iceberg, and Apache Hudi. They solve the same problems and differ mostly in mechanism, and the tuning knobs that matter fall into a handful of dimensions.</p><p><strong>File sizing.</strong> All three fight the small-files problem, but at different points in the write. Delta Lake does not size files during the write by default; its <code>OPTIMIZE</code> command bin-packs already-written small files into larger ones, available since Delta Lake 1.2.0<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup>. Iceberg rewrites files after the fact through its <code>rewrite_data_files</code> procedure, whose default <code>binpack</code> strategy coalesces small files<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup>. Hudi treats correct sizing at write time as &quot;a critical design decision&quot;: auto-sizing targets a 120 MB Parquet base file (<code>hoodie.parquet.max.file.size</code>) and pads any existing file at or below the 100 MB small-file limit (<code>hoodie.parquet.small.file.limit</code>) with new records rather than opening a fresh file<sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup>.</p><p><strong>Compaction.</strong> Delta&#39;s <code>OPTIMIZE</code> bin-packing is the compaction path, run manually or, since Delta Lake 3.1.0, as auto compaction that runs synchronously right after a write succeeds<sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup>. Iceberg uses <code>rewrite_data_files</code> as a stored procedure invoked from Spark SQL<sup class="footnote-ref"><a href="#fn2" id="fnref2:1">[2:1]</a></sup>. Hudi splits the job in two: <em>compaction</em> applies only to Merge-on-Read tables and merges the row-based delta logs back into base files (async by default, or inline via <code>hoodie.compact.inline = true</code>), while <em>clustering</em> is a separate data-layout service that stitches small files together<sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup><sup class="footnote-ref"><a href="#fn5" id="fnref5">[5]</a></sup>.</p><p><strong>Clustering.</strong> To co-locate rows that are queried together, Delta Lake offers Z-Ordering via <code>OPTIMIZE table ZORDER BY (cols)</code>; its effectiveness drops with each added column and it is not idempotent, since each run reclusters all files in the partition<sup class="footnote-ref"><a href="#fn1" id="fnref1:2">[1:2]</a></sup>. Delta also documents liquid clustering as a separate feature<sup class="footnote-ref"><a href="#fn1" id="fnref1:3">[1:3]</a></sup>. Iceberg clusters through <code>rewrite_data_files</code> with <code>strategy =&gt; &#39;sort&#39;</code> and a <code>sort_order</code> argument, including a <code>zorder(c1,c2)</code> form<sup class="footnote-ref"><a href="#fn2" id="fnref2:2">[2:2]</a></sup>. Hudi clustering rewrites file groups sorting by <code>hoodie.clustering.plan.strategy.sort.columns</code><sup class="footnote-ref"><a href="#fn5" id="fnref5:1">[5:1]</a></sup>.</p><p><strong>Metadata and manifest overhead.</strong> Delta records each change as a JSON commit in the transaction log and periodically compacts those commits into a Parquet checkpoint so readers reconstruct state without replaying every commit; checkpoints can be split multi-part (default 50,000 actions per part), and since Delta 3.0 log-compaction files aggregate a commit range to cut checkpoint frequency<sup class="footnote-ref"><a href="#fn1" id="fnref1:4">[1:4]</a></sup>. Iceberg writes a new metadata JSON file per change, tracks the manifests for each snapshot in a manifest list written fresh on every commit, and splits large tables across multiple manifests so query planning parallelizes<sup class="footnote-ref"><a href="#fn6" id="fnref6">[6]</a></sup>. Hudi records every write and table-service action as an instant on a timeline, with clustering and compaction writing their plans there before executing<sup class="footnote-ref"><a href="#fn5" id="fnref5:2">[5:2]</a></sup><sup class="footnote-ref"><a href="#fn4" id="fnref4:1">[4:1]</a></sup>.</p><p><strong>Snapshot and version expiry.</strong> Delta reclaims storage with <code>VACUUM</code>, which deletes data files past a retention threshold (default 7 days) but never log files; log files are pruned automatically after checkpoints, with a 30-day default set by <code>delta.logRetentionDuration</code><sup class="footnote-ref"><a href="#fn7" id="fnref7">[7]</a></sup>. Iceberg&#39;s <code>expire_snapshots</code> removes old snapshots and the files only they referenced (<code>older_than</code> default 5 days, <code>retain_last</code> default 1), and <code>remove_orphan_files</code> sweeps unreferenced files (<code>older_than</code> default 3 days)<sup class="footnote-ref"><a href="#fn2" id="fnref2:3">[2:3]</a></sup>. Hudi&#39;s cleaner runs automatically after each commit; the default <code>KEEP_LATEST_COMMITS</code> policy retains <code>hoodie.clean.commits.retained</code> commits (default 10)<sup class="footnote-ref"><a href="#fn8" id="fnref8">[8]</a></sup>.</p><p><strong>Row-level deletes: merge-on-read vs copy-on-write.</strong> By default, deleting one row in a Delta table rewrites the whole Parquet file that holds it. Deletion vectors avoid that by marking rows removed in a side file and applying the marks at read time; support landed incrementally (DELETE in 2.4.0, UPDATE in 3.0.0, on by default since 3.1.0)<sup class="footnote-ref"><a href="#fn9" id="fnref9">[9]</a></sup>. Iceberg took a spec-versioned route: v2 adds position and equality delete files, and v3 replaces position deletes with per-file deletion-vector bitmaps stored in the Puffin format<sup class="footnote-ref"><a href="#fn6" id="fnref6:1">[6:1]</a></sup>. Hudi frames the same trade-off as two table types: Copy-on-Write rewrites a base file on every change (fast reads, slower writes), while Merge-on-Read appends changes to log files merged at query time and compacted later (fast writes, some read cost)<sup class="footnote-ref"><a href="#fn10" id="fnref10">[10]</a></sup>.</p><h2 id="where-these-problems-show-up" tabindex="-1">Where these problems show up <a class="header-anchor" href="#where-these-problems-show-up" aria-label="Permalink to &quot;Where these problems show up&quot;">​</a></h2><p>Each of those knobs fails in its own recognizable way. The symptoms are the same ones the <a href="./data-formats.html">Data Formats</a> and <a href="./bottleneck-small-files.html">Small Files</a> pages describe, read through the table&#39;s own metadata. A table accumulating many sub-target files is a sizing problem: Hudi&#39;s own guidance ties small files to more tasks, more per-file open/close cost, and cloud object-store request-rate limits that trip because at least one request is issued per file regardless of size<sup class="footnote-ref"><a href="#fn3" id="fnref3:1">[3:1]</a></sup>.</p><p>Growing metadata is the second signal. Iceberg snapshots and metadata JSON files accumulate until expiry runs, and Iceberg recommends <code>expire_snapshots</code> specifically to keep metadata size small on top of freeing data files<sup class="footnote-ref"><a href="#fn6" id="fnref6:2">[6:2]</a></sup>. Delta&#39;s commit log grows until checkpoints and log compaction absorb it<sup class="footnote-ref"><a href="#fn1" id="fnref1:5">[1:5]</a></sup>. Hudi&#39;s timeline lengthens with every commit, clean, cluster, and compaction<sup class="footnote-ref"><a href="#fn5" id="fnref5:3">[5:3]</a></sup>. Delta&#39;s <code>deltaTable.history()</code> surfaces the per-commit version, timestamp, and operation for inspecting that growth<sup class="footnote-ref"><a href="#fn1" id="fnref1:6">[1:6]</a></sup>.</p><p>Read amplification is the third. On a Merge-on-Read Hudi table, uncompacted delta logs are merged at query time, so a table that has not compacted recently reads more slowly<sup class="footnote-ref"><a href="#fn4" id="fnref4:2">[4:2]</a></sup>. The Iceberg equivalent is a data file with many stacked position/equality delete files that a scan must apply<sup class="footnote-ref"><a href="#fn6" id="fnref6:3">[6:3]</a></sup>.</p><h2 id="what-each-dimension-costs" tabindex="-1">What each dimension costs <a class="header-anchor" href="#what-each-dimension-costs" aria-label="Permalink to &quot;What each dimension costs&quot;">​</a></h2><p>Those symptoms aren&#39;t cosmetic. Small files cost more in metadata and scheduling than their bytes suggest: a query scans many files for the same data, each file adds fixed overhead, and on object storage the per-file request pattern raises the odds of hitting per-prefix rate limits<sup class="footnote-ref"><a href="#fn3" id="fnref3:2">[3:2]</a></sup>. That is exactly the tension the compaction and clustering services exist to resolve, since ingestion favors many small files for low latency while queries favor fewer large ones<sup class="footnote-ref"><a href="#fn3" id="fnref3:3">[3:3]</a></sup>.</p><p>Metadata overhead is a planning tax. More data files mean more manifest entries, and small files inflate that metadata disproportionately and slow query planning, which is why Iceberg exposes <code>rewriteManifests</code> and compaction to cut it<sup class="footnote-ref"><a href="#fn6" id="fnref6:4">[6:4]</a></sup>. Unbounded snapshot and metadata retention keeps that footprint growing until expiry is run<sup class="footnote-ref"><a href="#fn6" id="fnref6:5">[6:5]</a></sup>.</p><p>Clustering decides how much data a query skips. Delta&#39;s Z-Ordering co-locates related values so data-skipping can prune more files, dramatically reducing bytes read<sup class="footnote-ref"><a href="#fn1" id="fnref1:7">[1:7]</a></sup>. The retention settings carry a correctness edge too: once Delta <code>VACUUM</code> runs, time travel to a version older than the retention window is gone<sup class="footnote-ref"><a href="#fn7" id="fnref7:1">[7:1]</a></sup>, and Iceberg warns that running <code>remove_orphan_files</code> with too short an interval can delete in-flight files and corrupt the table<sup class="footnote-ref"><a href="#fn6" id="fnref6:6">[6:6]</a></sup>.</p><h2 id="tuning-each-dimension" tabindex="-1">Tuning each dimension <a class="header-anchor" href="#tuning-each-dimension" aria-label="Permalink to &quot;Tuning each dimension&quot;">​</a></h2><p>Each dimension has a matching maintenance habit that keeps its cost down. Compact on a schedule. For Delta, run <code>OPTIMIZE</code> (optionally scoped with a <code>WHERE</code> partition predicate) or enable auto compaction on 3.1.0+<sup class="footnote-ref"><a href="#fn1" id="fnref1:8">[1:8]</a></sup>. For Iceberg, call <code>rewrite_data_files</code><sup class="footnote-ref"><a href="#fn2" id="fnref2:4">[2:4]</a></sup>. For Merge-on-Read Hudi, let async compaction run or force it inline with <code>hoodie.compact.inline = true</code>, and use clustering to consolidate small files independently of it<sup class="footnote-ref"><a href="#fn4" id="fnref4:3">[4:3]</a></sup><sup class="footnote-ref"><a href="#fn5" id="fnref5:4">[5:4]</a></sup>.</p><p>Cluster the columns you filter on. Use Delta <code>OPTIMIZE ... ZORDER BY (cols)</code> on a few high-cardinality predicate columns<sup class="footnote-ref"><a href="#fn1" id="fnref1:9">[1:9]</a></sup>, Iceberg <code>rewrite_data_files(strategy =&gt; &#39;sort&#39;, sort_order =&gt; &#39;...&#39;)</code> or its <code>zorder(...)</code> form<sup class="footnote-ref"><a href="#fn2" id="fnref2:5">[2:5]</a></sup>, or Hudi clustering with <code>hoodie.clustering.plan.strategy.sort.columns</code><sup class="footnote-ref"><a href="#fn5" id="fnref5:5">[5:5]</a></sup>.</p><p>Expire aggressively but safely. Run Delta <code>VACUUM</code> (keeping the 7-day floor unless you have a reason to override it, since the safety check exists to protect concurrent readers)<sup class="footnote-ref"><a href="#fn7" id="fnref7:2">[7:2]</a></sup>, Iceberg <code>expire_snapshots</code> plus periodic <code>remove_orphan_files</code> with an interval longer than your longest in-flight write<sup class="footnote-ref"><a href="#fn6" id="fnref6:7">[6:7]</a></sup>, and tune Hudi&#39;s cleaner via <code>hoodie.clean.commits.retained</code><sup class="footnote-ref"><a href="#fn8" id="fnref8:1">[8:1]</a></sup>.</p><p>Match the delete strategy to the workload. Enable Delta deletion vectors (<code>ALTER TABLE ... SET TBLPROPERTIES(&#39;delta.enableDeletionVectors&#39; = true)</code>) so DML marks rows instead of rewriting files, remembering the marks are applied physically only on <code>OPTIMIZE</code> or <code>REORG TABLE ... APPLY (PURGE)</code><sup class="footnote-ref"><a href="#fn9" id="fnref9:1">[9:1]</a></sup>. On Iceberg, prefer merge-on-read deletes for update-heavy tables and compact the delete files<sup class="footnote-ref"><a href="#fn6" id="fnref6:8">[6:8]</a></sup>. On Hudi, pick Copy-on-Write for read-heavy tables and Merge-on-Read for write-heavy or near-real-time ingestion<sup class="footnote-ref"><a href="#fn10" id="fnref10:1">[10:1]</a></sup>.</p><h2 id="sources" tabindex="-1">Sources <a class="header-anchor" href="#sources" aria-label="Permalink to &quot;Sources&quot;">​</a></h2><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><a href="https://docs.delta.io/latest/optimizations-oss.html" target="_blank" rel="noreferrer">Delta Lake: Optimizations (OSS)</a> <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a> <a href="#fnref1:2" class="footnote-backref">↩︎</a> <a href="#fnref1:3" class="footnote-backref">↩︎</a> <a href="#fnref1:4" class="footnote-backref">↩︎</a> <a href="#fnref1:5" class="footnote-backref">↩︎</a> <a href="#fnref1:6" class="footnote-backref">↩︎</a> <a href="#fnref1:7" class="footnote-backref">↩︎</a> <a href="#fnref1:8" class="footnote-backref">↩︎</a> <a href="#fnref1:9" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><a href="https://iceberg.apache.org/docs/latest/spark-procedures/" target="_blank" rel="noreferrer">Apache Iceberg: Spark Procedures</a> <a href="#fnref2" class="footnote-backref">↩︎</a> <a href="#fnref2:1" class="footnote-backref">↩︎</a> <a href="#fnref2:2" class="footnote-backref">↩︎</a> <a href="#fnref2:3" class="footnote-backref">↩︎</a> <a href="#fnref2:4" class="footnote-backref">↩︎</a> <a href="#fnref2:5" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><a href="https://hudi.apache.org/docs/file_sizing/" target="_blank" rel="noreferrer">Apache Hudi: File Sizing</a> <a href="#fnref3" class="footnote-backref">↩︎</a> <a href="#fnref3:1" class="footnote-backref">↩︎</a> <a href="#fnref3:2" class="footnote-backref">↩︎</a> <a href="#fnref3:3" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><a href="https://hudi.apache.org/docs/compaction/" target="_blank" rel="noreferrer">Apache Hudi: Compaction</a> <a href="#fnref4" class="footnote-backref">↩︎</a> <a href="#fnref4:1" class="footnote-backref">↩︎</a> <a href="#fnref4:2" class="footnote-backref">↩︎</a> <a href="#fnref4:3" class="footnote-backref">↩︎</a></p></li><li id="fn5" class="footnote-item"><p><a href="https://hudi.apache.org/docs/clustering/" target="_blank" rel="noreferrer">Apache Hudi: Clustering</a> <a href="#fnref5" class="footnote-backref">↩︎</a> <a href="#fnref5:1" class="footnote-backref">↩︎</a> <a href="#fnref5:2" class="footnote-backref">↩︎</a> <a href="#fnref5:3" class="footnote-backref">↩︎</a> <a href="#fnref5:4" class="footnote-backref">↩︎</a> <a href="#fnref5:5" class="footnote-backref">↩︎</a></p></li><li id="fn6" class="footnote-item"><p><a href="https://iceberg.apache.org/docs/latest/maintenance/" target="_blank" rel="noreferrer">Apache Iceberg: Maintenance</a> and <a href="https://iceberg.apache.org/spec/" target="_blank" rel="noreferrer">Table Spec</a> <a href="#fnref6" class="footnote-backref">↩︎</a> <a href="#fnref6:1" class="footnote-backref">↩︎</a> <a href="#fnref6:2" class="footnote-backref">↩︎</a> <a href="#fnref6:3" class="footnote-backref">↩︎</a> <a href="#fnref6:4" class="footnote-backref">↩︎</a> <a href="#fnref6:5" class="footnote-backref">↩︎</a> <a href="#fnref6:6" class="footnote-backref">↩︎</a> <a href="#fnref6:7" class="footnote-backref">↩︎</a> <a href="#fnref6:8" class="footnote-backref">↩︎</a></p></li><li id="fn7" class="footnote-item"><p><a href="https://docs.delta.io/latest/delta-utility.html" target="_blank" rel="noreferrer">Delta Lake: Table Utility Commands</a> <a href="#fnref7" class="footnote-backref">↩︎</a> <a href="#fnref7:1" class="footnote-backref">↩︎</a> <a href="#fnref7:2" class="footnote-backref">↩︎</a></p></li><li id="fn8" class="footnote-item"><p><a href="https://hudi.apache.org/docs/cleaning/" target="_blank" rel="noreferrer">Apache Hudi: Cleaning</a> <a href="#fnref8" class="footnote-backref">↩︎</a> <a href="#fnref8:1" class="footnote-backref">↩︎</a></p></li><li id="fn9" class="footnote-item"><p><a href="https://docs.delta.io/latest/delta-deletion-vectors.html" target="_blank" rel="noreferrer">Delta Lake: Deletion Vectors</a> <a href="#fnref9" class="footnote-backref">↩︎</a> <a href="#fnref9:1" class="footnote-backref">↩︎</a></p></li><li id="fn10" class="footnote-item"><p><a href="https://hudi.apache.org/docs/table_types/" target="_blank" rel="noreferrer">Apache Hudi: Table Types</a> <a href="#fnref10" class="footnote-backref">↩︎</a> <a href="#fnref10:1" class="footnote-backref">↩︎</a></p></li></ol></section></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/tuning-reference/data-formats.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>Data Formats</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="/docs/tuning-reference/caching.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>Caching & Persistence</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
26
- <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"9ToZXcTk\",\"contributor-guide_architecture_detector-contract.md\":\"CsCytCho\",\"contributor-guide_architecture_drill-down.md\":\"Bawx6NgP\",\"contributor-guide_architecture_impact-estimation.md\":\"sf9DU2bA\",\"contributor-guide_architecture_index.md\":\"3TO9ic6w\",\"contributor-guide_architecture_overview.md\":\"rhfxoGsI\",\"contributor-guide_architecture_state-and-history.md\":\"B8Wc_ZTg\",\"contributor-guide_architecture_widget-rendering.md\":\"DYwFeyjV\",\"contributor-guide_architecture_worker-protocol.md\":\"CBJkwH24\",\"contributor-guide_contributing.md\":\"B2E9ni79\",\"contributor-guide_development-setup.md\":\"s85j2l8b\",\"contributor-guide_testing.md\":\"Cnn6gz1C\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"Df1YMIHu\",\"tuning-reference_aqe.md\":\"BIsCtLzm\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"CEstB3Ia\",\"tuning-reference_bottleneck-cold-start.md\":\"CEuy-72y\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"CIohQDfn\",\"tuning-reference_bottleneck-failures.md\":\"CAsFf3RR\",\"tuning-reference_bottleneck-gc.md\":\"B4T3kJYV\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"BaJl__1W\",\"tuning-reference_bottleneck-memory-utilization.md\":\"DbP-SJZc\",\"tuning-reference_bottleneck-retry-waste.md\":\"D5JMjVOt\",\"tuning-reference_bottleneck-shuffle.md\":\"DDMQVOW6\",\"tuning-reference_bottleneck-skew.md\":\"B0MzhWcX\",\"tuning-reference_bottleneck-slow-host.md\":\"BlIo6UDW\",\"tuning-reference_bottleneck-small-files.md\":\"DHGMTmPs\",\"tuning-reference_bottleneck-spill.md\":\"PNH7mITt\",\"tuning-reference_bottleneck-straggler.md\":\"DIbXdAcQ\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"CB05v3zh\",\"tuning-reference_bottleneck-utilization.md\":\"DqC_iGkw\",\"tuning-reference_caching.md\":\"B7aQ8asB\",\"tuning-reference_cluster-config.md\":\"CD6vzYtD\",\"tuning-reference_config.md\":\"UvveiWG3\",\"tuning-reference_data-formats.md\":\"bjCAWH3N\",\"tuning-reference_index.md\":\"BQ_NooMV\",\"tuning-reference_intro.md\":\"CobD-lGB\",\"tuning-reference_joins.md\":\"BtKs_CuW\",\"tuning-reference_memory-model.md\":\"CWhvgevP\",\"tuning-reference_metrics.md\":\"mLOh7Apj\",\"tuning-reference_partitioning.md\":\"q0zKF_8X\",\"tuning-reference_pyspark.md\":\"DDCfvN9t\",\"tuning-reference_shuffle.md\":\"BZZ7R4Ix\",\"tuning-reference_spark-architecture.md\":\"Dwzm5avO\",\"tuning-reference_table-formats.md\":\"D6wj-2dX\",\"user-guide_alternative-log-retrieval.md\":\"ZHPSbBkF\",\"user-guide_getting-started.md\":\"Cpe0U-eO\",\"user-guide_mcp-tools.md\":\"DrphooJv\",\"user-guide_run-comparison.md\":\"CQc1aoU8\",\"user-guide_understanding-findings.md\":\"YBBLOwWi\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Alternative log retrieval\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
26
+ <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"VTcYTH4d\",\"contributor-guide_architecture_detector-contract.md\":\"CsqZHz_T\",\"contributor-guide_architecture_drill-down.md\":\"CFNCE8JB\",\"contributor-guide_architecture_impact-estimation.md\":\"HT9IJGIl\",\"contributor-guide_architecture_index.md\":\"3TO9ic6w\",\"contributor-guide_architecture_overview.md\":\"CTEqVeV4\",\"contributor-guide_architecture_state-and-history.md\":\"DOTVHjTV\",\"contributor-guide_architecture_widget-rendering.md\":\"BmpARZve\",\"contributor-guide_architecture_worker-protocol.md\":\"21pTYhFF\",\"contributor-guide_contributing.md\":\"B2E9ni79\",\"contributor-guide_development-setup.md\":\"CjgZJwjK\",\"contributor-guide_testing.md\":\"DTwhpoN2\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"D08Uaod7\",\"tuning-reference_aqe.md\":\"C1Ae6A8u\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"6wjKuJNE\",\"tuning-reference_bottleneck-cold-start.md\":\"BVQbsuHP\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"DtaYnaiF\",\"tuning-reference_bottleneck-failures.md\":\"Z1Tdmdd3\",\"tuning-reference_bottleneck-gc.md\":\"BCnXLf1n\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"CcvQj2g9\",\"tuning-reference_bottleneck-memory-utilization.md\":\"B-C0-8h-\",\"tuning-reference_bottleneck-retry-waste.md\":\"DCnhWiqj\",\"tuning-reference_bottleneck-shuffle.md\":\"B5OMJCd3\",\"tuning-reference_bottleneck-skew.md\":\"Cvo6NQm_\",\"tuning-reference_bottleneck-slow-host.md\":\"ja5qr3wq\",\"tuning-reference_bottleneck-small-files.md\":\"Crjtb0i2\",\"tuning-reference_bottleneck-spill.md\":\"Bl8KtSRg\",\"tuning-reference_bottleneck-straggler.md\":\"B0VN8reT\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"DQCftGhu\",\"tuning-reference_bottleneck-utilization.md\":\"BXDl30f9\",\"tuning-reference_caching.md\":\"CcaiMn_7\",\"tuning-reference_cluster-config.md\":\"DbWG1yZq\",\"tuning-reference_config.md\":\"YUqSpwFB\",\"tuning-reference_data-formats.md\":\"C5DX5xL4\",\"tuning-reference_index.md\":\"ZVIpQ6Dk\",\"tuning-reference_intro.md\":\"BSSaDlqx\",\"tuning-reference_joins.md\":\"BVhNCDnE\",\"tuning-reference_memory-model.md\":\"CfjFdgbl\",\"tuning-reference_metrics.md\":\"C64VNEL4\",\"tuning-reference_partitioning.md\":\"CtLK0xsS\",\"tuning-reference_pyspark.md\":\"D-92XgF8\",\"tuning-reference_shuffle.md\":\"Iq2v1oHJ\",\"tuning-reference_spark-architecture.md\":\"_qwnE_Sc\",\"tuning-reference_table-formats.md\":\"BSYEK-eN\",\"user-guide_alternative-log-retrieval.md\":\"ZHPSbBkF\",\"user-guide_getting-started.md\":\"CEa2ZX-l\",\"user-guide_mcp-tools.md\":\"dQwIvdTF\",\"user-guide_run-comparison.md\":\"ChLI3MK0\",\"user-guide_understanding-findings.md\":\"D1ccs60C\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Alternative log retrieval\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
27
27
 
28
28
  </body>
29
29
  </html>
package/public/docs/user-guide/alternative-log-retrieval.html CHANGED
@@ -9,9 +9,9 @@
9
9
  <link rel="preload stylesheet" href="/docs/assets/style.DSixAiZE.css" as="style">
10
10
  <link rel="preload stylesheet" href="/docs/vp-icons.css" as="style">
11
11
 
12
- <script type="module" src="/docs/assets/app.C5E-wy6e.js"></script>
12
+ <script type="module" src="/docs/assets/app.C6w47a9W.js"></script>
13
13
  <link rel="preload" href="/docs/assets/inter-roman-latin.Di8DUHzh.woff2" as="font" type="font/woff2" crossorigin="">
14
- <link rel="modulepreload" href="/docs/assets/chunks/theme.DQS9-w_Z.js">
14
+ <link rel="modulepreload" href="/docs/assets/chunks/theme.D-mE6j02.js">
15
15
  <link rel="modulepreload" href="/docs/assets/chunks/framework.DSg0KOwT.js">
16
16
  <link rel="modulepreload" href="/docs/assets/user-guide_alternative-log-retrieval.md.ZHPSbBkF.lean.js">
17
17
  <link rel="icon" type="image/svg+xml" href="/docs/favicon.svg">
@@ -23,7 +23,7 @@
23
23
  </head>
24
24
  <body>
25
25
  <div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>User Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/getting-started.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Getting started</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/understanding-findings.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Understanding findings</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/run-comparison.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Run comparison mode</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/mcp-tools.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>MCP tools reference</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/alternative-log-retrieval.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Alternative log retrieval</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_user-guide_alternative-log-retrieval" data-v-39a288b8><div><h1 id="alternative-ways-to-get-the-logs" tabindex="-1">Alternative ways to get the logs <a class="header-anchor" href="#alternative-ways-to-get-the-logs" aria-label="Permalink to &quot;Alternative ways to get the logs&quot;">​</a></h1><p><code>--shs-base-url</code> is the direct route: point SparkForensics at the History Server and it fetches the event log itself. This page covers what to do when that direct route isn&#39;t available.</p><h2 id="when-you-need-this" tabindex="-1">When you need this <a class="header-anchor" href="#when-you-need-this" aria-label="Permalink to &quot;When you need this&quot;">​</a></h2><p>The most common case is a Spark History Server (SHS) reachable only through an SSH bastion or jump host (the recipe below is the same regardless of what&#39;s issuing the SSH session), with the event logs themselves living on Kerberized HDFS and no direct HTTP path from your machine to SHS.</p><h2 id="why-shs-base-url-won-t-work-here" tabindex="-1">Why <code>--shs-base-url</code> won&#39;t work here <a class="header-anchor" href="#why-shs-base-url-won-t-work-here" aria-label="Permalink to &quot;Why `--shs-base-url` won&#39;t work here&quot;">​</a></h2><p>The CLI&#39;s <code>--shs-base-url</code>/<code>--app-id</code> flags and the MCP tools&#39; <code>{ shsBaseUrl, appId }</code> source both make a direct HTTP request from wherever SparkForensics runs to the History Server&#39;s REST API. If that address is only reachable from inside a bastion, that request never lands: there&#39;s no <code>--via-ssh</code> flag or other way to route it through an SSH session.</p><h2 id="the-recipe" tabindex="-1">The recipe <a class="header-anchor" href="#the-recipe" aria-label="Permalink to &quot;The recipe&quot;">​</a></h2><ol><li><p>SSH into the edge node (through the bastion, or whatever gets you there).</p></li><li><p>Pull the event log off HDFS onto local disk on the edge node:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">hdfs</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> dfs</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> -get</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> /path/to/spark-events/application_XXXX_XXXX</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> /tmp/application_XXXX_XXXX</span></span></code></pre></div></li><li><p>Copy that file back to your own machine through the same bastion, e.g. with <code>scp</code> or <code>sftp</code>:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">scp</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> edge-node:/tmp/application_XXXX_XXXX</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> ./application_XXXX_XXXX</span></span></code></pre></div></li><li><p>Point SparkForensics at the local copy instead of a <code>shsBaseUrl</code>:</p><ul><li>Browser: drop the file onto the landing page, same as any other run (see <a href="./getting-started.html">Getting started</a>).</li><li>CLI: <code>npx -p sparkforensics-cli sparkforensics-analyze ./application_XXXX_XXXX</code> (full flag list in <a href="./getting-started.html#ci-and-automation">Getting started</a>).</li><li>MCP: call a tool with <code>{ &quot;source&quot;: { &quot;path&quot;: &quot;./application_XXXX_XXXX&quot; } }</code> (tool reference in <a href="./mcp-tools.html">MCP tools reference</a>).</li></ul></li></ol><p>The file is the app&#39;s native input format either way: a single event-log file, optionally <code>.gz</code>/<code>.zstd</code>/<code>.lz4</code>/<code>.snappy</code>-compressed, or a rolling <code>eventlog_v2_*</code> directory pulled the same way. No conversion step needed.</p></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/user-guide/mcp-tools.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>MCP tools reference</span><!--]--></a></div><div class="pager" data-v-e257564d><!----></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
26
- <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"9ToZXcTk\",\"contributor-guide_architecture_detector-contract.md\":\"CsCytCho\",\"contributor-guide_architecture_drill-down.md\":\"Bawx6NgP\",\"contributor-guide_architecture_impact-estimation.md\":\"sf9DU2bA\",\"contributor-guide_architecture_index.md\":\"3TO9ic6w\",\"contributor-guide_architecture_overview.md\":\"rhfxoGsI\",\"contributor-guide_architecture_state-and-history.md\":\"B8Wc_ZTg\",\"contributor-guide_architecture_widget-rendering.md\":\"DYwFeyjV\",\"contributor-guide_architecture_worker-protocol.md\":\"CBJkwH24\",\"contributor-guide_contributing.md\":\"B2E9ni79\",\"contributor-guide_development-setup.md\":\"s85j2l8b\",\"contributor-guide_testing.md\":\"Cnn6gz1C\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"Df1YMIHu\",\"tuning-reference_aqe.md\":\"BIsCtLzm\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"CEstB3Ia\",\"tuning-reference_bottleneck-cold-start.md\":\"CEuy-72y\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"CIohQDfn\",\"tuning-reference_bottleneck-failures.md\":\"CAsFf3RR\",\"tuning-reference_bottleneck-gc.md\":\"B4T3kJYV\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"BaJl__1W\",\"tuning-reference_bottleneck-memory-utilization.md\":\"DbP-SJZc\",\"tuning-reference_bottleneck-retry-waste.md\":\"D5JMjVOt\",\"tuning-reference_bottleneck-shuffle.md\":\"DDMQVOW6\",\"tuning-reference_bottleneck-skew.md\":\"B0MzhWcX\",\"tuning-reference_bottleneck-slow-host.md\":\"BlIo6UDW\",\"tuning-reference_bottleneck-small-files.md\":\"DHGMTmPs\",\"tuning-reference_bottleneck-spill.md\":\"PNH7mITt\",\"tuning-reference_bottleneck-straggler.md\":\"DIbXdAcQ\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"CB05v3zh\",\"tuning-reference_bottleneck-utilization.md\":\"DqC_iGkw\",\"tuning-reference_caching.md\":\"B7aQ8asB\",\"tuning-reference_cluster-config.md\":\"CD6vzYtD\",\"tuning-reference_config.md\":\"UvveiWG3\",\"tuning-reference_data-formats.md\":\"bjCAWH3N\",\"tuning-reference_index.md\":\"BQ_NooMV\",\"tuning-reference_intro.md\":\"CobD-lGB\",\"tuning-reference_joins.md\":\"BtKs_CuW\",\"tuning-reference_memory-model.md\":\"CWhvgevP\",\"tuning-reference_metrics.md\":\"mLOh7Apj\",\"tuning-reference_partitioning.md\":\"q0zKF_8X\",\"tuning-reference_pyspark.md\":\"DDCfvN9t\",\"tuning-reference_shuffle.md\":\"BZZ7R4Ix\",\"tuning-reference_spark-architecture.md\":\"Dwzm5avO\",\"tuning-reference_table-formats.md\":\"D6wj-2dX\",\"user-guide_alternative-log-retrieval.md\":\"ZHPSbBkF\",\"user-guide_getting-started.md\":\"Cpe0U-eO\",\"user-guide_mcp-tools.md\":\"DrphooJv\",\"user-guide_run-comparison.md\":\"CQc1aoU8\",\"user-guide_understanding-findings.md\":\"YBBLOwWi\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Alternative log retrieval\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
26
+ <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"VTcYTH4d\",\"contributor-guide_architecture_detector-contract.md\":\"CsqZHz_T\",\"contributor-guide_architecture_drill-down.md\":\"CFNCE8JB\",\"contributor-guide_architecture_impact-estimation.md\":\"HT9IJGIl\",\"contributor-guide_architecture_index.md\":\"3TO9ic6w\",\"contributor-guide_architecture_overview.md\":\"CTEqVeV4\",\"contributor-guide_architecture_state-and-history.md\":\"DOTVHjTV\",\"contributor-guide_architecture_widget-rendering.md\":\"BmpARZve\",\"contributor-guide_architecture_worker-protocol.md\":\"21pTYhFF\",\"contributor-guide_contributing.md\":\"B2E9ni79\",\"contributor-guide_development-setup.md\":\"CjgZJwjK\",\"contributor-guide_testing.md\":\"DTwhpoN2\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"D08Uaod7\",\"tuning-reference_aqe.md\":\"C1Ae6A8u\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"6wjKuJNE\",\"tuning-reference_bottleneck-cold-start.md\":\"BVQbsuHP\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"DtaYnaiF\",\"tuning-reference_bottleneck-failures.md\":\"Z1Tdmdd3\",\"tuning-reference_bottleneck-gc.md\":\"BCnXLf1n\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"CcvQj2g9\",\"tuning-reference_bottleneck-memory-utilization.md\":\"B-C0-8h-\",\"tuning-reference_bottleneck-retry-waste.md\":\"DCnhWiqj\",\"tuning-reference_bottleneck-shuffle.md\":\"B5OMJCd3\",\"tuning-reference_bottleneck-skew.md\":\"Cvo6NQm_\",\"tuning-reference_bottleneck-slow-host.md\":\"ja5qr3wq\",\"tuning-reference_bottleneck-small-files.md\":\"Crjtb0i2\",\"tuning-reference_bottleneck-spill.md\":\"Bl8KtSRg\",\"tuning-reference_bottleneck-straggler.md\":\"B0VN8reT\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"DQCftGhu\",\"tuning-reference_bottleneck-utilization.md\":\"BXDl30f9\",\"tuning-reference_caching.md\":\"CcaiMn_7\",\"tuning-reference_cluster-config.md\":\"DbWG1yZq\",\"tuning-reference_config.md\":\"YUqSpwFB\",\"tuning-reference_data-formats.md\":\"C5DX5xL4\",\"tuning-reference_index.md\":\"ZVIpQ6Dk\",\"tuning-reference_intro.md\":\"BSSaDlqx\",\"tuning-reference_joins.md\":\"BVhNCDnE\",\"tuning-reference_memory-model.md\":\"CfjFdgbl\",\"tuning-reference_metrics.md\":\"C64VNEL4\",\"tuning-reference_partitioning.md\":\"CtLK0xsS\",\"tuning-reference_pyspark.md\":\"D-92XgF8\",\"tuning-reference_shuffle.md\":\"Iq2v1oHJ\",\"tuning-reference_spark-architecture.md\":\"_qwnE_Sc\",\"tuning-reference_table-formats.md\":\"BSYEK-eN\",\"user-guide_alternative-log-retrieval.md\":\"ZHPSbBkF\",\"user-guide_getting-started.md\":\"CEa2ZX-l\",\"user-guide_mcp-tools.md\":\"dQwIvdTF\",\"user-guide_run-comparison.md\":\"ChLI3MK0\",\"user-guide_understanding-findings.md\":\"D1ccs60C\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Alternative log retrieval\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
27
27
 
28
28
  </body>
29
29
  </html>