sparkforensics-server 0.2.4 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (173) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +5 -1
  3. package/package.json +1 -1
  4. package/public/assets/{AutoscalingChurn-BOiUUfuY.js → AutoscalingChurn-lxZHWIwq.js} +1 -1
  5. package/public/assets/CacheUtilization-DuAJ-Lve.js +1 -0
  6. package/public/assets/CachingOpportunity-DC1t7o2_.js +1 -0
  7. package/public/assets/{ColdStart-ZHxPrJmI.js → ColdStart-BOYXeiUZ.js} +1 -1
  8. package/public/assets/{ConfigAudit-BkVnq2XE.js → ConfigAudit-MCPrJomC.js} +1 -1
  9. package/public/assets/{CoreUsageArea-DobrSXk2.js → CoreUsageArea-ChNuqGA0.js} +1 -1
  10. package/public/assets/Dashboard-VuyUGvZx.js +1 -0
  11. package/public/assets/DuplicatePlanSubtree-JZut9xwq.js +1 -0
  12. package/public/assets/{DurationHistogram-DlQv4ZCB.js → DurationHistogram-_7C1w0l4.js} +1 -1
  13. package/public/assets/{EmptyState-DqnSxTjJ.js → EmptyState-D_ulfIIi.js} +1 -1
  14. package/public/assets/{ExecutorUtilization-CuQUxaCd.js → ExecutorUtilization-B08E76jT.js} +1 -1
  15. package/public/assets/GcPressure-BZMi2fTx.js +1 -0
  16. package/public/assets/{ImpactEstimate-Df_laG-n.js → ImpactEstimate-36WxuPdV.js} +1 -1
  17. package/public/assets/{IncompleteRun-CpIkS4XZ.js → IncompleteRun-t8JU7HRt.js} +1 -1
  18. package/public/assets/{JobFailures-DKbvYc4J.js → JobFailures-CguB1bec.js} +1 -1
  19. package/public/assets/{MemoryUtilization-C5kHmXIb.js → MemoryUtilization-DDk8lz9-.js} +1 -1
  20. package/public/assets/OverBroadcast-BHXLt6s7.js +1 -0
  21. package/public/assets/PartitionSizing-COj3uwV-.js +1 -0
  22. package/public/assets/{PlanExplorer-DjGTZifP.js → PlanExplorer-CnCdEY_i.js} +1 -1
  23. package/public/assets/PlanGraphRoute-BesZOW1n.js +1 -0
  24. package/public/assets/{RetryWaste-C0znHd_w.js → RetryWaste-CcU4ctAn.js} +1 -1
  25. package/public/assets/{RowPagination-2icPidMP.js → RowPagination-D2aIcfGv.js} +1 -1
  26. package/public/assets/{ShuffleIO-L_XSnJ_V.js → ShuffleIO-NPHkWbg1.js} +1 -1
  27. package/public/assets/{Skew-w4uV5H_g.js → Skew-CzztP-C4.js} +1 -1
  28. package/public/assets/SlowHost-Dh2KHypN.js +1 -0
  29. package/public/assets/SmallFiles-CgYSuZQw.js +1 -0
  30. package/public/assets/{SpeculationWaste-xCKwjz2K.js → SpeculationWaste-C8J5z1WS.js} +1 -1
  31. package/public/assets/Spill-DwTQPZ_L.js +1 -0
  32. package/public/assets/{StageFailed-De3vVDbc.js → StageFailed-aFeIEh3p.js} +1 -1
  33. package/public/assets/{StageFindingGroup-R4OW-g4U.js → StageFindingGroup-DQmAE3uk.js} +1 -1
  34. package/public/assets/{StageHeader-CGeWqC-0.js → StageHeader-D6GBbcZp.js} +1 -1
  35. package/public/assets/{StagePill-BwwKLcFl.js → StagePill-CSKyM8Kd.js} +1 -1
  36. package/public/assets/{StageShape-D4LmPVyV.js → StageShape-J6U4cUcM.js} +1 -1
  37. package/public/assets/{StageSlowness-8-z6qn4Q.js → StageSlowness-C5F2v_ZO.js} +1 -1
  38. package/public/assets/Straggler-DcP3dFGA.js +1 -0
  39. package/public/assets/TaskFailures-C-vIEheo.js +1 -0
  40. package/public/assets/{TinyTask-DYXks5sT.js → TinyTask-Be-pvTjJ.js} +1 -1
  41. package/public/assets/UnderBroadcast-DfjYGhoK.js +1 -0
  42. package/public/assets/{finding-anchor-CDbHtqKT.js → finding-anchor-vVODA_VC.js} +1 -1
  43. package/public/assets/{inbox-M_EuWNiK.js → inbox-LtbS_kBS.js} +1 -1
  44. package/public/assets/{index-pnImGbgQ.js → index-27MbqTFr.js} +5 -3
  45. package/public/assets/index-CWjjbtyo.css +2 -0
  46. package/public/assets/parser-worker-jAP0kWvS.js +122 -0
  47. package/public/assets/{select-C-MZGNpb.js → select-CC2Lyrmi.js} +1 -1
  48. package/public/assets/{toggle-group-C3nz7UQ_.js → toggle-group-lmgO21t9.js} +1 -1
  49. package/public/assets/useRecentFiles-qH5Ap4xm.js +12 -0
  50. package/public/assets/{useSortMode-DHlKZeWv.js → useSortMode-CvPY99UV.js} +1 -1
  51. package/public/docs/404.html +2 -2
  52. package/public/docs/assets/{app.DsFQcynM.js → app.C5E-wy6e.js} +1 -1
  53. package/public/docs/assets/chunks/@localSearchIndexroot.CJBoRv1b.js +1 -0
  54. package/public/docs/assets/chunks/{VPLocalSearchBox.oa8NbaiV.js → VPLocalSearchBox.DtyygpbL.js} +1 -1
  55. package/public/docs/assets/chunks/{theme.Ca4dR9dL.js → theme.DQS9-w_Z.js} +2 -2
  56. package/public/docs/assets/{contributor-guide_architecture_detector-contract.md.aQQfP7pj.js → contributor-guide_architecture_detector-contract.md.CsCytCho.js} +1 -1
  57. package/public/docs/assets/{contributor-guide_architecture_detector-contract.md.aQQfP7pj.lean.js → contributor-guide_architecture_detector-contract.md.CsCytCho.lean.js} +1 -1
  58. package/public/docs/assets/contributor-guide_architecture_overview.md.rhfxoGsI.js +1 -0
  59. package/public/docs/assets/contributor-guide_architecture_overview.md.rhfxoGsI.lean.js +1 -0
  60. package/public/docs/assets/{contributor-guide_architecture_state-and-history.md.C-xxn0q7.js → contributor-guide_architecture_state-and-history.md.B8Wc_ZTg.js} +1 -1
  61. package/public/docs/assets/contributor-guide_architecture_worker-protocol.md.CBJkwH24.js +6 -0
  62. package/public/docs/assets/{contributor-guide_architecture_worker-protocol.md.P7d1vtEJ.lean.js → contributor-guide_architecture_worker-protocol.md.CBJkwH24.lean.js} +1 -1
  63. package/public/docs/assets/contributor-guide_contributing.md.B2E9ni79.js +1 -0
  64. package/public/docs/assets/contributor-guide_contributing.md.B2E9ni79.lean.js +1 -0
  65. package/public/docs/assets/contributor-guide_development-setup.md.s85j2l8b.js +15 -0
  66. package/public/docs/assets/contributor-guide_development-setup.md.s85j2l8b.lean.js +1 -0
  67. package/public/docs/assets/contributor-guide_testing.md.Cnn6gz1C.js +1 -0
  68. package/public/docs/assets/contributor-guide_testing.md.Cnn6gz1C.lean.js +1 -0
  69. package/public/docs/assets/{user-guide_alternative-log-retrieval.md.B4tPGIal.js → user-guide_alternative-log-retrieval.md.ZHPSbBkF.js} +1 -1
  70. package/public/docs/assets/{user-guide_alternative-log-retrieval.md.B4tPGIal.lean.js → user-guide_alternative-log-retrieval.md.ZHPSbBkF.lean.js} +1 -1
  71. package/public/docs/assets/user-guide_getting-started.md.Cpe0U-eO.js +3 -0
  72. package/public/docs/assets/{user-guide_getting-started.md.BJvwLEIM.lean.js → user-guide_getting-started.md.Cpe0U-eO.lean.js} +1 -1
  73. package/public/docs/assets/{user-guide_mcp-tools.md.DGvolGzt.js → user-guide_mcp-tools.md.DrphooJv.js} +1 -1
  74. package/public/docs/assets/user-guide_understanding-findings.md.YBBLOwWi.js +1 -0
  75. package/public/docs/contributor-guide/architecture/board-widgets.html +3 -3
  76. package/public/docs/contributor-guide/architecture/detector-contract.html +5 -5
  77. package/public/docs/contributor-guide/architecture/drill-down.html +3 -3
  78. package/public/docs/contributor-guide/architecture/impact-estimation.html +3 -3
  79. package/public/docs/contributor-guide/architecture/index.html +3 -3
  80. package/public/docs/contributor-guide/architecture/overview.html +5 -5
  81. package/public/docs/contributor-guide/architecture/state-and-history.html +5 -5
  82. package/public/docs/contributor-guide/architecture/widget-rendering.html +3 -3
  83. package/public/docs/contributor-guide/architecture/worker-protocol.html +6 -6
  84. package/public/docs/contributor-guide/contributing.html +5 -5
  85. package/public/docs/contributor-guide/development-setup.html +11 -8
  86. package/public/docs/contributor-guide/testing.html +5 -5
  87. package/public/docs/hashmap.json +1 -1
  88. package/public/docs/index.html +3 -3
  89. package/public/docs/tuning-reference/anti-patterns.html +3 -3
  90. package/public/docs/tuning-reference/aqe.html +3 -3
  91. package/public/docs/tuning-reference/bottleneck-broadcast-sizing.html +3 -3
  92. package/public/docs/tuning-reference/bottleneck-cold-start.html +3 -3
  93. package/public/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html +3 -3
  94. package/public/docs/tuning-reference/bottleneck-failures.html +3 -3
  95. package/public/docs/tuning-reference/bottleneck-gc.html +3 -3
  96. package/public/docs/tuning-reference/bottleneck-job-failure-rate.html +3 -3
  97. package/public/docs/tuning-reference/bottleneck-memory-utilization.html +3 -3
  98. package/public/docs/tuning-reference/bottleneck-retry-waste.html +3 -3
  99. package/public/docs/tuning-reference/bottleneck-shuffle.html +3 -3
  100. package/public/docs/tuning-reference/bottleneck-skew.html +3 -3
  101. package/public/docs/tuning-reference/bottleneck-slow-host.html +3 -3
  102. package/public/docs/tuning-reference/bottleneck-small-files.html +3 -3
  103. package/public/docs/tuning-reference/bottleneck-spill.html +3 -3
  104. package/public/docs/tuning-reference/bottleneck-straggler.html +3 -3
  105. package/public/docs/tuning-reference/bottleneck-tiny-tasks.html +3 -3
  106. package/public/docs/tuning-reference/bottleneck-utilization.html +3 -3
  107. package/public/docs/tuning-reference/caching.html +3 -3
  108. package/public/docs/tuning-reference/cluster-config.html +3 -3
  109. package/public/docs/tuning-reference/config.html +3 -3
  110. package/public/docs/tuning-reference/data-formats.html +3 -3
  111. package/public/docs/tuning-reference/index.html +3 -3
  112. package/public/docs/tuning-reference/intro.html +3 -3
  113. package/public/docs/tuning-reference/joins.html +3 -3
  114. package/public/docs/tuning-reference/memory-model.html +3 -3
  115. package/public/docs/tuning-reference/metrics.html +3 -3
  116. package/public/docs/tuning-reference/partitioning.html +3 -3
  117. package/public/docs/tuning-reference/pyspark.html +3 -3
  118. package/public/docs/tuning-reference/shuffle.html +3 -3
  119. package/public/docs/tuning-reference/spark-architecture.html +3 -3
  120. package/public/docs/tuning-reference/table-formats.html +3 -3
  121. package/public/docs/user-guide/alternative-log-retrieval.html +5 -5
  122. package/public/docs/user-guide/getting-started.html +6 -6
  123. package/public/docs/user-guide/mcp-tools.html +5 -5
  124. package/public/docs/user-guide/run-comparison.html +3 -3
  125. package/public/docs/user-guide/understanding-findings.html +5 -5
  126. package/public/index.html +2 -2
  127. package/vendor-core/cli/native-zstd.js +2 -2
  128. package/vendor-core/detectors.js +16 -2
  129. package/vendor-core/docs-content/detection/fail.md +6 -2
  130. package/vendor-core/event-handlers.js +21 -0
  131. package/vendor-core/event-schemas.js +8 -0
  132. package/vendor-core/evidence-report.js +20 -1
  133. package/vendor-core/parser-worker.js +21 -4
  134. package/vendor-core/redact.js +24 -2
  135. package/vendor-core/run-comparison.js +8 -0
  136. package/vendor-core/shs-fetch.js +175 -65
  137. package/vendor-core/shs-load.js +1 -1
  138. package/vendor-core/stage-quantiles.js +12 -0
  139. package/vendor-core/task-failure.js +151 -0
  140. package/vendor-core/vendor/fflate.js +1 -1
  141. package/vendor-core/zip-archive.js +167 -0
  142. package/public/assets/CacheUtilization-DwX7lisd.js +0 -1
  143. package/public/assets/CachingOpportunity-DUE3GKpx.js +0 -1
  144. package/public/assets/Dashboard-y3_9iQJo.js +0 -1
  145. package/public/assets/DuplicatePlanSubtree-BSQZFeNq.js +0 -1
  146. package/public/assets/GcPressure-CLT5hkMJ.js +0 -1
  147. package/public/assets/OverBroadcast-C5pF66aq.js +0 -1
  148. package/public/assets/PartitionSizing-BvEJQ5M1.js +0 -1
  149. package/public/assets/PlanGraphRoute-DoIGA7_H.js +0 -1
  150. package/public/assets/SlowHost-D5tWJ1UR.js +0 -1
  151. package/public/assets/SmallFiles-CTtNiSOh.js +0 -1
  152. package/public/assets/Spill-BT8my24K.js +0 -1
  153. package/public/assets/Straggler-BIgxsowI.js +0 -1
  154. package/public/assets/TaskFailures-B9lLt6Rw.js +0 -1
  155. package/public/assets/UnderBroadcast-BWqWMpeG.js +0 -1
  156. package/public/assets/index-CjVklAdW.css +0 -2
  157. package/public/assets/parser-worker-C3BAO6z9.js +0 -117
  158. package/public/assets/useRecentFiles-0XuhS91q.js +0 -11
  159. package/public/docs/assets/chunks/@localSearchIndexroot.BEfYag5Q.js +0 -1
  160. package/public/docs/assets/contributor-guide_architecture_overview.md.BXQdgG5u.js +0 -1
  161. package/public/docs/assets/contributor-guide_architecture_overview.md.BXQdgG5u.lean.js +0 -1
  162. package/public/docs/assets/contributor-guide_architecture_worker-protocol.md.P7d1vtEJ.js +0 -6
  163. package/public/docs/assets/contributor-guide_contributing.md.CvRsdr6J.js +0 -1
  164. package/public/docs/assets/contributor-guide_contributing.md.CvRsdr6J.lean.js +0 -1
  165. package/public/docs/assets/contributor-guide_development-setup.md.DvMV1xhQ.js +0 -12
  166. package/public/docs/assets/contributor-guide_development-setup.md.DvMV1xhQ.lean.js +0 -1
  167. package/public/docs/assets/contributor-guide_testing.md.BU6vyA-9.js +0 -1
  168. package/public/docs/assets/contributor-guide_testing.md.BU6vyA-9.lean.js +0 -1
  169. package/public/docs/assets/user-guide_getting-started.md.BJvwLEIM.js +0 -3
  170. package/public/docs/assets/user-guide_understanding-findings.md.B29f0ucS.js +0 -1
  171. /package/public/docs/assets/{contributor-guide_architecture_state-and-history.md.C-xxn0q7.lean.js → contributor-guide_architecture_state-and-history.md.B8Wc_ZTg.lean.js} +0 -0
  172. /package/public/docs/assets/{user-guide_mcp-tools.md.DGvolGzt.lean.js → user-guide_mcp-tools.md.DrphooJv.lean.js} +0 -0
  173. /package/public/docs/assets/{user-guide_understanding-findings.md.B29f0ucS.lean.js → user-guide_understanding-findings.md.YBBLOwWi.lean.js} +0 -0
package/public/docs/tuning-reference/table-formats.html CHANGED
@@ -9,9 +9,9 @@
9
9
  <link rel="preload stylesheet" href="/docs/assets/style.DSixAiZE.css" as="style">
10
10
  <link rel="preload stylesheet" href="/docs/vp-icons.css" as="style">
11
11
 
12
- <script type="module" src="/docs/assets/app.DsFQcynM.js"></script>
12
+ <script type="module" src="/docs/assets/app.C5E-wy6e.js"></script>
13
13
  <link rel="preload" href="/docs/assets/inter-roman-latin.Di8DUHzh.woff2" as="font" type="font/woff2" crossorigin="">
14
- <link rel="modulepreload" href="/docs/assets/chunks/theme.Ca4dR9dL.js">
14
+ <link rel="modulepreload" href="/docs/assets/chunks/theme.DQS9-w_Z.js">
15
15
  <link rel="modulepreload" href="/docs/assets/chunks/framework.DSg0KOwT.js">
16
16
  <link rel="modulepreload" href="/docs/assets/tuning-reference_table-formats.md.D6wj-2dX.lean.js">
17
17
  <link rel="icon" type="image/svg+xml" href="/docs/favicon.svg">
@@ -23,7 +23,7 @@
23
23
  </head>
24
24
  <body>
25
25
  <div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Getting Started</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/intro.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Introduction</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Optimization Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/spark-architecture.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Execution Model</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/memory-model.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Management</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/partitioning.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Partitioning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/joins.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Join Optimization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/data-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Data Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/table-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Table Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/caching.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Caching & Persistence</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/pyspark.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>PySpark Specifics</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/aqe.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Adaptive Query Execution</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/cluster-config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cluster Tuning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/anti-patterns.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Anti-Patterns</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Detector Catalog</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-skew.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Skew</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle I/O</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-spill.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory / Disk Spill</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-gc.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>GC Pressure</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-cold-start.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cold Start</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Executor Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-slow-host.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Slow Host</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-failures.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Failures</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-straggler.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Stragglers</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-retry-waste.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Retry Waste</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-tiny-tasks.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Tiny Tasks</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-job-failure-rate.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Job Failure Rate</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-memory-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Duplicate Plan Subtree</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-small-files.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Small Files</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-broadcast-sizing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Broadcast Sizing</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Reference</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/metrics.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Metrics Glossary</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Config Quick-Reference</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_tuning-reference_table-formats" data-v-39a288b8><div><h1 id="table-formats" tabindex="-1">Table Formats <a class="header-anchor" href="#table-formats" aria-label="Permalink to &quot;Table Formats {#table-formats}&quot;">​</a></h1><h2 id="the-knobs-these-formats-share" tabindex="-1">The knobs these formats share <a class="header-anchor" href="#the-knobs-these-formats-share" aria-label="Permalink to &quot;The knobs these formats share&quot;">​</a></h2><p><a href="./data-formats.html">Parquet and ORC</a> stop at the file. A lakehouse <em>table format</em> wraps a directory of Parquet files with a transactional metadata layer that tracks which files belong to the table right now, so engines get ACID commits, time travel, schema evolution, and row-level updates over plain columnar files. The three in wide use are Delta Lake, Apache Iceberg, and Apache Hudi. They solve the same problems and differ mostly in mechanism, and the tuning knobs that matter fall into a handful of dimensions.</p><p><strong>File sizing.</strong> All three fight the small-files problem, but at different points in the write. Delta Lake does not size files during the write by default; its <code>OPTIMIZE</code> command bin-packs already-written small files into larger ones, available since Delta Lake 1.2.0<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup>. Iceberg rewrites files after the fact through its <code>rewrite_data_files</code> procedure, whose default <code>binpack</code> strategy coalesces small files<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup>. Hudi treats correct sizing at write time as &quot;a critical design decision&quot;: auto-sizing targets a 120 MB Parquet base file (<code>hoodie.parquet.max.file.size</code>) and pads any existing file at or below the 100 MB small-file limit (<code>hoodie.parquet.small.file.limit</code>) with new records rather than opening a fresh file<sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup>.</p><p><strong>Compaction.</strong> Delta&#39;s <code>OPTIMIZE</code> bin-packing is the compaction path, run manually or, since Delta Lake 3.1.0, as auto compaction that runs synchronously right after a write succeeds<sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup>. Iceberg uses <code>rewrite_data_files</code> as a stored procedure invoked from Spark SQL<sup class="footnote-ref"><a href="#fn2" id="fnref2:1">[2:1]</a></sup>. Hudi splits the job in two: <em>compaction</em> applies only to Merge-on-Read tables and merges the row-based delta logs back into base files (async by default, or inline via <code>hoodie.compact.inline = true</code>), while <em>clustering</em> is a separate data-layout service that stitches small files together<sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup><sup class="footnote-ref"><a href="#fn5" id="fnref5">[5]</a></sup>.</p><p><strong>Clustering.</strong> To co-locate rows that are queried together, Delta Lake offers Z-Ordering via <code>OPTIMIZE table ZORDER BY (cols)</code>; its effectiveness drops with each added column and it is not idempotent, since each run reclusters all files in the partition<sup class="footnote-ref"><a href="#fn1" id="fnref1:2">[1:2]</a></sup>. Delta also documents liquid clustering as a separate feature<sup class="footnote-ref"><a href="#fn1" id="fnref1:3">[1:3]</a></sup>. Iceberg clusters through <code>rewrite_data_files</code> with <code>strategy =&gt; &#39;sort&#39;</code> and a <code>sort_order</code> argument, including a <code>zorder(c1,c2)</code> form<sup class="footnote-ref"><a href="#fn2" id="fnref2:2">[2:2]</a></sup>. Hudi clustering rewrites file groups sorting by <code>hoodie.clustering.plan.strategy.sort.columns</code><sup class="footnote-ref"><a href="#fn5" id="fnref5:1">[5:1]</a></sup>.</p><p><strong>Metadata and manifest overhead.</strong> Delta records each change as a JSON commit in the transaction log and periodically compacts those commits into a Parquet checkpoint so readers reconstruct state without replaying every commit; checkpoints can be split multi-part (default 50,000 actions per part), and since Delta 3.0 log-compaction files aggregate a commit range to cut checkpoint frequency<sup class="footnote-ref"><a href="#fn1" id="fnref1:4">[1:4]</a></sup>. Iceberg writes a new metadata JSON file per change, tracks the manifests for each snapshot in a manifest list written fresh on every commit, and splits large tables across multiple manifests so query planning parallelizes<sup class="footnote-ref"><a href="#fn6" id="fnref6">[6]</a></sup>. Hudi records every write and table-service action as an instant on a timeline, with clustering and compaction writing their plans there before executing<sup class="footnote-ref"><a href="#fn5" id="fnref5:2">[5:2]</a></sup><sup class="footnote-ref"><a href="#fn4" id="fnref4:1">[4:1]</a></sup>.</p><p><strong>Snapshot and version expiry.</strong> Delta reclaims storage with <code>VACUUM</code>, which deletes data files past a retention threshold (default 7 days) but never log files; log files are pruned automatically after checkpoints, with a 30-day default set by <code>delta.logRetentionDuration</code><sup class="footnote-ref"><a href="#fn7" id="fnref7">[7]</a></sup>. Iceberg&#39;s <code>expire_snapshots</code> removes old snapshots and the files only they referenced (<code>older_than</code> default 5 days, <code>retain_last</code> default 1), and <code>remove_orphan_files</code> sweeps unreferenced files (<code>older_than</code> default 3 days)<sup class="footnote-ref"><a href="#fn2" id="fnref2:3">[2:3]</a></sup>. Hudi&#39;s cleaner runs automatically after each commit; the default <code>KEEP_LATEST_COMMITS</code> policy retains <code>hoodie.clean.commits.retained</code> commits (default 10)<sup class="footnote-ref"><a href="#fn8" id="fnref8">[8]</a></sup>.</p><p><strong>Row-level deletes: merge-on-read vs copy-on-write.</strong> By default, deleting one row in a Delta table rewrites the whole Parquet file that holds it. Deletion vectors avoid that by marking rows removed in a side file and applying the marks at read time; support landed incrementally (DELETE in 2.4.0, UPDATE in 3.0.0, on by default since 3.1.0)<sup class="footnote-ref"><a href="#fn9" id="fnref9">[9]</a></sup>. Iceberg took a spec-versioned route: v2 adds position and equality delete files, and v3 replaces position deletes with per-file deletion-vector bitmaps stored in the Puffin format<sup class="footnote-ref"><a href="#fn6" id="fnref6:1">[6:1]</a></sup>. Hudi frames the same trade-off as two table types: Copy-on-Write rewrites a base file on every change (fast reads, slower writes), while Merge-on-Read appends changes to log files merged at query time and compacted later (fast writes, some read cost)<sup class="footnote-ref"><a href="#fn10" id="fnref10">[10]</a></sup>.</p><h2 id="where-these-problems-show-up" tabindex="-1">Where these problems show up <a class="header-anchor" href="#where-these-problems-show-up" aria-label="Permalink to &quot;Where these problems show up&quot;">​</a></h2><p>Each of those knobs fails in its own recognizable way. The symptoms are the same ones the <a href="./data-formats.html">Data Formats</a> and <a href="./bottleneck-small-files.html">Small Files</a> pages describe, read through the table&#39;s own metadata. A table accumulating many sub-target files is a sizing problem: Hudi&#39;s own guidance ties small files to more tasks, more per-file open/close cost, and cloud object-store request-rate limits that trip because at least one request is issued per file regardless of size<sup class="footnote-ref"><a href="#fn3" id="fnref3:1">[3:1]</a></sup>.</p><p>Growing metadata is the second signal. Iceberg snapshots and metadata JSON files accumulate until expiry runs, and Iceberg recommends <code>expire_snapshots</code> specifically to keep metadata size small on top of freeing data files<sup class="footnote-ref"><a href="#fn6" id="fnref6:2">[6:2]</a></sup>. Delta&#39;s commit log grows until checkpoints and log compaction absorb it<sup class="footnote-ref"><a href="#fn1" id="fnref1:5">[1:5]</a></sup>. Hudi&#39;s timeline lengthens with every commit, clean, cluster, and compaction<sup class="footnote-ref"><a href="#fn5" id="fnref5:3">[5:3]</a></sup>. Delta&#39;s <code>deltaTable.history()</code> surfaces the per-commit version, timestamp, and operation for inspecting that growth<sup class="footnote-ref"><a href="#fn1" id="fnref1:6">[1:6]</a></sup>.</p><p>Read amplification is the third. On a Merge-on-Read Hudi table, uncompacted delta logs are merged at query time, so a table that has not compacted recently reads more slowly<sup class="footnote-ref"><a href="#fn4" id="fnref4:2">[4:2]</a></sup>. The Iceberg equivalent is a data file with many stacked position/equality delete files that a scan must apply<sup class="footnote-ref"><a href="#fn6" id="fnref6:3">[6:3]</a></sup>.</p><h2 id="what-each-dimension-costs" tabindex="-1">What each dimension costs <a class="header-anchor" href="#what-each-dimension-costs" aria-label="Permalink to &quot;What each dimension costs&quot;">​</a></h2><p>Those symptoms aren&#39;t cosmetic. Small files cost more in metadata and scheduling than their bytes suggest: a query scans many files for the same data, each file adds fixed overhead, and on object storage the per-file request pattern raises the odds of hitting per-prefix rate limits<sup class="footnote-ref"><a href="#fn3" id="fnref3:2">[3:2]</a></sup>. That is exactly the tension the compaction and clustering services exist to resolve, since ingestion favors many small files for low latency while queries favor fewer large ones<sup class="footnote-ref"><a href="#fn3" id="fnref3:3">[3:3]</a></sup>.</p><p>Metadata overhead is a planning tax. More data files mean more manifest entries, and small files inflate that metadata disproportionately and slow query planning, which is why Iceberg exposes <code>rewriteManifests</code> and compaction to cut it<sup class="footnote-ref"><a href="#fn6" id="fnref6:4">[6:4]</a></sup>. Unbounded snapshot and metadata retention keeps that footprint growing until expiry is run<sup class="footnote-ref"><a href="#fn6" id="fnref6:5">[6:5]</a></sup>.</p><p>Clustering decides how much data a query skips. Delta&#39;s Z-Ordering co-locates related values so data-skipping can prune more files, dramatically reducing bytes read<sup class="footnote-ref"><a href="#fn1" id="fnref1:7">[1:7]</a></sup>. The retention settings carry a correctness edge too: once Delta <code>VACUUM</code> runs, time travel to a version older than the retention window is gone<sup class="footnote-ref"><a href="#fn7" id="fnref7:1">[7:1]</a></sup>, and Iceberg warns that running <code>remove_orphan_files</code> with too short an interval can delete in-flight files and corrupt the table<sup class="footnote-ref"><a href="#fn6" id="fnref6:6">[6:6]</a></sup>.</p><h2 id="tuning-each-dimension" tabindex="-1">Tuning each dimension <a class="header-anchor" href="#tuning-each-dimension" aria-label="Permalink to &quot;Tuning each dimension&quot;">​</a></h2><p>Each dimension has a matching maintenance habit that keeps its cost down. Compact on a schedule. For Delta, run <code>OPTIMIZE</code> (optionally scoped with a <code>WHERE</code> partition predicate) or enable auto compaction on 3.1.0+<sup class="footnote-ref"><a href="#fn1" id="fnref1:8">[1:8]</a></sup>. For Iceberg, call <code>rewrite_data_files</code><sup class="footnote-ref"><a href="#fn2" id="fnref2:4">[2:4]</a></sup>. For Merge-on-Read Hudi, let async compaction run or force it inline with <code>hoodie.compact.inline = true</code>, and use clustering to consolidate small files independently of it<sup class="footnote-ref"><a href="#fn4" id="fnref4:3">[4:3]</a></sup><sup class="footnote-ref"><a href="#fn5" id="fnref5:4">[5:4]</a></sup>.</p><p>Cluster the columns you filter on. Use Delta <code>OPTIMIZE ... ZORDER BY (cols)</code> on a few high-cardinality predicate columns<sup class="footnote-ref"><a href="#fn1" id="fnref1:9">[1:9]</a></sup>, Iceberg <code>rewrite_data_files(strategy =&gt; &#39;sort&#39;, sort_order =&gt; &#39;...&#39;)</code> or its <code>zorder(...)</code> form<sup class="footnote-ref"><a href="#fn2" id="fnref2:5">[2:5]</a></sup>, or Hudi clustering with <code>hoodie.clustering.plan.strategy.sort.columns</code><sup class="footnote-ref"><a href="#fn5" id="fnref5:5">[5:5]</a></sup>.</p><p>Expire aggressively but safely. Run Delta <code>VACUUM</code> (keeping the 7-day floor unless you have a reason to override it, since the safety check exists to protect concurrent readers)<sup class="footnote-ref"><a href="#fn7" id="fnref7:2">[7:2]</a></sup>, Iceberg <code>expire_snapshots</code> plus periodic <code>remove_orphan_files</code> with an interval longer than your longest in-flight write<sup class="footnote-ref"><a href="#fn6" id="fnref6:7">[6:7]</a></sup>, and tune Hudi&#39;s cleaner via <code>hoodie.clean.commits.retained</code><sup class="footnote-ref"><a href="#fn8" id="fnref8:1">[8:1]</a></sup>.</p><p>Match the delete strategy to the workload. Enable Delta deletion vectors (<code>ALTER TABLE ... SET TBLPROPERTIES(&#39;delta.enableDeletionVectors&#39; = true)</code>) so DML marks rows instead of rewriting files, remembering the marks are applied physically only on <code>OPTIMIZE</code> or <code>REORG TABLE ... APPLY (PURGE)</code><sup class="footnote-ref"><a href="#fn9" id="fnref9:1">[9:1]</a></sup>. On Iceberg, prefer merge-on-read deletes for update-heavy tables and compact the delete files<sup class="footnote-ref"><a href="#fn6" id="fnref6:8">[6:8]</a></sup>. On Hudi, pick Copy-on-Write for read-heavy tables and Merge-on-Read for write-heavy or near-real-time ingestion<sup class="footnote-ref"><a href="#fn10" id="fnref10:1">[10:1]</a></sup>.</p><h2 id="sources" tabindex="-1">Sources <a class="header-anchor" href="#sources" aria-label="Permalink to &quot;Sources&quot;">​</a></h2><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><a href="https://docs.delta.io/latest/optimizations-oss.html" target="_blank" rel="noreferrer">Delta Lake: Optimizations (OSS)</a> <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a> <a href="#fnref1:2" class="footnote-backref">↩︎</a> <a href="#fnref1:3" class="footnote-backref">↩︎</a> <a href="#fnref1:4" class="footnote-backref">↩︎</a> <a href="#fnref1:5" class="footnote-backref">↩︎</a> <a href="#fnref1:6" class="footnote-backref">↩︎</a> <a href="#fnref1:7" class="footnote-backref">↩︎</a> <a href="#fnref1:8" class="footnote-backref">↩︎</a> <a href="#fnref1:9" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><a href="https://iceberg.apache.org/docs/latest/spark-procedures/" target="_blank" rel="noreferrer">Apache Iceberg: Spark Procedures</a> <a href="#fnref2" class="footnote-backref">↩︎</a> <a href="#fnref2:1" class="footnote-backref">↩︎</a> <a href="#fnref2:2" class="footnote-backref">↩︎</a> <a href="#fnref2:3" class="footnote-backref">↩︎</a> <a href="#fnref2:4" class="footnote-backref">↩︎</a> <a href="#fnref2:5" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><a href="https://hudi.apache.org/docs/file_sizing/" target="_blank" rel="noreferrer">Apache Hudi: File Sizing</a> <a href="#fnref3" class="footnote-backref">↩︎</a> <a href="#fnref3:1" class="footnote-backref">↩︎</a> <a href="#fnref3:2" class="footnote-backref">↩︎</a> <a href="#fnref3:3" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><a href="https://hudi.apache.org/docs/compaction/" target="_blank" rel="noreferrer">Apache Hudi: Compaction</a> <a href="#fnref4" class="footnote-backref">↩︎</a> <a href="#fnref4:1" class="footnote-backref">↩︎</a> <a href="#fnref4:2" class="footnote-backref">↩︎</a> <a href="#fnref4:3" class="footnote-backref">↩︎</a></p></li><li id="fn5" class="footnote-item"><p><a href="https://hudi.apache.org/docs/clustering/" target="_blank" rel="noreferrer">Apache Hudi: Clustering</a> <a href="#fnref5" class="footnote-backref">↩︎</a> <a href="#fnref5:1" class="footnote-backref">↩︎</a> <a href="#fnref5:2" class="footnote-backref">↩︎</a> <a href="#fnref5:3" class="footnote-backref">↩︎</a> <a href="#fnref5:4" class="footnote-backref">↩︎</a> <a href="#fnref5:5" class="footnote-backref">↩︎</a></p></li><li id="fn6" class="footnote-item"><p><a href="https://iceberg.apache.org/docs/latest/maintenance/" target="_blank" rel="noreferrer">Apache Iceberg: Maintenance</a> and <a href="https://iceberg.apache.org/spec/" target="_blank" rel="noreferrer">Table Spec</a> <a href="#fnref6" class="footnote-backref">↩︎</a> <a href="#fnref6:1" class="footnote-backref">↩︎</a> <a href="#fnref6:2" class="footnote-backref">↩︎</a> <a href="#fnref6:3" class="footnote-backref">↩︎</a> <a href="#fnref6:4" class="footnote-backref">↩︎</a> <a href="#fnref6:5" class="footnote-backref">↩︎</a> <a href="#fnref6:6" class="footnote-backref">↩︎</a> <a href="#fnref6:7" class="footnote-backref">↩︎</a> <a href="#fnref6:8" class="footnote-backref">↩︎</a></p></li><li id="fn7" class="footnote-item"><p><a href="https://docs.delta.io/latest/delta-utility.html" target="_blank" rel="noreferrer">Delta Lake: Table Utility Commands</a> <a href="#fnref7" class="footnote-backref">↩︎</a> <a href="#fnref7:1" class="footnote-backref">↩︎</a> <a href="#fnref7:2" class="footnote-backref">↩︎</a></p></li><li id="fn8" class="footnote-item"><p><a href="https://hudi.apache.org/docs/cleaning/" target="_blank" rel="noreferrer">Apache Hudi: Cleaning</a> <a href="#fnref8" class="footnote-backref">↩︎</a> <a href="#fnref8:1" class="footnote-backref">↩︎</a></p></li><li id="fn9" class="footnote-item"><p><a href="https://docs.delta.io/latest/delta-deletion-vectors.html" target="_blank" rel="noreferrer">Delta Lake: Deletion Vectors</a> <a href="#fnref9" class="footnote-backref">↩︎</a> <a href="#fnref9:1" class="footnote-backref">↩︎</a></p></li><li id="fn10" class="footnote-item"><p><a href="https://hudi.apache.org/docs/table_types/" target="_blank" rel="noreferrer">Apache Hudi: Table Types</a> <a href="#fnref10" class="footnote-backref">↩︎</a> <a href="#fnref10:1" class="footnote-backref">↩︎</a></p></li></ol></section></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/tuning-reference/data-formats.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>Data Formats</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="/docs/tuning-reference/caching.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>Caching & Persistence</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
26
- <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"9ToZXcTk\",\"contributor-guide_architecture_detector-contract.md\":\"aQQfP7pj\",\"contributor-guide_architecture_drill-down.md\":\"Bawx6NgP\",\"contributor-guide_architecture_impact-estimation.md\":\"sf9DU2bA\",\"contributor-guide_architecture_index.md\":\"3TO9ic6w\",\"contributor-guide_architecture_overview.md\":\"BXQdgG5u\",\"contributor-guide_architecture_state-and-history.md\":\"C-xxn0q7\",\"contributor-guide_architecture_widget-rendering.md\":\"DYwFeyjV\",\"contributor-guide_architecture_worker-protocol.md\":\"P7d1vtEJ\",\"contributor-guide_contributing.md\":\"CvRsdr6J\",\"contributor-guide_development-setup.md\":\"DvMV1xhQ\",\"contributor-guide_testing.md\":\"BU6vyA-9\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"Df1YMIHu\",\"tuning-reference_aqe.md\":\"BIsCtLzm\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"CEstB3Ia\",\"tuning-reference_bottleneck-cold-start.md\":\"CEuy-72y\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"CIohQDfn\",\"tuning-reference_bottleneck-failures.md\":\"CAsFf3RR\",\"tuning-reference_bottleneck-gc.md\":\"B4T3kJYV\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"BaJl__1W\",\"tuning-reference_bottleneck-memory-utilization.md\":\"DbP-SJZc\",\"tuning-reference_bottleneck-retry-waste.md\":\"D5JMjVOt\",\"tuning-reference_bottleneck-shuffle.md\":\"DDMQVOW6\",\"tuning-reference_bottleneck-skew.md\":\"B0MzhWcX\",\"tuning-reference_bottleneck-slow-host.md\":\"BlIo6UDW\",\"tuning-reference_bottleneck-small-files.md\":\"DHGMTmPs\",\"tuning-reference_bottleneck-spill.md\":\"PNH7mITt\",\"tuning-reference_bottleneck-straggler.md\":\"DIbXdAcQ\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"CB05v3zh\",\"tuning-reference_bottleneck-utilization.md\":\"DqC_iGkw\",\"tuning-reference_caching.md\":\"B7aQ8asB\",\"tuning-reference_cluster-config.md\":\"CD6vzYtD\",\"tuning-reference_config.md\":\"UvveiWG3\",\"tuning-reference_data-formats.md\":\"bjCAWH3N\",\"tuning-reference_index.md\":\"BQ_NooMV\",\"tuning-reference_intro.md\":\"CobD-lGB\",\"tuning-reference_joins.md\":\"BtKs_CuW\",\"tuning-reference_memory-model.md\":\"CWhvgevP\",\"tuning-reference_metrics.md\":\"mLOh7Apj\",\"tuning-reference_partitioning.md\":\"q0zKF_8X\",\"tuning-reference_pyspark.md\":\"DDCfvN9t\",\"tuning-reference_shuffle.md\":\"BZZ7R4Ix\",\"tuning-reference_spark-architecture.md\":\"Dwzm5avO\",\"tuning-reference_table-formats.md\":\"D6wj-2dX\",\"user-guide_alternative-log-retrieval.md\":\"B4tPGIal\",\"user-guide_getting-started.md\":\"BJvwLEIM\",\"user-guide_mcp-tools.md\":\"DGvolGzt\",\"user-guide_run-comparison.md\":\"CQc1aoU8\",\"user-guide_understanding-findings.md\":\"B29f0ucS\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Alternative log retrieval\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
26
+ <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"9ToZXcTk\",\"contributor-guide_architecture_detector-contract.md\":\"CsCytCho\",\"contributor-guide_architecture_drill-down.md\":\"Bawx6NgP\",\"contributor-guide_architecture_impact-estimation.md\":\"sf9DU2bA\",\"contributor-guide_architecture_index.md\":\"3TO9ic6w\",\"contributor-guide_architecture_overview.md\":\"rhfxoGsI\",\"contributor-guide_architecture_state-and-history.md\":\"B8Wc_ZTg\",\"contributor-guide_architecture_widget-rendering.md\":\"DYwFeyjV\",\"contributor-guide_architecture_worker-protocol.md\":\"CBJkwH24\",\"contributor-guide_contributing.md\":\"B2E9ni79\",\"contributor-guide_development-setup.md\":\"s85j2l8b\",\"contributor-guide_testing.md\":\"Cnn6gz1C\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"Df1YMIHu\",\"tuning-reference_aqe.md\":\"BIsCtLzm\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"CEstB3Ia\",\"tuning-reference_bottleneck-cold-start.md\":\"CEuy-72y\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"CIohQDfn\",\"tuning-reference_bottleneck-failures.md\":\"CAsFf3RR\",\"tuning-reference_bottleneck-gc.md\":\"B4T3kJYV\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"BaJl__1W\",\"tuning-reference_bottleneck-memory-utilization.md\":\"DbP-SJZc\",\"tuning-reference_bottleneck-retry-waste.md\":\"D5JMjVOt\",\"tuning-reference_bottleneck-shuffle.md\":\"DDMQVOW6\",\"tuning-reference_bottleneck-skew.md\":\"B0MzhWcX\",\"tuning-reference_bottleneck-slow-host.md\":\"BlIo6UDW\",\"tuning-reference_bottleneck-small-files.md\":\"DHGMTmPs\",\"tuning-reference_bottleneck-spill.md\":\"PNH7mITt\",\"tuning-reference_bottleneck-straggler.md\":\"DIbXdAcQ\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"CB05v3zh\",\"tuning-reference_bottleneck-utilization.md\":\"DqC_iGkw\",\"tuning-reference_caching.md\":\"B7aQ8asB\",\"tuning-reference_cluster-config.md\":\"CD6vzYtD\",\"tuning-reference_config.md\":\"UvveiWG3\",\"tuning-reference_data-formats.md\":\"bjCAWH3N\",\"tuning-reference_index.md\":\"BQ_NooMV\",\"tuning-reference_intro.md\":\"CobD-lGB\",\"tuning-reference_joins.md\":\"BtKs_CuW\",\"tuning-reference_memory-model.md\":\"CWhvgevP\",\"tuning-reference_metrics.md\":\"mLOh7Apj\",\"tuning-reference_partitioning.md\":\"q0zKF_8X\",\"tuning-reference_pyspark.md\":\"DDCfvN9t\",\"tuning-reference_shuffle.md\":\"BZZ7R4Ix\",\"tuning-reference_spark-architecture.md\":\"Dwzm5avO\",\"tuning-reference_table-formats.md\":\"D6wj-2dX\",\"user-guide_alternative-log-retrieval.md\":\"ZHPSbBkF\",\"user-guide_getting-started.md\":\"Cpe0U-eO\",\"user-guide_mcp-tools.md\":\"DrphooJv\",\"user-guide_run-comparison.md\":\"CQc1aoU8\",\"user-guide_understanding-findings.md\":\"YBBLOwWi\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Alternative log retrieval\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
27
27
 
28
28
  </body>
29
29
  </html>
package/public/docs/user-guide/alternative-log-retrieval.html CHANGED
@@ -9,11 +9,11 @@
9
9
  <link rel="preload stylesheet" href="/docs/assets/style.DSixAiZE.css" as="style">
10
10
  <link rel="preload stylesheet" href="/docs/vp-icons.css" as="style">
11
11
 
12
- <script type="module" src="/docs/assets/app.DsFQcynM.js"></script>
12
+ <script type="module" src="/docs/assets/app.C5E-wy6e.js"></script>
13
13
  <link rel="preload" href="/docs/assets/inter-roman-latin.Di8DUHzh.woff2" as="font" type="font/woff2" crossorigin="">
14
- <link rel="modulepreload" href="/docs/assets/chunks/theme.Ca4dR9dL.js">
14
+ <link rel="modulepreload" href="/docs/assets/chunks/theme.DQS9-w_Z.js">
15
15
  <link rel="modulepreload" href="/docs/assets/chunks/framework.DSg0KOwT.js">
16
- <link rel="modulepreload" href="/docs/assets/user-guide_alternative-log-retrieval.md.B4tPGIal.lean.js">
16
+ <link rel="modulepreload" href="/docs/assets/user-guide_alternative-log-retrieval.md.ZHPSbBkF.lean.js">
17
17
  <link rel="icon" type="image/svg+xml" href="/docs/favicon.svg">
18
18
  <link rel="preconnect" href="https://fonts.googleapis.com">
19
19
  <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin="">
@@ -22,8 +22,8 @@
22
22
  <script id="check-mac-os">document.documentElement.classList.toggle("mac",/Mac|iPhone|iPod|iPad/i.test(navigator.platform));</script>
23
23
  </head>
24
24
  <body>
25
- <div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>User Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/getting-started.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Getting started</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/understanding-findings.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Understanding findings</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/run-comparison.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Run comparison mode</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/mcp-tools.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>MCP tools reference</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/alternative-log-retrieval.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Alternative log retrieval</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_user-guide_alternative-log-retrieval" data-v-39a288b8><div><h1 id="alternative-ways-to-get-the-logs" tabindex="-1">Alternative ways to get the logs <a class="header-anchor" href="#alternative-ways-to-get-the-logs" aria-label="Permalink to &quot;Alternative ways to get the logs&quot;">​</a></h1><p><code>--shs-base-url</code> is the direct route: point SparkForensics at the History Server and it fetches the event log itself. This page covers what to do when that direct route isn&#39;t available.</p><h2 id="when-you-need-this" tabindex="-1">When you need this <a class="header-anchor" href="#when-you-need-this" aria-label="Permalink to &quot;When you need this&quot;">​</a></h2><p>The most common case is a Spark History Server (SHS) reachable only through an SSH bastion or jump host (the recipe below is the same regardless of what&#39;s issuing the SSH session), with the event logs themselves living on Kerberized HDFS and no direct HTTP path from your machine to SHS.</p><h2 id="why-shs-base-url-won-t-work-here" tabindex="-1">Why <code>--shs-base-url</code> won&#39;t work here <a class="header-anchor" href="#why-shs-base-url-won-t-work-here" aria-label="Permalink to &quot;Why `--shs-base-url` won&#39;t work here&quot;">​</a></h2><p>The CLI&#39;s <code>--shs-base-url</code>/<code>--app-id</code> flags and the MCP tools&#39; <code>{ shsBaseUrl, appId }</code> source both make a direct HTTP request from wherever SparkForensics runs to the History Server&#39;s REST API. If that address is only reachable from inside a bastion, that request never lands: there&#39;s no <code>--via-ssh</code> flag or other way to route it through an SSH session.</p><h2 id="the-recipe" tabindex="-1">The recipe <a class="header-anchor" href="#the-recipe" aria-label="Permalink to &quot;The recipe&quot;">​</a></h2><ol><li><p>SSH into the edge node (through the bastion, or whatever gets you there).</p></li><li><p>Pull the event log off HDFS onto local disk on the edge node:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">hdfs</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> dfs</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> -get</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> /path/to/spark-events/application_XXXX_XXXX</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> /tmp/application_XXXX_XXXX</span></span></code></pre></div></li><li><p>Copy that file back to your own machine through the same bastion, e.g. with <code>scp</code> or <code>sftp</code>:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">scp</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> edge-node:/tmp/application_XXXX_XXXX</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> ./application_XXXX_XXXX</span></span></code></pre></div></li><li><p>Point SparkForensics at the local copy instead of a <code>shsBaseUrl</code>:</p><ul><li>Browser: drop the file onto the landing page, same as any other run (see <a href="./getting-started.html">Getting started</a>).</li><li>CLI: <code>npx sparkforensics-analyze ./application_XXXX_XXXX</code> (full flag list in <a href="./getting-started.html#ci-and-automation">Getting started</a>).</li><li>MCP: call a tool with <code>{ &quot;source&quot;: { &quot;path&quot;: &quot;./application_XXXX_XXXX&quot; } }</code> (tool reference in <a href="./mcp-tools.html">MCP tools reference</a>).</li></ul></li></ol><p>The file is the app&#39;s native input format either way: a single event-log file, optionally <code>.gz</code>/<code>.zstd</code>/<code>.lz4</code>/<code>.snappy</code>-compressed, or a rolling <code>eventlog_v2_*</code> directory pulled the same way. No conversion step needed.</p></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/user-guide/mcp-tools.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>MCP tools reference</span><!--]--></a></div><div class="pager" data-v-e257564d><!----></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
26
- <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"9ToZXcTk\",\"contributor-guide_architecture_detector-contract.md\":\"aQQfP7pj\",\"contributor-guide_architecture_drill-down.md\":\"Bawx6NgP\",\"contributor-guide_architecture_impact-estimation.md\":\"sf9DU2bA\",\"contributor-guide_architecture_index.md\":\"3TO9ic6w\",\"contributor-guide_architecture_overview.md\":\"BXQdgG5u\",\"contributor-guide_architecture_state-and-history.md\":\"C-xxn0q7\",\"contributor-guide_architecture_widget-rendering.md\":\"DYwFeyjV\",\"contributor-guide_architecture_worker-protocol.md\":\"P7d1vtEJ\",\"contributor-guide_contributing.md\":\"CvRsdr6J\",\"contributor-guide_development-setup.md\":\"DvMV1xhQ\",\"contributor-guide_testing.md\":\"BU6vyA-9\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"Df1YMIHu\",\"tuning-reference_aqe.md\":\"BIsCtLzm\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"CEstB3Ia\",\"tuning-reference_bottleneck-cold-start.md\":\"CEuy-72y\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"CIohQDfn\",\"tuning-reference_bottleneck-failures.md\":\"CAsFf3RR\",\"tuning-reference_bottleneck-gc.md\":\"B4T3kJYV\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"BaJl__1W\",\"tuning-reference_bottleneck-memory-utilization.md\":\"DbP-SJZc\",\"tuning-reference_bottleneck-retry-waste.md\":\"D5JMjVOt\",\"tuning-reference_bottleneck-shuffle.md\":\"DDMQVOW6\",\"tuning-reference_bottleneck-skew.md\":\"B0MzhWcX\",\"tuning-reference_bottleneck-slow-host.md\":\"BlIo6UDW\",\"tuning-reference_bottleneck-small-files.md\":\"DHGMTmPs\",\"tuning-reference_bottleneck-spill.md\":\"PNH7mITt\",\"tuning-reference_bottleneck-straggler.md\":\"DIbXdAcQ\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"CB05v3zh\",\"tuning-reference_bottleneck-utilization.md\":\"DqC_iGkw\",\"tuning-reference_caching.md\":\"B7aQ8asB\",\"tuning-reference_cluster-config.md\":\"CD6vzYtD\",\"tuning-reference_config.md\":\"UvveiWG3\",\"tuning-reference_data-formats.md\":\"bjCAWH3N\",\"tuning-reference_index.md\":\"BQ_NooMV\",\"tuning-reference_intro.md\":\"CobD-lGB\",\"tuning-reference_joins.md\":\"BtKs_CuW\",\"tuning-reference_memory-model.md\":\"CWhvgevP\",\"tuning-reference_metrics.md\":\"mLOh7Apj\",\"tuning-reference_partitioning.md\":\"q0zKF_8X\",\"tuning-reference_pyspark.md\":\"DDCfvN9t\",\"tuning-reference_shuffle.md\":\"BZZ7R4Ix\",\"tuning-reference_spark-architecture.md\":\"Dwzm5avO\",\"tuning-reference_table-formats.md\":\"D6wj-2dX\",\"user-guide_alternative-log-retrieval.md\":\"B4tPGIal\",\"user-guide_getting-started.md\":\"BJvwLEIM\",\"user-guide_mcp-tools.md\":\"DGvolGzt\",\"user-guide_run-comparison.md\":\"CQc1aoU8\",\"user-guide_understanding-findings.md\":\"B29f0ucS\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Alternative log retrieval\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
25
+ <div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>User Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/getting-started.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Getting started</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/understanding-findings.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Understanding findings</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/run-comparison.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Run comparison mode</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/mcp-tools.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>MCP tools reference</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/alternative-log-retrieval.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Alternative log retrieval</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_user-guide_alternative-log-retrieval" data-v-39a288b8><div><h1 id="alternative-ways-to-get-the-logs" tabindex="-1">Alternative ways to get the logs <a class="header-anchor" href="#alternative-ways-to-get-the-logs" aria-label="Permalink to &quot;Alternative ways to get the logs&quot;">​</a></h1><p><code>--shs-base-url</code> is the direct route: point SparkForensics at the History Server and it fetches the event log itself. This page covers what to do when that direct route isn&#39;t available.</p><h2 id="when-you-need-this" tabindex="-1">When you need this <a class="header-anchor" href="#when-you-need-this" aria-label="Permalink to &quot;When you need this&quot;">​</a></h2><p>The most common case is a Spark History Server (SHS) reachable only through an SSH bastion or jump host (the recipe below is the same regardless of what&#39;s issuing the SSH session), with the event logs themselves living on Kerberized HDFS and no direct HTTP path from your machine to SHS.</p><h2 id="why-shs-base-url-won-t-work-here" tabindex="-1">Why <code>--shs-base-url</code> won&#39;t work here <a class="header-anchor" href="#why-shs-base-url-won-t-work-here" aria-label="Permalink to &quot;Why `--shs-base-url` won&#39;t work here&quot;">​</a></h2><p>The CLI&#39;s <code>--shs-base-url</code>/<code>--app-id</code> flags and the MCP tools&#39; <code>{ shsBaseUrl, appId }</code> source both make a direct HTTP request from wherever SparkForensics runs to the History Server&#39;s REST API. If that address is only reachable from inside a bastion, that request never lands: there&#39;s no <code>--via-ssh</code> flag or other way to route it through an SSH session.</p><h2 id="the-recipe" tabindex="-1">The recipe <a class="header-anchor" href="#the-recipe" aria-label="Permalink to &quot;The recipe&quot;">​</a></h2><ol><li><p>SSH into the edge node (through the bastion, or whatever gets you there).</p></li><li><p>Pull the event log off HDFS onto local disk on the edge node:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">hdfs</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> dfs</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> -get</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> /path/to/spark-events/application_XXXX_XXXX</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> /tmp/application_XXXX_XXXX</span></span></code></pre></div></li><li><p>Copy that file back to your own machine through the same bastion, e.g. with <code>scp</code> or <code>sftp</code>:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">scp</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> edge-node:/tmp/application_XXXX_XXXX</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> ./application_XXXX_XXXX</span></span></code></pre></div></li><li><p>Point SparkForensics at the local copy instead of a <code>shsBaseUrl</code>:</p><ul><li>Browser: drop the file onto the landing page, same as any other run (see <a href="./getting-started.html">Getting started</a>).</li><li>CLI: <code>npx -p sparkforensics-cli sparkforensics-analyze ./application_XXXX_XXXX</code> (full flag list in <a href="./getting-started.html#ci-and-automation">Getting started</a>).</li><li>MCP: call a tool with <code>{ &quot;source&quot;: { &quot;path&quot;: &quot;./application_XXXX_XXXX&quot; } }</code> (tool reference in <a href="./mcp-tools.html">MCP tools reference</a>).</li></ul></li></ol><p>The file is the app&#39;s native input format either way: a single event-log file, optionally <code>.gz</code>/<code>.zstd</code>/<code>.lz4</code>/<code>.snappy</code>-compressed, or a rolling <code>eventlog_v2_*</code> directory pulled the same way. No conversion step needed.</p></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/user-guide/mcp-tools.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>MCP tools reference</span><!--]--></a></div><div class="pager" data-v-e257564d><!----></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
26
+ <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"9ToZXcTk\",\"contributor-guide_architecture_detector-contract.md\":\"CsCytCho\",\"contributor-guide_architecture_drill-down.md\":\"Bawx6NgP\",\"contributor-guide_architecture_impact-estimation.md\":\"sf9DU2bA\",\"contributor-guide_architecture_index.md\":\"3TO9ic6w\",\"contributor-guide_architecture_overview.md\":\"rhfxoGsI\",\"contributor-guide_architecture_state-and-history.md\":\"B8Wc_ZTg\",\"contributor-guide_architecture_widget-rendering.md\":\"DYwFeyjV\",\"contributor-guide_architecture_worker-protocol.md\":\"CBJkwH24\",\"contributor-guide_contributing.md\":\"B2E9ni79\",\"contributor-guide_development-setup.md\":\"s85j2l8b\",\"contributor-guide_testing.md\":\"Cnn6gz1C\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"Df1YMIHu\",\"tuning-reference_aqe.md\":\"BIsCtLzm\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"CEstB3Ia\",\"tuning-reference_bottleneck-cold-start.md\":\"CEuy-72y\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"CIohQDfn\",\"tuning-reference_bottleneck-failures.md\":\"CAsFf3RR\",\"tuning-reference_bottleneck-gc.md\":\"B4T3kJYV\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"BaJl__1W\",\"tuning-reference_bottleneck-memory-utilization.md\":\"DbP-SJZc\",\"tuning-reference_bottleneck-retry-waste.md\":\"D5JMjVOt\",\"tuning-reference_bottleneck-shuffle.md\":\"DDMQVOW6\",\"tuning-reference_bottleneck-skew.md\":\"B0MzhWcX\",\"tuning-reference_bottleneck-slow-host.md\":\"BlIo6UDW\",\"tuning-reference_bottleneck-small-files.md\":\"DHGMTmPs\",\"tuning-reference_bottleneck-spill.md\":\"PNH7mITt\",\"tuning-reference_bottleneck-straggler.md\":\"DIbXdAcQ\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"CB05v3zh\",\"tuning-reference_bottleneck-utilization.md\":\"DqC_iGkw\",\"tuning-reference_caching.md\":\"B7aQ8asB\",\"tuning-reference_cluster-config.md\":\"CD6vzYtD\",\"tuning-reference_config.md\":\"UvveiWG3\",\"tuning-reference_data-formats.md\":\"bjCAWH3N\",\"tuning-reference_index.md\":\"BQ_NooMV\",\"tuning-reference_intro.md\":\"CobD-lGB\",\"tuning-reference_joins.md\":\"BtKs_CuW\",\"tuning-reference_memory-model.md\":\"CWhvgevP\",\"tuning-reference_metrics.md\":\"mLOh7Apj\",\"tuning-reference_partitioning.md\":\"q0zKF_8X\",\"tuning-reference_pyspark.md\":\"DDCfvN9t\",\"tuning-reference_shuffle.md\":\"BZZ7R4Ix\",\"tuning-reference_spark-architecture.md\":\"Dwzm5avO\",\"tuning-reference_table-formats.md\":\"D6wj-2dX\",\"user-guide_alternative-log-retrieval.md\":\"ZHPSbBkF\",\"user-guide_getting-started.md\":\"Cpe0U-eO\",\"user-guide_mcp-tools.md\":\"DrphooJv\",\"user-guide_run-comparison.md\":\"CQc1aoU8\",\"user-guide_understanding-findings.md\":\"YBBLOwWi\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Alternative log retrieval\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
27
27
 
28
28
  </body>
29
29
  </html>
package/public/docs/user-guide/getting-started.html CHANGED
@@ -9,11 +9,11 @@
9
9
  <link rel="preload stylesheet" href="/docs/assets/style.DSixAiZE.css" as="style">
10
10
  <link rel="preload stylesheet" href="/docs/vp-icons.css" as="style">
11
11
 
12
- <script type="module" src="/docs/assets/app.DsFQcynM.js"></script>
12
+ <script type="module" src="/docs/assets/app.C5E-wy6e.js"></script>
13
13
  <link rel="preload" href="/docs/assets/inter-roman-latin.Di8DUHzh.woff2" as="font" type="font/woff2" crossorigin="">
14
- <link rel="modulepreload" href="/docs/assets/chunks/theme.Ca4dR9dL.js">
14
+ <link rel="modulepreload" href="/docs/assets/chunks/theme.DQS9-w_Z.js">
15
15
  <link rel="modulepreload" href="/docs/assets/chunks/framework.DSg0KOwT.js">
16
- <link rel="modulepreload" href="/docs/assets/user-guide_getting-started.md.BJvwLEIM.lean.js">
16
+ <link rel="modulepreload" href="/docs/assets/user-guide_getting-started.md.Cpe0U-eO.lean.js">
17
17
  <link rel="icon" type="image/svg+xml" href="/docs/favicon.svg">
18
18
  <link rel="preconnect" href="https://fonts.googleapis.com">
19
19
  <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin="">
@@ -22,10 +22,10 @@
22
22
  <script id="check-mac-os">document.documentElement.classList.toggle("mac",/Mac|iPhone|iPod|iPad/i.test(navigator.platform));</script>
23
23
  </head>
24
24
  <body>
25
- <div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink active" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>User Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/getting-started.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Getting started</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/understanding-findings.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Understanding findings</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/run-comparison.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Run comparison mode</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/mcp-tools.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>MCP tools reference</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/alternative-log-retrieval.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Alternative log retrieval</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_user-guide_getting-started" data-v-39a288b8><div><h1 id="getting-started" tabindex="-1">Getting started <a class="header-anchor" href="#getting-started" aria-label="Permalink to &quot;Getting started&quot;">​</a></h1><p>SparkForensics reads a Spark History Server event log and turns it into a dashboard of flagged bottlenecks. No install and no account: open the app in a browser and drop a log in.</p><h2 id="load-a-run" tabindex="-1">Load a run <a class="header-anchor" href="#load-a-run" aria-label="Permalink to &quot;Load a run&quot;">​</a></h2><p>Drop a log file onto the landing page, or click <strong>Choose file</strong> to pick one. Parsing runs in a background worker, off the browser&#39;s main thread, so a multi-hundred-megabyte event stream doesn&#39;t freeze the tab.</p><p>No log of your own yet? Click <strong>Try a sample run</strong> on the landing page to load a bundled example run and see a populated dashboard right away.</p><p>The app takes a newline-delimited JSON event log (one JSON event per line, the format Spark writes to <code>spark.eventLog.dir</code>), either plain or gzip/Zstandard/LZ4/Snappy-compressed.</p><p>Click <strong>Other sources</strong> on the landing page for two more ways in:</p><ul><li><strong>Choose rolling-log folder</strong>, for an <code>eventlog_v2_*</code> rolling directory. Drop the folder or point the picker at it; the app reassembles its parts in order before parsing.</li><li><strong>Fetch from Spark History Server</strong>, to pull an application straight from a reachable History Server. This needs <strong>local-server mode</strong> (see below) and an application ID in one of the forms Spark itself uses: <code>application_&lt;timestamp&gt;_&lt;id&gt;</code>, <code>local-&lt;timestamp&gt;</code>, <code>app-&lt;id&gt;</code>, <code>spark-&lt;id&gt;</code>, or <code>driver-&lt;id&gt;</code>. If the fetch fails, the panel names the problem (server unreachable, application not found, local server not running, and so on) and suggests what to try next.</li></ul><p>Can&#39;t reach the History Server directly (it&#39;s only reachable through an SSH bastion)? See <a href="./alternative-log-retrieval.html">Alternative ways to get the logs</a>.</p><p>A History Server export needs one manual step. <code>GET /api/v1/applications/&lt;appId&gt;/logs</code> hands back a zip, and the drop zone does not unwrap zip containers. Extract the <code>.zstd</code> event log from that zip yourself and drop the extracted file in. It needs no further decompressing.</p><p>Files you have already loaded stay listed under <strong>Recent files</strong> on the landing page.</p><h3 id="local-server-mode" tabindex="-1">Local-server mode <a class="header-anchor" href="#local-server-mode" aria-label="Permalink to &quot;Local-server mode&quot;">​</a></h3><p>The plain browser app can&#39;t fetch from a History Server itself: the browser&#39;s CORS policy blocks a cross-origin request like that, and there&#39;s no server on the other end to proxy it. Local-server mode adds that server. Run it with:</p><div class="language-sh vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">npx</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> sparkforensics-server</span></span></code></pre></div><p>It listens on <code>http://127.0.0.1:4173</code> by default and binds to localhost only. Open that URL instead of the static build, and the <strong>Fetch from Spark History Server</strong> panel works because requests now go server-to-server. Full setup and its port/environment-variable overrides are in the <a href="https://github.com/shuffle-works/sparkforensics#readme" target="_blank" rel="noreferrer">project README</a>, which also covers what changes for a static (no-server) deploy.</p><h2 id="reading-the-dashboard" tabindex="-1">Reading the dashboard <a class="header-anchor" href="#reading-the-dashboard" aria-label="Permalink to &quot;Reading the dashboard&quot;">​</a></h2><p>When parsing finishes, the dashboard shows a board of widgets, each in its own card with a title. Every widget that flags a problem uses the same convention: a colored impact dot (critical / warning / info) and an ALL-CAPS tag for the bottleneck category (<code>SKEW</code>, <code>SPILL</code>, <code>GC</code>, and so on: see <a href="./understanding-findings.html">Understanding findings</a>). Widgets list every affected stage, not just the worst one. The dot&#39;s color tracks how much run time the finding could save you rather than how unusual the metric looks, so a small-looking anomaly with a big payoff can outrank a dramatic one that would barely move your run time.</p><p>A run scorecard (wall-clock, efficiency, wastage) always shows at the top. Below it, two tabs split the rest of the board:</p><ol><li><strong>Findings</strong>: every flagged finding and its detail widget, grouped by impact band (Critical, Warning, Info). Within a band, a recommendation row for a bottleneck type collapses into a summary row when it fires more than once; clicking the summary row expands its full list, and clicking any single row or widget jumps straight to that finding. Below the impact-band groups, memory and core-usage utilization always show, even on a clean run. Widgets that found nothing fold away into a &quot;Clean checks&quot; disclosure. This is the tab you land on.</li><li><strong>Full app report</strong>: the wall-clock and executor timelines, the stage table, and the reference-only cards.</li></ol><p>Click a finding&#39;s documentation link (or the topbar&#39;s <strong>Docs</strong> button) to open the reference material in a slide-in panel beside the dashboard: the dashboard stays visible and interactive, so you can check a metric against the reference without losing your place.</p><h3 id="advanced-view" tabindex="-1">Advanced view <a class="header-anchor" href="#advanced-view" aria-label="Permalink to &quot;Advanced view&quot;">​</a></h3><p>The topbar has an <strong>Advanced view</strong> toggle. It&#39;s off by default, which keeps each widget to the finding itself and what to do about it. Turn it on to also show confidence levels, supporting evidence, and documentation links for each finding, plus a few extra table columns. Your choice is remembered across runs.</p><h3 id="the-rest-of-the-topbar" tabindex="-1">The rest of the topbar <a class="header-anchor" href="#the-rest-of-the-topbar" aria-label="Permalink to &quot;The rest of the topbar&quot;">​</a></h3><p>Once a run is loaded, the topbar also carries a few more controls. <strong>New analysis</strong> goes back to the landing page to load another run. <strong>Plan graph</strong> opens an interactive node-and-edge view of the run&#39;s SQL execution plan, filterable down to I/O operators (scan, exchange), a broader &quot;basic&quot; set, or every operator. <strong>Export evidence</strong> downloads the current run&#39;s findings as a portable Markdown or JSON report, the same shape the CLI and MCP tools produce; turn on <strong>Redact identifiers</strong> first if the report is headed outside the environment that produced it, since that pseudonymizes the app id and any host/IP tokens. <strong>Keyboard shortcuts</strong> (press <code>?</code> from anywhere) lists every shortcut. Rounding it out: a <strong>Docs</strong> link and a theme toggle.</p><h2 id="compare-two-runs" tabindex="-1">Compare two runs <a class="header-anchor" href="#compare-two-runs" aria-label="Permalink to &quot;Compare two runs&quot;">​</a></h2><p>To compare a baseline run against a candidate, say to check whether a tuning change helped, see <a href="./run-comparison.html">Run comparison mode</a>.</p><h2 id="ci-and-automation" tabindex="-1">CI and automation <a class="header-anchor" href="#ci-and-automation" aria-label="Permalink to &quot;CI and automation&quot;">​</a></h2><p>In a pipeline or a script, use the <code>analyze</code> CLI instead of the browser dashboard. It parses the same event logs and exits non-zero when a run crosses a threshold you set, so it can gate a build:</p><div class="language-sh vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">npx</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> sparkforensics-analyze</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> &lt;</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">file</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">|</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">dir</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">&gt; [--max-runtime </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">ms]</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> [--max-skew </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">ratio]</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \</span></span>
25
+ <div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink active" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>User Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/getting-started.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Getting started</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/understanding-findings.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Understanding findings</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/run-comparison.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Run comparison mode</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/mcp-tools.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>MCP tools reference</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/alternative-log-retrieval.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Alternative log retrieval</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_user-guide_getting-started" data-v-39a288b8><div><h1 id="getting-started" tabindex="-1">Getting started <a class="header-anchor" href="#getting-started" aria-label="Permalink to &quot;Getting started&quot;">​</a></h1><p>SparkForensics reads a Spark History Server event log and turns it into a dashboard of flagged bottlenecks. No account needed.</p><h2 id="local-server-mode" tabindex="-1">Run it locally <a class="header-anchor" href="#local-server-mode" aria-label="Permalink to &quot;Run it locally {#local-server-mode}&quot;">​</a></h2><p>The recommended way to run SparkForensics on your machine:</p><div class="language-sh vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">npx</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> sparkforensics-server</span></span></code></pre></div><p>Open <code>http://127.0.0.1:4173</code> and drop a log in. This is <strong>local-server mode</strong>: it also fetches runs from a Spark History Server on your behalf. Port and environment-variable overrides, and what changes for a static deploy, are in the <a href="https://github.com/shuffle-works/sparkforensics#deploy-modes" target="_blank" rel="noreferrer">project README</a>.</p><p>The <a href="https://shuffle-works.github.io/sparkforensics/" target="_blank" rel="noreferrer">hosted demo</a> runs the same dashboard but can&#39;t fetch from a History Server. To work on SparkForensics itself, see <a href="./../contributor-guide/development-setup.html">Development setup</a>.</p><h2 id="load-a-run" tabindex="-1">Load a run <a class="header-anchor" href="#load-a-run" aria-label="Permalink to &quot;Load a run&quot;">​</a></h2><p>Drop a log file onto the landing page, or click <strong>Choose file</strong> to pick one. Parsing runs in a background worker, off the browser&#39;s main thread, so a multi-hundred-megabyte event stream doesn&#39;t freeze the tab.</p><p>No log of your own yet? Click <strong>Try a sample run</strong> on the landing page to load a bundled example run and see a populated dashboard right away.</p><p>The app takes a newline-delimited JSON event log (one JSON event per line, the format Spark writes to <code>spark.eventLog.dir</code>), either plain or gzip/Zstandard/LZ4/Snappy-compressed. It also takes the zip a Spark History Server hands back, from the Spark UI&#39;s download link or from <code>GET /api/v1/applications/&lt;appId&gt;/logs</code>, as-is: drop the <code>.zip</code> and the app unwraps the log inside it, or reassembles the parts of a rolling log. The zip must hold one attempt: for an application that ran more than once, download <code>/api/v1/applications/&lt;appId&gt;/&lt;attemptId&gt;/logs</code> instead.</p><p>Click <strong>Other sources</strong> on the landing page for two more ways in:</p><ul><li><strong>Choose rolling-log folder</strong>, for an <code>eventlog_v2_*</code> rolling directory. Drop the folder or point the picker at it; the app reassembles its parts in order before parsing.</li><li><strong>Fetch from Spark History Server</strong>, to pull an application straight from a reachable History Server. This needs <strong>local-server mode</strong> (see <a href="#local-server-mode">Run it locally</a>) and an application ID in one of the forms Spark itself uses: <code>application_&lt;timestamp&gt;_&lt;id&gt;</code>, <code>local-&lt;timestamp&gt;</code>, <code>app-&lt;id&gt;</code>, <code>spark-&lt;id&gt;</code>, or <code>driver-&lt;id&gt;</code>. If the fetch fails, the panel names the problem (server unreachable, application not found, local server not running, and so on) and suggests what to try next.</li></ul><p>Can&#39;t reach the History Server directly (it&#39;s only reachable through an SSH bastion)? See <a href="./alternative-log-retrieval.html">Alternative ways to get the logs</a>.</p><p>Files you have already loaded stay listed under <strong>Recent files</strong> on the landing page.</p><h2 id="reading-the-dashboard" tabindex="-1">Reading the dashboard <a class="header-anchor" href="#reading-the-dashboard" aria-label="Permalink to &quot;Reading the dashboard&quot;">​</a></h2><p>When parsing finishes, the dashboard shows a board of widgets, each in its own card with a title. Every widget that flags a problem uses the same convention: a colored impact dot (critical / warning / info) and an ALL-CAPS tag for the bottleneck category (<code>SKEW</code>, <code>SPILL</code>, <code>GC</code>, and so on: see <a href="./understanding-findings.html">Understanding findings</a>). Widgets list every affected stage, not just the worst one. The dot&#39;s color tracks how much run time the finding could save you rather than how unusual the metric looks, so a small-looking anomaly with a big payoff can outrank a dramatic one that would barely move your run time.</p><p>A run scorecard (wall-clock, efficiency, wastage) always shows at the top. Below it, two tabs split the rest of the board:</p><ol><li><strong>Findings</strong>: every flagged finding and its detail widget, grouped by impact band (Critical, Warning, Info). Within a band, a recommendation row for a bottleneck type collapses into a summary row when it fires more than once; clicking the summary row expands its full list, and clicking any single row or widget jumps straight to that finding. Below the impact-band groups, memory and core-usage utilization always show, even on a clean run. Widgets that found nothing fold away into a &quot;Clean checks&quot; disclosure. This is the tab you land on.</li><li><strong>Full app report</strong>: the wall-clock and executor timelines, the stage table, and the reference-only cards.</li></ol><p>Click a finding&#39;s documentation link (or the topbar&#39;s <strong>Docs</strong> button) to open the reference material in a slide-in panel beside the dashboard: the dashboard stays visible and interactive, so you can check a metric against the reference without losing your place.</p><h3 id="advanced-view" tabindex="-1">Advanced view <a class="header-anchor" href="#advanced-view" aria-label="Permalink to &quot;Advanced view&quot;">​</a></h3><p>The topbar has an <strong>Advanced view</strong> toggle. It&#39;s off by default, which keeps each widget to the finding itself and what to do about it. Turn it on to also show confidence levels, supporting evidence, and documentation links for each finding, plus a few extra table columns. Your choice is remembered across runs.</p><h3 id="the-rest-of-the-topbar" tabindex="-1">The rest of the topbar <a class="header-anchor" href="#the-rest-of-the-topbar" aria-label="Permalink to &quot;The rest of the topbar&quot;">​</a></h3><p>Once a run is loaded, the topbar also carries a few more controls. <strong>New analysis</strong> goes back to the landing page to load another run. <strong>Plan graph</strong> opens an interactive node-and-edge view of the run&#39;s SQL execution plan, filterable down to I/O operators (scan, exchange), a broader &quot;basic&quot; set, or every operator. <strong>Export evidence</strong> downloads the current run&#39;s findings as a portable Markdown or JSON report, the same shape the CLI and MCP tools produce; turn on <strong>Redact identifiers</strong> first if the report is headed outside the environment that produced it, since that pseudonymizes the app id and any host/IP tokens. <strong>Keyboard shortcuts</strong> (press <code>?</code> from anywhere) lists every shortcut. Rounding it out: a <strong>Docs</strong> link and a theme toggle.</p><h2 id="compare-two-runs" tabindex="-1">Compare two runs <a class="header-anchor" href="#compare-two-runs" aria-label="Permalink to &quot;Compare two runs&quot;">​</a></h2><p>To compare a baseline run against a candidate, say to check whether a tuning change helped, see <a href="./run-comparison.html">Run comparison mode</a>.</p><h2 id="ci-and-automation" tabindex="-1">CI and automation <a class="header-anchor" href="#ci-and-automation" aria-label="Permalink to &quot;CI and automation&quot;">​</a></h2><p>In a pipeline or a script, use the <code>analyze</code> CLI instead of the browser dashboard. It parses the same event logs and exits non-zero when a run crosses a threshold you set, so it can gate a build:</p><div class="language-sh vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">sh</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">npx</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> -p</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> sparkforensics-cli</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> sparkforensics-analyze</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> &lt;</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">file</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">|</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">dir</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">&gt; [--max-runtime </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">ms]</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> [--max-skew </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">ratio]</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \</span></span>
26
26
  <span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> [--max-spill </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">gb]</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> [--max-failed-task-rate </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">pct]</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> [--min-efficiency </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">pct]</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> \</span></span>
27
- <span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> [--out </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">path]</span></span></code></pre></div><p>Point it at a single event log or a directory of them. Pass <code>--out</code> to also write the findings to a file.</p><p>Pass <code>--export-html &lt;dir&gt;</code> to write a self-contained HTML dashboard for the run into <code>&lt;dir&gt;</code> (which must not already exist or must be empty). Open <code>&lt;dir&gt;/index.html</code> directly in a browser over <code>file://</code>, with no server, to get the same interactive dashboard offline. This makes it easy to archive or share a run. <code>--redact</code> applies to the exported report too.</p><p>The CLI also supports fetching a run directly from a reachable Spark History Server (<code>--shs-base-url</code>/<code>--app-id</code>/<code>--attempt-id</code>) instead of a local file, comparing a candidate run against a baseline with regression gating (<code>--baseline</code>/<code>--max-regression-pct</code>/<code>--regression-metric</code>/ <code>--fail-on-introduced</code>), redacting the app id and any host/IP tokens before sharing output (<code>--redact</code>), and narrowing the findings to certain impact bands, types, or a stage (<code>--impact</code>/<code>--type</code>/<code>--stage</code>). Run it with <code>--help</code> for the full flag list.</p><p>Same caveat as above: if the History Server is only reachable through an SSH bastion, <code>--shs-base-url</code> can&#39;t reach it either: see <a href="./alternative-log-retrieval.html">Alternative ways to get the logs</a>.</p><p>Running in Airflow instead of a plain CI pipeline? See <a href="https://github.com/shuffle-works/sparkforensics-operator" target="_blank" rel="noreferrer">sparkforensics-operator</a>, an Airflow operator that wraps the CLI and acts on the result after each Spark job, so you don&#39;t have to wire up the call yourself.</p><p>Want an AI assistant to diagnose a run directly, without the dashboard or a CI gate? See <a href="./mcp-tools.html">MCP tools reference</a>.</p></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><!----></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="/docs/user-guide/understanding-findings.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>Understanding findings</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
28
- <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"9ToZXcTk\",\"contributor-guide_architecture_detector-contract.md\":\"aQQfP7pj\",\"contributor-guide_architecture_drill-down.md\":\"Bawx6NgP\",\"contributor-guide_architecture_impact-estimation.md\":\"sf9DU2bA\",\"contributor-guide_architecture_index.md\":\"3TO9ic6w\",\"contributor-guide_architecture_overview.md\":\"BXQdgG5u\",\"contributor-guide_architecture_state-and-history.md\":\"C-xxn0q7\",\"contributor-guide_architecture_widget-rendering.md\":\"DYwFeyjV\",\"contributor-guide_architecture_worker-protocol.md\":\"P7d1vtEJ\",\"contributor-guide_contributing.md\":\"CvRsdr6J\",\"contributor-guide_development-setup.md\":\"DvMV1xhQ\",\"contributor-guide_testing.md\":\"BU6vyA-9\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"Df1YMIHu\",\"tuning-reference_aqe.md\":\"BIsCtLzm\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"CEstB3Ia\",\"tuning-reference_bottleneck-cold-start.md\":\"CEuy-72y\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"CIohQDfn\",\"tuning-reference_bottleneck-failures.md\":\"CAsFf3RR\",\"tuning-reference_bottleneck-gc.md\":\"B4T3kJYV\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"BaJl__1W\",\"tuning-reference_bottleneck-memory-utilization.md\":\"DbP-SJZc\",\"tuning-reference_bottleneck-retry-waste.md\":\"D5JMjVOt\",\"tuning-reference_bottleneck-shuffle.md\":\"DDMQVOW6\",\"tuning-reference_bottleneck-skew.md\":\"B0MzhWcX\",\"tuning-reference_bottleneck-slow-host.md\":\"BlIo6UDW\",\"tuning-reference_bottleneck-small-files.md\":\"DHGMTmPs\",\"tuning-reference_bottleneck-spill.md\":\"PNH7mITt\",\"tuning-reference_bottleneck-straggler.md\":\"DIbXdAcQ\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"CB05v3zh\",\"tuning-reference_bottleneck-utilization.md\":\"DqC_iGkw\",\"tuning-reference_caching.md\":\"B7aQ8asB\",\"tuning-reference_cluster-config.md\":\"CD6vzYtD\",\"tuning-reference_config.md\":\"UvveiWG3\",\"tuning-reference_data-formats.md\":\"bjCAWH3N\",\"tuning-reference_index.md\":\"BQ_NooMV\",\"tuning-reference_intro.md\":\"CobD-lGB\",\"tuning-reference_joins.md\":\"BtKs_CuW\",\"tuning-reference_memory-model.md\":\"CWhvgevP\",\"tuning-reference_metrics.md\":\"mLOh7Apj\",\"tuning-reference_partitioning.md\":\"q0zKF_8X\",\"tuning-reference_pyspark.md\":\"DDCfvN9t\",\"tuning-reference_shuffle.md\":\"BZZ7R4Ix\",\"tuning-reference_spark-architecture.md\":\"Dwzm5avO\",\"tuning-reference_table-formats.md\":\"D6wj-2dX\",\"user-guide_alternative-log-retrieval.md\":\"B4tPGIal\",\"user-guide_getting-started.md\":\"BJvwLEIM\",\"user-guide_mcp-tools.md\":\"DGvolGzt\",\"user-guide_run-comparison.md\":\"CQc1aoU8\",\"user-guide_understanding-findings.md\":\"B29f0ucS\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Alternative log retrieval\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
27
+ <span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> [--out </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">path]</span></span></code></pre></div><p>The command ships in the <code>sparkforensics-cli</code> package. To install it once: <code>npm i -g sparkforensics-cli</code>, then run <code>sparkforensics-analyze</code> directly.</p><p>Point it at a single event-log file, or at one run&#39;s <code>eventlog_v2_*</code> rolling-log directory; any other directory is rejected. Output goes to stdout; pass <code>--out &lt;path&gt;</code> to write it to that file instead.</p><p>Pass <code>--export-html &lt;dir&gt;</code> to write a self-contained HTML dashboard for the run into <code>&lt;dir&gt;</code> (which must not already exist or must be empty). Open <code>&lt;dir&gt;/index.html</code> directly in a browser over <code>file://</code>, with no server, to get the same interactive dashboard offline. This makes it easy to archive or share a run. <code>--redact</code> applies to the exported report too.</p><p>The CLI also supports fetching a run directly from a reachable Spark History Server (<code>--shs-base-url</code>/<code>--app-id</code>/<code>--attempt-id</code>) instead of a local file, comparing a candidate run against a baseline with regression gating (<code>--baseline</code>/<code>--max-regression-pct</code>/<code>--regression-metric</code>/ <code>--fail-on-introduced</code>), redacting the app id and any host/IP tokens before sharing output (<code>--redact</code>), and narrowing the findings to certain impact bands, types, or a stage (<code>--impact</code>/<code>--type</code>/<code>--stage</code>). Run it with <code>--help</code> for the full flag list.</p><h3 id="regression-metric-keys" tabindex="-1">Regression metric keys <a class="header-anchor" href="#regression-metric-keys" aria-label="Permalink to &quot;Regression metric keys&quot;">​</a></h3><p><code>--regression-metric</code> (and the MCP <code>evaluate_budgets</code> tool&#39;s <code>regressionMetric</code>) takes one of these keys; the default is <code>wallClock</code>:</p><ul><li><code>wallClock</code>: wall-clock duration</li><li><code>executorRunTime</code>: summed executor run time</li><li><code>shuffleSpill</code>: shuffle spill</li><li><code>diskSpill</code>: disk spill</li><li><code>gcTime</code>: JVM GC time</li><li><code>taskSkew</code>: p95 task skew</li><li><code>failedTaskRate</code>: failed-task rate</li></ul><p>Four more keys, <code>inputBytes</code>, <code>outputBytes</code>, <code>taskCount</code> and <code>executorsAdded</code>, measure workload volume rather than performance. They have no better or worse direction, so a regression budget on one of them reports <code>inconclusive</code> whenever the value changes, and passes when it doesn&#39;t.</p><p>Same caveat as above: if the History Server is only reachable through an SSH bastion, <code>--shs-base-url</code> can&#39;t reach it either: see <a href="./alternative-log-retrieval.html">Alternative ways to get the logs</a>.</p><p>Running in Airflow instead of a plain CI pipeline? See <a href="https://github.com/shuffle-works/sparkforensics-operator" target="_blank" rel="noreferrer">sparkforensics-operator</a>, an Airflow operator that wraps the CLI and acts on the result after each Spark job, so you don&#39;t have to wire up the call yourself.</p><p>Want an AI assistant to diagnose a run directly, without the dashboard or a CI gate? See <a href="./mcp-tools.html">MCP tools reference</a>.</p></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><!----></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="/docs/user-guide/understanding-findings.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>Understanding findings</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
28
+ <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"9ToZXcTk\",\"contributor-guide_architecture_detector-contract.md\":\"CsCytCho\",\"contributor-guide_architecture_drill-down.md\":\"Bawx6NgP\",\"contributor-guide_architecture_impact-estimation.md\":\"sf9DU2bA\",\"contributor-guide_architecture_index.md\":\"3TO9ic6w\",\"contributor-guide_architecture_overview.md\":\"rhfxoGsI\",\"contributor-guide_architecture_state-and-history.md\":\"B8Wc_ZTg\",\"contributor-guide_architecture_widget-rendering.md\":\"DYwFeyjV\",\"contributor-guide_architecture_worker-protocol.md\":\"CBJkwH24\",\"contributor-guide_contributing.md\":\"B2E9ni79\",\"contributor-guide_development-setup.md\":\"s85j2l8b\",\"contributor-guide_testing.md\":\"Cnn6gz1C\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"Df1YMIHu\",\"tuning-reference_aqe.md\":\"BIsCtLzm\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"CEstB3Ia\",\"tuning-reference_bottleneck-cold-start.md\":\"CEuy-72y\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"CIohQDfn\",\"tuning-reference_bottleneck-failures.md\":\"CAsFf3RR\",\"tuning-reference_bottleneck-gc.md\":\"B4T3kJYV\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"BaJl__1W\",\"tuning-reference_bottleneck-memory-utilization.md\":\"DbP-SJZc\",\"tuning-reference_bottleneck-retry-waste.md\":\"D5JMjVOt\",\"tuning-reference_bottleneck-shuffle.md\":\"DDMQVOW6\",\"tuning-reference_bottleneck-skew.md\":\"B0MzhWcX\",\"tuning-reference_bottleneck-slow-host.md\":\"BlIo6UDW\",\"tuning-reference_bottleneck-small-files.md\":\"DHGMTmPs\",\"tuning-reference_bottleneck-spill.md\":\"PNH7mITt\",\"tuning-reference_bottleneck-straggler.md\":\"DIbXdAcQ\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"CB05v3zh\",\"tuning-reference_bottleneck-utilization.md\":\"DqC_iGkw\",\"tuning-reference_caching.md\":\"B7aQ8asB\",\"tuning-reference_cluster-config.md\":\"CD6vzYtD\",\"tuning-reference_config.md\":\"UvveiWG3\",\"tuning-reference_data-formats.md\":\"bjCAWH3N\",\"tuning-reference_index.md\":\"BQ_NooMV\",\"tuning-reference_intro.md\":\"CobD-lGB\",\"tuning-reference_joins.md\":\"BtKs_CuW\",\"tuning-reference_memory-model.md\":\"CWhvgevP\",\"tuning-reference_metrics.md\":\"mLOh7Apj\",\"tuning-reference_partitioning.md\":\"q0zKF_8X\",\"tuning-reference_pyspark.md\":\"DDCfvN9t\",\"tuning-reference_shuffle.md\":\"BZZ7R4Ix\",\"tuning-reference_spark-architecture.md\":\"Dwzm5avO\",\"tuning-reference_table-formats.md\":\"D6wj-2dX\",\"user-guide_alternative-log-retrieval.md\":\"ZHPSbBkF\",\"user-guide_getting-started.md\":\"Cpe0U-eO\",\"user-guide_mcp-tools.md\":\"DrphooJv\",\"user-guide_run-comparison.md\":\"CQc1aoU8\",\"user-guide_understanding-findings.md\":\"YBBLOwWi\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Alternative log retrieval\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
29
29
 
30
30
  </body>
31
31
  </html>