sparkforensics-server 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (167) hide show
  1. package/index.js +147 -0
  2. package/lib/static-files.js +50 -0
  3. package/package.json +28 -0
  4. package/public/assets/AutoscalingChurn-CP5OjN6v.js +1 -0
  5. package/public/assets/CacheUtilization-BjyawXL6.js +1 -0
  6. package/public/assets/CachingOpportunity-DV2ABG2G.js +1 -0
  7. package/public/assets/ConfigAudit-DMiBhAoG.js +1 -0
  8. package/public/assets/CoreUsageArea-CGYIgPnt.js +1 -0
  9. package/public/assets/Dashboard-D1_IrWRs.js +12 -0
  10. package/public/assets/ExecutorTimeline-DkaAGqQT.js +1 -0
  11. package/public/assets/Failures-Crngoicu.js +1 -0
  12. package/public/assets/GcPressure-CFXm4BnA.js +1 -0
  13. package/public/assets/IncompleteRun-BWEJTb13.js +1 -0
  14. package/public/assets/JobFailures-BmjxCA-h.js +1 -0
  15. package/public/assets/MemoryUtilization-Bu2uQJbd.js +1 -0
  16. package/public/assets/PlanExplorer-DKUwF0uz.js +1 -0
  17. package/public/assets/PlanGraphRoute-BjPSugIG.js +1 -0
  18. package/public/assets/ShuffleIO--Rur3neu.js +1 -0
  19. package/public/assets/Spill-DIMpt8c2.js +1 -0
  20. package/public/assets/TaskSkew-BDr1DbeI.js +1 -0
  21. package/public/assets/charts-vendor-NMqKV0H4.js +59 -0
  22. package/public/assets/index-4id52G-A.js +9 -0
  23. package/public/assets/index-D9kwVYIO.css +1 -0
  24. package/public/assets/parser-worker-00O9WEQC.js +65 -0
  25. package/public/assets/plan-graph-vendor-BnuhLJ6X.css +1 -0
  26. package/public/assets/plan-graph-vendor-vZQHG2EJ.js +7 -0
  27. package/public/assets/react-vendor-u4pyvC5Y.js +9 -0
  28. package/public/assets/useRecentFiles-46iBU9zJ.js +5 -0
  29. package/public/docs/404.html +26 -0
  30. package/public/docs/assets/app.Bxy4_HVA.js +1 -0
  31. package/public/docs/assets/chunks/@localSearchIndexroot.DclgKtlT.js +1 -0
  32. package/public/docs/assets/chunks/VPLocalSearchBox.L5jNOIR3.js +9 -0
  33. package/public/docs/assets/chunks/framework.CNDaxUod.js +20 -0
  34. package/public/docs/assets/chunks/theme.CoT6y5bf.js +2 -0
  35. package/public/docs/assets/contributor-guide_architecture_board-widgets.md.BtjoE_1k.js +1 -0
  36. package/public/docs/assets/contributor-guide_architecture_board-widgets.md.BtjoE_1k.lean.js +1 -0
  37. package/public/docs/assets/contributor-guide_architecture_detector-contract.md.DQyCHQNM.js +1 -0
  38. package/public/docs/assets/contributor-guide_architecture_detector-contract.md.DQyCHQNM.lean.js +1 -0
  39. package/public/docs/assets/contributor-guide_architecture_drill-down.md.BKfyDQck.js +1 -0
  40. package/public/docs/assets/contributor-guide_architecture_drill-down.md.BKfyDQck.lean.js +1 -0
  41. package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.CzlXOnCq.js +1 -0
  42. package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.CzlXOnCq.lean.js +1 -0
  43. package/public/docs/assets/contributor-guide_architecture_index.md.94lNbF-4.js +1 -0
  44. package/public/docs/assets/contributor-guide_architecture_index.md.94lNbF-4.lean.js +1 -0
  45. package/public/docs/assets/contributor-guide_architecture_overview.md.mQFlN66o.js +1 -0
  46. package/public/docs/assets/contributor-guide_architecture_overview.md.mQFlN66o.lean.js +1 -0
  47. package/public/docs/assets/contributor-guide_architecture_state-and-history.md.DaqttKgQ.js +1 -0
  48. package/public/docs/assets/contributor-guide_architecture_state-and-history.md.DaqttKgQ.lean.js +1 -0
  49. package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.Bk6Iyj9S.js +1 -0
  50. package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.Bk6Iyj9S.lean.js +1 -0
  51. package/public/docs/assets/contributor-guide_architecture_worker-protocol.md.BaS_L10g.js +6 -0
  52. package/public/docs/assets/contributor-guide_architecture_worker-protocol.md.BaS_L10g.lean.js +1 -0
  53. package/public/docs/assets/contributor-guide_contributing.md.C-lKs70I.js +1 -0
  54. package/public/docs/assets/contributor-guide_contributing.md.C-lKs70I.lean.js +1 -0
  55. package/public/docs/assets/contributor-guide_development-setup.md.CpUMVtIK.js +12 -0
  56. package/public/docs/assets/contributor-guide_development-setup.md.CpUMVtIK.lean.js +1 -0
  57. package/public/docs/assets/contributor-guide_testing.md.C0q-BJfl.js +1 -0
  58. package/public/docs/assets/contributor-guide_testing.md.C0q-BJfl.lean.js +1 -0
  59. package/public/docs/assets/index.md.CETnDZzR.js +1 -0
  60. package/public/docs/assets/index.md.CETnDZzR.lean.js +1 -0
  61. package/public/docs/assets/inter-italic-cyrillic-ext.r48I6akx.woff2 +0 -0
  62. package/public/docs/assets/inter-italic-cyrillic.By2_1cv3.woff2 +0 -0
  63. package/public/docs/assets/inter-italic-greek-ext.1u6EdAuj.woff2 +0 -0
  64. package/public/docs/assets/inter-italic-greek.DJ8dCoTZ.woff2 +0 -0
  65. package/public/docs/assets/inter-italic-latin-ext.CN1xVJS-.woff2 +0 -0
  66. package/public/docs/assets/inter-italic-latin.C2AdPX0b.woff2 +0 -0
  67. package/public/docs/assets/inter-italic-vietnamese.BSbpV94h.woff2 +0 -0
  68. package/public/docs/assets/inter-roman-cyrillic-ext.BBPuwvHQ.woff2 +0 -0
  69. package/public/docs/assets/inter-roman-cyrillic.C5lxZ8CY.woff2 +0 -0
  70. package/public/docs/assets/inter-roman-greek-ext.CqjqNYQ-.woff2 +0 -0
  71. package/public/docs/assets/inter-roman-greek.BBVDIX6e.woff2 +0 -0
  72. package/public/docs/assets/inter-roman-latin-ext.4ZJIpNVo.woff2 +0 -0
  73. package/public/docs/assets/inter-roman-latin.Di8DUHzh.woff2 +0 -0
  74. package/public/docs/assets/inter-roman-vietnamese.BjW4sHH5.woff2 +0 -0
  75. package/public/docs/assets/style.D2mY0HUs.css +1 -0
  76. package/public/docs/assets/user-guide_getting-started.md.BJdD7a7_.js +3 -0
  77. package/public/docs/assets/user-guide_getting-started.md.BJdD7a7_.lean.js +1 -0
  78. package/public/docs/assets/user-guide_mcp-tools.md.DwYI1d6C.js +95 -0
  79. package/public/docs/assets/user-guide_mcp-tools.md.DwYI1d6C.lean.js +1 -0
  80. package/public/docs/assets/user-guide_run-comparison.md.5-oaOfTu.js +1 -0
  81. package/public/docs/assets/user-guide_run-comparison.md.5-oaOfTu.lean.js +1 -0
  82. package/public/docs/assets/user-guide_understanding-findings.md.CK2WZ0Tf.js +1 -0
  83. package/public/docs/assets/user-guide_understanding-findings.md.CK2WZ0Tf.lean.js +1 -0
  84. package/public/docs/contributor-guide/architecture/board-widgets.html +29 -0
  85. package/public/docs/contributor-guide/architecture/detector-contract.html +29 -0
  86. package/public/docs/contributor-guide/architecture/drill-down.html +29 -0
  87. package/public/docs/contributor-guide/architecture/impact-estimation.html +29 -0
  88. package/public/docs/contributor-guide/architecture/index.html +29 -0
  89. package/public/docs/contributor-guide/architecture/overview.html +29 -0
  90. package/public/docs/contributor-guide/architecture/state-and-history.html +29 -0
  91. package/public/docs/contributor-guide/architecture/widget-rendering.html +29 -0
  92. package/public/docs/contributor-guide/architecture/worker-protocol.html +34 -0
  93. package/public/docs/contributor-guide/contributing.html +29 -0
  94. package/public/docs/contributor-guide/development-setup.html +40 -0
  95. package/public/docs/contributor-guide/testing.html +29 -0
  96. package/public/docs/favicon.svg +4 -0
  97. package/public/docs/hashmap.json +1 -0
  98. package/public/docs/index.html +29 -0
  99. package/public/docs/user-guide/getting-started.html +31 -0
  100. package/public/docs/user-guide/mcp-tools.html +123 -0
  101. package/public/docs/user-guide/run-comparison.html +29 -0
  102. package/public/docs/user-guide/understanding-findings.html +29 -0
  103. package/public/docs/vp-icons.css +0 -0
  104. package/public/favicon.svg +4 -0
  105. package/public/index.html +36 -0
  106. package/public/vendor/spark-doc/anchors.json +205 -0
  107. package/public/vendor/spark-doc/index.html +4202 -0
  108. package/public/vendor/spark-doc/landing.html +418 -0
  109. package/public/vendor/spark-doc/meta.html +970 -0
  110. package/vendor-core/analyzer.js +167 -0
  111. package/vendor-core/assert-never.js +3 -0
  112. package/vendor-core/cli/budgets.js +203 -0
  113. package/vendor-core/cli/collect-run.js +107 -0
  114. package/vendor-core/core-count.js +68 -0
  115. package/vendor-core/core-locality-ratio.js +54 -0
  116. package/vendor-core/core-time-series.js +92 -0
  117. package/vendor-core/core-usage-locality.js +70 -0
  118. package/vendor-core/detectors.js +1989 -0
  119. package/vendor-core/docs-config.js +72 -0
  120. package/vendor-core/docs-site-config.js +23 -0
  121. package/vendor-core/efficiency-model.js +62 -0
  122. package/vendor-core/etl-phases.js +28 -0
  123. package/vendor-core/event-handlers.js +906 -0
  124. package/vendor-core/event-schemas.js +405 -0
  125. package/vendor-core/evidence-availability.js +121 -0
  126. package/vendor-core/evidence-report.js +459 -0
  127. package/vendor-core/finding-action-label.js +97 -0
  128. package/vendor-core/finding-filter-predicate.js +38 -0
  129. package/vendor-core/format-utils.js +167 -0
  130. package/vendor-core/impact-band.js +50 -0
  131. package/vendor-core/impact-estimator.js +428 -0
  132. package/vendor-core/ingest.js +139 -0
  133. package/vendor-core/job-groups.js +30 -0
  134. package/vendor-core/load-vendored.js +24 -0
  135. package/vendor-core/lz4-block.js +135 -0
  136. package/vendor-core/mcp-error.js +3 -0
  137. package/vendor-core/mcp-server-factory.js +115 -0
  138. package/vendor-core/mcp-tools.js +331 -0
  139. package/vendor-core/model-assembler.js +76 -0
  140. package/vendor-core/occupancy.js +202 -0
  141. package/vendor-core/parser-worker.js +249 -0
  142. package/vendor-core/plan-dot.js +25 -0
  143. package/vendor-core/plan-duration-attribution.js +185 -0
  144. package/vendor-core/plan-graph-model.js +171 -0
  145. package/vendor-core/plan-node-detail.js +159 -0
  146. package/vendor-core/plan-summary.js +233 -0
  147. package/vendor-core/plan-tree-walk.js +29 -0
  148. package/vendor-core/proxy.js +157 -0
  149. package/vendor-core/recommendation-rollup.js +197 -0
  150. package/vendor-core/redact.js +175 -0
  151. package/vendor-core/rolling-log-reassembly.js +52 -0
  152. package/vendor-core/run-aggregates.js +44 -0
  153. package/vendor-core/run-comparison.js +458 -0
  154. package/vendor-core/scaling-sim.js +73 -0
  155. package/vendor-core/session-snapshot.js +79 -0
  156. package/vendor-core/shs-fetch.js +196 -0
  157. package/vendor-core/shs-load.js +121 -0
  158. package/vendor-core/shs-request.js +101 -0
  159. package/vendor-core/shs-schemas.js +13 -0
  160. package/vendor-core/snappy-block.js +140 -0
  161. package/vendor-core/stage-quantiles.js +199 -0
  162. package/vendor-core/threshold-summary.js +35 -0
  163. package/vendor-core/types.js +286 -0
  164. package/vendor-core/vendor/fflate.js +2695 -0
  165. package/vendor-core/vendor/fzstd.js +768 -0
  166. package/vendor-core/wall-clock.js +36 -0
  167. package/vendor-core/wasted-core-hours.js +68 -0
package/public/docs/user-guide/run-comparison.html ADDED
@@ -0,0 +1,29 @@
1
+ <!DOCTYPE html>
2
+ <html lang="en-US" dir="ltr">
3
+ <head>
4
+ <meta charset="utf-8">
5
+ <meta name="viewport" content="width=device-width,initial-scale=1">
6
+ <title>Run comparison mode | SparkForensics</title>
7
+ <meta name="description" content="Docs for using and contributing to SparkForensics">
8
+ <meta name="generator" content="VitePress v1.6.4">
9
+ <link rel="preload stylesheet" href="/docs/assets/style.D2mY0HUs.css" as="style">
10
+ <link rel="preload stylesheet" href="/docs/vp-icons.css" as="style">
11
+
12
+ <script type="module" src="/docs/assets/app.Bxy4_HVA.js"></script>
13
+ <link rel="preload" href="/docs/assets/inter-roman-latin.Di8DUHzh.woff2" as="font" type="font/woff2" crossorigin="">
14
+ <link rel="modulepreload" href="/docs/assets/chunks/theme.CoT6y5bf.js">
15
+ <link rel="modulepreload" href="/docs/assets/chunks/framework.CNDaxUod.js">
16
+ <link rel="modulepreload" href="/docs/assets/user-guide_run-comparison.md.5-oaOfTu.lean.js">
17
+ <link rel="icon" type="image/svg+xml" href="/docs/favicon.svg">
18
+ <link rel="preconnect" href="https://fonts.googleapis.com">
19
+ <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin="">
20
+ <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Recursive:wght,CASL@400..700,0..1&amp;family=JetBrains+Mono:wght@400;500;600&amp;display=swap">
21
+ <script id="check-dark-mode">(()=>{const e=localStorage.getItem("vitepress-theme-appearance")||"auto",a=window.matchMedia("(prefers-color-scheme: dark)").matches;(!e||e==="auto"?a:e==="dark")&&document.documentElement.classList.add("dark")})();</script>
22
+ <script id="check-mac-os">document.documentElement.classList.toggle("mac",/Mac|iPhone|iPod|iPad/i.test(navigator.platform));</script>
23
+ </head>
24
+ <body>
25
+ <div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>User Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/getting-started.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Getting started</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/understanding-findings.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Understanding findings</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/run-comparison.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Run comparison mode</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/mcp-tools.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>MCP tools reference</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_user-guide_run-comparison" data-v-39a288b8><div><h1 id="run-comparison-mode" tabindex="-1">Run comparison mode <a class="header-anchor" href="#run-comparison-mode" aria-label="Permalink to &quot;Run comparison mode&quot;">​</a></h1><p>Put two runs side by side to see whether a tuning change helped.</p><h2 id="starting-a-comparison" tabindex="-1">Starting a comparison <a class="header-anchor" href="#starting-a-comparison" aria-label="Permalink to &quot;Starting a comparison&quot;">​</a></h2><p>From the landing page, click <strong>Compare two runs</strong>. Two slots appear, <strong>Run A</strong> and <strong>Run B</strong>.</p><ol><li>Load a file into each slot the same way you&#39;d load a single run (drag and drop, or <strong>Choose file</strong>).</li><li>Click <strong>Compare</strong>.</li></ol><p>The runs parse one after the other through the same background worker. When both finish, the comparison view opens.</p><h2 id="reading-the-comparison" tabindex="-1">Reading the comparison <a class="header-anchor" href="#reading-the-comparison" aria-label="Permalink to &quot;Reading the comparison&quot;">​</a></h2><p>The comparison page shows what changed between the two runs: findings that appeared or disappeared, plus metric deltas for duration, spill, GC time and so on.</p><p>For either run&#39;s full dashboard, click <strong>View run A dashboard</strong> or <strong>View run B dashboard</strong>. <strong>← Back to comparison</strong> takes you back.</p></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/user-guide/understanding-findings.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>Understanding findings</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="/docs/user-guide/mcp-tools.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>MCP tools reference</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
26
+ <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"BtjoE_1k\",\"contributor-guide_architecture_detector-contract.md\":\"DQyCHQNM\",\"contributor-guide_architecture_drill-down.md\":\"BKfyDQck\",\"contributor-guide_architecture_impact-estimation.md\":\"CzlXOnCq\",\"contributor-guide_architecture_index.md\":\"94lNbF-4\",\"contributor-guide_architecture_overview.md\":\"mQFlN66o\",\"contributor-guide_architecture_state-and-history.md\":\"DaqttKgQ\",\"contributor-guide_architecture_widget-rendering.md\":\"Bk6Iyj9S\",\"contributor-guide_architecture_worker-protocol.md\":\"BaS_L10g\",\"contributor-guide_contributing.md\":\"C-lKs70I\",\"contributor-guide_development-setup.md\":\"CpUMVtIK\",\"contributor-guide_testing.md\":\"C0q-BJfl\",\"index.md\":\"CETnDZzR\",\"user-guide_getting-started.md\":\"BJdD7a7_\",\"user-guide_mcp-tools.md\":\"DwYI1d6C\",\"user-guide_run-comparison.md\":\"5-oaOfTu\",\"user-guide_understanding-findings.md\":\"CK2WZ0Tf\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"}],\"sidebar\":{\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
27
+
28
+ </body>
29
+ </html>
package/public/docs/user-guide/understanding-findings.html ADDED
@@ -0,0 +1,29 @@
1
+ <!DOCTYPE html>
2
+ <html lang="en-US" dir="ltr">
3
+ <head>
4
+ <meta charset="utf-8">
5
+ <meta name="viewport" content="width=device-width,initial-scale=1">
6
+ <title>Understanding findings | SparkForensics</title>
7
+ <meta name="description" content="Docs for using and contributing to SparkForensics">
8
+ <meta name="generator" content="VitePress v1.6.4">
9
+ <link rel="preload stylesheet" href="/docs/assets/style.D2mY0HUs.css" as="style">
10
+ <link rel="preload stylesheet" href="/docs/vp-icons.css" as="style">
11
+
12
+ <script type="module" src="/docs/assets/app.Bxy4_HVA.js"></script>
13
+ <link rel="preload" href="/docs/assets/inter-roman-latin.Di8DUHzh.woff2" as="font" type="font/woff2" crossorigin="">
14
+ <link rel="modulepreload" href="/docs/assets/chunks/theme.CoT6y5bf.js">
15
+ <link rel="modulepreload" href="/docs/assets/chunks/framework.CNDaxUod.js">
16
+ <link rel="modulepreload" href="/docs/assets/user-guide_understanding-findings.md.CK2WZ0Tf.lean.js">
17
+ <link rel="icon" type="image/svg+xml" href="/docs/favicon.svg">
18
+ <link rel="preconnect" href="https://fonts.googleapis.com">
19
+ <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin="">
20
+ <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Recursive:wght,CASL@400..700,0..1&amp;family=JetBrains+Mono:wght@400;500;600&amp;display=swap">
21
+ <script id="check-dark-mode">(()=>{const e=localStorage.getItem("vitepress-theme-appearance")||"auto",a=window.matchMedia("(prefers-color-scheme: dark)").matches;(!e||e==="auto"?a:e==="dark")&&document.documentElement.classList.add("dark")})();</script>
22
+ <script id="check-mac-os">document.documentElement.classList.toggle("mac",/Mac|iPhone|iPod|iPad/i.test(navigator.platform));</script>
23
+ </head>
24
+ <body>
25
+ <div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>User Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/getting-started.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Getting started</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/understanding-findings.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Understanding findings</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/run-comparison.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Run comparison mode</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/mcp-tools.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>MCP tools reference</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_user-guide_understanding-findings" data-v-39a288b8><div><h1 id="understanding-findings" tabindex="-1">Understanding findings <a class="header-anchor" href="#understanding-findings" aria-label="Permalink to &quot;Understanding findings&quot;">​</a></h1><p>Every flagged problem carries a short ALL-CAPS tag. This page has one entry per tag: what it means, and what to do about it.</p><p>A few tags share their in-app &quot;Reference panel&quot; background reading with another tag, because the underlying Spark-tuning material overlaps: <code>SFAIL</code> with <code>FAIL</code>, <code>PART</code> with <code>SHFL</code>, <code>SPEC</code> with <code>STRAG</code>, and <code>CACHE</code>/<code>LOCAL</code> with <code>UTIL</code>.</p><h2 id="per-stage" tabindex="-1">Per-stage <a class="header-anchor" href="#per-stage" aria-label="Permalink to &quot;Per-stage&quot;">​</a></h2><h3 id="skew" tabindex="-1"><code>SKEW</code>: Task skew <a class="header-anchor" href="#skew" aria-label="Permalink to &quot;`SKEW`: Task skew {#skew}&quot;">​</a></h3><p>A small number of tasks take much longer than their peers in the same stage. For join-driven skew, enable AQE skew-join handling (<code>spark.sql.adaptive.skewJoin.enabled</code>); otherwise salt the key or repartition on a better key.</p><h3 id="shfl" tabindex="-1"><code>SHFL</code>: Shuffle I/O <a class="header-anchor" href="#shfl" aria-label="Permalink to &quot;`SHFL`: Shuffle I/O {#shfl}&quot;">​</a></h3><p>Tasks move a large amount of intermediate data between stages. Raise <code>spark.sql.shuffle.partitions</code>, or add a broadcast join.</p><h3 id="spill" tabindex="-1"><code>SPILL</code>: Memory and disk spill <a class="header-anchor" href="#spill" aria-label="Permalink to &quot;`SPILL`: Memory and disk spill {#spill}&quot;">​</a></h3><p>Tasks are writing data out of memory, which slows execution. Two spill patterns get flagged differently: skew spill, where a few heavy tasks spill while most don&#39;t (rebalance partitioning), and volume spill, where most tasks spill because the data genuinely exceeds available memory (add partitions).</p><h3 id="gc" tabindex="-1"><code>GC</code>: Garbage collection pressure <a class="header-anchor" href="#gc" aria-label="Permalink to &quot;`GC`: Garbage collection pressure {#gc}&quot;">​</a></h3><p>Tasks spend an unusually large share of time reclaiming memory. Reduce object creation: use primitive types, avoid UDFs, or raise executor memory.</p><h3 id="fail" tabindex="-1"><code>FAIL</code>: Failed tasks <a class="header-anchor" href="#fail" aria-label="Permalink to &quot;`FAIL`: Failed tasks {#fail}&quot;">​</a></h3><p>Tasks fail often enough to affect the stage. Failed tasks point to executor instability or data-driven errors: check driver logs for the dominant failure reason.</p><h3 id="sfail" tabindex="-1"><code>SFAIL</code>: Failed stage <a class="header-anchor" href="#sfail" aria-label="Permalink to &quot;`SFAIL`: Failed stage {#sfail}&quot;">​</a></h3><p>A stage attempt failed outright rather than losing individual tasks within it. Inspect the driver log for the failure reason and the job that triggered it.</p><h3 id="strag" tabindex="-1"><code>STRAG</code>: Straggler tasks <a class="header-anchor" href="#strag" aria-label="Permalink to &quot;`STRAG`: Straggler tasks {#strag}&quot;">​</a></h3><p>A few tasks run much slower than the rest of their stage. Likely causes: data skew that AQE&#39;s skew-join handling would catch, or data-locality issues.</p><h3 id="spec" tabindex="-1"><code>SPEC</code>: Speculation waste <a class="header-anchor" href="#spec" aria-label="Permalink to &quot;`SPEC`: Speculation waste {#spec}&quot;">​</a></h3><p>Speculative task attempts used a lot of executor time without confirming a genuine straggler. Self-flagged low-confidence: a design spike, not yet validated against real-world runs. If task durations are just naturally variable rather than genuine stragglers, tune <code>spark.speculation.multiplier</code>/<code>spark.speculation.quantile</code>.</p><h3 id="retry" tabindex="-1"><code>RETRY</code>: Retry waste <a class="header-anchor" href="#retry" aria-label="Permalink to &quot;`RETRY`: Retry waste {#retry}&quot;">​</a></h3><p>Repeated task attempts ate into execution time even though the stage completed. Investigate executor loss or fetch failures.</p><h3 id="tiny" tabindex="-1"><code>TINY</code>: Tiny tasks <a class="header-anchor" href="#tiny" aria-label="Permalink to &quot;`TINY`: Tiny tasks {#tiny}&quot;">​</a></h3><p>Many very short tasks add scheduling overhead out of proportion to the work each one does. Repartition to fewer, larger tasks.</p><h3 id="part" tabindex="-1"><code>PART</code>: Partition sizing <a class="header-anchor" href="#part" aria-label="Permalink to &quot;`PART`: Partition sizing {#part}&quot;">​</a></h3><p>Shuffle partitions are too large, too uneven, or too few for the work. A single shuffle partition over 5 GB, for example, will OOM or spill heavily: repartition to break it up before the stage runs.</p><h3 id="slow" tabindex="-1"><code>SLOW</code>: Stage slowness <a class="header-anchor" href="#slow" aria-label="Permalink to &quot;`SLOW`: Stage slowness {#slow}&quot;">​</a></h3><p>A stage is slow overall (high average executor runtime) without a more specific cause getting flagged. Profile its query plan and check for wide shuffles or expensive UDFs.</p><h3 id="shape" tabindex="-1"><code>SHAPE</code>: Stage shape <a class="header-anchor" href="#shape" aria-label="Permalink to &quot;`SHAPE`: Stage shape {#shape}&quot;">​</a></h3><p>The stage has an inefficient task count, output shape, or task-to-stage balance: for example, one straggler task taking a large fraction of the stage&#39;s wall-clock time.</p><h3 id="host" tabindex="-1"><code>HOST</code>: Slow host <a class="header-anchor" href="#host" aria-label="Permalink to &quot;`HOST`: Slow host {#host}&quot;">​</a></h3><p>One executor is much slower than its peers. Check that executor&#39;s logs: possible bad node, disk pressure, or NUMA misalignment.</p><h2 id="app-level" tabindex="-1">App-level <a class="header-anchor" href="#app-level" aria-label="Permalink to &quot;App-level&quot;">​</a></h2><h3 id="cold" tabindex="-1"><code>COLD</code>: Executor cold start <a class="header-anchor" href="#cold" aria-label="Permalink to &quot;`COLD`: Executor cold start {#cold}&quot;">​</a></h3><p>New executors take time to become available for work. Pre-warm the cluster, or use dynamic allocation.</p><h3 id="util" tabindex="-1"><code>UTIL</code>: Low utilization <a class="header-anchor" href="#util" aria-label="Permalink to &quot;`UTIL`: Low utilization {#util}&quot;">​</a></h3><p>Allocated executors sit idle for a large share of the application run. Consider a smaller cluster, or enable dynamic allocation.</p><h3 id="mem" tabindex="-1"><code>MEM</code>: Memory utilization <a class="header-anchor" href="#mem" aria-label="Permalink to &quot;`MEM`: Memory utilization {#mem}&quot;">​</a></h3><p>Executor memory or core capacity may be over- or under-provisioned. Some detail here needs <code>spark.eventLog.logStageExecutorMetrics=true</code> on the run being analyzed; without it, per-executor memory usage can&#39;t be broken down. Review <code>spark.executor.memory</code> and executor count if allocated memory sat largely idle over the run. That idle-memory variant is self-flagged low-confidence: it estimates waste from allocated-versus-used memory-time against an unverified 1.5x buffer ported from Dr. Elephant. Check it against the Spark UI before resizing anything.</p><h3 id="cache" tabindex="-1"><code>CACHE</code>: Caching opportunity <a class="header-anchor" href="#cache" aria-label="Permalink to &quot;`CACHE`: Caching opportunity {#cache}&quot;">​</a></h3><p>A reusable dataset (re-read via the same SQL relation more than once) may be worth persisting between stages. Self-flagged low-confidence: reuse is only inferred, from plan-scan identity across SQL executions, so confirm the reads really do hit the same data before you cache anything.</p><h3 id="cstor" tabindex="-1"><code>CSTOR</code>: Cache storage <a class="header-anchor" href="#cstor" aria-label="Permalink to &quot;`CSTOR`: Cache storage {#cstor}&quot;">​</a></h3><p>A persisted dataset is not fully cached in memory, or is spilling to disk. Raise executor memory, or shrink the cached dataset.</p><h3 id="local" tabindex="-1"><code>LOCAL</code>: Core usage locality <a class="header-anchor" href="#local" aria-label="Permalink to &quot;`LOCAL`: Core usage locality {#local}&quot;">​</a></h3><p>Tasks run without process- or node-local data placement more often than expected. Check <code>spark.locality.wait</code> settings and executor/data colocation. Self-flagged low-confidence: the non-local-ratio thresholds are unvalidated design-spike values, and no external tool publishes an equivalent metric to calibrate them against.</p><h3 id="chrn" tabindex="-1"><code>CHRN</code>: Autoscaling churn <a class="header-anchor" href="#chrn" aria-label="Permalink to &quot;`CHRN`: Autoscaling churn {#chrn}&quot;">​</a></h3><p>Executors are stood up and torn down again before they can do useful work: re-provisioning churn rather than normal scale-down. Raise <code>spark.dynamicAllocation.executorIdleTimeout</code>, or widen the <code>minExecutors</code>/<code>maxExecutors</code> bounds to reduce flapping. Self-flagged low-confidence: a design spike, not yet validated against real-world runs.</p><h3 id="jobs" tabindex="-1"><code>JOBS</code>: Job failure rate <a class="header-anchor" href="#jobs" aria-label="Permalink to &quot;`JOBS`: Job failure rate {#jobs}&quot;">​</a></h3><p>A large share of completed jobs did not succeed. Inspect the driver log for the failed job(s) and the stage failures that triggered them.</p><h3 id="incmp" tabindex="-1"><code>INCMP</code>: Incomplete run <a class="header-anchor" href="#incmp" aria-label="Permalink to &quot;`INCMP`: Incomplete run {#incmp}&quot;">​</a></h3><p>This event log never recorded an <code>ApplicationEnd</code> event: capture stopped before the run finished (an in-flight job, a rotated-away log, or a cut-short capture). Every other finding and metric on the board reflects only what was captured up to that point, not the full run.</p><h2 id="configuration-scope" tabindex="-1">Configuration scope <a class="header-anchor" href="#configuration-scope" aria-label="Permalink to &quot;Configuration scope&quot;">​</a></h2><h3 id="cfg" tabindex="-1"><code>CFG</code>: Configuration audit <a class="header-anchor" href="#cfg" aria-label="Permalink to &quot;`CFG`: Configuration audit {#cfg}&quot;">​</a></h3><p>Flags configuration settings that may cause reliability or efficiency problems, independent of any one stage&#39;s behavior. Four properties are audited today:</p><ul><li><code>spark.shuffle.service.enabled</code>: flagged when dynamic allocation is on but the external shuffle service is off, since shuffle data won&#39;t survive executor removal.</li><li><code>spark.dynamicAllocation.maxExecutors</code>: flagged for inverted bounds or a missing upper bound.</li><li><code>spark.serializer</code>: flagged when still on the default Java serializer; <code>org.apache.spark.serializer.KryoSerializer</code> is faster and produces smaller buffers.</li><li><code>spark.executor.memoryOverhead</code>: flagged when set below a safe floor.</li></ul><h2 id="sql-scope" tabindex="-1">SQL scope <a class="header-anchor" href="#sql-scope" aria-label="Permalink to &quot;SQL scope&quot;">​</a></h2><h3 id="plan" tabindex="-1"><code>PLAN</code>: Plan advisor <a class="header-anchor" href="#plan" aria-label="Permalink to &quot;`PLAN`: Plan advisor {#plan}&quot;">​</a></h3><p>Flags patterns in the SQL execution plan worth reviewing. Four checks share this tag:</p><ul><li>Duplicate plan subtree: the same subtree recomputed more than once in the plan.</li><li>Small files: reading an excessive number of small files.</li><li>Under-broadcast: the smaller side of a Sort Merge Join looks well under the broadcast threshold; consider a <code>broadcast()</code> hint or raising <code>spark.sql.autoBroadcastJoinThreshold</code>.</li><li>Over-broadcast: a broadcast exceeds the 1 GB threshold; check for a misapplied broadcast hint or a misconfigured <code>spark.sql.autoBroadcastJoinThreshold</code>.</li></ul></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/user-guide/getting-started.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>Getting started</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="/docs/user-guide/run-comparison.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>Run comparison mode</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
26
+ <script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"BtjoE_1k\",\"contributor-guide_architecture_detector-contract.md\":\"DQyCHQNM\",\"contributor-guide_architecture_drill-down.md\":\"BKfyDQck\",\"contributor-guide_architecture_impact-estimation.md\":\"CzlXOnCq\",\"contributor-guide_architecture_index.md\":\"94lNbF-4\",\"contributor-guide_architecture_overview.md\":\"mQFlN66o\",\"contributor-guide_architecture_state-and-history.md\":\"DaqttKgQ\",\"contributor-guide_architecture_widget-rendering.md\":\"Bk6Iyj9S\",\"contributor-guide_architecture_worker-protocol.md\":\"BaS_L10g\",\"contributor-guide_contributing.md\":\"C-lKs70I\",\"contributor-guide_development-setup.md\":\"CpUMVtIK\",\"contributor-guide_testing.md\":\"C0q-BJfl\",\"index.md\":\"CETnDZzR\",\"user-guide_getting-started.md\":\"BJdD7a7_\",\"user-guide_mcp-tools.md\":\"DwYI1d6C\",\"user-guide_run-comparison.md\":\"5-oaOfTu\",\"user-guide_understanding-findings.md\":\"CK2WZ0Tf\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"}],\"sidebar\":{\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
27
+
28
+ </body>
29
+ </html>
package/public/docs/vp-icons.css ADDED
File without changes
package/public/favicon.svg ADDED
@@ -0,0 +1,4 @@
1
+ <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 32 32" width="32" height="32">
2
+ <rect width="32" height="32" rx="7" fill="#181614"/>
3
+ <path d="M17.5 5 8 18h6.5l-1 9L24 14h-6.5l1-9z" fill="#ff6b2c"/>
4
+ </svg>
package/public/index.html ADDED
@@ -0,0 +1,36 @@
1
+ <!doctype html>
2
+ <html lang="en">
3
+ <head>
4
+ <meta charset="UTF-8" />
5
+ <meta name="viewport" content="width=device-width, initial-scale=1.0" />
6
+ <title>SparkForensics</title>
7
+ <meta name="description" content="SparkForensics is a zero-backend browser app that parses Spark History Server event logs to surface stage-level bottlenecks — skew, spill, GC, shuffle, cold start — without uploading cluster data anywhere.">
8
+ <link rel="icon" type="image/svg+xml" href="./favicon.svg">
9
+ <meta property="og:type" content="website">
10
+ <meta property="og:url" content="https://shuffle-works.github.io/sparkforensics/">
11
+ <meta property="og:title" content="SparkForensics">
12
+ <meta property="og:description" content="Zero-backend browser dashboard that parses Apache Spark event logs to diagnose performance bottlenecks: skew, spill, GC pressure, stragglers, cache utilization, and more.">
13
+ <link rel="preconnect" href="https://fonts.googleapis.com">
14
+ <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
15
+ <link href="https://fonts.googleapis.com/css2?family=Recursive:wght,CASL@400..700,0..1&family=JetBrains+Mono:wght@400;500;600&display=swap" rel="stylesheet">
16
+ <script>
17
+ try {
18
+ var stored = localStorage.getItem('shuffle-works-theme');
19
+ // Persisted choice wins; otherwise honor the OS preference. Mirrors
20
+ // initialTheme() in src/store/store.ts. Runs before paint to avoid a
21
+ // dark-then-light flash on prefers-light systems.
22
+ if (stored === 'light' || (stored == null && window.matchMedia('(prefers-color-scheme: light)').matches))
23
+ document.documentElement.setAttribute('data-theme', 'light');
24
+ } catch (_) {}
25
+ </script>
26
+ <script type="module" crossorigin src="./assets/index-4id52G-A.js"></script>
27
+ <link rel="modulepreload" crossorigin href="./assets/react-vendor-u4pyvC5Y.js">
28
+ <link rel="modulepreload" crossorigin href="./assets/plan-graph-vendor-vZQHG2EJ.js">
29
+ <link rel="modulepreload" crossorigin href="./assets/charts-vendor-NMqKV0H4.js">
30
+ <link rel="stylesheet" crossorigin href="./assets/plan-graph-vendor-BnuhLJ6X.css">
31
+ <link rel="stylesheet" crossorigin href="./assets/index-D9kwVYIO.css">
32
+ </head>
33
+ <body>
34
+ <div id="root"></div>
35
+ </body>
36
+ </html>
package/public/vendor/spark-doc/anchors.json ADDED
@@ -0,0 +1,205 @@
1
+ [
2
+ {
3
+ "anchor": "intro",
4
+ "section": "Getting Started",
5
+ "title": "Introduction",
6
+ "keywords": []
7
+ },
8
+ {
9
+ "anchor": "spark-architecture",
10
+ "section": "Optimization Guide",
11
+ "title": "Spark Execution Model",
12
+ "keywords": []
13
+ },
14
+ {
15
+ "anchor": "memory-model",
16
+ "section": "Optimization Guide",
17
+ "title": "Memory Management",
18
+ "keywords": []
19
+ },
20
+ {
21
+ "anchor": "partitioning",
22
+ "section": "Optimization Guide",
23
+ "title": "Partitioning",
24
+ "keywords": [
25
+ "repartition",
26
+ "coalesce",
27
+ "skew"
28
+ ]
29
+ },
30
+ {
31
+ "anchor": "joins",
32
+ "section": "Optimization Guide",
33
+ "title": "Join Optimization",
34
+ "keywords": [
35
+ "broadcast",
36
+ "shuffle-join",
37
+ "bucketing",
38
+ "aqe"
39
+ ]
40
+ },
41
+ {
42
+ "anchor": "shuffle",
43
+ "section": "Optimization Guide",
44
+ "title": "Shuffle",
45
+ "keywords": []
46
+ },
47
+ {
48
+ "anchor": "data-formats",
49
+ "section": "Optimization Guide",
50
+ "title": "Data Formats",
51
+ "keywords": []
52
+ },
53
+ {
54
+ "anchor": "table-formats",
55
+ "section": "Optimization Guide",
56
+ "title": "Table Formats",
57
+ "keywords": [
58
+ "delta",
59
+ "iceberg",
60
+ "hudi",
61
+ "compaction",
62
+ "optimize",
63
+ "z-order",
64
+ "deletion-vectors"
65
+ ]
66
+ },
67
+ {
68
+ "anchor": "caching",
69
+ "section": "Optimization Guide",
70
+ "title": "Caching & Persistence",
71
+ "keywords": []
72
+ },
73
+ {
74
+ "anchor": "pyspark",
75
+ "section": "Optimization Guide",
76
+ "title": "PySpark Specifics",
77
+ "keywords": []
78
+ },
79
+ {
80
+ "anchor": "aqe",
81
+ "section": "Optimization Guide",
82
+ "title": "Adaptive Query Execution",
83
+ "keywords": []
84
+ },
85
+ {
86
+ "anchor": "cluster-config",
87
+ "section": "Optimization Guide",
88
+ "title": "Cluster Tuning",
89
+ "keywords": []
90
+ },
91
+ {
92
+ "anchor": "anti-patterns",
93
+ "section": "Optimization Guide",
94
+ "title": "Anti-Patterns",
95
+ "keywords": []
96
+ },
97
+ {
98
+ "anchor": "bottleneck-skew",
99
+ "section": "Detector Catalog",
100
+ "title": "Task Skew",
101
+ "keywords": []
102
+ },
103
+ {
104
+ "anchor": "bottleneck-shuffle",
105
+ "section": "Detector Catalog",
106
+ "title": "Shuffle I/O",
107
+ "keywords": []
108
+ },
109
+ {
110
+ "anchor": "bottleneck-spill",
111
+ "section": "Detector Catalog",
112
+ "title": "Memory / Disk Spill",
113
+ "keywords": []
114
+ },
115
+ {
116
+ "anchor": "bottleneck-gc",
117
+ "section": "Detector Catalog",
118
+ "title": "GC Pressure",
119
+ "keywords": []
120
+ },
121
+ {
122
+ "anchor": "bottleneck-cold-start",
123
+ "section": "Detector Catalog",
124
+ "title": "Cold Start",
125
+ "keywords": []
126
+ },
127
+ {
128
+ "anchor": "bottleneck-utilization",
129
+ "section": "Detector Catalog",
130
+ "title": "Executor Utilization",
131
+ "keywords": []
132
+ },
133
+ {
134
+ "anchor": "bottleneck-slow-host",
135
+ "section": "Detector Catalog",
136
+ "title": "Slow Host",
137
+ "keywords": []
138
+ },
139
+ {
140
+ "anchor": "bottleneck-failures",
141
+ "section": "Detector Catalog",
142
+ "title": "Task Failures",
143
+ "keywords": []
144
+ },
145
+ {
146
+ "anchor": "bottleneck-straggler",
147
+ "section": "Detector Catalog",
148
+ "title": "Stragglers",
149
+ "keywords": []
150
+ },
151
+ {
152
+ "anchor": "bottleneck-retry-waste",
153
+ "section": "Detector Catalog",
154
+ "title": "Retry Waste",
155
+ "keywords": []
156
+ },
157
+ {
158
+ "anchor": "bottleneck-tiny-tasks",
159
+ "section": "Detector Catalog",
160
+ "title": "Tiny Tasks",
161
+ "keywords": []
162
+ },
163
+ {
164
+ "anchor": "bottleneck-job-failure-rate",
165
+ "section": "Detector Catalog",
166
+ "title": "Job Failure Rate",
167
+ "keywords": []
168
+ },
169
+ {
170
+ "anchor": "bottleneck-memory-utilization",
171
+ "section": "Detector Catalog",
172
+ "title": "Memory Utilization",
173
+ "keywords": []
174
+ },
175
+ {
176
+ "anchor": "bottleneck-duplicate-plan-subtree",
177
+ "section": "Detector Catalog",
178
+ "title": "Duplicate Plan Subtree",
179
+ "keywords": []
180
+ },
181
+ {
182
+ "anchor": "bottleneck-small-files",
183
+ "section": "Detector Catalog",
184
+ "title": "Small Files",
185
+ "keywords": []
186
+ },
187
+ {
188
+ "anchor": "bottleneck-broadcast-sizing",
189
+ "section": "Detector Catalog",
190
+ "title": "Broadcast Sizing",
191
+ "keywords": []
192
+ },
193
+ {
194
+ "anchor": "metrics",
195
+ "section": "Reference",
196
+ "title": "Metrics Glossary",
197
+ "keywords": []
198
+ },
199
+ {
200
+ "anchor": "config",
201
+ "section": "Reference",
202
+ "title": "Spark Config Quick-Reference",
203
+ "keywords": []
204
+ }
205
+ ]