sparkforensics-server 0.2.3 → 0.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/public/assets/{AutoscalingChurn-Bf5TN-s_.js → AutoscalingChurn-BOiUUfuY.js} +1 -1
- package/public/assets/{CacheUtilization-DRf_FNrK.js → CacheUtilization-DwX7lisd.js} +1 -1
- package/public/assets/{CachingOpportunity-BTJzIhtI.js → CachingOpportunity-DUE3GKpx.js} +1 -1
- package/public/assets/{ColdStart-DnFgnD3n.js → ColdStart-ZHxPrJmI.js} +1 -1
- package/public/assets/{ConfigAudit-B5FsOdmu.js → ConfigAudit-BkVnq2XE.js} +1 -1
- package/public/assets/{CoreUsageArea-CTksBPuq.js → CoreUsageArea-DobrSXk2.js} +1 -1
- package/public/assets/{Dashboard-BFMsMuc_.js → Dashboard-y3_9iQJo.js} +1 -1
- package/public/assets/{DuplicatePlanSubtree-BCt-Guuh.js → DuplicatePlanSubtree-BSQZFeNq.js} +1 -1
- package/public/assets/{DurationHistogram-xRcnND4k.js → DurationHistogram-DlQv4ZCB.js} +1 -1
- package/public/assets/{EmptyState-CnTrSjtU.js → EmptyState-DqnSxTjJ.js} +1 -1
- package/public/assets/{ExecutorUtilization-CP21XlUq.js → ExecutorUtilization-CuQUxaCd.js} +1 -1
- package/public/assets/{GcPressure-BoEJsLjb.js → GcPressure-CLT5hkMJ.js} +1 -1
- package/public/assets/{ImpactEstimate-BHcqJkcZ.js → ImpactEstimate-Df_laG-n.js} +1 -1
- package/public/assets/{IncompleteRun-b9FWrOHn.js → IncompleteRun-CpIkS4XZ.js} +1 -1
- package/public/assets/{JobFailures-C5FkTq3O.js → JobFailures-DKbvYc4J.js} +1 -1
- package/public/assets/{MemoryUtilization-D2W4AB_f.js → MemoryUtilization-C5kHmXIb.js} +1 -1
- package/public/assets/{OverBroadcast-Btu3xorf.js → OverBroadcast-C5pF66aq.js} +1 -1
- package/public/assets/{PartitionSizing-yZWhLN0e.js → PartitionSizing-BvEJQ5M1.js} +1 -1
- package/public/assets/{PlanExplorer-DSk1ivS4.js → PlanExplorer-DjGTZifP.js} +1 -1
- package/public/assets/{PlanGraphRoute-C3iOzUdB.js → PlanGraphRoute-DoIGA7_H.js} +1 -1
- package/public/assets/{RetryWaste-B4436kr1.js → RetryWaste-C0znHd_w.js} +1 -1
- package/public/assets/{RowPagination-oeWB8gBm.js → RowPagination-2icPidMP.js} +1 -1
- package/public/assets/{ShuffleIO-DUiv8EJe.js → ShuffleIO-L_XSnJ_V.js} +1 -1
- package/public/assets/{Skew-D2lB1oQc.js → Skew-w4uV5H_g.js} +1 -1
- package/public/assets/{SlowHost-Qb_8aa-W.js → SlowHost-D5tWJ1UR.js} +1 -1
- package/public/assets/{SmallFiles-u6suTgfj.js → SmallFiles-CTtNiSOh.js} +1 -1
- package/public/assets/{SpeculationWaste-vUCniiFP.js → SpeculationWaste-xCKwjz2K.js} +1 -1
- package/public/assets/{Spill-BskXFX2e.js → Spill-BT8my24K.js} +1 -1
- package/public/assets/{StageFailed-D9b83Jvy.js → StageFailed-De3vVDbc.js} +1 -1
- package/public/assets/{StageFindingGroup-DagqHOUz.js → StageFindingGroup-R4OW-g4U.js} +1 -1
- package/public/assets/{StageHeader-DibMSunO.js → StageHeader-CGeWqC-0.js} +1 -1
- package/public/assets/{StagePill-Dt6V9yVw.js → StagePill-BwwKLcFl.js} +1 -1
- package/public/assets/{StageShape-CfdrDUZa.js → StageShape-D4LmPVyV.js} +1 -1
- package/public/assets/{StageSlowness-BoGLIYKR.js → StageSlowness-8-z6qn4Q.js} +1 -1
- package/public/assets/{Straggler-DU1WSU-m.js → Straggler-BIgxsowI.js} +1 -1
- package/public/assets/{TaskFailures-DkY6zBNJ.js → TaskFailures-B9lLt6Rw.js} +1 -1
- package/public/assets/{TinyTask-Lsnt5mJF.js → TinyTask-DYXks5sT.js} +1 -1
- package/public/assets/{UnderBroadcast-DxHc0lwy.js → UnderBroadcast-BWqWMpeG.js} +1 -1
- package/public/assets/{finding-anchor-Bl1hRK5E.js → finding-anchor-CDbHtqKT.js} +1 -1
- package/public/assets/{inbox-DHIFHqcn.js → inbox-M_EuWNiK.js} +1 -1
- package/public/assets/{index-U654cmB8.js → index-pnImGbgQ.js} +3 -3
- package/public/assets/{parser-worker-5a9aUx3o.js → parser-worker-C3BAO6z9.js} +4 -2
- package/public/assets/{select-CtKaJ5Hx.js → select-C-MZGNpb.js} +1 -1
- package/public/assets/{toggle-group-DB2VfFIx.js → toggle-group-C3nz7UQ_.js} +1 -1
- package/public/assets/{useRecentFiles-BiYdWs8J.js → useRecentFiles-0XuhS91q.js} +4 -4
- package/public/assets/{useSortMode-BhPky1lW.js → useSortMode-DHlKZeWv.js} +1 -1
- package/public/assets/zstd-worker-C1Jq9lfb.js +1 -0
- package/public/docs/404.html +2 -2
- package/public/docs/assets/{app.BczRxXPr.js → app.DsFQcynM.js} +1 -1
- package/public/docs/assets/chunks/@localSearchIndexroot.BEfYag5Q.js +1 -0
- package/public/docs/assets/chunks/{VPLocalSearchBox.DXvkTVnC.js → VPLocalSearchBox.oa8NbaiV.js} +1 -1
- package/public/docs/assets/chunks/{theme.ByUdwrLG.js → theme.Ca4dR9dL.js} +2 -2
- package/public/docs/assets/{contributor-guide_architecture_detector-contract.md.B1vUqtry.js → contributor-guide_architecture_detector-contract.md.aQQfP7pj.js} +1 -1
- package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.sf9DU2bA.js +1 -0
- package/public/docs/assets/{contributor-guide_architecture_impact-estimation.md.DHVq1UPt.lean.js → contributor-guide_architecture_impact-estimation.md.sf9DU2bA.lean.js} +1 -1
- package/public/docs/assets/contributor-guide_architecture_overview.md.BXQdgG5u.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_overview.md.BXQdgG5u.lean.js +1 -0
- package/public/docs/assets/{contributor-guide_architecture_worker-protocol.md.C0SovZX8.js → contributor-guide_architecture_worker-protocol.md.P7d1vtEJ.js} +2 -2
- package/public/docs/assets/{contributor-guide_architecture_worker-protocol.md.C0SovZX8.lean.js → contributor-guide_architecture_worker-protocol.md.P7d1vtEJ.lean.js} +1 -1
- package/public/docs/assets/contributor-guide_testing.md.BU6vyA-9.js +1 -0
- package/public/docs/assets/contributor-guide_testing.md.BU6vyA-9.lean.js +1 -0
- package/public/docs/contributor-guide/architecture/board-widgets.html +3 -3
- package/public/docs/contributor-guide/architecture/detector-contract.html +5 -5
- package/public/docs/contributor-guide/architecture/drill-down.html +3 -3
- package/public/docs/contributor-guide/architecture/impact-estimation.html +5 -5
- package/public/docs/contributor-guide/architecture/index.html +3 -3
- package/public/docs/contributor-guide/architecture/overview.html +5 -5
- package/public/docs/contributor-guide/architecture/state-and-history.html +3 -3
- package/public/docs/contributor-guide/architecture/widget-rendering.html +3 -3
- package/public/docs/contributor-guide/architecture/worker-protocol.html +5 -5
- package/public/docs/contributor-guide/contributing.html +3 -3
- package/public/docs/contributor-guide/development-setup.html +3 -3
- package/public/docs/contributor-guide/testing.html +5 -5
- package/public/docs/hashmap.json +1 -1
- package/public/docs/index.html +3 -3
- package/public/docs/tuning-reference/anti-patterns.html +3 -3
- package/public/docs/tuning-reference/aqe.html +3 -3
- package/public/docs/tuning-reference/bottleneck-broadcast-sizing.html +3 -3
- package/public/docs/tuning-reference/bottleneck-cold-start.html +3 -3
- package/public/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html +3 -3
- package/public/docs/tuning-reference/bottleneck-failures.html +3 -3
- package/public/docs/tuning-reference/bottleneck-gc.html +3 -3
- package/public/docs/tuning-reference/bottleneck-job-failure-rate.html +3 -3
- package/public/docs/tuning-reference/bottleneck-memory-utilization.html +3 -3
- package/public/docs/tuning-reference/bottleneck-retry-waste.html +3 -3
- package/public/docs/tuning-reference/bottleneck-shuffle.html +3 -3
- package/public/docs/tuning-reference/bottleneck-skew.html +3 -3
- package/public/docs/tuning-reference/bottleneck-slow-host.html +3 -3
- package/public/docs/tuning-reference/bottleneck-small-files.html +3 -3
- package/public/docs/tuning-reference/bottleneck-spill.html +3 -3
- package/public/docs/tuning-reference/bottleneck-straggler.html +3 -3
- package/public/docs/tuning-reference/bottleneck-tiny-tasks.html +3 -3
- package/public/docs/tuning-reference/bottleneck-utilization.html +3 -3
- package/public/docs/tuning-reference/caching.html +3 -3
- package/public/docs/tuning-reference/cluster-config.html +3 -3
- package/public/docs/tuning-reference/config.html +3 -3
- package/public/docs/tuning-reference/data-formats.html +3 -3
- package/public/docs/tuning-reference/index.html +3 -3
- package/public/docs/tuning-reference/intro.html +3 -3
- package/public/docs/tuning-reference/joins.html +3 -3
- package/public/docs/tuning-reference/memory-model.html +3 -3
- package/public/docs/tuning-reference/metrics.html +3 -3
- package/public/docs/tuning-reference/partitioning.html +3 -3
- package/public/docs/tuning-reference/pyspark.html +3 -3
- package/public/docs/tuning-reference/shuffle.html +3 -3
- package/public/docs/tuning-reference/spark-architecture.html +3 -3
- package/public/docs/tuning-reference/table-formats.html +3 -3
- package/public/docs/user-guide/alternative-log-retrieval.html +3 -3
- package/public/docs/user-guide/getting-started.html +3 -3
- package/public/docs/user-guide/mcp-tools.html +3 -3
- package/public/docs/user-guide/run-comparison.html +3 -3
- package/public/docs/user-guide/understanding-findings.html +3 -3
- package/public/index.html +1 -1
- package/vendor-core/impact-estimator.js +23 -3
- package/vendor-core/occupancy.js +8 -7
- package/vendor-core/parser-worker.js +31 -15
- package/vendor-core/plan-summary.js +1 -1
- package/vendor-core/stage-quantiles.js +53 -1
- package/vendor-core/types.js +5 -0
- package/vendor-core/zstd-worker-client.js +180 -0
- package/vendor-core/zstd-worker.js +103 -0
- package/public/docs/assets/chunks/@localSearchIndexroot.c2HfrwDg.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.DHVq1UPt.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_overview.md.DkXdJ9jE.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_overview.md.DkXdJ9jE.lean.js +0 -1
- package/public/docs/assets/contributor-guide_testing.md.BohuQWvl.js +0 -1
- package/public/docs/assets/contributor-guide_testing.md.BohuQWvl.lean.js +0 -1
- /package/public/docs/assets/{contributor-guide_architecture_detector-contract.md.B1vUqtry.lean.js → contributor-guide_architecture_detector-contract.md.aQQfP7pj.lean.js} +0 -0
|
@@ -9,9 +9,9 @@
|
|
|
9
9
|
<link rel="preload stylesheet" href="/docs/assets/style.DSixAiZE.css" as="style">
|
|
10
10
|
<link rel="preload stylesheet" href="/docs/vp-icons.css" as="style">
|
|
11
11
|
|
|
12
|
-
<script type="module" src="/docs/assets/app.
|
|
12
|
+
<script type="module" src="/docs/assets/app.DsFQcynM.js"></script>
|
|
13
13
|
<link rel="preload" href="/docs/assets/inter-roman-latin.Di8DUHzh.woff2" as="font" type="font/woff2" crossorigin="">
|
|
14
|
-
<link rel="modulepreload" href="/docs/assets/chunks/theme.
|
|
14
|
+
<link rel="modulepreload" href="/docs/assets/chunks/theme.Ca4dR9dL.js">
|
|
15
15
|
<link rel="modulepreload" href="/docs/assets/chunks/framework.DSg0KOwT.js">
|
|
16
16
|
<link rel="modulepreload" href="/docs/assets/user-guide_run-comparison.md.CQc1aoU8.lean.js">
|
|
17
17
|
<link rel="icon" type="image/svg+xml" href="/docs/favicon.svg">
|
|
@@ -23,7 +23,7 @@
|
|
|
23
23
|
</head>
|
|
24
24
|
<body>
|
|
25
25
|
<div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>User Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/getting-started.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Getting started</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/understanding-findings.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Understanding findings</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/run-comparison.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Run comparison mode</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/mcp-tools.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>MCP tools reference</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/alternative-log-retrieval.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Alternative log retrieval</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_user-guide_run-comparison" data-v-39a288b8><div><h1 id="run-comparison-mode" tabindex="-1">Run comparison mode <a class="header-anchor" href="#run-comparison-mode" aria-label="Permalink to "Run comparison mode""></a></h1><p>Put two runs side by side to see whether a tuning change helped.</p><h2 id="starting-a-comparison" tabindex="-1">Starting a comparison <a class="header-anchor" href="#starting-a-comparison" aria-label="Permalink to "Starting a comparison""></a></h2><p>From the landing page, click <strong>Compare two runs</strong>. Two slots appear: <strong>Run A (baseline)</strong> and <strong>Run B (candidate)</strong>.</p><ol><li>Load a file into each slot the same way you'd load a single run (drag and drop, or <strong>Choose file</strong>).</li><li>Click <strong>Compare</strong>.</li></ol><p>The runs parse one after the other through the same background worker. When both finish, the comparison view opens.</p><h2 id="reading-the-comparison" tabindex="-1">Reading the comparison <a class="header-anchor" href="#reading-the-comparison" aria-label="Permalink to "Reading the comparison""></a></h2><p>The <strong>Metrics</strong> table covers the whole run: wall-clock duration, shuffle spill, task skew, failed-task rate, disk spill, GC time, input/output bytes, executor run-time, and task/executor counts, baseline against candidate with the delta. <strong>Findings by category</strong> lists which finding types appeared or disappeared between the two runs, with the affected stages for each.</p><p>Stage-level detail depends on matching a stage in the baseline to its counterpart in the candidate, and SparkForensics only does that automatically when a stage's identity (its position in the SQL plan) resolves to exactly one match on both sides; AQE's runtime replanning makes a plain stage-ID match unreliable. The <strong>Per-stage task skew</strong> table covers only stages matched this way, and the page states what percentage of stages that was. If matching is uncertain (for example, the two runs came from different application names), a warning banner says so; the metric deltas above it still hold; they don't depend on stage matching.</p><p>For everything else, matching is manual: the <strong>Pinned per-stage deltas</strong> widget lets you pick one stage from the baseline and one from the candidate yourself and pin the pair, then shows their duration, executor run-time, GC time, memory/disk spill, input/output bytes, task count, and failed tasks side by side. Pin as many pairs as you want to check.</p><p>For either run's full dashboard, click <strong>View run A dashboard</strong> or <strong>View run B dashboard</strong>. <strong>← Back to comparison</strong> takes you back.</p><h2 id="comparing-without-the-dashboard" tabindex="-1">Comparing without the dashboard <a class="header-anchor" href="#comparing-without-the-dashboard" aria-label="Permalink to "Comparing without the dashboard""></a></h2><p>The same comparison runs headlessly. The CLI's <code>--baseline</code> flag adds a comparison section to its output and can gate a build on it (<code>--max-regression-pct</code>, <code>--fail-on-introduced</code>; see <a href="./getting-started.html#ci-and-automation">Getting started</a>). The MCP server's <code>compare_runs</code> and <code>evaluate_budgets</code> tools do the same for an AI assistant; see <a href="./mcp-tools.html">MCP tools reference</a>.</p></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/user-guide/understanding-findings.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>Understanding findings</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="/docs/user-guide/mcp-tools.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>MCP tools reference</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
|
|
26
|
-
<script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"9ToZXcTk\",\"contributor-guide_architecture_detector-contract.md\":\"
|
|
26
|
+
<script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"9ToZXcTk\",\"contributor-guide_architecture_detector-contract.md\":\"aQQfP7pj\",\"contributor-guide_architecture_drill-down.md\":\"Bawx6NgP\",\"contributor-guide_architecture_impact-estimation.md\":\"sf9DU2bA\",\"contributor-guide_architecture_index.md\":\"3TO9ic6w\",\"contributor-guide_architecture_overview.md\":\"BXQdgG5u\",\"contributor-guide_architecture_state-and-history.md\":\"C-xxn0q7\",\"contributor-guide_architecture_widget-rendering.md\":\"DYwFeyjV\",\"contributor-guide_architecture_worker-protocol.md\":\"P7d1vtEJ\",\"contributor-guide_contributing.md\":\"CvRsdr6J\",\"contributor-guide_development-setup.md\":\"DvMV1xhQ\",\"contributor-guide_testing.md\":\"BU6vyA-9\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"Df1YMIHu\",\"tuning-reference_aqe.md\":\"BIsCtLzm\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"CEstB3Ia\",\"tuning-reference_bottleneck-cold-start.md\":\"CEuy-72y\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"CIohQDfn\",\"tuning-reference_bottleneck-failures.md\":\"CAsFf3RR\",\"tuning-reference_bottleneck-gc.md\":\"B4T3kJYV\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"BaJl__1W\",\"tuning-reference_bottleneck-memory-utilization.md\":\"DbP-SJZc\",\"tuning-reference_bottleneck-retry-waste.md\":\"D5JMjVOt\",\"tuning-reference_bottleneck-shuffle.md\":\"DDMQVOW6\",\"tuning-reference_bottleneck-skew.md\":\"B0MzhWcX\",\"tuning-reference_bottleneck-slow-host.md\":\"BlIo6UDW\",\"tuning-reference_bottleneck-small-files.md\":\"DHGMTmPs\",\"tuning-reference_bottleneck-spill.md\":\"PNH7mITt\",\"tuning-reference_bottleneck-straggler.md\":\"DIbXdAcQ\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"CB05v3zh\",\"tuning-reference_bottleneck-utilization.md\":\"DqC_iGkw\",\"tuning-reference_caching.md\":\"B7aQ8asB\",\"tuning-reference_cluster-config.md\":\"CD6vzYtD\",\"tuning-reference_config.md\":\"UvveiWG3\",\"tuning-reference_data-formats.md\":\"bjCAWH3N\",\"tuning-reference_index.md\":\"BQ_NooMV\",\"tuning-reference_intro.md\":\"CobD-lGB\",\"tuning-reference_joins.md\":\"BtKs_CuW\",\"tuning-reference_memory-model.md\":\"CWhvgevP\",\"tuning-reference_metrics.md\":\"mLOh7Apj\",\"tuning-reference_partitioning.md\":\"q0zKF_8X\",\"tuning-reference_pyspark.md\":\"DDCfvN9t\",\"tuning-reference_shuffle.md\":\"BZZ7R4Ix\",\"tuning-reference_spark-architecture.md\":\"Dwzm5avO\",\"tuning-reference_table-formats.md\":\"D6wj-2dX\",\"user-guide_alternative-log-retrieval.md\":\"B4tPGIal\",\"user-guide_getting-started.md\":\"BJvwLEIM\",\"user-guide_mcp-tools.md\":\"DGvolGzt\",\"user-guide_run-comparison.md\":\"CQc1aoU8\",\"user-guide_understanding-findings.md\":\"B29f0ucS\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Alternative log retrieval\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
|
|
27
27
|
|
|
28
28
|
</body>
|
|
29
29
|
</html>
|
|
@@ -9,9 +9,9 @@
|
|
|
9
9
|
<link rel="preload stylesheet" href="/docs/assets/style.DSixAiZE.css" as="style">
|
|
10
10
|
<link rel="preload stylesheet" href="/docs/vp-icons.css" as="style">
|
|
11
11
|
|
|
12
|
-
<script type="module" src="/docs/assets/app.
|
|
12
|
+
<script type="module" src="/docs/assets/app.DsFQcynM.js"></script>
|
|
13
13
|
<link rel="preload" href="/docs/assets/inter-roman-latin.Di8DUHzh.woff2" as="font" type="font/woff2" crossorigin="">
|
|
14
|
-
<link rel="modulepreload" href="/docs/assets/chunks/theme.
|
|
14
|
+
<link rel="modulepreload" href="/docs/assets/chunks/theme.Ca4dR9dL.js">
|
|
15
15
|
<link rel="modulepreload" href="/docs/assets/chunks/framework.DSg0KOwT.js">
|
|
16
16
|
<link rel="modulepreload" href="/docs/assets/user-guide_understanding-findings.md.B29f0ucS.lean.js">
|
|
17
17
|
<link rel="icon" type="image/svg+xml" href="/docs/favicon.svg">
|
|
@@ -23,7 +23,7 @@
|
|
|
23
23
|
</head>
|
|
24
24
|
<body>
|
|
25
25
|
<div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>User Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/getting-started.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Getting started</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/understanding-findings.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Understanding findings</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/run-comparison.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Run comparison mode</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/mcp-tools.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>MCP tools reference</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/user-guide/alternative-log-retrieval.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Alternative log retrieval</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_user-guide_understanding-findings" data-v-39a288b8><div><h1 id="understanding-findings" tabindex="-1">Understanding findings <a class="header-anchor" href="#understanding-findings" aria-label="Permalink to "Understanding findings""></a></h1><p>Every flagged problem carries a short ALL-CAPS tag. This page has one entry per tag: what it means, and what to do about it.</p><p>Turn on <a href="./getting-started.html#advanced-view">Advanced view</a> in the dashboard to see, per finding, the confidence level and a link to the same background reading collected here, opened in an in-app "Reference" panel next to the board. A few tags share that reading with another tag, because the underlying Spark-tuning material overlaps: <code>SFAIL</code> with <code>FAIL</code>, <code>PART</code> with <code>SHFL</code>, <code>SPEC</code> with <code>STRAG</code>, and <code>CACHE</code>/<code>LOCAL</code> with <code>UTIL</code>.</p><h2 id="per-stage" tabindex="-1">Per-stage <a class="header-anchor" href="#per-stage" aria-label="Permalink to "Per-stage""></a></h2><h3 id="skew" tabindex="-1"><code>SKEW</code>: Task skew <a class="header-anchor" href="#skew" aria-label="Permalink to "`SKEW`: Task skew {#skew}""></a></h3><p>A small number of tasks take much longer than their peers in the same stage. For join-driven skew, enable AQE skew-join handling (<code>spark.sql.adaptive.skewJoin.enabled</code>); otherwise salt the key or repartition on a better key.</p><h3 id="shfl" tabindex="-1"><code>SHFL</code>: Shuffle I/O <a class="header-anchor" href="#shfl" aria-label="Permalink to "`SHFL`: Shuffle I/O {#shfl}""></a></h3><p>Tasks move a large amount of intermediate data between stages. Raise <code>spark.sql.shuffle.partitions</code>, or add a broadcast join. Only flagged on stages that take at least 0.5% of the run.</p><h3 id="spill" tabindex="-1"><code>SPILL</code>: Memory and disk spill <a class="header-anchor" href="#spill" aria-label="Permalink to "`SPILL`: Memory and disk spill {#spill}""></a></h3><p>Tasks are writing data out of memory, which slows execution. Two spill patterns get flagged differently: skew spill, where a few heavy tasks spill while most don't (rebalance partitioning), and volume spill, where most tasks spill because the data genuinely exceeds available memory (add partitions). Only flagged on stages that take at least 0.5% of the run.</p><h3 id="gc" tabindex="-1"><code>GC</code>: Garbage collection pressure <a class="header-anchor" href="#gc" aria-label="Permalink to "`GC`: Garbage collection pressure {#gc}""></a></h3><p>Tasks spend an unusually large share of time reclaiming memory. Reduce object creation: use primitive types, avoid UDFs, or raise executor memory. A stage with very little GC gets an informational note that executor memory may be over-provisioned, only on stages that take at least 0.5% of the run.</p><h3 id="fail" tabindex="-1"><code>FAIL</code>: Failed tasks <a class="header-anchor" href="#fail" aria-label="Permalink to "`FAIL`: Failed tasks {#fail}""></a></h3><p>Tasks fail often enough to affect the stage. Failed tasks point to executor instability or data-driven errors: check driver logs for the dominant failure reason.</p><h3 id="sfail" tabindex="-1"><code>SFAIL</code>: Failed stage <a class="header-anchor" href="#sfail" aria-label="Permalink to "`SFAIL`: Failed stage {#sfail}""></a></h3><p>A stage attempt failed outright rather than losing individual tasks within it. Inspect the driver log for the failure reason and the job that triggered it.</p><h3 id="strag" tabindex="-1"><code>STRAG</code>: Straggler tasks <a class="header-anchor" href="#strag" aria-label="Permalink to "`STRAG`: Straggler tasks {#strag}""></a></h3><p>A few tasks run much slower than the rest of their stage. Rule out a GC pause or a slow shuffle fetch before assuming a hardware issue; if a skewed key is the real cause, that's a candidate for AQE's skew-join handling. Only flagged on stages that take at least 0.5% of the run.</p><h3 id="spec" tabindex="-1"><code>SPEC</code>: Speculation waste <a class="header-anchor" href="#spec" aria-label="Permalink to "`SPEC`: Speculation waste {#spec}""></a></h3><p>Speculative task attempts used a lot of executor time without confirming a genuine straggler. Self-flags a confidence that scales with how far the wasted time sits past the threshold: these thresholds are still a design spike, not yet validated against real-world runs. If task durations are just naturally variable rather than genuine stragglers, tune <code>spark.speculation.multiplier</code>/<code>spark.speculation.quantile</code>.</p><h3 id="retry" tabindex="-1"><code>RETRY</code>: Retry waste <a class="header-anchor" href="#retry" aria-label="Permalink to "`RETRY`: Retry waste {#retry}""></a></h3><p>Repeated task attempts ate into execution time even though the stage completed. Investigate executor loss or fetch failures.</p><h3 id="tiny" tabindex="-1"><code>TINY</code>: Tiny tasks <a class="header-anchor" href="#tiny" aria-label="Permalink to "`TINY`: Tiny tasks {#tiny}""></a></h3><p>Many very short tasks add scheduling overhead out of proportion to the work each one does. Repartition to fewer, larger tasks. Only flagged on stages that take at least 0.5% of the run.</p><h3 id="part" tabindex="-1"><code>PART</code>: Partition sizing <a class="header-anchor" href="#part" aria-label="Permalink to "`PART`: Partition sizing {#part}""></a></h3><p>Shuffle partitions are too large, too uneven, or too few for the work. A single shuffle partition over 5 GB, for example, will OOM or spill heavily: repartition to break it up before the stage runs.</p><h3 id="slow" tabindex="-1"><code>SLOW</code>: Stage slowness <a class="header-anchor" href="#slow" aria-label="Permalink to "`SLOW`: Stage slowness {#slow}""></a></h3><p>A stage ran long overall without a more specific cause getting flagged. Often a partition-count problem: raise parallelism via <code>spark.sql.shuffle.partitions</code> or <code>spark.default.parallelism</code>, or check for a large per-task data volume driving heavy shuffle and spill.</p><h3 id="shape" tabindex="-1"><code>SHAPE</code>: Stage shape <a class="header-anchor" href="#shape" aria-label="Permalink to "`SHAPE`: Stage shape {#shape}""></a></h3><p>The stage has an inefficient task count, output shape, or task-to-stage balance: for example, one straggler task taking a large fraction of the stage's wall-clock time. A too-low task count is only flagged on stages that take at least 0.5% of the run.</p><h3 id="host" tabindex="-1"><code>HOST</code>: Slow host <a class="header-anchor" href="#host" aria-label="Permalink to "`HOST`: Slow host {#host}""></a></h3><p>One executor is much slower than its peers. It may just hold data locality for its tasks or carry one heavy stage, rather than a hardware fault. Enable <code>spark.speculation</code> to relaunch a lagging task automatically. Only flagged on stages that take at least 0.5% of the run.</p><h2 id="app-level" tabindex="-1">App-level <a class="header-anchor" href="#app-level" aria-label="Permalink to "App-level""></a></h2><h3 id="cold" tabindex="-1"><code>COLD</code>: Executor cold start <a class="header-anchor" href="#cold" aria-label="Permalink to "`COLD`: Executor cold start {#cold}""></a></h3><p>New executors take time to become available for work. Pre-warm the cluster, or use dynamic allocation.</p><h3 id="util" tabindex="-1"><code>UTIL</code>: Low utilization <a class="header-anchor" href="#util" aria-label="Permalink to "`UTIL`: Low utilization {#util}""></a></h3><p>Allocated executors sit idle for a large share of the application run. Consider a smaller cluster, or enable dynamic allocation.</p><h3 id="mem" tabindex="-1"><code>MEM</code>: Memory utilization <a class="header-anchor" href="#mem" aria-label="Permalink to "`MEM`: Memory utilization {#mem}""></a></h3><p>Executor memory or core capacity may be over- or under-provisioned. Some detail here needs <code>spark.eventLog.logStageExecutorMetrics=true</code> on the run being analyzed; without it, per-executor memory usage can't be broken down. Review <code>spark.executor.memory</code> and executor count if allocated memory sat largely idle over the run. That idle-memory variant self-flags a confidence that scales with how far the estimated waste sits past a 1.5x buffer: it estimates waste from allocated-versus-used memory-time. Check it against the Spark UI before resizing anything.</p><h3 id="cache" tabindex="-1"><code>CACHE</code>: Caching opportunity <a class="header-anchor" href="#cache" aria-label="Permalink to "`CACHE`: Caching opportunity {#cache}""></a></h3><p>A reusable dataset (re-read via the same SQL relation more than once) may be worth persisting between stages. Self-flags a confidence that scales with how many executions reuse the same relation: reuse is only inferred, from plan-scan identity across SQL executions, so confirm the reads really do hit the same data before you cache anything.</p><h3 id="cstor" tabindex="-1"><code>CSTOR</code>: Cache storage <a class="header-anchor" href="#cstor" aria-label="Permalink to "`CSTOR`: Cache storage {#cstor}""></a></h3><p>A persisted dataset is not fully cached in memory, or is spilling to disk. Raise executor memory, or shrink the cached dataset.</p><h3 id="local" tabindex="-1"><code>LOCAL</code>: Core usage locality <a class="header-anchor" href="#local" aria-label="Permalink to "`LOCAL`: Core usage locality {#local}""></a></h3><p>Tasks run without process- or node-local data placement more often than expected. Check <code>spark.locality.wait</code> settings and executor/data colocation. Self-flags a confidence that scales with the non-local ratio and sample size: the thresholds are our own noise floor for this metric.</p><h3 id="chrn" tabindex="-1"><code>CHRN</code>: Autoscaling churn <a class="header-anchor" href="#chrn" aria-label="Permalink to "`CHRN`: Autoscaling churn {#chrn}""></a></h3><p>Executors are stood up and torn down again before they can do useful work: re-provisioning churn rather than normal scale-down. Raise <code>spark.dynamicAllocation.executorIdleTimeout</code>, or widen the <code>minExecutors</code>/<code>maxExecutors</code> bounds to reduce flapping. Self-flags a confidence that scales with how far the short-lived-executor share sits past the threshold: these thresholds are still a design spike, not yet validated against real-world runs.</p><h3 id="jobs" tabindex="-1"><code>JOBS</code>: Job failure rate <a class="header-anchor" href="#jobs" aria-label="Permalink to "`JOBS`: Job failure rate {#jobs}""></a></h3><p>A large share of completed jobs did not succeed. Inspect the driver log for the failed job(s) and the stage failures that triggered them.</p><h3 id="incmp" tabindex="-1"><code>INCMP</code>: Incomplete run <a class="header-anchor" href="#incmp" aria-label="Permalink to "`INCMP`: Incomplete run {#incmp}""></a></h3><p>This event log never recorded an <code>ApplicationEnd</code> event: capture stopped before the run finished (an in-flight job, a rotated-away log, or a cut-short capture). Every other finding and metric on the board reflects only what was captured up to that point, not the full run.</p><h2 id="configuration-scope" tabindex="-1">Configuration scope <a class="header-anchor" href="#configuration-scope" aria-label="Permalink to "Configuration scope""></a></h2><h3 id="cfg" tabindex="-1"><code>CFG</code>: Configuration audit <a class="header-anchor" href="#cfg" aria-label="Permalink to "`CFG`: Configuration audit {#cfg}""></a></h3><p>Flags configuration settings that may cause reliability or efficiency problems, independent of any one stage's behavior. Four properties are audited today:</p><ul><li><code>spark.shuffle.service.enabled</code>: flagged when dynamic allocation is on but the external shuffle service is off, since shuffle data won't survive executor removal.</li><li><code>spark.dynamicAllocation.maxExecutors</code>: flagged for inverted bounds or a missing upper bound.</li><li><code>spark.serializer</code>: flagged when still on the default Java serializer; <code>org.apache.spark.serializer.KryoSerializer</code> is faster and produces smaller buffers.</li><li><code>spark.executor.memoryOverhead</code>: flagged when set below a safe floor.</li></ul><h2 id="sql-scope" tabindex="-1">SQL scope <a class="header-anchor" href="#sql-scope" aria-label="Permalink to "SQL scope""></a></h2><h3 id="plan" tabindex="-1"><code>PLAN</code>: Plan advisor <a class="header-anchor" href="#plan" aria-label="Permalink to "`PLAN`: Plan advisor {#plan}""></a></h3><p>Flags patterns in the SQL execution plan worth reviewing. Four checks share this tag:</p><ul><li>Duplicate plan subtree: the same subtree recomputed more than once in the plan. When the repeats have the same shape but different filters, columns or tables, the finding stays informational and claims no time. Only flagged when the repeat's stages take at least 0.5% of the run.</li><li>Small files: reading an excessive number of small files.</li><li>Under-broadcast: the smaller side of a Sort Merge Join looks well under the broadcast threshold; consider a <code>broadcast()</code> hint or raising <code>spark.sql.autoBroadcastJoinThreshold</code>.</li><li>Over-broadcast: a broadcast exceeds the 1 GB threshold; check for a misapplied broadcast hint or a misconfigured <code>spark.sql.autoBroadcastJoinThreshold</code>.</li></ul></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/user-guide/getting-started.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>Getting started</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="/docs/user-guide/run-comparison.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>Run comparison mode</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
|
|
26
|
-
<script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"9ToZXcTk\",\"contributor-guide_architecture_detector-contract.md\":\"
|
|
26
|
+
<script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"9ToZXcTk\",\"contributor-guide_architecture_detector-contract.md\":\"aQQfP7pj\",\"contributor-guide_architecture_drill-down.md\":\"Bawx6NgP\",\"contributor-guide_architecture_impact-estimation.md\":\"sf9DU2bA\",\"contributor-guide_architecture_index.md\":\"3TO9ic6w\",\"contributor-guide_architecture_overview.md\":\"BXQdgG5u\",\"contributor-guide_architecture_state-and-history.md\":\"C-xxn0q7\",\"contributor-guide_architecture_widget-rendering.md\":\"DYwFeyjV\",\"contributor-guide_architecture_worker-protocol.md\":\"P7d1vtEJ\",\"contributor-guide_contributing.md\":\"CvRsdr6J\",\"contributor-guide_development-setup.md\":\"DvMV1xhQ\",\"contributor-guide_testing.md\":\"BU6vyA-9\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"Df1YMIHu\",\"tuning-reference_aqe.md\":\"BIsCtLzm\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"CEstB3Ia\",\"tuning-reference_bottleneck-cold-start.md\":\"CEuy-72y\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"CIohQDfn\",\"tuning-reference_bottleneck-failures.md\":\"CAsFf3RR\",\"tuning-reference_bottleneck-gc.md\":\"B4T3kJYV\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"BaJl__1W\",\"tuning-reference_bottleneck-memory-utilization.md\":\"DbP-SJZc\",\"tuning-reference_bottleneck-retry-waste.md\":\"D5JMjVOt\",\"tuning-reference_bottleneck-shuffle.md\":\"DDMQVOW6\",\"tuning-reference_bottleneck-skew.md\":\"B0MzhWcX\",\"tuning-reference_bottleneck-slow-host.md\":\"BlIo6UDW\",\"tuning-reference_bottleneck-small-files.md\":\"DHGMTmPs\",\"tuning-reference_bottleneck-spill.md\":\"PNH7mITt\",\"tuning-reference_bottleneck-straggler.md\":\"DIbXdAcQ\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"CB05v3zh\",\"tuning-reference_bottleneck-utilization.md\":\"DqC_iGkw\",\"tuning-reference_caching.md\":\"B7aQ8asB\",\"tuning-reference_cluster-config.md\":\"CD6vzYtD\",\"tuning-reference_config.md\":\"UvveiWG3\",\"tuning-reference_data-formats.md\":\"bjCAWH3N\",\"tuning-reference_index.md\":\"BQ_NooMV\",\"tuning-reference_intro.md\":\"CobD-lGB\",\"tuning-reference_joins.md\":\"BtKs_CuW\",\"tuning-reference_memory-model.md\":\"CWhvgevP\",\"tuning-reference_metrics.md\":\"mLOh7Apj\",\"tuning-reference_partitioning.md\":\"q0zKF_8X\",\"tuning-reference_pyspark.md\":\"DDCfvN9t\",\"tuning-reference_shuffle.md\":\"BZZ7R4Ix\",\"tuning-reference_spark-architecture.md\":\"Dwzm5avO\",\"tuning-reference_table-formats.md\":\"D6wj-2dX\",\"user-guide_alternative-log-retrieval.md\":\"B4tPGIal\",\"user-guide_getting-started.md\":\"BJvwLEIM\",\"user-guide_mcp-tools.md\":\"DGvolGzt\",\"user-guide_run-comparison.md\":\"CQc1aoU8\",\"user-guide_understanding-findings.md\":\"B29f0ucS\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Alternative log retrieval\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
|
|
27
27
|
|
|
28
28
|
</body>
|
|
29
29
|
</html>
|
package/public/index.html
CHANGED
|
@@ -23,7 +23,7 @@
|
|
|
23
23
|
document.documentElement.setAttribute('data-theme', 'light');
|
|
24
24
|
} catch (_) {}
|
|
25
25
|
</script>
|
|
26
|
-
<script type="module" crossorigin src="./assets/index-
|
|
26
|
+
<script type="module" crossorigin src="./assets/index-pnImGbgQ.js"></script>
|
|
27
27
|
<link rel="modulepreload" crossorigin href="./assets/rolldown-runtime-hePW80VL.js">
|
|
28
28
|
<link rel="modulepreload" crossorigin href="./assets/charts-vendor-0VMaLVLN.js">
|
|
29
29
|
<link rel="modulepreload" crossorigin href="./assets/plan-graph-vendor-B0sEms9Z.js">
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
|
|
2
|
+
import { nsToMs } from './format-utils.js';
|
|
2
3
|
import {
|
|
3
4
|
computeOccupancy, estimateSingleStage, estimateMultiStage, tailRecoveryMs, tailRemovedWorkMs, stragglerFixLongestTaskMs,
|
|
4
5
|
|
|
@@ -82,6 +83,24 @@ const EXECUTOR_STARTUP_OVERHEAD_MS = 15000;
|
|
|
82
83
|
// Assumed re-read throughput, shared by cachingOpportunity and cacheUtilization.
|
|
83
84
|
const RE_READ_THROUGHPUT_BPS = 125_000_000;
|
|
84
85
|
|
|
86
|
+
// Below this share of executorRunTime spent on CPU, a stage's tasks were idle, waiting on something
|
|
87
|
+
// outside Spark: on 14 real logs (2026-09-23) every non-Python stage under 1% was a JDBC read, a
|
|
88
|
+
// file listing or a Delta log read, while file writes, which more partitions do parallelize,
|
|
89
|
+
// start at 2%.
|
|
90
|
+
const IDLE_CPU_SHARE_MAX = 0.01;
|
|
91
|
+
|
|
92
|
+
// True when the stage's tasks spent under IDLE_CPU_SHARE_MAX of their run time on CPU. False when
|
|
93
|
+
// the share can't be trusted: no CPU time recorded (older Spark logs omit the metric), or Python
|
|
94
|
+
// code run through PythonRDD, whose worker-process CPU executorCpuTime (the JVM task thread's)
|
|
95
|
+
// never counts (such stages read 0.1% on the same logs while computing).
|
|
96
|
+
function tasksMostlyIdle(stage ) {
|
|
97
|
+
const runMs = stage.executorRunTime ?? 0;
|
|
98
|
+
const cpuMs = nsToMs(stage.executorCpuTime ?? 0);
|
|
99
|
+
if (runMs <= 0 || cpuMs <= 0) return false;
|
|
100
|
+
if (/PythonRDD/.test(stage.name ?? '') || /org\.apache\.spark\.api\.python\./.test(stage.details ?? '')) return false;
|
|
101
|
+
return cpuMs / runMs < IDLE_CPU_SHARE_MAX;
|
|
102
|
+
}
|
|
103
|
+
|
|
85
104
|
// skew and straggler claim time off the stage's longest task itself, so the occupancy clip must
|
|
86
105
|
// not floor them at that same task (see estimateSingleStage). detectors.ts's clippedWasteMs gates
|
|
87
106
|
// both detectors on the same option so the firing floor and the displayed estimate agree.
|
|
@@ -332,14 +351,15 @@ function computeEstimateForFinding(
|
|
|
332
351
|
// is queueing no partition count recovers. Splitting partitions splits the longest task
|
|
333
352
|
// too, hence TAIL_CLAIM's post-fix floor. Unknown cluster size: no defensible figure.
|
|
334
353
|
if (totalCores <= 0) return costOnly('modeled');
|
|
335
|
-
// A stage that read no input and no shuffle
|
|
336
|
-
// 1-task count stage open 27 minutes on 5s of CPU
|
|
354
|
+
// A stage that read no input and no shuffle, its tasks idle waiting on an external system,
|
|
355
|
+
// gains nothing from more partitions (a 1-task JDBC count stage open 27 minutes on 5s of CPU
|
|
356
|
+
// was claimed 99% recoverable): claim 0. Stages that read bytes keep their claim.
|
|
337
357
|
const readBytes = (stage.inputBytes ?? 0) + (stage.shuffleReadBytes ?? 0);
|
|
338
358
|
const activeMs = typeof stage.taskActiveMs === 'number'
|
|
339
359
|
? stage.taskActiveMs
|
|
340
360
|
: Math.max(0, (stage.completedAt ?? 0) - (stage.submittedAt ?? 0));
|
|
341
361
|
const taskCount = stage.taskCount ?? 0;
|
|
342
|
-
const wasteMs = readBytes
|
|
362
|
+
const wasteMs = readBytes <= 0 && tasksMostlyIdle(stage) ? 0 : activeMs * Math.max(0, 1 - taskCount / totalCores);
|
|
343
363
|
return singleStageImpact(wasteMs, finding.stageId, stages, occupancy, 'modeled', { value: wasteMs, unit: 'ms' }, TAIL_CLAIM);
|
|
344
364
|
}
|
|
345
365
|
case 'partitionSizing': {
|
package/vendor-core/occupancy.js
CHANGED
|
@@ -152,18 +152,19 @@ const SERIAL_GATE_THRESHOLD = 0.999;
|
|
|
152
152
|
|
|
153
153
|
|
|
154
154
|
|
|
155
|
+
|
|
155
156
|
|
|
156
157
|
|
|
157
158
|
|
|
158
159
|
|
|
159
|
-
// Wall-clock a skew/straggler fix recovers
|
|
160
|
-
//
|
|
161
|
-
//
|
|
162
|
-
//
|
|
163
|
-
//
|
|
164
|
-
//
|
|
165
|
-
// peak on one real stage), which doubled the claim.
|
|
160
|
+
// Wall-clock a skew/straggler fix recovers: finalizeStage's task-level replay
|
|
161
|
+
// (tailReplayRecoveryMs, computeTailReplayRecoveryMs) when the stage carries it. Without it (a
|
|
162
|
+
// stage built by hand, as in detector tests) an estimate from the slowest task's excess over the
|
|
163
|
+
// median: a lone straggler costs that excess; a tail of many slow tasks costs its summed excess
|
|
164
|
+
// (stragglerExcessMs) spread over the slots the stage had (peakConcurrentTasks), and the replay
|
|
165
|
+
// recovers about the larger of the two.
|
|
166
166
|
export function tailRecoveryMs(stage , singleTaskExcessMs ) {
|
|
167
|
+
if (stage.tailReplayRecoveryMs != null) return stage.tailReplayRecoveryMs;
|
|
167
168
|
const excessMs = stage.stragglerExcessMs ?? 0;
|
|
168
169
|
const slots = stage.peakConcurrentTasks ?? 0;
|
|
169
170
|
if (excessMs <= 0 || slots <= 0) return singleTaskExcessMs;
|
|
@@ -5,6 +5,7 @@ import { createSnappyBlockDecoder } from './snappy-block.js';
|
|
|
5
5
|
import { createState, dispatchLine, buildChunkDecoder, emitParseCompletion, } from './event-handlers.js';
|
|
6
6
|
import { TASK_FIELD_NAMES } from './stage-quantiles.js';
|
|
7
7
|
import { runParseFromUrl, sniffCodec } from './shs-fetch.js';
|
|
8
|
+
import { createWorkerZstdDecoders } from './zstd-worker-client.js';
|
|
8
9
|
|
|
9
10
|
export {
|
|
10
11
|
buildChunkDecoder, createState, normalizeSparkProperties, parseSparkMemoryMB, extractResources,
|
|
@@ -61,9 +62,11 @@ const PROGRESS_EMIT_LINES = 300;
|
|
|
61
62
|
// without touching the vendored files (mirrors shs-fetch.ts's shim).
|
|
62
63
|
|
|
63
64
|
|
|
64
|
-
// A Node decoder
|
|
65
|
+
// A Node decoder, or the browser's decompress worker (zstd-worker-client.ts), may decompress off
|
|
66
|
+
// the calling thread: streamFile awaits each push, and calls cancel() when it abandons the stream.
|
|
65
67
|
|
|
66
68
|
|
|
69
|
+
|
|
67
70
|
|
|
68
71
|
|
|
69
72
|
// Stream one File's (possibly compressed) bytes through the codec dispatch,
|
|
@@ -89,7 +92,7 @@ export async function streamFile(
|
|
|
89
92
|
const gunzip = codec === 'gz' ? new (Gunzip )((inflated) => onChunk(inflated, currentPct)) : null;
|
|
90
93
|
const lz4 = codec === 'lz4' ? createLz4BlockDecoder((inflated) => onChunk(inflated, currentPct)) : null;
|
|
91
94
|
const onZstdChunk = (inflated ) => onChunk(inflated, currentPct);
|
|
92
|
-
const zstd
|
|
95
|
+
const zstd = codec !== 'zstd' ? null
|
|
93
96
|
: zstdDecoder ? zstdDecoder(onZstdChunk)
|
|
94
97
|
: new (ZstdDecompress )(onZstdChunk);
|
|
95
98
|
const snappy = codec === 'snappy' ? createSnappyBlockDecoder((inflated) => onChunk(inflated, currentPct)) : null;
|
|
@@ -101,17 +104,22 @@ export async function streamFile(
|
|
|
101
104
|
const stepSize = Math.max(1, Math.min(chunkSize, Math.ceil(file.size / MIN_PROGRESS_STEPS)));
|
|
102
105
|
|
|
103
106
|
let offset = 0;
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
107
|
+
try {
|
|
108
|
+
while (offset < file.size) {
|
|
109
|
+
const start = offset;
|
|
110
|
+
const slice = new Uint8Array(await file.slice(start, start + stepSize).arrayBuffer());
|
|
111
|
+
offset += stepSize;
|
|
112
|
+
const final = offset >= file.size;
|
|
113
|
+
currentPct = start / file.size;
|
|
114
|
+
if (gunzip) gunzip.push(slice, final);
|
|
115
|
+
else if (lz4) lz4.push(slice);
|
|
116
|
+
else if (zstd) await zstd.push(slice, final);
|
|
117
|
+
else if (snappy) snappy.push(slice);
|
|
118
|
+
else onChunk(slice, currentPct);
|
|
119
|
+
}
|
|
120
|
+
} catch (e) {
|
|
121
|
+
zstd?.cancel?.();
|
|
122
|
+
throw e;
|
|
115
123
|
}
|
|
116
124
|
if (lz4) lz4.end();
|
|
117
125
|
if (snappy) snappy.end();
|
|
@@ -232,17 +240,25 @@ const isWorker = typeof WorkerGlobalScope !== 'undefined' && self instanceof Wor
|
|
|
232
240
|
|
|
233
241
|
if (isWorker) {
|
|
234
242
|
let workerState = null;
|
|
243
|
+
// Dropped zstd files decompress in a second worker, overlapping with parsing here. The
|
|
244
|
+
// `new Worker(new URL(...))` stays inline for Vite's worker detection (see ingest.ts). The SHS
|
|
245
|
+
// path (runParseFromUrl) decodes whole zip entries synchronously and keeps in-thread fzstd.
|
|
246
|
+
const zstdDecoder = createWorkerZstdDecoders(
|
|
247
|
+
() => new Worker(new URL('./zstd-worker.js', import.meta.url), { type: 'module' }),
|
|
248
|
+
(onChunk) => new (ZstdDecompress )(onChunk),
|
|
249
|
+
{ onFallback: (reason) => console.warn(`zstd: decompressing on the parse worker: ${reason}`) },
|
|
250
|
+
);
|
|
235
251
|
|
|
236
252
|
self.onmessage = async ({ data } ) => {
|
|
237
253
|
if (data.type === 'parse') {
|
|
238
254
|
workerState = createState();
|
|
239
|
-
await runParse(data.file, workerState);
|
|
255
|
+
await runParse(data.file, workerState, { zstdDecoder });
|
|
240
256
|
} else if (data.type === 'parseFromUrl') {
|
|
241
257
|
workerState = createState();
|
|
242
258
|
await runParseFromUrl(data.request, workerState);
|
|
243
259
|
} else if (data.type === 'parseFiles') {
|
|
244
260
|
workerState = createState();
|
|
245
|
-
await runParseFiles(data.files, workerState);
|
|
261
|
+
await runParseFiles(data.files, workerState, { zstdDecoder });
|
|
246
262
|
} else if (data.type === 'getTaskData') {
|
|
247
263
|
const { stageId, reqId } = data;
|
|
248
264
|
const stored = workerState?.taskStore.get(stageId) ?? new Float64Array(0);
|
|
@@ -45,7 +45,7 @@ function pushLongFilterWarning(result , len ) {
|
|
|
45
45
|
}
|
|
46
46
|
|
|
47
47
|
// Stable relation-identity key for a scan node: "<format>:<relation>" (e.g.
|
|
48
|
-
// "delta:mx.
|
|
48
|
+
// "delta:mx.store_map", "parquet:warehouse.sales", "jdbc:dw.dim_product")
|
|
49
49
|
// or null when the node is internal Delta metadata / a non-scan / un-nameable.
|
|
50
50
|
// The single source of truth for scan identity, shared by `visitScan` (summary)
|
|
51
51
|
// and the `cachingOpportunity` detector so the regexes live in one place.
|
|
@@ -150,6 +150,11 @@ export function finalizeStage(
|
|
|
150
150
|
}
|
|
151
151
|
}
|
|
152
152
|
|
|
153
|
+
const peakConcurrentTasks = computePeakConcurrentTasks(arr);
|
|
154
|
+
const tailReplayRecoveryMs = stragglerCount > 0
|
|
155
|
+
? computeTailReplayRecoveryMs(arr, p50, peakConcurrentTasks)
|
|
156
|
+
: 0; // no task over 4x P50: both replays schedule the same durations
|
|
157
|
+
|
|
153
158
|
const hostStatsArr = [...hostStats.entries()].map(
|
|
154
159
|
([host, s]) => ({ host, taskCount: s.taskCount, totalDuration: s.totalDuration })
|
|
155
160
|
);
|
|
@@ -166,10 +171,11 @@ export function finalizeStage(
|
|
|
166
171
|
const data = {
|
|
167
172
|
...stage,
|
|
168
173
|
hostStats: hostStatsArr, executorStats: executorStatsArr, failureReasons: failureReasonsArr, localityStats: localityStatsArr, stragglerCount, stragglerExcessMs, longestNonStragglerMs,
|
|
174
|
+
tailReplayRecoveryMs,
|
|
169
175
|
failedTaskSamples,
|
|
170
176
|
peakExecutionMemoryMax,
|
|
171
177
|
taskActiveMs: computeTaskActiveMs(arr),
|
|
172
|
-
peakConcurrentTasks
|
|
178
|
+
peakConcurrentTasks,
|
|
173
179
|
taskDurationP50: p50,
|
|
174
180
|
taskDurationP95: p95,
|
|
175
181
|
taskDurationMax: max,
|
|
@@ -234,6 +240,52 @@ export function computePeakConcurrentTasks(arr ) {
|
|
|
234
240
|
return peak;
|
|
235
241
|
}
|
|
236
242
|
|
|
243
|
+
// Wall-clock a tail fix recovers, replayed from the stage's own tasks: list scheduling (tasks in
|
|
244
|
+
// launch order, each on the slot that frees first) over `slots` slots, once with the real
|
|
245
|
+
// durations and once with every task over 4x P50 (the straggler definition above) capped at P50.
|
|
246
|
+
// The difference is the claim; dev/eval-tail-replay.mjs keeps an independent copy as its ground
|
|
247
|
+
// truth. Equal free times are interchangeable slots, so which one a min-heap picks can't change
|
|
248
|
+
// the result. Equal launch times keep the task array's order.
|
|
249
|
+
export function computeTailReplayRecoveryMs(arr , p50 , slots ) {
|
|
250
|
+
const taskCount = arr.length / FIELDS.STRIDE;
|
|
251
|
+
if (taskCount < 2 || !(p50 > 0)) return 0;
|
|
252
|
+
const order = new Uint32Array(taskCount);
|
|
253
|
+
for (let i = 0; i < taskCount; i++) order[i] = i;
|
|
254
|
+
order.sort((a, b) => arr[a * FIELDS.STRIDE + FIELDS.LAUNCH_TIME] - arr[b * FIELDS.STRIDE + FIELDS.LAUNCH_TIME] || a - b);
|
|
255
|
+
const free = new Float64Array(Math.max(1, Math.min(slots, taskCount)));
|
|
256
|
+
const capAboveMs = 4 * p50;
|
|
257
|
+
const actualEndMs = listScheduleEndMs(arr, order, free, Infinity, p50);
|
|
258
|
+
const fixedEndMs = listScheduleEndMs(arr, order, free, capAboveMs, p50);
|
|
259
|
+
return Math.max(0, actualEndMs - fixedEndMs);
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
// End of a list schedule over `free` (a min-heap of slot free times, reset here), each task's
|
|
263
|
+
// duration replaced by `cappedMs` when over `capAboveMs`.
|
|
264
|
+
function listScheduleEndMs(
|
|
265
|
+
arr , order , free , capAboveMs , cappedMs ,
|
|
266
|
+
) {
|
|
267
|
+
free.fill(0);
|
|
268
|
+
const n = free.length;
|
|
269
|
+
let endMs = 0;
|
|
270
|
+
for (let t = 0; t < order.length; t++) {
|
|
271
|
+
const duration = arr[order[t] * FIELDS.STRIDE + FIELDS.DURATION];
|
|
272
|
+
const finish = free[0] + (duration > capAboveMs ? cappedMs : duration);
|
|
273
|
+
if (finish > endMs) endMs = finish;
|
|
274
|
+
// Replace the root (earliest free slot) and sift it down.
|
|
275
|
+
let i = 0;
|
|
276
|
+
for (;;) {
|
|
277
|
+
const left = 2 * i + 1;
|
|
278
|
+
if (left >= n) break;
|
|
279
|
+
const child = left + 1 < n && free[left + 1] < free[left] ? left + 1 : left;
|
|
280
|
+
if (free[child] >= finish) break;
|
|
281
|
+
free[i] = free[child];
|
|
282
|
+
i = child;
|
|
283
|
+
}
|
|
284
|
+
free[i] = finish;
|
|
285
|
+
}
|
|
286
|
+
return endMs;
|
|
287
|
+
}
|
|
288
|
+
|
|
237
289
|
export function computeFieldQuantiles(arr , fieldIndex ) {
|
|
238
290
|
const taskCount = arr.length / FIELDS.STRIDE;
|
|
239
291
|
if (taskCount === 0) return { p50: 0, p95: 0, max: 0 };
|