sparkforensics-server 0.2.2 → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/public/assets/{AutoscalingChurn-B96FKx97.js → AutoscalingChurn-Bf5TN-s_.js} +1 -1
- package/public/assets/CacheUtilization-DRf_FNrK.js +1 -0
- package/public/assets/CachingOpportunity-BTJzIhtI.js +1 -0
- package/public/assets/{ColdStart-w8QYTNKo.js → ColdStart-DnFgnD3n.js} +1 -1
- package/public/assets/ConfigAudit-B5FsOdmu.js +1 -0
- package/public/assets/{CoreUsageArea-o3kqQwyi.js → CoreUsageArea-CTksBPuq.js} +1 -1
- package/public/assets/Dashboard-BFMsMuc_.js +1 -0
- package/public/assets/{DuplicatePlanSubtree-Cm_UU3lY.js → DuplicatePlanSubtree-BCt-Guuh.js} +1 -1
- package/public/assets/{DurationHistogram-ltF-7M0A.js → DurationHistogram-xRcnND4k.js} +1 -1
- package/public/assets/{EmptyState-BCA3xioa.js → EmptyState-CnTrSjtU.js} +1 -1
- package/public/assets/{ExecutorUtilization-Cn69i0uq.js → ExecutorUtilization-CP21XlUq.js} +1 -1
- package/public/assets/GcPressure-BoEJsLjb.js +1 -0
- package/public/assets/{ImpactEstimate-BGt-4WX5.js → ImpactEstimate-BHcqJkcZ.js} +1 -1
- package/public/assets/{IncompleteRun-s6uQ8OLf.js → IncompleteRun-b9FWrOHn.js} +1 -1
- package/public/assets/{JobFailures-DJ2dDvPA.js → JobFailures-C5FkTq3O.js} +1 -1
- package/public/assets/{MemoryUtilization-C9G7P94X.js → MemoryUtilization-D2W4AB_f.js} +1 -1
- package/public/assets/{OverBroadcast-DYauoYWQ.js → OverBroadcast-Btu3xorf.js} +1 -1
- package/public/assets/PartitionSizing-yZWhLN0e.js +1 -0
- package/public/assets/{PlanExplorer-_2oB5ouy.js → PlanExplorer-DSk1ivS4.js} +1 -1
- package/public/assets/PlanGraphRoute-C3iOzUdB.js +1 -0
- package/public/assets/{RetryWaste-D_m8ASwP.js → RetryWaste-B4436kr1.js} +1 -1
- package/public/assets/{RowPagination-BLsm8I6i.js → RowPagination-oeWB8gBm.js} +1 -1
- package/public/assets/ShuffleIO-DUiv8EJe.js +1 -0
- package/public/assets/{Skew-BGLJuZZR.js → Skew-D2lB1oQc.js} +1 -1
- package/public/assets/{SlowHost-ChdWVvGO.js → SlowHost-Qb_8aa-W.js} +1 -1
- package/public/assets/{SmallFiles-gTilBtjW.js → SmallFiles-u6suTgfj.js} +1 -1
- package/public/assets/{SpeculationWaste-CVzgJnSj.js → SpeculationWaste-vUCniiFP.js} +1 -1
- package/public/assets/{Spill-NGJGQaTB.js → Spill-BskXFX2e.js} +1 -1
- package/public/assets/{StageFailed-sGhki53z.js → StageFailed-D9b83Jvy.js} +1 -1
- package/public/assets/{StageFindingGroup-AyI7BMd3.js → StageFindingGroup-DagqHOUz.js} +1 -1
- package/public/assets/{StageHeader-BTzGkK8k.js → StageHeader-DibMSunO.js} +1 -1
- package/public/assets/{StagePill-BlMOW_pU.js → StagePill-Dt6V9yVw.js} +1 -1
- package/public/assets/{StageShape-DfOW93Z1.js → StageShape-CfdrDUZa.js} +1 -1
- package/public/assets/{StageSlowness-BMZ-K_5e.js → StageSlowness-BoGLIYKR.js} +1 -1
- package/public/assets/Straggler-DU1WSU-m.js +1 -0
- package/public/assets/{TaskFailures-DGDwUK2A.js → TaskFailures-DkY6zBNJ.js} +1 -1
- package/public/assets/{TinyTask-CfT8zZDr.js → TinyTask-Lsnt5mJF.js} +1 -1
- package/public/assets/{UnderBroadcast-CiOiJ0k1.js → UnderBroadcast-DxHc0lwy.js} +1 -1
- package/public/assets/{finding-anchor-DEV8JvgH.js → finding-anchor-Bl1hRK5E.js} +1 -1
- package/public/assets/{inbox-_QCYkBsP.js → inbox-DHIFHqcn.js} +1 -1
- package/public/assets/index-U654cmB8.js +9 -0
- package/public/assets/parser-worker-5a9aUx3o.js +115 -0
- package/public/assets/{select-kuy1MiAo.js → select-CtKaJ5Hx.js} +1 -1
- package/public/assets/{toggle-group-CUd-KWkp.js → toggle-group-DB2VfFIx.js} +1 -1
- package/public/assets/useRecentFiles-BiYdWs8J.js +11 -0
- package/public/assets/{useSortMode-BaeAA5cw.js → useSortMode-BhPky1lW.js} +1 -1
- package/public/docs/404.html +2 -2
- package/public/docs/assets/{app.APoq_UBs.js → app.BczRxXPr.js} +1 -1
- package/public/docs/assets/chunks/@localSearchIndexroot.c2HfrwDg.js +1 -0
- package/public/docs/assets/chunks/{VPLocalSearchBox.B3TPzlHq.js → VPLocalSearchBox.DXvkTVnC.js} +1 -1
- package/public/docs/assets/chunks/{theme.PavXO_vz.js → theme.ByUdwrLG.js} +2 -2
- package/public/docs/assets/contributor-guide_architecture_board-widgets.md.9ToZXcTk.js +1 -0
- package/public/docs/assets/{contributor-guide_architecture_board-widgets.md.CWpj01WU.lean.js → contributor-guide_architecture_board-widgets.md.9ToZXcTk.lean.js} +1 -1
- package/public/docs/assets/contributor-guide_architecture_detector-contract.md.B1vUqtry.js +1 -0
- package/public/docs/assets/{contributor-guide_architecture_detector-contract.md.CgzUsQ6W.lean.js → contributor-guide_architecture_detector-contract.md.B1vUqtry.lean.js} +1 -1
- package/public/docs/assets/{contributor-guide_architecture_drill-down.md.BtPdlM7r.js → contributor-guide_architecture_drill-down.md.Bawx6NgP.js} +1 -1
- package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.DHVq1UPt.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.DHVq1UPt.lean.js +1 -0
- package/public/docs/assets/{contributor-guide_architecture_overview.md.CehiRmGn.js → contributor-guide_architecture_overview.md.DkXdJ9jE.js} +1 -1
- package/public/docs/assets/contributor-guide_architecture_overview.md.DkXdJ9jE.lean.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.DYwFeyjV.js +1 -0
- package/public/docs/assets/{contributor-guide_architecture_worker-protocol.md.IbnfNrV3.js → contributor-guide_architecture_worker-protocol.md.C0SovZX8.js} +2 -2
- package/public/docs/assets/{contributor-guide_architecture_worker-protocol.md.IbnfNrV3.lean.js → contributor-guide_architecture_worker-protocol.md.C0SovZX8.lean.js} +1 -1
- package/public/docs/assets/{contributor-guide_development-setup.md.DvAN_9mK.js → contributor-guide_development-setup.md.DvMV1xhQ.js} +3 -3
- package/public/docs/assets/contributor-guide_development-setup.md.DvMV1xhQ.lean.js +1 -0
- package/public/docs/assets/{contributor-guide_testing.md.6rIKqSyY.js → contributor-guide_testing.md.BohuQWvl.js} +1 -1
- package/public/docs/assets/{contributor-guide_testing.md.6rIKqSyY.lean.js → contributor-guide_testing.md.BohuQWvl.lean.js} +1 -1
- package/public/docs/assets/{tuning-reference_bottleneck-failures.md.4z5BXGJ2.js → tuning-reference_bottleneck-failures.md.CAsFf3RR.js} +1 -1
- package/public/docs/assets/{tuning-reference_bottleneck-gc.md.DSxzZRK7.js → tuning-reference_bottleneck-gc.md.B4T3kJYV.js} +2 -2
- package/public/docs/assets/{tuning-reference_bottleneck-gc.md.DSxzZRK7.lean.js → tuning-reference_bottleneck-gc.md.B4T3kJYV.lean.js} +1 -1
- package/public/docs/assets/tuning-reference_bottleneck-shuffle.md.DDMQVOW6.js +12 -0
- package/public/docs/assets/{tuning-reference_bottleneck-shuffle.md.CM-nTmIH.lean.js → tuning-reference_bottleneck-shuffle.md.DDMQVOW6.lean.js} +1 -1
- package/public/docs/assets/{tuning-reference_bottleneck-skew.md.BdUwiDhn.js → tuning-reference_bottleneck-skew.md.B0MzhWcX.js} +4 -4
- package/public/docs/assets/{tuning-reference_bottleneck-skew.md.BdUwiDhn.lean.js → tuning-reference_bottleneck-skew.md.B0MzhWcX.lean.js} +1 -1
- package/public/docs/assets/{tuning-reference_bottleneck-small-files.md.B8kloyx8.js → tuning-reference_bottleneck-small-files.md.DHGMTmPs.js} +2 -2
- package/public/docs/assets/{tuning-reference_bottleneck-small-files.md.B8kloyx8.lean.js → tuning-reference_bottleneck-small-files.md.DHGMTmPs.lean.js} +1 -1
- package/public/docs/assets/{tuning-reference_bottleneck-straggler.md.DY36fHN5.js → tuning-reference_bottleneck-straggler.md.DIbXdAcQ.js} +2 -2
- package/public/docs/assets/{tuning-reference_bottleneck-straggler.md.DY36fHN5.lean.js → tuning-reference_bottleneck-straggler.md.DIbXdAcQ.lean.js} +1 -1
- package/public/docs/assets/{tuning-reference_bottleneck-tiny-tasks.md.QTV7O8kU.js → tuning-reference_bottleneck-tiny-tasks.md.CB05v3zh.js} +1 -1
- package/public/docs/assets/tuning-reference_bottleneck-utilization.md.DqC_iGkw.js +5 -0
- package/public/docs/assets/tuning-reference_bottleneck-utilization.md.DqC_iGkw.lean.js +1 -0
- package/public/docs/assets/{tuning-reference_cluster-config.md.ZVmDGsQ3.js → tuning-reference_cluster-config.md.CD6vzYtD.js} +1 -1
- package/public/docs/assets/{tuning-reference_cluster-config.md.ZVmDGsQ3.lean.js → tuning-reference_cluster-config.md.CD6vzYtD.lean.js} +1 -1
- package/public/docs/assets/{tuning-reference_memory-model.md.DhT-n4y3.js → tuning-reference_memory-model.md.CWhvgevP.js} +1 -1
- package/public/docs/assets/{tuning-reference_memory-model.md.DhT-n4y3.lean.js → tuning-reference_memory-model.md.CWhvgevP.lean.js} +1 -1
- package/public/docs/assets/{user-guide_mcp-tools.md.Vi3RoflJ.js → user-guide_mcp-tools.md.DGvolGzt.js} +1 -1
- package/public/docs/assets/user-guide_understanding-findings.md.B29f0ucS.js +1 -0
- package/public/docs/assets/{user-guide_understanding-findings.md.DL1UDhvR.lean.js → user-guide_understanding-findings.md.B29f0ucS.lean.js} +1 -1
- package/public/docs/contributor-guide/architecture/board-widgets.html +5 -5
- package/public/docs/contributor-guide/architecture/detector-contract.html +5 -5
- package/public/docs/contributor-guide/architecture/drill-down.html +5 -5
- package/public/docs/contributor-guide/architecture/impact-estimation.html +5 -5
- package/public/docs/contributor-guide/architecture/index.html +3 -3
- package/public/docs/contributor-guide/architecture/overview.html +5 -5
- package/public/docs/contributor-guide/architecture/state-and-history.html +3 -3
- package/public/docs/contributor-guide/architecture/widget-rendering.html +5 -5
- package/public/docs/contributor-guide/architecture/worker-protocol.html +6 -6
- package/public/docs/contributor-guide/contributing.html +3 -3
- package/public/docs/contributor-guide/development-setup.html +5 -5
- package/public/docs/contributor-guide/testing.html +5 -5
- package/public/docs/hashmap.json +1 -1
- package/public/docs/index.html +3 -3
- package/public/docs/tuning-reference/anti-patterns.html +3 -3
- package/public/docs/tuning-reference/aqe.html +3 -3
- package/public/docs/tuning-reference/bottleneck-broadcast-sizing.html +3 -3
- package/public/docs/tuning-reference/bottleneck-cold-start.html +3 -3
- package/public/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html +3 -3
- package/public/docs/tuning-reference/bottleneck-failures.html +5 -5
- package/public/docs/tuning-reference/bottleneck-gc.html +5 -5
- package/public/docs/tuning-reference/bottleneck-job-failure-rate.html +3 -3
- package/public/docs/tuning-reference/bottleneck-memory-utilization.html +3 -3
- package/public/docs/tuning-reference/bottleneck-retry-waste.html +3 -3
- package/public/docs/tuning-reference/bottleneck-shuffle.html +6 -6
- package/public/docs/tuning-reference/bottleneck-skew.html +7 -7
- package/public/docs/tuning-reference/bottleneck-slow-host.html +3 -3
- package/public/docs/tuning-reference/bottleneck-small-files.html +6 -6
- package/public/docs/tuning-reference/bottleneck-spill.html +3 -3
- package/public/docs/tuning-reference/bottleneck-straggler.html +5 -5
- package/public/docs/tuning-reference/bottleneck-tiny-tasks.html +5 -5
- package/public/docs/tuning-reference/bottleneck-utilization.html +6 -6
- package/public/docs/tuning-reference/caching.html +3 -3
- package/public/docs/tuning-reference/cluster-config.html +5 -5
- package/public/docs/tuning-reference/config.html +3 -3
- package/public/docs/tuning-reference/data-formats.html +3 -3
- package/public/docs/tuning-reference/index.html +3 -3
- package/public/docs/tuning-reference/intro.html +3 -3
- package/public/docs/tuning-reference/joins.html +3 -3
- package/public/docs/tuning-reference/memory-model.html +5 -5
- package/public/docs/tuning-reference/metrics.html +3 -3
- package/public/docs/tuning-reference/partitioning.html +3 -3
- package/public/docs/tuning-reference/pyspark.html +3 -3
- package/public/docs/tuning-reference/shuffle.html +3 -3
- package/public/docs/tuning-reference/spark-architecture.html +3 -3
- package/public/docs/tuning-reference/table-formats.html +3 -3
- package/public/docs/user-guide/alternative-log-retrieval.html +3 -3
- package/public/docs/user-guide/getting-started.html +3 -3
- package/public/docs/user-guide/mcp-tools.html +5 -5
- package/public/docs/user-guide/run-comparison.html +3 -3
- package/public/docs/user-guide/understanding-findings.html +5 -5
- package/public/index.html +1 -1
- package/vendor-core/cli/collect-run.js +3 -2
- package/vendor-core/cli/native-zstd.js +351 -0
- package/vendor-core/detectors.js +243 -53
- package/vendor-core/docs-config.js +34 -8
- package/vendor-core/docs-content/chapters/03-memory-model.md +39 -0
- package/vendor-core/docs-content/chapters/11-cluster-config.md +40 -0
- package/vendor-core/docs-content/detection/gc.md +2 -0
- package/vendor-core/docs-content/detection/host.md +2 -1
- package/vendor-core/docs-content/detection/plan.md +3 -1
- package/vendor-core/docs-content/detection/shape.md +2 -1
- package/vendor-core/docs-content/detection/shfl.md +2 -1
- package/vendor-core/docs-content/detection/spill.md +1 -1
- package/vendor-core/docs-content/detection/strag.md +2 -1
- package/vendor-core/docs-content/detection/tiny.md +2 -1
- package/vendor-core/docs-content/tuning/failures.md +1 -1
- package/vendor-core/docs-content/tuning/gc.md +11 -4
- package/vendor-core/docs-content/tuning/shuffle.md +25 -5
- package/vendor-core/docs-content/tuning/skew.md +14 -6
- package/vendor-core/docs-content/tuning/small-files.md +12 -7
- package/vendor-core/docs-content/tuning/straggler.md +34 -0
- package/vendor-core/docs-content/tuning/tiny-tasks.md +1 -1
- package/vendor-core/docs-content/tuning/utilization.md +57 -7
- package/vendor-core/docs-content/upstream.json +4 -0
- package/vendor-core/event-handlers.js +321 -69
- package/vendor-core/event-schemas.js +8 -6
- package/vendor-core/evidence-report.js +3 -1
- package/vendor-core/impact-estimator.js +150 -34
- package/vendor-core/mcp-tools.js +20 -7
- package/vendor-core/occupancy.js +70 -2
- package/vendor-core/parser-worker.js +28 -12
- package/vendor-core/plan-summary.js +4 -0
- package/vendor-core/run-comparison.js +1 -15
- package/vendor-core/shs-fetch.js +18 -7
- package/vendor-core/shs-load.js +2 -1
- package/vendor-core/stage-quantiles.js +59 -3
- package/vendor-core/string-hash.js +15 -0
- package/vendor-core/types.js +9 -1
- package/vendor-core/vendor/fzstd.js +94 -18
- package/public/assets/CacheUtilization-lKua3J1x.js +0 -1
- package/public/assets/CachingOpportunity-U0g-8HPV.js +0 -1
- package/public/assets/ConfigAudit-ClZp5BpV.js +0 -1
- package/public/assets/Dashboard-Bjl-MYMm.js +0 -1
- package/public/assets/GcPressure-CYsmU1d7.js +0 -1
- package/public/assets/PartitionSizing-Cz_B0-uG.js +0 -1
- package/public/assets/PlanGraphRoute-Cj-GWM8W.js +0 -1
- package/public/assets/ShuffleIO-pyDhxdaz.js +0 -1
- package/public/assets/Straggler-dffcAN7t.js +0 -1
- package/public/assets/index-3vlFVNCg.js +0 -9
- package/public/assets/parser-worker-B2bc0lWe.js +0 -112
- package/public/assets/useRecentFiles-D0OA_EOK.js +0 -11
- package/public/docs/assets/chunks/@localSearchIndexroot.CvM5fuxe.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_board-widgets.md.CWpj01WU.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_detector-contract.md.CgzUsQ6W.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.CooslVJt.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.CooslVJt.lean.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_overview.md.CehiRmGn.lean.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.R27gQrgY.js +0 -1
- package/public/docs/assets/contributor-guide_development-setup.md.DvAN_9mK.lean.js +0 -1
- package/public/docs/assets/tuning-reference_bottleneck-shuffle.md.CM-nTmIH.js +0 -12
- package/public/docs/assets/tuning-reference_bottleneck-utilization.md.DTiueZC3.js +0 -5
- package/public/docs/assets/tuning-reference_bottleneck-utilization.md.DTiueZC3.lean.js +0 -1
- package/public/docs/assets/user-guide_understanding-findings.md.DL1UDhvR.js +0 -1
- /package/public/docs/assets/{contributor-guide_architecture_drill-down.md.BtPdlM7r.lean.js → contributor-guide_architecture_drill-down.md.Bawx6NgP.lean.js} +0 -0
- /package/public/docs/assets/{contributor-guide_architecture_widget-rendering.md.R27gQrgY.lean.js → contributor-guide_architecture_widget-rendering.md.DYwFeyjV.lean.js} +0 -0
- /package/public/docs/assets/{tuning-reference_bottleneck-failures.md.4z5BXGJ2.lean.js → tuning-reference_bottleneck-failures.md.CAsFf3RR.lean.js} +0 -0
- /package/public/docs/assets/{tuning-reference_bottleneck-tiny-tasks.md.QTV7O8kU.lean.js → tuning-reference_bottleneck-tiny-tasks.md.CB05v3zh.lean.js} +0 -0
- /package/public/docs/assets/{user-guide_mcp-tools.md.Vi3RoflJ.lean.js → user-guide_mcp-tools.md.DGvolGzt.lean.js} +0 -0
|
@@ -9,11 +9,11 @@
|
|
|
9
9
|
<link rel="preload stylesheet" href="/docs/assets/style.DSixAiZE.css" as="style">
|
|
10
10
|
<link rel="preload stylesheet" href="/docs/vp-icons.css" as="style">
|
|
11
11
|
|
|
12
|
-
<script type="module" src="/docs/assets/app.
|
|
12
|
+
<script type="module" src="/docs/assets/app.BczRxXPr.js"></script>
|
|
13
13
|
<link rel="preload" href="/docs/assets/inter-roman-latin.Di8DUHzh.woff2" as="font" type="font/woff2" crossorigin="">
|
|
14
|
-
<link rel="modulepreload" href="/docs/assets/chunks/theme.
|
|
14
|
+
<link rel="modulepreload" href="/docs/assets/chunks/theme.ByUdwrLG.js">
|
|
15
15
|
<link rel="modulepreload" href="/docs/assets/chunks/framework.DSg0KOwT.js">
|
|
16
|
-
<link rel="modulepreload" href="/docs/assets/contributor-guide_architecture_impact-estimation.md.
|
|
16
|
+
<link rel="modulepreload" href="/docs/assets/contributor-guide_architecture_impact-estimation.md.DHVq1UPt.lean.js">
|
|
17
17
|
<link rel="icon" type="image/svg+xml" href="/docs/favicon.svg">
|
|
18
18
|
<link rel="preconnect" href="https://fonts.googleapis.com">
|
|
19
19
|
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin="">
|
|
@@ -22,8 +22,8 @@
|
|
|
22
22
|
<script id="check-mac-os">document.documentElement.classList.toggle("mac",/Mac|iPhone|iPod|iPad/i.test(navigator.platform));</script>
|
|
23
23
|
</head>
|
|
24
24
|
<body>
|
|
25
|
-
<div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Contributor Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/development-setup.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Development setup</p><!--]--></a><!----></div><!----></div><section class="VPSidebarItem level-1 collapsible collapsed is-link has-active" data-v-b3fd67f8><div class="item" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/" data-v-b3fd67f8><!--[--><h3 class="text" data-v-b3fd67f8>Architecture</h3><!--]--></a><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-b3fd67f8><span class="vpi-chevron-right caret-icon" data-v-b3fd67f8></span></div></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/overview.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Overview</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/worker-protocol.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Worker protocol</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/state-and-history.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>State & history intake</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/detector-contract.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Detector contract</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/impact-estimation.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Impact estimation</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/widget-rendering.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Widget rendering</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/board-widgets.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Board widgets</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/drill-down.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Drill-down</p><!--]--></a><!----></div><!----></div><!--]--></div></section><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/testing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Testing & verification</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/contributing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Contributing</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_contributor-guide_architecture_impact-estimation" data-v-39a288b8><div><h1 id="impact-estimation" tabindex="-1">Impact estimation <a class="header-anchor" href="#impact-estimation" aria-label="Permalink to "Impact estimation""></a></h1><p>Every finding covered by this section carries an optional <code>impactEstimate: {basis, wallClock, estimateMethod, rawWaste?}</code> (<code>src/types.ts</code>), attached by <code>src/impact-estimator.ts</code> as a post-pass after <code>DETECTORS</code> finishes (<code>src/analyzer.ts</code>). <code>estimateMethod</code> ('measured' | 'modeled' | 'none') is a distinct axis from the per-finding <code>confidence</code> field (<a href="./board-widgets.html#confidence-metadata">Confidence metadata</a>): <code>confidence</code> says how much to trust the finding itself, <code>estimateMethod</code> says how its impact number was derived. <code>'none'</code> marks a purely informational finding with no waste model at all (<code>configAudit</code>, <code>stageFailed</code>, <code>failures</code>, <code>incompleteRun</code>, <code>slowHost</code>'s byte-dimension multiDim shapes): it's distinct from <code>'measured'</code>/<code>'modeled'</code>, which both attach a real (if approximate) formula. <code>basis</code> is one of:</p><ul><li><code>'serial'</code>: the tied stage ran (effectively) alone; <code>wallClock</code> is a near-point estimate, <code>low === high</code>.</li><li><code>'contended'</code>: the tied stage shared wall-clock time with others; <code>wallClock</code> is an honest range, <code>high</code> optimistic (assumes the fix could still fully land), <code>low</code> the guaranteed floor.</li><li><code>'resourceOnly'</code>: no wall-clock claim is defensible (not stage-tied by nature, or the stage was excluded from the occupancy sweep), <code>wallClock: null</code>, but the formula's real signal survives in <code>rawWaste</code>.</li><li><code>'informational'</code>: no quantifiable magnitude at all, <code>wallClock: null</code>, no <code>rawWaste</code>.</li></ul><p><code>{basis: 'resourceOnly'|'informational', wallClock: null}</code> replaced the earlier design's <code>{low: 0, high: 0}</code>: that single value used to mean two incompatible things ("provably no wall-clock cost" and "the model gave up"), and 76-97% of stage-tied findings on real logs were the second case wearing the first case's clothing (2026-08-30 N1 redesign; see <code>docs/superpowers/specs/2026-08-30-critical-path-occupancy-redesign.md</code>).</p><p><code>rawWaste</code> is not exclusive to <code>resourceOnly</code>/<code>informational</code> findings: every <code>serial</code>/ <code>contended</code> finding carries it too (the only exceptions are <code>coldStart</code>, whose <code>wallClock</code> figure is already unclipped, and <code>estimateMethod: 'none'</code> findings, which have no formula at all), holding the formula's pre-clip magnitude in the formula's own natural unit (ms, bytes or core-ms). <code>wallClock</code> is what the occupancy model says is recoverable, which clips against the stage's own physical floor; <code>rawWaste</code> is what the stage really wasted either way. The two answer different questions.</p><h2 id="occupancy-weighted-attribution" tabindex="-1">Occupancy-weighted attribution <a class="header-anchor" href="#occupancy-weighted-attribution" aria-label="Permalink to "Occupancy-weighted attribution""></a></h2><p><code>src/occupancy.ts</code> sweeps every stage's observed <code>[submittedAt, completedAt)</code> window and splits each instant's wall-clock among concurrently-active stages proportional to <code>coreWeight(S) = stage.executorRunTime / stageDurationMs</code> (an average-concurrency proxy, held constant across the stage's whole window: this codebase has no per-task timestamps outside the parser worker to do better). Summing a stage's share across its own window gives its <code>occupancy(S)</code>; <code>gate(S) = occupancy(S) / duration(S) ∈ [0, 1]</code> is the single number that replaces the old CPM model's <code>onCriticalPath</code>/<code>slackMs</code>/<code>isUniquelyCritical</code>. 1.0 means the stage ran completely alone; 0 means the stage had zero <code>executorRunTime</code> while overlapping other, positive-weight stages, so it got no share of the shared window. Stages with <code>duration(S) <= 0</code> (Spark-skipped stages, or a malformed <code>submittedAt === completedAt</code>) are excluded from the sweep entirely.</p><p>This mechanism replaced a CPM (critical-path-method) graph over <code>parentIds</code> that produced near-zero on-critical-path membership on real logs (0.1-10.6% of stages, max graph depth 2 on 5 of 6 real logs measured): <code>parentIds</code> alone is too sparse a precedence signal for a meaningful longest-path computation. The same degeneracy fed <code>efficiency-model.ts</code>'s <code>floorInfiniteMs</code> ("floor with infinite executors"); rather than leave a second, unreconciled critical-path number in the codebase for a future UI to display next to the occupancy-based figures above, <code>criticalPathMs()</code>/<code>CriticalPathStage</code> (embedded in <code>efficiency-model.ts</code>, never a standalone module) were removed outright (no occupancy-based replacement: occupancy apportions observed concurrent time, it doesn't compute a dependency-graph longest path, so there's no drop-in equivalent). <code>efficiency-model.ts</code> now reports only <code>floorZeroSkewMs</code> (total task time / total cores) as its theoretical floor.</p><p><code>ceiling(S) = max(stage.taskDurationMax, stage.executorRunTime / totalCores)</code> is a physical floor on a stage's own duration: bounded below by its single longest task (unsplittable no matter how much parallelism exists) or by its core-work spread across every core in the cluster, whichever is larger. Every waste formula's raw claim is clipped against it before gate-weighting: <code>wasteMs_clipped(S) = min(wasteMs_claimed, max(0, duration(S) - ceiling(S)))</code>, so a finding can never claim to save more than the portion of the stage's observed duration that sits above its own unbeatable floor. This is what fixes historical overclaim bugs (a <code>tinyTask</code> finding claiming 407.5s on a 13.1s stage capped to 8.9s; a <code>shuffle</code> finding claiming 1939.9s on a 991.3s/1688.3s stage capped to 610.3s/250.1s).</p><p><code>analyzer.ts</code> feeds this <code>totalCores</code> from <code>src/core-count.ts</code>'s <code>computePeakConcurrentCores(app, executorsAdded, executorsRemoved)</code>, not the shared <code>computeTotalCores</code> helper. <code>computeTotalCores</code> sums every <code>ExecutorAdded</code> event's cores regardless of overlap, so under dynamic allocation or executor replacement it can far exceed the cores ever actually concurrent, which understates <code>ceiling(S)</code> and lets churn inflate a finding's claimed wall-clock. <code>computePeakConcurrentCores</code> instead sweeps add/remove events by timestamp and tracks the running total's peak, so a churned-through executor's cores are never double-counted against its replacement's. Same-timestamp events tie-break by delta ascending, so a removal applies before a same-instant replacement's addition (otherwise a same-instant swap would momentarily double-count both as concurrent). If every <code>executorsAdded</code> entry lacks <code>totalCores</code> the cores sweep peaks at zero and tells us nothing; the function then falls back to sweeping peak <em>executor count</em> instead (still concurrency-aware, just cores-blind) and multiplies by the configured per-executor core count, rather than falling back to <code>executorsAdded.length × cores</code>, which would reintroduce the exact cumulative-overcount-under-churn bug this function exists to avoid.</p><p>Per-finding estimate, using <code>wasteMs_clipped(S)</code>:</p><ul><li><code>gate(S) >= 0.999</code>: <code>basis: 'serial'</code>, <code>low = high = wasteMs_clipped(S)</code>.</li><li><code>gate(S) < 0.999</code>: <code>basis: 'contended'</code>, <code>high = wasteMs_clipped(S)</code>, <code>low = wasteMs_clipped(S) * gate(S)</code>.</li></ul><p>A finding spanning multiple stages (<code>stageIds</code>, plural) sums each stage's own estimate and caps the joint total at the union of just that finding's own stage windows (via <code>mergeIntervals</code>, <code>src/wall-clock.ts</code>): <code>high = min(Σ high_i, unionMs(stageIds))</code>, <code>low = min(Σ low_i, unionMs(stageIds))</code>. This is what prevents overclaiming when two or more of a finding's stages overlap in wall-clock time: a plain sum-and-cap, no CPM re-simulation. The union cap can force <code>low === high</code> numerically even when the constituent stages were individually contended (e.g. two fully-overlapping stages each at <code>gate</code> 0.5), so <code>basis</code> isn't derived from that numeric equality: a multi-stage finding gets <code>basis: 'serial'</code> only when every one of its per-stage estimates was itself <code>'serial'</code>; otherwise <code>'contended'</code>.</p><p>Regression guard: a finding whose stage (or, for a multi-stage finding, every one of its stages) ran alone (<code>gate >= 0.95</code>, deliberately looser than the <code>0.999</code> "serial" cutoff above: this guard exists to catch egregious false zeros, not to gate which <code>basis</code> a finding gets) with a real underlying magnitude (<code>rawWaste.value > 0</code>) must never report <code>wallClock.high === 0</code> or <code>basis: 'informational'/'resourceOnly'</code>. Covered by <code>tests/impact-estimator-real-log.test.js</code> against a real fixture; relies on <code>rawWaste</code> being attached to every serial/contended-capable formula (see above), so it's blind only to <code>coldStart</code> and the purely informational (<code>estimateMethod: 'none'</code>) finding types.</p><p>Real-log spot-check (2026-08-30, <code>grupo-semanal-beauty-application_1785266278671_91660.zstd</code>): median <code>gate</code> across stages was <code>≈0.34</code> (0.3428060791718594 exactly); <code>collectRun</code> plus the occupancy sweep together took <code>≈6,589</code>ms on the largest fixture measured (<code>run-compare-calimax-candidate-application_1784568768686_119096.zstd</code>, <code>1169</code> stages): parse-dominated, the sweep alone was not isolated by this measurement, but not a magnitude that suggests a regression either. <code>54</code> previously-<code>{0,0}</code> stage-tied findings on stages that ran effectively alone now report a real <code>wallClock</code> range instead.</p><h2 id="cross-finding-rollup-computestageunionms" tabindex="-1">Cross-finding rollup: <code>computeStageUnionMs</code> <a class="header-anchor" href="#cross-finding-rollup-computestageunionms" aria-label="Permalink to "Cross-finding rollup: `computeStageUnionMs`""></a></h2><p>The Findings tab's recommendation rollup (<code>FixTheseFirst.tsx</code>, built from <code>buildRecommendationRollup</code>, <code>src/recommendation-rollup.ts</code>) groups the filtered catalog by detector <code>type</code>, then needs its own cap for a group of several findings of that type, not just one finding's own <code>stageIds</code>. Summing each finding's already-clipped <code>wallClock.high</code> naively double-counts any stage two of those findings both touch. <code>computeStageUnionMs(stageIds, stages)</code> covers this: collect every stage touched by any finding in the group, merge their <code>[submittedAt, completedAt)</code> intervals, and sum the merged intervals' durations, so the group's wall-clock union holds regardless of how many findings' <code>stageIds</code> overlap. Stages missing either bound are skipped rather than defaulted to <code>0</code> (the same filter <code>computeWallClock</code> applies), so a truncated log (the case <code>incompleteRun</code> flags) can't contribute a negative interval and a negative recoverable-time figure.</p><p>It reuses the same <code>mergeIntervals</code> primitive (<code>src/wall-clock.ts</code>) that backs <code>src/occupancy.ts</code>'s per-finding <code>estimateMultiStage</code>/its internal <code>unionMs</code> sum, but is not an extension of that function: <code>estimateMultiStage</code> caps one finding's own multi-stage claim during the impact-estimation pass, before a <code>Finding</code> object even exists; <code>computeStageUnionMs</code> runs later, in the view layer, capping a naive sum <em>across</em> several already-estimated findings that happen to share a detector <code>type</code>. <code>buildTimeGroup</code> (same module) takes the smaller of the naive per-finding sum and this union figure as the group's <code>recoverableMsHigh</code>, falling back to the naive sum untouched when the group's findings carry no stage IDs at all (nothing to union against).</p><p>Within one <code>type</code> group, <code>buildRecommendationRollup</code> splits findings into up to three tiers, always rendered in this fixed order. <code>time</code> covers findings with a real <code>impactEstimate.wallClock</code> (<code>buildTimeGroup</code>, the union-capped figure above). <code>resource</code> covers findings with no <code>wallClock</code> but a <code>rawWaste</code> figure (<code>buildResourceGroup</code>), grouped again by <code>rawWaste.unit</code> so a <code>bytes</code> total never gets summed against a <code>coreHours</code> total under one type. <code>count</code> covers findings with neither (<code>buildCountGroup</code>), a plain per-impact-band tally with no magnitude claim at all. Each <code>RollupGroup</code> also carries its own <code>findings: Finding[]</code> (the exact members that fed the aggregate), which <code>FixTheseFirst.tsx</code> reads directly to pick a group's highest-impact member and to render its expanded, paginated list.</p><p>A type only contributes a tier when it has at least one finding of that kind; most types produce exactly one tier, but a type whose formula varies by <code>variant</code>/<code>rule</code> (e.g. <code>memoryUtilization</code>, see the coverage table below) can produce more than one.</p><p><code>cachingOpportunity</code> and <code>cacheUtilization</code> are both <code>cost-only</code>: <code>basis: 'resourceOnly'</code>, <code>wallClock: null</code>, but <code>rawWaste.unit</code> is <code>'ms'</code>, the same unit a real <code>wallClock</code> figure would use, because their formula's natural output happens to be time (a re-read cost), not because either finding makes a wall-clock claim. Left unlabeled, a <code>resource</code>-tier "ms" total sitting next to a <code>time</code>-tier "recoverable time" total would read as directly comparable when it isn't: the resource figure was never gate-clipped against any stage's occupancy, so it can exceed what the stage actually spent. <code>FixTheseFirst.tsx</code> calls this out via its trailing-stat copy: a <code>resource</code>- kind group (any unit, including <code>ms</code>) always reads "resource-cost projection", never "recoverable", so the two ms-shaped numbers are never mistaken for the same kind of claim.</p><h2 id="per-formula-spot-checks" tabindex="-1">Per-formula spot-checks <a class="header-anchor" href="#per-formula-spot-checks" aria-label="Permalink to "Per-formula spot-checks""></a></h2><table tabindex="0"><thead><tr><th>Detector</th><th>Formula basis</th><th>Spot-check</th></tr></thead><tbody><tr><td>gc</td><td><code>jvmGCTime / (executorRunTime / stageDurationMs)</code></td><td><code>grupo-semanal-beauty-application_1785266278671_91660.zstd</code>, stage 507: <code>jvmGCTime</code>=1080ms, <code>executorRunTime</code>=27509ms, <code>stageDurationMs</code>=56279ms → <code>wasteMs</code> = 1080 / (27509/56279) ≈ 2209.5ms. That's ≈3.9% of the stage's 56.3s wall-clock duration, matching the finding's own reported <code>gcPct</code> (3.9%) exactly, as the formula guarantees by construction. Under the occupancy model this stage's <code>gate</code> is <code>0.041</code> (0.04145044590332269 exactly): <code>basis: 'contended'</code>, <code>wallClock: {low: 91.6, high: 2209.5}</code> (91.5850382272182 / 2209.506706895925 exactly, per the Step 1 script's per-stage output).</td></tr><tr><td>shuffle</td><td><code>shuffleReadBytes / SHUFFLE_THROUGHPUT_BPS</code> (fallback tier; real-metrics parser not yet built)</td><td><code>ventas-mensual-multi-big-application_1785266278671_91510.zstd</code>, stage 99 (<code>SHFL</code> finding): <code>shuffleReadBytes</code>=204,172,518,504 → <code>wasteMs</code> = 204172518504 / 125,000,000 × 1000 ≈ 1,633,380ms, matching <code>rawWaste.value</code> exactly. Eyeball against the timeline: the stage's actual wall-clock duration is only 763,776ms, i.e. this run moved shuffle data at ≈267MB/s, roughly 2x the assumed 125MB/s (1Gbps) constant: expected for the fallback tier's deliberately conservative assumption, but worth knowing the modeled figure runs high on fast-network clusters. Under the occupancy model this stage's <code>gate</code> is <code>1</code>, <code>ceiling</code> is <code>≈434,568.1</code>ms (434568.1041666667 exactly): <code>wallClock</code> capped to <code>≈329,207.9</code>ms (329207.8958333333 exactly), since the raw 1,633,380ms claim vastly exceeds the stage's own 763,776ms duration.</td></tr><tr><td>spill</td><td><code>diskBytesSpilled / SPILL_IO_THROUGHPUT_BPS</code></td><td>Same run and stage (99): <code>diskBytesSpilled</code>=145,978,433,675 (note: the <code>SPILL</code> finding's own <code>value</code>/<code>metric</code> report <code>memoryBytesSpilled</code>=913,686,966,448, ~6x larger; the formula correctly uses the smaller disk figure, not that one) → <code>wasteMs</code> = 145978433675 / 200,000,000 × 1000 ≈ 729,892ms, matching <code>rawWaste.value</code> exactly. Eyeball: that's ≈191MB/s of implied disk throughput against the stage's 763,776ms actual duration, close to the assumed 200MB/s constant. Same ceiling-clip caveat as the shuffle row above applies here too, on the same stage.</td></tr></tbody></table><h2 id="overlap-caveat-skew-straggler" tabindex="-1">Overlap caveat: skew / straggler <a class="header-anchor" href="#overlap-caveat-skew-straggler" aria-label="Permalink to "Overlap caveat: skew / straggler""></a></h2><p><code>skew</code> (small-stage max-P50 fallback branch) and <code>straggler</code> can both fire on the same stage from the same single dominant outlier task, and each is clipped independently. This phase does not dedupe or suppress either: each keeps its own independently-computed <code>wallClock</code>. Do not sum <code>wallClock.high</code> across multiple findings on the same stage: if both fire together, they describe the same underlying waste, not two separate wastes. This overlap caveat is orthogonal to (and compounds with) the ceiling clip above: a stage with one dominant outlier task trips both detectors <em>and</em> has a small <code>ceiling</code>-derived recoverable room, since <code>ceiling</code> is itself <code>>= taskDurationMax</code>, the very quantity these two detectors are reacting to.</p><p><code>analyzer.ts</code>'s <code>flagSkewStragglerOverlap</code> (run after <code>deriveImpactBand</code>, once per <code>analyze()</code> call) surfaces this caveat to the reader instead of leaving it as an internal-only comment: whenever <code>skew</code>'s <code>max/median</code> branch and <code>straggler</code> both fire on the same <code>stageId</code>, it appends a "this overlaps with the X finding on this stage" sentence to both findings' <code>validationRequired</code> text (rather than suppressing either, so neither finding's own diagnostic value is lost). <code>skew</code>'s <code>P95/median</code> branch samples a different task from <code>straggler</code>'s own <code>taskDurationMax - taskDurationP50</code> delta, so it's excluded from the flag. The note rides the same confidence-caveat UI (<code>RowStatusCluster</code>) a reader already sees before trusting either finding's magnitude, since both detectors also carry a <code>confidence</code> field that scales <code>low</code>/<code>medium</code>/<code>high</code> off how far the finding sits past its own runtime-floor threshold (still unvalidated; see the confidence-disclosure note in detector-contract.md).</p><p><code>stageShape</code>'s <code>taskStageSkew</code> rule no longer participates in this caveat: it reports a <code>resourceOnly</code> idle-core-ms figure (see the coverage table below) instead of a wall-clock claim, so there's nothing left to double-count against <code>skew</code>/<code>straggler</code>. Its trigger condition (<code>taskDurationMax / stageDurationMs > skewWarn</code>) mathematically forces the occupancy-clipped wall-clock estimate to exactly zero on every firing (see <code>src/detectors.ts</code>'s <code>taskStageSkew</code> comment), which is why it was moved off the wall-clock path entirely rather than reconciled against the same ceiling clip as its two siblings above.</p><h2 id="per-finding-type-coverage" tabindex="-1">Per-finding-type coverage <a class="header-anchor" href="#per-finding-type-coverage" aria-label="Permalink to "Per-finding-type coverage""></a></h2><p>One row per distinct <code>type</code> string <code>src/detectors.ts</code> actually emits (cross-checked against <code>computeEstimateForFinding</code>'s <code>case</code> labels in <code>src/impact-estimator.ts</code>, not assumed from the prose here): every row below has a case, so the table itself is the coverage count, not a number restated here. <code>broadcastSizing</code> is a <code>DETECTORS</code> entry label only, and the plan-walk it drives emits <code>overBroadcast</code>/<code>underBroadcast</code> findings instead, so those two are the rows that appear, not <code>broadcastSizing</code> itself. Tag meanings: <code>measured</code> and <code>modeled</code> both produce a real, gate-clipped, non-<code>{0,0}</code><code>wallClock</code> (the difference is whether the formula's inputs are recorded per-stage fields or an assumed constant like a throughput figure); <code>cost-only</code> always reports <code>basis: 'resourceOnly'</code>, <code>wallClock: null</code> but carries its real signal in <code>rawWaste</code>; <code>informational-only</code> reports <code>basis: 'informational'</code>, <code>wallClock: null</code> with no <code>rawWaste</code> at all, since there's nothing quantifiable. A type with more than one tag fires a different formula per <code>variant</code>/<code>rule</code> on the same finding type; the basis column says which.</p><table tabindex="0"><thead><tr><th>Finding type</th><th>Scope</th><th>Tag</th><th>Basis</th></tr></thead><tbody><tr><td><code>retryWaste</code></td><td>stage</td><td>measured</td><td><code>retryWasteMs</code>, gate-clipped; pre-clip figure kept as <code>rawWaste</code> in <code>ms</code></td></tr><tr><td><code>speculationWaste</code></td><td>stage</td><td>measured</td><td><code>speculationWasteMs</code>, gate-clipped; pre-clip figure kept as <code>rawWaste</code> in <code>ms</code></td></tr><tr><td><code>coldStart</code></td><td>app</td><td>measured</td><td><code>gapSeconds × 1000</code>, unclipped, <code>basis: 'serial'</code> unconditionally (a pre-first-task gap can't overlap any stage)</td></tr><tr><td><code>gc</code></td><td>stage</td><td>modeled</td><td><code>jvmGCTime / (executorRunTime / stageDurationMs)</code>, gate-clipped: the concurrency division is an approximation, not a reconstruction, hence <code>modeled</code>; <code>rawWaste</code> in <code>coreMs</code> is the raw <code>jvmGCTime</code> sum before that conversion</td></tr><tr><td><code>skew</code></td><td>stage</td><td>measured</td><td><code>taskDurationP95</code> or <code>Max</code> minus <code>P50</code> (per <code>metric</code>), gate-clipped; pre-clip figure kept as <code>rawWaste</code> in <code>ms</code></td></tr><tr><td><code>straggler</code></td><td>stage</td><td>measured</td><td><code>taskDurationMax − taskDurationP50</code>, gate-clipped</td></tr><tr><td><code>stageShape</code></td><td>stage</td><td>cost-only</td><td>all three rules are <code>estimateMethod: 'measured'</code>, real per-stage fields, no assumed constant: <code>'lowParallelism'</code> → <code>rawWaste</code> in <code>coreMs</code> (idle cores × stage duration); <code>'dataExplosion'</code> → <code>rawWaste</code> in <code>bytes</code> (<code>outputBytes − inputBytes</code>); <code>'taskStageSkew'</code> → <code>rawWaste</code> in <code>coreMs</code> (<code>max(0, min(totalCores, taskCount) − 1) × (taskDurationMax − taskDurationP50)</code>, the cores idle during the straggler's tail at achieved concurrency)</td></tr><tr><td><code>slowHost</code></td><td>stage</td><td>measured / informational-only</td><td>duration-based variants (<code>hostMeanRatio</code>, <code>durationShare</code>, <code>multiDim</code>+<code>taskTime</code>): <code>value − taskDurationP50</code>, gate-clipped; byte-based <code>multiDim</code> dimensions: no formula yet</td></tr><tr><td><code>duplicatePlanSubtree</code></td><td>sql</td><td>measured</td><td>each contributing stage's real wall-clock duration × the redundant fraction <code>(occurrences − 1) / occurrences</code>, summed and capped at the finding's own <code>stageIds</code> union. <code>stageIds</code> is narrowed to the stages that actually ran the duplicated subtree's matched node instances (accumulator-ID evidence resolved onto each <code>PlanNode</code> at parse time, see <a href="./detector-contract.html#stage-id-attribution-for-plan-advisor-findings">Stage-ID attribution for Plan Advisor findings</a>), falling back to the whole execution's stages only when no matched instance has any accumulator coverage</td></tr><tr><td><code>shuffle</code></td><td>stage</td><td>modeled</td><td><code>shuffleReadBytes / SHUFFLE_THROUGHPUT_BPS</code> (assumed ~125MB/s), gate-clipped; <code>rawWaste</code> in <code>bytes</code> is the measured <code>shuffleReadBytes</code> behind it</td></tr><tr><td><code>spill</code></td><td>stage</td><td>modeled</td><td><code>diskBytesSpilled / SPILL_IO_THROUGHPUT_BPS</code> (assumed ~200MB/s), gate-clipped; <code>rawWaste</code> in <code>bytes</code> is <code>diskBytesSpilled</code>, which is the number the formula uses and not the <code>memoryBytesSpilled</code> the finding's own <code>metric</code> displays</td></tr><tr><td><code>stageSlowness</code></td><td>stage</td><td>modeled</td><td>stage duration minus the <code>stageSlowness</code> detector's own <code>infoMin</code> threshold, gate-clipped</td></tr><tr><td><code>partitionSizing</code></td><td>stage</td><td>modeled</td><td><code>maxPartitionTooBig</code>/<code>shufflePartitionSkew</code>: shuffle-throughput formulas, gate-clipped. <code>lowShuffleParallelism</code>: stage duration scaled down by the shortfall between actual and ideal-partition-count task counts (<code>stageDurationMs × (1 − taskCount / targetTaskCount)</code>), i.e. the serialized work more partitions would let run concurrently, not the scheduling cost of the tasks you'd add to fix it</td></tr><tr><td><code>tinyTask</code></td><td>stage</td><td>modeled</td><td>excess task count over 10% of the stage's actual count, × assumed per-task scheduling overhead, gate-clipped; pre-clip figure kept as <code>rawWaste</code> in <code>ms</code></td></tr><tr><td><code>smallFiles</code></td><td>sql</td><td>modeled / cost-only</td><td><code>excessFileCount × FILE_OPEN_OVERHEAD_MS</code>, summed and capped over <code>stageIds</code>'s union; with no <code>stageIds</code> to map to, cost-only with that same figure as <code>rawWaste</code> in <code>ms</code>. <code>stageIds</code> is narrowed the same way (see <a href="./detector-contract.html#stage-id-attribution-for-plan-advisor-findings">Stage-ID attribution for Plan Advisor findings</a>); falls back to the whole execution's stages when the flagged node(s) have no accumulator coverage.</td></tr><tr><td><code>overBroadcast</code></td><td>sql</td><td>modeled / cost-only</td><td><code>broadcastBytes / BROADCAST_BANDWIDTH_BPS</code>, summed and capped over <code>stageIds</code>'s union; cost-only with <code>rawWaste</code> in <code>ms</code> when not stage-mappable. <code>stageIds</code> is narrowed the same way; falls back to the whole execution's stages when the flagged node(s) have no accumulator coverage.</td></tr><tr><td><code>underBroadcast</code></td><td>sql</td><td>modeled / cost-only</td><td><code>smallerSideBytes / BROADCAST_BANDWIDTH_BPS</code>, summed and capped over <code>stageIds</code>'s union; cost-only with <code>rawWaste</code> in <code>ms</code> when not stage-mappable. <code>stageIds</code> is narrowed the same way; falls back to the whole execution's stages when the flagged node(s) have no accumulator coverage.</td></tr><tr><td><code>memoryUtilization</code></td><td>app</td><td>cost-only / informational-only</td><td>Three of the four variants report <code>rawWaste</code> in <code>mbSeconds</code>: <code>variant: 'wasteModel'</code> passes through its own <code>wastedMBSeconds</code>; <code>'idleCores'</code> uses <code>idleRateFraction × allocatedMB × peakExecutors × appDurationSeconds</code>; <code>'memoryBand'</code> with <code>rule: 'heapOverProvisioned'</code> uses <code>(allocatedBytes − heap) in MB × appDurationSeconds</code>. <code>'memoryBand'</code> with <code>rule: 'heapNearCapacity'</code> is an OOM-risk signal rather than a waste, and the <code>dataUnavailable</code> shape has no inputs at all: both informational-only</td></tr><tr><td><code>utilization</code></td><td>app</td><td>cost-only</td><td><code>rawWaste</code> in <code>coreHours</code>: <code>(1 − utilizationFraction) × appDurationMs × totalCores / 3.6e6</code></td></tr><tr><td><code>coreLocality</code></td><td>app</td><td>cost-only</td><td><code>rawWaste</code> in <code>coreMs</code>: <code>nonLocalTaskCount × NETWORK_FETCH_PENALTY_MS</code></td></tr><tr><td><code>autoscalingChurn</code></td><td>app</td><td>cost-only</td><td><code>rawWaste</code> in <code>coreHours</code>: <code>shortLivedExecutorCount × EXECUTOR_STARTUP_OVERHEAD_MS / 3.6e6</code></td></tr><tr><td><code>configAudit</code></td><td>config</td><td>informational-only</td><td>a config-drift check standing alone; no waste formula</td></tr><tr><td><code>jobFailureRate</code></td><td>app</td><td>cost-only</td><td><code>rawWaste</code> in <code>coreHours</code>: <code>failedJobCount × avgJobDurationMs / 3.6e6</code></td></tr><tr><td><code>cachingOpportunity</code></td><td>app</td><td>cost-only</td><td><code>rawWaste</code> in <code>ms</code>: <code>totalReadBytes / RE_READ_THROUGHPUT_BPS</code></td></tr><tr><td><code>cacheUtilization</code></td><td>app</td><td>cost-only</td><td><code>rawWaste</code> in <code>ms</code>: uncached-or-spilled bytes <code>/ RE_READ_THROUGHPUT_BPS</code>, where the never-cached partitions' bytes are extrapolated from the cached partitions' own average size (<code>memorySize + diskSize</code>, over <code>numCachedPartitions</code>), plus <code>diskSize</code> again for the already-cached-but-on-disk partitions' own re-read cost</td></tr><tr><td><code>stageFailed</code></td><td>stage</td><td>informational-only</td><td>no waste formula</td></tr><tr><td><code>failures</code></td><td>stage</td><td>informational-only</td><td>no waste formula</td></tr><tr><td><code>incompleteRun</code></td><td>app</td><td>informational-only</td><td>no waste formula</td></tr></tbody></table></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/contributor-guide/architecture/detector-contract.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>Detector contract</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="/docs/contributor-guide/architecture/widget-rendering.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>Widget rendering</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
|
|
26
|
-
<script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"
|
|
25
|
+
<div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Contributor Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/development-setup.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Development setup</p><!--]--></a><!----></div><!----></div><section class="VPSidebarItem level-1 collapsible collapsed is-link has-active" data-v-b3fd67f8><div class="item" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/" data-v-b3fd67f8><!--[--><h3 class="text" data-v-b3fd67f8>Architecture</h3><!--]--></a><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-b3fd67f8><span class="vpi-chevron-right caret-icon" data-v-b3fd67f8></span></div></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/overview.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Overview</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/worker-protocol.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Worker protocol</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/state-and-history.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>State & history intake</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/detector-contract.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Detector contract</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/impact-estimation.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Impact estimation</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/widget-rendering.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Widget rendering</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/board-widgets.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Board widgets</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/drill-down.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Drill-down</p><!--]--></a><!----></div><!----></div><!--]--></div></section><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/testing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Testing & verification</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/contributing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Contributing</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_contributor-guide_architecture_impact-estimation" data-v-39a288b8><div><h1 id="impact-estimation" tabindex="-1">Impact estimation <a class="header-anchor" href="#impact-estimation" aria-label="Permalink to "Impact estimation""></a></h1><p>Every finding covered by this section carries an optional <code>impactEstimate: {basis, wallClock, estimateMethod, rawWaste?}</code> (<code>src/types.ts</code>), attached by <code>src/impact-estimator.ts</code> as a post-pass after <code>DETECTORS</code> finishes (<code>src/analyzer.ts</code>). <code>estimateMethod</code> ('measured' | 'modeled' | 'none') is a distinct axis from the per-finding <code>confidence</code> field (<a href="./board-widgets.html#confidence-metadata">Confidence metadata</a>): <code>confidence</code> says how much to trust the finding itself, <code>estimateMethod</code> says how its impact number was derived. <code>'none'</code> marks a purely informational finding with no waste model at all (<code>configAudit</code>, <code>stageFailed</code>, <code>failures</code>, <code>incompleteRun</code>, <code>slowHost</code>'s byte-dimension multiDim shapes, <code>gc</code>'s low-GC direction): it's distinct from <code>'measured'</code>/<code>'modeled'</code>, which both attach a real (if approximate) formula. <code>basis</code> is one of:</p><ul><li><code>'serial'</code>: the tied stage ran (effectively) alone; <code>wallClock</code> is a near-point estimate, <code>low === high</code>.</li><li><code>'contended'</code>: the tied stage shared wall-clock time with others; <code>wallClock</code> is an honest range, <code>high</code> optimistic (assumes the fix could still fully land), <code>low</code> the guaranteed floor.</li><li><code>'resourceOnly'</code>: no wall-clock claim is defensible (not stage-tied by nature, or the stage was excluded from the occupancy sweep), <code>wallClock: null</code>, but the formula's real signal survives in <code>rawWaste</code>.</li><li><code>'informational'</code>: no quantifiable magnitude at all, <code>wallClock: null</code>, no <code>rawWaste</code>.</li></ul><p><code>{basis: 'resourceOnly'|'informational', wallClock: null}</code> replaced the earlier design's <code>{low: 0, high: 0}</code>: that single value used to mean two incompatible things ("provably no wall-clock cost" and "the model gave up"), and 76-97% of stage-tied findings on real logs were the second case wearing the first case's clothing (2026-08-30 N1 redesign; see <code>docs/superpowers/specs/2026-08-30-critical-path-occupancy-redesign.md</code>).</p><p><code>rawWaste</code> is not exclusive to <code>resourceOnly</code>/<code>informational</code> findings: every <code>serial</code>/ <code>contended</code> finding carries it too (the only exceptions are <code>coldStart</code>, whose <code>wallClock</code> figure is already unclipped, and <code>estimateMethod: 'none'</code> findings, which have no formula at all), holding the formula's pre-clip magnitude in the formula's own natural unit (ms, bytes or core-ms). <code>wallClock</code> is what the occupancy model says is recoverable, which clips against the stage's own physical floor; <code>rawWaste</code> is what the stage really wasted either way. The two answer different questions.</p><h2 id="occupancy-weighted-attribution" tabindex="-1">Occupancy-weighted attribution <a class="header-anchor" href="#occupancy-weighted-attribution" aria-label="Permalink to "Occupancy-weighted attribution""></a></h2><p><code>src/occupancy.ts</code> sweeps every stage's observed <code>[submittedAt, completedAt)</code> window and splits each instant's wall-clock among concurrently-active stages proportional to <code>coreWeight(S) = stage.executorRunTime / stageDurationMs</code> (an average-concurrency proxy, held constant across the stage's whole window: this codebase has no per-task timestamps outside the parser worker to do better). Summing a stage's share across its own window gives its <code>occupancy(S)</code>; <code>gate(S) = occupancy(S) / duration(S) ∈ [0, 1]</code> is the single number that replaces the old CPM model's <code>onCriticalPath</code>/<code>slackMs</code>/<code>isUniquelyCritical</code>. 1.0 means the stage ran completely alone; 0 means the stage had zero <code>executorRunTime</code> while overlapping other, positive-weight stages, so it got no share of the shared window. Stages with <code>duration(S) <= 0</code> (Spark-skipped stages, or a malformed <code>submittedAt === completedAt</code>) are excluded from the sweep entirely.</p><p>This mechanism replaced a CPM (critical-path-method) graph over <code>parentIds</code> that produced near-zero on-critical-path membership on real logs (0.1-10.6% of stages, max graph depth 2 on 5 of 6 real logs measured): <code>parentIds</code> alone is too sparse a precedence signal for a meaningful longest-path computation. The same degeneracy fed <code>efficiency-model.ts</code>'s <code>floorInfiniteMs</code> ("floor with infinite executors"); rather than leave a second, unreconciled critical-path number in the codebase for a future UI to display next to the occupancy-based figures above, <code>criticalPathMs()</code>/<code>CriticalPathStage</code> (embedded in <code>efficiency-model.ts</code>, never a standalone module) were removed outright (no occupancy-based replacement: occupancy apportions observed concurrent time, it doesn't compute a dependency-graph longest path, so there's no drop-in equivalent). <code>efficiency-model.ts</code> now reports only <code>floorZeroSkewMs</code> (total task time / total cores) as its theoretical floor.</p><p><code>ceiling(S) = max(stage.taskDurationMax, stage.executorRunTime / totalCores)</code> is a physical floor on a stage's own duration: bounded below by its single longest task (unsplittable no matter how much parallelism exists) or by its core-work spread across every core in the cluster, whichever is larger. Every waste formula's raw claim is clipped against it before gate-weighting: <code>wasteMs_clipped(S) = min(wasteMs_claimed, max(0, duration(S) - ceiling(S)))</code>, so a finding can never claim to save more than the portion of the stage's observed duration that sits above its own unbeatable floor. This is what fixes historical overclaim bugs (a <code>tinyTask</code> finding claiming 407.5s on a 13.1s stage capped to 8.9s; a <code>shuffle</code> finding claiming 1939.9s on a 991.3s/1688.3s stage capped to 610.3s/250.1s).</p><p><code>skew</code> and <code>straggler</code> are the exception (<code>estimateSingleStage</code>'s <code>shortensLongestTask</code> option, passed by both their <code>impact-estimator.ts</code> cases and <code>detectors.ts</code>'s <code>clippedWasteMs</code> runtime-floor gate, so firing and display agree; <code>stageSlowness</code>'s more-partitions estimate passes it too, since splitting partitions splits the longest task). Their claim shortens the stage's longest task itself, so <code>taskDurationMax</code> can't be their floor: clipping against it capped a stage gated by one straggler at <code>duration(S) − taskDurationMax</code>, about zero, exactly when the fix recovers the most. Their floor is instead the longest task the fix leaves plus the core work the fix leaves, <code>max(taskDurationMax − wasteMs_claimed, longestTaskAfterFixMs, (stage.executorRunTime − removed) / totalCores)</code>, where <code>removed</code> (<code>tailRemovedWorkMs</code>) is the larger of the longest task's excess over P50 and <code>stragglerExcessMs</code>, and <code>longestTaskAfterFixMs</code> is the longest task the fix leaves for <code>skew</code> and <code>straggler</code> (see below). Counting the stragglers' own run time as work the stage can't shed floored a stage whose tail is most of its core time near its observed duration: one real stage claimed 5.9s where the replay below recovers 38.7s, and now claims 38.7s. Scored against a list-scheduling replay of each flagged stage's own tasks (slots = the stage's observed peak concurrent tasks; recoverable = replay with actual durations minus replay with every task over 4× P50 capped at P50), across 765 skew/straggler findings on 14 real logs (2026-09-23, <code>dev/bench-analyze.mjs</code> snapshots): estimates more than 2× under the replay dropped from 199 to 11, estimates within 2× rose from 556 to 738, mean absolute error fell from 6.64s to 5.59s. Overclaims by more than 2× went from 10 to 16: stages where AQE or free slots absorbed the tail, which a stage-level model can't see.</p><p>The claim itself is <code>tailRecoveryMs</code> (<code>occupancy.ts</code>): the larger of the single-task delta above and <code>stragglerExcessMs / peakConcurrentTasks</code>, the summed excess over P50 of every task slower than 4× P50, spread over the most tasks the stage ever ran at once (both from <code>finalizeStage</code>). A lone straggler costs its own excess; a bimodal stage with hundreds of slow tasks (26% of 1400 on a real log) costs far more than its longest one, and the single-task delta claimed 38-92s where the replay recovered 443-638s. Peak, not average, concurrency: a tail-dominated stage runs few tasks for most of its span (5.7 average vs 14 peak on one real stage), and dividing by the average doubled the claim. Scored the same way over the 14 real logs plus the 17 complete corpus runs (non-info findings only): within 2× of the replay 56 of 78 → 79 of 85, more than 2× under 16 → 0, more than 2× over 6 → 6, mean absolute error 53.4s → 8.4s. The detectors' floor gates use the same figure, which lifted skew's recall from 0.60 to 0.73 at precision 0.97 → 0.98; straggler's scores didn't move. Taking the removed work out of the core-work floor, scored again over 65 runs (14 real logs plus the corpus, 103 non-info findings): within 2× 88 of 95 → 97 of 103, more than 2× under 1 → 0, more than 2× over 6 → 6, mean absolute error 7.55s → 5.18s; skew recall 0.76 → 0.81 at precision 0.98 → 0.96, straggler recall 0.71 → 0.76 at precision 0.94 → 0.92, either detector recall 0.85 → 0.90.</p><p><code>straggler</code>'s single-task delta runs from <code>taskDurationMax</code> down to the longest task its fix leaves (<code>stragglerFixLongestTaskMs</code>), not down to P50: the fix brings every task over 4× P50 to the median, so the stage still waits on its longest task at or under that (<code>longestNonStragglerMs</code>, from <code>finalizeStage</code>), and that task is also passed as a floor (<code>longestTaskAfterFixMs</code>). A speculation-driven finding with no task over 4× P50 keeps the P50 delta. <code>skew</code> keeps its P50 delta, since repartitioning may even out tasks under 4× P50 too, but takes the same <code>longestTaskAfterFixMs</code> floor: without it a 100 s stage whose next-longest task ran 39 s had skew claiming 90 s where straggler claimed 61 s. Scored the same way over the 65 runs: more than 2× over 6 → 4, mean absolute error 5.18s → 4.49s, straggler precision 0.92 → 0.94 at recall 0.76 → 0.74 (one stage at 0.61% of the run now estimates under the 0.5% floor, one below it no longer fires). The skew floor, scored the same way: within 2× 97 → 98 of 101, more than 2× over 4 → 3, mean absolute error 4.49s → 4.12s, precision and recall unchanged.</p><p><code>analyzer.ts</code> feeds this <code>totalCores</code> from <code>src/core-count.ts</code>'s <code>computePeakConcurrentCores(app, executorsAdded, executorsRemoved)</code>, not the shared <code>computeTotalCores</code> helper. <code>computeTotalCores</code> sums every <code>ExecutorAdded</code> event's cores regardless of overlap, so under dynamic allocation or executor replacement it can far exceed the cores ever actually concurrent, which understates <code>ceiling(S)</code> and lets churn inflate a finding's claimed wall-clock. <code>computePeakConcurrentCores</code> instead sweeps add/remove events by timestamp and tracks the running total's peak, so a churned-through executor's cores are never double-counted against its replacement's. Same-timestamp events tie-break by delta ascending, so a removal applies before a same-instant replacement's addition (otherwise a same-instant swap would momentarily double-count both as concurrent). If every <code>executorsAdded</code> entry lacks <code>totalCores</code> the cores sweep peaks at zero and tells us nothing; the function then falls back to sweeping peak <em>executor count</em> instead (still concurrency-aware, just cores-blind) and multiplies by the configured per-executor core count, rather than falling back to <code>executorsAdded.length × cores</code>, which would reintroduce the exact cumulative-overcount-under-churn bug this function exists to avoid.</p><p>Per-finding estimate, using <code>wasteMs_clipped(S)</code>:</p><ul><li><code>gate(S) >= 0.999</code>: <code>basis: 'serial'</code>, <code>low = high = wasteMs_clipped(S)</code>.</li><li><code>gate(S) < 0.999</code>: <code>basis: 'contended'</code>, <code>high = wasteMs_clipped(S)</code>, <code>low = wasteMs_clipped(S) * gate(S)</code>.</li></ul><p>A finding spanning multiple stages (<code>stageIds</code>, plural) sums each stage's own estimate and caps the joint total at the union of just that finding's own stage windows (via <code>mergeIntervals</code>, <code>src/wall-clock.ts</code>): <code>high = min(Σ high_i, unionMs(stageIds))</code>, <code>low = min(Σ low_i, unionMs(stageIds))</code>. This is what prevents overclaiming when two or more of a finding's stages overlap in wall-clock time: a plain sum-and-cap, no CPM re-simulation. The union cap can force <code>low === high</code> numerically even when the constituent stages were individually contended (e.g. two fully-overlapping stages each at <code>gate</code> 0.5), so <code>basis</code> isn't derived from that numeric equality: a multi-stage finding gets <code>basis: 'serial'</code> only when every one of its per-stage estimates was itself <code>'serial'</code>; otherwise <code>'contended'</code>.</p><p>Regression guard: a finding whose stage (or, for a multi-stage finding, every one of its stages) ran alone (<code>gate >= 0.95</code>, deliberately looser than the <code>0.999</code> "serial" cutoff above: this guard exists to catch egregious false zeros, not to gate which <code>basis</code> a finding gets) with a real underlying magnitude (<code>rawWaste.value > 0</code>) must never report <code>wallClock.high === 0</code> or <code>basis: 'informational'/'resourceOnly'</code>. Covered by <code>tests/impact-estimator-real-log.test.js</code> against a real fixture; relies on <code>rawWaste</code> being attached to every serial/contended-capable formula (see above), so it's blind only to <code>coldStart</code> and the purely informational (<code>estimateMethod: 'none'</code>) finding types.</p><p>Real-log spot-check (2026-08-30, <code>grupo-semanal-beauty-application_1785266278671_91660.zstd</code>): median <code>gate</code> across stages was <code>≈0.34</code> (0.3428060791718594 exactly); <code>collectRun</code> plus the occupancy sweep together took <code>≈6,589</code>ms on the largest fixture measured (<code>run-compare-calimax-candidate-application_1784568768686_119096.zstd</code>, <code>1169</code> stages): parse-dominated, the sweep alone was not isolated by this measurement, but not a magnitude that suggests a regression either. <code>54</code> previously-<code>{0,0}</code> stage-tied findings on stages that ran effectively alone now report a real <code>wallClock</code> range instead.</p><h2 id="cross-finding-rollup-computestageunionms" tabindex="-1">Cross-finding rollup: <code>computeStageUnionMs</code> <a class="header-anchor" href="#cross-finding-rollup-computestageunionms" aria-label="Permalink to "Cross-finding rollup: `computeStageUnionMs`""></a></h2><p>The Findings tab's recommendation rollup (<code>FixTheseFirst.tsx</code>, built from <code>buildRecommendationRollup</code>, <code>src/recommendation-rollup.ts</code>) groups the filtered catalog by detector <code>type</code>, then needs its own cap for a group of several findings of that type, not just one finding's own <code>stageIds</code>. Summing each finding's already-clipped <code>wallClock.high</code> naively double-counts any stage two of those findings both touch. <code>computeStageUnionMs(stageIds, stages)</code> covers this: collect every stage touched by any finding in the group, merge their <code>[submittedAt, completedAt)</code> intervals, and sum the merged intervals' durations, so the group's wall-clock union holds regardless of how many findings' <code>stageIds</code> overlap. Stages missing either bound are skipped rather than defaulted to <code>0</code> (the same filter <code>computeWallClock</code> applies), so a truncated log (the case <code>incompleteRun</code> flags) can't contribute a negative interval and a negative recoverable-time figure.</p><p>It reuses the same <code>mergeIntervals</code> primitive (<code>src/wall-clock.ts</code>) that backs <code>src/occupancy.ts</code>'s per-finding <code>estimateMultiStage</code>/its internal <code>unionMs</code> sum, but is not an extension of that function: <code>estimateMultiStage</code> caps one finding's own multi-stage claim during the impact-estimation pass, before a <code>Finding</code> object even exists; <code>computeStageUnionMs</code> runs later, in the view layer, capping a naive sum <em>across</em> several already-estimated findings that happen to share a detector <code>type</code>. <code>buildTimeGroup</code> (same module) takes the smaller of the naive per-finding sum and this union figure as the group's <code>recoverableMsHigh</code>, falling back to the naive sum untouched when the group's findings carry no stage IDs at all (nothing to union against).</p><p>Within one <code>type</code> group, <code>buildRecommendationRollup</code> splits findings into up to three tiers, always rendered in this fixed order. <code>time</code> covers findings with a real <code>impactEstimate.wallClock</code> (<code>buildTimeGroup</code>, the union-capped figure above). <code>resource</code> covers findings with no <code>wallClock</code> but a <code>rawWaste</code> figure (<code>buildResourceGroup</code>), grouped again by <code>rawWaste.unit</code> so a <code>bytes</code> total never gets summed against a <code>coreHours</code> total under one type. <code>count</code> covers findings with neither (<code>buildCountGroup</code>), a plain per-impact-band tally with no magnitude claim at all. Each <code>RollupGroup</code> also carries its own <code>findings: Finding[]</code> (the exact members that fed the aggregate), which <code>FixTheseFirst.tsx</code> reads directly to pick a group's highest-impact member and to render its expanded, paginated list.</p><p>A type only contributes a tier when it has at least one finding of that kind; most types produce exactly one tier, but a type whose formula varies by <code>variant</code>/<code>rule</code> (e.g. <code>memoryUtilization</code>, see the coverage table below) can produce more than one.</p><p><code>cachingOpportunity</code> and <code>cacheUtilization</code> are both <code>cost-only</code>: <code>basis: 'resourceOnly'</code>, <code>wallClock: null</code>, but <code>rawWaste.unit</code> is <code>'ms'</code>, the same unit a real <code>wallClock</code> figure would use, because their formula's natural output happens to be time (a re-read cost), not because either finding makes a wall-clock claim. Left unlabeled, a <code>resource</code>-tier "ms" total sitting next to a <code>time</code>-tier "recoverable time" total would read as directly comparable when it isn't: the resource figure was never gate-clipped against any stage's occupancy, so it can exceed what the stage actually spent. <code>FixTheseFirst.tsx</code> calls this out via its trailing-stat copy: a <code>resource</code>- kind group (any unit, including <code>ms</code>) always reads "resource-cost projection", never "recoverable", so the two ms-shaped numbers are never mistaken for the same kind of claim.</p><h2 id="per-formula-spot-checks" tabindex="-1">Per-formula spot-checks <a class="header-anchor" href="#per-formula-spot-checks" aria-label="Permalink to "Per-formula spot-checks""></a></h2><table tabindex="0"><thead><tr><th>Detector</th><th>Formula basis</th><th>Spot-check</th></tr></thead><tbody><tr><td>gc</td><td><code>jvmGCTime / (executorRunTime / stageDurationMs)</code></td><td><code>grupo-semanal-beauty-application_1785266278671_91660.zstd</code>, stage 507: <code>jvmGCTime</code>=1080ms, <code>executorRunTime</code>=27509ms, <code>stageDurationMs</code>=56279ms → <code>wasteMs</code> = 1080 / (27509/56279) ≈ 2209.5ms. That's ≈3.9% of the stage's 56.3s wall-clock duration, matching the finding's own reported <code>gcPct</code> (3.9%) exactly, as the formula guarantees by construction. Under the occupancy model this stage's <code>gate</code> is <code>0.041</code> (0.04145044590332269 exactly): <code>basis: 'contended'</code>, <code>wallClock: {low: 91.6, high: 2209.5}</code> (91.5850382272182 / 2209.506706895925 exactly, per the Step 1 script's per-stage output).</td></tr><tr><td>shuffle</td><td><code>shuffleReadBytes / (SHUFFLE_THROUGHPUT_BPS × executors that ran the stage)</code></td><td><code>ventas-mensual-multi-big-application_1785266278671_91510.zstd</code>, stage 99 (<code>SHFL</code> finding): <code>shuffleReadBytes</code>=204,172,518,504 over 8 executors → <code>wasteMs</code> = 204172518504 / (8 × 125,000,000) × 1000 ≈ 204,173ms, against the stage's 763,776ms duration. The tasks' own measured shuffle fetch wait on this stage is 25.2s of wall-clock (<code>fetchWaitTime</code> / average concurrency), so even the per-link model runs well above the network stall actually observed, and the claim is capped there: 25.2s, <code>measured</code>. The pre-2026-09-23 formula divided by one link's bandwidth (1,633,380ms, 2.1× the stage's whole duration); before the fetch-wait cap, the occupancy clip (gate <code>1</code>) held the per-link figure to <code>≈111,923.8</code>ms, the room above this stage's core-work floor.</td></tr><tr><td>spill</td><td><code>diskBytesSpilled / (SPILL_IO_THROUGHPUT_BPS × executors that ran the stage)</code></td><td>Same run and stage (99): <code>diskBytesSpilled</code>=145,978,433,675 (note: the <code>SPILL</code> finding's own <code>value</code>/<code>metric</code> report <code>memoryBytesSpilled</code>=913,686,966,448, ~6x larger; the formula correctly uses the smaller disk figure, not that one) over 8 executors → <code>wasteMs</code> = 145978433675 / (8 × 200,000,000) × 1000 ≈ 91,237ms (91236.521046875 exactly, matching <code>wallClock</code>, below the clip).</td></tr></tbody></table><h2 id="overlap-caveat-skew-straggler" tabindex="-1">Overlap caveat: skew / straggler <a class="header-anchor" href="#overlap-caveat-skew-straggler" aria-label="Permalink to "Overlap caveat: skew / straggler""></a></h2><p><code>skew</code> (small-stage max-P50 fallback branch) and <code>straggler</code> can both fire on the same stage from the same single dominant outlier task, and each is clipped independently. This phase does not dedupe or suppress either: each keeps its own independently-computed <code>wallClock</code>. Do not sum <code>wallClock.high</code> across multiple findings on the same stage: if both fire together, they describe the same underlying waste, not two separate wastes. Both are clipped with the post-fix floor described under <a href="#occupancy-weighted-attribution">Occupancy-weighted attribution</a>, not the plain <code>ceiling</code>, so a stage gated by one dominant outlier task reports that task's excess as recoverable instead of the near-zero room <code>ceiling >= taskDurationMax</code> would leave.</p><p><code>analyzer.ts</code>'s <code>flagSkewStragglerOverlap</code> (run after <code>deriveImpactBand</code>, once per <code>analyze()</code> call) surfaces this caveat to the reader instead of leaving it as an internal-only comment: whenever <code>skew</code>'s <code>max/median</code> branch and <code>straggler</code> both fire on the same <code>stageId</code>, it appends a "this overlaps with the X finding on this stage" sentence to both findings' <code>validationRequired</code> text (rather than suppressing either, so neither finding's own diagnostic value is lost). <code>skew</code>'s <code>P95/median</code> branch samples a different task from <code>straggler</code>'s own <code>taskDurationMax - taskDurationP50</code> delta, so it's excluded from the flag. The note rides the same confidence-caveat UI (<code>RowStatusCluster</code>) a reader already sees before trusting either finding's magnitude, since both detectors also carry a <code>confidence</code> field that scales <code>low</code>/<code>medium</code>/<code>high</code> off how far the finding sits past its own runtime-floor threshold (still unvalidated; see the confidence-disclosure note in detector-contract.md).</p><p><code>stageShape</code>'s <code>taskStageSkew</code> rule no longer participates in this caveat: it reports a <code>resourceOnly</code> idle-core-ms figure (see the coverage table below) instead of a wall-clock claim, so there's nothing left to double-count against <code>skew</code>/<code>straggler</code>. Its trigger condition (<code>taskDurationMax / stageDurationMs > skewWarn</code>) mathematically forces the occupancy-clipped wall-clock estimate to exactly zero on every firing (see <code>src/detectors.ts</code>'s <code>taskStageSkew</code> comment), which is why it was moved off the wall-clock path entirely rather than reconciled against the same ceiling clip as its two siblings above.</p><h2 id="per-finding-type-coverage" tabindex="-1">Per-finding-type coverage <a class="header-anchor" href="#per-finding-type-coverage" aria-label="Permalink to "Per-finding-type coverage""></a></h2><p>One row per distinct <code>type</code> string <code>src/detectors.ts</code> actually emits (cross-checked against <code>computeEstimateForFinding</code>'s <code>case</code> labels in <code>src/impact-estimator.ts</code>, not assumed from the prose here): every row below has a case, so the table itself is the coverage count, not a number restated here. <code>broadcastSizing</code> is a <code>DETECTORS</code> entry label only, and the plan-walk it drives emits <code>overBroadcast</code>/<code>underBroadcast</code> findings instead, so those two are the rows that appear, not <code>broadcastSizing</code> itself. Tag meanings: <code>measured</code> and <code>modeled</code> both produce a real, gate-clipped, non-<code>{0,0}</code><code>wallClock</code> (the difference is whether the formula's inputs are recorded per-stage fields or an assumed constant like a throughput figure); <code>cost-only</code> always reports <code>basis: 'resourceOnly'</code>, <code>wallClock: null</code> but carries its real signal in <code>rawWaste</code>; <code>informational-only</code> reports <code>basis: 'informational'</code>, <code>wallClock: null</code> with no <code>rawWaste</code> at all, since there's nothing quantifiable. A type with more than one tag fires a different formula per <code>variant</code>/<code>rule</code> on the same finding type; the basis column says which.</p><table tabindex="0"><thead><tr><th>Finding type</th><th>Scope</th><th>Tag</th><th>Basis</th></tr></thead><tbody><tr><td><code>retryWaste</code></td><td>stage</td><td>measured / modeled</td><td><code>retryWasteMs</code> is summed attempt time, and attempts of different tasks run side by side (one lost executor fails every task it was running at once). With every wasted attempt sampled (<code>retryTaskSamples</code>, capped at 20), modeled: the longest retry chain (highest sampled attempt number + 1) times the mean wasted attempt, or <code>retryWasteMs / peakConcurrentTasks</code> when larger, capped at <code>retryWasteMs</code>. Otherwise measured, <code>retryWasteMs</code> itself. Gate-clipped; the summed figure is kept as <code>rawWaste</code> in <code>ms</code>. On the 14 real logs the one non-info finding (4 first attempts on one lost executor, 41 slots) went from 146.6s to 36.6s</td></tr><tr><td><code>speculationWaste</code></td><td>stage</td><td>measured</td><td><code>speculationWasteMs</code>, gate-clipped; pre-clip figure kept as <code>rawWaste</code> in <code>ms</code></td></tr><tr><td><code>coldStart</code></td><td>app</td><td>measured</td><td><code>gapSeconds × 1000</code> (first stage submitted to first executor added), unclipped, <code>basis: 'serial'</code> unconditionally (a pre-first-task gap can't overlap any stage)</td></tr><tr><td><code>gc</code></td><td>stage</td><td>modeled / informational-only</td><td>high-GC: <code>jvmGCTime / (executorRunTime / stageDurationMs)</code>, gate-clipped: the concurrency division is an approximation, not a reconstruction, hence <code>modeled</code>; <code>rawWaste</code> in <code>coreMs</code> is the raw <code>jvmGCTime</code> sum before that conversion. Low-GC (<code>direction: 'low'</code>): informational-only, since its fix (less executor memory) raises GC rather than recovering it; it used to claim the stage's GC time as savings, which promoted 10 of 679 low-GC findings on 14 real logs to warning/critical</td></tr><tr><td><code>skew</code></td><td>stage</td><td>measured</td><td><code>tailRecoveryMs</code>: the larger of <code>taskDurationP95</code> or <code>Max</code> minus <code>P50</code> (per <code>metric</code>) and <code>stragglerExcessMs / peakConcurrentTasks</code>, gate-clipped against the post-fix floor (<code>shortensLongestTask</code>) with the longest task the fix leaves as a floor (<code>longestTaskAfterFixMs</code>, as for <code>straggler</code>); pre-clip figure kept as <code>rawWaste</code> in <code>ms</code></td></tr><tr><td><code>straggler</code></td><td>stage</td><td>measured</td><td><code>tailRecoveryMs</code>: the larger of <code>taskDurationMax</code> minus the longest task the fix leaves (<code>longestNonStragglerMs</code>, or <code>taskDurationP50</code> without stragglers) and <code>stragglerExcessMs / peakConcurrentTasks</code>, gate-clipped against the post-fix floor (<code>shortensLongestTask</code>, <code>longestTaskAfterFixMs</code>)</td></tr><tr><td><code>stageShape</code></td><td>stage</td><td>cost-only</td><td>all three rules are <code>estimateMethod: 'measured'</code>, real per-stage fields, no assumed constant: <code>'lowParallelism'</code> → <code>rawWaste</code> in <code>coreMs</code> (idle cores × stage duration); <code>'dataExplosion'</code> → <code>rawWaste</code> in <code>bytes</code> (<code>outputBytes − inputBytes</code>); <code>'taskStageSkew'</code> → <code>rawWaste</code> in <code>coreMs</code> (<code>max(0, min(totalCores, taskCount) − 1) × (taskDurationMax − taskDurationP50)</code>, the cores idle during the straggler's tail at achieved concurrency)</td></tr><tr><td><code>slowHost</code></td><td>stage</td><td>measured / informational-only</td><td>duration-based variants (<code>hostMeanRatio</code>, <code>durationShare</code>, <code>multiDim</code>+<code>taskTime</code>): <code>value − taskDurationP50</code>, gate-clipped; byte-based <code>multiDim</code> dimensions: no formula yet</td></tr><tr><td><code>duplicatePlanSubtree</code></td><td>sql</td><td>measured</td><td>per stage in <code>stageShares</code>: its task-active time (<code>taskActiveMs</code>, the union of its task intervals; submit-to-complete only when absent) × the repeated operators' share of that stage × the redundant fraction <code>(occurrences − 1) / occurrences</code>, summed and capped at the union of those stages' spans. A stage shared with other operators (the consuming join, the other join side) contributes only its share, so sibling groups can't claim one stage twice, and a stage left waiting for cores (2491 s open, 60 s of tasks on a real log) claims only its task time. No claim (<code>informational</code>) when the repeats' details differ (<code>occurrencesIdentical: false</code>) or no repeated operator has a stage (the execution-wide <code>stageIds</code> fallback stays for linking only). On the 14 real logs: 2307 min claimed before, including more duplicate time than the whole run on 3 logs (1661 of 458 min, 257 of 61, 338 of 68); 108 min after, critical 65 → 2, warning 54 → 21</td></tr><tr><td><code>shuffle</code></td><td>stage</td><td>modeled / measured</td><td><code>shuffleReadBytes / (SHUFFLE_THROUGHPUT_BPS × executors)</code> (assumed ~125MB/s per executor link; <code>executors</code> = <code>executorStats.length</code>, the executors that ran the stage, min 1), capped at the fetch wait the tasks measured in wall-clock (<code>fetchWaitTime / (executorRunTime / stage duration)</code>, the <code>gc</code> conversion; <code>measured</code> when that cap binds), gate-clipped; <code>rawWaste</code> in <code>bytes</code> is the measured <code>shuffleReadBytes</code> behind it. The link model can't see whether reads stalled the tasks: on 14 real logs (2026-09-23) it claimed 2780s over 284 shuffle findings where the capped figure is 256s, and 5 of the 10 non-info findings had about zero fetch wait (they are now <code>info</code>). Fetch wait misses disk reads and deserialization, so the cap is a floor on what a shuffle costs, not its total. Dividing by one link instead modeled the whole cluster as a single 1 Gbps pipe: on 14 real logs (2026-09-23) that claimed 721 minutes of shuffle+spill savings on a 458-minute app, on stages where 222 of 284 shuffle findings measured ~0s of task fetch wait</td></tr><tr><td><code>spill</code></td><td>stage</td><td>modeled</td><td><code>diskBytesSpilled / (SPILL_IO_THROUGHPUT_BPS × executors)</code> (assumed ~200MB/s per executor's local disk, same executor count as <code>shuffle</code>), gate-clipped; <code>rawWaste</code> in <code>bytes</code> is <code>diskBytesSpilled</code>, which is the number the formula uses and not the <code>memoryBytesSpilled</code> the finding's own <code>metric</code> displays</td></tr><tr><td><code>stageSlowness</code></td><td>stage</td><td>modeled</td><td>what more partitions (the finding's recommendation) could recover: the stage's task-active time (<code>taskActiveMs</code>, the union of its tasks' launch-to-finish intervals from <code>finalizeStage</code>) × <code>max(0, 1 − taskCount / totalCores)</code>, gate-clipped with the post-fix floor (<code>shortensLongestTask</code>: splitting partitions splits the longest task too). A stage that already ran at least as many tasks as the cluster had cores claims 0, and so does one that read no input and no shuffle bytes (no data for more partitions to split: a 1-task <code>count</code> stage open 27 minutes on 5s of CPU had claimed 99%); time a stage sat open with no task running (queued for slots) claims nothing. No cluster core count: informational. Replaced "stage duration minus the detector's 15-minute <code>infoMin</code>", which on 14 real logs (2026-09-23) graded 29 of 35 findings critical, including 1-task stages open 20-40 minutes whose only task ran under a second, while scoring a 27-minute single-task stage 0; before that no-read gate, 1 critical (that single-task stage), 1 warning, 33 info, claims 539 → 30 minutes</td></tr><tr><td><code>partitionSizing</code></td><td>stage</td><td>modeled</td><td><code>maxPartitionTooBig</code>/<code>shufflePartitionSkew</code>: shuffle-throughput formulas, gate-clipped. <code>lowShuffleParallelism</code>: stage duration scaled down by the shortfall between actual and ideal-partition-count task counts (<code>stageDurationMs × (1 − taskCount / targetTaskCount)</code>), i.e. the serialized work more partitions would let run concurrently, not the scheduling cost of the tasks you'd add to fix it</td></tr><tr><td><code>tinyTask</code></td><td>stage</td><td>measured / modeled</td><td>excess task count over 10% of the stage's actual count, × the stage's own measured per-task overhead (summed task wall time from <code>executorStats</code> minus <code>executorRunTime</code>, over <code>taskCount</code>), ÷ the stage's achieved task concurrency (task time ÷ stage duration, floored at 1), gate-clipped; <code>measured</code>. A stage with no <code>executorStats</code> or no measurable overhead falls back to the assumed 50ms per task, undivided (<code>modeled</code>). Pre-clip figure kept as <code>rawWaste</code> in <code>ms</code>. Measured overhead on 14 real logs ran 6-62ms per task, near the old constant, but the old formula summed it serially across tasks that actually ran in parallel: total claimed savings fell from 576s to 270s over 160 findings (2026-09-23)</td></tr><tr><td><code>smallFiles</code></td><td>sql</td><td>modeled / cost-only</td><td><code>excessFileCount × FILE_OPEN_OVERHEAD_MS</code>, divided for a read by the most tasks its stages ran at once (<code>peakConcurrentTasks</code>: tasks open their files in parallel; 91,344 files claimed 76 s on a 117 s stage that ran 314 tasks at once) and kept serial for a write (the job commit moves each file on the driver), summed and capped over <code>stageIds</code>'s union; with no <code>stageIds</code> to map to, cost-only with that same figure as <code>rawWaste</code> in <code>ms</code>. <code>stageIds</code> is narrowed the same way (see <a href="./detector-contract.html#stage-id-attribution-for-plan-advisor-findings">Stage-ID attribution for Plan Advisor findings</a>); falls back to the whole execution's stages when the flagged node(s) have no accumulator coverage.</td></tr><tr><td><code>overBroadcast</code></td><td>sql</td><td>modeled / cost-only</td><td><code>broadcastBytes / BROADCAST_BANDWIDTH_BPS</code>, summed and capped over <code>stageIds</code>'s union; cost-only with <code>rawWaste</code> in <code>ms</code> when not stage-mappable. <code>stageIds</code> is narrowed the same way; falls back to the whole execution's stages when the flagged node(s) have no accumulator coverage.</td></tr><tr><td><code>underBroadcast</code></td><td>sql</td><td>modeled / cost-only</td><td><code>smallerSideBytes / BROADCAST_BANDWIDTH_BPS</code>, summed and capped over <code>stageIds</code>'s union; cost-only with <code>rawWaste</code> in <code>ms</code> when not stage-mappable. <code>stageIds</code> is narrowed the same way; falls back to the whole execution's stages when the flagged node(s) have no accumulator coverage.</td></tr><tr><td><code>memoryUtilization</code></td><td>app</td><td>cost-only / informational-only</td><td>Three of the four variants report <code>rawWaste</code> in <code>mbSeconds</code>: <code>variant: 'wasteModel'</code> passes through its own <code>wastedMBSeconds</code>; <code>'idleCores'</code> uses <code>idleRateFraction × allocatedMB × peakExecutors × appDurationSeconds</code>; <code>'memoryBand'</code> with <code>rule: 'heapOverProvisioned'</code> uses <code>(allocatedBytes − heap) in MB × appDurationSeconds</code>. <code>'memoryBand'</code> with <code>rule: 'heapNearCapacity'</code> is an OOM-risk signal rather than a waste, and the <code>dataUnavailable</code> shape has no inputs at all: both informational-only</td></tr><tr><td><code>utilization</code></td><td>app</td><td>cost-only</td><td><code>rawWaste</code> in <code>coreHours</code>: <code>(1 − utilizationFraction) × appDurationMs × totalCores / 3.6e6</code></td></tr><tr><td><code>coreLocality</code></td><td>app</td><td>cost-only</td><td><code>rawWaste</code> in <code>coreMs</code>: <code>nonLocalTaskCount × NETWORK_FETCH_PENALTY_MS</code></td></tr><tr><td><code>autoscalingChurn</code></td><td>app</td><td>cost-only</td><td><code>rawWaste</code> in <code>coreHours</code>: <code>shortLivedExecutorCount × EXECUTOR_STARTUP_OVERHEAD_MS / 3.6e6</code></td></tr><tr><td><code>configAudit</code></td><td>config</td><td>informational-only</td><td>a config-drift check standing alone; no waste formula</td></tr><tr><td><code>jobFailureRate</code></td><td>app</td><td>cost-only</td><td><code>rawWaste</code> in <code>coreHours</code>: <code>failedJobCount × avgJobDurationMs / 3.6e6</code></td></tr><tr><td><code>cachingOpportunity</code></td><td>app</td><td>cost-only</td><td><code>rawWaste</code> in <code>ms</code>: <code>totalReadBytes / RE_READ_THROUGHPUT_BPS</code></td></tr><tr><td><code>cacheUtilization</code></td><td>app</td><td>cost-only</td><td><code>rawWaste</code> in <code>ms</code>: uncached-or-spilled bytes <code>/ RE_READ_THROUGHPUT_BPS</code>, where the never-cached partitions' bytes are extrapolated from the cached partitions' own average size (<code>memorySize + diskSize</code>, over <code>numCachedPartitions</code>), plus <code>diskSize</code> again for the already-cached-but-on-disk partitions' own re-read cost</td></tr><tr><td><code>stageFailed</code></td><td>stage</td><td>informational-only</td><td>no waste formula</td></tr><tr><td><code>failures</code></td><td>stage</td><td>informational-only</td><td>no waste formula</td></tr><tr><td><code>incompleteRun</code></td><td>app</td><td>informational-only</td><td>no waste formula</td></tr></tbody></table></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/contributor-guide/architecture/detector-contract.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>Detector contract</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="/docs/contributor-guide/architecture/widget-rendering.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>Widget rendering</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
|
|
26
|
+
<script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"9ToZXcTk\",\"contributor-guide_architecture_detector-contract.md\":\"B1vUqtry\",\"contributor-guide_architecture_drill-down.md\":\"Bawx6NgP\",\"contributor-guide_architecture_impact-estimation.md\":\"DHVq1UPt\",\"contributor-guide_architecture_index.md\":\"3TO9ic6w\",\"contributor-guide_architecture_overview.md\":\"DkXdJ9jE\",\"contributor-guide_architecture_state-and-history.md\":\"C-xxn0q7\",\"contributor-guide_architecture_widget-rendering.md\":\"DYwFeyjV\",\"contributor-guide_architecture_worker-protocol.md\":\"C0SovZX8\",\"contributor-guide_contributing.md\":\"CvRsdr6J\",\"contributor-guide_development-setup.md\":\"DvMV1xhQ\",\"contributor-guide_testing.md\":\"BohuQWvl\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"Df1YMIHu\",\"tuning-reference_aqe.md\":\"BIsCtLzm\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"CEstB3Ia\",\"tuning-reference_bottleneck-cold-start.md\":\"CEuy-72y\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"CIohQDfn\",\"tuning-reference_bottleneck-failures.md\":\"CAsFf3RR\",\"tuning-reference_bottleneck-gc.md\":\"B4T3kJYV\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"BaJl__1W\",\"tuning-reference_bottleneck-memory-utilization.md\":\"DbP-SJZc\",\"tuning-reference_bottleneck-retry-waste.md\":\"D5JMjVOt\",\"tuning-reference_bottleneck-shuffle.md\":\"DDMQVOW6\",\"tuning-reference_bottleneck-skew.md\":\"B0MzhWcX\",\"tuning-reference_bottleneck-slow-host.md\":\"BlIo6UDW\",\"tuning-reference_bottleneck-small-files.md\":\"DHGMTmPs\",\"tuning-reference_bottleneck-spill.md\":\"PNH7mITt\",\"tuning-reference_bottleneck-straggler.md\":\"DIbXdAcQ\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"CB05v3zh\",\"tuning-reference_bottleneck-utilization.md\":\"DqC_iGkw\",\"tuning-reference_caching.md\":\"B7aQ8asB\",\"tuning-reference_cluster-config.md\":\"CD6vzYtD\",\"tuning-reference_config.md\":\"UvveiWG3\",\"tuning-reference_data-formats.md\":\"bjCAWH3N\",\"tuning-reference_index.md\":\"BQ_NooMV\",\"tuning-reference_intro.md\":\"CobD-lGB\",\"tuning-reference_joins.md\":\"BtKs_CuW\",\"tuning-reference_memory-model.md\":\"CWhvgevP\",\"tuning-reference_metrics.md\":\"mLOh7Apj\",\"tuning-reference_partitioning.md\":\"q0zKF_8X\",\"tuning-reference_pyspark.md\":\"DDCfvN9t\",\"tuning-reference_shuffle.md\":\"BZZ7R4Ix\",\"tuning-reference_spark-architecture.md\":\"Dwzm5avO\",\"tuning-reference_table-formats.md\":\"D6wj-2dX\",\"user-guide_alternative-log-retrieval.md\":\"B4tPGIal\",\"user-guide_getting-started.md\":\"BJvwLEIM\",\"user-guide_mcp-tools.md\":\"DGvolGzt\",\"user-guide_run-comparison.md\":\"CQc1aoU8\",\"user-guide_understanding-findings.md\":\"B29f0ucS\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Alternative log retrieval\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
|
|
27
27
|
|
|
28
28
|
</body>
|
|
29
29
|
</html>
|
|
@@ -9,9 +9,9 @@
|
|
|
9
9
|
<link rel="preload stylesheet" href="/docs/assets/style.DSixAiZE.css" as="style">
|
|
10
10
|
<link rel="preload stylesheet" href="/docs/vp-icons.css" as="style">
|
|
11
11
|
|
|
12
|
-
<script type="module" src="/docs/assets/app.
|
|
12
|
+
<script type="module" src="/docs/assets/app.BczRxXPr.js"></script>
|
|
13
13
|
<link rel="preload" href="/docs/assets/inter-roman-latin.Di8DUHzh.woff2" as="font" type="font/woff2" crossorigin="">
|
|
14
|
-
<link rel="modulepreload" href="/docs/assets/chunks/theme.
|
|
14
|
+
<link rel="modulepreload" href="/docs/assets/chunks/theme.ByUdwrLG.js">
|
|
15
15
|
<link rel="modulepreload" href="/docs/assets/chunks/framework.DSg0KOwT.js">
|
|
16
16
|
<link rel="modulepreload" href="/docs/assets/contributor-guide_architecture_index.md.3TO9ic6w.lean.js">
|
|
17
17
|
<link rel="icon" type="image/svg+xml" href="/docs/favicon.svg">
|
|
@@ -23,7 +23,7 @@
|
|
|
23
23
|
</head>
|
|
24
24
|
<body>
|
|
25
25
|
<div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Contributor Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/development-setup.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Development setup</p><!--]--></a><!----></div><!----></div><section class="VPSidebarItem level-1 collapsible collapsed is-link" data-v-b3fd67f8><div class="item" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/" data-v-b3fd67f8><!--[--><h3 class="text" data-v-b3fd67f8>Architecture</h3><!--]--></a><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-b3fd67f8><span class="vpi-chevron-right caret-icon" data-v-b3fd67f8></span></div></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/overview.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Overview</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/worker-protocol.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Worker protocol</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/state-and-history.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>State & history intake</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/detector-contract.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Detector contract</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/impact-estimation.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Impact estimation</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/widget-rendering.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Widget rendering</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/board-widgets.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Board widgets</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/drill-down.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Drill-down</p><!--]--></a><!----></div><!----></div><!--]--></div></section><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/testing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Testing & verification</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/contributing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Contributing</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_contributor-guide_architecture_" data-v-39a288b8><div><h1 id="architecture" tabindex="-1">Architecture <a class="header-anchor" href="#architecture" aria-label="Permalink to "Architecture""></a></h1><p>These pages cover one concern each: the worker protocol, the state model, the detector contract, and the fixed widget rendering order.</p><p>Where to start:</p><ul><li><a href="./overview.html#two-actors">Two actors</a>: the parser worker and the view, and why the split exists.</li><li><a href="./detector-contract.html#detector-contract">Detector contract</a>: the interface every bottleneck detector implements. Read this before adding a finding.</li><li><a href="./impact-estimation.html#impact-estimation">Impact estimation</a>: how findings get a wall-clock/resource waste estimate attached.</li><li><a href="./../testing.html#testing-layout">Testing layout</a>: where tests live and what each layer covers.</li><li><a href="./../contributing.html">Contributing</a>: how architectural decisions get recorded as ADRs.</li></ul></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/contributor-guide/development-setup.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>Development setup</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="/docs/contributor-guide/architecture/overview.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>Overview</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
|
|
26
|
-
<script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"
|
|
26
|
+
<script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"9ToZXcTk\",\"contributor-guide_architecture_detector-contract.md\":\"B1vUqtry\",\"contributor-guide_architecture_drill-down.md\":\"Bawx6NgP\",\"contributor-guide_architecture_impact-estimation.md\":\"DHVq1UPt\",\"contributor-guide_architecture_index.md\":\"3TO9ic6w\",\"contributor-guide_architecture_overview.md\":\"DkXdJ9jE\",\"contributor-guide_architecture_state-and-history.md\":\"C-xxn0q7\",\"contributor-guide_architecture_widget-rendering.md\":\"DYwFeyjV\",\"contributor-guide_architecture_worker-protocol.md\":\"C0SovZX8\",\"contributor-guide_contributing.md\":\"CvRsdr6J\",\"contributor-guide_development-setup.md\":\"DvMV1xhQ\",\"contributor-guide_testing.md\":\"BohuQWvl\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"Df1YMIHu\",\"tuning-reference_aqe.md\":\"BIsCtLzm\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"CEstB3Ia\",\"tuning-reference_bottleneck-cold-start.md\":\"CEuy-72y\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"CIohQDfn\",\"tuning-reference_bottleneck-failures.md\":\"CAsFf3RR\",\"tuning-reference_bottleneck-gc.md\":\"B4T3kJYV\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"BaJl__1W\",\"tuning-reference_bottleneck-memory-utilization.md\":\"DbP-SJZc\",\"tuning-reference_bottleneck-retry-waste.md\":\"D5JMjVOt\",\"tuning-reference_bottleneck-shuffle.md\":\"DDMQVOW6\",\"tuning-reference_bottleneck-skew.md\":\"B0MzhWcX\",\"tuning-reference_bottleneck-slow-host.md\":\"BlIo6UDW\",\"tuning-reference_bottleneck-small-files.md\":\"DHGMTmPs\",\"tuning-reference_bottleneck-spill.md\":\"PNH7mITt\",\"tuning-reference_bottleneck-straggler.md\":\"DIbXdAcQ\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"CB05v3zh\",\"tuning-reference_bottleneck-utilization.md\":\"DqC_iGkw\",\"tuning-reference_caching.md\":\"B7aQ8asB\",\"tuning-reference_cluster-config.md\":\"CD6vzYtD\",\"tuning-reference_config.md\":\"UvveiWG3\",\"tuning-reference_data-formats.md\":\"bjCAWH3N\",\"tuning-reference_index.md\":\"BQ_NooMV\",\"tuning-reference_intro.md\":\"CobD-lGB\",\"tuning-reference_joins.md\":\"BtKs_CuW\",\"tuning-reference_memory-model.md\":\"CWhvgevP\",\"tuning-reference_metrics.md\":\"mLOh7Apj\",\"tuning-reference_partitioning.md\":\"q0zKF_8X\",\"tuning-reference_pyspark.md\":\"DDCfvN9t\",\"tuning-reference_shuffle.md\":\"BZZ7R4Ix\",\"tuning-reference_spark-architecture.md\":\"Dwzm5avO\",\"tuning-reference_table-formats.md\":\"D6wj-2dX\",\"user-guide_alternative-log-retrieval.md\":\"B4tPGIal\",\"user-guide_getting-started.md\":\"BJvwLEIM\",\"user-guide_mcp-tools.md\":\"DGvolGzt\",\"user-guide_run-comparison.md\":\"CQc1aoU8\",\"user-guide_understanding-findings.md\":\"B29f0ucS\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Alternative log retrieval\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
|
|
27
27
|
|
|
28
28
|
</body>
|
|
29
29
|
</html>
|