sparkforensics-server 0.5.2 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.js +16 -7
- package/package.json +1 -1
- package/public/assets/AutoscalingChurn-DenWKTv-.js +1 -0
- package/public/assets/CacheUtilization-DLz7NdjV.js +1 -0
- package/public/assets/CachingOpportunity-zpQIqhzm.js +1 -0
- package/public/assets/ChartTheme-DAiARbLP.js +8 -0
- package/public/assets/ColdStart-C-9HjxWl.js +1 -0
- package/public/assets/ConfigAudit-D0V9tcrJ.js +1 -0
- package/public/assets/CoreUsageArea-yeerrljd.js +1 -0
- package/public/assets/Dashboard-DkUPrHKV.js +1 -0
- package/public/assets/DisclosureContext-DK1_iz6J.js +1 -0
- package/public/assets/DuplicatePlanSubtree-xDNDbCd3.js +1 -0
- package/public/assets/DurationHistogram-B6jsE0Ov.js +1 -0
- package/public/assets/EmptyState-C1uL40VU.js +1 -0
- package/public/assets/ExecutorUtilization-DXDZKmSc.js +1 -0
- package/public/assets/GcPressure-MRf7I4IG.js +1 -0
- package/public/assets/ImpactEstimate-Dm3xWbzV.js +1 -0
- package/public/assets/IncompleteRun-CaLOhMtf.js +1 -0
- package/public/assets/JobFailures-CzmzlvhO.js +1 -0
- package/public/assets/MemoryUtilization-AWUMUjCH.js +1 -0
- package/public/assets/NestedLoopJoin-C9sNr6Ss.js +1 -0
- package/public/assets/OverBroadcast-BOmRE6ca.js +1 -0
- package/public/assets/PartitionHint-BL2EmJfE.js +1 -0
- package/public/assets/PartitionSizing-BzBySvUq.js +1 -0
- package/public/assets/PlanExplorer-DdiQRcJU.js +1 -0
- package/public/assets/PlanGraphRoute-CIH9gBuR.js +1 -0
- package/public/assets/PlanView-B1hxzFRs.js +2 -0
- package/public/assets/PythonUdf-DvgCCueD.js +1 -0
- package/public/assets/RetryWaste-uXXu_CyM.js +1 -0
- package/public/assets/RowPagination-C-LzWkI5.js +1 -0
- package/public/assets/RowStatusCluster-Dbtmte7l.js +1 -0
- package/public/assets/ShuffleIO-nALwzC1_.js +1 -0
- package/public/assets/Skew-CHMEKgvk.js +1 -0
- package/public/assets/SlowHost-yyvjpHk6.js +1 -0
- package/public/assets/SmallFiles-y6odJ97n.js +1 -0
- package/public/assets/SpeculationWaste-C-SD2LOt.js +1 -0
- package/public/assets/Spill-DHTMmOup.js +1 -0
- package/public/assets/StageFailed-CQ3DxC9R.js +1 -0
- package/public/assets/StageFindingGroup-BgYLyj5q.js +1 -0
- package/public/assets/StageHeader-CIETkjVZ.js +1 -0
- package/public/assets/StagePill--P7PDd3s.js +1 -0
- package/public/assets/StageShape-BIv8vx_o.js +1 -0
- package/public/assets/StageSlowness-MMqzERxN.js +1 -0
- package/public/assets/Straggler-BgrY7AyW.js +1 -0
- package/public/assets/TaskFailures-BHHyKYTs.js +1 -0
- package/public/assets/TinyTask-DDScNhCl.js +1 -0
- package/public/assets/UnderBroadcast-CGIC04TI.js +1 -0
- package/public/assets/WidgetLeadSummary-CXFZP00C.js +1 -0
- package/public/assets/button-DbpTz2yk.js +1 -0
- package/public/assets/charts-vendor-CtTxwTeu.js +59 -0
- package/public/assets/composite-C9Nu2EI9.js +1 -0
- package/public/assets/d3-vendor-DNlD2-cz.js +1 -0
- package/public/assets/detector-registry-Bp8hrLua.js +2 -0
- package/public/assets/downsample-WmNM1qc9.js +1 -0
- package/public/assets/finding-anchor-Dr40LkS9.js +1 -0
- package/public/assets/findings-of-type-BL2u3RGZ.js +1 -0
- package/public/assets/impact-sort-mC5ISgX8.js +1 -0
- package/public/assets/inbox-KPU1Gppm.js +1 -0
- package/public/assets/index-CaRVPvAT.css +2 -0
- package/public/assets/index-fDiwfuoO.js +14 -0
- package/public/assets/inertValue-L9VxV0VE.js +1 -0
- package/public/assets/instrument-sans-latin-ext-wdth-normal-C5E2Gvlv.woff2 +0 -0
- package/public/assets/instrument-sans-latin-wdth-normal-BVScPF0l.woff2 +0 -0
- package/public/assets/jetbrains-mono-latin-ext-wght-normal-DBQx-q_a.woff2 +0 -0
- package/public/assets/jetbrains-mono-latin-wght-normal-B9CIFXIH.woff2 +0 -0
- package/public/assets/layout.worker-CLcBHxVp.js +3 -0
- package/public/assets/{parser-worker-CA5RxqKc.js → parser-worker-CRIOPMOW.js} +8 -8
- package/public/assets/plan-graph-vendor-Ce6d4HU6.js +7 -0
- package/public/assets/react-vendor-C6KcUD36.js +9 -0
- package/public/assets/select-BHqOqun5.js +1 -0
- package/public/assets/toggle-DPeN6aNa.js +1 -0
- package/public/assets/toggle-group-Kl3uyHSr.js +1 -0
- package/public/assets/usePagedRows-BafD6bEN.js +1 -0
- package/public/assets/useRecentFiles-DzJZAoOT.js +3 -0
- package/public/assets/useRegisterFieldControl-pQv-ZDoU.js +1 -0
- package/public/assets/useScrollLock-BCxn_dMp.js +1 -0
- package/public/assets/useSortMode-DjrAN-z4.js +1 -0
- package/public/docs/404.html +3 -7
- package/public/docs/assets/{app.eYTPqTeq.js → app.1gvzd5Zb.js} +1 -1
- package/public/docs/assets/chunks/@localSearchIndexroot.DS2TQP5_.js +1 -0
- package/public/docs/assets/chunks/{VPLocalSearchBox.DsZrQKQR.js → VPLocalSearchBox.CNxT3s0I.js} +1 -1
- package/public/docs/assets/chunks/{theme.DsTNE2pE.js → theme.C5lLqUv7.js} +2 -2
- package/public/docs/assets/contributor-guide_architecture_board-widgets.md.CqKM_jEG.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_board-widgets.md.CqKM_jEG.lean.js +1 -0
- package/public/docs/assets/{contributor-guide_architecture_board-widgets_caching-and-memory.md.8bH1HnZp.js → contributor-guide_architecture_board-widgets_caching-and-memory.md.BYDo3ClR.js} +1 -1
- package/public/docs/assets/contributor-guide_architecture_board-widgets_plan-advisor.md.ajUAVdbc.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_board-widgets_visual-system-and-templating.md.WqY74T0T.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_board-widgets_visual-system-and-templating.md.WqY74T0T.lean.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_detector-contract.md.CHHgywoP.js +1 -0
- package/public/docs/assets/{contributor-guide_architecture_detector-contract.md.wj6Et6UD.lean.js → contributor-guide_architecture_detector-contract.md.CHHgywoP.lean.js} +1 -1
- package/public/docs/assets/contributor-guide_architecture_detector-contract_plan-attribution.md.DIz3_oz9.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_detector-contract_plan-attribution.md.DIz3_oz9.lean.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_detector-contract_threshold-tables.md.WH4tRHPL.js +1 -0
- package/public/docs/assets/{contributor-guide_architecture_detector-contract_threshold-tables.md.CKjWG56Y.lean.js → contributor-guide_architecture_detector-contract_threshold-tables.md.WH4tRHPL.lean.js} +1 -1
- package/public/docs/assets/{contributor-guide_architecture_drill-down.md.CqZPrdo9.js → contributor-guide_architecture_drill-down.md.yB-Hz2XT.js} +1 -1
- package/public/docs/assets/{contributor-guide_architecture_drill-down.md.CqZPrdo9.lean.js → contributor-guide_architecture_drill-down.md.yB-Hz2XT.lean.js} +1 -1
- package/public/docs/assets/contributor-guide_architecture_drill-down_plan-graph.md.BQZE04sw.js +1 -0
- package/public/docs/assets/{contributor-guide_architecture_drill-down_plan-graph.md.DwOozIEs.lean.js → contributor-guide_architecture_drill-down_plan-graph.md.BQZE04sw.lean.js} +1 -1
- package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.K71YB-Rr.js +1 -0
- package/public/docs/assets/{contributor-guide_architecture_impact-estimation.md.CZv_ZGOp.lean.js → contributor-guide_architecture_impact-estimation.md.K71YB-Rr.lean.js} +1 -1
- package/public/docs/assets/contributor-guide_architecture_impact-estimation_caveats-tuning-and-coverage.md.DLCAffwn.js +1 -0
- package/public/docs/assets/{contributor-guide_architecture_impact-estimation_caveats-tuning-and-coverage.md.BUucRc0A.lean.js → contributor-guide_architecture_impact-estimation_caveats-tuning-and-coverage.md.DLCAffwn.lean.js} +1 -1
- package/public/docs/assets/contributor-guide_architecture_impact-estimation_stage-union-rollup.md.JFrzDMwo.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_impact-estimation_stage-union-rollup.md.JFrzDMwo.lean.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_overview.md.nxkpOhgf.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_state-and-history_run-comparison-internals.md.U-9KMDFK.js +1 -0
- package/public/docs/assets/{contributor-guide_architecture_state-and-history_run-comparison-internals.md.CZA0JLtV.lean.js → contributor-guide_architecture_state-and-history_run-comparison-internals.md.U-9KMDFK.lean.js} +1 -1
- package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.BrSz8Ezh.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_widget-rendering_findings-tab.md.B7rjSF_h.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_widget-rendering_findings-tab.md.B7rjSF_h.lean.js +1 -0
- package/public/docs/assets/{contributor-guide_architecture_widget-rendering_list-sort-mode.md.BuHKgH0C.js → contributor-guide_architecture_widget-rendering_list-sort-mode.md.37hcZJ7q.js} +1 -1
- package/public/docs/assets/{contributor-guide_architecture_widget-rendering_row-expand-contract.md.ZT89u9f1.js → contributor-guide_architecture_widget-rendering_row-expand-contract.md.mK_VmAWk.js} +1 -1
- package/public/docs/assets/{contributor-guide_architecture_worker-protocol.md.BAnhi2-d.js → contributor-guide_architecture_worker-protocol.md.Br6OE7RW.js} +1 -1
- package/public/docs/assets/contributor-guide_architecture_worker-protocol.md.Br6OE7RW.lean.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_worker-protocol_cli-and-mcp.md.IVyaXrGe.js +1 -0
- package/public/docs/assets/{contributor-guide_architecture_worker-protocol_cli-and-mcp.md.DC1YqXBp.lean.js → contributor-guide_architecture_worker-protocol_cli-and-mcp.md.IVyaXrGe.lean.js} +1 -1
- package/public/docs/assets/{contributor-guide_architecture_worker-protocol_core-and-event-validation.md.DSVFPpWb.js → contributor-guide_architecture_worker-protocol_core-and-event-validation.md.DVSXn-bb.js} +1 -1
- package/public/docs/assets/{contributor-guide_architecture_worker-protocol_evidence-availability.md.CjimIDBY.js → contributor-guide_architecture_worker-protocol_evidence-availability.md.D3S5cyOW.js} +2 -2
- package/public/docs/assets/{contributor-guide_architecture_worker-protocol_evidence-availability.md.CjimIDBY.lean.js → contributor-guide_architecture_worker-protocol_evidence-availability.md.D3S5cyOW.lean.js} +1 -1
- package/public/docs/assets/contributor-guide_architecture_worker-protocol_evidence-report.md.CtmCU57I.js +1 -0
- package/public/docs/assets/{contributor-guide_architecture_worker-protocol_evidence-report.md.DVGZOWvW.lean.js → contributor-guide_architecture_worker-protocol_evidence-report.md.CtmCU57I.lean.js} +1 -1
- package/public/docs/assets/contributor-guide_contributing.md.DBW4drtZ.js +1 -0
- package/public/docs/assets/{contributor-guide_development-setup.md.D9bu4j_c.js → contributor-guide_development-setup.md.B21ZTDhn.js} +3 -3
- package/public/docs/assets/contributor-guide_development-setup.md.B21ZTDhn.lean.js +1 -0
- package/public/docs/assets/contributor-guide_testing.md.DpkozNoG.js +4 -0
- package/public/docs/assets/{contributor-guide_testing.md.By6XGR9a.lean.js → contributor-guide_testing.md.DpkozNoG.lean.js} +1 -1
- package/public/docs/assets/{index.md.CHJVslga.js → index.md.lMm7SUMz.js} +1 -1
- package/public/docs/assets/{index.md.CHJVslga.lean.js → index.md.lMm7SUMz.lean.js} +1 -1
- package/public/docs/assets/instrument-sans-latin-ext-wdth-normal.C5E2Gvlv.woff2 +0 -0
- package/public/docs/assets/instrument-sans-latin-wdth-normal.BVScPF0l.woff2 +0 -0
- package/public/docs/assets/jetbrains-mono-cyrillic-wght-normal.D73BlboJ.woff2 +0 -0
- package/public/docs/assets/jetbrains-mono-greek-wght-normal.Bw9x6K1M.woff2 +0 -0
- package/public/docs/assets/jetbrains-mono-latin-ext-wght-normal.DBQx-q_a.woff2 +0 -0
- package/public/docs/assets/jetbrains-mono-latin-wght-normal.B9CIFXIH.woff2 +0 -0
- package/public/docs/assets/jetbrains-mono-vietnamese-wght-normal.Bt-aOZkq.woff2 +0 -0
- package/public/docs/assets/style.Cfxqlj0y.css +1 -0
- package/public/docs/assets/user-guide_getting-started_ci-and-automation.md.C0pcQoNc.js +29 -0
- package/public/docs/assets/user-guide_getting-started_ci-and-automation.md.C0pcQoNc.lean.js +1 -0
- package/public/docs/assets/user-guide_getting-started_reading-the-dashboard.md.CGaxPX_1.js +1 -0
- package/public/docs/assets/{user-guide_getting-started_tuning-thresholds.md.B_MyjyZm.js → user-guide_getting-started_tuning-thresholds.md.BGVE-wrc.js} +2 -2
- package/public/docs/assets/{user-guide_getting-started_tuning-thresholds.md.B_MyjyZm.lean.js → user-guide_getting-started_tuning-thresholds.md.BGVE-wrc.lean.js} +1 -1
- package/public/docs/assets/{user-guide_mcp-tools.md.C65SjWUL.js → user-guide_mcp-tools.md.49r3L2YE.js} +1 -1
- package/public/docs/assets/{user-guide_mcp-tools_compare-runs.md.Cc_gLoix.js → user-guide_mcp-tools_compare-runs.md.BQ_kIptC.js} +11 -6
- package/public/docs/assets/user-guide_mcp-tools_compare-runs.md.BQ_kIptC.lean.js +1 -0
- package/public/docs/assets/user-guide_mcp-tools_diagnose-run.md.azCZXkH8.js +44 -0
- package/public/docs/assets/{user-guide_mcp-tools_diagnose-run.md.BP1vnHA5.lean.js → user-guide_mcp-tools_diagnose-run.md.azCZXkH8.lean.js} +1 -1
- package/public/docs/assets/{user-guide_mcp-tools_evaluate-budgets.md.BB0nDY3Z.js → user-guide_mcp-tools_evaluate-budgets.md.dpQpRQ7z.js} +2 -2
- package/public/docs/assets/{user-guide_mcp-tools_evaluate-budgets.md.BB0nDY3Z.lean.js → user-guide_mcp-tools_evaluate-budgets.md.dpQpRQ7z.lean.js} +1 -1
- package/public/docs/assets/{user-guide_mcp-tools_get-finding-evidence.md.rKyu4wts.js → user-guide_mcp-tools_get-finding-evidence.md.CH6FkNLX.js} +2 -1
- package/public/docs/assets/{user-guide_mcp-tools_get-finding-evidence.md.rKyu4wts.lean.js → user-guide_mcp-tools_get-finding-evidence.md.CH6FkNLX.lean.js} +1 -1
- package/public/docs/assets/{user-guide_mcp-tools_get-run-summary.md.DpxZ4ZNp.js → user-guide_mcp-tools_get-run-summary.md.EhaIsele.js} +1 -1
- package/public/docs/assets/user-guide_run-comparison.md.B_LOkdBf.js +1 -0
- package/public/docs/assets/user-guide_run-comparison.md.B_LOkdBf.lean.js +1 -0
- package/public/docs/assets/user-guide_understanding-findings.md.C4riKCKl.js +1 -0
- package/public/docs/assets/user-guide_understanding-findings.md.C4riKCKl.lean.js +1 -0
- package/public/docs/contributor-guide/architecture/board-widgets/caching-and-memory.html +6 -10
- package/public/docs/contributor-guide/architecture/board-widgets/plan-advisor.html +6 -10
- package/public/docs/contributor-guide/architecture/board-widgets/visual-system-and-templating.html +6 -10
- package/public/docs/contributor-guide/architecture/board-widgets.html +6 -10
- package/public/docs/contributor-guide/architecture/detector-contract/plan-attribution.html +6 -10
- package/public/docs/contributor-guide/architecture/detector-contract/threshold-tables.html +6 -10
- package/public/docs/contributor-guide/architecture/detector-contract.html +6 -10
- package/public/docs/contributor-guide/architecture/drill-down/plan-graph.html +6 -10
- package/public/docs/contributor-guide/architecture/drill-down.html +6 -10
- package/public/docs/contributor-guide/architecture/impact-estimation/caveats-tuning-and-coverage.html +6 -10
- package/public/docs/contributor-guide/architecture/impact-estimation/stage-union-rollup.html +6 -10
- package/public/docs/contributor-guide/architecture/impact-estimation.html +6 -10
- package/public/docs/contributor-guide/architecture/index.html +5 -9
- package/public/docs/contributor-guide/architecture/overview.html +6 -10
- package/public/docs/contributor-guide/architecture/state-and-history/history-server-intake.html +5 -9
- package/public/docs/contributor-guide/architecture/state-and-history/run-comparison-internals.html +6 -10
- package/public/docs/contributor-guide/architecture/state-and-history.html +5 -9
- package/public/docs/contributor-guide/architecture/widget-rendering/findings-tab.html +6 -10
- package/public/docs/contributor-guide/architecture/widget-rendering/investigation-routing.html +5 -9
- package/public/docs/contributor-guide/architecture/widget-rendering/list-sort-mode.html +6 -10
- package/public/docs/contributor-guide/architecture/widget-rendering/reference-section.html +5 -9
- package/public/docs/contributor-guide/architecture/widget-rendering/row-expand-contract.html +6 -10
- package/public/docs/contributor-guide/architecture/widget-rendering.html +6 -10
- package/public/docs/contributor-guide/architecture/worker-protocol/cli-and-mcp.html +6 -10
- package/public/docs/contributor-guide/architecture/worker-protocol/core-and-event-validation.html +6 -10
- package/public/docs/contributor-guide/architecture/worker-protocol/evidence-availability.html +7 -11
- package/public/docs/contributor-guide/architecture/worker-protocol/evidence-report.html +6 -10
- package/public/docs/contributor-guide/architecture/worker-protocol.html +6 -10
- package/public/docs/contributor-guide/contributing.html +6 -10
- package/public/docs/contributor-guide/development-setup.html +8 -12
- package/public/docs/contributor-guide/testing.html +9 -10
- package/public/docs/favicon.svg +2 -2
- package/public/docs/hashmap.json +1 -1
- package/public/docs/index.html +6 -10
- package/public/docs/logo-dark.svg +1 -0
- package/public/docs/logo-light.svg +1 -0
- package/public/docs/tuning-reference/anti-patterns.html +5 -9
- package/public/docs/tuning-reference/aqe.html +5 -9
- package/public/docs/tuning-reference/bottleneck-broadcast-sizing.html +5 -9
- package/public/docs/tuning-reference/bottleneck-cold-start.html +5 -9
- package/public/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html +5 -9
- package/public/docs/tuning-reference/bottleneck-failures.html +5 -9
- package/public/docs/tuning-reference/bottleneck-gc.html +5 -9
- package/public/docs/tuning-reference/bottleneck-job-failure-rate.html +5 -9
- package/public/docs/tuning-reference/bottleneck-memory-utilization.html +5 -9
- package/public/docs/tuning-reference/bottleneck-retry-waste.html +5 -9
- package/public/docs/tuning-reference/bottleneck-shuffle.html +5 -9
- package/public/docs/tuning-reference/bottleneck-skew.html +5 -9
- package/public/docs/tuning-reference/bottleneck-slow-host.html +5 -9
- package/public/docs/tuning-reference/bottleneck-small-files.html +5 -9
- package/public/docs/tuning-reference/bottleneck-spill.html +5 -9
- package/public/docs/tuning-reference/bottleneck-straggler.html +5 -9
- package/public/docs/tuning-reference/bottleneck-tiny-tasks.html +5 -9
- package/public/docs/tuning-reference/bottleneck-utilization.html +5 -9
- package/public/docs/tuning-reference/caching.html +5 -9
- package/public/docs/tuning-reference/cluster-config.html +5 -9
- package/public/docs/tuning-reference/config.html +5 -9
- package/public/docs/tuning-reference/data-formats.html +5 -9
- package/public/docs/tuning-reference/index.html +5 -9
- package/public/docs/tuning-reference/intro.html +5 -9
- package/public/docs/tuning-reference/joins.html +5 -9
- package/public/docs/tuning-reference/memory-model.html +5 -9
- package/public/docs/tuning-reference/metrics.html +5 -9
- package/public/docs/tuning-reference/partitioning.html +5 -9
- package/public/docs/tuning-reference/pyspark.html +5 -9
- package/public/docs/tuning-reference/shuffle.html +5 -9
- package/public/docs/tuning-reference/spark-architecture.html +5 -9
- package/public/docs/tuning-reference/table-formats.html +5 -9
- package/public/docs/user-guide/alternative-log-retrieval/managed-platforms.html +5 -9
- package/public/docs/user-guide/alternative-log-retrieval/ssh-bastion.html +5 -9
- package/public/docs/user-guide/alternative-log-retrieval.html +5 -9
- package/public/docs/user-guide/getting-started/ci-and-automation.html +17 -15
- package/public/docs/user-guide/getting-started/reading-the-dashboard.html +6 -10
- package/public/docs/user-guide/getting-started/regression-metric-keys.html +5 -9
- package/public/docs/user-guide/getting-started/tuning-thresholds.html +7 -11
- package/public/docs/user-guide/getting-started.html +5 -9
- package/public/docs/user-guide/mcp-tools/compare-runs.html +16 -15
- package/public/docs/user-guide/mcp-tools/diagnose-run.html +28 -11
- package/public/docs/user-guide/mcp-tools/evaluate-budgets.html +6 -10
- package/public/docs/user-guide/mcp-tools/get-finding-documentation.html +5 -9
- package/public/docs/user-guide/mcp-tools/get-finding-evidence.html +7 -10
- package/public/docs/user-guide/mcp-tools/get-reference-doc.html +5 -9
- package/public/docs/user-guide/mcp-tools/get-run-summary.html +6 -10
- package/public/docs/user-guide/mcp-tools/list-runs.html +5 -9
- package/public/docs/user-guide/mcp-tools.html +6 -10
- package/public/docs/user-guide/run-comparison/how-stages-are-matched.html +5 -9
- package/public/docs/user-guide/run-comparison.html +6 -10
- package/public/docs/user-guide/understanding-findings.html +6 -10
- package/public/export-template.html +46 -46
- package/public/favicon.svg +2 -2
- package/public/index.html +8 -10
- package/public/sample-runs/sample-run.ndjson.gz +0 -0
- package/vendor-core/allocation.js +126 -39
- package/vendor-core/analyzer.js +85 -58
- package/vendor-core/api-merge-writes.js +3 -0
- package/vendor-core/aqe-skew.js +347 -0
- package/vendor-core/cli/budgets.js +6 -4
- package/vendor-core/cli/collect-run-pair.js +89 -0
- package/vendor-core/cli/collect-run-worker.js +16 -0
- package/vendor-core/cli/collect-run.js +1 -2
- package/vendor-core/cli/native-zstd.js +12 -16
- package/vendor-core/comparability.js +234 -0
- package/vendor-core/comparison-output.js +15 -7
- package/vendor-core/comparison-verdict.js +105 -25
- package/vendor-core/core-source-hash.txt +1 -1
- package/vendor-core/core-time-series.js +0 -1
- package/vendor-core/core-usage-locality.js +1 -1
- package/vendor-core/delta-log-paths.js +10 -1
- package/vendor-core/detectors.js +1078 -416
- package/vendor-core/docs-config.js +0 -4
- package/vendor-core/docs-content/detection/cache.md +2 -1
- package/vendor-core/docs-content/detection/cfg.md +19 -11
- package/vendor-core/docs-content/detection/cold.md +2 -1
- package/vendor-core/docs-content/detection/gc.md +7 -3
- package/vendor-core/docs-content/detection/host.md +11 -1
- package/vendor-core/docs-content/detection/mem.md +17 -6
- package/vendor-core/docs-content/detection/part.md +3 -1
- package/vendor-core/docs-content/detection/plan.md +55 -13
- package/vendor-core/docs-content/detection/shfl.md +11 -5
- package/vendor-core/docs-content/detection/skew.md +48 -1
- package/vendor-core/docs-content/detection/slow.md +4 -2
- package/vendor-core/docs-content/detection/spec.md +6 -1
- package/vendor-core/docs-content/detection/spill.md +6 -3
- package/vendor-core/docs-content/detection/strag.md +19 -6
- package/vendor-core/docs-content/detection/tiny.md +9 -3
- package/vendor-core/docs-content/detection/util.md +6 -2
- package/vendor-core/effective-conf.js +7 -10
- package/vendor-core/efficiency-model.js +24 -16
- package/vendor-core/etl-phases.js +1 -2
- package/vendor-core/event-handlers.js +175 -57
- package/vendor-core/event-schemas.js +21 -16
- package/vendor-core/evidence-availability.js +9 -2
- package/vendor-core/evidence-report.js +118 -35
- package/vendor-core/executor-peaks.js +41 -0
- package/vendor-core/export-data.js +5 -5
- package/vendor-core/finding-identity.js +78 -0
- package/vendor-core/finding-presentation.js +129 -39
- package/vendor-core/finding-types.js +108 -30
- package/vendor-core/format-utils.js +14 -15
- package/vendor-core/html-export.js +1 -1
- package/vendor-core/impact-band.js +2 -2
- package/vendor-core/impact-format.js +47 -19
- package/vendor-core/impact-model.js +28 -17
- package/vendor-core/job-groups.js +4 -4
- package/vendor-core/mcp-server-factory.js +48 -12
- package/vendor-core/mcp-tools.js +179 -24
- package/vendor-core/median.js +2 -2
- package/vendor-core/nested-loop-join.js +73 -0
- package/vendor-core/occupancy.js +1 -2
- package/vendor-core/parser-worker.js +1 -1
- package/vendor-core/plan-graph-model.js +14 -22
- package/vendor-core/plan-node-detail.js +2 -3
- package/vendor-core/plan-summary.js +24 -19
- package/vendor-core/plan-tree-walk.js +6 -8
- package/vendor-core/python-stage.js +12 -1
- package/vendor-core/recommendation-rollup.js +13 -12
- package/vendor-core/redact.js +52 -46
- package/vendor-core/remediation.js +10 -5
- package/vendor-core/run-aggregates.js +6 -8
- package/vendor-core/run-comparison.js +226 -66
- package/vendor-core/run-identity.js +12 -0
- package/vendor-core/run-interpretation.js +21 -0
- package/vendor-core/run-metrics.js +2 -2
- package/vendor-core/run-output.js +44 -11
- package/vendor-core/run-totals.js +12 -0
- package/vendor-core/run-verdict.js +76 -3
- package/vendor-core/scaling-sim.js +3 -4
- package/vendor-core/snappy-block.js +2 -2
- package/vendor-core/spark-conf.js +168 -0
- package/vendor-core/sql-stages.js +2 -2
- package/vendor-core/stage-alignment.js +2 -12
- package/vendor-core/stage-identity.js +58 -3
- package/vendor-core/stage-label.js +14 -0
- package/vendor-core/stage-plan-nodes.js +17 -3
- package/vendor-core/stage-quantiles.js +141 -9
- package/vendor-core/stage-structure.js +17 -3
- package/vendor-core/threshold-overrides.js +9 -3
- package/vendor-core/types.js +70 -23
- package/vendor-core/wall-clock.js +8 -0
- package/vendor-core/wasted-core-hours.js +12 -12
- package/vendor-core/write-targets.js +146 -14
- package/vendor-core/zstd-worker-client.js +1 -1
- package/vendor-core/zstd-worker.js +1 -1
- package/public/assets/AutoscalingChurn-8t3U3isq.js +0 -1
- package/public/assets/CacheUtilization-CJs9tTum.js +0 -1
- package/public/assets/CachingOpportunity-D1NcHs0V.js +0 -1
- package/public/assets/ColdStart-CdRJ9gD8.js +0 -1
- package/public/assets/ConfigAudit-B5Gbe5js.js +0 -1
- package/public/assets/CoreUsageArea-a--NTRGY.js +0 -1
- package/public/assets/Dashboard-C1hNeX49.js +0 -2
- package/public/assets/DisclosureContext-ZpE6CkHx.js +0 -1
- package/public/assets/DuplicatePlanSubtree-dobXdz7E.js +0 -1
- package/public/assets/ExecutorUtilization-D9grwi7t.js +0 -1
- package/public/assets/GcPressure-8-G2ri10.js +0 -1
- package/public/assets/IncompleteRun-B91lky7a.js +0 -1
- package/public/assets/JobFailures-BsWvBMyJ.js +0 -1
- package/public/assets/MemoryUtilization-BM5schZI.js +0 -1
- package/public/assets/OverBroadcast-DdN8JIJH.js +0 -1
- package/public/assets/PartitionHint-CUvbnnxy.js +0 -1
- package/public/assets/PartitionSizing-CY5HDYtp.js +0 -1
- package/public/assets/PlanExplorer-DgHCDBD_.js +0 -1
- package/public/assets/PlanGraphRoute-DH3KJb8j.js +0 -1
- package/public/assets/RetryWaste-DOnE9vGF.js +0 -1
- package/public/assets/ShuffleIO-MMnDESkF.js +0 -1
- package/public/assets/Skew-BmZZJ5ms.js +0 -1
- package/public/assets/SlowHost-CsJzfEqo.js +0 -1
- package/public/assets/SmallFiles-BoK3R1Ex.js +0 -1
- package/public/assets/SpeculationWaste-CnVxB5AJ.js +0 -1
- package/public/assets/Spill-BhNicyBy.js +0 -1
- package/public/assets/StageFailed-mAhHU4l_.js +0 -1
- package/public/assets/StageFindingGroup-BEtPWcT0.js +0 -1
- package/public/assets/StageHeader-DJkYkrsU.js +0 -1
- package/public/assets/StageShape-BDxRknLp.js +0 -1
- package/public/assets/StageSlowness-BPRw-otN.js +0 -1
- package/public/assets/Straggler-CZCbnOWP.js +0 -1
- package/public/assets/TaskFailures-C57YcLFQ.js +0 -1
- package/public/assets/TinyTask-Bi84qU7n.js +0 -1
- package/public/assets/UnderBroadcast-DdQV7htE.js +0 -1
- package/public/assets/charts-vendor-0VMaLVLN.js +0 -59
- package/public/assets/finding-anchor-Cl6_5hfV.js +0 -1
- package/public/assets/impact-sort-CssZ6MT2.js +0 -1
- package/public/assets/index-2h_TgUAb.css +0 -2
- package/public/assets/index-B-03mG7K.js +0 -14
- package/public/assets/plan-graph-vendor-BYl-AzN5.js +0 -7
- package/public/assets/react-vendor-DEdfrRTJ.js +0 -9
- package/public/assets/select-CQjNhprm.js +0 -1
- package/public/assets/usePagedRows-Dyl6T8Ok.js +0 -1
- package/public/assets/useRecentFiles-XZEnLlal.js +0 -11
- package/public/assets/useSortMode-Dhvit8dt.js +0 -1
- package/public/docs/assets/chunks/@localSearchIndexroot.u_e5bQz4.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_board-widgets.md.YQcj5EI4.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_board-widgets.md.YQcj5EI4.lean.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_board-widgets_plan-advisor.md.RlzvUmNm.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_board-widgets_visual-system-and-templating.md.DYgUpIJB.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_board-widgets_visual-system-and-templating.md.DYgUpIJB.lean.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_detector-contract.md.wj6Et6UD.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_detector-contract_plan-attribution.md.yNex4bys.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_detector-contract_plan-attribution.md.yNex4bys.lean.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_detector-contract_threshold-tables.md.CKjWG56Y.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_drill-down_plan-graph.md.DwOozIEs.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.CZv_ZGOp.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_impact-estimation_caveats-tuning-and-coverage.md.BUucRc0A.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_impact-estimation_stage-union-rollup.md.DdCGtAGu.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_impact-estimation_stage-union-rollup.md.DdCGtAGu.lean.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_overview.md.C1sSvm9i.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_state-and-history_run-comparison-internals.md.CZA0JLtV.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.BOZ1azXG.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_widget-rendering_findings-tab.md.BQXycK6G.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_widget-rendering_findings-tab.md.BQXycK6G.lean.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_worker-protocol.md.BAnhi2-d.lean.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_worker-protocol_cli-and-mcp.md.DC1YqXBp.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_worker-protocol_evidence-report.md.DVGZOWvW.js +0 -1
- package/public/docs/assets/contributor-guide_contributing.md.CQMvGwFt.js +0 -1
- package/public/docs/assets/contributor-guide_development-setup.md.D9bu4j_c.lean.js +0 -1
- package/public/docs/assets/contributor-guide_testing.md.By6XGR9a.js +0 -1
- package/public/docs/assets/inter-italic-cyrillic-ext.r48I6akx.woff2 +0 -0
- package/public/docs/assets/inter-italic-cyrillic.By2_1cv3.woff2 +0 -0
- package/public/docs/assets/inter-italic-greek-ext.1u6EdAuj.woff2 +0 -0
- package/public/docs/assets/inter-italic-greek.DJ8dCoTZ.woff2 +0 -0
- package/public/docs/assets/inter-italic-latin-ext.CN1xVJS-.woff2 +0 -0
- package/public/docs/assets/inter-italic-latin.C2AdPX0b.woff2 +0 -0
- package/public/docs/assets/inter-italic-vietnamese.BSbpV94h.woff2 +0 -0
- package/public/docs/assets/inter-roman-cyrillic-ext.BBPuwvHQ.woff2 +0 -0
- package/public/docs/assets/inter-roman-cyrillic.C5lxZ8CY.woff2 +0 -0
- package/public/docs/assets/inter-roman-greek-ext.CqjqNYQ-.woff2 +0 -0
- package/public/docs/assets/inter-roman-greek.BBVDIX6e.woff2 +0 -0
- package/public/docs/assets/inter-roman-latin-ext.4ZJIpNVo.woff2 +0 -0
- package/public/docs/assets/inter-roman-latin.Di8DUHzh.woff2 +0 -0
- package/public/docs/assets/inter-roman-vietnamese.BjW4sHH5.woff2 +0 -0
- package/public/docs/assets/style.tmDhD3c8.css +0 -1
- package/public/docs/assets/user-guide_getting-started_ci-and-automation.md.OUQqw3mI.js +0 -23
- package/public/docs/assets/user-guide_getting-started_ci-and-automation.md.OUQqw3mI.lean.js +0 -1
- package/public/docs/assets/user-guide_getting-started_reading-the-dashboard.md.Obf3W3__.js +0 -1
- package/public/docs/assets/user-guide_mcp-tools_compare-runs.md.Cc_gLoix.lean.js +0 -1
- package/public/docs/assets/user-guide_mcp-tools_diagnose-run.md.BP1vnHA5.js +0 -23
- package/public/docs/assets/user-guide_run-comparison.md.BUALkohi.js +0 -1
- package/public/docs/assets/user-guide_run-comparison.md.BUALkohi.lean.js +0 -1
- package/public/docs/assets/user-guide_understanding-findings.md.C9YRruMT.js +0 -1
- package/public/docs/assets/user-guide_understanding-findings.md.C9YRruMT.lean.js +0 -1
- /package/public/docs/assets/{contributor-guide_architecture_board-widgets_caching-and-memory.md.8bH1HnZp.lean.js → contributor-guide_architecture_board-widgets_caching-and-memory.md.BYDo3ClR.lean.js} +0 -0
- /package/public/docs/assets/{contributor-guide_architecture_board-widgets_plan-advisor.md.RlzvUmNm.lean.js → contributor-guide_architecture_board-widgets_plan-advisor.md.ajUAVdbc.lean.js} +0 -0
- /package/public/docs/assets/{contributor-guide_architecture_overview.md.C1sSvm9i.lean.js → contributor-guide_architecture_overview.md.nxkpOhgf.lean.js} +0 -0
- /package/public/docs/assets/{contributor-guide_architecture_widget-rendering.md.BOZ1azXG.lean.js → contributor-guide_architecture_widget-rendering.md.BrSz8Ezh.lean.js} +0 -0
- /package/public/docs/assets/{contributor-guide_architecture_widget-rendering_list-sort-mode.md.BuHKgH0C.lean.js → contributor-guide_architecture_widget-rendering_list-sort-mode.md.37hcZJ7q.lean.js} +0 -0
- /package/public/docs/assets/{contributor-guide_architecture_widget-rendering_row-expand-contract.md.ZT89u9f1.lean.js → contributor-guide_architecture_widget-rendering_row-expand-contract.md.mK_VmAWk.lean.js} +0 -0
- /package/public/docs/assets/{contributor-guide_architecture_worker-protocol_core-and-event-validation.md.DSVFPpWb.lean.js → contributor-guide_architecture_worker-protocol_core-and-event-validation.md.DVSXn-bb.lean.js} +0 -0
- /package/public/docs/assets/{contributor-guide_contributing.md.CQMvGwFt.lean.js → contributor-guide_contributing.md.DBW4drtZ.lean.js} +0 -0
- /package/public/docs/assets/{user-guide_getting-started_reading-the-dashboard.md.Obf3W3__.lean.js → user-guide_getting-started_reading-the-dashboard.md.CGaxPX_1.lean.js} +0 -0
- /package/public/docs/assets/{user-guide_mcp-tools.md.C65SjWUL.lean.js → user-guide_mcp-tools.md.49r3L2YE.lean.js} +0 -0
- /package/public/docs/assets/{user-guide_mcp-tools_get-run-summary.md.DpxZ4ZNp.lean.js → user-guide_mcp-tools_get-run-summary.md.EhaIsele.lean.js} +0 -0
package/public/docs/index.html
CHANGED
|
@@ -6,25 +6,21 @@
|
|
|
6
6
|
<title>SparkForensics</title>
|
|
7
7
|
<meta name="description" content="Docs for using and contributing to SparkForensics">
|
|
8
8
|
<meta name="generator" content="VitePress v1.6.4">
|
|
9
|
-
<link rel="preload stylesheet" href="/docs/assets/style.
|
|
9
|
+
<link rel="preload stylesheet" href="/docs/assets/style.Cfxqlj0y.css" as="style">
|
|
10
10
|
<link rel="preload stylesheet" href="/docs/vp-icons.css" as="style">
|
|
11
11
|
|
|
12
|
-
<script type="module" src="/docs/assets/app.
|
|
13
|
-
<link rel="
|
|
14
|
-
<link rel="modulepreload" href="/docs/assets/chunks/theme.DsTNE2pE.js">
|
|
12
|
+
<script type="module" src="/docs/assets/app.1gvzd5Zb.js"></script>
|
|
13
|
+
<link rel="modulepreload" href="/docs/assets/chunks/theme.C5lLqUv7.js">
|
|
15
14
|
<link rel="modulepreload" href="/docs/assets/chunks/framework.DSg0KOwT.js">
|
|
16
|
-
<link rel="modulepreload" href="/docs/assets/index.md.
|
|
15
|
+
<link rel="modulepreload" href="/docs/assets/index.md.lMm7SUMz.lean.js">
|
|
17
16
|
<script>(()=>{const c=`${`[${["data","shuffle","product","bar"].join("-")}]`}, .VPNav, .VPLocalNav`,a=24,d=document.documentElement,l=t=>{const e=getComputedStyle(t);return e.display==="none"||t.getClientRects().length===0?0:e.position==="fixed"?t.getBoundingClientRect().bottom:e.position==="sticky"?(parseFloat(e.top)||0)+t.offsetHeight:0},u=()=>{try{return location.hash?document.getElementById(decodeURIComponent(location.hash.slice(1))):null}catch{return null}};let r=null;const s=()=>{const t=Array.from(document.querySelectorAll(c),l),e=Math.ceil(Math.max(0,...t))+a;if(e===r)return;const n=r;r=e,d.style.setProperty("--sf-anchor-offset",`${r}px`);const o=n===null?null:u();if(!o)return;const f=(parseFloat(getComputedStyle(o).paddingTop)||0)+2;Math.abs(o.getBoundingClientRect().top-n)<=f&&o.scrollIntoView()},i=()=>{const t=document.createElement("div");t.setAttribute("data-sf-anchor-offset",""),t.setAttribute("aria-hidden","true"),document.body.appendChild(t);const e=window.ResizeObserver?new ResizeObserver(s):null,n=()=>{e&&document.querySelectorAll(c).forEach(o=>e.observe(o)),s()};n(),new MutationObserver(n).observe(document.body,{childList:!0,subtree:!0}),window.addEventListener("resize",s),window.addEventListener("load",s)};document.readyState==="loading"?document.addEventListener("DOMContentLoaded",i,{once:!0}):i()})();</script>
|
|
18
17
|
<link rel="icon" type="image/svg+xml" href="/docs/favicon.svg">
|
|
19
|
-
<link rel="preconnect" href="https://fonts.googleapis.com">
|
|
20
|
-
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin="">
|
|
21
|
-
<link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Recursive:wght,CASL@400..700,0..1&family=JetBrains+Mono:wght@400;500;600&display=swap">
|
|
22
18
|
<script id="check-dark-mode">(()=>{const e=localStorage.getItem("vitepress-theme-appearance")||"auto",a=window.matchMedia("(prefers-color-scheme: dark)").matches;(!e||e==="auto"?a:e==="dark")&&document.documentElement.classList.add("dark")})();</script>
|
|
23
19
|
<script id="check-mac-os">document.documentElement.classList.toggle("mac",/Mac|iPhone|iPod|iPad/i.test(navigator.platform));</script>
|
|
24
20
|
</head>
|
|
25
21
|
<body>
|
|
26
|
-
<div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/
|
|
27
|
-
<script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"YQcj5EI4\",\"contributor-guide_architecture_board-widgets_caching-and-memory.md\":\"8bH1HnZp\",\"contributor-guide_architecture_board-widgets_plan-advisor.md\":\"RlzvUmNm\",\"contributor-guide_architecture_board-widgets_visual-system-and-templating.md\":\"DYgUpIJB\",\"contributor-guide_architecture_detector-contract.md\":\"wj6Et6UD\",\"contributor-guide_architecture_detector-contract_plan-attribution.md\":\"yNex4bys\",\"contributor-guide_architecture_detector-contract_threshold-tables.md\":\"CKjWG56Y\",\"contributor-guide_architecture_drill-down.md\":\"CqZPrdo9\",\"contributor-guide_architecture_drill-down_plan-graph.md\":\"DwOozIEs\",\"contributor-guide_architecture_impact-estimation.md\":\"CZv_ZGOp\",\"contributor-guide_architecture_impact-estimation_caveats-tuning-and-coverage.md\":\"BUucRc0A\",\"contributor-guide_architecture_impact-estimation_stage-union-rollup.md\":\"DdCGtAGu\",\"contributor-guide_architecture_index.md\":\"tymxf1Fh\",\"contributor-guide_architecture_overview.md\":\"C1sSvm9i\",\"contributor-guide_architecture_state-and-history.md\":\"DyCBdjuM\",\"contributor-guide_architecture_state-and-history_history-server-intake.md\":\"C298hrHZ\",\"contributor-guide_architecture_state-and-history_run-comparison-internals.md\":\"CZA0JLtV\",\"contributor-guide_architecture_widget-rendering.md\":\"BOZ1azXG\",\"contributor-guide_architecture_widget-rendering_findings-tab.md\":\"BQXycK6G\",\"contributor-guide_architecture_widget-rendering_investigation-routing.md\":\"BGdiyElu\",\"contributor-guide_architecture_widget-rendering_list-sort-mode.md\":\"BuHKgH0C\",\"contributor-guide_architecture_widget-rendering_reference-section.md\":\"DWqC80bN\",\"contributor-guide_architecture_widget-rendering_row-expand-contract.md\":\"ZT89u9f1\",\"contributor-guide_architecture_worker-protocol.md\":\"BAnhi2-d\",\"contributor-guide_architecture_worker-protocol_cli-and-mcp.md\":\"DC1YqXBp\",\"contributor-guide_architecture_worker-protocol_core-and-event-validation.md\":\"DSVFPpWb\",\"contributor-guide_architecture_worker-protocol_evidence-availability.md\":\"CjimIDBY\",\"contributor-guide_architecture_worker-protocol_evidence-report.md\":\"DVGZOWvW\",\"contributor-guide_contributing.md\":\"CQMvGwFt\",\"contributor-guide_development-setup.md\":\"D9bu4j_c\",\"contributor-guide_testing.md\":\"By6XGR9a\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"D08Uaod7\",\"tuning-reference_aqe.md\":\"C1Ae6A8u\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"6wjKuJNE\",\"tuning-reference_bottleneck-cold-start.md\":\"BVQbsuHP\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"DtaYnaiF\",\"tuning-reference_bottleneck-failures.md\":\"Z1Tdmdd3\",\"tuning-reference_bottleneck-gc.md\":\"BCnXLf1n\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"CcvQj2g9\",\"tuning-reference_bottleneck-memory-utilization.md\":\"B-C0-8h-\",\"tuning-reference_bottleneck-retry-waste.md\":\"DCnhWiqj\",\"tuning-reference_bottleneck-shuffle.md\":\"B5OMJCd3\",\"tuning-reference_bottleneck-skew.md\":\"Cvo6NQm_\",\"tuning-reference_bottleneck-slow-host.md\":\"ja5qr3wq\",\"tuning-reference_bottleneck-small-files.md\":\"Crjtb0i2\",\"tuning-reference_bottleneck-spill.md\":\"Bl8KtSRg\",\"tuning-reference_bottleneck-straggler.md\":\"B0VN8reT\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"DQCftGhu\",\"tuning-reference_bottleneck-utilization.md\":\"BXDl30f9\",\"tuning-reference_caching.md\":\"CcaiMn_7\",\"tuning-reference_cluster-config.md\":\"DbWG1yZq\",\"tuning-reference_config.md\":\"YUqSpwFB\",\"tuning-reference_data-formats.md\":\"C5DX5xL4\",\"tuning-reference_index.md\":\"ZVIpQ6Dk\",\"tuning-reference_intro.md\":\"BSSaDlqx\",\"tuning-reference_joins.md\":\"BVhNCDnE\",\"tuning-reference_memory-model.md\":\"CfjFdgbl\",\"tuning-reference_metrics.md\":\"C64VNEL4\",\"tuning-reference_partitioning.md\":\"CtLK0xsS\",\"tuning-reference_pyspark.md\":\"D-92XgF8\",\"tuning-reference_shuffle.md\":\"Iq2v1oHJ\",\"tuning-reference_spark-architecture.md\":\"_qwnE_Sc\",\"tuning-reference_table-formats.md\":\"BSYEK-eN\",\"user-guide_alternative-log-retrieval.md\":\"CJpYvHGo\",\"user-guide_alternative-log-retrieval_managed-platforms.md\":\"DvDwHNtC\",\"user-guide_alternative-log-retrieval_ssh-bastion.md\":\"Gszzlplv\",\"user-guide_getting-started.md\":\"DYHHHkQR\",\"user-guide_getting-started_ci-and-automation.md\":\"OUQqw3mI\",\"user-guide_getting-started_reading-the-dashboard.md\":\"Obf3W3__\",\"user-guide_getting-started_regression-metric-keys.md\":\"B_gzr2Mw\",\"user-guide_getting-started_tuning-thresholds.md\":\"B_MyjyZm\",\"user-guide_mcp-tools.md\":\"C65SjWUL\",\"user-guide_mcp-tools_compare-runs.md\":\"Cc_gLoix\",\"user-guide_mcp-tools_diagnose-run.md\":\"BP1vnHA5\",\"user-guide_mcp-tools_evaluate-budgets.md\":\"BB0nDY3Z\",\"user-guide_mcp-tools_get-finding-documentation.md\":\"BtTTHNV6\",\"user-guide_mcp-tools_get-finding-evidence.md\":\"rKyu4wts\",\"user-guide_mcp-tools_get-reference-doc.md\":\"Yac4tH3u\",\"user-guide_mcp-tools_get-run-summary.md\":\"DpxZ4ZNp\",\"user-guide_mcp-tools_list-runs.md\":\"DpQPeCM3\",\"user-guide_run-comparison.md\":\"BUALkohi\",\"user-guide_run-comparison_how-stages-are-matched.md\":\"DSXGu2tA\",\"user-guide_understanding-findings.md\":\"C9YRruMT\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\",\"collapsed\":true,\"items\":[{\"text\":\"Reading the dashboard\",\"link\":\"/user-guide/getting-started/reading-the-dashboard\"},{\"text\":\"CI and automation\",\"link\":\"/user-guide/getting-started/ci-and-automation\"},{\"text\":\"Tuning detector thresholds\",\"link\":\"/user-guide/getting-started/tuning-thresholds\"},{\"text\":\"Regression metric keys\",\"link\":\"/user-guide/getting-started/regression-metric-keys\"}]},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\",\"collapsed\":true,\"items\":[{\"text\":\"How stages are matched\",\"link\":\"/user-guide/run-comparison/how-stages-are-matched\"}]},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\",\"collapsed\":true,\"items\":[{\"text\":\"diagnose_run\",\"link\":\"/user-guide/mcp-tools/diagnose-run\"},{\"text\":\"get_run_summary\",\"link\":\"/user-guide/mcp-tools/get-run-summary\"},{\"text\":\"evaluate_budgets\",\"link\":\"/user-guide/mcp-tools/evaluate-budgets\"},{\"text\":\"compare_runs\",\"link\":\"/user-guide/mcp-tools/compare-runs\"},{\"text\":\"get_finding_evidence\",\"link\":\"/user-guide/mcp-tools/get-finding-evidence\"},{\"text\":\"get_finding_documentation\",\"link\":\"/user-guide/mcp-tools/get-finding-documentation\"},{\"text\":\"get_reference_doc\",\"link\":\"/user-guide/mcp-tools/get-reference-doc\"},{\"text\":\"list_runs\",\"link\":\"/user-guide/mcp-tools/list-runs\"}]},{\"text\":\"Finding your event log\",\"link\":\"/user-guide/alternative-log-retrieval\",\"collapsed\":true,\"items\":[{\"text\":\"Managed platforms\",\"link\":\"/user-guide/alternative-log-retrieval/managed-platforms\"},{\"text\":\"Behind an SSH bastion\",\"link\":\"/user-guide/alternative-log-retrieval/ssh-bastion\"}]}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\",\"collapsed\":true,\"items\":[{\"text\":\"Evidence availability\",\"link\":\"/contributor-guide/architecture/worker-protocol/evidence-availability\"},{\"text\":\"Evidence report and finding identity\",\"link\":\"/contributor-guide/architecture/worker-protocol/evidence-report\"},{\"text\":\"CLI and MCP server\",\"link\":\"/contributor-guide/architecture/worker-protocol/cli-and-mcp\"},{\"text\":\"Core and event validation\",\"link\":\"/contributor-guide/architecture/worker-protocol/core-and-event-validation\"}]},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\",\"collapsed\":true,\"items\":[{\"text\":\"Run comparison internals\",\"link\":\"/contributor-guide/architecture/state-and-history/run-comparison-internals\"},{\"text\":\"History Server intake\",\"link\":\"/contributor-guide/architecture/state-and-history/history-server-intake\"}]},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\",\"collapsed\":true,\"items\":[{\"text\":\"Plan attribution\",\"link\":\"/contributor-guide/architecture/detector-contract/plan-attribution\"},{\"text\":\"Threshold tables\",\"link\":\"/contributor-guide/architecture/detector-contract/threshold-tables\"}]},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\",\"collapsed\":true,\"items\":[{\"text\":\"Stage union rollup\",\"link\":\"/contributor-guide/architecture/impact-estimation/stage-union-rollup\"},{\"text\":\"Caveats, tuning and coverage\",\"link\":\"/contributor-guide/architecture/impact-estimation/caveats-tuning-and-coverage\"}]},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\",\"collapsed\":true,\"items\":[{\"text\":\"Findings tab\",\"link\":\"/contributor-guide/architecture/widget-rendering/findings-tab\"},{\"text\":\"List sort mode\",\"link\":\"/contributor-guide/architecture/widget-rendering/list-sort-mode\"},{\"text\":\"Row/expand contract\",\"link\":\"/contributor-guide/architecture/widget-rendering/row-expand-contract\"},{\"text\":\"ReferenceSection\",\"link\":\"/contributor-guide/architecture/widget-rendering/reference-section\"},{\"text\":\"Investigation routing\",\"link\":\"/contributor-guide/architecture/widget-rendering/investigation-routing\"}]},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\",\"collapsed\":true,\"items\":[{\"text\":\"Plan Advisor\",\"link\":\"/contributor-guide/architecture/board-widgets/plan-advisor\"},{\"text\":\"Caching and memory widgets\",\"link\":\"/contributor-guide/architecture/board-widgets/caching-and-memory\"},{\"text\":\"Visual system and templating\",\"link\":\"/contributor-guide/architecture/board-widgets/visual-system-and-templating\"}]},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\",\"collapsed\":true,\"items\":[{\"text\":\"Plan graph\",\"link\":\"/contributor-guide/architecture/drill-down/plan-graph\"}]}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":{\"selector\":\"[data-sf-anchor-offset]\",\"padding\":0},\"cleanUrls\":false}");</script>
|
|
22
|
+
<div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><!--[--><!--[--><img class="VPImage dark logo" src="/docs/logo-dark.svg" alt data-v-8426fc1a><!--]--><!--[--><img class="VPImage light logo" src="/docs/logo-light.svg" alt data-v-8426fc1a><!--]--><!--]--><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/.." target="_self" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Open app</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><!----><!----><div class="VPContent is-home" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPHome" data-v-1428d186 data-v-8b561e3d><!--[--><!--]--><div class="VPHero VPHomeHero" data-v-8b561e3d data-v-4f9c455b><div class="container" data-v-4f9c455b><div class="main" data-v-4f9c455b><!--[--><!--]--><!--[--><h1 class="heading" data-v-4f9c455b><span class="name clip" data-v-4f9c455b>SparkForensics</span><span class="text" data-v-4f9c455b>Docs</span></h1><p class="tagline" data-v-4f9c455b>Find out why your Spark job is slow, ranked by which fix saves the most time.</p><!--]--><!--[--><!--]--><!----><!--[--><!--]--></div><!----></div></div><!--[--><!--]--><!--[--><!--]--><div class="VPFeatures VPHomeFeatures" data-v-8b561e3d data-v-a6181336><div class="container" data-v-a6181336><div class="items" data-v-a6181336><!--[--><div class="grid-3 item" data-v-a6181336><a class="VPLink link no-icon VPFeature" href="/docs/user-guide/getting-started.html" data-v-a6181336 data-v-a3976bdc><!--[--><article class="box" data-v-a3976bdc><!----><h2 class="title" data-v-a3976bdc>User Guide</h2><p class="details" data-v-a3976bdc>Load a run, read the findings, compare two runs, use the MCP tools.</p><div class="link-text" data-v-a3976bdc><p class="link-text-value" data-v-a3976bdc>Get started <span class="vpi-arrow-right link-text-icon" data-v-a3976bdc></span></p></div></article><!--]--></a></div><div class="grid-3 item" data-v-a6181336><a class="VPLink link no-icon VPFeature" href="/docs/contributor-guide/development-setup.html" data-v-a6181336 data-v-a3976bdc><!--[--><article class="box" data-v-a3976bdc><!----><h2 class="title" data-v-a3976bdc>Contributor Guide</h2><p class="details" data-v-a3976bdc>Set up the project, read the architecture, run the tests, propose a change.</p><div class="link-text" data-v-a3976bdc><p class="link-text-value" data-v-a3976bdc>Start contributing <span class="vpi-arrow-right link-text-icon" data-v-a3976bdc></span></p></div></article><!--]--></a></div><div class="grid-3 item" data-v-a6181336><a class="VPLink link no-icon VPFeature" href="/docs/tuning-reference/" data-v-a6181336 data-v-a3976bdc><!--[--><article class="box" data-v-a3976bdc><!----><h2 class="title" data-v-a3976bdc>Tuning Reference</h2><p class="details" data-v-a3976bdc>Spark architecture, memory, shuffle and tuning notes behind every finding.</p><div class="link-text" data-v-a3976bdc><p class="link-text-value" data-v-a3976bdc>Open the reference <span class="vpi-arrow-right link-text-icon" data-v-a3976bdc></span></p></div></article><!--]--></a></div><!--]--></div></div></div><!--[--><!--]--><div class="vp-doc container" style="" data-v-8b561e3d data-v-8e2d4988><!--[--><div style="position:relative;" data-v-8b561e3d><div></div></div><!--]--></div></div></div><!----><!--[--><!--]--></div></div>
|
|
23
|
+
<script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"CqKM_jEG\",\"contributor-guide_architecture_board-widgets_caching-and-memory.md\":\"BYDo3ClR\",\"contributor-guide_architecture_board-widgets_plan-advisor.md\":\"ajUAVdbc\",\"contributor-guide_architecture_board-widgets_visual-system-and-templating.md\":\"WqY74T0T\",\"contributor-guide_architecture_detector-contract.md\":\"CHHgywoP\",\"contributor-guide_architecture_detector-contract_plan-attribution.md\":\"DIz3_oz9\",\"contributor-guide_architecture_detector-contract_threshold-tables.md\":\"WH4tRHPL\",\"contributor-guide_architecture_drill-down.md\":\"yB-Hz2XT\",\"contributor-guide_architecture_drill-down_plan-graph.md\":\"BQZE04sw\",\"contributor-guide_architecture_impact-estimation.md\":\"K71YB-Rr\",\"contributor-guide_architecture_impact-estimation_caveats-tuning-and-coverage.md\":\"DLCAffwn\",\"contributor-guide_architecture_impact-estimation_stage-union-rollup.md\":\"JFrzDMwo\",\"contributor-guide_architecture_index.md\":\"tymxf1Fh\",\"contributor-guide_architecture_overview.md\":\"nxkpOhgf\",\"contributor-guide_architecture_state-and-history.md\":\"DyCBdjuM\",\"contributor-guide_architecture_state-and-history_history-server-intake.md\":\"C298hrHZ\",\"contributor-guide_architecture_state-and-history_run-comparison-internals.md\":\"U-9KMDFK\",\"contributor-guide_architecture_widget-rendering.md\":\"BrSz8Ezh\",\"contributor-guide_architecture_widget-rendering_findings-tab.md\":\"B7rjSF_h\",\"contributor-guide_architecture_widget-rendering_investigation-routing.md\":\"BGdiyElu\",\"contributor-guide_architecture_widget-rendering_list-sort-mode.md\":\"37hcZJ7q\",\"contributor-guide_architecture_widget-rendering_reference-section.md\":\"DWqC80bN\",\"contributor-guide_architecture_widget-rendering_row-expand-contract.md\":\"mK_VmAWk\",\"contributor-guide_architecture_worker-protocol.md\":\"Br6OE7RW\",\"contributor-guide_architecture_worker-protocol_cli-and-mcp.md\":\"IVyaXrGe\",\"contributor-guide_architecture_worker-protocol_core-and-event-validation.md\":\"DVSXn-bb\",\"contributor-guide_architecture_worker-protocol_evidence-availability.md\":\"D3S5cyOW\",\"contributor-guide_architecture_worker-protocol_evidence-report.md\":\"CtmCU57I\",\"contributor-guide_contributing.md\":\"DBW4drtZ\",\"contributor-guide_development-setup.md\":\"B21ZTDhn\",\"contributor-guide_testing.md\":\"DpkozNoG\",\"index.md\":\"lMm7SUMz\",\"tuning-reference_anti-patterns.md\":\"D08Uaod7\",\"tuning-reference_aqe.md\":\"C1Ae6A8u\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"6wjKuJNE\",\"tuning-reference_bottleneck-cold-start.md\":\"BVQbsuHP\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"DtaYnaiF\",\"tuning-reference_bottleneck-failures.md\":\"Z1Tdmdd3\",\"tuning-reference_bottleneck-gc.md\":\"BCnXLf1n\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"CcvQj2g9\",\"tuning-reference_bottleneck-memory-utilization.md\":\"B-C0-8h-\",\"tuning-reference_bottleneck-retry-waste.md\":\"DCnhWiqj\",\"tuning-reference_bottleneck-shuffle.md\":\"B5OMJCd3\",\"tuning-reference_bottleneck-skew.md\":\"Cvo6NQm_\",\"tuning-reference_bottleneck-slow-host.md\":\"ja5qr3wq\",\"tuning-reference_bottleneck-small-files.md\":\"Crjtb0i2\",\"tuning-reference_bottleneck-spill.md\":\"Bl8KtSRg\",\"tuning-reference_bottleneck-straggler.md\":\"B0VN8reT\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"DQCftGhu\",\"tuning-reference_bottleneck-utilization.md\":\"BXDl30f9\",\"tuning-reference_caching.md\":\"CcaiMn_7\",\"tuning-reference_cluster-config.md\":\"DbWG1yZq\",\"tuning-reference_config.md\":\"YUqSpwFB\",\"tuning-reference_data-formats.md\":\"C5DX5xL4\",\"tuning-reference_index.md\":\"ZVIpQ6Dk\",\"tuning-reference_intro.md\":\"BSSaDlqx\",\"tuning-reference_joins.md\":\"BVhNCDnE\",\"tuning-reference_memory-model.md\":\"CfjFdgbl\",\"tuning-reference_metrics.md\":\"C64VNEL4\",\"tuning-reference_partitioning.md\":\"CtLK0xsS\",\"tuning-reference_pyspark.md\":\"D-92XgF8\",\"tuning-reference_shuffle.md\":\"Iq2v1oHJ\",\"tuning-reference_spark-architecture.md\":\"_qwnE_Sc\",\"tuning-reference_table-formats.md\":\"BSYEK-eN\",\"user-guide_alternative-log-retrieval.md\":\"CJpYvHGo\",\"user-guide_alternative-log-retrieval_managed-platforms.md\":\"DvDwHNtC\",\"user-guide_alternative-log-retrieval_ssh-bastion.md\":\"Gszzlplv\",\"user-guide_getting-started.md\":\"DYHHHkQR\",\"user-guide_getting-started_ci-and-automation.md\":\"C0pcQoNc\",\"user-guide_getting-started_reading-the-dashboard.md\":\"CGaxPX_1\",\"user-guide_getting-started_regression-metric-keys.md\":\"B_gzr2Mw\",\"user-guide_getting-started_tuning-thresholds.md\":\"BGVE-wrc\",\"user-guide_mcp-tools.md\":\"49r3L2YE\",\"user-guide_mcp-tools_compare-runs.md\":\"BQ_kIptC\",\"user-guide_mcp-tools_diagnose-run.md\":\"azCZXkH8\",\"user-guide_mcp-tools_evaluate-budgets.md\":\"dpQpRQ7z\",\"user-guide_mcp-tools_get-finding-documentation.md\":\"BtTTHNV6\",\"user-guide_mcp-tools_get-finding-evidence.md\":\"CH6FkNLX\",\"user-guide_mcp-tools_get-reference-doc.md\":\"Yac4tH3u\",\"user-guide_mcp-tools_get-run-summary.md\":\"EhaIsele\",\"user-guide_mcp-tools_list-runs.md\":\"DpQPeCM3\",\"user-guide_run-comparison.md\":\"B_LOkdBf\",\"user-guide_run-comparison_how-stages-are-matched.md\":\"DSXGu2tA\",\"user-guide_understanding-findings.md\":\"C4riKCKl\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":{\"light\":\"/logo-light.svg\",\"dark\":\"/logo-dark.svg\"},\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"},{\"text\":\"Open app\",\"link\":\"/..\",\"target\":\"_self\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\",\"collapsed\":true,\"items\":[{\"text\":\"Reading the dashboard\",\"link\":\"/user-guide/getting-started/reading-the-dashboard\"},{\"text\":\"CI and automation\",\"link\":\"/user-guide/getting-started/ci-and-automation\"},{\"text\":\"Tuning detector thresholds\",\"link\":\"/user-guide/getting-started/tuning-thresholds\"},{\"text\":\"Regression metric keys\",\"link\":\"/user-guide/getting-started/regression-metric-keys\"}]},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\",\"collapsed\":true,\"items\":[{\"text\":\"How stages are matched\",\"link\":\"/user-guide/run-comparison/how-stages-are-matched\"}]},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\",\"collapsed\":true,\"items\":[{\"text\":\"diagnose_run\",\"link\":\"/user-guide/mcp-tools/diagnose-run\"},{\"text\":\"get_run_summary\",\"link\":\"/user-guide/mcp-tools/get-run-summary\"},{\"text\":\"evaluate_budgets\",\"link\":\"/user-guide/mcp-tools/evaluate-budgets\"},{\"text\":\"compare_runs\",\"link\":\"/user-guide/mcp-tools/compare-runs\"},{\"text\":\"get_finding_evidence\",\"link\":\"/user-guide/mcp-tools/get-finding-evidence\"},{\"text\":\"get_finding_documentation\",\"link\":\"/user-guide/mcp-tools/get-finding-documentation\"},{\"text\":\"get_reference_doc\",\"link\":\"/user-guide/mcp-tools/get-reference-doc\"},{\"text\":\"list_runs\",\"link\":\"/user-guide/mcp-tools/list-runs\"}]},{\"text\":\"Finding your event log\",\"link\":\"/user-guide/alternative-log-retrieval\",\"collapsed\":true,\"items\":[{\"text\":\"Managed platforms\",\"link\":\"/user-guide/alternative-log-retrieval/managed-platforms\"},{\"text\":\"Behind an SSH bastion\",\"link\":\"/user-guide/alternative-log-retrieval/ssh-bastion\"}]}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\",\"collapsed\":true,\"items\":[{\"text\":\"Evidence availability\",\"link\":\"/contributor-guide/architecture/worker-protocol/evidence-availability\"},{\"text\":\"Evidence report and finding identity\",\"link\":\"/contributor-guide/architecture/worker-protocol/evidence-report\"},{\"text\":\"CLI and MCP server\",\"link\":\"/contributor-guide/architecture/worker-protocol/cli-and-mcp\"},{\"text\":\"Core and event validation\",\"link\":\"/contributor-guide/architecture/worker-protocol/core-and-event-validation\"}]},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\",\"collapsed\":true,\"items\":[{\"text\":\"Run comparison internals\",\"link\":\"/contributor-guide/architecture/state-and-history/run-comparison-internals\"},{\"text\":\"History Server intake\",\"link\":\"/contributor-guide/architecture/state-and-history/history-server-intake\"}]},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\",\"collapsed\":true,\"items\":[{\"text\":\"Plan attribution\",\"link\":\"/contributor-guide/architecture/detector-contract/plan-attribution\"},{\"text\":\"Threshold tables\",\"link\":\"/contributor-guide/architecture/detector-contract/threshold-tables\"}]},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\",\"collapsed\":true,\"items\":[{\"text\":\"Stage union rollup\",\"link\":\"/contributor-guide/architecture/impact-estimation/stage-union-rollup\"},{\"text\":\"Caveats, tuning and coverage\",\"link\":\"/contributor-guide/architecture/impact-estimation/caveats-tuning-and-coverage\"}]},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\",\"collapsed\":true,\"items\":[{\"text\":\"Findings tab\",\"link\":\"/contributor-guide/architecture/widget-rendering/findings-tab\"},{\"text\":\"List sort mode\",\"link\":\"/contributor-guide/architecture/widget-rendering/list-sort-mode\"},{\"text\":\"Row/expand contract\",\"link\":\"/contributor-guide/architecture/widget-rendering/row-expand-contract\"},{\"text\":\"ReferenceSection\",\"link\":\"/contributor-guide/architecture/widget-rendering/reference-section\"},{\"text\":\"Investigation routing\",\"link\":\"/contributor-guide/architecture/widget-rendering/investigation-routing\"}]},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\",\"collapsed\":true,\"items\":[{\"text\":\"Plan Advisor\",\"link\":\"/contributor-guide/architecture/board-widgets/plan-advisor\"},{\"text\":\"Caching and memory widgets\",\"link\":\"/contributor-guide/architecture/board-widgets/caching-and-memory\"},{\"text\":\"Visual system and templating\",\"link\":\"/contributor-guide/architecture/board-widgets/visual-system-and-templating\"}]},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\",\"collapsed\":true,\"items\":[{\"text\":\"Plan graph\",\"link\":\"/contributor-guide/architecture/drill-down/plan-graph\"}]}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":{\"selector\":\"[data-sf-anchor-offset]\",\"padding\":0},\"cleanUrls\":false}");</script>
|
|
28
24
|
|
|
29
25
|
</body>
|
|
30
26
|
</html>
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="24" height="24" fill="none" stroke="#e6e8ee" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M22 12h-2.48a2 2 0 0 0-1.93 1.46l-2.35 8.36a.25.25 0 0 1-.48 0L9.24 2.18a.25.25 0 0 0-.48 0l-2.35 8.36A2 2 0 0 1 4.49 12H2"/></svg>
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="24" height="24" fill="none" stroke="#16181d" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M22 12h-2.48a2 2 0 0 0-1.93 1.46l-2.35 8.36a.25.25 0 0 1-.48 0L9.24 2.18a.25.25 0 0 0-.48 0l-2.35 8.36A2 2 0 0 1 4.49 12H2"/></svg>
|
|
@@ -6,25 +6,21 @@
|
|
|
6
6
|
<title>Anti-Patterns | Spark Tuning Reference</title>
|
|
7
7
|
<meta name="description" content="Checklist of a dozen recurring Spark performance anti-patterns, each with what it is, how to detect it, why it hurts, and how to fix it.">
|
|
8
8
|
<meta name="generator" content="VitePress v1.6.4">
|
|
9
|
-
<link rel="preload stylesheet" href="/docs/assets/style.
|
|
9
|
+
<link rel="preload stylesheet" href="/docs/assets/style.Cfxqlj0y.css" as="style">
|
|
10
10
|
<link rel="preload stylesheet" href="/docs/vp-icons.css" as="style">
|
|
11
11
|
|
|
12
|
-
<script type="module" src="/docs/assets/app.
|
|
13
|
-
<link rel="
|
|
14
|
-
<link rel="modulepreload" href="/docs/assets/chunks/theme.DsTNE2pE.js">
|
|
12
|
+
<script type="module" src="/docs/assets/app.1gvzd5Zb.js"></script>
|
|
13
|
+
<link rel="modulepreload" href="/docs/assets/chunks/theme.C5lLqUv7.js">
|
|
15
14
|
<link rel="modulepreload" href="/docs/assets/chunks/framework.DSg0KOwT.js">
|
|
16
15
|
<link rel="modulepreload" href="/docs/assets/tuning-reference_anti-patterns.md.D08Uaod7.lean.js">
|
|
17
16
|
<script>(()=>{const c=`${`[${["data","shuffle","product","bar"].join("-")}]`}, .VPNav, .VPLocalNav`,a=24,d=document.documentElement,l=t=>{const e=getComputedStyle(t);return e.display==="none"||t.getClientRects().length===0?0:e.position==="fixed"?t.getBoundingClientRect().bottom:e.position==="sticky"?(parseFloat(e.top)||0)+t.offsetHeight:0},u=()=>{try{return location.hash?document.getElementById(decodeURIComponent(location.hash.slice(1))):null}catch{return null}};let r=null;const s=()=>{const t=Array.from(document.querySelectorAll(c),l),e=Math.ceil(Math.max(0,...t))+a;if(e===r)return;const n=r;r=e,d.style.setProperty("--sf-anchor-offset",`${r}px`);const o=n===null?null:u();if(!o)return;const f=(parseFloat(getComputedStyle(o).paddingTop)||0)+2;Math.abs(o.getBoundingClientRect().top-n)<=f&&o.scrollIntoView()},i=()=>{const t=document.createElement("div");t.setAttribute("data-sf-anchor-offset",""),t.setAttribute("aria-hidden","true"),document.body.appendChild(t);const e=window.ResizeObserver?new ResizeObserver(s):null,n=()=>{e&&document.querySelectorAll(c).forEach(o=>e.observe(o)),s()};n(),new MutationObserver(n).observe(document.body,{childList:!0,subtree:!0}),window.addEventListener("resize",s),window.addEventListener("load",s)};document.readyState==="loading"?document.addEventListener("DOMContentLoaded",i,{once:!0}):i()})();</script>
|
|
18
17
|
<link rel="icon" type="image/svg+xml" href="/docs/favicon.svg">
|
|
19
|
-
<link rel="preconnect" href="https://fonts.googleapis.com">
|
|
20
|
-
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin="">
|
|
21
|
-
<link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Recursive:wght,CASL@400..700,0..1&family=JetBrains+Mono:wght@400;500;600&display=swap">
|
|
22
18
|
<script id="check-dark-mode">(()=>{const e=localStorage.getItem("vitepress-theme-appearance")||"auto",a=window.matchMedia("(prefers-color-scheme: dark)").matches;(!e||e==="auto"?a:e==="dark")&&document.documentElement.classList.add("dark")})();</script>
|
|
23
19
|
<script id="check-mac-os">document.documentElement.classList.toggle("mac",/Mac|iPhone|iPod|iPad/i.test(navigator.platform));</script>
|
|
24
20
|
</head>
|
|
25
21
|
<body>
|
|
26
|
-
<div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Getting Started</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/intro.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Introduction</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Optimization Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/spark-architecture.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Execution Model</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/memory-model.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Management</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/partitioning.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Partitioning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/joins.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Join Optimization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/data-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Data Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/table-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Table Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/caching.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Caching & Persistence</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/pyspark.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>PySpark Specifics</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/aqe.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Adaptive Query Execution</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/cluster-config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cluster Tuning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/anti-patterns.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Anti-Patterns</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Detector Catalog</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-skew.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Skew</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle I/O</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-spill.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory / Disk Spill</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-gc.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>GC Pressure</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-cold-start.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cold Start</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Executor Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-slow-host.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Slow Host</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-failures.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Failures</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-straggler.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Stragglers</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-retry-waste.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Retry Waste</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-tiny-tasks.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Tiny Tasks</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-job-failure-rate.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Job Failure Rate</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-memory-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Duplicate Plan Subtree</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-small-files.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Small Files</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-broadcast-sizing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Broadcast Sizing</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Reference</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/metrics.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Metrics Glossary</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Config Quick-Reference</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_tuning-reference_anti-patterns" data-v-39a288b8><div><h1 id="anti-patterns" tabindex="-1">Anti-Patterns <a class="header-anchor" href="#anti-patterns" aria-label="Permalink to "Anti-Patterns {#anti-patterns}""></a></h1><p>Practitioner checklists on Spark performance converge on a recurring handful of mistakes rather than exotic ones. Each entry below covers what the mistake is, how to detect it, why it hurts, and how to fix it.</p><ol><li><p><strong>Staying on the RDD API instead of DataFrame/Dataset</strong></p><ul><li><strong>What:</strong> Building jobs on the RDD API (or RDD-based MLlib) instead of the DataFrame/Dataset API.</li><li><strong>Detect:</strong> Code that drives its logic through <code>.map</code>/<code>.filter</code> lambdas over RDDs, or that calls into the RDD-based MLlib API.</li><li><strong>Why:</strong> RDDs are lambda-driven, so Spark cannot see inside the closures. It can't apply predicate pushdown, filter reordering, Adaptive Query Execution, or cost-based join reordering. The RDD-based MLlib API is in maintenance mode for the same reason.<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup></li><li><strong>Fix:</strong> Use the DataFrame/Dataset API so Catalyst can optimize the query plan.</li></ul></li><li><p><strong>Reading CSV/JSON without an explicit schema</strong></p><ul><li><strong>What:</strong> Reading raw CSV or JSON files without supplying an explicit schema.</li><li><strong>Detect:</strong> A read step that spends noticeable time before any real work starts, because Spark is inferring types from the data itself.</li><li><strong>Why:</strong> Schema inference forces a full scan of the data just to determine types, and CSV/JSON don't support the column pruning, predicate pushdown, or stats-based file skipping that a columnar format does.<sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup></li><li><strong>Fix:</strong> Supply an explicit schema, or prefer Parquet for analytical workloads, where column pruning, predicate pushdown, and stats-based file skipping work out of the box.<sup class="footnote-ref"><a href="#fn1" id="fnref1:2">[1:2]</a></sup></li></ul></li><li><p><strong>Compressing input files with a non-splittable codec (GZIP)</strong></p><ul><li><strong>What:</strong> Storing input data as large GZIP files.</li><li><strong>Detect:</strong> One task/executor taking far longer than the rest on a stage reading GZIP-compressed input, while other executors sit idle.</li><li><strong>Why:</strong> A GZIP file can't be split across executors, so a single node has to decompress the whole file alone.<sup class="footnote-ref"><a href="#fn1" id="fnref1:3">[1:3]</a></sup></li><li><strong>Fix:</strong> Use a splittable codec instead: Snappy, LZ4, or ZSTD.<sup class="footnote-ref"><a href="#fn1" id="fnref1:4">[1:4]</a></sup></li></ul></li><li><p><strong>Storing data as JSON instead of a binary columnar/row format</strong></p><ul><li><strong>What:</strong> Persisting datasets as JSON rather than a binary columnar or row format.</li><li><strong>Detect:</strong> Read-heavy jobs where a large, recurring share of stage time goes to parsing the same JSON structures on every read.</li><li><strong>Why:</strong> JSON has to be re-parsed from text on every single read.<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup></li><li><strong>Fix:</strong> Store data as Avro, Parquet, Thrift, or Protobuf structs in a sequence file to avoid the repeated parsing cost.<sup class="footnote-ref"><a href="#fn2" id="fnref2:1">[2:1]</a></sup></li></ul></li><li><p><strong>Not registering custom classes with Kryo</strong></p><ul><li><strong>What:</strong> Using Kryo serialization without registering the application's custom classes.</li><li><strong>Detect:</strong> Larger-than-expected serialized record sizes, and poor efficiency when using a serialized cache storage level such as <code>MEMORY_SER</code>.</li><li><strong>Why:</strong> Unregistered classes increase serialized-record size and hurt serialized cache storage levels.<sup class="footnote-ref"><a href="#fn2" id="fnref2:2">[2:2]</a></sup></li><li><strong>Fix:</strong> Register the application's custom classes with Kryo.</li></ul></li><li><p><strong>Calling <code>collect()</code> on a large DataFrame</strong></p><ul><li><p><strong>What:</strong> Materializing an entire DataFrame into driver memory with <code>collect()</code>.</p></li><li><p><strong>Detect:</strong> A driver <code>OutOfMemoryError</code>, or a job that aborts once <code>spark.driver.maxResultSize</code> (default 1 GB) is exceeded. Spark also logs a warning once a single task's serialized result passes roughly 1 MB.<sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup><sup class="footnote-ref"><a href="#fn1" id="fnref1:5">[1:5]</a></sup></p></li><li><p><strong>Why:</strong> <code>collect()</code> gathers every <code>Row</code> from every partition into a single in-memory <code>Array</code> that lives entirely on the driver, not the cluster. The driver JVM has to hold the whole result set in its own heap.<sup class="footnote-ref"><a href="#fn3" id="fnref3:1">[3:1]</a></sup><sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup><sup class="footnote-ref"><a href="#fn5" id="fnref5">[5]</a></sup><sup class="footnote-ref"><a href="#fn6" id="fnref6">[6]</a></sup></p></li><li><p><strong>Fix:</strong> Use aggregations or <code>take(n)</code> instead of collecting the full result set.</p><blockquote><p><strong>PySpark:</strong> <code>toPandas()</code> performs the same driver-side collection as <code>collect()</code> for Python users, so avoid it on large DataFrames too.<sup class="footnote-ref"><a href="#fn6" id="fnref6:1">[6:1]</a></sup></p></blockquote></li></ul></li><li><p><strong>Assuming <code>cache()</code> + <code>count()</code> guarantees the DataFrame stays fully cached</strong></p><ul><li><strong>What:</strong> Treating <code>df.cache(); df.count()</code> as a guarantee that the DataFrame remains fully persisted for later actions.</li><li><strong>Detect:</strong> Later actions on a supposedly-cached DataFrame recompute from source instead of hitting the cache. There's no built-in visibility into how much of a DataFrame is still actually cached, so this typically only surfaces as unexpectedly slow recomputation.<sup class="footnote-ref"><a href="#fn7" id="fnref7">[7]</a></sup></li><li><strong>Why:</strong> <code>count()</code> does force every partition to be computed and attempted for caching, unlike <code>take</code>/<code>limit</code>, which only touch the partitions they need.<sup class="footnote-ref"><a href="#fn5" id="fnref5:1">[5:1]</a></sup><sup class="footnote-ref"><a href="#fn7" id="fnref7:1">[7:1]</a></sup> But partitions can't be fractionally cached: if there isn't room for all of them, the ones that don't fit are silently dropped and recomputed on next access.<sup class="footnote-ref"><a href="#fn5" id="fnref5:2">[5:2]</a></sup> Cached blocks also compete with execution for the same memory pool and can be evicted under pressure without warning, and caching is tied to the <em>analyzed</em> (pre-optimization) logical plan, so a semantically identical query with a different analyzed plan bypasses the cache entirely and recomputes from source.<sup class="footnote-ref"><a href="#fn7" id="fnref7:2">[7:2]</a></sup> Losing an executor drops any cached block that wasn't stored with a replicated storage level like <code>MEMORY_AND_DISK_2</code>.<sup class="footnote-ref"><a href="#fn7" id="fnref7:3">[7:3]</a></sup></li><li><strong>Fix:</strong> Keep using <code>.cache()</code> followed by <code>.count()</code> to force materialization, but don't assume that guarantees persistence (there's no built-in way to confirm how much of the DataFrame is actually cached<sup class="footnote-ref"><a href="#fn7" id="fnref7:4">[7:4]</a></sup>), and use a replicated storage level if losing a cached partition to executor failure would be costly.</li></ul></li><li><p><strong>Row-at-a-time Python UDFs</strong></p><ul><li><strong>What:</strong> Writing Python UDFs that operate one row at a time instead of vectorized Pandas UDFs.</li><li><strong>Detect:</strong> A UDF-heavy stage where most of the time goes to serialization rather than actual computation.<sup class="footnote-ref"><a href="#fn8" id="fnref8">[8]</a></sup></li><li><strong>Why:</strong> The overhead is serialization plus per-row JVM↔Python data movement, not raw Python interpreter speed. Row-at-a-time UDFs "suffer from high serialization and invocation overhead."<sup class="footnote-ref"><a href="#fn9" id="fnref9">[9]</a></sup> RDD-era Python UDFs pay a <em>double</em> serialization cost: Java/Scala objects are serialized, then re-serialized to Python via <code>cloudpickle</code> and back on every call.<sup class="footnote-ref"><a href="#fn8" id="fnref8:1">[8:1]</a></sup> A Databricks benchmark on a 10M-row DataFrame showed vectorized Pandas UDFs beating row-at-a-time UDFs "across the board, ranging from 3x to over 100x."<sup class="footnote-ref"><a href="#fn9" id="fnref9:1">[9:1]</a></sup></li><li><strong>Fix:</strong> Use vectorized Pandas UDFs instead of row-at-a-time UDFs: once data is transferred via Arrow, there's no need to serialize/pickle it, since it's already in a format consumable by the Python process.<sup class="footnote-ref"><a href="#fn5" id="fnref5:3">[5:3]</a></sup></li></ul></li><li><p><strong>Leaving <code>spark.sql.shuffle.partitions</code> at its default of 200</strong></p><ul><li><strong>What:</strong> Never tuning <code>spark.sql.shuffle.partitions</code> away from its default of 200 for wide transformations (<code>join</code>, <code>groupBy</code>, aggregations).</li><li><strong>Detect:</strong> In the Spark UI Stages tab, too few partitions for the data size shows up as "Spill (Memory)" or "Spill (Disk)" entries against a stage; too many partitions for the data size shows up as a large task/stage count where each task's duration is dominated by scheduling and bookkeeping, often paired with a large number of tiny output files.<sup class="footnote-ref"><a href="#fn10" id="fnref10">[10]</a></sup></li><li><strong>Why:</strong> 200 is a fixed default regardless of whether you're joining 5MB or 5TB.<sup class="footnote-ref"><a href="#fn10" id="fnref10:1">[10:1]</a></sup><sup class="footnote-ref"><a href="#fn11" id="fnref11">[11]</a></sup> Too few partitions means each one holds more data than fits comfortably in executor memory, which increases load per executor and leads to spills once partition size exceeds available memory.<sup class="footnote-ref"><a href="#fn10" id="fnref10:2">[10:2]</a></sup> Too many partitions on a small dataset can shrink tasks to around ten rows each, so "most of your CPUs will just be sitting there doing nothing."<sup class="footnote-ref"><a href="#fn10" id="fnref10:3">[10:3]</a></sup></li><li><strong>Fix:</strong> Increase <code>spark.sql.shuffle.partitions</code> if tasks are processing multiple GB each and spilling; decrease it if tasks finish in a couple of seconds and write tiny files. A rough starting target is 100–200 MB of data per task, tuned per dataset.<sup class="footnote-ref"><a href="#fn10" id="fnref10:4">[10:4]</a></sup></li></ul></li><li><p><strong>Broadcasting a table that's too large</strong></p><ul><li><strong>What:</strong> Forcing or allowing a broadcast join on a table that's too large to broadcast safely.</li><li><strong>Detect:</strong> A driver <code>OutOfMemoryError</code> during a broadcast join.</li><li><strong>Why:</strong> The small side of the join has to be collected onto the driver before it can be broadcast to every executor. That collection step is the same driver-memory operation as <code>.collect()</code>, and it's what fails, not executor-side replication or a serialization timeout: "if you try to broadcast something too large, you can crash your driver node (because that collect is expensive)."<sup class="footnote-ref"><a href="#fn4" id="fnref4:1">[4:1]</a></sup></li><li><strong>Fix:</strong> Use <code>spark.sql.autoBroadcastJoinThreshold</code> to control the maximum size Spark will broadcast, or increase driver memory.<sup class="footnote-ref"><a href="#fn4" id="fnref4:2">[4:2]</a></sup></li></ul></li><li><p><strong>Reaching for <code>coalesce(1)</code> before every single-file write</strong></p><ul><li><strong>What:</strong> Always using <code>coalesce(1)</code> instead of <code>repartition(1)</code> when a single output file is needed.</li><li><strong>Detect:</strong> The entire upstream stage (including filtering/transformation work that would otherwise run in parallel) collapses onto a single task/executor, leaving the rest of the cluster idle.</li><li><strong>Why:</strong> <code>coalesce</code> is a narrow transformation, so it "causes the upstream partitions in the entire stage to execute with the level of parallelism assigned by coalesce," fusing all preceding work down to one task.<sup class="footnote-ref"><a href="#fn6" id="fnref6:2">[6:2]</a></sup> <code>repartition</code> triggers a full shuffle instead, which inserts an explicit shuffle boundary so the upstream stage keeps its original parallelism.<sup class="footnote-ref"><a href="#fn4" id="fnref4:3">[4:3]</a></sup><sup class="footnote-ref"><a href="#fn6" id="fnref6:3">[6:3]</a></sup></li><li><strong>Fix:</strong> Use <code>repartition(1)</code> when there is meaningful upstream computation you don't want collapsed onto a single executor; reserve <code>coalesce(1)</code> for when the upstream stage is already cheap and paying for a shuffle would be wasted cost.<sup class="footnote-ref"><a href="#fn6" id="fnref6:4">[6:4]</a></sup></li></ul></li><li><p><strong>Not verifying predicate pushdown is actually happening</strong></p><ul><li><strong>What:</strong> Assuming filters and column projections are pushed down to the data source without checking the physical plan.</li><li><strong>Detect:</strong> Run <code>.explain()</code> and look at the <code>Scan</code> node for a <code>PushedFilters</code> marker. For example, <code>*Scan JDBCRel... PushedFilters: [*In(DEST_COUNTRY_NAME, [Anguilla, Sweden])]</code>. Column pruning is verified the same way: a <code>.select()</code> on one column should show a narrowed <code>ReadSchema</code> at the scan rather than a full-table scan.<sup class="footnote-ref"><a href="#fn4" id="fnref4:4">[4:4]</a></sup></li><li><strong>Why:</strong> Spark pushes down simple filters (column equality, <code>IN</code>, <code>IS NULL</code>) to JDBC sources automatically, but anything with a computed column or cast won't push down and gets evaluated in Spark after the full read.<sup class="footnote-ref"><a href="#fn1" id="fnref1:6">[1:6]</a></sup> Some sources also only partially handle a filter and leave Spark to re-evaluate it as a safety mechanism; that's still a "good" pushdown since the amount of data read is reduced.<sup class="footnote-ref"><a href="#fn6" id="fnref6:5">[6:5]</a></sup></li><li><strong>Fix:</strong> Check <code>explain</code> output (paired with <code>printSchema</code>)<sup class="footnote-ref"><a href="#fn6" id="fnref6:6">[6:6]</a></sup> for <code>PushedFilters</code> after writing a filter, and rewrite filters that use casts or computed columns as simple predicates where possible so they can push down.<sup class="footnote-ref"><a href="#fn4" id="fnref4:5">[4:5]</a></sup><sup class="footnote-ref"><a href="#fn1" id="fnref1:7">[1:7]</a></sup></li></ul></li></ol><h2 id="sources" tabindex="-1">Sources <a class="header-anchor" href="#sources" aria-label="Permalink to "Sources""></a></h2><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><a href="https://luminousmen.com/post/the-apache-spark-optimization-checklist" target="_blank" rel="noreferrer">The Apache Spark Optimization Checklist</a> <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a> <a href="#fnref1:2" class="footnote-backref">↩︎</a> <a href="#fnref1:3" class="footnote-backref">↩︎</a> <a href="#fnref1:4" class="footnote-backref">↩︎</a> <a href="#fnref1:5" class="footnote-backref">↩︎</a> <a href="#fnref1:6" class="footnote-backref">↩︎</a> <a href="#fnref1:7" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><a href="https://blog.cloudera.com/how-to-tune-your-apache-spark-jobs-part-2/" target="_blank" rel="noreferrer">How to Tune Your Apache Spark Jobs (Part 2)</a> <a href="#fnref2" class="footnote-backref">↩︎</a> <a href="#fnref2:1" class="footnote-backref">↩︎</a> <a href="#fnref2:2" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><em>Advanced Analytics with PySpark</em>, Tandon, Ryza, Laserson et al., ch. 2 <a href="#fnref3" class="footnote-backref">↩︎</a> <a href="#fnref3:1" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><em>Spark: The Definitive Guide</em>, Chambers & Zaharia, chs. 5, 8, 9, 18, 19 <a href="#fnref4" class="footnote-backref">↩︎</a> <a href="#fnref4:1" class="footnote-backref">↩︎</a> <a href="#fnref4:2" class="footnote-backref">↩︎</a> <a href="#fnref4:3" class="footnote-backref">↩︎</a> <a href="#fnref4:4" class="footnote-backref">↩︎</a> <a href="#fnref4:5" class="footnote-backref">↩︎</a></p></li><li id="fn5" class="footnote-item"><p><em>Learning Spark, 2nd Edition</em>, Damji, Wenig, Das & Lee, chs. 3, 5, 7 <a href="#fnref5" class="footnote-backref">↩︎</a> <a href="#fnref5:1" class="footnote-backref">↩︎</a> <a href="#fnref5:2" class="footnote-backref">↩︎</a> <a href="#fnref5:3" class="footnote-backref">↩︎</a></p></li><li id="fn6" class="footnote-item"><p><em>High Performance Spark, 2nd Edition</em>, Karau, Polak & Warren, chs. 5, 7 <a href="#fnref6" class="footnote-backref">↩︎</a> <a href="#fnref6:1" class="footnote-backref">↩︎</a> <a href="#fnref6:2" class="footnote-backref">↩︎</a> <a href="#fnref6:3" class="footnote-backref">↩︎</a> <a href="#fnref6:4" class="footnote-backref">↩︎</a> <a href="#fnref6:5" class="footnote-backref">↩︎</a> <a href="#fnref6:6" class="footnote-backref">↩︎</a></p></li><li id="fn7" class="footnote-item"><p><a href="https://luminousmen.com/post/explaining-the-mechanics-of-spark-caching" target="_blank" rel="noreferrer">Explaining the Mechanics of Spark Caching</a> <a href="#fnref7" class="footnote-backref">↩︎</a> <a href="#fnref7:1" class="footnote-backref">↩︎</a> <a href="#fnref7:2" class="footnote-backref">↩︎</a> <a href="#fnref7:3" class="footnote-backref">↩︎</a> <a href="#fnref7:4" class="footnote-backref">↩︎</a></p></li><li id="fn8" class="footnote-item"><p><a href="https://luminousmen.com/post/spark-tips-dataframe-api" target="_blank" rel="noreferrer">Spark Tips: DataFrame API</a> <a href="#fnref8" class="footnote-backref">↩︎</a> <a href="#fnref8:1" class="footnote-backref">↩︎</a></p></li><li id="fn9" class="footnote-item"><p><a href="https://www.databricks.com/blog/2017/10/30/introducing-vectorized-udfs-for-pyspark.html" target="_blank" rel="noreferrer">Introducing Vectorized UDFs for PySpark</a> <a href="#fnref9" class="footnote-backref">↩︎</a> <a href="#fnref9:1" class="footnote-backref">↩︎</a></p></li><li id="fn10" class="footnote-item"><p><a href="https://luminousmen.com/post/spark-partitions" target="_blank" rel="noreferrer">Spark Partitions</a> <a href="#fnref10" class="footnote-backref">↩︎</a> <a href="#fnref10:1" class="footnote-backref">↩︎</a> <a href="#fnref10:2" class="footnote-backref">↩︎</a> <a href="#fnref10:3" class="footnote-backref">↩︎</a> <a href="#fnref10:4" class="footnote-backref">↩︎</a></p></li><li id="fn11" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration — Spark</a> <a href="#fnref11" class="footnote-backref">↩︎</a></p></li></ol></section></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/tuning-reference/cluster-config.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>Cluster Tuning</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="/docs/tuning-reference/bottleneck-skew.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>Task Skew</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
|
|
27
|
-
<script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"YQcj5EI4\",\"contributor-guide_architecture_board-widgets_caching-and-memory.md\":\"8bH1HnZp\",\"contributor-guide_architecture_board-widgets_plan-advisor.md\":\"RlzvUmNm\",\"contributor-guide_architecture_board-widgets_visual-system-and-templating.md\":\"DYgUpIJB\",\"contributor-guide_architecture_detector-contract.md\":\"wj6Et6UD\",\"contributor-guide_architecture_detector-contract_plan-attribution.md\":\"yNex4bys\",\"contributor-guide_architecture_detector-contract_threshold-tables.md\":\"CKjWG56Y\",\"contributor-guide_architecture_drill-down.md\":\"CqZPrdo9\",\"contributor-guide_architecture_drill-down_plan-graph.md\":\"DwOozIEs\",\"contributor-guide_architecture_impact-estimation.md\":\"CZv_ZGOp\",\"contributor-guide_architecture_impact-estimation_caveats-tuning-and-coverage.md\":\"BUucRc0A\",\"contributor-guide_architecture_impact-estimation_stage-union-rollup.md\":\"DdCGtAGu\",\"contributor-guide_architecture_index.md\":\"tymxf1Fh\",\"contributor-guide_architecture_overview.md\":\"C1sSvm9i\",\"contributor-guide_architecture_state-and-history.md\":\"DyCBdjuM\",\"contributor-guide_architecture_state-and-history_history-server-intake.md\":\"C298hrHZ\",\"contributor-guide_architecture_state-and-history_run-comparison-internals.md\":\"CZA0JLtV\",\"contributor-guide_architecture_widget-rendering.md\":\"BOZ1azXG\",\"contributor-guide_architecture_widget-rendering_findings-tab.md\":\"BQXycK6G\",\"contributor-guide_architecture_widget-rendering_investigation-routing.md\":\"BGdiyElu\",\"contributor-guide_architecture_widget-rendering_list-sort-mode.md\":\"BuHKgH0C\",\"contributor-guide_architecture_widget-rendering_reference-section.md\":\"DWqC80bN\",\"contributor-guide_architecture_widget-rendering_row-expand-contract.md\":\"ZT89u9f1\",\"contributor-guide_architecture_worker-protocol.md\":\"BAnhi2-d\",\"contributor-guide_architecture_worker-protocol_cli-and-mcp.md\":\"DC1YqXBp\",\"contributor-guide_architecture_worker-protocol_core-and-event-validation.md\":\"DSVFPpWb\",\"contributor-guide_architecture_worker-protocol_evidence-availability.md\":\"CjimIDBY\",\"contributor-guide_architecture_worker-protocol_evidence-report.md\":\"DVGZOWvW\",\"contributor-guide_contributing.md\":\"CQMvGwFt\",\"contributor-guide_development-setup.md\":\"D9bu4j_c\",\"contributor-guide_testing.md\":\"By6XGR9a\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"D08Uaod7\",\"tuning-reference_aqe.md\":\"C1Ae6A8u\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"6wjKuJNE\",\"tuning-reference_bottleneck-cold-start.md\":\"BVQbsuHP\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"DtaYnaiF\",\"tuning-reference_bottleneck-failures.md\":\"Z1Tdmdd3\",\"tuning-reference_bottleneck-gc.md\":\"BCnXLf1n\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"CcvQj2g9\",\"tuning-reference_bottleneck-memory-utilization.md\":\"B-C0-8h-\",\"tuning-reference_bottleneck-retry-waste.md\":\"DCnhWiqj\",\"tuning-reference_bottleneck-shuffle.md\":\"B5OMJCd3\",\"tuning-reference_bottleneck-skew.md\":\"Cvo6NQm_\",\"tuning-reference_bottleneck-slow-host.md\":\"ja5qr3wq\",\"tuning-reference_bottleneck-small-files.md\":\"Crjtb0i2\",\"tuning-reference_bottleneck-spill.md\":\"Bl8KtSRg\",\"tuning-reference_bottleneck-straggler.md\":\"B0VN8reT\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"DQCftGhu\",\"tuning-reference_bottleneck-utilization.md\":\"BXDl30f9\",\"tuning-reference_caching.md\":\"CcaiMn_7\",\"tuning-reference_cluster-config.md\":\"DbWG1yZq\",\"tuning-reference_config.md\":\"YUqSpwFB\",\"tuning-reference_data-formats.md\":\"C5DX5xL4\",\"tuning-reference_index.md\":\"ZVIpQ6Dk\",\"tuning-reference_intro.md\":\"BSSaDlqx\",\"tuning-reference_joins.md\":\"BVhNCDnE\",\"tuning-reference_memory-model.md\":\"CfjFdgbl\",\"tuning-reference_metrics.md\":\"C64VNEL4\",\"tuning-reference_partitioning.md\":\"CtLK0xsS\",\"tuning-reference_pyspark.md\":\"D-92XgF8\",\"tuning-reference_shuffle.md\":\"Iq2v1oHJ\",\"tuning-reference_spark-architecture.md\":\"_qwnE_Sc\",\"tuning-reference_table-formats.md\":\"BSYEK-eN\",\"user-guide_alternative-log-retrieval.md\":\"CJpYvHGo\",\"user-guide_alternative-log-retrieval_managed-platforms.md\":\"DvDwHNtC\",\"user-guide_alternative-log-retrieval_ssh-bastion.md\":\"Gszzlplv\",\"user-guide_getting-started.md\":\"DYHHHkQR\",\"user-guide_getting-started_ci-and-automation.md\":\"OUQqw3mI\",\"user-guide_getting-started_reading-the-dashboard.md\":\"Obf3W3__\",\"user-guide_getting-started_regression-metric-keys.md\":\"B_gzr2Mw\",\"user-guide_getting-started_tuning-thresholds.md\":\"B_MyjyZm\",\"user-guide_mcp-tools.md\":\"C65SjWUL\",\"user-guide_mcp-tools_compare-runs.md\":\"Cc_gLoix\",\"user-guide_mcp-tools_diagnose-run.md\":\"BP1vnHA5\",\"user-guide_mcp-tools_evaluate-budgets.md\":\"BB0nDY3Z\",\"user-guide_mcp-tools_get-finding-documentation.md\":\"BtTTHNV6\",\"user-guide_mcp-tools_get-finding-evidence.md\":\"rKyu4wts\",\"user-guide_mcp-tools_get-reference-doc.md\":\"Yac4tH3u\",\"user-guide_mcp-tools_get-run-summary.md\":\"DpxZ4ZNp\",\"user-guide_mcp-tools_list-runs.md\":\"DpQPeCM3\",\"user-guide_run-comparison.md\":\"BUALkohi\",\"user-guide_run-comparison_how-stages-are-matched.md\":\"DSXGu2tA\",\"user-guide_understanding-findings.md\":\"C9YRruMT\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\",\"collapsed\":true,\"items\":[{\"text\":\"Reading the dashboard\",\"link\":\"/user-guide/getting-started/reading-the-dashboard\"},{\"text\":\"CI and automation\",\"link\":\"/user-guide/getting-started/ci-and-automation\"},{\"text\":\"Tuning detector thresholds\",\"link\":\"/user-guide/getting-started/tuning-thresholds\"},{\"text\":\"Regression metric keys\",\"link\":\"/user-guide/getting-started/regression-metric-keys\"}]},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\",\"collapsed\":true,\"items\":[{\"text\":\"How stages are matched\",\"link\":\"/user-guide/run-comparison/how-stages-are-matched\"}]},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\",\"collapsed\":true,\"items\":[{\"text\":\"diagnose_run\",\"link\":\"/user-guide/mcp-tools/diagnose-run\"},{\"text\":\"get_run_summary\",\"link\":\"/user-guide/mcp-tools/get-run-summary\"},{\"text\":\"evaluate_budgets\",\"link\":\"/user-guide/mcp-tools/evaluate-budgets\"},{\"text\":\"compare_runs\",\"link\":\"/user-guide/mcp-tools/compare-runs\"},{\"text\":\"get_finding_evidence\",\"link\":\"/user-guide/mcp-tools/get-finding-evidence\"},{\"text\":\"get_finding_documentation\",\"link\":\"/user-guide/mcp-tools/get-finding-documentation\"},{\"text\":\"get_reference_doc\",\"link\":\"/user-guide/mcp-tools/get-reference-doc\"},{\"text\":\"list_runs\",\"link\":\"/user-guide/mcp-tools/list-runs\"}]},{\"text\":\"Finding your event log\",\"link\":\"/user-guide/alternative-log-retrieval\",\"collapsed\":true,\"items\":[{\"text\":\"Managed platforms\",\"link\":\"/user-guide/alternative-log-retrieval/managed-platforms\"},{\"text\":\"Behind an SSH bastion\",\"link\":\"/user-guide/alternative-log-retrieval/ssh-bastion\"}]}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\",\"collapsed\":true,\"items\":[{\"text\":\"Evidence availability\",\"link\":\"/contributor-guide/architecture/worker-protocol/evidence-availability\"},{\"text\":\"Evidence report and finding identity\",\"link\":\"/contributor-guide/architecture/worker-protocol/evidence-report\"},{\"text\":\"CLI and MCP server\",\"link\":\"/contributor-guide/architecture/worker-protocol/cli-and-mcp\"},{\"text\":\"Core and event validation\",\"link\":\"/contributor-guide/architecture/worker-protocol/core-and-event-validation\"}]},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\",\"collapsed\":true,\"items\":[{\"text\":\"Run comparison internals\",\"link\":\"/contributor-guide/architecture/state-and-history/run-comparison-internals\"},{\"text\":\"History Server intake\",\"link\":\"/contributor-guide/architecture/state-and-history/history-server-intake\"}]},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\",\"collapsed\":true,\"items\":[{\"text\":\"Plan attribution\",\"link\":\"/contributor-guide/architecture/detector-contract/plan-attribution\"},{\"text\":\"Threshold tables\",\"link\":\"/contributor-guide/architecture/detector-contract/threshold-tables\"}]},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\",\"collapsed\":true,\"items\":[{\"text\":\"Stage union rollup\",\"link\":\"/contributor-guide/architecture/impact-estimation/stage-union-rollup\"},{\"text\":\"Caveats, tuning and coverage\",\"link\":\"/contributor-guide/architecture/impact-estimation/caveats-tuning-and-coverage\"}]},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\",\"collapsed\":true,\"items\":[{\"text\":\"Findings tab\",\"link\":\"/contributor-guide/architecture/widget-rendering/findings-tab\"},{\"text\":\"List sort mode\",\"link\":\"/contributor-guide/architecture/widget-rendering/list-sort-mode\"},{\"text\":\"Row/expand contract\",\"link\":\"/contributor-guide/architecture/widget-rendering/row-expand-contract\"},{\"text\":\"ReferenceSection\",\"link\":\"/contributor-guide/architecture/widget-rendering/reference-section\"},{\"text\":\"Investigation routing\",\"link\":\"/contributor-guide/architecture/widget-rendering/investigation-routing\"}]},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\",\"collapsed\":true,\"items\":[{\"text\":\"Plan Advisor\",\"link\":\"/contributor-guide/architecture/board-widgets/plan-advisor\"},{\"text\":\"Caching and memory widgets\",\"link\":\"/contributor-guide/architecture/board-widgets/caching-and-memory\"},{\"text\":\"Visual system and templating\",\"link\":\"/contributor-guide/architecture/board-widgets/visual-system-and-templating\"}]},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\",\"collapsed\":true,\"items\":[{\"text\":\"Plan graph\",\"link\":\"/contributor-guide/architecture/drill-down/plan-graph\"}]}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":{\"selector\":\"[data-sf-anchor-offset]\",\"padding\":0},\"cleanUrls\":false}");</script>
|
|
22
|
+
<div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><!--[--><!--[--><img class="VPImage dark logo" src="/docs/logo-dark.svg" alt data-v-8426fc1a><!--]--><!--[--><img class="VPImage light logo" src="/docs/logo-light.svg" alt data-v-8426fc1a><!--]--><!--]--><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/.." target="_self" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Open app</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Getting Started</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/intro.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Introduction</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Optimization Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/spark-architecture.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Execution Model</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/memory-model.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Management</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/partitioning.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Partitioning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/joins.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Join Optimization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/data-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Data Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/table-formats.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Table Formats</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/caching.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Caching & Persistence</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/pyspark.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>PySpark Specifics</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/aqe.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Adaptive Query Execution</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/cluster-config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cluster Tuning</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/anti-patterns.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Anti-Patterns</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Detector Catalog</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-skew.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Skew</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-shuffle.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Shuffle I/O</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-spill.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory / Disk Spill</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-gc.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>GC Pressure</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-cold-start.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Cold Start</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Executor Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-slow-host.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Slow Host</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-failures.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Task Failures</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-straggler.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Stragglers</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-retry-waste.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Retry Waste</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-tiny-tasks.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Tiny Tasks</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-job-failure-rate.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Job Failure Rate</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-memory-utilization.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Memory Utilization</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Duplicate Plan Subtree</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-small-files.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Small Files</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/bottleneck-broadcast-sizing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Broadcast Sizing</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Reference</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/metrics.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Metrics Glossary</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/tuning-reference/config.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Spark Config Quick-Reference</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_tuning-reference_anti-patterns" data-v-39a288b8><div><h1 id="anti-patterns" tabindex="-1">Anti-Patterns <a class="header-anchor" href="#anti-patterns" aria-label="Permalink to "Anti-Patterns {#anti-patterns}""></a></h1><p>Practitioner checklists on Spark performance converge on a recurring handful of mistakes rather than exotic ones. Each entry below covers what the mistake is, how to detect it, why it hurts, and how to fix it.</p><ol><li><p><strong>Staying on the RDD API instead of DataFrame/Dataset</strong></p><ul><li><strong>What:</strong> Building jobs on the RDD API (or RDD-based MLlib) instead of the DataFrame/Dataset API.</li><li><strong>Detect:</strong> Code that drives its logic through <code>.map</code>/<code>.filter</code> lambdas over RDDs, or that calls into the RDD-based MLlib API.</li><li><strong>Why:</strong> RDDs are lambda-driven, so Spark cannot see inside the closures. It can't apply predicate pushdown, filter reordering, Adaptive Query Execution, or cost-based join reordering. The RDD-based MLlib API is in maintenance mode for the same reason.<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup></li><li><strong>Fix:</strong> Use the DataFrame/Dataset API so Catalyst can optimize the query plan.</li></ul></li><li><p><strong>Reading CSV/JSON without an explicit schema</strong></p><ul><li><strong>What:</strong> Reading raw CSV or JSON files without supplying an explicit schema.</li><li><strong>Detect:</strong> A read step that spends noticeable time before any real work starts, because Spark is inferring types from the data itself.</li><li><strong>Why:</strong> Schema inference forces a full scan of the data just to determine types, and CSV/JSON don't support the column pruning, predicate pushdown, or stats-based file skipping that a columnar format does.<sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup></li><li><strong>Fix:</strong> Supply an explicit schema, or prefer Parquet for analytical workloads, where column pruning, predicate pushdown, and stats-based file skipping work out of the box.<sup class="footnote-ref"><a href="#fn1" id="fnref1:2">[1:2]</a></sup></li></ul></li><li><p><strong>Compressing input files with a non-splittable codec (GZIP)</strong></p><ul><li><strong>What:</strong> Storing input data as large GZIP files.</li><li><strong>Detect:</strong> One task/executor taking far longer than the rest on a stage reading GZIP-compressed input, while other executors sit idle.</li><li><strong>Why:</strong> A GZIP file can't be split across executors, so a single node has to decompress the whole file alone.<sup class="footnote-ref"><a href="#fn1" id="fnref1:3">[1:3]</a></sup></li><li><strong>Fix:</strong> Use a splittable codec instead: Snappy, LZ4, or ZSTD.<sup class="footnote-ref"><a href="#fn1" id="fnref1:4">[1:4]</a></sup></li></ul></li><li><p><strong>Storing data as JSON instead of a binary columnar/row format</strong></p><ul><li><strong>What:</strong> Persisting datasets as JSON rather than a binary columnar or row format.</li><li><strong>Detect:</strong> Read-heavy jobs where a large, recurring share of stage time goes to parsing the same JSON structures on every read.</li><li><strong>Why:</strong> JSON has to be re-parsed from text on every single read.<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup></li><li><strong>Fix:</strong> Store data as Avro, Parquet, Thrift, or Protobuf structs in a sequence file to avoid the repeated parsing cost.<sup class="footnote-ref"><a href="#fn2" id="fnref2:1">[2:1]</a></sup></li></ul></li><li><p><strong>Not registering custom classes with Kryo</strong></p><ul><li><strong>What:</strong> Using Kryo serialization without registering the application's custom classes.</li><li><strong>Detect:</strong> Larger-than-expected serialized record sizes, and poor efficiency when using a serialized cache storage level such as <code>MEMORY_SER</code>.</li><li><strong>Why:</strong> Unregistered classes increase serialized-record size and hurt serialized cache storage levels.<sup class="footnote-ref"><a href="#fn2" id="fnref2:2">[2:2]</a></sup></li><li><strong>Fix:</strong> Register the application's custom classes with Kryo.</li></ul></li><li><p><strong>Calling <code>collect()</code> on a large DataFrame</strong></p><ul><li><p><strong>What:</strong> Materializing an entire DataFrame into driver memory with <code>collect()</code>.</p></li><li><p><strong>Detect:</strong> A driver <code>OutOfMemoryError</code>, or a job that aborts once <code>spark.driver.maxResultSize</code> (default 1 GB) is exceeded. Spark also logs a warning once a single task's serialized result passes roughly 1 MB.<sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup><sup class="footnote-ref"><a href="#fn1" id="fnref1:5">[1:5]</a></sup></p></li><li><p><strong>Why:</strong> <code>collect()</code> gathers every <code>Row</code> from every partition into a single in-memory <code>Array</code> that lives entirely on the driver, not the cluster. The driver JVM has to hold the whole result set in its own heap.<sup class="footnote-ref"><a href="#fn3" id="fnref3:1">[3:1]</a></sup><sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup><sup class="footnote-ref"><a href="#fn5" id="fnref5">[5]</a></sup><sup class="footnote-ref"><a href="#fn6" id="fnref6">[6]</a></sup></p></li><li><p><strong>Fix:</strong> Use aggregations or <code>take(n)</code> instead of collecting the full result set.</p><blockquote><p><strong>PySpark:</strong> <code>toPandas()</code> performs the same driver-side collection as <code>collect()</code> for Python users, so avoid it on large DataFrames too.<sup class="footnote-ref"><a href="#fn6" id="fnref6:1">[6:1]</a></sup></p></blockquote></li></ul></li><li><p><strong>Assuming <code>cache()</code> + <code>count()</code> guarantees the DataFrame stays fully cached</strong></p><ul><li><strong>What:</strong> Treating <code>df.cache(); df.count()</code> as a guarantee that the DataFrame remains fully persisted for later actions.</li><li><strong>Detect:</strong> Later actions on a supposedly-cached DataFrame recompute from source instead of hitting the cache. There's no built-in visibility into how much of a DataFrame is still actually cached, so this typically only surfaces as unexpectedly slow recomputation.<sup class="footnote-ref"><a href="#fn7" id="fnref7">[7]</a></sup></li><li><strong>Why:</strong> <code>count()</code> does force every partition to be computed and attempted for caching, unlike <code>take</code>/<code>limit</code>, which only touch the partitions they need.<sup class="footnote-ref"><a href="#fn5" id="fnref5:1">[5:1]</a></sup><sup class="footnote-ref"><a href="#fn7" id="fnref7:1">[7:1]</a></sup> But partitions can't be fractionally cached: if there isn't room for all of them, the ones that don't fit are silently dropped and recomputed on next access.<sup class="footnote-ref"><a href="#fn5" id="fnref5:2">[5:2]</a></sup> Cached blocks also compete with execution for the same memory pool and can be evicted under pressure without warning, and caching is tied to the <em>analyzed</em> (pre-optimization) logical plan, so a semantically identical query with a different analyzed plan bypasses the cache entirely and recomputes from source.<sup class="footnote-ref"><a href="#fn7" id="fnref7:2">[7:2]</a></sup> Losing an executor drops any cached block that wasn't stored with a replicated storage level like <code>MEMORY_AND_DISK_2</code>.<sup class="footnote-ref"><a href="#fn7" id="fnref7:3">[7:3]</a></sup></li><li><strong>Fix:</strong> Keep using <code>.cache()</code> followed by <code>.count()</code> to force materialization, but don't assume that guarantees persistence (there's no built-in way to confirm how much of the DataFrame is actually cached<sup class="footnote-ref"><a href="#fn7" id="fnref7:4">[7:4]</a></sup>), and use a replicated storage level if losing a cached partition to executor failure would be costly.</li></ul></li><li><p><strong>Row-at-a-time Python UDFs</strong></p><ul><li><strong>What:</strong> Writing Python UDFs that operate one row at a time instead of vectorized Pandas UDFs.</li><li><strong>Detect:</strong> A UDF-heavy stage where most of the time goes to serialization rather than actual computation.<sup class="footnote-ref"><a href="#fn8" id="fnref8">[8]</a></sup></li><li><strong>Why:</strong> The overhead is serialization plus per-row JVM↔Python data movement, not raw Python interpreter speed. Row-at-a-time UDFs "suffer from high serialization and invocation overhead."<sup class="footnote-ref"><a href="#fn9" id="fnref9">[9]</a></sup> RDD-era Python UDFs pay a <em>double</em> serialization cost: Java/Scala objects are serialized, then re-serialized to Python via <code>cloudpickle</code> and back on every call.<sup class="footnote-ref"><a href="#fn8" id="fnref8:1">[8:1]</a></sup> A Databricks benchmark on a 10M-row DataFrame showed vectorized Pandas UDFs beating row-at-a-time UDFs "across the board, ranging from 3x to over 100x."<sup class="footnote-ref"><a href="#fn9" id="fnref9:1">[9:1]</a></sup></li><li><strong>Fix:</strong> Use vectorized Pandas UDFs instead of row-at-a-time UDFs: once data is transferred via Arrow, there's no need to serialize/pickle it, since it's already in a format consumable by the Python process.<sup class="footnote-ref"><a href="#fn5" id="fnref5:3">[5:3]</a></sup></li></ul></li><li><p><strong>Leaving <code>spark.sql.shuffle.partitions</code> at its default of 200</strong></p><ul><li><strong>What:</strong> Never tuning <code>spark.sql.shuffle.partitions</code> away from its default of 200 for wide transformations (<code>join</code>, <code>groupBy</code>, aggregations).</li><li><strong>Detect:</strong> In the Spark UI Stages tab, too few partitions for the data size shows up as "Spill (Memory)" or "Spill (Disk)" entries against a stage; too many partitions for the data size shows up as a large task/stage count where each task's duration is dominated by scheduling and bookkeeping, often paired with a large number of tiny output files.<sup class="footnote-ref"><a href="#fn10" id="fnref10">[10]</a></sup></li><li><strong>Why:</strong> 200 is a fixed default regardless of whether you're joining 5MB or 5TB.<sup class="footnote-ref"><a href="#fn10" id="fnref10:1">[10:1]</a></sup><sup class="footnote-ref"><a href="#fn11" id="fnref11">[11]</a></sup> Too few partitions means each one holds more data than fits comfortably in executor memory, which increases load per executor and leads to spills once partition size exceeds available memory.<sup class="footnote-ref"><a href="#fn10" id="fnref10:2">[10:2]</a></sup> Too many partitions on a small dataset can shrink tasks to around ten rows each, so "most of your CPUs will just be sitting there doing nothing."<sup class="footnote-ref"><a href="#fn10" id="fnref10:3">[10:3]</a></sup></li><li><strong>Fix:</strong> Increase <code>spark.sql.shuffle.partitions</code> if tasks are processing multiple GB each and spilling; decrease it if tasks finish in a couple of seconds and write tiny files. A rough starting target is 100–200 MB of data per task, tuned per dataset.<sup class="footnote-ref"><a href="#fn10" id="fnref10:4">[10:4]</a></sup></li></ul></li><li><p><strong>Broadcasting a table that's too large</strong></p><ul><li><strong>What:</strong> Forcing or allowing a broadcast join on a table that's too large to broadcast safely.</li><li><strong>Detect:</strong> A driver <code>OutOfMemoryError</code> during a broadcast join.</li><li><strong>Why:</strong> The small side of the join has to be collected onto the driver before it can be broadcast to every executor. That collection step is the same driver-memory operation as <code>.collect()</code>, and it's what fails, not executor-side replication or a serialization timeout: "if you try to broadcast something too large, you can crash your driver node (because that collect is expensive)."<sup class="footnote-ref"><a href="#fn4" id="fnref4:1">[4:1]</a></sup></li><li><strong>Fix:</strong> Use <code>spark.sql.autoBroadcastJoinThreshold</code> to control the maximum size Spark will broadcast, or increase driver memory.<sup class="footnote-ref"><a href="#fn4" id="fnref4:2">[4:2]</a></sup></li></ul></li><li><p><strong>Reaching for <code>coalesce(1)</code> before every single-file write</strong></p><ul><li><strong>What:</strong> Always using <code>coalesce(1)</code> instead of <code>repartition(1)</code> when a single output file is needed.</li><li><strong>Detect:</strong> The entire upstream stage (including filtering/transformation work that would otherwise run in parallel) collapses onto a single task/executor, leaving the rest of the cluster idle.</li><li><strong>Why:</strong> <code>coalesce</code> is a narrow transformation, so it "causes the upstream partitions in the entire stage to execute with the level of parallelism assigned by coalesce," fusing all preceding work down to one task.<sup class="footnote-ref"><a href="#fn6" id="fnref6:2">[6:2]</a></sup> <code>repartition</code> triggers a full shuffle instead, which inserts an explicit shuffle boundary so the upstream stage keeps its original parallelism.<sup class="footnote-ref"><a href="#fn4" id="fnref4:3">[4:3]</a></sup><sup class="footnote-ref"><a href="#fn6" id="fnref6:3">[6:3]</a></sup></li><li><strong>Fix:</strong> Use <code>repartition(1)</code> when there is meaningful upstream computation you don't want collapsed onto a single executor; reserve <code>coalesce(1)</code> for when the upstream stage is already cheap and paying for a shuffle would be wasted cost.<sup class="footnote-ref"><a href="#fn6" id="fnref6:4">[6:4]</a></sup></li></ul></li><li><p><strong>Not verifying predicate pushdown is actually happening</strong></p><ul><li><strong>What:</strong> Assuming filters and column projections are pushed down to the data source without checking the physical plan.</li><li><strong>Detect:</strong> Run <code>.explain()</code> and look at the <code>Scan</code> node for a <code>PushedFilters</code> marker. For example, <code>*Scan JDBCRel... PushedFilters: [*In(DEST_COUNTRY_NAME, [Anguilla, Sweden])]</code>. Column pruning is verified the same way: a <code>.select()</code> on one column should show a narrowed <code>ReadSchema</code> at the scan rather than a full-table scan.<sup class="footnote-ref"><a href="#fn4" id="fnref4:4">[4:4]</a></sup></li><li><strong>Why:</strong> Spark pushes down simple filters (column equality, <code>IN</code>, <code>IS NULL</code>) to JDBC sources automatically, but anything with a computed column or cast won't push down and gets evaluated in Spark after the full read.<sup class="footnote-ref"><a href="#fn1" id="fnref1:6">[1:6]</a></sup> Some sources also only partially handle a filter and leave Spark to re-evaluate it as a safety mechanism; that's still a "good" pushdown since the amount of data read is reduced.<sup class="footnote-ref"><a href="#fn6" id="fnref6:5">[6:5]</a></sup></li><li><strong>Fix:</strong> Check <code>explain</code> output (paired with <code>printSchema</code>)<sup class="footnote-ref"><a href="#fn6" id="fnref6:6">[6:6]</a></sup> for <code>PushedFilters</code> after writing a filter, and rewrite filters that use casts or computed columns as simple predicates where possible so they can push down.<sup class="footnote-ref"><a href="#fn4" id="fnref4:5">[4:5]</a></sup><sup class="footnote-ref"><a href="#fn1" id="fnref1:7">[1:7]</a></sup></li></ul></li></ol><h2 id="sources" tabindex="-1">Sources <a class="header-anchor" href="#sources" aria-label="Permalink to "Sources""></a></h2><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><a href="https://luminousmen.com/post/the-apache-spark-optimization-checklist" target="_blank" rel="noreferrer">The Apache Spark Optimization Checklist</a> <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a> <a href="#fnref1:2" class="footnote-backref">↩︎</a> <a href="#fnref1:3" class="footnote-backref">↩︎</a> <a href="#fnref1:4" class="footnote-backref">↩︎</a> <a href="#fnref1:5" class="footnote-backref">↩︎</a> <a href="#fnref1:6" class="footnote-backref">↩︎</a> <a href="#fnref1:7" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><a href="https://blog.cloudera.com/how-to-tune-your-apache-spark-jobs-part-2/" target="_blank" rel="noreferrer">How to Tune Your Apache Spark Jobs (Part 2)</a> <a href="#fnref2" class="footnote-backref">↩︎</a> <a href="#fnref2:1" class="footnote-backref">↩︎</a> <a href="#fnref2:2" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><em>Advanced Analytics with PySpark</em>, Tandon, Ryza, Laserson et al., ch. 2 <a href="#fnref3" class="footnote-backref">↩︎</a> <a href="#fnref3:1" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><em>Spark: The Definitive Guide</em>, Chambers & Zaharia, chs. 5, 8, 9, 18, 19 <a href="#fnref4" class="footnote-backref">↩︎</a> <a href="#fnref4:1" class="footnote-backref">↩︎</a> <a href="#fnref4:2" class="footnote-backref">↩︎</a> <a href="#fnref4:3" class="footnote-backref">↩︎</a> <a href="#fnref4:4" class="footnote-backref">↩︎</a> <a href="#fnref4:5" class="footnote-backref">↩︎</a></p></li><li id="fn5" class="footnote-item"><p><em>Learning Spark, 2nd Edition</em>, Damji, Wenig, Das & Lee, chs. 3, 5, 7 <a href="#fnref5" class="footnote-backref">↩︎</a> <a href="#fnref5:1" class="footnote-backref">↩︎</a> <a href="#fnref5:2" class="footnote-backref">↩︎</a> <a href="#fnref5:3" class="footnote-backref">↩︎</a></p></li><li id="fn6" class="footnote-item"><p><em>High Performance Spark, 2nd Edition</em>, Karau, Polak & Warren, chs. 5, 7 <a href="#fnref6" class="footnote-backref">↩︎</a> <a href="#fnref6:1" class="footnote-backref">↩︎</a> <a href="#fnref6:2" class="footnote-backref">↩︎</a> <a href="#fnref6:3" class="footnote-backref">↩︎</a> <a href="#fnref6:4" class="footnote-backref">↩︎</a> <a href="#fnref6:5" class="footnote-backref">↩︎</a> <a href="#fnref6:6" class="footnote-backref">↩︎</a></p></li><li id="fn7" class="footnote-item"><p><a href="https://luminousmen.com/post/explaining-the-mechanics-of-spark-caching" target="_blank" rel="noreferrer">Explaining the Mechanics of Spark Caching</a> <a href="#fnref7" class="footnote-backref">↩︎</a> <a href="#fnref7:1" class="footnote-backref">↩︎</a> <a href="#fnref7:2" class="footnote-backref">↩︎</a> <a href="#fnref7:3" class="footnote-backref">↩︎</a> <a href="#fnref7:4" class="footnote-backref">↩︎</a></p></li><li id="fn8" class="footnote-item"><p><a href="https://luminousmen.com/post/spark-tips-dataframe-api" target="_blank" rel="noreferrer">Spark Tips: DataFrame API</a> <a href="#fnref8" class="footnote-backref">↩︎</a> <a href="#fnref8:1" class="footnote-backref">↩︎</a></p></li><li id="fn9" class="footnote-item"><p><a href="https://www.databricks.com/blog/2017/10/30/introducing-vectorized-udfs-for-pyspark.html" target="_blank" rel="noreferrer">Introducing Vectorized UDFs for PySpark</a> <a href="#fnref9" class="footnote-backref">↩︎</a> <a href="#fnref9:1" class="footnote-backref">↩︎</a></p></li><li id="fn10" class="footnote-item"><p><a href="https://luminousmen.com/post/spark-partitions" target="_blank" rel="noreferrer">Spark Partitions</a> <a href="#fnref10" class="footnote-backref">↩︎</a> <a href="#fnref10:1" class="footnote-backref">↩︎</a> <a href="#fnref10:2" class="footnote-backref">↩︎</a> <a href="#fnref10:3" class="footnote-backref">↩︎</a> <a href="#fnref10:4" class="footnote-backref">↩︎</a></p></li><li id="fn11" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration — Spark</a> <a href="#fnref11" class="footnote-backref">↩︎</a></p></li></ol></section></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/tuning-reference/cluster-config.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>Cluster Tuning</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="/docs/tuning-reference/bottleneck-skew.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>Task Skew</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
|
|
23
|
+
<script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"CqKM_jEG\",\"contributor-guide_architecture_board-widgets_caching-and-memory.md\":\"BYDo3ClR\",\"contributor-guide_architecture_board-widgets_plan-advisor.md\":\"ajUAVdbc\",\"contributor-guide_architecture_board-widgets_visual-system-and-templating.md\":\"WqY74T0T\",\"contributor-guide_architecture_detector-contract.md\":\"CHHgywoP\",\"contributor-guide_architecture_detector-contract_plan-attribution.md\":\"DIz3_oz9\",\"contributor-guide_architecture_detector-contract_threshold-tables.md\":\"WH4tRHPL\",\"contributor-guide_architecture_drill-down.md\":\"yB-Hz2XT\",\"contributor-guide_architecture_drill-down_plan-graph.md\":\"BQZE04sw\",\"contributor-guide_architecture_impact-estimation.md\":\"K71YB-Rr\",\"contributor-guide_architecture_impact-estimation_caveats-tuning-and-coverage.md\":\"DLCAffwn\",\"contributor-guide_architecture_impact-estimation_stage-union-rollup.md\":\"JFrzDMwo\",\"contributor-guide_architecture_index.md\":\"tymxf1Fh\",\"contributor-guide_architecture_overview.md\":\"nxkpOhgf\",\"contributor-guide_architecture_state-and-history.md\":\"DyCBdjuM\",\"contributor-guide_architecture_state-and-history_history-server-intake.md\":\"C298hrHZ\",\"contributor-guide_architecture_state-and-history_run-comparison-internals.md\":\"U-9KMDFK\",\"contributor-guide_architecture_widget-rendering.md\":\"BrSz8Ezh\",\"contributor-guide_architecture_widget-rendering_findings-tab.md\":\"B7rjSF_h\",\"contributor-guide_architecture_widget-rendering_investigation-routing.md\":\"BGdiyElu\",\"contributor-guide_architecture_widget-rendering_list-sort-mode.md\":\"37hcZJ7q\",\"contributor-guide_architecture_widget-rendering_reference-section.md\":\"DWqC80bN\",\"contributor-guide_architecture_widget-rendering_row-expand-contract.md\":\"mK_VmAWk\",\"contributor-guide_architecture_worker-protocol.md\":\"Br6OE7RW\",\"contributor-guide_architecture_worker-protocol_cli-and-mcp.md\":\"IVyaXrGe\",\"contributor-guide_architecture_worker-protocol_core-and-event-validation.md\":\"DVSXn-bb\",\"contributor-guide_architecture_worker-protocol_evidence-availability.md\":\"D3S5cyOW\",\"contributor-guide_architecture_worker-protocol_evidence-report.md\":\"CtmCU57I\",\"contributor-guide_contributing.md\":\"DBW4drtZ\",\"contributor-guide_development-setup.md\":\"B21ZTDhn\",\"contributor-guide_testing.md\":\"DpkozNoG\",\"index.md\":\"lMm7SUMz\",\"tuning-reference_anti-patterns.md\":\"D08Uaod7\",\"tuning-reference_aqe.md\":\"C1Ae6A8u\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"6wjKuJNE\",\"tuning-reference_bottleneck-cold-start.md\":\"BVQbsuHP\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"DtaYnaiF\",\"tuning-reference_bottleneck-failures.md\":\"Z1Tdmdd3\",\"tuning-reference_bottleneck-gc.md\":\"BCnXLf1n\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"CcvQj2g9\",\"tuning-reference_bottleneck-memory-utilization.md\":\"B-C0-8h-\",\"tuning-reference_bottleneck-retry-waste.md\":\"DCnhWiqj\",\"tuning-reference_bottleneck-shuffle.md\":\"B5OMJCd3\",\"tuning-reference_bottleneck-skew.md\":\"Cvo6NQm_\",\"tuning-reference_bottleneck-slow-host.md\":\"ja5qr3wq\",\"tuning-reference_bottleneck-small-files.md\":\"Crjtb0i2\",\"tuning-reference_bottleneck-spill.md\":\"Bl8KtSRg\",\"tuning-reference_bottleneck-straggler.md\":\"B0VN8reT\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"DQCftGhu\",\"tuning-reference_bottleneck-utilization.md\":\"BXDl30f9\",\"tuning-reference_caching.md\":\"CcaiMn_7\",\"tuning-reference_cluster-config.md\":\"DbWG1yZq\",\"tuning-reference_config.md\":\"YUqSpwFB\",\"tuning-reference_data-formats.md\":\"C5DX5xL4\",\"tuning-reference_index.md\":\"ZVIpQ6Dk\",\"tuning-reference_intro.md\":\"BSSaDlqx\",\"tuning-reference_joins.md\":\"BVhNCDnE\",\"tuning-reference_memory-model.md\":\"CfjFdgbl\",\"tuning-reference_metrics.md\":\"C64VNEL4\",\"tuning-reference_partitioning.md\":\"CtLK0xsS\",\"tuning-reference_pyspark.md\":\"D-92XgF8\",\"tuning-reference_shuffle.md\":\"Iq2v1oHJ\",\"tuning-reference_spark-architecture.md\":\"_qwnE_Sc\",\"tuning-reference_table-formats.md\":\"BSYEK-eN\",\"user-guide_alternative-log-retrieval.md\":\"CJpYvHGo\",\"user-guide_alternative-log-retrieval_managed-platforms.md\":\"DvDwHNtC\",\"user-guide_alternative-log-retrieval_ssh-bastion.md\":\"Gszzlplv\",\"user-guide_getting-started.md\":\"DYHHHkQR\",\"user-guide_getting-started_ci-and-automation.md\":\"C0pcQoNc\",\"user-guide_getting-started_reading-the-dashboard.md\":\"CGaxPX_1\",\"user-guide_getting-started_regression-metric-keys.md\":\"B_gzr2Mw\",\"user-guide_getting-started_tuning-thresholds.md\":\"BGVE-wrc\",\"user-guide_mcp-tools.md\":\"49r3L2YE\",\"user-guide_mcp-tools_compare-runs.md\":\"BQ_kIptC\",\"user-guide_mcp-tools_diagnose-run.md\":\"azCZXkH8\",\"user-guide_mcp-tools_evaluate-budgets.md\":\"dpQpRQ7z\",\"user-guide_mcp-tools_get-finding-documentation.md\":\"BtTTHNV6\",\"user-guide_mcp-tools_get-finding-evidence.md\":\"CH6FkNLX\",\"user-guide_mcp-tools_get-reference-doc.md\":\"Yac4tH3u\",\"user-guide_mcp-tools_get-run-summary.md\":\"EhaIsele\",\"user-guide_mcp-tools_list-runs.md\":\"DpQPeCM3\",\"user-guide_run-comparison.md\":\"B_LOkdBf\",\"user-guide_run-comparison_how-stages-are-matched.md\":\"DSXGu2tA\",\"user-guide_understanding-findings.md\":\"C4riKCKl\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":{\"light\":\"/logo-light.svg\",\"dark\":\"/logo-dark.svg\"},\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"},{\"text\":\"Open app\",\"link\":\"/..\",\"target\":\"_self\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\",\"collapsed\":true,\"items\":[{\"text\":\"Reading the dashboard\",\"link\":\"/user-guide/getting-started/reading-the-dashboard\"},{\"text\":\"CI and automation\",\"link\":\"/user-guide/getting-started/ci-and-automation\"},{\"text\":\"Tuning detector thresholds\",\"link\":\"/user-guide/getting-started/tuning-thresholds\"},{\"text\":\"Regression metric keys\",\"link\":\"/user-guide/getting-started/regression-metric-keys\"}]},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\",\"collapsed\":true,\"items\":[{\"text\":\"How stages are matched\",\"link\":\"/user-guide/run-comparison/how-stages-are-matched\"}]},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\",\"collapsed\":true,\"items\":[{\"text\":\"diagnose_run\",\"link\":\"/user-guide/mcp-tools/diagnose-run\"},{\"text\":\"get_run_summary\",\"link\":\"/user-guide/mcp-tools/get-run-summary\"},{\"text\":\"evaluate_budgets\",\"link\":\"/user-guide/mcp-tools/evaluate-budgets\"},{\"text\":\"compare_runs\",\"link\":\"/user-guide/mcp-tools/compare-runs\"},{\"text\":\"get_finding_evidence\",\"link\":\"/user-guide/mcp-tools/get-finding-evidence\"},{\"text\":\"get_finding_documentation\",\"link\":\"/user-guide/mcp-tools/get-finding-documentation\"},{\"text\":\"get_reference_doc\",\"link\":\"/user-guide/mcp-tools/get-reference-doc\"},{\"text\":\"list_runs\",\"link\":\"/user-guide/mcp-tools/list-runs\"}]},{\"text\":\"Finding your event log\",\"link\":\"/user-guide/alternative-log-retrieval\",\"collapsed\":true,\"items\":[{\"text\":\"Managed platforms\",\"link\":\"/user-guide/alternative-log-retrieval/managed-platforms\"},{\"text\":\"Behind an SSH bastion\",\"link\":\"/user-guide/alternative-log-retrieval/ssh-bastion\"}]}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\",\"collapsed\":true,\"items\":[{\"text\":\"Evidence availability\",\"link\":\"/contributor-guide/architecture/worker-protocol/evidence-availability\"},{\"text\":\"Evidence report and finding identity\",\"link\":\"/contributor-guide/architecture/worker-protocol/evidence-report\"},{\"text\":\"CLI and MCP server\",\"link\":\"/contributor-guide/architecture/worker-protocol/cli-and-mcp\"},{\"text\":\"Core and event validation\",\"link\":\"/contributor-guide/architecture/worker-protocol/core-and-event-validation\"}]},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\",\"collapsed\":true,\"items\":[{\"text\":\"Run comparison internals\",\"link\":\"/contributor-guide/architecture/state-and-history/run-comparison-internals\"},{\"text\":\"History Server intake\",\"link\":\"/contributor-guide/architecture/state-and-history/history-server-intake\"}]},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\",\"collapsed\":true,\"items\":[{\"text\":\"Plan attribution\",\"link\":\"/contributor-guide/architecture/detector-contract/plan-attribution\"},{\"text\":\"Threshold tables\",\"link\":\"/contributor-guide/architecture/detector-contract/threshold-tables\"}]},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\",\"collapsed\":true,\"items\":[{\"text\":\"Stage union rollup\",\"link\":\"/contributor-guide/architecture/impact-estimation/stage-union-rollup\"},{\"text\":\"Caveats, tuning and coverage\",\"link\":\"/contributor-guide/architecture/impact-estimation/caveats-tuning-and-coverage\"}]},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\",\"collapsed\":true,\"items\":[{\"text\":\"Findings tab\",\"link\":\"/contributor-guide/architecture/widget-rendering/findings-tab\"},{\"text\":\"List sort mode\",\"link\":\"/contributor-guide/architecture/widget-rendering/list-sort-mode\"},{\"text\":\"Row/expand contract\",\"link\":\"/contributor-guide/architecture/widget-rendering/row-expand-contract\"},{\"text\":\"ReferenceSection\",\"link\":\"/contributor-guide/architecture/widget-rendering/reference-section\"},{\"text\":\"Investigation routing\",\"link\":\"/contributor-guide/architecture/widget-rendering/investigation-routing\"}]},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\",\"collapsed\":true,\"items\":[{\"text\":\"Plan Advisor\",\"link\":\"/contributor-guide/architecture/board-widgets/plan-advisor\"},{\"text\":\"Caching and memory widgets\",\"link\":\"/contributor-guide/architecture/board-widgets/caching-and-memory\"},{\"text\":\"Visual system and templating\",\"link\":\"/contributor-guide/architecture/board-widgets/visual-system-and-templating\"}]},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\",\"collapsed\":true,\"items\":[{\"text\":\"Plan graph\",\"link\":\"/contributor-guide/architecture/drill-down/plan-graph\"}]}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":{\"selector\":\"[data-sf-anchor-offset]\",\"padding\":0},\"cleanUrls\":false}");</script>
|
|
28
24
|
|
|
29
25
|
</body>
|
|
30
26
|
</html>
|