sparkforensics-server 0.5.0 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +3 -3
- package/public/assets/{AutoscalingChurn-9Zo6IMGS.js → AutoscalingChurn-5hDGIXi4.js} +1 -1
- package/public/assets/{CacheUtilization-MBDfMsKC.js → CacheUtilization-CT2SCnJJ.js} +1 -1
- package/public/assets/{CachingOpportunity-Cfw3wAFa.js → CachingOpportunity-D_O_WkbH.js} +1 -1
- package/public/assets/{ColdStart-BtS12b6_.js → ColdStart-B38OiAOp.js} +1 -1
- package/public/assets/{ConfigAudit-DNPV6EQg.js → ConfigAudit-BoplryCH.js} +1 -1
- package/public/assets/{CoreUsageArea-BObEeenE.js → CoreUsageArea-csgwNxkl.js} +1 -1
- package/public/assets/{Dashboard-CC_BXrNy.js → Dashboard-BgKodJl2.js} +2 -2
- package/public/assets/{DisclosureContext-Ds0hcZFn.js → DisclosureContext-B0siMVdg.js} +1 -1
- package/public/assets/{DuplicatePlanSubtree-vN3zqRsi.js → DuplicatePlanSubtree-BqS2A2j4.js} +1 -1
- package/public/assets/{ExecutorUtilization-CZObSLIj.js → ExecutorUtilization-Dp52QGWo.js} +1 -1
- package/public/assets/{GcPressure-ByJlLOua.js → GcPressure-CT1yzR_s.js} +1 -1
- package/public/assets/{IncompleteRun-MzqV4W8y.js → IncompleteRun-C7Qkjtjn.js} +1 -1
- package/public/assets/{JobFailures-LbMvAwAb.js → JobFailures-WHl3ELfh.js} +1 -1
- package/public/assets/{MemoryUtilization-DH4jYvJK.js → MemoryUtilization-D8IUDHJI.js} +1 -1
- package/public/assets/{OverBroadcast-euk6HKg4.js → OverBroadcast-6oSSAQ_R.js} +1 -1
- package/public/assets/{PartitionHint-DThaMMRT.js → PartitionHint-CUvbnnxy.js} +1 -1
- package/public/assets/{PartitionSizing-yIN-iZzx.js → PartitionSizing-BwdFL-mw.js} +1 -1
- package/public/assets/{PlanExplorer-Dd9WPK4S.js → PlanExplorer-DcxmiQWL.js} +1 -1
- package/public/assets/{PlanGraphRoute-NV5mCXRm.js → PlanGraphRoute-pqnHTt4i.js} +1 -1
- package/public/assets/{RetryWaste-CXH4D-xH.js → RetryWaste-BTbeHkJc.js} +1 -1
- package/public/assets/{ShuffleIO-CW7U68ph.js → ShuffleIO-CRhrYNE0.js} +1 -1
- package/public/assets/{Skew-PL79TppX.js → Skew-DSNXRG9V.js} +1 -1
- package/public/assets/{SlowHost-GBO73hCi.js → SlowHost-B7Lwu2EU.js} +1 -1
- package/public/assets/{SmallFiles-B5cUjEav.js → SmallFiles-C082c2rd.js} +1 -1
- package/public/assets/{SpeculationWaste-mxi3jgb0.js → SpeculationWaste-BS2vpadr.js} +1 -1
- package/public/assets/{Spill-CH1bNpxL.js → Spill-B4DVKHXx.js} +1 -1
- package/public/assets/{StageFailed-YvxC11ii.js → StageFailed-D3ac9Sou.js} +1 -1
- package/public/assets/{StageFindingGroup-B1A0nf-e.js → StageFindingGroup-D9naK109.js} +1 -1
- package/public/assets/{StageHeader-BMNmUy-s.js → StageHeader-pI1nTwVf.js} +1 -1
- package/public/assets/{StageShape-DO3mQ6NS.js → StageShape-eO5le8iG.js} +1 -1
- package/public/assets/{StageSlowness-SwRxfAEx.js → StageSlowness-BaNyfNvL.js} +1 -1
- package/public/assets/{Straggler-DHLTiBY-.js → Straggler-D5Fnok_y.js} +1 -1
- package/public/assets/{TaskFailures-Bj0QxiOc.js → TaskFailures-BWnC-K_G.js} +1 -1
- package/public/assets/{TinyTask-COUQQ5bQ.js → TinyTask-8_5lpq_2.js} +1 -1
- package/public/assets/{UnderBroadcast-RAUbQ6vw.js → UnderBroadcast-DJ0XSwth.js} +1 -1
- package/public/assets/{finding-anchor-Dl8mIHiU.js → finding-anchor-Cys8Z6Yn.js} +1 -1
- package/public/assets/{impact-sort-Eihua9dY.js → impact-sort-BTQndbof.js} +1 -1
- package/public/assets/index-DCeWge12.js +13 -0
- package/public/assets/parser-worker-DmeCnLWU.js +122 -0
- package/public/assets/{plan-graph-vendor-B0sEms9Z.js → plan-graph-vendor-BYl-AzN5.js} +1 -1
- package/public/assets/{react-vendor-BAKt3vny.js → react-vendor-DEdfrRTJ.js} +1 -1
- package/public/assets/{select-DkjFHlXR.js → select-DIYN6D3y.js} +1 -1
- package/public/assets/{usePagedRows-Bo-aSztx.js → usePagedRows-BxvdFnPk.js} +1 -1
- package/public/assets/{useRecentFiles-Bb4-ukVz.js → useRecentFiles-C3Ov47U7.js} +4 -4
- package/public/assets/{useSortMode-lDedD5a3.js → useSortMode-B8SRfAoG.js} +1 -1
- package/public/docs/404.html +2 -2
- package/public/docs/assets/{app.yjczoibf.js → app.D2GqAkhy.js} +1 -1
- package/public/docs/assets/chunks/@localSearchIndexroot.D4OdkQjy.js +1 -0
- package/public/docs/assets/chunks/{VPLocalSearchBox.DOBbPfix.js → VPLocalSearchBox.BS4cSmtW.js} +1 -1
- package/public/docs/assets/chunks/{theme.DFcAl8Ws.js → theme.BuxVpb0H.js} +2 -2
- package/public/docs/assets/contributor-guide_architecture_board-widgets.md.YQcj5EI4.js +1 -0
- package/public/docs/assets/{contributor-guide_architecture_board-widgets.md.D0Kkk_r5.lean.js → contributor-guide_architecture_board-widgets.md.YQcj5EI4.lean.js} +1 -1
- package/public/docs/assets/contributor-guide_architecture_board-widgets_caching-and-memory.md.8bH1HnZp.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_board-widgets_caching-and-memory.md.8bH1HnZp.lean.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_board-widgets_plan-advisor.md.RlzvUmNm.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_board-widgets_plan-advisor.md.RlzvUmNm.lean.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_board-widgets_visual-system-and-templating.md.DYgUpIJB.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_board-widgets_visual-system-and-templating.md.DYgUpIJB.lean.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_detector-contract.md.wj6Et6UD.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_detector-contract.md.wj6Et6UD.lean.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_detector-contract_plan-attribution.md.yNex4bys.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_detector-contract_plan-attribution.md.yNex4bys.lean.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_detector-contract_threshold-tables.md.fkPBXsU2.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_detector-contract_threshold-tables.md.fkPBXsU2.lean.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_drill-down.md.CqZPrdo9.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_drill-down.md.CqZPrdo9.lean.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_drill-down_plan-graph.md.DwOozIEs.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_drill-down_plan-graph.md.DwOozIEs.lean.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.CZv_ZGOp.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.CZv_ZGOp.lean.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_impact-estimation_caveats-tuning-and-coverage.md.BUucRc0A.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_impact-estimation_caveats-tuning-and-coverage.md.BUucRc0A.lean.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_impact-estimation_stage-union-rollup.md.DdCGtAGu.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_impact-estimation_stage-union-rollup.md.DdCGtAGu.lean.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_index.md.tymxf1Fh.js +1 -0
- package/public/docs/assets/{contributor-guide_architecture_index.md.B9yZieYL.lean.js → contributor-guide_architecture_index.md.tymxf1Fh.lean.js} +1 -1
- package/public/docs/assets/contributor-guide_architecture_overview.md.C1sSvm9i.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_state-and-history.md.DyCBdjuM.js +1 -0
- package/public/docs/assets/{contributor-guide_architecture_state-and-history.md.w4lwybdh.lean.js → contributor-guide_architecture_state-and-history.md.DyCBdjuM.lean.js} +1 -1
- package/public/docs/assets/contributor-guide_architecture_state-and-history_history-server-intake.md.C298hrHZ.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_state-and-history_history-server-intake.md.C298hrHZ.lean.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_state-and-history_run-comparison-internals.md.uuAKpY0Y.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_state-and-history_run-comparison-internals.md.uuAKpY0Y.lean.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.BOZ1azXG.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.BOZ1azXG.lean.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_widget-rendering_findings-tab.md.BQXycK6G.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_widget-rendering_findings-tab.md.BQXycK6G.lean.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_widget-rendering_investigation-routing.md.BGdiyElu.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_widget-rendering_investigation-routing.md.BGdiyElu.lean.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_widget-rendering_list-sort-mode.md.BuHKgH0C.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_widget-rendering_list-sort-mode.md.BuHKgH0C.lean.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_widget-rendering_reference-section.md.DWqC80bN.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_widget-rendering_reference-section.md.DWqC80bN.lean.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_widget-rendering_row-expand-contract.md.ZT89u9f1.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_widget-rendering_row-expand-contract.md.ZT89u9f1.lean.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_worker-protocol.md.BAnhi2-d.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_worker-protocol.md.BAnhi2-d.lean.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_worker-protocol_cli-and-mcp.md.Dk08VS7n.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_worker-protocol_cli-and-mcp.md.Dk08VS7n.lean.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_worker-protocol_core-and-event-validation.md.DSVFPpWb.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_worker-protocol_core-and-event-validation.md.DSVFPpWb.lean.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_worker-protocol_evidence-availability.md.CjimIDBY.js +6 -0
- package/public/docs/assets/contributor-guide_architecture_worker-protocol_evidence-availability.md.CjimIDBY.lean.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_worker-protocol_evidence-report.md.DVGZOWvW.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_worker-protocol_evidence-report.md.DVGZOWvW.lean.js +1 -0
- package/public/docs/assets/user-guide_alternative-log-retrieval.md.CJpYvHGo.js +4 -0
- package/public/docs/assets/user-guide_alternative-log-retrieval.md.CJpYvHGo.lean.js +1 -0
- package/public/docs/assets/user-guide_alternative-log-retrieval_managed-platforms.md.DvDwHNtC.js +4 -0
- package/public/docs/assets/user-guide_alternative-log-retrieval_managed-platforms.md.DvDwHNtC.lean.js +1 -0
- package/public/docs/assets/user-guide_alternative-log-retrieval_ssh-bastion.md.Gszzlplv.js +1 -0
- package/public/docs/assets/user-guide_alternative-log-retrieval_ssh-bastion.md.Gszzlplv.lean.js +1 -0
- package/public/docs/assets/user-guide_getting-started.md.DYHHHkQR.js +1 -0
- package/public/docs/assets/user-guide_getting-started.md.DYHHHkQR.lean.js +1 -0
- package/public/docs/assets/user-guide_getting-started_ci-and-automation.md.DpnfKz3x.js +22 -0
- package/public/docs/assets/user-guide_getting-started_ci-and-automation.md.DpnfKz3x.lean.js +1 -0
- package/public/docs/assets/user-guide_getting-started_reading-the-dashboard.md.Obf3W3__.js +1 -0
- package/public/docs/assets/user-guide_getting-started_reading-the-dashboard.md.Obf3W3__.lean.js +1 -0
- package/public/docs/assets/user-guide_getting-started_regression-metric-keys.md.B_gzr2Mw.js +1 -0
- package/public/docs/assets/user-guide_getting-started_regression-metric-keys.md.B_gzr2Mw.lean.js +1 -0
- package/public/docs/assets/user-guide_getting-started_tuning-thresholds.md.B_MyjyZm.js +4 -0
- package/public/docs/assets/user-guide_getting-started_tuning-thresholds.md.B_MyjyZm.lean.js +1 -0
- package/public/docs/assets/user-guide_mcp-tools.md.C65SjWUL.js +16 -0
- package/public/docs/assets/user-guide_mcp-tools.md.C65SjWUL.lean.js +1 -0
- package/public/docs/assets/user-guide_mcp-tools_compare-runs.md.CPowlb2q.js +55 -0
- package/public/docs/assets/user-guide_mcp-tools_compare-runs.md.CPowlb2q.lean.js +1 -0
- package/public/docs/assets/user-guide_mcp-tools_diagnose-run.md.BP1vnHA5.js +23 -0
- package/public/docs/assets/user-guide_mcp-tools_diagnose-run.md.BP1vnHA5.lean.js +1 -0
- package/public/docs/assets/user-guide_mcp-tools_evaluate-budgets.md.Dq5Azk8G.js +26 -0
- package/public/docs/assets/user-guide_mcp-tools_evaluate-budgets.md.Dq5Azk8G.lean.js +1 -0
- package/public/docs/assets/user-guide_mcp-tools_get-finding-documentation.md.BtTTHNV6.js +14 -0
- package/public/docs/assets/user-guide_mcp-tools_get-finding-documentation.md.BtTTHNV6.lean.js +1 -0
- package/public/docs/assets/user-guide_mcp-tools_get-finding-evidence.md.rKyu4wts.js +20 -0
- package/public/docs/assets/user-guide_mcp-tools_get-finding-evidence.md.rKyu4wts.lean.js +1 -0
- package/public/docs/assets/user-guide_mcp-tools_get-reference-doc.md.Yac4tH3u.js +5 -0
- package/public/docs/assets/user-guide_mcp-tools_get-reference-doc.md.Yac4tH3u.lean.js +1 -0
- package/public/docs/assets/user-guide_mcp-tools_get-run-summary.md.DpxZ4ZNp.js +15 -0
- package/public/docs/assets/user-guide_mcp-tools_get-run-summary.md.DpxZ4ZNp.lean.js +1 -0
- package/public/docs/assets/user-guide_mcp-tools_list-runs.md.DpQPeCM3.js +12 -0
- package/public/docs/assets/user-guide_mcp-tools_list-runs.md.DpQPeCM3.lean.js +1 -0
- package/public/docs/assets/{user-guide_run-comparison.md.qin0a4IX.js → user-guide_run-comparison.md.B6aNtDYF.js} +1 -1
- package/public/docs/assets/user-guide_run-comparison.md.B6aNtDYF.lean.js +1 -0
- package/public/docs/assets/user-guide_run-comparison_how-stages-are-matched.md.XBzQ1oEU.js +1 -0
- package/public/docs/assets/user-guide_run-comparison_how-stages-are-matched.md.XBzQ1oEU.lean.js +1 -0
- package/public/docs/assets/user-guide_understanding-findings.md.Dqt2H_C1.js +1 -0
- package/public/docs/contributor-guide/architecture/board-widgets/caching-and-memory.html +30 -0
- package/public/docs/contributor-guide/architecture/board-widgets/plan-advisor.html +30 -0
- package/public/docs/contributor-guide/architecture/board-widgets/visual-system-and-templating.html +30 -0
- package/public/docs/contributor-guide/architecture/board-widgets.html +5 -5
- package/public/docs/contributor-guide/architecture/detector-contract/plan-attribution.html +30 -0
- package/public/docs/contributor-guide/architecture/detector-contract/threshold-tables.html +30 -0
- package/public/docs/contributor-guide/architecture/detector-contract.html +5 -5
- package/public/docs/contributor-guide/architecture/drill-down/plan-graph.html +30 -0
- package/public/docs/contributor-guide/architecture/drill-down.html +5 -5
- package/public/docs/contributor-guide/architecture/impact-estimation/caveats-tuning-and-coverage.html +30 -0
- package/public/docs/contributor-guide/architecture/impact-estimation/stage-union-rollup.html +30 -0
- package/public/docs/contributor-guide/architecture/impact-estimation.html +5 -5
- package/public/docs/contributor-guide/architecture/index.html +5 -5
- package/public/docs/contributor-guide/architecture/overview.html +5 -5
- package/public/docs/contributor-guide/architecture/state-and-history/history-server-intake.html +30 -0
- package/public/docs/contributor-guide/architecture/state-and-history/run-comparison-internals.html +30 -0
- package/public/docs/contributor-guide/architecture/state-and-history.html +5 -5
- package/public/docs/contributor-guide/architecture/widget-rendering/findings-tab.html +30 -0
- package/public/docs/contributor-guide/architecture/widget-rendering/investigation-routing.html +30 -0
- package/public/docs/contributor-guide/architecture/widget-rendering/list-sort-mode.html +30 -0
- package/public/docs/contributor-guide/architecture/widget-rendering/reference-section.html +30 -0
- package/public/docs/contributor-guide/architecture/widget-rendering/row-expand-contract.html +30 -0
- package/public/docs/contributor-guide/architecture/widget-rendering.html +5 -5
- package/public/docs/contributor-guide/architecture/worker-protocol/cli-and-mcp.html +30 -0
- package/public/docs/contributor-guide/architecture/worker-protocol/core-and-event-validation.html +30 -0
- package/public/docs/contributor-guide/architecture/worker-protocol/evidence-availability.html +35 -0
- package/public/docs/contributor-guide/architecture/worker-protocol/evidence-report.html +30 -0
- package/public/docs/contributor-guide/architecture/worker-protocol.html +5 -10
- package/public/docs/contributor-guide/contributing.html +4 -4
- package/public/docs/contributor-guide/development-setup.html +4 -4
- package/public/docs/contributor-guide/testing.html +4 -4
- package/public/docs/hashmap.json +1 -1
- package/public/docs/index.html +3 -3
- package/public/docs/tuning-reference/anti-patterns.html +3 -3
- package/public/docs/tuning-reference/aqe.html +3 -3
- package/public/docs/tuning-reference/bottleneck-broadcast-sizing.html +3 -3
- package/public/docs/tuning-reference/bottleneck-cold-start.html +3 -3
- package/public/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html +3 -3
- package/public/docs/tuning-reference/bottleneck-failures.html +3 -3
- package/public/docs/tuning-reference/bottleneck-gc.html +3 -3
- package/public/docs/tuning-reference/bottleneck-job-failure-rate.html +3 -3
- package/public/docs/tuning-reference/bottleneck-memory-utilization.html +3 -3
- package/public/docs/tuning-reference/bottleneck-retry-waste.html +3 -3
- package/public/docs/tuning-reference/bottleneck-shuffle.html +3 -3
- package/public/docs/tuning-reference/bottleneck-skew.html +3 -3
- package/public/docs/tuning-reference/bottleneck-slow-host.html +3 -3
- package/public/docs/tuning-reference/bottleneck-small-files.html +3 -3
- package/public/docs/tuning-reference/bottleneck-spill.html +3 -3
- package/public/docs/tuning-reference/bottleneck-straggler.html +3 -3
- package/public/docs/tuning-reference/bottleneck-tiny-tasks.html +3 -3
- package/public/docs/tuning-reference/bottleneck-utilization.html +3 -3
- package/public/docs/tuning-reference/caching.html +3 -3
- package/public/docs/tuning-reference/cluster-config.html +3 -3
- package/public/docs/tuning-reference/config.html +3 -3
- package/public/docs/tuning-reference/data-formats.html +3 -3
- package/public/docs/tuning-reference/index.html +3 -3
- package/public/docs/tuning-reference/intro.html +3 -3
- package/public/docs/tuning-reference/joins.html +3 -3
- package/public/docs/tuning-reference/memory-model.html +3 -3
- package/public/docs/tuning-reference/metrics.html +3 -3
- package/public/docs/tuning-reference/partitioning.html +3 -3
- package/public/docs/tuning-reference/pyspark.html +3 -3
- package/public/docs/tuning-reference/shuffle.html +3 -3
- package/public/docs/tuning-reference/spark-architecture.html +3 -3
- package/public/docs/tuning-reference/table-formats.html +3 -3
- package/public/docs/user-guide/alternative-log-retrieval/managed-platforms.html +33 -0
- package/public/docs/user-guide/alternative-log-retrieval/ssh-bastion.html +30 -0
- package/public/docs/user-guide/alternative-log-retrieval.html +6 -9
- package/public/docs/user-guide/getting-started/ci-and-automation.html +51 -0
- package/public/docs/user-guide/getting-started/reading-the-dashboard.html +30 -0
- package/public/docs/user-guide/getting-started/regression-metric-keys.html +30 -0
- package/public/docs/user-guide/getting-started/tuning-thresholds.html +33 -0
- package/public/docs/user-guide/getting-started.html +5 -28
- package/public/docs/user-guide/mcp-tools/compare-runs.html +84 -0
- package/public/docs/user-guide/mcp-tools/diagnose-run.html +52 -0
- package/public/docs/user-guide/mcp-tools/evaluate-budgets.html +55 -0
- package/public/docs/user-guide/mcp-tools/get-finding-documentation.html +43 -0
- package/public/docs/user-guide/mcp-tools/get-finding-evidence.html +49 -0
- package/public/docs/user-guide/mcp-tools/get-reference-doc.html +34 -0
- package/public/docs/user-guide/mcp-tools/get-run-summary.html +44 -0
- package/public/docs/user-guide/mcp-tools/list-runs.html +41 -0
- package/public/docs/user-guide/mcp-tools.html +8 -140
- package/public/docs/user-guide/run-comparison/how-stages-are-matched.html +30 -0
- package/public/docs/user-guide/run-comparison.html +5 -5
- package/public/docs/user-guide/understanding-findings.html +5 -5
- package/public/export-template.html +44 -44
- package/public/index.html +4 -4
- package/vendor-core/analyzer.js +1 -1
- package/vendor-core/api-merge-writes.js +80 -0
- package/vendor-core/comparison-output.js +49 -0
- package/vendor-core/core-source-hash.txt +1 -1
- package/vendor-core/delta-commands.js +13 -0
- package/vendor-core/delta-log-paths.js +42 -0
- package/vendor-core/delta-targets.js +127 -0
- package/vendor-core/event-handlers.js +95 -9
- package/vendor-core/event-schemas.js +3 -1
- package/vendor-core/evidence-report.js +13 -4
- package/vendor-core/mcp-server-factory.js +13 -6
- package/vendor-core/mcp-tools.js +44 -35
- package/vendor-core/plan-graph-model.js +1 -1
- package/vendor-core/plan-summary.js +13 -10
- package/vendor-core/run-comparison.js +79 -120
- package/vendor-core/run-output.js +26 -0
- package/vendor-core/stage-alignment.js +262 -0
- package/vendor-core/stage-identity.js +112 -0
- package/vendor-core/types.js +5 -0
- package/vendor-core/write-target-args.js +48 -0
- package/vendor-core/write-targets.js +35 -52
- package/public/assets/index-DaC8A2FS.js +0 -13
- package/public/assets/parser-worker-CfDvPkwF.js +0 -122
- package/public/docs/assets/chunks/@localSearchIndexroot.r9imX8Aq.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_board-widgets.md.D0Kkk_r5.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_detector-contract.md.caA46z-u.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_detector-contract.md.caA46z-u.lean.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_drill-down.md.CdtlKHl-.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_drill-down.md.CdtlKHl-.lean.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.BNp14LtL.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.BNp14LtL.lean.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_index.md.B9yZieYL.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_overview.md.DBGo-bwn.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_state-and-history.md.w4lwybdh.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.CcvqxnZI.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.CcvqxnZI.lean.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_worker-protocol.md.CKeFttKj.js +0 -6
- package/public/docs/assets/contributor-guide_architecture_worker-protocol.md.CKeFttKj.lean.js +0 -1
- package/public/docs/assets/user-guide_alternative-log-retrieval.md.z-8npxNk.js +0 -7
- package/public/docs/assets/user-guide_alternative-log-retrieval.md.z-8npxNk.lean.js +0 -1
- package/public/docs/assets/user-guide_getting-started.md.BJW5oVhe.js +0 -24
- package/public/docs/assets/user-guide_getting-started.md.BJW5oVhe.lean.js +0 -1
- package/public/docs/assets/user-guide_mcp-tools.md.DcLiOa9t.js +0 -148
- package/public/docs/assets/user-guide_mcp-tools.md.DcLiOa9t.lean.js +0 -1
- package/public/docs/assets/user-guide_run-comparison.md.qin0a4IX.lean.js +0 -1
- package/public/docs/assets/user-guide_understanding-findings.md.DzReyCXb.js +0 -1
- /package/public/docs/assets/{contributor-guide_architecture_overview.md.DBGo-bwn.lean.js → contributor-guide_architecture_overview.md.C1sSvm9i.lean.js} +0 -0
- /package/public/docs/assets/{user-guide_understanding-findings.md.DzReyCXb.lean.js → user-guide_understanding-findings.md.Dqt2H_C1.lean.js} +0 -0
|
@@ -9,11 +9,11 @@
|
|
|
9
9
|
<link rel="preload stylesheet" href="/docs/assets/style.tmDhD3c8.css" as="style">
|
|
10
10
|
<link rel="preload stylesheet" href="/docs/vp-icons.css" as="style">
|
|
11
11
|
|
|
12
|
-
<script type="module" src="/docs/assets/app.
|
|
12
|
+
<script type="module" src="/docs/assets/app.D2GqAkhy.js"></script>
|
|
13
13
|
<link rel="preload" href="/docs/assets/inter-roman-latin.Di8DUHzh.woff2" as="font" type="font/woff2" crossorigin="">
|
|
14
|
-
<link rel="modulepreload" href="/docs/assets/chunks/theme.
|
|
14
|
+
<link rel="modulepreload" href="/docs/assets/chunks/theme.BuxVpb0H.js">
|
|
15
15
|
<link rel="modulepreload" href="/docs/assets/chunks/framework.DSg0KOwT.js">
|
|
16
|
-
<link rel="modulepreload" href="/docs/assets/contributor-guide_architecture_index.md.
|
|
16
|
+
<link rel="modulepreload" href="/docs/assets/contributor-guide_architecture_index.md.tymxf1Fh.lean.js">
|
|
17
17
|
<script>(()=>{const c=`${`[${["data","shuffle","product","bar"].join("-")}]`}, .VPNav, .VPLocalNav`,a=24,d=document.documentElement,l=t=>{const e=getComputedStyle(t);return e.display==="none"||t.getClientRects().length===0?0:e.position==="fixed"?t.getBoundingClientRect().bottom:e.position==="sticky"?(parseFloat(e.top)||0)+t.offsetHeight:0},u=()=>{try{return location.hash?document.getElementById(decodeURIComponent(location.hash.slice(1))):null}catch{return null}};let r=null;const s=()=>{const t=Array.from(document.querySelectorAll(c),l),e=Math.ceil(Math.max(0,...t))+a;if(e===r)return;const n=r;r=e,d.style.setProperty("--sf-anchor-offset",`${r}px`);const o=n===null?null:u();if(!o)return;const f=(parseFloat(getComputedStyle(o).paddingTop)||0)+2;Math.abs(o.getBoundingClientRect().top-n)<=f&&o.scrollIntoView()},i=()=>{const t=document.createElement("div");t.setAttribute("data-sf-anchor-offset",""),t.setAttribute("aria-hidden","true"),document.body.appendChild(t);const e=window.ResizeObserver?new ResizeObserver(s):null,n=()=>{e&&document.querySelectorAll(c).forEach(o=>e.observe(o)),s()};n(),new MutationObserver(n).observe(document.body,{childList:!0,subtree:!0}),window.addEventListener("resize",s),window.addEventListener("load",s)};document.readyState==="loading"?document.addEventListener("DOMContentLoaded",i,{once:!0}):i()})();</script>
|
|
18
18
|
<link rel="icon" type="image/svg+xml" href="/docs/favicon.svg">
|
|
19
19
|
<link rel="preconnect" href="https://fonts.googleapis.com">
|
|
@@ -23,8 +23,8 @@
|
|
|
23
23
|
<script id="check-mac-os">document.documentElement.classList.toggle("mac",/Mac|iPhone|iPod|iPad/i.test(navigator.platform));</script>
|
|
24
24
|
</head>
|
|
25
25
|
<body>
|
|
26
|
-
<div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Contributor Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/development-setup.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Development setup</p><!--]--></a><!----></div><!----></div><section class="VPSidebarItem level-1 collapsible collapsed is-link" data-v-b3fd67f8><div class="item" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/" data-v-b3fd67f8><!--[--><h3 class="text" data-v-b3fd67f8>Architecture</h3><!--]--></a><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-b3fd67f8><span class="vpi-chevron-right caret-icon" data-v-b3fd67f8></span></div></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/overview.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Overview</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/worker-protocol.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Worker protocol</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/state-and-history.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>State & history intake</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/detector-contract.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Detector contract</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/impact-estimation.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Impact estimation</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/widget-rendering.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Widget rendering</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/board-widgets.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Board widgets</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/drill-down.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Drill-down</p><!--]--></a><!----></div><!----></div><!--]--></div></section><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/testing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Testing & verification</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/contributing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Contributing</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_contributor-guide_architecture_" data-v-39a288b8><div><h1 id="architecture" tabindex="-1">Architecture <a class="header-anchor" href="#architecture" aria-label="Permalink to "Architecture""></a></h1><p>These pages cover one concern each: the two-actor overview, the worker protocol, state plus History Server intake, the detector contract, impact estimation, the fixed widget rendering order, the board's widgets, and the stage and plan drill-downs.</p><p>Where to start:</p><ul><li><a href="./overview.html#two-actors">Two actors</a>: the parser worker and the view, and why the split exists.</li><li><a href="./detector-contract.html#detector-contract">Detector contract</a>: the interface every bottleneck detector implements. Read this before adding a finding.</li><li><a href="./impact-estimation.html#impact-estimation">Impact estimation</a>: how findings get a wall-clock/resource waste estimate attached.</li><li><a href="./../testing.html#testing-layout">Testing layout</a>: where tests live and what each layer covers.</li><li><a href="./../contributing.html">Contributing</a>: where new docs go, changesets and the release flow.</li></ul></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/contributor-guide/development-setup.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>Development setup</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="/docs/contributor-guide/architecture/overview.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>Overview</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
|
|
27
|
-
<script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"D0Kkk_r5\",\"contributor-guide_architecture_detector-contract.md\":\"caA46z-u\",\"contributor-guide_architecture_drill-down.md\":\"CdtlKHl-\",\"contributor-guide_architecture_impact-estimation.md\":\"BNp14LtL\",\"contributor-guide_architecture_index.md\":\"B9yZieYL\",\"contributor-guide_architecture_overview.md\":\"DBGo-bwn\",\"contributor-guide_architecture_state-and-history.md\":\"w4lwybdh\",\"contributor-guide_architecture_widget-rendering.md\":\"CcvqxnZI\",\"contributor-guide_architecture_worker-protocol.md\":\"CKeFttKj\",\"contributor-guide_contributing.md\":\"CQMvGwFt\",\"contributor-guide_development-setup.md\":\"D9bu4j_c\",\"contributor-guide_testing.md\":\"By6XGR9a\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"D08Uaod7\",\"tuning-reference_aqe.md\":\"C1Ae6A8u\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"6wjKuJNE\",\"tuning-reference_bottleneck-cold-start.md\":\"BVQbsuHP\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"DtaYnaiF\",\"tuning-reference_bottleneck-failures.md\":\"Z1Tdmdd3\",\"tuning-reference_bottleneck-gc.md\":\"BCnXLf1n\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"CcvQj2g9\",\"tuning-reference_bottleneck-memory-utilization.md\":\"B-C0-8h-\",\"tuning-reference_bottleneck-retry-waste.md\":\"DCnhWiqj\",\"tuning-reference_bottleneck-shuffle.md\":\"B5OMJCd3\",\"tuning-reference_bottleneck-skew.md\":\"Cvo6NQm_\",\"tuning-reference_bottleneck-slow-host.md\":\"ja5qr3wq\",\"tuning-reference_bottleneck-small-files.md\":\"Crjtb0i2\",\"tuning-reference_bottleneck-spill.md\":\"Bl8KtSRg\",\"tuning-reference_bottleneck-straggler.md\":\"B0VN8reT\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"DQCftGhu\",\"tuning-reference_bottleneck-utilization.md\":\"BXDl30f9\",\"tuning-reference_caching.md\":\"CcaiMn_7\",\"tuning-reference_cluster-config.md\":\"DbWG1yZq\",\"tuning-reference_config.md\":\"YUqSpwFB\",\"tuning-reference_data-formats.md\":\"C5DX5xL4\",\"tuning-reference_index.md\":\"ZVIpQ6Dk\",\"tuning-reference_intro.md\":\"BSSaDlqx\",\"tuning-reference_joins.md\":\"BVhNCDnE\",\"tuning-reference_memory-model.md\":\"CfjFdgbl\",\"tuning-reference_metrics.md\":\"C64VNEL4\",\"tuning-reference_partitioning.md\":\"CtLK0xsS\",\"tuning-reference_pyspark.md\":\"D-92XgF8\",\"tuning-reference_shuffle.md\":\"Iq2v1oHJ\",\"tuning-reference_spark-architecture.md\":\"_qwnE_Sc\",\"tuning-reference_table-formats.md\":\"BSYEK-eN\",\"user-guide_alternative-log-retrieval.md\":\"z-8npxNk\",\"user-guide_getting-started.md\":\"BJW5oVhe\",\"user-guide_mcp-tools.md\":\"DcLiOa9t\",\"user-guide_run-comparison.md\":\"qin0a4IX\",\"user-guide_understanding-findings.md\":\"DzReyCXb\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Finding your event log\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":{\"selector\":\"[data-sf-anchor-offset]\",\"padding\":0},\"cleanUrls\":false}");</script>
|
|
26
|
+
<div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Contributor Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/development-setup.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Development setup</p><!--]--></a><!----></div><!----></div><section class="VPSidebarItem level-1 collapsible collapsed is-link" data-v-b3fd67f8><div class="item" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/" data-v-b3fd67f8><!--[--><h3 class="text" data-v-b3fd67f8>Architecture</h3><!--]--></a><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-b3fd67f8><span class="vpi-chevron-right caret-icon" data-v-b3fd67f8></span></div></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/overview.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Overview</p><!--]--></a><!----></div><!----></div><section class="VPSidebarItem level-2 collapsible collapsed is-link" data-v-b3fd67f8><div class="item" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/worker-protocol.html" data-v-b3fd67f8><!--[--><h4 class="text" data-v-b3fd67f8>Worker protocol</h4><!--]--></a><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-b3fd67f8><span class="vpi-chevron-right caret-icon" data-v-b3fd67f8></span></div></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-3 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/worker-protocol/evidence-availability.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Evidence availability</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-3 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/worker-protocol/evidence-report.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Evidence report and finding identity</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-3 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/worker-protocol/cli-and-mcp.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>CLI and MCP server</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-3 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/worker-protocol/core-and-event-validation.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Core and event validation</p><!--]--></a><!----></div><!----></div><!--]--></div></section><section class="VPSidebarItem level-2 collapsible collapsed is-link" data-v-b3fd67f8><div class="item" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/state-and-history.html" data-v-b3fd67f8><!--[--><h4 class="text" data-v-b3fd67f8>State & history intake</h4><!--]--></a><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-b3fd67f8><span class="vpi-chevron-right caret-icon" data-v-b3fd67f8></span></div></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-3 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/state-and-history/run-comparison-internals.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Run comparison internals</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-3 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/state-and-history/history-server-intake.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>History Server intake</p><!--]--></a><!----></div><!----></div><!--]--></div></section><section class="VPSidebarItem level-2 collapsible collapsed is-link" data-v-b3fd67f8><div class="item" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/detector-contract.html" data-v-b3fd67f8><!--[--><h4 class="text" data-v-b3fd67f8>Detector contract</h4><!--]--></a><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-b3fd67f8><span class="vpi-chevron-right caret-icon" data-v-b3fd67f8></span></div></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-3 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/detector-contract/plan-attribution.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Plan attribution</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-3 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/detector-contract/threshold-tables.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Threshold tables</p><!--]--></a><!----></div><!----></div><!--]--></div></section><section class="VPSidebarItem level-2 collapsible collapsed is-link" data-v-b3fd67f8><div class="item" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/impact-estimation.html" data-v-b3fd67f8><!--[--><h4 class="text" data-v-b3fd67f8>Impact estimation</h4><!--]--></a><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-b3fd67f8><span class="vpi-chevron-right caret-icon" data-v-b3fd67f8></span></div></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-3 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/impact-estimation/stage-union-rollup.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Stage union rollup</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-3 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/impact-estimation/caveats-tuning-and-coverage.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Caveats, tuning and coverage</p><!--]--></a><!----></div><!----></div><!--]--></div></section><section class="VPSidebarItem level-2 collapsible collapsed is-link" data-v-b3fd67f8><div class="item" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/widget-rendering.html" data-v-b3fd67f8><!--[--><h4 class="text" data-v-b3fd67f8>Widget rendering</h4><!--]--></a><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-b3fd67f8><span class="vpi-chevron-right caret-icon" data-v-b3fd67f8></span></div></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-3 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/widget-rendering/findings-tab.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Findings tab</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-3 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/widget-rendering/list-sort-mode.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>List sort mode</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-3 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/widget-rendering/row-expand-contract.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Row/expand contract</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-3 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/widget-rendering/reference-section.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>ReferenceSection</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-3 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/widget-rendering/investigation-routing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Investigation routing</p><!--]--></a><!----></div><!----></div><!--]--></div></section><section class="VPSidebarItem level-2 collapsible collapsed is-link" data-v-b3fd67f8><div class="item" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/board-widgets.html" data-v-b3fd67f8><!--[--><h4 class="text" data-v-b3fd67f8>Board widgets</h4><!--]--></a><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-b3fd67f8><span class="vpi-chevron-right caret-icon" data-v-b3fd67f8></span></div></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-3 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/board-widgets/plan-advisor.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Plan Advisor</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-3 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/board-widgets/caching-and-memory.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Caching and memory widgets</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-3 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/board-widgets/visual-system-and-templating.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Visual system and templating</p><!--]--></a><!----></div><!----></div><!--]--></div></section><section class="VPSidebarItem level-2 collapsible collapsed is-link" data-v-b3fd67f8><div class="item" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/drill-down.html" data-v-b3fd67f8><!--[--><h4 class="text" data-v-b3fd67f8>Drill-down</h4><!--]--></a><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-b3fd67f8><span class="vpi-chevron-right caret-icon" data-v-b3fd67f8></span></div></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-3 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/drill-down/plan-graph.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Plan graph</p><!--]--></a><!----></div><!----></div><!--]--></div></section><!--]--></div></section><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/testing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Testing & verification</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/contributing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Contributing</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_contributor-guide_architecture_" data-v-39a288b8><div><h1 id="architecture" tabindex="-1">Architecture <a class="header-anchor" href="#architecture" aria-label="Permalink to "Architecture""></a></h1><p>Each page covers one concern, and the larger ones split into child pages that the sidebar nests beneath them. Start with the two-actor overview, then the worker protocol, state and History Server intake, the detector contract, impact estimation, widget rendering, the board's widgets and the drill-downs.</p><p>Where to start:</p><ul><li><a href="./overview.html#two-actors">Two actors</a>: the parser worker and the view, and why the split exists.</li><li><a href="./detector-contract.html#detector-contract">Detector contract</a>: the interface every bottleneck detector implements. Read this before adding a finding.</li><li><a href="./impact-estimation.html#impact-estimation">Impact estimation</a>: how findings get a wall-clock/resource waste estimate attached.</li><li><a href="./../testing.html#testing-layout">Testing layout</a>: where tests live and what each layer covers.</li><li><a href="./../contributing.html">Contributing</a>: where new docs go, changesets and the release flow.</li></ul></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/contributor-guide/development-setup.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>Development setup</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="/docs/contributor-guide/architecture/overview.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>Overview</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
|
|
27
|
+
<script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"YQcj5EI4\",\"contributor-guide_architecture_board-widgets_caching-and-memory.md\":\"8bH1HnZp\",\"contributor-guide_architecture_board-widgets_plan-advisor.md\":\"RlzvUmNm\",\"contributor-guide_architecture_board-widgets_visual-system-and-templating.md\":\"DYgUpIJB\",\"contributor-guide_architecture_detector-contract.md\":\"wj6Et6UD\",\"contributor-guide_architecture_detector-contract_plan-attribution.md\":\"yNex4bys\",\"contributor-guide_architecture_detector-contract_threshold-tables.md\":\"fkPBXsU2\",\"contributor-guide_architecture_drill-down.md\":\"CqZPrdo9\",\"contributor-guide_architecture_drill-down_plan-graph.md\":\"DwOozIEs\",\"contributor-guide_architecture_impact-estimation.md\":\"CZv_ZGOp\",\"contributor-guide_architecture_impact-estimation_caveats-tuning-and-coverage.md\":\"BUucRc0A\",\"contributor-guide_architecture_impact-estimation_stage-union-rollup.md\":\"DdCGtAGu\",\"contributor-guide_architecture_index.md\":\"tymxf1Fh\",\"contributor-guide_architecture_overview.md\":\"C1sSvm9i\",\"contributor-guide_architecture_state-and-history.md\":\"DyCBdjuM\",\"contributor-guide_architecture_state-and-history_history-server-intake.md\":\"C298hrHZ\",\"contributor-guide_architecture_state-and-history_run-comparison-internals.md\":\"uuAKpY0Y\",\"contributor-guide_architecture_widget-rendering.md\":\"BOZ1azXG\",\"contributor-guide_architecture_widget-rendering_findings-tab.md\":\"BQXycK6G\",\"contributor-guide_architecture_widget-rendering_investigation-routing.md\":\"BGdiyElu\",\"contributor-guide_architecture_widget-rendering_list-sort-mode.md\":\"BuHKgH0C\",\"contributor-guide_architecture_widget-rendering_reference-section.md\":\"DWqC80bN\",\"contributor-guide_architecture_widget-rendering_row-expand-contract.md\":\"ZT89u9f1\",\"contributor-guide_architecture_worker-protocol.md\":\"BAnhi2-d\",\"contributor-guide_architecture_worker-protocol_cli-and-mcp.md\":\"Dk08VS7n\",\"contributor-guide_architecture_worker-protocol_core-and-event-validation.md\":\"DSVFPpWb\",\"contributor-guide_architecture_worker-protocol_evidence-availability.md\":\"CjimIDBY\",\"contributor-guide_architecture_worker-protocol_evidence-report.md\":\"DVGZOWvW\",\"contributor-guide_contributing.md\":\"CQMvGwFt\",\"contributor-guide_development-setup.md\":\"D9bu4j_c\",\"contributor-guide_testing.md\":\"By6XGR9a\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"D08Uaod7\",\"tuning-reference_aqe.md\":\"C1Ae6A8u\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"6wjKuJNE\",\"tuning-reference_bottleneck-cold-start.md\":\"BVQbsuHP\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"DtaYnaiF\",\"tuning-reference_bottleneck-failures.md\":\"Z1Tdmdd3\",\"tuning-reference_bottleneck-gc.md\":\"BCnXLf1n\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"CcvQj2g9\",\"tuning-reference_bottleneck-memory-utilization.md\":\"B-C0-8h-\",\"tuning-reference_bottleneck-retry-waste.md\":\"DCnhWiqj\",\"tuning-reference_bottleneck-shuffle.md\":\"B5OMJCd3\",\"tuning-reference_bottleneck-skew.md\":\"Cvo6NQm_\",\"tuning-reference_bottleneck-slow-host.md\":\"ja5qr3wq\",\"tuning-reference_bottleneck-small-files.md\":\"Crjtb0i2\",\"tuning-reference_bottleneck-spill.md\":\"Bl8KtSRg\",\"tuning-reference_bottleneck-straggler.md\":\"B0VN8reT\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"DQCftGhu\",\"tuning-reference_bottleneck-utilization.md\":\"BXDl30f9\",\"tuning-reference_caching.md\":\"CcaiMn_7\",\"tuning-reference_cluster-config.md\":\"DbWG1yZq\",\"tuning-reference_config.md\":\"YUqSpwFB\",\"tuning-reference_data-formats.md\":\"C5DX5xL4\",\"tuning-reference_index.md\":\"ZVIpQ6Dk\",\"tuning-reference_intro.md\":\"BSSaDlqx\",\"tuning-reference_joins.md\":\"BVhNCDnE\",\"tuning-reference_memory-model.md\":\"CfjFdgbl\",\"tuning-reference_metrics.md\":\"C64VNEL4\",\"tuning-reference_partitioning.md\":\"CtLK0xsS\",\"tuning-reference_pyspark.md\":\"D-92XgF8\",\"tuning-reference_shuffle.md\":\"Iq2v1oHJ\",\"tuning-reference_spark-architecture.md\":\"_qwnE_Sc\",\"tuning-reference_table-formats.md\":\"BSYEK-eN\",\"user-guide_alternative-log-retrieval.md\":\"CJpYvHGo\",\"user-guide_alternative-log-retrieval_managed-platforms.md\":\"DvDwHNtC\",\"user-guide_alternative-log-retrieval_ssh-bastion.md\":\"Gszzlplv\",\"user-guide_getting-started.md\":\"DYHHHkQR\",\"user-guide_getting-started_ci-and-automation.md\":\"DpnfKz3x\",\"user-guide_getting-started_reading-the-dashboard.md\":\"Obf3W3__\",\"user-guide_getting-started_regression-metric-keys.md\":\"B_gzr2Mw\",\"user-guide_getting-started_tuning-thresholds.md\":\"B_MyjyZm\",\"user-guide_mcp-tools.md\":\"C65SjWUL\",\"user-guide_mcp-tools_compare-runs.md\":\"CPowlb2q\",\"user-guide_mcp-tools_diagnose-run.md\":\"BP1vnHA5\",\"user-guide_mcp-tools_evaluate-budgets.md\":\"Dq5Azk8G\",\"user-guide_mcp-tools_get-finding-documentation.md\":\"BtTTHNV6\",\"user-guide_mcp-tools_get-finding-evidence.md\":\"rKyu4wts\",\"user-guide_mcp-tools_get-reference-doc.md\":\"Yac4tH3u\",\"user-guide_mcp-tools_get-run-summary.md\":\"DpxZ4ZNp\",\"user-guide_mcp-tools_list-runs.md\":\"DpQPeCM3\",\"user-guide_run-comparison.md\":\"B6aNtDYF\",\"user-guide_run-comparison_how-stages-are-matched.md\":\"XBzQ1oEU\",\"user-guide_understanding-findings.md\":\"Dqt2H_C1\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\",\"collapsed\":true,\"items\":[{\"text\":\"Reading the dashboard\",\"link\":\"/user-guide/getting-started/reading-the-dashboard\"},{\"text\":\"CI and automation\",\"link\":\"/user-guide/getting-started/ci-and-automation\"},{\"text\":\"Tuning detector thresholds\",\"link\":\"/user-guide/getting-started/tuning-thresholds\"},{\"text\":\"Regression metric keys\",\"link\":\"/user-guide/getting-started/regression-metric-keys\"}]},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\",\"collapsed\":true,\"items\":[{\"text\":\"How stages are matched\",\"link\":\"/user-guide/run-comparison/how-stages-are-matched\"}]},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\",\"collapsed\":true,\"items\":[{\"text\":\"diagnose_run\",\"link\":\"/user-guide/mcp-tools/diagnose-run\"},{\"text\":\"get_run_summary\",\"link\":\"/user-guide/mcp-tools/get-run-summary\"},{\"text\":\"evaluate_budgets\",\"link\":\"/user-guide/mcp-tools/evaluate-budgets\"},{\"text\":\"compare_runs\",\"link\":\"/user-guide/mcp-tools/compare-runs\"},{\"text\":\"get_finding_evidence\",\"link\":\"/user-guide/mcp-tools/get-finding-evidence\"},{\"text\":\"get_finding_documentation\",\"link\":\"/user-guide/mcp-tools/get-finding-documentation\"},{\"text\":\"get_reference_doc\",\"link\":\"/user-guide/mcp-tools/get-reference-doc\"},{\"text\":\"list_runs\",\"link\":\"/user-guide/mcp-tools/list-runs\"}]},{\"text\":\"Finding your event log\",\"link\":\"/user-guide/alternative-log-retrieval\",\"collapsed\":true,\"items\":[{\"text\":\"Managed platforms\",\"link\":\"/user-guide/alternative-log-retrieval/managed-platforms\"},{\"text\":\"Behind an SSH bastion\",\"link\":\"/user-guide/alternative-log-retrieval/ssh-bastion\"}]}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\",\"collapsed\":true,\"items\":[{\"text\":\"Evidence availability\",\"link\":\"/contributor-guide/architecture/worker-protocol/evidence-availability\"},{\"text\":\"Evidence report and finding identity\",\"link\":\"/contributor-guide/architecture/worker-protocol/evidence-report\"},{\"text\":\"CLI and MCP server\",\"link\":\"/contributor-guide/architecture/worker-protocol/cli-and-mcp\"},{\"text\":\"Core and event validation\",\"link\":\"/contributor-guide/architecture/worker-protocol/core-and-event-validation\"}]},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\",\"collapsed\":true,\"items\":[{\"text\":\"Run comparison internals\",\"link\":\"/contributor-guide/architecture/state-and-history/run-comparison-internals\"},{\"text\":\"History Server intake\",\"link\":\"/contributor-guide/architecture/state-and-history/history-server-intake\"}]},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\",\"collapsed\":true,\"items\":[{\"text\":\"Plan attribution\",\"link\":\"/contributor-guide/architecture/detector-contract/plan-attribution\"},{\"text\":\"Threshold tables\",\"link\":\"/contributor-guide/architecture/detector-contract/threshold-tables\"}]},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\",\"collapsed\":true,\"items\":[{\"text\":\"Stage union rollup\",\"link\":\"/contributor-guide/architecture/impact-estimation/stage-union-rollup\"},{\"text\":\"Caveats, tuning and coverage\",\"link\":\"/contributor-guide/architecture/impact-estimation/caveats-tuning-and-coverage\"}]},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\",\"collapsed\":true,\"items\":[{\"text\":\"Findings tab\",\"link\":\"/contributor-guide/architecture/widget-rendering/findings-tab\"},{\"text\":\"List sort mode\",\"link\":\"/contributor-guide/architecture/widget-rendering/list-sort-mode\"},{\"text\":\"Row/expand contract\",\"link\":\"/contributor-guide/architecture/widget-rendering/row-expand-contract\"},{\"text\":\"ReferenceSection\",\"link\":\"/contributor-guide/architecture/widget-rendering/reference-section\"},{\"text\":\"Investigation routing\",\"link\":\"/contributor-guide/architecture/widget-rendering/investigation-routing\"}]},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\",\"collapsed\":true,\"items\":[{\"text\":\"Plan Advisor\",\"link\":\"/contributor-guide/architecture/board-widgets/plan-advisor\"},{\"text\":\"Caching and memory widgets\",\"link\":\"/contributor-guide/architecture/board-widgets/caching-and-memory\"},{\"text\":\"Visual system and templating\",\"link\":\"/contributor-guide/architecture/board-widgets/visual-system-and-templating\"}]},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\",\"collapsed\":true,\"items\":[{\"text\":\"Plan graph\",\"link\":\"/contributor-guide/architecture/drill-down/plan-graph\"}]}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":{\"selector\":\"[data-sf-anchor-offset]\",\"padding\":0},\"cleanUrls\":false}");</script>
|
|
28
28
|
|
|
29
29
|
</body>
|
|
30
30
|
</html>
|
|
@@ -9,11 +9,11 @@
|
|
|
9
9
|
<link rel="preload stylesheet" href="/docs/assets/style.tmDhD3c8.css" as="style">
|
|
10
10
|
<link rel="preload stylesheet" href="/docs/vp-icons.css" as="style">
|
|
11
11
|
|
|
12
|
-
<script type="module" src="/docs/assets/app.
|
|
12
|
+
<script type="module" src="/docs/assets/app.D2GqAkhy.js"></script>
|
|
13
13
|
<link rel="preload" href="/docs/assets/inter-roman-latin.Di8DUHzh.woff2" as="font" type="font/woff2" crossorigin="">
|
|
14
|
-
<link rel="modulepreload" href="/docs/assets/chunks/theme.
|
|
14
|
+
<link rel="modulepreload" href="/docs/assets/chunks/theme.BuxVpb0H.js">
|
|
15
15
|
<link rel="modulepreload" href="/docs/assets/chunks/framework.DSg0KOwT.js">
|
|
16
|
-
<link rel="modulepreload" href="/docs/assets/contributor-guide_architecture_overview.md.
|
|
16
|
+
<link rel="modulepreload" href="/docs/assets/contributor-guide_architecture_overview.md.C1sSvm9i.lean.js">
|
|
17
17
|
<script>(()=>{const c=`${`[${["data","shuffle","product","bar"].join("-")}]`}, .VPNav, .VPLocalNav`,a=24,d=document.documentElement,l=t=>{const e=getComputedStyle(t);return e.display==="none"||t.getClientRects().length===0?0:e.position==="fixed"?t.getBoundingClientRect().bottom:e.position==="sticky"?(parseFloat(e.top)||0)+t.offsetHeight:0},u=()=>{try{return location.hash?document.getElementById(decodeURIComponent(location.hash.slice(1))):null}catch{return null}};let r=null;const s=()=>{const t=Array.from(document.querySelectorAll(c),l),e=Math.ceil(Math.max(0,...t))+a;if(e===r)return;const n=r;r=e,d.style.setProperty("--sf-anchor-offset",`${r}px`);const o=n===null?null:u();if(!o)return;const f=(parseFloat(getComputedStyle(o).paddingTop)||0)+2;Math.abs(o.getBoundingClientRect().top-n)<=f&&o.scrollIntoView()},i=()=>{const t=document.createElement("div");t.setAttribute("data-sf-anchor-offset",""),t.setAttribute("aria-hidden","true"),document.body.appendChild(t);const e=window.ResizeObserver?new ResizeObserver(s):null,n=()=>{e&&document.querySelectorAll(c).forEach(o=>e.observe(o)),s()};n(),new MutationObserver(n).observe(document.body,{childList:!0,subtree:!0}),window.addEventListener("resize",s),window.addEventListener("load",s)};document.readyState==="loading"?document.addEventListener("DOMContentLoaded",i,{once:!0}):i()})();</script>
|
|
18
18
|
<link rel="icon" type="image/svg+xml" href="/docs/favicon.svg">
|
|
19
19
|
<link rel="preconnect" href="https://fonts.googleapis.com">
|
|
@@ -23,8 +23,8 @@
|
|
|
23
23
|
<script id="check-mac-os">document.documentElement.classList.toggle("mac",/Mac|iPhone|iPod|iPad/i.test(navigator.platform));</script>
|
|
24
24
|
</head>
|
|
25
25
|
<body>
|
|
26
|
-
<div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Contributor Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/development-setup.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Development setup</p><!--]--></a><!----></div><!----></div><section class="VPSidebarItem level-1 collapsible collapsed is-link has-active" data-v-b3fd67f8><div class="item" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/" data-v-b3fd67f8><!--[--><h3 class="text" data-v-b3fd67f8>Architecture</h3><!--]--></a><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-b3fd67f8><span class="vpi-chevron-right caret-icon" data-v-b3fd67f8></span></div></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/overview.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Overview</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/worker-protocol.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Worker protocol</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/state-and-history.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>State & history intake</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/detector-contract.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Detector contract</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/impact-estimation.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Impact estimation</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/widget-rendering.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Widget rendering</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/board-widgets.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Board widgets</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/drill-down.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Drill-down</p><!--]--></a><!----></div><!----></div><!--]--></div></section><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/testing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Testing & verification</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/contributing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Contributing</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_contributor-guide_architecture_overview" data-v-39a288b8><div><h1 id="architecture-overview" tabindex="-1">Architecture overview <a class="header-anchor" href="#architecture-overview" aria-label="Permalink to "Architecture overview""></a></h1><h2 id="core-invariant" tabindex="-1">Core invariant <a class="header-anchor" href="#core-invariant" aria-label="Permalink to "Core invariant""></a></h2><p>Task data must never live on the main thread. A Spark job can emit millions of <code>SparkListenerTaskEnd</code> events (240 MB+ of metrics); parsing them on the UI thread freezes Chrome. The design below enforces that in both deploy modes: the static zero-backend app, and the optional local-server mode in <code>packages/server/</code>, which also proxies Spark History Server fetches to sidestep CORS.</p><h2 id="two-actors" tabindex="-1">Two actors <a class="header-anchor" href="#two-actors" aria-label="Permalink to "Two actors""></a></h2><h3 id="main-thread" tabindex="-1">Main thread <a class="header-anchor" href="#main-thread" aria-label="Permalink to "Main thread""></a></h3><p>Owns the DOM (React) and a summary <code>AppModel</code> (jobs, stages, sql, executors, no raw tasks), held in a Zustand store (<code>src/store/store.ts</code>).</p><p><code>src/store/useIngest.ts</code> is the composition root. It owns the ingest client's lifecycle (<code>createIngestClient</code>, <code>@sparkforensics/core/ingest.ts</code>) and wires <code>model-assembler.ts</code>'s <code>createModelCallbacks</code> (worker message → <code>AppModel</code>) into store updates: <code>onProgress</code> → <code>parse</code>; <code>onDone</code> → derives <code>evidenceAvailability</code> from the final normalized model plus <code>done.skippedLines</code>, runs <code>analyzer.ts</code>, sets <code>catalog</code>, and prefetches flagged-stage task data.</p><p><code>src/App.tsx</code> routes on store state: a two-run compare load, the plan-graph route and the run comparison take precedence; otherwise idle/error → <code>CompareLanding</code> (one or two <code>DropZone</code>s), parsing → a live progress readout (an SHS parse keeps its intake form instead), ready → the lazy-loaded <code>Dashboard</code>. <code>src/view/Dashboard.tsx</code> renders widgets from <code>src/view/detector-registry.tsx</code>'s <code>REGISTRY</code> (see <a href="./widget-rendering.html">Widget rendering</a>).</p><h3 id="worker-web-worker-packages-core" tabindex="-1">Worker (Web Worker, <code>packages/core</code>) <a class="header-anchor" href="#worker-web-worker-packages-core" aria-label="Permalink to "Worker (Web Worker, `packages/core`)""></a></h3><p>Owns the file and <code>taskStore: Map<stageId, Float64Array></code>, keyed by 8-field stride <code>[duration, gcTime, memSpilled, diskSpilled, shuffleRead, shuffleWrite, launchTime, finishTime]</code>. Six modules, one of them the rolling-log helper:</p><ul><li><code>packages/core/src/stage-quantiles.ts</code>, a pure leaf, exporting <code>finalizeStage</code>, <code>computeFieldQuantiles</code>, <code>computeDurationQuantiles</code>, <code>classifySpill</code>, <code>FIELDS</code>, and <code>TASK_FIELD_NAMES</code> for the task-field packed-array layout.</li><li><code>packages/core/src/event-handlers.ts</code>, holding <code>processEvent</code> and the per-event handler functions, <code>accumulateTask</code>, <code>createState</code>, <code>dispatchLine</code>, <code>buildChunkDecoder</code>, <code>emitParseCompletion</code>, <code>collectStageExecutorMetrics</code>, <code>collectLateSpeculationWaste</code>.</li><li><code>packages/core/src/shs-fetch.ts</code>, the SHS zip-fetch/decompress path: <code>runParseFromUrl</code>, <code>decodeShsArchive</code>, <code>parseZipArchive</code>, <code>sniffCodec</code>.</li><li><code>packages/core/src/rolling-log-reassembly.ts</code>, import-free on purpose so the main-thread <code>DropZone</code> can use it: <code>naturalCompare</code>, <code>reassembleRollingEntries</code>.</li><li><code>packages/core/src/zip-archive.ts</code>, the random-access zip reader both zip paths use: <code>isZip</code>, <code>listZipEntries</code>, <code>streamZipEntry</code>.</li><li><code>packages/core/src/parser-worker.ts</code>, a thin entrypoint (<code>streamFile</code>, <code>runParse</code>, <code>runParseFiles</code>, the <code>isWorker</code>/<code>self.onmessage</code> bus) that re-exports the public parts of <code>event-handlers.ts</code>, <code>stage-quantiles.ts</code>, <code>shs-fetch.ts</code> and <code>rolling-log-reassembly.ts</code>. It is the only piece needing the File/Blob streaming API.</li></ul><p>Dropped zstd files also run a nested decompress worker, <code>packages/core/src/zstd-worker.ts</code>, driven from the parse worker by <code>packages/core/src/zstd-worker-client.ts</code> (see <a href="./worker-protocol.html#decompress-worker">Decompress worker</a>).</p><h2 id="streaming" tabindex="-1">Streaming <a class="header-anchor" href="#streaming" aria-label="Permalink to "Streaming""></a></h2><p>The worker reads the <code>File</code> in 512 KB slices (smaller for small files, so every file gets at least 100 progress steps) via <code>file.slice(...).arrayBuffer()</code>, decodes each decompressed chunk with one streaming <code>TextDecoder</code>, splits on <code>\n</code>, JSON-parses and dispatches per event type. Quantiles (P50/P95/max) and spill classification (<code>skew | volume | unclassified</code>) are computed at <code>SparkListenerStageCompleted</code> time, before posting the stage's <code>{ type: 'stage' }</code> message to main.</p><p>When <code>spark.eventLog.logStageExecutorMetrics=true</code> (default <code>false</code>), <code>SparkListenerStageExecutorMetrics</code> events populate <code>stage.executorMetrics: Map<execId, {...}></code> with the 23 raw peak-memory/GC fields verbatim (camelCased), consumed by the <code>memoryUtilization</code> detector's per-executor memory bands (see <a href="./board-widgets.html">Memory Utilization</a>).</p><p>These events can arrive <em>after</em> <code>SparkListenerStageCompleted</code> for the same stage, so the per-stage <code>stage</code> message posted at completion time never carries them. Instead, <code>collectStageExecutorMetrics(state)</code> walks every stage once more just before <code>done</code>, and the worker re-posts a single <code>stageExecutorMetrics</code> message (<code>Map<stageId, Map<execId, metrics>></code>, empty when the log has no such data). Main-thread consumers get every stage's metrics without a per-stage race.</p><p>A TaskEnd that arrives after its stage's StageCompleted is otherwise dropped, since the finalized stage already posted its stats. The exception is the losing copy of a speculative race: Spark kills it only once the stage finishes ("Stage cancelled: Stage finished"), so its TaskEnd is normally late. <code>accumulateTask</code> adds its time to the stage's <code>speculationWasteMs</code> and <code>speculationWastedAttempts</code> and nothing else, pairing it with a speculative winner kept for that purpose. <code>collectLateSpeculationWaste(state)</code> re-posts the updated totals of those stages once, just before <code>done</code>, as a <code>stageSpeculationWaste</code> message (<code>Map<stageId, { speculationWasteMs, speculationWastedAttempts }></code>, empty when no speculative attempt ended late). Any other late task of a failed attempt (a zombie or killed task) leaves the stage's own figures alone and adds to its <code>lateAttemptWork</code>, which only the CLI's <code>metrics</code> block reads; <code>collectLateAttemptWork(state)</code> re-posts it as a <code>stageLateAttemptWork</code> message (<code>Map<stageId, StageAttemptTotals></code>). <code>emitParseCompletion</code> sends the speculation totals after <code>runAggregates</code> (whole-run aggregates computed from <code>taskStore</code>) and before the final <code>app</code> and <code>done</code> messages.</p><p>Compressed logs are decompressed as they stream. <code>sniffCodec</code> reads the leading magic bytes: gzip (<code>1f 8b</code>), Zstandard (<code>28 b5 2f fd</code>, Spark's <code>spark.io.compression.codec=zstd</code>), Spark's custom <code>LZ4Block</code> framing, or Spark's Snappy framing (<code>org.xerial.snappy</code>'s <code>\x82SNAPPY\0</code> header, <code>spark.io.compression.codec=snappy</code>). Only a zip entry falls back to its name's suffix (see <a href="#zip-archives">Zip archives</a>). Both the dropped-file path and the SHS-fetch path stream block-by-block (fflate <code>Gunzip</code> / fzstd <code>Decompress</code> / the LZ4Block decoder / the Snappy block decoder) to keep one decompressed chunk live at a time. In the browser, a dropped zstd file decompresses in a second worker while the parse worker parses earlier output, with a three-slice window bounding what is queued between them (see <a href="./worker-protocol.html#decompress-worker">Decompress worker</a>). The Node CLI/MCP paths (<code>collectRun</code> for local files, <code>shs-load.ts</code> for SHS archives) swap fzstd for Node's native zlib zstd where the running Node has it (22.15+/23.8+), through the <code>zstdDecoder</code> option of <code>runParse</code>/<code>decodeShsArchive</code>: <code>packages/core/src/cli/native-zstd.ts</code> decompresses one frame at a time, since Node's own decoders stop after a stream's first frame and Spark writes thousands of small ones. A frame declaring more than 64 MiB of content, one still incomplete after 64 MiB compressed, a malformed one, or a truncated tail goes to fzstd instead. Spark's frames declare no content size, so each one's output is capped at 64 MiB as it decodes: past that, the inline decoder hands the frame to fzstd and the threaded one streams it, pausing while 64 MiB wait to be parsed. For local files (<code>nodeParseCodecs</code>, <code>createThreadedZstdDecoder</code>), frames of 64 KB or more compressed decompress on libuv's threadpool, up to 4 at a time, while the main thread parses earlier output; <code>streamFile</code> awaits each <code>push</code>, and chunks still arrive in stream order. SHS archives fetched from a History Server keep the inline decoder (<code>nodeArchiveCodecs</code>).</p><p>Rolling <code>eventlog_v2_*</code> directories (Spark's multi-file event-log format, <code>events_<index>_<appId>(.codec)</code> files plus a zero-byte <code>appstatus_*</code> completion marker and, periodically, one <code>*.compact</code> merge file) are reassembled into parse order by <code>reassembleRollingEntries(names)</code>: drop the marker, drop every non-compact <code>events_*</code> file at or below the most recent <code>.compact</code> file's index, sort the rest numerically, and throw if an index is missing. The zip path (<code>parseZipArchive</code>) calls it on member names; for a dropped folder, <code>DropZone</code> (and the CLI's <code>collectRun</code>) call it before handing the ordered files to <code>runParseFiles</code>.</p><p><code>runParseFiles</code> streams each file in the resulting order through the same chunked codec dispatch as <code>runParse</code>, with a fresh decompressor per file, since codecs never span a roll boundary. The NDJSON line-decoder is the exception: it stays alive across all files, because nothing guarantees a roll boundary lands on a line boundary.</p><h2 id="zip-archives" tabindex="-1">Zip archives <a class="header-anchor" href="#zip-archives" aria-label="Permalink to "Zip archives""></a></h2><p>A History Server download is a zip, whether fetched by <code>runParseFromUrl</code> or dropped as a file (the Spark UI's download link, or <code>GET /api/v1/applications/<appId>/logs</code> saved to disk). Both go through <code>parseZipArchive</code> (<code>packages/core/src/shs-fetch.ts</code>): <code>runParse</code> checks for the zip signature before its codec dispatch, and <code>decodeShsArchive</code> wraps the downloaded bytes in the same <code>slice()</code> interface a <code>File</code> has. The two differ only in how they report failure (a plain <code>error</code> message for a dropped file, <code>{ source: 'shs', code: 'invalid-event-log' }</code> for a fetch).</p><p><code>packages/core/src/zip-archive.ts</code> reads the central directory from the archive's tail, then streams one entry at a time through fflate's <code>UnzipInflate</code> in <code>chunkSize</code> slices. A dropped file is read a slice at a time, and no decompressed entry is ever held whole (the SHS fetch still buffers its download before parsing). It avoids fflate's forward-scanning <code>Unzip</code> on purpose: the History Server writes its zip with Java's <code>ZipOutputStream</code>, which leaves each local header's sizes blank and appends a data descriptor, so <code>Unzip</code> would have to find each entry's end by scanning compressed bytes for a signature, and it would hand rolling parts over in archive order (<code>events_10</code> before <code>events_2</code>). A single-file log is one entry at the root. A rolling log's parts sit under an <code>eventlog_v2_<appId>/</code> directory entry and are matched by base name. The zip must hold a single application attempt. Without an attempt ID the History Server packs every attempt into one download (several root entries, or several <code>eventlog_v2_</code> directories), and <code>parseZipArchive</code> rejects that before parsing with the attempt count and a pointer to <code>GET /api/v1/applications/<appId>/<attemptId>/logs</code>.</p><p>Each entry's bytes then go through the codec's streaming decoder, sniffed from the entry's first bytes, with the entry name's suffix as fallback. One-shotting the decompression of an entry (fzstd's <code>decompress()</code>, fflate's <code>gunzipSync()</code>) would allocate its full declared content size as one ArrayBuffer, which fails outright for a multi-GB event log. Decompressors are vendored under <code>packages/core/src/vendor/</code> (<code>fflate.js</code>, <code>fzstd.js</code>) plus <code>packages/core/src/lz4-block.ts</code> and <code>packages/core/src/snappy-block.ts</code>.</p></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/contributor-guide/architecture/" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>Architecture</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="/docs/contributor-guide/architecture/worker-protocol.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>Worker protocol</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
|
|
27
|
-
<script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"D0Kkk_r5\",\"contributor-guide_architecture_detector-contract.md\":\"caA46z-u\",\"contributor-guide_architecture_drill-down.md\":\"CdtlKHl-\",\"contributor-guide_architecture_impact-estimation.md\":\"BNp14LtL\",\"contributor-guide_architecture_index.md\":\"B9yZieYL\",\"contributor-guide_architecture_overview.md\":\"DBGo-bwn\",\"contributor-guide_architecture_state-and-history.md\":\"w4lwybdh\",\"contributor-guide_architecture_widget-rendering.md\":\"CcvqxnZI\",\"contributor-guide_architecture_worker-protocol.md\":\"CKeFttKj\",\"contributor-guide_contributing.md\":\"CQMvGwFt\",\"contributor-guide_development-setup.md\":\"D9bu4j_c\",\"contributor-guide_testing.md\":\"By6XGR9a\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"D08Uaod7\",\"tuning-reference_aqe.md\":\"C1Ae6A8u\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"6wjKuJNE\",\"tuning-reference_bottleneck-cold-start.md\":\"BVQbsuHP\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"DtaYnaiF\",\"tuning-reference_bottleneck-failures.md\":\"Z1Tdmdd3\",\"tuning-reference_bottleneck-gc.md\":\"BCnXLf1n\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"CcvQj2g9\",\"tuning-reference_bottleneck-memory-utilization.md\":\"B-C0-8h-\",\"tuning-reference_bottleneck-retry-waste.md\":\"DCnhWiqj\",\"tuning-reference_bottleneck-shuffle.md\":\"B5OMJCd3\",\"tuning-reference_bottleneck-skew.md\":\"Cvo6NQm_\",\"tuning-reference_bottleneck-slow-host.md\":\"ja5qr3wq\",\"tuning-reference_bottleneck-small-files.md\":\"Crjtb0i2\",\"tuning-reference_bottleneck-spill.md\":\"Bl8KtSRg\",\"tuning-reference_bottleneck-straggler.md\":\"B0VN8reT\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"DQCftGhu\",\"tuning-reference_bottleneck-utilization.md\":\"BXDl30f9\",\"tuning-reference_caching.md\":\"CcaiMn_7\",\"tuning-reference_cluster-config.md\":\"DbWG1yZq\",\"tuning-reference_config.md\":\"YUqSpwFB\",\"tuning-reference_data-formats.md\":\"C5DX5xL4\",\"tuning-reference_index.md\":\"ZVIpQ6Dk\",\"tuning-reference_intro.md\":\"BSSaDlqx\",\"tuning-reference_joins.md\":\"BVhNCDnE\",\"tuning-reference_memory-model.md\":\"CfjFdgbl\",\"tuning-reference_metrics.md\":\"C64VNEL4\",\"tuning-reference_partitioning.md\":\"CtLK0xsS\",\"tuning-reference_pyspark.md\":\"D-92XgF8\",\"tuning-reference_shuffle.md\":\"Iq2v1oHJ\",\"tuning-reference_spark-architecture.md\":\"_qwnE_Sc\",\"tuning-reference_table-formats.md\":\"BSYEK-eN\",\"user-guide_alternative-log-retrieval.md\":\"z-8npxNk\",\"user-guide_getting-started.md\":\"BJW5oVhe\",\"user-guide_mcp-tools.md\":\"DcLiOa9t\",\"user-guide_run-comparison.md\":\"qin0a4IX\",\"user-guide_understanding-findings.md\":\"DzReyCXb\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Finding your event log\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":{\"selector\":\"[data-sf-anchor-offset]\",\"padding\":0},\"cleanUrls\":false}");</script>
|
|
26
|
+
<div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Contributor Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/development-setup.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Development setup</p><!--]--></a><!----></div><!----></div><section class="VPSidebarItem level-1 collapsible collapsed is-link has-active" data-v-b3fd67f8><div class="item" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/" data-v-b3fd67f8><!--[--><h3 class="text" data-v-b3fd67f8>Architecture</h3><!--]--></a><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-b3fd67f8><span class="vpi-chevron-right caret-icon" data-v-b3fd67f8></span></div></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/overview.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Overview</p><!--]--></a><!----></div><!----></div><section class="VPSidebarItem level-2 collapsible collapsed is-link" data-v-b3fd67f8><div class="item" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/worker-protocol.html" data-v-b3fd67f8><!--[--><h4 class="text" data-v-b3fd67f8>Worker protocol</h4><!--]--></a><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-b3fd67f8><span class="vpi-chevron-right caret-icon" data-v-b3fd67f8></span></div></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-3 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/worker-protocol/evidence-availability.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Evidence availability</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-3 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/worker-protocol/evidence-report.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Evidence report and finding identity</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-3 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/worker-protocol/cli-and-mcp.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>CLI and MCP server</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-3 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/worker-protocol/core-and-event-validation.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Core and event validation</p><!--]--></a><!----></div><!----></div><!--]--></div></section><section class="VPSidebarItem level-2 collapsible collapsed is-link" data-v-b3fd67f8><div class="item" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/state-and-history.html" data-v-b3fd67f8><!--[--><h4 class="text" data-v-b3fd67f8>State & history intake</h4><!--]--></a><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-b3fd67f8><span class="vpi-chevron-right caret-icon" data-v-b3fd67f8></span></div></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-3 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/state-and-history/run-comparison-internals.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Run comparison internals</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-3 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/state-and-history/history-server-intake.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>History Server intake</p><!--]--></a><!----></div><!----></div><!--]--></div></section><section class="VPSidebarItem level-2 collapsible collapsed is-link" data-v-b3fd67f8><div class="item" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/detector-contract.html" data-v-b3fd67f8><!--[--><h4 class="text" data-v-b3fd67f8>Detector contract</h4><!--]--></a><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-b3fd67f8><span class="vpi-chevron-right caret-icon" data-v-b3fd67f8></span></div></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-3 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/detector-contract/plan-attribution.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Plan attribution</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-3 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/detector-contract/threshold-tables.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Threshold tables</p><!--]--></a><!----></div><!----></div><!--]--></div></section><section class="VPSidebarItem level-2 collapsible collapsed is-link" data-v-b3fd67f8><div class="item" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/impact-estimation.html" data-v-b3fd67f8><!--[--><h4 class="text" data-v-b3fd67f8>Impact estimation</h4><!--]--></a><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-b3fd67f8><span class="vpi-chevron-right caret-icon" data-v-b3fd67f8></span></div></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-3 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/impact-estimation/stage-union-rollup.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Stage union rollup</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-3 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/impact-estimation/caveats-tuning-and-coverage.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Caveats, tuning and coverage</p><!--]--></a><!----></div><!----></div><!--]--></div></section><section class="VPSidebarItem level-2 collapsible collapsed is-link" data-v-b3fd67f8><div class="item" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/widget-rendering.html" data-v-b3fd67f8><!--[--><h4 class="text" data-v-b3fd67f8>Widget rendering</h4><!--]--></a><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-b3fd67f8><span class="vpi-chevron-right caret-icon" data-v-b3fd67f8></span></div></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-3 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/widget-rendering/findings-tab.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Findings tab</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-3 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/widget-rendering/list-sort-mode.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>List sort mode</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-3 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/widget-rendering/row-expand-contract.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Row/expand contract</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-3 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/widget-rendering/reference-section.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>ReferenceSection</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-3 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/widget-rendering/investigation-routing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Investigation routing</p><!--]--></a><!----></div><!----></div><!--]--></div></section><section class="VPSidebarItem level-2 collapsible collapsed is-link" data-v-b3fd67f8><div class="item" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/board-widgets.html" data-v-b3fd67f8><!--[--><h4 class="text" data-v-b3fd67f8>Board widgets</h4><!--]--></a><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-b3fd67f8><span class="vpi-chevron-right caret-icon" data-v-b3fd67f8></span></div></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-3 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/board-widgets/plan-advisor.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Plan Advisor</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-3 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/board-widgets/caching-and-memory.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Caching and memory widgets</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-3 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/board-widgets/visual-system-and-templating.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Visual system and templating</p><!--]--></a><!----></div><!----></div><!--]--></div></section><section class="VPSidebarItem level-2 collapsible collapsed is-link" data-v-b3fd67f8><div class="item" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/drill-down.html" data-v-b3fd67f8><!--[--><h4 class="text" data-v-b3fd67f8>Drill-down</h4><!--]--></a><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-b3fd67f8><span class="vpi-chevron-right caret-icon" data-v-b3fd67f8></span></div></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-3 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/drill-down/plan-graph.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Plan graph</p><!--]--></a><!----></div><!----></div><!--]--></div></section><!--]--></div></section><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/testing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Testing & verification</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/contributing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Contributing</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_contributor-guide_architecture_overview" data-v-39a288b8><div><h1 id="architecture-overview" tabindex="-1">Architecture overview <a class="header-anchor" href="#architecture-overview" aria-label="Permalink to "Architecture overview""></a></h1><h2 id="core-invariant" tabindex="-1">Core invariant <a class="header-anchor" href="#core-invariant" aria-label="Permalink to "Core invariant""></a></h2><p>Task data must never live on the main thread. A Spark job can emit millions of <code>SparkListenerTaskEnd</code> events (240 MB+ of metrics); parsing them on the UI thread freezes Chrome. The design below enforces that in both deploy modes: the static zero-backend app, and the optional local-server mode in <code>packages/server/</code>, which also proxies Spark History Server fetches to sidestep CORS.</p><h2 id="two-actors" tabindex="-1">Two actors <a class="header-anchor" href="#two-actors" aria-label="Permalink to "Two actors""></a></h2><h3 id="main-thread" tabindex="-1">Main thread <a class="header-anchor" href="#main-thread" aria-label="Permalink to "Main thread""></a></h3><p>Owns the DOM (React) and a summary <code>AppModel</code> (jobs, stages, sql, executors, no raw tasks), held in a Zustand store (<code>src/store/store.ts</code>).</p><p><code>src/store/useIngest.ts</code> is the composition root. It owns the ingest client's lifecycle (<code>createIngestClient</code>, <code>@sparkforensics/core/ingest.ts</code>) and wires <code>model-assembler.ts</code>'s <code>createModelCallbacks</code> (worker message → <code>AppModel</code>) into store updates: <code>onProgress</code> → <code>parse</code>; <code>onDone</code> → derives <code>evidenceAvailability</code> from the final normalized model plus <code>done.skippedLines</code>, runs <code>analyzer.ts</code>, sets <code>catalog</code>, and prefetches flagged-stage task data.</p><p><code>src/App.tsx</code> routes on store state: a two-run compare load, the plan-graph route and the run comparison take precedence; otherwise idle/error → <code>CompareLanding</code> (one or two <code>DropZone</code>s), parsing → a live progress readout (an SHS parse keeps its intake form instead), ready → the lazy-loaded <code>Dashboard</code>. <code>src/view/Dashboard.tsx</code> renders widgets from <code>src/view/detector-registry.tsx</code>'s <code>REGISTRY</code> (see <a href="./widget-rendering.html">Widget rendering</a>).</p><h3 id="worker-web-worker-packages-core" tabindex="-1">Worker (Web Worker, <code>packages/core</code>) <a class="header-anchor" href="#worker-web-worker-packages-core" aria-label="Permalink to "Worker (Web Worker, `packages/core`)""></a></h3><p>Owns the file and <code>taskStore: Map<stageId, Float64Array></code>, keyed by 8-field stride <code>[duration, gcTime, memSpilled, diskSpilled, shuffleRead, shuffleWrite, launchTime, finishTime]</code>. Six modules, one of them the rolling-log helper:</p><ul><li><code>packages/core/src/stage-quantiles.ts</code>, a pure leaf, exporting <code>finalizeStage</code>, <code>computeFieldQuantiles</code>, <code>computeDurationQuantiles</code>, <code>classifySpill</code>, <code>FIELDS</code>, and <code>TASK_FIELD_NAMES</code> for the task-field packed-array layout.</li><li><code>packages/core/src/event-handlers.ts</code>, holding <code>processEvent</code> and the per-event handler functions, <code>accumulateTask</code>, <code>createState</code>, <code>dispatchLine</code>, <code>buildChunkDecoder</code>, <code>emitParseCompletion</code>, <code>collectStageExecutorMetrics</code>, <code>collectLateSpeculationWaste</code>.</li><li><code>packages/core/src/shs-fetch.ts</code>, the SHS zip-fetch/decompress path: <code>runParseFromUrl</code>, <code>decodeShsArchive</code>, <code>parseZipArchive</code>, <code>sniffCodec</code>.</li><li><code>packages/core/src/rolling-log-reassembly.ts</code>, import-free on purpose so the main-thread <code>DropZone</code> can use it: <code>naturalCompare</code>, <code>reassembleRollingEntries</code>.</li><li><code>packages/core/src/zip-archive.ts</code>, the random-access zip reader both zip paths use: <code>isZip</code>, <code>listZipEntries</code>, <code>streamZipEntry</code>.</li><li><code>packages/core/src/parser-worker.ts</code>, a thin entrypoint (<code>streamFile</code>, <code>runParse</code>, <code>runParseFiles</code>, the <code>isWorker</code>/<code>self.onmessage</code> bus) that re-exports the public parts of <code>event-handlers.ts</code>, <code>stage-quantiles.ts</code>, <code>shs-fetch.ts</code> and <code>rolling-log-reassembly.ts</code>. It is the only piece needing the File/Blob streaming API.</li></ul><p>Dropped zstd files also run a nested decompress worker, <code>packages/core/src/zstd-worker.ts</code>, driven from the parse worker by <code>packages/core/src/zstd-worker-client.ts</code> (see <a href="./worker-protocol.html#decompress-worker">Decompress worker</a>).</p><h2 id="streaming" tabindex="-1">Streaming <a class="header-anchor" href="#streaming" aria-label="Permalink to "Streaming""></a></h2><p>The worker reads the <code>File</code> in 512 KB slices (smaller for small files, so every file gets at least 100 progress steps) via <code>file.slice(...).arrayBuffer()</code>, decodes each decompressed chunk with one streaming <code>TextDecoder</code>, splits on <code>\n</code>, JSON-parses and dispatches per event type. Quantiles (P50/P95/max) and spill classification (<code>skew | volume | unclassified</code>) are computed at <code>SparkListenerStageCompleted</code> time, before posting the stage's <code>{ type: 'stage' }</code> message to main.</p><p>When <code>spark.eventLog.logStageExecutorMetrics=true</code> (default <code>false</code>), <code>SparkListenerStageExecutorMetrics</code> events populate <code>stage.executorMetrics: Map<execId, {...}></code> with the 23 raw peak-memory/GC fields verbatim (camelCased), consumed by the <code>memoryUtilization</code> detector's per-executor memory bands (see <a href="./board-widgets/caching-and-memory.html">Memory Utilization</a>).</p><p>These events can arrive <em>after</em> <code>SparkListenerStageCompleted</code> for the same stage, so the per-stage <code>stage</code> message posted at completion time never carries them. Instead, <code>collectStageExecutorMetrics(state)</code> walks every stage once more just before <code>done</code>, and the worker re-posts a single <code>stageExecutorMetrics</code> message (<code>Map<stageId, Map<execId, metrics>></code>, empty when the log has no such data). Main-thread consumers get every stage's metrics without a per-stage race.</p><p>A TaskEnd that arrives after its stage's StageCompleted is otherwise dropped, since the finalized stage already posted its stats. The exception is the losing copy of a speculative race: Spark kills it only once the stage finishes ("Stage cancelled: Stage finished"), so its TaskEnd is normally late. <code>accumulateTask</code> adds its time to the stage's <code>speculationWasteMs</code> and <code>speculationWastedAttempts</code> and nothing else, pairing it with a speculative winner kept for that purpose. <code>collectLateSpeculationWaste(state)</code> re-posts the updated totals of those stages once, just before <code>done</code>, as a <code>stageSpeculationWaste</code> message (<code>Map<stageId, { speculationWasteMs, speculationWastedAttempts }></code>, empty when no speculative attempt ended late). Any other late task of a failed attempt (a zombie or killed task) leaves the stage's own figures alone and adds to its <code>lateAttemptWork</code>, which only the CLI's <code>metrics</code> block reads; <code>collectLateAttemptWork(state)</code> re-posts it as a <code>stageLateAttemptWork</code> message (<code>Map<stageId, StageAttemptTotals></code>). <code>emitParseCompletion</code> sends the speculation totals after <code>runAggregates</code> (whole-run aggregates computed from <code>taskStore</code>) and before the final <code>app</code> and <code>done</code> messages.</p><p>Compressed logs are decompressed as they stream. <code>sniffCodec</code> reads the leading magic bytes: gzip (<code>1f 8b</code>), Zstandard (<code>28 b5 2f fd</code>, Spark's <code>spark.io.compression.codec=zstd</code>), Spark's custom <code>LZ4Block</code> framing, or Spark's Snappy framing (<code>org.xerial.snappy</code>'s <code>\x82SNAPPY\0</code> header, <code>spark.io.compression.codec=snappy</code>). Only a zip entry falls back to its name's suffix (see <a href="#zip-archives">Zip archives</a>). Both the dropped-file path and the SHS-fetch path stream block-by-block (fflate <code>Gunzip</code> / fzstd <code>Decompress</code> / the LZ4Block decoder / the Snappy block decoder) to keep one decompressed chunk live at a time. In the browser, a dropped zstd file decompresses in a second worker while the parse worker parses earlier output, with a three-slice window bounding what is queued between them (see <a href="./worker-protocol.html#decompress-worker">Decompress worker</a>). The Node CLI/MCP paths (<code>collectRun</code> for local files, <code>shs-load.ts</code> for SHS archives) swap fzstd for Node's native zlib zstd where the running Node has it (22.15+/23.8+), through the <code>zstdDecoder</code> option of <code>runParse</code>/<code>decodeShsArchive</code>: <code>packages/core/src/cli/native-zstd.ts</code> decompresses one frame at a time, since Node's own decoders stop after a stream's first frame and Spark writes thousands of small ones. A frame declaring more than 64 MiB of content, one still incomplete after 64 MiB compressed, a malformed one, or a truncated tail goes to fzstd instead. Spark's frames declare no content size, so each one's output is capped at 64 MiB as it decodes: past that, the inline decoder hands the frame to fzstd and the threaded one streams it, pausing while 64 MiB wait to be parsed. For local files (<code>nodeParseCodecs</code>, <code>createThreadedZstdDecoder</code>), frames of 64 KB or more compressed decompress on libuv's threadpool, up to 4 at a time, while the main thread parses earlier output; <code>streamFile</code> awaits each <code>push</code>, and chunks still arrive in stream order. SHS archives fetched from a History Server keep the inline decoder (<code>nodeArchiveCodecs</code>).</p><p>Rolling <code>eventlog_v2_*</code> directories (Spark's multi-file event-log format, <code>events_<index>_<appId>(.codec)</code> files plus a zero-byte <code>appstatus_*</code> completion marker and, periodically, one <code>*.compact</code> merge file) are reassembled into parse order by <code>reassembleRollingEntries(names)</code>: drop the marker, drop every non-compact <code>events_*</code> file at or below the most recent <code>.compact</code> file's index, sort the rest numerically, and throw if an index is missing. The zip path (<code>parseZipArchive</code>) calls it on member names; for a dropped folder, <code>DropZone</code> (and the CLI's <code>collectRun</code>) call it before handing the ordered files to <code>runParseFiles</code>.</p><p><code>runParseFiles</code> streams each file in the resulting order through the same chunked codec dispatch as <code>runParse</code>, with a fresh decompressor per file, since codecs never span a roll boundary. The NDJSON line-decoder is the exception: it stays alive across all files, because nothing guarantees a roll boundary lands on a line boundary.</p><h2 id="zip-archives" tabindex="-1">Zip archives <a class="header-anchor" href="#zip-archives" aria-label="Permalink to "Zip archives""></a></h2><p>A History Server download is a zip, whether fetched by <code>runParseFromUrl</code> or dropped as a file (the Spark UI's download link, or <code>GET /api/v1/applications/<appId>/logs</code> saved to disk). Both go through <code>parseZipArchive</code> (<code>packages/core/src/shs-fetch.ts</code>): <code>runParse</code> checks for the zip signature before its codec dispatch, and <code>decodeShsArchive</code> wraps the downloaded bytes in the same <code>slice()</code> interface a <code>File</code> has. The two differ only in how they report failure (a plain <code>error</code> message for a dropped file, <code>{ source: 'shs', code: 'invalid-event-log' }</code> for a fetch).</p><p><code>packages/core/src/zip-archive.ts</code> reads the central directory from the archive's tail, then streams one entry at a time through fflate's <code>UnzipInflate</code> in <code>chunkSize</code> slices. A dropped file is read a slice at a time, and no decompressed entry is ever held whole (the SHS fetch still buffers its download before parsing). It avoids fflate's forward-scanning <code>Unzip</code> on purpose: the History Server writes its zip with Java's <code>ZipOutputStream</code>, which leaves each local header's sizes blank and appends a data descriptor, so <code>Unzip</code> would have to find each entry's end by scanning compressed bytes for a signature, and it would hand rolling parts over in archive order (<code>events_10</code> before <code>events_2</code>). A single-file log is one entry at the root. A rolling log's parts sit under an <code>eventlog_v2_<appId>/</code> directory entry and are matched by base name. The zip must hold a single application attempt. Without an attempt ID the History Server packs every attempt into one download (several root entries, or several <code>eventlog_v2_</code> directories), and <code>parseZipArchive</code> rejects that before parsing with the attempt count and a pointer to <code>GET /api/v1/applications/<appId>/<attemptId>/logs</code>.</p><p>Each entry's bytes then go through the codec's streaming decoder, sniffed from the entry's first bytes, with the entry name's suffix as fallback. One-shotting the decompression of an entry (fzstd's <code>decompress()</code>, fflate's <code>gunzipSync()</code>) would allocate its full declared content size as one ArrayBuffer, which fails outright for a multi-GB event log. Decompressors are vendored under <code>packages/core/src/vendor/</code> (<code>fflate.js</code>, <code>fzstd.js</code>) plus <code>packages/core/src/lz4-block.ts</code> and <code>packages/core/src/snappy-block.ts</code>.</p></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/contributor-guide/architecture/" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>Architecture</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="/docs/contributor-guide/architecture/worker-protocol.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>Worker protocol</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
|
|
27
|
+
<script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"YQcj5EI4\",\"contributor-guide_architecture_board-widgets_caching-and-memory.md\":\"8bH1HnZp\",\"contributor-guide_architecture_board-widgets_plan-advisor.md\":\"RlzvUmNm\",\"contributor-guide_architecture_board-widgets_visual-system-and-templating.md\":\"DYgUpIJB\",\"contributor-guide_architecture_detector-contract.md\":\"wj6Et6UD\",\"contributor-guide_architecture_detector-contract_plan-attribution.md\":\"yNex4bys\",\"contributor-guide_architecture_detector-contract_threshold-tables.md\":\"fkPBXsU2\",\"contributor-guide_architecture_drill-down.md\":\"CqZPrdo9\",\"contributor-guide_architecture_drill-down_plan-graph.md\":\"DwOozIEs\",\"contributor-guide_architecture_impact-estimation.md\":\"CZv_ZGOp\",\"contributor-guide_architecture_impact-estimation_caveats-tuning-and-coverage.md\":\"BUucRc0A\",\"contributor-guide_architecture_impact-estimation_stage-union-rollup.md\":\"DdCGtAGu\",\"contributor-guide_architecture_index.md\":\"tymxf1Fh\",\"contributor-guide_architecture_overview.md\":\"C1sSvm9i\",\"contributor-guide_architecture_state-and-history.md\":\"DyCBdjuM\",\"contributor-guide_architecture_state-and-history_history-server-intake.md\":\"C298hrHZ\",\"contributor-guide_architecture_state-and-history_run-comparison-internals.md\":\"uuAKpY0Y\",\"contributor-guide_architecture_widget-rendering.md\":\"BOZ1azXG\",\"contributor-guide_architecture_widget-rendering_findings-tab.md\":\"BQXycK6G\",\"contributor-guide_architecture_widget-rendering_investigation-routing.md\":\"BGdiyElu\",\"contributor-guide_architecture_widget-rendering_list-sort-mode.md\":\"BuHKgH0C\",\"contributor-guide_architecture_widget-rendering_reference-section.md\":\"DWqC80bN\",\"contributor-guide_architecture_widget-rendering_row-expand-contract.md\":\"ZT89u9f1\",\"contributor-guide_architecture_worker-protocol.md\":\"BAnhi2-d\",\"contributor-guide_architecture_worker-protocol_cli-and-mcp.md\":\"Dk08VS7n\",\"contributor-guide_architecture_worker-protocol_core-and-event-validation.md\":\"DSVFPpWb\",\"contributor-guide_architecture_worker-protocol_evidence-availability.md\":\"CjimIDBY\",\"contributor-guide_architecture_worker-protocol_evidence-report.md\":\"DVGZOWvW\",\"contributor-guide_contributing.md\":\"CQMvGwFt\",\"contributor-guide_development-setup.md\":\"D9bu4j_c\",\"contributor-guide_testing.md\":\"By6XGR9a\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"D08Uaod7\",\"tuning-reference_aqe.md\":\"C1Ae6A8u\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"6wjKuJNE\",\"tuning-reference_bottleneck-cold-start.md\":\"BVQbsuHP\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"DtaYnaiF\",\"tuning-reference_bottleneck-failures.md\":\"Z1Tdmdd3\",\"tuning-reference_bottleneck-gc.md\":\"BCnXLf1n\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"CcvQj2g9\",\"tuning-reference_bottleneck-memory-utilization.md\":\"B-C0-8h-\",\"tuning-reference_bottleneck-retry-waste.md\":\"DCnhWiqj\",\"tuning-reference_bottleneck-shuffle.md\":\"B5OMJCd3\",\"tuning-reference_bottleneck-skew.md\":\"Cvo6NQm_\",\"tuning-reference_bottleneck-slow-host.md\":\"ja5qr3wq\",\"tuning-reference_bottleneck-small-files.md\":\"Crjtb0i2\",\"tuning-reference_bottleneck-spill.md\":\"Bl8KtSRg\",\"tuning-reference_bottleneck-straggler.md\":\"B0VN8reT\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"DQCftGhu\",\"tuning-reference_bottleneck-utilization.md\":\"BXDl30f9\",\"tuning-reference_caching.md\":\"CcaiMn_7\",\"tuning-reference_cluster-config.md\":\"DbWG1yZq\",\"tuning-reference_config.md\":\"YUqSpwFB\",\"tuning-reference_data-formats.md\":\"C5DX5xL4\",\"tuning-reference_index.md\":\"ZVIpQ6Dk\",\"tuning-reference_intro.md\":\"BSSaDlqx\",\"tuning-reference_joins.md\":\"BVhNCDnE\",\"tuning-reference_memory-model.md\":\"CfjFdgbl\",\"tuning-reference_metrics.md\":\"C64VNEL4\",\"tuning-reference_partitioning.md\":\"CtLK0xsS\",\"tuning-reference_pyspark.md\":\"D-92XgF8\",\"tuning-reference_shuffle.md\":\"Iq2v1oHJ\",\"tuning-reference_spark-architecture.md\":\"_qwnE_Sc\",\"tuning-reference_table-formats.md\":\"BSYEK-eN\",\"user-guide_alternative-log-retrieval.md\":\"CJpYvHGo\",\"user-guide_alternative-log-retrieval_managed-platforms.md\":\"DvDwHNtC\",\"user-guide_alternative-log-retrieval_ssh-bastion.md\":\"Gszzlplv\",\"user-guide_getting-started.md\":\"DYHHHkQR\",\"user-guide_getting-started_ci-and-automation.md\":\"DpnfKz3x\",\"user-guide_getting-started_reading-the-dashboard.md\":\"Obf3W3__\",\"user-guide_getting-started_regression-metric-keys.md\":\"B_gzr2Mw\",\"user-guide_getting-started_tuning-thresholds.md\":\"B_MyjyZm\",\"user-guide_mcp-tools.md\":\"C65SjWUL\",\"user-guide_mcp-tools_compare-runs.md\":\"CPowlb2q\",\"user-guide_mcp-tools_diagnose-run.md\":\"BP1vnHA5\",\"user-guide_mcp-tools_evaluate-budgets.md\":\"Dq5Azk8G\",\"user-guide_mcp-tools_get-finding-documentation.md\":\"BtTTHNV6\",\"user-guide_mcp-tools_get-finding-evidence.md\":\"rKyu4wts\",\"user-guide_mcp-tools_get-reference-doc.md\":\"Yac4tH3u\",\"user-guide_mcp-tools_get-run-summary.md\":\"DpxZ4ZNp\",\"user-guide_mcp-tools_list-runs.md\":\"DpQPeCM3\",\"user-guide_run-comparison.md\":\"B6aNtDYF\",\"user-guide_run-comparison_how-stages-are-matched.md\":\"XBzQ1oEU\",\"user-guide_understanding-findings.md\":\"Dqt2H_C1\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\",\"collapsed\":true,\"items\":[{\"text\":\"Reading the dashboard\",\"link\":\"/user-guide/getting-started/reading-the-dashboard\"},{\"text\":\"CI and automation\",\"link\":\"/user-guide/getting-started/ci-and-automation\"},{\"text\":\"Tuning detector thresholds\",\"link\":\"/user-guide/getting-started/tuning-thresholds\"},{\"text\":\"Regression metric keys\",\"link\":\"/user-guide/getting-started/regression-metric-keys\"}]},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\",\"collapsed\":true,\"items\":[{\"text\":\"How stages are matched\",\"link\":\"/user-guide/run-comparison/how-stages-are-matched\"}]},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\",\"collapsed\":true,\"items\":[{\"text\":\"diagnose_run\",\"link\":\"/user-guide/mcp-tools/diagnose-run\"},{\"text\":\"get_run_summary\",\"link\":\"/user-guide/mcp-tools/get-run-summary\"},{\"text\":\"evaluate_budgets\",\"link\":\"/user-guide/mcp-tools/evaluate-budgets\"},{\"text\":\"compare_runs\",\"link\":\"/user-guide/mcp-tools/compare-runs\"},{\"text\":\"get_finding_evidence\",\"link\":\"/user-guide/mcp-tools/get-finding-evidence\"},{\"text\":\"get_finding_documentation\",\"link\":\"/user-guide/mcp-tools/get-finding-documentation\"},{\"text\":\"get_reference_doc\",\"link\":\"/user-guide/mcp-tools/get-reference-doc\"},{\"text\":\"list_runs\",\"link\":\"/user-guide/mcp-tools/list-runs\"}]},{\"text\":\"Finding your event log\",\"link\":\"/user-guide/alternative-log-retrieval\",\"collapsed\":true,\"items\":[{\"text\":\"Managed platforms\",\"link\":\"/user-guide/alternative-log-retrieval/managed-platforms\"},{\"text\":\"Behind an SSH bastion\",\"link\":\"/user-guide/alternative-log-retrieval/ssh-bastion\"}]}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\",\"collapsed\":true,\"items\":[{\"text\":\"Evidence availability\",\"link\":\"/contributor-guide/architecture/worker-protocol/evidence-availability\"},{\"text\":\"Evidence report and finding identity\",\"link\":\"/contributor-guide/architecture/worker-protocol/evidence-report\"},{\"text\":\"CLI and MCP server\",\"link\":\"/contributor-guide/architecture/worker-protocol/cli-and-mcp\"},{\"text\":\"Core and event validation\",\"link\":\"/contributor-guide/architecture/worker-protocol/core-and-event-validation\"}]},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\",\"collapsed\":true,\"items\":[{\"text\":\"Run comparison internals\",\"link\":\"/contributor-guide/architecture/state-and-history/run-comparison-internals\"},{\"text\":\"History Server intake\",\"link\":\"/contributor-guide/architecture/state-and-history/history-server-intake\"}]},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\",\"collapsed\":true,\"items\":[{\"text\":\"Plan attribution\",\"link\":\"/contributor-guide/architecture/detector-contract/plan-attribution\"},{\"text\":\"Threshold tables\",\"link\":\"/contributor-guide/architecture/detector-contract/threshold-tables\"}]},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\",\"collapsed\":true,\"items\":[{\"text\":\"Stage union rollup\",\"link\":\"/contributor-guide/architecture/impact-estimation/stage-union-rollup\"},{\"text\":\"Caveats, tuning and coverage\",\"link\":\"/contributor-guide/architecture/impact-estimation/caveats-tuning-and-coverage\"}]},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\",\"collapsed\":true,\"items\":[{\"text\":\"Findings tab\",\"link\":\"/contributor-guide/architecture/widget-rendering/findings-tab\"},{\"text\":\"List sort mode\",\"link\":\"/contributor-guide/architecture/widget-rendering/list-sort-mode\"},{\"text\":\"Row/expand contract\",\"link\":\"/contributor-guide/architecture/widget-rendering/row-expand-contract\"},{\"text\":\"ReferenceSection\",\"link\":\"/contributor-guide/architecture/widget-rendering/reference-section\"},{\"text\":\"Investigation routing\",\"link\":\"/contributor-guide/architecture/widget-rendering/investigation-routing\"}]},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\",\"collapsed\":true,\"items\":[{\"text\":\"Plan Advisor\",\"link\":\"/contributor-guide/architecture/board-widgets/plan-advisor\"},{\"text\":\"Caching and memory widgets\",\"link\":\"/contributor-guide/architecture/board-widgets/caching-and-memory\"},{\"text\":\"Visual system and templating\",\"link\":\"/contributor-guide/architecture/board-widgets/visual-system-and-templating\"}]},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\",\"collapsed\":true,\"items\":[{\"text\":\"Plan graph\",\"link\":\"/contributor-guide/architecture/drill-down/plan-graph\"}]}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":{\"selector\":\"[data-sf-anchor-offset]\",\"padding\":0},\"cleanUrls\":false}");</script>
|
|
28
28
|
|
|
29
29
|
</body>
|
|
30
30
|
</html>
|