sparkforensics-server 0.5.2 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.js +16 -7
- package/package.json +1 -1
- package/public/assets/AutoscalingChurn-DenWKTv-.js +1 -0
- package/public/assets/CacheUtilization-DLz7NdjV.js +1 -0
- package/public/assets/CachingOpportunity-zpQIqhzm.js +1 -0
- package/public/assets/ChartTheme-DAiARbLP.js +8 -0
- package/public/assets/ColdStart-C-9HjxWl.js +1 -0
- package/public/assets/ConfigAudit-D0V9tcrJ.js +1 -0
- package/public/assets/CoreUsageArea-yeerrljd.js +1 -0
- package/public/assets/Dashboard-DkUPrHKV.js +1 -0
- package/public/assets/DisclosureContext-DK1_iz6J.js +1 -0
- package/public/assets/DuplicatePlanSubtree-xDNDbCd3.js +1 -0
- package/public/assets/DurationHistogram-B6jsE0Ov.js +1 -0
- package/public/assets/EmptyState-C1uL40VU.js +1 -0
- package/public/assets/ExecutorUtilization-DXDZKmSc.js +1 -0
- package/public/assets/GcPressure-MRf7I4IG.js +1 -0
- package/public/assets/ImpactEstimate-Dm3xWbzV.js +1 -0
- package/public/assets/IncompleteRun-CaLOhMtf.js +1 -0
- package/public/assets/JobFailures-CzmzlvhO.js +1 -0
- package/public/assets/MemoryUtilization-AWUMUjCH.js +1 -0
- package/public/assets/NestedLoopJoin-C9sNr6Ss.js +1 -0
- package/public/assets/OverBroadcast-BOmRE6ca.js +1 -0
- package/public/assets/PartitionHint-BL2EmJfE.js +1 -0
- package/public/assets/PartitionSizing-BzBySvUq.js +1 -0
- package/public/assets/PlanExplorer-DdiQRcJU.js +1 -0
- package/public/assets/PlanGraphRoute-CIH9gBuR.js +1 -0
- package/public/assets/PlanView-B1hxzFRs.js +2 -0
- package/public/assets/PythonUdf-DvgCCueD.js +1 -0
- package/public/assets/RetryWaste-uXXu_CyM.js +1 -0
- package/public/assets/RowPagination-C-LzWkI5.js +1 -0
- package/public/assets/RowStatusCluster-Dbtmte7l.js +1 -0
- package/public/assets/ShuffleIO-nALwzC1_.js +1 -0
- package/public/assets/Skew-CHMEKgvk.js +1 -0
- package/public/assets/SlowHost-yyvjpHk6.js +1 -0
- package/public/assets/SmallFiles-y6odJ97n.js +1 -0
- package/public/assets/SpeculationWaste-C-SD2LOt.js +1 -0
- package/public/assets/Spill-DHTMmOup.js +1 -0
- package/public/assets/StageFailed-CQ3DxC9R.js +1 -0
- package/public/assets/StageFindingGroup-BgYLyj5q.js +1 -0
- package/public/assets/StageHeader-CIETkjVZ.js +1 -0
- package/public/assets/StagePill--P7PDd3s.js +1 -0
- package/public/assets/StageShape-BIv8vx_o.js +1 -0
- package/public/assets/StageSlowness-MMqzERxN.js +1 -0
- package/public/assets/Straggler-BgrY7AyW.js +1 -0
- package/public/assets/TaskFailures-BHHyKYTs.js +1 -0
- package/public/assets/TinyTask-DDScNhCl.js +1 -0
- package/public/assets/UnderBroadcast-CGIC04TI.js +1 -0
- package/public/assets/WidgetLeadSummary-CXFZP00C.js +1 -0
- package/public/assets/button-DbpTz2yk.js +1 -0
- package/public/assets/charts-vendor-CtTxwTeu.js +59 -0
- package/public/assets/composite-C9Nu2EI9.js +1 -0
- package/public/assets/d3-vendor-DNlD2-cz.js +1 -0
- package/public/assets/detector-registry-Bp8hrLua.js +2 -0
- package/public/assets/downsample-WmNM1qc9.js +1 -0
- package/public/assets/finding-anchor-Dr40LkS9.js +1 -0
- package/public/assets/findings-of-type-BL2u3RGZ.js +1 -0
- package/public/assets/impact-sort-mC5ISgX8.js +1 -0
- package/public/assets/inbox-KPU1Gppm.js +1 -0
- package/public/assets/index-CaRVPvAT.css +2 -0
- package/public/assets/index-fDiwfuoO.js +14 -0
- package/public/assets/inertValue-L9VxV0VE.js +1 -0
- package/public/assets/instrument-sans-latin-ext-wdth-normal-C5E2Gvlv.woff2 +0 -0
- package/public/assets/instrument-sans-latin-wdth-normal-BVScPF0l.woff2 +0 -0
- package/public/assets/jetbrains-mono-latin-ext-wght-normal-DBQx-q_a.woff2 +0 -0
- package/public/assets/jetbrains-mono-latin-wght-normal-B9CIFXIH.woff2 +0 -0
- package/public/assets/layout.worker-CLcBHxVp.js +3 -0
- package/public/assets/{parser-worker-CA5RxqKc.js → parser-worker-CRIOPMOW.js} +8 -8
- package/public/assets/plan-graph-vendor-Ce6d4HU6.js +7 -0
- package/public/assets/react-vendor-C6KcUD36.js +9 -0
- package/public/assets/select-BHqOqun5.js +1 -0
- package/public/assets/toggle-DPeN6aNa.js +1 -0
- package/public/assets/toggle-group-Kl3uyHSr.js +1 -0
- package/public/assets/usePagedRows-BafD6bEN.js +1 -0
- package/public/assets/useRecentFiles-DzJZAoOT.js +3 -0
- package/public/assets/useRegisterFieldControl-pQv-ZDoU.js +1 -0
- package/public/assets/useScrollLock-BCxn_dMp.js +1 -0
- package/public/assets/useSortMode-DjrAN-z4.js +1 -0
- package/public/docs/404.html +3 -7
- package/public/docs/assets/{app.eYTPqTeq.js → app.1gvzd5Zb.js} +1 -1
- package/public/docs/assets/chunks/@localSearchIndexroot.DS2TQP5_.js +1 -0
- package/public/docs/assets/chunks/{VPLocalSearchBox.DsZrQKQR.js → VPLocalSearchBox.CNxT3s0I.js} +1 -1
- package/public/docs/assets/chunks/{theme.DsTNE2pE.js → theme.C5lLqUv7.js} +2 -2
- package/public/docs/assets/contributor-guide_architecture_board-widgets.md.CqKM_jEG.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_board-widgets.md.CqKM_jEG.lean.js +1 -0
- package/public/docs/assets/{contributor-guide_architecture_board-widgets_caching-and-memory.md.8bH1HnZp.js → contributor-guide_architecture_board-widgets_caching-and-memory.md.BYDo3ClR.js} +1 -1
- package/public/docs/assets/contributor-guide_architecture_board-widgets_plan-advisor.md.ajUAVdbc.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_board-widgets_visual-system-and-templating.md.WqY74T0T.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_board-widgets_visual-system-and-templating.md.WqY74T0T.lean.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_detector-contract.md.CHHgywoP.js +1 -0
- package/public/docs/assets/{contributor-guide_architecture_detector-contract.md.wj6Et6UD.lean.js → contributor-guide_architecture_detector-contract.md.CHHgywoP.lean.js} +1 -1
- package/public/docs/assets/contributor-guide_architecture_detector-contract_plan-attribution.md.DIz3_oz9.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_detector-contract_plan-attribution.md.DIz3_oz9.lean.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_detector-contract_threshold-tables.md.WH4tRHPL.js +1 -0
- package/public/docs/assets/{contributor-guide_architecture_detector-contract_threshold-tables.md.CKjWG56Y.lean.js → contributor-guide_architecture_detector-contract_threshold-tables.md.WH4tRHPL.lean.js} +1 -1
- package/public/docs/assets/{contributor-guide_architecture_drill-down.md.CqZPrdo9.js → contributor-guide_architecture_drill-down.md.yB-Hz2XT.js} +1 -1
- package/public/docs/assets/{contributor-guide_architecture_drill-down.md.CqZPrdo9.lean.js → contributor-guide_architecture_drill-down.md.yB-Hz2XT.lean.js} +1 -1
- package/public/docs/assets/contributor-guide_architecture_drill-down_plan-graph.md.BQZE04sw.js +1 -0
- package/public/docs/assets/{contributor-guide_architecture_drill-down_plan-graph.md.DwOozIEs.lean.js → contributor-guide_architecture_drill-down_plan-graph.md.BQZE04sw.lean.js} +1 -1
- package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.K71YB-Rr.js +1 -0
- package/public/docs/assets/{contributor-guide_architecture_impact-estimation.md.CZv_ZGOp.lean.js → contributor-guide_architecture_impact-estimation.md.K71YB-Rr.lean.js} +1 -1
- package/public/docs/assets/contributor-guide_architecture_impact-estimation_caveats-tuning-and-coverage.md.DLCAffwn.js +1 -0
- package/public/docs/assets/{contributor-guide_architecture_impact-estimation_caveats-tuning-and-coverage.md.BUucRc0A.lean.js → contributor-guide_architecture_impact-estimation_caveats-tuning-and-coverage.md.DLCAffwn.lean.js} +1 -1
- package/public/docs/assets/contributor-guide_architecture_impact-estimation_stage-union-rollup.md.JFrzDMwo.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_impact-estimation_stage-union-rollup.md.JFrzDMwo.lean.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_overview.md.nxkpOhgf.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_state-and-history_run-comparison-internals.md.U-9KMDFK.js +1 -0
- package/public/docs/assets/{contributor-guide_architecture_state-and-history_run-comparison-internals.md.CZA0JLtV.lean.js → contributor-guide_architecture_state-and-history_run-comparison-internals.md.U-9KMDFK.lean.js} +1 -1
- package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.BrSz8Ezh.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_widget-rendering_findings-tab.md.B7rjSF_h.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_widget-rendering_findings-tab.md.B7rjSF_h.lean.js +1 -0
- package/public/docs/assets/{contributor-guide_architecture_widget-rendering_list-sort-mode.md.BuHKgH0C.js → contributor-guide_architecture_widget-rendering_list-sort-mode.md.37hcZJ7q.js} +1 -1
- package/public/docs/assets/{contributor-guide_architecture_widget-rendering_row-expand-contract.md.ZT89u9f1.js → contributor-guide_architecture_widget-rendering_row-expand-contract.md.mK_VmAWk.js} +1 -1
- package/public/docs/assets/{contributor-guide_architecture_worker-protocol.md.BAnhi2-d.js → contributor-guide_architecture_worker-protocol.md.Br6OE7RW.js} +1 -1
- package/public/docs/assets/contributor-guide_architecture_worker-protocol.md.Br6OE7RW.lean.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_worker-protocol_cli-and-mcp.md.IVyaXrGe.js +1 -0
- package/public/docs/assets/{contributor-guide_architecture_worker-protocol_cli-and-mcp.md.DC1YqXBp.lean.js → contributor-guide_architecture_worker-protocol_cli-and-mcp.md.IVyaXrGe.lean.js} +1 -1
- package/public/docs/assets/{contributor-guide_architecture_worker-protocol_core-and-event-validation.md.DSVFPpWb.js → contributor-guide_architecture_worker-protocol_core-and-event-validation.md.DVSXn-bb.js} +1 -1
- package/public/docs/assets/{contributor-guide_architecture_worker-protocol_evidence-availability.md.CjimIDBY.js → contributor-guide_architecture_worker-protocol_evidence-availability.md.D3S5cyOW.js} +2 -2
- package/public/docs/assets/{contributor-guide_architecture_worker-protocol_evidence-availability.md.CjimIDBY.lean.js → contributor-guide_architecture_worker-protocol_evidence-availability.md.D3S5cyOW.lean.js} +1 -1
- package/public/docs/assets/contributor-guide_architecture_worker-protocol_evidence-report.md.CtmCU57I.js +1 -0
- package/public/docs/assets/{contributor-guide_architecture_worker-protocol_evidence-report.md.DVGZOWvW.lean.js → contributor-guide_architecture_worker-protocol_evidence-report.md.CtmCU57I.lean.js} +1 -1
- package/public/docs/assets/contributor-guide_contributing.md.DBW4drtZ.js +1 -0
- package/public/docs/assets/{contributor-guide_development-setup.md.D9bu4j_c.js → contributor-guide_development-setup.md.B21ZTDhn.js} +3 -3
- package/public/docs/assets/contributor-guide_development-setup.md.B21ZTDhn.lean.js +1 -0
- package/public/docs/assets/contributor-guide_testing.md.DpkozNoG.js +4 -0
- package/public/docs/assets/{contributor-guide_testing.md.By6XGR9a.lean.js → contributor-guide_testing.md.DpkozNoG.lean.js} +1 -1
- package/public/docs/assets/{index.md.CHJVslga.js → index.md.lMm7SUMz.js} +1 -1
- package/public/docs/assets/{index.md.CHJVslga.lean.js → index.md.lMm7SUMz.lean.js} +1 -1
- package/public/docs/assets/instrument-sans-latin-ext-wdth-normal.C5E2Gvlv.woff2 +0 -0
- package/public/docs/assets/instrument-sans-latin-wdth-normal.BVScPF0l.woff2 +0 -0
- package/public/docs/assets/jetbrains-mono-cyrillic-wght-normal.D73BlboJ.woff2 +0 -0
- package/public/docs/assets/jetbrains-mono-greek-wght-normal.Bw9x6K1M.woff2 +0 -0
- package/public/docs/assets/jetbrains-mono-latin-ext-wght-normal.DBQx-q_a.woff2 +0 -0
- package/public/docs/assets/jetbrains-mono-latin-wght-normal.B9CIFXIH.woff2 +0 -0
- package/public/docs/assets/jetbrains-mono-vietnamese-wght-normal.Bt-aOZkq.woff2 +0 -0
- package/public/docs/assets/style.Cfxqlj0y.css +1 -0
- package/public/docs/assets/user-guide_getting-started_ci-and-automation.md.C0pcQoNc.js +29 -0
- package/public/docs/assets/user-guide_getting-started_ci-and-automation.md.C0pcQoNc.lean.js +1 -0
- package/public/docs/assets/user-guide_getting-started_reading-the-dashboard.md.CGaxPX_1.js +1 -0
- package/public/docs/assets/{user-guide_getting-started_tuning-thresholds.md.B_MyjyZm.js → user-guide_getting-started_tuning-thresholds.md.BGVE-wrc.js} +2 -2
- package/public/docs/assets/{user-guide_getting-started_tuning-thresholds.md.B_MyjyZm.lean.js → user-guide_getting-started_tuning-thresholds.md.BGVE-wrc.lean.js} +1 -1
- package/public/docs/assets/{user-guide_mcp-tools.md.C65SjWUL.js → user-guide_mcp-tools.md.49r3L2YE.js} +1 -1
- package/public/docs/assets/{user-guide_mcp-tools_compare-runs.md.Cc_gLoix.js → user-guide_mcp-tools_compare-runs.md.BQ_kIptC.js} +11 -6
- package/public/docs/assets/user-guide_mcp-tools_compare-runs.md.BQ_kIptC.lean.js +1 -0
- package/public/docs/assets/user-guide_mcp-tools_diagnose-run.md.azCZXkH8.js +44 -0
- package/public/docs/assets/{user-guide_mcp-tools_diagnose-run.md.BP1vnHA5.lean.js → user-guide_mcp-tools_diagnose-run.md.azCZXkH8.lean.js} +1 -1
- package/public/docs/assets/{user-guide_mcp-tools_evaluate-budgets.md.BB0nDY3Z.js → user-guide_mcp-tools_evaluate-budgets.md.dpQpRQ7z.js} +2 -2
- package/public/docs/assets/{user-guide_mcp-tools_evaluate-budgets.md.BB0nDY3Z.lean.js → user-guide_mcp-tools_evaluate-budgets.md.dpQpRQ7z.lean.js} +1 -1
- package/public/docs/assets/{user-guide_mcp-tools_get-finding-evidence.md.rKyu4wts.js → user-guide_mcp-tools_get-finding-evidence.md.CH6FkNLX.js} +2 -1
- package/public/docs/assets/{user-guide_mcp-tools_get-finding-evidence.md.rKyu4wts.lean.js → user-guide_mcp-tools_get-finding-evidence.md.CH6FkNLX.lean.js} +1 -1
- package/public/docs/assets/{user-guide_mcp-tools_get-run-summary.md.DpxZ4ZNp.js → user-guide_mcp-tools_get-run-summary.md.EhaIsele.js} +1 -1
- package/public/docs/assets/user-guide_run-comparison.md.B_LOkdBf.js +1 -0
- package/public/docs/assets/user-guide_run-comparison.md.B_LOkdBf.lean.js +1 -0
- package/public/docs/assets/user-guide_understanding-findings.md.C4riKCKl.js +1 -0
- package/public/docs/assets/user-guide_understanding-findings.md.C4riKCKl.lean.js +1 -0
- package/public/docs/contributor-guide/architecture/board-widgets/caching-and-memory.html +6 -10
- package/public/docs/contributor-guide/architecture/board-widgets/plan-advisor.html +6 -10
- package/public/docs/contributor-guide/architecture/board-widgets/visual-system-and-templating.html +6 -10
- package/public/docs/contributor-guide/architecture/board-widgets.html +6 -10
- package/public/docs/contributor-guide/architecture/detector-contract/plan-attribution.html +6 -10
- package/public/docs/contributor-guide/architecture/detector-contract/threshold-tables.html +6 -10
- package/public/docs/contributor-guide/architecture/detector-contract.html +6 -10
- package/public/docs/contributor-guide/architecture/drill-down/plan-graph.html +6 -10
- package/public/docs/contributor-guide/architecture/drill-down.html +6 -10
- package/public/docs/contributor-guide/architecture/impact-estimation/caveats-tuning-and-coverage.html +6 -10
- package/public/docs/contributor-guide/architecture/impact-estimation/stage-union-rollup.html +6 -10
- package/public/docs/contributor-guide/architecture/impact-estimation.html +6 -10
- package/public/docs/contributor-guide/architecture/index.html +5 -9
- package/public/docs/contributor-guide/architecture/overview.html +6 -10
- package/public/docs/contributor-guide/architecture/state-and-history/history-server-intake.html +5 -9
- package/public/docs/contributor-guide/architecture/state-and-history/run-comparison-internals.html +6 -10
- package/public/docs/contributor-guide/architecture/state-and-history.html +5 -9
- package/public/docs/contributor-guide/architecture/widget-rendering/findings-tab.html +6 -10
- package/public/docs/contributor-guide/architecture/widget-rendering/investigation-routing.html +5 -9
- package/public/docs/contributor-guide/architecture/widget-rendering/list-sort-mode.html +6 -10
- package/public/docs/contributor-guide/architecture/widget-rendering/reference-section.html +5 -9
- package/public/docs/contributor-guide/architecture/widget-rendering/row-expand-contract.html +6 -10
- package/public/docs/contributor-guide/architecture/widget-rendering.html +6 -10
- package/public/docs/contributor-guide/architecture/worker-protocol/cli-and-mcp.html +6 -10
- package/public/docs/contributor-guide/architecture/worker-protocol/core-and-event-validation.html +6 -10
- package/public/docs/contributor-guide/architecture/worker-protocol/evidence-availability.html +7 -11
- package/public/docs/contributor-guide/architecture/worker-protocol/evidence-report.html +6 -10
- package/public/docs/contributor-guide/architecture/worker-protocol.html +6 -10
- package/public/docs/contributor-guide/contributing.html +6 -10
- package/public/docs/contributor-guide/development-setup.html +8 -12
- package/public/docs/contributor-guide/testing.html +9 -10
- package/public/docs/favicon.svg +2 -2
- package/public/docs/hashmap.json +1 -1
- package/public/docs/index.html +6 -10
- package/public/docs/logo-dark.svg +1 -0
- package/public/docs/logo-light.svg +1 -0
- package/public/docs/tuning-reference/anti-patterns.html +5 -9
- package/public/docs/tuning-reference/aqe.html +5 -9
- package/public/docs/tuning-reference/bottleneck-broadcast-sizing.html +5 -9
- package/public/docs/tuning-reference/bottleneck-cold-start.html +5 -9
- package/public/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html +5 -9
- package/public/docs/tuning-reference/bottleneck-failures.html +5 -9
- package/public/docs/tuning-reference/bottleneck-gc.html +5 -9
- package/public/docs/tuning-reference/bottleneck-job-failure-rate.html +5 -9
- package/public/docs/tuning-reference/bottleneck-memory-utilization.html +5 -9
- package/public/docs/tuning-reference/bottleneck-retry-waste.html +5 -9
- package/public/docs/tuning-reference/bottleneck-shuffle.html +5 -9
- package/public/docs/tuning-reference/bottleneck-skew.html +5 -9
- package/public/docs/tuning-reference/bottleneck-slow-host.html +5 -9
- package/public/docs/tuning-reference/bottleneck-small-files.html +5 -9
- package/public/docs/tuning-reference/bottleneck-spill.html +5 -9
- package/public/docs/tuning-reference/bottleneck-straggler.html +5 -9
- package/public/docs/tuning-reference/bottleneck-tiny-tasks.html +5 -9
- package/public/docs/tuning-reference/bottleneck-utilization.html +5 -9
- package/public/docs/tuning-reference/caching.html +5 -9
- package/public/docs/tuning-reference/cluster-config.html +5 -9
- package/public/docs/tuning-reference/config.html +5 -9
- package/public/docs/tuning-reference/data-formats.html +5 -9
- package/public/docs/tuning-reference/index.html +5 -9
- package/public/docs/tuning-reference/intro.html +5 -9
- package/public/docs/tuning-reference/joins.html +5 -9
- package/public/docs/tuning-reference/memory-model.html +5 -9
- package/public/docs/tuning-reference/metrics.html +5 -9
- package/public/docs/tuning-reference/partitioning.html +5 -9
- package/public/docs/tuning-reference/pyspark.html +5 -9
- package/public/docs/tuning-reference/shuffle.html +5 -9
- package/public/docs/tuning-reference/spark-architecture.html +5 -9
- package/public/docs/tuning-reference/table-formats.html +5 -9
- package/public/docs/user-guide/alternative-log-retrieval/managed-platforms.html +5 -9
- package/public/docs/user-guide/alternative-log-retrieval/ssh-bastion.html +5 -9
- package/public/docs/user-guide/alternative-log-retrieval.html +5 -9
- package/public/docs/user-guide/getting-started/ci-and-automation.html +17 -15
- package/public/docs/user-guide/getting-started/reading-the-dashboard.html +6 -10
- package/public/docs/user-guide/getting-started/regression-metric-keys.html +5 -9
- package/public/docs/user-guide/getting-started/tuning-thresholds.html +7 -11
- package/public/docs/user-guide/getting-started.html +5 -9
- package/public/docs/user-guide/mcp-tools/compare-runs.html +16 -15
- package/public/docs/user-guide/mcp-tools/diagnose-run.html +28 -11
- package/public/docs/user-guide/mcp-tools/evaluate-budgets.html +6 -10
- package/public/docs/user-guide/mcp-tools/get-finding-documentation.html +5 -9
- package/public/docs/user-guide/mcp-tools/get-finding-evidence.html +7 -10
- package/public/docs/user-guide/mcp-tools/get-reference-doc.html +5 -9
- package/public/docs/user-guide/mcp-tools/get-run-summary.html +6 -10
- package/public/docs/user-guide/mcp-tools/list-runs.html +5 -9
- package/public/docs/user-guide/mcp-tools.html +6 -10
- package/public/docs/user-guide/run-comparison/how-stages-are-matched.html +5 -9
- package/public/docs/user-guide/run-comparison.html +6 -10
- package/public/docs/user-guide/understanding-findings.html +6 -10
- package/public/export-template.html +46 -46
- package/public/favicon.svg +2 -2
- package/public/index.html +8 -10
- package/public/sample-runs/sample-run.ndjson.gz +0 -0
- package/vendor-core/allocation.js +126 -39
- package/vendor-core/analyzer.js +85 -58
- package/vendor-core/api-merge-writes.js +3 -0
- package/vendor-core/aqe-skew.js +347 -0
- package/vendor-core/cli/budgets.js +6 -4
- package/vendor-core/cli/collect-run-pair.js +89 -0
- package/vendor-core/cli/collect-run-worker.js +16 -0
- package/vendor-core/cli/collect-run.js +1 -2
- package/vendor-core/cli/native-zstd.js +12 -16
- package/vendor-core/comparability.js +234 -0
- package/vendor-core/comparison-output.js +15 -7
- package/vendor-core/comparison-verdict.js +105 -25
- package/vendor-core/core-source-hash.txt +1 -1
- package/vendor-core/core-time-series.js +0 -1
- package/vendor-core/core-usage-locality.js +1 -1
- package/vendor-core/delta-log-paths.js +10 -1
- package/vendor-core/detectors.js +1078 -416
- package/vendor-core/docs-config.js +0 -4
- package/vendor-core/docs-content/detection/cache.md +2 -1
- package/vendor-core/docs-content/detection/cfg.md +19 -11
- package/vendor-core/docs-content/detection/cold.md +2 -1
- package/vendor-core/docs-content/detection/gc.md +7 -3
- package/vendor-core/docs-content/detection/host.md +11 -1
- package/vendor-core/docs-content/detection/mem.md +17 -6
- package/vendor-core/docs-content/detection/part.md +3 -1
- package/vendor-core/docs-content/detection/plan.md +55 -13
- package/vendor-core/docs-content/detection/shfl.md +11 -5
- package/vendor-core/docs-content/detection/skew.md +48 -1
- package/vendor-core/docs-content/detection/slow.md +4 -2
- package/vendor-core/docs-content/detection/spec.md +6 -1
- package/vendor-core/docs-content/detection/spill.md +6 -3
- package/vendor-core/docs-content/detection/strag.md +19 -6
- package/vendor-core/docs-content/detection/tiny.md +9 -3
- package/vendor-core/docs-content/detection/util.md +6 -2
- package/vendor-core/effective-conf.js +7 -10
- package/vendor-core/efficiency-model.js +24 -16
- package/vendor-core/etl-phases.js +1 -2
- package/vendor-core/event-handlers.js +175 -57
- package/vendor-core/event-schemas.js +21 -16
- package/vendor-core/evidence-availability.js +9 -2
- package/vendor-core/evidence-report.js +118 -35
- package/vendor-core/executor-peaks.js +41 -0
- package/vendor-core/export-data.js +5 -5
- package/vendor-core/finding-identity.js +78 -0
- package/vendor-core/finding-presentation.js +129 -39
- package/vendor-core/finding-types.js +108 -30
- package/vendor-core/format-utils.js +14 -15
- package/vendor-core/html-export.js +1 -1
- package/vendor-core/impact-band.js +2 -2
- package/vendor-core/impact-format.js +47 -19
- package/vendor-core/impact-model.js +28 -17
- package/vendor-core/job-groups.js +4 -4
- package/vendor-core/mcp-server-factory.js +48 -12
- package/vendor-core/mcp-tools.js +179 -24
- package/vendor-core/median.js +2 -2
- package/vendor-core/nested-loop-join.js +73 -0
- package/vendor-core/occupancy.js +1 -2
- package/vendor-core/parser-worker.js +1 -1
- package/vendor-core/plan-graph-model.js +14 -22
- package/vendor-core/plan-node-detail.js +2 -3
- package/vendor-core/plan-summary.js +24 -19
- package/vendor-core/plan-tree-walk.js +6 -8
- package/vendor-core/python-stage.js +12 -1
- package/vendor-core/recommendation-rollup.js +13 -12
- package/vendor-core/redact.js +52 -46
- package/vendor-core/remediation.js +10 -5
- package/vendor-core/run-aggregates.js +6 -8
- package/vendor-core/run-comparison.js +226 -66
- package/vendor-core/run-identity.js +12 -0
- package/vendor-core/run-interpretation.js +21 -0
- package/vendor-core/run-metrics.js +2 -2
- package/vendor-core/run-output.js +44 -11
- package/vendor-core/run-totals.js +12 -0
- package/vendor-core/run-verdict.js +76 -3
- package/vendor-core/scaling-sim.js +3 -4
- package/vendor-core/snappy-block.js +2 -2
- package/vendor-core/spark-conf.js +168 -0
- package/vendor-core/sql-stages.js +2 -2
- package/vendor-core/stage-alignment.js +2 -12
- package/vendor-core/stage-identity.js +58 -3
- package/vendor-core/stage-label.js +14 -0
- package/vendor-core/stage-plan-nodes.js +17 -3
- package/vendor-core/stage-quantiles.js +141 -9
- package/vendor-core/stage-structure.js +17 -3
- package/vendor-core/threshold-overrides.js +9 -3
- package/vendor-core/types.js +70 -23
- package/vendor-core/wall-clock.js +8 -0
- package/vendor-core/wasted-core-hours.js +12 -12
- package/vendor-core/write-targets.js +146 -14
- package/vendor-core/zstd-worker-client.js +1 -1
- package/vendor-core/zstd-worker.js +1 -1
- package/public/assets/AutoscalingChurn-8t3U3isq.js +0 -1
- package/public/assets/CacheUtilization-CJs9tTum.js +0 -1
- package/public/assets/CachingOpportunity-D1NcHs0V.js +0 -1
- package/public/assets/ColdStart-CdRJ9gD8.js +0 -1
- package/public/assets/ConfigAudit-B5Gbe5js.js +0 -1
- package/public/assets/CoreUsageArea-a--NTRGY.js +0 -1
- package/public/assets/Dashboard-C1hNeX49.js +0 -2
- package/public/assets/DisclosureContext-ZpE6CkHx.js +0 -1
- package/public/assets/DuplicatePlanSubtree-dobXdz7E.js +0 -1
- package/public/assets/ExecutorUtilization-D9grwi7t.js +0 -1
- package/public/assets/GcPressure-8-G2ri10.js +0 -1
- package/public/assets/IncompleteRun-B91lky7a.js +0 -1
- package/public/assets/JobFailures-BsWvBMyJ.js +0 -1
- package/public/assets/MemoryUtilization-BM5schZI.js +0 -1
- package/public/assets/OverBroadcast-DdN8JIJH.js +0 -1
- package/public/assets/PartitionHint-CUvbnnxy.js +0 -1
- package/public/assets/PartitionSizing-CY5HDYtp.js +0 -1
- package/public/assets/PlanExplorer-DgHCDBD_.js +0 -1
- package/public/assets/PlanGraphRoute-DH3KJb8j.js +0 -1
- package/public/assets/RetryWaste-DOnE9vGF.js +0 -1
- package/public/assets/ShuffleIO-MMnDESkF.js +0 -1
- package/public/assets/Skew-BmZZJ5ms.js +0 -1
- package/public/assets/SlowHost-CsJzfEqo.js +0 -1
- package/public/assets/SmallFiles-BoK3R1Ex.js +0 -1
- package/public/assets/SpeculationWaste-CnVxB5AJ.js +0 -1
- package/public/assets/Spill-BhNicyBy.js +0 -1
- package/public/assets/StageFailed-mAhHU4l_.js +0 -1
- package/public/assets/StageFindingGroup-BEtPWcT0.js +0 -1
- package/public/assets/StageHeader-DJkYkrsU.js +0 -1
- package/public/assets/StageShape-BDxRknLp.js +0 -1
- package/public/assets/StageSlowness-BPRw-otN.js +0 -1
- package/public/assets/Straggler-CZCbnOWP.js +0 -1
- package/public/assets/TaskFailures-C57YcLFQ.js +0 -1
- package/public/assets/TinyTask-Bi84qU7n.js +0 -1
- package/public/assets/UnderBroadcast-DdQV7htE.js +0 -1
- package/public/assets/charts-vendor-0VMaLVLN.js +0 -59
- package/public/assets/finding-anchor-Cl6_5hfV.js +0 -1
- package/public/assets/impact-sort-CssZ6MT2.js +0 -1
- package/public/assets/index-2h_TgUAb.css +0 -2
- package/public/assets/index-B-03mG7K.js +0 -14
- package/public/assets/plan-graph-vendor-BYl-AzN5.js +0 -7
- package/public/assets/react-vendor-DEdfrRTJ.js +0 -9
- package/public/assets/select-CQjNhprm.js +0 -1
- package/public/assets/usePagedRows-Dyl6T8Ok.js +0 -1
- package/public/assets/useRecentFiles-XZEnLlal.js +0 -11
- package/public/assets/useSortMode-Dhvit8dt.js +0 -1
- package/public/docs/assets/chunks/@localSearchIndexroot.u_e5bQz4.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_board-widgets.md.YQcj5EI4.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_board-widgets.md.YQcj5EI4.lean.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_board-widgets_plan-advisor.md.RlzvUmNm.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_board-widgets_visual-system-and-templating.md.DYgUpIJB.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_board-widgets_visual-system-and-templating.md.DYgUpIJB.lean.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_detector-contract.md.wj6Et6UD.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_detector-contract_plan-attribution.md.yNex4bys.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_detector-contract_plan-attribution.md.yNex4bys.lean.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_detector-contract_threshold-tables.md.CKjWG56Y.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_drill-down_plan-graph.md.DwOozIEs.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.CZv_ZGOp.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_impact-estimation_caveats-tuning-and-coverage.md.BUucRc0A.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_impact-estimation_stage-union-rollup.md.DdCGtAGu.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_impact-estimation_stage-union-rollup.md.DdCGtAGu.lean.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_overview.md.C1sSvm9i.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_state-and-history_run-comparison-internals.md.CZA0JLtV.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.BOZ1azXG.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_widget-rendering_findings-tab.md.BQXycK6G.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_widget-rendering_findings-tab.md.BQXycK6G.lean.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_worker-protocol.md.BAnhi2-d.lean.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_worker-protocol_cli-and-mcp.md.DC1YqXBp.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_worker-protocol_evidence-report.md.DVGZOWvW.js +0 -1
- package/public/docs/assets/contributor-guide_contributing.md.CQMvGwFt.js +0 -1
- package/public/docs/assets/contributor-guide_development-setup.md.D9bu4j_c.lean.js +0 -1
- package/public/docs/assets/contributor-guide_testing.md.By6XGR9a.js +0 -1
- package/public/docs/assets/inter-italic-cyrillic-ext.r48I6akx.woff2 +0 -0
- package/public/docs/assets/inter-italic-cyrillic.By2_1cv3.woff2 +0 -0
- package/public/docs/assets/inter-italic-greek-ext.1u6EdAuj.woff2 +0 -0
- package/public/docs/assets/inter-italic-greek.DJ8dCoTZ.woff2 +0 -0
- package/public/docs/assets/inter-italic-latin-ext.CN1xVJS-.woff2 +0 -0
- package/public/docs/assets/inter-italic-latin.C2AdPX0b.woff2 +0 -0
- package/public/docs/assets/inter-italic-vietnamese.BSbpV94h.woff2 +0 -0
- package/public/docs/assets/inter-roman-cyrillic-ext.BBPuwvHQ.woff2 +0 -0
- package/public/docs/assets/inter-roman-cyrillic.C5lxZ8CY.woff2 +0 -0
- package/public/docs/assets/inter-roman-greek-ext.CqjqNYQ-.woff2 +0 -0
- package/public/docs/assets/inter-roman-greek.BBVDIX6e.woff2 +0 -0
- package/public/docs/assets/inter-roman-latin-ext.4ZJIpNVo.woff2 +0 -0
- package/public/docs/assets/inter-roman-latin.Di8DUHzh.woff2 +0 -0
- package/public/docs/assets/inter-roman-vietnamese.BjW4sHH5.woff2 +0 -0
- package/public/docs/assets/style.tmDhD3c8.css +0 -1
- package/public/docs/assets/user-guide_getting-started_ci-and-automation.md.OUQqw3mI.js +0 -23
- package/public/docs/assets/user-guide_getting-started_ci-and-automation.md.OUQqw3mI.lean.js +0 -1
- package/public/docs/assets/user-guide_getting-started_reading-the-dashboard.md.Obf3W3__.js +0 -1
- package/public/docs/assets/user-guide_mcp-tools_compare-runs.md.Cc_gLoix.lean.js +0 -1
- package/public/docs/assets/user-guide_mcp-tools_diagnose-run.md.BP1vnHA5.js +0 -23
- package/public/docs/assets/user-guide_run-comparison.md.BUALkohi.js +0 -1
- package/public/docs/assets/user-guide_run-comparison.md.BUALkohi.lean.js +0 -1
- package/public/docs/assets/user-guide_understanding-findings.md.C9YRruMT.js +0 -1
- package/public/docs/assets/user-guide_understanding-findings.md.C9YRruMT.lean.js +0 -1
- /package/public/docs/assets/{contributor-guide_architecture_board-widgets_caching-and-memory.md.8bH1HnZp.lean.js → contributor-guide_architecture_board-widgets_caching-and-memory.md.BYDo3ClR.lean.js} +0 -0
- /package/public/docs/assets/{contributor-guide_architecture_board-widgets_plan-advisor.md.RlzvUmNm.lean.js → contributor-guide_architecture_board-widgets_plan-advisor.md.ajUAVdbc.lean.js} +0 -0
- /package/public/docs/assets/{contributor-guide_architecture_overview.md.C1sSvm9i.lean.js → contributor-guide_architecture_overview.md.nxkpOhgf.lean.js} +0 -0
- /package/public/docs/assets/{contributor-guide_architecture_widget-rendering.md.BOZ1azXG.lean.js → contributor-guide_architecture_widget-rendering.md.BrSz8Ezh.lean.js} +0 -0
- /package/public/docs/assets/{contributor-guide_architecture_widget-rendering_list-sort-mode.md.BuHKgH0C.lean.js → contributor-guide_architecture_widget-rendering_list-sort-mode.md.37hcZJ7q.lean.js} +0 -0
- /package/public/docs/assets/{contributor-guide_architecture_widget-rendering_row-expand-contract.md.ZT89u9f1.lean.js → contributor-guide_architecture_widget-rendering_row-expand-contract.md.mK_VmAWk.lean.js} +0 -0
- /package/public/docs/assets/{contributor-guide_architecture_worker-protocol_core-and-event-validation.md.DSVFPpWb.lean.js → contributor-guide_architecture_worker-protocol_core-and-event-validation.md.DVSXn-bb.lean.js} +0 -0
- /package/public/docs/assets/{contributor-guide_contributing.md.CQMvGwFt.lean.js → contributor-guide_contributing.md.DBW4drtZ.lean.js} +0 -0
- /package/public/docs/assets/{user-guide_getting-started_reading-the-dashboard.md.Obf3W3__.lean.js → user-guide_getting-started_reading-the-dashboard.md.CGaxPX_1.lean.js} +0 -0
- /package/public/docs/assets/{user-guide_mcp-tools.md.C65SjWUL.lean.js → user-guide_mcp-tools.md.49r3L2YE.lean.js} +0 -0
- /package/public/docs/assets/{user-guide_mcp-tools_get-run-summary.md.DpxZ4ZNp.lean.js → user-guide_mcp-tools_get-run-summary.md.EhaIsele.lean.js} +0 -0
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
// Best-effort summary of a resolved plan tree (
|
|
1
|
+
// Best-effort summary of a resolved plan tree (`resolvePlanTree` in event-handlers.ts).
|
|
2
2
|
// Unparseable fragments are silently skipped, never surfaced as an error.
|
|
3
3
|
|
|
4
4
|
|
|
@@ -56,14 +56,14 @@ export function isDeltaLogRead(name , detail ) {
|
|
|
56
56
|
}
|
|
57
57
|
|
|
58
58
|
// Stable relation-identity key for a scan node: "<format>:<relation>" (e.g.
|
|
59
|
-
// "delta:
|
|
59
|
+
// "delta:db.t", "parquet:db.a", "jdbc:db.b")
|
|
60
60
|
// or null when the node is internal Delta metadata / a non-scan / un-nameable.
|
|
61
61
|
// The single source of truth for scan identity, shared by `visitScan` (summary)
|
|
62
62
|
// and the `cachingOpportunity` detector so the regexes live in one place.
|
|
63
63
|
export function scanRelationId(name , detail ) {
|
|
64
|
-
//
|
|
65
|
-
//
|
|
66
|
-
//
|
|
64
|
+
// Named catalog table, untruncated, from the nodeName (the Location: path is truncated at
|
|
65
|
+
// ~100 chars and points at _delta_log for Delta tables). All catalog tables surface as
|
|
66
|
+
// `spark_catalog.<db>.<table>`.
|
|
67
67
|
let format = null, table = null;
|
|
68
68
|
const nameM = name.match(/^Scan\s+(parquet|orc|csv|json)\s+(spark_catalog\.\S+)$/i);
|
|
69
69
|
// Fast reject: every branch below that returns a key needs this name match, a FileScan
|
|
@@ -81,12 +81,12 @@ export function scanRelationId(name , detail ) {
|
|
|
81
81
|
return `${format}:${table.replace(/^spark_catalog\./, '')}`;
|
|
82
82
|
}
|
|
83
83
|
|
|
84
|
-
//
|
|
84
|
+
// Internal Delta metadata (see isDeltaLogRead): no relation.
|
|
85
85
|
if (isDeltaLogRead(name, detail)) return null;
|
|
86
86
|
|
|
87
|
-
//
|
|
88
|
-
//
|
|
89
|
-
//
|
|
87
|
+
// Anonymous real file read (InMemoryFileIndex, no catalog name): best-effort Location basename.
|
|
88
|
+
// Preserves temp/snapshot self-reuse; a truncated Location can still fragment these, which is
|
|
89
|
+
// unrecoverable.
|
|
90
90
|
const fileM = detail.match(/FileScan\s+(parquet|orc|csv|json)\b/i);
|
|
91
91
|
if (fileM) {
|
|
92
92
|
const locM = detail.match(/Location:[^[]*\[([^\]]+)\]/);
|
|
@@ -94,15 +94,14 @@ export function scanRelationId(name , detail ) {
|
|
|
94
94
|
return `${fileM[1].toLowerCase()}:${pathBasename(locM[1].split(',')[0].trim())}`;
|
|
95
95
|
}
|
|
96
96
|
|
|
97
|
-
//
|
|
98
|
-
//
|
|
97
|
+
// JDBC: the inner SQL's first real table, lowercased. null when the first post-FROM token is a
|
|
98
|
+
// subquery `(` or there is no single table (CTE/UNION).
|
|
99
99
|
if (/JDBCRelation/.test(detail)) {
|
|
100
100
|
const fromM = jdbcSql(detail).match(/\bFROM\s+(\(|[A-Za-z_][\w.]*)/i);
|
|
101
101
|
if (!fromM || fromM[1] === '(') return null;
|
|
102
102
|
return `jdbc:${fromM[1].toLowerCase()}`;
|
|
103
103
|
}
|
|
104
104
|
|
|
105
|
-
// 5. Non-scan / unrecognized.
|
|
106
105
|
return null;
|
|
107
106
|
}
|
|
108
107
|
|
|
@@ -139,9 +138,17 @@ function visitScan(name , detail , result ) {
|
|
|
139
138
|
return true;
|
|
140
139
|
}
|
|
141
140
|
|
|
141
|
+
// <JoinOp> [leftKeys], [rightKeys], <JoinType>[, BuildSide]; the join type is Spark's JoinType.sql
|
|
142
|
+
// name (Inner, LeftOuter, RightOuter, FullOuter, LeftSemi, LeftAnti, Cross, ExistenceJoin(...)).
|
|
143
|
+
const JOIN_DETAIL = /^(SortMergeJoin|BroadcastHashJoin|ShuffledHashJoin|BroadcastNestedLoopJoin)\s+(\[[^\]]*\]),\s*(\[[^\]]*\]),\s*(\w+)/;
|
|
144
|
+
|
|
145
|
+
/** The join type a keyed join node's detail names, or null when the detail is not a join line. */
|
|
146
|
+
export function parseJoinType(detail ) {
|
|
147
|
+
return JOIN_DETAIL.exec(detail)?.[4] ?? null;
|
|
148
|
+
}
|
|
149
|
+
|
|
142
150
|
function visitJoin(_name , detail , result ) {
|
|
143
|
-
|
|
144
|
-
const m = detail.match(/^(SortMergeJoin|BroadcastHashJoin|ShuffledHashJoin|BroadcastNestedLoopJoin)\s+(\[[^\]]*\]),\s*(\[[^\]]*\]),\s*(\w+)/);
|
|
151
|
+
const m = detail.match(JOIN_DETAIL);
|
|
145
152
|
if (m) {
|
|
146
153
|
result.joins.push({ joinType: m[1], leftKeys: parseColList(m[2]), rightKeys: parseColList(m[3]) });
|
|
147
154
|
if (/^Cross$/i.test(m[4])) pushCrossJoinWarning(result, 'Cross join');
|
|
@@ -217,13 +224,12 @@ export function describePlanNode(
|
|
|
217
224
|
pushCrossJoinWarning(result, 'CartesianProduct');
|
|
218
225
|
} else if (visitScan(name, detail, result)) {
|
|
219
226
|
recognizedType = 'scan';
|
|
220
|
-
// handled
|
|
221
227
|
} else if (/Join/i.test(name) && visitJoin(name, detail, result)) {
|
|
222
228
|
recognizedType = 'join';
|
|
223
|
-
//
|
|
229
|
+
// A Cross join type may also have pushed a crossJoin warning.
|
|
224
230
|
} else if (/Aggregate/i.test(name) && visitAgg(name, detail, result)) {
|
|
225
231
|
recognizedType = 'agg';
|
|
226
|
-
//
|
|
232
|
+
// A partial-aggregation pass pushes nothing into result.aggs.
|
|
227
233
|
} else if (isExchangeNode(node)) {
|
|
228
234
|
recognizedType = 'exchange';
|
|
229
235
|
visitExchange(detail, result);
|
|
@@ -242,10 +248,9 @@ export function describePlanNode(
|
|
|
242
248
|
const fields = result.scans[0] ?? result.joins[0] ?? result.aggs[0] ?? result.exchanges[0] ?? null;
|
|
243
249
|
const warning = result.warnings[0] ?? null;
|
|
244
250
|
|
|
245
|
-
//
|
|
251
|
+
// An agg or filter with no content or warning is not described.
|
|
246
252
|
if (!recognizedType) return null;
|
|
247
253
|
if ((recognizedType === 'agg' || recognizedType === 'filter') && !kind && !warning) return null;
|
|
248
254
|
|
|
249
|
-
// For exchange or other types, or if there's content/warning, return the object
|
|
250
255
|
return { kind, warning, ...(fields ?? {}) };
|
|
251
256
|
}
|
|
@@ -1,12 +1,10 @@
|
|
|
1
|
-
// Canonical iterative walk over a resolved planTree (
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
// they all share, so it must not import from detectors.js (or vice versa).
|
|
1
|
+
// Canonical iterative walk over a resolved planTree (resolvePlanTree in event-handlers.ts). Every
|
|
2
|
+
// plan-*.ts consumer and the sql-scope detectors in detectors.ts need the same traversal: this is
|
|
3
|
+
// the single implementation they all share, so it must not import from detectors.ts (or vice versa).
|
|
5
4
|
//
|
|
6
|
-
// Pre-order, left-to-right child order (children are reverse-pushed so the
|
|
7
|
-
//
|
|
8
|
-
//
|
|
9
|
-
// per-node state (plan-dot.js's edges, plan-duration-attribution.js's segment
|
|
5
|
+
// Pre-order, left-to-right child order (children are reverse-pushed so the leftmost pops first):
|
|
6
|
+
// plan-summary.ts's finding order depends on this. `visit(node, parent)` gets the parent so callers
|
|
7
|
+
// needing per-edge or per-node state (plan-dot.ts's edges, plan-duration-attribution.ts's segment
|
|
10
8
|
// index) can track it themselves via a Map keyed by node.
|
|
11
9
|
export function walkPlanTree (
|
|
12
10
|
root ,
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { planNodesOfStage } from './stage-plan-nodes.js';
|
|
2
|
-
|
|
2
|
+
|
|
3
3
|
|
|
4
4
|
// Plan operators that hand rows to a Python worker process: the row-at-a-time and Arrow Python UDF
|
|
5
5
|
// evaluators, the pandas/Arrow grouped and map operators, and the PythonRDD scan of an RDD
|
|
@@ -9,6 +9,17 @@ import { planNodesOfStage } from './stage-plan-nodes.js';
|
|
|
9
9
|
// with "*(n) " in some plan strings.
|
|
10
10
|
const PYTHON_PLAN_NODE = /^(?:\*\(\d+\)\s*)?(?:PythonRDD|BatchEvalPython|ArrowEvalPython|ArrowAggregatePython|ArrowWindowPython|\w+InPandas|\w+InArrow)\w*\b/;
|
|
11
11
|
|
|
12
|
+
// The row-at-a-time Python UDF evaluator: each row is pickled to a Python worker and its result
|
|
13
|
+
// pickled back. The Arrow-based ArrowEvalPython is deliberately not matched, and neither are the
|
|
14
|
+
// UDTF variants (BatchEvalPythonUDTF), which spark.sql.execution.pythonUDF.arrow.enabled does not
|
|
15
|
+
// govern.
|
|
16
|
+
const BATCH_EVAL_PYTHON_NODE = /^(?:\*\(\d+\)\s*)?BatchEvalPython$/;
|
|
17
|
+
|
|
18
|
+
/** True for a plan node that runs row-at-a-time Python UDFs (BatchEvalPython). */
|
|
19
|
+
export function isBatchEvalPythonNode(node ) {
|
|
20
|
+
return BATCH_EVAL_PYTHON_NODE.test(node.name ?? '');
|
|
21
|
+
}
|
|
22
|
+
|
|
12
23
|
// An RDD lambda or map function has no SQL plan to match, and a stage whose plan could not be
|
|
13
24
|
// matched is left with nothing but its name and call site.
|
|
14
25
|
const PYTHON_STAGE_NAME = /PythonRDD/;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// Explicit .ts extensions: plain Node's ESM resolver (the runtime CLI/MCP path
|
|
2
2
|
// runs under) requires the exact specifier, unlike a bundler.
|
|
3
3
|
import { mergeIntervals } from './intervals.js';
|
|
4
|
-
import { formatRawWaste, formatWallClockRange, readsAsZero, worstImpactBand, IMPACT_BAND_ORDER } from './format-utils.js';
|
|
4
|
+
import { asCoreTime, formatRawWaste, formatWallClockRange, readsAsZero, worstImpactBand, IMPACT_BAND_ORDER } from './format-utils.js';
|
|
5
5
|
|
|
6
6
|
|
|
7
7
|
|
|
@@ -25,7 +25,7 @@ export function computeStageUnionMs(
|
|
|
25
25
|
// incomplete/truncated log (the case `incompleteRun` flags) leaves
|
|
26
26
|
// `completedAt` unset, and coercing it to 0 would contribute a hugely
|
|
27
27
|
// negative interval and a nonsense negative recoverable-time figure.
|
|
28
|
-
// Same filter `computeWallClock` (
|
|
28
|
+
// Same filter `computeWallClock` (wall-clock.ts) applies.
|
|
29
29
|
if (stage.submittedAt == null || stage.completedAt == null) continue;
|
|
30
30
|
intervals.push([stage.submittedAt, stage.completedAt]);
|
|
31
31
|
}
|
|
@@ -126,7 +126,7 @@ export function buildRecommendationRollup(
|
|
|
126
126
|
|
|
127
127
|
/** A resource group's summed waste as the board and CLI print it; null when it reads as zero. */
|
|
128
128
|
export function resourceGroupTotal(group ) {
|
|
129
|
-
const text = formatRawWaste({ value: group.total, unit: group.unit });
|
|
129
|
+
const text = formatRawWaste(asCoreTime({ value: group.total, unit: group.unit }));
|
|
130
130
|
return readsAsZero(text) ? null : text;
|
|
131
131
|
}
|
|
132
132
|
|
|
@@ -161,10 +161,10 @@ export function rollupGroupStat(group )
|
|
|
161
161
|
* else that decides whether a REGISTRY type has "something to show" (an
|
|
162
162
|
* active widget card vs. a Clean-checks line): memoryUtilization's
|
|
163
163
|
* memoryBand/dataUnavailable variant reports a missing-evidence caveat
|
|
164
|
-
* (
|
|
164
|
+
* (the log carries no executor heap peaks: Spark before 3.0 or local mode), not an
|
|
165
165
|
* optimization or a clean bill of health; the exact same fact already lives
|
|
166
166
|
* in the Evidence availability ledger's own `executorMetrics` entry
|
|
167
|
-
* (
|
|
167
|
+
* (evidence-availability.ts), so it belongs there, not as its own card. */
|
|
168
168
|
export function isRealFinding(finding ) {
|
|
169
169
|
if (finding.type === 'memoryUtilization' && finding.variant === 'memoryBand' && finding.dataUnavailable) return false;
|
|
170
170
|
return true;
|
|
@@ -173,7 +173,9 @@ export function isRealFinding(finding ) {
|
|
|
173
173
|
/** True when a finding is a candidate for the "fix these first" ranking.
|
|
174
174
|
* Shared by evidence-report.ts (CLI/MCP path) and FixTheseFirst.tsx (the
|
|
175
175
|
* dashboard widget) so these hardcoded exclusions can't drift between the two
|
|
176
|
-
* surfaces.
|
|
176
|
+
* surfaces. The verdict's finding count is the eligible findings, so it leaves out a log-completeness
|
|
177
|
+
* note (incompleteRun) and a missing-evidence caveat; the verdict wording says so (`verdictSummary`).
|
|
178
|
+
* Doesn't check `REGISTRY[finding.type] != null`: `REGISTRY` lives
|
|
177
179
|
* in a `.tsx` file, not importable from this core module, so FixTheseFirst.tsx
|
|
178
180
|
* layers that extra check on top of this one. */
|
|
179
181
|
export function isEligible(finding ) {
|
|
@@ -192,10 +194,9 @@ export function isEligible(finding ) {
|
|
|
192
194
|
/** Ranking tier for a single finding: time-based findings (a real `wallClock`
|
|
193
195
|
* claim) are the only ones whose magnitudes share a unit, so they are the
|
|
194
196
|
* only ones ranked numerically. `resourceOnly` findings carry `rawWaste` in
|
|
195
|
-
* whatever unit their detector emitted, which
|
|
196
|
-
*
|
|
197
|
-
*
|
|
198
|
-
* finding. Informational findings (no magnitude at all) rank last, also by
|
|
197
|
+
* whatever unit their detector emitted, which cannot be compared against a
|
|
198
|
+
* time figure or against another resourceOnly finding's own raw magnitude;
|
|
199
|
+
* they rank by impact band instead, below every time-based finding. Informational findings (no magnitude at all) rank last, also by
|
|
199
200
|
* impact band. */
|
|
200
201
|
const TIER_TIME = 0;
|
|
201
202
|
const TIER_RESOURCE = 1;
|
|
@@ -211,8 +212,8 @@ function tierOf(finding ) {
|
|
|
211
212
|
/** Ranks a group's members so the "representative" finding (the one whose
|
|
212
213
|
* action label/tag donate to the group's row) is always the highest-impact
|
|
213
214
|
* one, not just the first one `buildRecommendationRollup` happened to
|
|
214
|
-
* collect.
|
|
215
|
-
*
|
|
215
|
+
* collect. Picks each type-group's own highest-impact member and orders
|
|
216
|
+
* that group's expanded list; cross-group ordering is
|
|
216
217
|
* `buildRecommendationRollup`'s own job, not this function's. */
|
|
217
218
|
export function rankFindings(findings ) {
|
|
218
219
|
return [...findings].sort((a, b) => {
|
package/vendor-core/redact.js
CHANGED
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
|
|
10
10
|
import { decodeCollections, encodeCollections, } from './export-data.js';
|
|
11
11
|
|
|
12
|
+
import { isDetectorConfKey, REDACTED_CONF_VALUE } from './spark-conf.js';
|
|
12
13
|
import { REDACTED_TEXT, redactTaskFailureGroup, } from './task-failure.js';
|
|
13
14
|
|
|
14
15
|
// Host / IP identifier patterns. Used to enumerate host names that surface only
|
|
@@ -23,13 +24,10 @@ const HOST_PATTERNS = [
|
|
|
23
24
|
/\b(?:\d{1,3}\.){3}\d{1,3}\b/g, // bare IPv4: 10.1.2.3
|
|
24
25
|
];
|
|
25
26
|
|
|
26
|
-
// Spark application id (e.g. application_1690000000000_0001). Besides
|
|
27
|
-
//
|
|
28
|
-
//
|
|
29
|
-
//
|
|
30
|
-
// or plan detail. redactComparison has no structured app-id field at all
|
|
31
|
-
// (baselineLabel/candidateLabel are caller-supplied labels, not Spark app
|
|
32
|
-
// ids), so free text is its *only* source of app ids.
|
|
27
|
+
// Spark application id (e.g. application_1690000000000_0001). Besides the
|
|
28
|
+
// structured summary.app.id field, an app id can surface as a free-text token
|
|
29
|
+
// in evidence or recommendation strings. redactComparison has no structured
|
|
30
|
+
// app-id field, so free text is its only source of app ids.
|
|
33
31
|
const APP_ID_PATTERNS = [/\bapplication_\d{10,}_\d+\b/g];
|
|
34
32
|
|
|
35
33
|
// Walk every string in the tree once, collecting matches for each `{ patterns,
|
|
@@ -56,12 +54,11 @@ function scanTokens(node , sinks
|
|
|
56
54
|
}
|
|
57
55
|
|
|
58
56
|
// Recursively collects every string value found under a key literally named
|
|
59
|
-
// `host`, anywhere in the tree. Host names surface at several depths
|
|
60
|
-
// finding's own `host`, `evidence.host`,
|
|
61
|
-
// `evidence.
|
|
62
|
-
//
|
|
63
|
-
//
|
|
64
|
-
// than enumerating known paths keeps any future nested `host` field covered.
|
|
57
|
+
// `host`, anywhere in the tree. Host names surface at several depths (a
|
|
58
|
+
// finding's own `host`, `evidence.host`, `evidence.failedTaskDetails[].host`,
|
|
59
|
+
// `evidence.retriedTaskDetails[].host`), and only some are host-pattern-shaped
|
|
60
|
+
// (a plain FQDN like `worker-3.internal` matches neither HOST_PATTERNS).
|
|
61
|
+
// Walking by key name keeps any nested `host` field covered.
|
|
65
62
|
function collectHostFields(node , hosts ) {
|
|
66
63
|
if (Array.isArray(node)) {
|
|
67
64
|
for (const n of node) collectHostFields(n, hosts);
|
|
@@ -77,9 +74,7 @@ function collectHostFields(node , hosts ) {
|
|
|
77
74
|
|
|
78
75
|
// A failed-task error's message and stack text can carry file paths and data values that no
|
|
79
76
|
// host/app-id pattern recognizes, so they are dropped rather than pseudonymized: every array under
|
|
80
|
-
// a key named `failureGroups`
|
|
81
|
-
// redactTaskFailureGroup applied. Walking by key name, like collectHostFields, needs no path list.
|
|
82
|
-
// Returns a fresh tree.
|
|
77
|
+
// a key named `failureGroups` gets redactTaskFailureGroup applied. Returns a fresh tree.
|
|
83
78
|
function redactFailureGroups (node ) {
|
|
84
79
|
if (Array.isArray(node)) return node.map((n) => redactFailureGroups(n)) ;
|
|
85
80
|
if (node && typeof node === 'object') {
|
|
@@ -95,10 +90,8 @@ function redactFailureGroups (node ) {
|
|
|
95
90
|
}
|
|
96
91
|
|
|
97
92
|
// A stage's failure reason is Spark's free-text message and can carry file paths and data values
|
|
98
|
-
// too, so it is replaced outright
|
|
99
|
-
// `
|
|
100
|
-
// fallback reason), and the report's `failureReason` copy. Walking by key name, like
|
|
101
|
-
// collectHostFields, needs no path list. Returns a fresh tree.
|
|
93
|
+
// too, so it is replaced outright: a `stageFailed` finding's `valueText`, a stage's
|
|
94
|
+
// `stageFailureReason`, a job's `exception` and the report's `failureReason`. Returns a fresh tree.
|
|
102
95
|
const REASON_KEYS = new Set(['stageFailureReason', 'exception', 'failureReason']);
|
|
103
96
|
function redactStageFailureReasons (node ) {
|
|
104
97
|
if (Array.isArray(node)) return node.map((n) => redactStageFailureReasons(n)) ;
|
|
@@ -123,13 +116,11 @@ function redactFailureText (node ) {
|
|
|
123
116
|
// spark.yarn.am.hostname) carry plain FQDN host names that neither
|
|
124
117
|
// HOST_PATTERNS matches (no IP/EC2 shape) nor collectHostFields's by-key-name
|
|
125
118
|
// walk catches (the literal key is the dotted Spark property name, never
|
|
126
|
-
// `host` itself). app.config
|
|
127
|
-
//
|
|
128
|
-
// Known gap: a hostname
|
|
129
|
-
//
|
|
130
|
-
//
|
|
131
|
-
// (...AmIpFilter.param.PROXY_HOSTS, ...RM_HA_URLS) all carry real hostnames
|
|
132
|
-
// through un-redacted today.
|
|
119
|
+
// `host` itself). app.config and each SQL execution's modifiedConfigs are
|
|
120
|
+
// flat Record<string, string> dicts that only the run-tree redaction ships.
|
|
121
|
+
// Known gap: a hostname under a differently-named key is not caught by this
|
|
122
|
+
// suffix check (spark.master, spark.yarn.historyServer.address and the YARN
|
|
123
|
+
// proxy/HA keys); ci-and-automation.md lists them.
|
|
133
124
|
function collectConfigHostValues(config , hosts ) {
|
|
134
125
|
if (!config) return;
|
|
135
126
|
for (const [key, value] of Object.entries(config)) {
|
|
@@ -138,11 +129,9 @@ function collectConfigHostValues(config , hos
|
|
|
138
129
|
}
|
|
139
130
|
}
|
|
140
131
|
|
|
141
|
-
// Known locations of the identifiers
|
|
142
|
-
//
|
|
143
|
-
//
|
|
144
|
-
// scan for hosts/IPs and app ids that appear only inside string values (e.g. a
|
|
145
|
-
// finding's evidence/recommendation text).
|
|
132
|
+
// Known locations of the identifiers: the app id lives at summary.app.id; host
|
|
133
|
+
// names live under any `host` key in the findings tree (see collectHostFields),
|
|
134
|
+
// plus a free-text scan for hosts/IPs and app ids inside string values.
|
|
146
135
|
|
|
147
136
|
|
|
148
137
|
|
|
@@ -227,15 +216,10 @@ export function redactReport (input ) {
|
|
|
227
216
|
return app ? { ...out, summary: { ...out.summary, app: withRedactedName(app) } } : out;
|
|
228
217
|
}
|
|
229
218
|
|
|
230
|
-
// Run-comparison counterpart:
|
|
231
|
-
// (baselineLabel/candidateLabel are caller-supplied labels, not Spark app
|
|
232
|
-
// ids), but stage names surface throughout the tree (FindingsDeltaRow.stages,
|
|
233
|
-
// baseStages/candStages[].name) and, like the evidence report's findings, can
|
|
219
|
+
// Run-comparison counterpart: stage names surface throughout the tree and can
|
|
234
220
|
// carry a host/IP token or an app id as free text (e.g. `collect at
|
|
235
221
|
// application_1690000000000_0001 worker-10-1-2-3.scala:45`). Scans the whole
|
|
236
|
-
//
|
|
237
|
-
// future CompareRunsResult field carrying free text is covered without this
|
|
238
|
-
// function needing to change.
|
|
222
|
+
// tree rather than enumerating fields, so any free-text field is covered.
|
|
239
223
|
export function redactComparison (comparison ) {
|
|
240
224
|
const hosts = new Set ();
|
|
241
225
|
const appIds = new Set ();
|
|
@@ -244,14 +228,34 @@ export function redactComparison (comparison ) {
|
|
|
244
228
|
}
|
|
245
229
|
|
|
246
230
|
// HTML-export counterpart to redactReport/redactComparison. Unlike
|
|
247
|
-
// redactReport
|
|
248
|
-
//
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
231
|
+
// redactReport, this also walks executors.added/removed for their literal
|
|
232
|
+
// `host` field, since raw executor records reach the export, not just findings.
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
// A per-query setting is whatever the job passed to spark.conf.set: a path, a bucket name, a host. A
|
|
240
|
+
// detector reads only the Spark SQL tuning keys, so every other value is replaced by Spark's own
|
|
241
|
+
// placeholder for a hidden value and keeps no job-written text.
|
|
242
|
+
function withoutJobSetQueryValues (exec ) {
|
|
243
|
+
if (!exec.modifiedConfigs) return exec;
|
|
244
|
+
const kept = Object.fromEntries(Object.entries(exec.modifiedConfigs)
|
|
245
|
+
.map(([key, value]) => [key, isDetectorConfKey(key) ? value : REDACTED_CONF_VALUE]));
|
|
246
|
+
return { ...exec, modifiedConfigs: kept };
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
// An executor's removal reason is Spark's free-text message and can carry a path or a data value. No
|
|
250
|
+
// surface displays it. It is replaced by field, not through REASON_KEYS: `reason` also names unrelated
|
|
251
|
+
// fields elsewhere, and the key walk would blank those.
|
|
252
|
+
function withoutExecutorRemovalReasons (data ) {
|
|
253
|
+
const removed = data.executors.removed.map((e) => ({ ...e, reason: REDACTED_TEXT }));
|
|
254
|
+
return { ...data, executors: { ...data.executors, removed } };
|
|
255
|
+
}
|
|
252
256
|
|
|
253
257
|
function redactRunTree (input ) {
|
|
254
|
-
const data = redactFailureText(input);
|
|
258
|
+
const data = withoutExecutorRemovalReasons(redactFailureText(input));
|
|
255
259
|
const appIds = new Set ();
|
|
256
260
|
const hosts = new Set ();
|
|
257
261
|
const appId = data.app?.id;
|
|
@@ -261,8 +265,10 @@ function redactRunTree (input ) {
|
|
|
261
265
|
collectHostFields(data.catalog, hosts);
|
|
262
266
|
collectHostFields(data.configFindings, hosts);
|
|
263
267
|
collectConfigHostValues(data.app?.config, hosts);
|
|
268
|
+
for (const exec of data.sql ?? []) collectConfigHostValues(exec.modifiedConfigs, hosts);
|
|
264
269
|
scanTokens(data, [{ patterns: HOST_PATTERNS, out: hosts }, { patterns: APP_ID_PATTERNS, out: appIds }]);
|
|
265
|
-
const
|
|
270
|
+
const replaced = applyReplacements(data, { appIds, hosts });
|
|
271
|
+
const out = replaced.sql ? { ...replaced, sql: replaced.sql.map(withoutJobSetQueryValues) } : replaced;
|
|
266
272
|
if (!out.app) return out;
|
|
267
273
|
const app = withRedactedName(out.app);
|
|
268
274
|
// spark.app.name repeats the name in the config the HTML export ships.
|
|
@@ -1,20 +1,25 @@
|
|
|
1
1
|
// Constructors for the structured `remediation` a detector attaches next to its prose
|
|
2
2
|
// `recommendation`: only where the detector already names the Spark property, never an invented one.
|
|
3
|
-
|
|
3
|
+
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
|
|
6
6
|
|
|
7
7
|
/** The property should be raised; `suggested` is the value when the detector computed one. */
|
|
8
|
-
export function increaseConf(key , suggested = null)
|
|
8
|
+
export function increaseConf(key , suggested = null) {
|
|
9
9
|
return { kind: 'conf', key, direction: 'increase', suggested };
|
|
10
10
|
}
|
|
11
11
|
|
|
12
12
|
/** The property should be lowered; `suggested` is the value when the detector computed one. */
|
|
13
|
-
export function decreaseConf(key , suggested = null)
|
|
13
|
+
export function decreaseConf(key , suggested = null) {
|
|
14
14
|
return { kind: 'conf', key, direction: 'decrease', suggested };
|
|
15
15
|
}
|
|
16
16
|
|
|
17
17
|
/** The property should take a specific value (a switch or a class name), not move up or down. */
|
|
18
|
-
export function setConf(key , suggested = null)
|
|
18
|
+
export function setConf(key , suggested = null) {
|
|
19
19
|
return { kind: 'conf', key, direction: 'set', suggested };
|
|
20
20
|
}
|
|
21
|
+
|
|
22
|
+
/** No property fixes it: the job's code or data has to change, as `hint` says. */
|
|
23
|
+
export function codeFix(hint ) {
|
|
24
|
+
return { kind: 'code', hint };
|
|
25
|
+
}
|
|
@@ -1,12 +1,10 @@
|
|
|
1
|
-
// Whole-run aggregation pass. Runs INSIDE parser-worker.
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
// worker-agnostic so it is unit-testable without a Worker.
|
|
1
|
+
// Whole-run aggregation pass. Runs INSIDE parser-worker.ts over the retained per-task
|
|
2
|
+
// launch/finish timestamps in taskStore, so raw task data never reaches the main thread: only the
|
|
3
|
+
// compact summary below is posted. Pure and worker-agnostic so it is unit-testable without a Worker.
|
|
5
4
|
//
|
|
6
|
-
// Uses computeCoreTimeSeries in coreCount mode (O(n log n)) to derive the
|
|
7
|
-
//
|
|
8
|
-
//
|
|
9
|
-
// tasks (see core-time-series.js PERF note).
|
|
5
|
+
// Uses computeCoreTimeSeries in coreCount mode (O(n log n)) to derive the busy-core histogram; the
|
|
6
|
+
// time-bucket mode is intentionally NOT used here because it is O(buckets × segments) and a whole
|
|
7
|
+
// run can hold millions of tasks.
|
|
10
8
|
|
|
11
9
|
import { FIELDS } from './stage-quantiles.js';
|
|
12
10
|
import { computeCoreTimeSeries } from './core-time-series.js';
|