sparkforensics-server 0.6.0 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.js +16 -7
- package/package.json +1 -1
- package/public/assets/AutoscalingChurn-DenWKTv-.js +1 -0
- package/public/assets/CacheUtilization-DLz7NdjV.js +1 -0
- package/public/assets/CachingOpportunity-zpQIqhzm.js +1 -0
- package/public/assets/ChartTheme-DAiARbLP.js +8 -0
- package/public/assets/ColdStart-C-9HjxWl.js +1 -0
- package/public/assets/ConfigAudit-D0V9tcrJ.js +1 -0
- package/public/assets/CoreUsageArea-yeerrljd.js +1 -0
- package/public/assets/Dashboard-DkUPrHKV.js +1 -0
- package/public/assets/DisclosureContext-DK1_iz6J.js +1 -0
- package/public/assets/DuplicatePlanSubtree-xDNDbCd3.js +1 -0
- package/public/assets/DurationHistogram-B6jsE0Ov.js +1 -0
- package/public/assets/EmptyState-C1uL40VU.js +1 -0
- package/public/assets/ExecutorUtilization-DXDZKmSc.js +1 -0
- package/public/assets/GcPressure-MRf7I4IG.js +1 -0
- package/public/assets/ImpactEstimate-Dm3xWbzV.js +1 -0
- package/public/assets/IncompleteRun-CaLOhMtf.js +1 -0
- package/public/assets/JobFailures-CzmzlvhO.js +1 -0
- package/public/assets/MemoryUtilization-AWUMUjCH.js +1 -0
- package/public/assets/NestedLoopJoin-C9sNr6Ss.js +1 -0
- package/public/assets/OverBroadcast-BOmRE6ca.js +1 -0
- package/public/assets/PartitionHint-BL2EmJfE.js +1 -0
- package/public/assets/PartitionSizing-BzBySvUq.js +1 -0
- package/public/assets/PlanExplorer-DdiQRcJU.js +1 -0
- package/public/assets/PlanGraphRoute-CIH9gBuR.js +1 -0
- package/public/assets/PlanView-B1hxzFRs.js +2 -0
- package/public/assets/PythonUdf-DvgCCueD.js +1 -0
- package/public/assets/RetryWaste-uXXu_CyM.js +1 -0
- package/public/assets/RowPagination-C-LzWkI5.js +1 -0
- package/public/assets/RowStatusCluster-Dbtmte7l.js +1 -0
- package/public/assets/ShuffleIO-nALwzC1_.js +1 -0
- package/public/assets/Skew-CHMEKgvk.js +1 -0
- package/public/assets/SlowHost-yyvjpHk6.js +1 -0
- package/public/assets/SmallFiles-y6odJ97n.js +1 -0
- package/public/assets/SpeculationWaste-C-SD2LOt.js +1 -0
- package/public/assets/Spill-DHTMmOup.js +1 -0
- package/public/assets/StageFailed-CQ3DxC9R.js +1 -0
- package/public/assets/StageFindingGroup-BgYLyj5q.js +1 -0
- package/public/assets/StageHeader-CIETkjVZ.js +1 -0
- package/public/assets/StagePill--P7PDd3s.js +1 -0
- package/public/assets/StageShape-BIv8vx_o.js +1 -0
- package/public/assets/StageSlowness-MMqzERxN.js +1 -0
- package/public/assets/Straggler-BgrY7AyW.js +1 -0
- package/public/assets/TaskFailures-BHHyKYTs.js +1 -0
- package/public/assets/TinyTask-DDScNhCl.js +1 -0
- package/public/assets/UnderBroadcast-CGIC04TI.js +1 -0
- package/public/assets/WidgetLeadSummary-CXFZP00C.js +1 -0
- package/public/assets/button-DbpTz2yk.js +1 -0
- package/public/assets/charts-vendor-CtTxwTeu.js +59 -0
- package/public/assets/composite-C9Nu2EI9.js +1 -0
- package/public/assets/d3-vendor-DNlD2-cz.js +1 -0
- package/public/assets/detector-registry-Bp8hrLua.js +2 -0
- package/public/assets/downsample-WmNM1qc9.js +1 -0
- package/public/assets/finding-anchor-Dr40LkS9.js +1 -0
- package/public/assets/findings-of-type-BL2u3RGZ.js +1 -0
- package/public/assets/impact-sort-mC5ISgX8.js +1 -0
- package/public/assets/inbox-KPU1Gppm.js +1 -0
- package/public/assets/index-CaRVPvAT.css +2 -0
- package/public/assets/index-fDiwfuoO.js +14 -0
- package/public/assets/inertValue-L9VxV0VE.js +1 -0
- package/public/assets/layout.worker-CLcBHxVp.js +3 -0
- package/public/assets/{parser-worker-CA5RxqKc.js → parser-worker-CRIOPMOW.js} +8 -8
- package/public/assets/plan-graph-vendor-Ce6d4HU6.js +7 -0
- package/public/assets/react-vendor-C6KcUD36.js +9 -0
- package/public/assets/select-BHqOqun5.js +1 -0
- package/public/assets/toggle-DPeN6aNa.js +1 -0
- package/public/assets/toggle-group-Kl3uyHSr.js +1 -0
- package/public/assets/usePagedRows-BafD6bEN.js +1 -0
- package/public/assets/useRecentFiles-DzJZAoOT.js +3 -0
- package/public/assets/useRegisterFieldControl-pQv-ZDoU.js +1 -0
- package/public/assets/useScrollLock-BCxn_dMp.js +1 -0
- package/public/assets/useSortMode-DjrAN-z4.js +1 -0
- package/public/docs/404.html +3 -3
- package/public/docs/assets/{app.CnWL6bKV.js → app.1gvzd5Zb.js} +1 -1
- package/public/docs/assets/chunks/@localSearchIndexroot.DS2TQP5_.js +1 -0
- package/public/docs/assets/chunks/{VPLocalSearchBox.C0IBKeRQ.js → VPLocalSearchBox.CNxT3s0I.js} +1 -1
- package/public/docs/assets/chunks/{theme.BH-2HuRf.js → theme.C5lLqUv7.js} +2 -2
- package/public/docs/assets/contributor-guide_architecture_board-widgets.md.CqKM_jEG.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_board-widgets.md.CqKM_jEG.lean.js +1 -0
- package/public/docs/assets/{contributor-guide_architecture_board-widgets_caching-and-memory.md.CQgn1t_M.js → contributor-guide_architecture_board-widgets_caching-and-memory.md.BYDo3ClR.js} +1 -1
- package/public/docs/assets/contributor-guide_architecture_board-widgets_plan-advisor.md.ajUAVdbc.js +1 -0
- package/public/docs/assets/{contributor-guide_architecture_board-widgets_visual-system-and-templating.md.wulOueeB.js → contributor-guide_architecture_board-widgets_visual-system-and-templating.md.WqY74T0T.js} +1 -1
- package/public/docs/assets/contributor-guide_architecture_board-widgets_visual-system-and-templating.md.WqY74T0T.lean.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_detector-contract.md.CHHgywoP.js +1 -0
- package/public/docs/assets/{contributor-guide_architecture_detector-contract.md.N4hhenrA.lean.js → contributor-guide_architecture_detector-contract.md.CHHgywoP.lean.js} +1 -1
- package/public/docs/assets/contributor-guide_architecture_detector-contract_plan-attribution.md.DIz3_oz9.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_detector-contract_plan-attribution.md.DIz3_oz9.lean.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_detector-contract_threshold-tables.md.WH4tRHPL.js +1 -0
- package/public/docs/assets/{contributor-guide_architecture_detector-contract_threshold-tables.md.Dp3c40rx.lean.js → contributor-guide_architecture_detector-contract_threshold-tables.md.WH4tRHPL.lean.js} +1 -1
- package/public/docs/assets/{contributor-guide_architecture_drill-down.md.CqZPrdo9.js → contributor-guide_architecture_drill-down.md.yB-Hz2XT.js} +1 -1
- package/public/docs/assets/{contributor-guide_architecture_drill-down.md.CqZPrdo9.lean.js → contributor-guide_architecture_drill-down.md.yB-Hz2XT.lean.js} +1 -1
- package/public/docs/assets/contributor-guide_architecture_drill-down_plan-graph.md.BQZE04sw.js +1 -0
- package/public/docs/assets/{contributor-guide_architecture_drill-down_plan-graph.md.DwOozIEs.lean.js → contributor-guide_architecture_drill-down_plan-graph.md.BQZE04sw.lean.js} +1 -1
- package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.K71YB-Rr.js +1 -0
- package/public/docs/assets/{contributor-guide_architecture_impact-estimation.md.DfvbqEUu.lean.js → contributor-guide_architecture_impact-estimation.md.K71YB-Rr.lean.js} +1 -1
- package/public/docs/assets/contributor-guide_architecture_impact-estimation_caveats-tuning-and-coverage.md.DLCAffwn.js +1 -0
- package/public/docs/assets/{contributor-guide_architecture_impact-estimation_caveats-tuning-and-coverage.md.Cwm1jHmn.lean.js → contributor-guide_architecture_impact-estimation_caveats-tuning-and-coverage.md.DLCAffwn.lean.js} +1 -1
- package/public/docs/assets/contributor-guide_architecture_impact-estimation_stage-union-rollup.md.JFrzDMwo.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_impact-estimation_stage-union-rollup.md.JFrzDMwo.lean.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_overview.md.nxkpOhgf.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_state-and-history_run-comparison-internals.md.U-9KMDFK.js +1 -0
- package/public/docs/assets/{contributor-guide_architecture_state-and-history_run-comparison-internals.md.CZA0JLtV.lean.js → contributor-guide_architecture_state-and-history_run-comparison-internals.md.U-9KMDFK.lean.js} +1 -1
- package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.BrSz8Ezh.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_widget-rendering_findings-tab.md.B7rjSF_h.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_widget-rendering_findings-tab.md.B7rjSF_h.lean.js +1 -0
- package/public/docs/assets/{contributor-guide_architecture_widget-rendering_list-sort-mode.md.BuHKgH0C.js → contributor-guide_architecture_widget-rendering_list-sort-mode.md.37hcZJ7q.js} +1 -1
- package/public/docs/assets/{contributor-guide_architecture_widget-rendering_row-expand-contract.md.ZT89u9f1.js → contributor-guide_architecture_widget-rendering_row-expand-contract.md.mK_VmAWk.js} +1 -1
- package/public/docs/assets/{contributor-guide_architecture_worker-protocol.md.BAnhi2-d.js → contributor-guide_architecture_worker-protocol.md.Br6OE7RW.js} +1 -1
- package/public/docs/assets/contributor-guide_architecture_worker-protocol.md.Br6OE7RW.lean.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_worker-protocol_cli-and-mcp.md.IVyaXrGe.js +1 -0
- package/public/docs/assets/{contributor-guide_architecture_worker-protocol_cli-and-mcp.md.BX1XnFbq.lean.js → contributor-guide_architecture_worker-protocol_cli-and-mcp.md.IVyaXrGe.lean.js} +1 -1
- package/public/docs/assets/{contributor-guide_architecture_worker-protocol_core-and-event-validation.md.DSVFPpWb.js → contributor-guide_architecture_worker-protocol_core-and-event-validation.md.DVSXn-bb.js} +1 -1
- package/public/docs/assets/{contributor-guide_architecture_worker-protocol_evidence-availability.md.CjimIDBY.js → contributor-guide_architecture_worker-protocol_evidence-availability.md.D3S5cyOW.js} +2 -2
- package/public/docs/assets/{contributor-guide_architecture_worker-protocol_evidence-availability.md.CjimIDBY.lean.js → contributor-guide_architecture_worker-protocol_evidence-availability.md.D3S5cyOW.lean.js} +1 -1
- package/public/docs/assets/contributor-guide_architecture_worker-protocol_evidence-report.md.CtmCU57I.js +1 -0
- package/public/docs/assets/{contributor-guide_architecture_worker-protocol_evidence-report.md.CiQl1MLb.lean.js → contributor-guide_architecture_worker-protocol_evidence-report.md.CtmCU57I.lean.js} +1 -1
- package/public/docs/assets/contributor-guide_contributing.md.DBW4drtZ.js +1 -0
- package/public/docs/assets/contributor-guide_development-setup.md.B21ZTDhn.js +15 -0
- package/public/docs/assets/contributor-guide_development-setup.md.B21ZTDhn.lean.js +1 -0
- package/public/docs/assets/contributor-guide_testing.md.DpkozNoG.js +4 -0
- package/public/docs/assets/{contributor-guide_testing.md.By6XGR9a.lean.js → contributor-guide_testing.md.DpkozNoG.lean.js} +1 -1
- package/public/docs/assets/{index.md.CHJVslga.js → index.md.lMm7SUMz.js} +1 -1
- package/public/docs/assets/{index.md.CHJVslga.lean.js → index.md.lMm7SUMz.lean.js} +1 -1
- package/public/docs/assets/{style.CyK9Ss73.css → style.Cfxqlj0y.css} +1 -1
- package/public/docs/assets/user-guide_getting-started_ci-and-automation.md.C0pcQoNc.js +29 -0
- package/public/docs/assets/{user-guide_getting-started_ci-and-automation.md.BEepbDDL.lean.js → user-guide_getting-started_ci-and-automation.md.C0pcQoNc.lean.js} +1 -1
- package/public/docs/assets/user-guide_getting-started_reading-the-dashboard.md.CGaxPX_1.js +1 -0
- package/public/docs/assets/{user-guide_getting-started_tuning-thresholds.md.B_MyjyZm.js → user-guide_getting-started_tuning-thresholds.md.BGVE-wrc.js} +2 -2
- package/public/docs/assets/{user-guide_getting-started_tuning-thresholds.md.B_MyjyZm.lean.js → user-guide_getting-started_tuning-thresholds.md.BGVE-wrc.lean.js} +1 -1
- package/public/docs/assets/{user-guide_mcp-tools.md.C65SjWUL.js → user-guide_mcp-tools.md.49r3L2YE.js} +1 -1
- package/public/docs/assets/{user-guide_mcp-tools_compare-runs.md.Cc_gLoix.js → user-guide_mcp-tools_compare-runs.md.BQ_kIptC.js} +11 -6
- package/public/docs/assets/user-guide_mcp-tools_compare-runs.md.BQ_kIptC.lean.js +1 -0
- package/public/docs/assets/user-guide_mcp-tools_diagnose-run.md.azCZXkH8.js +44 -0
- package/public/docs/assets/{user-guide_mcp-tools_diagnose-run.md.BP1vnHA5.lean.js → user-guide_mcp-tools_diagnose-run.md.azCZXkH8.lean.js} +1 -1
- package/public/docs/assets/{user-guide_mcp-tools_evaluate-budgets.md.BB0nDY3Z.js → user-guide_mcp-tools_evaluate-budgets.md.dpQpRQ7z.js} +2 -2
- package/public/docs/assets/{user-guide_mcp-tools_evaluate-budgets.md.BB0nDY3Z.lean.js → user-guide_mcp-tools_evaluate-budgets.md.dpQpRQ7z.lean.js} +1 -1
- package/public/docs/assets/{user-guide_mcp-tools_get-finding-evidence.md.rKyu4wts.js → user-guide_mcp-tools_get-finding-evidence.md.CH6FkNLX.js} +2 -1
- package/public/docs/assets/{user-guide_mcp-tools_get-finding-evidence.md.rKyu4wts.lean.js → user-guide_mcp-tools_get-finding-evidence.md.CH6FkNLX.lean.js} +1 -1
- package/public/docs/assets/user-guide_run-comparison.md.B_LOkdBf.js +1 -0
- package/public/docs/assets/user-guide_run-comparison.md.B_LOkdBf.lean.js +1 -0
- package/public/docs/assets/user-guide_understanding-findings.md.C4riKCKl.js +1 -0
- package/public/docs/assets/user-guide_understanding-findings.md.C4riKCKl.lean.js +1 -0
- package/public/docs/contributor-guide/architecture/board-widgets/caching-and-memory.html +6 -6
- package/public/docs/contributor-guide/architecture/board-widgets/plan-advisor.html +6 -6
- package/public/docs/contributor-guide/architecture/board-widgets/visual-system-and-templating.html +6 -6
- package/public/docs/contributor-guide/architecture/board-widgets.html +6 -6
- package/public/docs/contributor-guide/architecture/detector-contract/plan-attribution.html +6 -6
- package/public/docs/contributor-guide/architecture/detector-contract/threshold-tables.html +6 -6
- package/public/docs/contributor-guide/architecture/detector-contract.html +6 -6
- package/public/docs/contributor-guide/architecture/drill-down/plan-graph.html +6 -6
- package/public/docs/contributor-guide/architecture/drill-down.html +6 -6
- package/public/docs/contributor-guide/architecture/impact-estimation/caveats-tuning-and-coverage.html +6 -6
- package/public/docs/contributor-guide/architecture/impact-estimation/stage-union-rollup.html +6 -6
- package/public/docs/contributor-guide/architecture/impact-estimation.html +6 -6
- package/public/docs/contributor-guide/architecture/index.html +5 -5
- package/public/docs/contributor-guide/architecture/overview.html +6 -6
- package/public/docs/contributor-guide/architecture/state-and-history/history-server-intake.html +5 -5
- package/public/docs/contributor-guide/architecture/state-and-history/run-comparison-internals.html +6 -6
- package/public/docs/contributor-guide/architecture/state-and-history.html +5 -5
- package/public/docs/contributor-guide/architecture/widget-rendering/findings-tab.html +6 -6
- package/public/docs/contributor-guide/architecture/widget-rendering/investigation-routing.html +5 -5
- package/public/docs/contributor-guide/architecture/widget-rendering/list-sort-mode.html +6 -6
- package/public/docs/contributor-guide/architecture/widget-rendering/reference-section.html +5 -5
- package/public/docs/contributor-guide/architecture/widget-rendering/row-expand-contract.html +6 -6
- package/public/docs/contributor-guide/architecture/widget-rendering.html +6 -6
- package/public/docs/contributor-guide/architecture/worker-protocol/cli-and-mcp.html +6 -6
- package/public/docs/contributor-guide/architecture/worker-protocol/core-and-event-validation.html +6 -6
- package/public/docs/contributor-guide/architecture/worker-protocol/evidence-availability.html +7 -7
- package/public/docs/contributor-guide/architecture/worker-protocol/evidence-report.html +6 -6
- package/public/docs/contributor-guide/architecture/worker-protocol.html +6 -6
- package/public/docs/contributor-guide/contributing.html +6 -6
- package/public/docs/contributor-guide/development-setup.html +8 -8
- package/public/docs/contributor-guide/testing.html +9 -6
- package/public/docs/hashmap.json +1 -1
- package/public/docs/index.html +6 -6
- package/public/docs/logo-dark.svg +1 -0
- package/public/docs/logo-light.svg +1 -0
- package/public/docs/tuning-reference/anti-patterns.html +5 -5
- package/public/docs/tuning-reference/aqe.html +5 -5
- package/public/docs/tuning-reference/bottleneck-broadcast-sizing.html +5 -5
- package/public/docs/tuning-reference/bottleneck-cold-start.html +5 -5
- package/public/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html +5 -5
- package/public/docs/tuning-reference/bottleneck-failures.html +5 -5
- package/public/docs/tuning-reference/bottleneck-gc.html +5 -5
- package/public/docs/tuning-reference/bottleneck-job-failure-rate.html +5 -5
- package/public/docs/tuning-reference/bottleneck-memory-utilization.html +5 -5
- package/public/docs/tuning-reference/bottleneck-retry-waste.html +5 -5
- package/public/docs/tuning-reference/bottleneck-shuffle.html +5 -5
- package/public/docs/tuning-reference/bottleneck-skew.html +5 -5
- package/public/docs/tuning-reference/bottleneck-slow-host.html +5 -5
- package/public/docs/tuning-reference/bottleneck-small-files.html +5 -5
- package/public/docs/tuning-reference/bottleneck-spill.html +5 -5
- package/public/docs/tuning-reference/bottleneck-straggler.html +5 -5
- package/public/docs/tuning-reference/bottleneck-tiny-tasks.html +5 -5
- package/public/docs/tuning-reference/bottleneck-utilization.html +5 -5
- package/public/docs/tuning-reference/caching.html +5 -5
- package/public/docs/tuning-reference/cluster-config.html +5 -5
- package/public/docs/tuning-reference/config.html +5 -5
- package/public/docs/tuning-reference/data-formats.html +5 -5
- package/public/docs/tuning-reference/index.html +5 -5
- package/public/docs/tuning-reference/intro.html +5 -5
- package/public/docs/tuning-reference/joins.html +5 -5
- package/public/docs/tuning-reference/memory-model.html +5 -5
- package/public/docs/tuning-reference/metrics.html +5 -5
- package/public/docs/tuning-reference/partitioning.html +5 -5
- package/public/docs/tuning-reference/pyspark.html +5 -5
- package/public/docs/tuning-reference/shuffle.html +5 -5
- package/public/docs/tuning-reference/spark-architecture.html +5 -5
- package/public/docs/tuning-reference/table-formats.html +5 -5
- package/public/docs/user-guide/alternative-log-retrieval/managed-platforms.html +5 -5
- package/public/docs/user-guide/alternative-log-retrieval/ssh-bastion.html +5 -5
- package/public/docs/user-guide/alternative-log-retrieval.html +5 -5
- package/public/docs/user-guide/getting-started/ci-and-automation.html +16 -11
- package/public/docs/user-guide/getting-started/reading-the-dashboard.html +6 -6
- package/public/docs/user-guide/getting-started/regression-metric-keys.html +5 -5
- package/public/docs/user-guide/getting-started/tuning-thresholds.html +7 -7
- package/public/docs/user-guide/getting-started.html +5 -5
- package/public/docs/user-guide/mcp-tools/compare-runs.html +16 -11
- package/public/docs/user-guide/mcp-tools/diagnose-run.html +28 -7
- package/public/docs/user-guide/mcp-tools/evaluate-budgets.html +6 -6
- package/public/docs/user-guide/mcp-tools/get-finding-documentation.html +5 -5
- package/public/docs/user-guide/mcp-tools/get-finding-evidence.html +7 -6
- package/public/docs/user-guide/mcp-tools/get-reference-doc.html +5 -5
- package/public/docs/user-guide/mcp-tools/get-run-summary.html +5 -5
- package/public/docs/user-guide/mcp-tools/list-runs.html +5 -5
- package/public/docs/user-guide/mcp-tools.html +6 -6
- package/public/docs/user-guide/run-comparison/how-stages-are-matched.html +5 -5
- package/public/docs/user-guide/run-comparison.html +6 -6
- package/public/docs/user-guide/understanding-findings.html +6 -6
- package/public/export-template.html +45 -45
- package/public/index.html +8 -7
- package/public/sample-runs/sample-run.ndjson.gz +0 -0
- package/vendor-core/allocation.js +25 -23
- package/vendor-core/analyzer.js +85 -58
- package/vendor-core/api-merge-writes.js +2 -0
- package/vendor-core/aqe-skew.js +347 -0
- package/vendor-core/cli/budgets.js +6 -4
- package/vendor-core/cli/collect-run-pair.js +89 -0
- package/vendor-core/cli/collect-run-worker.js +16 -0
- package/vendor-core/cli/collect-run.js +1 -2
- package/vendor-core/cli/native-zstd.js +12 -16
- package/vendor-core/comparability.js +234 -0
- package/vendor-core/comparison-output.js +15 -7
- package/vendor-core/comparison-verdict.js +105 -25
- package/vendor-core/core-source-hash.txt +1 -1
- package/vendor-core/core-time-series.js +0 -1
- package/vendor-core/core-usage-locality.js +1 -1
- package/vendor-core/detectors.js +1007 -380
- package/vendor-core/docs-config.js +0 -4
- package/vendor-core/docs-content/detection/cache.md +2 -1
- package/vendor-core/docs-content/detection/cfg.md +19 -11
- package/vendor-core/docs-content/detection/cold.md +2 -1
- package/vendor-core/docs-content/detection/gc.md +7 -3
- package/vendor-core/docs-content/detection/host.md +11 -1
- package/vendor-core/docs-content/detection/mem.md +17 -6
- package/vendor-core/docs-content/detection/part.md +3 -1
- package/vendor-core/docs-content/detection/plan.md +55 -13
- package/vendor-core/docs-content/detection/shfl.md +11 -5
- package/vendor-core/docs-content/detection/skew.md +47 -1
- package/vendor-core/docs-content/detection/slow.md +4 -2
- package/vendor-core/docs-content/detection/spec.md +6 -1
- package/vendor-core/docs-content/detection/spill.md +6 -3
- package/vendor-core/docs-content/detection/strag.md +19 -6
- package/vendor-core/docs-content/detection/tiny.md +9 -3
- package/vendor-core/effective-conf.js +7 -10
- package/vendor-core/efficiency-model.js +2 -2
- package/vendor-core/etl-phases.js +1 -2
- package/vendor-core/event-handlers.js +175 -57
- package/vendor-core/event-schemas.js +21 -16
- package/vendor-core/evidence-availability.js +9 -2
- package/vendor-core/evidence-report.js +118 -35
- package/vendor-core/executor-peaks.js +41 -0
- package/vendor-core/export-data.js +5 -5
- package/vendor-core/finding-identity.js +78 -0
- package/vendor-core/finding-presentation.js +116 -33
- package/vendor-core/finding-types.js +88 -21
- package/vendor-core/format-utils.js +14 -15
- package/vendor-core/html-export.js +1 -1
- package/vendor-core/impact-band.js +2 -2
- package/vendor-core/impact-format.js +47 -19
- package/vendor-core/impact-model.js +26 -15
- package/vendor-core/job-groups.js +4 -4
- package/vendor-core/mcp-server-factory.js +48 -12
- package/vendor-core/mcp-tools.js +179 -24
- package/vendor-core/median.js +2 -2
- package/vendor-core/nested-loop-join.js +73 -0
- package/vendor-core/occupancy.js +1 -2
- package/vendor-core/parser-worker.js +1 -1
- package/vendor-core/plan-graph-model.js +14 -22
- package/vendor-core/plan-node-detail.js +2 -3
- package/vendor-core/plan-summary.js +24 -19
- package/vendor-core/plan-tree-walk.js +6 -8
- package/vendor-core/python-stage.js +12 -1
- package/vendor-core/recommendation-rollup.js +13 -12
- package/vendor-core/redact.js +52 -46
- package/vendor-core/run-aggregates.js +6 -8
- package/vendor-core/run-comparison.js +226 -66
- package/vendor-core/run-identity.js +12 -0
- package/vendor-core/run-interpretation.js +21 -0
- package/vendor-core/run-metrics.js +2 -2
- package/vendor-core/run-output.js +32 -10
- package/vendor-core/run-totals.js +12 -0
- package/vendor-core/run-verdict.js +76 -3
- package/vendor-core/scaling-sim.js +3 -4
- package/vendor-core/snappy-block.js +2 -2
- package/vendor-core/spark-conf.js +168 -0
- package/vendor-core/sql-stages.js +2 -2
- package/vendor-core/stage-alignment.js +2 -12
- package/vendor-core/stage-identity.js +58 -3
- package/vendor-core/stage-label.js +14 -0
- package/vendor-core/stage-plan-nodes.js +17 -3
- package/vendor-core/stage-quantiles.js +141 -9
- package/vendor-core/stage-structure.js +17 -3
- package/vendor-core/threshold-overrides.js +9 -3
- package/vendor-core/types.js +58 -21
- package/vendor-core/wasted-core-hours.js +0 -1
- package/vendor-core/write-targets.js +114 -14
- package/vendor-core/zstd-worker-client.js +1 -1
- package/vendor-core/zstd-worker.js +1 -1
- package/public/assets/AutoscalingChurn-CRA2Mb7F.js +0 -1
- package/public/assets/CacheUtilization-Bh9OY_V9.js +0 -1
- package/public/assets/CachingOpportunity-oCiQwmTJ.js +0 -1
- package/public/assets/ColdStart-BJrtjfmn.js +0 -1
- package/public/assets/ConfigAudit-DMvKxySP.js +0 -1
- package/public/assets/CoreUsageArea-REC2zsKw.js +0 -1
- package/public/assets/Dashboard-BHZ76JCu.js +0 -2
- package/public/assets/DisclosureContext-OO7X1HAx.js +0 -1
- package/public/assets/DuplicatePlanSubtree-BgG-L-cK.js +0 -1
- package/public/assets/ExecutorUtilization-D2KhSVbY.js +0 -1
- package/public/assets/GcPressure-D1BTE8qh.js +0 -1
- package/public/assets/IncompleteRun-DD40xUpy.js +0 -1
- package/public/assets/JobFailures-YwNzCF19.js +0 -1
- package/public/assets/MemoryUtilization-9b8ee_3_.js +0 -1
- package/public/assets/OverBroadcast-CfctS8a8.js +0 -1
- package/public/assets/PartitionHint-CUvbnnxy.js +0 -1
- package/public/assets/PartitionSizing-VG8kBgG5.js +0 -1
- package/public/assets/PlanExplorer-DmubTAOP.js +0 -1
- package/public/assets/PlanGraphRoute-DctPZjS0.js +0 -1
- package/public/assets/RetryWaste-CnpM8bB7.js +0 -1
- package/public/assets/ShuffleIO-D13vGmas.js +0 -1
- package/public/assets/Skew-DlagstYP.js +0 -1
- package/public/assets/SlowHost-cSkLRZ_e.js +0 -1
- package/public/assets/SmallFiles-B-kERMcq.js +0 -1
- package/public/assets/SpeculationWaste-DaysAzXO.js +0 -1
- package/public/assets/Spill-H44Gz4US.js +0 -1
- package/public/assets/StageFailed-DCfii9h_.js +0 -1
- package/public/assets/StageFindingGroup-BXJ0Mj5n.js +0 -1
- package/public/assets/StageHeader--oN2J-oD.js +0 -1
- package/public/assets/StageShape-IZ80CZiJ.js +0 -1
- package/public/assets/StageSlowness--BRIaBc1.js +0 -1
- package/public/assets/Straggler-Ctq0Df_o.js +0 -1
- package/public/assets/TaskFailures-Bks_gqhL.js +0 -1
- package/public/assets/TinyTask-BssfzGFB.js +0 -1
- package/public/assets/UnderBroadcast-DsScrxc-.js +0 -1
- package/public/assets/charts-vendor-0VMaLVLN.js +0 -59
- package/public/assets/finding-anchor-BYW1oz5R.js +0 -1
- package/public/assets/impact-sort-xzy0g-Vh.js +0 -1
- package/public/assets/index-DhpRGMZc.css +0 -2
- package/public/assets/index-DwqkFUFk.js +0 -14
- package/public/assets/plan-graph-vendor-BYl-AzN5.js +0 -7
- package/public/assets/react-vendor-DfpuPQZf.js +0 -9
- package/public/assets/select-C8edjYTq.js +0 -1
- package/public/assets/usePagedRows-CsuUFeYx.js +0 -1
- package/public/assets/useRecentFiles-BvkKo3JN.js +0 -11
- package/public/assets/useSortMode-joX71BZN.js +0 -1
- package/public/docs/assets/chunks/@localSearchIndexroot.tE5pz54z.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_board-widgets.md.YQcj5EI4.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_board-widgets.md.YQcj5EI4.lean.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_board-widgets_plan-advisor.md.RlzvUmNm.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_board-widgets_visual-system-and-templating.md.wulOueeB.lean.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_detector-contract.md.N4hhenrA.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_detector-contract_plan-attribution.md.yNex4bys.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_detector-contract_plan-attribution.md.yNex4bys.lean.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_detector-contract_threshold-tables.md.Dp3c40rx.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_drill-down_plan-graph.md.DwOozIEs.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.DfvbqEUu.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_impact-estimation_caveats-tuning-and-coverage.md.Cwm1jHmn.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_impact-estimation_stage-union-rollup.md.DdCGtAGu.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_impact-estimation_stage-union-rollup.md.DdCGtAGu.lean.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_overview.md.C1sSvm9i.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_state-and-history_run-comparison-internals.md.CZA0JLtV.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.BOZ1azXG.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_widget-rendering_findings-tab.md.BQXycK6G.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_widget-rendering_findings-tab.md.BQXycK6G.lean.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_worker-protocol.md.BAnhi2-d.lean.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_worker-protocol_cli-and-mcp.md.BX1XnFbq.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_worker-protocol_evidence-report.md.CiQl1MLb.js +0 -1
- package/public/docs/assets/contributor-guide_contributing.md.CQMvGwFt.js +0 -1
- package/public/docs/assets/contributor-guide_development-setup.md.DQdzera7.js +0 -15
- package/public/docs/assets/contributor-guide_development-setup.md.DQdzera7.lean.js +0 -1
- package/public/docs/assets/contributor-guide_testing.md.By6XGR9a.js +0 -1
- package/public/docs/assets/user-guide_getting-started_ci-and-automation.md.BEepbDDL.js +0 -24
- package/public/docs/assets/user-guide_getting-started_reading-the-dashboard.md.Obf3W3__.js +0 -1
- package/public/docs/assets/user-guide_mcp-tools_compare-runs.md.Cc_gLoix.lean.js +0 -1
- package/public/docs/assets/user-guide_mcp-tools_diagnose-run.md.BP1vnHA5.js +0 -23
- package/public/docs/assets/user-guide_run-comparison.md.BUALkohi.js +0 -1
- package/public/docs/assets/user-guide_run-comparison.md.BUALkohi.lean.js +0 -1
- package/public/docs/assets/user-guide_understanding-findings.md.3C7KCEZO.js +0 -1
- package/public/docs/assets/user-guide_understanding-findings.md.3C7KCEZO.lean.js +0 -1
- /package/public/docs/assets/{contributor-guide_architecture_board-widgets_caching-and-memory.md.CQgn1t_M.lean.js → contributor-guide_architecture_board-widgets_caching-and-memory.md.BYDo3ClR.lean.js} +0 -0
- /package/public/docs/assets/{contributor-guide_architecture_board-widgets_plan-advisor.md.RlzvUmNm.lean.js → contributor-guide_architecture_board-widgets_plan-advisor.md.ajUAVdbc.lean.js} +0 -0
- /package/public/docs/assets/{contributor-guide_architecture_overview.md.C1sSvm9i.lean.js → contributor-guide_architecture_overview.md.nxkpOhgf.lean.js} +0 -0
- /package/public/docs/assets/{contributor-guide_architecture_widget-rendering.md.BOZ1azXG.lean.js → contributor-guide_architecture_widget-rendering.md.BrSz8Ezh.lean.js} +0 -0
- /package/public/docs/assets/{contributor-guide_architecture_widget-rendering_list-sort-mode.md.BuHKgH0C.lean.js → contributor-guide_architecture_widget-rendering_list-sort-mode.md.37hcZJ7q.lean.js} +0 -0
- /package/public/docs/assets/{contributor-guide_architecture_widget-rendering_row-expand-contract.md.ZT89u9f1.lean.js → contributor-guide_architecture_widget-rendering_row-expand-contract.md.mK_VmAWk.lean.js} +0 -0
- /package/public/docs/assets/{contributor-guide_architecture_worker-protocol_core-and-event-validation.md.DSVFPpWb.lean.js → contributor-guide_architecture_worker-protocol_core-and-event-validation.md.DVSXn-bb.lean.js} +0 -0
- /package/public/docs/assets/{contributor-guide_contributing.md.CQMvGwFt.lean.js → contributor-guide_contributing.md.DBW4drtZ.lean.js} +0 -0
- /package/public/docs/assets/{user-guide_getting-started_reading-the-dashboard.md.Obf3W3__.lean.js → user-guide_getting-started_reading-the-dashboard.md.CGaxPX_1.lean.js} +0 -0
- /package/public/docs/assets/{user-guide_mcp-tools.md.C65SjWUL.lean.js → user-guide_mcp-tools.md.49r3L2YE.lean.js} +0 -0
|
@@ -25,7 +25,7 @@ export function buildHtmlExportData(
|
|
|
25
25
|
skippedLines ,
|
|
26
26
|
{ redact, buildId, producer } ,
|
|
27
27
|
) {
|
|
28
|
-
const raw = { appModel, catalog, configFindings: auditConfig(appModel.app) };
|
|
28
|
+
const raw = { appModel, catalog, configFindings: auditConfig(appModel.app, appModel.stages) };
|
|
29
29
|
const run = redact ? redactRunModel(raw.appModel, raw.catalog, raw.configFindings) : raw;
|
|
30
30
|
const interpretation = interpretRun(run.appModel, run.catalog, run.configFindings);
|
|
31
31
|
const provenance = { coreVersion: CORE_VERSION, buildId, producer };
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
|
|
2
2
|
|
|
3
3
|
// The run-wide noise floor, as a share of the run's duration, that grades every wall-clock
|
|
4
|
-
// estimate
|
|
5
|
-
//
|
|
4
|
+
// estimate. skew and straggler default their firing floors to these same figures (detectors.ts), so
|
|
5
|
+
// a finding they admit grades at least warning here. Source status: threshold-tables.md.
|
|
6
6
|
export const IMPACT_FLOOR_PCT_WARN = 0.005;
|
|
7
7
|
export const IMPACT_FLOOR_PCT_CRIT = 0.02;
|
|
8
8
|
|
|
@@ -1,18 +1,33 @@
|
|
|
1
1
|
// Savings figures as every surface prints them: the dashboard's widgets and verdict, and the
|
|
2
2
|
// CLI/MCP evidence report. One formatter set, so units and rounding never drift between paths.
|
|
3
|
-
|
|
3
|
+
// A figure that rounds to zero ("0s", "0.0 core-h") reads as a measured nothing, so every
|
|
4
|
+
// formatter here shows no figure for it (see readsAsZero in format-utils.ts).
|
|
5
|
+
import { asCoreTime, fmtMs, formatRawWaste, formatWallClockRange, readsAsZero } from './format-utils.js';
|
|
4
6
|
|
|
5
7
|
|
|
8
|
+
/** The raw figure as the surfaces print it: a cost-only (`resourceOnly`) task-time figure in core
|
|
9
|
+
* time, anything else as the detector emitted it. `rawWaste` itself stays untouched. */
|
|
10
|
+
function shownRawWaste(estimate ) {
|
|
11
|
+
const rawWaste = estimate.rawWaste;
|
|
12
|
+
return rawWaste && estimate.basis === 'resourceOnly' ? asCoreTime(rawWaste) : rawWaste;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
/** " (modeled)" after a cost-only figure that rests on an assumed constant, so a modeled core
|
|
16
|
+
* time never reads as a measured one. Empty for any other estimate. */
|
|
17
|
+
function certaintyNote(estimate ) {
|
|
18
|
+
return estimate.basis === 'resourceOnly' && estimate.estimateMethod === 'modeled' ? ' (modeled)' : '';
|
|
19
|
+
}
|
|
20
|
+
|
|
6
21
|
/** A finding's one-line savings figure: the wall-clock range for a time-based
|
|
7
22
|
* finding, the raw resource figure for a `resourceOnly` one, or nothing for a
|
|
8
|
-
* purely informational estimate
|
|
9
|
-
* core-h" reads as a measured nothing). */
|
|
23
|
+
* purely informational estimate. */
|
|
10
24
|
export function impactFigure(finding ) {
|
|
11
25
|
const estimate = finding.impactEstimate;
|
|
12
26
|
if (!estimate) return null;
|
|
13
27
|
if (estimate.wallClock) return formatWallClockRange(estimate.wallClock.low, estimate.wallClock.high);
|
|
14
|
-
|
|
15
|
-
|
|
28
|
+
const rawWaste = shownRawWaste(estimate);
|
|
29
|
+
if (rawWaste) {
|
|
30
|
+
const text = formatRawWaste(rawWaste);
|
|
16
31
|
return readsAsZero(text) ? null : text;
|
|
17
32
|
}
|
|
18
33
|
return null;
|
|
@@ -25,7 +40,7 @@ export function rawWasteMeaning(rawWaste )
|
|
|
25
40
|
case 'mbSeconds': return 'of unused executor memory';
|
|
26
41
|
case 'coreHours':
|
|
27
42
|
case 'coreMs': return rawWaste.idle ? 'of idle core capacity' : 'of core time';
|
|
28
|
-
case 'bytes': return 'of extra data written';
|
|
43
|
+
case 'bytes': return rawWaste.reread ? 'of input read by the repeated scans' : 'of extra data written';
|
|
29
44
|
case 'ms': return 'of task time';
|
|
30
45
|
default: return null;
|
|
31
46
|
}
|
|
@@ -40,30 +55,34 @@ export function savingsMeaning(finding ) {
|
|
|
40
55
|
const estimate = finding.impactEstimate;
|
|
41
56
|
if (!estimate) return null;
|
|
42
57
|
if (estimate.wallClock) return 'of run time';
|
|
43
|
-
|
|
58
|
+
const meaning = rawWasteMeaning(estimate.rawWaste);
|
|
59
|
+
return meaning && `${meaning}${certaintyNote(estimate)}`;
|
|
44
60
|
}
|
|
45
61
|
|
|
46
62
|
/** A finding's "Potential savings" figure as the widget board shows it: the
|
|
47
63
|
* wall-clock range, or the raw waste only when there is no wall-clock claim,
|
|
48
64
|
* never both (they would read as two competing numbers). A zero-value
|
|
49
|
-
* estimate is suppressed like an informational one
|
|
50
|
-
*
|
|
65
|
+
* estimate is suppressed like an informational one. `meaning` says what the
|
|
66
|
+
* shown figure counts. */
|
|
51
67
|
export function impactEstimateFigure(estimate ) {
|
|
52
68
|
if (!estimate) return null;
|
|
53
69
|
const highText = estimate.wallClock && estimate.wallClock.high > 0 ? fmtMs(estimate.wallClock.high) : null;
|
|
54
70
|
if (highText && !readsAsZero(highText)) {
|
|
55
71
|
return { text: formatWallClockRange(estimate.wallClock .low, estimate.wallClock .high), meaning: 'of run time' };
|
|
56
72
|
}
|
|
57
|
-
const
|
|
58
|
-
|
|
73
|
+
const rawWaste = shownRawWaste(estimate);
|
|
74
|
+
const rawWasteText = rawWaste && rawWaste.value > 0 ? formatRawWaste(rawWaste) : null;
|
|
75
|
+
if (rawWasteText && !readsAsZero(rawWasteText)) {
|
|
76
|
+
const meaning = rawWasteMeaning(estimate.rawWaste);
|
|
77
|
+
return { text: rawWasteText, meaning: meaning && `${meaning}${certaintyNote(estimate)}` };
|
|
78
|
+
}
|
|
59
79
|
return null;
|
|
60
80
|
}
|
|
61
81
|
|
|
62
82
|
/** Compact single-value form for dense lists (the stage table's finding chips,
|
|
63
83
|
* the stage dialog): the high-end wall-clock figure, or the raw-waste figure
|
|
64
84
|
* when there's no wall-clock claim, or `null` for a purely informational or a
|
|
65
|
-
* zero-value estimate
|
|
66
|
-
* measured nothing). */
|
|
85
|
+
* zero-value estimate. */
|
|
67
86
|
export function impactEstimateCompact(estimate ) {
|
|
68
87
|
if (!estimate) return null;
|
|
69
88
|
if (estimate.wallClock) {
|
|
@@ -71,10 +90,11 @@ export function impactEstimateCompact(estimate )
|
|
|
71
90
|
const text = fmtMs(estimate.wallClock.high);
|
|
72
91
|
return readsAsZero(text) ? null : text;
|
|
73
92
|
}
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
93
|
+
const rawWaste = shownRawWaste(estimate);
|
|
94
|
+
if (rawWaste) {
|
|
95
|
+
if (rawWaste.value <= 0) return null;
|
|
96
|
+
const text = formatRawWaste(rawWaste);
|
|
97
|
+
return readsAsZero(text) ? null : `${text}${certaintyNote(estimate)}`;
|
|
78
98
|
}
|
|
79
99
|
return null;
|
|
80
100
|
}
|
|
@@ -88,14 +108,22 @@ export function impactEstimateCompact(estimate )
|
|
|
88
108
|
* formatting and zero rules as the step's own savings figure, which it does
|
|
89
109
|
* not repeat: the step already shows it. */
|
|
90
110
|
export function estimateProvenance(finding ) {
|
|
111
|
+
const sentence = provenanceSentence(finding);
|
|
112
|
+
const assumptions = finding.impactEstimate?.assumptions;
|
|
113
|
+
if (!sentence || !assumptions) return sentence;
|
|
114
|
+
return `${sentence} ${assumptions[0].toUpperCase()}${assumptions.slice(1)}.`;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
function provenanceSentence(finding ) {
|
|
91
118
|
const estimate = finding.impactEstimate;
|
|
92
119
|
if (!estimate || estimate.estimateMethod === 'none') return null;
|
|
93
120
|
const method = `${estimate.estimateMethod[0].toUpperCase()}${estimate.estimateMethod.slice(1)}`;
|
|
94
|
-
const
|
|
121
|
+
const shown = shownRawWaste(estimate);
|
|
122
|
+
const rawWaste = shown && shown.value > 0 ? shown : null;
|
|
95
123
|
const raw = rawWaste && !readsAsZero(formatRawWaste(rawWaste)) ? formatRawWaste(rawWaste) : null;
|
|
96
124
|
const wallClock = estimate.wallClock;
|
|
97
125
|
if (estimate.basis === 'resourceOnly') {
|
|
98
|
-
return raw ? `${method}; ${raw} wasted, which may not shorten the run.` : null;
|
|
126
|
+
return raw ? `${method}; ${raw} ${rawWasteMeaning(estimate.rawWaste) ?? 'wasted'}, which may not shorten the run.` : null;
|
|
99
127
|
}
|
|
100
128
|
if (!wallClock || wallClock.high <= 0) return null;
|
|
101
129
|
const highText = formatWallClockRange(wallClock.high, wallClock.high);
|
|
@@ -19,6 +19,10 @@ import {
|
|
|
19
19
|
|
|
20
20
|
|
|
21
21
|
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
|
|
22
26
|
|
|
23
27
|
|
|
24
28
|
// Assumed shuffle-network throughput per executor link, ~1 Gbps. Starting assumption, unvalidated.
|
|
@@ -43,10 +47,8 @@ export function measuredTaskOverhead(stage )
|
|
|
43
47
|
|
|
44
48
|
// Both constants above are single-device figures (one NIC, one local disk). A stage's shuffle
|
|
45
49
|
// reads and spills are spread over every executor that ran its tasks, each moving its own share
|
|
46
|
-
// in parallel, so the stage's aggregate bandwidth scales with that executor count.
|
|
47
|
-
//
|
|
48
|
-
// real logs that claimed up to 1,870s of shuffle time on stages whose tasks measured ~0s of
|
|
49
|
-
// shuffle fetch wait (222 of 284 shuffle findings). No executor data falls back to one device.
|
|
50
|
+
// in parallel, so the stage's aggregate bandwidth scales with that executor count. No executor
|
|
51
|
+
// data falls back to one device.
|
|
50
52
|
export function stageIoParallelism(stage ) {
|
|
51
53
|
const executors = Array.isArray(stage.executorStats) ? stage.executorStats.length : 0;
|
|
52
54
|
return Math.max(1, executors);
|
|
@@ -55,9 +57,7 @@ export function stageIoParallelism(stage ) {
|
|
|
55
57
|
// Wall-clock the stage's tasks spent blocked fetching shuffle blocks: fetchWaitTime is a
|
|
56
58
|
// cross-task sum like executorRunTime, so dividing by the stage's average concurrency converts it
|
|
57
59
|
// (the gc estimate's conversion). Null when the stage has no run time or duration to convert with.
|
|
58
|
-
//
|
|
59
|
-
// this, with about zero fetch wait on 5 of the 10 non-info ones: reads that overlapped compute
|
|
60
|
-
// stalled nothing.
|
|
60
|
+
// The link model caps at this because reads that overlap compute stall nothing.
|
|
61
61
|
export function fetchWaitWallClockMs(stage ) {
|
|
62
62
|
const fetchWaitMs = stage.fetchWaitTime;
|
|
63
63
|
const runTimeMs = stage.executorRunTime ?? 0;
|
|
@@ -66,10 +66,9 @@ export function fetchWaitWallClockMs(stage ) {
|
|
|
66
66
|
return fetchWaitMs / (runTimeMs / durationMs);
|
|
67
67
|
}
|
|
68
68
|
|
|
69
|
-
// Wall-clock a stage's wasted (retried) attempts cost it. Each one
|
|
70
|
-
// attempts of different tasks
|
|
71
|
-
// at once
|
|
72
|
-
// at once, claimed as 146.6s). So the stage lost at most its longest retry chain, the most attempts
|
|
69
|
+
// Wall-clock a stage's wasted (retried) attempts cost it. Each one delays only its own task, and
|
|
70
|
+
// attempts of different tasks run side by side: one lost executor fails every task it was running
|
|
71
|
+
// at once. So the stage loses at most its longest retry chain, the most attempts
|
|
73
72
|
// one task wasted (its highest attempt number + 1) times the mean wasted attempt, or their summed
|
|
74
73
|
// time spread over its slots, whichever is larger. Without a sample of every wasted attempt
|
|
75
74
|
// (retryTaskSamples is capped), the chain isn't known: the summed time.
|
|
@@ -99,10 +98,8 @@ export const EXECUTOR_STARTUP_OVERHEAD_MS = 15000;
|
|
|
99
98
|
// Assumed re-read throughput, shared by cachingOpportunity and cacheUtilization.
|
|
100
99
|
export const RE_READ_THROUGHPUT_BPS = 125_000_000;
|
|
101
100
|
|
|
102
|
-
// Below this share of executorRunTime spent on CPU, a stage's tasks
|
|
103
|
-
// outside Spark:
|
|
104
|
-
// file listing or a Delta log read, while file writes, which more partitions do parallelize,
|
|
105
|
-
// start at 2%.
|
|
101
|
+
// Below this share of executorRunTime spent on CPU, a stage's tasks are idle, waiting on something
|
|
102
|
+
// outside Spark (rationale: impact-estimation/caveats-tuning-and-coverage.md, Idle CPU share).
|
|
106
103
|
const IDLE_CPU_SHARE_MAX = 0.01;
|
|
107
104
|
|
|
108
105
|
// True when the stage's tasks spent under IDLE_CPU_SHARE_MAX of their run time on CPU. False when
|
|
@@ -130,6 +127,20 @@ export function costOnly(estimateMethod , rawWaste
|
|
|
130
127
|
: { basis: 'informational', wallClock: null, estimateMethod };
|
|
131
128
|
}
|
|
132
129
|
|
|
130
|
+
/** The estimate with what its model assumes and leaves out (`ImpactEstimate.assumptions`). */
|
|
131
|
+
export function withAssumptions(estimate , assumptions ) {
|
|
132
|
+
return { ...estimate, assumptions };
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/** The throughput assumption behind a per-executor I/O model (stageIoParallelism), in the words
|
|
136
|
+
* `ImpactEstimate.assumptions` carries: the rate and how many executors it was spread over. */
|
|
137
|
+
export function ioModelAssumption(bytesPerSecond , stage ) {
|
|
138
|
+
const executors = Array.isArray(stage.executorStats) ? stage.executorStats.length : 0;
|
|
139
|
+
const rate = `${Math.round(bytesPerSecond / 1e6)} MB/s`;
|
|
140
|
+
if (executors === 0) return `assumes ${rate} on one device, as the stage has no per-executor data`;
|
|
141
|
+
return `assumes ${rate} per executor over ${executors} ${executors === 1 ? 'executor' : 'executors'}`;
|
|
142
|
+
}
|
|
143
|
+
|
|
133
144
|
/** The estimate() of an entry whose findings have no waste model at all. */
|
|
134
145
|
export function noWasteModel() {
|
|
135
146
|
return costOnly('none');
|
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
//
|
|
2
|
-
//
|
|
3
|
-
//
|
|
1
|
+
// Concurrent-job-group reliability guard. Groups jobs by SQL execution id (jobs with none form
|
|
2
|
+
// singleton groups). If two DIFFERENT groups' intervals overlap, wall-clock-based estimates become
|
|
3
|
+
// unreliable. Jobs WITHIN a group overlapping (AQE) is normal.
|
|
4
4
|
export function checkConcurrentJobGroups(jobs ) {
|
|
5
5
|
const list = jobs ? [...jobs.values()] : [];
|
|
6
6
|
// Build one interval per group: [min submission, max completion].
|
|
7
|
-
const groups = new Map();
|
|
7
|
+
const groups = new Map();
|
|
8
8
|
let singletonSeq = 0;
|
|
9
9
|
for (const j of list) {
|
|
10
10
|
if (j.submissionTime == null || j.completionTime == null) continue;
|
|
@@ -2,10 +2,12 @@ import { z } from 'zod';
|
|
|
2
2
|
import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
|
3
3
|
|
|
4
4
|
import {
|
|
5
|
-
resolveOrCreateRun, diagnoseRun, getRunSummary, compareRuns, getFindingEvidence, getFindingDocumentation, getReferenceDoc, evaluateBudgetsForRun,
|
|
5
|
+
resolveOrCreateRun, diagnoseRun, diagnoseHeadline, getRunSummary, compareRuns, getFindingEvidence, getFindingDocumentation, getReferenceDoc, evaluateBudgetsForRun,
|
|
6
|
+
|
|
6
7
|
} from './mcp-tools.js';
|
|
7
8
|
import { listRuns } from './list-runs.js';
|
|
8
9
|
|
|
10
|
+
|
|
9
11
|
import { MAX_NORMALIZE_PATTERN_LENGTH, MAX_NORMALIZE_PATTERNS } from './stage-alignment.js';
|
|
10
12
|
|
|
11
13
|
const sourceSchema = z.union([
|
|
@@ -42,7 +44,7 @@ function toolErrorResult(error )
|
|
|
42
44
|
};
|
|
43
45
|
}
|
|
44
46
|
|
|
45
|
-
//
|
|
47
|
+
// compareRuns may carry a `markdown` field (only when the caller asked for
|
|
46
48
|
// format: 'md'); it never belongs in structuredContent (whose shape must stay stable regardless of
|
|
47
49
|
// format), so it's stripped here and, when present, becomes content[0].text instead of JSON.stringify.
|
|
48
50
|
function toolResultWithMarkdown (promise ) {
|
|
@@ -55,16 +57,40 @@ function toolResultWithMarkdown (promise
|
|
|
55
57
|
);
|
|
56
58
|
}
|
|
57
59
|
|
|
60
|
+
// diagnose_run's own mapping: with `format: 'md'` the Markdown is the content and structuredContent
|
|
61
|
+
// shrinks to a few headline fields, so a client that stores it does not store the whole report.
|
|
62
|
+
function diagnoseToolResult(promise ) {
|
|
63
|
+
return promise.then(
|
|
64
|
+
(value) => {
|
|
65
|
+
const { markdown, ...structuredContent } = value;
|
|
66
|
+
return markdown !== undefined
|
|
67
|
+
? toCallToolResult(markdown, diagnoseHeadline(value))
|
|
68
|
+
: toCallToolResult(JSON.stringify(structuredContent), structuredContent );
|
|
69
|
+
},
|
|
70
|
+
toolErrorResult,
|
|
71
|
+
);
|
|
72
|
+
}
|
|
73
|
+
|
|
58
74
|
// Counterpart for the tools that never produce a markdown field: its own mapping, so there's no
|
|
59
75
|
// need to cast T to pretend it might carry a `markdown` field it never does.
|
|
60
76
|
function toolResult (promise ) {
|
|
61
77
|
return promise.then((value) => toCallToolResult(JSON.stringify(value), value ), toolErrorResult);
|
|
62
78
|
}
|
|
63
79
|
|
|
80
|
+
// The handshake version of a server built without a `generator`: no package to read a version from.
|
|
81
|
+
const UNKNOWN_VERSION = '0.0.0';
|
|
82
|
+
|
|
83
|
+
// Shared by every tool that returns a run handle.
|
|
84
|
+
const RUN_ID_NOTE = 'runId is a handle for the run this server process has cached. It differs per tool by design (compare_runs returns runIdA and runIdB, evaluate_budgets the runId of its first run) and per server process, so match runs by app.id and app.attemptId, not by runId.';
|
|
85
|
+
|
|
64
86
|
/** `thresholds`: the bin's --thresholds overrides, applied to every analyzing tool for the life
|
|
65
|
-
* of the server. Omitted, every tool runs the specification defaults.
|
|
66
|
-
|
|
67
|
-
|
|
87
|
+
* of the server. Omitted, every tool runs the specification defaults.
|
|
88
|
+
* `generator`: the host package's name, version and core build id. The server reports its version
|
|
89
|
+
* in the MCP handshake and every diagnose_run and compare_runs result carries the block. */
|
|
90
|
+
export function createMcpServer(
|
|
91
|
+
{ thresholds, generator } = {},
|
|
92
|
+
) {
|
|
93
|
+
const server = new McpServer({ name: 'sparkforensics', version: generator?.version ?? UNKNOWN_VERSION });
|
|
68
94
|
|
|
69
95
|
server.registerTool('list_runs', {
|
|
70
96
|
description: 'List candidate Spark event-log runs from a local directory or a Spark History Server, before diagnosing one with the other tools.',
|
|
@@ -72,16 +98,22 @@ export function createMcpServer({ thresholds }
|
|
|
72
98
|
}, (params) => toolResult(listRuns(params)));
|
|
73
99
|
|
|
74
100
|
server.registerTool('diagnose_run', {
|
|
75
|
-
description: 'Diagnose a Spark run: the dashboard verdict (title, summary, and the top places to look, ranked by potential savings
|
|
101
|
+
description: 'Diagnose a Spark run: the dashboard verdict (title, summary, and the top places to look, ranked by potential savings, each with its lead finding), findings, an impact-ranked fix recommendation rollup, and the writeTargets, metrics and effectiveConf blocks of the CLI report. view "compact" (the default) lists one short row per finding and leaves out the clean-check and not-checked lists; view "full" returns every finding with its remediation text and evidence, and those lists. Every finding row has a key that stays the same for the same finding in a rerun of the job (compare_runs rows list the same keys), and an id for get_finding_evidence. With format "md" the content is Markdown and structuredContent holds a few headline fields. When the server was started with --thresholds, findings and clean checks from a tuned detector carry tunedThresholds, and their impact estimates are uncalibrated. The result names the Spark application (app) and the build that wrote it (generator). ' + RUN_ID_NOTE,
|
|
76
102
|
inputSchema: {
|
|
77
103
|
...runRefSchema, redact: z.boolean().optional(),
|
|
104
|
+
view: z.enum(['compact', 'full']).optional()
|
|
105
|
+
.describe('compact (default): one short row per finding. full: the CLI report\'s rows, clean checks and not-checked checks.'),
|
|
78
106
|
include: z.array(z.enum(['summary', 'evidenceAvailability', 'detectors'])).optional(),
|
|
79
107
|
impactBand: z.array(z.string()).optional(), type: z.array(z.string()).optional(), stageId: z.number().int().optional(),
|
|
108
|
+
minBand: z.enum(['critical', 'warning', 'info']).optional()
|
|
109
|
+
.describe('List only findings at this band or a more severe one.'),
|
|
110
|
+
maxFindings: z.number().int().nonnegative().optional()
|
|
111
|
+
.describe('List at most this many findings, most severe first. findingsOmitted counts the ones left out; each verdict step still carries its lead finding.'),
|
|
80
112
|
...formatSchema,
|
|
81
113
|
},
|
|
82
|
-
}, ({ source, runId, redact, include, format, impactBand, type, stageId }) =>
|
|
114
|
+
}, ({ source, runId, redact, include, format, impactBand, type, stageId, view, minBand, maxFindings }) => diagnoseToolResult(
|
|
83
115
|
resolveOrCreateRun({ source, runId }).then(({ runId: id }) =>
|
|
84
|
-
diagnoseRun(id, { redact, include, markdown: format === 'md', impactBand, type, stageId, thresholds })),
|
|
116
|
+
diagnoseRun(id, { redact, include, markdown: format === 'md', impactBand, type, stageId, thresholds, generator, view: view ?? 'compact', minBand, maxFindings })),
|
|
85
117
|
));
|
|
86
118
|
|
|
87
119
|
server.registerTool('get_run_summary', {
|
|
@@ -92,7 +124,7 @@ export function createMcpServer({ thresholds }
|
|
|
92
124
|
));
|
|
93
125
|
|
|
94
126
|
server.registerTool('compare_runs', {
|
|
95
|
-
description: 'Compare two runs: the verdict, metric deltas (metrics), categorized findings delta (findings)
|
|
127
|
+
description: 'Compare two runs: the verdict, metric deltas (metrics), categorized findings delta (findings), the stage pairs (stagePairs) with the share of executor run time they cover (runtimeCoverage), and comparability: the configuration differences, plan equality, data volume, executor capacity and the factors that bridge the wall-clock difference. confidence is ok, low or insufficient. metricDeltas and findingsDelta repeat metrics and findings under their earlier names and will be removed. The result names both Spark applications (baselineApp, candidateApp) and the build that wrote it (generator). Each findings row lists its stage names in stages and the same names without call-site directories in stageLabels. ' + RUN_ID_NOTE,
|
|
96
128
|
inputSchema: {
|
|
97
129
|
runIdA: z.string().optional(), sourceA: sourceSchema.optional(),
|
|
98
130
|
...secondRunRefSchema,
|
|
@@ -101,14 +133,18 @@ export function createMcpServer({ thresholds }
|
|
|
101
133
|
.describe('Extra blocks to return. stagePairs: one row per paired stage with its deltas, which can be large.'),
|
|
102
134
|
normalizePath: z.array(z.string().max(MAX_NORMALIZE_PATTERN_LENGTH)).max(MAX_NORMALIZE_PATTERNS).optional()
|
|
103
135
|
.describe('Regular expressions whose every match in a plan node\'s text is replaced with a fixed token before stages are paired, so run-specific text (an output directory that differs per run) does not keep the same stage from pairing. A generic pattern, not only for paths. Affects stage pairing, runtimeCoverage and confidence, never findings. A pattern that backtracks catastrophically can stall the server: keep them simple and anchored.'),
|
|
136
|
+
ignoreConf: z.array(z.string().max(MAX_NORMALIZE_PATTERN_LENGTH)).max(MAX_NORMALIZE_PATTERNS).optional()
|
|
137
|
+
.describe('Regular expressions matched against Spark configuration keys. A key that matches is left out of comparability.conf, for keys that differ by design such as a per-run output path. Per-run identity keys (application id, start time, driver address) are always left out.'),
|
|
138
|
+
noiseBand: z.number().min(0).max(1).optional()
|
|
139
|
+
.describe('Share of the baseline value under which a metric row is marked withinNoise (default 0.02). Never changes a row\'s direction.'),
|
|
104
140
|
...formatSchema,
|
|
105
141
|
},
|
|
106
|
-
}, ({ runIdA, sourceA, runIdB, sourceB, redact, include, normalizePath, format }) => toolResultWithMarkdown(
|
|
107
|
-
compareRuns({ runId: runIdA, source: sourceA }, { runId: runIdB, source: sourceB }, { redact, markdown: format === 'md', thresholds, normalizePath, include }),
|
|
142
|
+
}, ({ runIdA, sourceA, runIdB, sourceB, redact, include, normalizePath, ignoreConf, noiseBand, format }) => toolResultWithMarkdown(
|
|
143
|
+
compareRuns({ runId: runIdA, source: sourceA }, { runId: runIdB, source: sourceB }, { redact, markdown: format === 'md', thresholds, normalizePath, ignoreConf, noiseBand, include, generator }),
|
|
108
144
|
));
|
|
109
145
|
|
|
110
146
|
server.registerTool('evaluate_budgets', {
|
|
111
|
-
description: 'Evaluate a run against pass/fail thresholds, optionally against a baseline run for regression budgets. With two runs, absolute budgets apply to the candidate (sourceB/runIdB), matching the CLI. A run with no ApplicationEnd always adds an inconclusive run-complete result.',
|
|
147
|
+
description: 'Evaluate a run against pass/fail thresholds, optionally against a baseline run for regression budgets. With two runs, absolute budgets apply to the candidate (sourceB/runIdB), matching the CLI. A run with no ApplicationEnd always adds an inconclusive run-complete result. ' + RUN_ID_NOTE,
|
|
112
148
|
inputSchema: {
|
|
113
149
|
source: sourceSchema.optional().describe('The run to evaluate. When sourceB/runIdB is given, this is the regression baseline instead, and the absolute budgets apply to sourceB/runIdB.'),
|
|
114
150
|
runId: runRefSchema.runId.describe('Same as `source`, referencing an already-resolved run by id.'),
|