sparkforensics-cli 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -0
- package/bin/sparkforensics-analyze.mjs +113 -48
- package/export-template/docs/404.html +25 -0
- package/export-template/docs/assets/app.CndaAS6v.js +1 -0
- package/export-template/docs/assets/aqe-loop.IwQSATHw.svg +1 -0
- package/export-template/docs/assets/aqe-loop.dark.DGbaxqJE.svg +1 -0
- package/export-template/docs/assets/broadcast-vs-shuffle.Db4WY1XK.svg +1 -0
- package/export-template/docs/assets/broadcast-vs-shuffle.dark.C7Bxs0mG.svg +1 -0
- package/export-template/docs/assets/cache-lifecycle.dark.B-hS7AgU.svg +1 -0
- package/export-template/docs/assets/cache-lifecycle.rEOVYQNU.svg +1 -0
- package/export-template/docs/assets/chunks/@localSearchIndexroot.DNY8bVcl.js +1 -0
- package/export-template/docs/assets/chunks/VPLocalSearchBox.yJbZbsEo.js +9 -0
- package/export-template/docs/assets/chunks/duplicate-plan-subtree.dark.Cdp70QhV.js +1 -0
- package/export-template/docs/assets/chunks/framework.DSg0KOwT.js +20 -0
- package/export-template/docs/assets/chunks/retry-escalation-ladder.dark.DHipdJgZ.js +1 -0
- package/export-template/docs/assets/chunks/theme.Df2VAG9w.js +2 -0
- package/export-template/docs/assets/cold-start-timeline.DxC_Sc7w.svg +1 -0
- package/export-template/docs/assets/cold-start-timeline.dark.CZ17YcAG.svg +1 -0
- package/export-template/docs/assets/columnar-layout.PghGeOEA.svg +1 -0
- package/export-template/docs/assets/columnar-layout.dark.BVNlz0ff.svg +1 -0
- package/export-template/docs/assets/container-memory.DIO0AnIm.svg +1 -0
- package/export-template/docs/assets/container-memory.dark.CP-5zuCl.svg +1 -0
- package/export-template/docs/assets/contributor-guide_architecture_board-widgets.md.B-OsL91z.js +1 -0
- package/export-template/docs/assets/contributor-guide_architecture_board-widgets.md.B-OsL91z.lean.js +1 -0
- package/export-template/docs/assets/contributor-guide_architecture_detector-contract.md.BOeH4d1J.js +1 -0
- package/export-template/docs/assets/contributor-guide_architecture_detector-contract.md.BOeH4d1J.lean.js +1 -0
- package/export-template/docs/assets/contributor-guide_architecture_drill-down.md.BtPdlM7r.js +1 -0
- package/export-template/docs/assets/contributor-guide_architecture_drill-down.md.BtPdlM7r.lean.js +1 -0
- package/export-template/docs/assets/contributor-guide_architecture_impact-estimation.md.DYCDPgkh.js +1 -0
- package/export-template/docs/assets/contributor-guide_architecture_impact-estimation.md.DYCDPgkh.lean.js +1 -0
- package/export-template/docs/assets/contributor-guide_architecture_index.md.3TO9ic6w.js +1 -0
- package/export-template/docs/assets/contributor-guide_architecture_index.md.3TO9ic6w.lean.js +1 -0
- package/export-template/docs/assets/contributor-guide_architecture_overview.md.CehiRmGn.js +1 -0
- package/export-template/docs/assets/contributor-guide_architecture_overview.md.CehiRmGn.lean.js +1 -0
- package/export-template/docs/assets/contributor-guide_architecture_state-and-history.md.m3S3UdMk.js +1 -0
- package/export-template/docs/assets/contributor-guide_architecture_state-and-history.md.m3S3UdMk.lean.js +1 -0
- package/export-template/docs/assets/contributor-guide_architecture_widget-rendering.md.DbqPf2OT.js +1 -0
- package/export-template/docs/assets/contributor-guide_architecture_widget-rendering.md.DbqPf2OT.lean.js +1 -0
- package/export-template/docs/assets/contributor-guide_architecture_worker-protocol.md.B93qJ_tT.js +6 -0
- package/export-template/docs/assets/contributor-guide_architecture_worker-protocol.md.B93qJ_tT.lean.js +1 -0
- package/export-template/docs/assets/contributor-guide_contributing.md.CvRsdr6J.js +1 -0
- package/export-template/docs/assets/contributor-guide_contributing.md.CvRsdr6J.lean.js +1 -0
- package/export-template/docs/assets/contributor-guide_development-setup.md.DvAN_9mK.js +12 -0
- package/export-template/docs/assets/contributor-guide_development-setup.md.DvAN_9mK.lean.js +1 -0
- package/export-template/docs/assets/contributor-guide_testing.md.6rIKqSyY.js +1 -0
- package/export-template/docs/assets/contributor-guide_testing.md.6rIKqSyY.lean.js +1 -0
- package/export-template/docs/assets/dag-stages.DSz_S937.svg +1 -0
- package/export-template/docs/assets/dag-stages.dark.F72UzxH4.svg +1 -0
- package/export-template/docs/assets/driver-executor.D5pQ7YN1.svg +1 -0
- package/export-template/docs/assets/driver-executor.dark.BmX9cPvh.svg +1 -0
- package/export-template/docs/assets/duplicate-plan-subtree.B4cvN6fj.svg +1 -0
- package/export-template/docs/assets/duplicate-plan-subtree.dark.Dw8wS0Ag.svg +1 -0
- package/export-template/docs/assets/index.md.CHJVslga.js +1 -0
- package/export-template/docs/assets/index.md.CHJVslga.lean.js +1 -0
- package/export-template/docs/assets/inter-italic-cyrillic-ext.r48I6akx.woff2 +0 -0
- package/export-template/docs/assets/inter-italic-cyrillic.By2_1cv3.woff2 +0 -0
- package/export-template/docs/assets/inter-italic-greek-ext.1u6EdAuj.woff2 +0 -0
- package/export-template/docs/assets/inter-italic-greek.DJ8dCoTZ.woff2 +0 -0
- package/export-template/docs/assets/inter-italic-latin-ext.CN1xVJS-.woff2 +0 -0
- package/export-template/docs/assets/inter-italic-latin.C2AdPX0b.woff2 +0 -0
- package/export-template/docs/assets/inter-italic-vietnamese.BSbpV94h.woff2 +0 -0
- package/export-template/docs/assets/inter-roman-cyrillic-ext.BBPuwvHQ.woff2 +0 -0
- package/export-template/docs/assets/inter-roman-cyrillic.C5lxZ8CY.woff2 +0 -0
- package/export-template/docs/assets/inter-roman-greek-ext.CqjqNYQ-.woff2 +0 -0
- package/export-template/docs/assets/inter-roman-greek.BBVDIX6e.woff2 +0 -0
- package/export-template/docs/assets/inter-roman-latin-ext.4ZJIpNVo.woff2 +0 -0
- package/export-template/docs/assets/inter-roman-latin.Di8DUHzh.woff2 +0 -0
- package/export-template/docs/assets/inter-roman-vietnamese.BjW4sHH5.woff2 +0 -0
- package/export-template/docs/assets/join-strategy.C_FvrCEo.svg +1 -0
- package/export-template/docs/assets/join-strategy.dark.ChMLnNII.svg +1 -0
- package/export-template/docs/assets/memory-borrowing.BqQRJg0u.svg +1 -0
- package/export-template/docs/assets/memory-borrowing.dark.Yhh20O9C.svg +1 -0
- package/export-template/docs/assets/memory-regions.XHvO7jHG.svg +1 -0
- package/export-template/docs/assets/memory-regions.dark.D4TP9_08.svg +1 -0
- package/export-template/docs/assets/repartition-vs-coalesce.BovLRrpj.svg +1 -0
- package/export-template/docs/assets/repartition-vs-coalesce.dark.BhAczKZQ.svg +1 -0
- package/export-template/docs/assets/retry-escalation-ladder.DyTKJJmZ.svg +1 -0
- package/export-template/docs/assets/retry-escalation-ladder.dark.BdsabtU3.svg +1 -0
- package/export-template/docs/assets/shuffle-map-reduce.KuOEZVmg.svg +1 -0
- package/export-template/docs/assets/shuffle-map-reduce.dark.BgQZnFSb.svg +1 -0
- package/export-template/docs/assets/spill-classification.BU2euYDO.svg +1 -0
- package/export-template/docs/assets/spill-classification.dark.D7i1M40d.svg +1 -0
- package/export-template/docs/assets/style.DXOMCXxn.css +1 -0
- package/export-template/docs/assets/tuning-reference_anti-patterns.md.Df1YMIHu.js +1 -0
- package/export-template/docs/assets/tuning-reference_anti-patterns.md.Df1YMIHu.lean.js +1 -0
- package/export-template/docs/assets/tuning-reference_aqe.md.BIsCtLzm.js +1 -0
- package/export-template/docs/assets/tuning-reference_aqe.md.BIsCtLzm.lean.js +1 -0
- package/export-template/docs/assets/tuning-reference_bottleneck-broadcast-sizing.md.CEstB3Ia.js +1 -0
- package/export-template/docs/assets/tuning-reference_bottleneck-broadcast-sizing.md.CEstB3Ia.lean.js +1 -0
- package/export-template/docs/assets/tuning-reference_bottleneck-cold-start.md.CEuy-72y.js +7 -0
- package/export-template/docs/assets/tuning-reference_bottleneck-cold-start.md.CEuy-72y.lean.js +1 -0
- package/export-template/docs/assets/tuning-reference_bottleneck-duplicate-plan-subtree.md.CIohQDfn.js +1 -0
- package/export-template/docs/assets/tuning-reference_bottleneck-duplicate-plan-subtree.md.CIohQDfn.lean.js +1 -0
- package/export-template/docs/assets/tuning-reference_bottleneck-failures.md.4z5BXGJ2.js +6 -0
- package/export-template/docs/assets/tuning-reference_bottleneck-failures.md.4z5BXGJ2.lean.js +1 -0
- package/export-template/docs/assets/tuning-reference_bottleneck-gc.md.DSxzZRK7.js +6 -0
- package/export-template/docs/assets/tuning-reference_bottleneck-gc.md.DSxzZRK7.lean.js +1 -0
- package/export-template/docs/assets/tuning-reference_bottleneck-job-failure-rate.md.BaJl__1W.js +8 -0
- package/export-template/docs/assets/tuning-reference_bottleneck-job-failure-rate.md.BaJl__1W.lean.js +1 -0
- package/export-template/docs/assets/tuning-reference_bottleneck-memory-utilization.md.DbP-SJZc.js +7 -0
- package/export-template/docs/assets/tuning-reference_bottleneck-memory-utilization.md.DbP-SJZc.lean.js +1 -0
- package/export-template/docs/assets/tuning-reference_bottleneck-retry-waste.md.D5JMjVOt.js +1 -0
- package/export-template/docs/assets/tuning-reference_bottleneck-retry-waste.md.D5JMjVOt.lean.js +1 -0
- package/export-template/docs/assets/tuning-reference_bottleneck-shuffle.md.CM-nTmIH.js +12 -0
- package/export-template/docs/assets/tuning-reference_bottleneck-shuffle.md.CM-nTmIH.lean.js +1 -0
- package/export-template/docs/assets/tuning-reference_bottleneck-skew.md.BdUwiDhn.js +14 -0
- package/export-template/docs/assets/tuning-reference_bottleneck-skew.md.BdUwiDhn.lean.js +1 -0
- package/export-template/docs/assets/tuning-reference_bottleneck-slow-host.md.BlIo6UDW.js +7 -0
- package/export-template/docs/assets/tuning-reference_bottleneck-slow-host.md.BlIo6UDW.lean.js +1 -0
- package/export-template/docs/assets/tuning-reference_bottleneck-small-files.md.B8kloyx8.js +5 -0
- package/export-template/docs/assets/tuning-reference_bottleneck-small-files.md.B8kloyx8.lean.js +1 -0
- package/export-template/docs/assets/tuning-reference_bottleneck-spill.md.PNH7mITt.js +6 -0
- package/export-template/docs/assets/tuning-reference_bottleneck-spill.md.PNH7mITt.lean.js +1 -0
- package/export-template/docs/assets/tuning-reference_bottleneck-straggler.md.DY36fHN5.js +7 -0
- package/export-template/docs/assets/tuning-reference_bottleneck-straggler.md.DY36fHN5.lean.js +1 -0
- package/export-template/docs/assets/tuning-reference_bottleneck-tiny-tasks.md.QTV7O8kU.js +8 -0
- package/export-template/docs/assets/tuning-reference_bottleneck-tiny-tasks.md.QTV7O8kU.lean.js +1 -0
- package/export-template/docs/assets/tuning-reference_bottleneck-utilization.md.DTiueZC3.js +5 -0
- package/export-template/docs/assets/tuning-reference_bottleneck-utilization.md.DTiueZC3.lean.js +1 -0
- package/export-template/docs/assets/tuning-reference_caching.md.B7aQ8asB.js +1 -0
- package/export-template/docs/assets/tuning-reference_caching.md.B7aQ8asB.lean.js +1 -0
- package/export-template/docs/assets/tuning-reference_cluster-config.md.ZVmDGsQ3.js +1 -0
- package/export-template/docs/assets/tuning-reference_cluster-config.md.ZVmDGsQ3.lean.js +1 -0
- package/export-template/docs/assets/tuning-reference_config.md.UvveiWG3.js +1 -0
- package/export-template/docs/assets/tuning-reference_config.md.UvveiWG3.lean.js +1 -0
- package/export-template/docs/assets/tuning-reference_data-formats.md.bjCAWH3N.js +1 -0
- package/export-template/docs/assets/tuning-reference_data-formats.md.bjCAWH3N.lean.js +1 -0
- package/export-template/docs/assets/tuning-reference_index.md.BQ_NooMV.js +1 -0
- package/export-template/docs/assets/tuning-reference_index.md.BQ_NooMV.lean.js +1 -0
- package/export-template/docs/assets/tuning-reference_intro.md.CobD-lGB.js +1 -0
- package/export-template/docs/assets/tuning-reference_intro.md.CobD-lGB.lean.js +1 -0
- package/export-template/docs/assets/tuning-reference_joins.md.BtKs_CuW.js +1 -0
- package/export-template/docs/assets/tuning-reference_joins.md.BtKs_CuW.lean.js +1 -0
- package/export-template/docs/assets/tuning-reference_memory-model.md.DhT-n4y3.js +1 -0
- package/export-template/docs/assets/tuning-reference_memory-model.md.DhT-n4y3.lean.js +1 -0
- package/export-template/docs/assets/tuning-reference_metrics.md.mLOh7Apj.js +1 -0
- package/export-template/docs/assets/tuning-reference_metrics.md.mLOh7Apj.lean.js +1 -0
- package/export-template/docs/assets/tuning-reference_partitioning.md.q0zKF_8X.js +1 -0
- package/export-template/docs/assets/tuning-reference_partitioning.md.q0zKF_8X.lean.js +1 -0
- package/export-template/docs/assets/tuning-reference_pyspark.md.DDCfvN9t.js +6 -0
- package/export-template/docs/assets/tuning-reference_pyspark.md.DDCfvN9t.lean.js +1 -0
- package/export-template/docs/assets/tuning-reference_shuffle.md.BZZ7R4Ix.js +1 -0
- package/export-template/docs/assets/tuning-reference_shuffle.md.BZZ7R4Ix.lean.js +1 -0
- package/export-template/docs/assets/tuning-reference_spark-architecture.md.Dwzm5avO.js +1 -0
- package/export-template/docs/assets/tuning-reference_spark-architecture.md.Dwzm5avO.lean.js +1 -0
- package/export-template/docs/assets/tuning-reference_table-formats.md.D6wj-2dX.js +1 -0
- package/export-template/docs/assets/tuning-reference_table-formats.md.D6wj-2dX.lean.js +1 -0
- package/export-template/docs/assets/udf-execution-models.BUFDICuG.svg +1 -0
- package/export-template/docs/assets/udf-execution-models.dark.YTNS6GDq.svg +1 -0
- package/export-template/docs/assets/user-guide_alternative-log-retrieval.md.sU3KGarf.js +1 -0
- package/export-template/docs/assets/user-guide_alternative-log-retrieval.md.sU3KGarf.lean.js +1 -0
- package/export-template/docs/assets/user-guide_getting-started.md.DtEM37MK.js +3 -0
- package/export-template/docs/assets/user-guide_getting-started.md.DtEM37MK.lean.js +1 -0
- package/export-template/docs/assets/user-guide_mcp-tools.md.C8MiIu7F.js +125 -0
- package/export-template/docs/assets/user-guide_mcp-tools.md.C8MiIu7F.lean.js +1 -0
- package/export-template/docs/assets/user-guide_run-comparison.md.S0TWWmLY.js +1 -0
- package/export-template/docs/assets/user-guide_run-comparison.md.S0TWWmLY.lean.js +1 -0
- package/export-template/docs/assets/user-guide_understanding-findings.md.D0R_Y-R2.js +1 -0
- package/export-template/docs/assets/user-guide_understanding-findings.md.D0R_Y-R2.lean.js +1 -0
- package/export-template/docs/contributor-guide/architecture/board-widgets.html +25 -0
- package/export-template/docs/contributor-guide/architecture/detector-contract.html +25 -0
- package/export-template/docs/contributor-guide/architecture/drill-down.html +25 -0
- package/export-template/docs/contributor-guide/architecture/impact-estimation.html +25 -0
- package/export-template/docs/contributor-guide/architecture/index.html +25 -0
- package/export-template/docs/contributor-guide/architecture/overview.html +25 -0
- package/export-template/docs/contributor-guide/architecture/state-and-history.html +25 -0
- package/export-template/docs/contributor-guide/architecture/widget-rendering.html +25 -0
- package/export-template/docs/contributor-guide/architecture/worker-protocol.html +30 -0
- package/export-template/docs/contributor-guide/contributing.html +25 -0
- package/export-template/docs/contributor-guide/development-setup.html +36 -0
- package/export-template/docs/contributor-guide/testing.html +25 -0
- package/export-template/docs/favicon.svg +4 -0
- package/export-template/docs/hashmap.json +1 -0
- package/export-template/docs/index.html +25 -0
- package/export-template/docs/package.json +1 -0
- package/export-template/docs/tuning-reference/anti-patterns.html +25 -0
- package/export-template/docs/tuning-reference/aqe.html +25 -0
- package/export-template/docs/tuning-reference/bottleneck-broadcast-sizing.html +25 -0
- package/export-template/docs/tuning-reference/bottleneck-cold-start.html +31 -0
- package/export-template/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html +25 -0
- package/export-template/docs/tuning-reference/bottleneck-failures.html +30 -0
- package/export-template/docs/tuning-reference/bottleneck-gc.html +30 -0
- package/export-template/docs/tuning-reference/bottleneck-job-failure-rate.html +32 -0
- package/export-template/docs/tuning-reference/bottleneck-memory-utilization.html +31 -0
- package/export-template/docs/tuning-reference/bottleneck-retry-waste.html +25 -0
- package/export-template/docs/tuning-reference/bottleneck-shuffle.html +36 -0
- package/export-template/docs/tuning-reference/bottleneck-skew.html +38 -0
- package/export-template/docs/tuning-reference/bottleneck-slow-host.html +31 -0
- package/export-template/docs/tuning-reference/bottleneck-small-files.html +29 -0
- package/export-template/docs/tuning-reference/bottleneck-spill.html +30 -0
- package/export-template/docs/tuning-reference/bottleneck-straggler.html +31 -0
- package/export-template/docs/tuning-reference/bottleneck-tiny-tasks.html +32 -0
- package/export-template/docs/tuning-reference/bottleneck-utilization.html +29 -0
- package/export-template/docs/tuning-reference/caching.html +25 -0
- package/export-template/docs/tuning-reference/cluster-config.html +25 -0
- package/export-template/docs/tuning-reference/config.html +25 -0
- package/export-template/docs/tuning-reference/data-formats.html +25 -0
- package/export-template/docs/tuning-reference/index.html +25 -0
- package/export-template/docs/tuning-reference/intro.html +25 -0
- package/export-template/docs/tuning-reference/joins.html +25 -0
- package/export-template/docs/tuning-reference/memory-model.html +25 -0
- package/export-template/docs/tuning-reference/metrics.html +25 -0
- package/export-template/docs/tuning-reference/partitioning.html +25 -0
- package/export-template/docs/tuning-reference/pyspark.html +30 -0
- package/export-template/docs/tuning-reference/shuffle.html +25 -0
- package/export-template/docs/tuning-reference/spark-architecture.html +25 -0
- package/export-template/docs/tuning-reference/table-formats.html +25 -0
- package/export-template/docs/user-guide/alternative-log-retrieval.html +25 -0
- package/export-template/docs/user-guide/getting-started.html +27 -0
- package/export-template/docs/user-guide/mcp-tools.html +149 -0
- package/export-template/docs/user-guide/run-comparison.html +25 -0
- package/export-template/docs/user-guide/understanding-findings.html +25 -0
- package/export-template/docs/vp-icons.css +0 -0
- package/export-template/favicon.svg +4 -0
- package/export-template/index.html +115 -0
- package/export-template/parser-worker-QqyEE4m9.js +64 -0
- package/package.json +16 -3
- package/vendor-core/analyzer.js +74 -74
- package/vendor-core/cli/budgets.js +13 -27
- package/vendor-core/cli/collect-run.js +43 -19
- package/vendor-core/core-count.js +25 -27
- package/vendor-core/core-locality-ratio.js +4 -11
- package/vendor-core/core-time-series.js +6 -12
- package/vendor-core/core-usage-locality.js +3 -4
- package/vendor-core/detectors.js +256 -375
- package/vendor-core/docs-config.js +69 -21
- package/vendor-core/docs-content/chapters/01-intro.md +32 -0
- package/vendor-core/docs-content/chapters/02-spark-architecture.md +76 -0
- package/vendor-core/docs-content/chapters/03-memory-model.md +73 -0
- package/vendor-core/docs-content/chapters/04-partitioning.md +65 -0
- package/vendor-core/docs-content/chapters/05-joins.md +62 -0
- package/vendor-core/docs-content/chapters/06-shuffle.md +59 -0
- package/vendor-core/docs-content/chapters/07-data-formats.md +81 -0
- package/vendor-core/docs-content/chapters/07b-table-formats.md +56 -0
- package/vendor-core/docs-content/chapters/08-caching.md +58 -0
- package/vendor-core/docs-content/chapters/09-pyspark.md +78 -0
- package/vendor-core/docs-content/chapters/10-aqe.md +167 -0
- package/vendor-core/docs-content/chapters/11-cluster-config.md +170 -0
- package/vendor-core/docs-content/chapters/12-anti-patterns.md +171 -0
- package/vendor-core/docs-content/chapters/14-metrics.md +87 -0
- package/vendor-core/docs-content/chapters/15-config.md +93 -0
- package/vendor-core/docs-content/chapters/nav-index.json +370 -0
- package/vendor-core/docs-content/detection/cache.md +6 -0
- package/vendor-core/docs-content/detection/cfg.md +15 -0
- package/vendor-core/docs-content/detection/chrn.md +7 -0
- package/vendor-core/docs-content/detection/cold.md +4 -0
- package/vendor-core/docs-content/detection/cstor.md +4 -0
- package/vendor-core/docs-content/detection/fail.md +5 -0
- package/vendor-core/docs-content/detection/gc.md +4 -0
- package/vendor-core/docs-content/detection/host.md +5 -0
- package/vendor-core/docs-content/detection/incmp.md +6 -0
- package/vendor-core/docs-content/detection/jobs.md +4 -0
- package/vendor-core/docs-content/detection/local.md +7 -0
- package/vendor-core/docs-content/detection/mem.md +10 -0
- package/vendor-core/docs-content/detection/part.md +5 -0
- package/vendor-core/docs-content/detection/plan.md +14 -0
- package/vendor-core/docs-content/detection/retry.md +4 -0
- package/vendor-core/docs-content/detection/sfail.md +5 -0
- package/vendor-core/docs-content/detection/shape.md +5 -0
- package/vendor-core/docs-content/detection/shfl.md +4 -0
- package/vendor-core/docs-content/detection/skew.md +6 -0
- package/vendor-core/docs-content/detection/slow.md +6 -0
- package/vendor-core/docs-content/detection/spec.md +7 -0
- package/vendor-core/docs-content/detection/spill.md +7 -0
- package/vendor-core/docs-content/detection/strag.md +5 -0
- package/vendor-core/docs-content/detection/tiny.md +4 -0
- package/vendor-core/docs-content/detection/util.md +4 -0
- package/vendor-core/docs-content/diagrams/aqe-loop.dark.svg +1 -0
- package/vendor-core/docs-content/diagrams/aqe-loop.svg +1 -0
- package/vendor-core/docs-content/diagrams/broadcast-vs-shuffle.dark.svg +1 -0
- package/vendor-core/docs-content/diagrams/broadcast-vs-shuffle.svg +1 -0
- package/vendor-core/docs-content/diagrams/cache-lifecycle.dark.svg +1 -0
- package/vendor-core/docs-content/diagrams/cache-lifecycle.svg +1 -0
- package/vendor-core/docs-content/diagrams/cold-start-timeline.dark.svg +1 -0
- package/vendor-core/docs-content/diagrams/cold-start-timeline.svg +1 -0
- package/vendor-core/docs-content/diagrams/columnar-layout.dark.svg +1 -0
- package/vendor-core/docs-content/diagrams/columnar-layout.svg +1 -0
- package/vendor-core/docs-content/diagrams/container-memory.dark.svg +1 -0
- package/vendor-core/docs-content/diagrams/container-memory.svg +1 -0
- package/vendor-core/docs-content/diagrams/dag-stages.dark.svg +1 -0
- package/vendor-core/docs-content/diagrams/dag-stages.svg +1 -0
- package/vendor-core/docs-content/diagrams/driver-executor.dark.svg +1 -0
- package/vendor-core/docs-content/diagrams/driver-executor.svg +1 -0
- package/vendor-core/docs-content/diagrams/duplicate-plan-subtree.dark.svg +1 -0
- package/vendor-core/docs-content/diagrams/duplicate-plan-subtree.svg +1 -0
- package/vendor-core/docs-content/diagrams/join-strategy.dark.svg +1 -0
- package/vendor-core/docs-content/diagrams/join-strategy.svg +1 -0
- package/vendor-core/docs-content/diagrams/memory-borrowing.dark.svg +1 -0
- package/vendor-core/docs-content/diagrams/memory-borrowing.svg +1 -0
- package/vendor-core/docs-content/diagrams/memory-regions.dark.svg +1 -0
- package/vendor-core/docs-content/diagrams/memory-regions.svg +1 -0
- package/vendor-core/docs-content/diagrams/repartition-vs-coalesce.dark.svg +1 -0
- package/vendor-core/docs-content/diagrams/repartition-vs-coalesce.svg +1 -0
- package/vendor-core/docs-content/diagrams/retry-escalation-ladder.dark.svg +1 -0
- package/vendor-core/docs-content/diagrams/retry-escalation-ladder.svg +1 -0
- package/vendor-core/docs-content/diagrams/shuffle-map-reduce.dark.svg +1 -0
- package/vendor-core/docs-content/diagrams/shuffle-map-reduce.svg +1 -0
- package/vendor-core/docs-content/diagrams/spill-classification.dark.svg +1 -0
- package/vendor-core/docs-content/diagrams/spill-classification.svg +1 -0
- package/vendor-core/docs-content/diagrams/udf-execution-models.dark.svg +1 -0
- package/vendor-core/docs-content/diagrams/udf-execution-models.svg +1 -0
- package/vendor-core/docs-content/tuning/broadcast-sizing.md +78 -0
- package/vendor-core/docs-content/tuning/cold-start.md +81 -0
- package/vendor-core/docs-content/tuning/duplicate-plan-subtree.md +45 -0
- package/vendor-core/docs-content/tuning/failures.md +124 -0
- package/vendor-core/docs-content/tuning/gc.md +110 -0
- package/vendor-core/docs-content/tuning/job-failure-rate.md +101 -0
- package/vendor-core/docs-content/tuning/memory-utilization.md +58 -0
- package/vendor-core/docs-content/tuning/retry-waste.md +90 -0
- package/vendor-core/docs-content/tuning/shuffle.md +154 -0
- package/vendor-core/docs-content/tuning/skew.md +123 -0
- package/vendor-core/docs-content/tuning/slow-host.md +117 -0
- package/vendor-core/docs-content/tuning/small-files.md +99 -0
- package/vendor-core/docs-content/tuning/spill.md +114 -0
- package/vendor-core/docs-content/tuning/straggler.md +103 -0
- package/vendor-core/docs-content/tuning/tiny-tasks.md +94 -0
- package/vendor-core/docs-content/tuning/utilization.md +90 -0
- package/vendor-core/docs-site-config.js +10 -17
- package/vendor-core/efficiency-model.js +7 -13
- package/vendor-core/etl-phases.js +3 -5
- package/vendor-core/event-handlers.js +232 -134
- package/vendor-core/event-schemas.js +48 -114
- package/vendor-core/evidence-availability.js +5 -10
- package/vendor-core/evidence-report.js +72 -122
- package/vendor-core/export-data.js +48 -0
- package/vendor-core/finding-action-label.js +4 -10
- package/vendor-core/finding-filter-predicate.js +3 -7
- package/vendor-core/finding-generic-recommendation.js +112 -0
- package/vendor-core/finding-names.js +51 -0
- package/vendor-core/format-utils.js +112 -38
- package/vendor-core/impact-band.js +18 -24
- package/vendor-core/impact-estimator.js +38 -74
- package/vendor-core/ingest.js +7 -13
- package/vendor-core/job-groups.js +3 -6
- package/vendor-core/list-runs.js +278 -0
- package/vendor-core/load-vendored.js +6 -12
- package/vendor-core/log-header-peek.js +81 -0
- package/vendor-core/lz4-block.js +4 -6
- package/vendor-core/mcp-server-factory.js +38 -8
- package/vendor-core/mcp-tools.js +105 -76
- package/vendor-core/model-assembler.js +8 -16
- package/vendor-core/occupancy.js +5 -9
- package/vendor-core/parser-worker.js +18 -27
- package/vendor-core/plan-dot.js +2 -5
- package/vendor-core/plan-duration-attribution.js +78 -29
- package/vendor-core/plan-graph-model.js +126 -69
- package/vendor-core/plan-node-detail.js +31 -17
- package/vendor-core/plan-summary.js +19 -8
- package/vendor-core/recommendation-rollup.js +35 -39
- package/vendor-core/redact.js +72 -16
- package/vendor-core/rolling-log-reassembly.js +4 -6
- package/vendor-core/run-comparison.js +65 -70
- package/vendor-core/scaling-sim.js +5 -7
- package/vendor-core/session-snapshot.js +1 -1
- package/vendor-core/shs-fetch.js +4 -6
- package/vendor-core/shs-load.js +9 -13
- package/vendor-core/shs-request.js +1 -1
- package/vendor-core/stage-quantiles.js +14 -0
- package/vendor-core/types.js +78 -18
- package/vendor-core/wasted-core-hours.js +7 -12
|
@@ -1,26 +1,24 @@
|
|
|
1
1
|
// Flattens a resolved planTree into a { nodes, edges } graph shape for the
|
|
2
2
|
// React Flow + Dagre plan graph view ("Plan graph view" in
|
|
3
|
-
// docs-site/contributor-guide/architecture/drill-down.md).
|
|
4
|
-
//
|
|
5
|
-
//
|
|
6
|
-
//
|
|
7
|
-
// half
|
|
8
|
-
//
|
|
9
|
-
// logic over nodes never double-counts one Exchange.
|
|
3
|
+
// docs-site/contributor-guide/architecture/drill-down.md). The Exchange
|
|
4
|
+
// write/read split is done upstream now, in resolvePlanTree
|
|
5
|
+
// (event-handlers.ts): this file just walks the already-split tree and maps
|
|
6
|
+
// each real PlanNode onto a PlanGraphNodeData. sourceNodeId still lets a
|
|
7
|
+
// write half point back to its read half's id, so aggregate logic over
|
|
8
|
+
// nodes can group a pair without special-casing exchangeRole everywhere.
|
|
10
9
|
import { walkPlanTree } from './plan-tree-walk.js';
|
|
11
10
|
import {
|
|
12
11
|
classifyNode,
|
|
13
12
|
buildDurationMap,
|
|
14
13
|
parseOperatorDetail,
|
|
15
14
|
getPrimaryMetric,
|
|
15
|
+
formatPlanMetricValue,
|
|
16
16
|
} from './plan-node-detail.js';
|
|
17
17
|
import { computeSegments, mapSegmentsToStagesForDisplay } from './plan-duration-attribution.js';
|
|
18
18
|
import { stageIdsForSqlExec } from './detectors.js';
|
|
19
|
+
|
|
19
20
|
|
|
20
|
-
|
|
21
|
-
function isExchangeNode(node ) {
|
|
22
|
-
return classifyNode(node.name) === 'exchange';
|
|
23
|
-
}
|
|
21
|
+
|
|
24
22
|
|
|
25
23
|
function makeGraphNode({
|
|
26
24
|
id,
|
|
@@ -30,6 +28,8 @@ function makeGraphNode({
|
|
|
30
28
|
segmentIndex,
|
|
31
29
|
splitRole,
|
|
32
30
|
durationShare,
|
|
31
|
+
exclusiveDurationShare,
|
|
32
|
+
findings,
|
|
33
33
|
}
|
|
34
34
|
|
|
35
35
|
|
|
@@ -38,6 +38,8 @@ function makeGraphNode({
|
|
|
38
38
|
|
|
39
39
|
|
|
40
40
|
|
|
41
|
+
|
|
42
|
+
|
|
41
43
|
) {
|
|
42
44
|
return {
|
|
43
45
|
id,
|
|
@@ -49,21 +51,29 @@ function makeGraphNode({
|
|
|
49
51
|
segmentIndex,
|
|
50
52
|
splitRole,
|
|
51
53
|
durationShare: durationShare ?? null,
|
|
54
|
+
exclusiveDurationShare: exclusiveDurationShare ?? null,
|
|
55
|
+
findings,
|
|
56
|
+
metrics: (planNode.metrics ?? []).map((m) => ({ name: m.name, value: formatPlanMetricValue(m) })),
|
|
57
|
+
// Drop a detail that only repeats the operator name (no extra information
|
|
58
|
+
// for the card); parseOperatorDetail already treats that case as empty.
|
|
59
|
+
detailText: planNode.detail && planNode.detail !== planNode.name ? planNode.detail : '',
|
|
52
60
|
};
|
|
53
61
|
}
|
|
54
62
|
|
|
55
63
|
export function buildPlanGraphModel(
|
|
56
64
|
planTree ,
|
|
57
|
-
opts
|
|
65
|
+
opts
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
,
|
|
58
72
|
) {
|
|
59
|
-
const { scope, stageId, appModel } = opts;
|
|
73
|
+
const { scope, stageId, appModel, findings = [], durationMode = 'exclusive' } = opts;
|
|
60
74
|
if (!planTree) return { nodes: [], edges: [], segmentIndex: null, segmentCount: 0, scope: 'full', segmentStageIds: new Map () };
|
|
61
75
|
|
|
62
76
|
const stage = appModel.stages.get(stageId);
|
|
63
|
-
// Coalesce undefined -> null so `sqlExecutionId` has a stable `number | null`
|
|
64
|
-
// type: every branch below narrows off the same variable instead of
|
|
65
|
-
// re-deriving it from `stage`, which TS can't re-narrow after the `stage?.`
|
|
66
|
-
// optional-chain expression above collapses to a plain boolean.
|
|
67
77
|
const sqlExecutionId = stage?.sqlExecutionId ?? null;
|
|
68
78
|
const sqlExec = sqlExecutionId != null ? appModel.sql.get(sqlExecutionId) : null;
|
|
69
79
|
|
|
@@ -71,20 +81,21 @@ export function buildPlanGraphModel(
|
|
|
71
81
|
const segmentCount = segments.filter(Boolean).length;
|
|
72
82
|
|
|
73
83
|
const durationMap =
|
|
74
|
-
sqlExec && sqlExecutionId != null ? buildDurationMap(planTree, appModel, sqlExec, sqlExecutionId) : null;
|
|
75
|
-
|
|
76
|
-
//
|
|
77
|
-
//
|
|
78
|
-
//
|
|
79
|
-
//
|
|
80
|
-
//
|
|
81
|
-
//
|
|
82
|
-
//
|
|
83
|
-
//
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
84
|
+
sqlExec && sqlExecutionId != null ? buildDurationMap(planTree, appModel, sqlExec, sqlExecutionId, durationMode) : null;
|
|
85
|
+
// A second, always-exclusive map, independent of the requested `durationMode`.
|
|
86
|
+
// Exclusive shares are a true, non-overlapping partition of each paired
|
|
87
|
+
// stage's wall time (computeExclusiveSharesForPairs), so their sum is a
|
|
88
|
+
// stable normalization total. PlanGraphCanvas needs this alongside
|
|
89
|
+
// `durationMap` (the mode-selected values actually shown) because summing
|
|
90
|
+
// `durationMap` itself would double/triple-count under 'inclusive' mode,
|
|
91
|
+
// where an ancestor's share already folds in every descendant's share.
|
|
92
|
+
// durationMode === 'exclusive' already computed the same map above; only
|
|
93
|
+
// redo the work when the requested mode actually differs.
|
|
94
|
+
const exclusiveDurationMap =
|
|
95
|
+
durationMode === 'exclusive'
|
|
96
|
+
? durationMap
|
|
97
|
+
: sqlExec && sqlExecutionId != null ? buildDurationMap(planTree, appModel, sqlExec, sqlExecutionId, 'exclusive') : null;
|
|
98
|
+
|
|
88
99
|
const segmentStageIds = new Map ();
|
|
89
100
|
const linkedStageIds =
|
|
90
101
|
sqlExec && sqlExecutionId != null ? stageIdsForSqlExec(sqlExecutionId, appModel.stages) : [];
|
|
@@ -99,50 +110,99 @@ export function buildPlanGraphModel(
|
|
|
99
110
|
}
|
|
100
111
|
}
|
|
101
112
|
|
|
102
|
-
//
|
|
103
|
-
//
|
|
104
|
-
//
|
|
105
|
-
//
|
|
106
|
-
//
|
|
107
|
-
|
|
108
|
-
|
|
113
|
+
// resolvePlanTree (event-handlers.ts) now prefixes every id with its owning
|
|
114
|
+
// execution, but this filter stays as defense in depth: a finding is scoped
|
|
115
|
+
// to the execution whose graph is being built before it's indexed by node
|
|
116
|
+
// id, so a stale/synthetic finding whose planNodeIds happen to collide with
|
|
117
|
+
// this tree's ids (e.g. hand-built test fixtures, or data from before the
|
|
118
|
+
// prefix existed) still can't badge onto the wrong tree. No linked SQL
|
|
119
|
+
// execution means no findings attach, matching the sqlExec-falsy behavior
|
|
120
|
+
// used elsewhere in this function (durationMap, linkedStageIds).
|
|
121
|
+
const findingsByNodeId = new Map ();
|
|
122
|
+
if (sqlExecutionId != null) {
|
|
123
|
+
for (const finding of findings) {
|
|
124
|
+
if (finding.executionId !== sqlExecutionId) continue;
|
|
125
|
+
for (const nodeId of finding.planNodeIds ?? []) {
|
|
126
|
+
const list = findingsByNodeId.get(nodeId);
|
|
127
|
+
if (list) list.push(finding);
|
|
128
|
+
else findingsByNodeId.set(nodeId, [finding]);
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
|
|
109
133
|
const allNodes = [];
|
|
110
134
|
const allEdges = [];
|
|
111
135
|
|
|
136
|
+
// Both halves of one split Exchange share a sourceNodeId (the read half's id:
|
|
137
|
+
// the read half points at itself, the write half at its read parent). Collect
|
|
138
|
+
// them under that key during the walk so the post-pass below can cross-link
|
|
139
|
+
// the pair and mirror the shuffle-boundary bytes onto both halves, even
|
|
140
|
+
// though only one half is visible from any single-segment view.
|
|
141
|
+
const splitPairs = new Map ();
|
|
142
|
+
|
|
112
143
|
walkPlanTree(planTree, (node, parent) => {
|
|
113
|
-
|
|
144
|
+
// resolvePlanTree always sets id; safe downstream of it.
|
|
145
|
+
const id = node.id ;
|
|
114
146
|
const category = classifyNode(node.name);
|
|
115
|
-
// computeSegments (above) already walked this same planTree with the
|
|
116
|
-
// same `{ dedupe: true }` traversal, so every node this walk visits is
|
|
117
|
-
// guaranteed to already have a segOf entry; the `!` reflects that
|
|
118
|
-
// invariant, not an actual nullable field.
|
|
119
147
|
const seg = segOf.get(node) ;
|
|
120
|
-
const parentId = parent ?
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
148
|
+
const parentId = parent ? parent.id : null;
|
|
149
|
+
const sourceNodeId = node.exchangeRole === 'write' && parent ? parent.id : id;
|
|
150
|
+
|
|
151
|
+
const graphNode = makeGraphNode({
|
|
152
|
+
id,
|
|
153
|
+
sourceNodeId,
|
|
154
|
+
planNode: node,
|
|
155
|
+
category,
|
|
156
|
+
segmentIndex: seg,
|
|
157
|
+
splitRole: node.exchangeRole ?? null,
|
|
158
|
+
durationShare: durationMap?.get(node),
|
|
159
|
+
exclusiveDurationShare: exclusiveDurationMap?.get(node),
|
|
160
|
+
findings: findingsByNodeId.get(id) ?? [],
|
|
161
|
+
});
|
|
162
|
+
allNodes.push(graphNode);
|
|
163
|
+
if (node.exchangeRole) {
|
|
164
|
+
const entry = splitPairs.get(sourceNodeId) ?? {};
|
|
165
|
+
entry[node.exchangeRole] = graphNode;
|
|
166
|
+
splitPairs.set(sourceNodeId, entry);
|
|
167
|
+
}
|
|
168
|
+
if (parentId) {
|
|
169
|
+
const edge = { id: `${parentId}->${id}`, source: parentId, target: id };
|
|
170
|
+
// The read->write pairing edge is the shuffle boundary: weight it by the
|
|
171
|
+
// bytes written across it. The write half lives in the producer segment,
|
|
172
|
+
// and its stage's shuffleWriteBytes is exactly the data materialized to
|
|
173
|
+
// this shuffle, falling back to shuffleReadBytes when the write side
|
|
174
|
+
// wasn't captured. Zero/absent on both means no weight, so a broadcast
|
|
175
|
+
// (which moves little by design) or an empty exchange stays a plain edge.
|
|
176
|
+
if (node.exchangeRole === 'write') {
|
|
177
|
+
const producingStageId = segmentStageIds.get(seg);
|
|
178
|
+
const producingStage = producingStageId != null ? appModel.stages.get(producingStageId) : undefined;
|
|
179
|
+
const shuffleBytes = producingStage?.shuffleWriteBytes || producingStage?.shuffleReadBytes;
|
|
180
|
+
if (shuffleBytes) {
|
|
181
|
+
edge.shuffleBytes = shuffleBytes;
|
|
182
|
+
const entry = splitPairs.get(sourceNodeId) ?? {};
|
|
183
|
+
entry.shuffleBytes = shuffleBytes;
|
|
184
|
+
splitPairs.set(sourceNodeId, entry);
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
allEdges.push(edge);
|
|
143
188
|
}
|
|
144
189
|
}, { dedupe: true });
|
|
145
190
|
|
|
191
|
+
// Cross-link each Exchange pair and mirror the shuffle-boundary bytes onto
|
|
192
|
+
// both halves, so the detail card can reach the partner and show the volume
|
|
193
|
+
// whichever half is open. Runs over the full node set before any scope
|
|
194
|
+
// filtering, so pairedNodeId stays set even when the partner is scoped out.
|
|
195
|
+
for (const { read, write, shuffleBytes } of splitPairs.values()) {
|
|
196
|
+
if (read && write) {
|
|
197
|
+
read.pairedNodeId = write.id;
|
|
198
|
+
write.pairedNodeId = read.id;
|
|
199
|
+
}
|
|
200
|
+
if (shuffleBytes != null) {
|
|
201
|
+
if (read) read.exchangeShuffleBytes = shuffleBytes;
|
|
202
|
+
if (write) write.exchangeShuffleBytes = shuffleBytes;
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
|
|
146
206
|
if (scope === 'full') {
|
|
147
207
|
return { nodes: allNodes, edges: allEdges, segmentIndex: null, segmentCount, scope: 'full', segmentStageIds };
|
|
148
208
|
}
|
|
@@ -153,9 +213,6 @@ export function buildPlanGraphModel(
|
|
|
153
213
|
}
|
|
154
214
|
|
|
155
215
|
if (targetSegmentIndex === null) {
|
|
156
|
-
// Segment lookup failed (no sql/stage linkage, or this stage's segment
|
|
157
|
-
// fell outside the Math.min(segments, stages) truncation), fall back
|
|
158
|
-
// to full-plan scope rather than showing nothing.
|
|
159
216
|
return { nodes: allNodes, edges: allEdges, segmentIndex: null, segmentCount, scope: 'full', segmentStageIds };
|
|
160
217
|
}
|
|
161
218
|
|
|
@@ -1,13 +1,12 @@
|
|
|
1
1
|
// Per-operator plan-node rendering helpers for the stage-detail modal's
|
|
2
|
-
// hierarchical plan tree.
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
// best-effort: unparseable fragments are silently skipped, never thrown.
|
|
2
|
+
// hierarchical plan tree. Pure string-parsing / tree-walking, no DOM.
|
|
3
|
+
// Plan-text parsing is best-effort: unparseable fragments are silently
|
|
4
|
+
// skipped, never thrown.
|
|
6
5
|
|
|
7
6
|
import { pathBasename, formatBytes, formatDuration } from './format-utils.js';
|
|
8
|
-
import { attributeStageDurationToPlan } from './plan-duration-attribution.js';
|
|
7
|
+
import { attributeStageDurationToPlan, attributeStageDurationToPlanInclusive } from './plan-duration-attribution.js';
|
|
9
8
|
import { stageIdsForSqlExec } from './detectors.js';
|
|
10
|
-
|
|
9
|
+
|
|
11
10
|
|
|
12
11
|
const BOILERPLATE_PREFIXES = ['serializefromobject', 'deserializetoobject', 'mapelements',
|
|
13
12
|
'mappartitions', 'inputadapter', 'columnartorow', 'rowtocolumnar', 'union'];
|
|
@@ -27,6 +26,14 @@ export function classifyNode(
|
|
|
27
26
|
return 'transform';
|
|
28
27
|
}
|
|
29
28
|
|
|
29
|
+
export function isExchangeNode(node ) {
|
|
30
|
+
return node.exchangeRole !== undefined || classifyNode(node.name) === 'exchange';
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export function isBroadcastExchangeNode(name ) {
|
|
34
|
+
return name === 'BroadcastExchange';
|
|
35
|
+
}
|
|
36
|
+
|
|
30
37
|
export function parseOperatorDetail(name , simpleString ) {
|
|
31
38
|
if (!simpleString || simpleString === name) return '';
|
|
32
39
|
const n = (name ?? '').toLowerCase();
|
|
@@ -112,11 +119,16 @@ export function parseOperatorDetail(name , simpleString )
|
|
|
112
119
|
return fmtM ? fmtM[1] : '';
|
|
113
120
|
}
|
|
114
121
|
|
|
115
|
-
|
|
122
|
+
// Some operators (e.g. InMemoryTableScan) have no dedicated branch above, so
|
|
123
|
+
// their simpleString falls through here unparsed and repeats the operator
|
|
124
|
+
// name verbatim as a prefix ("InMemoryTableScan [DATE#1, COUNTRY#2, ...]"),
|
|
125
|
+
// which would otherwise duplicate the bold node name shown right next to it.
|
|
126
|
+
const escapedName = name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
127
|
+
const stripped = stripAliases(simpleString).replace(new RegExp(`^${escapedName}\\s*`, 'i'), '').trim();
|
|
116
128
|
return stripped.length > 80 ? stripped.slice(0, 80) + '…' : stripped;
|
|
117
129
|
}
|
|
118
130
|
|
|
119
|
-
export function
|
|
131
|
+
export function formatPlanMetricValue(metric ) {
|
|
120
132
|
const { value, metricType } = metric;
|
|
121
133
|
if (typeof value === 'string') return value;
|
|
122
134
|
if (metricType === 'size') return formatBytes(value);
|
|
@@ -131,17 +143,18 @@ export function getPrimaryMetric(
|
|
|
131
143
|
if (!metrics || metrics.length === 0) return '';
|
|
132
144
|
const priority = ['number of output rows', 'size of files read', 'spill size', 'duration'];
|
|
133
145
|
const found = priority.map(n => metrics.find(m => m.name === n)).find(Boolean) ?? metrics[0];
|
|
134
|
-
return
|
|
146
|
+
return formatPlanMetricValue(found);
|
|
135
147
|
}
|
|
136
148
|
|
|
137
|
-
// Approximate per-node wall-time
|
|
138
|
-
// attributeStageDurationToPlan. Returns null when there's nothing to attribute
|
|
139
|
-
// (no plan tree, or the SQL execution
|
|
149
|
+
// Approximate per-node wall-time via segment attribution (see
|
|
150
|
+
// attributeStageDurationToPlan). Returns null when there's nothing to attribute
|
|
151
|
+
// (no plan tree, or the SQL execution carries no stageIds list).
|
|
140
152
|
export function buildDurationMap(
|
|
141
153
|
planTree ,
|
|
142
154
|
appModel ,
|
|
143
155
|
sqlExec ,
|
|
144
156
|
executionId ,
|
|
157
|
+
durationMode = 'exclusive',
|
|
145
158
|
) {
|
|
146
159
|
if (!planTree || !sqlExec) return null;
|
|
147
160
|
const stageIds = stageIdsForSqlExec(executionId, appModel.stages);
|
|
@@ -151,9 +164,10 @@ export function buildDurationMap(
|
|
|
151
164
|
const s = appModel.stages.get(id);
|
|
152
165
|
if (s) stagesById.set(id, { submittedAt: s.submittedAt, completedAt: s.completedAt });
|
|
153
166
|
}
|
|
154
|
-
// attributeStageDurationToPlan
|
|
155
|
-
//
|
|
156
|
-
//
|
|
157
|
-
|
|
158
|
-
|
|
167
|
+
// attributeStageDurationToPlan reads its 3rd arg's own `.stageIds` field;
|
|
168
|
+
// override it with the derived array rather than the (always-empty on real
|
|
169
|
+
// data) one already on `sqlExec`.
|
|
170
|
+
return durationMode === 'inclusive'
|
|
171
|
+
? attributeStageDurationToPlanInclusive(planTree, stagesById, { ...sqlExec, stageIds })
|
|
172
|
+
: attributeStageDurationToPlan(planTree, stagesById, { ...sqlExec, stageIds });
|
|
159
173
|
}
|
|
@@ -1,11 +1,10 @@
|
|
|
1
1
|
// Best-effort summary of a resolved plan tree (parser-worker `resolvePlanTree`).
|
|
2
|
-
//
|
|
3
|
-
// silently skipped, never surfaced as an error. This replaces the old regex
|
|
4
|
-
// `plan-extractor.js`: the single plan representation is the tree.
|
|
2
|
+
// Unparseable fragments are silently skipped, never surfaced as an error.
|
|
5
3
|
|
|
6
4
|
|
|
7
5
|
import { pathBasename } from './format-utils.js';
|
|
8
6
|
import { walkPlanTree } from './plan-tree-walk.js';
|
|
7
|
+
import { isExchangeNode } from './plan-node-detail.js';
|
|
9
8
|
|
|
10
9
|
|
|
11
10
|
|
|
@@ -70,8 +69,8 @@ export function scanRelationId(name , detail ) {
|
|
|
70
69
|
// 2. Internal Delta metadata (the _delta_log state scan and anonymous
|
|
71
70
|
// commit/checkpoint reads) → null. Location: is often truncated to `Del...`,
|
|
72
71
|
// so also key off DeltaLogFileIndex or the Delta-log action column signature.
|
|
73
|
-
//
|
|
74
|
-
//
|
|
72
|
+
// (Column-signature heuristic: a future Spark renaming those action columns
|
|
73
|
+
// would leak a noise row; acceptable.)
|
|
75
74
|
if (/_delta_log/.test(name) || /_delta_log/.test(detail) ||
|
|
76
75
|
/DeltaLogFileIndex/i.test(name + detail) ||
|
|
77
76
|
/checkpointMetadata#|sidecar#|commitInfo#/.test(name + detail)) {
|
|
@@ -181,7 +180,7 @@ export function summarizePlanTree(planTree ) {
|
|
|
181
180
|
else if (visitScan(name, detail, result)) { /* handled */ }
|
|
182
181
|
else if (/Join/i.test(name) && visitJoin(name, detail, result)) { /* handled */ }
|
|
183
182
|
else if (/Aggregate/i.test(name) && visitAgg(name, detail, result)) { /* handled */ }
|
|
184
|
-
else if (
|
|
183
|
+
else if (n.exchangeRole === 'read') visitExchange(detail, result);
|
|
185
184
|
else if (/^Filter/i.test(name)) {
|
|
186
185
|
const condM = detail.match(/^Filter\s+([\s\S]+)/);
|
|
187
186
|
if (condM && condM[1].trim().length > 1000) pushLongFilterWarning(result, condM[1].trim().length);
|
|
@@ -204,18 +203,25 @@ export function describePlanNode(
|
|
|
204
203
|
const name = node.name ?? '';
|
|
205
204
|
const detail = node.detail ?? '';
|
|
206
205
|
const result = { scans: [], joins: [], aggs: [], exchanges: [], warnings: [] };
|
|
206
|
+
let recognizedType = null;
|
|
207
207
|
|
|
208
208
|
if (/CartesianProduct/i.test(name) || /CartesianProduct/i.test(detail)) {
|
|
209
|
+
recognizedType = 'cartesian';
|
|
209
210
|
pushCrossJoinWarning(result, 'CartesianProduct');
|
|
210
211
|
} else if (visitScan(name, detail, result)) {
|
|
212
|
+
recognizedType = 'scan';
|
|
211
213
|
// handled
|
|
212
214
|
} else if (/Join/i.test(name) && visitJoin(name, detail, result)) {
|
|
215
|
+
recognizedType = 'join';
|
|
213
216
|
// handled (a Cross join type may also have pushed a crossJoin warning)
|
|
214
217
|
} else if (/Aggregate/i.test(name) && visitAgg(name, detail, result)) {
|
|
218
|
+
recognizedType = 'agg';
|
|
215
219
|
// handled (a partial-aggregation pass pushes nothing into result.aggs)
|
|
216
|
-
} else if (
|
|
220
|
+
} else if (isExchangeNode(node)) {
|
|
221
|
+
recognizedType = 'exchange';
|
|
217
222
|
visitExchange(detail, result);
|
|
218
223
|
} else if (/^Filter/i.test(name)) {
|
|
224
|
+
recognizedType = 'filter';
|
|
219
225
|
const condM = detail.match(/^Filter\s+([\s\S]+)/);
|
|
220
226
|
if (condM && condM[1].trim().length > 1000) pushLongFilterWarning(result, condM[1].trim().length);
|
|
221
227
|
}
|
|
@@ -224,10 +230,15 @@ export function describePlanNode(
|
|
|
224
230
|
: result.joins[0] ? 'join'
|
|
225
231
|
: result.aggs[0] ? 'agg'
|
|
226
232
|
: result.exchanges[0] ? 'exchange'
|
|
233
|
+
: recognizedType === 'exchange' && node.exchangeRole !== 'write' ? 'exchange' // ReusedExchange recognized as exchange but no pattern match; write halves stay null
|
|
227
234
|
: null;
|
|
228
235
|
const fields = result.scans[0] ?? result.joins[0] ?? result.aggs[0] ?? result.exchanges[0] ?? null;
|
|
229
236
|
const warning = result.warnings[0] ?? null;
|
|
230
237
|
|
|
231
|
-
if
|
|
238
|
+
// Return null if no recognized type, or if it's agg/filter with no content/warning
|
|
239
|
+
if (!recognizedType) return null;
|
|
240
|
+
if ((recognizedType === 'agg' || recognizedType === 'filter') && !kind && !warning) return null;
|
|
241
|
+
|
|
242
|
+
// For exchange or other types, or if there's content/warning, return the object
|
|
232
243
|
return { kind, warning, ...(fields ?? {}) };
|
|
233
244
|
}
|
|
@@ -1,10 +1,5 @@
|
|
|
1
|
-
// Explicit .ts extensions
|
|
2
|
-
// the
|
|
3
|
-
// under, now that src/evidence-report.ts imports this module too - requires
|
|
4
|
-
// the exact specifier, not an extensionless one a bundler would resolve for
|
|
5
|
-
// you): this file's prior only consumer, src/view/widgets/FixTheseFirst.tsx,
|
|
6
|
-
// went through Vite, which tolerates the extensionless form, so this never
|
|
7
|
-
// surfaced until the CLI/MCP path started importing it.
|
|
1
|
+
// Explicit .ts extensions: plain Node's ESM resolver (the runtime CLI/MCP path
|
|
2
|
+
// runs under) requires the exact specifier, unlike a bundler.
|
|
8
3
|
import { mergeIntervals } from './wall-clock.js';
|
|
9
4
|
import { worstImpactBand, IMPACT_BAND_ORDER } from './format-utils.js';
|
|
10
5
|
|
|
@@ -14,13 +9,10 @@ import { worstImpactBand, IMPACT_BAND_ORDER } from './format-utils.js';
|
|
|
14
9
|
|
|
15
10
|
|
|
16
11
|
|
|
17
|
-
/**
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
* across different findings of the same detector type). Reuses the generic
|
|
22
|
-
* mergeIntervals primitive (src/wall-clock.ts) rather than widening the
|
|
23
|
-
* single-finding-scoped estimateMultiStage/unionMs in src/occupancy.ts. */
|
|
12
|
+
/** Cross-finding grouping: given every stage touched by a group of findings,
|
|
13
|
+
* merge their intervals once across the combined set, instead of summing each
|
|
14
|
+
* finding's own already-clipped wallClock figure (which double-counts
|
|
15
|
+
* concurrent stages across different findings of the same detector type). */
|
|
24
16
|
export function computeStageUnionMs(
|
|
25
17
|
stageIds ,
|
|
26
18
|
stages ,
|
|
@@ -124,36 +116,41 @@ export function buildRecommendationRollup(
|
|
|
124
116
|
const kindDelta = KIND_ORDER[a.kind] - KIND_ORDER[b.kind];
|
|
125
117
|
if (kindDelta !== 0) return kindDelta;
|
|
126
118
|
if (a.kind === 'time' && b.kind === 'time') return b.recoverableMsHigh - a.recoverableMsHigh;
|
|
127
|
-
// resource/count groups carry no shared magnitude to rank by (
|
|
128
|
-
//
|
|
129
|
-
// instead, the same tie-break FixTheseFirst used pre-consolidation.
|
|
119
|
+
// resource/count groups carry no shared magnitude to rank by (comparing
|
|
120
|
+
// incompatible units is forbidden); break ties by worst impact band instead.
|
|
130
121
|
const aImpactBand = IMPACT_BAND_ORDER[worstImpactBand(a.findings) ?? 'info'] ?? 9;
|
|
131
122
|
const bImpactBand = IMPACT_BAND_ORDER[worstImpactBand(b.findings) ?? 'info'] ?? 9;
|
|
132
123
|
return aImpactBand - bImpactBand;
|
|
133
124
|
});
|
|
134
125
|
}
|
|
135
126
|
|
|
136
|
-
/** True when a finding is
|
|
137
|
-
*
|
|
138
|
-
*
|
|
139
|
-
*
|
|
140
|
-
*
|
|
141
|
-
*
|
|
142
|
-
*
|
|
143
|
-
*
|
|
127
|
+
/** True when a finding is real evidence of an issue, as opposed to a mere
|
|
128
|
+
* evidence-unavailable caveat. Shared by `isEligible` below and by anything
|
|
129
|
+
* else that decides whether a REGISTRY type has "something to show" (an
|
|
130
|
+
* active widget card vs. a Clean-checks line): memoryUtilization's
|
|
131
|
+
* memoryBand/dataUnavailable variant reports a missing-evidence caveat
|
|
132
|
+
* (spark.eventLog.logStageExecutorMetrics=true not on for this run), not an
|
|
133
|
+
* optimization or a clean bill of health; the exact same fact already lives
|
|
134
|
+
* in the Evidence availability ledger's own `executorMetrics` entry
|
|
135
|
+
* (src/evidence-availability.ts), so it belongs there, not as its own card. */
|
|
136
|
+
export function isRealFinding(finding ) {
|
|
137
|
+
if (finding.type === 'memoryUtilization' && finding.variant === 'memoryBand' && finding.dataUnavailable) return false;
|
|
138
|
+
return true;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/** True when a finding is a candidate for the "fix these first" ranking.
|
|
142
|
+
* Shared by evidence-report.ts (CLI/MCP path) and FixTheseFirst.tsx (the
|
|
143
|
+
* dashboard widget) so these hardcoded exclusions can't drift between the two
|
|
144
|
+
* surfaces. Doesn't check `REGISTRY[finding.type] != null`: `REGISTRY` lives
|
|
145
|
+
* in a `.tsx` file, not importable from this core module, so FixTheseFirst.tsx
|
|
146
|
+
* layers that extra check on top of this one. */
|
|
144
147
|
export function isEligible(finding ) {
|
|
145
148
|
// incompleteRun is a pipeline-completeness signal, not an addressable fix:
|
|
146
|
-
// hardcoded here by type, not read from Detector.fixEffort.
|
|
147
|
-
//
|
|
148
|
-
//
|
|
149
|
+
// hardcoded here by type, not read from Detector.fixEffort. Unlike
|
|
150
|
+
// `isRealFinding`, this exclusion doesn't apply beyond the rollup: an
|
|
151
|
+
// incompleteRun finding still backs its own ordinary active widget card.
|
|
149
152
|
if (finding.type === 'incompleteRun') return false;
|
|
150
|
-
|
|
151
|
-
// -evidence caveat (spark.eventLog.logStageExecutorMetrics=true not on for
|
|
152
|
-
// this run), not an optimization: the exact same fact already lives in the
|
|
153
|
-
// Evidence availability ledger's own `executorMetrics` entry
|
|
154
|
-
// (src/evidence-availability.ts), so it belongs there, not here.
|
|
155
|
-
if (finding.type === 'memoryUtilization' && finding.variant === 'memoryBand' && finding.dataUnavailable) return false;
|
|
156
|
-
return true;
|
|
153
|
+
return isRealFinding(finding);
|
|
157
154
|
}
|
|
158
155
|
|
|
159
156
|
/** Ranking tier for a single finding: time-based findings (a real `wallClock`
|
|
@@ -186,10 +183,9 @@ export function rankFindings(findings ) {
|
|
|
186
183
|
const tierDelta = tierOf(a) - tierOf(b);
|
|
187
184
|
if (tierDelta !== 0) return tierDelta;
|
|
188
185
|
if (tierOf(a) === TIER_TIME) {
|
|
189
|
-
// Same unit (ms) on both sides
|
|
190
|
-
// .
|
|
191
|
-
//
|
|
192
|
-
// to investigate" target all agree on the same figure.
|
|
186
|
+
// Same unit (ms) on both sides. Rank on .high, matching deriveImpactBand
|
|
187
|
+
// and triage-target.ts, so ordering, badge color, and the "click to
|
|
188
|
+
// investigate" target all agree on the same figure.
|
|
193
189
|
return b.impactEstimate .wallClock .high - a.impactEstimate .wallClock .high;
|
|
194
190
|
}
|
|
195
191
|
return (IMPACT_BAND_ORDER[a.impactBand] ?? 9) - (IMPACT_BAND_ORDER[b.impactBand] ?? 9);
|