sparkforensics-server 0.1.0 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -0
- package/index.js +41 -50
- package/lib/static-files.js +15 -10
- package/package.json +23 -3
- package/public/assets/AutoscalingChurn-B96FKx97.js +1 -0
- package/public/assets/CacheUtilization-lKua3J1x.js +1 -0
- package/public/assets/CachingOpportunity-U0g-8HPV.js +1 -0
- package/public/assets/ColdStart-w8QYTNKo.js +1 -0
- package/public/assets/ConfigAudit-ClZp5BpV.js +1 -0
- package/public/assets/CoreUsageArea-o3kqQwyi.js +1 -0
- package/public/assets/Dashboard-Bjl-MYMm.js +1 -0
- package/public/assets/DuplicatePlanSubtree-Cm_UU3lY.js +1 -0
- package/public/assets/DurationHistogram-ltF-7M0A.js +1 -0
- package/public/assets/EmptyState-BCA3xioa.js +1 -0
- package/public/assets/ExecutorUtilization-Cn69i0uq.js +1 -0
- package/public/assets/GcPressure-CYsmU1d7.js +1 -0
- package/public/assets/ImpactEstimate-BGt-4WX5.js +1 -0
- package/public/assets/IncompleteRun-s6uQ8OLf.js +1 -0
- package/public/assets/JobFailures-DJ2dDvPA.js +1 -0
- package/public/assets/MemoryUtilization-C9G7P94X.js +1 -0
- package/public/assets/OverBroadcast-DYauoYWQ.js +1 -0
- package/public/assets/PartitionSizing-Cz_B0-uG.js +1 -0
- package/public/assets/PlanExplorer-_2oB5ouy.js +1 -0
- package/public/assets/PlanGraphRoute-Cj-GWM8W.js +1 -0
- package/public/assets/RetryWaste-D_m8ASwP.js +1 -0
- package/public/assets/RowPagination-BLsm8I6i.js +1 -0
- package/public/assets/ShuffleIO-pyDhxdaz.js +1 -0
- package/public/assets/Skew-BGLJuZZR.js +1 -0
- package/public/assets/SlowHost-ChdWVvGO.js +1 -0
- package/public/assets/SmallFiles-gTilBtjW.js +1 -0
- package/public/assets/SpeculationWaste-CVzgJnSj.js +1 -0
- package/public/assets/Spill-NGJGQaTB.js +1 -0
- package/public/assets/StageFailed-sGhki53z.js +1 -0
- package/public/assets/StageFindingGroup-AyI7BMd3.js +1 -0
- package/public/assets/StageHeader-BTzGkK8k.js +1 -0
- package/public/assets/StagePill-BlMOW_pU.js +1 -0
- package/public/assets/StageShape-DfOW93Z1.js +1 -0
- package/public/assets/StageSlowness-BMZ-K_5e.js +1 -0
- package/public/assets/Straggler-dffcAN7t.js +1 -0
- package/public/assets/TaskFailures-DGDwUK2A.js +1 -0
- package/public/assets/TinyTask-CfT8zZDr.js +1 -0
- package/public/assets/UnderBroadcast-CiOiJ0k1.js +1 -0
- package/public/assets/WidgetLeadSummary-BdUUi7L5.js +1 -0
- package/public/assets/charts-vendor-0VMaLVLN.js +59 -0
- package/public/assets/downsample-WmNM1qc9.js +1 -0
- package/public/assets/finding-anchor-DEV8JvgH.js +1 -0
- package/public/assets/impact-sort-DxaUheWV.js +1 -0
- package/public/assets/inbox-_QCYkBsP.js +1 -0
- package/public/assets/index-3vlFVNCg.js +9 -0
- package/public/assets/index-CjVklAdW.css +2 -0
- package/public/assets/parser-worker-B2bc0lWe.js +112 -0
- package/public/assets/plan-graph-vendor-B0sEms9Z.js +7 -0
- package/public/assets/plan-graph-vendor-DLioOiRN.css +1 -0
- package/public/assets/react-vendor-D2TLlSzI.js +9 -0
- package/public/assets/rolldown-runtime-hePW80VL.js +1 -0
- package/public/assets/select-kuy1MiAo.js +1 -0
- package/public/assets/toggle-group-CUd-KWkp.js +1 -0
- package/public/assets/usePagedRows-jNUWgVTD.js +1 -0
- package/public/assets/useRecentFiles-D0OA_EOK.js +11 -0
- package/public/assets/useSortMode-BaeAA5cw.js +1 -0
- package/public/docs/404.html +3 -3
- package/public/docs/assets/{app.Bxy4_HVA.js → app.APoq_UBs.js} +1 -1
- package/public/docs/assets/aqe-loop.IwQSATHw.svg +1 -0
- package/public/docs/assets/aqe-loop.dark.DGbaxqJE.svg +1 -0
- package/public/docs/assets/broadcast-vs-shuffle.Db4WY1XK.svg +1 -0
- package/public/docs/assets/broadcast-vs-shuffle.dark.C7Bxs0mG.svg +1 -0
- package/public/docs/assets/cache-lifecycle.dark.B-hS7AgU.svg +1 -0
- package/public/docs/assets/cache-lifecycle.rEOVYQNU.svg +1 -0
- package/public/docs/assets/chunks/@localSearchIndexroot.CvM5fuxe.js +1 -0
- package/public/docs/assets/chunks/{VPLocalSearchBox.L5jNOIR3.js → VPLocalSearchBox.B3TPzlHq.js} +1 -1
- package/public/docs/assets/chunks/duplicate-plan-subtree.dark.Cdp70QhV.js +1 -0
- package/public/docs/assets/chunks/{framework.CNDaxUod.js → framework.DSg0KOwT.js} +1 -1
- package/public/docs/assets/chunks/retry-escalation-ladder.dark.DHipdJgZ.js +1 -0
- package/public/docs/assets/chunks/theme.PavXO_vz.js +2 -0
- package/public/docs/assets/cold-start-timeline.DxC_Sc7w.svg +1 -0
- package/public/docs/assets/cold-start-timeline.dark.CZ17YcAG.svg +1 -0
- package/public/docs/assets/columnar-layout.PghGeOEA.svg +1 -0
- package/public/docs/assets/columnar-layout.dark.BVNlz0ff.svg +1 -0
- package/public/docs/assets/container-memory.DIO0AnIm.svg +1 -0
- package/public/docs/assets/container-memory.dark.CP-5zuCl.svg +1 -0
- package/public/docs/assets/contributor-guide_architecture_board-widgets.md.CWpj01WU.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_board-widgets.md.CWpj01WU.lean.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_detector-contract.md.CgzUsQ6W.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_detector-contract.md.CgzUsQ6W.lean.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_drill-down.md.BtPdlM7r.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_drill-down.md.BtPdlM7r.lean.js +1 -0
- package/public/docs/assets/{contributor-guide_architecture_impact-estimation.md.CzlXOnCq.js → contributor-guide_architecture_impact-estimation.md.CooslVJt.js} +1 -1
- package/public/docs/assets/{contributor-guide_architecture_impact-estimation.md.CzlXOnCq.lean.js → contributor-guide_architecture_impact-estimation.md.CooslVJt.lean.js} +1 -1
- package/public/docs/assets/{contributor-guide_architecture_index.md.94lNbF-4.js → contributor-guide_architecture_index.md.3TO9ic6w.js} +1 -1
- package/public/docs/assets/{contributor-guide_architecture_index.md.94lNbF-4.lean.js → contributor-guide_architecture_index.md.3TO9ic6w.lean.js} +1 -1
- package/public/docs/assets/{contributor-guide_architecture_overview.md.mQFlN66o.js → contributor-guide_architecture_overview.md.CehiRmGn.js} +1 -1
- package/public/docs/assets/{contributor-guide_architecture_overview.md.mQFlN66o.lean.js → contributor-guide_architecture_overview.md.CehiRmGn.lean.js} +1 -1
- package/public/docs/assets/contributor-guide_architecture_state-and-history.md.C-xxn0q7.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_state-and-history.md.C-xxn0q7.lean.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.R27gQrgY.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.R27gQrgY.lean.js +1 -0
- package/public/docs/assets/{contributor-guide_architecture_worker-protocol.md.BaS_L10g.js → contributor-guide_architecture_worker-protocol.md.IbnfNrV3.js} +2 -2
- package/public/docs/assets/{contributor-guide_architecture_worker-protocol.md.BaS_L10g.lean.js → contributor-guide_architecture_worker-protocol.md.IbnfNrV3.lean.js} +1 -1
- package/public/docs/assets/contributor-guide_contributing.md.CvRsdr6J.js +1 -0
- package/public/docs/assets/contributor-guide_contributing.md.CvRsdr6J.lean.js +1 -0
- package/public/docs/assets/{contributor-guide_development-setup.md.CpUMVtIK.js → contributor-guide_development-setup.md.DvAN_9mK.js} +1 -1
- package/public/docs/assets/{contributor-guide_development-setup.md.CpUMVtIK.lean.js → contributor-guide_development-setup.md.DvAN_9mK.lean.js} +1 -1
- package/public/docs/assets/{contributor-guide_testing.md.C0q-BJfl.js → contributor-guide_testing.md.6rIKqSyY.js} +1 -1
- package/public/docs/assets/{contributor-guide_testing.md.C0q-BJfl.lean.js → contributor-guide_testing.md.6rIKqSyY.lean.js} +1 -1
- package/public/docs/assets/dag-stages.DSz_S937.svg +1 -0
- package/public/docs/assets/dag-stages.dark.F72UzxH4.svg +1 -0
- package/public/docs/assets/driver-executor.D5pQ7YN1.svg +1 -0
- package/public/docs/assets/driver-executor.dark.BmX9cPvh.svg +1 -0
- package/public/docs/assets/duplicate-plan-subtree.B4cvN6fj.svg +1 -0
- package/public/docs/assets/duplicate-plan-subtree.dark.Dw8wS0Ag.svg +1 -0
- package/public/docs/assets/{index.md.CETnDZzR.js → index.md.CHJVslga.js} +1 -1
- package/public/docs/assets/{index.md.CETnDZzR.lean.js → index.md.CHJVslga.lean.js} +1 -1
- package/public/docs/assets/join-strategy.C_FvrCEo.svg +1 -0
- package/public/docs/assets/join-strategy.dark.ChMLnNII.svg +1 -0
- package/public/docs/assets/memory-borrowing.BqQRJg0u.svg +1 -0
- package/public/docs/assets/memory-borrowing.dark.Yhh20O9C.svg +1 -0
- package/public/docs/assets/memory-regions.XHvO7jHG.svg +1 -0
- package/public/docs/assets/memory-regions.dark.D4TP9_08.svg +1 -0
- package/public/docs/assets/repartition-vs-coalesce.BovLRrpj.svg +1 -0
- package/public/docs/assets/repartition-vs-coalesce.dark.BhAczKZQ.svg +1 -0
- package/public/docs/assets/retry-escalation-ladder.DyTKJJmZ.svg +1 -0
- package/public/docs/assets/retry-escalation-ladder.dark.BdsabtU3.svg +1 -0
- package/public/docs/assets/shuffle-map-reduce.KuOEZVmg.svg +1 -0
- package/public/docs/assets/shuffle-map-reduce.dark.BgQZnFSb.svg +1 -0
- package/public/docs/assets/spill-classification.BU2euYDO.svg +1 -0
- package/public/docs/assets/spill-classification.dark.D7i1M40d.svg +1 -0
- package/public/docs/assets/{style.D2mY0HUs.css → style.DSixAiZE.css} +1 -1
- package/public/docs/assets/tuning-reference_anti-patterns.md.Df1YMIHu.js +1 -0
- package/public/docs/assets/tuning-reference_anti-patterns.md.Df1YMIHu.lean.js +1 -0
- package/public/docs/assets/tuning-reference_aqe.md.BIsCtLzm.js +1 -0
- package/public/docs/assets/tuning-reference_aqe.md.BIsCtLzm.lean.js +1 -0
- package/public/docs/assets/tuning-reference_bottleneck-broadcast-sizing.md.CEstB3Ia.js +1 -0
- package/public/docs/assets/tuning-reference_bottleneck-broadcast-sizing.md.CEstB3Ia.lean.js +1 -0
- package/public/docs/assets/tuning-reference_bottleneck-cold-start.md.CEuy-72y.js +7 -0
- package/public/docs/assets/tuning-reference_bottleneck-cold-start.md.CEuy-72y.lean.js +1 -0
- package/public/docs/assets/tuning-reference_bottleneck-duplicate-plan-subtree.md.CIohQDfn.js +1 -0
- package/public/docs/assets/tuning-reference_bottleneck-duplicate-plan-subtree.md.CIohQDfn.lean.js +1 -0
- package/public/docs/assets/tuning-reference_bottleneck-failures.md.4z5BXGJ2.js +6 -0
- package/public/docs/assets/tuning-reference_bottleneck-failures.md.4z5BXGJ2.lean.js +1 -0
- package/public/docs/assets/tuning-reference_bottleneck-gc.md.DSxzZRK7.js +6 -0
- package/public/docs/assets/tuning-reference_bottleneck-gc.md.DSxzZRK7.lean.js +1 -0
- package/public/docs/assets/tuning-reference_bottleneck-job-failure-rate.md.BaJl__1W.js +8 -0
- package/public/docs/assets/tuning-reference_bottleneck-job-failure-rate.md.BaJl__1W.lean.js +1 -0
- package/public/docs/assets/tuning-reference_bottleneck-memory-utilization.md.DbP-SJZc.js +7 -0
- package/public/docs/assets/tuning-reference_bottleneck-memory-utilization.md.DbP-SJZc.lean.js +1 -0
- package/public/docs/assets/tuning-reference_bottleneck-retry-waste.md.D5JMjVOt.js +1 -0
- package/public/docs/assets/tuning-reference_bottleneck-retry-waste.md.D5JMjVOt.lean.js +1 -0
- package/public/docs/assets/tuning-reference_bottleneck-shuffle.md.CM-nTmIH.js +12 -0
- package/public/docs/assets/tuning-reference_bottleneck-shuffle.md.CM-nTmIH.lean.js +1 -0
- package/public/docs/assets/tuning-reference_bottleneck-skew.md.BdUwiDhn.js +14 -0
- package/public/docs/assets/tuning-reference_bottleneck-skew.md.BdUwiDhn.lean.js +1 -0
- package/public/docs/assets/tuning-reference_bottleneck-slow-host.md.BlIo6UDW.js +7 -0
- package/public/docs/assets/tuning-reference_bottleneck-slow-host.md.BlIo6UDW.lean.js +1 -0
- package/public/docs/assets/tuning-reference_bottleneck-small-files.md.B8kloyx8.js +5 -0
- package/public/docs/assets/tuning-reference_bottleneck-small-files.md.B8kloyx8.lean.js +1 -0
- package/public/docs/assets/tuning-reference_bottleneck-spill.md.PNH7mITt.js +6 -0
- package/public/docs/assets/tuning-reference_bottleneck-spill.md.PNH7mITt.lean.js +1 -0
- package/public/docs/assets/tuning-reference_bottleneck-straggler.md.DY36fHN5.js +7 -0
- package/public/docs/assets/tuning-reference_bottleneck-straggler.md.DY36fHN5.lean.js +1 -0
- package/public/docs/assets/tuning-reference_bottleneck-tiny-tasks.md.QTV7O8kU.js +8 -0
- package/public/docs/assets/tuning-reference_bottleneck-tiny-tasks.md.QTV7O8kU.lean.js +1 -0
- package/public/docs/assets/tuning-reference_bottleneck-utilization.md.DTiueZC3.js +5 -0
- package/public/docs/assets/tuning-reference_bottleneck-utilization.md.DTiueZC3.lean.js +1 -0
- package/public/docs/assets/tuning-reference_caching.md.B7aQ8asB.js +1 -0
- package/public/docs/assets/tuning-reference_caching.md.B7aQ8asB.lean.js +1 -0
- package/public/docs/assets/tuning-reference_cluster-config.md.ZVmDGsQ3.js +1 -0
- package/public/docs/assets/tuning-reference_cluster-config.md.ZVmDGsQ3.lean.js +1 -0
- package/public/docs/assets/tuning-reference_config.md.UvveiWG3.js +1 -0
- package/public/docs/assets/tuning-reference_config.md.UvveiWG3.lean.js +1 -0
- package/public/docs/assets/tuning-reference_data-formats.md.bjCAWH3N.js +1 -0
- package/public/docs/assets/tuning-reference_data-formats.md.bjCAWH3N.lean.js +1 -0
- package/public/docs/assets/tuning-reference_index.md.BQ_NooMV.js +1 -0
- package/public/docs/assets/tuning-reference_index.md.BQ_NooMV.lean.js +1 -0
- package/public/docs/assets/tuning-reference_intro.md.CobD-lGB.js +1 -0
- package/public/docs/assets/tuning-reference_intro.md.CobD-lGB.lean.js +1 -0
- package/public/docs/assets/tuning-reference_joins.md.BtKs_CuW.js +1 -0
- package/public/docs/assets/tuning-reference_joins.md.BtKs_CuW.lean.js +1 -0
- package/public/docs/assets/tuning-reference_memory-model.md.DhT-n4y3.js +1 -0
- package/public/docs/assets/tuning-reference_memory-model.md.DhT-n4y3.lean.js +1 -0
- package/public/docs/assets/tuning-reference_metrics.md.mLOh7Apj.js +1 -0
- package/public/docs/assets/tuning-reference_metrics.md.mLOh7Apj.lean.js +1 -0
- package/public/docs/assets/tuning-reference_partitioning.md.q0zKF_8X.js +1 -0
- package/public/docs/assets/tuning-reference_partitioning.md.q0zKF_8X.lean.js +1 -0
- package/public/docs/assets/tuning-reference_pyspark.md.DDCfvN9t.js +6 -0
- package/public/docs/assets/tuning-reference_pyspark.md.DDCfvN9t.lean.js +1 -0
- package/public/docs/assets/tuning-reference_shuffle.md.BZZ7R4Ix.js +1 -0
- package/public/docs/assets/tuning-reference_shuffle.md.BZZ7R4Ix.lean.js +1 -0
- package/public/docs/assets/tuning-reference_spark-architecture.md.Dwzm5avO.js +1 -0
- package/public/docs/assets/tuning-reference_spark-architecture.md.Dwzm5avO.lean.js +1 -0
- package/public/docs/assets/tuning-reference_table-formats.md.D6wj-2dX.js +1 -0
- package/public/docs/assets/tuning-reference_table-formats.md.D6wj-2dX.lean.js +1 -0
- package/public/docs/assets/udf-execution-models.BUFDICuG.svg +1 -0
- package/public/docs/assets/udf-execution-models.dark.YTNS6GDq.svg +1 -0
- package/public/docs/assets/user-guide_alternative-log-retrieval.md.B4tPGIal.js +1 -0
- package/public/docs/assets/user-guide_alternative-log-retrieval.md.B4tPGIal.lean.js +1 -0
- package/public/docs/assets/user-guide_getting-started.md.BJvwLEIM.js +3 -0
- package/public/docs/assets/user-guide_getting-started.md.BJvwLEIM.lean.js +1 -0
- package/public/docs/assets/{user-guide_mcp-tools.md.DwYI1d6C.js → user-guide_mcp-tools.md.Vi3RoflJ.js} +35 -5
- package/public/docs/assets/{user-guide_mcp-tools.md.DwYI1d6C.lean.js → user-guide_mcp-tools.md.Vi3RoflJ.lean.js} +1 -1
- package/public/docs/assets/user-guide_run-comparison.md.CQc1aoU8.js +1 -0
- package/public/docs/assets/user-guide_run-comparison.md.CQc1aoU8.lean.js +1 -0
- package/public/docs/assets/user-guide_understanding-findings.md.DL1UDhvR.js +1 -0
- package/public/docs/assets/user-guide_understanding-findings.md.DL1UDhvR.lean.js +1 -0
- package/public/docs/contributor-guide/architecture/board-widgets.html +7 -7
- package/public/docs/contributor-guide/architecture/detector-contract.html +7 -7
- package/public/docs/contributor-guide/architecture/drill-down.html +7 -7
- package/public/docs/contributor-guide/architecture/impact-estimation.html +7 -7
- package/public/docs/contributor-guide/architecture/index.html +7 -7
- package/public/docs/contributor-guide/architecture/overview.html +7 -7
- package/public/docs/contributor-guide/architecture/state-and-history.html +7 -7
- package/public/docs/contributor-guide/architecture/widget-rendering.html +7 -7
- package/public/docs/contributor-guide/architecture/worker-protocol.html +8 -8
- package/public/docs/contributor-guide/contributing.html +7 -7
- package/public/docs/contributor-guide/development-setup.html +7 -7
- package/public/docs/contributor-guide/testing.html +7 -7
- package/public/docs/hashmap.json +1 -1
- package/public/docs/index.html +7 -7
- package/public/docs/tuning-reference/anti-patterns.html +29 -0
- package/public/docs/tuning-reference/aqe.html +29 -0
- package/public/docs/tuning-reference/bottleneck-broadcast-sizing.html +29 -0
- package/public/docs/tuning-reference/bottleneck-cold-start.html +35 -0
- package/public/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html +30 -0
- package/public/docs/tuning-reference/bottleneck-failures.html +35 -0
- package/public/docs/tuning-reference/bottleneck-gc.html +34 -0
- package/public/docs/tuning-reference/bottleneck-job-failure-rate.html +37 -0
- package/public/docs/tuning-reference/bottleneck-memory-utilization.html +35 -0
- package/public/docs/tuning-reference/bottleneck-retry-waste.html +30 -0
- package/public/docs/tuning-reference/bottleneck-shuffle.html +40 -0
- package/public/docs/tuning-reference/bottleneck-skew.html +42 -0
- package/public/docs/tuning-reference/bottleneck-slow-host.html +35 -0
- package/public/docs/tuning-reference/bottleneck-small-files.html +33 -0
- package/public/docs/tuning-reference/bottleneck-spill.html +34 -0
- package/public/docs/tuning-reference/bottleneck-straggler.html +35 -0
- package/public/docs/tuning-reference/bottleneck-tiny-tasks.html +36 -0
- package/public/docs/tuning-reference/bottleneck-utilization.html +34 -0
- package/public/docs/tuning-reference/caching.html +29 -0
- package/public/docs/tuning-reference/cluster-config.html +29 -0
- package/public/docs/tuning-reference/config.html +29 -0
- package/public/docs/tuning-reference/data-formats.html +29 -0
- package/public/docs/tuning-reference/index.html +29 -0
- package/public/docs/tuning-reference/intro.html +29 -0
- package/public/docs/tuning-reference/joins.html +29 -0
- package/public/docs/tuning-reference/memory-model.html +29 -0
- package/public/docs/tuning-reference/metrics.html +29 -0
- package/public/docs/tuning-reference/partitioning.html +29 -0
- package/public/docs/tuning-reference/pyspark.html +34 -0
- package/public/docs/tuning-reference/shuffle.html +29 -0
- package/public/docs/tuning-reference/spark-architecture.html +29 -0
- package/public/docs/tuning-reference/table-formats.html +29 -0
- package/public/docs/user-guide/alternative-log-retrieval.html +29 -0
- package/public/docs/user-guide/getting-started.html +8 -8
- package/public/docs/user-guide/mcp-tools.html +41 -11
- package/public/docs/user-guide/run-comparison.html +7 -7
- package/public/docs/user-guide/understanding-findings.html +7 -7
- package/public/index.html +8 -7
- package/public/sample-runs/sample-run.ndjson.gz +0 -0
- package/vendor-core/analyzer.js +74 -74
- package/vendor-core/cli/budgets.js +13 -27
- package/vendor-core/cli/collect-run.js +43 -19
- package/vendor-core/core-count.js +25 -27
- package/vendor-core/core-locality-ratio.js +4 -11
- package/vendor-core/core-time-series.js +6 -12
- package/vendor-core/core-usage-locality.js +3 -4
- package/vendor-core/detectors.js +395 -389
- package/vendor-core/docs-config.js +69 -21
- package/vendor-core/docs-content/chapters/01-intro.md +32 -0
- package/vendor-core/docs-content/chapters/02-spark-architecture.md +76 -0
- package/vendor-core/docs-content/chapters/03-memory-model.md +73 -0
- package/vendor-core/docs-content/chapters/04-partitioning.md +65 -0
- package/vendor-core/docs-content/chapters/05-joins.md +62 -0
- package/vendor-core/docs-content/chapters/06-shuffle.md +59 -0
- package/vendor-core/docs-content/chapters/07-data-formats.md +81 -0
- package/vendor-core/docs-content/chapters/07b-table-formats.md +56 -0
- package/vendor-core/docs-content/chapters/08-caching.md +58 -0
- package/vendor-core/docs-content/chapters/09-pyspark.md +78 -0
- package/vendor-core/docs-content/chapters/10-aqe.md +167 -0
- package/vendor-core/docs-content/chapters/11-cluster-config.md +170 -0
- package/vendor-core/docs-content/chapters/12-anti-patterns.md +171 -0
- package/vendor-core/docs-content/chapters/14-metrics.md +87 -0
- package/vendor-core/docs-content/chapters/15-config.md +93 -0
- package/vendor-core/docs-content/chapters/nav-index.json +370 -0
- package/vendor-core/docs-content/detection/cache.md +7 -0
- package/vendor-core/docs-content/detection/cfg.md +15 -0
- package/vendor-core/docs-content/detection/chrn.md +9 -0
- package/vendor-core/docs-content/detection/cold.md +4 -0
- package/vendor-core/docs-content/detection/cstor.md +4 -0
- package/vendor-core/docs-content/detection/fail.md +5 -0
- package/vendor-core/docs-content/detection/gc.md +4 -0
- package/vendor-core/docs-content/detection/host.md +5 -0
- package/vendor-core/docs-content/detection/incmp.md +6 -0
- package/vendor-core/docs-content/detection/jobs.md +4 -0
- package/vendor-core/docs-content/detection/local.md +6 -0
- package/vendor-core/docs-content/detection/mem.md +10 -0
- package/vendor-core/docs-content/detection/part.md +5 -0
- package/vendor-core/docs-content/detection/plan.md +14 -0
- package/vendor-core/docs-content/detection/retry.md +4 -0
- package/vendor-core/docs-content/detection/sfail.md +5 -0
- package/vendor-core/docs-content/detection/shape.md +5 -0
- package/vendor-core/docs-content/detection/shfl.md +4 -0
- package/vendor-core/docs-content/detection/skew.md +6 -0
- package/vendor-core/docs-content/detection/slow.md +6 -0
- package/vendor-core/docs-content/detection/spec.md +8 -0
- package/vendor-core/docs-content/detection/spill.md +7 -0
- package/vendor-core/docs-content/detection/strag.md +5 -0
- package/vendor-core/docs-content/detection/tiny.md +4 -0
- package/vendor-core/docs-content/detection/util.md +4 -0
- package/vendor-core/docs-content/diagrams/aqe-loop.dark.svg +1 -0
- package/vendor-core/docs-content/diagrams/aqe-loop.svg +1 -0
- package/vendor-core/docs-content/diagrams/broadcast-vs-shuffle.dark.svg +1 -0
- package/vendor-core/docs-content/diagrams/broadcast-vs-shuffle.svg +1 -0
- package/vendor-core/docs-content/diagrams/cache-lifecycle.dark.svg +1 -0
- package/vendor-core/docs-content/diagrams/cache-lifecycle.svg +1 -0
- package/vendor-core/docs-content/diagrams/cold-start-timeline.dark.svg +1 -0
- package/vendor-core/docs-content/diagrams/cold-start-timeline.svg +1 -0
- package/vendor-core/docs-content/diagrams/columnar-layout.dark.svg +1 -0
- package/vendor-core/docs-content/diagrams/columnar-layout.svg +1 -0
- package/vendor-core/docs-content/diagrams/container-memory.dark.svg +1 -0
- package/vendor-core/docs-content/diagrams/container-memory.svg +1 -0
- package/vendor-core/docs-content/diagrams/dag-stages.dark.svg +1 -0
- package/vendor-core/docs-content/diagrams/dag-stages.svg +1 -0
- package/vendor-core/docs-content/diagrams/driver-executor.dark.svg +1 -0
- package/vendor-core/docs-content/diagrams/driver-executor.svg +1 -0
- package/vendor-core/docs-content/diagrams/duplicate-plan-subtree.dark.svg +1 -0
- package/vendor-core/docs-content/diagrams/duplicate-plan-subtree.svg +1 -0
- package/vendor-core/docs-content/diagrams/join-strategy.dark.svg +1 -0
- package/vendor-core/docs-content/diagrams/join-strategy.svg +1 -0
- package/vendor-core/docs-content/diagrams/memory-borrowing.dark.svg +1 -0
- package/vendor-core/docs-content/diagrams/memory-borrowing.svg +1 -0
- package/vendor-core/docs-content/diagrams/memory-regions.dark.svg +1 -0
- package/vendor-core/docs-content/diagrams/memory-regions.svg +1 -0
- package/vendor-core/docs-content/diagrams/repartition-vs-coalesce.dark.svg +1 -0
- package/vendor-core/docs-content/diagrams/repartition-vs-coalesce.svg +1 -0
- package/vendor-core/docs-content/diagrams/retry-escalation-ladder.dark.svg +1 -0
- package/vendor-core/docs-content/diagrams/retry-escalation-ladder.svg +1 -0
- package/vendor-core/docs-content/diagrams/shuffle-map-reduce.dark.svg +1 -0
- package/vendor-core/docs-content/diagrams/shuffle-map-reduce.svg +1 -0
- package/vendor-core/docs-content/diagrams/spill-classification.dark.svg +1 -0
- package/vendor-core/docs-content/diagrams/spill-classification.svg +1 -0
- package/vendor-core/docs-content/diagrams/udf-execution-models.dark.svg +1 -0
- package/vendor-core/docs-content/diagrams/udf-execution-models.svg +1 -0
- package/vendor-core/docs-content/tuning/broadcast-sizing.md +78 -0
- package/vendor-core/docs-content/tuning/cold-start.md +81 -0
- package/vendor-core/docs-content/tuning/duplicate-plan-subtree.md +45 -0
- package/vendor-core/docs-content/tuning/failures.md +124 -0
- package/vendor-core/docs-content/tuning/gc.md +110 -0
- package/vendor-core/docs-content/tuning/job-failure-rate.md +101 -0
- package/vendor-core/docs-content/tuning/memory-utilization.md +58 -0
- package/vendor-core/docs-content/tuning/retry-waste.md +90 -0
- package/vendor-core/docs-content/tuning/shuffle.md +154 -0
- package/vendor-core/docs-content/tuning/skew.md +123 -0
- package/vendor-core/docs-content/tuning/slow-host.md +117 -0
- package/vendor-core/docs-content/tuning/small-files.md +99 -0
- package/vendor-core/docs-content/tuning/spill.md +114 -0
- package/vendor-core/docs-content/tuning/straggler.md +103 -0
- package/vendor-core/docs-content/tuning/tiny-tasks.md +94 -0
- package/vendor-core/docs-content/tuning/utilization.md +90 -0
- package/vendor-core/docs-site-config.js +10 -17
- package/vendor-core/efficiency-model.js +7 -13
- package/vendor-core/etl-phases.js +3 -5
- package/vendor-core/event-handlers.js +232 -134
- package/vendor-core/event-schemas.js +48 -114
- package/vendor-core/evidence-availability.js +5 -10
- package/vendor-core/evidence-report.js +73 -123
- package/vendor-core/export-data.js +48 -0
- package/vendor-core/finding-action-label.js +4 -10
- package/vendor-core/finding-filter-predicate.js +3 -7
- package/vendor-core/finding-generic-recommendation.js +112 -0
- package/vendor-core/finding-names.js +51 -0
- package/vendor-core/format-utils.js +112 -38
- package/vendor-core/impact-band.js +18 -24
- package/vendor-core/impact-estimator.js +38 -74
- package/vendor-core/ingest.js +7 -13
- package/vendor-core/job-groups.js +3 -6
- package/vendor-core/list-runs.js +278 -0
- package/vendor-core/load-vendored.js +6 -12
- package/vendor-core/log-header-peek.js +81 -0
- package/vendor-core/lz4-block.js +4 -6
- package/vendor-core/mcp-server-factory.js +38 -8
- package/vendor-core/mcp-tools.js +105 -76
- package/vendor-core/model-assembler.js +8 -16
- package/vendor-core/occupancy.js +5 -9
- package/vendor-core/parser-worker.js +20 -29
- package/vendor-core/plan-dot.js +2 -5
- package/vendor-core/plan-duration-attribution.js +78 -29
- package/vendor-core/plan-graph-model.js +126 -69
- package/vendor-core/plan-node-detail.js +31 -17
- package/vendor-core/plan-summary.js +19 -8
- package/vendor-core/recommendation-rollup.js +35 -39
- package/vendor-core/redact.js +72 -16
- package/vendor-core/rolling-log-reassembly.js +4 -6
- package/vendor-core/run-comparison.js +86 -72
- package/vendor-core/scaling-sim.js +5 -7
- package/vendor-core/session-snapshot.js +1 -1
- package/vendor-core/shs-fetch.js +4 -6
- package/vendor-core/shs-load.js +9 -13
- package/vendor-core/shs-request.js +1 -1
- package/vendor-core/stage-quantiles.js +14 -0
- package/vendor-core/types.js +78 -18
- package/vendor-core/wasted-core-hours.js +7 -12
- package/public/assets/AutoscalingChurn-CP5OjN6v.js +0 -1
- package/public/assets/CacheUtilization-BjyawXL6.js +0 -1
- package/public/assets/CachingOpportunity-DV2ABG2G.js +0 -1
- package/public/assets/ConfigAudit-DMiBhAoG.js +0 -1
- package/public/assets/CoreUsageArea-CGYIgPnt.js +0 -1
- package/public/assets/Dashboard-D1_IrWRs.js +0 -12
- package/public/assets/ExecutorTimeline-DkaAGqQT.js +0 -1
- package/public/assets/Failures-Crngoicu.js +0 -1
- package/public/assets/GcPressure-CFXm4BnA.js +0 -1
- package/public/assets/IncompleteRun-BWEJTb13.js +0 -1
- package/public/assets/JobFailures-BmjxCA-h.js +0 -1
- package/public/assets/MemoryUtilization-Bu2uQJbd.js +0 -1
- package/public/assets/PlanExplorer-DKUwF0uz.js +0 -1
- package/public/assets/PlanGraphRoute-BjPSugIG.js +0 -1
- package/public/assets/ShuffleIO--Rur3neu.js +0 -1
- package/public/assets/Spill-DIMpt8c2.js +0 -1
- package/public/assets/TaskSkew-BDr1DbeI.js +0 -1
- package/public/assets/charts-vendor-NMqKV0H4.js +0 -59
- package/public/assets/index-4id52G-A.js +0 -9
- package/public/assets/index-D9kwVYIO.css +0 -1
- package/public/assets/parser-worker-00O9WEQC.js +0 -65
- package/public/assets/plan-graph-vendor-BnuhLJ6X.css +0 -1
- package/public/assets/plan-graph-vendor-vZQHG2EJ.js +0 -7
- package/public/assets/react-vendor-u4pyvC5Y.js +0 -9
- package/public/assets/useRecentFiles-46iBU9zJ.js +0 -5
- package/public/docs/assets/chunks/@localSearchIndexroot.DclgKtlT.js +0 -1
- package/public/docs/assets/chunks/theme.CoT6y5bf.js +0 -2
- package/public/docs/assets/contributor-guide_architecture_board-widgets.md.BtjoE_1k.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_board-widgets.md.BtjoE_1k.lean.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_detector-contract.md.DQyCHQNM.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_detector-contract.md.DQyCHQNM.lean.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_drill-down.md.BKfyDQck.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_drill-down.md.BKfyDQck.lean.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_state-and-history.md.DaqttKgQ.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_state-and-history.md.DaqttKgQ.lean.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.Bk6Iyj9S.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.Bk6Iyj9S.lean.js +0 -1
- package/public/docs/assets/contributor-guide_contributing.md.C-lKs70I.js +0 -1
- package/public/docs/assets/contributor-guide_contributing.md.C-lKs70I.lean.js +0 -1
- package/public/docs/assets/user-guide_getting-started.md.BJdD7a7_.js +0 -3
- package/public/docs/assets/user-guide_getting-started.md.BJdD7a7_.lean.js +0 -1
- package/public/docs/assets/user-guide_run-comparison.md.5-oaOfTu.js +0 -1
- package/public/docs/assets/user-guide_run-comparison.md.5-oaOfTu.lean.js +0 -1
- package/public/docs/assets/user-guide_understanding-findings.md.CK2WZ0Tf.js +0 -1
- package/public/docs/assets/user-guide_understanding-findings.md.CK2WZ0Tf.lean.js +0 -1
- package/public/vendor/spark-doc/anchors.json +0 -205
- package/public/vendor/spark-doc/index.html +0 -4202
- package/public/vendor/spark-doc/landing.html +0 -418
- package/public/vendor/spark-doc/meta.html +0 -970
|
@@ -6,14 +6,14 @@
|
|
|
6
6
|
<title>Worker protocol | SparkForensics</title>
|
|
7
7
|
<meta name="description" content="Docs for using and contributing to SparkForensics">
|
|
8
8
|
<meta name="generator" content="VitePress v1.6.4">
|
|
9
|
-
<link rel="preload stylesheet" href="/docs/assets/style.
|
|
9
|
+
<link rel="preload stylesheet" href="/docs/assets/style.DSixAiZE.css" as="style">
|
|
10
10
|
<link rel="preload stylesheet" href="/docs/vp-icons.css" as="style">
|
|
11
11
|
|
|
12
|
-
<script type="module" src="/docs/assets/app.
|
|
12
|
+
<script type="module" src="/docs/assets/app.APoq_UBs.js"></script>
|
|
13
13
|
<link rel="preload" href="/docs/assets/inter-roman-latin.Di8DUHzh.woff2" as="font" type="font/woff2" crossorigin="">
|
|
14
|
-
<link rel="modulepreload" href="/docs/assets/chunks/theme.
|
|
15
|
-
<link rel="modulepreload" href="/docs/assets/chunks/framework.
|
|
16
|
-
<link rel="modulepreload" href="/docs/assets/contributor-guide_architecture_worker-protocol.md.
|
|
14
|
+
<link rel="modulepreload" href="/docs/assets/chunks/theme.PavXO_vz.js">
|
|
15
|
+
<link rel="modulepreload" href="/docs/assets/chunks/framework.DSg0KOwT.js">
|
|
16
|
+
<link rel="modulepreload" href="/docs/assets/contributor-guide_architecture_worker-protocol.md.IbnfNrV3.lean.js">
|
|
17
17
|
<link rel="icon" type="image/svg+xml" href="/docs/favicon.svg">
|
|
18
18
|
<link rel="preconnect" href="https://fonts.googleapis.com">
|
|
19
19
|
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin="">
|
|
@@ -22,13 +22,13 @@
|
|
|
22
22
|
<script id="check-mac-os">document.documentElement.classList.toggle("mac",/Mac|iPhone|iPod|iPad/i.test(navigator.platform));</script>
|
|
23
23
|
</head>
|
|
24
24
|
<body>
|
|
25
|
-
<div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Contributor Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/development-setup.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Development setup</p><!--]--></a><!----></div><!----></div><section class="VPSidebarItem level-1 collapsible collapsed is-link has-active" data-v-b3fd67f8><div class="item" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/" data-v-b3fd67f8><!--[--><h3 class="text" data-v-b3fd67f8>Architecture</h3><!--]--></a><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-b3fd67f8><span class="vpi-chevron-right caret-icon" data-v-b3fd67f8></span></div></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/overview.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Overview</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/worker-protocol.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Worker protocol</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/state-and-history.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>State & history intake</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/detector-contract.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Detector contract</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/impact-estimation.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Impact estimation</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/widget-rendering.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Widget rendering</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/board-widgets.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Board widgets</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/drill-down.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Drill-down</p><!--]--></a><!----></div><!----></div><!--]--></div></section><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/testing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Testing & verification</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/contributing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Contributing</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_contributor-guide_architecture_worker-protocol" data-v-39a288b8><div><h1 id="worker-protocol" tabindex="-1">Worker protocol <a class="header-anchor" href="#worker-protocol" aria-label="Permalink to "Worker protocol""></a></h1><h2 id="message-protocol" tabindex="-1">Message protocol <a class="header-anchor" href="#message-protocol" aria-label="Permalink to "Message protocol""></a></h2><p>Worker to main:</p><ul><li><code>app</code>, <code>stage</code>, <code>sql</code>, <code>executor</code>, <code>job</code>, <code>progress</code>: progressive, posted as the parser reads.</li><li><code>sqlPlan</code> (<code>{ type: 'sqlPlan', data: { executionId, planTree } }</code>): posted by <code>endSqlExecution</code> once a SQL execution's plan tree resolves.</li><li><code>runAggregates</code>: one whole-run core-time-series summary (busy-core-ms, peak concurrency, per-stage task-duration sums), emitted just before <code>done</code>.</li><li><code>stageExecutorMetrics</code>: the post-completion re-post described in <a href="./overview.html#streaming">Streaming</a>, also emitted just before <code>done</code>.</li><li><code>done</code>, <code>taskData</code>, <code>error</code>.</li></ul><p>A History Server failure is always the typed, display-safe payload <code>{ type: 'error', source: 'shs', code }</code>, where <code>code</code> is one of <code>local-server-unavailable</code>, <code>upstream-unreachable</code>, <code>application-not-found</code>, <code>access-or-upstream-failure</code>, or <code>invalid-event-log</code>. It never carries an upstream message, URL, status, or response body.</p><p>Evidence availability has no dedicated worker message: the final <code>app</code> message carries a compact <code>evidenceInputs</code> counter summary, and <code>done.skippedLines</code> supplies its parse-integrity input.</p><p>Main to worker: <code>parse</code>, <code>parseFiles(files)</code> (rolling <code>eventlog_v2_*</code> directories, one continuous stream across files), <code>parseFromUrl({ baseUrl, appId, attemptId })</code>, and <code>getTaskData(stageId)</code>. The SHS request object is normalized before it reaches the worker: <code>{ baseUrl: string, appId: string, attemptId: string | null }</code>.</p><p><code>taskData</code> uses structured-clone (<code>.slice()</code>) so the worker retains its own <code>Float64Array</code> for re-renders.</p><p>Post-parse prefetch: after <code>done</code>, main runs <code>analyzer.ts</code> to build the bottleneck catalog, then fires parallel <code>getTaskData</code> for every flagged stage so their widgets render immediately. Unflagged stages are on-demand.</p><h3 id="evidence-availability-worker-input" tabindex="-1">Evidence-availability worker input <a class="header-anchor" href="#evidence-availability-worker-input" aria-label="Permalink to "Evidence-availability worker input""></a></h3><p><code>packages/core/src/event-handlers.ts</code>'s <code>createState()</code>/per-event handlers start and increment this fixed <code>evidenceInputs</code> shape on the normalized app model:</p><div class="language-text vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">text</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span>environmentUpdates, applicationEnds, stageSubmissions, rddStorageSnapshots,</span></span>
|
|
25
|
+
<div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Contributor Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/development-setup.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Development setup</p><!--]--></a><!----></div><!----></div><section class="VPSidebarItem level-1 collapsible collapsed is-link has-active" data-v-b3fd67f8><div class="item" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/" data-v-b3fd67f8><!--[--><h3 class="text" data-v-b3fd67f8>Architecture</h3><!--]--></a><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-b3fd67f8><span class="vpi-chevron-right caret-icon" data-v-b3fd67f8></span></div></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/overview.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Overview</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/worker-protocol.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Worker protocol</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/state-and-history.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>State & history intake</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/detector-contract.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Detector contract</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/impact-estimation.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Impact estimation</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/widget-rendering.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Widget rendering</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/board-widgets.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Board widgets</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/drill-down.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Drill-down</p><!--]--></a><!----></div><!----></div><!--]--></div></section><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/testing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Testing & verification</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/contributing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Contributing</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_contributor-guide_architecture_worker-protocol" data-v-39a288b8><div><h1 id="worker-protocol" tabindex="-1">Worker protocol <a class="header-anchor" href="#worker-protocol" aria-label="Permalink to "Worker protocol""></a></h1><h2 id="message-protocol" tabindex="-1">Message protocol <a class="header-anchor" href="#message-protocol" aria-label="Permalink to "Message protocol""></a></h2><p>Worker to main:</p><ul><li><code>app</code>, <code>stage</code>, <code>sql</code>, <code>executor</code>, <code>job</code>, <code>progress</code>: progressive, posted as the parser reads.</li><li><code>sqlPlan</code> (<code>{ type: 'sqlPlan', data: { executionId, planTree } }</code>): posted by <code>endSqlExecution</code> once a SQL execution's plan tree resolves.</li><li><code>runAggregates</code>: one whole-run core-time-series summary (busy-core-ms, peak concurrency, per-stage task-duration sums), emitted just before <code>done</code>.</li><li><code>stageExecutorMetrics</code>: the post-completion re-post described in <a href="./overview.html#streaming">Streaming</a>, also emitted just before <code>done</code>.</li><li><code>done</code>, <code>taskData</code>, <code>error</code>.</li></ul><p>A History Server failure is always the typed, display-safe payload <code>{ type: 'error', source: 'shs', code }</code>, where <code>code</code> is one of <code>local-server-unavailable</code>, <code>upstream-unreachable</code>, <code>application-not-found</code>, <code>access-or-upstream-failure</code>, or <code>invalid-event-log</code>. It never carries an upstream message, URL, status, or response body.</p><p>Evidence availability has no dedicated worker message: the final <code>app</code> message carries a compact <code>evidenceInputs</code> counter summary, and <code>done.skippedLines</code> supplies its parse-integrity input.</p><p>Main to worker: <code>parse</code>, <code>parseFiles(files)</code> (rolling <code>eventlog_v2_*</code> directories, one continuous stream across files), <code>parseFromUrl({ baseUrl, appId, attemptId })</code>, and <code>getTaskData(stageId)</code>. The SHS request object is normalized before it reaches the worker: <code>{ baseUrl: string, appId: string, attemptId: string | null }</code>.</p><p><code>taskData</code> uses structured-clone (<code>.slice()</code>) so the worker retains its own <code>Float64Array</code> for re-renders.</p><p>Post-parse prefetch: after <code>done</code>, main runs <code>analyzer.ts</code> to build the bottleneck catalog, then fires parallel <code>getTaskData</code> for every flagged stage so their widgets render immediately. Unflagged stages are on-demand.</p><h3 id="evidence-availability-worker-input" tabindex="-1">Evidence-availability worker input <a class="header-anchor" href="#evidence-availability-worker-input" aria-label="Permalink to "Evidence-availability worker input""></a></h3><p><code>packages/core/src/event-handlers.ts</code>'s <code>createState()</code>/per-event handlers start and increment this fixed <code>evidenceInputs</code> shape on the normalized app model:</p><div class="language-text vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">text</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span>environmentUpdates, applicationEnds, stageSubmissions, rddStorageSnapshots,</span></span>
|
|
26
26
|
<span class="line"><span>sqlExecutions, resolvedSqlPlans, executorMetricRows, taskRecords</span></span></code></pre></div><p>The counters are a structured-clone-safe summary: they contain counts, not raw events, task records, host names, SQL text, paths, or Spark-property values. The final <code>app</code> message snapshots the counters after all input is processed, immediately before <code>done</code>; a consumer must not add a second worker message just for the ledger. <code>app</code> messages are also emitted mid-parse from <code>SparkListenerApplicationStart</code> and <code>SparkListenerEnvironmentUpdate</code>; those snapshots are partial and non-authoritative. Only the terminal <code>app</code> message emitted by <code>emitParseCompletion</code> (the one immediately before <code>done</code>) should be used for evidence-availability conclusions. On the main thread, <code>useIngest</code> combines the normalized <code>AppModel</code> with <code>done.skippedLines</code>, derives the ledger before calling <code>analyze()</code>, and stores it as <code>appModel.evidenceAvailability</code>.</p><h3 id="evidence-availability-contract-v1" tabindex="-1">Evidence-availability contract (V1) <a class="header-anchor" href="#evidence-availability-contract-v1" aria-label="Permalink to "Evidence-availability contract (V1)""></a></h3><p><code>packages/core/src/evidence-availability.ts</code> is the single reusable taxonomy for the browser, future report output, and future headless consumers. Its serialized ledger shape is <code>{ schemaVersion: 1, entries: EvidenceAvailabilityEntry[] }</code>. Entries are ordered by the following fixed eight-key enum:</p><div class="language-text vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">text</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span>executorMetrics, rddStorageSnapshots, sqlPlan, sparkConfiguration,</span></span>
|
|
27
27
|
<span class="line"><span>taskCoreTime, infrastructureContext, sourceContext, costContext</span></span></code></pre></div><p>Each entry has the stable fields <code>key</code>, <code>state</code>, <code>reasonCode</code>, <code>summary</code>, and optional <code>evidence</code>. The closed V1 state enum is:</p><div class="language-text vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">text</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span>present, disabled, notEmitted, notApplicable, outsideEventLog, unknown</span></span></code></pre></div><p>The closed V1 reason-code enum is:</p><div class="language-text vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">text</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span>observed, explicitlyDisabled, noObservedExecutorMetrics,</span></span>
|
|
28
28
|
<span class="line"><span>noObservedStageSubmission, noRddStorageSnapshot, noResolvedSqlPlan,</span></span>
|
|
29
29
|
<span class="line"><span>noSqlExecution, noEnvironmentUpdate, noTaskRecords, noUsableCoreTimeAggregate,</span></span>
|
|
30
|
-
<span class="line"><span>outsideEventLogScope, parseIncomplete</span></span></code></pre></div><p><code>summary</code> is domain-agnostic presentation copy derived from the reason code; the enum values, not labels, are the machine contract. Optional <code>evidence</code> is limited to safe compact provenance of the exact <code>{ eventType, count }</code> shape. <code>eventType</code> values are pinned to the exact <code>evidenceInputs</code> counter keys; they are not shortened or aliased. It must never include raw Spark-property values, host names, paths, SQL text, task records, raw-event identifiers, or other event payloads.</p><table tabindex="0"><thead><tr><th>Key</th><th>Proven <code>present</code> evidence</th><th>Trustworthy absence / boundary result</th></tr></thead><tbody><tr><td><code>executorMetrics</code></td><td><code>executorMetricRows > 0</code></td><td>Explicit observed <code>spark.eventLog.logStageExecutorMetrics=false</code> is <code>disabled</code> / <code>explicitlyDisabled</code>; otherwise <code>notEmitted</code> / <code>noObservedExecutorMetrics</code>. Observed metrics win over the explicit setting.</td></tr><tr><td><code>rddStorageSnapshots</code></td><td><code>rddStorageSnapshots > 0</code> from <code>SparkListenerStageSubmitted</code></td><td>No stage submissions is <code>notEmitted</code> / <code>noObservedStageSubmission</code>; stages submitted without any <code>RDD Info</code> rows is <code>notEmitted</code> / <code>noRddStorageSnapshot</code>. This is RDD snapshot evidence, not block-update telemetry.</td></tr><tr><td><code>sqlPlan</code></td><td><code>resolvedSqlPlans > 0</code></td><td>A SQL execution without a resolved plan is <code>notEmitted</code> / <code>noResolvedSqlPlan</code>; no SQL execution is <code>notApplicable</code> / <code>noSqlExecution</code>.</td></tr><tr><td><code>sparkConfiguration</code></td><td><code>environmentUpdates > 0</code></td><td><code>notEmitted</code> / <code>noEnvironmentUpdate</code>; a normalized empty config object alone is not evidence.</td></tr><tr><td><code>taskCoreTime</code></td><td><code>taskRecords > 0</code> and a usable <code>runAggregates.perStage</code> task count</td><td>No task records is <code>notEmitted</code> / <code>noTaskRecords</code>; task records with no usable aggregate are <code>notEmitted</code> / <code>noUsableCoreTimeAggregate</code>. An empty aggregate object is not usable evidence.</td></tr><tr><td><code>infrastructureContext</code></td><td>none</td><td>Always <code>outsideEventLog</code> / <code>outsideEventLogScope</code>.</td></tr><tr><td><code>sourceContext</code></td><td>none</td><td>Always <code>outsideEventLog</code> / <code>outsideEventLogScope</code>.</td></tr><tr><td><code>costContext</code></td><td>none</td><td>Always <code>outsideEventLog</code> / <code>outsideEventLogScope</code>.</td></tr></tbody></table><p>Parse integrity is fail-closed. A parse is trustworthy only when it skipped no malformed lines and observed an application-end event. For an untrustworthy parse, proof already observed remains <code>present</code>, and an explicitly observed executor-metrics disablement remains <code>disabled</code>; every conclusion based on absence or irrelevance becomes <code>unknown</code> / <code>parseIncomplete</code>. The three outside-event-log product boundaries remain <code>outsideEventLog</code>. In particular, <code>notEmitted</code> and <code>notApplicable</code> are never favorable measurements inferred from an incomplete parse.</p><p>Availability is model metadata, not a detector or finding: it never changes detector thresholds or ordering, impact band, catalog/scorecard counts, no-bottleneck behavior, or detector suppression. <code>session-snapshot.ts</code> captures and restores <code>evidenceAvailability</code> with the normalized model, so a recent-file switch retains the same ledger without reparsing.</p><h3 id="portable-evidence-report-v1" tabindex="-1">Portable evidence report (V1) <a class="header-anchor" href="#portable-evidence-report-v1" aria-label="Permalink to "Portable evidence report (V1)""></a></h3><p><code>packages/core/src/evidence-report.ts</code>'s <code>buildEvidenceReport(appModel, { redact })</code> returns <code>{ markdown, json }</code>: a self-contained, byte-stable document that runs the detectors over an <code>appModel</code> and serializes the result for sharing outside the tool. Raw task records are never included; identifier redaction (app id + host names → <code>app-1</code>/<code>host-1</code> pseudonyms via <code>packages/core/src/redact.ts</code>) is opt-in with <code>{ redact: true }</code>. The JSON is pinned by <code>EVIDENCE_SCHEMA_VERSION</code> (currently <code>3</code>, surfaced as <code>json.schemaVersion</code>) and has this fixed top-level key order:</p><div class="language-text vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">text</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span>schemaVersion, summary, evidenceAvailability, detectors, findings, recommendations, cleanChecks</span></span></code></pre></div><ul><li><code>summary</code> is the run header: <code>{ app: { id, name, sparkVersion }, stageCount, jobCount, sqlExecutionCount, findingCount, impactBandCounts }</code>.</li><li><code>evidenceAvailability</code> is the ledger above (or <code>null</code> when absent).</li><li><code>detectors</code> is <code>detectorCatalog()</code> output: one <code>{ type, version, scope, thresholds, docAnchor }</code> per detector, in <code>DETECTORS</code> order, so the exact threshold set that produced each finding travels with the evidence.</li><li><code>findings</code> are deterministically sorted rows (impact band → type → stage → id), each with a stable <code>id</code>, <code>tag</code>, core columns, an always-present <code>actionLabel</code>, and an <code>evidence</code> sub-object for non-core fields; <code>confidence</code>/ <code>validationRequired</code>/<code>docAnchor</code> appear only when the detector emitted them.</li><li><code>recommendations</code> is the impact-ranked <code>buildRecommendationRollup</code> output (<code>packages/core/src/recommendation-rollup.ts</code>), and <code>cleanChecks</code> lists every detector type that fired zero findings this run: see the 2026-09-03 update below.</li></ul><p>Determinism holds because detector order, finding sort, and object key order are all fixed, so a given <code>appModel</code> serializes identically across calls.</p><p>Redaction enumerates hosts two ways: from each finding's structured <code>host</code> field (which the builder nests under <code>evidence.host</code>), and by scanning every string value for EC2-style hostnames / bare IPv4 tokens. So identifiers that surface only in free text (recommendation copy, a <code>stageFailed</code> failure-reason value) are pseudonymized too. Pseudonym numbering uses a numeric-aware sort, so re-redacting an already-redacted report is a no-op even past <code>host-10</code>.</p><p>The Markdown rendering mirrors the JSON's AC3 field set: each finding block prints its <code>detector version</code>, its sorted <code>evidence</code> entries (byte-magnitude keys humanized), and the report ends with a <code>## Detectors</code> catalog carrying the version + threshold set. A finding's <code>impactEstimate</code> (when its <code>basis</code> isn't <code>'informational'</code>) prints as its own <code>- impact: </code> line (<code>Est. <low>-<high></code> and/or the raw-waste figure, plus <code>estimateMethod</code>), via <code>renderImpactEstimate</code>/<code>formatWallClockRange</code>/<code>formatRawWaste</code> in <code>packages/core/src/evidence-report.ts</code>. <code>EvidenceExport</code> names downloads <code>evidence-<appId>[-redacted].<md|json></code>, taking the app id from the (already pseudonymized when redacting) report so a redacted file never leaks the real id and is never name-identical to a raw export.</p><p>Decision 9 (design spec): <code>Finding</code>'s <code>impactEstimate</code> field, plus the <code>utilizationFraction</code>/<code>memorySize</code>/<code>diskSize</code>/<code>numCachedPartitions</code>/<code>numPartitions</code> instrumentation fields the impact estimator reads, were added without bumping <code>EVIDENCE_SCHEMA_VERSION</code> past <code>1</code>. Both additions are purely optional and ride <code>Finding</code>'s existing optional-field-plus-catch-all convention, so an evidence report built before these fields existed still deserializes and compares byte-for-byte against one built after: nothing about the schema's stability guarantee changed, only its surface grew. A future reader who notices <code>impactEstimate</code> in the JSON without a schema bump is looking at this deliberate call, not an oversight.</p><p>2026-08-30 update: <code>EVIDENCE_SCHEMA_VERSION</code> was bumped to <code>2</code> for the occupancy-weighted attribution redesign (see <a href="./impact-estimation.html#occupancy-weighted-attribution">Occupancy-weighted attribution</a>): <code>ImpactEstimate</code>'s shape changed from <code>{low, high}</code> to <code>{basis, wallClock, estimateMethod, rawWaste?}</code>, a real, non-additive breaking change to a field this same Decision 9 previously shipped without a bump. <code>impactEstimate</code> had zero consumers outside <code>packages/core/src/impact-estimator.ts</code> and its own tests at the time of this bump (confirmed by grep across <code>src/view/*</code> and <code>packages/core/src/evidence-report.ts</code>), so no other code needed migrating alongside it.</p><p><code>FindingRow</code>'s schema was extended to surface <code>impactEstimate</code> as a first-class column without further bumping <code>EVIDENCE_SCHEMA_VERSION</code> past <code>2</code>, consistent with <code>Finding</code>'s existing optional-field-plus-catch-all convention. It appears after the pinned core columns (<code>id</code>, <code>type</code>, <code>impactBand</code>, <code>stageId</code>, <code>metric</code>, <code>value</code>, <code>recommendation</code>, <code>detectorVersion</code>, plus optional but pinned <code>confidence</code>, <code>validationRequired</code>, <code>docAnchor</code>) without displacing any of them; byte-for-byte deserializability of existing reports is preserved. <code>FindingRow.impactEstimate</code> carries the full contract documented in <a href="./impact-estimation.html#occupancy-weighted-attribution">Impact estimation</a> (basis, wallClock, estimateMethod, rawWaste).</p><p>2026-09-03 update: the "Fix These First" dashboard redesign (impact-ranked recommendation rollup, short per-finding action labels, a clean-checks table) was UI-only when it shipped; this update ports the underlying data into <code>buildEvidenceReport()</code> so the CLI (<code>packages/cli/bin/sparkforensics-analyze.mjs</code>) and the MCP tool (<code>diagnoseRun</code> in <code>packages/core/src/mcp-tools.ts</code>) get it too, not just the web markdown/JSON download. Three additions, all purely additive, so this does not bump <code>EVIDENCE_SCHEMA_VERSION</code> past <code>2</code>, the same rationale as the <code>impactEstimate</code> addition immediately above: <code>FindingRow.actionLabel</code> is now always present (a short imperative label like "Reduce shuffle size"), sourced from a new core module, <code>packages/core/src/finding-action-label.ts</code>'s <code>coreFindingActionLabel</code>, extracted from the (type, discriminant) switch statement that used to live only in the view layer (<code>src/view/finding-action-label.ts</code>, which now wraps the core function and layers its own <code>REGISTRY</code> fallback on top). <code>EvidenceReportJson.recommendations</code> is the same impact-ranked <code>buildRecommendationRollup</code> grouping (<code>packages/core/src/recommendation-rollup.ts</code>) that <code>FixTheseFirst.tsx</code> renders, so CLI/MCP/download consumers get the same "what's the highest-impact fix" ranking the dashboard shows, without changing the existing <code>findings</code> array's own impact-band-sorted order at all. <code>EvidenceReportJson.cleanChecks</code> lists every detector type that fired zero findings this run, each with <code>getThresholdSummary</code>'s one-line "what would have tripped it" sentence; unlike the dashboard's <code>Alerts.tsx</code> clean-checks table, it deliberately includes the "always-mounted" reference types (<code>memoryUtilization</code>, <code>utilization</code>, <code>coreLocality</code>) even when they have no findings, since a flat evidence report has no separate always-visible surface for them to already appear on the way those widgets do on the board.</p><h3 id="finding-identity" tabindex="-1">Finding identity <a class="header-anchor" href="#finding-identity" aria-label="Permalink to "Finding identity""></a></h3><p>Every finding <code>analyzer.ts</code> emits carries a stable <code>id</code> (<code>push()</code>'s <code>findingId()</code> choke point, FNV-1a hash over <code>type | location | metric | value | discriminators</code>) and a <code>detectorVersion</code> (from the emitting <code>DETECTORS</code> entry). The two are deliberately decoupled: <code>id</code> is derived only from a finding's evidence tuple, never from <code>detectorVersion</code>, so a threshold or logic tweak that bumps a detector's <code>version</code> does not change the <code>id</code> of findings it still emits at the same location/metric. A saved reference (dashboard bookmark, exported report row, future URL-restored filter) keeps pointing at the same logical finding across detector revisions; <code>detectorVersion</code> is separate provenance metadata for "which ruleset produced this," not part of identity. See <code>packages/core/test/analyzer-finding-identity.test.js</code> for the decoupling proof.</p><p><code>id</code> stability holds for equivalent reruns of the same schema/analyzer version on the same input. Changing the id-derivation rule itself (the hash algorithm, or which fields feed the location key/discriminators) is an intentional breaking change and must bump <code>EVIDENCE_SCHEMA_VERSION</code> (<code>packages/core/src/evidence-report.ts</code>); there is no separate id-scheme version.</p><h3 id="headless-analysis-cli-v1" tabindex="-1">Headless analysis CLI (V1) <a class="header-anchor" href="#headless-analysis-cli-v1" aria-label="Permalink to "Headless analysis CLI (V1)""></a></h3><p><code>packages/cli/bin/sparkforensics-analyze.mjs</code> runs the same parser + detector contracts outside the browser, for CI. It accepts a single event-log file or a rolling-log directory, drives <code>runParse</code>/<code>runParseFiles</code> (<code>packages/core/src/parser-worker.ts</code>) with a Node-only File-like shim (<code>packages/core/src/cli/collect-run.ts</code>), and writes the exact <code>buildEvidenceReport</code> JSON schema described above: one format, not a second. Alternatively, <code>--shs-base-url <url> --app-id <id> [--attempt-id <id>]</code> fetches the run from a Spark History Server instead (mutually exclusive with the positional file/directory argument), calling <code>resolveFromShs</code> (<code>packages/core/src/shs-load.ts</code>, shared with the MCP server's SHS source path below) directly; no dependency on the <code>packages/server</code> package. A failed SHS fetch reports its message to <code>stderr</code> and exits <code>2</code>, same as a local file that can't be parsed.</p><p>Optional CLI-flag budgets (<code>--max-runtime <ms></code>, <code>--max-spill <gb></code>, <code>--max-skew <ratio></code>, <code>--max-failed-task-rate <pct></code>, <code>--min-efficiency <pct></code>) are evaluated in <code>packages/core/src/cli/budgets.ts</code> against the existing finding catalog (<code>analyze()</code>) and <code>computeEfficiencyModel</code>; there is no second rule engine. A budget whose required evidence is missing (e.g. the run never emitted <code>ApplicationEnd</code>, or has no usable per-task <code>runAggregates</code>) is reported as inconclusive (<code>stderr</code> warning) rather than silently passing, and gets its own exit code distinct from both pass and violation. Exit codes: <code>0</code> pass, <code>1</code> a configured budget was violated, <code>2</code> the input could not be parsed at all, <code>3</code> no violations but at least one budget was inconclusive. A violation always wins over an inconclusive result in the same run (exit <code>1</code>, not <code>3</code>).</p><h3 id="mcp-server-v1" tabindex="-1">MCP server (V1) <a class="header-anchor" href="#mcp-server-v1" aria-label="Permalink to "MCP server (V1)""></a></h3><p><code>packages/core/src/mcp-server-factory.ts</code>'s <code>createMcpServer()</code> registers 5 tools: <code>diagnose_run</code> (thresholded findings + remediation text), <code>get_run_summary</code> (app/stage/job/sql counts and duration, no findings), <code>compare_runs</code> (categorized findings delta + metric deltas between two runs), <code>evaluate_budgets</code> (pass/fail budget thresholds against one run, optionally with a second run for regression/fail-on-introduced budgets: the MCP side of the CLI's <code>evaluateBudgets()</code> gating), and <code>get_finding_evidence</code> (raw evidence bundle for one finding, for drill-down after <code>diagnose_run</code>). None re-implement detector logic; all repackage <code>analyze</code>/<code>buildEvidenceReport</code>/<code>compareRuns</code>/<code>captureSnapshot</code>/<code>evaluateBudgets</code> from <code>packages/core/src/mcp-tools.ts</code>, which resolves a <code>source</code> (event-log <code>path</code>, or an SHS <code>shsBaseUrl</code>/<code>appId</code>/<code>attemptId</code> triple) into a cached <code>AppModel</code>. The SHS source path calls <code>resolveFromShs</code> (<code>packages/core/src/shs-load.ts</code>, also used directly by the CLI's <code>--shs-base-url</code> mode above), which reuses two extractions shared with the browser ingestion flow: <code>fetchShsEventLog</code> (<code>packages/core/src/proxy.js</code>) fetches the zip, and <code>decodeShsArchive</code> (<code>packages/core/src/shs-fetch.ts</code>, re-exported from <code>parser-worker.ts</code>'s barrel) decodes it into a parsed <code>AppModel</code>, the same split <code>runParseFromUrl</code> itself calls into.</p><p>Two transports connect to that one factory: <code>packages/mcp/bin/sparkforensics-mcp.mjs</code> (stdio, for local MCP clients) and <code>packages/server/index.js</code>'s <code>/mcp</code> route (streamable HTTP, for the local server). The run cache (<code>packages/core/src/mcp-tools.ts</code>) is a module-level LRU (cap 8 and 15-minute idle TTL by default, overridable via <code>SPARKFORENSICS_MCP_CACHE_CAP</code> and <code>SPARKFORENSICS_MCP_CACHE_TTL_MS</code>, lazily swept on access) keyed by resolved source (path mtime, or SHS baseUrl+appId+attemptId), so a client mints a <code>runId</code> once via <code>resolveOrCreateRun</code> and reuses it across subsequent tool calls instead of re-parsing.</p><p>Every tool failure comes back as <code>{isError: true, content: [...], structuredContent: {code}}</code>, never an HTTP-status-shaped error; <code>code</code> is one of the 5 existing SHS codes (<code>SHS_ERROR_CODES</code>, <code>packages/core/src/shs-request.js</code>) plus <code>run-not-found</code>, <code>finding-not-found</code>, <code>invalid-event-log</code> (also covers a nonexistent <code>path</code> source), and <code>archive-too-large</code> (SHS archive over the <code>SPARKFORENSICS_MAX_ARCHIVE_BYTES</code> byte cap, default 1 GiB, because the MCP path buffers the whole archive in memory, unlike the streaming <code>/shs-proxy</code> route).</p><p><code>scripts/vendor-core.mjs</code> (shared by <code>packages/cli</code>, <code>packages/mcp</code>, and <code>packages/server</code>'s <code>prepack</code> scripts) vendors <code>packages/core/src/</code> wholesale into each package's own <code>vendor-core/</code> at pack time, pre-stripping TypeScript to plain <code>.js</code> (Node's native TS stripping refuses to run on <code>.ts</code> files under <code>node_modules</code>, which is exactly where a published <code>vendor-core/</code> lands). Each package's bin/entry point resolves its needed module from <code>vendor-core/</code> if present, else falls back to the real <code>packages/core/src/</code> sibling loaded as <code>.ts</code> directly (<code>packages/server/index.js</code> mirrors <code>resolveStaticRoot</code>'s <code>public/</code>-vs-<code>../dist</code> pattern for this same fallback).</p><h2 id="typescript-core-and-runtime-event-validation" tabindex="-1">TypeScript core and runtime event validation <a class="header-anchor" href="#typescript-core-and-runtime-event-validation" aria-label="Permalink to "TypeScript core and runtime event validation""></a></h2><p>All of <code>src/</code> (browser SPA) and <code>packages/core/src/</code> (shared analysis logic) is strict TypeScript; the only plain-<code>.js</code> holdouts are the two vendored third-party decompressors, <code>packages/core/src/vendor/fflate.js</code> and <code>packages/core/src/vendor/fzstd.js</code> (left untouched deliberately: vendored code, not project code). Every remaining import of a same-repo module uses a <code>.ts</code> specifier (e.g. <code>import { dispatchLine } from './event-handlers.ts'</code>), not <code>.js</code>: the CLI and MCP entrypoints (<code>packages/cli/bin/sparkforensics-analyze.mjs</code>, <code>packages/mcp/bin/sparkforensics-mcp.mjs</code>) run under plain Node's ESM resolver, which cannot remap a <code>.js</code> specifier to a same-named <code>.ts</code> file the way Vite/Vitest's bundler-style resolver can. <code>tsconfig.json</code> sets <code>"allowImportingTsExtensions": true</code> to make this legal.</p><p>Event schema validation lives in <code>packages/core/src/event-schemas.ts</code>: one zod schema per <code>SparkListener*</code> event variant the parser understands (15 total: <code>LogStart</code>, <code>ApplicationStart</code>, <code>EnvironmentUpdate</code>, <code>ApplicationEnd</code>, <code>JobStart</code>, <code>JobEnd</code>, <code>StageSubmitted</code>, <code>StageCompleted</code>, <code>StageExecutorMetrics</code>, <code>TaskEnd</code>, the three SQL-execution-UI listener events, <code>ExecutorAdded</code>, <code>ExecutorRemoved</code>), combined into <code>SparkEventSchema = z.discriminatedUnion('Event', [...])</code>. <code>processEvent</code>'s switch (<code>event-handlers.ts</code>) consumes the resulting <code>SparkEvent</code> union type directly, so a schema change and a handler's expectations can't silently drift apart.</p><p>The recursive <code>sparkPlanInfo.children</code> tree carried on SQL-execution-start events is validated by <code>parseSparkPlanInfoTree</code>, an iterative, explicit heap-allocated-stack parser, deliberately not <code>z.lazy()</code>. A real plan tree's depth is unbounded, and attacker-uncontrolled input should never hand zod's own recursive schema resolution an arbitrarily deep structure to walk. The iterative parser shallow-validates one node at a time via <code>z.object(...)</code> and builds the tree itself, capping at <code>MAX_PLAN_DEPTH = 500</code> (throws past that). This mirrors <code>packages/core/src/plan-tree-walk.ts</code>'s <code>walkPlanTree</code>, which uses the same iterative-over-recursive approach for the <em>resolved</em> tree; <code>event-schemas.ts</code> applies it one layer earlier, to the raw JSON before it becomes a tree at all.</p><p>There are two external-data boundaries, the two places this codebase parses data it does not control. Both now run that data through a schema, and both treat a validation failure the same way: a silent skip, not a distinct error.</p><ul><li><code>dispatchLine</code> (<code>event-handlers.ts</code>): after <code>JSON.parse</code> succeeds, a line whose <code>Event</code> value is one of the 15 modeled types but fails that type's own schema now increments <code>skippedLines</code> (previously: no shape validation existed at all, and a malformed event silently corrupted downstream state with no signal anywhere). An <code>Event</code> value outside the 15 modeled types is still silently ignored without incrementing <code>skippedLines</code>, unchanged from before migration (see the note below on why the broader design was rejected).</li><li><code>shs-fetch.ts</code>: on a non-OK response from the local SHS proxy, the JSON error envelope is validated against <code>ShsProxyErrorBodySchema</code> (<code>packages/core/src/shs-schemas.ts</code>, <code>{ code: string }</code>, <code>.passthrough()</code>). A body that isn't valid JSON, or is JSON but fails that schema, falls back to the generic <code>access-or-upstream-failure</code> code: the same silent-skip treatment as a malformed log line, collapsing what could have been a separate "malformed SHS response" error code into the existing generic one.</li></ul><p>Neither boundary distinguishes "malformed JSON" from "wrong shape" from "unrecognized variant" in what it reports outward: all three collapse into the same skip/fallback path. That was a deliberate scope decision. The correction below says why the <em>line</em> boundary stops there rather than flagging every unrecognized event type.</p><blockquote><p>Correction made during migration, not part of the original design: an earlier draft counted <em>any</em> <code>Event</code> value outside the 15 modeled types toward <code>skippedLines</code>, not just ones that fail their own schema. Running that design against real Spark event logs (which always contain plenty of ordinary event types this tool has never modeled: <code>TaskStart</code>, <code>BlockManagerAdded</code>, <code>ExecutorMetricsUpdate</code>, and others, always silently ignored pre-migration) pushed <code>skippedLines</code> from 0 to the tens of thousands on completely healthy logs. That would have tripped <code>evidence-availability.ts</code>'s fail-closed <code>trustworthy</code> gate and shown a false "malformed JSON" warning on every real file. Caught in review against real fixtures, not synthetic ones; fixed to the narrower rule described above before merging. <code>evidence-availability.ts</code>'s gate itself (<code>trustworthy = skippedLines === 0 && applicationEnds > 0</code>) needed no code change once this was fixed: it was already correct, the input feeding it was not.</p></blockquote><p>The exhaustiveness convention is <code>packages/core/src/assert-never.ts</code>. <code>assertNever(x: never): never</code> throws at runtime and, more importantly, fails <code>tsc</code> at compile time if <code>x</code> is not actually <code>never</code>, i.e. if some case of a union type isn't handled. Used today at the <code>default</code> arm of <code>event-handlers.ts</code>'s <code>processEvent</code> switch (over <code>SparkEvent</code>) and <code>analyzer.ts</code>'s scope-dispatch switch (over a detector's <code>scope: 'stage' | 'sql' | 'app' | 'config'</code>). It is the project's standard pattern for any future exhaustive switch or dispatch over a closed union: add <code>default: return assertNever(x);</code> (or the closest non-returning equivalent) so that adding a new union member without updating every consumer becomes a compile error instead of a silent runtime gap.</p></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/contributor-guide/architecture/overview.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>Overview</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="/docs/contributor-guide/architecture/state-and-history.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>State & history intake</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
|
|
31
|
-
<script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"
|
|
30
|
+
<span class="line"><span>outsideEventLogScope, parseIncomplete</span></span></code></pre></div><p><code>summary</code> is domain-agnostic presentation copy derived from the reason code; the enum values, not labels, are the machine contract. Optional <code>evidence</code> is limited to safe compact provenance of the exact <code>{ eventType, count }</code> shape. <code>eventType</code> values are pinned to the exact <code>evidenceInputs</code> counter keys; they are not shortened or aliased. It must never include raw Spark-property values, host names, paths, SQL text, task records, raw-event identifiers, or other event payloads.</p><table tabindex="0"><thead><tr><th>Key</th><th>Proven <code>present</code> evidence</th><th>Trustworthy absence / boundary result</th></tr></thead><tbody><tr><td><code>executorMetrics</code></td><td><code>executorMetricRows > 0</code></td><td>Explicit observed <code>spark.eventLog.logStageExecutorMetrics=false</code> is <code>disabled</code> / <code>explicitlyDisabled</code>; otherwise <code>notEmitted</code> / <code>noObservedExecutorMetrics</code>. Observed metrics win over the explicit setting.</td></tr><tr><td><code>rddStorageSnapshots</code></td><td><code>rddStorageSnapshots > 0</code> from <code>SparkListenerStageSubmitted</code></td><td>No stage submissions is <code>notEmitted</code> / <code>noObservedStageSubmission</code>; stages submitted without any <code>RDD Info</code> rows is <code>notEmitted</code> / <code>noRddStorageSnapshot</code>. This is RDD snapshot evidence, not block-update telemetry.</td></tr><tr><td><code>sqlPlan</code></td><td><code>resolvedSqlPlans > 0</code></td><td>A SQL execution without a resolved plan is <code>notEmitted</code> / <code>noResolvedSqlPlan</code>; no SQL execution is <code>notApplicable</code> / <code>noSqlExecution</code>.</td></tr><tr><td><code>sparkConfiguration</code></td><td><code>environmentUpdates > 0</code></td><td><code>notEmitted</code> / <code>noEnvironmentUpdate</code>; a normalized empty config object alone is not evidence.</td></tr><tr><td><code>taskCoreTime</code></td><td><code>taskRecords > 0</code> and a usable <code>runAggregates.perStage</code> task count</td><td>No task records is <code>notEmitted</code> / <code>noTaskRecords</code>; task records with no usable aggregate are <code>notEmitted</code> / <code>noUsableCoreTimeAggregate</code>. An empty aggregate object is not usable evidence.</td></tr><tr><td><code>infrastructureContext</code></td><td>none</td><td>Always <code>outsideEventLog</code> / <code>outsideEventLogScope</code>.</td></tr><tr><td><code>sourceContext</code></td><td>none</td><td>Always <code>outsideEventLog</code> / <code>outsideEventLogScope</code>.</td></tr><tr><td><code>costContext</code></td><td>none</td><td>Always <code>outsideEventLog</code> / <code>outsideEventLogScope</code>.</td></tr></tbody></table><p>Parse integrity is fail-closed. A parse is trustworthy only when it skipped no malformed lines and observed an application-end event. For an untrustworthy parse, proof already observed remains <code>present</code>, and an explicitly observed executor-metrics disablement remains <code>disabled</code>; every conclusion based on absence or irrelevance becomes <code>unknown</code> / <code>parseIncomplete</code>. The three outside-event-log product boundaries remain <code>outsideEventLog</code>. In particular, <code>notEmitted</code> and <code>notApplicable</code> are never favorable measurements inferred from an incomplete parse.</p><p>Availability is model metadata, not a detector or finding: it never changes detector thresholds or ordering, impact band, catalog/scorecard counts, no-bottleneck behavior, or detector suppression. <code>session-snapshot.ts</code> captures and restores <code>evidenceAvailability</code> with the normalized model, so a recent-file switch retains the same ledger without reparsing.</p><h3 id="portable-evidence-report-v1" tabindex="-1">Portable evidence report (V1) <a class="header-anchor" href="#portable-evidence-report-v1" aria-label="Permalink to "Portable evidence report (V1)""></a></h3><p><code>packages/core/src/evidence-report.ts</code>'s <code>buildEvidenceReport(appModel, { redact })</code> returns <code>{ markdown, json }</code>: a self-contained, byte-stable document that runs the detectors over an <code>appModel</code> and serializes the result for sharing outside the tool. Raw task records are never included; identifier redaction (app id + host names → <code>app-1</code>/<code>host-1</code> pseudonyms via <code>packages/core/src/redact.ts</code>) is opt-in with <code>{ redact: true }</code>. The JSON is pinned by <code>EVIDENCE_SCHEMA_VERSION</code> (currently <code>3</code>, surfaced as <code>json.schemaVersion</code>) and has this fixed top-level key order:</p><div class="language-text vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">text</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span>schemaVersion, summary, evidenceAvailability, detectors, findings, recommendations, cleanChecks</span></span></code></pre></div><ul><li><code>summary</code> is the run header: <code>{ app: { id, name, sparkVersion }, stageCount, jobCount, sqlExecutionCount, findingCount, impactBandCounts }</code>.</li><li><code>evidenceAvailability</code> is the ledger above (or <code>null</code> when absent).</li><li><code>detectors</code> is <code>detectorCatalog()</code> output: one <code>{ type, version, scope, thresholds, docAnchor }</code> per detector, in <code>DETECTORS</code> order, so the exact threshold set that produced each finding travels with the evidence.</li><li><code>findings</code> are deterministically sorted rows (impact band → type → stage → id), each with a stable <code>id</code>, <code>tag</code>, core columns, an always-present <code>actionLabel</code>, and an <code>evidence</code> sub-object for non-core fields; <code>confidence</code>/ <code>validationRequired</code>/<code>docAnchor</code> appear only when the detector emitted them.</li><li><code>recommendations</code> is the impact-ranked <code>buildRecommendationRollup</code> output (<code>packages/core/src/recommendation-rollup.ts</code>), and <code>cleanChecks</code> lists every detector type that fired zero findings this run: see the 2026-09-03 update below.</li></ul><p>Determinism holds because detector order, finding sort, and object key order are all fixed, so a given <code>appModel</code> serializes identically across calls.</p><p>Redaction enumerates hosts two ways: by walking the findings tree for every string value under a key literally named <code>host</code> (<code>evidence.host</code>, <code>evidence.failedTaskDetails[].host</code>, <code>evidence.retriedTaskDetails[].host</code>, and any future nested <code>host</code> field, all covered without enumerating paths), and by scanning every string value for EC2-style hostnames / bare IPv4 tokens. So identifiers that surface only in free text (recommendation copy, a <code>stageFailed</code> failure-reason value) are pseudonymized too. Pseudonym numbering uses a numeric-aware sort, so re-redacting an already-redacted report is a no-op even past <code>host-10</code>.</p><p>The Markdown rendering mirrors the JSON's AC3 field set: each finding block prints its <code>detector version</code>, its sorted <code>evidence</code> entries (byte-magnitude keys humanized), and the report ends with a <code>## Detectors</code> catalog carrying the version + threshold set. A finding's <code>impactEstimate</code> (when its <code>basis</code> isn't <code>'informational'</code>) prints as its own <code>- impact: </code> line (<code>Estimated <low>-<high></code> and/or the raw-waste figure, plus <code>estimateMethod</code>), via <code>renderImpactEstimate</code>/<code>formatWallClockRange</code>/<code>formatRawWaste</code> in <code>packages/core/src/evidence-report.ts</code>. <code>EvidenceExport</code> names downloads <code>evidence-<appId>[-redacted].<md|json></code>, taking the app id from the (already pseudonymized when redacting) report so a redacted file never leaks the real id and is never name-identical to a raw export.</p><p>Decision 9 (design spec): <code>Finding</code>'s <code>impactEstimate</code> field, plus the <code>utilizationFraction</code>/<code>memorySize</code>/<code>diskSize</code>/<code>numCachedPartitions</code>/<code>numPartitions</code> instrumentation fields the impact estimator reads, were added without bumping <code>EVIDENCE_SCHEMA_VERSION</code> past <code>1</code>. Both additions are purely optional and ride <code>Finding</code>'s existing optional-field-plus-catch-all convention, so an evidence report built before these fields existed still deserializes and compares byte-for-byte against one built after: nothing about the schema's stability guarantee changed, only its surface grew. A future reader who notices <code>impactEstimate</code> in the JSON without a schema bump is looking at this deliberate call, not an oversight.</p><p>2026-08-30 update: <code>EVIDENCE_SCHEMA_VERSION</code> was bumped to <code>2</code> for the occupancy-weighted attribution redesign (see <a href="./impact-estimation.html#occupancy-weighted-attribution">Occupancy-weighted attribution</a>): <code>ImpactEstimate</code>'s shape changed from <code>{low, high}</code> to <code>{basis, wallClock, estimateMethod, rawWaste?}</code>, a real, non-additive breaking change to a field this same Decision 9 previously shipped without a bump. <code>impactEstimate</code> had zero consumers outside <code>packages/core/src/impact-estimator.ts</code> and its own tests at the time of this bump (confirmed by grep across <code>src/view/*</code> and <code>packages/core/src/evidence-report.ts</code>), so no other code needed migrating alongside it.</p><p><code>FindingRow</code>'s schema was extended to surface <code>impactEstimate</code> as a first-class column without further bumping <code>EVIDENCE_SCHEMA_VERSION</code> past <code>2</code>, consistent with <code>Finding</code>'s existing optional-field-plus-catch-all convention. It appears after the pinned core columns (<code>id</code>, <code>type</code>, <code>impactBand</code>, <code>stageId</code>, <code>metric</code>, <code>value</code>, <code>recommendation</code>, <code>detectorVersion</code>, plus optional but pinned <code>confidence</code>, <code>validationRequired</code>, <code>docAnchor</code>) without displacing any of them; byte-for-byte deserializability of existing reports is preserved. <code>FindingRow.impactEstimate</code> carries the full contract documented in <a href="./impact-estimation.html#occupancy-weighted-attribution">Impact estimation</a> (basis, wallClock, estimateMethod, rawWaste).</p><p>2026-09-03 update: the "Fix These First" dashboard redesign (impact-ranked recommendation rollup, short per-finding action labels, a clean-checks table) was UI-only when it shipped; this update ports the underlying data into <code>buildEvidenceReport()</code> so the CLI (<code>packages/cli/bin/sparkforensics-analyze.mjs</code>) and the MCP tool (<code>diagnoseRun</code> in <code>packages/core/src/mcp-tools.ts</code>) get it too, not just the web markdown/JSON download. Three additions, all purely additive, so this does not bump <code>EVIDENCE_SCHEMA_VERSION</code> past <code>2</code>, the same rationale as the <code>impactEstimate</code> addition immediately above: <code>FindingRow.actionLabel</code> is now always present (a short imperative label like "Reduce shuffle size"), sourced from a new core module, <code>packages/core/src/finding-action-label.ts</code>'s <code>coreFindingActionLabel</code>, extracted from the (type, discriminant) switch statement that used to live only in the view layer (<code>src/view/finding-action-label.ts</code>, which now wraps the core function and layers its own <code>REGISTRY</code> fallback on top). <code>EvidenceReportJson.recommendations</code> is the same impact-ranked <code>buildRecommendationRollup</code> grouping (<code>packages/core/src/recommendation-rollup.ts</code>) that <code>FixTheseFirst.tsx</code> renders, so CLI/MCP/download consumers get the same "what's the highest-impact fix" ranking the dashboard shows, without changing the existing <code>findings</code> array's own impact-band-sorted order at all. <code>EvidenceReportJson.cleanChecks</code> lists every detector type that fired zero findings this run, each with <code>getThresholdSummary</code>'s one-line "what would have tripped it" sentence; unlike the dashboard's <code>Alerts.tsx</code> clean-checks table, it deliberately includes the one "always-mounted" reference type (<code>coreLocality</code>) even when it has no findings, since a flat evidence report has no separate always-visible surface for it to already appear on the way that widget does on the board.</p><h3 id="finding-identity" tabindex="-1">Finding identity <a class="header-anchor" href="#finding-identity" aria-label="Permalink to "Finding identity""></a></h3><p>Every finding <code>analyzer.ts</code> emits carries a stable <code>id</code> (<code>push()</code>'s <code>findingId()</code> choke point, FNV-1a hash over <code>type | location | metric | value | discriminators</code>) and a <code>detectorVersion</code> (from the emitting <code>DETECTORS</code> entry). The two are deliberately decoupled: <code>id</code> is derived only from a finding's evidence tuple, never from <code>detectorVersion</code>, so a threshold or logic tweak that bumps a detector's <code>version</code> does not change the <code>id</code> of findings it still emits at the same location/metric. A saved reference (dashboard bookmark, exported report row, future URL-restored filter) keeps pointing at the same logical finding across detector revisions; <code>detectorVersion</code> is separate provenance metadata for "which ruleset produced this," not part of identity. See <code>packages/core/test/analyzer-finding-identity.test.js</code> for the decoupling proof.</p><p><code>id</code> stability holds for equivalent reruns of the same schema/analyzer version on the same input. Changing the id-derivation rule itself (the hash algorithm, or which fields feed the location key/discriminators) is an intentional breaking change and must bump <code>EVIDENCE_SCHEMA_VERSION</code> (<code>packages/core/src/evidence-report.ts</code>); there is no separate id-scheme version.</p><h3 id="headless-analysis-cli-v1" tabindex="-1">Headless analysis CLI (V1) <a class="header-anchor" href="#headless-analysis-cli-v1" aria-label="Permalink to "Headless analysis CLI (V1)""></a></h3><p><code>packages/cli/bin/sparkforensics-analyze.mjs</code> runs the same parser + detector contracts outside the browser, for CI. It accepts a single event-log file or a rolling-log directory, drives <code>runParse</code>/<code>runParseFiles</code> (<code>packages/core/src/parser-worker.ts</code>) with a Node-only File-like shim (<code>packages/core/src/cli/collect-run.ts</code>), and writes the exact <code>buildEvidenceReport</code> JSON schema described above: one format, not a second. Alternatively, <code>--shs-base-url <url> --app-id <id> [--attempt-id <id>]</code> fetches the run from a Spark History Server instead (mutually exclusive with the positional file/directory argument), calling <code>resolveFromShs</code> (<code>packages/core/src/shs-load.ts</code>, shared with the MCP server's SHS source path below) directly; no dependency on the <code>packages/server</code> package. A failed SHS fetch reports its message to <code>stderr</code> and exits <code>2</code>, same as a local file that can't be parsed.</p><p>Optional CLI-flag budgets (<code>--max-runtime <ms></code>, <code>--max-spill <gb></code>, <code>--max-skew <ratio></code>, <code>--max-failed-task-rate <pct></code>, <code>--min-efficiency <pct></code>) are evaluated in <code>packages/core/src/cli/budgets.ts</code> against the existing finding catalog (<code>analyze()</code>) and <code>computeEfficiencyModel</code>; there is no second rule engine. A budget whose required evidence is missing (e.g. the run never emitted <code>ApplicationEnd</code>, or has no usable per-task <code>runAggregates</code>) is reported as inconclusive (<code>stderr</code> warning) rather than silently passing, and gets its own exit code distinct from both pass and violation. Exit codes: <code>0</code> pass, <code>1</code> a configured budget was violated, <code>2</code> the input could not be parsed at all, <code>3</code> no violations but at least one budget was inconclusive. A violation always wins over an inconclusive result in the same run (exit <code>1</code>, not <code>3</code>).</p><h3 id="mcp-server-v1" tabindex="-1">MCP server (V1) <a class="header-anchor" href="#mcp-server-v1" aria-label="Permalink to "MCP server (V1)""></a></h3><p><code>packages/core/src/mcp-server-factory.ts</code>'s <code>createMcpServer()</code> registers 6 tools: <code>diagnose_run</code> (thresholded findings + remediation text), <code>get_run_summary</code> (app/stage/job/sql counts and duration, no findings), <code>compare_runs</code> (categorized findings delta + metric deltas between two runs), <code>evaluate_budgets</code> (pass/fail budget thresholds against one run, optionally with a second run for regression/fail-on-introduced budgets: the MCP side of the CLI's <code>evaluateBudgets()</code> gating), <code>get_finding_evidence</code> (raw evidence bundle for one finding, for drill-down after <code>diagnose_run</code>), and <code>get_finding_documentation</code> (detection/tuning reference docs for one finding type, independent of any run). None re-implement detector logic; all repackage <code>analyze</code>/<code>buildEvidenceReport</code>/<code>compareRuns</code>/<code>captureSnapshot</code>/<code>evaluateBudgets</code> from <code>packages/core/src/mcp-tools.ts</code>, which resolves a <code>source</code> (event-log <code>path</code>, or an SHS <code>shsBaseUrl</code>/<code>appId</code>/<code>attemptId</code> triple) into a cached <code>AppModel</code>. The SHS source path calls <code>resolveFromShs</code> (<code>packages/core/src/shs-load.ts</code>, also used directly by the CLI's <code>--shs-base-url</code> mode above), which reuses two extractions shared with the browser ingestion flow: <code>fetchShsEventLog</code> (<code>packages/core/src/proxy.js</code>) fetches the zip, and <code>decodeShsArchive</code> (<code>packages/core/src/shs-fetch.ts</code>, re-exported from <code>parser-worker.ts</code>'s barrel) decodes it into a parsed <code>AppModel</code>, the same split <code>runParseFromUrl</code> itself calls into.</p><p>Two transports connect to that one factory: <code>packages/mcp/bin/sparkforensics-mcp.mjs</code> (stdio, for local MCP clients) and <code>packages/server/index.js</code>'s <code>/mcp</code> route (streamable HTTP, for the local server). The run cache (<code>packages/core/src/mcp-tools.ts</code>) is a module-level LRU (cap 8 and 15-minute idle TTL by default, overridable via <code>SPARKFORENSICS_MCP_CACHE_CAP</code> and <code>SPARKFORENSICS_MCP_CACHE_TTL_MS</code>, lazily swept on access) keyed by resolved source (path mtime, or SHS baseUrl+appId+attemptId), so a client mints a <code>runId</code> once via <code>resolveOrCreateRun</code> and reuses it across subsequent tool calls instead of re-parsing.</p><p>Every tool failure comes back as <code>{isError: true, content: [...], structuredContent: {code}}</code>, never an HTTP-status-shaped error; <code>code</code> is one of the 5 existing SHS codes (<code>SHS_ERROR_CODES</code>, <code>packages/core/src/shs-request.js</code>) plus <code>run-not-found</code>, <code>finding-not-found</code>, <code>invalid-event-log</code> (also covers a nonexistent <code>path</code> source), and <code>archive-too-large</code> (SHS archive over the <code>SPARKFORENSICS_MAX_ARCHIVE_BYTES</code> byte cap, default 1 GiB, because the MCP path buffers the whole archive in memory, unlike the streaming <code>/shs-proxy</code> route).</p><p><code>scripts/vendor-core.mjs</code> (shared by <code>packages/cli</code>, <code>packages/mcp</code>, and <code>packages/server</code>'s <code>prepack</code> scripts) vendors <code>packages/core/src/</code> wholesale into each package's own <code>vendor-core/</code> at pack time, pre-stripping TypeScript to plain <code>.js</code> (Node's native TS stripping refuses to run on <code>.ts</code> files under <code>node_modules</code>, which is exactly where a published <code>vendor-core/</code> lands). Each package's bin/entry point resolves its needed module from <code>vendor-core/</code> if present, else falls back to the real <code>packages/core/src/</code> sibling loaded as <code>.ts</code> directly (<code>packages/server/index.js</code> mirrors <code>resolveStaticRoot</code>'s <code>public/</code>-vs-<code>../dist</code> pattern for this same fallback).</p><h2 id="typescript-core-and-runtime-event-validation" tabindex="-1">TypeScript core and runtime event validation <a class="header-anchor" href="#typescript-core-and-runtime-event-validation" aria-label="Permalink to "TypeScript core and runtime event validation""></a></h2><p>All of <code>src/</code> (browser SPA) and <code>packages/core/src/</code> (shared analysis logic) is strict TypeScript; the only plain-<code>.js</code> holdouts are the two vendored third-party decompressors, <code>packages/core/src/vendor/fflate.js</code> and <code>packages/core/src/vendor/fzstd.js</code> (left untouched deliberately: vendored code, not project code). Every remaining import of a same-repo module uses a <code>.ts</code> specifier (e.g. <code>import { dispatchLine } from './event-handlers.ts'</code>), not <code>.js</code>: the CLI and MCP entrypoints (<code>packages/cli/bin/sparkforensics-analyze.mjs</code>, <code>packages/mcp/bin/sparkforensics-mcp.mjs</code>) run under plain Node's ESM resolver, which cannot remap a <code>.js</code> specifier to a same-named <code>.ts</code> file the way Vite/Vitest's bundler-style resolver can. <code>tsconfig.json</code> sets <code>"allowImportingTsExtensions": true</code> to make this legal.</p><p>Event schema validation lives in <code>packages/core/src/event-schemas.ts</code>: one zod schema per <code>SparkListener*</code> event variant the parser understands (15 total: <code>LogStart</code>, <code>ApplicationStart</code>, <code>EnvironmentUpdate</code>, <code>ApplicationEnd</code>, <code>JobStart</code>, <code>JobEnd</code>, <code>StageSubmitted</code>, <code>StageCompleted</code>, <code>StageExecutorMetrics</code>, <code>TaskEnd</code>, the three SQL-execution-UI listener events, <code>ExecutorAdded</code>, <code>ExecutorRemoved</code>), combined into <code>SparkEventSchema = z.discriminatedUnion('Event', [...])</code>. <code>processEvent</code>'s switch (<code>event-handlers.ts</code>) consumes the resulting <code>SparkEvent</code> union type directly, so a schema change and a handler's expectations can't silently drift apart.</p><p>The recursive <code>sparkPlanInfo.children</code> tree carried on SQL-execution-start events is validated by <code>parseSparkPlanInfoTree</code>, an iterative, explicit heap-allocated-stack parser, deliberately not <code>z.lazy()</code>. A real plan tree's depth is unbounded, and attacker-uncontrolled input should never hand zod's own recursive schema resolution an arbitrarily deep structure to walk. The iterative parser shallow-validates one node at a time via <code>z.object(...)</code> and builds the tree itself, capping at <code>MAX_PLAN_DEPTH = 500</code> (throws past that). This mirrors <code>packages/core/src/plan-tree-walk.ts</code>'s <code>walkPlanTree</code>, which uses the same iterative-over-recursive approach for the <em>resolved</em> tree; <code>event-schemas.ts</code> applies it one layer earlier, to the raw JSON before it becomes a tree at all.</p><p>There are two external-data boundaries, the two places this codebase parses data it does not control. Both now run that data through a schema, and both treat a validation failure the same way: a silent skip, not a distinct error.</p><ul><li><code>dispatchLine</code> (<code>event-handlers.ts</code>): after <code>JSON.parse</code> succeeds, a line whose <code>Event</code> value is one of the 15 modeled types but fails that type's own schema now increments <code>skippedLines</code> (previously: no shape validation existed at all, and a malformed event silently corrupted downstream state with no signal anywhere). An <code>Event</code> value outside the 15 modeled types is still silently ignored without incrementing <code>skippedLines</code>, unchanged from before migration (see the note below on why the broader design was rejected).</li><li><code>shs-fetch.ts</code>: on a non-OK response from the local SHS proxy, the JSON error envelope is validated against <code>ShsProxyErrorBodySchema</code> (<code>packages/core/src/shs-schemas.ts</code>, <code>{ code: string }</code>, <code>.passthrough()</code>). A body that isn't valid JSON, or is JSON but fails that schema, falls back to the generic <code>access-or-upstream-failure</code> code: the same silent-skip treatment as a malformed log line, collapsing what could have been a separate "malformed SHS response" error code into the existing generic one.</li></ul><p>Neither boundary distinguishes "malformed JSON" from "wrong shape" from "unrecognized variant" in what it reports outward: all three collapse into the same skip/fallback path. That was a deliberate scope decision. The correction below says why the <em>line</em> boundary stops there rather than flagging every unrecognized event type.</p><blockquote><p>Correction made during migration, not part of the original design: an earlier draft counted <em>any</em> <code>Event</code> value outside the 15 modeled types toward <code>skippedLines</code>, not just ones that fail their own schema. Running that design against real Spark event logs (which always contain plenty of ordinary event types this tool has never modeled: <code>TaskStart</code>, <code>BlockManagerAdded</code>, <code>ExecutorMetricsUpdate</code>, and others, always silently ignored pre-migration) pushed <code>skippedLines</code> from 0 to the tens of thousands on completely healthy logs. That would have tripped <code>evidence-availability.ts</code>'s fail-closed <code>trustworthy</code> gate and shown a false "malformed JSON" warning on every real file. Caught in review against real fixtures, not synthetic ones; fixed to the narrower rule described above before merging. <code>evidence-availability.ts</code>'s gate itself (<code>trustworthy = skippedLines === 0 && applicationEnds > 0</code>) needed no code change once this was fixed: it was already correct, the input feeding it was not.</p></blockquote><p>The exhaustiveness convention is <code>packages/core/src/assert-never.ts</code>. <code>assertNever(x: never): never</code> throws at runtime and, more importantly, fails <code>tsc</code> at compile time if <code>x</code> is not actually <code>never</code>, i.e. if some case of a union type isn't handled. Used today at the <code>default</code> arm of <code>event-handlers.ts</code>'s <code>processEvent</code> switch (over <code>SparkEvent</code>) and <code>analyzer.ts</code>'s scope-dispatch switch (over a detector's <code>scope: 'stage' | 'sql' | 'app' | 'config'</code>). It is the project's standard pattern for any future exhaustive switch or dispatch over a closed union: add <code>default: return assertNever(x);</code> (or the closest non-returning equivalent) so that adding a new union member without updating every consumer becomes a compile error instead of a silent runtime gap.</p></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/contributor-guide/architecture/overview.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>Overview</span><!--]--></a></div><div class="pager" data-v-e257564d><a class="VPLink link pager-link next" href="/docs/contributor-guide/architecture/state-and-history.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Next page</span><span class="title" data-v-e257564d>State & history intake</span><!--]--></a></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
|
|
31
|
+
<script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"CWpj01WU\",\"contributor-guide_architecture_detector-contract.md\":\"CgzUsQ6W\",\"contributor-guide_architecture_drill-down.md\":\"BtPdlM7r\",\"contributor-guide_architecture_impact-estimation.md\":\"CooslVJt\",\"contributor-guide_architecture_index.md\":\"3TO9ic6w\",\"contributor-guide_architecture_overview.md\":\"CehiRmGn\",\"contributor-guide_architecture_state-and-history.md\":\"C-xxn0q7\",\"contributor-guide_architecture_widget-rendering.md\":\"R27gQrgY\",\"contributor-guide_architecture_worker-protocol.md\":\"IbnfNrV3\",\"contributor-guide_contributing.md\":\"CvRsdr6J\",\"contributor-guide_development-setup.md\":\"DvAN_9mK\",\"contributor-guide_testing.md\":\"6rIKqSyY\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"Df1YMIHu\",\"tuning-reference_aqe.md\":\"BIsCtLzm\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"CEstB3Ia\",\"tuning-reference_bottleneck-cold-start.md\":\"CEuy-72y\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"CIohQDfn\",\"tuning-reference_bottleneck-failures.md\":\"4z5BXGJ2\",\"tuning-reference_bottleneck-gc.md\":\"DSxzZRK7\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"BaJl__1W\",\"tuning-reference_bottleneck-memory-utilization.md\":\"DbP-SJZc\",\"tuning-reference_bottleneck-retry-waste.md\":\"D5JMjVOt\",\"tuning-reference_bottleneck-shuffle.md\":\"CM-nTmIH\",\"tuning-reference_bottleneck-skew.md\":\"BdUwiDhn\",\"tuning-reference_bottleneck-slow-host.md\":\"BlIo6UDW\",\"tuning-reference_bottleneck-small-files.md\":\"B8kloyx8\",\"tuning-reference_bottleneck-spill.md\":\"PNH7mITt\",\"tuning-reference_bottleneck-straggler.md\":\"DY36fHN5\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"QTV7O8kU\",\"tuning-reference_bottleneck-utilization.md\":\"DTiueZC3\",\"tuning-reference_caching.md\":\"B7aQ8asB\",\"tuning-reference_cluster-config.md\":\"ZVmDGsQ3\",\"tuning-reference_config.md\":\"UvveiWG3\",\"tuning-reference_data-formats.md\":\"bjCAWH3N\",\"tuning-reference_index.md\":\"BQ_NooMV\",\"tuning-reference_intro.md\":\"CobD-lGB\",\"tuning-reference_joins.md\":\"BtKs_CuW\",\"tuning-reference_memory-model.md\":\"DhT-n4y3\",\"tuning-reference_metrics.md\":\"mLOh7Apj\",\"tuning-reference_partitioning.md\":\"q0zKF_8X\",\"tuning-reference_pyspark.md\":\"DDCfvN9t\",\"tuning-reference_shuffle.md\":\"BZZ7R4Ix\",\"tuning-reference_spark-architecture.md\":\"Dwzm5avO\",\"tuning-reference_table-formats.md\":\"D6wj-2dX\",\"user-guide_alternative-log-retrieval.md\":\"B4tPGIal\",\"user-guide_getting-started.md\":\"BJvwLEIM\",\"user-guide_mcp-tools.md\":\"Vi3RoflJ\",\"user-guide_run-comparison.md\":\"CQc1aoU8\",\"user-guide_understanding-findings.md\":\"DL1UDhvR\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Alternative log retrieval\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
|
|
32
32
|
|
|
33
33
|
</body>
|
|
34
34
|
</html>
|
|
@@ -6,14 +6,14 @@
|
|
|
6
6
|
<title>Contributing | SparkForensics</title>
|
|
7
7
|
<meta name="description" content="Docs for using and contributing to SparkForensics">
|
|
8
8
|
<meta name="generator" content="VitePress v1.6.4">
|
|
9
|
-
<link rel="preload stylesheet" href="/docs/assets/style.
|
|
9
|
+
<link rel="preload stylesheet" href="/docs/assets/style.DSixAiZE.css" as="style">
|
|
10
10
|
<link rel="preload stylesheet" href="/docs/vp-icons.css" as="style">
|
|
11
11
|
|
|
12
|
-
<script type="module" src="/docs/assets/app.
|
|
12
|
+
<script type="module" src="/docs/assets/app.APoq_UBs.js"></script>
|
|
13
13
|
<link rel="preload" href="/docs/assets/inter-roman-latin.Di8DUHzh.woff2" as="font" type="font/woff2" crossorigin="">
|
|
14
|
-
<link rel="modulepreload" href="/docs/assets/chunks/theme.
|
|
15
|
-
<link rel="modulepreload" href="/docs/assets/chunks/framework.
|
|
16
|
-
<link rel="modulepreload" href="/docs/assets/contributor-guide_contributing.md.
|
|
14
|
+
<link rel="modulepreload" href="/docs/assets/chunks/theme.PavXO_vz.js">
|
|
15
|
+
<link rel="modulepreload" href="/docs/assets/chunks/framework.DSg0KOwT.js">
|
|
16
|
+
<link rel="modulepreload" href="/docs/assets/contributor-guide_contributing.md.CvRsdr6J.lean.js">
|
|
17
17
|
<link rel="icon" type="image/svg+xml" href="/docs/favicon.svg">
|
|
18
18
|
<link rel="preconnect" href="https://fonts.googleapis.com">
|
|
19
19
|
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin="">
|
|
@@ -22,8 +22,8 @@
|
|
|
22
22
|
<script id="check-mac-os">document.documentElement.classList.toggle("mac",/Mac|iPhone|iPod|iPad/i.test(navigator.platform));</script>
|
|
23
23
|
</head>
|
|
24
24
|
<body>
|
|
25
|
-
<div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Contributor Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/development-setup.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Development setup</p><!--]--></a><!----></div><!----></div><section class="VPSidebarItem level-1 collapsible collapsed is-link" data-v-b3fd67f8><div class="item" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/" data-v-b3fd67f8><!--[--><h3 class="text" data-v-b3fd67f8>Architecture</h3><!--]--></a><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-b3fd67f8><span class="vpi-chevron-right caret-icon" data-v-b3fd67f8></span></div></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/overview.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Overview</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/worker-protocol.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Worker protocol</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/state-and-history.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>State & history intake</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/detector-contract.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Detector contract</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/impact-estimation.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Impact estimation</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/widget-rendering.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Widget rendering</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/board-widgets.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Board widgets</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/drill-down.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Drill-down</p><!--]--></a><!----></div><!----></div><!--]--></div></section><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/testing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Testing & verification</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/contributing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Contributing</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_contributor-guide_contributing" data-v-39a288b8><div><h1 id="contributing" tabindex="-1">Contributing <a class="header-anchor" href="#contributing" aria-label="Permalink to "Contributing""></a></h1><h2 id="where-does-new-content-go" tabindex="-1">Where does new content go? <a class="header-anchor" href="#where-does-new-content-go" aria-label="Permalink to "Where does new content go?""></a></h2><ul><li><code>docs/</code> is for internal engineering records: ADRs, audits, competitive research. It stays flat, unstructured markdown, read by contributors through GitHub's own file viewer.</li><li><code>docs-site/</code> (this site) is the published documentation for users and contributors. If you're writing something a user or a first-time contributor should read, it belongs here, not in <code>docs/</code>.</li></ul><h2 id="linting-before-you-commit" tabindex="-1">Linting before you commit <a class="header-anchor" href="#linting-before-you-commit" aria-label="Permalink to "Linting before you commit""></a></h2><p><code>npm install</code> runs the <code>prepare</code> script, which points git at <code>.githooks/</code> (<code>git config core.hooksPath .githooks</code>). After that, every commit runs <code>.githooks/pre-commit</code>: it lints any staged <code>.js</code>/<code>.mjs</code> files with eslint and blocks the commit if linting fails. <code>git commit --no-verify</code> skips it. To run the same linter by hand, use <code>npm run lint</code>.</p><h2 id="changesets-and-releases" tabindex="-1">Changesets and releases <a class="header-anchor" href="#changesets-and-releases" aria-label="Permalink to "Changesets and releases""></a></h2><p><code>packages/cli</code>, <code>packages/mcp</code>, and <code>packages/server</code> publish to npm; a PR that touches any of them needs a changeset. Run <code>npx changeset add</code>, answer its prompts, and commit the generated <code>.changeset/*.md</code> file. CI's <code>changeset</code> job (<code>scripts/check-changeset.sh</code>) fails the PR otherwise, with instructions to fix it.</p><p>A release itself is two merges, not one: merging your feature PR into <code>main</code> runs <code>.github/workflows/release.yml</code>, which opens (or updates) a "Version Packages" PR collecting all pending changesets. Nothing publishes yet. Merging <em>that</em> PR is what actually bumps versions and runs <code>npm publish</code> for whichever packages had pending changesets.</p><p>Maintainers: this pipeline depends on the three package names being claimed on npmjs.com and Trusted Publishing being registered against this repo (a one-time manual bootstrap step, tracked separately). Until that's done, every push to <code>main</code> will fail at the publish step. Check whether it's been done before merging a "Version Packages" PR.</p><h2 id="proposing-an-architectural-decision" tabindex="-1">Proposing an architectural decision <a class="header-anchor" href="#proposing-an-architectural-decision" aria-label="Permalink to "Proposing an architectural decision""></a></h2><p>Architectural decisions are recorded as ADRs under <code>docs/adr/</code>. See <a href="https://github.com/shuffle-works/sparkforensics/blob/main/docs/adr/README.md" target="_blank" rel="noreferrer"><code>docs/adr/README.md</code></a> for the naming convention and the required sections. An accepted decision is never rewritten: a change to it becomes a successor record.</p><p>For the full setup and test workflow, see <a href="./development-setup.html">Development setup</a> and <a href="./testing.html">Testing & verification</a>.</p></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/contributor-guide/testing.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>Testing & verification</span><!--]--></a></div><div class="pager" data-v-e257564d><!----></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
|
|
26
|
-
<script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"
|
|
25
|
+
<div id="app"><div class="Layout" data-v-5d98c3a5><!--[--><!--]--><!--[--><span tabindex="-1" data-v-0b0ada53></span><a href="#VPContent" class="VPSkipLink visually-hidden" data-v-0b0ada53>Skip to content</a><!--]--><!----><header class="VPNav" data-v-5d98c3a5 data-v-ae24b3ad><div class="VPNavBar" data-v-ae24b3ad data-v-6aa21345><div class="wrapper" data-v-6aa21345><div class="container" data-v-6aa21345><div class="title" data-v-6aa21345><div class="VPNavBarTitle has-sidebar" data-v-6aa21345 data-v-1168a8e4><a class="title" href="/docs/" data-v-1168a8e4><!--[--><!--]--><!--[--><img class="VPImage logo" src="/docs/favicon.svg" alt data-v-8426fc1a><!--]--><span data-v-1168a8e4>SparkForensics</span><!--[--><!--]--></a></div></div><div class="content" data-v-6aa21345><div class="content-body" data-v-6aa21345><!--[--><!--]--><div class="VPNavBarSearch search" data-v-6aa21345><!--[--><!----><div id="local-search"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><span class="vp-icon DocSearch-Search-Icon"></span><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"><kbd class="DocSearch-Button-Key"></kbd><kbd class="DocSearch-Button-Key">K</kbd></span></button></div><!--]--></div><nav aria-labelledby="main-nav-aria-label" class="VPNavBarMenu menu" data-v-6aa21345 data-v-dc692963><span id="main-nav-aria-label" class="visually-hidden" data-v-dc692963> Main Navigation </span><!--[--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/user-guide/getting-started.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>User Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/contributor-guide/development-setup.html" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Contributor Guide</span><!--]--></a><!--]--><!--[--><a class="VPLink link VPNavBarMenuLink" href="/docs/tuning-reference/" tabindex="0" data-v-dc692963 data-v-e56f3d57><!--[--><span data-v-e56f3d57>Tuning Reference</span><!--]--></a><!--]--><!--]--></nav><!----><div class="VPNavBarAppearance appearance" data-v-6aa21345 data-v-6c893767><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-6c893767 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div><!----><div class="VPFlyout VPNavBarExtra extra" data-v-6aa21345 data-v-bb2aa2f0 data-v-cf11d7a2><button type="button" class="button" aria-haspopup="true" aria-expanded="false" aria-label="extra navigation" data-v-cf11d7a2><span class="vpi-more-horizontal icon" data-v-cf11d7a2></span></button><div class="menu" data-v-cf11d7a2><div class="VPMenu" data-v-cf11d7a2 data-v-b98bc113><!----><!--[--><!--[--><!----><div class="group" data-v-bb2aa2f0><div class="item appearance" data-v-bb2aa2f0><p class="label" data-v-bb2aa2f0>Appearance</p><div class="appearance-action" data-v-bb2aa2f0><button class="VPSwitch VPSwitchAppearance" type="button" role="switch" title aria-checked="false" data-v-bb2aa2f0 data-v-5337faa4 data-v-1d5665e3><span class="check" data-v-1d5665e3><span class="icon" data-v-1d5665e3><!--[--><span class="vpi-sun sun" data-v-5337faa4></span><span class="vpi-moon moon" data-v-5337faa4></span><!--]--></span></span></button></div></div></div><!----><!--]--><!--]--></div></div></div><!--[--><!--]--><button type="button" class="VPNavBarHamburger hamburger" aria-label="mobile navigation" aria-expanded="false" aria-controls="VPNavScreen" data-v-6aa21345 data-v-e5dd9c1c><span class="container" data-v-e5dd9c1c><span class="top" data-v-e5dd9c1c></span><span class="middle" data-v-e5dd9c1c></span><span class="bottom" data-v-e5dd9c1c></span></span></button></div></div></div></div><div class="divider" data-v-6aa21345><div class="divider-line" data-v-6aa21345></div></div></div><!----></header><div class="VPLocalNav has-sidebar empty" data-v-5d98c3a5 data-v-a6f0e41e><div class="container" data-v-a6f0e41e><button class="menu" aria-expanded="false" aria-controls="VPSidebarNav" data-v-a6f0e41e><span class="vpi-align-left menu-icon" data-v-a6f0e41e></span><span class="menu-text" data-v-a6f0e41e>Menu</span></button><div class="VPLocalNavOutlineDropdown" style="--vp-vh:0px;" data-v-a6f0e41e data-v-8a42e2b4><button data-v-8a42e2b4>Return to top</button><!----></div></div></div><aside class="VPSidebar" data-v-5d98c3a5 data-v-319d5ca6><div class="curtain" data-v-319d5ca6></div><nav class="nav" id="VPSidebarNav" aria-labelledby="sidebar-aria-label" tabindex="-1" data-v-319d5ca6><span class="visually-hidden" id="sidebar-aria-label" data-v-319d5ca6> Sidebar Navigation </span><!--[--><!--]--><!--[--><div class="no-transition group" data-v-c40bc020><section class="VPSidebarItem level-0 has-active" data-v-c40bc020 data-v-b3fd67f8><div class="item" role="button" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><h2 class="text" data-v-b3fd67f8>Contributor Guide</h2><!----></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/development-setup.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Development setup</p><!--]--></a><!----></div><!----></div><section class="VPSidebarItem level-1 collapsible collapsed is-link" data-v-b3fd67f8><div class="item" tabindex="0" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/" data-v-b3fd67f8><!--[--><h3 class="text" data-v-b3fd67f8>Architecture</h3><!--]--></a><div class="caret" role="button" aria-label="toggle section" tabindex="0" data-v-b3fd67f8><span class="vpi-chevron-right caret-icon" data-v-b3fd67f8></span></div></div><div class="items" data-v-b3fd67f8><!--[--><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/overview.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Overview</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/worker-protocol.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Worker protocol</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/state-and-history.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>State & history intake</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/detector-contract.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Detector contract</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/impact-estimation.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Impact estimation</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/widget-rendering.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Widget rendering</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/board-widgets.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Board widgets</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-2 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/architecture/drill-down.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Drill-down</p><!--]--></a><!----></div><!----></div><!--]--></div></section><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/testing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Testing & verification</p><!--]--></a><!----></div><!----></div><div class="VPSidebarItem level-1 is-link" data-v-b3fd67f8><div class="item" data-v-b3fd67f8><div class="indicator" data-v-b3fd67f8></div><a class="VPLink link link" href="/docs/contributor-guide/contributing.html" data-v-b3fd67f8><!--[--><p class="text" data-v-b3fd67f8>Contributing</p><!--]--></a><!----></div><!----></div><!--]--></div></section></div><!--]--><!--[--><!--]--></nav></aside><div class="VPContent has-sidebar" id="VPContent" data-v-5d98c3a5 data-v-1428d186><div class="VPDoc has-sidebar has-aside" data-v-1428d186 data-v-39a288b8><!--[--><!--]--><div class="container" data-v-39a288b8><div class="aside" data-v-39a288b8><div class="aside-curtain" data-v-39a288b8></div><div class="aside-container" data-v-39a288b8><div class="aside-content" data-v-39a288b8><div class="VPDocAside" data-v-39a288b8 data-v-3f215769><!--[--><!--]--><!--[--><!--]--><nav aria-labelledby="doc-outline-aria-label" class="VPDocAsideOutline" data-v-3f215769 data-v-a5bbad30><div class="content" data-v-a5bbad30><div class="outline-marker" data-v-a5bbad30></div><div aria-level="2" class="outline-title" id="doc-outline-aria-label" role="heading" data-v-a5bbad30>On this page</div><ul class="VPDocOutlineItem root" data-v-a5bbad30 data-v-b933a997><!--[--><!--]--></ul></div></nav><!--[--><!--]--><div class="spacer" data-v-3f215769></div><!--[--><!--]--><!----><!--[--><!--]--><!--[--><!--]--></div></div></div></div><div class="content" data-v-39a288b8><div class="content-container" data-v-39a288b8><!--[--><!--]--><main class="main" data-v-39a288b8><div style="position:relative;" class="vp-doc _docs_contributor-guide_contributing" data-v-39a288b8><div><h1 id="contributing" tabindex="-1">Contributing <a class="header-anchor" href="#contributing" aria-label="Permalink to "Contributing""></a></h1><h2 id="where-does-new-content-go" tabindex="-1">Where does new content go? <a class="header-anchor" href="#where-does-new-content-go" aria-label="Permalink to "Where does new content go?""></a></h2><ul><li><code>docs/</code> is for internal engineering records: ADRs, audits, competitive research. It stays flat, unstructured markdown, read by contributors through GitHub's own file viewer.</li><li><code>docs-site/</code> (this site) is the published documentation for users and contributors. If you're writing something a user or a first-time contributor should read, it belongs here, not in <code>docs/</code>.</li></ul><h2 id="linting-before-you-commit" tabindex="-1">Linting before you commit <a class="header-anchor" href="#linting-before-you-commit" aria-label="Permalink to "Linting before you commit""></a></h2><p><code>npm install</code> runs the <code>prepare</code> script, which points git at <code>.githooks/</code> (<code>git config core.hooksPath .githooks</code>). After that, every commit runs <code>.githooks/pre-commit</code>: it lints any staged <code>.js</code>/<code>.mjs</code> files with eslint and blocks the commit if linting fails. <code>git commit --no-verify</code> skips it. To run the same linter by hand, use <code>npm run lint</code>.</p><h2 id="changesets-and-releases" tabindex="-1">Changesets and releases <a class="header-anchor" href="#changesets-and-releases" aria-label="Permalink to "Changesets and releases""></a></h2><p>A changeset is required for a PR that touches <code>packages/cli</code>, <code>packages/mcp</code>, <code>packages/server</code>, or the root site app (<code>src/</code>, <code>index.html</code>, <code>vite.config.ts</code>, or the root <code>package.json</code>). Run <code>npx changeset add</code>, answer its prompts, and commit the generated <code>.changeset/*.md</code> file. CI's <code>changeset</code> job (<code>scripts/check-changeset.sh</code>) fails the PR otherwise, with instructions to fix it. <code>docs-site/</code> and <code>docs/</code> are exempt: doc-only changes don't need one.</p><p>Needing a changeset doesn't mean getting published, though. Only <code>packages/cli</code>, <code>packages/mcp</code>, and <code>packages/server</code> publish to npm. The root site (<code>sparkforensics</code> itself) is <code>"private": true</code> and never published; its changeset only bumps its <code>package.json</code> version and writes a <code>CHANGELOG.md</code> entry, via <code>.changeset/config.json</code>'s <code>privatePackages.version</code> setting. <code>@sparkforensics/core</code> is the other private package but stays out of this entirely (<code>.changeset/config.json</code>'s <code>ignore</code> list): it's vendored into cli/mcp by filesystem copy at pack time, not read as a real dependency, so versioning it would have no consumer.</p><p>A release itself is two merges, not one: merging your feature PR into <code>main</code> runs <code>.github/workflows/release.yml</code>, which opens (or updates) a "Version Packages" PR collecting all pending changesets. Nothing publishes yet. Merging <em>that</em> PR is what actually bumps versions, writes changelogs, and runs <code>npm publish</code> for whichever of the three publishable packages had pending changesets.</p><p>Maintainers: this pipeline depends on the three package names being claimed on npmjs.com and Trusted Publishing being registered against this repo (a one-time manual bootstrap step, tracked separately). Until that's done, every push to <code>main</code> will fail at the publish step. Check whether it's been done before merging a "Version Packages" PR.</p><p>For the full setup and test workflow, see <a href="./development-setup.html">Development setup</a> and <a href="./testing.html">Testing & verification</a>.</p></div></div></main><footer class="VPDocFooter" data-v-39a288b8 data-v-e257564d><!--[--><!--]--><!----><nav class="prev-next" aria-labelledby="doc-footer-aria-label" data-v-e257564d><span class="visually-hidden" id="doc-footer-aria-label" data-v-e257564d>Pager</span><div class="pager" data-v-e257564d><a class="VPLink link pager-link prev" href="/docs/contributor-guide/testing.html" data-v-e257564d><!--[--><span class="desc" data-v-e257564d>Previous page</span><span class="title" data-v-e257564d>Testing & verification</span><!--]--></a></div><div class="pager" data-v-e257564d><!----></div></nav></footer><!--[--><!--]--></div></div></div><!--[--><!--]--></div></div><!----><!--[--><!--]--></div></div>
|
|
26
|
+
<script>window.__VP_HASH_MAP__=JSON.parse("{\"contributor-guide_architecture_board-widgets.md\":\"CWpj01WU\",\"contributor-guide_architecture_detector-contract.md\":\"CgzUsQ6W\",\"contributor-guide_architecture_drill-down.md\":\"BtPdlM7r\",\"contributor-guide_architecture_impact-estimation.md\":\"CooslVJt\",\"contributor-guide_architecture_index.md\":\"3TO9ic6w\",\"contributor-guide_architecture_overview.md\":\"CehiRmGn\",\"contributor-guide_architecture_state-and-history.md\":\"C-xxn0q7\",\"contributor-guide_architecture_widget-rendering.md\":\"R27gQrgY\",\"contributor-guide_architecture_worker-protocol.md\":\"IbnfNrV3\",\"contributor-guide_contributing.md\":\"CvRsdr6J\",\"contributor-guide_development-setup.md\":\"DvAN_9mK\",\"contributor-guide_testing.md\":\"6rIKqSyY\",\"index.md\":\"CHJVslga\",\"tuning-reference_anti-patterns.md\":\"Df1YMIHu\",\"tuning-reference_aqe.md\":\"BIsCtLzm\",\"tuning-reference_bottleneck-broadcast-sizing.md\":\"CEstB3Ia\",\"tuning-reference_bottleneck-cold-start.md\":\"CEuy-72y\",\"tuning-reference_bottleneck-duplicate-plan-subtree.md\":\"CIohQDfn\",\"tuning-reference_bottleneck-failures.md\":\"4z5BXGJ2\",\"tuning-reference_bottleneck-gc.md\":\"DSxzZRK7\",\"tuning-reference_bottleneck-job-failure-rate.md\":\"BaJl__1W\",\"tuning-reference_bottleneck-memory-utilization.md\":\"DbP-SJZc\",\"tuning-reference_bottleneck-retry-waste.md\":\"D5JMjVOt\",\"tuning-reference_bottleneck-shuffle.md\":\"CM-nTmIH\",\"tuning-reference_bottleneck-skew.md\":\"BdUwiDhn\",\"tuning-reference_bottleneck-slow-host.md\":\"BlIo6UDW\",\"tuning-reference_bottleneck-small-files.md\":\"B8kloyx8\",\"tuning-reference_bottleneck-spill.md\":\"PNH7mITt\",\"tuning-reference_bottleneck-straggler.md\":\"DY36fHN5\",\"tuning-reference_bottleneck-tiny-tasks.md\":\"QTV7O8kU\",\"tuning-reference_bottleneck-utilization.md\":\"DTiueZC3\",\"tuning-reference_caching.md\":\"B7aQ8asB\",\"tuning-reference_cluster-config.md\":\"ZVmDGsQ3\",\"tuning-reference_config.md\":\"UvveiWG3\",\"tuning-reference_data-formats.md\":\"bjCAWH3N\",\"tuning-reference_index.md\":\"BQ_NooMV\",\"tuning-reference_intro.md\":\"CobD-lGB\",\"tuning-reference_joins.md\":\"BtKs_CuW\",\"tuning-reference_memory-model.md\":\"DhT-n4y3\",\"tuning-reference_metrics.md\":\"mLOh7Apj\",\"tuning-reference_partitioning.md\":\"q0zKF_8X\",\"tuning-reference_pyspark.md\":\"DDCfvN9t\",\"tuning-reference_shuffle.md\":\"BZZ7R4Ix\",\"tuning-reference_spark-architecture.md\":\"Dwzm5avO\",\"tuning-reference_table-formats.md\":\"D6wj-2dX\",\"user-guide_alternative-log-retrieval.md\":\"B4tPGIal\",\"user-guide_getting-started.md\":\"BJvwLEIM\",\"user-guide_mcp-tools.md\":\"Vi3RoflJ\",\"user-guide_run-comparison.md\":\"CQc1aoU8\",\"user-guide_understanding-findings.md\":\"DL1UDhvR\"}");window.__VP_SITE_DATA__=JSON.parse("{\"lang\":\"en-US\",\"dir\":\"ltr\",\"title\":\"SparkForensics\",\"description\":\"Docs for using and contributing to SparkForensics\",\"base\":\"/docs/\",\"head\":[],\"router\":{\"prefetchLinks\":true},\"appearance\":true,\"themeConfig\":{\"logo\":\"/favicon.svg\",\"nav\":[{\"text\":\"User Guide\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Contributor Guide\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Tuning Reference\",\"link\":\"/tuning-reference/\"}],\"sidebar\":{\"/tuning-reference/\":[{\"text\":\"Getting Started\",\"items\":[{\"text\":\"Introduction\",\"link\":\"/tuning-reference/intro\"}]},{\"text\":\"Optimization Guide\",\"items\":[{\"text\":\"Spark Execution Model\",\"link\":\"/tuning-reference/spark-architecture\"},{\"text\":\"Memory Management\",\"link\":\"/tuning-reference/memory-model\"},{\"text\":\"Partitioning\",\"link\":\"/tuning-reference/partitioning\"},{\"text\":\"Join Optimization\",\"link\":\"/tuning-reference/joins\"},{\"text\":\"Shuffle\",\"link\":\"/tuning-reference/shuffle\"},{\"text\":\"Data Formats\",\"link\":\"/tuning-reference/data-formats\"},{\"text\":\"Table Formats\",\"link\":\"/tuning-reference/table-formats\"},{\"text\":\"Caching & Persistence\",\"link\":\"/tuning-reference/caching\"},{\"text\":\"PySpark Specifics\",\"link\":\"/tuning-reference/pyspark\"},{\"text\":\"Adaptive Query Execution\",\"link\":\"/tuning-reference/aqe\"},{\"text\":\"Cluster Tuning\",\"link\":\"/tuning-reference/cluster-config\"},{\"text\":\"Anti-Patterns\",\"link\":\"/tuning-reference/anti-patterns\"}]},{\"text\":\"Detector Catalog\",\"items\":[{\"text\":\"Task Skew\",\"link\":\"/tuning-reference/bottleneck-skew\"},{\"text\":\"Shuffle I/O\",\"link\":\"/tuning-reference/bottleneck-shuffle\"},{\"text\":\"Memory / Disk Spill\",\"link\":\"/tuning-reference/bottleneck-spill\"},{\"text\":\"GC Pressure\",\"link\":\"/tuning-reference/bottleneck-gc\"},{\"text\":\"Cold Start\",\"link\":\"/tuning-reference/bottleneck-cold-start\"},{\"text\":\"Executor Utilization\",\"link\":\"/tuning-reference/bottleneck-utilization\"},{\"text\":\"Slow Host\",\"link\":\"/tuning-reference/bottleneck-slow-host\"},{\"text\":\"Task Failures\",\"link\":\"/tuning-reference/bottleneck-failures\"},{\"text\":\"Stragglers\",\"link\":\"/tuning-reference/bottleneck-straggler\"},{\"text\":\"Retry Waste\",\"link\":\"/tuning-reference/bottleneck-retry-waste\"},{\"text\":\"Tiny Tasks\",\"link\":\"/tuning-reference/bottleneck-tiny-tasks\"},{\"text\":\"Job Failure Rate\",\"link\":\"/tuning-reference/bottleneck-job-failure-rate\"},{\"text\":\"Memory Utilization\",\"link\":\"/tuning-reference/bottleneck-memory-utilization\"},{\"text\":\"Duplicate Plan Subtree\",\"link\":\"/tuning-reference/bottleneck-duplicate-plan-subtree\"},{\"text\":\"Small Files\",\"link\":\"/tuning-reference/bottleneck-small-files\"},{\"text\":\"Broadcast Sizing\",\"link\":\"/tuning-reference/bottleneck-broadcast-sizing\"}]},{\"text\":\"Reference\",\"items\":[{\"text\":\"Metrics Glossary\",\"link\":\"/tuning-reference/metrics\"},{\"text\":\"Spark Config Quick-Reference\",\"link\":\"/tuning-reference/config\"}]}],\"/user-guide/\":[{\"text\":\"User Guide\",\"items\":[{\"text\":\"Getting started\",\"link\":\"/user-guide/getting-started\"},{\"text\":\"Understanding findings\",\"link\":\"/user-guide/understanding-findings\"},{\"text\":\"Run comparison mode\",\"link\":\"/user-guide/run-comparison\"},{\"text\":\"MCP tools reference\",\"link\":\"/user-guide/mcp-tools\"},{\"text\":\"Alternative log retrieval\",\"link\":\"/user-guide/alternative-log-retrieval\"}]}],\"/contributor-guide/\":[{\"text\":\"Contributor Guide\",\"items\":[{\"text\":\"Development setup\",\"link\":\"/contributor-guide/development-setup\"},{\"text\":\"Architecture\",\"link\":\"/contributor-guide/architecture/\",\"collapsed\":true,\"items\":[{\"text\":\"Overview\",\"link\":\"/contributor-guide/architecture/overview\"},{\"text\":\"Worker protocol\",\"link\":\"/contributor-guide/architecture/worker-protocol\"},{\"text\":\"State & history intake\",\"link\":\"/contributor-guide/architecture/state-and-history\"},{\"text\":\"Detector contract\",\"link\":\"/contributor-guide/architecture/detector-contract\"},{\"text\":\"Impact estimation\",\"link\":\"/contributor-guide/architecture/impact-estimation\"},{\"text\":\"Widget rendering\",\"link\":\"/contributor-guide/architecture/widget-rendering\"},{\"text\":\"Board widgets\",\"link\":\"/contributor-guide/architecture/board-widgets\"},{\"text\":\"Drill-down\",\"link\":\"/contributor-guide/architecture/drill-down\"}]},{\"text\":\"Testing & verification\",\"link\":\"/contributor-guide/testing\"},{\"text\":\"Contributing\",\"link\":\"/contributor-guide/contributing\"}]}]},\"search\":{\"provider\":\"local\"}},\"locales\":{},\"scrollOffset\":134,\"cleanUrls\":false}");</script>
|
|
27
27
|
|
|
28
28
|
</body>
|
|
29
29
|
</html>
|