sparkforensics-server 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.js +3 -0
- package/package.json +1 -1
- package/public/assets/AutoscalingChurn-9Zo6IMGS.js +1 -0
- package/public/assets/CacheUtilization-MBDfMsKC.js +1 -0
- package/public/assets/CachingOpportunity-Cfw3wAFa.js +1 -0
- package/public/assets/ColdStart-BtS12b6_.js +1 -0
- package/public/assets/ConfigAudit-DNPV6EQg.js +1 -0
- package/public/assets/CoreUsageArea-BObEeenE.js +1 -0
- package/public/assets/Dashboard-CC_BXrNy.js +2 -0
- package/public/assets/DisclosureContext-Ds0hcZFn.js +1 -0
- package/public/assets/DuplicatePlanSubtree-vN3zqRsi.js +1 -0
- package/public/assets/ExecutorUtilization-CZObSLIj.js +1 -0
- package/public/assets/GcPressure-ByJlLOua.js +1 -0
- package/public/assets/IncompleteRun-MzqV4W8y.js +1 -0
- package/public/assets/JobFailures-LbMvAwAb.js +1 -0
- package/public/assets/MemoryUtilization-DH4jYvJK.js +1 -0
- package/public/assets/OverBroadcast-euk6HKg4.js +1 -0
- package/public/assets/PartitionHint-DThaMMRT.js +1 -0
- package/public/assets/PartitionSizing-yIN-iZzx.js +1 -0
- package/public/assets/{PlanExplorer-DdmPkKO1.js → PlanExplorer-Dd9WPK4S.js} +1 -1
- package/public/assets/PlanGraphRoute-NV5mCXRm.js +1 -0
- package/public/assets/RetryWaste-CXH4D-xH.js +1 -0
- package/public/assets/ShuffleIO-CW7U68ph.js +1 -0
- package/public/assets/Skew-PL79TppX.js +1 -0
- package/public/assets/SlowHost-GBO73hCi.js +1 -0
- package/public/assets/SmallFiles-B5cUjEav.js +1 -0
- package/public/assets/SpeculationWaste-mxi3jgb0.js +1 -0
- package/public/assets/Spill-CH1bNpxL.js +1 -0
- package/public/assets/StageFailed-YvxC11ii.js +1 -0
- package/public/assets/StageFindingGroup-B1A0nf-e.js +1 -0
- package/public/assets/{StageHeader-BSV8ALhT.js → StageHeader-BMNmUy-s.js} +1 -1
- package/public/assets/StageShape-DO3mQ6NS.js +1 -0
- package/public/assets/StageSlowness-SwRxfAEx.js +1 -0
- package/public/assets/Straggler-DHLTiBY-.js +1 -0
- package/public/assets/TaskFailures-Bj0QxiOc.js +1 -0
- package/public/assets/{TinyTask-CIPRxWfT.js → TinyTask-COUQQ5bQ.js} +1 -1
- package/public/assets/UnderBroadcast-RAUbQ6vw.js +1 -0
- package/public/assets/{finding-anchor-DF1cSHO1.js → finding-anchor-Dl8mIHiU.js} +1 -1
- package/public/assets/{impact-sort-7z0ZcBZf.js → impact-sort-Eihua9dY.js} +1 -1
- package/public/assets/index-DaC8A2FS.js +13 -0
- package/public/assets/{parser-worker-Q5KCPdKT.js → parser-worker-CfDvPkwF.js} +5 -5
- package/public/assets/{select-Cy5pmbrh.js → select-DkjFHlXR.js} +1 -1
- package/public/assets/{usePagedRows-BjPOpUU8.js → usePagedRows-Bo-aSztx.js} +1 -1
- package/public/assets/useRecentFiles-Bb4-ukVz.js +11 -0
- package/public/assets/{useSortMode-CU0vzThu.js → useSortMode-lDedD5a3.js} +1 -1
- package/public/docs/404.html +4 -3
- package/public/docs/assets/{app.C6w47a9W.js → app.yjczoibf.js} +1 -1
- package/public/docs/assets/chunks/@localSearchIndexroot.r9imX8Aq.js +1 -0
- package/public/docs/assets/chunks/{VPLocalSearchBox.BcTNvT55.js → VPLocalSearchBox.DOBbPfix.js} +1 -1
- package/public/docs/assets/chunks/{theme.D-mE6j02.js → theme.DFcAl8Ws.js} +2 -2
- package/public/docs/assets/contributor-guide_architecture_board-widgets.md.D0Kkk_r5.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_board-widgets.md.D0Kkk_r5.lean.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_detector-contract.md.caA46z-u.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_drill-down.md.CdtlKHl-.js +1 -0
- package/public/docs/assets/{contributor-guide_architecture_drill-down.md.CFNCE8JB.lean.js → contributor-guide_architecture_drill-down.md.CdtlKHl-.lean.js} +1 -1
- package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.BNp14LtL.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.BNp14LtL.lean.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_index.md.B9yZieYL.js +1 -0
- package/public/docs/assets/{contributor-guide_architecture_index.md.3TO9ic6w.lean.js → contributor-guide_architecture_index.md.B9yZieYL.lean.js} +1 -1
- package/public/docs/assets/contributor-guide_architecture_overview.md.DBGo-bwn.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_overview.md.DBGo-bwn.lean.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_state-and-history.md.w4lwybdh.js +1 -0
- package/public/docs/assets/{contributor-guide_architecture_state-and-history.md.DOTVHjTV.lean.js → contributor-guide_architecture_state-and-history.md.w4lwybdh.lean.js} +1 -1
- package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.CcvqxnZI.js +1 -0
- package/public/docs/assets/{contributor-guide_architecture_widget-rendering.md.BmpARZve.lean.js → contributor-guide_architecture_widget-rendering.md.CcvqxnZI.lean.js} +1 -1
- package/public/docs/assets/contributor-guide_architecture_worker-protocol.md.CKeFttKj.js +6 -0
- package/public/docs/assets/{contributor-guide_architecture_worker-protocol.md.21pTYhFF.lean.js → contributor-guide_architecture_worker-protocol.md.CKeFttKj.lean.js} +1 -1
- package/public/docs/assets/contributor-guide_contributing.md.CQMvGwFt.js +1 -0
- package/public/docs/assets/{contributor-guide_contributing.md.B2E9ni79.lean.js → contributor-guide_contributing.md.CQMvGwFt.lean.js} +1 -1
- package/public/docs/assets/contributor-guide_development-setup.md.D9bu4j_c.js +15 -0
- package/public/docs/assets/contributor-guide_development-setup.md.D9bu4j_c.lean.js +1 -0
- package/public/docs/assets/contributor-guide_testing.md.By6XGR9a.js +1 -0
- package/public/docs/assets/{contributor-guide_testing.md.DTwhpoN2.lean.js → contributor-guide_testing.md.By6XGR9a.lean.js} +1 -1
- package/public/docs/assets/{style.DSixAiZE.css → style.tmDhD3c8.css} +1 -1
- package/public/docs/assets/user-guide_alternative-log-retrieval.md.z-8npxNk.js +7 -0
- package/public/docs/assets/user-guide_alternative-log-retrieval.md.z-8npxNk.lean.js +1 -0
- package/public/docs/assets/user-guide_getting-started.md.BJW5oVhe.js +24 -0
- package/public/docs/assets/user-guide_getting-started.md.BJW5oVhe.lean.js +1 -0
- package/public/docs/assets/{user-guide_mcp-tools.md.dQwIvdTF.js → user-guide_mcp-tools.md.DcLiOa9t.js} +40 -23
- package/public/docs/assets/{user-guide_mcp-tools.md.dQwIvdTF.lean.js → user-guide_mcp-tools.md.DcLiOa9t.lean.js} +1 -1
- package/public/docs/assets/user-guide_run-comparison.md.qin0a4IX.js +1 -0
- package/public/docs/assets/{user-guide_run-comparison.md.ChLI3MK0.lean.js → user-guide_run-comparison.md.qin0a4IX.lean.js} +1 -1
- package/public/docs/assets/user-guide_understanding-findings.md.DzReyCXb.js +1 -0
- package/public/docs/contributor-guide/architecture/board-widgets.html +7 -6
- package/public/docs/contributor-guide/architecture/detector-contract.html +7 -6
- package/public/docs/contributor-guide/architecture/drill-down.html +7 -6
- package/public/docs/contributor-guide/architecture/impact-estimation.html +7 -6
- package/public/docs/contributor-guide/architecture/index.html +7 -6
- package/public/docs/contributor-guide/architecture/overview.html +7 -6
- package/public/docs/contributor-guide/architecture/state-and-history.html +7 -6
- package/public/docs/contributor-guide/architecture/widget-rendering.html +7 -6
- package/public/docs/contributor-guide/architecture/worker-protocol.html +10 -9
- package/public/docs/contributor-guide/contributing.html +7 -6
- package/public/docs/contributor-guide/development-setup.html +11 -10
- package/public/docs/contributor-guide/testing.html +7 -6
- package/public/docs/hashmap.json +1 -1
- package/public/docs/index.html +5 -4
- package/public/docs/tuning-reference/anti-patterns.html +5 -4
- package/public/docs/tuning-reference/aqe.html +5 -4
- package/public/docs/tuning-reference/bottleneck-broadcast-sizing.html +5 -4
- package/public/docs/tuning-reference/bottleneck-cold-start.html +5 -4
- package/public/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html +5 -4
- package/public/docs/tuning-reference/bottleneck-failures.html +5 -4
- package/public/docs/tuning-reference/bottleneck-gc.html +5 -4
- package/public/docs/tuning-reference/bottleneck-job-failure-rate.html +5 -4
- package/public/docs/tuning-reference/bottleneck-memory-utilization.html +5 -4
- package/public/docs/tuning-reference/bottleneck-retry-waste.html +5 -4
- package/public/docs/tuning-reference/bottleneck-shuffle.html +5 -4
- package/public/docs/tuning-reference/bottleneck-skew.html +5 -4
- package/public/docs/tuning-reference/bottleneck-slow-host.html +5 -4
- package/public/docs/tuning-reference/bottleneck-small-files.html +5 -4
- package/public/docs/tuning-reference/bottleneck-spill.html +5 -4
- package/public/docs/tuning-reference/bottleneck-straggler.html +5 -4
- package/public/docs/tuning-reference/bottleneck-tiny-tasks.html +5 -4
- package/public/docs/tuning-reference/bottleneck-utilization.html +5 -4
- package/public/docs/tuning-reference/caching.html +5 -4
- package/public/docs/tuning-reference/cluster-config.html +5 -4
- package/public/docs/tuning-reference/config.html +5 -4
- package/public/docs/tuning-reference/data-formats.html +5 -4
- package/public/docs/tuning-reference/index.html +5 -4
- package/public/docs/tuning-reference/intro.html +5 -4
- package/public/docs/tuning-reference/joins.html +5 -4
- package/public/docs/tuning-reference/memory-model.html +5 -4
- package/public/docs/tuning-reference/metrics.html +5 -4
- package/public/docs/tuning-reference/partitioning.html +5 -4
- package/public/docs/tuning-reference/pyspark.html +5 -4
- package/public/docs/tuning-reference/shuffle.html +5 -4
- package/public/docs/tuning-reference/spark-architecture.html +5 -4
- package/public/docs/tuning-reference/table-formats.html +5 -4
- package/public/docs/user-guide/alternative-log-retrieval.html +14 -7
- package/public/docs/user-guide/getting-started.html +27 -8
- package/public/docs/user-guide/mcp-tools.html +46 -28
- package/public/docs/user-guide/run-comparison.html +7 -6
- package/public/docs/user-guide/understanding-findings.html +7 -6
- package/public/export-template.html +44 -44
- package/public/index.html +2 -2
- package/vendor-core/allocation.js +106 -0
- package/vendor-core/analyzer.js +13 -13
- package/vendor-core/cli/budgets.js +23 -9
- package/vendor-core/cli/collect-run.js +11 -4
- package/vendor-core/cli/regression-budgets.js +83 -0
- package/vendor-core/comparison-verdict.js +22 -22
- package/vendor-core/core-source-hash.txt +1 -1
- package/vendor-core/detectors.js +202 -68
- package/vendor-core/docs-content/detection/cache.md +3 -2
- package/vendor-core/docs-content/detection/cfg.md +9 -8
- package/vendor-core/docs-content/detection/chrn.md +1 -2
- package/vendor-core/docs-content/detection/cold.md +4 -2
- package/vendor-core/docs-content/detection/fail.md +3 -2
- package/vendor-core/docs-content/detection/gc.md +3 -2
- package/vendor-core/docs-content/detection/host.md +2 -1
- package/vendor-core/docs-content/detection/local.md +1 -1
- package/vendor-core/docs-content/detection/mem.md +5 -2
- package/vendor-core/docs-content/detection/plan.md +2 -1
- package/vendor-core/docs-content/detection/sfail.md +2 -1
- package/vendor-core/docs-content/detection/shape.md +5 -4
- package/vendor-core/docs-content/detection/skew.md +3 -1
- package/vendor-core/docs-content/detection/slow.md +2 -2
- package/vendor-core/docs-content/detection/spec.md +2 -3
- package/vendor-core/docs-content/detection/spill.md +1 -1
- package/vendor-core/docs-site-config.js +1 -1
- package/vendor-core/effective-conf.js +107 -0
- package/vendor-core/efficiency-model.js +8 -6
- package/vendor-core/event-handlers.js +160 -47
- package/vendor-core/event-schemas.js +2 -0
- package/vendor-core/evidence-report.js +18 -10
- package/vendor-core/finding-generic-recommendation.js +20 -1
- package/vendor-core/finding-names.js +7 -0
- package/vendor-core/finding-presentation.js +61 -26
- package/vendor-core/finding-tag-help.js +1 -1
- package/vendor-core/finding-types.js +12 -0
- package/vendor-core/format-utils.js +4 -3
- package/vendor-core/impact-estimator.js +20 -2
- package/vendor-core/impact-format.js +14 -13
- package/vendor-core/impact-model.js +27 -5
- package/vendor-core/ingest.js +4 -2
- package/vendor-core/list-runs.js +5 -2
- package/vendor-core/mcp-tools.js +1 -1
- package/vendor-core/model-assembler.js +23 -1
- package/vendor-core/parser-worker.js +1 -1
- package/vendor-core/proxy.js +3 -1
- package/vendor-core/python-stage.js +25 -0
- package/vendor-core/recommendation-rollup.js +16 -9
- package/vendor-core/redact.js +51 -10
- package/vendor-core/remediation.js +20 -0
- package/vendor-core/run-comparison.js +43 -24
- package/vendor-core/run-interpretation.js +2 -1
- package/vendor-core/run-metrics.js +198 -0
- package/vendor-core/run-totals.js +24 -0
- package/vendor-core/run-verdict.js +3 -4
- package/vendor-core/scorecard-estimates.js +1 -0
- package/vendor-core/session-snapshot.js +7 -0
- package/vendor-core/shs-schemas.js +2 -2
- package/vendor-core/spark-memory.js +17 -0
- package/vendor-core/stage-plan-nodes.js +18 -0
- package/vendor-core/stage-quantiles.js +4 -0
- package/vendor-core/types.js +49 -1
- package/vendor-core/wasted-core-hours.js +10 -7
- package/vendor-core/write-targets.js +312 -0
- package/public/assets/AutoscalingChurn-CUSIvvab.js +0 -1
- package/public/assets/CacheUtilization-Ce0K6_X2.js +0 -1
- package/public/assets/CachingOpportunity-Cn-LmRA4.js +0 -1
- package/public/assets/ColdStart-BoiYzEyV.js +0 -1
- package/public/assets/ConfigAudit-BoJc6PAw.js +0 -1
- package/public/assets/CoreUsageArea-BFfXKu7K.js +0 -1
- package/public/assets/Dashboard-Bo_1jGqH.js +0 -2
- package/public/assets/DisclosureContext-LYzHroHO.js +0 -1
- package/public/assets/DuplicatePlanSubtree-BaAuGAoo.js +0 -1
- package/public/assets/ExecutorUtilization-OE6VMFkG.js +0 -1
- package/public/assets/GcPressure-AoL_ZNrV.js +0 -1
- package/public/assets/IncompleteRun-Bv6IzgsQ.js +0 -1
- package/public/assets/JobFailures-DF3U2z1f.js +0 -1
- package/public/assets/MemoryUtilization-B65OVZlq.js +0 -1
- package/public/assets/OverBroadcast-DXbJREDl.js +0 -1
- package/public/assets/PartitionSizing-DZLnjy0d.js +0 -1
- package/public/assets/PlanGraphRoute-C-BGmC6j.js +0 -1
- package/public/assets/RetryWaste-BbEPsdMw.js +0 -1
- package/public/assets/ShuffleIO-wZB94LS8.js +0 -1
- package/public/assets/Skew-SpQE7zeJ.js +0 -1
- package/public/assets/SlowHost-DwDBCTvr.js +0 -1
- package/public/assets/SmallFiles-yjY_6sT1.js +0 -1
- package/public/assets/SpeculationWaste-73PDmj0c.js +0 -1
- package/public/assets/Spill-DEw0Kd1i.js +0 -1
- package/public/assets/StageFailed-BEfXdEJc.js +0 -1
- package/public/assets/StageFindingGroup-Bzm2TY03.js +0 -1
- package/public/assets/StageShape-CzR4jHOt.js +0 -1
- package/public/assets/StageSlowness-vMPsnAaa.js +0 -1
- package/public/assets/Straggler-Cf__OGl5.js +0 -1
- package/public/assets/TaskFailures-BZ8kLZiO.js +0 -1
- package/public/assets/UnderBroadcast-CBm8ZPoS.js +0 -1
- package/public/assets/index-CEM9bz_x.js +0 -13
- package/public/assets/useRecentFiles-BrBl-AP1.js +0 -11
- package/public/docs/assets/chunks/@localSearchIndexroot.DbZy5EtM.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_board-widgets.md.VTcYTH4d.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_board-widgets.md.VTcYTH4d.lean.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_detector-contract.md.CsqZHz_T.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_drill-down.md.CFNCE8JB.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.HT9IJGIl.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.HT9IJGIl.lean.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_index.md.3TO9ic6w.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_overview.md.CTEqVeV4.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_overview.md.CTEqVeV4.lean.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_state-and-history.md.DOTVHjTV.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.BmpARZve.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_worker-protocol.md.21pTYhFF.js +0 -6
- package/public/docs/assets/contributor-guide_contributing.md.B2E9ni79.js +0 -1
- package/public/docs/assets/contributor-guide_development-setup.md.CjgZJwjK.js +0 -15
- package/public/docs/assets/contributor-guide_development-setup.md.CjgZJwjK.lean.js +0 -1
- package/public/docs/assets/contributor-guide_testing.md.DTwhpoN2.js +0 -1
- package/public/docs/assets/user-guide_alternative-log-retrieval.md.ZHPSbBkF.js +0 -1
- package/public/docs/assets/user-guide_alternative-log-retrieval.md.ZHPSbBkF.lean.js +0 -1
- package/public/docs/assets/user-guide_getting-started.md.CEa2ZX-l.js +0 -6
- package/public/docs/assets/user-guide_getting-started.md.CEa2ZX-l.lean.js +0 -1
- package/public/docs/assets/user-guide_run-comparison.md.ChLI3MK0.js +0 -1
- package/public/docs/assets/user-guide_understanding-findings.md.D1ccs60C.js +0 -1
- /package/public/docs/assets/{contributor-guide_architecture_detector-contract.md.CsqZHz_T.lean.js → contributor-guide_architecture_detector-contract.md.caA46z-u.lean.js} +0 -0
- /package/public/docs/assets/{user-guide_understanding-findings.md.D1ccs60C.lean.js → user-guide_understanding-findings.md.DzReyCXb.lean.js} +0 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
import{_ as o,o as t,c as d,a5 as a}from"./chunks/framework.DSg0KOwT.js";const p=JSON.parse('{"title":"Widget rendering","description":"","frontmatter":{},"headers":[],"relativePath":"contributor-guide/architecture/widget-rendering.md","filePath":"contributor-guide/architecture/widget-rendering.md"}'),i={name:"contributor-guide/architecture/widget-rendering.md"};function n(c,e,s,r,l,h){return t(),d("div",null,[...e[0]||(e[0]=[a('<h1 id="widget-rendering" tabindex="-1">Widget rendering <a class="header-anchor" href="#widget-rendering" aria-label="Permalink to "Widget rendering""></a></h1><h2 id="widget-rendering-order" tabindex="-1">Render order <a class="header-anchor" href="#widget-rendering-order" aria-label="Permalink to "Render order {#widget-rendering-order}""></a></h2><p><code>src/view/Dashboard.tsx</code>'s <code>FilteredBoard</code> renders inside a <code><main></code> that opens with <code>SampleRunNotice</code> (only while the bundled sample run is open), then <code>RunVerdict</code> (built from the unfiltered catalog), then a single <code>Scorecard</code> strip, then <code>FindingFilterBar</code> (Advanced view, or any active filter) and (only when an active filter empties both finding streams) <code>NoMatchBanner</code>, then (when the active filter doesn't empty the board) a two-tab <code>Tabs</code> (<code>src/components/ui/tabs.tsx</code>, a base-ui primitive): <strong>Findings</strong> and <strong>Full app report</strong>, the first grouped by impact band and the second always reachable. Both <code>TabsContent</code> panels pass <code>keepMounted</code>, so the inactive one is hidden (native <code>hidden</code> attribute) rather than unmounted: switching tabs keeps each widget's local state (e.g. ImpactBoard's expanded group) and scroll position.</p><p><code>region</code> on <code>RegistryEntry</code> (<code>src/view/detector-registry.tsx</code>) is read only to decide whether a widget always mounts: <code>isAlwaysMountedType()</code> flags the one <code>reference</code>-region type carved out as an always-mounted exception, <code>coreLocality</code> → <code>CoreUsageArea</code>. The other <code>reference</code>-region types stay ordinary finding-gated instead: <code>cacheUtilization</code>, <code>memoryUtilization</code>, and <code>utilization</code> are a product decision: a clean run on any of them isn't evidence worth surfacing unconditionally, so each collapses to a plain clean-check line like any other detector. The Findings tab (<code>src/view/widgets/ImpactBoard.tsx</code>) groups its content into two tiers: impact-banded rows and cards for every <code>REGISTRY</code> component and every recommendation row with at least one finding; and a collapsed "Clean checks" disclosure covering every remaining type with zero findings, built per detector <em>type</em> (<code>Object.keys(REGISTRY)</code>). The one always-mounted exception lives in Full app report (see "ReferenceSection" below), which stays structural-only.</p><p>Tags carry their own docs links; there is no separate legend widget. <code>TagBadge</code> (<code>src/view/ImpactBadge.tsx</code>) resolves its own tooltip. Its documentation anchor is the caller's <code>docAnchor</code> prop when that is a known anchor (call sites holding the finding pass <code>finding.docAnchor</code>; a widget header or grouped row passes <code>sharedDocAnchor(findings)</code>), else the type's single known anchor (the run interpretation's <code>detectors[type].docAnchor</code>, from <code>docAnchorForType</code> in <code>packages/core/src/detector-docs.ts</code>). The prop matters for <code>configAudit</code>, whose four entries carry different anchors, so the type lookup finds none. With an anchor, the pill itself links into the docs panel, and (density <code>advanced</code> only) a second icon link opens that tag's entry in <code>docs-site/user-guide/understanding-findings.md</code> (<code>findingGuideUrl</code>, <code>packages/core/src/docs-site-config.ts</code>) in the same in-app docs panel; its <code>target="_blank"</code> only applies to a modifier-click or when no <code>DocsProvider</code> is mounted. When a type has no vendor-doc anchor (e.g. <code>incompleteRun</code>), the pill links straight to that same guide entry instead of rendering as inert text, and the second icon link is skipped as redundant. Either way, a linked pill gets a visible dotted underline, not just a hover tooltip. <code>TagBadge</code>'s <code>plainBadge</code> prop suppresses both links together; Stage Summary's per-stage tag chips are the one caller that sets it, on a chip that sits inside its own investigate button. Findings-tab rows leave it unset, so their badge links stay real <code><a></code>s. The HTML export renders no badge links at all (<code>exportMode</code>).</p><h3 id="findings-tab" tabindex="-1">Findings tab <a class="header-anchor" href="#findings-tab" aria-label="Permalink to "Findings tab""></a></h3><p>Rendered by <code>FilteredBoard</code>'s <code>TabsContent value="findings"</code> (<code>src/view/widgets/ImpactBoard.tsx</code>): recommendation rows and active widget cards in one impact-ranked board. The recommendation-rollup logic lives in <code>src/view/widgets/FixTheseFirst.tsx</code> (its own file with directly-testable exports, not rendered as a standalone page section by <code>Dashboard.tsx</code>) and the active-widget logic lives in <code>src/view/widgets/Alerts.tsx</code> (same: not rendered standalone). <code>ImpactBoard</code> calls <code>FixTheseFirst.tsx</code>'s exported <code>useFixTheseFirstData</code> (eligible findings, rollup groups, the top triage target) and <code>Alerts.tsx</code>'s exported <code>computeActiveWidgets</code> (the ranked active-<code>REGISTRY</code> list). The rollup and its ranking are computed in core by <code>interpretRun</code> (<code>rankedRollup</code>/<code>rankFindings</code>, <code>packages/core/src/recommendation-rollup.ts</code>); <code>useFixTheseFirstData</code> reads it through <code>boardRollup</code> (<code>src/view/interpretation.ts</code>: the interpretation's carried groups when unfiltered, core <code>rankedRollup</code> recomputed over the kept findings under a filter). <code>ImpactBoard</code> then buckets rows by each group's <code>band</code> (its representative member's impact band, after core <code>rankFindings</code>, the same finding whose impact band its own badge already shows) and cards by each active widget's own <code>worstImpactBand</code>.</p><p>Eligible findings for the rollup are <code>catalog</code> ∪ <code>configFindings</code> that pass core <code>isEligible</code> (<code>packages/core/src/recommendation-rollup.ts</code>) and have a display type (<code>FINDING_DISPLAY_ORDER</code>): <code>incompleteRun</code> (a pipeline-completeness caveat, not an addressable fix) and every <code>dataUnavailable</code> evidence caveat (e.g. <code>memoryUtilization</code>'s <code>memoryBand</code> variant, already covered by Evidence availability's own <code>executorMetrics</code> entry, <code>packages/core/src/evidence-availability.ts</code>) are excluded. Findings are grouped strictly by <code>finding.type</code> via <code>buildRecommendationRollup</code> (<code>packages/core/src/recommendation-rollup.ts</code>, further split within a type by impact kind and, for <code>resource</code>, unit: never merged across types); each resulting group becomes one row: a type with exactly one finding renders that finding directly, a type with more than one collapses into a summary row. Within an impact band, group order comes from <code>buildRecommendationRollup</code>'s own sort: <code>time</code> groups (a real <code>wallClock</code> claim) first, ranked among themselves by their union-capped <code>recoverableMsHigh</code> descending; then <code>resource</code> groups (<code>rawWaste</code> but no <code>wallClock</code>); then <code>count</code> groups (neither), both of the latter two ranked by worst impact band, never by their incomparable raw magnitudes. A summary row's tag badge and action label come from the group's own highest-impact member (via the same three-tier comparator, core <code>rankFindings</code>); its sentence is the type-level <code>coreFindingGenericRecommendation</code>, not one member's own numbers. The trailing stat depends on the group's kind (<code>×N · <time> recoverable</code>, <code>×N · <total></code> with the summed resource waste the CLI report also prints, or <code>×N</code> alone, adding a per-impact-band tally only when the group mixes bands). Clicking it expands straight to the group's full, impact-ranked list, with no intermediate "worst-K" step, paginated at 10 rows per page (<code>data-testid="fix-these-first-group-row"</code>; no pager renders for a group of 10 or fewer findings; it appears once a group exceeds 10). A band's rollup rows render as a headerless three-column <code>Table</code> (<code>src/components/ui/table.tsx</code>): a single-finding row (<code>FindingRow</code>, <code>data-testid="fix-these-first-row"</code>, <code>data-finding-type</code>) has three <code>TableCell</code>s: the impact dot + ALL-CAPS tag as a real <code>TagBadge</code> (not <code>plainBadge</code>: nothing wraps it, so its own docs links stay real <code><a></code>s, same as everywhere else on the board); a text block inside its own nested <code><button></code> (a short imperative action label, e.g. "Reduce shuffle size", from <code>findingActionLabel</code> (<code>packages/core/src/finding-action-label.ts</code>), over the finding's own full <code>recommendation</code> sentence in smaller muted text, both wrapping rather than truncating); and a right-aligned monospace stage reference + impact figure (e.g. <code>St.49 · 20.1s</code>, via the shared <code>formatWallClockRange</code>/<code>formatRawWaste</code> in <code>packages/core/src/format-utils.ts</code>). That inner button, not the row, is the click target: it routes via <code>selectTriageTargetForFinding</code> (<code>src/view/triage-target.ts</code>), the same per-finding resolver Stage Summary Table's own control uses (see "First investigation routing" below); a <code>TypeGroupRow</code>'s own inner button toggles its expand state instead (<code>aria-expanded</code>). An expanded group's members (<code>FindingInstanceRow</code>) omit the badge and span its cell (<code>colSpan={2}</code>): the location line, then the recommendation clamped to one line, then the figure in the third column. <code>ImpactBoard</code> owns which single group is expanded (<code>expandedGroupKey</code>); a <code>TypeGroupRow</code> owns only its page.</p><p>Each impact band (<code>ImpactBoard.tsx</code>'s own <code>ImpactGroup</code>, one call per entry of <code>IMPACT_BAND_ORDER_LIST = ['critical', 'warning', 'info']</code>) is a <code><section aria-labelledby></code> pointing at its <code><h2 id="impact-band-<band>-heading" tabIndex={-1}></code> (Critical, Warning, Info; the top bar's count chip focuses it), a sibling of the panel's sr-only <code><h2>Findings</h2></code>, and renders nothing (not even the heading) when it has neither a rollup row nor an active widget: a run with no critical findings has no "Critical" heading or section at all. Inside a band, rollup rows render first as the headerless <code>Table</code> described above, followed by that band's active <code>REGISTRY</code> widget cards (<code>computeActiveWidgets</code>'s ranked list, filtered to this impact band) in their own <code>WidgetGrid</code>: every one of <code>orderedWidgets()</code>'s <code>REGISTRY</code> components <em>except</em> the one always-mounted one below, with at least one finding in <code>catalog</code> ∪ <code>configFindings</code>. Within a band, active widgets keep <code>orderedWidgets()</code>'s own order: the interpretation's <code>DetectorInfo</code> sorted by region (<code>action</code> first), then ascending detector <code>order</code>. In Basic view a band with both rows and cards folds its <code>WidgetGrid</code> behind one "Show the evidence (N cards)" disclosure, unmounted while closed; it opens itself (and stays open) when the active route target (<code>useActiveRouteTarget</code>) is one of its cards, and the card, mounting with the route still pending, opens and scrolls itself through <code>registerWidget</code>. A band with cards but no rows, and every band in Advanced view, shows its grid directly. <code>cacheUtilization</code>, <code>memoryUtilization</code>, and <code>utilization</code> are <code>reference</code>-region types but aren't always-mounted exceptions, so a Cache Storage, Memory Utilization, or Executor Utilization card with an active finding surfaces in its own impact band like any other active widget.</p><p>Below the impact bands, <code>Alerts.tsx</code>'s exported <code>CleanChecks</code> renders a collapsed "Clean checks" disclosure of <code>CleanCheckRow</code> lines (<code>src/view/widgets/CleanCheckRow.tsx</code>: the type's tag chip and label, plus in Advanced view the threshold it was held to, the interpretation's <code>detectors[type].thresholdSummary</code> from <code>getThresholdSummary</code>, which states the detector's own numbers) built per detector <em>type</em> (every <code>REGISTRY</code> key except that one always-mounted key), under one "Every check below passed." line (or "No check could run on this log." when none did) and grouped by detector scope (Per-stage, App-level, SQL plan, Config checks). Types the log could not check (an <code>isEvidenceCaveat</code> finding of that type, every per-stage type when no stage finished, or the run-span types <code>RUN_SPAN_CHECK_TYPES</code> on an <code>incompleteRun</code> log, the same rule that keeps the verdict from calling the run clean) render first under <strong>Not checked on this log</strong> as <code>CleanCheckRow status="notRun"</code>, drawn neutral rather than clean green, after the interpretation's <code>coverage.gaps</code> (<code>verdictGaps</code>) lines saying why and naming the setting to turn on. A clean run lands <code>cacheUtilization</code>, <code>memoryUtilization</code>, and <code>utilization</code> here too, same as any ordinary action-region type. Caching Opportunities, Config Audit, and the four Plan Advisor widgets (Redundant Plan Subtree, Excessive Small Files, Missed Broadcast Join, Oversized Broadcast Join) render through the ordinary active/clean paths above (see <a href="./board-widgets.html#board-widgets-app-and-plan-widgets">App- and plan-level widgets</a>). Core Usage by Locality (<code>coreLocality</code>, resolving to <code>CoreUsageArea</code>) is not in the Findings tab at all: it mounts unconditionally from <code>appModel</code> at the head of the Full app report's reference grid (<code>alwaysMountedWidgets()</code>/<code>isAlwaysMountedType()</code>), carrying its own impact-band indicator when a finding is active instead of collapsing to a clean-check line on a clean run. A <code>coreLocality</code> finding still lists in its impact band's recommendation rows, and its <strong>Show evidence</strong> switches to the Full app report tab.</p><h3 id="per-widget-list-sort-mode" tabindex="-1">Per-widget list sort mode <a class="header-anchor" href="#per-widget-list-sort-mode" aria-label="Permalink to "Per-widget list sort mode""></a></h3><p><code>src/view/impact-sort.ts</code> (<code>sumWallClockLow</code>, <code>hasSortableImpact</code>, <code>canToggleSort</code>, <code>byImpactDesc</code>, <code>stageIdOf</code>, <code>byStageAsc</code>), <code>src/view/useSortMode.ts</code> and <code>src/view/SortModeToggle.tsx</code> are a shared, opt-in set a Findings-tab active widget can use to let its own expanded, multi-row list default to potential-savings order (<code>wallClock.low</code> descending, the guaranteed-floor bound, not the optimistic <code>high</code>) instead of stage number. Each of the widgets below gets its <code>sortMode</code> (<code>SortMode</code>, <code>'impact' | 'stage'</code>, default <code>'impact'</code>) from <code>useSortMode</code>, which forces impact order at Basic density, and renders a <code>SortModeToggle</code> in the <code>WidgetCard</code> <code>badges</code> slot, so it sits on the title row itself rather than floating in the body: a button alongside the widget's tag badges (a <code>WidgetCard</code> header renders <code>badges</code> beside the title heading, never inside another control). Every widget below carries the same <code>canToggleSort</code> check, so the toggle only actually renders for a type whose finding can carry a wall-clock estimate: <code>Skew.tsx</code>, <code>TinyTask.tsx</code> (<code>stageShape</code>'s own rules never produce one, so <code>StageShape.tsx</code> carries the same check but it never fires), <code>ShuffleIO.tsx</code> (narrowed to <code>shuffle</code>), <code>PartitionSizing.tsx</code>, <code>Spill.tsx</code>, <code>GcPressure.tsx</code> (both its high-GC and low-GC sections, one shared toggle), <code>RetryWaste.tsx</code> (its siblings <code>StageFailed.tsx</code>/<code>TaskFailures.tsx</code> carry the same check, but <code>stageFailed</code>/<code>failures</code> are <code>estimateMethod: 'none'</code>, so it never fires there either), <code>SlowHost.tsx</code>, <code>StageSlowness.tsx</code>, <code>Straggler.tsx</code>, <code>SpeculationWaste.tsx</code>, <code>ColdStart.tsx</code> (one widget per type, each sorting only its own flat issue list), and <code>DuplicatePlanSubtree.tsx</code>/ <code>SmallFiles.tsx</code>/<code>UnderBroadcast.tsx</code>/<code>OverBroadcast.tsx</code> (each reorders its own list by <code>stageIdOf</code>, the lowest stage id its finding touches). It is a two-segment <code>ToggleGroup</code> (Impact / Stage; optional <code>stageLabel</code>, default "Stage", which no caller overrides), wrapped in <code>AdvancedOnly</code>: stage number is the one axis every one of these widgets' items can be compared on. Neither mode sorts by impact band, which ranks findings by how bad they are, not by how much fixing them would save. A per-stage detector's own <code>finding.stageId</code> is the sort key directly; a sql-scope finding that spans several stages (<code>duplicatePlanSubtree</code>, <code>smallFiles</code>, <code>underBroadcast</code>, <code>overBroadcast</code>, via <code>stageIds</code>) sorts by the lowest stage id it touches (<code>stageIdOf</code>). The toggle renders only when <code>canToggleSort</code> holds: <code>hasSortableImpact</code> finds a wall-clock claim in the list, there is more than one row, and the card is open; re-sorting a list with no wall-clock claim would be a silent no-op. Both comparators return <code>0</code> when neither side has a comparable value, so those items keep their prior relative order (<code>Array.prototype.sort</code>'s stability) rather than being shuffled. <code>FixTheseFirst</code>'s own expanded group list (see "Findings tab" above) sorts by impact on its own and does not use this pattern; it has no stage-order toggle.</p><h3 id="gold-standard-row-expand-contract" tabindex="-1">Gold Standard row/expand contract <a class="header-anchor" href="#gold-standard-row-expand-contract" aria-label="Permalink to "Gold Standard row/expand contract""></a></h3><p>Every <code>REGISTRY</code> widget renders its content as N≥1 rows, using <code>Skew.tsx</code> as the reference implementation. A widget's data shape (chart, table, single app-scoped scalar) is never by itself a reason to skip this: the four named exceptions below (one histogram toggle, three partial exceptions) are the only ones. Any further exception must get its own entry here, justified in the same PR that introduces it.</p><ul><li>Tier A (universal): <code>WidgetCard</code> chrome, an impact-band left-border + dot, ALL-CAPS <code>TagBadge</code>s, a <code>SortModeToggle</code> when <code>canToggleSort</code> is true, a <code>finding-anchor</code> ref on any focusable/deep-linkable unit, and an early <code>null</code> return on an empty catalog filter (except Core Usage by Locality, the one always-mounted widget that renders unconditionally from <code>appModel</code> via <code>alwaysMountedWidgets()</code>/<code>isAlwaysMountedType()</code> rather than early-returning on an empty catalog filter).</li><li>Tier B (the row/expand pattern): a row's collapsed state shows its core metric(s), <code>ImpactEstimate</code>, any config-hint code snippet/list, and docs links, all unconditionally. The fix is stated once per card: <code>WidgetCard</code>'s <code>fixFor</code> prints each distinct <code>coreFindingGenericRecommendation</code> above the body, so rows don't repeat a recommendation that restates their number. Every finding-row widget passes it, Shuffle I/O and the Skew, Stage Shape and Tiny Tasks groups (<code>StageFindingGroup</code>) included; Incomplete Run is the one that doesn't. Where the detector's sentence is a row's only measurement (Excessive Small Files, Missed Broadcast Join, Oversized Broadcast Join, Config Audit), the row shows its measured half, split off by core's <code>recommendationParts</code>. Three rows keep their own sentence because their type has no generic fix for them: Incomplete Run, and a data-unavailable Memory Utilization or Cache Storage row, which names the setting to enable. Confidence and evidence are unconditional too: <code>RowStatusCluster</code> (<code>src/view/RowStatusCluster.tsx</code>) is a single, fixed-position pill combining both. Every widget that carries one wraps it in <code>AdvancedOnly</code> (<code>src/view/AdvancedOnly.tsx</code>), so it only renders at the Advanced density tier: this is a content-visibility gate (Basic vs. Advanced), not a per-row collapse, and it's applied consistently across every adopting widget. No widget gates confidence or evidence behind a per-row click. A widget-header <code>RowStatusCluster</code> (passed as <code>WidgetCard</code>'s <code>statusBadge</code> prop) carries one further, unrelated gate on top of <code>AdvancedOnly</code>: <code>open && statusBadge</code> (<code>WidgetCard.tsx</code>'s header) keeps it out of the collapsed header, matching the comment on <code>statusBadge</code> itself ("kept separate so tag/impact badges stay visible in the collapsed summary while a <code>RowStatusCluster</code>-style ... marker doesn't"). This is the card's own expand/collapse, not a separate reveal-on-click built for the cluster, and it composes with density as an AND: a header cluster needs both Advanced tier and an expanded card. A row with neither renders no cluster at all (<code>RowStatusCluster</code> returns <code>null</code>). Confidence renders as plain, non-interactive text ("<code>{confidence} confidence</code>", no further suffix) with a <code>title</code> + <code>aria-describedby</code> tooltip carrying the full <code>validationRequired</code> detail: never itself a click target. Evidence renders as the cluster's one real click target: a button whose visible text is just the evidence label (e.g. "SQL plan") but whose accessible name always carries the "Evidence: ..." prefix via an explicit <code>aria-label</code>, regardless of what else sits nearby. Clicking it calls <code>revealEvidence</code>. <code>RowStatusCluster</code>'s fixed slot is the row's own stage-pill/title line (<code>justify-between</code>, cluster right-aligned), the same position in every adopting widget, not floating with the row's detail below. Lists longer than <code>VISIBLE_LIMIT</code> (6, <code>packages/core/src/format-utils.ts</code>) get page-based navigation (Previous/Next, <code>usePagedRows</code>/<code>RowPagination</code>, <code>src/view/usePagedRows.ts</code>/<code>src/view/RowPagination.tsx</code>) instead of rendering unconditionally: the one sanctioned cap mechanism across every Tier B widget. A Tier B widget whose rows are also triage-routable (wires <code>useFindingAnchor</code>) must pass its <code>routeIndex</code> (the routed finding's position in the paginated list) into <code>usePagedRows</code>, or a deep-linked route to a finding beyond the first page silently degrades to focusing the widget's disclosure title instead of the row.</li></ul><p>The following widgets carry a <code>RowStatusCluster</code>, all Advanced-only: <code>Spill.tsx</code> (confidence only), <code>ConfigAudit.tsx</code>'s widget-header control (evidence only, a widget-wide constant rather than per-finding data, since the evidence key doesn't vary per row; also the control shown in the widget's empty-findings/no-data states), <code>MemoryUtilization.tsx</code>'s widget header (confidence + evidence, one marker, since <code>executorMetrics</code> is widget-wide; <code>ExecutorUtilization.tsx</code>'s rows carry neither), each of the four Plan Advisor widgets' (<code>DuplicatePlanSubtree.tsx</code>, <code>SmallFiles.tsx</code>, <code>UnderBroadcast.tsx</code>, <code>OverBroadcast.tsx</code>) widget header (confidence + <code>sqlPlan</code> evidence, one marker per widget since each is its own finding type; no per-row control), <code>ScalingSim.tsx</code> (two call sites, evidence only), <code>CoreUsageArea.tsx</code>, <code>EfficiencyModel.tsx</code> and <code>PlanView.tsx</code> (three call sites) (these last confidence-only; all standalone rather than per-finding-row: a card-header badge, a widget-level single marker, a plan-tree node/summary-row marker, or an "unavailable data" message: the same component, same visual language, regardless of where it sits); and, because <code>skew</code>/<code>straggler</code>/<code>gc</code> disclose their own unvalidated noise-floor thresholds (confidence only, per-row, no <code>evidenceKey</code> passed), <code>GcPressure.tsx</code>'s rows, <code>Straggler.tsx</code>'s rows, <code>CachingOpportunity.tsx</code>'s rows (the <code>cachingOpportunity</code> detector scales <code>confidence</code> per finding via <code>cachingReuseConfidence</code>, so the badge sits next to each row rather than as a single caveat below the table), and the shared <code>StageFindingGroup.tsx</code> row (adopted by <code>Skew.tsx</code>/<code>StageShape.tsx</code>/<code>TinyTask.tsx</code>, though only <code>skew</code> findings carry a <code>confidence</code> field).</p><p>One named exception:</p><ul><li><code>Skew.tsx</code>/<code>StageShape.tsx</code>/<code>TinyTask.tsx</code>: their per-row histogram toggle (<code>ExpandToggleButton</code> + <code>useExpandableRow</code>) gates a lazily-fetched duration histogram, unrelated to confidence/evidence. It sits alongside <code>RowStatusCluster</code> in the shared <code>StageFindingGroup.tsx</code> row; the two controls coexist per row. <code>ExpandToggleButton</code> is single-purpose (always the task-detail toggle) since these three are its only adopters.</li></ul><p>No toggle at all, unconditional content, since these widgets carry no confidence/evidence display: <code>IncompleteRun.tsx</code>, <code>ShuffleIO.tsx</code>, <code>PartitionSizing.tsx</code>, <code>StageFailed.tsx</code>, <code>TaskFailures.tsx</code>, <code>RetryWaste.tsx</code>, <code>ColdStart.tsx</code>, <code>SlowHost.tsx</code>, <code>StageSlowness.tsx</code>, <code>SpeculationWaste.tsx</code>, <code>ExecutorCountChart.tsx</code>, <code>ExecutorUtilization.tsx</code>, <code>JobFailures.tsx</code>, <code>CacheUtilization.tsx</code> and <code>AutoscalingChurn.tsx</code>.</p><p>Three named exceptions to specific pieces of the contract, not to the row wrapper or pagination, which still apply to all three:</p><ul><li><code>CoreUsageArea.tsx</code>'s non-local-stage list is a derived stat breakdown (<code>computeCoreLocalityRatio</code>'s <code>topStages</code>), not a <code>Finding[]</code>: there is no per-row finding or recommendation to reveal. Paginated like every other Tier B list, but with no per-row expand.</li><li><code>CacheUtilization.tsx</code>'s RDD <code><Table></code> holds reference columns (name, storage level, partitions, memory, disk bytes) with no recommendation attached to any row. Paginated like every other Tier B list, but with no per-row expand. Its separate finding list below the table is full Tier B (unconditional, no toggle, per the paragraph above).</li><li><code>IncompleteRun.tsx</code> leads with no metric of its own: this app-scoped, single-finding widget has no natural lead figure distinct from its title/badge, unlike every other Gold Standard widget, which leads with a percentage, duration, or count.</li></ul><h3 id="referencesection" tabindex="-1">ReferenceSection <a class="header-anchor" href="#referencesection" aria-label="Permalink to "ReferenceSection""></a></h3><p>Rendered by <code>FilteredBoard</code>'s <code>TabsContent value="full-report"</code>, tab label "Full app report". A plain <code><div></code>, structural-only, reading none of <code>REGISTRY</code>/<code>orderedWidgets()</code> at all. Its heading is a visually-hidden (<code>sr-only</code>) <code><h2>Full app report</h2></code>, matching the tab's own label for the accessibility-tree heading outline without visually duplicating the tab text. Selecting the tab is the disclosure; the panel stays mounted but hidden while Findings is active (see "Render order" above). Its exact order is WallClock → Timeline → Executor Count Over Time (<code>ExecutorCountChart.tsx</code>, the executor add/remove count chart; not driven by any finding, so it isn't a <code>REGISTRY</code> entry) → StageTable → a <code>WidgetGrid</code> holding Core Usage by Locality (the one always-mounted <code>REGISTRY</code> card) → Evidence availability → ETL Phase Attribution → What-If Executor Scaling → Compute Efficiency → Core-Usage Distribution. Scorecard is not part of this section; it renders once, above the tabs themselves, in <code>FilteredBoard</code> (<code>src/view/Dashboard.tsx</code>), so it stays visible regardless of which tab is active rather than living inside either one (a three-tile run-info row: Wall-clock, Efficiency, Unused core time; see <a href="./board-widgets.html#board-widgets-app-and-plan-widgets">App- and plan-level widgets</a>). WallClock, Timeline, Executor Count Over Time and StageTable render expanded. The grid tiles beside them (Core Usage by Locality, Evidence availability and the report lenses) are <code>collapsedTile</code>s: they start collapsed at a uniform height and widen to the full row when opened; <code>revealEvidence</code> opens the Evidence availability tile. Core Usage by Locality is the only detector-driven <code>REGISTRY</code> card in this section, always mounted; Memory Utilization, Executor Utilization, and Cache Storage live in the Findings tab above and surface there only when they have an active finding.</p><p>The Evidence availability card is the persistent, non-impact-band ledger <a href="./worker-protocol.html#evidence-availability-contract">defined in the worker protocol</a>, not an alert or detector widget. An <code>Evidence: …</code> control appears only where a conclusion or unavailable report lens declares a relevant ledger dependency. <code>revealEvidence</code> (<code>src/view/EvidenceAvailabilityContext.tsx</code>) bumps a <code>revealRequest</code> counter, which <code>DashboardContent</code> (<code>src/view/Dashboard.tsx</code>) watches in a <code>useLayoutEffect</code> and translates into <code>setActiveTab('full-report')</code>. Each click is a new request, so it switches tabs again after the reader returns to Findings. The layout effect makes the tab visible before the provider's queued focus runs, since a row inside the hidden, still-mounted panel can't take focus. Mouse and keyboard activation switches to the Full app report tab, opens the ledger card, then focuses the referenced stable entry id (<code>evidence-availability-<key></code>). The control explains evidence availability; it does not promise an unavailable signal would have produced a finding.</p><p><code>WidgetGrid</code> (both inside a Findings impact band and inside <code>ReferenceSection</code>): collapsed cards occupy one responsive column; opening a card expands it across the full row. Nested <code>WidgetCard</code>s are isolated from the parent grid state. A Findings-tab <code>FixTheseFirst</code> rollup row is a table row, not a <code>WidgetGrid</code> card, and carries no collapse/expand state of its own (<code>ImpactBoard</code> holds which group is expanded; a <code>TypeGroupRow</code> owns only its page). Full app report's non-<code>REGISTRY</code> tiles (ETL Phase Attribution, What-If Executor Scaling, Compute Efficiency, Core-Usage Distribution) sit in the same <code>WidgetGrid</code> as Core Usage by Locality and Evidence availability; only Core Usage by Locality's item carries a <code>widgetId</code> (see "First investigation routing" below), so it is the grid's one route destination.</p><p>Firm constraint: <code>orderedWidgets(detectors)</code> (in <code>src/view/detector-registry.tsx</code>) sorts the run interpretation's per-type <code>DetectorInfo</code> entries by region (<code>action</code> before <code>reference</code>), then ascending detector <code>order</code>; it does not walk the static <code>DETECTORS</code> import. Every <code>REGISTRY</code> entry maps to its own unique component, so it needs no component-identity dedup. <code>cacheUtilization</code>, <code>memoryUtilization</code>, and <code>utilization</code> (<code>reference</code>-region) can reach the active grid alongside <code>duplicatePlanSubtree</code> (<code>action</code>-region) and any other active finding, since none of them is the one always-mounted exception filtered out before that grid. <code>computeActiveWidgets</code> (<code>Alerts.tsx</code>) is its main consumer, filtered through <code>isAlwaysMountedType()</code> to exclude that one always-mounted component; the clean-check list bypasses <code>orderedWidgets()</code> entirely, iterating <code>Object.keys(REGISTRY)</code> per type instead (see "Findings tab" above). <code>DETECTORS</code>' own array order and iteration, plus its cross-detector <code>suppressedBy</code> logic (e.g. <code>stageSlowness</code> deferring to <code>slowHost</code>, see <a href="./detector-contract.html#detector-contract">Detector contract</a>), live entirely in <code>packages/core/src/detectors.ts</code>/<code>packages/core/src/analyzer.ts</code>. Which React component each finding type resolves to is registry-owned.</p><h3 id="first-investigation-routing" tabindex="-1">First investigation routing <a class="header-anchor" href="#first-investigation-routing" aria-label="Permalink to "First investigation routing""></a></h3><p>The dashboard has a view-only per-finding route, not a single first-investigation pick: every Findings-tab recommendation row (rendered by <code>FixTheseFirst.tsx</code>'s row components inside <code>ImpactBoard</code>) and the visible-finding control in Stage Summary (<code>StageTable.tsx</code>) each independently resolve and request their own target. <code>detector-registry.tsx</code> owns each routeable finding type's stable <code>widgetId</code>; its label comes from core <code>FINDING_PRESENTATION</code> (<code>findingName</code>) and its title from the widget itself, not from runtime position or raw detector type. Eligible targets have a mapped, routeable registry entry and a non-empty trimmed recommendation (<code>triageTargetFor</code>, <code>src/view/triage-target.ts</code>). The target's widget still renders every affected stage in its local order.</p><p><code>src/view/triage-target.ts</code> also exports <code>rankTriageTargets</code>, which orders <code>catalog</code> by the run interpretation's <code>savingsRank</code>. That rank comes from core's <code>rankBySavings</code> (<code>packages/core/src/run-verdict.ts</code>, shared with the CLI/MCP verdict), which orders every routeable finding by potential savings (<code>impactEstimate.wallClock.high</code>), a quantified estimate ahead of an unquantified one, then impact band, then widget display order (<code>FINDING_DISPLAY_ORDER</code>, core's copy of <code>orderedWidgets()</code>, which a <code>tests/view/detector-registry.test.tsx</code> case keeps equal), then catalog order; <code>selectTriageTarget</code> is its first entry. The occupancy-weighted impact estimator gives every finding a comparable <code>impactEstimate.wallClock</code> figure, and ranking by it keeps a "start here" pick off a <code>skew</code>/<code>straggler</code> finding ranked <code>critical</code> on a ratio basis whose occupancy-clipped recoverable time is near zero, when a lower-severity finding beside it has an order-of-magnitude larger recoverable-time estimate. Impact band only breaks ties. <code>RunVerdict</code>'s next steps are built from this ranking (see "Full render sequence" below).</p><p>The route is re-derived from the current catalog before each asynchronous step. Its identity is object-reference equality against the current catalog (<code>catalog.includes(finding)</code> in <code>selectTriageTargetForFinding</code>, <code>src/view/triage-target.ts</code>) rather than a derived key: a new parse yields new finding object references, so reference presence alone detects staleness. It is not persisted and does not extend the core <code>Finding</code> contract; cross-session consumers (exports, URL-restored state) use the core <code>Finding.id</code> instead (see <a href="./worker-protocol.html#finding-identity">Finding identity</a>).</p><p><code>Dashboard</code> owns disclosure and navigation. Every routeable <code>REGISTRY</code> widget lives in the Findings tab except the always-mounted Core Usage by Locality, which lives in Full app report (see "Render order" above), so <code>requestRoute</code> (<code>DashboardContent</code>, <code>src/view/Dashboard.tsx</code>) starts by switching to the tab holding the target's widget (<code>isAlwaysMountedType(target.finding.type)</code> picks Full app report), whether the request came from a control already on that tab or from the other. Both panels stay mounted, but a route can still mount its target card fresh in the same commit as the route landing: a Basic-view band's "Show the evidence" fold mounts its cards only when opened, and opens itself when a route targets one of them (see "Findings tab" above). Two routing paths have to account for that: <code>reportWidgetOpen</code> does not clear a route just because the target <code>WidgetGridItem</code> hasn't registered yet in this commit (a child <code>WidgetCard</code> reports its own open state before its parent <code>WidgetGridItem</code>'s registration effect runs on a fresh mount, so treating "not registered yet" as "stale" would drop the route before the parent even gets a chance to register it); and a paginated widget that must reveal a routed row passes its <code>routeIndex</code> into <code>usePagedRows</code> (<code>src/view/usePagedRows.ts</code>), which computes the page jump during render, not only in a post-commit effect, so the row exists in the DOM in time for this same commit's anchor lookup instead of one commit later.</p><p>After the target card commits open, an anchored row (<code>useFindingAnchor</code>) is scrolled to center and focused, then flashed for 2 s; without one, the card wrapper scrolls with a top margin below the fixed header and its disclosure button receives visible, temporary route focus (using instant scrolling for reduced motion). Requests use a latest-request-wins token (<code>tokenRef</code>, <code>DashboardContent</code>) that guards every step and revalidates; a stale, missing, unmounted, changed-catalog, or changed-active-file target cancels without redirecting, scrolling, or moving focus. <code>registerWidget</code>'s cleanup deletes a registration only if the map still holds that same object, so StrictMode double-mounts don't drop a live one.</p><p>This route deliberately does not cover detector/parser or threshold changes, Stage Detail routing, or generic badges, tags, chips, or StagePills as controls. Config Audit findings route like any other: they resolve against <code>catalog</code> ∪ <code>configFindings</code>.</p><h3 id="widget-placement" tabindex="-1">Widget placement <a class="header-anchor" href="#widget-placement" aria-label="Permalink to "Widget placement""></a></h3><p>The Plan Explorer (<code>src/view/widgets/PlanExplorer.tsx</code>) is embedded inside a flagged stage's own row, not as a standalone widget. It has four call sites (<code>ShuffleIO.tsx</code>, <code>PartitionSizing.tsx</code>, <code>Spill.tsx</code>, and the shared <code>StageFindingGroup.tsx</code> row behind <code>Skew.tsx</code>/<code>StageShape.tsx</code>/<code>TinyTask.tsx</code>), each Advanced-density only and only when the stage's plan tree resolves. The first three have no row-expansion toggle, so the trigger renders as a direct sibling in the row body; <code>StageFindingGroup.tsx</code> renders it inside the expanded Task detail. The same stage's plan is also reachable from <code>StageDetailDialog</code>.</p><p>Plan detection logic belongs in a <code>packages/core/src/detectors.ts</code> entry (<code>scope:'sql'</code>) consuming <code>planTree</code>. <code>packages/core/src/plan-summary.ts</code> is display-only summarization; do not add detection heuristics there. It is best-effort: <code>summarizePlanTree</code> reads each resolved <code>planTree</code> node's <code>detail</code> with lenient regexes and silently omits a fragment it can't parse, never surfacing an error.</p><p><code>StageHeader.tsx</code> (<code>StagePill</code> plus a <code><span></code> naming the stage) vs. bare <code>StagePill</code>/<code>StagePillGroup</code> is a deliberate row-density choice, not an inconsistency: <code>StageHeader</code> is for single-stage-per-row detail widgets where naming the stage adds context (<code>SlowHost.tsx</code>, <code>StageSlowness.tsx</code>, <code>Straggler.tsx</code>, <code>SpeculationWaste.tsx</code>, <code>GcPressure.tsx</code>, <code>Skew.tsx</code>, <code>StageShape.tsx</code>, <code>TinyTask.tsx</code>, <code>PartitionSizing.tsx</code>), while bare <code>StagePill</code>/<code>StagePillGroup</code> is for compact, multi-row lists where many stages appear per widget (<code>StageFailed.tsx</code>, <code>TaskFailures.tsx</code>, <code>RetryWaste.tsx</code>, the four Plan Advisor widgets, <code>Spill.tsx</code>, and <code>StageTable.tsx</code>). <code>ShuffleIO.tsx</code> uses both in different parts of its own row, which is fine: it isn't a violation of the convention above.</p><p>Stage Summary Table (<code>src/view/widgets/StageTable.tsx</code>) defaults to the <strong>top 10 stages by duration</strong>, with a header toggle to <strong>problems only</strong> (stages carrying a finding). It renders inside the Full app report tab, not as a standalone board section.</p><h3 id="full-render-sequence" tabindex="-1">Full render sequence <a class="header-anchor" href="#full-render-sequence" aria-label="Permalink to "Full render sequence""></a></h3><p>Top to bottom, in <code>Dashboard.tsx</code>'s <code>FilteredBoard</code>:</p><ol><li><code>SampleRunNotice</code> (<code>src/view/SampleRunNotice.tsx</code>), only while the bundled sample run is open (the landing's <strong>Try a sample run</strong> loads it under <code>SAMPLE_RUN_ID</code>, <code>src/view/sample-run.ts</code>) and never in the export bundle: says the board shows the sample, with <strong>Load my event log</strong> and a docs-panel link to the log-retrieval guide.</li><li><code>RunVerdict</code> (<code>src/view/widgets/RunVerdict.tsx</code>): the run's verdict title, a summary sentence, and up to three numbered next steps built by <code>buildRunVerdict</code> (<code>packages/core/src/run-verdict.ts</code>, the same code the CLI/MCP evidence report's <code>verdict</code> runs) via <code>buildNextSteps</code>. Steps group routeable eligible findings by location (one stage, one multi-stage finding type, or one app-level finding type and variant), ordered by <code>rankBySavings</code> (potential savings, then impact band, then widget order), the same ranking every other component uses; on a run with a failed job, failure steps (<code>stageFailed</code>, <code>jobFailureRate</code>) move ahead (see below), and no other type jumps the order. Each step shows its tag, action label, stage and savings, then "What's happening:" (the measurement) and "What to try:" (the fix), split from the recommendation by core <code>recommendationParts</code>; a step that merged other types at its location adds "Also flagged here, likely the same cause: ...", and the summary says once that same-stage findings are grouped. Each step has <strong>Show evidence</strong>, <strong>Stage N details</strong> (when it has one stage) and <strong>Copy</strong>; "N more places under Findings" counts the places past the step limit. Basic view adds a collapsed "New to Spark tuning?" primer. An idle-capacity step (<code>utilization</code>, or <code>memoryUtilization</code>'s <code>idleCores</code> variant only, never its heap variants) titles the verdict only when it ranks first; otherwise an idle share of at least <code>IDLE_NOTABLE_PCT</code> (40%) adds one summary sentence. The idle share (<code>verdictIdlePct</code>) is the figure that idle-capacity step itself reports, falling back to the Scorecard's Unused core time figure only when no step carries one, so the title and the step never disagree. Each step's savings figure is followed by what it counts (<code>savingsMeaning</code> in <code>packages/core/src/impact-format.ts</code>: run time for a wall-clock figure, otherwise the resource its <code>rawWaste</code> unit measures), and <strong>Copy next steps</strong> copies the whole verdict as a plain-text checklist (<code>planCopyText</code> in core <code>run-verdict.ts</code>). Always the unfiltered catalog: a board filter never changes the verdict. A step's <strong>Show evidence</strong> on a finding the active filter hides clears only the filter dimensions that hide it (<code>excludingDimensions</code> in <code>src/view/finding-filter.ts</code>, synced to the URL as usual) and shows a one-line notice naming what it cleared; the Topbar count chip's jump to its impact band (<code>jumpToFindings</code> in <code>Dashboard.tsx</code>) uses the same path. The clean-run message ("No findings to fix right now.") lives here and shows only when no finding at all was emitted and the log lacked nothing a check needs (below); an <code>incompleteRun</code> finding always adds a sentence saying the figures cover only the captured part of the run, and titles the verdict ("This log looks incomplete ...") when no other finding is eligible. Job results (<code>summarizeRunOutcome</code> in <code>packages/core/src/run-outcome.ts</code>) set the run outcome: with a failed job the title says the run failed, the verdict quotes the first line of Spark's recorded reason (a failed job's <code>stageFailed</code> value first, then any <code>stageFailed</code>, then the job exception), the run is never called clean, and <code>buildNextSteps</code> ranks <code>stageFailed</code> and <code>jobFailureRate</code> steps first (a failed job's stage leading). Evidence caveats (a finding with <code>dataUnavailable</code>, or one <code>isRealFinding</code> drops), a log with no finished stage, and an <code>incompleteRun</code> log (whose <code>RUN_SPAN_CHECK_TYPES</code> had no run length to measure) are gaps (<code>verdictGaps</code>, <code>packages/core/src/check-coverage.ts</code>): any gap keeps the run from being called clean, and a log with no finished stage and no finding gets its own title. The verdict card does not list the gaps; the Clean checks disclosure's "Not checked on this log" group does, each caveat by its own recommendation text, which names the setting to enable. In Advanced view each step adds an "Estimate:" line from <code>estimateProvenance</code> (<code>packages/core/src/impact-format.ts</code>, carried as the interpretation's per-finding <code>savings.provenance</code>: method, basis as a point figure or a floor-to-high range, ms raw waste only when the floor clipped it, a non-time raw waste as the resource measured; nothing for <code>estimateMethod: 'none'</code> or a zero figure), the finding's <code>confidence</code> when not <code>high</code>, and the list ends with the ordering rule when it has more than one step.</li><li><code>Scorecard</code>: a three-tile run-info row (Wall-clock, Efficiency, Unused core time; Basic view captions say what each measures, and for Efficiency and Unused core time which direction is better; Advanced view shows the raw run/idle breakdown), rendered once regardless of which tab is active.</li><li><code>FindingFilterBar</code>, only in Advanced view or while a filter is active (plus <code>NoMatchBanner</code> when the active filter empties both finding streams).</li><li>A two-tab <code>Tabs</code> (skipped entirely when the active filter empties both finding streams; <code>NoMatchBanner</code> above already covers that case), tab labels <strong>Findings</strong> and <strong>Full app report</strong>: <ul><li><strong>Findings</strong> (<code>ImpactBoard</code>): one <code><section></code> per impact band in <code>Critical</code> → <code>Warning</code> → <code>Info</code> order, each rendering nothing when it has neither a recommendation row nor an active widget: a recommendation-rollup <code>Table</code> (one row per eligible-finding type, a display type, <code>incompleteRun</code> and evidence caveats excluded: a direct row for a type with one finding, an expandable, paginated summary row for a type with more than one) followed by a <code>WidgetGrid</code> of that band's active <code>REGISTRY</code> widgets (ranked by widget order: the interpretation's <code>DetectorInfo</code> by region, action-region components ahead of reference-region ones, then ascending detector order); then, below every impact band, a collapsed "Clean checks" disclosure of <code>CleanCheckRow</code> lines built per detector type (every remaining <code>REGISTRY</code> key with zero findings).</li><li><strong>Full app report</strong> (<code>ReferenceSection</code>): WallClock → Timeline → Executor Count Over Time → StageTable → a grid of Core Usage by Locality → Evidence availability → fixed report-lens tail (ETL Phase Attribution → What-If Executor Scaling → Compute Efficiency → Core-Usage Distribution). Structural-only, as above. Hidden (not unmounted) while Findings is active (see "Render order" above).</li></ul></li></ol><p>Report lenses carry no impact-band chip and self-hide when their input data is absent. Every widget self-wraps in <code>WidgetCard</code> (<code>src/view/WidgetCard.tsx</code>: an <code><h3></code> title, one level below the board's <code><h2></code> section headers) for the document outline, except Scorecard, which renders its own header band, and RunVerdict, a plain <code><section></code> with its own <code><h2></code> title.</p><p>Every card is collapsible: the title sits in a <code>CollapsibleTrigger</code> button with a chevron (<code>WidgetCard.tsx</code>), and each widget sets its own <code>defaultCollapsed</code> (most default to <code>true</code>; <code>ImpactBoard</code>'s action-region grid forces <code>defaultCollapsed</code> on every <code>REGISTRY</code> widget instance it mounts, so a marginal finding doesn't default open). Route navigation ("jump to this finding") goes through the grid coordinator instead of a fixed <code>tabIndex</code>: <code>WidgetGrid.tsx</code> tracks each card's disclosure button and bumps <code>openRequestGeneration</code> to force a collapsed card open and focus its trigger when the target finding has no anchored row of its own. How much a card shows beyond that is a board-wide choice, set by the Basic/Advanced density tier in the topbar.</p>',43)])])}const u=o(i,[["render",n]]);export{p as __pageData,u as default};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{_ as o,o as t,c as d,a5 as a}from"./chunks/framework.DSg0KOwT.js";const p=JSON.parse('{"title":"Widget rendering","description":"","frontmatter":{},"headers":[],"relativePath":"contributor-guide/architecture/widget-rendering.md","filePath":"contributor-guide/architecture/widget-rendering.md"}'),i={name:"contributor-guide/architecture/widget-rendering.md"};function n(c,e,r,
|
|
1
|
+
import{_ as o,o as t,c as d,a5 as a}from"./chunks/framework.DSg0KOwT.js";const p=JSON.parse('{"title":"Widget rendering","description":"","frontmatter":{},"headers":[],"relativePath":"contributor-guide/architecture/widget-rendering.md","filePath":"contributor-guide/architecture/widget-rendering.md"}'),i={name:"contributor-guide/architecture/widget-rendering.md"};function n(c,e,s,r,l,h){return t(),d("div",null,[...e[0]||(e[0]=[a("",43)])])}const u=o(i,[["render",n]]);export{p as __pageData,u as default};
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import{_ as o,o as d,c as t,a5 as c}from"./chunks/framework.DSg0KOwT.js";const u=JSON.parse('{"title":"Worker protocol","description":"","frontmatter":{},"headers":[],"relativePath":"contributor-guide/architecture/worker-protocol.md","filePath":"contributor-guide/architecture/worker-protocol.md"}'),a={name:"contributor-guide/architecture/worker-protocol.md"};function s(r,e,n,i,l,p){return d(),t("div",null,[...e[0]||(e[0]=[c(`<h1 id="worker-protocol" tabindex="-1">Worker protocol <a class="header-anchor" href="#worker-protocol" aria-label="Permalink to "Worker protocol""></a></h1><h2 id="message-protocol" tabindex="-1">Message protocol <a class="header-anchor" href="#message-protocol" aria-label="Permalink to "Message protocol""></a></h2><p>Worker to main:</p><ul><li><code>app</code>, <code>stage</code>, <code>sql</code>, <code>executor</code>, <code>job</code>, <code>progress</code>: progressive, posted as the parser reads.</li><li><code>sqlPlan</code> (<code>{ type: 'sqlPlan', data: { executionId, planTree } }</code>): posted by <code>endSqlExecution</code> once a SQL execution's plan tree resolves. A repeated <code>SQLExecutionEnd</code> for that execution posts nothing, so no later <code>sql</code> message replaces the entry holding its <code>planTree</code>.</li><li><code>runAggregates</code>: one whole-run core-time-series summary (busy-core-ms, peak concurrency, per-stage task-duration sums), emitted just before <code>done</code>.</li><li><code>stageExecutorMetrics</code>: the post-completion re-post described in <a href="./overview.html#streaming">Streaming</a>, also emitted just before <code>done</code>.</li><li><code>stageSpeculationWaste</code>: the speculation totals of stages that gained waste from a TaskEnd after their StageCompleted, also described in <a href="./overview.html#streaming">Streaming</a> and emitted just before <code>done</code>.</li><li><code>stageLateAttemptWork</code>: the work of a failed stage attempt's tasks that ended after its StageCompleted, also described in <a href="./overview.html#streaming">Streaming</a> and emitted just before <code>done</code>.</li><li><code>done</code>, <code>taskData</code>, <code>error</code>.</li></ul><p><code>emitParseCompletion</code> posts the tail in a fixed order: any deferred AQE <code>sql</code> updates, <code>progress</code> (<code>pct: 1</code>), <code>stageLateAttemptWork</code>, <code>runAggregates</code>, <code>stageSpeculationWaste</code>, <code>stageExecutorMetrics</code>, the terminal <code>app</code>, then <code>done</code>.</p><p>A History Server failure is always the typed payload <code>{ type: 'error', source: 'shs', code, message? }</code>, where <code>code</code> is one of <code>local-server-unavailable</code>, <code>upstream-unreachable</code>, <code>application-not-found</code>, <code>access-or-upstream-failure</code>, or <code>invalid-event-log</code>. Fetch and HTTP failures carry only <code>code</code>: never an upstream message, URL, status, or response body. An <code>invalid-event-log</code> from the archive decode can carry a <code>message</code> written by <code>decodeShsArchive</code> (<code>packages/core/src/shs-fetch.ts</code>): the zip is unreadable, holds several application attempts or no event log, or an entry failed to decompress, in which case it names the zip entry and includes the decoder's error text. The intake shows that message under the recovery text.</p><p>Evidence availability has no dedicated worker message: the final <code>app</code> message carries a compact <code>evidenceInputs</code> counter summary, and <code>done.skippedLines</code> supplies its parse-integrity input.</p><p>Main to worker: <code>{ type: 'parse', file }</code>, <code>{ type: 'parseFiles', files }</code> (rolling <code>eventlog_v2_*</code> directories, one continuous stream across files), <code>{ type: 'parseFromUrl', request }</code> with <code>request = { baseUrl, appId, attemptId }</code>, and <code>{ type: 'getTaskData', stageId, reqId }</code>; <code>taskData</code> echoes <code>stageId</code> and <code>reqId</code> with <code>metrics</code> and <code>fieldNames</code>. The SHS request object is normalized before it reaches the worker: <code>{ baseUrl: string, appId: string, attemptId: string | null }</code>.</p><p>Files are read in 512 KiB slices (smaller for small files, so every file gets at least 100 reads), and <code>progress { pct, linesProcessed }</code> posts every 300 lines. <code>parseFromUrl</code> reports the download as 0-0.5, then <code>pct: null</code> every 2000 lines while parsing.</p><p><code>taskData</code> uses structured-clone (<code>.slice()</code>) so the worker retains its own <code>Float64Array</code> for re-renders.</p><p>Post-parse prefetch: after <code>done</code>, main runs <code>analyzer.ts</code> to build the bottleneck catalog, then fires parallel <code>getTaskData</code> for every flagged stage so their widgets render immediately. Unflagged stages are on-demand.</p><h3 id="decompress-worker" tabindex="-1">Decompress worker <a class="header-anchor" href="#decompress-worker" aria-label="Permalink to "Decompress worker""></a></h3><p>A dropped zstd file (<code>parse</code>, and each zstd file of a <code>parseFiles</code> directory) is decompressed in a second, nested worker, <code>packages/core/src/zstd-worker.ts</code>, so fzstd and the NDJSON parser run at the same time. The parse worker starts it on the first zstd file and reuses it for the rest of the parse. It dies with the parse worker, so the page's <code>terminate()</code> also cancels it. Other codecs and the SHS path (<code>parseFromUrl</code>) decompress on the parse worker. A dropped History Server zip (<code>parse</code>) streams its zstd entries through the decompress worker too.</p><p><code>packages/core/src/zstd-worker-client.ts</code> is the parse-worker end: it plugs into <code>streamFile</code> as the <code>zstdDecoder</code> option, like the Node CLI's native decoder. The messages' buffers move by transfer rather than structured-clone copy (a read slice that is a view of a larger buffer is copied once first, and decoded output is copied once into a 1 MiB batch):</p><ul><li>Parse to decompress: <code>start { id }</code> opens a stream, and replaces any open one. <code>data { id, seq, bytes, final }</code> carries one compressed read slice. <code>cancel { id }</code> drops the stream.</li><li>Decompress to parse: <code>ready</code> once, at startup. <code>chunk { id, bytes, length }</code> carries decoded output in batches of up to 1 MiB. <code>consumed { id, seq }</code> follows the last <code>chunk</code> of slice <code>seq</code>. <code>error { id, message }</code> ends the stream, and its message becomes the usual <code>Could not decompress</code> error.</li></ul><p>Flow control is a window of three input slices: <code>push()</code> resolves while fewer than three slices wait for their <code>consumed</code>. Each <code>chunk</code> is parsed in its message handler, so a slice is acknowledged only after its output was parsed, and at most three slices' output is ever queued. The final <code>push()</code> resolves once every slice is acknowledged. When <code>streamFile</code> gives up mid-stream (for example a failed file read) it calls <code>cancel()</code>. A parse exception inside a <code>chunk</code> handler makes the client post <code>cancel</code> itself and fail the stream, which surfaces on the next <code>push()</code>.</p><p>When the nested worker cannot start (<code>new Worker</code> throws or its script fails to load), the parse worker logs a warning and decodes with in-thread fzstd, with identical output. A crash after startup fails the stream it was decoding, and later streams fall back the same way. The progress <code>pct</code> is the read position, so it can run up to the window ahead of the slice being parsed. The self-contained <code>file://</code> export never parses in the browser (it opens with its run already analyzed), so it never starts either worker.</p><h3 id="zstd-in-the-browser-and-in-node" tabindex="-1">zstd in the browser and in Node <a class="header-anchor" href="#zstd-in-the-browser-and-in-node" aria-label="Permalink to "zstd in the browser and in Node""></a></h3><p>The browser decodes zstd with the vendored fzstd, not <code>DecompressionStream</code>. The Node CLI and MCP path (<code>collectRun</code>, <code>shs-load.ts</code>) uses <code>packages/core/src/cli/native-zstd.ts</code> instead, which walks frame boundaries itself: Node's own zstd decoders stop after the first frame, and Spark writes thousands of small ones. A parser change that depends on chunk shape must hold for both. Native chunks are whole frames, often one full event line and up to tens of MB (<code>buildChunkDecoder</code> decodes those in 512 KiB slices). For local files, frames of 64 KB or more decompress off the main thread and arrive as 256 KB pieces, so a native <code>push()</code> is async and <code>streamFile</code> awaits it. fzstd's chunks are views of one reused buffer, valid only until its <code>ondata</code> callback returns: copy one before keeping it. On a Node without native zstd (before 22.15/23.8), and for a frame past 64 MiB, the Node path falls back to fzstd. The SHS archive loader decodes frames inline, and only <code>collectRun</code> uses the off-thread path.</p><h3 id="evidence-availability-worker-input" tabindex="-1">Evidence-availability worker input <a class="header-anchor" href="#evidence-availability-worker-input" aria-label="Permalink to "Evidence-availability worker input""></a></h3><p><code>packages/core/src/event-handlers.ts</code>'s <code>createState()</code>/per-event handlers start and increment this fixed <code>evidenceInputs</code> shape on the normalized app model:</p><div class="language-text vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">text</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span>environmentUpdates, applicationEnds, stageSubmissions, rddStorageSnapshots,</span></span>
|
|
2
|
+
<span class="line"><span>sqlExecutions, resolvedSqlPlans, executorMetricRows, taskRecords</span></span></code></pre></div><p>The counters are a structured-clone-safe summary: they contain counts, not raw events, task records, host names, SQL text, paths, or Spark-property values. The final <code>app</code> message snapshots the counters after all input is processed, immediately before <code>done</code>; a consumer must not add a second worker message just for the ledger. <code>app</code> messages are also emitted mid-parse from <code>SparkListenerApplicationStart</code>, <code>SparkListenerApplicationEnd</code>, and a <code>SparkListenerEnvironmentUpdate</code> that arrives after the start; those snapshots are partial and non-authoritative. Only the terminal <code>app</code> message emitted by <code>emitParseCompletion</code> (the one immediately before <code>done</code>) should be used for evidence-availability conclusions. On the main thread, <code>useIngest</code> combines the normalized <code>AppModel</code> with <code>done.skippedLines</code>, derives the ledger before calling <code>analyze()</code>, and stores it as <code>appModel.evidenceAvailability</code>.</p><h3 id="evidence-availability-contract" tabindex="-1">Evidence-availability contract <a class="header-anchor" href="#evidence-availability-contract" aria-label="Permalink to "Evidence-availability contract""></a></h3><p><code>packages/core/src/evidence-availability.ts</code> is the single reusable taxonomy for the browser, the evidence report, and the headless CLI and MCP consumers. Its serialized ledger shape is <code>{ schemaVersion: 1, entries: EvidenceAvailabilityEntry[] }</code>. Entries are ordered by the following fixed eight-key enum:</p><div class="language-text vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">text</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span>executorMetrics, rddStorageSnapshots, sqlPlan, sparkConfiguration,</span></span>
|
|
3
|
+
<span class="line"><span>taskCoreTime, infrastructureContext, sourceContext, costContext</span></span></code></pre></div><p>Each entry has the stable fields <code>key</code>, <code>state</code>, <code>reasonCode</code>, <code>summary</code>, and optional <code>evidence</code>. The closed state enum is:</p><div class="language-text vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">text</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span>present, disabled, notEmitted, notApplicable, outsideEventLog, unknown</span></span></code></pre></div><p>The closed reason-code enum is:</p><div class="language-text vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">text</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span>observed, explicitlyDisabled, noObservedExecutorMetrics,</span></span>
|
|
4
|
+
<span class="line"><span>noObservedStageSubmission, noRddStorageSnapshot, noResolvedSqlPlan,</span></span>
|
|
5
|
+
<span class="line"><span>noSqlExecution, noEnvironmentUpdate, noTaskRecords, noUsableCoreTimeAggregate,</span></span>
|
|
6
|
+
<span class="line"><span>outsideEventLogScope, parseIncomplete</span></span></code></pre></div><p><code>summary</code> is domain-agnostic presentation copy derived from the reason code; the enum values, not labels, are the machine contract. Optional <code>evidence</code> is limited to safe compact provenance of the exact <code>{ eventType, count }</code> shape. <code>eventType</code> values are pinned to the exact <code>evidenceInputs</code> counter keys; they are not shortened or aliased. It must never include raw Spark-property values, host names, paths, SQL text, task records, raw-event identifiers, or other event payloads.</p><table tabindex="0"><thead><tr><th>Key</th><th>Proven <code>present</code> evidence</th><th>Trustworthy absence / boundary result</th></tr></thead><tbody><tr><td><code>executorMetrics</code></td><td><code>executorMetricRows > 0</code></td><td>Explicit observed <code>spark.eventLog.logStageExecutorMetrics=false</code> is <code>disabled</code> / <code>explicitlyDisabled</code>; otherwise <code>notEmitted</code> / <code>noObservedExecutorMetrics</code>. Observed metrics win over the explicit setting.</td></tr><tr><td><code>rddStorageSnapshots</code></td><td><code>rddStorageSnapshots > 0</code> from <code>SparkListenerStageSubmitted</code></td><td>No stage submissions is <code>notEmitted</code> / <code>noObservedStageSubmission</code>; stages submitted without any <code>RDD Info</code> rows is <code>notEmitted</code> / <code>noRddStorageSnapshot</code>. This is RDD snapshot evidence, not block-update telemetry.</td></tr><tr><td><code>sqlPlan</code></td><td><code>resolvedSqlPlans > 0</code></td><td>A SQL execution without a resolved plan is <code>notEmitted</code> / <code>noResolvedSqlPlan</code>; no SQL execution is <code>notApplicable</code> / <code>noSqlExecution</code>.</td></tr><tr><td><code>sparkConfiguration</code></td><td><code>environmentUpdates > 0</code></td><td><code>notEmitted</code> / <code>noEnvironmentUpdate</code>; a normalized empty config object alone is not evidence.</td></tr><tr><td><code>taskCoreTime</code></td><td><code>taskRecords > 0</code> and a usable <code>runAggregates.perStage</code> task count</td><td>No task records is <code>notEmitted</code> / <code>noTaskRecords</code>; task records with no usable aggregate are <code>notEmitted</code> / <code>noUsableCoreTimeAggregate</code>. An empty aggregate object is not usable evidence.</td></tr><tr><td><code>infrastructureContext</code></td><td>none</td><td>Always <code>outsideEventLog</code> / <code>outsideEventLogScope</code>.</td></tr><tr><td><code>sourceContext</code></td><td>none</td><td>Always <code>outsideEventLog</code> / <code>outsideEventLogScope</code>.</td></tr><tr><td><code>costContext</code></td><td>none</td><td>Always <code>outsideEventLog</code> / <code>outsideEventLogScope</code>.</td></tr></tbody></table><p>Parse integrity is fail-closed. A parse is trustworthy only when it skipped no malformed lines and observed an application-end event. For an untrustworthy parse, proof already observed remains <code>present</code>, and an explicitly observed executor-metrics disablement remains <code>disabled</code>; every conclusion based on absence or irrelevance becomes <code>unknown</code> / <code>parseIncomplete</code>. The three outside-event-log product boundaries remain <code>outsideEventLog</code>. In particular, <code>notEmitted</code> and <code>notApplicable</code> are never favorable measurements inferred from an incomplete parse.</p><p>Availability is model metadata, not a detector or finding: it never changes detector thresholds or ordering, impact band, catalog/scorecard counts, no-bottleneck behavior, or detector suppression. <code>session-snapshot.ts</code> captures and restores <code>evidenceAvailability</code> with the normalized model, so a recent-file switch retains the same ledger without reparsing.</p><h3 id="portable-evidence-report" tabindex="-1">Portable evidence report <a class="header-anchor" href="#portable-evidence-report" aria-label="Permalink to "Portable evidence report""></a></h3><p><code>packages/core/src/evidence-report.ts</code>'s <code>buildEvidenceReport(appModel, { redact })</code> returns <code>{ markdown, json }</code>: a self-contained, byte-stable document that runs the detectors over an <code>appModel</code> and serializes the result for sharing outside the tool. Raw task records are never included; identifier redaction (app id + host names → <code>app-1</code>/<code>host-1</code> pseudonyms via <code>packages/core/src/redact.ts</code>, with the app name and <code>spark.app.name</code> replaced by the app id's pseudonym) is opt-in with <code>{ redact: true }</code>. The JSON is pinned by <code>EVIDENCE_SCHEMA_VERSION</code> (<code>packages/core/src/evidence-report.ts</code>, surfaced as <code>json.schemaVersion</code>) and has this fixed top-level key order:</p><div class="language-text vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">text</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span>schemaVersion, summary, verdict, evidenceAvailability, detectors, findings, writeTargets, recommendations, cleanChecks, notRunChecks</span></span></code></pre></div><ul><li><code>summary</code> is the run header: <code>{ app: { id, name, sparkVersion }, stageCount, jobCount, sqlExecutionCount, findingCount, impactBandCounts, actionableFindingCount, actionableImpactBandCounts, clean, outcome, runShape }</code>. <code>findingCount</code> counts every row in <code>findings</code>; the <code>actionable*</code> counts leave out evidence caveats and the <code>incompleteRun</code> row, the same set the dashboard's top bar and verdict count. <code>clean</code> is <code>isCleanRun</code> from <code>packages/core/src/check-coverage.ts</code>, and <code>outcome</code> is <code>{ failedJobs, totalJobs, failureReason, failureReasonStageId }</code> from <code>summarizeRunOutcome</code> (<code>packages/core/src/run-outcome.ts</code>), the job results the dashboard verdict leads with; the Markdown prints it as an <code>- Outcome:</code> header line. <code>runShape</code> (<code>computeRunShape</code>, <code>packages/core/src/run-shape.ts</code>) is the Scorecard's wall-clock, Efficiency and Unused core time, the ETL phases' summed stage time and Core Usage by Locality's peak busy cores, each <code>null</code> where the dashboard shows none; MCP <code>get_run_summary</code> returns it too.</li><li><code>verdict</code> is the dashboard's run verdict from <code>buildRunVerdict</code> (<code>packages/core/src/run-verdict.ts</code>): <code>{ title, summary, steps, remainingPlaces, copyText }</code>. <code>steps</code> holds the first three places to look in the verdict card's order, each <code>{ key, stageId, type, tag, leadFindingId, actionLabel, recommendation, impact, impactMeaning, relatedTypes, text }</code>, where <code>text</code> is that step's line of the "Copy next steps" checklist and <code>copyText</code> is the whole checklist. The Markdown opens with it as <code>## Verdict</code>. The <code>recommendations</code> rollup stays alongside: it ranks fix types, the verdict ranks places.</li><li><code>evidenceAvailability</code> is the ledger above (or <code>null</code> when absent).</li><li><code>detectors</code> is <code>detectorCatalog()</code> output: one <code>{ type, version, scope, thresholds, docAnchor }</code> per detector, in <code>DETECTORS</code> order, so the exact threshold set that produced each finding travels with the evidence. On a tuned run (see the tuned-run paragraph below) a tuned row's <code>thresholds</code> are the ones the run used, plus <code>tunedThresholds</code>.</li><li><code>findings</code> are deterministically sorted rows (impact band → type → stage → id), each with a stable <code>id</code>, <code>tag</code>, core columns, an always-present <code>actionLabel</code>, and an <code>evidence</code> sub-object holding that finding type's declared evidence fields (see the per-type evidence paragraph below); <code>confidence</code>/ <code>validationRequired</code>/<code>docAnchor</code> appear only when the detector emitted them.</li><li><code>writeTargets</code> is <code>extractWriteTargets</code> output (<code>packages/core/src/write-targets.ts</code>): every SQL write command with its path or table. The field contract is in the user guide's <a href="./../../user-guide/getting-started.html#write-targets">Write targets</a>.</li><li><code>recommendations</code> is the impact-ranked <code>buildRecommendationRollup</code> output (<code>packages/core/src/recommendation-rollup.ts</code>), and <code>cleanChecks</code> lists every detector type that fired zero findings this run and could run: see the rollup and clean-checks paragraph below.</li><li><code>notRunChecks</code> lists the zero-finding types the log lacked the data to run, each with a <code>reason</code>: see the not-run checks paragraph below.</li></ul><p>Determinism holds because detector order, finding sort, and object key order are all fixed, so a given <code>appModel</code> serializes identically across calls.</p><p>Redaction enumerates hosts two ways: by walking the findings tree for every string value under a key literally named <code>host</code> (<code>evidence.host</code>, <code>evidence.failedTaskDetails[].host</code>, <code>evidence.retriedTaskDetails[].host</code>, and any nested <code>host</code> field, all covered without enumerating paths), and by scanning every string value for EC2-style hostnames / bare IPv4 tokens. So identifiers that surface only in free text (recommendation copy, a <code>stageFailed</code> failure reason in <code>valueText</code>) are pseudonymized too. Pseudonym numbering uses a numeric-aware sort, so re-redacting an already-redacted report is a no-op even past <code>host-10</code>.</p><p>Failed-task error text is dropped rather than pseudonymized, since a message or stack trace can carry file paths and data values that no host pattern matches: every array under a key named <code>failureGroups</code> has its <code>message</code> replaced and the message text stripped from its <code>stackExcerpt</code> (<code>redactTaskFailureGroup</code> in <code>task-failure.ts</code>). That covers the evidence report and both the findings and the stage records of the HTML export. A stage's failure reason is replaced outright with <code>[redacted]</code> wherever it appears: a <code>stageFailed</code> finding's <code>valueText</code>, a stage record's <code>stageFailureReason</code>, a job's <code>exception</code> and the report's <code>failureReason</code> (<code>redactStageFailureReasons</code> in <code>redact.ts</code>).</p><p>The Markdown rendering mirrors the JSON's field set: each finding block prints its <code>detector version</code>, its sorted <code>evidence</code> entries (byte-magnitude keys humanized), and the report ends with a <code>## Detectors</code> catalog carrying the version + threshold set. A finding's <code>impactEstimate</code> prints as its own <code>- impact: </code> line worded as the dashboard's "Potential savings": the wall-clock range, or the raw waste only when there is no range, followed by what it counts ("of run time", "of unused executor memory") and <code>estimateMethod</code>, and nothing for an informational or zero estimate. A <code>- estimate: </code> line carries the Advanced view's provenance sentence (<code>estimateProvenance</code>). Both come from <code>packages/core/src/impact-format.ts</code>, the formatters every dashboard surface uses, so memory reads in GB-h from 0.1 GB-h up, core time in core-s or core-h, and a time figure has no "Estimated" prefix. JSON finding rows carry the same figure as <code>impact</code>/<code>impactMeaning</code> when there is one, and each <code>recommendations</code> row an <code>impactMeaning</code> next to its <code>impact</code>, which is null for a resource figure that rounds to zero. <code>EvidenceExport</code> names downloads <code>evidence-<appId>[-redacted].<md|json|html></code>, taking the app id from the (already pseudonymized when redacting) report so a redacted file never leaks the real id and is never name-identical to a raw export.</p><p>A finding row carries <code>impactEstimate</code> when the finding has one, with the full contract documented in <a href="./impact-estimation.html#occupancy-weighted-attribution">Impact estimation</a> (basis, wallClock, estimateMethod, rawWaste, coreTimeMs). It appears after the pinned core columns (<code>id</code>, <code>type</code>, <code>impactBand</code>, <code>stageId</code>, <code>metric</code>, <code>value</code>, <code>recommendation</code>, <code>detectorVersion</code>, plus optional but pinned <code>confidence</code>, <code>validationRequired</code>, <code>docAnchor</code>) without displacing any of them. The full row shape is <code>FindingRowColumns</code> in <code>evidence-report.ts</code>, which also carries <code>name</code>, <code>tag</code>, <code>valueText</code>, <code>remediation</code>, <code>actionLabel</code>, <code>impact</code>/<code>impactMeaning</code> and, on a tuned run, <code>tunedThresholds</code>.</p><p>The report carries the same recommendation data the dashboard's "Fix These First" view shows, so the CLI (<code>packages/cli/bin/sparkforensics-analyze.mjs</code>) and the MCP tool (<code>diagnoseRun</code> in <code>packages/core/src/mcp-tools.ts</code>) get it too, not just the web markdown/JSON download. <code>FindingRow.actionLabel</code> is always present (a short imperative label like "Reduce shuffle size"), from <code>findingActionLabel</code> (<code>packages/core/src/finding-action-label.ts</code>), which reads the type's <code>actionLabel</code> in <code>finding-presentation.ts</code> and falls back to the type's name. The dashboard, the run verdict and the report rows all call it. <code>EvidenceReportJson.recommendations</code> is the same impact-ranked <code>buildRecommendationRollup</code> grouping (<code>packages/core/src/recommendation-rollup.ts</code>) that <code>FixTheseFirst.tsx</code> renders, so CLI/MCP/download consumers get the same "what's the highest-impact fix" ranking the dashboard shows, without changing the <code>findings</code> array's own impact-band-sorted order at all. <code>EvidenceReportJson.cleanChecks</code> lists every detector type that fired zero findings this run, each with <code>getThresholdSummary</code>'s one-line "what would have tripped it" sentence; unlike the dashboard's <code>Alerts.tsx</code> clean-checks table, it deliberately includes the one "always-mounted" reference type (<code>coreLocality</code>) even when it has no findings, since a flat evidence report has no separate always-visible surface for it to already appear on the way that widget does on the board.</p><p>A check the log could not run is not listed as clean. <code>packages/core/src/check-coverage.ts</code> holds the one rule, shared with the dashboard's verdict, top bar and Clean checks: a type whose only finding is an evidence caveat, every <code>scope: 'stage'</code> type on a log where no stage recorded an end, and the run-span types (<code>utilization</code>, <code>memoryUtilization</code>, <code>autoscalingChurn</code>) on a log with no ApplicationEnd. Those types are listed in <code>notRunChecks</code> instead of <code>cleanChecks</code>, each <code>{ type, tag, thresholdSummary, reason }</code>, where <code>reason</code> is the caveat's own recommendation (it names the setting to turn on) or the log-wide sentence. The Markdown has a <code>## Not checked on this log</code> section above <code>## Clean checks</code>, and a <code>Findings to act on</code> header line.</p><p><code>Finding</code> is a union discriminated on <code>type</code>, one member per emitted finding type (<code>packages/core/src/finding-types.ts</code>), and a finding row's <code>evidence</code> is an explicit per-type projection. Each type's <code><Type>Evidence</code> interface names its public fields, and <code>EVIDENCE_KEYS</code> in <code>evidence-report.ts</code> lists the same keys, checked both ways at compile time, so a field a detector adds only for another core module does not become report contract. Rows leave out these fields: <code>stageShape</code>'s <code>totalCores</code>; <code>utilization</code>'s <code>utilizationFraction</code>, <code>appDurationMs</code> and <code>totalCores</code>; <code>memoryUtilization</code>'s <code>idleRateFraction</code>, <code>allocatedMB</code>, <code>peakExecutors</code>, <code>appDurationMs</code> and <code>allocatedBytes</code>; <code>retryWaste</code>'s <code>extended</code> display copy. The impact estimator reads them on the finding. <code>value</code> is always numeric or <code>null</code>: the text-valued findings (<code>stageFailed</code>'s failure reason, <code>configAudit</code>'s current setting, <code>incompleteRun</code>'s <code>missing</code>) carry their text in a <code>valueText</code> column, present only on those rows, and the Markdown prints it where <code>value</code> would go. Renaming or removing an evidence field is a breaking change and needs a schema bump. <code>cleanChecks</code>/<code>notRunChecks</code> list emitted finding types, the set the dashboard's Clean checks shows: <code>overBroadcast</code> and <code>underBroadcast</code> in place of the <code>broadcastSizing</code> detector entry. A finding row's <code>actionLabel</code> for a (type, discriminant) combination with no label of its own is the type's name, the same fallback the verdict step uses.</p><p><code>buildEvidenceReport(appModel, { thresholds })</code> runs the detectors with a user's validated overrides (the CLI's and MCP server's <code>--thresholds</code>; see <a href="./detector-contract.html#tuning-thresholds">Tuning thresholds</a>). Only a tuned run adds keys: <code>tunedThresholds</code> (<code>{ <name>: { value, default } }</code>) on each finding row and each <code>cleanChecks</code>/<code>notRunChecks</code> entry from a detector an override moved off its defaults (or whose <code>suppressedBy</code> detector it moved), on that detector's <code>detectors</code> row, and as <code>summary.tunedThresholds</code> keyed by detector type. The Markdown adds a <code>- Tuned thresholds:</code> header line, a <code>- tuned thresholds:</code> line per affected finding, and marks tuned catalog rows and clean checks. A default run's report carries none of these keys, and every one is optional. The report caches key on the overrides object as well as the <code>appModel</code>, so one model's default and tuned reports never mix. Finding ids ignore thresholds: a finding a tuned run still emits keeps the id it has in a default run.</p><h3 id="finding-identity" tabindex="-1">Finding identity <a class="header-anchor" href="#finding-identity" aria-label="Permalink to "Finding identity""></a></h3><p>Every finding <code>analyzer.ts</code> emits carries a stable <code>id</code> (<code>push()</code>'s <code>findingId()</code> choke point, FNV-1a hash over <code>type | location | metric | value | discriminators</code>) and a <code>detectorVersion</code> (from the emitting <code>DETECTORS</code> entry). The two are deliberately decoupled: <code>id</code> is derived only from a finding's evidence tuple, never from <code>detectorVersion</code>, so a threshold or logic tweak that bumps a detector's <code>version</code> does not change the <code>id</code> of findings it still emits at the same location/metric. A saved reference (dashboard bookmark, exported report row) keeps pointing at the same logical finding across detector revisions; <code>detectorVersion</code> is separate provenance metadata for "which ruleset produced this," not part of identity. See <code>packages/core/test/analyzer-finding-identity.test.js</code> for the decoupling proof.</p><p>The discriminators are declared per finding type (<code>ID_DISCRIMINATORS</code> in <code>analyzer.ts</code>, each key checked against that type's fields) and joined in the fixed <code>DISCRIMINATOR_SLOTS</code> order, and the value slot takes <code>value</code>, else <code>valueText</code>.</p><p><code>id</code> stability holds for equivalent reruns of the same schema/analyzer version on the same input. Changing the id-derivation rule itself (the hash algorithm, or which fields feed the location key/discriminators) is an intentional breaking change and must bump <code>EVIDENCE_SCHEMA_VERSION</code> (<code>packages/core/src/evidence-report.ts</code>); there is no separate id-scheme version.</p><h3 id="headless-analysis-cli" tabindex="-1">Headless analysis CLI <a class="header-anchor" href="#headless-analysis-cli" aria-label="Permalink to "Headless analysis CLI""></a></h3><p><code>packages/cli/bin/sparkforensics-analyze.mjs</code> runs the same parser + detector contracts outside the browser, for CI. It accepts a single event-log file or a rolling-log directory, drives <code>runParse</code>/<code>runParseFiles</code> (<code>packages/core/src/parser-worker.ts</code>) with a Node-only File-like shim (<code>nodeFileFromPath</code> in <code>packages/core/src/cli/collect-run.ts</code>), and writes the exact <code>buildEvidenceReport</code> JSON schema described above: one format, not a second. The shim reads each requested slice with a positioned <code>readSync</code>, so a local log is never loaded whole and has no 2 GiB size limit; <code>collectRun</code> closes every descriptor it opened once parsing settles, on success and on error. Alternatively, <code>--shs-base-url <url> --app-id <id> [--attempt-id <id>]</code> fetches the run from a Spark History Server instead (mutually exclusive with the positional file/directory argument), calling <code>resolveFromShs</code> (<code>packages/core/src/shs-load.ts</code>, shared with the MCP server's SHS source path below) directly; no dependency on the <code>packages/server</code> package. The three flags are checked first with <code>validateShsRequest</code> (<code>packages/core/src/shs-request.js</code>), and a malformed one is a usage error (exit <code>2</code>) before any fetch. A failed SHS fetch reports its message to <code>stderr</code> and exits <code>4</code>, same as a candidate file that can't be parsed.</p><p>Regression budgets beyond the <code>--max-regression-pct</code> pair come from repeated <code>--regression-budget <metric>:<pct></code> flags and a <code>--budgets</code> file, parsed in <code>packages/core/src/cli/regression-budgets.ts</code> and passed to <code>evaluateBudgets()</code> as <code>regressionBudgets</code>; the legacy pair is one more budget, and a metric budgeted twice is a usage error. With two or more positional candidates (or <code>--format ndjson</code>), the CLI's <code>runMultiLog</code> parses and analyzes the baseline once, then evaluates each candidate in turn and writes one NDJSON line per candidate (<code>log</code>, <code>status</code>, <code>exitCode</code>, <code>error</code>, <code>budgets</code>, <code>candidate</code>, <code>comparison</code>). A candidate that can't be read or parsed gets a <code>status: "error"</code> line with <code>exitCode</code> <code>4</code> (<code>6</code> for an internal failure on that candidate); the exit code is the worst line (<code>6</code> > <code>5</code> > <code>4</code> > <code>1</code> > <code>3</code> > <code>0</code>, <code>EXIT_SEVERITY</code>). A baseline that can't be read exits <code>5</code> with no lines. Under <code>--redact</code>, <code>log</code> and the <code>stderr</code> prefixes are <code>candidate-<n></code> (1-based argument position) and an error line's message is generic, so no candidate path, which usually carries the app id, reaches the output. In both modes <code>--redact</code> also swaps the <code>stderr</code> text of a baseline or candidate failure (SHS fetch included) for <code>redactedFailure</code>'s role-only message.</p><p>Optional CLI-flag budgets (<code>--max-runtime <ms></code>, <code>--max-spill <gb></code>, <code>--max-skew <ratio></code>, <code>--max-failed-task-rate <pct></code>, <code>--min-efficiency <pct></code>) are evaluated in <code>packages/core/src/cli/budgets.ts</code> against the existing finding catalog (<code>analyze()</code>) and <code>computeEfficiencyModel</code>; there is no second rule engine. <code>--min-efficiency</code> compares <code>100 - wastagePct</code> (busy core time, the complement of the dashboard's Unused core time) and says so in its detail ("Busy core time 26% below budget 90%."); it is not the Scorecard's Efficiency tile (<code>stagesActive / total</code>). A budget whose required evidence is missing (e.g. the run never emitted <code>ApplicationEnd</code>, or has no usable per-task <code>runAggregates</code>) is reported as inconclusive (<code>stderr</code> warning) rather than silently passing, and gets its own exit code distinct from both pass and violation. Exit codes: <code>0</code> pass, <code>1</code> a configured budget was violated, <code>2</code> a usage error (unknown or value-less flag, unknown <code>--regression-metric</code> key, an unreadable or invalid <code>--thresholds</code> or <code>--budgets</code> file), <code>3</code> no violations but at least one budget was inconclusive, <code>4</code> the candidate could not be read or parsed (or its SHS fetch failed), <code>5</code> the <code>--baseline</code> could not be read or parsed, <code>6</code> an internal error (an unhandled failure, a failed <code>--export-html</code> export). <code>main</code> catches anything unhandled and exits <code>6</code> rather than letting Node exit <code>1</code>, which would read as a violation. When several apply the worst wins: <code>6</code>, <code>5</code>, <code>4</code>, <code>1</code>, <code>3</code>, <code>0</code>; a violation always wins over an inconclusive result in the same run (exit <code>1</code>, not <code>3</code>). <code>evaluateBudgets()</code> also always adds an inconclusive <code>run-complete</code> result when the catalog has an <code>incompleteRun</code> finding, so a run with no <code>ApplicationEnd</code> exits <code>3</code> even with no budget flags. With <code>--baseline</code>, the absolute budgets and this check apply to the candidate run; the MCP <code>evaluate_budgets</code> tool below uses the same function with the same roles.</p><h3 id="mcp-server" tabindex="-1">MCP server <a class="header-anchor" href="#mcp-server" aria-label="Permalink to "MCP server""></a></h3><p><code>packages/core/src/mcp-server-factory.ts</code>'s <code>createMcpServer()</code> registers 8 tools: <code>list_runs</code> (candidate runs in a local directory or on a Spark History Server, to pick one before diagnosing it), <code>diagnose_run</code> (thresholded findings + remediation text), <code>get_run_summary</code> (app/stage/job/sql counts and duration, no findings), <code>compare_runs</code> (the comparison verdict from <code>comparisonVerdict</code> in <code>packages/core/src/comparison-verdict.ts</code>, the dashboard comparison page's own headline, plus categorized findings delta + metric deltas between two runs; <code>CompareRunsResult.jobOutcomes</code> carries each run's failed jobs and incomplete flag for it), <code>evaluate_budgets</code> (pass/fail budget thresholds against one run, optionally with a second run for regression/fail-on-introduced budgets: the MCP side of the CLI's <code>evaluateBudgets()</code> gating), <code>get_finding_evidence</code> (raw evidence bundle for one finding, for drill-down after <code>diagnose_run</code>), <code>get_finding_documentation</code> (detection/tuning reference docs for one finding type, independent of any run), and <code>get_reference_doc</code> (a full tuning-reference chapter or bottleneck page by doc anchor, e.g. <code>#joins</code>). None re-implement detector logic: <code>list_runs</code> lists candidates (<code>packages/core/src/list-runs.ts</code>), and the rest repackage <code>analyze</code>/<code>buildEvidenceReport</code>/<code>compareRuns</code>/<code>captureSnapshot</code>/<code>evaluateBudgets</code> from <code>packages/core/src/mcp-tools.ts</code>, which resolves a <code>source</code> (event-log <code>path</code>, or an SHS <code>shsBaseUrl</code>/<code>appId</code>/<code>attemptId</code> triple) into a cached <code>AppModel</code>. The SHS source path calls <code>resolveFromShs</code> (<code>packages/core/src/shs-load.ts</code>, also used directly by the CLI's <code>--shs-base-url</code> mode above), which reuses two extractions shared with the browser ingestion flow: <code>fetchShsEventLog</code> (<code>packages/core/src/proxy.js</code>, the same upstream fetch the local server's <code>/shs-proxy</code> route uses) fetches the zip, and <code>decodeShsArchive</code> (<code>packages/core/src/shs-fetch.ts</code>, re-exported from <code>parser-worker.ts</code>'s barrel) streams it through the parser as worker messages, which <code>collectViaDispatch</code> assembles into an <code>AppModel</code>. <code>runParseFromUrl</code> calls the same <code>decodeShsArchive</code> after fetching through the proxy.</p><p>Two transports connect to that one factory: <code>packages/mcp/bin/sparkforensics-mcp.mjs</code> (stdio, for local MCP clients) and <code>packages/server/index.js</code>'s <code>/mcp</code> route (streamable HTTP, for the local server). The run cache (<code>packages/core/src/mcp-tools.ts</code>) is a module-level LRU (cap 8 and 15-minute idle TTL by default, overridable via <code>SPARKFORENSICS_MCP_CACHE_CAP</code> and <code>SPARKFORENSICS_MCP_CACHE_TTL_MS</code>, lazily swept on access) keyed by resolved source (path plus mtime, ctime and size, or SHS baseUrl+appId+attemptId), so a client mints a <code>runId</code> once via <code>resolveOrCreateRun</code> and reuses it across subsequent tool calls instead of re-parsing.</p><p>Every tool failure comes back as <code>{isError: true, content: [...], structuredContent: {code}}</code>, never an HTTP-status-shaped error; <code>code</code> is one of the 5 existing SHS codes (<code>SHS_ERROR_CODES</code>, <code>packages/core/src/shs-request.js</code>) plus <code>run-not-found</code>, <code>finding-not-found</code>, <code>invalid-event-log</code> (also covers a nonexistent <code>path</code> source), <code>archive-too-large</code> (SHS archive over the <code>SPARKFORENSICS_MAX_ARCHIVE_BYTES</code> byte cap, default 1 GiB, because the MCP path buffers the whole archive in memory, unlike the streaming <code>/shs-proxy</code> route), <code>invalid-type</code>, <code>invalid-anchor</code> (documentation tools), <code>directory-not-found</code>, <code>invalid-date-filter</code>, and <code>invalid-shs-base-url</code> (<code>list_runs</code>). An error without a code reports <code>access-or-upstream-failure</code>. A stalled SHS archive body fails as <code>upstream-unreachable</code> after <code>SPARKFORENSICS_SHS_TIMEOUT_MS</code> (default 30 s) without data.</p><p><code>scripts/vendor-core.mjs</code> (shared by <code>packages/cli</code>, <code>packages/mcp</code>, and <code>packages/server</code>'s <code>prepack</code> scripts) vendors <code>packages/core/src/</code> wholesale into each package's own <code>vendor-core/</code> at pack time, pre-stripping TypeScript to plain <code>.js</code> (Node's native TS stripping refuses to run on <code>.ts</code> files under <code>node_modules</code>, which is exactly where a published <code>vendor-core/</code> lands). Each package's bin/entry point resolves its needed module through <code>packages/core/src/load-vendored.js</code>: from <code>vendor-core/</code> if present, else from the real <code>packages/core/src/</code> sibling loaded as <code>.ts</code> directly. In a monorepo checkout a leftover <code>vendor-core/</code> is used only while its <code>core-source-hash.txt</code> (written by <code>vendor-core.mjs</code>) matches <code>packages/core/src</code>; otherwise the bin warns on stderr and loads <code>packages/core/src</code> (<code>packages/server/index.js</code> mirrors <code>resolveStaticRoot</code>'s <code>public/</code>-vs-<code>../dist</code> pattern for this same fallback).</p><h2 id="typescript-core-and-runtime-event-validation" tabindex="-1">TypeScript core and runtime event validation <a class="header-anchor" href="#typescript-core-and-runtime-event-validation" aria-label="Permalink to "TypeScript core and runtime event validation""></a></h2><p>All of <code>src/</code> (browser SPA) and <code>packages/core/src/</code> (shared analysis logic) is strict TypeScript, except for a few plain-<code>.js</code> files: the two vendored third-party decompressors, <code>packages/core/src/vendor/fflate.js</code> and <code>packages/core/src/vendor/fzstd.js</code> (left untouched deliberately: vendored code, not project code), <code>load-vendored.js</code> (copied byte-for-byte into <code>vendor-core/</code>, so it must run without TS stripping), and the SHS helpers <code>shs-request.js</code> and <code>proxy.js</code>, which the local server imports as plain JS. Every remaining import of a same-repo <code>.ts</code> module uses a <code>.ts</code> specifier (e.g. <code>import { dispatchLine } from './event-handlers.ts'</code>), not <code>.js</code>: the CLI and MCP entrypoints (<code>packages/cli/bin/sparkforensics-analyze.mjs</code>, <code>packages/mcp/bin/sparkforensics-mcp.mjs</code>) run under plain Node's ESM resolver, which cannot remap a <code>.js</code> specifier to a same-named <code>.ts</code> file the way Vite/Vitest's bundler-style resolver can. <code>tsconfig.json</code> sets <code>"allowImportingTsExtensions": true</code> to make this legal.</p><p>Event schema validation lives in <code>packages/core/src/event-schemas.ts</code>: one zod schema per <code>SparkListener*</code> event variant the parser understands (17 total: <code>LogStart</code>, <code>ApplicationStart</code>, <code>EnvironmentUpdate</code>, <code>ApplicationEnd</code>, <code>JobStart</code>, <code>JobEnd</code>, <code>StageSubmitted</code>, <code>StageCompleted</code>, <code>StageExecutorMetrics</code>, <code>TaskEnd</code>, the four SQL-execution-UI listener events, <code>ExecutorAdded</code>, <code>ExecutorRemoved</code>, <code>BlockUpdated</code>), combined into <code>SparkEventSchema = z.discriminatedUnion('Event', [...])</code>. <code>processEvent</code>'s switch (<code>event-handlers.ts</code>) consumes the resulting <code>SparkEvent</code> union type directly, so a schema change and a handler's expectations can't silently drift apart.</p><p>The recursive <code>sparkPlanInfo.children</code> tree carried on SQL-execution-start events is validated by <code>parseSparkPlanInfoTree</code>, an iterative, explicit heap-allocated-stack parser, deliberately not <code>z.lazy()</code>. A real plan tree's depth is unbounded, and attacker-uncontrolled input should never hand zod's own recursive schema resolution an arbitrarily deep structure to walk. The iterative parser shallow-validates one node at a time via <code>z.object(...)</code> and builds the tree itself, capping at <code>MAX_PLAN_DEPTH = 500</code> (throws past that). This mirrors <code>packages/core/src/plan-tree-walk.ts</code>'s <code>walkPlanTree</code>, which uses the same iterative-over-recursive approach for the <em>resolved</em> tree; <code>event-schemas.ts</code> applies it one layer earlier, to the raw JSON before it becomes a tree at all.</p><p>There are two external-data boundaries, the two places this codebase parses data it does not control. Both run that data through a schema, and both treat a validation failure the same way: a silent skip, not a distinct error.</p><ul><li><code>dispatchLine</code> (<code>event-handlers.ts</code>): after <code>JSON.parse</code> succeeds, a line whose <code>Event</code> value is one of the 17 modeled types but fails that type's own schema increments <code>skippedLines</code>. An <code>Event</code> value outside the 17 modeled types is silently ignored without incrementing <code>skippedLines</code>: real Spark logs always carry plenty of ordinary event types this tool does not model (<code>TaskStart</code>, <code>BlockManagerAdded</code>, <code>ExecutorMetricsUpdate</code>, and others), and counting them would trip <code>evidence-availability.ts</code>'s fail-closed <code>trustworthy</code> gate on healthy logs. One exception: an AQE update that a later update for the same open execution supersedes is never parsed (<code>deferAdaptiveUpdate</code>, see <a href="./detector-contract.html">the detector contract</a>), so a malformed superseded update is not counted.</li><li><code>shs-fetch.ts</code>: on a non-OK response from the local SHS proxy, the JSON error envelope is validated against <code>ShsProxyErrorBodySchema</code> (<code>packages/core/src/shs-schemas.ts</code>, <code>{ code: string }</code>, <code>.passthrough()</code>). A body that isn't valid JSON, or is JSON but fails that schema, falls back to the generic <code>access-or-upstream-failure</code> code: the same silent-skip treatment as a malformed log line. There is no separate "malformed SHS response" error code.</li></ul><p>Neither boundary distinguishes "malformed JSON" from "wrong shape" from "unrecognized variant" in what it reports outward: all three collapse into the same skip/fallback path. That is a deliberate scope decision.</p><p>The exhaustiveness convention is <code>packages/core/src/assert-never.ts</code>. <code>assertNever(x: never): never</code> throws at runtime and, more importantly, fails <code>tsc</code> at compile time if <code>x</code> is not actually <code>never</code>, i.e. if some case of a union type isn't handled. Used at the <code>default</code> arm of <code>event-handlers.ts</code>'s <code>processEvent</code> switch (over <code>SparkEvent</code>) and <code>analyzer.ts</code>'s scope-dispatch switch (over a detector's <code>scope: 'stage' | 'sql' | 'app' | 'config'</code>). It is the project's standard pattern for any future exhaustive switch or dispatch over a closed union: add <code>default: return assertNever(x);</code> (or the closest non-returning equivalent) so that adding a new union member without updating every consumer becomes a compile error instead of a silent runtime gap.</p>`,68)])])}const m=o(a,[["render",s]]);export{u as __pageData,m as default};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{_ as o,o as
|
|
1
|
+
import{_ as o,o as d,c as t,a5 as c}from"./chunks/framework.DSg0KOwT.js";const u=JSON.parse('{"title":"Worker protocol","description":"","frontmatter":{},"headers":[],"relativePath":"contributor-guide/architecture/worker-protocol.md","filePath":"contributor-guide/architecture/worker-protocol.md"}'),a={name:"contributor-guide/architecture/worker-protocol.md"};function s(r,e,n,i,l,p){return d(),t("div",null,[...e[0]||(e[0]=[c("",68)])])}const m=o(a,[["render",s]]);export{u as __pageData,m as default};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
import{_ as s,o,c as a,a5 as i}from"./chunks/framework.DSg0KOwT.js";const g=JSON.parse('{"title":"Contributing","description":"","frontmatter":{},"headers":[],"relativePath":"contributor-guide/contributing.md","filePath":"contributor-guide/contributing.md"}'),t={name:"contributor-guide/contributing.md"};function n(c,e,d,r,h,p){return o(),a("div",null,[...e[0]||(e[0]=[i('<h1 id="contributing" tabindex="-1">Contributing <a class="header-anchor" href="#contributing" aria-label="Permalink to "Contributing""></a></h1><h2 id="where-does-new-content-go" tabindex="-1">Where does new content go? <a class="header-anchor" href="#where-does-new-content-go" aria-label="Permalink to "Where does new content go?""></a></h2><ul><li><code>docs/</code> is for internal engineering records, such as the design and product notes in <code>docs/DESIGN.md</code> and <code>docs/PRODUCT.md</code>. It stays flat, unstructured markdown, read through GitHub's own file viewer. Working plans and specs (<code>docs/plans/</code>, <code>docs/specs/</code>, <code>docs/superpowers/</code>), like audits in <code>docs/audit/</code>, are gitignored: never commit them.</li><li><code>docs-site/</code> (this site) is the published documentation for users and contributors. If you're writing something a user or a first-time contributor should read, it belongs here, not in <code>docs/</code>.</li></ul><h2 id="linting-before-you-commit" tabindex="-1">Linting before you commit <a class="header-anchor" href="#linting-before-you-commit" aria-label="Permalink to "Linting before you commit""></a></h2><p><code>npm install</code> runs the <code>prepare</code> script, which points git at <code>.githooks/</code> (<code>git config core.hooksPath .githooks</code>). After that, every commit runs <code>.githooks/pre-commit</code>: it lints any staged <code>.js</code>/<code>.mjs</code> files with eslint and blocks the commit if linting fails. <code>git commit --no-verify</code> skips it. To run the same linter by hand, use <code>npm run lint</code>.</p><h2 id="changesets-and-releases" tabindex="-1">Changesets and releases <a class="header-anchor" href="#changesets-and-releases" aria-label="Permalink to "Changesets and releases""></a></h2><p>A changeset is required for a PR that touches <code>packages/core</code>, <code>packages/cli</code>, <code>packages/mcp</code>, <code>packages/server</code>, <code>packages/analyze</code>, <code>packages/sparkforensics</code>, or the root site app (<code>src/</code>, <code>index.html</code>, <code>vite.config.ts</code>, or the root <code>package.json</code>). Run <code>npx changeset add</code>, answer its prompts, and commit the generated <code>.changeset/*.md</code> file. CI's <code>changeset</code> job (<code>scripts/check-changeset.sh</code>) fails the PR otherwise, with instructions to fix it. <code>docs-site/</code> and <code>docs/</code> are exempt: doc-only changes don't need one.</p><p>Needing a changeset doesn't mean getting published, though. Only <code>packages/cli</code>, <code>packages/mcp</code>, <code>packages/server</code>, and the two alias packages publish to npm. The aliases, <code>packages/analyze</code> (<code>sparkforensics-analyze</code>) and <code>packages/sparkforensics</code> (<code>sparkforensics</code>), hold no code of their own: each depends on <code>sparkforensics-cli</code> and runs its <code>sparkforensics-analyze</code> bin, so <code>npx sparkforensics-analyze</code> works outside a checkout and nobody else can publish under either name. <code>.changeset/config.json</code>'s <code>fixed</code> group keeps their versions equal to <code>sparkforensics-cli</code>'s, so every CLI release republishes them. The root site (<code>sparkforensics-web</code>) is <code>"private": true</code> and never published; its changeset only bumps its <code>package.json</code> version and writes a <code>CHANGELOG.md</code> entry, via <code>.changeset/config.json</code>'s <code>privatePackages.version</code> setting. <code>@sparkforensics/core</code> is the other private package but stays out of this entirely (<code>.changeset/config.json</code>'s <code>ignore</code> list): it's vendored into each publishable code package (<code>packages/{cli,mcp,server}</code>) by filesystem copy at pack time (<code>scripts/vendor-core.mjs</code>), not read as a real dependency, so versioning it would have no consumer.</p><p>A release itself is two merges, not one: merging your feature PR into <code>main</code> runs <code>.github/workflows/release.yml</code>, which opens (or updates) a "Version Packages" PR collecting all pending changesets. Nothing publishes at this step. Merging <em>that</em> PR lands the version bumps and changelog entries it carries, and the release run that follows runs the full build and test suite, then <code>npm publish</code> for each publishable package whose new version isn't on npm yet.</p><h3 id="publishing-a-new-package-name-maintainers" tabindex="-1">Publishing a new package name (maintainers) <a class="header-anchor" href="#publishing-a-new-package-name-maintainers" aria-label="Permalink to "Publishing a new package name (maintainers)""></a></h3><p>The release workflow publishes through npm Trusted Publishing, which can't create a package name that doesn't exist on npmjs.com yet. A new publishable package needs one manual publish from a maintainer's npm account first, then a Trusted Publisher entry. All five current packages (<code>sparkforensics-cli</code>, <code>sparkforensics-mcp</code>, <code>sparkforensics-server</code>, <code>sparkforensics-analyze</code> and <code>sparkforensics</code>) have their Trusted Publisher entries; this only applies to a package added later. Until it's done, that package's publish step fails and the release run exits non-zero; packages that don't depend on it still publish.</p><p>First add the package to <code>PACKAGES</code> in <code>scripts/publish-packages.mjs</code> and to the path regex in <code>scripts/check-changeset.sh</code> (the comments in both list every place to keep in sync); an alias of the CLI also goes in <code>.changeset/config.json</code>'s <code>fixed</code> group. Then:</p><ol><li><p>From a clean checkout of <code>main</code>, logged in with <code>npm login</code>, publish the new package at the version in its <code>package.json</code>:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">cd</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> packages/</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"><</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">new-packag</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">e</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">></span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> && </span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">npm</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> publish</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --access</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> public</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">)</span></span></code></pre></div></li><li><p>On npmjs.com, open the package's <strong>Settings</strong>, add a <strong>Trusted Publisher</strong> for GitHub Actions with organization <code>shuffle-works</code>, repository <code>sparkforensics</code>, workflow filename <code>release.yml</code> and no environment, and save.</p></li><li><p>Optional: in the same settings page, set publishing access to require two-factor authentication and disallow tokens, so only the workflow can publish from then on.</p></li></ol><p><code>scripts/publish-packages.mjs</code> skips a version that's already on the registry, so the next release run won't try to republish the bootstrap version.</p><p>For the full setup and test workflow, see <a href="./development-setup.html">Development setup</a> and <a href="./testing.html">Testing & verification</a>.</p>',15)])])}const u=s(t,[["render",n]]);export{g as __pageData,u as default};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{_ as s,o,c as a,a5 as i}from"./chunks/framework.DSg0KOwT.js";const g=JSON.parse('{"title":"Contributing","description":"","frontmatter":{},"headers":[],"relativePath":"contributor-guide/contributing.md","filePath":"contributor-guide/contributing.md"}'),t={name:"contributor-guide/contributing.md"};function n(c,e,r,
|
|
1
|
+
import{_ as s,o,c as a,a5 as i}from"./chunks/framework.DSg0KOwT.js";const g=JSON.parse('{"title":"Contributing","description":"","frontmatter":{},"headers":[],"relativePath":"contributor-guide/contributing.md","filePath":"contributor-guide/contributing.md"}'),t={name:"contributor-guide/contributing.md"};function n(c,e,d,r,h,p){return o(),a("div",null,[...e[0]||(e[0]=[i("",15)])])}const u=s(t,[["render",n]]);export{g as __pageData,u as default};
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import{_ as s,o as t,c as i,a5 as a}from"./chunks/framework.DSg0KOwT.js";const k=JSON.parse('{"title":"Development setup","description":"","frontmatter":{},"headers":[],"relativePath":"contributor-guide/development-setup.md","filePath":"contributor-guide/development-setup.md"}'),o={name:"contributor-guide/development-setup.md"};function n(d,e,r,c,p,h){return t(),i("div",null,[...e[0]||(e[0]=[a(`<h1 id="development-setup" tabindex="-1">Development setup <a class="header-anchor" href="#development-setup" aria-label="Permalink to "Development setup""></a></h1><p>This page is for working on SparkForensics itself. To run the dashboard as a user, see <a href="./../user-guide/getting-started.html#local-server-mode">Run it locally</a>.</p><p>You need Node 22.18+ (CI uses the version in <code>.nvmrc</code>) and <code>git</code> on your PATH: the first test or build run fetches the tuning reference with it. Clone the repo, then:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">git</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> submodule</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> update</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --init</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> dev/log-corpus</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"> # or: git clone --recurse-submodules</span></span>
|
|
2
|
+
<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">npm</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> install</span></span>
|
|
3
|
+
<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">npm</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> test</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"> # vitest run: the root suite (site app and dev tooling)</span></span>
|
|
4
|
+
<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">npm</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> run</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> test:watch</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"> # vitest, watch mode</span></span>
|
|
5
|
+
<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">npm</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> run</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> test:core</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"> # packages/core's suite: detectors, analyzer, parser, MCP tools</span></span>
|
|
6
|
+
<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">npm</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> run</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> test:cli</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"> # packages/cli's suite (packs and spawns the real tarball)</span></span>
|
|
7
|
+
<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">npm</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> run</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> test:mcp</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"> # packages/mcp's suite (packs and spawns the real tarball)</span></span>
|
|
8
|
+
<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">npm</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> run</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> dev</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"> # Vite dev server for the app itself</span></span>
|
|
9
|
+
<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">npm</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> run</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> build</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"> # production build, outputs dist/ (plus dist/export-template.html, the HTML download's template)</span></span>
|
|
10
|
+
<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">npm</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> run</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> preview</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"> # serves dist/ locally</span></span>
|
|
11
|
+
<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">npx</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> tsc</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --noEmit</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"> # typecheck (strict TypeScript across src/, tests/view/ and packages/core/src/)</span></span>
|
|
12
|
+
<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">npm</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> run</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> lint</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"> # eslint over the repo's JS (.ts/.tsx are covered only by tsc)</span></span>
|
|
13
|
+
<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">node</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> packages/cli/bin/sparkforensics-analyze.mjs</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> <</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">file</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">|</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">rolling-log-dir</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">> </span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># run the CLI analyzer against a log, outside the browser</span></span></code></pre></div><p><code>npm run test:coverage</code>, at the root or inside <code>packages/core</code>, <code>cli</code>, <code>mcp</code> or <code>server</code>, runs that suite with <code>--coverage</code> (v8 provider, <code>lcov</code> and <code>text</code> reporters, plus <code>html</code> at the root; written to <code><pkg>/coverage/lcov.info</code>). CI's <code>build</code>, <code>core</code>, <code>cli</code>, <code>mcp</code> and <code>server</code> jobs all run through it and upload to Coveralls, merged by a final <code>finish</code> job.</p><p>The dashboard's <strong>Download HTML dashboard</strong> fetches <code>export-template.html</code> from beside the app, so under <code>npm run dev</code> it fails with an error toast; use <code>npm run build && npm run preview</code> to try it.</p><p><code>npm install</code> also runs the <code>prepare</code> script, which points git at <code>.githooks/</code>. After that, a pre-commit hook lints staged JS files on every commit. Fix what it reports, or skip it with <code>git commit --no-verify</code>.</p><p>The tuning reference under <code>packages/core/src/docs-content/{chapters,tuning,diagrams}</code> isn't committed. It's generated from the <code>shuffle-works/spark-tuning-reference</code> commit pinned in <code>packages/core/src/docs-content/upstream.json</code>: the first <code>npm test</code>, <code>npm run docs:dev</code> or <code>npm run build</code> fetches it over HTTPS (about 2s, no credentials) and later runs reuse the gitignored copy while it matches the pin. <code>npm run docs:fetch</code> does the same step on its own. Never edit those folders: fix the content upstream, then <code>npm run docs:bump</code> to move the pin. Running the MCP or CLI bins from source (<code>node packages/*/bin/...</code>) doesn't fetch it, so run <code>npm run docs:fetch</code> first: without it the MCP's tuning-reference lookups come back empty. <code>npm run docs:bump [-- <sha>]</code> moves the pin (default: upstream head), regenerates the docs and writes the changeset; put the compare link it prints in the PR body. <code>npm run docs:bump -- --check [<sha>]</code> runs the anchor gate and writes nothing; <code>.github/workflows/tuning-reference-drift.yml</code> runs it with <code>test:core</code> every week.</p><p>Offline, a cached copy keeps working. If the pin has moved since, <code>npm test</code> and <code>docs:dev</code> warn and use the older copy, while <code>docs:build</code>, <code>npm pack</code> and anything with <code>CI</code> set fail. To build from a local upstream checkout instead (offline, or to preview upstream edits in this docs site before they merge), set <code>SPARK_TUNING_REFERENCE_DIR</code>:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">SPARK_TUNING_REFERENCE_DIR</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">=</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">../spark-tuning-reference</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;"> npm</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> run</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> docs:dev</span></span></code></pre></div><p>The same checks apply to that checkout. <code>npm pack</code> and CI accept it only when it sits at the pinned commit with no uncommitted <code>content/</code> changes.</p><p><code>dev/log-corpus</code> is a git submodule pointing at the public <code>spark-event-corpus-data</code> repo. It's optional locally: if you skip the <code>git submodule update</code> step above, the corpus-backed tests (such as <code>packages/server/test/shs-proxy-fixture.test.js</code>) report as skipped rather than failed, which is expected, not a bug. CI's <code>core</code> and <code>server</code> jobs check it out and run them (plus the <a href="./testing.html#corpus-regression-snapshot">corpus regression snapshot</a> in <code>core</code>); the <code>build</code>, <code>cli</code> and <code>mcp</code> jobs don't, so a corpus-backed test belongs in <code>packages/core</code> or <code>packages/server</code>.</p><p>The <code>packages/server/</code> package (the optional local-server deploy mode) keeps its own dependencies and test suite. Run it from the repo root: a second <code>npm install</code> inside <code>packages/server</code> prunes the shared root <code>node_modules</code>'s workspace symlinks.</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">npm</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> run</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> test:server</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"> # runs packages/server's own suite, not part of \`npm test\`</span></span></code></pre></div><p>Packing or releasing <code>packages/server/</code> builds the docs site for you: its <code>prepack</code> runs <code>packages/server/scripts/copy-frontend.js</code>, which runs the root <code>npm run build</code> (<code>docs:build</code> included), so the packed artifact always ships <code>/docs/</code>.</p><p>This docs site has its own dev loop:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">npm</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> run</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> docs:dev</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"> # VitePress dev server for docs-site/</span></span>
|
|
14
|
+
<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">npm</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> run</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> docs:build</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"> # production build; also checks internal links/anchors</span></span>
|
|
15
|
+
<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">npm</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> run</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> docs:preview</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"> # serves the built docs site locally</span></span></code></pre></div><h2 id="generated-and-vendored-files" tabindex="-1">Generated and vendored files <a class="header-anchor" href="#generated-and-vendored-files" aria-label="Permalink to "Generated and vendored files""></a></h2><ul><li><code>packages/core/src/docs-content/detection/*.md</code> is generated from <code>docs-site/user-guide/understanding-findings.md</code>. After editing that guide, run <code>npm run split-detection-docs</code> and commit the output, or <code>tests/detection-docs-split.test.js</code> fails.</li><li><code>packages/{cli,mcp,server}/vendor-core/</code> (gitignored) is rebuilt only at <code>prepack</code>. In the monorepo the bins load <code>packages/core/src/load-vendored.js</code>, which uses a leftover <code>vendor-core/</code> only while its <code>core-source-hash.txt</code> matches <code>packages/core/src</code>; otherwise it warns on stderr and runs the source.</li><li>The CLI's <code>export-template/</code> has no such check. Without it, <code>--export-html</code> from source uses the repo-root <code>dist-export/</code> from <code>npm run build:export</code> (not produced by <code>npm run build</code>); a leftover <code>export-template/</code> wins over that, so rebuild it with <code>node scripts/vendor-export-template.mjs</code> or delete it before trusting a local <code>--export-html</code>.</li></ul><h2 id="the-docs-site-s-anchor-clicks" tabindex="-1">The docs site's anchor clicks <a class="header-anchor" href="#the-docs-site-s-anchor-clicks" aria-label="Permalink to "The docs site's anchor clicks""></a></h2><p>VitePress installs a <code>window</code>-level, capture-phase click listener (<code>node_modules/vitepress/dist/client/app/router.js</code>) that catches clicks on any <code><a href="#..."></code> pointing at the current page and scrolls to the target itself, before a handler the theme adds on that element runs, so <code>preventDefault()</code> there is too late. An interactive control built over in-page anchors, such as the footnote-marker popovers in <code>docs-site/.vitepress/theme/citation-chips.ts</code>, must drop the element's <code>href</code> and restore <code>role</code>, <code>tabindex</code> and keyboard handling by hand.</p><p>That router scrolls with its own offset, while a deep link, and every click on the published tuning reference (which ships without hydration), is a native fragment jump that follows <code>scroll-padding-top</code>. Both read one measured value: <code>docs-site/.vitepress/anchor-offset.js</code>, inlined into every page's head, sizes it from the pinned chrome on screen, which includes the Shuffle Works product bar on the published site. Never write that bar's marker attribute literally in the docs source, prose included: the hub skips injecting its bar into any built page that contains it.</p><h2 id="driving-the-app-in-a-browser" tabindex="-1">Driving the app in a browser <a class="header-anchor" href="#driving-the-app-in-a-browser" aria-label="Permalink to "Driving the app in a browser""></a></h2><ul><li>Sample logs: download one from a Spark History Server with <code>curl -o app.zip "<baseUrl>/api/v1/applications/<appId>/logs"</code> and drop the zip in as-is (single-file and rolling logs both unwrap), or the <code>.zstd</code> event log inside it. Private logs follow the naming rule in <a href="./testing.html">Testing</a>.</li><li>Loading a file under Playwright: the drop zone prefers <code>showOpenFilePicker</code>, a native dialog that localhost Chromium can't drive. Run <code>await page.evaluate(() => delete window.showOpenFilePicker)</code> before clicking <strong>Choose file</strong>, then <code>setInputFiles</code> on <code>[data-testid=file-input]</code> or handle the <code>filechooser</code> event.</li><li>Run comparison: on the landing page, click <strong>Compare two runs</strong>, fill the <strong>Baseline</strong> and <strong>Candidate</strong> slots the same way (each has its own <code>[data-testid=file-input]</code>), then click <strong>Compare</strong>. Both runs parse one after the other through the one worker. From a run's dashboard, the topbar's <strong>Compare with another run</strong> opens the same view with that run in the baseline slot (a <code>cached</code> <code>RunSource</code>, not parsed again). <strong>View baseline/candidate dashboard</strong> drills into one run, and <strong>← Back to comparison</strong> returns.</li><li>The landing page's <strong>Try a sample run</strong> (<code>src/view/DropZone.tsx</code>) loads a gzip-compressed corpus log from <code>public/sample-runs/</code>: the corpus candidate with the most findings under <code>sparkforensics-analyze --format json</code>. Rerun that scan over every candidate, against the current corpus, before swapping the bundled sample.</li><li>Screenshots on a PR: the <code>gh</code> token can't use GitHub's browser-only attachment uploader. Commit the PNGs to a throwaway asset branch and embed them by commit SHA as <code>https://raw.githubusercontent.com/<owner>/<repo>/<sha>/<path>.png</code>. The repo is public, so every reader's browser loads them, and the binaries stay out of the feature diff.</li></ul>`,24)])])}const u=s(o,[["render",n]]);export{k as __pageData,u as default};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
import{_ as s,o as t,c as i,a5 as a}from"./chunks/framework.DSg0KOwT.js";const k=JSON.parse('{"title":"Development setup","description":"","frontmatter":{},"headers":[],"relativePath":"contributor-guide/development-setup.md","filePath":"contributor-guide/development-setup.md"}'),o={name:"contributor-guide/development-setup.md"};function n(d,e,r,c,p,h){return t(),i("div",null,[...e[0]||(e[0]=[a("",24)])])}const u=s(o,[["render",n]]);export{k as __pageData,u as default};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
import{_ as t,o,c as s,a5 as a}from"./chunks/framework.DSg0KOwT.js";const u=JSON.parse('{"title":"Testing & verification","description":"","frontmatter":{},"headers":[],"relativePath":"contributor-guide/testing.md","filePath":"contributor-guide/testing.md"}'),d={name:"contributor-guide/testing.md"};function c(r,e,i,n,l,p){return o(),s("div",null,[...e[0]||(e[0]=[a('<h1 id="testing-verification" tabindex="-1">Testing & verification <a class="header-anchor" href="#testing-verification" aria-label="Permalink to "Testing & verification""></a></h1><p>Before opening a PR:</p><ol><li>Run the full suite: <code>npm test</code> (root), plus <code>npm run test:core</code>, <code>npm run test:cli</code>, <code>npm run test:mcp</code> and <code>npm run test:server</code> for the packages you touched. Each of those four packages has its own Vitest config and suite, none of them covered by the root <code>npm test</code>. Run <code>npm run build</code> first on a fresh checkout: <code>tests/vite-entry.test.js</code> checks the production bundle in <code>dist/</code>.</li><li>Typecheck: <code>npx tsc --noEmit</code>.</li><li>Lint: <code>npm run lint</code>. A pre-commit hook (<code>.githooks/pre-commit</code>) runs eslint on staged <code>.js</code>/<code>.mjs</code> files only, so run the full lint yourself before committing. Neither lints <code>.ts</code>/<code>.tsx</code> (see <code>eslint.config.js</code>): <code>tsc</code> is their only check.</li><li>For any UI change, verify it in a real browser (Playwright or headless Chrome). Unit and jsdom tests check code correctness; they don't prove a feature works end-to-end. Drive the app, exercise the golden path and the edge cases, and look for regressions elsewhere on the dashboard.</li><li>Report the passing test count when you're done, not "tests pass."</li></ol><p>Sample Spark event logs for manual testing live outside this repo. Keep them in a sibling <code>../spark-log-examples/</code> directory: real <code>.zstd</code> event logs plus several <code>run-compare-*-baseline-*</code>/<code>run-compare-*-candidate-*</code> pairs for run comparison. They are private and this repo is public: never put their file names, app IDs, app names, tables or paths in tracked files, commits or PR text. Use the neutral <code>private-log-NN</code> labels under <a href="#test-fixtures">Test fixtures</a>, and synthetic IDs such as <code>application_0000000000000_0001</code>.</p><h2 id="testing-layout" tabindex="-1">Testing layout <a class="header-anchor" href="#testing-layout" aria-label="Permalink to "Testing layout""></a></h2><p>Vitest throughout. Core logic tests (<code>packages/core/test/*.test.js</code>, covering parser, analyzer, detectors, format-utils, etc.) run under Node via <code>npm run test:core</code>. The root <code>npm test</code> runs two environments in one run: root-level tooling tests (<code>tests/*.test.js</code>, <code>tests/*.test.ts</code>) and pure view-logic tests (<code>tests/view/*.test.ts</code>) under Node, and component tests (<code>tests/view/*.test.tsx</code>), which declare <code>// @vitest-environment jsdom</code> and use React Testing Library (<code>render</code>/<code>screen</code>/<code>userEvent</code>). Mount components through React Testing Library, not hand-rolled DOM code.</p><p><code>packages/cli</code>, <code>packages/mcp</code> and <code>packages/server</code> are separate npm packages, each with its own Vitest config and suite, not run by the root suite. Each has a pack-and-spawn test (for example <code>packages/server/test/bin.test.js</code>) that packs and installs the real published tarball and spawns the real <code>.bin</code> entry point, exercising the production build end-to-end.</p><p><code>npm run test:core</code>, <code>test:cli</code> and <code>test:mcp</code> run with v8 coverage; CI uploads every suite's lcov to Coveralls. No coverage threshold is enforced (see the comment in <code>vitest.config.js</code>).</p><p>Two checks guard invariants that a future edit could silently break. The <code>detector contract</code> suite in <code>packages/core/test/analyzer.test.js</code> asserts every <code>DETECTORS</code> entry's contract fields, that <code>detectorInfoByType</code> covers exactly the emitted types, that each entry's thresholds are frozen, and that the only <code>suppressedBy</code> pair is <code>stageSlowness</code> -> <code>slowHost</code>, both stage-scope (see <a href="./architecture/detector-contract.html#cross-detector-suppression">Cross-detector suppression</a>; declaration order does not matter). <code>REGISTRY</code> and <code>FINDING_PRESENTATION</code> completeness is a compile-time check, not a test: both are typed against the emitted <code>FindingType</code> union, so <code>npx tsc --noEmit</code> fails on a missing or extra key (see <a href="./architecture/detector-contract.html#detector-contract">Detector contract</a>).</p><h2 id="test-fixtures" tabindex="-1">Test fixtures <a class="header-anchor" href="#test-fixtures" aria-label="Permalink to "Test fixtures""></a></h2><p>Real Spark event logs from private workloads live under <code>examples/</code>, gitignored: they're large, and their file names and contents identify the jobs that produced them, so they never enter this public repo. Tests, docs and scripts refer to them only by the neutral labels below. A test that reads one skips when its file is missing, so the suite passes without any of them. To run those tests, copy or symlink your own logs into <code>examples/</code> under these names:</p><table tabindex="0"><thead><tr><th>Label</th><th>Log it stands for</th><th>Used by</th></tr></thead><tbody><tr><td><code>private-log-01.zstd</code></td><td>~12 MB zstd run with Plan Advisor findings</td><td><code>impact-estimator-real-log.test.js</code>, <code>plan-node-stage-mapping-real-log.test.js</code>, spot-checks in <a href="./architecture/impact-estimation.html">Impact estimation</a></td></tr><tr><td><code>private-log-02.zstd</code></td><td>~2 MB zstd run with a heavy shuffle-and-spill stage</td><td><code>plan-node-stage-mapping-real-log.test.js</code>, spot-checks in <a href="./architecture/impact-estimation.html">Impact estimation</a></td></tr><tr><td><code>private-log-03.zstd</code></td><td>~28 MB zstd run with about 1,200 stages</td><td>timing spot-check in <a href="./architecture/impact-estimation.html">Impact estimation</a></td></tr><tr><td><code>private-log-04.zstd</code></td><td>~9 MB zstd run whose stage 394 plan became <code>plan-graph-stage-394.json</code></td><td><code>scripts/extract-plan-fixture.mjs</code></td></tr><tr><td><code>private-log-05</code></td><td>~76 MB uncompressed NDJSON run, 62 stages</td><td><code>parser-worker.test.js</code>; also a fast end-to-end smoke test</td></tr><tr><td><code>private-log-06</code></td><td>~1.3 GB uncompressed NDJSON run</td><td><code>detectors-plan.test.js</code>; also a stress test for the Web Worker streaming path</td></tr><tr><td><code>private-log-07</code></td><td>~120 MB log from a Spark History Server download</td><td>none: check the <code>LZ4Block</code> codec decoder by hand; <code>packages/core/test/lz4-block.test.js</code> uses a small hardcoded byte array instead</td></tr></tbody></table><p><code>scripts/extract-plan-fixture.mjs</code> anonymizes every table, column, path and literal before it writes the committed fixture; keep it that way if you extract another one.</p><p>Of the finding types <code>retryWaste</code>, <code>autoscalingChurn</code>, <code>failures</code>, <code>stageFailed</code> and <code>jobFailureRate</code>, only <code>retryWaste</code> and <code>autoscalingChurn</code> fire on any of the real logs in <code>../spark-log-examples/</code>. <code>failures</code>, <code>stageFailed</code>, and <code>jobFailureRate</code> fire on none of them; they are covered by unit tests only (<code>tests/view/task-failures.test.tsx</code>, <code>tests/view/stage-failed.test.tsx</code>, <code>tests/view/job-failures.test.tsx</code>), not against a real log.</p><p><code>dev/log-corpus/</code> is a git submodule pointing at the public <code>spark-event-corpus-data</code> repo, pinned to a tag. It backs the corpus-backed tests in <code>packages/core/test/</code> (for example <code>list-runs.test.js</code>, <code>task-failure.test.js</code>) and <code>packages/server/test/shs-proxy-fixture.test.js</code> (which loads a real <code>*-parquet-baseline.ndjson</code> fixture and asserts the <code>/shs-proxy</code> route streams it back unmodified), plus the <a href="#corpus-regression-snapshot">corpus regression snapshot</a>. Populate it locally with <code>git submodule update --init dev/log-corpus</code>; without it, those tests skip and the snapshot check has no logs (see <a href="./development-setup.html">Development setup</a>). CI's <code>core</code>, <code>server</code> and <code>node18</code> jobs check it out.</p><h3 id="corpus-regression-snapshot" tabindex="-1">Corpus regression snapshot <a class="header-anchor" href="#corpus-regression-snapshot" aria-label="Permalink to "Corpus regression snapshot""></a></h3><p><code>dev/corpus-snapshot.json</code> records every finding (type, location, band, confidence, value, impact estimate) that the detectors produce on each public corpus log, plus its stage, SQL execution and skipped-line counts, with timings and memory left out so it is deterministic. CI's <code>core</code> job re-analyzes the corpus and fails on any difference, printing the added, removed, re-banded and re-estimated findings and changed counts per log:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">node</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> dev/bench-analyze.mjs</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --check</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> dev/corpus-snapshot.json</span></span></code></pre></div><p>When a detector, threshold or estimator change is meant to alter findings, read that diff, confirm each change is the one you intended, then refresh the snapshot and commit it with the change:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">node</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> dev/bench-analyze.mjs</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --update</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> dev/corpus-snapshot.json</span></span></code></pre></div><p>Both default to <code>dev/log-corpus/logs</code> and its <code>external/</code> folder. Only public corpus logs belong in the snapshot: never pass private logs to <code>--update</code>.</p><h3 id="detector-and-estimate-tools" tabindex="-1">Detector and estimate tools <a class="header-anchor" href="#detector-and-estimate-tools" aria-label="Permalink to "Detector and estimate tools""></a></h3><p>Record before and after numbers from these in the commit that changes a detector, threshold, estimate or the vendored fzstd.</p><ul><li><code>node dev/bench-analyze.mjs [--repeat N] --out snap.json <file|dir>...</code> snapshots every finding (band, estimate) plus parse and analyze timings, one child process per log. <code>--diff a.json b.json [--verbose]</code> shows the findings added, removed or re-banded between two snapshots.</li><li><code>node --max-old-space-size=12000 dev/eval-tail-replay.mjs [--set detector.threshold=value] [--verbose] <file|dir>...</code> scores skew and straggler against a task-level replay (precision, recall, estimate error). <code>--verbose</code> lists each miss, false positive and estimate more than 2x off.</li><li><code>node dev/fuzz-fzstd.mjs --upstream <pristine fzstd esm/index.mjs> <log.zstd>...</code> checks the locally patched <code>packages/core/src/vendor/fzstd.js</code> against upstream, on whole logs and on randomly corrupted prefixes. Run it after any fzstd edit.</li></ul><p>CI also runs a <code>node18</code> job, because the published <code>cli</code>, <code>mcp</code> and <code>server</code> packages declare <code>engines.node >=18</code> while vitest needs Node 22+. It packs the three tarballs on <code>.nvmrc</code>'s Node, installs them on Node 18, runs the CLI over every corpus log plus a zstd copy of one, checks that the MCP server answers <code>initialize</code>, and starts <code>sparkforensics-server</code> to check it serves the app and answers <code>initialize</code> on <code>/mcp</code>.</p><h2 id="test-suite-growth-discipline" tabindex="-1">Test-suite growth discipline <a class="header-anchor" href="#test-suite-growth-discipline" aria-label="Permalink to "Test-suite growth discipline""></a></h2><p>Widget and detector test files collect copy-pasted boilerplate fast. Before adding a widget test file, or a per-widget or per-detector test:</p><ul><li>Cross-cutting widget behavior (impact/stage sort-toggle default and flip, density-gated visibility, 6-at-a-time pagination) belongs in a shared helper under <code>tests/view/_shared/</code>, called from each widget's test file. It holds <code>testFinding</code> (<code>finding.ts</code>), <code>installInterpretation</code> and the sort-order helpers (<code>sort-order-toggle.ts</code>); add a density or pagination helper there the first time two widget tests would share one. Extract into it only when a test body is identical to an existing one apart from the widget name, fixture or label. A widget with a real difference (a <code>React.lazy</code>-loaded chunk, a custom row finder) keeps its own test.</li><li><code>packages/core/test/fixtures/</code> holds the shared stage and app fixture factories (<code>makeStage</code>/<code>makeApp</code>). Import them instead of pasting a local copy into a new core test file.</li><li>Before adding a small test file, check whether a sibling file's <code>describe</code> block already covers that surface. A detector-catalog shape check, for example, belongs in <code>analyzer.test.js</code>'s <code>detector contract</code> block.</li><li>Calling an exported pure function directly, or calling the documented event reducer (<code>processEvent</code> in <code>packages/core/src/event-handlers.ts</code>, re-exported from <code>parser-worker.ts</code>; see the <a href="./architecture/worker-protocol.html">worker protocol</a>) with one event and inspecting the state, is normal unit testing. Don't rewrite reducer-style tests into NDJSON-through-<code>runParse</code> integration tests to avoid touching internals: that costs readability for no real safety.</li></ul>',28)])])}const g=t(d,[["render",c]]);export{u as __pageData,g as default};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{_ as t,o,c as s,a5 as a}from"./chunks/framework.DSg0KOwT.js";const u=JSON.parse('{"title":"Testing & verification","description":"","frontmatter":{},"headers":[],"relativePath":"contributor-guide/testing.md","filePath":"contributor-guide/testing.md"}'),d={name:"contributor-guide/testing.md"};function
|
|
1
|
+
import{_ as t,o,c as s,a5 as a}from"./chunks/framework.DSg0KOwT.js";const u=JSON.parse('{"title":"Testing & verification","description":"","frontmatter":{},"headers":[],"relativePath":"contributor-guide/testing.md","filePath":"contributor-guide/testing.md"}'),d={name:"contributor-guide/testing.md"};function c(r,e,i,n,l,p){return o(),s("div",null,[...e[0]||(e[0]=[a("",28)])])}const g=t(d,[["render",c]]);export{u as __pageData,g as default};
|