sparkforensics-server 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -0
- package/index.js +40 -49
- package/lib/static-files.js +10 -6
- package/package.json +20 -1
- package/public/assets/AutoscalingChurn-CFJkiqxW.js +1 -0
- package/public/assets/CacheUtilization-kTKZBzDA.js +1 -0
- package/public/assets/CachingOpportunity-CU70EWcF.js +1 -0
- package/public/assets/ColdStart-hOI-WOvm.js +1 -0
- package/public/assets/ConfigAudit-B4oAksZY.js +1 -0
- package/public/assets/CoreUsageArea-D-dR3SQk.js +1 -0
- package/public/assets/Dashboard-MDhohsq4.js +4 -0
- package/public/assets/DuplicatePlanSubtree-54RKsfI7.js +1 -0
- package/public/assets/DurationHistogram-CviQyPNM.js +1 -0
- package/public/assets/EmptyState-v2ehFncc.js +1 -0
- package/public/assets/ExecutorUtilization-vHyFHgL6.js +1 -0
- package/public/assets/GcPressure-DINuhcno.js +1 -0
- package/public/assets/ImpactEstimate-C6GzFOY9.js +1 -0
- package/public/assets/IncompleteRun-a1kuCnU6.js +1 -0
- package/public/assets/JobFailures-CYwPfUhR.js +1 -0
- package/public/assets/MemoryUtilization-Dlfj0e_f.js +1 -0
- package/public/assets/OverBroadcast-BMu7Y8l9.js +1 -0
- package/public/assets/PartitionSizing-CvMGfSVB.js +1 -0
- package/public/assets/PlanExplorer-xYFme_Rx.js +1 -0
- package/public/assets/PlanGraphRoute-a3OGg6ET.js +1 -0
- package/public/assets/RetryWaste-JKRjOZ-J.js +1 -0
- package/public/assets/RowPagination-DypWbnic.js +1 -0
- package/public/assets/ShuffleIO-Mi8HiVu-.js +1 -0
- package/public/assets/Skew-C3iOScTC.js +1 -0
- package/public/assets/SlowHost-DKBg18v-.js +1 -0
- package/public/assets/SmallFiles-CCCB6AIu.js +1 -0
- package/public/assets/SpeculationWaste-DUNupw0_.js +1 -0
- package/public/assets/Spill-BlLdatik.js +1 -0
- package/public/assets/StageFailed-qtY7C7Yo.js +1 -0
- package/public/assets/StageFindingGroup-CoVdEZYs.js +1 -0
- package/public/assets/StageHeader-uDBLHgdg.js +1 -0
- package/public/assets/StagePill-CYCMdlIx.js +1 -0
- package/public/assets/StageShape-zs0mAjQJ.js +1 -0
- package/public/assets/StageSlowness-C7vg0u5Q.js +1 -0
- package/public/assets/Straggler-4BRGAQP4.js +1 -0
- package/public/assets/TaskFailures-CSjO23Kg.js +1 -0
- package/public/assets/TinyTask-6Zuovcn5.js +1 -0
- package/public/assets/UnderBroadcast-CePjo0ZV.js +1 -0
- package/public/assets/WidgetLeadSummary-DRNRVvn7.js +1 -0
- package/public/assets/{charts-vendor-NMqKV0H4.js → charts-vendor-Cf1-wzeH.js} +2 -2
- package/public/assets/downsample-D8XmJk6i.js +1 -0
- package/public/assets/finding-anchor-4TmpXd7j.js +1 -0
- package/public/assets/impact-sort-Cq6KOLqh.js +1 -0
- package/public/assets/inbox-DBFovbUX.js +1 -0
- package/public/assets/index-CHjV2-jB.js +9 -0
- package/public/assets/index-DMEtaP4u.css +1 -0
- package/public/assets/parser-worker-BMPPE37w.js +64 -0
- package/public/assets/plan-graph-vendor-B96a-e4N.js +7 -0
- package/public/assets/react-vendor-DA6opS-q.js +9 -0
- package/public/assets/select-D5fK-za0.js +1 -0
- package/public/assets/toggle-group-lbIXRG0W.js +1 -0
- package/public/assets/usePagedRows-CFoZrPr4.js +1 -0
- package/public/assets/useRecentFiles-BPGQ6XII.js +13 -0
- package/public/assets/useSortMode-BOPpfLhJ.js +1 -0
- package/public/docs/404.html +3 -3
- package/public/docs/assets/{app.Bxy4_HVA.js → app.BgzsaTS3.js} +1 -1
- package/public/docs/assets/aqe-loop.IwQSATHw.svg +1 -0
- package/public/docs/assets/aqe-loop.dark.DGbaxqJE.svg +1 -0
- package/public/docs/assets/broadcast-vs-shuffle.Db4WY1XK.svg +1 -0
- package/public/docs/assets/broadcast-vs-shuffle.dark.C7Bxs0mG.svg +1 -0
- package/public/docs/assets/cache-lifecycle.dark.B-hS7AgU.svg +1 -0
- package/public/docs/assets/cache-lifecycle.rEOVYQNU.svg +1 -0
- package/public/docs/assets/chunks/@localSearchIndexroot.PQ5JepCp.js +1 -0
- package/public/docs/assets/chunks/{VPLocalSearchBox.L5jNOIR3.js → VPLocalSearchBox.DOq3t-On.js} +1 -1
- package/public/docs/assets/chunks/duplicate-plan-subtree.dark.Cdp70QhV.js +1 -0
- package/public/docs/assets/chunks/{framework.CNDaxUod.js → framework.DSg0KOwT.js} +1 -1
- package/public/docs/assets/chunks/retry-escalation-ladder.dark.DHipdJgZ.js +1 -0
- package/public/docs/assets/chunks/theme.BP14rGBx.js +2 -0
- package/public/docs/assets/cold-start-timeline.DxC_Sc7w.svg +1 -0
- package/public/docs/assets/cold-start-timeline.dark.CZ17YcAG.svg +1 -0
- package/public/docs/assets/columnar-layout.PghGeOEA.svg +1 -0
- package/public/docs/assets/columnar-layout.dark.BVNlz0ff.svg +1 -0
- package/public/docs/assets/container-memory.DIO0AnIm.svg +1 -0
- package/public/docs/assets/container-memory.dark.CP-5zuCl.svg +1 -0
- package/public/docs/assets/contributor-guide_architecture_board-widgets.md.B-OsL91z.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_board-widgets.md.B-OsL91z.lean.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_detector-contract.md.BOeH4d1J.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_detector-contract.md.BOeH4d1J.lean.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_drill-down.md.BtPdlM7r.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_drill-down.md.BtPdlM7r.lean.js +1 -0
- package/public/docs/assets/{contributor-guide_architecture_impact-estimation.md.CzlXOnCq.js → contributor-guide_architecture_impact-estimation.md.DYCDPgkh.js} +1 -1
- package/public/docs/assets/{contributor-guide_architecture_impact-estimation.md.CzlXOnCq.lean.js → contributor-guide_architecture_impact-estimation.md.DYCDPgkh.lean.js} +1 -1
- package/public/docs/assets/{contributor-guide_architecture_index.md.94lNbF-4.js → contributor-guide_architecture_index.md.3TO9ic6w.js} +1 -1
- package/public/docs/assets/{contributor-guide_architecture_index.md.94lNbF-4.lean.js → contributor-guide_architecture_index.md.3TO9ic6w.lean.js} +1 -1
- package/public/docs/assets/{contributor-guide_architecture_overview.md.mQFlN66o.js → contributor-guide_architecture_overview.md.CehiRmGn.js} +1 -1
- package/public/docs/assets/{contributor-guide_architecture_overview.md.mQFlN66o.lean.js → contributor-guide_architecture_overview.md.CehiRmGn.lean.js} +1 -1
- package/public/docs/assets/contributor-guide_architecture_state-and-history.md.m3S3UdMk.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_state-and-history.md.m3S3UdMk.lean.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.DbqPf2OT.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.DbqPf2OT.lean.js +1 -0
- package/public/docs/assets/{contributor-guide_architecture_worker-protocol.md.BaS_L10g.js → contributor-guide_architecture_worker-protocol.md.B93qJ_tT.js} +2 -2
- package/public/docs/assets/{contributor-guide_architecture_worker-protocol.md.BaS_L10g.lean.js → contributor-guide_architecture_worker-protocol.md.B93qJ_tT.lean.js} +1 -1
- package/public/docs/assets/contributor-guide_contributing.md.CvRsdr6J.js +1 -0
- package/public/docs/assets/contributor-guide_contributing.md.CvRsdr6J.lean.js +1 -0
- package/public/docs/assets/{contributor-guide_development-setup.md.CpUMVtIK.js → contributor-guide_development-setup.md.DvAN_9mK.js} +1 -1
- package/public/docs/assets/{contributor-guide_development-setup.md.CpUMVtIK.lean.js → contributor-guide_development-setup.md.DvAN_9mK.lean.js} +1 -1
- package/public/docs/assets/{contributor-guide_testing.md.C0q-BJfl.js → contributor-guide_testing.md.6rIKqSyY.js} +1 -1
- package/public/docs/assets/{contributor-guide_testing.md.C0q-BJfl.lean.js → contributor-guide_testing.md.6rIKqSyY.lean.js} +1 -1
- package/public/docs/assets/dag-stages.DSz_S937.svg +1 -0
- package/public/docs/assets/dag-stages.dark.F72UzxH4.svg +1 -0
- package/public/docs/assets/driver-executor.D5pQ7YN1.svg +1 -0
- package/public/docs/assets/driver-executor.dark.BmX9cPvh.svg +1 -0
- package/public/docs/assets/duplicate-plan-subtree.B4cvN6fj.svg +1 -0
- package/public/docs/assets/duplicate-plan-subtree.dark.Dw8wS0Ag.svg +1 -0
- package/public/docs/assets/{index.md.CETnDZzR.js → index.md.CHJVslga.js} +1 -1
- package/public/docs/assets/{index.md.CETnDZzR.lean.js → index.md.CHJVslga.lean.js} +1 -1
- package/public/docs/assets/join-strategy.C_FvrCEo.svg +1 -0
- package/public/docs/assets/join-strategy.dark.ChMLnNII.svg +1 -0
- package/public/docs/assets/memory-borrowing.BqQRJg0u.svg +1 -0
- package/public/docs/assets/memory-borrowing.dark.Yhh20O9C.svg +1 -0
- package/public/docs/assets/memory-regions.XHvO7jHG.svg +1 -0
- package/public/docs/assets/memory-regions.dark.D4TP9_08.svg +1 -0
- package/public/docs/assets/repartition-vs-coalesce.BovLRrpj.svg +1 -0
- package/public/docs/assets/repartition-vs-coalesce.dark.BhAczKZQ.svg +1 -0
- package/public/docs/assets/retry-escalation-ladder.DyTKJJmZ.svg +1 -0
- package/public/docs/assets/retry-escalation-ladder.dark.BdsabtU3.svg +1 -0
- package/public/docs/assets/shuffle-map-reduce.KuOEZVmg.svg +1 -0
- package/public/docs/assets/shuffle-map-reduce.dark.BgQZnFSb.svg +1 -0
- package/public/docs/assets/spill-classification.BU2euYDO.svg +1 -0
- package/public/docs/assets/spill-classification.dark.D7i1M40d.svg +1 -0
- package/public/docs/assets/{style.D2mY0HUs.css → style.DXOMCXxn.css} +1 -1
- package/public/docs/assets/tuning-reference_anti-patterns.md.Df1YMIHu.js +1 -0
- package/public/docs/assets/tuning-reference_anti-patterns.md.Df1YMIHu.lean.js +1 -0
- package/public/docs/assets/tuning-reference_aqe.md.BIsCtLzm.js +1 -0
- package/public/docs/assets/tuning-reference_aqe.md.BIsCtLzm.lean.js +1 -0
- package/public/docs/assets/tuning-reference_bottleneck-broadcast-sizing.md.CEstB3Ia.js +1 -0
- package/public/docs/assets/tuning-reference_bottleneck-broadcast-sizing.md.CEstB3Ia.lean.js +1 -0
- package/public/docs/assets/tuning-reference_bottleneck-cold-start.md.CEuy-72y.js +7 -0
- package/public/docs/assets/tuning-reference_bottleneck-cold-start.md.CEuy-72y.lean.js +1 -0
- package/public/docs/assets/tuning-reference_bottleneck-duplicate-plan-subtree.md.CIohQDfn.js +1 -0
- package/public/docs/assets/tuning-reference_bottleneck-duplicate-plan-subtree.md.CIohQDfn.lean.js +1 -0
- package/public/docs/assets/tuning-reference_bottleneck-failures.md.4z5BXGJ2.js +6 -0
- package/public/docs/assets/tuning-reference_bottleneck-failures.md.4z5BXGJ2.lean.js +1 -0
- package/public/docs/assets/tuning-reference_bottleneck-gc.md.DSxzZRK7.js +6 -0
- package/public/docs/assets/tuning-reference_bottleneck-gc.md.DSxzZRK7.lean.js +1 -0
- package/public/docs/assets/tuning-reference_bottleneck-job-failure-rate.md.BaJl__1W.js +8 -0
- package/public/docs/assets/tuning-reference_bottleneck-job-failure-rate.md.BaJl__1W.lean.js +1 -0
- package/public/docs/assets/tuning-reference_bottleneck-memory-utilization.md.DbP-SJZc.js +7 -0
- package/public/docs/assets/tuning-reference_bottleneck-memory-utilization.md.DbP-SJZc.lean.js +1 -0
- package/public/docs/assets/tuning-reference_bottleneck-retry-waste.md.D5JMjVOt.js +1 -0
- package/public/docs/assets/tuning-reference_bottleneck-retry-waste.md.D5JMjVOt.lean.js +1 -0
- package/public/docs/assets/tuning-reference_bottleneck-shuffle.md.CM-nTmIH.js +12 -0
- package/public/docs/assets/tuning-reference_bottleneck-shuffle.md.CM-nTmIH.lean.js +1 -0
- package/public/docs/assets/tuning-reference_bottleneck-skew.md.BdUwiDhn.js +14 -0
- package/public/docs/assets/tuning-reference_bottleneck-skew.md.BdUwiDhn.lean.js +1 -0
- package/public/docs/assets/tuning-reference_bottleneck-slow-host.md.BlIo6UDW.js +7 -0
- package/public/docs/assets/tuning-reference_bottleneck-slow-host.md.BlIo6UDW.lean.js +1 -0
- package/public/docs/assets/tuning-reference_bottleneck-small-files.md.B8kloyx8.js +5 -0
- package/public/docs/assets/tuning-reference_bottleneck-small-files.md.B8kloyx8.lean.js +1 -0
- package/public/docs/assets/tuning-reference_bottleneck-spill.md.PNH7mITt.js +6 -0
- package/public/docs/assets/tuning-reference_bottleneck-spill.md.PNH7mITt.lean.js +1 -0
- package/public/docs/assets/tuning-reference_bottleneck-straggler.md.DY36fHN5.js +7 -0
- package/public/docs/assets/tuning-reference_bottleneck-straggler.md.DY36fHN5.lean.js +1 -0
- package/public/docs/assets/tuning-reference_bottleneck-tiny-tasks.md.QTV7O8kU.js +8 -0
- package/public/docs/assets/tuning-reference_bottleneck-tiny-tasks.md.QTV7O8kU.lean.js +1 -0
- package/public/docs/assets/tuning-reference_bottleneck-utilization.md.DTiueZC3.js +5 -0
- package/public/docs/assets/tuning-reference_bottleneck-utilization.md.DTiueZC3.lean.js +1 -0
- package/public/docs/assets/tuning-reference_caching.md.B7aQ8asB.js +1 -0
- package/public/docs/assets/tuning-reference_caching.md.B7aQ8asB.lean.js +1 -0
- package/public/docs/assets/tuning-reference_cluster-config.md.ZVmDGsQ3.js +1 -0
- package/public/docs/assets/tuning-reference_cluster-config.md.ZVmDGsQ3.lean.js +1 -0
- package/public/docs/assets/tuning-reference_config.md.UvveiWG3.js +1 -0
- package/public/docs/assets/tuning-reference_config.md.UvveiWG3.lean.js +1 -0
- package/public/docs/assets/tuning-reference_data-formats.md.bjCAWH3N.js +1 -0
- package/public/docs/assets/tuning-reference_data-formats.md.bjCAWH3N.lean.js +1 -0
- package/public/docs/assets/tuning-reference_index.md.BQ_NooMV.js +1 -0
- package/public/docs/assets/tuning-reference_index.md.BQ_NooMV.lean.js +1 -0
- package/public/docs/assets/tuning-reference_intro.md.CobD-lGB.js +1 -0
- package/public/docs/assets/tuning-reference_intro.md.CobD-lGB.lean.js +1 -0
- package/public/docs/assets/tuning-reference_joins.md.BtKs_CuW.js +1 -0
- package/public/docs/assets/tuning-reference_joins.md.BtKs_CuW.lean.js +1 -0
- package/public/docs/assets/tuning-reference_memory-model.md.DhT-n4y3.js +1 -0
- package/public/docs/assets/tuning-reference_memory-model.md.DhT-n4y3.lean.js +1 -0
- package/public/docs/assets/tuning-reference_metrics.md.mLOh7Apj.js +1 -0
- package/public/docs/assets/tuning-reference_metrics.md.mLOh7Apj.lean.js +1 -0
- package/public/docs/assets/tuning-reference_partitioning.md.q0zKF_8X.js +1 -0
- package/public/docs/assets/tuning-reference_partitioning.md.q0zKF_8X.lean.js +1 -0
- package/public/docs/assets/tuning-reference_pyspark.md.DDCfvN9t.js +6 -0
- package/public/docs/assets/tuning-reference_pyspark.md.DDCfvN9t.lean.js +1 -0
- package/public/docs/assets/tuning-reference_shuffle.md.BZZ7R4Ix.js +1 -0
- package/public/docs/assets/tuning-reference_shuffle.md.BZZ7R4Ix.lean.js +1 -0
- package/public/docs/assets/tuning-reference_spark-architecture.md.Dwzm5avO.js +1 -0
- package/public/docs/assets/tuning-reference_spark-architecture.md.Dwzm5avO.lean.js +1 -0
- package/public/docs/assets/tuning-reference_table-formats.md.D6wj-2dX.js +1 -0
- package/public/docs/assets/tuning-reference_table-formats.md.D6wj-2dX.lean.js +1 -0
- package/public/docs/assets/udf-execution-models.BUFDICuG.svg +1 -0
- package/public/docs/assets/udf-execution-models.dark.YTNS6GDq.svg +1 -0
- package/public/docs/assets/user-guide_alternative-log-retrieval.md.sU3KGarf.js +1 -0
- package/public/docs/assets/user-guide_alternative-log-retrieval.md.sU3KGarf.lean.js +1 -0
- package/public/docs/assets/user-guide_getting-started.md.DtEM37MK.js +3 -0
- package/public/docs/assets/user-guide_getting-started.md.DtEM37MK.lean.js +1 -0
- package/public/docs/assets/{user-guide_mcp-tools.md.DwYI1d6C.js → user-guide_mcp-tools.md.C8MiIu7F.js} +35 -5
- package/public/docs/assets/{user-guide_mcp-tools.md.DwYI1d6C.lean.js → user-guide_mcp-tools.md.C8MiIu7F.lean.js} +1 -1
- package/public/docs/assets/{user-guide_run-comparison.md.5-oaOfTu.js → user-guide_run-comparison.md.S0TWWmLY.js} +1 -1
- package/public/docs/assets/{user-guide_run-comparison.md.5-oaOfTu.lean.js → user-guide_run-comparison.md.S0TWWmLY.lean.js} +1 -1
- package/public/docs/assets/user-guide_understanding-findings.md.D0R_Y-R2.js +1 -0
- package/public/docs/assets/{user-guide_understanding-findings.md.CK2WZ0Tf.lean.js → user-guide_understanding-findings.md.D0R_Y-R2.lean.js} +1 -1
- package/public/docs/contributor-guide/architecture/board-widgets.html +7 -7
- package/public/docs/contributor-guide/architecture/detector-contract.html +7 -7
- package/public/docs/contributor-guide/architecture/drill-down.html +7 -7
- package/public/docs/contributor-guide/architecture/impact-estimation.html +7 -7
- package/public/docs/contributor-guide/architecture/index.html +7 -7
- package/public/docs/contributor-guide/architecture/overview.html +7 -7
- package/public/docs/contributor-guide/architecture/state-and-history.html +7 -7
- package/public/docs/contributor-guide/architecture/widget-rendering.html +7 -7
- package/public/docs/contributor-guide/architecture/worker-protocol.html +8 -8
- package/public/docs/contributor-guide/contributing.html +7 -7
- package/public/docs/contributor-guide/development-setup.html +7 -7
- package/public/docs/contributor-guide/testing.html +7 -7
- package/public/docs/hashmap.json +1 -1
- package/public/docs/index.html +7 -7
- package/public/docs/tuning-reference/anti-patterns.html +29 -0
- package/public/docs/tuning-reference/aqe.html +29 -0
- package/public/docs/tuning-reference/bottleneck-broadcast-sizing.html +29 -0
- package/public/docs/tuning-reference/bottleneck-cold-start.html +35 -0
- package/public/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html +30 -0
- package/public/docs/tuning-reference/bottleneck-failures.html +35 -0
- package/public/docs/tuning-reference/bottleneck-gc.html +34 -0
- package/public/docs/tuning-reference/bottleneck-job-failure-rate.html +37 -0
- package/public/docs/tuning-reference/bottleneck-memory-utilization.html +35 -0
- package/public/docs/tuning-reference/bottleneck-retry-waste.html +30 -0
- package/public/docs/tuning-reference/bottleneck-shuffle.html +40 -0
- package/public/docs/tuning-reference/bottleneck-skew.html +42 -0
- package/public/docs/tuning-reference/bottleneck-slow-host.html +35 -0
- package/public/docs/tuning-reference/bottleneck-small-files.html +33 -0
- package/public/docs/tuning-reference/bottleneck-spill.html +34 -0
- package/public/docs/tuning-reference/bottleneck-straggler.html +35 -0
- package/public/docs/tuning-reference/bottleneck-tiny-tasks.html +36 -0
- package/public/docs/tuning-reference/bottleneck-utilization.html +34 -0
- package/public/docs/tuning-reference/caching.html +29 -0
- package/public/docs/tuning-reference/cluster-config.html +29 -0
- package/public/docs/tuning-reference/config.html +29 -0
- package/public/docs/tuning-reference/data-formats.html +29 -0
- package/public/docs/tuning-reference/index.html +29 -0
- package/public/docs/tuning-reference/intro.html +29 -0
- package/public/docs/tuning-reference/joins.html +29 -0
- package/public/docs/tuning-reference/memory-model.html +29 -0
- package/public/docs/tuning-reference/metrics.html +29 -0
- package/public/docs/tuning-reference/partitioning.html +29 -0
- package/public/docs/tuning-reference/pyspark.html +34 -0
- package/public/docs/tuning-reference/shuffle.html +29 -0
- package/public/docs/tuning-reference/spark-architecture.html +29 -0
- package/public/docs/tuning-reference/table-formats.html +29 -0
- package/public/docs/user-guide/alternative-log-retrieval.html +29 -0
- package/public/docs/user-guide/getting-started.html +8 -8
- package/public/docs/user-guide/mcp-tools.html +41 -11
- package/public/docs/user-guide/run-comparison.html +7 -7
- package/public/docs/user-guide/understanding-findings.html +7 -7
- package/public/index.html +6 -6
- package/vendor-core/analyzer.js +74 -74
- package/vendor-core/cli/budgets.js +13 -27
- package/vendor-core/cli/collect-run.js +43 -19
- package/vendor-core/core-count.js +25 -27
- package/vendor-core/core-locality-ratio.js +4 -11
- package/vendor-core/core-time-series.js +6 -12
- package/vendor-core/core-usage-locality.js +3 -4
- package/vendor-core/detectors.js +256 -375
- package/vendor-core/docs-config.js +69 -21
- package/vendor-core/docs-content/chapters/01-intro.md +32 -0
- package/vendor-core/docs-content/chapters/02-spark-architecture.md +76 -0
- package/vendor-core/docs-content/chapters/03-memory-model.md +73 -0
- package/vendor-core/docs-content/chapters/04-partitioning.md +65 -0
- package/vendor-core/docs-content/chapters/05-joins.md +62 -0
- package/vendor-core/docs-content/chapters/06-shuffle.md +59 -0
- package/vendor-core/docs-content/chapters/07-data-formats.md +81 -0
- package/vendor-core/docs-content/chapters/07b-table-formats.md +56 -0
- package/vendor-core/docs-content/chapters/08-caching.md +58 -0
- package/vendor-core/docs-content/chapters/09-pyspark.md +78 -0
- package/vendor-core/docs-content/chapters/10-aqe.md +167 -0
- package/vendor-core/docs-content/chapters/11-cluster-config.md +170 -0
- package/vendor-core/docs-content/chapters/12-anti-patterns.md +171 -0
- package/vendor-core/docs-content/chapters/14-metrics.md +87 -0
- package/vendor-core/docs-content/chapters/15-config.md +93 -0
- package/vendor-core/docs-content/chapters/nav-index.json +370 -0
- package/vendor-core/docs-content/detection/cache.md +6 -0
- package/vendor-core/docs-content/detection/cfg.md +15 -0
- package/vendor-core/docs-content/detection/chrn.md +7 -0
- package/vendor-core/docs-content/detection/cold.md +4 -0
- package/vendor-core/docs-content/detection/cstor.md +4 -0
- package/vendor-core/docs-content/detection/fail.md +5 -0
- package/vendor-core/docs-content/detection/gc.md +4 -0
- package/vendor-core/docs-content/detection/host.md +5 -0
- package/vendor-core/docs-content/detection/incmp.md +6 -0
- package/vendor-core/docs-content/detection/jobs.md +4 -0
- package/vendor-core/docs-content/detection/local.md +7 -0
- package/vendor-core/docs-content/detection/mem.md +10 -0
- package/vendor-core/docs-content/detection/part.md +5 -0
- package/vendor-core/docs-content/detection/plan.md +14 -0
- package/vendor-core/docs-content/detection/retry.md +4 -0
- package/vendor-core/docs-content/detection/sfail.md +5 -0
- package/vendor-core/docs-content/detection/shape.md +5 -0
- package/vendor-core/docs-content/detection/shfl.md +4 -0
- package/vendor-core/docs-content/detection/skew.md +6 -0
- package/vendor-core/docs-content/detection/slow.md +6 -0
- package/vendor-core/docs-content/detection/spec.md +7 -0
- package/vendor-core/docs-content/detection/spill.md +7 -0
- package/vendor-core/docs-content/detection/strag.md +5 -0
- package/vendor-core/docs-content/detection/tiny.md +4 -0
- package/vendor-core/docs-content/detection/util.md +4 -0
- package/vendor-core/docs-content/diagrams/aqe-loop.dark.svg +1 -0
- package/vendor-core/docs-content/diagrams/aqe-loop.svg +1 -0
- package/vendor-core/docs-content/diagrams/broadcast-vs-shuffle.dark.svg +1 -0
- package/vendor-core/docs-content/diagrams/broadcast-vs-shuffle.svg +1 -0
- package/vendor-core/docs-content/diagrams/cache-lifecycle.dark.svg +1 -0
- package/vendor-core/docs-content/diagrams/cache-lifecycle.svg +1 -0
- package/vendor-core/docs-content/diagrams/cold-start-timeline.dark.svg +1 -0
- package/vendor-core/docs-content/diagrams/cold-start-timeline.svg +1 -0
- package/vendor-core/docs-content/diagrams/columnar-layout.dark.svg +1 -0
- package/vendor-core/docs-content/diagrams/columnar-layout.svg +1 -0
- package/vendor-core/docs-content/diagrams/container-memory.dark.svg +1 -0
- package/vendor-core/docs-content/diagrams/container-memory.svg +1 -0
- package/vendor-core/docs-content/diagrams/dag-stages.dark.svg +1 -0
- package/vendor-core/docs-content/diagrams/dag-stages.svg +1 -0
- package/vendor-core/docs-content/diagrams/driver-executor.dark.svg +1 -0
- package/vendor-core/docs-content/diagrams/driver-executor.svg +1 -0
- package/vendor-core/docs-content/diagrams/duplicate-plan-subtree.dark.svg +1 -0
- package/vendor-core/docs-content/diagrams/duplicate-plan-subtree.svg +1 -0
- package/vendor-core/docs-content/diagrams/join-strategy.dark.svg +1 -0
- package/vendor-core/docs-content/diagrams/join-strategy.svg +1 -0
- package/vendor-core/docs-content/diagrams/memory-borrowing.dark.svg +1 -0
- package/vendor-core/docs-content/diagrams/memory-borrowing.svg +1 -0
- package/vendor-core/docs-content/diagrams/memory-regions.dark.svg +1 -0
- package/vendor-core/docs-content/diagrams/memory-regions.svg +1 -0
- package/vendor-core/docs-content/diagrams/repartition-vs-coalesce.dark.svg +1 -0
- package/vendor-core/docs-content/diagrams/repartition-vs-coalesce.svg +1 -0
- package/vendor-core/docs-content/diagrams/retry-escalation-ladder.dark.svg +1 -0
- package/vendor-core/docs-content/diagrams/retry-escalation-ladder.svg +1 -0
- package/vendor-core/docs-content/diagrams/shuffle-map-reduce.dark.svg +1 -0
- package/vendor-core/docs-content/diagrams/shuffle-map-reduce.svg +1 -0
- package/vendor-core/docs-content/diagrams/spill-classification.dark.svg +1 -0
- package/vendor-core/docs-content/diagrams/spill-classification.svg +1 -0
- package/vendor-core/docs-content/diagrams/udf-execution-models.dark.svg +1 -0
- package/vendor-core/docs-content/diagrams/udf-execution-models.svg +1 -0
- package/vendor-core/docs-content/tuning/broadcast-sizing.md +78 -0
- package/vendor-core/docs-content/tuning/cold-start.md +81 -0
- package/vendor-core/docs-content/tuning/duplicate-plan-subtree.md +45 -0
- package/vendor-core/docs-content/tuning/failures.md +124 -0
- package/vendor-core/docs-content/tuning/gc.md +110 -0
- package/vendor-core/docs-content/tuning/job-failure-rate.md +101 -0
- package/vendor-core/docs-content/tuning/memory-utilization.md +58 -0
- package/vendor-core/docs-content/tuning/retry-waste.md +90 -0
- package/vendor-core/docs-content/tuning/shuffle.md +154 -0
- package/vendor-core/docs-content/tuning/skew.md +123 -0
- package/vendor-core/docs-content/tuning/slow-host.md +117 -0
- package/vendor-core/docs-content/tuning/small-files.md +99 -0
- package/vendor-core/docs-content/tuning/spill.md +114 -0
- package/vendor-core/docs-content/tuning/straggler.md +103 -0
- package/vendor-core/docs-content/tuning/tiny-tasks.md +94 -0
- package/vendor-core/docs-content/tuning/utilization.md +90 -0
- package/vendor-core/docs-site-config.js +10 -17
- package/vendor-core/efficiency-model.js +7 -13
- package/vendor-core/etl-phases.js +3 -5
- package/vendor-core/event-handlers.js +232 -134
- package/vendor-core/event-schemas.js +48 -114
- package/vendor-core/evidence-availability.js +5 -10
- package/vendor-core/evidence-report.js +72 -122
- package/vendor-core/export-data.js +48 -0
- package/vendor-core/finding-action-label.js +4 -10
- package/vendor-core/finding-filter-predicate.js +3 -7
- package/vendor-core/finding-generic-recommendation.js +112 -0
- package/vendor-core/finding-names.js +51 -0
- package/vendor-core/format-utils.js +112 -38
- package/vendor-core/impact-band.js +18 -24
- package/vendor-core/impact-estimator.js +38 -74
- package/vendor-core/ingest.js +7 -13
- package/vendor-core/job-groups.js +3 -6
- package/vendor-core/list-runs.js +278 -0
- package/vendor-core/load-vendored.js +6 -12
- package/vendor-core/log-header-peek.js +81 -0
- package/vendor-core/lz4-block.js +4 -6
- package/vendor-core/mcp-server-factory.js +38 -8
- package/vendor-core/mcp-tools.js +105 -76
- package/vendor-core/model-assembler.js +8 -16
- package/vendor-core/occupancy.js +5 -9
- package/vendor-core/parser-worker.js +18 -27
- package/vendor-core/plan-dot.js +2 -5
- package/vendor-core/plan-duration-attribution.js +78 -29
- package/vendor-core/plan-graph-model.js +126 -69
- package/vendor-core/plan-node-detail.js +31 -17
- package/vendor-core/plan-summary.js +19 -8
- package/vendor-core/recommendation-rollup.js +35 -39
- package/vendor-core/redact.js +72 -16
- package/vendor-core/rolling-log-reassembly.js +4 -6
- package/vendor-core/run-comparison.js +65 -70
- package/vendor-core/scaling-sim.js +5 -7
- package/vendor-core/session-snapshot.js +1 -1
- package/vendor-core/shs-fetch.js +4 -6
- package/vendor-core/shs-load.js +9 -13
- package/vendor-core/shs-request.js +1 -1
- package/vendor-core/stage-quantiles.js +14 -0
- package/vendor-core/types.js +78 -18
- package/vendor-core/wasted-core-hours.js +7 -12
- package/public/assets/AutoscalingChurn-CP5OjN6v.js +0 -1
- package/public/assets/CacheUtilization-BjyawXL6.js +0 -1
- package/public/assets/CachingOpportunity-DV2ABG2G.js +0 -1
- package/public/assets/ConfigAudit-DMiBhAoG.js +0 -1
- package/public/assets/CoreUsageArea-CGYIgPnt.js +0 -1
- package/public/assets/Dashboard-D1_IrWRs.js +0 -12
- package/public/assets/ExecutorTimeline-DkaAGqQT.js +0 -1
- package/public/assets/Failures-Crngoicu.js +0 -1
- package/public/assets/GcPressure-CFXm4BnA.js +0 -1
- package/public/assets/IncompleteRun-BWEJTb13.js +0 -1
- package/public/assets/JobFailures-BmjxCA-h.js +0 -1
- package/public/assets/MemoryUtilization-Bu2uQJbd.js +0 -1
- package/public/assets/PlanExplorer-DKUwF0uz.js +0 -1
- package/public/assets/PlanGraphRoute-BjPSugIG.js +0 -1
- package/public/assets/ShuffleIO--Rur3neu.js +0 -1
- package/public/assets/Spill-DIMpt8c2.js +0 -1
- package/public/assets/TaskSkew-BDr1DbeI.js +0 -1
- package/public/assets/index-4id52G-A.js +0 -9
- package/public/assets/index-D9kwVYIO.css +0 -1
- package/public/assets/parser-worker-00O9WEQC.js +0 -65
- package/public/assets/plan-graph-vendor-vZQHG2EJ.js +0 -7
- package/public/assets/react-vendor-u4pyvC5Y.js +0 -9
- package/public/assets/useRecentFiles-46iBU9zJ.js +0 -5
- package/public/docs/assets/chunks/@localSearchIndexroot.DclgKtlT.js +0 -1
- package/public/docs/assets/chunks/theme.CoT6y5bf.js +0 -2
- package/public/docs/assets/contributor-guide_architecture_board-widgets.md.BtjoE_1k.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_board-widgets.md.BtjoE_1k.lean.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_detector-contract.md.DQyCHQNM.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_detector-contract.md.DQyCHQNM.lean.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_drill-down.md.BKfyDQck.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_drill-down.md.BKfyDQck.lean.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_state-and-history.md.DaqttKgQ.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_state-and-history.md.DaqttKgQ.lean.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.Bk6Iyj9S.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.Bk6Iyj9S.lean.js +0 -1
- package/public/docs/assets/contributor-guide_contributing.md.C-lKs70I.js +0 -1
- package/public/docs/assets/contributor-guide_contributing.md.C-lKs70I.lean.js +0 -1
- package/public/docs/assets/user-guide_getting-started.md.BJdD7a7_.js +0 -3
- package/public/docs/assets/user-guide_getting-started.md.BJdD7a7_.lean.js +0 -1
- package/public/docs/assets/user-guide_understanding-findings.md.CK2WZ0Tf.js +0 -1
- package/public/vendor/spark-doc/anchors.json +0 -205
- package/public/vendor/spark-doc/index.html +0 -4202
- package/public/vendor/spark-doc/landing.html +0 -418
- package/public/vendor/spark-doc/meta.html +0 -970
|
@@ -1,970 +0,0 @@
|
|
|
1
|
-
<!DOCTYPE html>
|
|
2
|
-
<html lang="en">
|
|
3
|
-
<head>
|
|
4
|
-
<meta charset="UTF-8">
|
|
5
|
-
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
6
|
-
<title>How This Site Works</title>
|
|
7
|
-
<script>
|
|
8
|
-
(() => {
|
|
9
|
-
try {
|
|
10
|
-
const theme = localStorage.getItem('spark-tuning-reference-theme');
|
|
11
|
-
if (theme === 'light' || theme === 'dark') {
|
|
12
|
-
document.documentElement.setAttribute('data-theme', theme);
|
|
13
|
-
}
|
|
14
|
-
} catch (_) {
|
|
15
|
-
// Storage can be unavailable in a sandboxed SparkForensics iframe.
|
|
16
|
-
}
|
|
17
|
-
})();
|
|
18
|
-
</script>
|
|
19
|
-
<style>
|
|
20
|
-
@import url('https://fonts.googleapis.com/css2?family=Recursive:wght@400;500;600;700&family=JetBrains+Mono:wght@400;500;600&display=swap');
|
|
21
|
-
|
|
22
|
-
:root {
|
|
23
|
-
color-scheme: dark;
|
|
24
|
-
--color-bg: #080b0d;
|
|
25
|
-
--color-surface: #11171a;
|
|
26
|
-
--color-surface-raised: #182124;
|
|
27
|
-
--color-surface-code: #05080a;
|
|
28
|
-
--color-text: #f4f4f5;
|
|
29
|
-
--color-text-muted: #a9b5b4;
|
|
30
|
-
--color-border: #233035;
|
|
31
|
-
--color-accent: #ff6b2c;
|
|
32
|
-
--color-accent-hover: #ff8a4f;
|
|
33
|
-
--color-signal: #39d6b5;
|
|
34
|
-
--color-signal-orange: var(--color-accent);
|
|
35
|
-
--color-signal-teal: var(--color-signal);
|
|
36
|
-
--color-info: var(--color-signal-teal);
|
|
37
|
-
--color-warning: var(--color-signal-orange);
|
|
38
|
-
--color-critical: #ff8576;
|
|
39
|
-
|
|
40
|
-
--font-sans: 'Recursive', -apple-system, BlinkMacSystemFont, sans-serif;
|
|
41
|
-
--font-mono: 'JetBrains Mono', ui-monospace, monospace;
|
|
42
|
-
|
|
43
|
-
--space-1: 4px;
|
|
44
|
-
--space-2: 8px;
|
|
45
|
-
--space-3: 16px;
|
|
46
|
-
--space-4: 24px;
|
|
47
|
-
--space-5: 32px;
|
|
48
|
-
--space-6: 48px;
|
|
49
|
-
|
|
50
|
-
--sidebar-width: 260px;
|
|
51
|
-
--content-max-width: 760px;
|
|
52
|
-
--transition-fast: 160ms cubic-bezier(0.22, 1, 0.36, 1);
|
|
53
|
-
}
|
|
54
|
-
|
|
55
|
-
:root[data-theme="light"] {
|
|
56
|
-
color-scheme: light;
|
|
57
|
-
--color-bg: #f7f8f4;
|
|
58
|
-
--color-surface: #ffffff;
|
|
59
|
-
--color-surface-raised: #edf2ef;
|
|
60
|
-
--color-surface-code: #e9efec;
|
|
61
|
-
--color-text: #151b1e;
|
|
62
|
-
--color-text-muted: #4f5f60;
|
|
63
|
-
--color-border: #dbe2df;
|
|
64
|
-
--color-accent: #bf4413;
|
|
65
|
-
--color-accent-hover: #aa3510;
|
|
66
|
-
--color-signal: #007e6b;
|
|
67
|
-
--color-signal-orange: var(--color-accent);
|
|
68
|
-
--color-signal-teal: var(--color-signal);
|
|
69
|
-
--color-info: var(--color-signal-teal);
|
|
70
|
-
--color-warning: var(--color-signal-orange);
|
|
71
|
-
--color-critical: #c34235;
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
* {
|
|
75
|
-
box-sizing: border-box;
|
|
76
|
-
}
|
|
77
|
-
|
|
78
|
-
html {
|
|
79
|
-
scroll-behavior: smooth;
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
body {
|
|
83
|
-
margin: 0;
|
|
84
|
-
background: var(--color-bg);
|
|
85
|
-
color: var(--color-text);
|
|
86
|
-
font-family: var(--font-sans);
|
|
87
|
-
font-size: 16px;
|
|
88
|
-
line-height: 1.6;
|
|
89
|
-
-webkit-font-smoothing: antialiased;
|
|
90
|
-
}
|
|
91
|
-
|
|
92
|
-
a {
|
|
93
|
-
color: var(--color-accent);
|
|
94
|
-
text-decoration: none;
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
a:hover {
|
|
98
|
-
color: var(--color-accent-hover);
|
|
99
|
-
text-decoration: underline;
|
|
100
|
-
}
|
|
101
|
-
|
|
102
|
-
a[href^="http"]::after {
|
|
103
|
-
content: "\2197";
|
|
104
|
-
display: inline-block;
|
|
105
|
-
margin-left: 0.2em;
|
|
106
|
-
font-size: 0.85em;
|
|
107
|
-
color: var(--color-text-muted);
|
|
108
|
-
}
|
|
109
|
-
|
|
110
|
-
:focus-visible {
|
|
111
|
-
outline: 3px solid var(--color-signal-teal);
|
|
112
|
-
outline-offset: 3px;
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
/* Layout */
|
|
116
|
-
|
|
117
|
-
.layout {
|
|
118
|
-
display: flex;
|
|
119
|
-
min-height: 100dvh;
|
|
120
|
-
align-items: flex-start;
|
|
121
|
-
}
|
|
122
|
-
|
|
123
|
-
.sidebar {
|
|
124
|
-
position: sticky;
|
|
125
|
-
top: 0;
|
|
126
|
-
width: var(--sidebar-width);
|
|
127
|
-
flex-shrink: 0;
|
|
128
|
-
height: 100dvh;
|
|
129
|
-
overflow-y: auto;
|
|
130
|
-
padding: var(--space-3);
|
|
131
|
-
border-right: 1px solid var(--color-border);
|
|
132
|
-
background: var(--color-surface);
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
.sidebar-header {
|
|
136
|
-
display: flex;
|
|
137
|
-
align-items: center;
|
|
138
|
-
justify-content: space-between;
|
|
139
|
-
gap: var(--space-2);
|
|
140
|
-
margin-bottom: var(--space-4);
|
|
141
|
-
padding: var(--space-2) 0 var(--space-3);
|
|
142
|
-
border-bottom: 1px solid var(--color-border);
|
|
143
|
-
}
|
|
144
|
-
|
|
145
|
-
.site-identity {
|
|
146
|
-
display: inline-flex;
|
|
147
|
-
align-items: center;
|
|
148
|
-
min-width: 0;
|
|
149
|
-
gap: var(--space-2);
|
|
150
|
-
color: var(--color-text);
|
|
151
|
-
font-size: 13px;
|
|
152
|
-
font-weight: 700;
|
|
153
|
-
letter-spacing: -0.01em;
|
|
154
|
-
line-height: 1.15;
|
|
155
|
-
}
|
|
156
|
-
|
|
157
|
-
.site-identity:hover {
|
|
158
|
-
color: var(--color-text);
|
|
159
|
-
text-decoration: none;
|
|
160
|
-
}
|
|
161
|
-
|
|
162
|
-
.brand-mark {
|
|
163
|
-
font-family: var(--font-mono);
|
|
164
|
-
font-weight: 800;
|
|
165
|
-
color: var(--color-text);
|
|
166
|
-
}
|
|
167
|
-
|
|
168
|
-
.site-name {
|
|
169
|
-
overflow: hidden;
|
|
170
|
-
text-overflow: ellipsis;
|
|
171
|
-
}
|
|
172
|
-
|
|
173
|
-
.theme-toggle {
|
|
174
|
-
display: inline-flex;
|
|
175
|
-
width: 44px;
|
|
176
|
-
min-height: 44px;
|
|
177
|
-
align-items: center;
|
|
178
|
-
justify-content: center;
|
|
179
|
-
flex: 0 0 auto;
|
|
180
|
-
border: 1px solid var(--color-border);
|
|
181
|
-
border-radius: 7px;
|
|
182
|
-
background: transparent;
|
|
183
|
-
color: var(--color-text);
|
|
184
|
-
cursor: pointer;
|
|
185
|
-
transition: background var(--transition-fast), border-color var(--transition-fast), color var(--transition-fast);
|
|
186
|
-
}
|
|
187
|
-
|
|
188
|
-
.theme-toggle:hover {
|
|
189
|
-
border-color: var(--color-signal-teal);
|
|
190
|
-
color: var(--color-signal-teal);
|
|
191
|
-
}
|
|
192
|
-
|
|
193
|
-
.theme-icon {
|
|
194
|
-
width: 1.15rem;
|
|
195
|
-
height: 1.15rem;
|
|
196
|
-
fill: none;
|
|
197
|
-
stroke: currentColor;
|
|
198
|
-
stroke-width: 1.8;
|
|
199
|
-
stroke-linecap: round;
|
|
200
|
-
}
|
|
201
|
-
|
|
202
|
-
.theme-icon.moon { display: none; }
|
|
203
|
-
:root[data-theme="light"] .theme-icon.sun { display: none; }
|
|
204
|
-
:root[data-theme="light"] .theme-icon.moon { display: block; }
|
|
205
|
-
|
|
206
|
-
.content {
|
|
207
|
-
flex: 1;
|
|
208
|
-
min-width: 0;
|
|
209
|
-
padding: var(--space-6) var(--space-5);
|
|
210
|
-
}
|
|
211
|
-
|
|
212
|
-
.doc-section {
|
|
213
|
-
max-width: var(--content-max-width);
|
|
214
|
-
margin: 0 auto var(--space-6);
|
|
215
|
-
}
|
|
216
|
-
|
|
217
|
-
.doc-section:first-of-type {
|
|
218
|
-
margin-top: 0;
|
|
219
|
-
}
|
|
220
|
-
|
|
221
|
-
/* Content group titles (mirror sidebar nav groups, so scrolling the page
|
|
222
|
-
surfaces the same top-level structure the sidebar shows) */
|
|
223
|
-
|
|
224
|
-
.content-group-title {
|
|
225
|
-
max-width: var(--content-max-width);
|
|
226
|
-
margin: var(--space-6) auto var(--space-4);
|
|
227
|
-
padding-bottom: var(--space-2);
|
|
228
|
-
border-bottom: 1px solid var(--color-border);
|
|
229
|
-
font-size: 13px;
|
|
230
|
-
font-weight: 600;
|
|
231
|
-
text-transform: uppercase;
|
|
232
|
-
letter-spacing: 0.08em;
|
|
233
|
-
color: var(--color-signal-teal);
|
|
234
|
-
}
|
|
235
|
-
|
|
236
|
-
.content-group-title-first {
|
|
237
|
-
margin-top: 0;
|
|
238
|
-
}
|
|
239
|
-
|
|
240
|
-
/* Accessibility utilities */
|
|
241
|
-
|
|
242
|
-
.visually-hidden {
|
|
243
|
-
position: absolute;
|
|
244
|
-
width: 1px;
|
|
245
|
-
height: 1px;
|
|
246
|
-
padding: 0;
|
|
247
|
-
margin: -1px;
|
|
248
|
-
overflow: hidden;
|
|
249
|
-
clip: rect(0, 0, 0, 0);
|
|
250
|
-
white-space: nowrap;
|
|
251
|
-
border: 0;
|
|
252
|
-
}
|
|
253
|
-
|
|
254
|
-
.skip-link {
|
|
255
|
-
position: absolute;
|
|
256
|
-
top: -100%;
|
|
257
|
-
left: var(--space-3);
|
|
258
|
-
z-index: 30;
|
|
259
|
-
padding: var(--space-2) var(--space-3);
|
|
260
|
-
background: var(--color-bg);
|
|
261
|
-
color: var(--color-text);
|
|
262
|
-
border: 1px solid var(--color-border);
|
|
263
|
-
border-radius: 6px;
|
|
264
|
-
}
|
|
265
|
-
|
|
266
|
-
.skip-link:focus {
|
|
267
|
-
top: var(--space-3);
|
|
268
|
-
}
|
|
269
|
-
|
|
270
|
-
/* Nav */
|
|
271
|
-
|
|
272
|
-
.nav-toggle {
|
|
273
|
-
display: none;
|
|
274
|
-
position: fixed;
|
|
275
|
-
top: var(--space-3);
|
|
276
|
-
left: var(--space-3);
|
|
277
|
-
z-index: 20;
|
|
278
|
-
width: 44px;
|
|
279
|
-
height: 44px;
|
|
280
|
-
align-items: center;
|
|
281
|
-
justify-content: center;
|
|
282
|
-
background: var(--color-surface);
|
|
283
|
-
border: 1px solid var(--color-border);
|
|
284
|
-
border-radius: 6px;
|
|
285
|
-
color: var(--color-text);
|
|
286
|
-
cursor: pointer;
|
|
287
|
-
}
|
|
288
|
-
|
|
289
|
-
.nav-search {
|
|
290
|
-
display: block;
|
|
291
|
-
width: 100%;
|
|
292
|
-
box-sizing: border-box;
|
|
293
|
-
margin-bottom: var(--space-3);
|
|
294
|
-
padding: var(--space-1) var(--space-2);
|
|
295
|
-
font-size: 14px;
|
|
296
|
-
font-family: inherit;
|
|
297
|
-
color: var(--color-text);
|
|
298
|
-
background: var(--color-surface-raised);
|
|
299
|
-
border: 1px solid var(--color-border);
|
|
300
|
-
border-radius: 6px;
|
|
301
|
-
}
|
|
302
|
-
|
|
303
|
-
.nav-search::placeholder {
|
|
304
|
-
color: var(--color-text-muted);
|
|
305
|
-
}
|
|
306
|
-
|
|
307
|
-
.nav-group {
|
|
308
|
-
margin-bottom: var(--space-4);
|
|
309
|
-
}
|
|
310
|
-
|
|
311
|
-
.nav-group-title {
|
|
312
|
-
font-size: 12px;
|
|
313
|
-
font-weight: 600;
|
|
314
|
-
text-transform: uppercase;
|
|
315
|
-
letter-spacing: 0.05em;
|
|
316
|
-
color: var(--color-text-muted);
|
|
317
|
-
margin-bottom: var(--space-2);
|
|
318
|
-
padding: 0 var(--space-2);
|
|
319
|
-
}
|
|
320
|
-
|
|
321
|
-
.nav-list {
|
|
322
|
-
list-style: none;
|
|
323
|
-
margin: 0;
|
|
324
|
-
padding: 0;
|
|
325
|
-
}
|
|
326
|
-
|
|
327
|
-
.nav-link {
|
|
328
|
-
display: block;
|
|
329
|
-
padding: var(--space-1) var(--space-2);
|
|
330
|
-
border-radius: 6px;
|
|
331
|
-
color: var(--color-text);
|
|
332
|
-
font-size: 14px;
|
|
333
|
-
line-height: 1.4;
|
|
334
|
-
transition: background var(--transition-fast), color var(--transition-fast);
|
|
335
|
-
}
|
|
336
|
-
|
|
337
|
-
.nav-link:hover {
|
|
338
|
-
background: var(--color-surface-raised);
|
|
339
|
-
text-decoration: none;
|
|
340
|
-
}
|
|
341
|
-
|
|
342
|
-
.nav-link.active {
|
|
343
|
-
color: var(--color-signal-orange);
|
|
344
|
-
font-weight: 600;
|
|
345
|
-
background: color-mix(in srgb, var(--color-signal-orange) 14%, transparent);
|
|
346
|
-
}
|
|
347
|
-
|
|
348
|
-
/* Anchor-not-found fallback */
|
|
349
|
-
|
|
350
|
-
.anchor-fallback[hidden] {
|
|
351
|
-
display: none;
|
|
352
|
-
}
|
|
353
|
-
|
|
354
|
-
.anchor-fallback {
|
|
355
|
-
display: flex;
|
|
356
|
-
align-items: center;
|
|
357
|
-
justify-content: space-between;
|
|
358
|
-
gap: var(--space-3);
|
|
359
|
-
max-width: var(--content-max-width);
|
|
360
|
-
margin: 0 auto var(--space-4);
|
|
361
|
-
padding: var(--space-2) var(--space-3);
|
|
362
|
-
background: color-mix(in srgb, var(--color-signal-orange) 10%, var(--color-surface));
|
|
363
|
-
border: 1px solid var(--color-border);
|
|
364
|
-
border-radius: 6px;
|
|
365
|
-
font-size: 14px;
|
|
366
|
-
}
|
|
367
|
-
|
|
368
|
-
.anchor-fallback button {
|
|
369
|
-
background: none;
|
|
370
|
-
border: none;
|
|
371
|
-
color: var(--color-text-muted);
|
|
372
|
-
font-size: 18px;
|
|
373
|
-
line-height: 1;
|
|
374
|
-
cursor: pointer;
|
|
375
|
-
min-width: 44px;
|
|
376
|
-
min-height: 44px;
|
|
377
|
-
}
|
|
378
|
-
|
|
379
|
-
/* Prose */
|
|
380
|
-
|
|
381
|
-
h1, h2, h3, h4 {
|
|
382
|
-
font-weight: 600;
|
|
383
|
-
line-height: 1.3;
|
|
384
|
-
margin: var(--space-5) 0 var(--space-2);
|
|
385
|
-
}
|
|
386
|
-
|
|
387
|
-
.doc-section > h2:first-child,
|
|
388
|
-
.doc-section > h3:first-child,
|
|
389
|
-
.doc-section > h4:first-child {
|
|
390
|
-
margin-top: 0;
|
|
391
|
-
}
|
|
392
|
-
|
|
393
|
-
/* Each section's own Markdown headings are demoted a level in build.py
|
|
394
|
-
(bump_headings) so the page has exactly one real <h1>; sizes below keep the
|
|
395
|
-
original visual scale (former h1/h2/h3 sizes) despite the semantic shift. */
|
|
396
|
-
h1 { font-size: 32px; }
|
|
397
|
-
h2 { font-size: 32px; }
|
|
398
|
-
h3 { font-size: 24px; }
|
|
399
|
-
h4 { font-size: 18px; }
|
|
400
|
-
|
|
401
|
-
p, ul, ol {
|
|
402
|
-
margin: 0 0 var(--space-3);
|
|
403
|
-
}
|
|
404
|
-
|
|
405
|
-
p { color: var(--color-text); }
|
|
406
|
-
|
|
407
|
-
/* PySpark callouts (blockquotes) */
|
|
408
|
-
|
|
409
|
-
blockquote {
|
|
410
|
-
margin: var(--space-4) 0;
|
|
411
|
-
padding: var(--space-3);
|
|
412
|
-
border: 1px solid var(--color-border);
|
|
413
|
-
background: color-mix(in srgb, var(--color-signal-teal) 8%, var(--color-surface));
|
|
414
|
-
border-radius: 6px;
|
|
415
|
-
}
|
|
416
|
-
|
|
417
|
-
blockquote p:last-child {
|
|
418
|
-
margin-bottom: 0;
|
|
419
|
-
}
|
|
420
|
-
|
|
421
|
-
/* Code */
|
|
422
|
-
|
|
423
|
-
code {
|
|
424
|
-
font-family: var(--font-mono);
|
|
425
|
-
font-size: 0.9em;
|
|
426
|
-
background: var(--color-surface-code);
|
|
427
|
-
padding: 0.15em 0.4em;
|
|
428
|
-
border-radius: 4px;
|
|
429
|
-
overflow-wrap: anywhere;
|
|
430
|
-
}
|
|
431
|
-
|
|
432
|
-
pre {
|
|
433
|
-
background: var(--color-surface-code);
|
|
434
|
-
border: 1px solid var(--color-border);
|
|
435
|
-
border-radius: 6px;
|
|
436
|
-
padding: var(--space-3);
|
|
437
|
-
overflow-x: auto;
|
|
438
|
-
}
|
|
439
|
-
|
|
440
|
-
pre code {
|
|
441
|
-
background: none;
|
|
442
|
-
padding: 0;
|
|
443
|
-
font-size: 14px;
|
|
444
|
-
}
|
|
445
|
-
|
|
446
|
-
/* Tables */
|
|
447
|
-
|
|
448
|
-
.table-scroll {
|
|
449
|
-
margin: var(--space-4) 0;
|
|
450
|
-
overflow-x: auto;
|
|
451
|
-
}
|
|
452
|
-
|
|
453
|
-
table {
|
|
454
|
-
width: 100%;
|
|
455
|
-
border-collapse: collapse;
|
|
456
|
-
margin: 0;
|
|
457
|
-
font-size: 14px;
|
|
458
|
-
}
|
|
459
|
-
|
|
460
|
-
th, td {
|
|
461
|
-
text-align: left;
|
|
462
|
-
padding: var(--space-2) var(--space-3);
|
|
463
|
-
border-bottom: 1px solid var(--color-border);
|
|
464
|
-
vertical-align: top;
|
|
465
|
-
}
|
|
466
|
-
|
|
467
|
-
th {
|
|
468
|
-
font-weight: 600;
|
|
469
|
-
color: var(--color-text);
|
|
470
|
-
border-bottom: 2px solid var(--color-border);
|
|
471
|
-
}
|
|
472
|
-
|
|
473
|
-
/* Footnotes */
|
|
474
|
-
|
|
475
|
-
.footnote {
|
|
476
|
-
font-size: 13px;
|
|
477
|
-
color: var(--color-text-muted);
|
|
478
|
-
border-top: 1px solid var(--color-border);
|
|
479
|
-
padding-top: var(--space-3);
|
|
480
|
-
margin-top: var(--space-5);
|
|
481
|
-
}
|
|
482
|
-
|
|
483
|
-
.footnote li {
|
|
484
|
-
margin-bottom: var(--space-1);
|
|
485
|
-
}
|
|
486
|
-
|
|
487
|
-
sup {
|
|
488
|
-
line-height: 0;
|
|
489
|
-
}
|
|
490
|
-
|
|
491
|
-
/* Severity dots & tags */
|
|
492
|
-
|
|
493
|
-
.severity-dot {
|
|
494
|
-
display: inline-block;
|
|
495
|
-
width: 8px;
|
|
496
|
-
height: 8px;
|
|
497
|
-
border-radius: 50%;
|
|
498
|
-
margin-right: var(--space-1);
|
|
499
|
-
}
|
|
500
|
-
|
|
501
|
-
.severity-dot.info { background: var(--color-info); }
|
|
502
|
-
.severity-dot.warning { background: var(--color-warning); }
|
|
503
|
-
.severity-dot.critical { background: var(--color-critical); }
|
|
504
|
-
|
|
505
|
-
.tag {
|
|
506
|
-
display: inline-block;
|
|
507
|
-
font-family: var(--font-mono);
|
|
508
|
-
font-size: 11px;
|
|
509
|
-
font-weight: 600;
|
|
510
|
-
text-transform: uppercase;
|
|
511
|
-
letter-spacing: 0.03em;
|
|
512
|
-
padding: 0.1em 0.5em;
|
|
513
|
-
border: 1px solid var(--color-border);
|
|
514
|
-
border-radius: 4px;
|
|
515
|
-
color: var(--color-signal-teal);
|
|
516
|
-
}
|
|
517
|
-
|
|
518
|
-
/* Responsive */
|
|
519
|
-
|
|
520
|
-
@media (max-width: 900px) {
|
|
521
|
-
.nav-toggle {
|
|
522
|
-
display: flex;
|
|
523
|
-
}
|
|
524
|
-
|
|
525
|
-
.nav-link {
|
|
526
|
-
min-height: 44px;
|
|
527
|
-
display: flex;
|
|
528
|
-
align-items: center;
|
|
529
|
-
}
|
|
530
|
-
|
|
531
|
-
.sidebar {
|
|
532
|
-
position: fixed;
|
|
533
|
-
top: 0;
|
|
534
|
-
left: 0;
|
|
535
|
-
z-index: 10;
|
|
536
|
-
transform: translateX(-100%);
|
|
537
|
-
box-shadow: 8px 0 8px rgba(0, 0, 0, 0.18);
|
|
538
|
-
transition: transform var(--transition-fast);
|
|
539
|
-
}
|
|
540
|
-
|
|
541
|
-
.sidebar.sidebar-open {
|
|
542
|
-
transform: translateX(0);
|
|
543
|
-
}
|
|
544
|
-
|
|
545
|
-
.content {
|
|
546
|
-
/* top padding clears the fixed nav-toggle button (44px + its own space-3 offset)
|
|
547
|
-
plus a gap, so it never overlaps the first heading/group title */
|
|
548
|
-
padding: calc(var(--space-3) * 2 + 44px) var(--space-3) var(--space-3);
|
|
549
|
-
}
|
|
550
|
-
}
|
|
551
|
-
|
|
552
|
-
@media (prefers-reduced-motion: reduce) {
|
|
553
|
-
html {
|
|
554
|
-
scroll-behavior: auto;
|
|
555
|
-
}
|
|
556
|
-
|
|
557
|
-
* {
|
|
558
|
-
transition-duration: 0.01ms !important;
|
|
559
|
-
}
|
|
560
|
-
}
|
|
561
|
-
|
|
562
|
-
</style>
|
|
563
|
-
</head>
|
|
564
|
-
<body>
|
|
565
|
-
<a href="#main-content" class="skip-link">Skip to content</a>
|
|
566
|
-
<h1 class="visually-hidden">How This Site Works</h1>
|
|
567
|
-
<button type="button" class="nav-toggle" id="nav-toggle" aria-expanded="false" aria-controls="sidebar" aria-label="Toggle navigation">
|
|
568
|
-
<svg width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" aria-hidden="true">
|
|
569
|
-
<line x1="3" y1="6" x2="21" y2="6"></line>
|
|
570
|
-
<line x1="3" y1="12" x2="21" y2="12"></line>
|
|
571
|
-
<line x1="3" y1="18" x2="21" y2="18"></line>
|
|
572
|
-
</svg>
|
|
573
|
-
</button>
|
|
574
|
-
|
|
575
|
-
<div class="layout">
|
|
576
|
-
<nav class="sidebar" id="sidebar" aria-label="Sections">
|
|
577
|
-
<div class="sidebar-header">
|
|
578
|
-
<a class="site-identity" href="#pipeline-overview" aria-label="How This Site Works introduction">
|
|
579
|
-
<span class="brand-mark" aria-hidden="true">S/</span>
|
|
580
|
-
<span class="site-name">How This Site Works</span>
|
|
581
|
-
</a>
|
|
582
|
-
<button type="button" class="theme-toggle" id="theme-toggle" aria-label="Switch to light theme" aria-pressed="false">
|
|
583
|
-
<svg class="theme-icon sun" viewBox="0 0 24 24" aria-hidden="true"><circle cx="12" cy="12" r="3.5"></circle><path d="M12 2v2M12 20v2M4.93 4.93l1.41 1.41M17.66 17.66l1.41 1.41M2 12h2M20 12h2M4.93 19.07l1.41-1.41M17.66 6.34l1.41-1.41"></path></svg>
|
|
584
|
-
<svg class="theme-icon moon" viewBox="0 0 24 24" aria-hidden="true"><path d="M20.5 14.2A8.7 8.7 0 0 1 9.8 3.5 8.7 8.7 0 1 0 20.5 14.2Z"></path></svg>
|
|
585
|
-
</button>
|
|
586
|
-
</div>
|
|
587
|
-
<input type="search" id="nav-search" class="nav-search" placeholder="Filter sections…" aria-label="Filter sections">
|
|
588
|
-
|
|
589
|
-
<div class="nav-group">
|
|
590
|
-
<div class="nav-group-title">Pipeline Overview</div>
|
|
591
|
-
<ul class="nav-list">
|
|
592
|
-
|
|
593
|
-
<li><a href="#pipeline-overview" class="nav-link">Pipeline Overview</a></li>
|
|
594
|
-
|
|
595
|
-
</ul>
|
|
596
|
-
</div>
|
|
597
|
-
|
|
598
|
-
<div class="nav-group">
|
|
599
|
-
<div class="nav-group-title">Registry & Retrieval</div>
|
|
600
|
-
<ul class="nav-list">
|
|
601
|
-
|
|
602
|
-
<li><a href="#registry-and-retrieval" class="nav-link">Registry & Retrieval</a></li>
|
|
603
|
-
|
|
604
|
-
</ul>
|
|
605
|
-
</div>
|
|
606
|
-
|
|
607
|
-
<div class="nav-group">
|
|
608
|
-
<div class="nav-group-title">Agents & Orchestration</div>
|
|
609
|
-
<ul class="nav-list">
|
|
610
|
-
|
|
611
|
-
<li><a href="#agents-and-orchestration" class="nav-link">Agents & Orchestration</a></li>
|
|
612
|
-
|
|
613
|
-
</ul>
|
|
614
|
-
</div>
|
|
615
|
-
|
|
616
|
-
<div class="nav-group">
|
|
617
|
-
<div class="nav-group-title">Content, Staleness & Build</div>
|
|
618
|
-
<ul class="nav-list">
|
|
619
|
-
|
|
620
|
-
<li><a href="#content-staleness-and-build" class="nav-link">Content, Staleness & Build</a></li>
|
|
621
|
-
|
|
622
|
-
</ul>
|
|
623
|
-
</div>
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
</nav>
|
|
627
|
-
|
|
628
|
-
<main class="content" id="main-content" tabindex="-1">
|
|
629
|
-
<div class="anchor-fallback" id="anchor-fallback" hidden role="status">
|
|
630
|
-
Section not found — showing introduction.
|
|
631
|
-
<button type="button" id="anchor-fallback-dismiss" aria-label="Dismiss">×</button>
|
|
632
|
-
</div>
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
<div class="content-group-title content-group-title-first">Pipeline Overview</div>
|
|
636
|
-
|
|
637
|
-
<section id="pipeline-overview" class="doc-section">
|
|
638
|
-
<h2>Pipeline Overview</h2>
|
|
639
|
-
<h3>What this repository builds</h3>
|
|
640
|
-
<p>This repository runs a question-granular, citation-backed answering pipeline that turns a registry of research questions into a reviewed reference site, and it does so twice: once for a Spark Tuning Reference and once for the pipeline's own architecture<sup id="fnref:3-architecture"><a class="footnote-ref" href="#fn:3-architecture" title="architecture — references-meta/docs/architecture.md">1</a></sup>. Selection between the two happens via <code>--corpus {spark,meta}</code> on every pipeline script, defaulting to <code>spark</code>. <code>scripts/corpus.py</code> holds a <code>Corpus</code> dataclass with <code>SPARK</code>/<code>META</code> profiles, and every path a script touches (registry, per-question answers dir, section-projection dir, content-state sidecar, content manifest, references dir, Chroma path/collection, embedding model, build output, site title) comes from the selected profile rather than a hardcoded constant<sup id="fnref2:3-architecture"><a class="footnote-ref" href="#fn:3-architecture" title="architecture — references-meta/docs/architecture.md">1</a></sup>. The <code>spark</code> corpus is the Spark Tuning Reference itself, built over <code>references/</code> and indexed with the asymmetric question→passage retriever <code>multi-qa-mpnet-base-dot-v1</code>; its build output is <code>index.html</code><sup id="fnref3:3-architecture"><a class="footnote-ref" href="#fn:3-architecture" title="architecture — references-meta/docs/architecture.md">1</a></sup>. The <code>meta</code> corpus is this project's own architecture doc (<code>docs/architecture.md</code>, <code>docs/runbook.md</code>, <code>docs/rules/*.md</code>, <code>AGENTS.md</code>, and the agent/skill files that make up the pipeline). It is indexed via symlinks under <code>references-meta/</code> with the general symmetric model <code>all-mpnet-base-v2</code> into a separate Chroma collection, and its build output is <code>meta.html</code><sup id="fnref4:3-architecture"><a class="footnote-ref" href="#fn:3-architecture" title="architecture — references-meta/docs/architecture.md">1</a></sup>. The two corpora never share a registry, a Chroma collection, or a content manifest; each is an independent instance of the same pipeline shape<sup id="fnref5:3-architecture"><a class="footnote-ref" href="#fn:3-architecture" title="architecture — references-meta/docs/architecture.md">1</a></sup>.</p>
|
|
641
|
-
<p>Within either corpus, the atomic unit of the pipeline is the question, not the section: the registry (<code>research/questions.yaml</code>, or its meta counterpart) is the single source of truth holding sections and questions, and every other artifact is a derived projection, never edited directly as a source of truth<sup id="fnref6:3-architecture"><a class="footnote-ref" href="#fn:3-architecture" title="architecture — references-meta/docs/architecture.md">1</a></sup>. Content flows one way through the pipeline: registry → per-question answer files (prose plus footnotes and their own <code>## Sources</code>) → section projections (an ordered concatenation of a section's question answers, with sources merged and deduped) → content pages (brief-driven on first generation, human-owned after that) → the built site, and a downstream artifact is never treated as a source of truth for an upstream one<sup id="fnref7:3-architecture"><a class="footnote-ref" href="#fn:3-architecture" title="architecture — references-meta/docs/architecture.md">1</a></sup>. The project's own <code>AGENTS.md</code> restates the operator-facing consequence: every content page is real cited prose, not a placeholder, and new or changed content still flows registry → answer → projection → page<sup id="fnref:3-agents-md"><a class="footnote-ref" href="#fn:3-agents-md" title="agents-md — references-meta/AGENTS.md">2</a></sup>.</p>
|
|
642
|
-
<p>The build output is produced regardless of whether every content page is current. A committed hash sidecar tracks each content page against the sha256 of the section projection it was last generated or accepted from, classifying pages as ungenerated, stale, or ok; the build script checks this before rendering and prints a non-fatal warning banner listing any stale or ungenerated pages, but the build still produces <code>index.html</code> either way<sup id="fnref8:3-architecture"><a class="footnote-ref" href="#fn:3-architecture" title="architecture — references-meta/docs/architecture.md">1</a></sup>. Recompute never triggers regeneration on its own; it only flags a page stale, since content pages are human-owned once generated<sup id="fnref9:3-architecture"><a class="footnote-ref" href="#fn:3-architecture" title="architecture — references-meta/docs/architecture.md">1</a></sup>.</p>
|
|
643
|
-
<p>On the Spark side, the resulting pages carry a further constraint tied to why the site exists: its manifest-level anchor IDs are a frozen cross-repo Integration Contract with the sibling <code>sparkforensics</code> repo, so renaming or removing one is a breaking change, and content assembly must read anchors from <code>content/manifest.yaml</code> rather than deriving them from a filename or heading<sup id="fnref:3-rules-content"><a class="footnote-ref" href="#fn:3-rules-content" title="rules-content — references-meta/docs/rules/content.md">3</a></sup>. That contract applies only to the Spark corpus; <code>content/meta/manifest.yaml</code> and <code>meta.html</code> carry no such contract, and their anchors may change freely<sup id="fnref2:3-rules-content"><a class="footnote-ref" href="#fn:3-rules-content" title="rules-content — references-meta/docs/rules/content.md">3</a></sup>. <code>docs/anchor-map.md</code> reconciles the two, mapping each research answer file to its content file and frozen anchor<sup id="fnref:3-anchor-map"><a class="footnote-ref" href="#fn:3-anchor-map" title="anchor-map — references-meta/docs/anchor-map.md">4</a></sup>.</p>
|
|
644
|
-
<h3>The one-way derive chain</h3>
|
|
645
|
-
<p>The chain runs in one direction, front to back:</p>
|
|
646
|
-
<p><code>research/questions.yaml</code> → <code>research/answers/q/<id>.md</code> → <code>research/answers/<section-id>.md</code> → <code>content/*.md</code> (and <code>content/bottlenecks/*.md</code>) → <code>index.html</code><sup id="fnref10:3-architecture"><a class="footnote-ref" href="#fn:3-architecture" title="architecture — references-meta/docs/architecture.md">1</a></sup>.</p>
|
|
647
|
-
<p>Each stage has a role. <code>questions.yaml</code> is the registry of sections and questions with their statuses. Each per-question answer file holds that question's prose plus its own <code>[^id]</code> footnotes and <code>## Sources</code> block. The section file is a projection: an ordered concatenation of a section's question answers, with <code>## Sources</code> merged and deduped, and it is the same path the content manifest's <code>assembled_from:</code> already points at. Content pages are brief-driven and only first-generated from that projection, after which they become human-owned. The build turns the content pages into <code>index.html</code><sup id="fnref11:3-architecture"><a class="footnote-ref" href="#fn:3-architecture" title="architecture — references-meta/docs/architecture.md">1</a></sup>.</p>
|
|
648
|
-
<p>The architecture forbids treating any downstream artifact as a source of truth because <code>research/questions.yaml</code> is the single source of truth for the whole pipeline. Every other artifact (per-question answers, section projections, content pages) is a derived projection, regenerable from the registry and the per-question answer files, never edited as a source of truth itself<sup id="fnref12:3-architecture"><a class="footnote-ref" href="#fn:3-architecture" title="architecture — references-meta/docs/architecture.md">1</a></sup>. The registry rules state this directly: never edit a downstream artifact as if it were the source of truth, since per-question answers, section projections, and content pages are all derived projections regenerable from the registry, and recompute always starts at <code>questions.yaml</code> and flows forward<sup id="fnref:3-rules-registry"><a class="footnote-ref" href="#fn:3-rules-registry" title="rules-registry — references-meta/docs/rules/registry.md">5</a></sup>. The derive-direction diagram states the same principle as an invariant: every arrow is one-way, and a downstream artifact (a section projection, a content page, the built site) is never a source of truth for an upstream one<sup id="fnref13:3-architecture"><a class="footnote-ref" href="#fn:3-architecture" title="architecture — references-meta/docs/architecture.md">1</a></sup>. The project's own <code>AGENTS.md</code> restates the practical consequence for operators: new or changed content still flows registry → answer → projection → page, and <code>questions.yaml</code> remains the authoritative question registry rather than any prose plan or derived array<sup id="fnref2:3-agents-md"><a class="footnote-ref" href="#fn:3-agents-md" title="agents-md — references-meta/AGENTS.md">2</a></sup>.</p>
|
|
649
|
-
<h3>Why the question, not the section, is the atomic unit</h3>
|
|
650
|
-
<p><code>docs/architecture.md</code> states the design directly: the atomic unit of the answering pipeline is the <strong>question</strong>, not the section. It goes on to name <code>research/questions.yaml</code> as the single source of truth, holding <code>sections</code> (id, num, title, default_k) as a grouping label and <code>questions</code> (id, section, text, version, k, status) as the working records. Every other artifact, from per-question answer files up through section projections and content pages, is a derived projection regenerable from the registry and those answer files, never a source of truth itself<sup id="fnref14:3-architecture"><a class="footnote-ref" href="#fn:3-architecture" title="architecture — references-meta/docs/architecture.md">1</a></sup>.</p>
|
|
651
|
-
<p>That per-question state (its own <code>version</code>, <code>k</code>, <code>status</code>) is what makes independent recompute possible. <code>add_question.py</code> mints an id and appends a single <code>pending</code> record without mutating any existing record, and <code>recompute_question.py</code> flips exactly one record back to <code>pending</code> and backs up only that question's old answer file, touching no other record or file<sup id="fnref2:3-rules-registry"><a class="footnote-ref" href="#fn:3-rules-registry" title="rules-registry — references-meta/docs/rules/registry.md">5</a></sup>. If the section were the atomic unit, recomputing one question's answer would force treating the whole section's state as dirty; scoping status and versioning to the question lets one question be re-answered, re-versioned, or given its own <code>k</code> without disturbing its siblings.</p>
|
|
652
|
-
<p>The rest of the pipeline is built around that same grain: the Draft/Verify/Redraft agents are each scoped to a single question, and the <code>answer-question</code> skill runs one question end to end: Draft → Verify → [Redraft → re-Verify] → Humanize → <code>check_prose</code> → <code>project_section</code> → <code>content_state</code> ... → commit<sup id="fnref15:3-architecture"><a class="footnote-ref" href="#fn:3-architecture" title="architecture — references-meta/docs/architecture.md">1</a></sup>. The meta corpus's own <code>answer-meta-question</code> skill mirrors this: its input is one question id, and it dispatches the Draft/Verify agents with a one-item question list equal to the record's text<sup id="fnref:3-skill-answer-meta-question"><a class="footnote-ref" href="#fn:3-skill-answer-meta-question" title="skill-answer-meta-question — references-meta/skills/answer-meta-question.md">6</a></sup>. Only after a question is answered does <code>project_section</code> roll it into the section file, by concatenating that section's question answers in registry order and merging their <code>## Sources</code> blocks<sup id="fnref16:3-architecture"><a class="footnote-ref" href="#fn:3-architecture" title="architecture — references-meta/docs/architecture.md">1</a></sup>. The section projection is assembled from finished question-level work, not answered as a unit itself.</p>
|
|
653
|
-
<h3>Sources</h3>
|
|
654
|
-
<div class="footnote">
|
|
655
|
-
<hr />
|
|
656
|
-
<ol>
|
|
657
|
-
<li id="fn:3-architecture">
|
|
658
|
-
<p>architecture — references-meta/docs/architecture.md <a class="footnote-backref" href="#fnref:3-architecture" title="Jump back to footnote 1 in the text">↩</a><a class="footnote-backref" href="#fnref2:3-architecture" title="Jump back to footnote 1 in the text">↩</a><a class="footnote-backref" href="#fnref3:3-architecture" title="Jump back to footnote 1 in the text">↩</a><a class="footnote-backref" href="#fnref4:3-architecture" title="Jump back to footnote 1 in the text">↩</a><a class="footnote-backref" href="#fnref5:3-architecture" title="Jump back to footnote 1 in the text">↩</a><a class="footnote-backref" href="#fnref6:3-architecture" title="Jump back to footnote 1 in the text">↩</a><a class="footnote-backref" href="#fnref7:3-architecture" title="Jump back to footnote 1 in the text">↩</a><a class="footnote-backref" href="#fnref8:3-architecture" title="Jump back to footnote 1 in the text">↩</a><a class="footnote-backref" href="#fnref9:3-architecture" title="Jump back to footnote 1 in the text">↩</a><a class="footnote-backref" href="#fnref10:3-architecture" title="Jump back to footnote 1 in the text">↩</a><a class="footnote-backref" href="#fnref11:3-architecture" title="Jump back to footnote 1 in the text">↩</a><a class="footnote-backref" href="#fnref12:3-architecture" title="Jump back to footnote 1 in the text">↩</a><a class="footnote-backref" href="#fnref13:3-architecture" title="Jump back to footnote 1 in the text">↩</a><a class="footnote-backref" href="#fnref14:3-architecture" title="Jump back to footnote 1 in the text">↩</a><a class="footnote-backref" href="#fnref15:3-architecture" title="Jump back to footnote 1 in the text">↩</a><a class="footnote-backref" href="#fnref16:3-architecture" title="Jump back to footnote 1 in the text">↩</a></p>
|
|
659
|
-
</li>
|
|
660
|
-
<li id="fn:3-agents-md">
|
|
661
|
-
<p>agents-md — references-meta/AGENTS.md <a class="footnote-backref" href="#fnref:3-agents-md" title="Jump back to footnote 2 in the text">↩</a><a class="footnote-backref" href="#fnref2:3-agents-md" title="Jump back to footnote 2 in the text">↩</a></p>
|
|
662
|
-
</li>
|
|
663
|
-
<li id="fn:3-rules-content">
|
|
664
|
-
<p>rules-content — references-meta/docs/rules/content.md <a class="footnote-backref" href="#fnref:3-rules-content" title="Jump back to footnote 3 in the text">↩</a><a class="footnote-backref" href="#fnref2:3-rules-content" title="Jump back to footnote 3 in the text">↩</a></p>
|
|
665
|
-
</li>
|
|
666
|
-
<li id="fn:3-anchor-map">
|
|
667
|
-
<p>anchor-map — references-meta/docs/anchor-map.md <a class="footnote-backref" href="#fnref:3-anchor-map" title="Jump back to footnote 4 in the text">↩</a></p>
|
|
668
|
-
</li>
|
|
669
|
-
<li id="fn:3-rules-registry">
|
|
670
|
-
<p>rules-registry — references-meta/docs/rules/registry.md <a class="footnote-backref" href="#fnref:3-rules-registry" title="Jump back to footnote 5 in the text">↩</a><a class="footnote-backref" href="#fnref2:3-rules-registry" title="Jump back to footnote 5 in the text">↩</a></p>
|
|
671
|
-
</li>
|
|
672
|
-
<li id="fn:3-skill-answer-meta-question">
|
|
673
|
-
<p>skill-answer-meta-question — references-meta/skills/answer-meta-question.md <a class="footnote-backref" href="#fnref:3-skill-answer-meta-question" title="Jump back to footnote 6 in the text">↩</a></p>
|
|
674
|
-
</li>
|
|
675
|
-
</ol>
|
|
676
|
-
</div>
|
|
677
|
-
</section>
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
<div class="content-group-title">Registry & Retrieval</div>
|
|
681
|
-
|
|
682
|
-
<section id="registry-and-retrieval" class="doc-section">
|
|
683
|
-
<h2>Registry & Retrieval</h2>
|
|
684
|
-
<h3>The registry's data model</h3>
|
|
685
|
-
<p><code>research/questions.yaml</code> holds two top-level collections. A section record has <code>id</code>, <code>num</code>, <code>title</code>, and <code>default_k</code>. A question record has <code>id</code>, <code>section</code>, <code>text</code>, <code>version</code>, <code>k</code>, and <code>status</code><sup id="fnref:4-architecture"><a class="footnote-ref" href="#fn:4-architecture" title="architecture — references-meta/docs/architecture.md">1</a></sup>.</p>
|
|
686
|
-
<p><code>id</code> is stable once minted: <code>scripts/add_question.py</code> appends a new <code>pending</code> record and prints its id<sup id="fnref:4-runbook"><a class="footnote-ref" href="#fn:4-runbook" title="runbook — references-meta/docs/runbook.md">3</a></sup>, but never mutates an existing record's <code>id</code>, <code>section</code>, or <code>text</code>. A reworded or dropped question gets a new id rather than a rewritten one<sup id="fnref:4-rules-registry"><a class="footnote-ref" href="#fn:4-rules-registry" title="rules-registry — references-meta/docs/rules/registry.md">2</a></sup>.</p>
|
|
687
|
-
<p><code>version</code> is one of <code>3.0</code> / <code>3.5</code> / <code>4.0</code> / <code>n/a</code>, passed through to <code>ask.py --version</code><sup id="fnref2:4-rules-registry"><a class="footnote-ref" href="#fn:4-rules-registry" title="rules-registry — references-meta/docs/rules/registry.md">2</a></sup>. It has a default rather than being absent: <code>add_question</code> takes an optional <code>--version X</code> flag and falls back to <code>n/a</code> when omitted<sup id="fnref2:4-runbook"><a class="footnote-ref" href="#fn:4-runbook" title="runbook — references-meta/docs/runbook.md">3</a></sup>. The meta corpus's own registry (<code>research/meta/questions.yaml</code>) doesn't use the bucket concept at all. Every question there is <code>n/a</code><sup id="fnref:4-agent-meta-draft"><a class="footnote-ref" href="#fn:4-agent-meta-draft" title="agent-meta-draft — references-meta/agents/meta-section-draft.md">4</a></sup>.</p>
|
|
688
|
-
<p><code>k</code> is the field that's genuinely optional at the per-question level: <code>add_question</code> takes an optional <code>--k N</code> that overrides the section's <code>default_k</code> for that one question only; when a question's <code>k</code> is left null, the answering pipeline reads the section's <code>default_k</code> instead<sup id="fnref:4-skill-answer-question"><a class="footnote-ref" href="#fn:4-skill-answer-question" title="skill-answer-question — references-meta/skills/answer-question.md">5</a></sup>.</p>
|
|
689
|
-
<p><code>status</code> starts as <code>pending</code> on append and is later flipped by the answering pipeline to <code>answered</code> if verification passes with no unresolved markers, or <code>needs-review</code> otherwise<sup id="fnref2:4-skill-answer-question"><a class="footnote-ref" href="#fn:4-skill-answer-question" title="skill-answer-question — references-meta/skills/answer-question.md">5</a></sup>. <code>scripts/recompute_question.py</code> can also flip an existing record's status back to <code>pending</code> while backing up its old answer file, without touching any other record or file<sup id="fnref3:4-rules-registry"><a class="footnote-ref" href="#fn:4-rules-registry" title="rules-registry — references-meta/docs/rules/registry.md">2</a></sup>.</p>
|
|
690
|
-
<h3>Why retrieval differs by corpus</h3>
|
|
691
|
-
<p>For the Spark corpus (the <code>spark</code> profile, <code>scripts/ask.py</code>'s default), retrieval is indexed over <code>references/</code> with <code>multi-qa-mpnet-base-dot-v1</code>, which the architecture doc itself labels "the asymmetric question→passage retriever"<sup id="fnref2:4-architecture"><a class="footnote-ref" href="#fn:4-architecture" title="architecture — references-meta/docs/architecture.md">1</a></sup>. That label is contrasted directly with the other corpus this pipeline runs: the meta corpus (this doc's own corpus) uses a different, symmetric embedding model, <code>all-mpnet-base-v2</code>, instead<sup id="fnref3:4-architecture"><a class="footnote-ref" href="#fn:4-architecture" title="architecture — references-meta/docs/architecture.md">1</a></sup>. The research rules single out <code>multi-qa-mpnet-base-dot-v1</code>'s asymmetric, question-to-passage nature as the reason the retriever's absolute similarity scores run low. Relevance is computed as <code>2·cos − 1</code>, so rank matters more than the raw number<sup id="fnref:4-rules-research"><a class="footnote-ref" href="#fn:4-rules-research" title="rules-research — references-meta/docs/rules/research.md">6</a></sup>. That's also why the gap-marking gate for Spark-corpus drafting tells writers not to mark a claim <code>[UNRESOURCED]</code> just because a retrieved chunk's relevance score is low or slightly negative: with this asymmetric model, a positive-or-slightly-negative-relevance chunk can still visibly answer the question and should be extracted, and <code>[UNRESOURCED]</code> is reserved for when no retrieved chunk supports any part of the answer at all<sup id="fnref2:4-rules-research"><a class="footnote-ref" href="#fn:4-rules-research" title="rules-research — references-meta/docs/rules/research.md">6</a></sup>.</p>
|
|
692
|
-
<p>The two corpora are selected via <code>--corpus {spark,meta}</code> on every pipeline script, and <code>scripts/corpus.py</code>'s <code>Corpus</code> dataclass keeps every path and setting for each profile fully separate: registry, per-question answers dir, section-projection dir, content-state sidecar, content manifest, references dir, Chroma path/collection, embedding model, build output, and site title all come from the selected profile rather than a hardcoded constant<sup id="fnref4:4-architecture"><a class="footnote-ref" href="#fn:4-architecture" title="architecture — references-meta/docs/architecture.md">1</a></sup>. The Spark corpus is indexed with the asymmetric question→passage retriever <code>multi-qa-mpnet-base-dot-v1</code>, chosen for retrieving over <code>references/</code>, while the meta corpus is indexed with the general symmetric model <code>all-mpnet-base-v2</code> into its own Chroma collection, <code>meta_docs</code> in <code>.chromadb-meta/</code><sup id="fnref5:4-architecture"><a class="footnote-ref" href="#fn:4-architecture" title="architecture — references-meta/docs/architecture.md">1</a></sup>. That model choice tracks what each corpus actually contains: the meta corpus has no book sources<sup id="fnref3:4-rules-research"><a class="footnote-ref" href="#fn:4-rules-research" title="rules-research — references-meta/docs/rules/research.md">6</a></sup> and consists of general-purpose prose (docs, rules, agent/skill files describing this repo's own pipeline)<sup id="fnref6:4-architecture"><a class="footnote-ref" href="#fn:4-architecture" title="architecture — references-meta/docs/architecture.md">1</a></sup>, which is why it uses a symmetric embedding model rather than the Spark corpus's asymmetric one<sup id="fnref4:4-rules-research"><a class="footnote-ref" href="#fn:4-rules-research" title="rules-research — references-meta/docs/rules/research.md">6</a></sup>.</p>
|
|
693
|
-
<p>Beyond the embedding model, the two instances share no state at any layer: no registry, no Chroma collection, no content manifest. Each is a fully independent instance of the same pipeline shape (registry question → RAG draft/verify/redraft → section projection → content page → build)<sup id="fnref7:4-architecture"><a class="footnote-ref" href="#fn:4-architecture" title="architecture — references-meta/docs/architecture.md">1</a></sup>.</p>
|
|
694
|
-
<h3>The append-only registry</h3>
|
|
695
|
-
<p><code>research/questions.yaml</code> never has an existing record mutated: new questions are added by appending a fresh <code>pending</code> record, and existing records keep their <code>id</code>, <code>section</code>, and <code>text</code> forever once written<sup id="fnref4:4-rules-registry"><a class="footnote-ref" href="#fn:4-rules-registry" title="rules-registry — references-meta/docs/rules/registry.md">2</a></sup>. Two scripts enforce this:</p>
|
|
696
|
-
<ul>
|
|
697
|
-
<li><code>scripts/add_question.py</code> mints a new id and appends a <code>pending</code> record; it never mutates an existing record<sup id="fnref6:4-rules-registry"><a class="footnote-ref" href="#fn:4-rules-registry" title="rules-registry — references-meta/docs/rules/registry.md">2</a></sup><sup id="fnref10:4-architecture"><a class="footnote-ref" href="#fn:4-architecture" title="architecture — references-meta/docs/architecture.md">1</a></sup>.</li>
|
|
698
|
-
<li><code>scripts/recompute_question.py</code> flips one existing record's status to <code>pending</code> and backs up its old answer file, but touches no other record or file<sup id="fnref7:4-rules-registry"><a class="footnote-ref" href="#fn:4-rules-registry" title="rules-registry — references-meta/docs/rules/registry.md">2</a></sup><sup id="fnref11:4-architecture"><a class="footnote-ref" href="#fn:4-architecture" title="architecture — references-meta/docs/architecture.md">1</a></sup>.</li>
|
|
699
|
-
</ul>
|
|
700
|
-
<p>Separately, <code>scripts/registry.py</code> loads, saves, and validates <code>questions.yaml</code> for the <code>Registry</code>/<code>Section</code>/<code>Question</code> dataclasses<sup id="fnref8:4-architecture"><a class="footnote-ref" href="#fn:4-architecture" title="architecture — references-meta/docs/architecture.md">1</a></sup>. The rules also forbid hand-editing an existing record's <code>id</code>, <code>section</code>, or <code>text</code>; a changed question is added as a new record instead of being rewritten in place<sup id="fnref5:4-rules-registry"><a class="footnote-ref" href="#fn:4-rules-registry" title="rules-registry — references-meta/docs/rules/registry.md">2</a></sup>. Because of this, the rest of the pipeline treats <code>questions.yaml</code> as the one-way source of truth: recompute always starts at the registry and flows forward through per-question answers, section projections, and content pages, never the other direction<sup id="fnref9:4-architecture"><a class="footnote-ref" href="#fn:4-architecture" title="architecture — references-meta/docs/architecture.md">1</a></sup>.</p>
|
|
701
|
-
<h3>Sources</h3>
|
|
702
|
-
<div class="footnote">
|
|
703
|
-
<hr />
|
|
704
|
-
<ol>
|
|
705
|
-
<li id="fn:4-architecture">
|
|
706
|
-
<p>architecture — references-meta/docs/architecture.md <a class="footnote-backref" href="#fnref:4-architecture" title="Jump back to footnote 1 in the text">↩</a><a class="footnote-backref" href="#fnref2:4-architecture" title="Jump back to footnote 1 in the text">↩</a><a class="footnote-backref" href="#fnref3:4-architecture" title="Jump back to footnote 1 in the text">↩</a><a class="footnote-backref" href="#fnref4:4-architecture" title="Jump back to footnote 1 in the text">↩</a><a class="footnote-backref" href="#fnref5:4-architecture" title="Jump back to footnote 1 in the text">↩</a><a class="footnote-backref" href="#fnref6:4-architecture" title="Jump back to footnote 1 in the text">↩</a><a class="footnote-backref" href="#fnref7:4-architecture" title="Jump back to footnote 1 in the text">↩</a><a class="footnote-backref" href="#fnref8:4-architecture" title="Jump back to footnote 1 in the text">↩</a><a class="footnote-backref" href="#fnref9:4-architecture" title="Jump back to footnote 1 in the text">↩</a><a class="footnote-backref" href="#fnref10:4-architecture" title="Jump back to footnote 1 in the text">↩</a><a class="footnote-backref" href="#fnref11:4-architecture" title="Jump back to footnote 1 in the text">↩</a></p>
|
|
707
|
-
</li>
|
|
708
|
-
<li id="fn:4-rules-registry">
|
|
709
|
-
<p>rules-registry — references-meta/docs/rules/registry.md <a class="footnote-backref" href="#fnref:4-rules-registry" title="Jump back to footnote 2 in the text">↩</a><a class="footnote-backref" href="#fnref2:4-rules-registry" title="Jump back to footnote 2 in the text">↩</a><a class="footnote-backref" href="#fnref3:4-rules-registry" title="Jump back to footnote 2 in the text">↩</a><a class="footnote-backref" href="#fnref4:4-rules-registry" title="Jump back to footnote 2 in the text">↩</a><a class="footnote-backref" href="#fnref5:4-rules-registry" title="Jump back to footnote 2 in the text">↩</a><a class="footnote-backref" href="#fnref6:4-rules-registry" title="Jump back to footnote 2 in the text">↩</a><a class="footnote-backref" href="#fnref7:4-rules-registry" title="Jump back to footnote 2 in the text">↩</a></p>
|
|
710
|
-
</li>
|
|
711
|
-
<li id="fn:4-runbook">
|
|
712
|
-
<p>runbook — references-meta/docs/runbook.md <a class="footnote-backref" href="#fnref:4-runbook" title="Jump back to footnote 3 in the text">↩</a><a class="footnote-backref" href="#fnref2:4-runbook" title="Jump back to footnote 3 in the text">↩</a></p>
|
|
713
|
-
</li>
|
|
714
|
-
<li id="fn:4-agent-meta-draft">
|
|
715
|
-
<p>agent-meta-draft — references-meta/agents/meta-section-draft.md <a class="footnote-backref" href="#fnref:4-agent-meta-draft" title="Jump back to footnote 4 in the text">↩</a></p>
|
|
716
|
-
</li>
|
|
717
|
-
<li id="fn:4-skill-answer-question">
|
|
718
|
-
<p>skill-answer-question — references-meta/skills/answer-question.md <a class="footnote-backref" href="#fnref:4-skill-answer-question" title="Jump back to footnote 5 in the text">↩</a><a class="footnote-backref" href="#fnref2:4-skill-answer-question" title="Jump back to footnote 5 in the text">↩</a></p>
|
|
719
|
-
</li>
|
|
720
|
-
<li id="fn:4-rules-research">
|
|
721
|
-
<p>rules-research — references-meta/docs/rules/research.md <a class="footnote-backref" href="#fnref:4-rules-research" title="Jump back to footnote 6 in the text">↩</a><a class="footnote-backref" href="#fnref2:4-rules-research" title="Jump back to footnote 6 in the text">↩</a><a class="footnote-backref" href="#fnref3:4-rules-research" title="Jump back to footnote 6 in the text">↩</a><a class="footnote-backref" href="#fnref4:4-rules-research" title="Jump back to footnote 6 in the text">↩</a></p>
|
|
722
|
-
</li>
|
|
723
|
-
</ol>
|
|
724
|
-
</div>
|
|
725
|
-
</section>
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
<div class="content-group-title">Agents & Orchestration</div>
|
|
729
|
-
|
|
730
|
-
<section id="agents-and-orchestration" class="doc-section">
|
|
731
|
-
<h2>Agents & Orchestration</h2>
|
|
732
|
-
<h3>Draft, Verify, and Redraft</h3>
|
|
733
|
-
<p>The pipeline splits judgment-requiring work (retrieval, drafting, verifying citations, fixing issues, humanizing prose) into separate agents rather than folding it into deterministic scripts, because that work has no single correct output given its inputs<sup id="fnref:5-architecture"><a class="footnote-ref" href="#fn:5-architecture" title="architecture — references-meta/docs/architecture.md">1</a></sup>. For the meta corpus these are <code>meta-section-draft</code>, <code>meta-section-verify</code>, and <code>meta-section-redraft</code>, mirrors of the Spark corpus's <code>spark-section-draft</code>/<code>spark-section-verify</code>/<code>spark-section-redraft</code>, adapted to the meta corpus's paths and <code>--corpus meta</code> invocations but not sharing prompt text with them<sup id="fnref2:5-architecture"><a class="footnote-ref" href="#fn:5-architecture" title="architecture — references-meta/docs/architecture.md">1</a></sup>.</p>
|
|
734
|
-
<p><strong>Draft</strong> produces one citation-backed answer per question from the local RAG index. It writes the finished answer using an <code>## <question text></code> heading with prose beneath, footnotes every factual claim with the source ID <code>ask.py</code> printed, and closes with a <code>## Sources</code> section<sup id="fnref:5-agent-spark-draft"><a class="footnote-ref" href="#fn:5-agent-spark-draft" title="agent-spark-draft — references-meta/agents/spark-section-draft.md">2</a></sup>. For the meta corpus this same agent role is described as drafting one section of the meta-doc "one citation-backed answer per question," acting as the Draft stage of the meta answering pipeline<sup id="fnref:5-agent-meta-draft"><a class="footnote-ref" href="#fn:5-agent-meta-draft" title="agent-meta-draft — references-meta/agents/meta-section-draft.md">3</a></sup>.</p>
|
|
735
|
-
<p><strong>Verify</strong> checks a drafted section: it validates gaps, checks every cited claim against its underlying chunk, and checks that the <code>## Sources</code> block is formatted correctly<sup id="fnref:5-agent-meta-verify"><a class="footnote-ref" href="#fn:5-agent-meta-verify" title="agent-meta-verify — references-meta/agents/meta-section-verify.md">4</a></sup>.</p>
|
|
736
|
-
<p><strong>Redraft</strong> is dispatched only if Verify reports issues, and only once, taking those issues as input; after it runs, the section is re-verified a single time. If it still fails after that one redraft-and-re-verify cycle, the answer keeps whatever <code>[NEEDS-REVIEW: ...]</code> markers remain, and the question's registry record is set to <code>needs-review</code> rather than <code>answered</code><sup id="fnref:5-skill-answer-meta-question"><a class="footnote-ref" href="#fn:5-skill-answer-meta-question" title="skill-answer-meta-question — references-meta/skills/answer-meta-question.md">5</a></sup>.</p>
|
|
737
|
-
<p>The <code>answer-meta-question</code> skill runs these stages in a fixed sequence for a single question: read the registry record, dispatch Draft, dispatch Verify, dispatch Redraft-if-needed followed by one re-verify, then Humanize, project the section, update the record status, refresh content state, and finally build and commit<sup id="fnref2:5-skill-answer-meta-question"><a class="footnote-ref" href="#fn:5-skill-answer-meta-question" title="skill-answer-meta-question — references-meta/skills/answer-meta-question.md">5</a></sup>. This order (Draft → Verify → an optional Redraft-then-re-Verify loop → Humanize → downstream projection and build steps → commit) is the same sequence the answering pipeline documents for orchestrating one question end to end<sup id="fnref3:5-architecture"><a class="footnote-ref" href="#fn:5-architecture" title="architecture — references-meta/docs/architecture.md">1</a></sup>.</p>
|
|
738
|
-
<h3>Why citations are scoped per question</h3>
|
|
739
|
-
<p>The verify agent's citation check requires two things for every footnoted claim: the source ID must appear on THAT question's own <code>citable source_ids</code> line, and the chunk excerpt under that question must actually support the claim. A footnote whose source ID is not on that question's <code>citable source_ids</code> line is a FAIL even if the underlying fact is true. The claim-support check against that question's own chunk text is the real gate, not the mere existence of the ID somewhere in the corpus<sup id="fnref:5-agent-spark-verify"><a class="footnote-ref" href="#fn:5-agent-spark-verify" title="agent-spark-verify — references-meta/agents/spark-section-verify.md">6</a></sup><sup id="fnref2:5-agent-meta-verify"><a class="footnote-ref" href="#fn:5-agent-meta-verify" title="agent-meta-verify — references-meta/agents/meta-section-verify.md">4</a></sup>.</p>
|
|
740
|
-
<p>The reason the same ID can't cross questions is that retrieval is per-question: a source ID retrieved under a different question's block does not license citing it here, because the chunk <em>text</em> returned for that ID differs per question<sup id="fnref:5-rules-research"><a class="footnote-ref" href="#fn:5-rules-research" title="rules-research — references-meta/docs/rules/research.md">7</a></sup>. The Draft stage states the same constraint from the writer's side: answer each question from its own retrieval block only, since a source can be retrieved for several questions but with different chunks, and a fact seen under one question's chunks does not license citing that source under another question unless that question's own chunk text states it. The held-out gate that motivated this rule caught exactly this failure: a true <code>peakExecutionMemory</code> definition retrieved under one question was cited under a different question whose own chunks only held an executor-level table<sup id="fnref2:5-agent-spark-draft"><a class="footnote-ref" href="#fn:5-agent-spark-draft" title="agent-spark-draft — references-meta/agents/spark-section-draft.md">2</a></sup>.</p>
|
|
741
|
-
<p>This is why Redraft treats a "misattributed footnote" (source ID not on that question's <code>citable source_ids</code> line) as something to fix rather than wave through: either re-cite to a source ID that is on that question's line and supports the claim, or mark the claim <code>[NEEDS-REVIEW: <reason>]</code> and remove the wrong footnote, never leaving an orphan footnote<sup id="fnref:5-agent-meta-redraft"><a class="footnote-ref" href="#fn:5-agent-meta-redraft" title="agent-meta-redraft — references-meta/agents/meta-section-redraft.md">8</a></sup><sup id="fnref:5-agent-spark-redraft"><a class="footnote-ref" href="#fn:5-agent-spark-redraft" title="agent-spark-redraft — references-meta/agents/spark-section-redraft.md">9</a></sup>.</p>
|
|
742
|
-
<h3>Humanize and its deterministic backstop</h3>
|
|
743
|
-
<p>The Humanize stage applies prose style to a drafted answer. It is a separate step run by the <code>writing-prose-like-a-human-for-agents:prose-humanizer</code> agent (an installed plugin), not by the Draft/Verify/Redraft agents themselves<sup id="fnref:5-rules-editorial"><a class="footnote-ref" href="#fn:5-rules-editorial" title="rules-editorial — references-meta/docs/rules/editorial.md">10</a></sup><sup id="fnref4:5-architecture"><a class="footnote-ref" href="#fn:5-architecture" title="architecture — references-meta/docs/architecture.md">1</a></sup>. In the per-question skill flow it sits as step 5 of 8, after Draft, Verify, and any Redraft/re-Verify cycle, and before <code>check_prose</code>, <code>project_section</code>, and <code>content_state</code><sup id="fnref5:5-architecture"><a class="footnote-ref" href="#fn:5-architecture" title="architecture — references-meta/docs/architecture.md">1</a></sup><sup id="fnref2:5-rules-editorial"><a class="footnote-ref" href="#fn:5-rules-editorial" title="rules-editorial — references-meta/docs/rules/editorial.md">10</a></sup>. For the meta corpus specifically, the humanizer runs against <code>research/meta/answers/q/<id>.md</code> (copied first to a unique scratch path) under hard constraints: it must never touch <code>[^id]</code> footnotes, the <code>## Sources</code> block, <code>[UNRESOURCED]</code>/<code>[NEEDS-REVIEW]</code> markers, or the <code>## <question></code> heading<sup id="fnref3:5-skill-answer-meta-question"><a class="footnote-ref" href="#fn:5-skill-answer-meta-question" title="skill-answer-meta-question — references-meta/skills/answer-meta-question.md">5</a></sup>.</p>
|
|
744
|
-
<p>The deterministic backstop is <code>scripts/check_prose.py</code>, described as a "no-LLM lint/backstop" for the Humanize stage<sup id="fnref6:5-architecture"><a class="footnote-ref" href="#fn:5-architecture" title="architecture — references-meta/docs/architecture.md">1</a></sup>. It runs in two forms after every humanize pass. Plain <code>check_prose.py <file></code> lints the humanized file for leftover AI-tell vocabulary (significance inflation, inflated verbs, structural tics such as conclusion phrases or additive transitions, vague attribution), bans curly quotes, and caps em dashes at zero<sup id="fnref3:5-rules-editorial"><a class="footnote-ref" href="#fn:5-rules-editorial" title="rules-editorial — references-meta/docs/rules/editorial.md">10</a></sup><sup id="fnref7:5-architecture"><a class="footnote-ref" href="#fn:5-architecture" title="architecture — references-meta/docs/architecture.md">1</a></sup>. <code>check_prose.py --diff <before> <after></code> instead checks that the humanizer didn't touch a protected element: <code>[^id]</code> footnotes, the <code>## Sources</code> block, <code>[UNRESOURCED]</code>/<code>[NEEDS-REVIEW]</code> markers, the <code>## <question></code> heading, or any numeric/config token<sup id="fnref4:5-rules-editorial"><a class="footnote-ref" href="#fn:5-rules-editorial" title="rules-editorial — references-meta/docs/rules/editorial.md">10</a></sup><sup id="fnref8:5-architecture"><a class="footnote-ref" href="#fn:5-architecture" title="architecture — references-meta/docs/architecture.md">1</a></sup>. For the meta pipeline, both invocations run as <code>uv run python -m scripts.check_prose --diff <scratch-copy> research/meta/answers/q/<id>.md</code> and <code>uv run python -m scripts.check_prose research/meta/answers/q/<id>.md</code><sup id="fnref4:5-skill-answer-meta-question"><a class="footnote-ref" href="#fn:5-skill-answer-meta-question" title="skill-answer-meta-question — references-meta/skills/answer-meta-question.md">5</a></sup>.</p>
|
|
745
|
-
<p>The vocabulary/structural list check_prose applies is a literal blocklist rather than semantic understanding, so it can false-positive on legitimate technical usage (the rule's own example is a Spark phrase about broadcast joins that trips the blocklist even though it is idiomatic)<sup id="fnref5:5-rules-editorial"><a class="footnote-ref" href="#fn:5-rules-editorial" title="rules-editorial — references-meta/docs/rules/editorial.md">10</a></sup>. A lint failure is treated as a signal to re-dispatch the humanizer once rather than an automatic hard block<sup id="fnref6:5-rules-editorial"><a class="footnote-ref" href="#fn:5-rules-editorial" title="rules-editorial — references-meta/docs/rules/editorial.md">10</a></sup>; in the meta procedure, if either check fails the humanizer is re-dispatched once with the reported issues and both checks re-run, and if still failing the issues are left in place and noted in the final status<sup id="fnref5:5-skill-answer-meta-question"><a class="footnote-ref" href="#fn:5-skill-answer-meta-question" title="skill-answer-meta-question — references-meta/skills/answer-meta-question.md">5</a></sup>.</p>
|
|
746
|
-
<h3>Sources</h3>
|
|
747
|
-
<div class="footnote">
|
|
748
|
-
<hr />
|
|
749
|
-
<ol>
|
|
750
|
-
<li id="fn:5-architecture">
|
|
751
|
-
<p>architecture — references-meta/docs/architecture.md <a class="footnote-backref" href="#fnref:5-architecture" title="Jump back to footnote 1 in the text">↩</a><a class="footnote-backref" href="#fnref2:5-architecture" title="Jump back to footnote 1 in the text">↩</a><a class="footnote-backref" href="#fnref3:5-architecture" title="Jump back to footnote 1 in the text">↩</a><a class="footnote-backref" href="#fnref4:5-architecture" title="Jump back to footnote 1 in the text">↩</a><a class="footnote-backref" href="#fnref5:5-architecture" title="Jump back to footnote 1 in the text">↩</a><a class="footnote-backref" href="#fnref6:5-architecture" title="Jump back to footnote 1 in the text">↩</a><a class="footnote-backref" href="#fnref7:5-architecture" title="Jump back to footnote 1 in the text">↩</a><a class="footnote-backref" href="#fnref8:5-architecture" title="Jump back to footnote 1 in the text">↩</a></p>
|
|
752
|
-
</li>
|
|
753
|
-
<li id="fn:5-agent-spark-draft">
|
|
754
|
-
<p>agent-spark-draft — references-meta/agents/spark-section-draft.md <a class="footnote-backref" href="#fnref:5-agent-spark-draft" title="Jump back to footnote 2 in the text">↩</a><a class="footnote-backref" href="#fnref2:5-agent-spark-draft" title="Jump back to footnote 2 in the text">↩</a></p>
|
|
755
|
-
</li>
|
|
756
|
-
<li id="fn:5-agent-meta-draft">
|
|
757
|
-
<p>agent-meta-draft — references-meta/agents/meta-section-draft.md <a class="footnote-backref" href="#fnref:5-agent-meta-draft" title="Jump back to footnote 3 in the text">↩</a></p>
|
|
758
|
-
</li>
|
|
759
|
-
<li id="fn:5-agent-meta-verify">
|
|
760
|
-
<p>agent-meta-verify — references-meta/agents/meta-section-verify.md <a class="footnote-backref" href="#fnref:5-agent-meta-verify" title="Jump back to footnote 4 in the text">↩</a><a class="footnote-backref" href="#fnref2:5-agent-meta-verify" title="Jump back to footnote 4 in the text">↩</a></p>
|
|
761
|
-
</li>
|
|
762
|
-
<li id="fn:5-skill-answer-meta-question">
|
|
763
|
-
<p>skill-answer-meta-question — references-meta/skills/answer-meta-question.md <a class="footnote-backref" href="#fnref:5-skill-answer-meta-question" title="Jump back to footnote 5 in the text">↩</a><a class="footnote-backref" href="#fnref2:5-skill-answer-meta-question" title="Jump back to footnote 5 in the text">↩</a><a class="footnote-backref" href="#fnref3:5-skill-answer-meta-question" title="Jump back to footnote 5 in the text">↩</a><a class="footnote-backref" href="#fnref4:5-skill-answer-meta-question" title="Jump back to footnote 5 in the text">↩</a><a class="footnote-backref" href="#fnref5:5-skill-answer-meta-question" title="Jump back to footnote 5 in the text">↩</a></p>
|
|
764
|
-
</li>
|
|
765
|
-
<li id="fn:5-agent-spark-verify">
|
|
766
|
-
<p>agent-spark-verify — references-meta/agents/spark-section-verify.md <a class="footnote-backref" href="#fnref:5-agent-spark-verify" title="Jump back to footnote 6 in the text">↩</a></p>
|
|
767
|
-
</li>
|
|
768
|
-
<li id="fn:5-rules-research">
|
|
769
|
-
<p>rules-research — references-meta/docs/rules/research.md <a class="footnote-backref" href="#fnref:5-rules-research" title="Jump back to footnote 7 in the text">↩</a></p>
|
|
770
|
-
</li>
|
|
771
|
-
<li id="fn:5-agent-meta-redraft">
|
|
772
|
-
<p>agent-meta-redraft — references-meta/agents/meta-section-redraft.md <a class="footnote-backref" href="#fnref:5-agent-meta-redraft" title="Jump back to footnote 8 in the text">↩</a></p>
|
|
773
|
-
</li>
|
|
774
|
-
<li id="fn:5-agent-spark-redraft">
|
|
775
|
-
<p>agent-spark-redraft — references-meta/agents/spark-section-redraft.md <a class="footnote-backref" href="#fnref:5-agent-spark-redraft" title="Jump back to footnote 9 in the text">↩</a></p>
|
|
776
|
-
</li>
|
|
777
|
-
<li id="fn:5-rules-editorial">
|
|
778
|
-
<p>rules-editorial — references-meta/docs/rules/editorial.md <a class="footnote-backref" href="#fnref:5-rules-editorial" title="Jump back to footnote 10 in the text">↩</a><a class="footnote-backref" href="#fnref2:5-rules-editorial" title="Jump back to footnote 10 in the text">↩</a><a class="footnote-backref" href="#fnref3:5-rules-editorial" title="Jump back to footnote 10 in the text">↩</a><a class="footnote-backref" href="#fnref4:5-rules-editorial" title="Jump back to footnote 10 in the text">↩</a><a class="footnote-backref" href="#fnref5:5-rules-editorial" title="Jump back to footnote 10 in the text">↩</a><a class="footnote-backref" href="#fnref6:5-rules-editorial" title="Jump back to footnote 10 in the text">↩</a></p>
|
|
779
|
-
</li>
|
|
780
|
-
</ol>
|
|
781
|
-
</div>
|
|
782
|
-
</section>
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
<div class="content-group-title">Content, Staleness & Build</div>
|
|
786
|
-
|
|
787
|
-
<section id="content-staleness-and-build" class="doc-section">
|
|
788
|
-
<h2>Content, Staleness & Build</h2>
|
|
789
|
-
<h3>Ungenerated vs. stale</h3>
|
|
790
|
-
<p><code>content_state.py</code> distinguishes the two states by comparing a page's recorded hash in the committed sidecar <code>research/content-state.yaml</code> against the sha256 of the section projection it was assembled from<sup id="fnref:6-architecture"><a class="footnote-ref" href="#fn:6-architecture" title="architecture — references-meta/docs/architecture.md">1</a></sup>. A page with no entry in that sidecar has never been generated from its section projection, and <code>content_state.py</code> reports it <strong>ungenerated</strong><sup id="fnref:6-runbook"><a class="footnote-ref" href="#fn:6-runbook" title="runbook — references-meta/docs/runbook.md">2</a></sup>. A page has an entry whose recorded hash no longer matches the current projection hash when the section's underlying answers changed since the page was generated or last accepted; <code>content_state.py</code> reports it <strong>stale</strong><sup id="fnref2:6-architecture"><a class="footnote-ref" href="#fn:6-architecture" title="architecture — references-meta/docs/architecture.md">1</a></sup><sup id="fnref2:6-runbook"><a class="footnote-ref" href="#fn:6-runbook" title="runbook — references-meta/docs/runbook.md">2</a></sup>.</p>
|
|
791
|
-
<p>The required human action differs accordingly. For an <strong>ungenerated</strong> page, the pipeline generates it: the answer-meta-question skill reads the page's <code>brief:</code> and <code>assembled_from:</code> entries from <code>content/meta/manifest.yaml</code>, writes the page from the projection, humanizes it, then runs <code>content_state --accept <page></code> to stamp its hash<sup id="fnref:6-skill-answer-meta-question"><a class="footnote-ref" href="#fn:6-skill-answer-meta-question" title="skill-answer-meta-question — references-meta/skills/answer-meta-question.md">3</a></sup>. For a <strong>stale</strong> page, nothing is overwritten automatically. The pipeline does not edit the page; a human hand-edits it to reflect the changed answers, and only then does <code>content_state.py --accept <page></code> re-stamp the hash so the page is no longer flagged<sup id="fnref3:6-runbook"><a class="footnote-ref" href="#fn:6-runbook" title="runbook — references-meta/docs/runbook.md">2</a></sup><sup id="fnref3:6-architecture"><a class="footnote-ref" href="#fn:6-architecture" title="architecture — references-meta/docs/architecture.md">1</a></sup>. This follows from content pages being human-owned once generated: the pipeline never silently overwrites a generated page<sup id="fnref:6-rules-content"><a class="footnote-ref" href="#fn:6-rules-content" title="rules-content — references-meta/docs/rules/content.md">4</a></sup>.</p>
|
|
792
|
-
<h3>Anchors as a frozen cross-repo contract</h3>
|
|
793
|
-
<p><code>content/manifest.yaml</code> is the single authority for section order, titles, and anchor IDs; content assembly reads anchors from it and never derives them from a Markdown heading or a filename<sup id="fnref2:6-rules-content"><a class="footnote-ref" href="#fn:6-rules-content" title="rules-content — references-meta/docs/rules/content.md">4</a></sup><sup id="fnref:6-anchor-map"><a class="footnote-ref" href="#fn:6-anchor-map" title="anchor-map — references-meta/docs/anchor-map.md">5</a></sup>. Because the anchors are looked up rather than generated from content, other repos can depend on them staying stable, so the project documents them as a frozen cross-repo Integration Contract with the sibling repo <code>sparkforensics</code><sup id="fnref3:6-rules-content"><a class="footnote-ref" href="#fn:6-rules-content" title="rules-content — references-meta/docs/rules/content.md">4</a></sup><sup id="fnref2:6-anchor-map"><a class="footnote-ref" href="#fn:6-anchor-map" title="anchor-map — references-meta/docs/anchor-map.md">5</a></sup>. Renaming or removing an anchor is a breaking change, and if the two repos' descriptions of an anchor diverge, the sibling repo's own contract doc wins<sup id="fnref4:6-rules-content"><a class="footnote-ref" href="#fn:6-rules-content" title="rules-content — references-meta/docs/rules/content.md">4</a></sup>.</p>
|
|
794
|
-
<p>The contract covers two sets of IDs: the 26 manifest-level anchors, one per <code>content/manifest.yaml</code> entry (enforced by <code>scripts/smoke_content.py</code>'s 26-section count check), and 4 config-page sub-anchors nested inside the single <code>config</code> manifest entry (<code>config-shuffle-service</code>, <code>config-autoscale-bounds</code>, <code>config-serializer</code>, <code>config-memory-overhead</code>). These four aren't manifest entries themselves but are also enforced by <code>scripts/smoke_content.py</code><sup id="fnref5:6-rules-content"><a class="footnote-ref" href="#fn:6-rules-content" title="rules-content — references-meta/docs/rules/content.md">4</a></sup>. The contract applies only to <code>content/manifest.yaml</code>/<code>index.html</code>, i.e. the Spark corpus; <code>content/meta/manifest.yaml</code> and <code>meta.html</code> carry no such contract, so this meta corpus's own anchors may change freely<sup id="fnref6:6-rules-content"><a class="footnote-ref" href="#fn:6-rules-content" title="rules-content — references-meta/docs/rules/content.md">4</a></sup>.</p>
|
|
795
|
-
<h3>The build's non-fatal staleness warning</h3>
|
|
796
|
-
<p>Before rendering, <code>build.py</code> calls <code>content_state.report(...)</code>, which compares each content page's recorded hash (from <code>research/content-state.yaml</code>) against the current section-projection hash and classifies the page as ungenerated (no recorded entry), stale (hash differs), or up to date (hash matches)<sup id="fnref4:6-architecture"><a class="footnote-ref" href="#fn:6-architecture" title="architecture — references-meta/docs/architecture.md">1</a></sup>. If any pages come back ungenerated or stale, <code>build.py</code> prints a non-fatal warning banner listing them, but proceeds to produce <code>index.html</code> regardless<sup id="fnref5:6-architecture"><a class="footnote-ref" href="#fn:6-architecture" title="architecture — references-meta/docs/architecture.md">1</a></sup><sup id="fnref4:6-runbook"><a class="footnote-ref" href="#fn:6-runbook" title="runbook — references-meta/docs/runbook.md">2</a></sup>.</p>
|
|
797
|
-
<p>The warning is non-fatal because content pages are human-owned once generated: the pipeline never overwrites a page's prose automatically, even when its upstream answers have changed<sup id="fnref6:6-architecture"><a class="footnote-ref" href="#fn:6-architecture" title="architecture — references-meta/docs/architecture.md">1</a></sup>. A stale or ungenerated flag just means the built page's text may not reflect the latest answers yet. Fixing it requires a human to edit the page (or run first-gen for an ungenerated one) and then re-stamp its hash with <code>content_state.py --accept <page></code> so it reports up to date again<sup id="fnref5:6-runbook"><a class="footnote-ref" href="#fn:6-runbook" title="runbook — references-meta/docs/runbook.md">2</a></sup><sup id="fnref7:6-architecture"><a class="footnote-ref" href="#fn:6-architecture" title="architecture — references-meta/docs/architecture.md">1</a></sup>. Recompute itself never triggers first-gen; it only ever flags a page stale, so content pages stay human-owned<sup id="fnref8:6-architecture"><a class="footnote-ref" href="#fn:6-architecture" title="architecture — references-meta/docs/architecture.md">1</a></sup>.</p>
|
|
798
|
-
<h3>Sources</h3>
|
|
799
|
-
<div class="footnote">
|
|
800
|
-
<hr />
|
|
801
|
-
<ol>
|
|
802
|
-
<li id="fn:6-architecture">
|
|
803
|
-
<p>architecture — references-meta/docs/architecture.md <a class="footnote-backref" href="#fnref:6-architecture" title="Jump back to footnote 1 in the text">↩</a><a class="footnote-backref" href="#fnref2:6-architecture" title="Jump back to footnote 1 in the text">↩</a><a class="footnote-backref" href="#fnref3:6-architecture" title="Jump back to footnote 1 in the text">↩</a><a class="footnote-backref" href="#fnref4:6-architecture" title="Jump back to footnote 1 in the text">↩</a><a class="footnote-backref" href="#fnref5:6-architecture" title="Jump back to footnote 1 in the text">↩</a><a class="footnote-backref" href="#fnref6:6-architecture" title="Jump back to footnote 1 in the text">↩</a><a class="footnote-backref" href="#fnref7:6-architecture" title="Jump back to footnote 1 in the text">↩</a><a class="footnote-backref" href="#fnref8:6-architecture" title="Jump back to footnote 1 in the text">↩</a></p>
|
|
804
|
-
</li>
|
|
805
|
-
<li id="fn:6-runbook">
|
|
806
|
-
<p>runbook — references-meta/docs/runbook.md <a class="footnote-backref" href="#fnref:6-runbook" title="Jump back to footnote 2 in the text">↩</a><a class="footnote-backref" href="#fnref2:6-runbook" title="Jump back to footnote 2 in the text">↩</a><a class="footnote-backref" href="#fnref3:6-runbook" title="Jump back to footnote 2 in the text">↩</a><a class="footnote-backref" href="#fnref4:6-runbook" title="Jump back to footnote 2 in the text">↩</a><a class="footnote-backref" href="#fnref5:6-runbook" title="Jump back to footnote 2 in the text">↩</a></p>
|
|
807
|
-
</li>
|
|
808
|
-
<li id="fn:6-skill-answer-meta-question">
|
|
809
|
-
<p>skill-answer-meta-question — references-meta/skills/answer-meta-question.md <a class="footnote-backref" href="#fnref:6-skill-answer-meta-question" title="Jump back to footnote 3 in the text">↩</a></p>
|
|
810
|
-
</li>
|
|
811
|
-
<li id="fn:6-rules-content">
|
|
812
|
-
<p>rules-content — references-meta/docs/rules/content.md <a class="footnote-backref" href="#fnref:6-rules-content" title="Jump back to footnote 4 in the text">↩</a><a class="footnote-backref" href="#fnref2:6-rules-content" title="Jump back to footnote 4 in the text">↩</a><a class="footnote-backref" href="#fnref3:6-rules-content" title="Jump back to footnote 4 in the text">↩</a><a class="footnote-backref" href="#fnref4:6-rules-content" title="Jump back to footnote 4 in the text">↩</a><a class="footnote-backref" href="#fnref5:6-rules-content" title="Jump back to footnote 4 in the text">↩</a><a class="footnote-backref" href="#fnref6:6-rules-content" title="Jump back to footnote 4 in the text">↩</a></p>
|
|
813
|
-
</li>
|
|
814
|
-
<li id="fn:6-anchor-map">
|
|
815
|
-
<p>anchor-map — references-meta/docs/anchor-map.md <a class="footnote-backref" href="#fnref:6-anchor-map" title="Jump back to footnote 5 in the text">↩</a><a class="footnote-backref" href="#fnref2:6-anchor-map" title="Jump back to footnote 5 in the text">↩</a></p>
|
|
816
|
-
</li>
|
|
817
|
-
</ol>
|
|
818
|
-
</div>
|
|
819
|
-
</section>
|
|
820
|
-
|
|
821
|
-
</main>
|
|
822
|
-
</div>
|
|
823
|
-
|
|
824
|
-
<script>
|
|
825
|
-
// postMessage bridge to the embedding sparkforensics app.
|
|
826
|
-
// Contract: Integration Contract with sparkforensics (anchor IDs frozen).
|
|
827
|
-
//
|
|
828
|
-
// SDOC-021 handshake nonce: the parent loads this iframe at an opaque
|
|
829
|
-
// (file:// / null) origin, so an origin allowlist can't be the trust anchor.
|
|
830
|
-
// Instead the parent passes a per-load token as ?tok=... in the iframe URL; we
|
|
831
|
-
// only accept inbound messages carrying that same token and echo it on every
|
|
832
|
-
// message we post. postMessage target stays '*' (parent origin is opaque) — the
|
|
833
|
-
// nonce is what makes '*' safe.
|
|
834
|
-
|
|
835
|
-
// Pure, side-effect-free trust decision so it can be unit-tested in Node.
|
|
836
|
-
// Transitional: only a truly ABSENT token (docs opened directly, or a pre-nonce
|
|
837
|
-
// parent that hasn't shipped its ?tok= side yet) falls back to accepting
|
|
838
|
-
// everything so nothing breaks before sparkforensics ships the matching change. A present-
|
|
839
|
-
// but-empty token (?tok= with no value) is a real, if degenerate, token and
|
|
840
|
-
// still gates. `expectedTok` comes from URLSearchParams.get('tok'), which is
|
|
841
|
-
// null when absent and '' when present-but-empty; `== null` covers null and the
|
|
842
|
-
// undefined passed when isTrusted is called with no token at all.
|
|
843
|
-
// Remove the null-expectedTok branch once sparkforensics always injects ?tok=.
|
|
844
|
-
function isTrusted(expectedTok, data) {
|
|
845
|
-
if (expectedTok == null) return true; // transitional: no handshake configured
|
|
846
|
-
return !!data && data.tok === expectedTok;
|
|
847
|
-
}
|
|
848
|
-
|
|
849
|
-
// CommonJS export for tests. Inert in the browser: this file is inlined into a
|
|
850
|
-
// classic <script> (see templates/page.html.jinja2) where `module` is
|
|
851
|
-
// undefined, so a bare ESM `export` would be a parse error — this guard isn't.
|
|
852
|
-
if (typeof module !== 'undefined' && module.exports) {
|
|
853
|
-
module.exports = { isTrusted };
|
|
854
|
-
}
|
|
855
|
-
|
|
856
|
-
// Browser wiring only — skipped when the module is required in Node for tests.
|
|
857
|
-
if (typeof window !== 'undefined' && typeof document !== 'undefined') {
|
|
858
|
-
// Handshake token from the query string (?tok=...). NOT the hash — the hash is
|
|
859
|
-
// reserved for #anchor navigation. Kept in this closure for the page's life.
|
|
860
|
-
const expectedTok = new URLSearchParams(window.location.search).get('tok');
|
|
861
|
-
|
|
862
|
-
window.addEventListener('message', (e) => {
|
|
863
|
-
if (!isTrusted(expectedTok, e.data)) return;
|
|
864
|
-
if (e.data?.type === 'theme') {
|
|
865
|
-
document.documentElement.setAttribute('data-theme', e.data.theme);
|
|
866
|
-
}
|
|
867
|
-
if (e.data?.type === 'navigate') {
|
|
868
|
-
document.querySelector(e.data.anchor)?.scrollIntoView({ behavior: 'smooth' });
|
|
869
|
-
}
|
|
870
|
-
});
|
|
871
|
-
|
|
872
|
-
document.addEventListener('DOMContentLoaded', () => {
|
|
873
|
-
window.parent.postMessage({ type: 'ready', tok: expectedTok }, '*');
|
|
874
|
-
|
|
875
|
-
const hash = window.location.hash;
|
|
876
|
-
if (hash && !document.querySelector(hash)) {
|
|
877
|
-
document.querySelector('#intro')?.scrollIntoView();
|
|
878
|
-
const fallback = document.getElementById('anchor-fallback');
|
|
879
|
-
if (fallback) {
|
|
880
|
-
fallback.hidden = false;
|
|
881
|
-
document.getElementById('anchor-fallback-dismiss')?.addEventListener('click', () => {
|
|
882
|
-
fallback.hidden = true;
|
|
883
|
-
});
|
|
884
|
-
}
|
|
885
|
-
}
|
|
886
|
-
|
|
887
|
-
const toggle = document.getElementById('nav-toggle');
|
|
888
|
-
const sidebar = document.getElementById('sidebar');
|
|
889
|
-
if (toggle && sidebar) {
|
|
890
|
-
toggle.addEventListener('click', () => {
|
|
891
|
-
const isOpen = sidebar.classList.toggle('sidebar-open');
|
|
892
|
-
toggle.setAttribute('aria-expanded', String(isOpen));
|
|
893
|
-
});
|
|
894
|
-
sidebar.querySelectorAll('.nav-link').forEach((link) => {
|
|
895
|
-
link.addEventListener('click', () => {
|
|
896
|
-
sidebar.classList.remove('sidebar-open');
|
|
897
|
-
toggle.setAttribute('aria-expanded', 'false');
|
|
898
|
-
});
|
|
899
|
-
});
|
|
900
|
-
}
|
|
901
|
-
|
|
902
|
-
const navSearch = document.getElementById('nav-search');
|
|
903
|
-
if (navSearch) {
|
|
904
|
-
navSearch.addEventListener('input', () => {
|
|
905
|
-
const query = navSearch.value.trim().toLowerCase();
|
|
906
|
-
document.querySelectorAll('.nav-group').forEach((group) => {
|
|
907
|
-
let anyVisible = false;
|
|
908
|
-
group.querySelectorAll('.nav-list > li').forEach((item) => {
|
|
909
|
-
const matches = !query || item.textContent.toLowerCase().includes(query);
|
|
910
|
-
item.hidden = !matches;
|
|
911
|
-
anyVisible = anyVisible || matches;
|
|
912
|
-
});
|
|
913
|
-
group.hidden = !anyVisible;
|
|
914
|
-
});
|
|
915
|
-
});
|
|
916
|
-
}
|
|
917
|
-
|
|
918
|
-
const navLinks = document.querySelectorAll('.nav-link');
|
|
919
|
-
const setActive = (anchor) => {
|
|
920
|
-
navLinks.forEach((link) => {
|
|
921
|
-
link.classList.toggle('active', link.getAttribute('href') === `#${anchor}`);
|
|
922
|
-
});
|
|
923
|
-
};
|
|
924
|
-
const sections = document.querySelectorAll('.doc-section[id]');
|
|
925
|
-
if (navLinks.length && sections.length && 'IntersectionObserver' in window) {
|
|
926
|
-
const observer = new IntersectionObserver(
|
|
927
|
-
(entries) => {
|
|
928
|
-
const visible = entries.filter((entry) => entry.isIntersecting);
|
|
929
|
-
if (visible.length) {
|
|
930
|
-
const topmost = visible.reduce((a, b) =>
|
|
931
|
-
a.boundingClientRect.top <= b.boundingClientRect.top ? a : b
|
|
932
|
-
);
|
|
933
|
-
setActive(topmost.target.id);
|
|
934
|
-
}
|
|
935
|
-
},
|
|
936
|
-
{ rootMargin: '0px 0px -70% 0px', threshold: 0 }
|
|
937
|
-
);
|
|
938
|
-
sections.forEach((section) => observer.observe(section));
|
|
939
|
-
}
|
|
940
|
-
});
|
|
941
|
-
}
|
|
942
|
-
|
|
943
|
-
</script>
|
|
944
|
-
<script>
|
|
945
|
-
(() => {
|
|
946
|
-
const root = document.documentElement;
|
|
947
|
-
const toggle = document.getElementById('theme-toggle');
|
|
948
|
-
|
|
949
|
-
const syncThemeToggle = () => {
|
|
950
|
-
const isLight = root.getAttribute('data-theme') === 'light';
|
|
951
|
-
toggle?.setAttribute('aria-pressed', String(isLight));
|
|
952
|
-
toggle?.setAttribute('aria-label', `Switch to ${isLight ? 'dark' : 'light'} theme`);
|
|
953
|
-
};
|
|
954
|
-
|
|
955
|
-
syncThemeToggle();
|
|
956
|
-
new MutationObserver(syncThemeToggle).observe(root, { attributes: true, attributeFilter: ['data-theme'] });
|
|
957
|
-
|
|
958
|
-
toggle?.addEventListener('click', () => {
|
|
959
|
-
const theme = root.getAttribute('data-theme') === 'light' ? 'dark' : 'light';
|
|
960
|
-
root.setAttribute('data-theme', theme);
|
|
961
|
-
try {
|
|
962
|
-
localStorage.setItem('spark-tuning-reference-theme', theme);
|
|
963
|
-
} catch (_) {
|
|
964
|
-
// The visual preference still works when persistent storage is blocked.
|
|
965
|
-
}
|
|
966
|
-
});
|
|
967
|
-
})();
|
|
968
|
-
</script>
|
|
969
|
-
</body>
|
|
970
|
-
</html>
|