mostlyright-data 0.9.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mostlyright/data_harness/__init__.py +158 -0
- mostlyright/data_harness/acquisition/__init__.py +55 -0
- mostlyright/data_harness/acquisition/http.py +2773 -0
- mostlyright/data_harness/acquisition/parsing.py +809 -0
- mostlyright/data_harness/acquisition/ranges.py +495 -0
- mostlyright/data_harness/acquisition/result_download.py +360 -0
- mostlyright/data_harness/acquisition/retention_admission.py +248 -0
- mostlyright/data_harness/acquisition/sandbox.py +4888 -0
- mostlyright/data_harness/acquisition/url_policy.py +530 -0
- mostlyright/data_harness/agent_runtime.py +2743 -0
- mostlyright/data_harness/assets/logo-ink.svg +31 -0
- mostlyright/data_harness/backends/__init__.py +28 -0
- mostlyright/data_harness/backends/pandas_backend.py +350 -0
- mostlyright/data_harness/backends/polars_backend.py +366 -0
- mostlyright/data_harness/backends/protocol.py +124 -0
- mostlyright/data_harness/backends/reference.py +83 -0
- mostlyright/data_harness/backends/registry.py +55 -0
- mostlyright/data_harness/backends/restrictions.py +126 -0
- mostlyright/data_harness/canonical.py +333 -0
- mostlyright/data_harness/catalog_job.py +625 -0
- mostlyright/data_harness/cli.py +5398 -0
- mostlyright/data_harness/contracts.py +53 -0
- mostlyright/data_harness/coordinator.py +1307 -0
- mostlyright/data_harness/deploy.py +924 -0
- mostlyright/data_harness/deploy_target.py +312 -0
- mostlyright/data_harness/deployment_evidence.py +1067 -0
- mostlyright/data_harness/event_presentation.py +576 -0
- mostlyright/data_harness/events.py +2152 -0
- mostlyright/data_harness/fast_delimited.py +239 -0
- mostlyright/data_harness/fleet.py +237 -0
- mostlyright/data_harness/formats.py +236 -0
- mostlyright/data_harness/governors.py +1163 -0
- mostlyright/data_harness/hosted_bootstrap.py +972 -0
- mostlyright/data_harness/hosted_crawler.py +1115 -0
- mostlyright/data_harness/hosted_crawler_container_smoke.py +351 -0
- mostlyright/data_harness/hosted_crawler_fetch.py +423 -0
- mostlyright/data_harness/hosted_crawler_job.py +1277 -0
- mostlyright/data_harness/hosted_crawler_protocol.py +676 -0
- mostlyright/data_harness/hosted_dataset.py +1500 -0
- mostlyright/data_harness/hosted_deploy.py +3037 -0
- mostlyright/data_harness/hosted_handoff.py +62 -0
- mostlyright/data_harness/hosted_ingestion_contract.py +504 -0
- mostlyright/data_harness/hosted_ingestion_job.py +356 -0
- mostlyright/data_harness/hosted_ingestion_job_smoke.py +40 -0
- mostlyright/data_harness/hosted_session_container_smoke.py +194 -0
- mostlyright/data_harness/hosted_session_worker.py +3554 -0
- mostlyright/data_harness/hosted_session_worker_job_smoke.py +46 -0
- mostlyright/data_harness/hosted_worker.py +6784 -0
- mostlyright/data_harness/ingestion/__init__.py +56 -0
- mostlyright/data_harness/ingestion/contracts.py +461 -0
- mostlyright/data_harness/ingestion/faults.py +42 -0
- mostlyright/data_harness/ingestion/gcs_store.py +1162 -0
- mostlyright/data_harness/ingestion/spool.py +130 -0
- mostlyright/data_harness/ingestion/store.py +885 -0
- mostlyright/data_harness/key_seam.py +434 -0
- mostlyright/data_harness/linux_process_boundary.py +262 -0
- mostlyright/data_harness/local_contracts.py +2880 -0
- mostlyright/data_harness/local_search/__init__.py +5 -0
- mostlyright/data_harness/local_search/build_index.py +1087 -0
- mostlyright/data_harness/local_search/contracts.py +920 -0
- mostlyright/data_harness/local_search/query_trace.py +266 -0
- mostlyright/data_harness/local_search/retrieval.py +700 -0
- mostlyright/data_harness/local_search/sealed.py +474 -0
- mostlyright/data_harness/local_search/service.py +784 -0
- mostlyright/data_harness/nbrender/CONTRACT.md +212 -0
- mostlyright/data_harness/nbrender/__init__.py +12 -0
- mostlyright/data_harness/nbrender/chrome.py +359 -0
- mostlyright/data_harness/nbrender/code_body.py +266 -0
- mostlyright/data_harness/nbrender/document.py +407 -0
- mostlyright/data_harness/nbrender/frame.py +275 -0
- mostlyright/data_harness/nbrender/interactive.py +337 -0
- mostlyright/data_harness/nbrender/markdown_body.py +477 -0
- mostlyright/data_harness/nbrender/mr_components.py +134 -0
- mostlyright/data_harness/nbrender/outputs_data.py +595 -0
- mostlyright/data_harness/nbrender/outputs_rich.py +906 -0
- mostlyright/data_harness/nbrender/outputs_source.py +260 -0
- mostlyright/data_harness/nbrender/outputs_stage.py +176 -0
- mostlyright/data_harness/nbrender/outputs_text.py +400 -0
- mostlyright/data_harness/nbrender/parse.py +394 -0
- mostlyright/data_harness/nbrender/status.py +40 -0
- mostlyright/data_harness/nbrender/tokens.py +1295 -0
- mostlyright/data_harness/notebook.py +1710 -0
- mostlyright/data_harness/offline.py +2049 -0
- mostlyright/data_harness/operation_registry.py +1007 -0
- mostlyright/data_harness/operator_setup.py +239 -0
- mostlyright/data_harness/pipeline.py +6428 -0
- mostlyright/data_harness/plan_graph.py +2026 -0
- mostlyright/data_harness/preparation/__init__.py +104 -0
- mostlyright/data_harness/preparation/contracts.py +1017 -0
- mostlyright/data_harness/preparation/engine.py +221 -0
- mostlyright/data_harness/preparation/errors.py +14 -0
- mostlyright/data_harness/preparation/gates.py +751 -0
- mostlyright/data_harness/preparation/joins.py +574 -0
- mostlyright/data_harness/preparation/profile.py +384 -0
- mostlyright/data_harness/preparation/table.py +217 -0
- mostlyright/data_harness/preparation/transforms.py +568 -0
- mostlyright/data_harness/progress_events.py +534 -0
- mostlyright/data_harness/readers/__init__.py +46 -0
- mostlyright/data_harness/readers/containers.py +963 -0
- mostlyright/data_harness/readers/contracts.py +542 -0
- mostlyright/data_harness/readers/delimited.py +257 -0
- mostlyright/data_harness/readers/grib2/__init__.py +33 -0
- mostlyright/data_harness/readers/grib2/admission.py +722 -0
- mostlyright/data_harness/readers/grib2/decode.py +1009 -0
- mostlyright/data_harness/readers/grib2/geometry.py +1133 -0
- mostlyright/data_harness/readers/grib2/portable_math.py +501 -0
- mostlyright/data_harness/readers/json_tabular.py +485 -0
- mostlyright/data_harness/readers/registry.py +514 -0
- mostlyright/data_harness/readers/samples/README.md +110 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/cities.csv.gz +0 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/cities.csv.gz +0 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/cities.tar +0 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/cities.tar +0 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/cities.zip +0 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/expected.json +25 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/expected.json +25 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/expected.json +25 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/cities.csv +3 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/expected.json +23 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/cities.tsv +5 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/expected.json +25 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/expected.json +30 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/observations.csv +5 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/expected.json +44 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/stations.json +1 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/expected.json +48 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/observations.ndjson +4 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/cities.xlsx +0 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/expected.json +24 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/expected.json +27 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/expected.json +27 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/README.md +20 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/expected.json +55 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/gfs-2m-temperature.grib2 +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/expected.json +54 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/expected.json +54 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/hrrr-categorical-rain.grib2 +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/expected.json +54 -0
- mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
- mostlyright/data_harness/readers/samples.py +582 -0
- mostlyright/data_harness/readers/spreadsheet.py +803 -0
- mostlyright/data_harness/readers/tabular.py +510 -0
- mostlyright/data_harness/recipe.py +5321 -0
- mostlyright/data_harness/repair/__init__.py +78 -0
- mostlyright/data_harness/repair/adapters.py +274 -0
- mostlyright/data_harness/repair/contracts.py +872 -0
- mostlyright/data_harness/repair/coordinator.py +1099 -0
- mostlyright/data_harness/repair/errors.py +16 -0
- mostlyright/data_harness/review.py +2533 -0
- mostlyright/data_harness/rowset.py +283 -0
- mostlyright/data_harness/serving.py +1975 -0
- mostlyright/data_harness/serving_edge.py +590 -0
- mostlyright/data_harness/serving_http.py +1031 -0
- mostlyright/data_harness/session_probes.py +759 -0
- mostlyright/data_harness/signing.py +101 -0
- mostlyright/data_harness/source_discovery.py +898 -0
- mostlyright/data_harness/sources/__init__.py +209 -0
- mostlyright/data_harness/sources/_adapter_steps.py +213 -0
- mostlyright/data_harness/sources/adapters.py +1214 -0
- mostlyright/data_harness/sources/cadence.py +1428 -0
- mostlyright/data_harness/sources/cadence_emission.py +453 -0
- mostlyright/data_harness/sources/cadence_history.py +546 -0
- mostlyright/data_harness/sources/catalog/__init__.py +17 -0
- mostlyright/data_harness/sources/catalog/admission.py +477 -0
- mostlyright/data_harness/sources/catalog/authoring.py +1701 -0
- mostlyright/data_harness/sources/catalog/authoring_policy.py +701 -0
- mostlyright/data_harness/sources/catalog/authoring_shards.py +1217 -0
- mostlyright/data_harness/sources/catalog/bounded_io.py +231 -0
- mostlyright/data_harness/sources/catalog/channel.py +523 -0
- mostlyright/data_harness/sources/catalog/channel_client.py +296 -0
- mostlyright/data_harness/sources/catalog/contracts.py +825 -0
- mostlyright/data_harness/sources/catalog/coverage.py +137 -0
- mostlyright/data_harness/sources/catalog/delta.py +1340 -0
- mostlyright/data_harness/sources/catalog/embedding.py +532 -0
- mostlyright/data_harness/sources/catalog/entry_v2.py +1182 -0
- mostlyright/data_harness/sources/catalog/fill.py +3889 -0
- mostlyright/data_harness/sources/catalog/fill_partitions.py +459 -0
- mostlyright/data_harness/sources/catalog/fill_staging.py +1105 -0
- mostlyright/data_harness/sources/catalog/gating.py +374 -0
- mostlyright/data_harness/sources/catalog/generation_receipt.py +1607 -0
- mostlyright/data_harness/sources/catalog/harvest/__init__.py +7 -0
- mostlyright/data_harness/sources/catalog/harvest/ckan.py +384 -0
- mostlyright/data_harness/sources/catalog/harvest/datagov_v4.py +798 -0
- mostlyright/data_harness/sources/catalog/harvest/protocol.py +964 -0
- mostlyright/data_harness/sources/catalog/harvest/sdmx.py +445 -0
- mostlyright/data_harness/sources/catalog/harvest/stac.py +384 -0
- mostlyright/data_harness/sources/catalog/health.py +447 -0
- mostlyright/data_harness/sources/catalog/hosted_catalog.py +105 -0
- mostlyright/data_harness/sources/catalog/identity_history.py +1549 -0
- mostlyright/data_harness/sources/catalog/neural.py +1618 -0
- mostlyright/data_harness/sources/catalog/packed_catalog.py +2345 -0
- mostlyright/data_harness/sources/catalog/packed_retrieval.py +1517 -0
- mostlyright/data_harness/sources/catalog/packed_writer.py +2802 -0
- mostlyright/data_harness/sources/catalog/query_trace.py +1037 -0
- mostlyright/data_harness/sources/catalog/recommend.py +171 -0
- mostlyright/data_harness/sources/catalog/retrieval.py +230 -0
- mostlyright/data_harness/sources/catalog/retrieval_manifest.py +995 -0
- mostlyright/data_harness/sources/catalog/rights_decisions.py +254 -0
- mostlyright/data_harness/sources/catalog/sealed.py +560 -0
- mostlyright/data_harness/sources/catalog/search.py +230 -0
- mostlyright/data_harness/sources/catalog/streaming_delta.py +1097 -0
- mostlyright/data_harness/sources/catalog/update.py +891 -0
- mostlyright/data_harness/sources/collections.py +815 -0
- mostlyright/data_harness/sources/contracts.py +2223 -0
- mostlyright/data_harness/sources/deletion.py +761 -0
- mostlyright/data_harness/sources/fitness.py +162 -0
- mostlyright/data_harness/sources/governance.py +163 -0
- mostlyright/data_harness/sources/hosted.py +173 -0
- mostlyright/data_harness/sources/integration.py +218 -0
- mostlyright/data_harness/sources/range_reader.py +418 -0
- mostlyright/data_harness/sources/registry.py +514 -0
- mostlyright/data_harness/sources/rights_rule.py +59 -0
- mostlyright/data_harness/sources/source_cadence_vectors.v1.json +1 -0
- mostlyright/data_harness/sources/sports.py +521 -0
- mostlyright/data_harness/sources/stream.py +524 -0
- mostlyright/data_harness/sources/stream_connector.py +418 -0
- mostlyright/data_harness/sources/stream_recorder.py +1404 -0
- mostlyright/data_harness/studio_boundary.py +2019 -0
- mostlyright/data_harness/thin/__init__.py +37 -0
- mostlyright/data_harness/thin/acquire.py +1137 -0
- mostlyright/data_harness/thin/acquire_cancel.py +579 -0
- mostlyright/data_harness/thin/approvals.py +617 -0
- mostlyright/data_harness/thin/commands.py +406 -0
- mostlyright/data_harness/thin/download.py +194 -0
- mostlyright/data_harness/thin/narrative.py +589 -0
- mostlyright/data_harness/thin/parity.py +1070 -0
- mostlyright/data_harness/thin/propose.py +2759 -0
- mostlyright/data_harness/thin/research.py +1663 -0
- mostlyright/data_harness/thin/router.py +924 -0
- mostlyright/data_harness/thin/runs.py +519 -0
- mostlyright/data_harness/thin/session.py +281 -0
- mostlyright/data_harness/thin/stream.py +501 -0
- mostlyright/data_harness/thin/transport.py +187 -0
- mostlyright/data_harness/thin/vocabulary.py +368 -0
- mostlyright/data_harness/thin/workers.py +164 -0
- mostlyright/data_harness/ucum/TABLE-PIN.json +40 -0
- mostlyright/data_harness/ucum/ucum-subset.v1.json +632 -0
- mostlyright/data_harness/unit_flow.py +927 -0
- mostlyright/data_harness/units.py +572 -0
- mostlyright/data_harness/ux/__init__.py +9 -0
- mostlyright/data_harness/ux/approve.py +485 -0
- mostlyright/data_harness/ux/author_yaml.py +597 -0
- mostlyright/data_harness/ux/cloud_auth.py +447 -0
- mostlyright/data_harness/ux/commands/__init__.py +260 -0
- mostlyright/data_harness/ux/commands/approve.py +136 -0
- mostlyright/data_harness/ux/commands/auth.py +744 -0
- mostlyright/data_harness/ux/commands/author.py +79 -0
- mostlyright/data_harness/ux/commands/catalog_author.py +403 -0
- mostlyright/data_harness/ux/commands/catalog_fill.py +523 -0
- mostlyright/data_harness/ux/commands/catalog_harvest.py +545 -0
- mostlyright/data_harness/ux/commands/catalog_publish.py +1838 -0
- mostlyright/data_harness/ux/commands/catalog_search.py +71 -0
- mostlyright/data_harness/ux/commands/catalog_update.py +437 -0
- mostlyright/data_harness/ux/commands/deploy.py +134 -0
- mostlyright/data_harness/ux/commands/deploy_dataset.py +98 -0
- mostlyright/data_harness/ux/commands/deploy_plan.py +105 -0
- mostlyright/data_harness/ux/commands/deploy_status.py +104 -0
- mostlyright/data_harness/ux/commands/diff.py +74 -0
- mostlyright/data_harness/ux/commands/index.py +84 -0
- mostlyright/data_harness/ux/commands/inventory.py +47 -0
- mostlyright/data_harness/ux/commands/list_builds.py +143 -0
- mostlyright/data_harness/ux/commands/login.py +63 -0
- mostlyright/data_harness/ux/commands/peek.py +236 -0
- mostlyright/data_harness/ux/commands/plan_check.py +90 -0
- mostlyright/data_harness/ux/commands/preflight.py +97 -0
- mostlyright/data_harness/ux/commands/record.py +107 -0
- mostlyright/data_harness/ux/commands/review_setup.py +47 -0
- mostlyright/data_harness/ux/commands/search.py +440 -0
- mostlyright/data_harness/ux/commands/show.py +61 -0
- mostlyright/data_harness/ux/commands/whoami.py +37 -0
- mostlyright/data_harness/ux/credential_native.py +551 -0
- mostlyright/data_harness/ux/credential_store.py +1055 -0
- mostlyright/data_harness/ux/credentials.py +631 -0
- mostlyright/data_harness/ux/diffing.py +444 -0
- mostlyright/data_harness/ux/headline.py +671 -0
- mostlyright/data_harness/ux/hosted_acquisition.py +974 -0
- mostlyright/data_harness/ux/hosted_run_status.py +619 -0
- mostlyright/data_harness/ux/inventory.py +427 -0
- mostlyright/data_harness/ux/local_review.py +375 -0
- mostlyright/data_harness/ux/login.py +691 -0
- mostlyright/data_harness/ux/path_kind.py +147 -0
- mostlyright/data_harness/ux/peek.py +1000 -0
- mostlyright/data_harness/ux/plain_file.py +178 -0
- mostlyright/data_harness/ux/plan_check.py +311 -0
- mostlyright/data_harness/ux/preflight.py +918 -0
- mostlyright/data_harness/ux/readers.py +1124 -0
- mostlyright/data_harness/ux/remediation.py +2195 -0
- mostlyright/data_harness/ux/render.py +657 -0
- mostlyright/data_harness/ux/workload.py +1077 -0
- mostlyright/data_harness/viewer.py +3713 -0
- mostlyright/data_harness/visual_run/__init__.py +83 -0
- mostlyright/data_harness/visual_run/authoring.py +235 -0
- mostlyright/data_harness/visual_run/contracts.py +673 -0
- mostlyright/data_harness/visual_run/materialize.py +486 -0
- mostlyright/data_harness/visual_run/observations.py +874 -0
- mostlyright/data_harness/visual_run/query.py +259 -0
- mostlyright/data_harness/visual_run/reducer.py +280 -0
- mostlyright/data_harness/visual_run/sdk.py +892 -0
- mostlyright/data_harness/visual_run/store.py +584 -0
- mostlyright/data_harness/visual_run/transport.py +239 -0
- mostlyright/data_harness/watch.py +2999 -0
- mostlyright_data-0.9.0.dist-info/METADATA +607 -0
- mostlyright_data-0.9.0.dist-info/RECORD +314 -0
- mostlyright_data-0.9.0.dist-info/WHEEL +4 -0
- mostlyright_data-0.9.0.dist-info/entry_points.txt +12 -0
|
@@ -0,0 +1,2999 @@
|
|
|
1
|
+
"""The live run view for ``mr-data watch``: a localhost page that tails one run's event feed.
|
|
2
|
+
|
|
3
|
+
Watch is the live run view; the notebook sidecar (``mr-data view``) is the finished dataset view.
|
|
4
|
+
They share HTTP serving conventions but no files. Watch uses ``ThreadingHTTPServer``, an ephemeral
|
|
5
|
+
loopback port, a 0.25 s SSE poll, and a disabled access log.
|
|
6
|
+
|
|
7
|
+
Watch is a spectator. No route on this server mutates anything, and there is no POST handler at
|
|
8
|
+
all: v1 approval is display-only, and the ABSENCE of a mutating route is the mitigation for the
|
|
9
|
+
approval-authority threat. Nothing here writes to the run directory, to ``state.json``, or to the
|
|
10
|
+
feed; deleting the feed loses nothing but the show.
|
|
11
|
+
|
|
12
|
+
One thing is fixed here that the merged viewer still gets wrong: rendering on the request thread.
|
|
13
|
+
Watch is ALWAYS reading a file another process is appending to, so the watcher thread reads, parses
|
|
14
|
+
and renders, and the handler only writes bytes it was handed and cannot fail on bad feed bytes.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import errno
|
|
20
|
+
import hashlib
|
|
21
|
+
import hmac
|
|
22
|
+
import ipaddress
|
|
23
|
+
import json
|
|
24
|
+
import os
|
|
25
|
+
import re
|
|
26
|
+
import socket
|
|
27
|
+
import threading
|
|
28
|
+
import time
|
|
29
|
+
import urllib.parse
|
|
30
|
+
import webbrowser
|
|
31
|
+
from collections.abc import Callable, Mapping, Sequence
|
|
32
|
+
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
33
|
+
from pathlib import Path
|
|
34
|
+
from typing import Any, NamedTuple
|
|
35
|
+
|
|
36
|
+
from mostlyright.data_harness import events
|
|
37
|
+
from mostlyright.data_harness.event_presentation import (
|
|
38
|
+
EVENT_SENTENCES as _EVENT_SENTENCES,
|
|
39
|
+
)
|
|
40
|
+
from mostlyright.data_harness.event_presentation import (
|
|
41
|
+
STAGE_RAIL,
|
|
42
|
+
STOPPED_EVENTS,
|
|
43
|
+
sentence,
|
|
44
|
+
)
|
|
45
|
+
from mostlyright.data_harness.nbrender.parse import esc
|
|
46
|
+
|
|
47
|
+
EVENT_SENTENCES = _EVENT_SENTENCES
|
|
48
|
+
|
|
49
|
+
# Keep the live-run palette independent of notebook renderer imports.
|
|
50
|
+
PAPER = "#F5F2EB" # page ground
|
|
51
|
+
SUB_PAPER = "#FAF8F3" # card fill
|
|
52
|
+
SURFACE = "#EDE9DF" # rail fill, command block fill
|
|
53
|
+
BORDER = "#E2DDD0" # every 1px border
|
|
54
|
+
INK = "#17160F" # headings, reached labels
|
|
55
|
+
BODY = "#3B382F" # body text
|
|
56
|
+
MUTED = "#57534A" # secondary text
|
|
57
|
+
FAINT = "#6E6959" # tertiary text, unreached labels
|
|
58
|
+
DIM = "#837D6E" # the "receipt appears later" note
|
|
59
|
+
COBALT = "#2B5FE3" # links, reached marks, the copy button
|
|
60
|
+
ORANGE = "#F15B22" # a build that stopped
|
|
61
|
+
FONT_SANS = '"Space Grotesk", "Helvetica Neue", sans-serif'
|
|
62
|
+
FONT_MONO = '"JetBrains Mono", Menlo, monospace'
|
|
63
|
+
|
|
64
|
+
# The watcher and SSE loop use the same interval.
|
|
65
|
+
_POLL_SECONDS = 0.25
|
|
66
|
+
|
|
67
|
+
# The manifest is a file another local user may have written. Read it with a ceiling rather than
|
|
68
|
+
# allocating whatever is on disk.
|
|
69
|
+
_MAX_MANIFEST_BYTES = 4 * 1024 * 1024
|
|
70
|
+
|
|
71
|
+
# These errnos mean the candidate path is absent, not a directory, or a symlink. Other errors fail
|
|
72
|
+
# the current poll without invalidating an already pinned descriptor.
|
|
73
|
+
_NAME_HOLDS_NO_CANDIDATE = frozenset({errno.ENOENT, errno.ENOTDIR, errno.ELOOP})
|
|
74
|
+
|
|
75
|
+
# `manifest.json` is a real file inside the run but is NOT one of `manifest["members"]` -- a
|
|
76
|
+
# manifest cannot carry its own digest -- so the member set carries it explicitly.
|
|
77
|
+
_MANIFEST_NAME = "manifest.json"
|
|
78
|
+
|
|
79
|
+
# The ceiling on a single served receipt. Every member of a real run is far below it; the cap
|
|
80
|
+
# exists because the run directory is a place another local user can write (T-30-24).
|
|
81
|
+
MAX_RECEIPT_BYTES = 2 * 1024 * 1024
|
|
82
|
+
|
|
83
|
+
# Proving a member is the file the run recorded means hashing it, and a dataset is the one member
|
|
84
|
+
# that can be large. It is read in chunks so the cost is time and not memory.
|
|
85
|
+
_MEMBER_HASH_CHUNK_BYTES = 1024 * 1024
|
|
86
|
+
|
|
87
|
+
# Bound the total bytes and member count verified in one pass. Members are considered in manifest
|
|
88
|
+
# order; a member larger than the remaining byte budget is skipped.
|
|
89
|
+
MAX_VERIFIED_TOTAL_BYTES = 256 * 1024 * 1024
|
|
90
|
+
MAX_MANIFEST_MEMBERS = 4096
|
|
91
|
+
|
|
92
|
+
# A settled run must still notice a member changed without turning the maximum accepted receipt
|
|
93
|
+
# into a permanent syscall budget. Page-authoritative members are checked every tick; the rest are
|
|
94
|
+
# walked as a rotating slice, so every accepted member is revisited within a bounded number of
|
|
95
|
+
# ticks while one tick never opens all 4,096. Receipt requests remain stronger still: they hash the
|
|
96
|
+
# requested bytes synchronously and never serve from this change-detection cache.
|
|
97
|
+
MAX_SETTLED_MEMBER_PROBES_PER_POLL = 256
|
|
98
|
+
|
|
99
|
+
# These members can change the main page's cards, approval decision, rail, or fixed event receipts.
|
|
100
|
+
# They stay outside the rotating delay. The event mapping is the normative source for build-step
|
|
101
|
+
# evidence; the five authored/governance records are the pre-build surface declared by STAGE_RAIL.
|
|
102
|
+
_PAGE_AUTHORITY_MEMBERS = frozenset(events.EVIDENCE_MEMBER.values()) | frozenset(
|
|
103
|
+
{
|
|
104
|
+
"question.json",
|
|
105
|
+
"requirements.json",
|
|
106
|
+
"source-proposals.json",
|
|
107
|
+
"recipe.json",
|
|
108
|
+
"recipe-approval.json",
|
|
109
|
+
}
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
# The pacing clamp for `--replay`: a run that sat idle overnight must not make a demo sit idle
|
|
113
|
+
# overnight, so no released gap is ever longer than this before the speed divisor is applied.
|
|
114
|
+
MAX_REPLAY_GAP_SECONDS = 5.0
|
|
115
|
+
|
|
116
|
+
# A rederived feed has no clock -- every projected record carries `at: 0.0` -- so a rederived
|
|
117
|
+
# replay chooses a readable rhythm rather than inventing timings it does not have.
|
|
118
|
+
REDERIVED_REPLAY_STEP_SECONDS = 0.35
|
|
119
|
+
|
|
120
|
+
# The four refusal sentences. Plain language, no internal vocabulary, and -- for the two tamper
|
|
121
|
+
# cases -- deliberately identical, so a refusal never tells a prober WHICH check it failed.
|
|
122
|
+
_NOT_FINISHED = "This build has not finished yet, so there is no receipt to show."
|
|
123
|
+
_NOT_A_RECEIPT = "This build has no receipt by that name."
|
|
124
|
+
_NOT_THE_RECORDED_FILE = "The file on disk is not the file this build recorded, so it is not shown."
|
|
125
|
+
_TOO_LARGE = "This receipt is too large to show here."
|
|
126
|
+
|
|
127
|
+
# The member-path character class, copied deliberately from `events._MEMBER_PATH_RE` rather than
|
|
128
|
+
# imported: a manifest is a file on disk that another local user may have written, so the member
|
|
129
|
+
# names it offers are untrusted strings even though they arrive through the run's own receipt.
|
|
130
|
+
_MEMBER_PATH_RE = re.compile(r"\A[A-Za-z0-9._/-]{1,128}\Z")
|
|
131
|
+
|
|
132
|
+
# The shape a recorded SHA-256 digest has: 64 lowercase hex characters, which is what
|
|
133
|
+
# `hashlib.sha256(...).hexdigest()` and `canonical.sha256_bytes` produce and therefore the only
|
|
134
|
+
# string a recorded digest can be.
|
|
135
|
+
_HEX_DIGEST_RE = re.compile(r"\A[0-9a-f]{64}\Z")
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def _is_member_path(value: Any) -> bool:
|
|
139
|
+
"""True when ``value`` is a candidate-relative member path and nothing else."""
|
|
140
|
+
|
|
141
|
+
if not isinstance(value, str) or not _MEMBER_PATH_RE.match(value):
|
|
142
|
+
return False
|
|
143
|
+
if value.startswith("/") or ".." in value.split("/"):
|
|
144
|
+
return False
|
|
145
|
+
return True
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def _is_recorded_digest(value: Any) -> bool:
|
|
149
|
+
"""True when ``value`` is the shape a recorded SHA-256 digest has, and nothing else."""
|
|
150
|
+
|
|
151
|
+
return isinstance(value, str) and _HEX_DIGEST_RE.match(value) is not None
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def _matches_digest(found: Any, recorded: Any) -> bool:
|
|
155
|
+
"""Return whether both values are lowercase SHA-256 digests with equal bytes.
|
|
156
|
+
|
|
157
|
+
Validate both operands first because ``hmac.compare_digest`` raises for non-ASCII strings.
|
|
158
|
+
Invalid values are verification failures.
|
|
159
|
+
"""
|
|
160
|
+
|
|
161
|
+
return (
|
|
162
|
+
_is_recorded_digest(found)
|
|
163
|
+
and _is_recorded_digest(recorded)
|
|
164
|
+
and hmac.compare_digest(found, recorded)
|
|
165
|
+
)
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
# Shared nonblocking, no-follow regular-file opener.
|
|
169
|
+
_open_regular_member = events.open_regular_file
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _writable_path_text(value: str | os.PathLike[str]) -> str:
|
|
173
|
+
"""Return a path, or one component of it, as text a served page can actually encode.
|
|
174
|
+
|
|
175
|
+
The run-directory name is also a display-string boundary. ``cli`` hands
|
|
176
|
+
``resolve_target`` whatever ``os.fsdecode`` made of ``argv``, POSIX decodes an argument that is
|
|
177
|
+
not valid UTF-8 through ``surrogateescape``, and ``mr-data watch /data/runs/caf<0xe9>-run`` puts
|
|
178
|
+
``'caf\\udce9-run'`` into ``_render_header``'s "Build:" line on EVERY page and into
|
|
179
|
+
``approve_command``'s copyable line -- with no member, no feed and no adversary. The directory
|
|
180
|
+
need not exist when the watcher starts.
|
|
181
|
+
|
|
182
|
+
``render_page`` does not encode the result. Invalid text would let polling succeed and then make
|
|
183
|
+
``_write_html`` fail during
|
|
184
|
+
``markup.encode("utf-8")`` BEFORE ``send_response``, inside the handler thread where
|
|
185
|
+
``BaseHTTPRequestHandler`` has no ``except``; ``/events`` keeps writing version frames and the
|
|
186
|
+
tab's ``fetch('/')`` rejects into the live script's empty ``.catch``. ``/`` is dead for the life
|
|
187
|
+
of the server and the page says nothing.
|
|
188
|
+
|
|
189
|
+
Apply ``events.writable_text`` at this boundary, not in ``_write_html``. It changes display
|
|
190
|
+
text, never the ``Path``: every filesystem
|
|
191
|
+
call in this module goes through a descriptor opened from the real name, so a directory whose
|
|
192
|
+
name really does carry those bytes is still read, and only what a page writes is bounded.
|
|
193
|
+
"""
|
|
194
|
+
|
|
195
|
+
# This is display-only path algebra. ``os.fspath`` made the source gate classify the
|
|
196
|
+
# conversion itself as a filesystem lookup even though no name is resolved here.
|
|
197
|
+
return events.writable_text(str(value))
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def _file_identity(info: os.stat_result) -> tuple[int, int, int, int, int]:
|
|
201
|
+
"""Bind a digest proof to the file snapshot it measured.
|
|
202
|
+
|
|
203
|
+
Size and mtime are writer-controlled and therefore only hints. POSIX ctime is the invalidation
|
|
204
|
+
token here: ordinary file APIs advance it for a content, mode, link or timestamp change and do
|
|
205
|
+
not let that writer restore its old value. The digest remains the proof of bytes; this tuple
|
|
206
|
+
says when that proof must be paid again.
|
|
207
|
+
"""
|
|
208
|
+
|
|
209
|
+
return (info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns, info.st_ctime_ns)
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def _feed_rejection_proof(
|
|
213
|
+
path: Path,
|
|
214
|
+
) -> tuple[str, tuple[int, int, int, int, int], str] | None:
|
|
215
|
+
"""Return stable identity plus bounded content evidence for one rejected feed snapshot.
|
|
216
|
+
|
|
217
|
+
Size alone cannot identify bytes: an atomic replacement may carry a valid feed of the exact
|
|
218
|
+
same length. The cache therefore binds both descriptor identity and a digest of the bounded
|
|
219
|
+
bytes read from that descriptor. A file that changes during the read earns no reusable proof.
|
|
220
|
+
"""
|
|
221
|
+
|
|
222
|
+
try:
|
|
223
|
+
descriptor = events.open_regular_file(path)
|
|
224
|
+
except OSError:
|
|
225
|
+
return None
|
|
226
|
+
try:
|
|
227
|
+
identity_before = _file_identity(os.fstat(descriptor))
|
|
228
|
+
remaining = events.MAX_FEED_BYTES + 1
|
|
229
|
+
digest = hashlib.sha256()
|
|
230
|
+
while remaining:
|
|
231
|
+
chunk = os.read(descriptor, min(64 * 1024, remaining))
|
|
232
|
+
if not chunk:
|
|
233
|
+
break
|
|
234
|
+
digest.update(chunk)
|
|
235
|
+
remaining -= len(chunk)
|
|
236
|
+
identity_after = _file_identity(os.fstat(descriptor))
|
|
237
|
+
except OSError:
|
|
238
|
+
return None
|
|
239
|
+
finally:
|
|
240
|
+
os.close(descriptor)
|
|
241
|
+
if identity_after != identity_before:
|
|
242
|
+
return None
|
|
243
|
+
# Display/cache-key conversion only. The descriptor open above is the sole name resolution.
|
|
244
|
+
return str(path), identity_after, digest.hexdigest()
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
class ReceiptRefused(Exception):
|
|
248
|
+
"""A receipt could not be served. Carries the status and the one plain sentence to show.
|
|
249
|
+
|
|
250
|
+
Every refusal renders a page with the sentence and NOTHING of the file's content: a member
|
|
251
|
+
whose bytes do not match the digest the run recorded has already failed the only test that
|
|
252
|
+
makes it a receipt, and showing it anyway would be showing an unlabelled claim.
|
|
253
|
+
"""
|
|
254
|
+
|
|
255
|
+
def __init__(self, status: int, sentence: str) -> None:
|
|
256
|
+
super().__init__(sentence)
|
|
257
|
+
self.status = status
|
|
258
|
+
self.sentence = sentence
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
class _Target(NamedTuple):
|
|
262
|
+
"""The two directories a watcher needs: the run it describes and the feed it tails."""
|
|
263
|
+
|
|
264
|
+
run_dir: Path
|
|
265
|
+
feed_dir: Path
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def resolve_target(run_dir: str | os.PathLike[str]) -> _Target:
|
|
269
|
+
"""Return ``(run_dir, feed_dir)`` for a run directory.
|
|
270
|
+
|
|
271
|
+
The feed lives in the runs PARENT (``events.run_feed_dir`` puts it there), because a failed
|
|
272
|
+
build has no run directory at all and a file inside the run directory would be a new file
|
|
273
|
+
inside a closed member set. It is scoped by the run's own name, so a sibling run's feed is not
|
|
274
|
+
in this directory and cannot be mistaken for this run's. Neither directory need exist: the
|
|
275
|
+
primary scenario is a watcher started BEFORE the build, when neither does.
|
|
276
|
+
"""
|
|
277
|
+
|
|
278
|
+
resolved = Path(run_dir).absolute()
|
|
279
|
+
return _Target(resolved, events.run_feed_dir(resolved))
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
class _WatchState:
|
|
283
|
+
"""Everything the request path must not do: read the feed, parse it, and render the page.
|
|
284
|
+
|
|
285
|
+
The member set is an INPUT to each render, not a property of the server, because the run it
|
|
286
|
+
describes does not exist yet when the server starts. ``poll`` passes the current value on every
|
|
287
|
+
render and never a value captured at construction.
|
|
288
|
+
"""
|
|
289
|
+
|
|
290
|
+
def __init__(
|
|
291
|
+
self,
|
|
292
|
+
target: _Target | str | os.PathLike[str],
|
|
293
|
+
*,
|
|
294
|
+
replay: float | None = None,
|
|
295
|
+
clock: Callable[[], float] = time.monotonic,
|
|
296
|
+
) -> None:
|
|
297
|
+
self.target = target if isinstance(target, _Target) else resolve_target(target)
|
|
298
|
+
self.run_dir = self.target.run_dir
|
|
299
|
+
self.feed_dir = self.target.feed_dir
|
|
300
|
+
|
|
301
|
+
self._lock = threading.Lock()
|
|
302
|
+
self._version = 0
|
|
303
|
+
self._last_seq = 0
|
|
304
|
+
# A page before any I/O has happened: `render_page` reads no file of its own, so this
|
|
305
|
+
# cannot raise, and a request that arrives before the first poll gets a document rather
|
|
306
|
+
# than an empty body.
|
|
307
|
+
self._html = render_page(
|
|
308
|
+
[], run_dir=self.run_dir, members_present=frozenset(), now=time.time()
|
|
309
|
+
)
|
|
310
|
+
# The last render that SUCCEEDED, with no stall note spliced into it. `_html` is what a
|
|
311
|
+
# request is served and may carry the note; this is what the note is spliced onto, so a
|
|
312
|
+
# stall that lasts a thousand polls still yields one note and not a thousand.
|
|
313
|
+
self._good_html = self._html
|
|
314
|
+
self._offset = 0 # bytes of the last successfully parsed feed read
|
|
315
|
+
self._fingerprint: tuple[Any, ...] | None = None
|
|
316
|
+
self._rejected_feed: tuple[str, tuple[int, int, int, int, int], str] | None = None
|
|
317
|
+
self._failed_polls = 0
|
|
318
|
+
# The class name of the last failure, or None while the last poll cycle completed. Only the
|
|
319
|
+
# NAME: an exception's message can carry bytes off disk, and this one is rendered.
|
|
320
|
+
self._failure_reason: str | None = None
|
|
321
|
+
|
|
322
|
+
self._manifest: Mapping[str, Any] | None = None
|
|
323
|
+
self._members_present: frozenset[str] = frozenset()
|
|
324
|
+
self._member_index: dict[str, Mapping[str, Any]] = {}
|
|
325
|
+
self._candidate_fd: int | None = None
|
|
326
|
+
self._candidate_identity: tuple[int, int] | None = None
|
|
327
|
+
self._manifest_file_identity: tuple[int, int, int, int, int] | None = None
|
|
328
|
+
# True when a manifest is on disk and does NOT carry its own fingerprint. Distinct from
|
|
329
|
+
# "no manifest yet", which is every run before it exists. It is a verdict about the bytes
|
|
330
|
+
# THIS poll read, so it is re-derived like every other one and never held past a poll that
|
|
331
|
+
# found the file intact again.
|
|
332
|
+
self._manifest_disowned = False
|
|
333
|
+
# The members this reader has PROVED under the currently pinned candidate, and the ones it
|
|
334
|
+
# has read whole and found not to be what the run recorded. Between them and the manifest's
|
|
335
|
+
# own index they say whether the answer is complete; see `_refresh_run`.
|
|
336
|
+
self._verified: dict[str, tuple[int, int, int, int, int]] = {}
|
|
337
|
+
self._measured_wrong: dict[str, tuple[int, int, int, int, int] | None] = {}
|
|
338
|
+
self._settled_probe_cursor = 0
|
|
339
|
+
|
|
340
|
+
# Replay state. `replay` is None for a live watch and a positive speed multiplier for a
|
|
341
|
+
# paced one; the released count is the only thing pacing changes, so live and paced share
|
|
342
|
+
# one renderer, one channel and one code path.
|
|
343
|
+
self._replay = replay
|
|
344
|
+
self._clock = clock
|
|
345
|
+
self._released = 0
|
|
346
|
+
self._released_at: float | None = None
|
|
347
|
+
# The records the last successful rederivation returned, retained only with the complete
|
|
348
|
+
# descriptor-relative proof beside them. A changed candidate, manifest, member decision,
|
|
349
|
+
# or directory namespace makes the pair unusable; refusals are never retained.
|
|
350
|
+
self._rederived: list[dict[str, Any]] | None = None
|
|
351
|
+
self._rederived_proof: tuple[Any, ...] | None = None
|
|
352
|
+
self._records_source = "none"
|
|
353
|
+
|
|
354
|
+
try:
|
|
355
|
+
self.poll()
|
|
356
|
+
except BaseException:
|
|
357
|
+
# `poll` may have adopted the candidate directory before a process-level exception.
|
|
358
|
+
# The caller cannot close an object whose constructor never returned, so ownership
|
|
359
|
+
# has to unwind here. Cleanup is secondary and must not replace the initiating error.
|
|
360
|
+
try:
|
|
361
|
+
self.close()
|
|
362
|
+
except BaseException:
|
|
363
|
+
pass
|
|
364
|
+
raise
|
|
365
|
+
|
|
366
|
+
# -- reads -----------------------------------------------------------------------------------
|
|
367
|
+
|
|
368
|
+
def snapshot(self) -> tuple[int, int, str]:
|
|
369
|
+
"""Return one consistent ``(version, last_seq, html)`` snapshot.
|
|
370
|
+
|
|
371
|
+
Every ``_html`` update increments ``_version`` so open clients observe page changes.
|
|
372
|
+
"""
|
|
373
|
+
|
|
374
|
+
with self._lock:
|
|
375
|
+
return self._version, self._last_seq, self._html
|
|
376
|
+
|
|
377
|
+
def evidence_index(self) -> tuple[Mapping[str, Any] | None, frozenset[str]]:
|
|
378
|
+
"""Return the memoized manifest and member set as a CONSISTENT pair, under the lock."""
|
|
379
|
+
|
|
380
|
+
with self._lock:
|
|
381
|
+
return self._manifest, self._members_present
|
|
382
|
+
|
|
383
|
+
def members_present(self) -> frozenset[str]:
|
|
384
|
+
"""The member paths this run carries, or an empty set before the manifest exists."""
|
|
385
|
+
|
|
386
|
+
return self.evidence_index()[1]
|
|
387
|
+
|
|
388
|
+
def member_index(self) -> dict[str, Mapping[str, Any]]:
|
|
389
|
+
"""The run's own allowlist: member path to the entry the run recorded for it.
|
|
390
|
+
|
|
391
|
+
This is an ALLOWLIST, never a path sanitizer. A sanitizer is a list of attacks someone
|
|
392
|
+
thought of; an allowlist is the set of files this run actually recorded. A name that is not
|
|
393
|
+
a key here never reaches the filesystem at all (T-30-21).
|
|
394
|
+
"""
|
|
395
|
+
|
|
396
|
+
with self._lock:
|
|
397
|
+
return dict(self._member_index)
|
|
398
|
+
|
|
399
|
+
def recorded_digests(self) -> dict[str, str]:
|
|
400
|
+
"""Return each member path and the SHA-256 recorded in the manifest."""
|
|
401
|
+
|
|
402
|
+
return {
|
|
403
|
+
member: entry["sha256"]
|
|
404
|
+
for member, entry in self.member_index().items()
|
|
405
|
+
if _is_recorded_digest(entry.get("sha256"))
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
def _duplicate_candidate_fd(self) -> int | None:
|
|
409
|
+
"""Own a stable reference to the pinned candidate, or return ``None``.
|
|
410
|
+
|
|
411
|
+
Reading the integer under the lock is not ownership. Another thread may drop the run,
|
|
412
|
+
close that descriptor and let the kernel reuse the same number before an ``openat`` uses
|
|
413
|
+
it. Duplication happens while the retained descriptor is protected by the same lock that
|
|
414
|
+
detaches it in ``_drop_run``; the caller then owns an independent reference and must close
|
|
415
|
+
it after its complete descriptor-rooted operation.
|
|
416
|
+
"""
|
|
417
|
+
|
|
418
|
+
with self._lock:
|
|
419
|
+
if self._candidate_fd is None:
|
|
420
|
+
return None
|
|
421
|
+
try:
|
|
422
|
+
return events.duplicate_directory(self._candidate_fd)
|
|
423
|
+
except OSError:
|
|
424
|
+
# A descriptor-pressure failure is a fact about this read, not about the run.
|
|
425
|
+
return None
|
|
426
|
+
|
|
427
|
+
# -- the verified read ------------------------------------------------------------------------
|
|
428
|
+
|
|
429
|
+
def read_member(self, member: str, cited: str | None = None) -> bytes:
|
|
430
|
+
"""Return one member's bytes, or raise ``ReceiptRefused`` with the sentence to show.
|
|
431
|
+
|
|
432
|
+
The order is the whole security argument and it does not commute: resolve the name against
|
|
433
|
+
the run's own allowlist FIRST, then open relative to the retained directory descriptor,
|
|
434
|
+
then hash, then compare. A name that fails the allowlist causes no filesystem call, and a
|
|
435
|
+
file whose bytes fail their fingerprint renders nothing.
|
|
436
|
+
"""
|
|
437
|
+
|
|
438
|
+
manifest, _members = self.evidence_index()
|
|
439
|
+
if manifest is None:
|
|
440
|
+
with self._lock:
|
|
441
|
+
disowned = self._manifest_disowned
|
|
442
|
+
# Two different facts, and they must not be told as one. A run that does not exist has
|
|
443
|
+
# no receipt YET; a run whose receipt is not its own receipt has failed the only test
|
|
444
|
+
# that makes it a receipt, and that reads exactly like a member whose bytes changed --
|
|
445
|
+
# same status, same sentence -- so a refusal never tells a prober which check it failed.
|
|
446
|
+
if disowned:
|
|
447
|
+
raise ReceiptRefused(409, _NOT_THE_RECORDED_FILE)
|
|
448
|
+
raise ReceiptRefused(404, _NOT_FINISHED)
|
|
449
|
+
|
|
450
|
+
if member == _MANIFEST_NAME:
|
|
451
|
+
# ONE named branch, taken before the allowlist lookup, and not a special case a later
|
|
452
|
+
# reader may tidy away. `manifest.json` is NOT an entry in `manifest["members"]` and
|
|
453
|
+
# structurally cannot be: `members` is built from the expected file set and the
|
|
454
|
+
# manifest is written afterwards, so a file would have to contain its own fingerprint.
|
|
455
|
+
# Looking it up in the allowlist therefore returns nothing, and comparing it against
|
|
456
|
+
# "the recorded fingerprint" compares against a value that does not exist.
|
|
457
|
+
return self._verify_manifest_self(self._read_member_bytes(_MANIFEST_NAME), cited)
|
|
458
|
+
|
|
459
|
+
entry = self.member_index().get(member)
|
|
460
|
+
if entry is None:
|
|
461
|
+
raise ReceiptRefused(404, _NOT_A_RECEIPT)
|
|
462
|
+
recorded = entry.get("sha256")
|
|
463
|
+
if not _is_recorded_digest(recorded):
|
|
464
|
+
# Checked BEFORE the read so a receipt carrying a value that is not a digest costs no
|
|
465
|
+
# filesystem call and no hash, and reported as tamper because that is what it is.
|
|
466
|
+
raise ReceiptRefused(409, _NOT_THE_RECORDED_FILE)
|
|
467
|
+
|
|
468
|
+
raw = self._read_member_bytes(member)
|
|
469
|
+
found = hashlib.sha256(raw).hexdigest()
|
|
470
|
+
if not _matches_digest(found, recorded):
|
|
471
|
+
raise ReceiptRefused(409, _NOT_THE_RECORDED_FILE)
|
|
472
|
+
if cited is not None and not _matches_digest(found, cited):
|
|
473
|
+
# `cited` came off the query string, so it is the least trusted value on this path.
|
|
474
|
+
raise ReceiptRefused(409, _NOT_THE_RECORDED_FILE)
|
|
475
|
+
return raw
|
|
476
|
+
|
|
477
|
+
def _read_member_bytes(self, member: str) -> bytes:
|
|
478
|
+
"""Open ``member`` RELATIVE to the retained descriptor, never by string concatenation."""
|
|
479
|
+
|
|
480
|
+
fd = self._duplicate_candidate_fd()
|
|
481
|
+
if fd is None:
|
|
482
|
+
raise ReceiptRefused(404, _NOT_FINISHED)
|
|
483
|
+
try:
|
|
484
|
+
try:
|
|
485
|
+
handle_fd = _open_regular_member(member, dir_fd=fd)
|
|
486
|
+
with os.fdopen(handle_fd, "rb") as handle:
|
|
487
|
+
raw = handle.read(MAX_RECEIPT_BYTES + 1)
|
|
488
|
+
except OSError as exc:
|
|
489
|
+
raise ReceiptRefused(404, _NOT_A_RECEIPT) from exc
|
|
490
|
+
finally:
|
|
491
|
+
os.close(fd)
|
|
492
|
+
if len(raw) > MAX_RECEIPT_BYTES:
|
|
493
|
+
raise ReceiptRefused(413, _TOO_LARGE)
|
|
494
|
+
return raw
|
|
495
|
+
|
|
496
|
+
def _durable_candidate_digest(self) -> str | None:
|
|
497
|
+
"""The candidate digest the WORKSPACE durably recorded, or ``None`` when there is none.
|
|
498
|
+
|
|
499
|
+
The second authority, and the only one on this disk that is not the manifest talking about
|
|
500
|
+
itself. ``offline`` writes ``state['candidate']['candidate_digest']`` and re-checks it at
|
|
501
|
+
three linearization points, and an enrolled review binds it under a signature. ``watch``
|
|
502
|
+
consulted neither, which is why ``_verify_manifest_self`` proves coherence and not
|
|
503
|
+
authenticity.
|
|
504
|
+
|
|
505
|
+
Read through the hardened reader, relative to a descriptor for the workspace, so
|
|
506
|
+
``.mr-data`` cannot be a symlink out of it. A run directory that is not a workspace's
|
|
507
|
+
``result/`` simply has no such file, and this returns ``None``: absence of a second
|
|
508
|
+
authority is reported as absence, never as agreement.
|
|
509
|
+
"""
|
|
510
|
+
|
|
511
|
+
_status, digest = self._durable_candidate_digest_status()
|
|
512
|
+
return digest
|
|
513
|
+
|
|
514
|
+
def _durable_candidate_digest_status(self) -> tuple[str, str | None]:
|
|
515
|
+
"""Return ``(absent|unreadable|present, digest)`` for the workspace authority.
|
|
516
|
+
|
|
517
|
+
``absent`` is deliberately not an approval verdict. It only distinguishes a workspace
|
|
518
|
+
without a durable authority from one whose authority is present but unreadable; callers
|
|
519
|
+
that need corroboration must require ``present`` and an agreeing digest. Otherwise deleting
|
|
520
|
+
the whole control directory -- including across a Watch restart -- manufactures approval.
|
|
521
|
+
"""
|
|
522
|
+
|
|
523
|
+
try:
|
|
524
|
+
workspace_fd = events.open_directory(self.run_dir.parent)
|
|
525
|
+
except OSError:
|
|
526
|
+
return "unreadable", None
|
|
527
|
+
try:
|
|
528
|
+
try:
|
|
529
|
+
state_fd = events.open_directory(".mr-data", dir_fd=workspace_fd)
|
|
530
|
+
except OSError as exc:
|
|
531
|
+
if exc.errno == errno.ENOENT:
|
|
532
|
+
return "absent", None
|
|
533
|
+
return "unreadable", None
|
|
534
|
+
try:
|
|
535
|
+
raw = events.read_regular_bytes(
|
|
536
|
+
"state.json",
|
|
537
|
+
dir_fd=state_fd,
|
|
538
|
+
max_bytes=_MAX_MANIFEST_BYTES,
|
|
539
|
+
)
|
|
540
|
+
finally:
|
|
541
|
+
os.close(state_fd)
|
|
542
|
+
candidate = json.loads(raw.decode("utf-8")).get("candidate")
|
|
543
|
+
except (OSError, ValueError, AttributeError, RecursionError):
|
|
544
|
+
return "unreadable", None
|
|
545
|
+
finally:
|
|
546
|
+
os.close(workspace_fd)
|
|
547
|
+
digest = candidate.get("candidate_digest") if isinstance(candidate, Mapping) else None
|
|
548
|
+
if not _is_recorded_digest(digest):
|
|
549
|
+
return "unreadable", None
|
|
550
|
+
return "present", digest
|
|
551
|
+
|
|
552
|
+
def _approval_is_corroborated(self) -> bool:
|
|
553
|
+
"""True only when the second authority corroborates the approval summary's receipt.
|
|
554
|
+
|
|
555
|
+
The summary is the one part of this page that talks a person into an action (T-30-27), and
|
|
556
|
+
the receipt underneath it proves only coherence: whoever writes ``manifest.json`` writes
|
|
557
|
+
its ``candidate_digest`` too, so an adversary who reseals is not detected by recomputing
|
|
558
|
+
it. This is the cross-check that is available -- the workspace's durable
|
|
559
|
+
``state.json`` -- and its verdict is deliberately fail closed:
|
|
560
|
+
|
|
561
|
+
* the durable digest AGREES with the manifest's: render;
|
|
562
|
+
* the durable digest DISAGREES: render nothing, because two authorities on this disk
|
|
563
|
+
disagree about which candidate this is and a summary is a claim about a specific one;
|
|
564
|
+
* there is NO readable durable digest (a bare run directory or a workspace mid-write):
|
|
565
|
+
render no approval summary. Candidate-local bytes prove only their own coherence, and
|
|
566
|
+
process-local memory cannot distinguish a genuinely bare run from an authority an
|
|
567
|
+
attacker deleted before this Watch process started.
|
|
568
|
+
|
|
569
|
+
Nothing here is memoized, and the ANSWER is part of the poll's fingerprint. Those two are
|
|
570
|
+
one sentence, not two: a disagreement while the seal is mid-flight -- the candidate
|
|
571
|
+
directory renamed into place before ``state.json`` is republished -- must cost that render
|
|
572
|
+
and not that candidate, so the next poll asks again AND acts on a different answer. Asked
|
|
573
|
+
below the fingerprint gate, as an argument to ``render_page``, the next poll returned
|
|
574
|
+
before asking; see ``_poll_once``.
|
|
575
|
+
"""
|
|
576
|
+
|
|
577
|
+
manifest, _members = self.evidence_index()
|
|
578
|
+
if manifest is None:
|
|
579
|
+
return False
|
|
580
|
+
durable_status, durable = self._durable_candidate_digest_status()
|
|
581
|
+
if durable_status != "present" or durable is None:
|
|
582
|
+
return False
|
|
583
|
+
# No shape check here: the comparator owns it, on both operands. A proof two methods away
|
|
584
|
+
# -- `_verify_manifest_self` refuses a manifest whose `candidate_digest` is not a digest
|
|
585
|
+
# before `self._manifest` is ever set -- is not a guard at the comparison.
|
|
586
|
+
return _matches_digest(manifest.get("candidate_digest"), durable)
|
|
587
|
+
|
|
588
|
+
def _verify_manifest_self(self, raw: bytes, cited: str | None) -> bytes:
|
|
589
|
+
"""Verify that the run's receipt is COHERENT, by recomputing its fingerprint over itself.
|
|
590
|
+
|
|
591
|
+
COHERENCE, NOT AUTHENTICITY, and the difference is the whole reading of this method. The
|
|
592
|
+
seal computes the fingerprint over the receipt minus that one field, so verifying it means
|
|
593
|
+
doing exactly that again and comparing -- against a value in the same file, written by
|
|
594
|
+
whoever wrote the file. It detects an edit that forgot to reseal. It detects nothing at all
|
|
595
|
+
from an adversary who reseals, and the adversary this module names throughout is a local
|
|
596
|
+
user who can write the run directory (T-30-24), which is to say one who can reseal.
|
|
597
|
+
|
|
598
|
+
There is no other file inside the candidate that records this file's fingerprint, so this
|
|
599
|
+
is the strongest statement available from the candidate alone. The authority that is NOT
|
|
600
|
+
inside the candidate is the workspace's durable ``state.json``; see
|
|
601
|
+
``_durable_candidate_digest`` and ``_approval_is_corroborated``, which is where the one
|
|
602
|
+
claim on this page that asks a person to act is gated on it.
|
|
603
|
+
"""
|
|
604
|
+
|
|
605
|
+
# Deferred: `watch` must stay importable without pulling `pipeline` in at module load.
|
|
606
|
+
from mostlyright.data_harness.canonical import sha256_bytes
|
|
607
|
+
from mostlyright.data_harness.pipeline import canonical_json_line_bytes
|
|
608
|
+
|
|
609
|
+
try:
|
|
610
|
+
payload = json.loads(raw.decode("utf-8"))
|
|
611
|
+
recorded = payload.pop("candidate_digest")
|
|
612
|
+
except (ValueError, TypeError, KeyError, AttributeError, RecursionError) as exc:
|
|
613
|
+
raise ReceiptRefused(409, _NOT_THE_RECORDED_FILE) from exc
|
|
614
|
+
if not _is_recorded_digest(recorded):
|
|
615
|
+
raise ReceiptRefused(409, _NOT_THE_RECORDED_FILE)
|
|
616
|
+
try:
|
|
617
|
+
recomputed = sha256_bytes(canonical_json_line_bytes(payload))
|
|
618
|
+
except (TypeError, ValueError) as exc:
|
|
619
|
+
raise ReceiptRefused(409, _NOT_THE_RECORDED_FILE) from exc
|
|
620
|
+
if not _matches_digest(recomputed, recorded):
|
|
621
|
+
raise ReceiptRefused(409, _NOT_THE_RECORDED_FILE)
|
|
622
|
+
if cited is not None and not _matches_digest(recorded, cited):
|
|
623
|
+
raise ReceiptRefused(409, _NOT_THE_RECORDED_FILE)
|
|
624
|
+
return raw
|
|
625
|
+
|
|
626
|
+
# -- the poll --------------------------------------------------------------------------------
|
|
627
|
+
|
|
628
|
+
def poll(self) -> None:
|
|
629
|
+
"""Refresh run state, feed records, and rendered HTML when inputs change.
|
|
630
|
+
|
|
631
|
+
An exception increments failed-poll state and preserves the previous valid page. The page
|
|
632
|
+
reports the exception class and failure count.
|
|
633
|
+
"""
|
|
634
|
+
|
|
635
|
+
try:
|
|
636
|
+
self._poll_once()
|
|
637
|
+
except Exception as exc:
|
|
638
|
+
self._record_failed_poll(exc)
|
|
639
|
+
else:
|
|
640
|
+
self._clear_failed_poll()
|
|
641
|
+
|
|
642
|
+
def _record_failed_poll(self, exc: BaseException) -> None:
|
|
643
|
+
"""Count a cycle that could not complete, and say so on the page that stopped moving.
|
|
644
|
+
|
|
645
|
+
The version moves with the page. Splicing the note in without it left the note reachable
|
|
646
|
+
only by a hard reload: ``/events`` writes a ``data:`` frame when the version changes and a
|
|
647
|
+
comment ping otherwise, and a comment fires nothing in a browser, so the tab ``mr-data
|
|
648
|
+
watch`` opened -- the only view this command produces -- sat on a frozen page and was told
|
|
649
|
+
nothing. A note about a stall that the stalled page cannot deliver is not a note.
|
|
650
|
+
"""
|
|
651
|
+
|
|
652
|
+
with self._lock:
|
|
653
|
+
self._failed_polls += 1
|
|
654
|
+
self._failure_reason = type(exc).__name__
|
|
655
|
+
self._html = _with_stall_note(self._good_html, self._failed_polls, self._failure_reason)
|
|
656
|
+
self._version += 1
|
|
657
|
+
|
|
658
|
+
def _clear_failed_poll(self) -> None:
|
|
659
|
+
"""Take the note off once a cycle completes again.
|
|
660
|
+
|
|
661
|
+
Cleared on a COMPLETED cycle, not on a fresh render: most polls change nothing and return
|
|
662
|
+
early by design, and a page that only recovered when something new happened would keep
|
|
663
|
+
telling a reader it was stuck long after it had started reading again.
|
|
664
|
+
|
|
665
|
+
The count is not reset. It is the accounting of how many reads of this run have failed, and
|
|
666
|
+
that is a fact about the run, not about the current streak.
|
|
667
|
+
|
|
668
|
+
The version moves here for the same reason it moves when the note goes on: a tab that was
|
|
669
|
+
told the page had stopped must be told it started again, or the note outlives the stall.
|
|
670
|
+
"""
|
|
671
|
+
|
|
672
|
+
with self._lock:
|
|
673
|
+
if self._failure_reason is None:
|
|
674
|
+
return
|
|
675
|
+
self._failure_reason = None
|
|
676
|
+
self._html = self._good_html
|
|
677
|
+
self._version += 1
|
|
678
|
+
|
|
679
|
+
def _poll_once(self) -> None:
|
|
680
|
+
"""One poll cycle. Every raise it can produce is caught by ``poll``."""
|
|
681
|
+
|
|
682
|
+
self._refresh_run()
|
|
683
|
+
# This verdict is asked on every cycle, before any feed memo can return. A rejected feed
|
|
684
|
+
# may suppress another read of the same bad bytes, but it cannot freeze the authority that
|
|
685
|
+
# decides whether the approval summary is safe to show.
|
|
686
|
+
corroborated = self._approval_is_corroborated()
|
|
687
|
+
|
|
688
|
+
path, size, records = self._current_feed()
|
|
689
|
+
key = None if path is None else (str(path), size)
|
|
690
|
+
rejection_proof = _feed_rejection_proof(path) if path is not None and not records else None
|
|
691
|
+
with self._lock:
|
|
692
|
+
rejected = self._rejected_feed
|
|
693
|
+
fingerprint_before = self._fingerprint
|
|
694
|
+
members = self._members_present
|
|
695
|
+
identity = self._candidate_identity
|
|
696
|
+
recorded_digests = tuple(
|
|
697
|
+
sorted(
|
|
698
|
+
(member, entry["sha256"])
|
|
699
|
+
for member, entry in self._member_index.items()
|
|
700
|
+
if _is_recorded_digest(entry.get("sha256"))
|
|
701
|
+
)
|
|
702
|
+
)
|
|
703
|
+
feed_rejected = rejection_proof is not None and rejection_proof == rejected
|
|
704
|
+
if feed_rejected:
|
|
705
|
+
# These exact bytes did not parse last tick and have not changed since. Re-reading them
|
|
706
|
+
# every 0.25 s would be work with a known answer.
|
|
707
|
+
records = []
|
|
708
|
+
|
|
709
|
+
if (
|
|
710
|
+
not feed_rejected
|
|
711
|
+
and path is not None
|
|
712
|
+
and size > 0
|
|
713
|
+
and not records
|
|
714
|
+
and not events.feed_is_readable(path)
|
|
715
|
+
):
|
|
716
|
+
# These bytes are not a feed at all: a foreign header, a truncation, or a half-written
|
|
717
|
+
# first line. Memoize and keep serving the last good render.
|
|
718
|
+
#
|
|
719
|
+
# READ AS NOTHING is not the same state as NARRATED NOTHING, and only the first belongs
|
|
720
|
+
# here. An attempt file that parses cleanly and carries no record is routine -- an
|
|
721
|
+
# idempotent re-run arms a feed and writes only its header -- and it never changes size,
|
|
722
|
+
# so memoizing it against `(path, size)` blocked the receipts fallback below for the
|
|
723
|
+
# life of the server and rendered a sealed, verifying run as "Nothing has happened yet."
|
|
724
|
+
proof_after = _feed_rejection_proof(path)
|
|
725
|
+
with self._lock:
|
|
726
|
+
self._rejected_feed = (
|
|
727
|
+
rejection_proof
|
|
728
|
+
if rejection_proof is not None and rejection_proof == proof_after
|
|
729
|
+
else None
|
|
730
|
+
)
|
|
731
|
+
feed_rejected = True
|
|
732
|
+
|
|
733
|
+
if not feed_rejected:
|
|
734
|
+
with self._lock:
|
|
735
|
+
self._rejected_feed = None
|
|
736
|
+
|
|
737
|
+
if feed_rejected and identity is None:
|
|
738
|
+
# A live build has no receipts to fall back to. The corroboration verdict above was
|
|
739
|
+
# still re-asked, but there is no page input it can safely replace the last good feed
|
|
740
|
+
# render with, so keep serving it until readable bytes or a candidate exists.
|
|
741
|
+
return
|
|
742
|
+
|
|
743
|
+
source = "feed" if records else "none"
|
|
744
|
+
if not records:
|
|
745
|
+
rederived = self._rederived_records(identity)
|
|
746
|
+
if rederived:
|
|
747
|
+
records, source = rederived, "receipts"
|
|
748
|
+
self._records_source = source
|
|
749
|
+
|
|
750
|
+
visible = self._release(records, source)
|
|
751
|
+
|
|
752
|
+
# Include every page input in the fingerprint so a changed verification result rerenders.
|
|
753
|
+
fingerprint = (
|
|
754
|
+
key,
|
|
755
|
+
source,
|
|
756
|
+
len(visible),
|
|
757
|
+
visible[-1]["seq"] if visible else 0,
|
|
758
|
+
visible[-1]["at"] if visible else 0.0,
|
|
759
|
+
identity,
|
|
760
|
+
members,
|
|
761
|
+
recorded_digests,
|
|
762
|
+
corroborated,
|
|
763
|
+
)
|
|
764
|
+
if fingerprint == fingerprint_before:
|
|
765
|
+
return
|
|
766
|
+
|
|
767
|
+
# `poll` records render failures and preserves the previous valid page.
|
|
768
|
+
markup = render_page(
|
|
769
|
+
visible,
|
|
770
|
+
run_dir=self.run_dir,
|
|
771
|
+
members_present=members,
|
|
772
|
+
now=time.time(),
|
|
773
|
+
read_member=self.read_member,
|
|
774
|
+
recorded_digests=dict(recorded_digests),
|
|
775
|
+
approval_corroborated=corroborated,
|
|
776
|
+
)
|
|
777
|
+
|
|
778
|
+
with self._lock:
|
|
779
|
+
self._offset = size if path is not None else 0
|
|
780
|
+
self._fingerprint = fingerprint
|
|
781
|
+
self._version += 1
|
|
782
|
+
self._last_seq = int(visible[-1]["seq"]) if visible else 0
|
|
783
|
+
self._html = markup
|
|
784
|
+
self._good_html = markup
|
|
785
|
+
|
|
786
|
+
# -- the record source and the paced release --------------------------------------------------
|
|
787
|
+
|
|
788
|
+
def _rederived_records(self, identity: tuple[int, int] | None) -> list[dict[str, Any]]:
|
|
789
|
+
"""Project this run's feed from its own receipts when no feed file exists.
|
|
790
|
+
|
|
791
|
+
The retained candidate descriptor, namespace proof, and member identities must remain
|
|
792
|
+
unchanged before cached records are reused. Actual member bytes and member count are bounded
|
|
793
|
+
by the same limits as direct verification. Failed verification is not cached.
|
|
794
|
+
"""
|
|
795
|
+
|
|
796
|
+
if identity is None:
|
|
797
|
+
return []
|
|
798
|
+
fd = self._duplicate_candidate_fd()
|
|
799
|
+
with self._lock:
|
|
800
|
+
manifest = self._manifest
|
|
801
|
+
if fd is None or manifest is None:
|
|
802
|
+
# Nothing pinned this instant, or no member set established, so there is nothing to ask
|
|
803
|
+
# about and nothing to bound the asking with. The next poll decides.
|
|
804
|
+
if fd is not None:
|
|
805
|
+
os.close(fd)
|
|
806
|
+
return []
|
|
807
|
+
try:
|
|
808
|
+
if not self._rederivation_fits_the_budget(fd):
|
|
809
|
+
return []
|
|
810
|
+
proof_before = self._rederivation_proof(fd)
|
|
811
|
+
with self._lock:
|
|
812
|
+
cached = None if self._rederived is None else list(self._rederived)
|
|
813
|
+
cached_proof = self._rederived_proof
|
|
814
|
+
if proof_before is not None and cached is not None and proof_before == cached_proof:
|
|
815
|
+
return cached
|
|
816
|
+
try:
|
|
817
|
+
rederived = events.rederive_feed(self.run_dir, pinned_candidate_fd=fd)
|
|
818
|
+
except Exception:
|
|
819
|
+
# Retry failed verification on the next poll.
|
|
820
|
+
rederived = []
|
|
821
|
+
proof_after = self._rederivation_proof(fd)
|
|
822
|
+
if proof_before is None or proof_after != proof_before:
|
|
823
|
+
rederived = []
|
|
824
|
+
proof_after = None
|
|
825
|
+
with self._lock:
|
|
826
|
+
self._rederived = list(rederived)
|
|
827
|
+
# A refusal is never memoized. The next poll must be able to observe a repair
|
|
828
|
+
# without depending on an incomplete proxy for the state that caused the refusal.
|
|
829
|
+
self._rederived_proof = proof_after if rederived else None
|
|
830
|
+
return rederived
|
|
831
|
+
finally:
|
|
832
|
+
os.close(fd)
|
|
833
|
+
|
|
834
|
+
def _rederivation_proof(self, fd: int) -> tuple[Any, ...] | None:
|
|
835
|
+
"""Complete cheap identity paired with a successful receipts replay.
|
|
836
|
+
|
|
837
|
+
File contents retain the rotating descriptor probes in ``_run_is_settled``. Directory
|
|
838
|
+
namespace is cheaper and must be complete: adding an unrecorded member changes the child
|
|
839
|
+
set and invalidates the cached projection.
|
|
840
|
+
"""
|
|
841
|
+
|
|
842
|
+
with self._lock:
|
|
843
|
+
if self._manifest is None or self._manifest_file_identity is None:
|
|
844
|
+
return None
|
|
845
|
+
candidate_identity = self._candidate_identity
|
|
846
|
+
manifest_identity = self._manifest_file_identity
|
|
847
|
+
index = dict(self._member_index)
|
|
848
|
+
verified = tuple(sorted(self._verified.items()))
|
|
849
|
+
wrong = tuple(
|
|
850
|
+
sorted(
|
|
851
|
+
(member, () if decided is None else decided)
|
|
852
|
+
for member, decided in self._measured_wrong.items()
|
|
853
|
+
)
|
|
854
|
+
)
|
|
855
|
+
try:
|
|
856
|
+
namespace = events.snapshot_candidate_namespace(fd, tuple(index))
|
|
857
|
+
except OSError:
|
|
858
|
+
return None
|
|
859
|
+
return candidate_identity, manifest_identity, verified, wrong, namespace
|
|
860
|
+
|
|
861
|
+
def _rederivation_fits_the_budget(self, fd: int) -> bool:
|
|
862
|
+
"""Whether re-proving this candidate costs less than the ceiling the sibling reader uses.
|
|
863
|
+
|
|
864
|
+
Measured through the PINNED descriptor with open-then-fstat, so the size is the object's and
|
|
865
|
+
not the name's, and so the answer cannot be lied to by the manifest that is about to be
|
|
866
|
+
read: the pipeline's own pre-read ceiling is asked of the DECLARED byte counts, and the
|
|
867
|
+
declared counts are exactly what an attacker writes. A member this cannot open is counted as
|
|
868
|
+
nothing, because the rederivation will not read it either -- it will refuse the whole feed.
|
|
869
|
+
|
|
870
|
+
The sum is over the member set the run declares and the projection reads it TWICE (once in
|
|
871
|
+
``verify_candidate``'s snapshot, once to project), so a set that fits here is bounded at
|
|
872
|
+
both passes.
|
|
873
|
+
"""
|
|
874
|
+
|
|
875
|
+
index = self.member_index()
|
|
876
|
+
if len(index) > MAX_MANIFEST_MEMBERS:
|
|
877
|
+
return False
|
|
878
|
+
total = 0
|
|
879
|
+
for member in index:
|
|
880
|
+
try:
|
|
881
|
+
handle_fd = _open_regular_member(member, dir_fd=fd)
|
|
882
|
+
except OSError:
|
|
883
|
+
continue
|
|
884
|
+
try:
|
|
885
|
+
total += os.fstat(handle_fd).st_size
|
|
886
|
+
finally:
|
|
887
|
+
os.close(handle_fd)
|
|
888
|
+
if total > MAX_VERIFIED_TOTAL_BYTES:
|
|
889
|
+
return False
|
|
890
|
+
return True
|
|
891
|
+
|
|
892
|
+
def _release(self, records: list[dict[str, Any]], source: str) -> list[dict[str, Any]]:
|
|
893
|
+
"""Return the prefix of ``records`` the page may show right now.
|
|
894
|
+
|
|
895
|
+
With no ``replay`` speed this is every record, which is what a live watch wants. With a
|
|
896
|
+
speed it is the records whose original inter-arrival gap has elapsed, divided by the speed
|
|
897
|
+
and clamped, so the existing channel and the existing renderer drive the replay with no new
|
|
898
|
+
machinery. Reaching the end stops releasing and leaves the final state on screen; it does
|
|
899
|
+
not loop and does not clear.
|
|
900
|
+
"""
|
|
901
|
+
|
|
902
|
+
if self._replay is None or not records:
|
|
903
|
+
return records
|
|
904
|
+
|
|
905
|
+
now = self._clock()
|
|
906
|
+
if self._released_at is None:
|
|
907
|
+
self._released = 1
|
|
908
|
+
self._released_at = now
|
|
909
|
+
return records[:1]
|
|
910
|
+
|
|
911
|
+
while self._released < len(records):
|
|
912
|
+
if source == "receipts":
|
|
913
|
+
gap = REDERIVED_REPLAY_STEP_SECONDS
|
|
914
|
+
else:
|
|
915
|
+
previous = float(records[self._released - 1]["at"])
|
|
916
|
+
following = float(records[self._released]["at"])
|
|
917
|
+
gap = min(max(following - previous, 0.0), MAX_REPLAY_GAP_SECONDS)
|
|
918
|
+
delay = gap / self._replay
|
|
919
|
+
if now - self._released_at < delay:
|
|
920
|
+
break
|
|
921
|
+
self._released += 1
|
|
922
|
+
self._released_at += delay
|
|
923
|
+
if self._released >= len(records):
|
|
924
|
+
# Caught up. A feed that is still being written continues from here in live mode --
|
|
925
|
+
# one code path, two modes -- so the release clock must not stay in the past.
|
|
926
|
+
self._released_at = max(self._released_at, now)
|
|
927
|
+
return records[: self._released]
|
|
928
|
+
|
|
929
|
+
def _current_feed(self) -> tuple[Path | None, int, list[dict[str, Any]]]:
|
|
930
|
+
"""Return the attempt being watched: ``(path, size, records)``, or ``(None, 0, [])``.
|
|
931
|
+
|
|
932
|
+
Resolve the attempt on every poll. Selection uses the sealed candidate digest when present;
|
|
933
|
+
otherwise it uses the newest active attempt according to ``events.select_narrated_feed``.
|
|
934
|
+
"""
|
|
935
|
+
|
|
936
|
+
with self._lock:
|
|
937
|
+
manifest = self._manifest
|
|
938
|
+
digest = manifest.get("candidate_digest") if isinstance(manifest, Mapping) else None
|
|
939
|
+
return events.select_narrated_feed(
|
|
940
|
+
self.feed_dir,
|
|
941
|
+
sealed_candidate_digest=digest if isinstance(digest, str) else None,
|
|
942
|
+
)
|
|
943
|
+
|
|
944
|
+
# -- the run directory -----------------------------------------------------------------------
|
|
945
|
+
|
|
946
|
+
def _refresh_run(self) -> None:
|
|
947
|
+
"""Load, keep or drop the run's manifest and member set. Called before every render.
|
|
948
|
+
|
|
949
|
+
The run directory does NOT exist while the build runs: ``build_candidate`` stages into a
|
|
950
|
+
hidden temporary directory and renames into place only after sealing. A manifest memoized
|
|
951
|
+
at construction is therefore permanently empty for exactly the build the user is watching.
|
|
952
|
+
|
|
953
|
+
Open the directory on every poll and measure identity from that descriptor. An unchanged
|
|
954
|
+
directory closes the new descriptor and retains the existing pinned reference.
|
|
955
|
+
|
|
956
|
+
Manifest content is not cached against directory identity. Incomplete reads are retried:
|
|
957
|
+
|
|
958
|
+
* complete means a manifest that parses and verifies, and every member it names either
|
|
959
|
+
proved present or READ WHOLE and found not to be the bytes the run recorded;
|
|
960
|
+
* a manifest that will not parse or will not verify is not complete, so the next poll reads
|
|
961
|
+
the file again;
|
|
962
|
+
* a member whose open or read RAISED measured nothing, so it is not complete either. That
|
|
963
|
+
covers the transient errnos above and it covers a run still arriving on disk.
|
|
964
|
+
|
|
965
|
+
A completed digest mismatch remains valid only for the measured file identity.
|
|
966
|
+
"""
|
|
967
|
+
|
|
968
|
+
candidate = self.run_dir / "candidate"
|
|
969
|
+
try:
|
|
970
|
+
opened = events.open_directory(candidate)
|
|
971
|
+
except OSError as exc:
|
|
972
|
+
if exc.errno in _NAME_HOLDS_NO_CANDIDATE:
|
|
973
|
+
# Absent, removed, or no longer a directory.
|
|
974
|
+
self._drop_run()
|
|
975
|
+
return
|
|
976
|
+
|
|
977
|
+
adopted = False
|
|
978
|
+
try:
|
|
979
|
+
info = os.fstat(opened)
|
|
980
|
+
identity = (info.st_dev, info.st_ino)
|
|
981
|
+
with self._lock:
|
|
982
|
+
retained_identity = self._candidate_identity
|
|
983
|
+
if retained_identity is not None and identity != retained_identity:
|
|
984
|
+
# The directory at this path was REPLACED -- a rebuild. Drop the descriptor, the
|
|
985
|
+
# manifest and the member set before loading anything, so a new run is never served
|
|
986
|
+
# through the previous run's allowlist.
|
|
987
|
+
self._drop_run()
|
|
988
|
+
|
|
989
|
+
with self._lock:
|
|
990
|
+
if self._candidate_fd is None:
|
|
991
|
+
# Pin the descriptor used to measure `identity`.
|
|
992
|
+
self._candidate_fd = opened
|
|
993
|
+
self._candidate_identity = identity
|
|
994
|
+
adopted = True
|
|
995
|
+
finally:
|
|
996
|
+
if not adopted:
|
|
997
|
+
os.close(opened)
|
|
998
|
+
|
|
999
|
+
owned = self._duplicate_candidate_fd()
|
|
1000
|
+
if owned is None:
|
|
1001
|
+
return
|
|
1002
|
+
try:
|
|
1003
|
+
self._refresh_pinned_run(owned)
|
|
1004
|
+
finally:
|
|
1005
|
+
os.close(owned)
|
|
1006
|
+
|
|
1007
|
+
def _refresh_pinned_run(self, pinned: int) -> None:
|
|
1008
|
+
"""Refresh manifest state through one directory reference owned for the whole operation."""
|
|
1009
|
+
|
|
1010
|
+
if self._run_is_settled(pinned):
|
|
1011
|
+
return
|
|
1012
|
+
|
|
1013
|
+
try:
|
|
1014
|
+
raw, manifest_file_identity = self._read_manifest_snapshot(pinned)
|
|
1015
|
+
except OSError:
|
|
1016
|
+
# The seal is mid-flight, absent, or unreadable this instant. An earlier accepted
|
|
1017
|
+
# manifest cannot remain the page's authority while its own receipt route says 404,
|
|
1018
|
+
# so revoke every conclusion derived from it and retry next tick.
|
|
1019
|
+
self._invalidate_installed_manifest(disowned=False)
|
|
1020
|
+
return
|
|
1021
|
+
|
|
1022
|
+
try:
|
|
1023
|
+
manifest = json.loads(raw.decode("utf-8"))
|
|
1024
|
+
# The member names come out of a file another local user may have written, so they are
|
|
1025
|
+
# re-checked against the member-path shape before any of them can become an allowlist
|
|
1026
|
+
# key. A name that is not a member path is dropped, not repaired.
|
|
1027
|
+
index = {
|
|
1028
|
+
entry["path"]: dict(entry)
|
|
1029
|
+
for entry in manifest["members"]
|
|
1030
|
+
if isinstance(entry, Mapping) and _is_member_path(entry.get("path"))
|
|
1031
|
+
}
|
|
1032
|
+
except (ValueError, TypeError, KeyError, RecursionError):
|
|
1033
|
+
# A half-written file. NOT memoized: this is a verdict about bytes, and the candidate
|
|
1034
|
+
# directory's inode -- the only key this class has -- cannot see those bytes change.
|
|
1035
|
+
# It DOES revoke an earlier accepted manifest immediately. Keeping the installed
|
|
1036
|
+
# allowlist/page while the current receipt is invalid lets one page describe bytes
|
|
1037
|
+
# that its own receipt endpoint refuses.
|
|
1038
|
+
self._invalidate_installed_manifest(disowned=True)
|
|
1039
|
+
return
|
|
1040
|
+
|
|
1041
|
+
# A NAME IN A FILE IS NOT EVIDENCE. Everything downstream of `members_present` -- the rail,
|
|
1042
|
+
# the receipt links, the cards, the freeze summary -- reads that set as "this run really
|
|
1043
|
+
# carries this file", and the rail turns membership alone into a done mark. So the set is
|
|
1044
|
+
# built from what this reader has ESTABLISHED, in two steps that do not commute:
|
|
1045
|
+
#
|
|
1046
|
+
# 1. The receipt must be COHERENT. `manifest.json` records its fingerprint over itself,
|
|
1047
|
+
# so it is recomputed and compared before a single member name is believed. Appending
|
|
1048
|
+
# fabricated entries and leaving that fingerprint alone breaks it, and a manifest that
|
|
1049
|
+
# fails here contributes no members at all -- not even `manifest.json`.
|
|
1050
|
+
# 2. Each member must be the bytes THIS receipt recorded. A name whose file is absent,
|
|
1051
|
+
# truncated or altered is dropped from the set, so a partial copy narrates the members
|
|
1052
|
+
# it still has and claims nothing for the ones it lost.
|
|
1053
|
+
#
|
|
1054
|
+
# WHAT THESE TWO STEPS DO NOT DO, stated here because this comment used to claim it: they
|
|
1055
|
+
# do not establish that the receipt is the run's. Whoever writes the file writes its
|
|
1056
|
+
# fingerprint, so a local user who RESEALS passes step 1, and step 2 then measures their
|
|
1057
|
+
# members against their own manifest. A forged, fully self-consistent manifest naming
|
|
1058
|
+
# attacker-written `recipe.json` and `recipe-approval.json` members renders as a verified
|
|
1059
|
+
# member set. The claim on this page that a forgery could turn into an action -- the
|
|
1060
|
+
# approval summary -- is therefore gated separately on the workspace's durable digest; see
|
|
1061
|
+
# `_approval_is_corroborated`. Other content describes verified candidate members.
|
|
1062
|
+
try:
|
|
1063
|
+
self._verify_manifest_self(raw, None)
|
|
1064
|
+
except ReceiptRefused:
|
|
1065
|
+
# Also not memoized, and for the same reason. The flag says what THIS poll found, so a
|
|
1066
|
+
# manifest restored on disk stops being disowned on the next tick rather than never.
|
|
1067
|
+
self._invalidate_installed_manifest(disowned=True)
|
|
1068
|
+
return
|
|
1069
|
+
|
|
1070
|
+
with self._lock:
|
|
1071
|
+
if self._manifest_file_identity != manifest_file_identity:
|
|
1072
|
+
self._verified.clear()
|
|
1073
|
+
self._measured_wrong.clear()
|
|
1074
|
+
self._settled_probe_cursor = 0
|
|
1075
|
+
self._manifest_file_identity = manifest_file_identity
|
|
1076
|
+
self._verify_members(pinned, index)
|
|
1077
|
+
|
|
1078
|
+
with self._lock:
|
|
1079
|
+
self._manifest = manifest
|
|
1080
|
+
self._members_present = frozenset(self._verified) | {_MANIFEST_NAME}
|
|
1081
|
+
self._member_index = index
|
|
1082
|
+
self._manifest_disowned = False
|
|
1083
|
+
|
|
1084
|
+
def _run_is_settled(self, fd: int) -> bool:
|
|
1085
|
+
"""Whether every installed digest decision still names the same kernel snapshot.
|
|
1086
|
+
|
|
1087
|
+
The digest is paid once for a snapshot. Each later poll opens and fstats the manifest, every
|
|
1088
|
+
page-authoritative member, and one rotating bounded slice of the remaining decisions. Any
|
|
1089
|
+
device, inode, size, mtime OR unforgeable ctime change invalidates the decision and routes
|
|
1090
|
+
the new snapshot through bounded hashing. Missing and unreadable proved objects become
|
|
1091
|
+
unsettled, so manifest revocation and copy-in-progress recovery keep their fail-closed
|
|
1092
|
+
behaviour. The rotation means every accepted member is revisited, without making 4,096
|
|
1093
|
+
opens the permanent price of watching an unchanged run.
|
|
1094
|
+
"""
|
|
1095
|
+
|
|
1096
|
+
with self._lock:
|
|
1097
|
+
if self._manifest is None or self._manifest_file_identity is None:
|
|
1098
|
+
return False
|
|
1099
|
+
manifest_identity = self._manifest_file_identity
|
|
1100
|
+
index = dict(self._member_index)
|
|
1101
|
+
verified = dict(self._verified)
|
|
1102
|
+
wrong = dict(self._measured_wrong)
|
|
1103
|
+
|
|
1104
|
+
try:
|
|
1105
|
+
manifest_fd = _open_regular_member(_MANIFEST_NAME, dir_fd=fd)
|
|
1106
|
+
except OSError:
|
|
1107
|
+
return False
|
|
1108
|
+
try:
|
|
1109
|
+
if _file_identity(os.fstat(manifest_fd)) != manifest_identity:
|
|
1110
|
+
return False
|
|
1111
|
+
finally:
|
|
1112
|
+
os.close(manifest_fd)
|
|
1113
|
+
|
|
1114
|
+
complete = not (set(index) - set(verified) - set(wrong))
|
|
1115
|
+
decided = sorted(
|
|
1116
|
+
set(verified)
|
|
1117
|
+
| {member for member, decided_identity in wrong.items() if decided_identity is not None}
|
|
1118
|
+
)
|
|
1119
|
+
priority = [member for member in decided if member in _PAGE_AUTHORITY_MEMBERS]
|
|
1120
|
+
background = [member for member in decided if member not in _PAGE_AUTHORITY_MEMBERS]
|
|
1121
|
+
remaining = max(0, MAX_SETTLED_MEMBER_PROBES_PER_POLL - len(priority))
|
|
1122
|
+
if background and remaining:
|
|
1123
|
+
with self._lock:
|
|
1124
|
+
start = self._settled_probe_cursor % len(background)
|
|
1125
|
+
count = min(remaining, len(background))
|
|
1126
|
+
self._settled_probe_cursor = (start + count) % len(background)
|
|
1127
|
+
rotated = background[start:] + background[:start]
|
|
1128
|
+
probes = priority + rotated[:count]
|
|
1129
|
+
else:
|
|
1130
|
+
probes = priority
|
|
1131
|
+
|
|
1132
|
+
changed = False
|
|
1133
|
+
for member in probes:
|
|
1134
|
+
decided_identity = verified.get(member)
|
|
1135
|
+
was_verified = decided_identity is not None
|
|
1136
|
+
if decided_identity is None:
|
|
1137
|
+
decided_identity = wrong[member]
|
|
1138
|
+
try:
|
|
1139
|
+
member_fd = _open_regular_member(member, dir_fd=fd)
|
|
1140
|
+
except OSError:
|
|
1141
|
+
if was_verified:
|
|
1142
|
+
with self._lock:
|
|
1143
|
+
self._verified.pop(member, None)
|
|
1144
|
+
changed = True
|
|
1145
|
+
# An already-refused member becoming absent does not make the page less true. Keep
|
|
1146
|
+
# its refusal; when a file appears here again its identity is compared on its next
|
|
1147
|
+
# rotating probe and a changed snapshot is re-proved.
|
|
1148
|
+
continue
|
|
1149
|
+
try:
|
|
1150
|
+
unchanged = _file_identity(os.fstat(member_fd)) == decided_identity
|
|
1151
|
+
finally:
|
|
1152
|
+
os.close(member_fd)
|
|
1153
|
+
if not unchanged:
|
|
1154
|
+
with self._lock:
|
|
1155
|
+
if was_verified:
|
|
1156
|
+
self._verified.pop(member, None)
|
|
1157
|
+
else:
|
|
1158
|
+
self._measured_wrong.pop(member, None)
|
|
1159
|
+
changed = True
|
|
1160
|
+
return complete and not changed
|
|
1161
|
+
|
|
1162
|
+
def _verify_members(self, fd: int, index: Mapping[str, Mapping[str, Any]]) -> None:
|
|
1163
|
+
"""Verify undecided members under ``fd`` within the per-pass limits.
|
|
1164
|
+
|
|
1165
|
+
Digests are streamed and reused only while file identity is unchanged. Open or read errors
|
|
1166
|
+
remain undecided and are retried on the next poll.
|
|
1167
|
+
"""
|
|
1168
|
+
|
|
1169
|
+
budget = MAX_VERIFIED_TOTAL_BYTES
|
|
1170
|
+
for member, entry in list(index.items())[:MAX_MANIFEST_MEMBERS]:
|
|
1171
|
+
if member in self._verified or member in self._measured_wrong:
|
|
1172
|
+
continue
|
|
1173
|
+
recorded = entry.get("sha256")
|
|
1174
|
+
size = entry.get("bytes")
|
|
1175
|
+
if not _is_recorded_digest(recorded) or type(size) is not int or size < 0:
|
|
1176
|
+
# The ENTRY is unusable, and the entry is part of the manifest this pass already
|
|
1177
|
+
# proved coherent. Nothing about the file on disk can change that, so it is decided.
|
|
1178
|
+
self._measured_wrong[member] = None
|
|
1179
|
+
continue
|
|
1180
|
+
if size > budget:
|
|
1181
|
+
# Not called present and not called absent: this reader declines to hash it. The
|
|
1182
|
+
# rail still lights the step from the member's OWN event, which is evidence of the
|
|
1183
|
+
# same fact from the producer rather than from this reader. Undecided, so the next
|
|
1184
|
+
# pass -- which starts with a full budget -- offers it the remainder again.
|
|
1185
|
+
continue
|
|
1186
|
+
digest = hashlib.sha256()
|
|
1187
|
+
read = 0
|
|
1188
|
+
try:
|
|
1189
|
+
handle_fd = _open_regular_member(member, dir_fd=fd)
|
|
1190
|
+
with os.fdopen(handle_fd, "rb") as handle:
|
|
1191
|
+
identity_before = _file_identity(os.fstat(handle.fileno()))
|
|
1192
|
+
while True:
|
|
1193
|
+
chunk = handle.read(_MEMBER_HASH_CHUNK_BYTES)
|
|
1194
|
+
if not chunk:
|
|
1195
|
+
break
|
|
1196
|
+
read += len(chunk)
|
|
1197
|
+
if read > size:
|
|
1198
|
+
break
|
|
1199
|
+
digest.update(chunk)
|
|
1200
|
+
identity_after = _file_identity(os.fstat(handle.fileno()))
|
|
1201
|
+
except OSError:
|
|
1202
|
+
# Absent, unreadable, a symlink or a FIFO where a member should be. This poll could
|
|
1203
|
+
# not look; it is not a fact about the run, and the next poll asks again.
|
|
1204
|
+
budget -= read
|
|
1205
|
+
continue
|
|
1206
|
+
budget -= read
|
|
1207
|
+
if identity_before != identity_after:
|
|
1208
|
+
# The name changed while it was being measured. Neither the digest nor a refusal
|
|
1209
|
+
# is attached to that mixed read; the next poll opens the resulting snapshot.
|
|
1210
|
+
continue
|
|
1211
|
+
if read != size or not _matches_digest(digest.hexdigest(), recorded):
|
|
1212
|
+
self._measured_wrong[member] = identity_after
|
|
1213
|
+
continue
|
|
1214
|
+
self._measured_wrong.pop(member, None)
|
|
1215
|
+
self._verified[member] = identity_after
|
|
1216
|
+
|
|
1217
|
+
def _read_manifest_snapshot(self, fd: int) -> tuple[bytes, tuple[int, int, int, int, int]]:
|
|
1218
|
+
"""Read ``manifest.json`` RELATIVE to the retained descriptor, never by path.
|
|
1219
|
+
|
|
1220
|
+
Holding the descriptor rather than re-opening by path per request is what makes a receipt
|
|
1221
|
+
immune to a path swapped underneath it. A missing or non-regular manifest is retried on the
|
|
1222
|
+
next poll.
|
|
1223
|
+
"""
|
|
1224
|
+
|
|
1225
|
+
handle_fd = _open_regular_member(_MANIFEST_NAME, dir_fd=fd)
|
|
1226
|
+
with os.fdopen(handle_fd, "rb") as handle:
|
|
1227
|
+
identity_before = _file_identity(os.fstat(handle.fileno()))
|
|
1228
|
+
raw = handle.read(_MAX_MANIFEST_BYTES + 1)
|
|
1229
|
+
identity_after = _file_identity(os.fstat(handle.fileno()))
|
|
1230
|
+
if len(raw) > _MAX_MANIFEST_BYTES:
|
|
1231
|
+
raise OSError("manifest is larger than this reader accepts")
|
|
1232
|
+
if identity_before != identity_after:
|
|
1233
|
+
raise OSError("manifest changed while this reader measured it")
|
|
1234
|
+
return raw, identity_after
|
|
1235
|
+
|
|
1236
|
+
def _drop_run(self) -> None:
|
|
1237
|
+
# Detach the retained descriptor while holding the same lock used to duplicate it. A
|
|
1238
|
+
# reader therefore either owns its duplicate before this point or observes no candidate;
|
|
1239
|
+
# it can never carry a bare descriptor number across this close and into kernel reuse.
|
|
1240
|
+
with self._lock:
|
|
1241
|
+
candidate_fd = self._candidate_fd
|
|
1242
|
+
self._candidate_fd = None
|
|
1243
|
+
self._candidate_identity = None
|
|
1244
|
+
self._rederived = None
|
|
1245
|
+
self._rederived_proof = None
|
|
1246
|
+
self._manifest = None
|
|
1247
|
+
self._members_present = frozenset()
|
|
1248
|
+
self._member_index = {}
|
|
1249
|
+
self._manifest_disowned = False
|
|
1250
|
+
self._manifest_file_identity = None
|
|
1251
|
+
self._verified = {}
|
|
1252
|
+
self._measured_wrong = {}
|
|
1253
|
+
self._settled_probe_cursor = 0
|
|
1254
|
+
if candidate_fd is not None:
|
|
1255
|
+
try:
|
|
1256
|
+
os.close(candidate_fd)
|
|
1257
|
+
except OSError: # pragma: no cover - closing a descriptor twice is the only shape
|
|
1258
|
+
pass
|
|
1259
|
+
|
|
1260
|
+
def _invalidate_installed_manifest(self, *, disowned: bool) -> None:
|
|
1261
|
+
"""Revoke every conclusion derived from manifest bytes this poll found invalid.
|
|
1262
|
+
|
|
1263
|
+
The candidate descriptor remains pinned so a restored receipt can recover on the next
|
|
1264
|
+
poll. Only the installed authority, allowlist and member decisions are cleared; retaining
|
|
1265
|
+
any one of them would keep the stale page or a stale receipt route alive.
|
|
1266
|
+
"""
|
|
1267
|
+
|
|
1268
|
+
self._rederived = None
|
|
1269
|
+
self._rederived_proof = None
|
|
1270
|
+
with self._lock:
|
|
1271
|
+
self._manifest = None
|
|
1272
|
+
self._members_present = frozenset()
|
|
1273
|
+
self._member_index = {}
|
|
1274
|
+
self._manifest_disowned = disowned
|
|
1275
|
+
self._manifest_file_identity = None
|
|
1276
|
+
self._verified.clear()
|
|
1277
|
+
self._measured_wrong.clear()
|
|
1278
|
+
self._settled_probe_cursor = 0
|
|
1279
|
+
|
|
1280
|
+
def close(self) -> None:
|
|
1281
|
+
"""Release the retained descriptor. Called on server shutdown."""
|
|
1282
|
+
|
|
1283
|
+
self._drop_run()
|
|
1284
|
+
|
|
1285
|
+
|
|
1286
|
+
class _WatchServer(ThreadingHTTPServer):
|
|
1287
|
+
daemon_threads = True
|
|
1288
|
+
|
|
1289
|
+
def __init__(self, address: tuple[str, int], handler: type, state: _WatchState) -> None:
|
|
1290
|
+
super().__init__(address, handler)
|
|
1291
|
+
self.watch_state = state
|
|
1292
|
+
self.watch_stopped = False
|
|
1293
|
+
self.watch_thread: threading.Thread | None = None
|
|
1294
|
+
|
|
1295
|
+
|
|
1296
|
+
class _WatchServerV6(_WatchServer):
|
|
1297
|
+
address_family = socket.AF_INET6
|
|
1298
|
+
|
|
1299
|
+
|
|
1300
|
+
class _WatchHandler(BaseHTTPRequestHandler):
|
|
1301
|
+
# The server exposes read-only GET routes. Approval remains a terminal operation.
|
|
1302
|
+
|
|
1303
|
+
def log_message(self, *_args: object) -> None: # silence default stderr access logging
|
|
1304
|
+
return
|
|
1305
|
+
|
|
1306
|
+
@property
|
|
1307
|
+
def _state(self) -> _WatchState:
|
|
1308
|
+
return self.server.watch_state # type: ignore[attr-defined]
|
|
1309
|
+
|
|
1310
|
+
def do_GET(self) -> None: # BaseHTTPRequestHandler's fixed dispatch name
|
|
1311
|
+
if not self._request_targets_this_loopback_server():
|
|
1312
|
+
self.send_error(421)
|
|
1313
|
+
return
|
|
1314
|
+
path = self.path.split("?", 1)[0]
|
|
1315
|
+
if path == "/":
|
|
1316
|
+
self._serve_index()
|
|
1317
|
+
elif path == "/events":
|
|
1318
|
+
self._serve_events()
|
|
1319
|
+
elif path == "/receipt":
|
|
1320
|
+
self._serve_receipt()
|
|
1321
|
+
else:
|
|
1322
|
+
self.send_error(404)
|
|
1323
|
+
|
|
1324
|
+
def _request_targets_this_loopback_server(self) -> bool:
|
|
1325
|
+
"""Reject DNS-rebound and cross-origin reads before any run content is written."""
|
|
1326
|
+
|
|
1327
|
+
bound_port = int(self.server.server_address[1])
|
|
1328
|
+
try:
|
|
1329
|
+
bound_address = ipaddress.ip_address(str(self.server.server_address[0]))
|
|
1330
|
+
except ValueError:
|
|
1331
|
+
return False
|
|
1332
|
+
|
|
1333
|
+
def local_authority(value: str, *, origin: bool) -> bool:
|
|
1334
|
+
try:
|
|
1335
|
+
parsed = urllib.parse.urlsplit(value if origin else f"//{value}")
|
|
1336
|
+
hostname = parsed.hostname
|
|
1337
|
+
port = parsed.port
|
|
1338
|
+
except ValueError:
|
|
1339
|
+
return False
|
|
1340
|
+
if parsed.username is not None or parsed.password is not None:
|
|
1341
|
+
return False
|
|
1342
|
+
if hostname is None or port != bound_port:
|
|
1343
|
+
return False
|
|
1344
|
+
if hostname == "localhost":
|
|
1345
|
+
return True
|
|
1346
|
+
try:
|
|
1347
|
+
requested = ipaddress.ip_address(hostname)
|
|
1348
|
+
except ValueError:
|
|
1349
|
+
return False
|
|
1350
|
+
return requested.is_loopback and requested == bound_address
|
|
1351
|
+
|
|
1352
|
+
host = self.headers.get("Host")
|
|
1353
|
+
if host is None or not local_authority(host, origin=False):
|
|
1354
|
+
return False
|
|
1355
|
+
origin = self.headers.get("Origin")
|
|
1356
|
+
return origin is None or local_authority(origin, origin=True)
|
|
1357
|
+
|
|
1358
|
+
def _serve_index(self) -> None:
|
|
1359
|
+
"""Write the memoized bytes. Performs no file I/O and cannot raise on bad feed bytes."""
|
|
1360
|
+
|
|
1361
|
+
_version, _last_seq, markup = self._state.snapshot()
|
|
1362
|
+
self._write_html(200, markup)
|
|
1363
|
+
|
|
1364
|
+
def _serve_receipt(self) -> None:
|
|
1365
|
+
"""Serve one receipt, verified against the fingerprint the run recorded for it.
|
|
1366
|
+
|
|
1367
|
+
The manifest, the allowlist and the directory descriptor are read from ``_WatchState`` on
|
|
1368
|
+
every request and never captured at handler construction because they may change while the
|
|
1369
|
+
build runs.
|
|
1370
|
+
"""
|
|
1371
|
+
|
|
1372
|
+
query = urllib.parse.parse_qs(urllib.parse.urlparse(self.path).query)
|
|
1373
|
+
member = (query.get("member") or [""])[0]
|
|
1374
|
+
cited_values = query.get("sha256") or []
|
|
1375
|
+
cited = cited_values[0] if cited_values else None
|
|
1376
|
+
|
|
1377
|
+
try:
|
|
1378
|
+
raw = self._state.read_member(member, cited)
|
|
1379
|
+
except ReceiptRefused as refusal:
|
|
1380
|
+
self._write_html(refusal.status, render_refusal(refusal.sentence))
|
|
1381
|
+
return
|
|
1382
|
+
except Exception: # pragma: no cover - every refusal shape above is typed
|
|
1383
|
+
self._write_html(404, render_refusal(_NOT_A_RECEIPT))
|
|
1384
|
+
return
|
|
1385
|
+
|
|
1386
|
+
try:
|
|
1387
|
+
markup = render_receipt(member, raw)
|
|
1388
|
+
except Exception: # pragma: no cover - render_receipt degrades rather than raising
|
|
1389
|
+
self._write_html(404, render_refusal(_NOT_A_RECEIPT))
|
|
1390
|
+
return
|
|
1391
|
+
self._write_html(200, markup)
|
|
1392
|
+
|
|
1393
|
+
def _write_html(self, status: int, markup: str) -> None:
|
|
1394
|
+
body = markup.encode("utf-8")
|
|
1395
|
+
self.send_response(status)
|
|
1396
|
+
self.send_header("Content-Type", "text/html; charset=utf-8")
|
|
1397
|
+
self.send_header("Content-Length", str(len(body)))
|
|
1398
|
+
self.end_headers()
|
|
1399
|
+
self.wfile.write(body)
|
|
1400
|
+
|
|
1401
|
+
def _serve_events(self) -> None:
|
|
1402
|
+
self.send_response(200)
|
|
1403
|
+
self.send_header("Content-Type", "text/event-stream")
|
|
1404
|
+
self.send_header("Cache-Control", "no-cache")
|
|
1405
|
+
self.send_header("Connection", "keep-alive")
|
|
1406
|
+
self.end_headers()
|
|
1407
|
+
last = self._state.snapshot()[0]
|
|
1408
|
+
try:
|
|
1409
|
+
while not getattr(self.server, "watch_stopped", False):
|
|
1410
|
+
version, last_seq, _markup = self._state.snapshot()
|
|
1411
|
+
if version != last:
|
|
1412
|
+
last = version
|
|
1413
|
+
# The version leads, and the sequence number rides along. The client treats a
|
|
1414
|
+
# frame it has already seen as nothing to do, so the value on the wire has to
|
|
1415
|
+
# change whenever the served page changes -- and the sequence number alone does
|
|
1416
|
+
# not: a failed poll puts a stall note on the page and adds no record, so a tab
|
|
1417
|
+
# deduping on the sequence threw away the one frame that mattered. It is still
|
|
1418
|
+
# a state identity and not a reload signal: a full reload on every event is a
|
|
1419
|
+
# flicker, and the page should feel like a record of events accumulating, not a
|
|
1420
|
+
# dashboard refreshing.
|
|
1421
|
+
self.wfile.write(f"data: {version}.{last_seq}\n\n".encode())
|
|
1422
|
+
else:
|
|
1423
|
+
self.wfile.write(b": ping\n\n")
|
|
1424
|
+
self.wfile.flush()
|
|
1425
|
+
time.sleep(_POLL_SECONDS)
|
|
1426
|
+
except (BrokenPipeError, ConnectionResetError, OSError):
|
|
1427
|
+
return
|
|
1428
|
+
|
|
1429
|
+
|
|
1430
|
+
def _watch_loop(server: _WatchServer) -> None:
|
|
1431
|
+
state = server.watch_state
|
|
1432
|
+
while not server.watch_stopped:
|
|
1433
|
+
state.poll()
|
|
1434
|
+
time.sleep(_POLL_SECONDS)
|
|
1435
|
+
|
|
1436
|
+
|
|
1437
|
+
def _shutdown_watch_server(server: _WatchServer) -> None:
|
|
1438
|
+
"""Stop and join the sole poll thread before closing anything it can still reach."""
|
|
1439
|
+
|
|
1440
|
+
server.watch_stopped = True
|
|
1441
|
+
failure: BaseException | None = None
|
|
1442
|
+
thread = server.watch_thread
|
|
1443
|
+
if thread is not None and thread.ident is not None:
|
|
1444
|
+
try:
|
|
1445
|
+
thread.join()
|
|
1446
|
+
except BaseException as error:
|
|
1447
|
+
failure = error
|
|
1448
|
+
try:
|
|
1449
|
+
server.watch_state.close()
|
|
1450
|
+
except BaseException as error:
|
|
1451
|
+
if failure is None:
|
|
1452
|
+
failure = error
|
|
1453
|
+
try:
|
|
1454
|
+
server.server_close()
|
|
1455
|
+
except BaseException as error:
|
|
1456
|
+
if failure is None:
|
|
1457
|
+
failure = error
|
|
1458
|
+
if failure is not None:
|
|
1459
|
+
raise failure
|
|
1460
|
+
|
|
1461
|
+
|
|
1462
|
+
def _build_server(
|
|
1463
|
+
target: _Target | str | os.PathLike[str],
|
|
1464
|
+
host: str,
|
|
1465
|
+
port: int,
|
|
1466
|
+
*,
|
|
1467
|
+
replay: float | None = None,
|
|
1468
|
+
) -> tuple[_WatchServer, str]:
|
|
1469
|
+
"""Bind a loopback server over the run's feed and start its watcher thread."""
|
|
1470
|
+
|
|
1471
|
+
state = _WatchState(target, replay=replay)
|
|
1472
|
+
server: _WatchServer | None = None
|
|
1473
|
+
try:
|
|
1474
|
+
address = ipaddress.ip_address(host)
|
|
1475
|
+
server_type = _WatchServerV6 if address.version == 6 else _WatchServer
|
|
1476
|
+
server = server_type((address.compressed, port), _WatchHandler, state)
|
|
1477
|
+
bound_host = str(server.server_address[0])
|
|
1478
|
+
url_host = f"[{bound_host}]" if address.version == 6 else bound_host
|
|
1479
|
+
url = f"http://{url_host}:{server.server_address[1]}/"
|
|
1480
|
+
server.watch_thread = threading.Thread(target=_watch_loop, args=(server,), daemon=True)
|
|
1481
|
+
server.watch_thread.start()
|
|
1482
|
+
return server, url
|
|
1483
|
+
except BaseException:
|
|
1484
|
+
try:
|
|
1485
|
+
if server is not None:
|
|
1486
|
+
_shutdown_watch_server(server)
|
|
1487
|
+
else:
|
|
1488
|
+
state.close()
|
|
1489
|
+
except BaseException:
|
|
1490
|
+
# Construction/startup is the first failure. Cleanup still runs compoundly above,
|
|
1491
|
+
# but a secondary failure must not replace the one the caller needs to diagnose.
|
|
1492
|
+
pass
|
|
1493
|
+
raise
|
|
1494
|
+
|
|
1495
|
+
|
|
1496
|
+
def serve_watch(
|
|
1497
|
+
target: _Target | str | os.PathLike[str],
|
|
1498
|
+
*,
|
|
1499
|
+
host: str = "127.0.0.1",
|
|
1500
|
+
port: int = 0,
|
|
1501
|
+
open_browser: bool = False,
|
|
1502
|
+
replay: float | None = None,
|
|
1503
|
+
announce: Callable[[str], None] | None = None,
|
|
1504
|
+
) -> None:
|
|
1505
|
+
"""Serve the live run view on ``localhost``. ``KeyboardInterrupt`` (Ctrl-C) shuts it down.
|
|
1506
|
+
|
|
1507
|
+
The served-URL line is flushed immediately: the port is ephemeral, so a caller that starts this
|
|
1508
|
+
in the background has to read that line to learn the port, and a block-buffered pipe would
|
|
1509
|
+
withhold it for the entire life of the server.
|
|
1510
|
+
|
|
1511
|
+
``replay`` paces a finished run's records on their original inter-arrival gaps divided by the
|
|
1512
|
+
given speed. Replay opens nothing for writing: it re-runs no build, appends to no feed and
|
|
1513
|
+
touches no byte of the run directory.
|
|
1514
|
+
"""
|
|
1515
|
+
|
|
1516
|
+
server, url = _build_server(target, host, port, replay=replay)
|
|
1517
|
+
try:
|
|
1518
|
+
if announce is None:
|
|
1519
|
+
print(f"mr-data watch: serving {url} (Ctrl-C to stop)", flush=True)
|
|
1520
|
+
else:
|
|
1521
|
+
announce(url)
|
|
1522
|
+
if open_browser:
|
|
1523
|
+
webbrowser.open(url)
|
|
1524
|
+
try:
|
|
1525
|
+
server.serve_forever()
|
|
1526
|
+
except KeyboardInterrupt:
|
|
1527
|
+
pass
|
|
1528
|
+
finally:
|
|
1529
|
+
_shutdown_watch_server(server)
|
|
1530
|
+
|
|
1531
|
+
|
|
1532
|
+
# ------------------------------------------------------------------------------------------------
|
|
1533
|
+
# The page
|
|
1534
|
+
# ------------------------------------------------------------------------------------------------
|
|
1535
|
+
|
|
1536
|
+
|
|
1537
|
+
_STOPPED_EVENTS = STOPPED_EVENTS
|
|
1538
|
+
|
|
1539
|
+
|
|
1540
|
+
def _rail_positions() -> dict[str, tuple[int, int]]:
|
|
1541
|
+
"""Map every event name onto its position in the rail. A stage header sits at step ``-1``."""
|
|
1542
|
+
|
|
1543
|
+
positions: dict[str, tuple[int, int]] = {}
|
|
1544
|
+
for stage_index, stage in enumerate(STAGE_RAIL):
|
|
1545
|
+
for name in stage["events"]:
|
|
1546
|
+
positions[name] = (stage_index, -1)
|
|
1547
|
+
for step_index, step in enumerate(stage["steps"]):
|
|
1548
|
+
for name in step["events"]:
|
|
1549
|
+
positions[name] = (stage_index, step_index)
|
|
1550
|
+
return positions
|
|
1551
|
+
|
|
1552
|
+
|
|
1553
|
+
_EVENT_POSITIONS = _rail_positions()
|
|
1554
|
+
|
|
1555
|
+
|
|
1556
|
+
def _rail_members(element: Mapping[str, Any]) -> tuple[str, ...]:
|
|
1557
|
+
"""The files that prove this rail element happened, or an empty tuple when it has none.
|
|
1558
|
+
|
|
1559
|
+
A step declares no ``members`` key of its own: the member its event points at IS its proof, and
|
|
1560
|
+
``_rail_member`` already resolves it from ``events.EVIDENCE_MEMBER``.
|
|
1561
|
+
"""
|
|
1562
|
+
|
|
1563
|
+
declared = element.get("members")
|
|
1564
|
+
if declared:
|
|
1565
|
+
return tuple(declared)
|
|
1566
|
+
member = _rail_member(element["events"])
|
|
1567
|
+
return (member,) if member is not None else ()
|
|
1568
|
+
|
|
1569
|
+
|
|
1570
|
+
# Written with %-placeholders rather than an f-string so CSS braces stay unescaped and readable.
|
|
1571
|
+
# Every colour and type value is an imported token; nothing here invents a hex. Classes are
|
|
1572
|
+
# `mr-watch-` prefixed kebab-case with `is-<state>` for transient state, following the notebook
|
|
1573
|
+
# renderer's class convention without touching any of its files.
|
|
1574
|
+
_CSS = """
|
|
1575
|
+
body { margin: 0; padding: 32px 24px 56px; background: %(paper)s; color: %(ink)s;
|
|
1576
|
+
font-family: %(sans)s; }
|
|
1577
|
+
.mr-watch-page { max-width: 860px; margin: 0 auto; }
|
|
1578
|
+
.mr-watch-title { font-size: 25px; font-weight: 600; letter-spacing: -0.02em; margin: 0 0 6px;
|
|
1579
|
+
color: %(ink)s; }
|
|
1580
|
+
.mr-watch-run { font-family: %(mono)s; font-size: 11.5px; color: %(muted)s; margin: 0 0 4px; }
|
|
1581
|
+
.mr-watch-elapsed { font-family: %(mono)s; font-size: 11.5px; color: %(faint)s; margin: 0 0 26px; }
|
|
1582
|
+
.mr-watch-rail { list-style: none; margin: 0 0 28px; padding: 16px 20px; border-radius: 12px;
|
|
1583
|
+
border: 1px solid %(border)s; background: %(surface)s; }
|
|
1584
|
+
.mr-watch-stage { display: block; padding: 5px 0; }
|
|
1585
|
+
.mr-watch-stage-mark, .mr-watch-step-mark { display: inline-block; width: 8px; height: 8px;
|
|
1586
|
+
border-radius: 2px; margin-right: 10px; vertical-align: 1px; background: %(border)s; }
|
|
1587
|
+
.mr-watch-stage.is-done > .mr-watch-stage-mark { background: %(cobalt)s; }
|
|
1588
|
+
.mr-watch-step.is-done > .mr-watch-step-mark { background: %(cobalt)s; }
|
|
1589
|
+
.mr-watch-stage-label { font-size: 15px; font-weight: 600; color: %(faint)s; }
|
|
1590
|
+
.mr-watch-stage.is-done > .mr-watch-stage-label { color: %(ink)s; }
|
|
1591
|
+
.mr-watch-steps { list-style: none; margin: 6px 0 6px 18px; padding: 0; }
|
|
1592
|
+
.mr-watch-step { display: block; padding: 3px 0; }
|
|
1593
|
+
.mr-watch-step-label { font-family: %(mono)s; font-size: 12px; color: %(faint)s; }
|
|
1594
|
+
.mr-watch-step.is-done > .mr-watch-step-label { color: %(body)s; }
|
|
1595
|
+
.mr-watch-log { list-style: none; margin: 0; padding: 0; border-top: 1px solid %(border)s; }
|
|
1596
|
+
.mr-watch-event { display: flex; gap: 14px; align-items: baseline; padding: 7px 2px;
|
|
1597
|
+
border-bottom: 1px solid %(border)s; }
|
|
1598
|
+
.mr-watch-event.is-stopped > .mr-watch-text { color: %(orange)s; }
|
|
1599
|
+
.mr-watch-time { font-family: %(mono)s; font-size: 11px; color: %(faint)s; min-width: 56px;
|
|
1600
|
+
font-variant-numeric: tabular-nums; }
|
|
1601
|
+
.mr-watch-text { font-size: 14px; color: %(body)s; flex: 1; }
|
|
1602
|
+
.mr-watch-receipt { font-family: %(mono)s; font-size: 11px; color: %(muted)s; }
|
|
1603
|
+
.mr-watch-receipt-link { font-family: %(mono)s; font-size: 11px; color: %(cobalt)s; }
|
|
1604
|
+
.mr-watch-later { font-family: %(mono)s; font-size: 11px; color: %(dim)s; }
|
|
1605
|
+
.mr-watch-empty { font-family: %(mono)s; font-size: 12px; color: %(faint)s; }
|
|
1606
|
+
.mr-watch-stalled { font-size: 13.5px; color: %(orange)s; border: 1px solid %(orange)s;
|
|
1607
|
+
border-radius: 8px; background: %(sub_paper)s; padding: 10px 12px; margin: 0 0 20px; }
|
|
1608
|
+
.mr-watch-cards { margin: 0 0 28px; }
|
|
1609
|
+
.mr-watch-card { border: 1px solid %(border)s; border-radius: 12px; background: %(sub_paper)s;
|
|
1610
|
+
padding: 16px 20px; margin: 0 0 14px; }
|
|
1611
|
+
.mr-watch-card-title { font-size: 16px; font-weight: 600; margin: 0 0 8px; color: %(ink)s; }
|
|
1612
|
+
.mr-watch-card-lead { font-size: 14px; color: %(body)s; margin: 0 0 10px; }
|
|
1613
|
+
.mr-watch-list { margin: 0 0 8px; padding-left: 20px; }
|
|
1614
|
+
.mr-watch-list li { font-size: 13.5px; color: %(body)s; padding: 2px 0; }
|
|
1615
|
+
.mr-watch-sub { border-top: 1px solid %(border)s; margin-top: 10px; padding-top: 10px; }
|
|
1616
|
+
.mr-watch-sub-title { font-size: 14px; font-weight: 600; color: %(ink)s; margin: 0 0 4px; }
|
|
1617
|
+
.mr-watch-defs { margin: 0; font-size: 13.5px; color: %(body)s; }
|
|
1618
|
+
.mr-watch-defs dt { font-weight: 600; color: %(muted)s; font-size: 12px; margin-top: 6px; }
|
|
1619
|
+
.mr-watch-defs dd { margin: 1px 0 0; }
|
|
1620
|
+
.mr-watch-table { border-collapse: collapse; width: 100%%; font-size: 13px; margin: 6px 0 2px; }
|
|
1621
|
+
.mr-watch-table th { text-align: left; font-size: 11.5px; color: %(muted)s; font-weight: 600;
|
|
1622
|
+
border-bottom: 1px solid %(border)s; padding: 4px 10px 4px 0; }
|
|
1623
|
+
.mr-watch-table td { color: %(body)s; border-bottom: 1px solid %(border)s; padding: 4px 10px 4px 0;
|
|
1624
|
+
vertical-align: top; }
|
|
1625
|
+
.mr-watch-approve { border: 1px solid %(cobalt)s; border-radius: 12px; background: %(sub_paper)s;
|
|
1626
|
+
padding: 16px 20px; margin: 0 0 24px; }
|
|
1627
|
+
.mr-watch-note { font-size: 13px; color: %(muted)s; margin: 10px 0 8px; }
|
|
1628
|
+
.mr-watch-command { display: block; font-family: %(mono)s; font-size: 11.5px; color: %(ink)s;
|
|
1629
|
+
background: %(surface)s; border: 1px solid %(border)s; border-radius: 8px; padding: 10px 12px;
|
|
1630
|
+
overflow-x: auto; white-space: pre; }
|
|
1631
|
+
.mr-watch-copy { font-family: %(sans)s; font-size: 12.5px; color: %(paper)s; background: %(cobalt)s;
|
|
1632
|
+
border: 1px solid %(cobalt)s; border-radius: 8px; padding: 6px 14px; margin-top: 10px;
|
|
1633
|
+
cursor: pointer; }
|
|
1634
|
+
.mr-watch-back { font-size: 12.5px; color: %(cobalt)s; display: inline-block; margin-bottom: 18px; }
|
|
1635
|
+
.mr-watch-refusal { font-size: 15px; color: %(orange)s; margin: 0; }
|
|
1636
|
+
.mr-watch-pre { font-family: %(mono)s; font-size: 11.5px; color: %(body)s; white-space: pre-wrap;
|
|
1637
|
+
word-break: break-all; margin: 0; }
|
|
1638
|
+
"""
|
|
1639
|
+
|
|
1640
|
+
# The whole client: one EventSource, one 1 Hz recompute of the elapsed text. No framework, no
|
|
1641
|
+
# bundler, no build step, no network. Wrapped in try/catch so live update is best-effort -- a page
|
|
1642
|
+
# whose script failed is still a correct record of everything that had happened when it was served.
|
|
1643
|
+
_LIVE_SCRIPT = (
|
|
1644
|
+
"<script>\n"
|
|
1645
|
+
"(function () {\n"
|
|
1646
|
+
" try {\n"
|
|
1647
|
+
" var seen = null;\n"
|
|
1648
|
+
" var source = new EventSource('/events');\n"
|
|
1649
|
+
" source.onmessage = function (event) {\n"
|
|
1650
|
+
" if (event.data === seen) { return; }\n"
|
|
1651
|
+
" seen = event.data;\n"
|
|
1652
|
+
" fetch('/').then(function (response) { return response.text(); })\n"
|
|
1653
|
+
" .then(function (text) {\n"
|
|
1654
|
+
" var parsed = new DOMParser().parseFromString(text, 'text/html');\n"
|
|
1655
|
+
" document.body.innerHTML = parsed.body.innerHTML;\n"
|
|
1656
|
+
" })\n"
|
|
1657
|
+
" .catch(function () { /* the next event tries again */ });\n"
|
|
1658
|
+
" };\n"
|
|
1659
|
+
" var retick = function () {\n"
|
|
1660
|
+
" var node = document.querySelector('.mr-watch-elapsed');\n"
|
|
1661
|
+
" if (!node) { return; }\n"
|
|
1662
|
+
" var at = parseFloat(node.getAttribute('data-at'));\n"
|
|
1663
|
+
" if (!(at > 0)) { return; }\n"
|
|
1664
|
+
" var seconds = Math.max(0, Math.round(Date.now() / 1000 - at));\n"
|
|
1665
|
+
" node.textContent = 'Last event ' + seconds + ' seconds ago';\n"
|
|
1666
|
+
" };\n"
|
|
1667
|
+
" window.setInterval(retick, 1000);\n"
|
|
1668
|
+
" document.addEventListener('click', function (clicked) {\n"
|
|
1669
|
+
" var button = clicked.target.closest('.mr-watch-copy');\n"
|
|
1670
|
+
" if (!button) { return; }\n"
|
|
1671
|
+
" var target = document.getElementById(button.getAttribute('data-copy-target'));\n"
|
|
1672
|
+
" if (!target) { return; }\n"
|
|
1673
|
+
" var text = target.textContent;\n"
|
|
1674
|
+
" var done = function () {\n"
|
|
1675
|
+
" button.textContent = 'Copied';\n"
|
|
1676
|
+
" window.setTimeout(function () { button.textContent = 'Copy'; }, 2000);\n"
|
|
1677
|
+
" };\n"
|
|
1678
|
+
" if (navigator.clipboard && navigator.clipboard.writeText) {\n"
|
|
1679
|
+
" navigator.clipboard.writeText(text).then(done, function () {});\n"
|
|
1680
|
+
" return;\n"
|
|
1681
|
+
" }\n"
|
|
1682
|
+
" var scratch = document.createElement('textarea');\n"
|
|
1683
|
+
" scratch.value = text;\n"
|
|
1684
|
+
" document.body.appendChild(scratch);\n"
|
|
1685
|
+
" scratch.select();\n"
|
|
1686
|
+
" try { document.execCommand('copy'); done(); } catch (error) { /* nothing to do */ }\n"
|
|
1687
|
+
" document.body.removeChild(scratch);\n"
|
|
1688
|
+
" });\n"
|
|
1689
|
+
" } catch (error) { /* live update is best-effort */ }\n"
|
|
1690
|
+
"})();\n"
|
|
1691
|
+
"</script>"
|
|
1692
|
+
)
|
|
1693
|
+
|
|
1694
|
+
|
|
1695
|
+
def stylesheet() -> str:
|
|
1696
|
+
return (
|
|
1697
|
+
"<style>"
|
|
1698
|
+
+ _CSS
|
|
1699
|
+
% {
|
|
1700
|
+
"paper": PAPER,
|
|
1701
|
+
"ink": INK,
|
|
1702
|
+
"body": BODY,
|
|
1703
|
+
"muted": MUTED,
|
|
1704
|
+
"faint": FAINT,
|
|
1705
|
+
"dim": DIM,
|
|
1706
|
+
"border": BORDER,
|
|
1707
|
+
"surface": SURFACE,
|
|
1708
|
+
"sub_paper": SUB_PAPER,
|
|
1709
|
+
"cobalt": COBALT,
|
|
1710
|
+
"orange": ORANGE,
|
|
1711
|
+
"sans": FONT_SANS,
|
|
1712
|
+
"mono": FONT_MONO,
|
|
1713
|
+
}
|
|
1714
|
+
+ "</style>"
|
|
1715
|
+
)
|
|
1716
|
+
|
|
1717
|
+
|
|
1718
|
+
def _seen_events(records: Sequence[Mapping[str, Any]]) -> frozenset[str]:
|
|
1719
|
+
"""Return the rail event names present in the feed."""
|
|
1720
|
+
|
|
1721
|
+
return frozenset(
|
|
1722
|
+
str(record.get("event"))
|
|
1723
|
+
for record in records
|
|
1724
|
+
if str(record.get("event")) in _EVENT_POSITIONS
|
|
1725
|
+
)
|
|
1726
|
+
|
|
1727
|
+
|
|
1728
|
+
def _state_class(
|
|
1729
|
+
element: Mapping[str, Any],
|
|
1730
|
+
seen: frozenset[str],
|
|
1731
|
+
members_present: frozenset[str],
|
|
1732
|
+
) -> str:
|
|
1733
|
+
"""A rail item is done when its OWN evidence says so, and never on a later item's strength.
|
|
1734
|
+
|
|
1735
|
+
Two ways to be done, and no third: one of the element's own events is in the feed, or every
|
|
1736
|
+
file it names is in the sealed member set. Anything else waits. There is no animated state --
|
|
1737
|
+
progress on this page is events that happened, never a spinner.
|
|
1738
|
+
"""
|
|
1739
|
+
|
|
1740
|
+
if any(name in seen for name in element["events"]):
|
|
1741
|
+
return "is-done"
|
|
1742
|
+
members = _rail_members(element)
|
|
1743
|
+
if members and all(member in members_present for member in members):
|
|
1744
|
+
return "is-done"
|
|
1745
|
+
return "is-waiting"
|
|
1746
|
+
|
|
1747
|
+
|
|
1748
|
+
def _sentence(event: str, facts: Mapping[str, Any]) -> str:
|
|
1749
|
+
return sentence(event, facts)
|
|
1750
|
+
|
|
1751
|
+
|
|
1752
|
+
def _pointer_of(record: Mapping[str, Any]) -> tuple[str | None, str | None]:
|
|
1753
|
+
"""Return ``(member, sha256)`` for a record's evidence pointer, either half possibly ``None``.
|
|
1754
|
+
|
|
1755
|
+
A record sealed AFTER the fact carries both halves in ``evidence``; a record emitted DURING the
|
|
1756
|
+
build carries only the member name it will be provable against, in ``facts["evidence_member"]``.
|
|
1757
|
+
"""
|
|
1758
|
+
|
|
1759
|
+
evidence = record.get("evidence")
|
|
1760
|
+
if isinstance(evidence, Mapping):
|
|
1761
|
+
member = evidence.get("member")
|
|
1762
|
+
sha256 = evidence.get("sha256")
|
|
1763
|
+
if isinstance(member, str):
|
|
1764
|
+
return member, sha256 if isinstance(sha256, str) else None
|
|
1765
|
+
facts = record.get("facts")
|
|
1766
|
+
member = facts.get("evidence_member") if isinstance(facts, Mapping) else None
|
|
1767
|
+
return (member if isinstance(member, str) else None), None
|
|
1768
|
+
|
|
1769
|
+
|
|
1770
|
+
def receipt_href(member: str, sha256: str | None = None) -> str:
|
|
1771
|
+
"""Build the link to one receipt. Both values are percent-quoted before they reach the URL."""
|
|
1772
|
+
|
|
1773
|
+
query = {"member": member}
|
|
1774
|
+
if sha256:
|
|
1775
|
+
query["sha256"] = sha256
|
|
1776
|
+
return "/receipt?" + urllib.parse.urlencode(query, quote_via=urllib.parse.quote)
|
|
1777
|
+
|
|
1778
|
+
|
|
1779
|
+
def _receipt(
|
|
1780
|
+
record: Mapping[str, Any],
|
|
1781
|
+
members_present: frozenset[str],
|
|
1782
|
+
recorded_digests: Mapping[str, str] | None = None,
|
|
1783
|
+
) -> str:
|
|
1784
|
+
"""Return a receipt link when the member and manifest digest are verified.
|
|
1785
|
+
|
|
1786
|
+
A feed digest that disagrees with the manifest suppresses the link.
|
|
1787
|
+
"""
|
|
1788
|
+
|
|
1789
|
+
member, sha256 = _pointer_of(record)
|
|
1790
|
+
if member is None:
|
|
1791
|
+
return ""
|
|
1792
|
+
if member not in members_present:
|
|
1793
|
+
return '<span class="mr-watch-later">receipt appears when the build finishes</span>'
|
|
1794
|
+
recorded = (recorded_digests or {}).get(member)
|
|
1795
|
+
if sha256 is not None and recorded is not None and not _matches_digest(sha256, recorded):
|
|
1796
|
+
return ""
|
|
1797
|
+
return (
|
|
1798
|
+
f'<a class="mr-watch-receipt-link" href="{esc(receipt_href(member, recorded))}"'
|
|
1799
|
+
f' data-member="{esc(member)}">Receipt: {esc(member)}</a>'
|
|
1800
|
+
)
|
|
1801
|
+
|
|
1802
|
+
|
|
1803
|
+
def render_stall_note(failures: int, reason: str) -> str:
|
|
1804
|
+
"""The sentence a view that has stopped reading owes its reader.
|
|
1805
|
+
|
|
1806
|
+
Watch has no spinner: the page shows what happened, and where nothing has happened it says so.
|
|
1807
|
+
A page whose reader has stopped being able to read is the same
|
|
1808
|
+
obligation one step further on -- it is still showing a true record, but of a moment that has
|
|
1809
|
+
passed, and a reader has no way to tell that from a build that is simply taking its time.
|
|
1810
|
+
|
|
1811
|
+
The failure's own NAME is on the page and its message is not. A name comes from this codebase;
|
|
1812
|
+
a message can carry bytes out of the run directory, which is a place another local user writes.
|
|
1813
|
+
"""
|
|
1814
|
+
|
|
1815
|
+
times = "once" if failures == 1 else f"{failures} times"
|
|
1816
|
+
return (
|
|
1817
|
+
f'<p class="mr-watch-stalled" data-failed-polls="{esc(failures)}"'
|
|
1818
|
+
f' data-reason="{esc(reason)}">'
|
|
1819
|
+
f"This page has stopped updating. Reading this build failed {esc(times)}, "
|
|
1820
|
+
f"the last time with {esc(reason)}. "
|
|
1821
|
+
"What you see below is the last thing this page could read."
|
|
1822
|
+
"</p>"
|
|
1823
|
+
)
|
|
1824
|
+
|
|
1825
|
+
|
|
1826
|
+
# The stall note is spliced in after the header rather than rendered with the page, because the page
|
|
1827
|
+
# it belongs on is by definition one this reader could NOT re-render. Every page this module builds
|
|
1828
|
+
# opens with exactly one header, so the first close tag is the anchor.
|
|
1829
|
+
_STALL_ANCHOR = "</header>"
|
|
1830
|
+
|
|
1831
|
+
|
|
1832
|
+
def _with_stall_note(markup: str, failures: int, reason: str) -> str:
|
|
1833
|
+
"""Return the last good page with the stall note under its header."""
|
|
1834
|
+
|
|
1835
|
+
head, anchor, tail = markup.partition(_STALL_ANCHOR)
|
|
1836
|
+
note = render_stall_note(failures, reason)
|
|
1837
|
+
if not anchor: # pragma: no cover - every page this module renders carries a header
|
|
1838
|
+
return note + markup
|
|
1839
|
+
return head + anchor + note + tail
|
|
1840
|
+
|
|
1841
|
+
|
|
1842
|
+
def _render_header(run_dir: Path, records: Sequence[Mapping[str, Any]], now: float) -> str:
|
|
1843
|
+
last_at = float(records[-1]["at"]) if records else 0.0
|
|
1844
|
+
if records and last_at > 0:
|
|
1845
|
+
text = f"Last event {max(0, int(now - last_at))} seconds ago"
|
|
1846
|
+
elif records:
|
|
1847
|
+
# Rederived records have no source timestamp.
|
|
1848
|
+
text = "Rebuilt from the files this build recorded"
|
|
1849
|
+
else:
|
|
1850
|
+
text = "No events yet"
|
|
1851
|
+
return (
|
|
1852
|
+
'<header class="mr-watch-header">'
|
|
1853
|
+
'<h1 class="mr-watch-title">Watch</h1>'
|
|
1854
|
+
f'<p class="mr-watch-run">Build: {esc(_writable_path_text(run_dir.name))}</p>'
|
|
1855
|
+
f'<p class="mr-watch-elapsed" data-at="{esc(f"{last_at:.3f}")}">{esc(text)}</p>'
|
|
1856
|
+
"</header>"
|
|
1857
|
+
)
|
|
1858
|
+
|
|
1859
|
+
|
|
1860
|
+
def _rail_member(names: Sequence[str]) -> str | None:
|
|
1861
|
+
"""The receipt a rail element stands for: the first of its events with a fixed pointer.
|
|
1862
|
+
|
|
1863
|
+
``member_sealed`` and ``build_sealed`` have no fixed member -- the first varies per member and
|
|
1864
|
+
the second anchors on the run's own fingerprint -- so the "Receipt recorded" step falls through
|
|
1865
|
+
to the run's own receipt, which is exactly what a reader clicking it wants to see.
|
|
1866
|
+
"""
|
|
1867
|
+
|
|
1868
|
+
for name in names:
|
|
1869
|
+
member = events.EVIDENCE_MEMBER.get(name)
|
|
1870
|
+
if member is not None:
|
|
1871
|
+
return member
|
|
1872
|
+
if "build_sealed" in names:
|
|
1873
|
+
return _MANIFEST_NAME
|
|
1874
|
+
return None
|
|
1875
|
+
|
|
1876
|
+
|
|
1877
|
+
def _rail_label(label: str, css: str, names: Sequence[str], members_present: frozenset[str]) -> str:
|
|
1878
|
+
"""Render one rail element's label, linked to its receipt once that receipt exists."""
|
|
1879
|
+
|
|
1880
|
+
member = _rail_member(names)
|
|
1881
|
+
if member is None or member not in members_present:
|
|
1882
|
+
return f'<span class="{css}">{esc(label)}</span>'
|
|
1883
|
+
return (
|
|
1884
|
+
f'<a class="{css}" href="{esc(receipt_href(member))}"'
|
|
1885
|
+
f' data-member="{esc(member)}">{esc(label)}</a>'
|
|
1886
|
+
)
|
|
1887
|
+
|
|
1888
|
+
|
|
1889
|
+
def _render_rail(seen: frozenset[str], members_present: frozenset[str]) -> str:
|
|
1890
|
+
items: list[str] = []
|
|
1891
|
+
for stage in STAGE_RAIL:
|
|
1892
|
+
stage_state = _state_class(stage, seen, members_present)
|
|
1893
|
+
steps = "".join(
|
|
1894
|
+
f'<li class="mr-watch-step {_state_class(step, seen, members_present)}"'
|
|
1895
|
+
f' data-rail="{esc(stage["stage"])}.{esc(step["step"])}">'
|
|
1896
|
+
'<span class="mr-watch-step-mark"></span>'
|
|
1897
|
+
+ _rail_label(step["label"], "mr-watch-step-label", step["events"], members_present)
|
|
1898
|
+
+ "</li>"
|
|
1899
|
+
for step in stage["steps"]
|
|
1900
|
+
)
|
|
1901
|
+
items.append(
|
|
1902
|
+
f'<li class="mr-watch-stage {stage_state}" data-rail="{esc(stage["stage"])}">'
|
|
1903
|
+
'<span class="mr-watch-stage-mark"></span>'
|
|
1904
|
+
+ _rail_label(stage["label"], "mr-watch-stage-label", stage["events"], members_present)
|
|
1905
|
+
+ (f'<ol class="mr-watch-steps">{steps}</ol>' if steps else "")
|
|
1906
|
+
+ "</li>"
|
|
1907
|
+
)
|
|
1908
|
+
return '<ol class="mr-watch-rail">' + "".join(items) + "</ol>"
|
|
1909
|
+
|
|
1910
|
+
|
|
1911
|
+
def _render_log(
|
|
1912
|
+
records: Sequence[Mapping[str, Any]],
|
|
1913
|
+
members_present: frozenset[str],
|
|
1914
|
+
recorded_digests: Mapping[str, str] | None = None,
|
|
1915
|
+
) -> str:
|
|
1916
|
+
if not records:
|
|
1917
|
+
return '<p class="mr-watch-empty">Nothing has happened yet.</p>'
|
|
1918
|
+
started = float(records[0]["at"])
|
|
1919
|
+
items: list[str] = []
|
|
1920
|
+
for record in records:
|
|
1921
|
+
event = str(record.get("event", ""))
|
|
1922
|
+
facts = record.get("facts")
|
|
1923
|
+
offset = max(0, int(float(record.get("at", started)) - started))
|
|
1924
|
+
classes = "mr-watch-event is-stopped" if event in _STOPPED_EVENTS else "mr-watch-event"
|
|
1925
|
+
items.append(
|
|
1926
|
+
f'<li class="{classes}">'
|
|
1927
|
+
f'<span class="mr-watch-time">+{esc(offset)}s</span>'
|
|
1928
|
+
f'<span class="mr-watch-text">'
|
|
1929
|
+
f"{esc(_sentence(event, facts if isinstance(facts, Mapping) else {}))}</span>"
|
|
1930
|
+
f"{_receipt(record, members_present, recorded_digests)}"
|
|
1931
|
+
"</li>"
|
|
1932
|
+
)
|
|
1933
|
+
return '<ol class="mr-watch-log">' + "".join(items) + "</ol>"
|
|
1934
|
+
|
|
1935
|
+
|
|
1936
|
+
def render_page(
|
|
1937
|
+
records: Sequence[Mapping[str, Any]],
|
|
1938
|
+
*,
|
|
1939
|
+
run_dir: Path,
|
|
1940
|
+
members_present: frozenset[str],
|
|
1941
|
+
now: float,
|
|
1942
|
+
read_member: Callable[[str], bytes] | None = None,
|
|
1943
|
+
recorded_digests: Mapping[str, str] | None = None,
|
|
1944
|
+
approval_corroborated: bool = True,
|
|
1945
|
+
) -> str:
|
|
1946
|
+
"""Render the whole page. It opens no file of its own and takes every input as an argument.
|
|
1947
|
+
|
|
1948
|
+
``read_member`` is the one exception and it is a PARAMETER, not a capability this function
|
|
1949
|
+
reaches for: the cards and the approval summary are claims about what the run recorded, so they
|
|
1950
|
+
are built from the same fingerprint-verified read ``/receipt`` uses and can never render bytes
|
|
1951
|
+
that failed their check. With no reader the page is the rail and the event log, which is exactly
|
|
1952
|
+
what a watcher started before the build should show.
|
|
1953
|
+
|
|
1954
|
+
Every interpolated value crosses ``esc`` because feed and evidence strings may contain markup.
|
|
1955
|
+
|
|
1956
|
+
``recorded_digests`` is the run's OWN digest for each member -- ``member_index`` -- and it is
|
|
1957
|
+
what every receipt link uses.
|
|
1958
|
+
"""
|
|
1959
|
+
|
|
1960
|
+
seen = _seen_events(records)
|
|
1961
|
+
approval = ""
|
|
1962
|
+
cards = ""
|
|
1963
|
+
if read_member is not None and approval_corroborated:
|
|
1964
|
+
approval = _safe(lambda: render_approval(run_dir, members_present, read_member))
|
|
1965
|
+
if read_member is not None:
|
|
1966
|
+
cards = _safe(lambda: render_cards(members_present, read_member))
|
|
1967
|
+
return (
|
|
1968
|
+
"<!doctype html>\n"
|
|
1969
|
+
'<html lang="en"><head><meta charset="utf-8">'
|
|
1970
|
+
'<meta name="viewport" content="width=device-width, initial-scale=1">'
|
|
1971
|
+
"<title>Watch</title>"
|
|
1972
|
+
f"{stylesheet()}"
|
|
1973
|
+
"</head><body>\n"
|
|
1974
|
+
'<div class="mr-watch-page">'
|
|
1975
|
+
f"{_render_header(run_dir, records, now)}"
|
|
1976
|
+
f"{_render_rail(seen, members_present)}"
|
|
1977
|
+
f"{approval}"
|
|
1978
|
+
f"{cards}"
|
|
1979
|
+
f"{_render_log(records, members_present, recorded_digests)}"
|
|
1980
|
+
"</div>\n"
|
|
1981
|
+
f"{_LIVE_SCRIPT}\n"
|
|
1982
|
+
"</body></html>\n"
|
|
1983
|
+
)
|
|
1984
|
+
|
|
1985
|
+
|
|
1986
|
+
def _safe(render: Callable[[], str]) -> str:
|
|
1987
|
+
"""Render a section, or render nothing. A section is never allowed to blank the page.
|
|
1988
|
+
|
|
1989
|
+
The run directory is a place another local user can write, so a member can be any bytes at all.
|
|
1990
|
+
A card that raised on a shape it did not expect would take down the event log with it, and the
|
|
1991
|
+
event log is the part a reader most needs when something has gone wrong.
|
|
1992
|
+
"""
|
|
1993
|
+
|
|
1994
|
+
try:
|
|
1995
|
+
return render()
|
|
1996
|
+
except Exception:
|
|
1997
|
+
return ""
|
|
1998
|
+
|
|
1999
|
+
|
|
2000
|
+
# ------------------------------------------------------------------------------------------------
|
|
2001
|
+
# Plain vocabulary
|
|
2002
|
+
# ------------------------------------------------------------------------------------------------
|
|
2003
|
+
#
|
|
2004
|
+
# Two vocabularies, one direction. The evidence a receipt renders is internal contract JSON, and its
|
|
2005
|
+
# field names are engineering-precise on purpose. None of them may reach a page. The rule this file
|
|
2006
|
+
# follows is to map every field to a plain phrase and omit fields without a plain rendering.
|
|
2007
|
+
|
|
2008
|
+
# A word that, appearing in a key, means that key has no plain rendering and must be dropped.
|
|
2009
|
+
_INTERNAL_WORDS = frozenset(
|
|
2010
|
+
{
|
|
2011
|
+
"acquisition",
|
|
2012
|
+
"adapter",
|
|
2013
|
+
"attempt",
|
|
2014
|
+
"candidate",
|
|
2015
|
+
"capability",
|
|
2016
|
+
"caps",
|
|
2017
|
+
"classification",
|
|
2018
|
+
"credential",
|
|
2019
|
+
"crawler",
|
|
2020
|
+
"decode",
|
|
2021
|
+
"digest",
|
|
2022
|
+
"digests",
|
|
2023
|
+
"golden",
|
|
2024
|
+
"governance",
|
|
2025
|
+
"id",
|
|
2026
|
+
"ids",
|
|
2027
|
+
"lawful",
|
|
2028
|
+
"manifest",
|
|
2029
|
+
"materialization",
|
|
2030
|
+
"normalizer",
|
|
2031
|
+
"policy",
|
|
2032
|
+
"predecessor",
|
|
2033
|
+
"principal",
|
|
2034
|
+
"producer",
|
|
2035
|
+
"promotion",
|
|
2036
|
+
"sandbox",
|
|
2037
|
+
"schema",
|
|
2038
|
+
"sealed",
|
|
2039
|
+
"semantics",
|
|
2040
|
+
"sha256",
|
|
2041
|
+
"sha256s",
|
|
2042
|
+
"template",
|
|
2043
|
+
"verifier",
|
|
2044
|
+
"version",
|
|
2045
|
+
"watermark",
|
|
2046
|
+
"workspace",
|
|
2047
|
+
}
|
|
2048
|
+
)
|
|
2049
|
+
|
|
2050
|
+
# Keys whose plain phrase is worth stating explicitly rather than deriving.
|
|
2051
|
+
_PLAIN_KEYS: Mapping[str, str] = {
|
|
2052
|
+
"bytes": "Size in bytes",
|
|
2053
|
+
"cardinality": "Shape of the match",
|
|
2054
|
+
"cleaning": "Cleaning",
|
|
2055
|
+
"columns": "Columns",
|
|
2056
|
+
"created_at": "Recorded at",
|
|
2057
|
+
"evidence": "What backs it",
|
|
2058
|
+
"expected": "Expected",
|
|
2059
|
+
"grain": "One row per",
|
|
2060
|
+
"join": "Join",
|
|
2061
|
+
"keys": "Joined on",
|
|
2062
|
+
"left_rows": "Rows on the left",
|
|
2063
|
+
"left_source": "Left side",
|
|
2064
|
+
"matched_left_rows": "Rows that matched",
|
|
2065
|
+
"null_counts": "Blanks per column",
|
|
2066
|
+
"observed": "Observed",
|
|
2067
|
+
"operations": "What was done to it",
|
|
2068
|
+
"origin": "Where it came from",
|
|
2069
|
+
"output_intent": "Meant for",
|
|
2070
|
+
"output_rows": "Rows out",
|
|
2071
|
+
"passed": "Result",
|
|
2072
|
+
"permissions": "You may",
|
|
2073
|
+
"population": "Who is in it",
|
|
2074
|
+
"quality": "Checks",
|
|
2075
|
+
"questions": "Questions",
|
|
2076
|
+
"relative_path": "File",
|
|
2077
|
+
"rights": "Rights",
|
|
2078
|
+
"right_rows": "Rows on the right",
|
|
2079
|
+
"right_source": "Right side",
|
|
2080
|
+
"row_count": "Rows",
|
|
2081
|
+
"row_multiplier": "Row multiplier",
|
|
2082
|
+
"rows": "Rows",
|
|
2083
|
+
"select": "Columns kept",
|
|
2084
|
+
"sources": "Sources",
|
|
2085
|
+
"stable_order": "Row order",
|
|
2086
|
+
"status": "State",
|
|
2087
|
+
"text": "Text",
|
|
2088
|
+
"types": "Types",
|
|
2089
|
+
"unmatched_left_rows": "Rows that did not match",
|
|
2090
|
+
}
|
|
2091
|
+
|
|
2092
|
+
# Contract values that have a plain reading. Anything not here is humanized, never dropped: a value
|
|
2093
|
+
# is data the run recorded, and dropping data would be editing the receipt.
|
|
2094
|
+
_RIGHTS_STATUS: Mapping[str, str] = {
|
|
2095
|
+
"project_owned": "owned by this project",
|
|
2096
|
+
"public_domain": "in the public domain",
|
|
2097
|
+
"licensed": "used under a licence",
|
|
2098
|
+
"permitted": "cleared for this use",
|
|
2099
|
+
"restricted": "restricted",
|
|
2100
|
+
"unknown": "not decided yet",
|
|
2101
|
+
}
|
|
2102
|
+
|
|
2103
|
+
_RIGHTS_DECISION: Mapping[str, str] = {
|
|
2104
|
+
"approved": "cleared for this use",
|
|
2105
|
+
"conditional": "conditions attached",
|
|
2106
|
+
"denied": "not usable",
|
|
2107
|
+
"blocked": "not usable",
|
|
2108
|
+
"pending": "not decided yet",
|
|
2109
|
+
}
|
|
2110
|
+
|
|
2111
|
+
_PERMISSIONS: Mapping[str, str] = {
|
|
2112
|
+
"local_use": "use it here",
|
|
2113
|
+
"redistribute": "share it on",
|
|
2114
|
+
"host": "publish it",
|
|
2115
|
+
}
|
|
2116
|
+
|
|
2117
|
+
_HOW_FETCHED: Mapping[str, str] = {
|
|
2118
|
+
"checked_in_fixture": "a file kept with the project",
|
|
2119
|
+
"user_file": "a file you supplied",
|
|
2120
|
+
"download": "downloaded over the network",
|
|
2121
|
+
"api": "read from an interface",
|
|
2122
|
+
}
|
|
2123
|
+
|
|
2124
|
+
_MATCH_SHAPE: Mapping[str, str] = {
|
|
2125
|
+
"one_to_one": "one row to one row",
|
|
2126
|
+
"many_to_one": "many rows to one row",
|
|
2127
|
+
"one_to_many": "one row to many rows",
|
|
2128
|
+
"many_to_many": "many rows to many rows",
|
|
2129
|
+
}
|
|
2130
|
+
|
|
2131
|
+
_JOIN_KIND: Mapping[str, str] = {
|
|
2132
|
+
"left": "kept every row on the left",
|
|
2133
|
+
"inner": "kept only rows on both sides",
|
|
2134
|
+
}
|
|
2135
|
+
|
|
2136
|
+
_OUTPUT_INTENT: Mapping[str, str] = {
|
|
2137
|
+
"local_use": "use here only",
|
|
2138
|
+
"redistribute": "sharing on",
|
|
2139
|
+
"host": "publishing",
|
|
2140
|
+
}
|
|
2141
|
+
|
|
2142
|
+
# The five checks a Recipe run must pass (`recipe.REQUIRED_CHECK_INVENTORY`) plus the three a plan
|
|
2143
|
+
# declares, each with the sentence a person reads. A check id is an internal name by design; this is
|
|
2144
|
+
# the map that keeps it from reaching a page.
|
|
2145
|
+
_PLAIN_CHECKS: Mapping[str, str] = {
|
|
2146
|
+
"EXACT_CANDIDATE_REPLAY": "The build reproduces exactly, byte for byte",
|
|
2147
|
+
"RECIPE_DRIFT": "Nothing has drifted from the Recipe",
|
|
2148
|
+
"SOURCE_AUTHORITY": "Every source is the one the Recipe names",
|
|
2149
|
+
"OUTPUT_SEMANTICS": "Every column still means what was approved",
|
|
2150
|
+
"TEMPORAL_INTEGRITY": "The time window still holds",
|
|
2151
|
+
}
|
|
2152
|
+
|
|
2153
|
+
|
|
2154
|
+
def _humanize(value: Any) -> str:
|
|
2155
|
+
"""Render one contract value as words: underscores out, first letter up."""
|
|
2156
|
+
|
|
2157
|
+
text = str(value).replace("_", " ").replace("-", " ").strip()
|
|
2158
|
+
return text[:1].upper() + text[1:] if text else ""
|
|
2159
|
+
|
|
2160
|
+
|
|
2161
|
+
def _plain_label(key: Any) -> str | None:
|
|
2162
|
+
"""Return the plain heading for one field name, or ``None`` when it has no plain reading."""
|
|
2163
|
+
|
|
2164
|
+
if not isinstance(key, str) or not key:
|
|
2165
|
+
return None
|
|
2166
|
+
known = _PLAIN_KEYS.get(key)
|
|
2167
|
+
if known is not None:
|
|
2168
|
+
return known
|
|
2169
|
+
words = key.replace("-", " ").replace("_", " ").split()
|
|
2170
|
+
if not words or any(word.lower() in _INTERNAL_WORDS for word in words):
|
|
2171
|
+
return None
|
|
2172
|
+
text = " ".join(words)
|
|
2173
|
+
return text[:1].upper() + text[1:]
|
|
2174
|
+
|
|
2175
|
+
|
|
2176
|
+
def _plain_from(table: Mapping[str, str], value: Any) -> str:
|
|
2177
|
+
known = table.get(str(value))
|
|
2178
|
+
return known if known is not None else _humanize(value)
|
|
2179
|
+
|
|
2180
|
+
|
|
2181
|
+
def _plain_operation(operation: Any) -> str:
|
|
2182
|
+
text = str(operation)
|
|
2183
|
+
if text.startswith("cast:"):
|
|
2184
|
+
return f"read as {text.split(':', 1)[1]}"
|
|
2185
|
+
if text == "trim":
|
|
2186
|
+
return "trimmed of stray spaces"
|
|
2187
|
+
return _humanize(text)
|
|
2188
|
+
|
|
2189
|
+
|
|
2190
|
+
def _plain_check(check_id: Any, expected: Any) -> str:
|
|
2191
|
+
"""Name one check the way a person would say it.
|
|
2192
|
+
|
|
2193
|
+
Known internal check identifiers map to explicit operator-facing descriptions.
|
|
2194
|
+
"""
|
|
2195
|
+
|
|
2196
|
+
text = str(check_id)
|
|
2197
|
+
known = _PLAIN_CHECKS.get(text)
|
|
2198
|
+
if known is not None:
|
|
2199
|
+
return known
|
|
2200
|
+
if text == "MIN_ROWS":
|
|
2201
|
+
return f"At least {str(expected).lstrip('>=')} rows"
|
|
2202
|
+
if text.startswith("NOT_NULL_"):
|
|
2203
|
+
return f"No blanks in {text[len('NOT_NULL_') :]}"
|
|
2204
|
+
if text == "UNIQUE_GRAIN":
|
|
2205
|
+
return "One row per key, with no repeats"
|
|
2206
|
+
words = text.replace("_", " ").replace("-", " ").split()
|
|
2207
|
+
if not words or any(word.lower() in _INTERNAL_WORDS for word in words):
|
|
2208
|
+
return "A check this build requires"
|
|
2209
|
+
return _humanize(text)
|
|
2210
|
+
|
|
2211
|
+
|
|
2212
|
+
def _joined(values: Any, empty: str = "none") -> str:
|
|
2213
|
+
if not isinstance(values, (list, tuple)) or not values:
|
|
2214
|
+
return empty
|
|
2215
|
+
return ", ".join(str(value) for value in values)
|
|
2216
|
+
|
|
2217
|
+
|
|
2218
|
+
# ------------------------------------------------------------------------------------------------
|
|
2219
|
+
# Receipts
|
|
2220
|
+
# ------------------------------------------------------------------------------------------------
|
|
2221
|
+
|
|
2222
|
+
|
|
2223
|
+
def _document(title: str, body: str, *, script: str = "") -> str:
|
|
2224
|
+
return (
|
|
2225
|
+
"<!doctype html>\n"
|
|
2226
|
+
'<html lang="en"><head><meta charset="utf-8">'
|
|
2227
|
+
'<meta name="viewport" content="width=device-width, initial-scale=1">'
|
|
2228
|
+
f"<title>{esc(title)}</title>"
|
|
2229
|
+
f"{stylesheet()}"
|
|
2230
|
+
"</head><body>\n"
|
|
2231
|
+
f'<div class="mr-watch-page">{body}</div>\n'
|
|
2232
|
+
f"{script}"
|
|
2233
|
+
"</body></html>\n"
|
|
2234
|
+
)
|
|
2235
|
+
|
|
2236
|
+
|
|
2237
|
+
def _back() -> str:
|
|
2238
|
+
return '<a class="mr-watch-back" href="/">Back to the build</a>'
|
|
2239
|
+
|
|
2240
|
+
|
|
2241
|
+
def render_refusal(sentence: str) -> str:
|
|
2242
|
+
"""Render a refusal: the one plain sentence, the way back, and NONE of the file's content."""
|
|
2243
|
+
|
|
2244
|
+
return _document(
|
|
2245
|
+
"Receipt",
|
|
2246
|
+
_back() + f'<p class="mr-watch-refusal">{esc(sentence)}</p>',
|
|
2247
|
+
)
|
|
2248
|
+
|
|
2249
|
+
|
|
2250
|
+
def _definitions(pairs: Sequence[tuple[str, str]]) -> str:
|
|
2251
|
+
items = "".join(
|
|
2252
|
+
f"<dt>{esc(label)}</dt><dd>{esc(value)}</dd>" for label, value in pairs if label
|
|
2253
|
+
)
|
|
2254
|
+
return f'<dl class="mr-watch-defs">{items}</dl>' if items else ""
|
|
2255
|
+
|
|
2256
|
+
|
|
2257
|
+
def _table(headings: Sequence[str], rows: Sequence[Sequence[Any]]) -> str:
|
|
2258
|
+
head = "".join(f"<th>{esc(heading)}</th>" for heading in headings)
|
|
2259
|
+
body = "".join(
|
|
2260
|
+
"<tr>" + "".join(f"<td>{esc(cell)}</td>" for cell in row) + "</tr>" for row in rows
|
|
2261
|
+
)
|
|
2262
|
+
return (
|
|
2263
|
+
f'<table class="mr-watch-table"><thead><tr>{head}</tr></thead><tbody>{body}</tbody></table>'
|
|
2264
|
+
)
|
|
2265
|
+
|
|
2266
|
+
|
|
2267
|
+
def _lines(sentences: Sequence[str]) -> str:
|
|
2268
|
+
items = "".join(f"<li>{esc(sentence)}</li>" for sentence in sentences if sentence)
|
|
2269
|
+
return f'<ul class="mr-watch-list">{items}</ul>' if items else ""
|
|
2270
|
+
|
|
2271
|
+
|
|
2272
|
+
def _render_run_receipt(payload: Mapping[str, Any]) -> str:
|
|
2273
|
+
members = payload.get("members")
|
|
2274
|
+
members = members if isinstance(members, list) else []
|
|
2275
|
+
total = sum(entry.get("bytes", 0) for entry in members if isinstance(entry, Mapping))
|
|
2276
|
+
return "".join(
|
|
2277
|
+
[
|
|
2278
|
+
'<h1 class="mr-watch-card-title">The receipt for this whole build</h1>',
|
|
2279
|
+
'<p class="mr-watch-card-lead">This is what the build recorded about itself. '
|
|
2280
|
+
"Every file below was fingerprinted when the build finished, and the fingerprint of "
|
|
2281
|
+
"the whole set is the one line to quote when someone asks which build this was.</p>",
|
|
2282
|
+
_definitions(
|
|
2283
|
+
[
|
|
2284
|
+
("Fingerprint of the whole build", str(payload.get("candidate_digest", ""))),
|
|
2285
|
+
("Files recorded", str(len(members))),
|
|
2286
|
+
("Total size in bytes", str(total)),
|
|
2287
|
+
("Built with engine", str(payload.get("engine_version", ""))),
|
|
2288
|
+
]
|
|
2289
|
+
),
|
|
2290
|
+
_table(
|
|
2291
|
+
["File", "Size in bytes", "Fingerprint"],
|
|
2292
|
+
[
|
|
2293
|
+
[entry.get("path", ""), entry.get("bytes", ""), entry.get("sha256", "")]
|
|
2294
|
+
for entry in members
|
|
2295
|
+
if isinstance(entry, Mapping)
|
|
2296
|
+
],
|
|
2297
|
+
),
|
|
2298
|
+
]
|
|
2299
|
+
)
|
|
2300
|
+
|
|
2301
|
+
|
|
2302
|
+
def _render_join(payload: Mapping[str, Any]) -> str:
|
|
2303
|
+
matched = payload.get("matched_left_rows")
|
|
2304
|
+
left = payload.get("left_rows")
|
|
2305
|
+
unmatched = payload.get("unmatched_left_rows")
|
|
2306
|
+
keys = _joined(payload.get("keys"))
|
|
2307
|
+
return "".join(
|
|
2308
|
+
[
|
|
2309
|
+
'<h1 class="mr-watch-card-title">The join</h1>',
|
|
2310
|
+
f'<p class="mr-watch-card-lead">{esc(f"Joined on {keys}.")} '
|
|
2311
|
+
f"{esc(f'{matched} of {left} rows matched; {unmatched} did not.')}</p>",
|
|
2312
|
+
_definitions(
|
|
2313
|
+
[
|
|
2314
|
+
("Joined on", keys),
|
|
2315
|
+
("Left side", str(payload.get("left_source", ""))),
|
|
2316
|
+
("Right side", str(payload.get("right_source", ""))),
|
|
2317
|
+
("Rows on the left", str(left)),
|
|
2318
|
+
("Rows on the right", str(payload.get("right_rows", ""))),
|
|
2319
|
+
("Rows that matched", str(matched)),
|
|
2320
|
+
("Rows that did not match", str(unmatched)),
|
|
2321
|
+
("Rows out", str(payload.get("output_rows", ""))),
|
|
2322
|
+
("Row multiplier", str(payload.get("row_multiplier", ""))),
|
|
2323
|
+
("Shape of the match", _plain_from(_MATCH_SHAPE, payload.get("cardinality"))),
|
|
2324
|
+
("Row order", _humanize(payload.get("stable_order", ""))),
|
|
2325
|
+
]
|
|
2326
|
+
),
|
|
2327
|
+
]
|
|
2328
|
+
)
|
|
2329
|
+
|
|
2330
|
+
|
|
2331
|
+
def _render_quality(payload: Mapping[str, Any]) -> str:
|
|
2332
|
+
checks = payload.get("checks")
|
|
2333
|
+
checks = checks if isinstance(checks, list) else []
|
|
2334
|
+
passed = sum(
|
|
2335
|
+
1 for check in checks if isinstance(check, Mapping) and check.get("passed") is True
|
|
2336
|
+
)
|
|
2337
|
+
rows = [
|
|
2338
|
+
[
|
|
2339
|
+
_plain_check(check.get("check_id"), check.get("expected")),
|
|
2340
|
+
"pass" if check.get("passed") is True else "fail",
|
|
2341
|
+
check.get("observed", ""),
|
|
2342
|
+
check.get("expected", ""),
|
|
2343
|
+
]
|
|
2344
|
+
for check in checks
|
|
2345
|
+
if isinstance(check, Mapping)
|
|
2346
|
+
]
|
|
2347
|
+
return "".join(
|
|
2348
|
+
[
|
|
2349
|
+
'<h1 class="mr-watch-card-title">The checks</h1>',
|
|
2350
|
+
f'<p class="mr-watch-card-lead">{esc(f"{passed} of {len(checks)} checks passed.")}</p>',
|
|
2351
|
+
_table(["Check", "Result", "Observed", "Expected"], rows),
|
|
2352
|
+
]
|
|
2353
|
+
)
|
|
2354
|
+
|
|
2355
|
+
|
|
2356
|
+
def _render_sources(payload: Any) -> str:
|
|
2357
|
+
entries = payload if isinstance(payload, list) else []
|
|
2358
|
+
cards: list[str] = []
|
|
2359
|
+
for entry in entries:
|
|
2360
|
+
if not isinstance(entry, Mapping):
|
|
2361
|
+
continue
|
|
2362
|
+
rights = entry.get("rights")
|
|
2363
|
+
rights = rights if isinstance(rights, Mapping) else {}
|
|
2364
|
+
permissions = rights.get("permissions")
|
|
2365
|
+
cards.append(
|
|
2366
|
+
'<div class="mr-watch-sub">'
|
|
2367
|
+
f'<p class="mr-watch-sub-title">{esc(entry.get("source_id", ""))}</p>'
|
|
2368
|
+
+ _definitions(
|
|
2369
|
+
[
|
|
2370
|
+
("Where it came from", str(entry.get("origin", ""))),
|
|
2371
|
+
("File", str(entry.get("relative_path", ""))),
|
|
2372
|
+
(
|
|
2373
|
+
"How it was fetched",
|
|
2374
|
+
_plain_from(_HOW_FETCHED, entry.get("acquisition_method")),
|
|
2375
|
+
),
|
|
2376
|
+
("Rights", _plain_from(_RIGHTS_STATUS, rights.get("status"))),
|
|
2377
|
+
(
|
|
2378
|
+
"You may",
|
|
2379
|
+
", ".join(_plain_from(_PERMISSIONS, item) for item in permissions)
|
|
2380
|
+
if isinstance(permissions, list) and permissions
|
|
2381
|
+
else "nothing recorded",
|
|
2382
|
+
),
|
|
2383
|
+
("What backs that", str(rights.get("evidence", ""))),
|
|
2384
|
+
("Size in bytes", str(entry.get("bytes", ""))),
|
|
2385
|
+
("Fingerprint", str(entry.get("sha256", ""))),
|
|
2386
|
+
]
|
|
2387
|
+
)
|
|
2388
|
+
+ "</div>"
|
|
2389
|
+
)
|
|
2390
|
+
return (
|
|
2391
|
+
'<h1 class="mr-watch-card-title">The sources this build read</h1>'
|
|
2392
|
+
f'<p class="mr-watch-card-lead">{esc(f"{len(cards)} sources were read.")}</p>'
|
|
2393
|
+
+ "".join(cards)
|
|
2394
|
+
)
|
|
2395
|
+
|
|
2396
|
+
|
|
2397
|
+
def _profile_table(profile: Mapping[str, Any]) -> str:
|
|
2398
|
+
columns = profile.get("columns")
|
|
2399
|
+
columns = columns if isinstance(columns, list) else []
|
|
2400
|
+
types = profile.get("types")
|
|
2401
|
+
types = types if isinstance(types, Mapping) else {}
|
|
2402
|
+
blanks = profile.get("null_counts")
|
|
2403
|
+
blanks = blanks if isinstance(blanks, Mapping) else {}
|
|
2404
|
+
return _table(
|
|
2405
|
+
["Column", "Type", "Blanks"],
|
|
2406
|
+
[[name, types.get(name, ""), blanks.get(name, "")] for name in columns],
|
|
2407
|
+
)
|
|
2408
|
+
|
|
2409
|
+
|
|
2410
|
+
def _render_profile(payload: Mapping[str, Any]) -> str:
|
|
2411
|
+
return "".join(
|
|
2412
|
+
[
|
|
2413
|
+
'<h1 class="mr-watch-card-title">The dataset this build produced</h1>',
|
|
2414
|
+
f'<p class="mr-watch-card-lead">{esc(str(payload.get("row_count", 0)))} rows.</p>',
|
|
2415
|
+
_profile_table(payload),
|
|
2416
|
+
]
|
|
2417
|
+
)
|
|
2418
|
+
|
|
2419
|
+
|
|
2420
|
+
def _render_source_profiles(payload: Mapping[str, Any]) -> str:
|
|
2421
|
+
parts = ['<h1 class="mr-watch-card-title">What each source looked like</h1>']
|
|
2422
|
+
for source_id in sorted(payload):
|
|
2423
|
+
shapes = payload[source_id]
|
|
2424
|
+
if not isinstance(shapes, Mapping):
|
|
2425
|
+
continue
|
|
2426
|
+
parts.append(
|
|
2427
|
+
f'<div class="mr-watch-sub"><p class="mr-watch-sub-title">{esc(source_id)}</p>'
|
|
2428
|
+
)
|
|
2429
|
+
for stage, label in (("raw", "As it was read"), ("cleaned", "After cleaning")):
|
|
2430
|
+
shape = shapes.get(stage)
|
|
2431
|
+
if isinstance(shape, Mapping):
|
|
2432
|
+
parts.append(f"<p><strong>{esc(label)}</strong></p>")
|
|
2433
|
+
parts.append(f'<p class="mr-watch-note">{esc(shape.get("row_count", 0))} rows</p>')
|
|
2434
|
+
parts.append(_profile_table(shape))
|
|
2435
|
+
parts.append("</div>")
|
|
2436
|
+
return "".join(parts)
|
|
2437
|
+
|
|
2438
|
+
|
|
2439
|
+
def _render_lineage(payload: Mapping[str, Any]) -> str:
|
|
2440
|
+
sentences = []
|
|
2441
|
+
for column in sorted(payload):
|
|
2442
|
+
entry = payload[column]
|
|
2443
|
+
if not isinstance(entry, Mapping):
|
|
2444
|
+
continue
|
|
2445
|
+
operations = entry.get("operations")
|
|
2446
|
+
applied = (
|
|
2447
|
+
", ".join(_plain_operation(item) for item in operations)
|
|
2448
|
+
if isinstance(operations, list) and operations
|
|
2449
|
+
else "nothing"
|
|
2450
|
+
)
|
|
2451
|
+
sentences.append(
|
|
2452
|
+
f"{column} comes from {entry.get('source_column', '')} in "
|
|
2453
|
+
f"{entry.get('source_id', '')}; it was {applied}."
|
|
2454
|
+
)
|
|
2455
|
+
return (
|
|
2456
|
+
'<h1 class="mr-watch-card-title">Where every column came from</h1>'
|
|
2457
|
+
'<p class="mr-watch-card-lead">One line per column in the finished dataset.</p>'
|
|
2458
|
+
+ _lines(sentences)
|
|
2459
|
+
)
|
|
2460
|
+
|
|
2461
|
+
|
|
2462
|
+
def _plan_summary(plan: Mapping[str, Any]) -> list[tuple[str, str]]:
|
|
2463
|
+
sources = plan.get("sources")
|
|
2464
|
+
sources = sources if isinstance(sources, list) else []
|
|
2465
|
+
join = plan.get("join")
|
|
2466
|
+
join = join if isinstance(join, Mapping) else {}
|
|
2467
|
+
quality = plan.get("quality")
|
|
2468
|
+
quality = quality if isinstance(quality, Mapping) else {}
|
|
2469
|
+
cleaning = plan.get("cleaning")
|
|
2470
|
+
cleaning = cleaning if isinstance(cleaning, list) else []
|
|
2471
|
+
return [
|
|
2472
|
+
(
|
|
2473
|
+
"Sources",
|
|
2474
|
+
", ".join(
|
|
2475
|
+
str(source.get("id", "")) for source in sources if isinstance(source, Mapping)
|
|
2476
|
+
)
|
|
2477
|
+
or "none",
|
|
2478
|
+
),
|
|
2479
|
+
("One row per", _joined(plan.get("grain"))),
|
|
2480
|
+
("Columns kept", _joined(plan.get("select"))),
|
|
2481
|
+
("Cleaning", f"{len(cleaning)} steps"),
|
|
2482
|
+
(
|
|
2483
|
+
"Join",
|
|
2484
|
+
f"{join.get('left', '')} to {join.get('right', '')} on {_joined(join.get('on'))}, "
|
|
2485
|
+
f"{_plain_from(_JOIN_KIND, join.get('kind'))}, "
|
|
2486
|
+
f"{_plain_from(_MATCH_SHAPE, join.get('cardinality'))}",
|
|
2487
|
+
),
|
|
2488
|
+
("Meant for", _plain_from(_OUTPUT_INTENT, plan.get("output_intent", ""))),
|
|
2489
|
+
(
|
|
2490
|
+
"Checks",
|
|
2491
|
+
f"at least {quality.get('min_rows', 0)} rows; no blanks in "
|
|
2492
|
+
f"{_joined(quality.get('not_null'))}",
|
|
2493
|
+
),
|
|
2494
|
+
]
|
|
2495
|
+
|
|
2496
|
+
|
|
2497
|
+
def _cleaning_lines(plan: Mapping[str, Any]) -> list[str]:
|
|
2498
|
+
cleaning = plan.get("cleaning")
|
|
2499
|
+
cleaning = cleaning if isinstance(cleaning, list) else []
|
|
2500
|
+
lines: list[str] = []
|
|
2501
|
+
for step in cleaning:
|
|
2502
|
+
if not isinstance(step, Mapping):
|
|
2503
|
+
continue
|
|
2504
|
+
columns = step.get("columns")
|
|
2505
|
+
if isinstance(columns, Mapping):
|
|
2506
|
+
named = ", ".join(f"{key} as {value}" for key, value in columns.items())
|
|
2507
|
+
else:
|
|
2508
|
+
named = _joined(columns)
|
|
2509
|
+
lines.append(f"{_plain_operation(step.get('operation'))}: {named} in {step.get('source')}")
|
|
2510
|
+
return lines
|
|
2511
|
+
|
|
2512
|
+
|
|
2513
|
+
def _render_plan(payload: Mapping[str, Any]) -> str:
|
|
2514
|
+
return "".join(
|
|
2515
|
+
[
|
|
2516
|
+
'<h1 class="mr-watch-card-title">What this build was asked to do</h1>',
|
|
2517
|
+
f'<p class="mr-watch-card-lead">{esc(payload.get("question", ""))}</p>',
|
|
2518
|
+
_definitions(_plan_summary(payload)),
|
|
2519
|
+
"<p><strong>Cleaning, in order</strong></p>",
|
|
2520
|
+
_lines(_cleaning_lines(payload)),
|
|
2521
|
+
]
|
|
2522
|
+
)
|
|
2523
|
+
|
|
2524
|
+
|
|
2525
|
+
def _render_generic(payload: Any) -> str:
|
|
2526
|
+
"""The fallback for a file with no renderer of its own.
|
|
2527
|
+
|
|
2528
|
+
It renders only fields that HAVE a plain reading and drops the rest, because the alternative --
|
|
2529
|
+
dumping the field names the contract uses -- is the jargon leak this product refuses. A file
|
|
2530
|
+
whose every field is internal renders its size and fingerprint and nothing else, which is an
|
|
2531
|
+
honest thing for a page to say.
|
|
2532
|
+
"""
|
|
2533
|
+
|
|
2534
|
+
if not isinstance(payload, Mapping):
|
|
2535
|
+
return '<p class="mr-watch-card-lead">This file was recorded for this build.</p>'
|
|
2536
|
+
pairs: list[tuple[str, str]] = []
|
|
2537
|
+
for key in payload:
|
|
2538
|
+
label = _plain_label(key)
|
|
2539
|
+
if label is None:
|
|
2540
|
+
continue
|
|
2541
|
+
value = payload[key]
|
|
2542
|
+
if isinstance(value, Mapping):
|
|
2543
|
+
rendered = f"{len(value)} entries"
|
|
2544
|
+
elif isinstance(value, list):
|
|
2545
|
+
rendered = (
|
|
2546
|
+
_joined(value)
|
|
2547
|
+
if all(not isinstance(v, (dict, list)) for v in value)
|
|
2548
|
+
else (f"{len(value)} entries")
|
|
2549
|
+
)
|
|
2550
|
+
else:
|
|
2551
|
+
rendered = str(value)
|
|
2552
|
+
pairs.append((label, rendered))
|
|
2553
|
+
return (
|
|
2554
|
+
'<p class="mr-watch-card-lead">This file was recorded for this build.</p>'
|
|
2555
|
+
+ _definitions(pairs)
|
|
2556
|
+
)
|
|
2557
|
+
|
|
2558
|
+
|
|
2559
|
+
def _render_opaque(member: str, raw: bytes) -> str:
|
|
2560
|
+
"""A data file's receipt is its size and its fingerprint, not its bytes.
|
|
2561
|
+
|
|
2562
|
+
Rendering the content of an arbitrary recorded file would put untrusted bytes of unknown
|
|
2563
|
+
encoding on a page for no gain: the thing that makes a data file trustworthy is that its
|
|
2564
|
+
fingerprint matches what the build recorded, and that is what this page states.
|
|
2565
|
+
"""
|
|
2566
|
+
|
|
2567
|
+
return "".join(
|
|
2568
|
+
[
|
|
2569
|
+
'<h1 class="mr-watch-card-title">A data file from this build</h1>',
|
|
2570
|
+
'<p class="mr-watch-card-lead">This file holds data rather than a written record, '
|
|
2571
|
+
"so what is shown is what the build recorded about it.</p>",
|
|
2572
|
+
_definitions(
|
|
2573
|
+
[
|
|
2574
|
+
("File", member),
|
|
2575
|
+
("Size in bytes", str(len(raw))),
|
|
2576
|
+
("Fingerprint", hashlib.sha256(raw).hexdigest()),
|
|
2577
|
+
]
|
|
2578
|
+
),
|
|
2579
|
+
]
|
|
2580
|
+
)
|
|
2581
|
+
|
|
2582
|
+
|
|
2583
|
+
_RECEIPT_RENDERERS: Mapping[str, Callable[[Any], str]] = {
|
|
2584
|
+
"manifest.json": _render_run_receipt,
|
|
2585
|
+
"plan.json": _render_plan,
|
|
2586
|
+
"evidence/join.json": _render_join,
|
|
2587
|
+
"evidence/quality.json": _render_quality,
|
|
2588
|
+
"evidence/sources.json": _render_sources,
|
|
2589
|
+
"evidence/profile.json": _render_profile,
|
|
2590
|
+
"evidence/source_profiles.json": _render_source_profiles,
|
|
2591
|
+
"evidence/lineage.json": _render_lineage,
|
|
2592
|
+
}
|
|
2593
|
+
|
|
2594
|
+
|
|
2595
|
+
def render_receipt(member: str, raw: Any) -> str:
|
|
2596
|
+
"""Render one verified receipt as a page a person can read.
|
|
2597
|
+
|
|
2598
|
+
``raw`` is the member's bytes, already verified against the fingerprint the build recorded for
|
|
2599
|
+
it. A renderer that raises on a shape it did not expect degrades to the generic renderer; a
|
|
2600
|
+
malformed file must produce a plain page, never a traceback.
|
|
2601
|
+
|
|
2602
|
+
VERIFIED IS NOT WRITABLE. Everything that reaches the page here crosses ``_writable_payload``
|
|
2603
|
+
first -- the member name included, for the opaque path below -- because a member whose bytes
|
|
2604
|
+
match the digest the run recorded can still carry a lone surrogate that no surface can encode;
|
|
2605
|
+
see that function.
|
|
2606
|
+
"""
|
|
2607
|
+
|
|
2608
|
+
member = events.writable_text(member)
|
|
2609
|
+
payload: Any
|
|
2610
|
+
if isinstance(raw, (bytes, bytearray)):
|
|
2611
|
+
try:
|
|
2612
|
+
payload = _writable_payload(json.loads(bytes(raw).decode("utf-8")))
|
|
2613
|
+
except (ValueError, UnicodeDecodeError, RecursionError):
|
|
2614
|
+
return _document("Receipt", _back() + _render_opaque(member, bytes(raw)))
|
|
2615
|
+
else:
|
|
2616
|
+
payload = _writable_payload(raw)
|
|
2617
|
+
|
|
2618
|
+
renderer = _RECEIPT_RENDERERS.get(member)
|
|
2619
|
+
body = ""
|
|
2620
|
+
if renderer is not None:
|
|
2621
|
+
try:
|
|
2622
|
+
body = renderer(payload)
|
|
2623
|
+
except Exception:
|
|
2624
|
+
body = ""
|
|
2625
|
+
if not body:
|
|
2626
|
+
body = _AUTHORED_RECEIPTS.get(member, _render_generic)(payload)
|
|
2627
|
+
return _document("Receipt", _back() + body)
|
|
2628
|
+
|
|
2629
|
+
|
|
2630
|
+
# ------------------------------------------------------------------------------------------------
|
|
2631
|
+
# Authored artifact cards
|
|
2632
|
+
# ------------------------------------------------------------------------------------------------
|
|
2633
|
+
|
|
2634
|
+
|
|
2635
|
+
def _card(title: str, lead: str, body: str) -> str:
|
|
2636
|
+
return (
|
|
2637
|
+
'<section class="mr-watch-card">'
|
|
2638
|
+
f'<h2 class="mr-watch-card-title">{esc(title)}</h2>'
|
|
2639
|
+
+ (f'<p class="mr-watch-card-lead">{esc(lead)}</p>' if lead else "")
|
|
2640
|
+
+ body
|
|
2641
|
+
+ "</section>"
|
|
2642
|
+
)
|
|
2643
|
+
|
|
2644
|
+
|
|
2645
|
+
def _question_card(question: Any, requirements: Any = None) -> str:
|
|
2646
|
+
if not isinstance(question, Mapping):
|
|
2647
|
+
return ""
|
|
2648
|
+
criteria = []
|
|
2649
|
+
if isinstance(requirements, Mapping):
|
|
2650
|
+
listed = requirements.get("success_criteria")
|
|
2651
|
+
if isinstance(listed, list):
|
|
2652
|
+
criteria = [str(item) for item in listed]
|
|
2653
|
+
body = ("<p><strong>It is done when</strong></p>" + _lines(criteria)) if criteria else ""
|
|
2654
|
+
return _card("The question", str(question.get("text", "")), body)
|
|
2655
|
+
|
|
2656
|
+
|
|
2657
|
+
def _requirements_card(requirements: Any) -> str:
|
|
2658
|
+
if not isinstance(requirements, Mapping):
|
|
2659
|
+
return ""
|
|
2660
|
+
sentences: list[str] = []
|
|
2661
|
+
population = requirements.get("population")
|
|
2662
|
+
if population:
|
|
2663
|
+
sentences.append(str(population))
|
|
2664
|
+
window = requirements.get("time_range")
|
|
2665
|
+
if isinstance(window, Mapping):
|
|
2666
|
+
sentences.append(
|
|
2667
|
+
f"It covers {window.get('start_inclusive', '')} up to "
|
|
2668
|
+
f"{window.get('end_exclusive', '')}."
|
|
2669
|
+
)
|
|
2670
|
+
grain = requirements.get("output_grain")
|
|
2671
|
+
if isinstance(grain, Mapping) and grain.get("description"):
|
|
2672
|
+
sentences.append(str(grain["description"]))
|
|
2673
|
+
fields = requirements.get("required_fields")
|
|
2674
|
+
if isinstance(fields, list) and fields:
|
|
2675
|
+
sentences.append(f"Every row carries {_joined(fields)}.")
|
|
2676
|
+
feasibility = requirements.get("feasibility")
|
|
2677
|
+
if isinstance(feasibility, Mapping) and feasibility.get("narrative"):
|
|
2678
|
+
sentences.append(str(feasibility["narrative"]))
|
|
2679
|
+
if not sentences:
|
|
2680
|
+
return ""
|
|
2681
|
+
return _card("What it has to satisfy", "", _lines(sentences))
|
|
2682
|
+
|
|
2683
|
+
|
|
2684
|
+
def _proposals_card(proposals: Any) -> str:
|
|
2685
|
+
listed: Any = None
|
|
2686
|
+
if isinstance(proposals, Mapping):
|
|
2687
|
+
listed = proposals.get("source_proposals")
|
|
2688
|
+
elif isinstance(proposals, list):
|
|
2689
|
+
listed = proposals
|
|
2690
|
+
if not isinstance(listed, list) or not listed:
|
|
2691
|
+
return ""
|
|
2692
|
+
blocks: list[str] = []
|
|
2693
|
+
for proposal in listed:
|
|
2694
|
+
if not isinstance(proposal, Mapping):
|
|
2695
|
+
continue
|
|
2696
|
+
coverage = proposal.get("historical_coverage")
|
|
2697
|
+
evidence = proposal.get("evidence")
|
|
2698
|
+
argument = (
|
|
2699
|
+
"The agent chose this one."
|
|
2700
|
+
if proposal.get("fitness_decision") == "selected"
|
|
2701
|
+
else f"The agent's reading: {_humanize(proposal.get('fitness_decision'))}."
|
|
2702
|
+
)
|
|
2703
|
+
if isinstance(coverage, Mapping):
|
|
2704
|
+
argument += (
|
|
2705
|
+
f" It covers {coverage.get('start_inclusive', '')} up to "
|
|
2706
|
+
f"{coverage.get('end_exclusive', '')}."
|
|
2707
|
+
)
|
|
2708
|
+
if isinstance(evidence, list) and evidence:
|
|
2709
|
+
argument += f" {len(evidence)} pieces of backing were recorded."
|
|
2710
|
+
blocks.append(
|
|
2711
|
+
'<div class="mr-watch-sub">'
|
|
2712
|
+
f'<p class="mr-watch-sub-title">{esc(proposal.get("display_name", ""))}</p>'
|
|
2713
|
+
+ _definitions(
|
|
2714
|
+
[
|
|
2715
|
+
("What it is", _humanize(proposal.get("data_format", ""))),
|
|
2716
|
+
("Where it came from", str(proposal.get("locator", ""))),
|
|
2717
|
+
("Why it fits", argument),
|
|
2718
|
+
("Rights", _plain_from(_RIGHTS_DECISION, proposal.get("rights_status"))),
|
|
2719
|
+
]
|
|
2720
|
+
)
|
|
2721
|
+
+ "</div>"
|
|
2722
|
+
)
|
|
2723
|
+
if not blocks:
|
|
2724
|
+
return ""
|
|
2725
|
+
return _card(
|
|
2726
|
+
"Sources the agent proposed",
|
|
2727
|
+
f"{len(blocks)} proposed.",
|
|
2728
|
+
"".join(blocks),
|
|
2729
|
+
)
|
|
2730
|
+
|
|
2731
|
+
|
|
2732
|
+
def _recipe_summary(recipe: Mapping[str, Any]) -> list[tuple[str, str]]:
|
|
2733
|
+
plan = recipe.get("transform_plan")
|
|
2734
|
+
plan = plan if isinstance(plan, Mapping) else {}
|
|
2735
|
+
validation = recipe.get("validation")
|
|
2736
|
+
validation = validation if isinstance(validation, Mapping) else {}
|
|
2737
|
+
pairs = _plan_summary(plan)
|
|
2738
|
+
required = validation.get("required_check_ids")
|
|
2739
|
+
if isinstance(required, list) and required:
|
|
2740
|
+
# Semicolons, not commas: every one of these is a sentence, and comma-joining sentences
|
|
2741
|
+
# produces a paragraph a reader cannot parse into its parts.
|
|
2742
|
+
pairs.append(
|
|
2743
|
+
(
|
|
2744
|
+
"Checks that must pass",
|
|
2745
|
+
"; ".join(_plain_check(item, "") for item in required),
|
|
2746
|
+
)
|
|
2747
|
+
)
|
|
2748
|
+
return pairs
|
|
2749
|
+
|
|
2750
|
+
|
|
2751
|
+
def _recipe_card(recipe: Any) -> str:
|
|
2752
|
+
if not isinstance(recipe, Mapping):
|
|
2753
|
+
return ""
|
|
2754
|
+
plan = recipe.get("transform_plan")
|
|
2755
|
+
plan = plan if isinstance(plan, Mapping) else {}
|
|
2756
|
+
return _card(
|
|
2757
|
+
"The Recipe",
|
|
2758
|
+
"These are the locked instructions this build followed. They do not change between runs.",
|
|
2759
|
+
_definitions(_recipe_summary(recipe))
|
|
2760
|
+
+ "<p><strong>Cleaning, in order</strong></p>"
|
|
2761
|
+
+ _lines(_cleaning_lines(plan)),
|
|
2762
|
+
)
|
|
2763
|
+
|
|
2764
|
+
|
|
2765
|
+
# The four authored artifacts render the SAME body whether they are reached as a card on the index
|
|
2766
|
+
# or as a receipt of their own, so a reader never meets two readings of one file.
|
|
2767
|
+
_AUTHORED_RECEIPTS: Mapping[str, Callable[[Any], str]] = {
|
|
2768
|
+
"question.json": _question_card,
|
|
2769
|
+
"requirements.json": _requirements_card,
|
|
2770
|
+
"source-proposals.json": _proposals_card,
|
|
2771
|
+
"recipe.json": _recipe_card,
|
|
2772
|
+
}
|
|
2773
|
+
|
|
2774
|
+
_CARD_MEMBERS = ("question.json", "requirements.json", "source-proposals.json", "recipe.json")
|
|
2775
|
+
|
|
2776
|
+
|
|
2777
|
+
def _writable_payload(value: Any) -> Any:
|
|
2778
|
+
"""Normalize every string so the rendered payload is UTF-8 encodable."""
|
|
2779
|
+
|
|
2780
|
+
if isinstance(value, str):
|
|
2781
|
+
return events.writable_text(value)
|
|
2782
|
+
if isinstance(value, Mapping):
|
|
2783
|
+
return {_writable_payload(key): _writable_payload(item) for key, item in value.items()}
|
|
2784
|
+
if isinstance(value, list):
|
|
2785
|
+
return [_writable_payload(item) for item in value]
|
|
2786
|
+
return value
|
|
2787
|
+
|
|
2788
|
+
|
|
2789
|
+
def _member_payload(read_member: Callable[[str], bytes], member: str) -> Any:
|
|
2790
|
+
"""Read and parse one member through the verified read, or return ``None``.
|
|
2791
|
+
|
|
2792
|
+
Verified is not writable: see ``_writable_payload`` for why a member that passed every check
|
|
2793
|
+
the run makes of it still cannot be put on a page unchanged.
|
|
2794
|
+
"""
|
|
2795
|
+
|
|
2796
|
+
try:
|
|
2797
|
+
raw = read_member(member)
|
|
2798
|
+
except Exception:
|
|
2799
|
+
return None
|
|
2800
|
+
try:
|
|
2801
|
+
return _writable_payload(json.loads(raw.decode("utf-8")))
|
|
2802
|
+
except (ValueError, UnicodeDecodeError, RecursionError):
|
|
2803
|
+
return None
|
|
2804
|
+
|
|
2805
|
+
|
|
2806
|
+
def render_cards(members_present: frozenset[str], read_member: Callable[[str], bytes]) -> str:
|
|
2807
|
+
"""Render each verified authored artifact that is currently present."""
|
|
2808
|
+
|
|
2809
|
+
loaded: dict[str, Any] = {}
|
|
2810
|
+
for member in _CARD_MEMBERS:
|
|
2811
|
+
if member in members_present:
|
|
2812
|
+
payload = _member_payload(read_member, member)
|
|
2813
|
+
if payload is not None:
|
|
2814
|
+
loaded[member] = payload
|
|
2815
|
+
if not loaded:
|
|
2816
|
+
return ""
|
|
2817
|
+
|
|
2818
|
+
cards: list[str] = []
|
|
2819
|
+
if "question.json" in loaded:
|
|
2820
|
+
cards.append(
|
|
2821
|
+
_safe(lambda: _question_card(loaded["question.json"], loaded.get("requirements.json")))
|
|
2822
|
+
)
|
|
2823
|
+
if "requirements.json" in loaded:
|
|
2824
|
+
cards.append(_safe(lambda: _requirements_card(loaded["requirements.json"])))
|
|
2825
|
+
if "source-proposals.json" in loaded:
|
|
2826
|
+
cards.append(_safe(lambda: _proposals_card(loaded["source-proposals.json"])))
|
|
2827
|
+
if "recipe.json" in loaded:
|
|
2828
|
+
cards.append(_safe(lambda: _recipe_card(loaded["recipe.json"])))
|
|
2829
|
+
rendered = "".join(card for card in cards if card)
|
|
2830
|
+
return f'<section class="mr-watch-cards">{rendered}</section>' if rendered else ""
|
|
2831
|
+
|
|
2832
|
+
|
|
2833
|
+
# ------------------------------------------------------------------------------------------------
|
|
2834
|
+
# The approval gate -- display only
|
|
2835
|
+
# ------------------------------------------------------------------------------------------------
|
|
2836
|
+
#
|
|
2837
|
+
# The displayed command uses the explicit recipe approval interface. Tests bind the command and
|
|
2838
|
+
# option names to the CLI parser.
|
|
2839
|
+
APPROVE_COMMAND_NAME = "recipe-approve"
|
|
2840
|
+
APPROVE_COMMAND_TEMPLATE = (
|
|
2841
|
+
"mr-data recipe-approve"
|
|
2842
|
+
" --recipe {recipe}"
|
|
2843
|
+
" --approval-id APPROVAL_ID"
|
|
2844
|
+
" --approved-by YOUR_NAME"
|
|
2845
|
+
" --approved-at TIMESTAMP"
|
|
2846
|
+
" --approval-policy-digest APPROVAL_RULES_FINGERPRINT"
|
|
2847
|
+
" --output {output}"
|
|
2848
|
+
)
|
|
2849
|
+
|
|
2850
|
+
_APPROVE_COMMAND_ID = "mr-watch-approve-command"
|
|
2851
|
+
|
|
2852
|
+
|
|
2853
|
+
def approve_command(run_dir: Path) -> str:
|
|
2854
|
+
"""Return the approval command with normalized recipe and output paths."""
|
|
2855
|
+
|
|
2856
|
+
return APPROVE_COMMAND_TEMPLATE.format(
|
|
2857
|
+
recipe=_writable_path_text(run_dir / "candidate" / "recipe.json"),
|
|
2858
|
+
output=_writable_path_text(run_dir / "recipe-approval.json"),
|
|
2859
|
+
)
|
|
2860
|
+
|
|
2861
|
+
|
|
2862
|
+
# Relationship between an approval record and its recipe digest.
|
|
2863
|
+
_BINDS = "binds"
|
|
2864
|
+
_DISAGREES = "disagrees"
|
|
2865
|
+
_CLAIMS_NOTHING = "claims-nothing"
|
|
2866
|
+
|
|
2867
|
+
|
|
2868
|
+
def _recipe_digest(recipe: Mapping[str, Any]) -> str:
|
|
2869
|
+
"""The Recipe's own canonical digest, computed exactly as ``RecipeApproval`` computes it."""
|
|
2870
|
+
|
|
2871
|
+
try:
|
|
2872
|
+
# Deferred: `watch` must stay importable without pulling `canonical` in at module load.
|
|
2873
|
+
from mostlyright.data_harness.canonical import canonical_json_bytes, sha256_bytes
|
|
2874
|
+
|
|
2875
|
+
return sha256_bytes(canonical_json_bytes(dict(recipe)))
|
|
2876
|
+
except Exception:
|
|
2877
|
+
return ""
|
|
2878
|
+
|
|
2879
|
+
|
|
2880
|
+
def _approval_record(
|
|
2881
|
+
read_member: Callable[[str], bytes], members_present: frozenset[str]
|
|
2882
|
+
) -> Mapping[str, Any] | None:
|
|
2883
|
+
"""The run's approval record, read once through the verified read the rest of the page uses."""
|
|
2884
|
+
|
|
2885
|
+
if "recipe-approval.json" not in members_present:
|
|
2886
|
+
return None
|
|
2887
|
+
record = _member_payload(read_member, "recipe-approval.json")
|
|
2888
|
+
return record if isinstance(record, Mapping) else None
|
|
2889
|
+
|
|
2890
|
+
|
|
2891
|
+
def _approval_binding(record: Mapping[str, Any] | None, computed: str) -> str:
|
|
2892
|
+
"""Whether ``record`` binds an approval to the Recipe whose derived digest is ``computed``.
|
|
2893
|
+
|
|
2894
|
+
Through ``_matches_digest``, so a claim that is not a digest at all gets the same answer as a
|
|
2895
|
+
claim that is the wrong digest. ``computed`` is "" when the Recipe yields no canonical digest,
|
|
2896
|
+
and the comparator refuses that operand too: nothing can be proven to bind a Recipe whose own
|
|
2897
|
+
fingerprint cannot be derived.
|
|
2898
|
+
"""
|
|
2899
|
+
|
|
2900
|
+
claimed = None if record is None else record.get("recipe_digest")
|
|
2901
|
+
if not isinstance(claimed, str) or not claimed:
|
|
2902
|
+
return _CLAIMS_NOTHING
|
|
2903
|
+
return _BINDS if _matches_digest(computed, claimed) else _DISAGREES
|
|
2904
|
+
|
|
2905
|
+
|
|
2906
|
+
def _recipe_fingerprint(
|
|
2907
|
+
read_member: Callable[[str], bytes],
|
|
2908
|
+
members_present: frozenset[str],
|
|
2909
|
+
recipe: Mapping[str, Any],
|
|
2910
|
+
) -> str:
|
|
2911
|
+
"""Return the canonical recipe digest unless the approval records a different digest."""
|
|
2912
|
+
|
|
2913
|
+
return _fingerprint_line(recipe, _approval_record(read_member, members_present))
|
|
2914
|
+
|
|
2915
|
+
|
|
2916
|
+
def _fingerprint_line(recipe: Mapping[str, Any], record: Mapping[str, Any] | None) -> str:
|
|
2917
|
+
"""The value ``_recipe_fingerprint`` returns, from a record the caller has already read."""
|
|
2918
|
+
|
|
2919
|
+
computed = _recipe_digest(recipe)
|
|
2920
|
+
if not computed:
|
|
2921
|
+
return ""
|
|
2922
|
+
return "" if _approval_binding(record, computed) == _DISAGREES else computed
|
|
2923
|
+
|
|
2924
|
+
|
|
2925
|
+
def render_approval(
|
|
2926
|
+
run_dir: Path,
|
|
2927
|
+
members_present: frozenset[str],
|
|
2928
|
+
read_member: Callable[[str], bytes],
|
|
2929
|
+
) -> str:
|
|
2930
|
+
"""Render the verified recipe and its approval state, or return an empty string.
|
|
2931
|
+
|
|
2932
|
+
Approved output requires an approval digest that matches the canonical recipe digest. A
|
|
2933
|
+
mismatch is reported without an approval command. Approval itself remains a terminal action.
|
|
2934
|
+
"""
|
|
2935
|
+
|
|
2936
|
+
if "recipe.json" not in members_present:
|
|
2937
|
+
return ""
|
|
2938
|
+
recipe = _member_payload(read_member, "recipe.json")
|
|
2939
|
+
if not isinstance(recipe, Mapping):
|
|
2940
|
+
return ""
|
|
2941
|
+
|
|
2942
|
+
# One binding result controls both the displayed digest and approval state.
|
|
2943
|
+
record = _approval_record(read_member, members_present)
|
|
2944
|
+
binding = _approval_binding(record, _recipe_digest(recipe))
|
|
2945
|
+
|
|
2946
|
+
pairs = _recipe_summary(recipe)
|
|
2947
|
+
fingerprint = _fingerprint_line(recipe, record)
|
|
2948
|
+
if fingerprint:
|
|
2949
|
+
pairs.append(("Fingerprint of the Recipe", fingerprint))
|
|
2950
|
+
receipt = _member_payload(read_member, _MANIFEST_NAME)
|
|
2951
|
+
if isinstance(receipt, Mapping) and receipt.get("candidate_digest"):
|
|
2952
|
+
# Kept, under a label that says whose fingerprint it is. It identifies the build, not the
|
|
2953
|
+
# Recipe, and a reader who quotes one for the other has quoted the wrong thing.
|
|
2954
|
+
pairs.append(("Fingerprint of this build", str(receipt["candidate_digest"])))
|
|
2955
|
+
|
|
2956
|
+
approval = (
|
|
2957
|
+
record if binding == _BINDS and record is not None and record.get("approved_by") else None
|
|
2958
|
+
)
|
|
2959
|
+
|
|
2960
|
+
if approval is not None:
|
|
2961
|
+
return (
|
|
2962
|
+
'<section class="mr-watch-approve">'
|
|
2963
|
+
'<h2 class="mr-watch-card-title">What this build was built from</h2>'
|
|
2964
|
+
'<p class="mr-watch-card-lead">This Recipe was approved before the build ran, and the '
|
|
2965
|
+
"build followed it exactly. Nothing on this page changes that.</p>"
|
|
2966
|
+
+ _definitions(pairs)
|
|
2967
|
+
+ f'<p class="mr-watch-note">Approved by {esc(approval.get("approved_by", ""))} '
|
|
2968
|
+
f"on {esc(approval.get('approved_at', ''))}.</p>" + "</section>"
|
|
2969
|
+
)
|
|
2970
|
+
|
|
2971
|
+
if binding == _DISAGREES:
|
|
2972
|
+
return (
|
|
2973
|
+
'<section class="mr-watch-approve">'
|
|
2974
|
+
'<h2 class="mr-watch-card-title">Approval does not match this Recipe</h2>'
|
|
2975
|
+
'<p class="mr-watch-card-lead">The approval on disk binds a different Recipe. '
|
|
2976
|
+
"This build cannot present it as approval for the Recipe shown here.</p>"
|
|
2977
|
+
+ _definitions(pairs)
|
|
2978
|
+
+ '<p class="mr-watch-note">No approval command is shown while these records '
|
|
2979
|
+
"disagree. Check the Recipe and approval receipts before taking any action.</p>"
|
|
2980
|
+
"</section>"
|
|
2981
|
+
)
|
|
2982
|
+
|
|
2983
|
+
command = approve_command(run_dir)
|
|
2984
|
+
return (
|
|
2985
|
+
'<section class="mr-watch-approve">'
|
|
2986
|
+
'<h2 class="mr-watch-card-title">What you are about to freeze</h2>'
|
|
2987
|
+
'<p class="mr-watch-card-lead">Freezing turns these instructions into a Recipe that runs '
|
|
2988
|
+
"the same way every time. Read what is below before you do it.</p>"
|
|
2989
|
+
+ _definitions(pairs)
|
|
2990
|
+
+ '<p class="mr-watch-note">Approving happens in your terminal. This page only shows you '
|
|
2991
|
+
"what you would be approving. Copy the line below and run it yourself.</p>"
|
|
2992
|
+
+ '<p class="mr-watch-note">Four values in the line are yours to supply, and they are '
|
|
2993
|
+
"written in capitals: a name for this approval, your own name, the time you approved, and "
|
|
2994
|
+
"the fingerprint of the approval rules you followed.</p>"
|
|
2995
|
+
+ f'<code class="mr-watch-command" id="{_APPROVE_COMMAND_ID}">{esc(command)}</code>'
|
|
2996
|
+
+ f'<button type="button" class="mr-watch-copy" data-copy-target="{_APPROVE_COMMAND_ID}">'
|
|
2997
|
+
"Copy</button>"
|
|
2998
|
+
"</section>"
|
|
2999
|
+
)
|