mostlyright-data 0.9.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mostlyright/data_harness/__init__.py +158 -0
- mostlyright/data_harness/acquisition/__init__.py +55 -0
- mostlyright/data_harness/acquisition/http.py +2773 -0
- mostlyright/data_harness/acquisition/parsing.py +809 -0
- mostlyright/data_harness/acquisition/ranges.py +495 -0
- mostlyright/data_harness/acquisition/result_download.py +360 -0
- mostlyright/data_harness/acquisition/retention_admission.py +248 -0
- mostlyright/data_harness/acquisition/sandbox.py +4888 -0
- mostlyright/data_harness/acquisition/url_policy.py +530 -0
- mostlyright/data_harness/agent_runtime.py +2743 -0
- mostlyright/data_harness/assets/logo-ink.svg +31 -0
- mostlyright/data_harness/backends/__init__.py +28 -0
- mostlyright/data_harness/backends/pandas_backend.py +350 -0
- mostlyright/data_harness/backends/polars_backend.py +366 -0
- mostlyright/data_harness/backends/protocol.py +124 -0
- mostlyright/data_harness/backends/reference.py +83 -0
- mostlyright/data_harness/backends/registry.py +55 -0
- mostlyright/data_harness/backends/restrictions.py +126 -0
- mostlyright/data_harness/canonical.py +333 -0
- mostlyright/data_harness/catalog_job.py +625 -0
- mostlyright/data_harness/cli.py +5398 -0
- mostlyright/data_harness/contracts.py +53 -0
- mostlyright/data_harness/coordinator.py +1307 -0
- mostlyright/data_harness/deploy.py +924 -0
- mostlyright/data_harness/deploy_target.py +312 -0
- mostlyright/data_harness/deployment_evidence.py +1067 -0
- mostlyright/data_harness/event_presentation.py +576 -0
- mostlyright/data_harness/events.py +2152 -0
- mostlyright/data_harness/fast_delimited.py +239 -0
- mostlyright/data_harness/fleet.py +237 -0
- mostlyright/data_harness/formats.py +236 -0
- mostlyright/data_harness/governors.py +1163 -0
- mostlyright/data_harness/hosted_bootstrap.py +972 -0
- mostlyright/data_harness/hosted_crawler.py +1115 -0
- mostlyright/data_harness/hosted_crawler_container_smoke.py +351 -0
- mostlyright/data_harness/hosted_crawler_fetch.py +423 -0
- mostlyright/data_harness/hosted_crawler_job.py +1277 -0
- mostlyright/data_harness/hosted_crawler_protocol.py +676 -0
- mostlyright/data_harness/hosted_dataset.py +1500 -0
- mostlyright/data_harness/hosted_deploy.py +3037 -0
- mostlyright/data_harness/hosted_handoff.py +62 -0
- mostlyright/data_harness/hosted_ingestion_contract.py +504 -0
- mostlyright/data_harness/hosted_ingestion_job.py +356 -0
- mostlyright/data_harness/hosted_ingestion_job_smoke.py +40 -0
- mostlyright/data_harness/hosted_session_container_smoke.py +194 -0
- mostlyright/data_harness/hosted_session_worker.py +3554 -0
- mostlyright/data_harness/hosted_session_worker_job_smoke.py +46 -0
- mostlyright/data_harness/hosted_worker.py +6784 -0
- mostlyright/data_harness/ingestion/__init__.py +56 -0
- mostlyright/data_harness/ingestion/contracts.py +461 -0
- mostlyright/data_harness/ingestion/faults.py +42 -0
- mostlyright/data_harness/ingestion/gcs_store.py +1162 -0
- mostlyright/data_harness/ingestion/spool.py +130 -0
- mostlyright/data_harness/ingestion/store.py +885 -0
- mostlyright/data_harness/key_seam.py +434 -0
- mostlyright/data_harness/linux_process_boundary.py +262 -0
- mostlyright/data_harness/local_contracts.py +2880 -0
- mostlyright/data_harness/local_search/__init__.py +5 -0
- mostlyright/data_harness/local_search/build_index.py +1087 -0
- mostlyright/data_harness/local_search/contracts.py +920 -0
- mostlyright/data_harness/local_search/query_trace.py +266 -0
- mostlyright/data_harness/local_search/retrieval.py +700 -0
- mostlyright/data_harness/local_search/sealed.py +474 -0
- mostlyright/data_harness/local_search/service.py +784 -0
- mostlyright/data_harness/nbrender/CONTRACT.md +212 -0
- mostlyright/data_harness/nbrender/__init__.py +12 -0
- mostlyright/data_harness/nbrender/chrome.py +359 -0
- mostlyright/data_harness/nbrender/code_body.py +266 -0
- mostlyright/data_harness/nbrender/document.py +407 -0
- mostlyright/data_harness/nbrender/frame.py +275 -0
- mostlyright/data_harness/nbrender/interactive.py +337 -0
- mostlyright/data_harness/nbrender/markdown_body.py +477 -0
- mostlyright/data_harness/nbrender/mr_components.py +134 -0
- mostlyright/data_harness/nbrender/outputs_data.py +595 -0
- mostlyright/data_harness/nbrender/outputs_rich.py +906 -0
- mostlyright/data_harness/nbrender/outputs_source.py +260 -0
- mostlyright/data_harness/nbrender/outputs_stage.py +176 -0
- mostlyright/data_harness/nbrender/outputs_text.py +400 -0
- mostlyright/data_harness/nbrender/parse.py +394 -0
- mostlyright/data_harness/nbrender/status.py +40 -0
- mostlyright/data_harness/nbrender/tokens.py +1295 -0
- mostlyright/data_harness/notebook.py +1710 -0
- mostlyright/data_harness/offline.py +2049 -0
- mostlyright/data_harness/operation_registry.py +1007 -0
- mostlyright/data_harness/operator_setup.py +239 -0
- mostlyright/data_harness/pipeline.py +6428 -0
- mostlyright/data_harness/plan_graph.py +2026 -0
- mostlyright/data_harness/preparation/__init__.py +104 -0
- mostlyright/data_harness/preparation/contracts.py +1017 -0
- mostlyright/data_harness/preparation/engine.py +221 -0
- mostlyright/data_harness/preparation/errors.py +14 -0
- mostlyright/data_harness/preparation/gates.py +751 -0
- mostlyright/data_harness/preparation/joins.py +574 -0
- mostlyright/data_harness/preparation/profile.py +384 -0
- mostlyright/data_harness/preparation/table.py +217 -0
- mostlyright/data_harness/preparation/transforms.py +568 -0
- mostlyright/data_harness/progress_events.py +534 -0
- mostlyright/data_harness/readers/__init__.py +46 -0
- mostlyright/data_harness/readers/containers.py +963 -0
- mostlyright/data_harness/readers/contracts.py +542 -0
- mostlyright/data_harness/readers/delimited.py +257 -0
- mostlyright/data_harness/readers/grib2/__init__.py +33 -0
- mostlyright/data_harness/readers/grib2/admission.py +722 -0
- mostlyright/data_harness/readers/grib2/decode.py +1009 -0
- mostlyright/data_harness/readers/grib2/geometry.py +1133 -0
- mostlyright/data_harness/readers/grib2/portable_math.py +501 -0
- mostlyright/data_harness/readers/json_tabular.py +485 -0
- mostlyright/data_harness/readers/registry.py +514 -0
- mostlyright/data_harness/readers/samples/README.md +110 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/cities.csv.gz +0 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/cities.csv.gz +0 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/cities.tar +0 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/cities.tar +0 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/cities.zip +0 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/expected.json +25 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/expected.json +25 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/expected.json +25 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/cities.csv +3 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/expected.json +23 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/cities.tsv +5 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/expected.json +25 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/expected.json +30 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/observations.csv +5 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/expected.json +44 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/stations.json +1 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/expected.json +48 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/observations.ndjson +4 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/cities.xlsx +0 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/expected.json +24 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/expected.json +27 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/expected.json +27 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/README.md +20 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/expected.json +55 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/gfs-2m-temperature.grib2 +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/expected.json +54 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/expected.json +54 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/hrrr-categorical-rain.grib2 +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/expected.json +54 -0
- mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
- mostlyright/data_harness/readers/samples.py +582 -0
- mostlyright/data_harness/readers/spreadsheet.py +803 -0
- mostlyright/data_harness/readers/tabular.py +510 -0
- mostlyright/data_harness/recipe.py +5321 -0
- mostlyright/data_harness/repair/__init__.py +78 -0
- mostlyright/data_harness/repair/adapters.py +274 -0
- mostlyright/data_harness/repair/contracts.py +872 -0
- mostlyright/data_harness/repair/coordinator.py +1099 -0
- mostlyright/data_harness/repair/errors.py +16 -0
- mostlyright/data_harness/review.py +2533 -0
- mostlyright/data_harness/rowset.py +283 -0
- mostlyright/data_harness/serving.py +1975 -0
- mostlyright/data_harness/serving_edge.py +590 -0
- mostlyright/data_harness/serving_http.py +1031 -0
- mostlyright/data_harness/session_probes.py +759 -0
- mostlyright/data_harness/signing.py +101 -0
- mostlyright/data_harness/source_discovery.py +898 -0
- mostlyright/data_harness/sources/__init__.py +209 -0
- mostlyright/data_harness/sources/_adapter_steps.py +213 -0
- mostlyright/data_harness/sources/adapters.py +1214 -0
- mostlyright/data_harness/sources/cadence.py +1428 -0
- mostlyright/data_harness/sources/cadence_emission.py +453 -0
- mostlyright/data_harness/sources/cadence_history.py +546 -0
- mostlyright/data_harness/sources/catalog/__init__.py +17 -0
- mostlyright/data_harness/sources/catalog/admission.py +477 -0
- mostlyright/data_harness/sources/catalog/authoring.py +1701 -0
- mostlyright/data_harness/sources/catalog/authoring_policy.py +701 -0
- mostlyright/data_harness/sources/catalog/authoring_shards.py +1217 -0
- mostlyright/data_harness/sources/catalog/bounded_io.py +231 -0
- mostlyright/data_harness/sources/catalog/channel.py +523 -0
- mostlyright/data_harness/sources/catalog/channel_client.py +296 -0
- mostlyright/data_harness/sources/catalog/contracts.py +825 -0
- mostlyright/data_harness/sources/catalog/coverage.py +137 -0
- mostlyright/data_harness/sources/catalog/delta.py +1340 -0
- mostlyright/data_harness/sources/catalog/embedding.py +532 -0
- mostlyright/data_harness/sources/catalog/entry_v2.py +1182 -0
- mostlyright/data_harness/sources/catalog/fill.py +3889 -0
- mostlyright/data_harness/sources/catalog/fill_partitions.py +459 -0
- mostlyright/data_harness/sources/catalog/fill_staging.py +1105 -0
- mostlyright/data_harness/sources/catalog/gating.py +374 -0
- mostlyright/data_harness/sources/catalog/generation_receipt.py +1607 -0
- mostlyright/data_harness/sources/catalog/harvest/__init__.py +7 -0
- mostlyright/data_harness/sources/catalog/harvest/ckan.py +384 -0
- mostlyright/data_harness/sources/catalog/harvest/datagov_v4.py +798 -0
- mostlyright/data_harness/sources/catalog/harvest/protocol.py +964 -0
- mostlyright/data_harness/sources/catalog/harvest/sdmx.py +445 -0
- mostlyright/data_harness/sources/catalog/harvest/stac.py +384 -0
- mostlyright/data_harness/sources/catalog/health.py +447 -0
- mostlyright/data_harness/sources/catalog/hosted_catalog.py +105 -0
- mostlyright/data_harness/sources/catalog/identity_history.py +1549 -0
- mostlyright/data_harness/sources/catalog/neural.py +1618 -0
- mostlyright/data_harness/sources/catalog/packed_catalog.py +2345 -0
- mostlyright/data_harness/sources/catalog/packed_retrieval.py +1517 -0
- mostlyright/data_harness/sources/catalog/packed_writer.py +2802 -0
- mostlyright/data_harness/sources/catalog/query_trace.py +1037 -0
- mostlyright/data_harness/sources/catalog/recommend.py +171 -0
- mostlyright/data_harness/sources/catalog/retrieval.py +230 -0
- mostlyright/data_harness/sources/catalog/retrieval_manifest.py +995 -0
- mostlyright/data_harness/sources/catalog/rights_decisions.py +254 -0
- mostlyright/data_harness/sources/catalog/sealed.py +560 -0
- mostlyright/data_harness/sources/catalog/search.py +230 -0
- mostlyright/data_harness/sources/catalog/streaming_delta.py +1097 -0
- mostlyright/data_harness/sources/catalog/update.py +891 -0
- mostlyright/data_harness/sources/collections.py +815 -0
- mostlyright/data_harness/sources/contracts.py +2223 -0
- mostlyright/data_harness/sources/deletion.py +761 -0
- mostlyright/data_harness/sources/fitness.py +162 -0
- mostlyright/data_harness/sources/governance.py +163 -0
- mostlyright/data_harness/sources/hosted.py +173 -0
- mostlyright/data_harness/sources/integration.py +218 -0
- mostlyright/data_harness/sources/range_reader.py +418 -0
- mostlyright/data_harness/sources/registry.py +514 -0
- mostlyright/data_harness/sources/rights_rule.py +59 -0
- mostlyright/data_harness/sources/source_cadence_vectors.v1.json +1 -0
- mostlyright/data_harness/sources/sports.py +521 -0
- mostlyright/data_harness/sources/stream.py +524 -0
- mostlyright/data_harness/sources/stream_connector.py +418 -0
- mostlyright/data_harness/sources/stream_recorder.py +1404 -0
- mostlyright/data_harness/studio_boundary.py +2019 -0
- mostlyright/data_harness/thin/__init__.py +37 -0
- mostlyright/data_harness/thin/acquire.py +1137 -0
- mostlyright/data_harness/thin/acquire_cancel.py +579 -0
- mostlyright/data_harness/thin/approvals.py +617 -0
- mostlyright/data_harness/thin/commands.py +406 -0
- mostlyright/data_harness/thin/download.py +194 -0
- mostlyright/data_harness/thin/narrative.py +589 -0
- mostlyright/data_harness/thin/parity.py +1070 -0
- mostlyright/data_harness/thin/propose.py +2759 -0
- mostlyright/data_harness/thin/research.py +1663 -0
- mostlyright/data_harness/thin/router.py +924 -0
- mostlyright/data_harness/thin/runs.py +519 -0
- mostlyright/data_harness/thin/session.py +281 -0
- mostlyright/data_harness/thin/stream.py +501 -0
- mostlyright/data_harness/thin/transport.py +187 -0
- mostlyright/data_harness/thin/vocabulary.py +368 -0
- mostlyright/data_harness/thin/workers.py +164 -0
- mostlyright/data_harness/ucum/TABLE-PIN.json +40 -0
- mostlyright/data_harness/ucum/ucum-subset.v1.json +632 -0
- mostlyright/data_harness/unit_flow.py +927 -0
- mostlyright/data_harness/units.py +572 -0
- mostlyright/data_harness/ux/__init__.py +9 -0
- mostlyright/data_harness/ux/approve.py +485 -0
- mostlyright/data_harness/ux/author_yaml.py +597 -0
- mostlyright/data_harness/ux/cloud_auth.py +447 -0
- mostlyright/data_harness/ux/commands/__init__.py +260 -0
- mostlyright/data_harness/ux/commands/approve.py +136 -0
- mostlyright/data_harness/ux/commands/auth.py +744 -0
- mostlyright/data_harness/ux/commands/author.py +79 -0
- mostlyright/data_harness/ux/commands/catalog_author.py +403 -0
- mostlyright/data_harness/ux/commands/catalog_fill.py +523 -0
- mostlyright/data_harness/ux/commands/catalog_harvest.py +545 -0
- mostlyright/data_harness/ux/commands/catalog_publish.py +1838 -0
- mostlyright/data_harness/ux/commands/catalog_search.py +71 -0
- mostlyright/data_harness/ux/commands/catalog_update.py +437 -0
- mostlyright/data_harness/ux/commands/deploy.py +134 -0
- mostlyright/data_harness/ux/commands/deploy_dataset.py +98 -0
- mostlyright/data_harness/ux/commands/deploy_plan.py +105 -0
- mostlyright/data_harness/ux/commands/deploy_status.py +104 -0
- mostlyright/data_harness/ux/commands/diff.py +74 -0
- mostlyright/data_harness/ux/commands/index.py +84 -0
- mostlyright/data_harness/ux/commands/inventory.py +47 -0
- mostlyright/data_harness/ux/commands/list_builds.py +143 -0
- mostlyright/data_harness/ux/commands/login.py +63 -0
- mostlyright/data_harness/ux/commands/peek.py +236 -0
- mostlyright/data_harness/ux/commands/plan_check.py +90 -0
- mostlyright/data_harness/ux/commands/preflight.py +97 -0
- mostlyright/data_harness/ux/commands/record.py +107 -0
- mostlyright/data_harness/ux/commands/review_setup.py +47 -0
- mostlyright/data_harness/ux/commands/search.py +440 -0
- mostlyright/data_harness/ux/commands/show.py +61 -0
- mostlyright/data_harness/ux/commands/whoami.py +37 -0
- mostlyright/data_harness/ux/credential_native.py +551 -0
- mostlyright/data_harness/ux/credential_store.py +1055 -0
- mostlyright/data_harness/ux/credentials.py +631 -0
- mostlyright/data_harness/ux/diffing.py +444 -0
- mostlyright/data_harness/ux/headline.py +671 -0
- mostlyright/data_harness/ux/hosted_acquisition.py +974 -0
- mostlyright/data_harness/ux/hosted_run_status.py +619 -0
- mostlyright/data_harness/ux/inventory.py +427 -0
- mostlyright/data_harness/ux/local_review.py +375 -0
- mostlyright/data_harness/ux/login.py +691 -0
- mostlyright/data_harness/ux/path_kind.py +147 -0
- mostlyright/data_harness/ux/peek.py +1000 -0
- mostlyright/data_harness/ux/plain_file.py +178 -0
- mostlyright/data_harness/ux/plan_check.py +311 -0
- mostlyright/data_harness/ux/preflight.py +918 -0
- mostlyright/data_harness/ux/readers.py +1124 -0
- mostlyright/data_harness/ux/remediation.py +2195 -0
- mostlyright/data_harness/ux/render.py +657 -0
- mostlyright/data_harness/ux/workload.py +1077 -0
- mostlyright/data_harness/viewer.py +3713 -0
- mostlyright/data_harness/visual_run/__init__.py +83 -0
- mostlyright/data_harness/visual_run/authoring.py +235 -0
- mostlyright/data_harness/visual_run/contracts.py +673 -0
- mostlyright/data_harness/visual_run/materialize.py +486 -0
- mostlyright/data_harness/visual_run/observations.py +874 -0
- mostlyright/data_harness/visual_run/query.py +259 -0
- mostlyright/data_harness/visual_run/reducer.py +280 -0
- mostlyright/data_harness/visual_run/sdk.py +892 -0
- mostlyright/data_harness/visual_run/store.py +584 -0
- mostlyright/data_harness/visual_run/transport.py +239 -0
- mostlyright/data_harness/watch.py +2999 -0
- mostlyright_data-0.9.0.dist-info/METADATA +607 -0
- mostlyright_data-0.9.0.dist-info/RECORD +314 -0
- mostlyright_data-0.9.0.dist-info/WHEEL +4 -0
- mostlyright_data-0.9.0.dist-info/entry_points.txt +12 -0
|
@@ -0,0 +1,1607 @@
|
|
|
1
|
+
"""The composite generation receipt, and the bounded reads every stage of it is built from.
|
|
2
|
+
|
|
3
|
+
One provider-scale generation is produced by three separate mutations -- harvest, author, publish
|
|
4
|
+
-- and the only durable evidence a downstream verifier may read is the composite receipt this module
|
|
5
|
+
composes. It is authoritative rather than descriptive: it carries the *exact bytes* of the harvest
|
|
6
|
+
and author receipts it was handed, together with their fingerprints, so nothing downstream has to
|
|
7
|
+
reconstruct a stage from counts somebody transcribed.
|
|
8
|
+
|
|
9
|
+
Every untrusted evidence document that reaches it is opened the same way. The parent chain is
|
|
10
|
+
walked with ``O_DIRECTORY|O_NOFOLLOW`` and retained, the final component is opened
|
|
11
|
+
``O_NOFOLLOW``, the descriptor is confirmed to be a single-link regular file inside its byte cap,
|
|
12
|
+
the bytes are read through that one descriptor, and the descriptor's identity is confirmed
|
|
13
|
+
unchanged afterwards. A link, a swap during the read, a directory, a device, or one byte over the
|
|
14
|
+
cap is a refusal before anything is parsed. The sole narrower exception is a wheel-installed
|
|
15
|
+
distribution member: installers may hardlink an authenticated cache member into ``site-packages``,
|
|
16
|
+
so that reader permits a regular hardlink while retaining the same no-follow, bounded, stable
|
|
17
|
+
descriptor checks; the resulting complete distribution digest must still equal the exact wheel
|
|
18
|
+
coordinate.
|
|
19
|
+
|
|
20
|
+
The caps below are fixed. There is deliberately no argument anywhere on the three commands that
|
|
21
|
+
widens one: a caller who could raise the composite-receipt ceiling could publish evidence nobody
|
|
22
|
+
can read back inside the budget the reader states.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
import ctypes
|
|
28
|
+
import errno
|
|
29
|
+
import fcntl
|
|
30
|
+
import hashlib
|
|
31
|
+
import importlib.metadata as metadata
|
|
32
|
+
import io
|
|
33
|
+
import os
|
|
34
|
+
import stat
|
|
35
|
+
import zipfile
|
|
36
|
+
from collections.abc import Callable, Iterator, Mapping, Sequence
|
|
37
|
+
from contextlib import contextmanager
|
|
38
|
+
from contextvars import ContextVar
|
|
39
|
+
from dataclasses import dataclass
|
|
40
|
+
from pathlib import Path
|
|
41
|
+
from typing import Any
|
|
42
|
+
|
|
43
|
+
from mostlyright.data_harness.canonical import (
|
|
44
|
+
CanonicalJSONError,
|
|
45
|
+
canonical_json_bytes,
|
|
46
|
+
canonical_sha256,
|
|
47
|
+
parse_canonical_json,
|
|
48
|
+
sha256_bytes,
|
|
49
|
+
)
|
|
50
|
+
from mostlyright.data_harness.sources.contracts import SourceContractError
|
|
51
|
+
|
|
52
|
+
#: The composite receipt this module composes, and the two stage receipts it binds.
|
|
53
|
+
GENERATION_RECEIPT_SCHEMA = "mr-data-catalog-generation.v1"
|
|
54
|
+
HARVEST_RECEIPT_SCHEMA = "mr-data-catalog-harvest.v1"
|
|
55
|
+
HARVEST_PREFLIGHT_SCHEMA = "mr-data-catalog-harvest-preflight.v1"
|
|
56
|
+
|
|
57
|
+
#: The committed tree one generation was produced from: wheel inputs, product source, tests,
|
|
58
|
+
#: verifier scripts, the Makefile and the lock. It deliberately names no narrative document.
|
|
59
|
+
EXECUTION_COORDINATE_SCHEMA = "mr-data-execution-coordinate.v1"
|
|
60
|
+
|
|
61
|
+
# --- the fixed caps, none of which any caller may widen ------------------------------------------
|
|
62
|
+
|
|
63
|
+
MAX_STAGE_RECEIPT_BYTES = 8_388_608
|
|
64
|
+
MAX_STAGE_RECEIPT_DEPTH = 16
|
|
65
|
+
MAX_STAGE_RECEIPT_MEMBERS = 512
|
|
66
|
+
MAX_STAGE_RECEIPT_ITEMS = 1024
|
|
67
|
+
|
|
68
|
+
MAX_EXECUTION_MANIFEST_BYTES = 16_777_216
|
|
69
|
+
MAX_EXECUTION_MANIFEST_DEPTH = 8
|
|
70
|
+
MAX_EXECUTION_MANIFEST_FILES = 20_000
|
|
71
|
+
MAX_EXECUTION_PATH_BYTES = 512
|
|
72
|
+
MAX_DISTRIBUTION_FILES = 10_000
|
|
73
|
+
MAX_DISTRIBUTION_MEMBER_BYTES = 33_554_432
|
|
74
|
+
MAX_DISTRIBUTION_TOTAL_BYTES = 33_554_432
|
|
75
|
+
MAX_DISTRIBUTION_COMPRESSED_BYTES = 67_108_864
|
|
76
|
+
# One expansion guard, shared with every other archive this package screens: an archive
|
|
77
|
+
# may not inflate past this floor unless it stays within this ratio of its packed size.
|
|
78
|
+
ARCHIVE_EXPANSION_FLOOR_BYTES = 4_194_304
|
|
79
|
+
ARCHIVE_EXPANSION_RATIO = 100
|
|
80
|
+
MAX_WHEEL_BYTES = 67_108_864
|
|
81
|
+
|
|
82
|
+
MAX_PREDECESSOR_HEAD_BYTES = 4_194_304
|
|
83
|
+
MAX_PREDECESSOR_HEAD_DEPTH = 16
|
|
84
|
+
MAX_PREDECESSOR_RANGE_DESCRIPTORS = 1024
|
|
85
|
+
|
|
86
|
+
MAX_RANGE_MANIFEST_BYTES = 65_536
|
|
87
|
+
MAX_RANGE_MANIFEST_MEMBERS = 24
|
|
88
|
+
|
|
89
|
+
MAX_COMPOSITE_RECEIPT_BYTES = 33_554_432
|
|
90
|
+
MAX_COMPOSITE_RECEIPT_DEPTH = 20
|
|
91
|
+
MAX_COMPOSITE_RANGE_DESCRIPTORS = 1024
|
|
92
|
+
MAX_COMPOSITE_ARRAY_ITEMS = 8192
|
|
93
|
+
|
|
94
|
+
_OPEN_MEMBER = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_CLOEXEC", 0)
|
|
95
|
+
_OPEN_DIRECTORY = (
|
|
96
|
+
os.O_RDONLY
|
|
97
|
+
| getattr(os, "O_DIRECTORY", 0)
|
|
98
|
+
| getattr(os, "O_NOFOLLOW", 0)
|
|
99
|
+
| getattr(os, "O_CLOEXEC", 0)
|
|
100
|
+
)
|
|
101
|
+
_OPEN_LOCK = os.O_RDWR | os.O_CREAT | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_CLOEXEC", 0)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
@dataclass(frozen=True)
|
|
105
|
+
class ReceiptInstallation:
|
|
106
|
+
"""The exact receipt parent retained for one locked mutation window."""
|
|
107
|
+
|
|
108
|
+
destination: Path
|
|
109
|
+
parent_descriptor: int
|
|
110
|
+
parent_identity: tuple[int, int]
|
|
111
|
+
name: str
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
_ACTIVE_RECEIPT_INSTALLATIONS: ContextVar[tuple[ReceiptInstallation, ...]] = ContextVar(
|
|
115
|
+
"active_receipt_installations", default=()
|
|
116
|
+
)
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
class GenerationReceiptRefused(SourceContractError):
|
|
120
|
+
"""A stable refusal of a bounded input, an execution coordinate, or a composite receipt."""
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _absolute_path(path: Path) -> Path:
|
|
124
|
+
raw = os.fspath(path)
|
|
125
|
+
if not raw or "\x00" in raw:
|
|
126
|
+
raise OSError
|
|
127
|
+
return Path(os.path.abspath(raw))
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _open_directory_chain(path: Path, *, create: bool = False) -> list[int]:
|
|
131
|
+
"""Open one absolute directory path without following any component."""
|
|
132
|
+
|
|
133
|
+
absolute = _absolute_path(path)
|
|
134
|
+
descriptors = [os.open("/", _OPEN_DIRECTORY)]
|
|
135
|
+
try:
|
|
136
|
+
for component in absolute.parts[1:]:
|
|
137
|
+
parent = descriptors[-1]
|
|
138
|
+
if create:
|
|
139
|
+
try:
|
|
140
|
+
os.mkdir(component, mode=0o700, dir_fd=parent)
|
|
141
|
+
except FileExistsError:
|
|
142
|
+
pass
|
|
143
|
+
descriptors.append(os.open(component, _OPEN_DIRECTORY, dir_fd=parent))
|
|
144
|
+
except BaseException:
|
|
145
|
+
for descriptor in reversed(descriptors):
|
|
146
|
+
os.close(descriptor)
|
|
147
|
+
raise
|
|
148
|
+
return descriptors
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _same_identity(left: os.stat_result, right: os.stat_result) -> bool:
|
|
152
|
+
return (left.st_dev, left.st_ino, left.st_size, left.st_mtime_ns, left.st_ctime_ns) == (
|
|
153
|
+
right.st_dev,
|
|
154
|
+
right.st_ino,
|
|
155
|
+
right.st_size,
|
|
156
|
+
right.st_mtime_ns,
|
|
157
|
+
right.st_ctime_ns,
|
|
158
|
+
)
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
@contextmanager
|
|
162
|
+
def publication_output_lock(
|
|
163
|
+
root: Path,
|
|
164
|
+
*,
|
|
165
|
+
label: str,
|
|
166
|
+
code_prefix: str,
|
|
167
|
+
refusal: type[SourceContractError] = GenerationReceiptRefused,
|
|
168
|
+
) -> Iterator[int]:
|
|
169
|
+
"""Retain and exclusively lock one private no-follow publication root."""
|
|
170
|
+
|
|
171
|
+
descriptors: list[int] = []
|
|
172
|
+
lock_descriptor: int | None = None
|
|
173
|
+
try:
|
|
174
|
+
absolute = _absolute_path(root)
|
|
175
|
+
if absolute == Path("/"):
|
|
176
|
+
raise OSError
|
|
177
|
+
descriptors = _open_directory_chain(absolute.parent, create=True)
|
|
178
|
+
parent_descriptor = descriptors[-1]
|
|
179
|
+
name = absolute.name
|
|
180
|
+
if name in {"", ".", ".."}:
|
|
181
|
+
raise OSError
|
|
182
|
+
# Lock a stable parent/name coordinate. Locking the root inode itself would let a rename
|
|
183
|
+
# plus replacement create a second lock domain at the same pathname.
|
|
184
|
+
lock_descriptor = os.open(
|
|
185
|
+
f".{name}.publication.lock", _OPEN_LOCK, 0o600, dir_fd=parent_descriptor
|
|
186
|
+
)
|
|
187
|
+
lock_info = os.fstat(lock_descriptor)
|
|
188
|
+
if not stat.S_ISREG(lock_info.st_mode) or lock_info.st_nlink != 1:
|
|
189
|
+
raise OSError
|
|
190
|
+
os.fchmod(lock_descriptor, 0o600)
|
|
191
|
+
fcntl.flock(lock_descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
192
|
+
try:
|
|
193
|
+
os.mkdir(name, mode=0o700, dir_fd=parent_descriptor)
|
|
194
|
+
except FileExistsError:
|
|
195
|
+
pass
|
|
196
|
+
root_descriptor = os.open(name, _OPEN_DIRECTORY, dir_fd=parent_descriptor)
|
|
197
|
+
descriptors.append(root_descriptor)
|
|
198
|
+
info = os.fstat(root_descriptor)
|
|
199
|
+
if not stat.S_ISDIR(info.st_mode):
|
|
200
|
+
raise OSError
|
|
201
|
+
os.fchmod(root_descriptor, 0o700)
|
|
202
|
+
except OSError as error:
|
|
203
|
+
if lock_descriptor is not None:
|
|
204
|
+
try:
|
|
205
|
+
fcntl.flock(lock_descriptor, fcntl.LOCK_UN)
|
|
206
|
+
except OSError:
|
|
207
|
+
pass
|
|
208
|
+
os.close(lock_descriptor)
|
|
209
|
+
for descriptor in reversed(descriptors):
|
|
210
|
+
os.close(descriptor)
|
|
211
|
+
code = (
|
|
212
|
+
f"{code_prefix}_OUTPUT_LOCKED"
|
|
213
|
+
if error.errno in {errno.EACCES, errno.EAGAIN}
|
|
214
|
+
else f"{code_prefix}_OUTPUT_PATH"
|
|
215
|
+
)
|
|
216
|
+
detail = (
|
|
217
|
+
"another invocation owns this output root"
|
|
218
|
+
if code.endswith("_LOCKED")
|
|
219
|
+
else "output root must be a retained no-follow directory path"
|
|
220
|
+
)
|
|
221
|
+
raise refusal(
|
|
222
|
+
code,
|
|
223
|
+
label,
|
|
224
|
+
detail,
|
|
225
|
+
) from error
|
|
226
|
+
try:
|
|
227
|
+
require_output_root_identity(
|
|
228
|
+
absolute,
|
|
229
|
+
root_descriptor,
|
|
230
|
+
label=label,
|
|
231
|
+
code_prefix=code_prefix,
|
|
232
|
+
refusal=refusal,
|
|
233
|
+
)
|
|
234
|
+
yield root_descriptor
|
|
235
|
+
finally:
|
|
236
|
+
assert lock_descriptor is not None
|
|
237
|
+
fcntl.flock(lock_descriptor, fcntl.LOCK_UN)
|
|
238
|
+
os.close(lock_descriptor)
|
|
239
|
+
for descriptor in reversed(descriptors):
|
|
240
|
+
os.close(descriptor)
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def require_output_root_identity(
|
|
244
|
+
root: Path,
|
|
245
|
+
descriptor: int,
|
|
246
|
+
*,
|
|
247
|
+
label: str,
|
|
248
|
+
code_prefix: str,
|
|
249
|
+
refusal: type[SourceContractError] = GenerationReceiptRefused,
|
|
250
|
+
) -> None:
|
|
251
|
+
"""Require the named root to remain the exact retained, no-follow directory."""
|
|
252
|
+
|
|
253
|
+
try:
|
|
254
|
+
named = os.stat(_absolute_path(root), follow_symlinks=False)
|
|
255
|
+
retained = os.fstat(descriptor)
|
|
256
|
+
except OSError as error:
|
|
257
|
+
raise refusal(
|
|
258
|
+
f"{code_prefix}_OUTPUT_PATH",
|
|
259
|
+
label,
|
|
260
|
+
"output root no longer names the retained directory",
|
|
261
|
+
) from error
|
|
262
|
+
if (
|
|
263
|
+
not stat.S_ISDIR(named.st_mode)
|
|
264
|
+
or not stat.S_ISDIR(retained.st_mode)
|
|
265
|
+
or (named.st_dev, named.st_ino) != (retained.st_dev, retained.st_ino)
|
|
266
|
+
):
|
|
267
|
+
raise refusal(
|
|
268
|
+
f"{code_prefix}_OUTPUT_PATH",
|
|
269
|
+
label,
|
|
270
|
+
"output root no longer names the retained directory",
|
|
271
|
+
)
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
@contextmanager
|
|
275
|
+
def retained_child_directory(
|
|
276
|
+
parent_descriptor: int,
|
|
277
|
+
name: str,
|
|
278
|
+
*,
|
|
279
|
+
create: bool,
|
|
280
|
+
label: str,
|
|
281
|
+
code_prefix: str,
|
|
282
|
+
refusal: type[SourceContractError] = GenerationReceiptRefused,
|
|
283
|
+
) -> Iterator[int]:
|
|
284
|
+
"""Open one private child relative to a retained parent, without reopening its pathname."""
|
|
285
|
+
|
|
286
|
+
descriptor = -1
|
|
287
|
+
try:
|
|
288
|
+
if not name or name in {".", ".."} or "/" in name or "\x00" in name:
|
|
289
|
+
raise OSError
|
|
290
|
+
if create:
|
|
291
|
+
try:
|
|
292
|
+
os.mkdir(name, mode=0o700, dir_fd=parent_descriptor)
|
|
293
|
+
except FileExistsError:
|
|
294
|
+
pass
|
|
295
|
+
descriptor = os.open(name, _OPEN_DIRECTORY, dir_fd=parent_descriptor)
|
|
296
|
+
info = os.fstat(descriptor)
|
|
297
|
+
if not stat.S_ISDIR(info.st_mode) or stat.S_IMODE(info.st_mode) & 0o077:
|
|
298
|
+
raise OSError
|
|
299
|
+
except OSError as error:
|
|
300
|
+
if descriptor >= 0:
|
|
301
|
+
os.close(descriptor)
|
|
302
|
+
raise refusal(
|
|
303
|
+
f"{code_prefix}_OUTPUT_PATH", label, "output child must be a private directory"
|
|
304
|
+
) from error
|
|
305
|
+
try:
|
|
306
|
+
yield descriptor
|
|
307
|
+
finally:
|
|
308
|
+
os.close(descriptor)
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
@contextmanager
|
|
312
|
+
def retained_input_directory(
|
|
313
|
+
root: Path,
|
|
314
|
+
*,
|
|
315
|
+
label: str,
|
|
316
|
+
code_prefix: str,
|
|
317
|
+
refusal: type[SourceContractError] = GenerationReceiptRefused,
|
|
318
|
+
) -> Iterator[int]:
|
|
319
|
+
"""Retain one existing private input root without following any path component."""
|
|
320
|
+
|
|
321
|
+
descriptors: list[int] = []
|
|
322
|
+
try:
|
|
323
|
+
descriptors = _open_directory_chain(root)
|
|
324
|
+
descriptor = descriptors[-1]
|
|
325
|
+
info = os.fstat(descriptor)
|
|
326
|
+
if not stat.S_ISDIR(info.st_mode) or stat.S_IMODE(info.st_mode) & 0o077:
|
|
327
|
+
raise OSError
|
|
328
|
+
except OSError as error:
|
|
329
|
+
for opened in reversed(descriptors):
|
|
330
|
+
os.close(opened)
|
|
331
|
+
raise refusal(
|
|
332
|
+
f"{code_prefix}_INPUT_PATH",
|
|
333
|
+
label,
|
|
334
|
+
"input root must be an existing retained private no-follow directory",
|
|
335
|
+
) from error
|
|
336
|
+
try:
|
|
337
|
+
yield descriptor
|
|
338
|
+
finally:
|
|
339
|
+
for opened in reversed(descriptors):
|
|
340
|
+
os.close(opened)
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
@contextmanager
|
|
344
|
+
def receipt_installation_lock(
|
|
345
|
+
path: Path,
|
|
346
|
+
*,
|
|
347
|
+
label: str,
|
|
348
|
+
code_prefix: str,
|
|
349
|
+
refusal: type[SourceContractError] = GenerationReceiptRefused,
|
|
350
|
+
) -> Iterator[ReceiptInstallation]:
|
|
351
|
+
"""Exclude another cooperative writer for the receipt's whole mutation window."""
|
|
352
|
+
|
|
353
|
+
descriptors: list[int] = []
|
|
354
|
+
try:
|
|
355
|
+
destination = _absolute_path(path)
|
|
356
|
+
descriptors = _open_directory_chain(destination.parent)
|
|
357
|
+
descriptor = descriptors[-1]
|
|
358
|
+
opened = os.fstat(descriptor)
|
|
359
|
+
if not stat.S_ISDIR(opened.st_mode):
|
|
360
|
+
raise OSError
|
|
361
|
+
name = destination.name
|
|
362
|
+
if name in {"", ".", ".."}:
|
|
363
|
+
raise OSError
|
|
364
|
+
fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
365
|
+
except OSError as error:
|
|
366
|
+
for opened_descriptor in reversed(descriptors):
|
|
367
|
+
os.close(opened_descriptor)
|
|
368
|
+
raise refusal(
|
|
369
|
+
f"{code_prefix}_RECEIPT_LOCKED",
|
|
370
|
+
label,
|
|
371
|
+
"another invocation owns this receipt destination",
|
|
372
|
+
) from error
|
|
373
|
+
installation = ReceiptInstallation(
|
|
374
|
+
destination=destination,
|
|
375
|
+
parent_descriptor=descriptor,
|
|
376
|
+
parent_identity=(opened.st_dev, opened.st_ino),
|
|
377
|
+
name=name,
|
|
378
|
+
)
|
|
379
|
+
token = _ACTIVE_RECEIPT_INSTALLATIONS.set((*_ACTIVE_RECEIPT_INSTALLATIONS.get(), installation))
|
|
380
|
+
try:
|
|
381
|
+
_require_receipt_parent_identity(
|
|
382
|
+
installation, label=label, code_prefix=code_prefix, refusal=refusal
|
|
383
|
+
)
|
|
384
|
+
yield installation
|
|
385
|
+
finally:
|
|
386
|
+
_ACTIVE_RECEIPT_INSTALLATIONS.reset(token)
|
|
387
|
+
fcntl.flock(descriptor, fcntl.LOCK_UN)
|
|
388
|
+
for opened_descriptor in reversed(descriptors):
|
|
389
|
+
os.close(opened_descriptor)
|
|
390
|
+
|
|
391
|
+
|
|
392
|
+
def _require_receipt_parent_identity(
|
|
393
|
+
installation: ReceiptInstallation,
|
|
394
|
+
*,
|
|
395
|
+
label: str,
|
|
396
|
+
code_prefix: str,
|
|
397
|
+
refusal: type[SourceContractError],
|
|
398
|
+
) -> None:
|
|
399
|
+
"""Require the receipt pathname to retain the locked parent directory."""
|
|
400
|
+
|
|
401
|
+
try:
|
|
402
|
+
retained = os.fstat(installation.parent_descriptor)
|
|
403
|
+
# The named-parent lookup is deliberately last. Callers use this helper as their final
|
|
404
|
+
# filesystem operation before reporting success, so no retained-descriptor observation may
|
|
405
|
+
# follow and hide a pathname replacement that happened during it.
|
|
406
|
+
named = os.stat(installation.destination.parent, follow_symlinks=False)
|
|
407
|
+
except OSError as error:
|
|
408
|
+
raise refusal(
|
|
409
|
+
f"{code_prefix}_RECEIPT_DESTINATION",
|
|
410
|
+
label,
|
|
411
|
+
"receipt parent no longer names the retained locked directory",
|
|
412
|
+
) from error
|
|
413
|
+
if (
|
|
414
|
+
not stat.S_ISDIR(named.st_mode)
|
|
415
|
+
or not stat.S_ISDIR(retained.st_mode)
|
|
416
|
+
or (named.st_dev, named.st_ino) != installation.parent_identity
|
|
417
|
+
or (retained.st_dev, retained.st_ino) != installation.parent_identity
|
|
418
|
+
):
|
|
419
|
+
raise refusal(
|
|
420
|
+
f"{code_prefix}_RECEIPT_DESTINATION",
|
|
421
|
+
label,
|
|
422
|
+
"receipt parent no longer names the retained locked directory",
|
|
423
|
+
)
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
def is_digest(value: Any) -> bool:
|
|
427
|
+
"""Whether this value is a lowercase SHA-256, which is the only fingerprint spelling here."""
|
|
428
|
+
|
|
429
|
+
return (
|
|
430
|
+
isinstance(value, str) and len(value) == 64 and all(c in "0123456789abcdef" for c in value)
|
|
431
|
+
)
|
|
432
|
+
|
|
433
|
+
|
|
434
|
+
def is_commit(value: Any) -> bool:
|
|
435
|
+
"""Whether this value is a full lowercase 40-character commit identifier."""
|
|
436
|
+
|
|
437
|
+
return (
|
|
438
|
+
isinstance(value, str) and len(value) == 40 and all(c in "0123456789abcdef" for c in value)
|
|
439
|
+
)
|
|
440
|
+
|
|
441
|
+
|
|
442
|
+
# ------------------------------------------------------------------------------------------------
|
|
443
|
+
# Bounded reads
|
|
444
|
+
# ------------------------------------------------------------------------------------------------
|
|
445
|
+
|
|
446
|
+
|
|
447
|
+
def read_bounded_document(path: Path, *, maximum: int, label: str) -> bytes:
|
|
448
|
+
"""Read one document through a retained no-follow parent chain and a stable descriptor.
|
|
449
|
+
|
|
450
|
+
The name is resolved component by component so that no directory in the chain may be a link
|
|
451
|
+
swapped underneath the read, and the file's own identity is compared before and after: a
|
|
452
|
+
document that moved while it was being read never becomes evidence.
|
|
453
|
+
"""
|
|
454
|
+
|
|
455
|
+
raw_path = os.fspath(path)
|
|
456
|
+
if not raw_path or "\x00" in raw_path:
|
|
457
|
+
raise GenerationReceiptRefused("GENERATION_INPUT_PATH", label, "path is not a usable name")
|
|
458
|
+
absolute = Path(os.path.abspath(raw_path))
|
|
459
|
+
if absolute.name in {"", ".", ".."}:
|
|
460
|
+
raise GenerationReceiptRefused(
|
|
461
|
+
"GENERATION_INPUT_PATH", label, "path does not name one document"
|
|
462
|
+
)
|
|
463
|
+
owned: list[int] = []
|
|
464
|
+
descriptor = -1
|
|
465
|
+
try:
|
|
466
|
+
parent = os.open("/", _OPEN_DIRECTORY)
|
|
467
|
+
owned.append(parent)
|
|
468
|
+
for component in absolute.parts[1:-1]:
|
|
469
|
+
parent = os.open(component, _OPEN_DIRECTORY, dir_fd=parent)
|
|
470
|
+
owned.append(parent)
|
|
471
|
+
descriptor = os.open(absolute.name, _OPEN_MEMBER, dir_fd=parent)
|
|
472
|
+
except OSError as error:
|
|
473
|
+
for opened in reversed(owned):
|
|
474
|
+
os.close(opened)
|
|
475
|
+
raise GenerationReceiptRefused(
|
|
476
|
+
"GENERATION_INPUT_PATH",
|
|
477
|
+
label,
|
|
478
|
+
"must be reachable without following one link, from a retained parent",
|
|
479
|
+
) from error
|
|
480
|
+
try:
|
|
481
|
+
before = os.fstat(descriptor)
|
|
482
|
+
if not stat.S_ISREG(before.st_mode) or before.st_nlink != 1:
|
|
483
|
+
raise GenerationReceiptRefused(
|
|
484
|
+
"GENERATION_INPUT_PATH", label, "must be a single-link regular file"
|
|
485
|
+
)
|
|
486
|
+
if before.st_size > maximum:
|
|
487
|
+
raise GenerationReceiptRefused(
|
|
488
|
+
"GENERATION_INPUT_LIMIT", label, f"must be at most {maximum} bytes"
|
|
489
|
+
)
|
|
490
|
+
chunks: list[bytes] = []
|
|
491
|
+
remaining = maximum + 1
|
|
492
|
+
while remaining > 0:
|
|
493
|
+
chunk = os.read(descriptor, min(remaining, 1 << 16))
|
|
494
|
+
if not chunk:
|
|
495
|
+
break
|
|
496
|
+
chunks.append(chunk)
|
|
497
|
+
remaining -= len(chunk)
|
|
498
|
+
after = os.fstat(descriptor)
|
|
499
|
+
if (before.st_dev, before.st_ino, before.st_size, before.st_mtime_ns) != (
|
|
500
|
+
after.st_dev,
|
|
501
|
+
after.st_ino,
|
|
502
|
+
after.st_size,
|
|
503
|
+
after.st_mtime_ns,
|
|
504
|
+
):
|
|
505
|
+
raise GenerationReceiptRefused(
|
|
506
|
+
"GENERATION_INPUT_RACE", label, "document changed while it was being read"
|
|
507
|
+
)
|
|
508
|
+
finally:
|
|
509
|
+
os.close(descriptor)
|
|
510
|
+
for opened in reversed(owned):
|
|
511
|
+
os.close(opened)
|
|
512
|
+
raw = b"".join(chunks)
|
|
513
|
+
if len(raw) > maximum:
|
|
514
|
+
raise GenerationReceiptRefused(
|
|
515
|
+
"GENERATION_INPUT_LIMIT", label, f"must be at most {maximum} bytes"
|
|
516
|
+
)
|
|
517
|
+
return raw
|
|
518
|
+
|
|
519
|
+
|
|
520
|
+
def _read_installed_distribution_member(path: Path, *, maximum: int, label: str) -> bytes:
|
|
521
|
+
"""Read one installer-owned member without rejecting a legitimate cache hardlink.
|
|
522
|
+
|
|
523
|
+
Unlike :func:`read_bounded_document`, this helper is used only for the closed
|
|
524
|
+
``importlib.metadata`` distribution inventory that is subsequently compared with the exact
|
|
525
|
+
wheel coordinate. It deliberately permits ``st_nlink > 1`` because uv and other installers
|
|
526
|
+
may install wheel payloads by hardlink. It still refuses symlinks and special files, retains
|
|
527
|
+
every no-follow parent descriptor, enforces the byte cap, and requires the complete file
|
|
528
|
+
identity (including link count and ctime) and its current parent binding to remain stable
|
|
529
|
+
during the read.
|
|
530
|
+
"""
|
|
531
|
+
|
|
532
|
+
raw_path = os.fspath(path)
|
|
533
|
+
if not raw_path or "\x00" in raw_path:
|
|
534
|
+
raise GenerationReceiptRefused("GENERATION_INPUT_PATH", label, "path is not a usable name")
|
|
535
|
+
absolute = Path(os.path.abspath(raw_path))
|
|
536
|
+
if absolute.name in {"", ".", ".."}:
|
|
537
|
+
raise GenerationReceiptRefused(
|
|
538
|
+
"GENERATION_INPUT_PATH", label, "path does not name one document"
|
|
539
|
+
)
|
|
540
|
+
owned: list[int] = []
|
|
541
|
+
descriptor = -1
|
|
542
|
+
try:
|
|
543
|
+
parent = os.open("/", _OPEN_DIRECTORY)
|
|
544
|
+
owned.append(parent)
|
|
545
|
+
for component in absolute.parts[1:-1]:
|
|
546
|
+
parent = os.open(component, _OPEN_DIRECTORY, dir_fd=parent)
|
|
547
|
+
owned.append(parent)
|
|
548
|
+
descriptor = os.open(
|
|
549
|
+
absolute.name,
|
|
550
|
+
_OPEN_MEMBER | getattr(os, "O_NONBLOCK", 0),
|
|
551
|
+
dir_fd=parent,
|
|
552
|
+
)
|
|
553
|
+
except OSError as error:
|
|
554
|
+
for opened in reversed(owned):
|
|
555
|
+
os.close(opened)
|
|
556
|
+
raise GenerationReceiptRefused(
|
|
557
|
+
"GENERATION_INPUT_PATH",
|
|
558
|
+
label,
|
|
559
|
+
"must be reachable without following one link, from a retained parent",
|
|
560
|
+
) from error
|
|
561
|
+
try:
|
|
562
|
+
before = os.fstat(descriptor)
|
|
563
|
+
if not stat.S_ISREG(before.st_mode) or before.st_nlink < 1:
|
|
564
|
+
raise GenerationReceiptRefused(
|
|
565
|
+
"GENERATION_INPUT_PATH", label, "must be a regular installed file"
|
|
566
|
+
)
|
|
567
|
+
if before.st_size > maximum:
|
|
568
|
+
raise GenerationReceiptRefused(
|
|
569
|
+
"GENERATION_INPUT_LIMIT", label, f"must be at most {maximum} bytes"
|
|
570
|
+
)
|
|
571
|
+
chunks: list[bytes] = []
|
|
572
|
+
remaining = maximum + 1
|
|
573
|
+
while remaining > 0:
|
|
574
|
+
chunk = os.read(descriptor, min(remaining, 1 << 16))
|
|
575
|
+
if not chunk:
|
|
576
|
+
break
|
|
577
|
+
chunks.append(chunk)
|
|
578
|
+
remaining -= len(chunk)
|
|
579
|
+
after = os.fstat(descriptor)
|
|
580
|
+
before_identity = (
|
|
581
|
+
before.st_dev,
|
|
582
|
+
before.st_ino,
|
|
583
|
+
before.st_mode,
|
|
584
|
+
before.st_nlink,
|
|
585
|
+
before.st_size,
|
|
586
|
+
before.st_mtime_ns,
|
|
587
|
+
before.st_ctime_ns,
|
|
588
|
+
)
|
|
589
|
+
after_identity = (
|
|
590
|
+
after.st_dev,
|
|
591
|
+
after.st_ino,
|
|
592
|
+
after.st_mode,
|
|
593
|
+
after.st_nlink,
|
|
594
|
+
after.st_size,
|
|
595
|
+
after.st_mtime_ns,
|
|
596
|
+
after.st_ctime_ns,
|
|
597
|
+
)
|
|
598
|
+
raw = b"".join(chunks)
|
|
599
|
+
current_chain: list[int] = []
|
|
600
|
+
try:
|
|
601
|
+
current_chain = _open_directory_chain(absolute.parent)
|
|
602
|
+
current_parent = os.fstat(current_chain[-1])
|
|
603
|
+
captured_parent = os.fstat(owned[-1])
|
|
604
|
+
current_member = os.stat(
|
|
605
|
+
absolute.name,
|
|
606
|
+
dir_fd=current_chain[-1],
|
|
607
|
+
follow_symlinks=False,
|
|
608
|
+
)
|
|
609
|
+
except OSError as error:
|
|
610
|
+
raise GenerationReceiptRefused(
|
|
611
|
+
"GENERATION_INPUT_RACE", label, "installed file path changed while it was read"
|
|
612
|
+
) from error
|
|
613
|
+
finally:
|
|
614
|
+
for opened in reversed(current_chain):
|
|
615
|
+
os.close(opened)
|
|
616
|
+
current_identity = (
|
|
617
|
+
current_member.st_dev,
|
|
618
|
+
current_member.st_ino,
|
|
619
|
+
current_member.st_mode,
|
|
620
|
+
current_member.st_nlink,
|
|
621
|
+
current_member.st_size,
|
|
622
|
+
current_member.st_mtime_ns,
|
|
623
|
+
current_member.st_ctime_ns,
|
|
624
|
+
)
|
|
625
|
+
if (
|
|
626
|
+
before_identity != after_identity
|
|
627
|
+
or current_identity != after_identity
|
|
628
|
+
or (current_parent.st_dev, current_parent.st_ino)
|
|
629
|
+
!= (captured_parent.st_dev, captured_parent.st_ino)
|
|
630
|
+
or len(raw) != before.st_size
|
|
631
|
+
):
|
|
632
|
+
raise GenerationReceiptRefused(
|
|
633
|
+
"GENERATION_INPUT_RACE", label, "installed file or path changed while it was read"
|
|
634
|
+
)
|
|
635
|
+
return raw
|
|
636
|
+
finally:
|
|
637
|
+
if descriptor >= 0:
|
|
638
|
+
os.close(descriptor)
|
|
639
|
+
for opened in reversed(owned):
|
|
640
|
+
os.close(opened)
|
|
641
|
+
|
|
642
|
+
|
|
643
|
+
def enforce_shape(
|
|
644
|
+
value: Any,
|
|
645
|
+
*,
|
|
646
|
+
label: str,
|
|
647
|
+
max_depth: int,
|
|
648
|
+
max_members: int,
|
|
649
|
+
max_items: int,
|
|
650
|
+
code: str = "GENERATION_INPUT_SHAPE",
|
|
651
|
+
) -> None:
|
|
652
|
+
"""Refuse a parsed document deeper, wider, or longer than its stated structural bounds."""
|
|
653
|
+
|
|
654
|
+
stack: list[tuple[Any, int]] = [(value, 1)]
|
|
655
|
+
while stack:
|
|
656
|
+
current, depth = stack.pop()
|
|
657
|
+
if depth > max_depth:
|
|
658
|
+
raise GenerationReceiptRefused(code, label, f"nesting exceeds {max_depth} levels")
|
|
659
|
+
if isinstance(current, Mapping):
|
|
660
|
+
if len(current) > max_members:
|
|
661
|
+
raise GenerationReceiptRefused(
|
|
662
|
+
code, label, f"an object exceeds {max_members} members"
|
|
663
|
+
)
|
|
664
|
+
stack.extend((member, depth + 1) for member in current.values())
|
|
665
|
+
elif isinstance(current, Sequence) and not isinstance(current, str | bytes | bytearray):
|
|
666
|
+
if len(current) > max_items:
|
|
667
|
+
raise GenerationReceiptRefused(code, label, f"an array exceeds {max_items} items")
|
|
668
|
+
stack.extend((member, depth + 1) for member in current)
|
|
669
|
+
|
|
670
|
+
|
|
671
|
+
def load_bounded_json(
|
|
672
|
+
path: Path,
|
|
673
|
+
*,
|
|
674
|
+
label: str,
|
|
675
|
+
maximum: int,
|
|
676
|
+
max_depth: int,
|
|
677
|
+
max_members: int,
|
|
678
|
+
max_items: int,
|
|
679
|
+
expected_sha256: str | None = None,
|
|
680
|
+
shape_code: str = "GENERATION_INPUT_SHAPE",
|
|
681
|
+
) -> tuple[bytes, dict[str, Any]]:
|
|
682
|
+
"""Read, fingerprint, parse, and shape-check one bounded canonical JSON document."""
|
|
683
|
+
|
|
684
|
+
if expected_sha256 is not None and not is_digest(expected_sha256):
|
|
685
|
+
raise GenerationReceiptRefused(
|
|
686
|
+
"GENERATION_INPUT_DIGEST", label, "must be a lowercase SHA-256"
|
|
687
|
+
)
|
|
688
|
+
raw = read_bounded_document(path, maximum=maximum, label=label)
|
|
689
|
+
if expected_sha256 is not None and sha256_bytes(raw) != expected_sha256:
|
|
690
|
+
raise GenerationReceiptRefused(
|
|
691
|
+
"GENERATION_INPUT_DIGEST", label, "document bytes differ from the required digest"
|
|
692
|
+
)
|
|
693
|
+
try:
|
|
694
|
+
parsed = parse_canonical_json(raw)
|
|
695
|
+
except CanonicalJSONError as error:
|
|
696
|
+
raise GenerationReceiptRefused(
|
|
697
|
+
shape_code, label, "document is not canonical JSON"
|
|
698
|
+
) from error
|
|
699
|
+
if not isinstance(parsed, dict):
|
|
700
|
+
raise GenerationReceiptRefused(shape_code, label, "document must be one JSON object")
|
|
701
|
+
enforce_shape(
|
|
702
|
+
parsed,
|
|
703
|
+
label=label,
|
|
704
|
+
max_depth=max_depth,
|
|
705
|
+
max_members=max_members,
|
|
706
|
+
max_items=max_items,
|
|
707
|
+
code=shape_code,
|
|
708
|
+
)
|
|
709
|
+
return raw, parsed
|
|
710
|
+
|
|
711
|
+
|
|
712
|
+
def load_bounded_json_at(
|
|
713
|
+
parent_descriptor: int,
|
|
714
|
+
name: str,
|
|
715
|
+
*,
|
|
716
|
+
label: str,
|
|
717
|
+
maximum: int,
|
|
718
|
+
max_depth: int,
|
|
719
|
+
max_members: int,
|
|
720
|
+
max_items: int,
|
|
721
|
+
shape_code: str = "GENERATION_INPUT_SHAPE",
|
|
722
|
+
) -> tuple[bytes, dict[str, Any]]:
|
|
723
|
+
"""Read one bounded canonical JSON member relative to an already retained directory."""
|
|
724
|
+
|
|
725
|
+
try:
|
|
726
|
+
raw, _identity = _read_bounded_at(parent_descriptor, name, maximum=maximum)
|
|
727
|
+
except OSError as error:
|
|
728
|
+
raise GenerationReceiptRefused(
|
|
729
|
+
"GENERATION_INPUT_PATH", label, "member is not a bounded single-link regular file"
|
|
730
|
+
) from error
|
|
731
|
+
try:
|
|
732
|
+
parsed = parse_canonical_json(raw)
|
|
733
|
+
except CanonicalJSONError as error:
|
|
734
|
+
raise GenerationReceiptRefused(
|
|
735
|
+
shape_code, label, "document is not canonical JSON"
|
|
736
|
+
) from error
|
|
737
|
+
if not isinstance(parsed, dict):
|
|
738
|
+
raise GenerationReceiptRefused(shape_code, label, "document must be one JSON object")
|
|
739
|
+
enforce_shape(
|
|
740
|
+
parsed,
|
|
741
|
+
label=label,
|
|
742
|
+
max_depth=max_depth,
|
|
743
|
+
max_members=max_members,
|
|
744
|
+
max_items=max_items,
|
|
745
|
+
code=shape_code,
|
|
746
|
+
)
|
|
747
|
+
return raw, parsed
|
|
748
|
+
|
|
749
|
+
|
|
750
|
+
def load_stage_receipt(
|
|
751
|
+
path: Path,
|
|
752
|
+
expected_sha256: str,
|
|
753
|
+
*,
|
|
754
|
+
label: str,
|
|
755
|
+
schemas: frozenset[str],
|
|
756
|
+
) -> tuple[bytes, dict[str, Any]]:
|
|
757
|
+
"""One component receipt, read under the fixed stage caps and its caller-retained digest."""
|
|
758
|
+
|
|
759
|
+
raw, payload = load_bounded_json(
|
|
760
|
+
path,
|
|
761
|
+
label=label,
|
|
762
|
+
maximum=MAX_STAGE_RECEIPT_BYTES,
|
|
763
|
+
max_depth=MAX_STAGE_RECEIPT_DEPTH,
|
|
764
|
+
max_members=MAX_STAGE_RECEIPT_MEMBERS,
|
|
765
|
+
max_items=MAX_STAGE_RECEIPT_ITEMS,
|
|
766
|
+
expected_sha256=expected_sha256,
|
|
767
|
+
)
|
|
768
|
+
if payload.get("schema_version") not in schemas:
|
|
769
|
+
raise GenerationReceiptRefused(
|
|
770
|
+
"GENERATION_INPUT_SCHEMA", label, "receipt does not name one accepted stage schema"
|
|
771
|
+
)
|
|
772
|
+
return raw, payload
|
|
773
|
+
|
|
774
|
+
|
|
775
|
+
def load_execution_coordinate(path: Path, expected_commit: str) -> dict[str, Any]:
|
|
776
|
+
"""The committed tree this generation was produced from, named by its own exact commit."""
|
|
777
|
+
|
|
778
|
+
if not is_commit(expected_commit):
|
|
779
|
+
raise GenerationReceiptRefused(
|
|
780
|
+
"GENERATION_EXECUTION_COMMIT",
|
|
781
|
+
"execution_commit",
|
|
782
|
+
"must be a full lowercase 40-character commit",
|
|
783
|
+
)
|
|
784
|
+
_raw, payload = load_bounded_json(
|
|
785
|
+
path,
|
|
786
|
+
label="execution_manifest",
|
|
787
|
+
maximum=MAX_EXECUTION_MANIFEST_BYTES,
|
|
788
|
+
max_depth=MAX_EXECUTION_MANIFEST_DEPTH,
|
|
789
|
+
max_members=MAX_EXECUTION_MANIFEST_FILES,
|
|
790
|
+
max_items=MAX_EXECUTION_MANIFEST_FILES,
|
|
791
|
+
shape_code="GENERATION_EXECUTION_SHAPE",
|
|
792
|
+
)
|
|
793
|
+
if payload.get("schema_version") != EXECUTION_COORDINATE_SCHEMA:
|
|
794
|
+
raise GenerationReceiptRefused(
|
|
795
|
+
"GENERATION_EXECUTION_SHAPE",
|
|
796
|
+
"execution_manifest",
|
|
797
|
+
"manifest does not name the execution coordinate schema",
|
|
798
|
+
)
|
|
799
|
+
files = payload.get("files")
|
|
800
|
+
if not isinstance(files, list) or not files or len(files) > MAX_EXECUTION_MANIFEST_FILES:
|
|
801
|
+
raise GenerationReceiptRefused(
|
|
802
|
+
"GENERATION_EXECUTION_SHAPE",
|
|
803
|
+
"execution_manifest.files",
|
|
804
|
+
f"must name 1 to {MAX_EXECUTION_MANIFEST_FILES} committed files",
|
|
805
|
+
)
|
|
806
|
+
seen: set[str] = set()
|
|
807
|
+
for index, item in enumerate(files):
|
|
808
|
+
member = f"execution_manifest.files[{index}]"
|
|
809
|
+
if not isinstance(item, dict) or set(item) != {"path", "sha256"}:
|
|
810
|
+
raise GenerationReceiptRefused(
|
|
811
|
+
"GENERATION_EXECUTION_SHAPE", member, "each file is one path and one fingerprint"
|
|
812
|
+
)
|
|
813
|
+
name = item["path"]
|
|
814
|
+
if (
|
|
815
|
+
not isinstance(name, str)
|
|
816
|
+
or not name
|
|
817
|
+
or len(name.encode("utf-8")) > MAX_EXECUTION_PATH_BYTES
|
|
818
|
+
or name in seen
|
|
819
|
+
):
|
|
820
|
+
raise GenerationReceiptRefused(
|
|
821
|
+
"GENERATION_EXECUTION_SHAPE",
|
|
822
|
+
member,
|
|
823
|
+
f"each path is stated once and is at most {MAX_EXECUTION_PATH_BYTES} bytes",
|
|
824
|
+
)
|
|
825
|
+
seen.add(name)
|
|
826
|
+
if not is_digest(item["sha256"]):
|
|
827
|
+
raise GenerationReceiptRefused(
|
|
828
|
+
"GENERATION_EXECUTION_SHAPE", member, "each file states a lowercase SHA-256"
|
|
829
|
+
)
|
|
830
|
+
body = {key: value for key, value in payload.items() if key != "root_sha256"}
|
|
831
|
+
if payload.get("root_sha256") != canonical_sha256(body):
|
|
832
|
+
raise GenerationReceiptRefused(
|
|
833
|
+
"GENERATION_EXECUTION_SHAPE",
|
|
834
|
+
"execution_manifest.root_sha256",
|
|
835
|
+
"the manifest does not reproduce its own root fingerprint",
|
|
836
|
+
)
|
|
837
|
+
if payload.get("commit") != expected_commit:
|
|
838
|
+
raise GenerationReceiptRefused(
|
|
839
|
+
"GENERATION_EXECUTION_COMMIT",
|
|
840
|
+
"execution_manifest.commit",
|
|
841
|
+
"the manifest was written for another execution commit",
|
|
842
|
+
)
|
|
843
|
+
coordinate: dict[str, Any] = {
|
|
844
|
+
"commit": expected_commit,
|
|
845
|
+
"root_sha256": payload["root_sha256"],
|
|
846
|
+
"file_count": len(files),
|
|
847
|
+
}
|
|
848
|
+
installed = payload.get("installed_runtime")
|
|
849
|
+
if installed is not None:
|
|
850
|
+
if not isinstance(installed, dict) or set(installed) != {
|
|
851
|
+
"wheel_sha256",
|
|
852
|
+
"distribution_sha256",
|
|
853
|
+
"distribution_file_count",
|
|
854
|
+
}:
|
|
855
|
+
raise GenerationReceiptRefused(
|
|
856
|
+
"GENERATION_EXECUTION_SHAPE",
|
|
857
|
+
"execution_manifest.installed_runtime",
|
|
858
|
+
"installed runtime must be one exact wheel/distribution coordinate",
|
|
859
|
+
)
|
|
860
|
+
for member in ("wheel_sha256", "distribution_sha256"):
|
|
861
|
+
if not is_digest(installed.get(member)):
|
|
862
|
+
raise GenerationReceiptRefused(
|
|
863
|
+
"GENERATION_EXECUTION_SHAPE",
|
|
864
|
+
f"execution_manifest.installed_runtime.{member}",
|
|
865
|
+
"must be a lowercase SHA-256",
|
|
866
|
+
)
|
|
867
|
+
count = installed.get("distribution_file_count")
|
|
868
|
+
if type(count) is not int or count <= 0 or count > MAX_DISTRIBUTION_FILES:
|
|
869
|
+
raise GenerationReceiptRefused(
|
|
870
|
+
"GENERATION_EXECUTION_SHAPE",
|
|
871
|
+
"execution_manifest.installed_runtime.distribution_file_count",
|
|
872
|
+
"must be a positive bounded file count",
|
|
873
|
+
)
|
|
874
|
+
observed = installed_distribution_coordinate()
|
|
875
|
+
if observed != {
|
|
876
|
+
"distribution_sha256": installed["distribution_sha256"],
|
|
877
|
+
"distribution_file_count": count,
|
|
878
|
+
}:
|
|
879
|
+
raise GenerationReceiptRefused(
|
|
880
|
+
"GENERATION_EXECUTION_DISTRIBUTION",
|
|
881
|
+
"execution_manifest.installed_runtime",
|
|
882
|
+
"the executing installed distribution differs from the wheel coordinate",
|
|
883
|
+
)
|
|
884
|
+
coordinate["installed_runtime"] = dict(installed)
|
|
885
|
+
return coordinate
|
|
886
|
+
|
|
887
|
+
|
|
888
|
+
def _distribution_member(path: str, dist_info: str) -> bool:
|
|
889
|
+
if path.startswith("mostlyright/"):
|
|
890
|
+
return True
|
|
891
|
+
if not path.startswith(f"{dist_info}/"):
|
|
892
|
+
return False
|
|
893
|
+
relative = path[len(dist_info) + 1 :]
|
|
894
|
+
return relative not in {
|
|
895
|
+
"RECORD",
|
|
896
|
+
"INSTALLER",
|
|
897
|
+
"REQUESTED",
|
|
898
|
+
"direct_url.json",
|
|
899
|
+
"uv_cache.json",
|
|
900
|
+
}
|
|
901
|
+
|
|
902
|
+
|
|
903
|
+
def _distribution_coordinate(files: Sequence[Mapping[str, str]], version: str) -> dict[str, Any]:
|
|
904
|
+
ordered = sorted((dict(item) for item in files), key=lambda item: item["path"])
|
|
905
|
+
if not ordered or len(ordered) > MAX_DISTRIBUTION_FILES:
|
|
906
|
+
raise GenerationReceiptRefused(
|
|
907
|
+
"GENERATION_EXECUTION_DISTRIBUTION",
|
|
908
|
+
"installed_runtime.distribution",
|
|
909
|
+
"the installed distribution file inventory is absent or over its cap",
|
|
910
|
+
)
|
|
911
|
+
body = {"name": "mostlyright-data", "version": version, "files": ordered}
|
|
912
|
+
return {
|
|
913
|
+
"distribution_sha256": canonical_sha256(body),
|
|
914
|
+
"distribution_file_count": len(ordered),
|
|
915
|
+
}
|
|
916
|
+
|
|
917
|
+
|
|
918
|
+
def wheel_runtime_coordinate(
|
|
919
|
+
path: Path, *, expected_product_sha256s: Mapping[str, str] | None = None
|
|
920
|
+
) -> dict[str, Any]:
|
|
921
|
+
"""Fingerprint one wheel and the portable distribution bytes it must install."""
|
|
922
|
+
|
|
923
|
+
try:
|
|
924
|
+
wheel_raw = read_bounded_document(path, maximum=MAX_WHEEL_BYTES, label="installed_wheel")
|
|
925
|
+
with zipfile.ZipFile(io.BytesIO(wheel_raw)) as archive:
|
|
926
|
+
names = archive.namelist()
|
|
927
|
+
if len(names) != len(set(names)):
|
|
928
|
+
raise GenerationReceiptRefused(
|
|
929
|
+
"GENERATION_EXECUTION_DISTRIBUTION",
|
|
930
|
+
"installed_wheel",
|
|
931
|
+
"wheel carries duplicate member names",
|
|
932
|
+
)
|
|
933
|
+
records = [name for name in names if name.endswith(".dist-info/RECORD")]
|
|
934
|
+
if len(records) != 1 or len(names) > MAX_DISTRIBUTION_FILES:
|
|
935
|
+
raise GenerationReceiptRefused(
|
|
936
|
+
"GENERATION_EXECUTION_DISTRIBUTION",
|
|
937
|
+
"installed_wheel",
|
|
938
|
+
"wheel must carry one bounded distribution RECORD",
|
|
939
|
+
)
|
|
940
|
+
dist_info = records[0].split("/", 1)[0]
|
|
941
|
+
for info in archive.infolist():
|
|
942
|
+
name = info.filename
|
|
943
|
+
member_type = stat.S_IFMT(info.external_attr >> 16)
|
|
944
|
+
if (
|
|
945
|
+
"\\" in name
|
|
946
|
+
or name.startswith("/")
|
|
947
|
+
or "//" in name
|
|
948
|
+
or any(part in {".", ".."} for part in name.split("/"))
|
|
949
|
+
or not (name.startswith("mostlyright/") or name.startswith(f"{dist_info}/"))
|
|
950
|
+
or member_type not in {0, stat.S_IFREG, stat.S_IFDIR}
|
|
951
|
+
or bool(info.flag_bits & 0x1)
|
|
952
|
+
):
|
|
953
|
+
raise GenerationReceiptRefused(
|
|
954
|
+
"GENERATION_EXECUTION_DISTRIBUTION",
|
|
955
|
+
"installed_wheel",
|
|
956
|
+
"wheel carries a payload outside its closed installed roots",
|
|
957
|
+
)
|
|
958
|
+
metadata_name = f"{dist_info}/METADATA"
|
|
959
|
+
version = ""
|
|
960
|
+
if metadata_name in names:
|
|
961
|
+
for line in archive.read(metadata_name).decode("utf-8").splitlines():
|
|
962
|
+
if line.startswith("Version: "):
|
|
963
|
+
version = line.removeprefix("Version: ").strip()
|
|
964
|
+
break
|
|
965
|
+
if not version:
|
|
966
|
+
raise GenerationReceiptRefused(
|
|
967
|
+
"GENERATION_EXECUTION_DISTRIBUTION",
|
|
968
|
+
"installed_wheel",
|
|
969
|
+
"wheel metadata states no distribution version",
|
|
970
|
+
)
|
|
971
|
+
selected_infos: list[zipfile.ZipInfo] = []
|
|
972
|
+
total_bytes = 0
|
|
973
|
+
total_compressed_bytes = 0
|
|
974
|
+
for name in names:
|
|
975
|
+
if not _distribution_member(name, dist_info):
|
|
976
|
+
continue
|
|
977
|
+
info = archive.getinfo(name)
|
|
978
|
+
if info.is_dir() or info.file_size > MAX_DISTRIBUTION_MEMBER_BYTES:
|
|
979
|
+
raise GenerationReceiptRefused(
|
|
980
|
+
"GENERATION_EXECUTION_DISTRIBUTION",
|
|
981
|
+
"installed_wheel",
|
|
982
|
+
"wheel distribution member is not a bounded regular payload",
|
|
983
|
+
)
|
|
984
|
+
total_bytes += info.file_size
|
|
985
|
+
total_compressed_bytes += info.compress_size
|
|
986
|
+
selected_infos.append(info)
|
|
987
|
+
if (
|
|
988
|
+
total_bytes > MAX_DISTRIBUTION_TOTAL_BYTES
|
|
989
|
+
or total_compressed_bytes > MAX_DISTRIBUTION_COMPRESSED_BYTES
|
|
990
|
+
or total_bytes
|
|
991
|
+
> max(
|
|
992
|
+
ARCHIVE_EXPANSION_FLOOR_BYTES,
|
|
993
|
+
total_compressed_bytes * ARCHIVE_EXPANSION_RATIO,
|
|
994
|
+
)
|
|
995
|
+
):
|
|
996
|
+
raise GenerationReceiptRefused(
|
|
997
|
+
"GENERATION_EXECUTION_DISTRIBUTION",
|
|
998
|
+
"installed_wheel",
|
|
999
|
+
"wheel distribution exceeds its aggregate expansion budget",
|
|
1000
|
+
)
|
|
1001
|
+
selected = [
|
|
1002
|
+
{"path": info.filename, "sha256": sha256_bytes(archive.read(info))}
|
|
1003
|
+
for info in selected_infos
|
|
1004
|
+
]
|
|
1005
|
+
if expected_product_sha256s is not None:
|
|
1006
|
+
actual_product_sha256s = {
|
|
1007
|
+
info.filename: sha256_bytes(archive.read(info))
|
|
1008
|
+
for info in selected_infos
|
|
1009
|
+
if info.filename.startswith("mostlyright/")
|
|
1010
|
+
}
|
|
1011
|
+
if not expected_product_sha256s or actual_product_sha256s != dict(
|
|
1012
|
+
expected_product_sha256s
|
|
1013
|
+
):
|
|
1014
|
+
raise GenerationReceiptRefused(
|
|
1015
|
+
"GENERATION_EXECUTION_SOURCE",
|
|
1016
|
+
"installed_wheel",
|
|
1017
|
+
"wheel product bytes differ from the committed source coordinate",
|
|
1018
|
+
)
|
|
1019
|
+
except GenerationReceiptRefused:
|
|
1020
|
+
raise
|
|
1021
|
+
except (OSError, zipfile.BadZipFile, UnicodeError) as error:
|
|
1022
|
+
raise GenerationReceiptRefused(
|
|
1023
|
+
"GENERATION_EXECUTION_DISTRIBUTION",
|
|
1024
|
+
"installed_wheel",
|
|
1025
|
+
"wheel is not a readable installed distribution",
|
|
1026
|
+
) from error
|
|
1027
|
+
return {
|
|
1028
|
+
"wheel_sha256": hashlib.sha256(wheel_raw).hexdigest(),
|
|
1029
|
+
**_distribution_coordinate(selected, version),
|
|
1030
|
+
}
|
|
1031
|
+
|
|
1032
|
+
|
|
1033
|
+
def installed_distribution_coordinate() -> dict[str, Any]:
|
|
1034
|
+
"""Fingerprint the wheel-owned bytes and refuse an editable/source-tree import."""
|
|
1035
|
+
|
|
1036
|
+
try:
|
|
1037
|
+
distribution = metadata.distribution("mostlyright-data")
|
|
1038
|
+
root = Path(distribution.locate_file("")).resolve(strict=True)
|
|
1039
|
+
origin = Path(__file__).resolve(strict=True)
|
|
1040
|
+
origin.relative_to(root)
|
|
1041
|
+
if not ({"site-packages", "dist-packages"} & set(root.parts)):
|
|
1042
|
+
raise ValueError
|
|
1043
|
+
paths = [item.as_posix() for item in distribution.files or ()]
|
|
1044
|
+
records = [path for path in paths if path.endswith(".dist-info/RECORD")]
|
|
1045
|
+
if len(records) != 1:
|
|
1046
|
+
raise ValueError
|
|
1047
|
+
dist_info = records[0].split("/", 1)[0]
|
|
1048
|
+
selected: list[dict[str, str]] = []
|
|
1049
|
+
total_bytes = 0
|
|
1050
|
+
for name in paths:
|
|
1051
|
+
if not _distribution_member(name, dist_info):
|
|
1052
|
+
continue
|
|
1053
|
+
member = Path(distribution.locate_file(name))
|
|
1054
|
+
try:
|
|
1055
|
+
member.resolve(strict=True).relative_to(root)
|
|
1056
|
+
except (OSError, ValueError):
|
|
1057
|
+
raise GenerationReceiptRefused(
|
|
1058
|
+
"GENERATION_EXECUTION_DISTRIBUTION",
|
|
1059
|
+
"installed_runtime.distribution",
|
|
1060
|
+
"installed distribution member escapes its package root",
|
|
1061
|
+
) from None
|
|
1062
|
+
raw = _read_installed_distribution_member(
|
|
1063
|
+
member,
|
|
1064
|
+
maximum=MAX_DISTRIBUTION_MEMBER_BYTES,
|
|
1065
|
+
label="installed_distribution.member",
|
|
1066
|
+
)
|
|
1067
|
+
total_bytes += len(raw)
|
|
1068
|
+
if total_bytes > MAX_DISTRIBUTION_TOTAL_BYTES:
|
|
1069
|
+
raise GenerationReceiptRefused(
|
|
1070
|
+
"GENERATION_EXECUTION_DISTRIBUTION",
|
|
1071
|
+
"installed_runtime.distribution",
|
|
1072
|
+
"installed distribution exceeds its aggregate byte budget",
|
|
1073
|
+
)
|
|
1074
|
+
selected.append({"path": name, "sha256": sha256_bytes(raw)})
|
|
1075
|
+
except (OSError, ValueError, metadata.PackageNotFoundError) as error:
|
|
1076
|
+
raise GenerationReceiptRefused(
|
|
1077
|
+
"GENERATION_EXECUTION_DISTRIBUTION",
|
|
1078
|
+
"installed_runtime.distribution",
|
|
1079
|
+
"catalog execution must originate from one installed wheel distribution",
|
|
1080
|
+
) from error
|
|
1081
|
+
return _distribution_coordinate(selected, distribution.version)
|
|
1082
|
+
|
|
1083
|
+
|
|
1084
|
+
def root_coordinate(root: Path) -> str:
|
|
1085
|
+
"""One output root's coordinate: the digest of its resolved name, never the name itself.
|
|
1086
|
+
|
|
1087
|
+
A receipt binds the root it describes so that a resume cannot be pointed at another one, and a
|
|
1088
|
+
digest binds it without writing an executor-local absolute path into durable evidence.
|
|
1089
|
+
"""
|
|
1090
|
+
|
|
1091
|
+
return canonical_sha256({"output_root": str(_absolute_path(root))})
|
|
1092
|
+
|
|
1093
|
+
|
|
1094
|
+
# ------------------------------------------------------------------------------------------------
|
|
1095
|
+
# Installing a receipt, and replacing exactly one typed-incomplete predecessor
|
|
1096
|
+
# ------------------------------------------------------------------------------------------------
|
|
1097
|
+
|
|
1098
|
+
|
|
1099
|
+
def install_receipt(
|
|
1100
|
+
path: Path,
|
|
1101
|
+
payload: Mapping[str, Any],
|
|
1102
|
+
*,
|
|
1103
|
+
maximum: int,
|
|
1104
|
+
label: str,
|
|
1105
|
+
code_prefix: str,
|
|
1106
|
+
refusal: type[SourceContractError] = GenerationReceiptRefused,
|
|
1107
|
+
replaces: Mapping[str, Any] | None = None,
|
|
1108
|
+
installation: ReceiptInstallation | None = None,
|
|
1109
|
+
commit_guard: Callable[[], None] | None = None,
|
|
1110
|
+
) -> bytes:
|
|
1111
|
+
"""Install one complete receipt with append-only failure handling.
|
|
1112
|
+
|
|
1113
|
+
Every invocation first fsyncs the complete bytes under a unique private name. Publication is
|
|
1114
|
+
one atomic no-replace rename, so the authoritative name is absent or complete. Replacement
|
|
1115
|
+
first moves the exact authorized predecessor into immutable history. A failed post-install
|
|
1116
|
+
guard moves the visible bytes to a unique quarantine and restores the predecessor without
|
|
1117
|
+
overwriting or deleting any name. Private remnants are never authoritative and never block a
|
|
1118
|
+
retry. The final descriptor/name identity check is the last filesystem action before success.
|
|
1119
|
+
"""
|
|
1120
|
+
|
|
1121
|
+
destination = _absolute_path(path)
|
|
1122
|
+
if installation is None:
|
|
1123
|
+
installation = next(
|
|
1124
|
+
(
|
|
1125
|
+
active
|
|
1126
|
+
for active in reversed(_ACTIVE_RECEIPT_INSTALLATIONS.get())
|
|
1127
|
+
if active.destination == destination
|
|
1128
|
+
),
|
|
1129
|
+
None,
|
|
1130
|
+
)
|
|
1131
|
+
if installation is None:
|
|
1132
|
+
with receipt_installation_lock(
|
|
1133
|
+
destination, label=label, code_prefix=code_prefix, refusal=refusal
|
|
1134
|
+
) as retained_parent:
|
|
1135
|
+
return install_receipt(
|
|
1136
|
+
destination,
|
|
1137
|
+
payload,
|
|
1138
|
+
maximum=maximum,
|
|
1139
|
+
label=label,
|
|
1140
|
+
code_prefix=code_prefix,
|
|
1141
|
+
refusal=refusal,
|
|
1142
|
+
replaces=replaces,
|
|
1143
|
+
installation=retained_parent,
|
|
1144
|
+
commit_guard=commit_guard,
|
|
1145
|
+
)
|
|
1146
|
+
if installation.destination != destination:
|
|
1147
|
+
raise refusal(
|
|
1148
|
+
f"{code_prefix}_RECEIPT_DESTINATION",
|
|
1149
|
+
label,
|
|
1150
|
+
"receipt installation does not own this destination",
|
|
1151
|
+
)
|
|
1152
|
+
|
|
1153
|
+
raw = canonical_json_bytes(payload)
|
|
1154
|
+
if len(raw) > maximum:
|
|
1155
|
+
raise refusal(f"{code_prefix}_RECEIPT_LIMIT", label, "receipt exceeds its byte bound")
|
|
1156
|
+
parent_descriptor = installation.parent_descriptor
|
|
1157
|
+
name = installation.name
|
|
1158
|
+
staged = _private_receipt_name(name, "staging", sha256_bytes(raw))
|
|
1159
|
+
retained_name: str | None = None
|
|
1160
|
+
predecessor_source: str | None = None
|
|
1161
|
+
predecessor_claimed = False
|
|
1162
|
+
published = False
|
|
1163
|
+
installed_identity: os.stat_result | None = None
|
|
1164
|
+
try:
|
|
1165
|
+
_require_receipt_parent_identity(
|
|
1166
|
+
installation, label=label, code_prefix=code_prefix, refusal=refusal
|
|
1167
|
+
)
|
|
1168
|
+
flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0)
|
|
1169
|
+
descriptor = os.open(staged, flags, 0o600, dir_fd=parent_descriptor)
|
|
1170
|
+
_write_all(descriptor, raw)
|
|
1171
|
+
staged_raw, staged_identity = _read_bounded_at(parent_descriptor, staged, maximum=maximum)
|
|
1172
|
+
staged_named_identity = os.stat(staged, dir_fd=parent_descriptor, follow_symlinks=False)
|
|
1173
|
+
if staged_raw != raw or not _same_identity(staged_identity, staged_named_identity):
|
|
1174
|
+
raise refusal(
|
|
1175
|
+
f"{code_prefix}_RECEIPT_READBACK",
|
|
1176
|
+
label,
|
|
1177
|
+
"private receipt staging changed before publication",
|
|
1178
|
+
)
|
|
1179
|
+
if commit_guard is not None:
|
|
1180
|
+
commit_guard()
|
|
1181
|
+
if replaces is not None:
|
|
1182
|
+
expected = replaces.get("receipt_sha256")
|
|
1183
|
+
if not is_digest(expected):
|
|
1184
|
+
raise refusal(
|
|
1185
|
+
f"{code_prefix}_RESUME_RECEIPT",
|
|
1186
|
+
label,
|
|
1187
|
+
"the retained receipt has no exact fingerprint",
|
|
1188
|
+
)
|
|
1189
|
+
retained_name = f".{name}.retained.{expected}"
|
|
1190
|
+
current, current_identity = _read_bounded_at(parent_descriptor, name, maximum=maximum)
|
|
1191
|
+
if sha256_bytes(current) != expected:
|
|
1192
|
+
raise refusal(
|
|
1193
|
+
f"{code_prefix}_RESUME_RECEIPT",
|
|
1194
|
+
label,
|
|
1195
|
+
"the retained receipt changed before replacement",
|
|
1196
|
+
)
|
|
1197
|
+
named_identity = os.stat(name, dir_fd=parent_descriptor, follow_symlinks=False)
|
|
1198
|
+
if not _same_identity(current_identity, named_identity):
|
|
1199
|
+
raise refusal(
|
|
1200
|
+
f"{code_prefix}_RESUME_RECEIPT",
|
|
1201
|
+
label,
|
|
1202
|
+
"the retained receipt changed before replacement",
|
|
1203
|
+
)
|
|
1204
|
+
_require_receipt_parent_identity(
|
|
1205
|
+
installation, label=label, code_prefix=code_prefix, refusal=refusal
|
|
1206
|
+
)
|
|
1207
|
+
claim_name = _private_receipt_name(name, "predecessor", expected)
|
|
1208
|
+
_rename_noreplace(parent_descriptor, name, claim_name)
|
|
1209
|
+
predecessor_claimed = True
|
|
1210
|
+
predecessor_source = claim_name
|
|
1211
|
+
os.fsync(parent_descriptor)
|
|
1212
|
+
claimed, claimed_identity = _read_bounded_at(
|
|
1213
|
+
parent_descriptor, claim_name, maximum=maximum
|
|
1214
|
+
)
|
|
1215
|
+
claimed_named_identity = os.stat(
|
|
1216
|
+
claim_name, dir_fd=parent_descriptor, follow_symlinks=False
|
|
1217
|
+
)
|
|
1218
|
+
if (
|
|
1219
|
+
claimed != current
|
|
1220
|
+
or sha256_bytes(claimed) != expected
|
|
1221
|
+
or (current_identity.st_dev, current_identity.st_ino)
|
|
1222
|
+
!= (claimed_identity.st_dev, claimed_identity.st_ino)
|
|
1223
|
+
or not _same_identity(claimed_identity, claimed_named_identity)
|
|
1224
|
+
):
|
|
1225
|
+
raise refusal(
|
|
1226
|
+
f"{code_prefix}_RESUME_RECEIPT",
|
|
1227
|
+
label,
|
|
1228
|
+
"the retained receipt changed before replacement",
|
|
1229
|
+
)
|
|
1230
|
+
try:
|
|
1231
|
+
_rename_noreplace(parent_descriptor, claim_name, retained_name)
|
|
1232
|
+
predecessor_source = retained_name
|
|
1233
|
+
os.fsync(parent_descriptor)
|
|
1234
|
+
except OSError as error:
|
|
1235
|
+
if error.errno != errno.EEXIST:
|
|
1236
|
+
raise
|
|
1237
|
+
archived, archived_identity = _read_bounded_at(
|
|
1238
|
+
parent_descriptor, retained_name, maximum=maximum
|
|
1239
|
+
)
|
|
1240
|
+
archived_named_identity = os.stat(
|
|
1241
|
+
retained_name, dir_fd=parent_descriptor, follow_symlinks=False
|
|
1242
|
+
)
|
|
1243
|
+
if (
|
|
1244
|
+
archived != current
|
|
1245
|
+
or sha256_bytes(archived) != expected
|
|
1246
|
+
or not _same_identity(archived_identity, archived_named_identity)
|
|
1247
|
+
):
|
|
1248
|
+
raise refusal(
|
|
1249
|
+
f"{code_prefix}_RESUME_RECEIPT",
|
|
1250
|
+
label,
|
|
1251
|
+
"the retained receipt archive differs from the authorized predecessor",
|
|
1252
|
+
)
|
|
1253
|
+
_require_receipt_parent_identity(
|
|
1254
|
+
installation, label=label, code_prefix=code_prefix, refusal=refusal
|
|
1255
|
+
)
|
|
1256
|
+
_rename_noreplace(parent_descriptor, staged, name)
|
|
1257
|
+
published = True
|
|
1258
|
+
os.fsync(parent_descriptor)
|
|
1259
|
+
installed, installed_identity = _read_bounded_at(parent_descriptor, name, maximum=maximum)
|
|
1260
|
+
if installed != raw:
|
|
1261
|
+
raise refusal(
|
|
1262
|
+
f"{code_prefix}_RECEIPT_READBACK", label, "installed receipt readback differs"
|
|
1263
|
+
)
|
|
1264
|
+
_require_receipt_parent_identity(
|
|
1265
|
+
installation, label=label, code_prefix=code_prefix, refusal=refusal
|
|
1266
|
+
)
|
|
1267
|
+
if commit_guard is not None:
|
|
1268
|
+
commit_guard()
|
|
1269
|
+
final, final_identity = _read_bounded_at(parent_descriptor, name, maximum=maximum)
|
|
1270
|
+
if (
|
|
1271
|
+
final != raw
|
|
1272
|
+
or installed_identity is None
|
|
1273
|
+
or not _same_identity(installed_identity, final_identity)
|
|
1274
|
+
):
|
|
1275
|
+
raise refusal(
|
|
1276
|
+
f"{code_prefix}_RECEIPT_READBACK",
|
|
1277
|
+
label,
|
|
1278
|
+
"installed receipt changed before commit completed",
|
|
1279
|
+
)
|
|
1280
|
+
final_named_identity = os.stat(name, dir_fd=parent_descriptor, follow_symlinks=False)
|
|
1281
|
+
if not _same_identity(final_identity, final_named_identity):
|
|
1282
|
+
raise refusal(
|
|
1283
|
+
f"{code_prefix}_RECEIPT_READBACK",
|
|
1284
|
+
label,
|
|
1285
|
+
"installed receipt changed before commit completed",
|
|
1286
|
+
)
|
|
1287
|
+
_require_receipt_parent_identity(
|
|
1288
|
+
installation, label=label, code_prefix=code_prefix, refusal=refusal
|
|
1289
|
+
)
|
|
1290
|
+
return raw
|
|
1291
|
+
except (OSError, SourceContractError) as error:
|
|
1292
|
+
try:
|
|
1293
|
+
if published:
|
|
1294
|
+
_quarantine_receipt(
|
|
1295
|
+
parent_descriptor,
|
|
1296
|
+
name,
|
|
1297
|
+
maximum=maximum,
|
|
1298
|
+
)
|
|
1299
|
+
if predecessor_claimed:
|
|
1300
|
+
assert predecessor_source is not None
|
|
1301
|
+
_restore_receipt_from_history(
|
|
1302
|
+
parent_descriptor,
|
|
1303
|
+
predecessor_source,
|
|
1304
|
+
name,
|
|
1305
|
+
maximum=maximum,
|
|
1306
|
+
)
|
|
1307
|
+
except OSError as recovery_error:
|
|
1308
|
+
code = (
|
|
1309
|
+
f"{code_prefix}_RESUME_RECEIPT"
|
|
1310
|
+
if replaces is not None
|
|
1311
|
+
else f"{code_prefix}_RECEIPT_DESTINATION"
|
|
1312
|
+
)
|
|
1313
|
+
raise refusal(
|
|
1314
|
+
code,
|
|
1315
|
+
label,
|
|
1316
|
+
"receipt publication failed and its prior visible state could not be restored",
|
|
1317
|
+
) from recovery_error
|
|
1318
|
+
if isinstance(error, SourceContractError):
|
|
1319
|
+
raise
|
|
1320
|
+
code = (
|
|
1321
|
+
f"{code_prefix}_RESUME_RECEIPT"
|
|
1322
|
+
if replaces is not None
|
|
1323
|
+
else f"{code_prefix}_RECEIPT_DESTINATION"
|
|
1324
|
+
)
|
|
1325
|
+
detail = (
|
|
1326
|
+
"the retained receipt changed before replacement"
|
|
1327
|
+
if replaces is not None
|
|
1328
|
+
else "receipt destination must be an absent no-follow path"
|
|
1329
|
+
)
|
|
1330
|
+
raise refusal(code, label, detail) from error
|
|
1331
|
+
|
|
1332
|
+
|
|
1333
|
+
def _private_receipt_name(destination: str, role: str, digest: str) -> str:
|
|
1334
|
+
"""Return an invocation-private, non-authoritative member name."""
|
|
1335
|
+
|
|
1336
|
+
return f".{destination}.{role}.{digest}.{os.urandom(16).hex()}"
|
|
1337
|
+
|
|
1338
|
+
|
|
1339
|
+
def _quarantine_receipt(parent: int, destination: str, *, maximum: int) -> str | None:
|
|
1340
|
+
"""Move the current public name into append-only quarantine without deleting its bytes."""
|
|
1341
|
+
|
|
1342
|
+
quarantine = _private_receipt_name(destination, "quarantine", os.urandom(32).hex())
|
|
1343
|
+
try:
|
|
1344
|
+
_rename_noreplace(parent, destination, quarantine)
|
|
1345
|
+
except FileNotFoundError:
|
|
1346
|
+
return None
|
|
1347
|
+
os.fsync(parent)
|
|
1348
|
+
_raw, identity = _read_bounded_at(parent, quarantine, maximum=maximum)
|
|
1349
|
+
named_identity = os.stat(quarantine, dir_fd=parent, follow_symlinks=False)
|
|
1350
|
+
if not _same_identity(identity, named_identity):
|
|
1351
|
+
raise OSError(errno.EIO, "quarantined receipt changed")
|
|
1352
|
+
return quarantine
|
|
1353
|
+
|
|
1354
|
+
|
|
1355
|
+
def _restore_receipt_from_history(
|
|
1356
|
+
parent: int,
|
|
1357
|
+
source: str,
|
|
1358
|
+
destination: str,
|
|
1359
|
+
*,
|
|
1360
|
+
maximum: int,
|
|
1361
|
+
) -> None:
|
|
1362
|
+
"""Copy immutable predecessor bytes through a private stage and restore them no-replace."""
|
|
1363
|
+
|
|
1364
|
+
raw, source_identity = _read_bounded_at(parent, source, maximum=maximum)
|
|
1365
|
+
source_named_identity = os.stat(source, dir_fd=parent, follow_symlinks=False)
|
|
1366
|
+
if not _same_identity(source_identity, source_named_identity):
|
|
1367
|
+
raise OSError(errno.EIO, "receipt history changed")
|
|
1368
|
+
staged = _private_receipt_name(destination, "restoring", sha256_bytes(raw))
|
|
1369
|
+
flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0)
|
|
1370
|
+
descriptor = os.open(staged, flags, 0o600, dir_fd=parent)
|
|
1371
|
+
_write_all(descriptor, raw)
|
|
1372
|
+
staged_raw, staged_identity = _read_bounded_at(parent, staged, maximum=maximum)
|
|
1373
|
+
staged_named_identity = os.stat(staged, dir_fd=parent, follow_symlinks=False)
|
|
1374
|
+
if staged_raw != raw or not _same_identity(staged_identity, staged_named_identity):
|
|
1375
|
+
raise OSError(errno.EIO, "receipt restoration stage changed")
|
|
1376
|
+
_rename_noreplace(parent, staged, destination)
|
|
1377
|
+
os.fsync(parent)
|
|
1378
|
+
restored, restored_identity = _read_bounded_at(parent, destination, maximum=maximum)
|
|
1379
|
+
restored_named_identity = os.stat(destination, dir_fd=parent, follow_symlinks=False)
|
|
1380
|
+
if restored != raw or not _same_identity(restored_identity, restored_named_identity):
|
|
1381
|
+
raise OSError(errno.EIO, "restored receipt changed")
|
|
1382
|
+
|
|
1383
|
+
|
|
1384
|
+
def _rename_noreplace(parent: int, source: str, destination: str) -> None:
|
|
1385
|
+
"""Rename one regular member only when the destination is absent."""
|
|
1386
|
+
|
|
1387
|
+
library = ctypes.CDLL(None, use_errno=True)
|
|
1388
|
+
source_bytes = os.fsencode(source)
|
|
1389
|
+
destination_bytes = os.fsencode(destination)
|
|
1390
|
+
if hasattr(library, "renameat2"):
|
|
1391
|
+
rename = library.renameat2
|
|
1392
|
+
rename.argtypes = [
|
|
1393
|
+
ctypes.c_int,
|
|
1394
|
+
ctypes.c_char_p,
|
|
1395
|
+
ctypes.c_int,
|
|
1396
|
+
ctypes.c_char_p,
|
|
1397
|
+
ctypes.c_uint,
|
|
1398
|
+
]
|
|
1399
|
+
rename.restype = ctypes.c_int
|
|
1400
|
+
result = rename(parent, source_bytes, parent, destination_bytes, 1)
|
|
1401
|
+
elif hasattr(library, "renameatx_np"):
|
|
1402
|
+
rename = library.renameatx_np
|
|
1403
|
+
rename.argtypes = [
|
|
1404
|
+
ctypes.c_int,
|
|
1405
|
+
ctypes.c_char_p,
|
|
1406
|
+
ctypes.c_int,
|
|
1407
|
+
ctypes.c_char_p,
|
|
1408
|
+
ctypes.c_uint,
|
|
1409
|
+
]
|
|
1410
|
+
rename.restype = ctypes.c_int
|
|
1411
|
+
result = rename(parent, source_bytes, parent, destination_bytes, 4)
|
|
1412
|
+
else:
|
|
1413
|
+
raise OSError(errno.ENOTSUP, "atomic no-replace rename is unavailable")
|
|
1414
|
+
if result != 0:
|
|
1415
|
+
value = ctypes.get_errno()
|
|
1416
|
+
raise OSError(value, os.strerror(value), destination)
|
|
1417
|
+
|
|
1418
|
+
|
|
1419
|
+
def _read_bounded_at(parent: int, name: str, *, maximum: int) -> tuple[bytes, os.stat_result]:
|
|
1420
|
+
descriptor = os.open(name, _OPEN_MEMBER, dir_fd=parent)
|
|
1421
|
+
try:
|
|
1422
|
+
before = os.fstat(descriptor)
|
|
1423
|
+
if not stat.S_ISREG(before.st_mode) or before.st_nlink != 1 or before.st_size > maximum:
|
|
1424
|
+
raise OSError
|
|
1425
|
+
chunks: list[bytes] = []
|
|
1426
|
+
remaining = maximum + 1
|
|
1427
|
+
while remaining > 0:
|
|
1428
|
+
chunk = os.read(descriptor, min(remaining, 1 << 16))
|
|
1429
|
+
if not chunk:
|
|
1430
|
+
break
|
|
1431
|
+
chunks.append(chunk)
|
|
1432
|
+
remaining -= len(chunk)
|
|
1433
|
+
after = os.fstat(descriptor)
|
|
1434
|
+
if not _same_identity(before, after):
|
|
1435
|
+
raise OSError
|
|
1436
|
+
raw = b"".join(chunks)
|
|
1437
|
+
if len(raw) > maximum:
|
|
1438
|
+
raise OSError
|
|
1439
|
+
return raw, after
|
|
1440
|
+
finally:
|
|
1441
|
+
os.close(descriptor)
|
|
1442
|
+
|
|
1443
|
+
|
|
1444
|
+
def _write_all(descriptor: int, raw: bytes) -> None:
|
|
1445
|
+
try:
|
|
1446
|
+
written = 0
|
|
1447
|
+
while written < len(raw):
|
|
1448
|
+
count = os.write(descriptor, raw[written:])
|
|
1449
|
+
if count <= 0:
|
|
1450
|
+
raise OSError(errno.EIO, "receipt write made no progress")
|
|
1451
|
+
written += count
|
|
1452
|
+
os.fsync(descriptor)
|
|
1453
|
+
finally:
|
|
1454
|
+
os.close(descriptor)
|
|
1455
|
+
|
|
1456
|
+
|
|
1457
|
+
def retained_incomplete_receipt(
|
|
1458
|
+
path: Path,
|
|
1459
|
+
*,
|
|
1460
|
+
maximum: int,
|
|
1461
|
+
label: str,
|
|
1462
|
+
schema: str,
|
|
1463
|
+
incomplete_status: str,
|
|
1464
|
+
output_root: Path,
|
|
1465
|
+
code_prefix: str,
|
|
1466
|
+
refusal: type[SourceContractError] = GenerationReceiptRefused,
|
|
1467
|
+
) -> dict[str, Any] | None:
|
|
1468
|
+
"""The typed-incomplete receipt a resume is allowed to replace, or ``None`` if there is none.
|
|
1469
|
+
|
|
1470
|
+
A resume that finds nothing at the destination is an ordinary first write. A resume that
|
|
1471
|
+
finds a receipt which is complete, written under another schema, or bound to another output
|
|
1472
|
+
root is refused rather than overwritten: replacing evidence is only ever allowed to continue
|
|
1473
|
+
the exact interrupted invocation that wrote it.
|
|
1474
|
+
"""
|
|
1475
|
+
|
|
1476
|
+
try:
|
|
1477
|
+
raw = read_bounded_document(Path(path), maximum=maximum, label=label)
|
|
1478
|
+
except GenerationReceiptRefused as error:
|
|
1479
|
+
if error.code == "GENERATION_INPUT_PATH" and not Path(path).exists():
|
|
1480
|
+
return None
|
|
1481
|
+
raise
|
|
1482
|
+
try:
|
|
1483
|
+
payload = parse_canonical_json(raw)
|
|
1484
|
+
except CanonicalJSONError as error:
|
|
1485
|
+
raise refusal(
|
|
1486
|
+
f"{code_prefix}_RESUME_RECEIPT", label, "retained receipt is not canonical JSON"
|
|
1487
|
+
) from error
|
|
1488
|
+
if (
|
|
1489
|
+
not isinstance(payload, dict)
|
|
1490
|
+
or payload.get("schema_version") != schema
|
|
1491
|
+
or payload.get("status") != incomplete_status
|
|
1492
|
+
):
|
|
1493
|
+
raise refusal(
|
|
1494
|
+
f"{code_prefix}_RECEIPT_DESTINATION",
|
|
1495
|
+
label,
|
|
1496
|
+
"only this stage's own typed-incomplete receipt may be replaced",
|
|
1497
|
+
)
|
|
1498
|
+
if payload.get("output_root_sha256") != root_coordinate(output_root):
|
|
1499
|
+
raise refusal(
|
|
1500
|
+
f"{code_prefix}_RESUME_RECEIPT",
|
|
1501
|
+
label,
|
|
1502
|
+
"the retained typed-incomplete receipt describes another output root",
|
|
1503
|
+
)
|
|
1504
|
+
return {**payload, "receipt_sha256": sha256_bytes(raw)}
|
|
1505
|
+
|
|
1506
|
+
|
|
1507
|
+
# ------------------------------------------------------------------------------------------------
|
|
1508
|
+
# The composite receipt
|
|
1509
|
+
# ------------------------------------------------------------------------------------------------
|
|
1510
|
+
|
|
1511
|
+
|
|
1512
|
+
def compose_generation_receipt(body: Mapping[str, Any]) -> dict[str, Any]:
|
|
1513
|
+
"""Seal one composite receipt: shape-check it, then bind it under its own coordinate."""
|
|
1514
|
+
|
|
1515
|
+
payload = {key: value for key, value in body.items() if key != "coordinate_sha256"}
|
|
1516
|
+
enforce_shape(
|
|
1517
|
+
payload,
|
|
1518
|
+
label="generation_receipt",
|
|
1519
|
+
max_depth=MAX_COMPOSITE_RECEIPT_DEPTH,
|
|
1520
|
+
max_members=MAX_COMPOSITE_ARRAY_ITEMS,
|
|
1521
|
+
max_items=MAX_COMPOSITE_ARRAY_ITEMS,
|
|
1522
|
+
code="GENERATION_RECEIPT_SHAPE",
|
|
1523
|
+
)
|
|
1524
|
+
receipt = {**payload, "coordinate_sha256": canonical_sha256(payload)}
|
|
1525
|
+
raw = canonical_json_bytes(receipt)
|
|
1526
|
+
if len(raw) > MAX_COMPOSITE_RECEIPT_BYTES:
|
|
1527
|
+
raise GenerationReceiptRefused(
|
|
1528
|
+
"GENERATION_RECEIPT_LIMIT",
|
|
1529
|
+
"generation_receipt",
|
|
1530
|
+
f"the composite receipt is at most {MAX_COMPOSITE_RECEIPT_BYTES} bytes",
|
|
1531
|
+
)
|
|
1532
|
+
return receipt
|
|
1533
|
+
|
|
1534
|
+
|
|
1535
|
+
def verify_generation_receipt(receipt: Mapping[str, Any]) -> None:
|
|
1536
|
+
"""Re-derive one composite receipt against itself: schema, shape, and its own coordinate.
|
|
1537
|
+
|
|
1538
|
+
This is the check that costs no I/O. It says the document in front of you is a whole
|
|
1539
|
+
composite receipt that reproduces its own fingerprint -- not that the generation it describes
|
|
1540
|
+
is still installed, which is what the readback block inside it is for.
|
|
1541
|
+
"""
|
|
1542
|
+
|
|
1543
|
+
if not isinstance(receipt, Mapping):
|
|
1544
|
+
raise GenerationReceiptRefused(
|
|
1545
|
+
"GENERATION_RECEIPT_SCHEMA", "generation_receipt", "must be one object"
|
|
1546
|
+
)
|
|
1547
|
+
if receipt.get("schema_version") != GENERATION_RECEIPT_SCHEMA:
|
|
1548
|
+
raise GenerationReceiptRefused(
|
|
1549
|
+
"GENERATION_RECEIPT_SCHEMA", "generation_receipt", "receipt names another schema"
|
|
1550
|
+
)
|
|
1551
|
+
for member in (
|
|
1552
|
+
"status",
|
|
1553
|
+
"components",
|
|
1554
|
+
"execution",
|
|
1555
|
+
"staging",
|
|
1556
|
+
"policy",
|
|
1557
|
+
"author",
|
|
1558
|
+
"classification",
|
|
1559
|
+
"publication",
|
|
1560
|
+
"readback",
|
|
1561
|
+
"limits",
|
|
1562
|
+
"coordinate_sha256",
|
|
1563
|
+
):
|
|
1564
|
+
if member not in receipt:
|
|
1565
|
+
raise GenerationReceiptRefused(
|
|
1566
|
+
"GENERATION_RECEIPT_SCHEMA",
|
|
1567
|
+
f"generation_receipt.{member}",
|
|
1568
|
+
"a composite receipt states every one of its blocks",
|
|
1569
|
+
)
|
|
1570
|
+
body = {key: value for key, value in receipt.items() if key != "coordinate_sha256"}
|
|
1571
|
+
enforce_shape(
|
|
1572
|
+
body,
|
|
1573
|
+
label="generation_receipt",
|
|
1574
|
+
max_depth=MAX_COMPOSITE_RECEIPT_DEPTH,
|
|
1575
|
+
max_members=MAX_COMPOSITE_ARRAY_ITEMS,
|
|
1576
|
+
max_items=MAX_COMPOSITE_ARRAY_ITEMS,
|
|
1577
|
+
code="GENERATION_RECEIPT_SHAPE",
|
|
1578
|
+
)
|
|
1579
|
+
components = receipt["components"]
|
|
1580
|
+
for stage in ("harvest", "author"):
|
|
1581
|
+
embedded = components.get(f"{stage}_receipt")
|
|
1582
|
+
stated = components.get(f"{stage}_receipt_sha256")
|
|
1583
|
+
if not isinstance(embedded, Mapping) or not is_digest(stated):
|
|
1584
|
+
raise GenerationReceiptRefused(
|
|
1585
|
+
"GENERATION_RECEIPT_COMPONENT",
|
|
1586
|
+
f"generation_receipt.components.{stage}_receipt",
|
|
1587
|
+
"a composite receipt carries each component receipt and its exact fingerprint",
|
|
1588
|
+
)
|
|
1589
|
+
if sha256_bytes(canonical_json_bytes(embedded)) != stated:
|
|
1590
|
+
raise GenerationReceiptRefused(
|
|
1591
|
+
"GENERATION_RECEIPT_COMPONENT",
|
|
1592
|
+
f"generation_receipt.components.{stage}_receipt",
|
|
1593
|
+
"the carried receipt is not the bytes its fingerprint names",
|
|
1594
|
+
)
|
|
1595
|
+
ranges = receipt["publication"].get("range_manifest_sha256s", ())
|
|
1596
|
+
if not isinstance(ranges, list) or len(ranges) > MAX_COMPOSITE_RANGE_DESCRIPTORS:
|
|
1597
|
+
raise GenerationReceiptRefused(
|
|
1598
|
+
"GENERATION_RECEIPT_SHAPE",
|
|
1599
|
+
"generation_receipt.publication.range_manifest_sha256s",
|
|
1600
|
+
f"a generation states at most {MAX_COMPOSITE_RANGE_DESCRIPTORS} ranges",
|
|
1601
|
+
)
|
|
1602
|
+
if receipt["coordinate_sha256"] != canonical_sha256(body):
|
|
1603
|
+
raise GenerationReceiptRefused(
|
|
1604
|
+
"GENERATION_RECEIPT_COORDINATE",
|
|
1605
|
+
"generation_receipt.coordinate_sha256",
|
|
1606
|
+
"the receipt does not reproduce its own coordinate",
|
|
1607
|
+
)
|