mostlyright-data 0.9.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mostlyright/data_harness/__init__.py +158 -0
- mostlyright/data_harness/acquisition/__init__.py +55 -0
- mostlyright/data_harness/acquisition/http.py +2773 -0
- mostlyright/data_harness/acquisition/parsing.py +809 -0
- mostlyright/data_harness/acquisition/ranges.py +495 -0
- mostlyright/data_harness/acquisition/result_download.py +360 -0
- mostlyright/data_harness/acquisition/retention_admission.py +248 -0
- mostlyright/data_harness/acquisition/sandbox.py +4888 -0
- mostlyright/data_harness/acquisition/url_policy.py +530 -0
- mostlyright/data_harness/agent_runtime.py +2743 -0
- mostlyright/data_harness/assets/logo-ink.svg +31 -0
- mostlyright/data_harness/backends/__init__.py +28 -0
- mostlyright/data_harness/backends/pandas_backend.py +350 -0
- mostlyright/data_harness/backends/polars_backend.py +366 -0
- mostlyright/data_harness/backends/protocol.py +124 -0
- mostlyright/data_harness/backends/reference.py +83 -0
- mostlyright/data_harness/backends/registry.py +55 -0
- mostlyright/data_harness/backends/restrictions.py +126 -0
- mostlyright/data_harness/canonical.py +333 -0
- mostlyright/data_harness/catalog_job.py +625 -0
- mostlyright/data_harness/cli.py +5398 -0
- mostlyright/data_harness/contracts.py +53 -0
- mostlyright/data_harness/coordinator.py +1307 -0
- mostlyright/data_harness/deploy.py +924 -0
- mostlyright/data_harness/deploy_target.py +312 -0
- mostlyright/data_harness/deployment_evidence.py +1067 -0
- mostlyright/data_harness/event_presentation.py +576 -0
- mostlyright/data_harness/events.py +2152 -0
- mostlyright/data_harness/fast_delimited.py +239 -0
- mostlyright/data_harness/fleet.py +237 -0
- mostlyright/data_harness/formats.py +236 -0
- mostlyright/data_harness/governors.py +1163 -0
- mostlyright/data_harness/hosted_bootstrap.py +972 -0
- mostlyright/data_harness/hosted_crawler.py +1115 -0
- mostlyright/data_harness/hosted_crawler_container_smoke.py +351 -0
- mostlyright/data_harness/hosted_crawler_fetch.py +423 -0
- mostlyright/data_harness/hosted_crawler_job.py +1277 -0
- mostlyright/data_harness/hosted_crawler_protocol.py +676 -0
- mostlyright/data_harness/hosted_dataset.py +1500 -0
- mostlyright/data_harness/hosted_deploy.py +3037 -0
- mostlyright/data_harness/hosted_handoff.py +62 -0
- mostlyright/data_harness/hosted_ingestion_contract.py +504 -0
- mostlyright/data_harness/hosted_ingestion_job.py +356 -0
- mostlyright/data_harness/hosted_ingestion_job_smoke.py +40 -0
- mostlyright/data_harness/hosted_session_container_smoke.py +194 -0
- mostlyright/data_harness/hosted_session_worker.py +3554 -0
- mostlyright/data_harness/hosted_session_worker_job_smoke.py +46 -0
- mostlyright/data_harness/hosted_worker.py +6784 -0
- mostlyright/data_harness/ingestion/__init__.py +56 -0
- mostlyright/data_harness/ingestion/contracts.py +461 -0
- mostlyright/data_harness/ingestion/faults.py +42 -0
- mostlyright/data_harness/ingestion/gcs_store.py +1162 -0
- mostlyright/data_harness/ingestion/spool.py +130 -0
- mostlyright/data_harness/ingestion/store.py +885 -0
- mostlyright/data_harness/key_seam.py +434 -0
- mostlyright/data_harness/linux_process_boundary.py +262 -0
- mostlyright/data_harness/local_contracts.py +2880 -0
- mostlyright/data_harness/local_search/__init__.py +5 -0
- mostlyright/data_harness/local_search/build_index.py +1087 -0
- mostlyright/data_harness/local_search/contracts.py +920 -0
- mostlyright/data_harness/local_search/query_trace.py +266 -0
- mostlyright/data_harness/local_search/retrieval.py +700 -0
- mostlyright/data_harness/local_search/sealed.py +474 -0
- mostlyright/data_harness/local_search/service.py +784 -0
- mostlyright/data_harness/nbrender/CONTRACT.md +212 -0
- mostlyright/data_harness/nbrender/__init__.py +12 -0
- mostlyright/data_harness/nbrender/chrome.py +359 -0
- mostlyright/data_harness/nbrender/code_body.py +266 -0
- mostlyright/data_harness/nbrender/document.py +407 -0
- mostlyright/data_harness/nbrender/frame.py +275 -0
- mostlyright/data_harness/nbrender/interactive.py +337 -0
- mostlyright/data_harness/nbrender/markdown_body.py +477 -0
- mostlyright/data_harness/nbrender/mr_components.py +134 -0
- mostlyright/data_harness/nbrender/outputs_data.py +595 -0
- mostlyright/data_harness/nbrender/outputs_rich.py +906 -0
- mostlyright/data_harness/nbrender/outputs_source.py +260 -0
- mostlyright/data_harness/nbrender/outputs_stage.py +176 -0
- mostlyright/data_harness/nbrender/outputs_text.py +400 -0
- mostlyright/data_harness/nbrender/parse.py +394 -0
- mostlyright/data_harness/nbrender/status.py +40 -0
- mostlyright/data_harness/nbrender/tokens.py +1295 -0
- mostlyright/data_harness/notebook.py +1710 -0
- mostlyright/data_harness/offline.py +2049 -0
- mostlyright/data_harness/operation_registry.py +1007 -0
- mostlyright/data_harness/operator_setup.py +239 -0
- mostlyright/data_harness/pipeline.py +6428 -0
- mostlyright/data_harness/plan_graph.py +2026 -0
- mostlyright/data_harness/preparation/__init__.py +104 -0
- mostlyright/data_harness/preparation/contracts.py +1017 -0
- mostlyright/data_harness/preparation/engine.py +221 -0
- mostlyright/data_harness/preparation/errors.py +14 -0
- mostlyright/data_harness/preparation/gates.py +751 -0
- mostlyright/data_harness/preparation/joins.py +574 -0
- mostlyright/data_harness/preparation/profile.py +384 -0
- mostlyright/data_harness/preparation/table.py +217 -0
- mostlyright/data_harness/preparation/transforms.py +568 -0
- mostlyright/data_harness/progress_events.py +534 -0
- mostlyright/data_harness/readers/__init__.py +46 -0
- mostlyright/data_harness/readers/containers.py +963 -0
- mostlyright/data_harness/readers/contracts.py +542 -0
- mostlyright/data_harness/readers/delimited.py +257 -0
- mostlyright/data_harness/readers/grib2/__init__.py +33 -0
- mostlyright/data_harness/readers/grib2/admission.py +722 -0
- mostlyright/data_harness/readers/grib2/decode.py +1009 -0
- mostlyright/data_harness/readers/grib2/geometry.py +1133 -0
- mostlyright/data_harness/readers/grib2/portable_math.py +501 -0
- mostlyright/data_harness/readers/json_tabular.py +485 -0
- mostlyright/data_harness/readers/registry.py +514 -0
- mostlyright/data_harness/readers/samples/README.md +110 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/cities.csv.gz +0 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/cities.csv.gz +0 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/cities.tar +0 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/cities.tar +0 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/cities.zip +0 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/expected.json +25 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/expected.json +25 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/expected.json +25 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/cities.csv +3 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/expected.json +23 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/cities.tsv +5 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/expected.json +25 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/expected.json +30 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/observations.csv +5 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/expected.json +44 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/stations.json +1 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/expected.json +48 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/observations.ndjson +4 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/cities.xlsx +0 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/expected.json +24 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/expected.json +27 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/expected.json +27 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/README.md +20 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/expected.json +55 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/gfs-2m-temperature.grib2 +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/expected.json +54 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/expected.json +54 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/hrrr-categorical-rain.grib2 +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/expected.json +54 -0
- mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
- mostlyright/data_harness/readers/samples.py +582 -0
- mostlyright/data_harness/readers/spreadsheet.py +803 -0
- mostlyright/data_harness/readers/tabular.py +510 -0
- mostlyright/data_harness/recipe.py +5321 -0
- mostlyright/data_harness/repair/__init__.py +78 -0
- mostlyright/data_harness/repair/adapters.py +274 -0
- mostlyright/data_harness/repair/contracts.py +872 -0
- mostlyright/data_harness/repair/coordinator.py +1099 -0
- mostlyright/data_harness/repair/errors.py +16 -0
- mostlyright/data_harness/review.py +2533 -0
- mostlyright/data_harness/rowset.py +283 -0
- mostlyright/data_harness/serving.py +1975 -0
- mostlyright/data_harness/serving_edge.py +590 -0
- mostlyright/data_harness/serving_http.py +1031 -0
- mostlyright/data_harness/session_probes.py +759 -0
- mostlyright/data_harness/signing.py +101 -0
- mostlyright/data_harness/source_discovery.py +898 -0
- mostlyright/data_harness/sources/__init__.py +209 -0
- mostlyright/data_harness/sources/_adapter_steps.py +213 -0
- mostlyright/data_harness/sources/adapters.py +1214 -0
- mostlyright/data_harness/sources/cadence.py +1428 -0
- mostlyright/data_harness/sources/cadence_emission.py +453 -0
- mostlyright/data_harness/sources/cadence_history.py +546 -0
- mostlyright/data_harness/sources/catalog/__init__.py +17 -0
- mostlyright/data_harness/sources/catalog/admission.py +477 -0
- mostlyright/data_harness/sources/catalog/authoring.py +1701 -0
- mostlyright/data_harness/sources/catalog/authoring_policy.py +701 -0
- mostlyright/data_harness/sources/catalog/authoring_shards.py +1217 -0
- mostlyright/data_harness/sources/catalog/bounded_io.py +231 -0
- mostlyright/data_harness/sources/catalog/channel.py +523 -0
- mostlyright/data_harness/sources/catalog/channel_client.py +296 -0
- mostlyright/data_harness/sources/catalog/contracts.py +825 -0
- mostlyright/data_harness/sources/catalog/coverage.py +137 -0
- mostlyright/data_harness/sources/catalog/delta.py +1340 -0
- mostlyright/data_harness/sources/catalog/embedding.py +532 -0
- mostlyright/data_harness/sources/catalog/entry_v2.py +1182 -0
- mostlyright/data_harness/sources/catalog/fill.py +3889 -0
- mostlyright/data_harness/sources/catalog/fill_partitions.py +459 -0
- mostlyright/data_harness/sources/catalog/fill_staging.py +1105 -0
- mostlyright/data_harness/sources/catalog/gating.py +374 -0
- mostlyright/data_harness/sources/catalog/generation_receipt.py +1607 -0
- mostlyright/data_harness/sources/catalog/harvest/__init__.py +7 -0
- mostlyright/data_harness/sources/catalog/harvest/ckan.py +384 -0
- mostlyright/data_harness/sources/catalog/harvest/datagov_v4.py +798 -0
- mostlyright/data_harness/sources/catalog/harvest/protocol.py +964 -0
- mostlyright/data_harness/sources/catalog/harvest/sdmx.py +445 -0
- mostlyright/data_harness/sources/catalog/harvest/stac.py +384 -0
- mostlyright/data_harness/sources/catalog/health.py +447 -0
- mostlyright/data_harness/sources/catalog/hosted_catalog.py +105 -0
- mostlyright/data_harness/sources/catalog/identity_history.py +1549 -0
- mostlyright/data_harness/sources/catalog/neural.py +1618 -0
- mostlyright/data_harness/sources/catalog/packed_catalog.py +2345 -0
- mostlyright/data_harness/sources/catalog/packed_retrieval.py +1517 -0
- mostlyright/data_harness/sources/catalog/packed_writer.py +2802 -0
- mostlyright/data_harness/sources/catalog/query_trace.py +1037 -0
- mostlyright/data_harness/sources/catalog/recommend.py +171 -0
- mostlyright/data_harness/sources/catalog/retrieval.py +230 -0
- mostlyright/data_harness/sources/catalog/retrieval_manifest.py +995 -0
- mostlyright/data_harness/sources/catalog/rights_decisions.py +254 -0
- mostlyright/data_harness/sources/catalog/sealed.py +560 -0
- mostlyright/data_harness/sources/catalog/search.py +230 -0
- mostlyright/data_harness/sources/catalog/streaming_delta.py +1097 -0
- mostlyright/data_harness/sources/catalog/update.py +891 -0
- mostlyright/data_harness/sources/collections.py +815 -0
- mostlyright/data_harness/sources/contracts.py +2223 -0
- mostlyright/data_harness/sources/deletion.py +761 -0
- mostlyright/data_harness/sources/fitness.py +162 -0
- mostlyright/data_harness/sources/governance.py +163 -0
- mostlyright/data_harness/sources/hosted.py +173 -0
- mostlyright/data_harness/sources/integration.py +218 -0
- mostlyright/data_harness/sources/range_reader.py +418 -0
- mostlyright/data_harness/sources/registry.py +514 -0
- mostlyright/data_harness/sources/rights_rule.py +59 -0
- mostlyright/data_harness/sources/source_cadence_vectors.v1.json +1 -0
- mostlyright/data_harness/sources/sports.py +521 -0
- mostlyright/data_harness/sources/stream.py +524 -0
- mostlyright/data_harness/sources/stream_connector.py +418 -0
- mostlyright/data_harness/sources/stream_recorder.py +1404 -0
- mostlyright/data_harness/studio_boundary.py +2019 -0
- mostlyright/data_harness/thin/__init__.py +37 -0
- mostlyright/data_harness/thin/acquire.py +1137 -0
- mostlyright/data_harness/thin/acquire_cancel.py +579 -0
- mostlyright/data_harness/thin/approvals.py +617 -0
- mostlyright/data_harness/thin/commands.py +406 -0
- mostlyright/data_harness/thin/download.py +194 -0
- mostlyright/data_harness/thin/narrative.py +589 -0
- mostlyright/data_harness/thin/parity.py +1070 -0
- mostlyright/data_harness/thin/propose.py +2759 -0
- mostlyright/data_harness/thin/research.py +1663 -0
- mostlyright/data_harness/thin/router.py +924 -0
- mostlyright/data_harness/thin/runs.py +519 -0
- mostlyright/data_harness/thin/session.py +281 -0
- mostlyright/data_harness/thin/stream.py +501 -0
- mostlyright/data_harness/thin/transport.py +187 -0
- mostlyright/data_harness/thin/vocabulary.py +368 -0
- mostlyright/data_harness/thin/workers.py +164 -0
- mostlyright/data_harness/ucum/TABLE-PIN.json +40 -0
- mostlyright/data_harness/ucum/ucum-subset.v1.json +632 -0
- mostlyright/data_harness/unit_flow.py +927 -0
- mostlyright/data_harness/units.py +572 -0
- mostlyright/data_harness/ux/__init__.py +9 -0
- mostlyright/data_harness/ux/approve.py +485 -0
- mostlyright/data_harness/ux/author_yaml.py +597 -0
- mostlyright/data_harness/ux/cloud_auth.py +447 -0
- mostlyright/data_harness/ux/commands/__init__.py +260 -0
- mostlyright/data_harness/ux/commands/approve.py +136 -0
- mostlyright/data_harness/ux/commands/auth.py +744 -0
- mostlyright/data_harness/ux/commands/author.py +79 -0
- mostlyright/data_harness/ux/commands/catalog_author.py +403 -0
- mostlyright/data_harness/ux/commands/catalog_fill.py +523 -0
- mostlyright/data_harness/ux/commands/catalog_harvest.py +545 -0
- mostlyright/data_harness/ux/commands/catalog_publish.py +1838 -0
- mostlyright/data_harness/ux/commands/catalog_search.py +71 -0
- mostlyright/data_harness/ux/commands/catalog_update.py +437 -0
- mostlyright/data_harness/ux/commands/deploy.py +134 -0
- mostlyright/data_harness/ux/commands/deploy_dataset.py +98 -0
- mostlyright/data_harness/ux/commands/deploy_plan.py +105 -0
- mostlyright/data_harness/ux/commands/deploy_status.py +104 -0
- mostlyright/data_harness/ux/commands/diff.py +74 -0
- mostlyright/data_harness/ux/commands/index.py +84 -0
- mostlyright/data_harness/ux/commands/inventory.py +47 -0
- mostlyright/data_harness/ux/commands/list_builds.py +143 -0
- mostlyright/data_harness/ux/commands/login.py +63 -0
- mostlyright/data_harness/ux/commands/peek.py +236 -0
- mostlyright/data_harness/ux/commands/plan_check.py +90 -0
- mostlyright/data_harness/ux/commands/preflight.py +97 -0
- mostlyright/data_harness/ux/commands/record.py +107 -0
- mostlyright/data_harness/ux/commands/review_setup.py +47 -0
- mostlyright/data_harness/ux/commands/search.py +440 -0
- mostlyright/data_harness/ux/commands/show.py +61 -0
- mostlyright/data_harness/ux/commands/whoami.py +37 -0
- mostlyright/data_harness/ux/credential_native.py +551 -0
- mostlyright/data_harness/ux/credential_store.py +1055 -0
- mostlyright/data_harness/ux/credentials.py +631 -0
- mostlyright/data_harness/ux/diffing.py +444 -0
- mostlyright/data_harness/ux/headline.py +671 -0
- mostlyright/data_harness/ux/hosted_acquisition.py +974 -0
- mostlyright/data_harness/ux/hosted_run_status.py +619 -0
- mostlyright/data_harness/ux/inventory.py +427 -0
- mostlyright/data_harness/ux/local_review.py +375 -0
- mostlyright/data_harness/ux/login.py +691 -0
- mostlyright/data_harness/ux/path_kind.py +147 -0
- mostlyright/data_harness/ux/peek.py +1000 -0
- mostlyright/data_harness/ux/plain_file.py +178 -0
- mostlyright/data_harness/ux/plan_check.py +311 -0
- mostlyright/data_harness/ux/preflight.py +918 -0
- mostlyright/data_harness/ux/readers.py +1124 -0
- mostlyright/data_harness/ux/remediation.py +2195 -0
- mostlyright/data_harness/ux/render.py +657 -0
- mostlyright/data_harness/ux/workload.py +1077 -0
- mostlyright/data_harness/viewer.py +3713 -0
- mostlyright/data_harness/visual_run/__init__.py +83 -0
- mostlyright/data_harness/visual_run/authoring.py +235 -0
- mostlyright/data_harness/visual_run/contracts.py +673 -0
- mostlyright/data_harness/visual_run/materialize.py +486 -0
- mostlyright/data_harness/visual_run/observations.py +874 -0
- mostlyright/data_harness/visual_run/query.py +259 -0
- mostlyright/data_harness/visual_run/reducer.py +280 -0
- mostlyright/data_harness/visual_run/sdk.py +892 -0
- mostlyright/data_harness/visual_run/store.py +584 -0
- mostlyright/data_harness/visual_run/transport.py +239 -0
- mostlyright/data_harness/watch.py +2999 -0
- mostlyright_data-0.9.0.dist-info/METADATA +607 -0
- mostlyright_data-0.9.0.dist-info/RECORD +314 -0
- mostlyright_data-0.9.0.dist-info/WHEEL +4 -0
- mostlyright_data-0.9.0.dist-info/entry_points.txt +12 -0
|
@@ -0,0 +1,2773 @@
|
|
|
1
|
+
"""Pinned-peer HTTPS retrieval with redirect and resource bounds."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import contextlib
|
|
6
|
+
import errno
|
|
7
|
+
import hashlib
|
|
8
|
+
import http.client
|
|
9
|
+
import json
|
|
10
|
+
import os
|
|
11
|
+
import re
|
|
12
|
+
import socket
|
|
13
|
+
import ssl
|
|
14
|
+
import stat
|
|
15
|
+
import threading
|
|
16
|
+
import time
|
|
17
|
+
from collections.abc import Callable, Iterator, Mapping
|
|
18
|
+
from dataclasses import dataclass, replace
|
|
19
|
+
from pathlib import Path, PurePosixPath
|
|
20
|
+
from typing import Protocol
|
|
21
|
+
from urllib.parse import urljoin, urlsplit
|
|
22
|
+
|
|
23
|
+
try:
|
|
24
|
+
import fcntl
|
|
25
|
+
except ImportError: # pragma: no cover - sealing already needs dir_fd links, which Windows lacks
|
|
26
|
+
fcntl = None # type: ignore[assignment]
|
|
27
|
+
|
|
28
|
+
from mostlyright.data_harness.acquisition.retention_admission import (
|
|
29
|
+
configured_retention_evidence_root,
|
|
30
|
+
require_snapshot_admission,
|
|
31
|
+
)
|
|
32
|
+
from mostlyright.data_harness.acquisition.url_policy import (
|
|
33
|
+
AcquisitionSecurityError,
|
|
34
|
+
EgressPolicy,
|
|
35
|
+
Resolver,
|
|
36
|
+
ValidatedTarget,
|
|
37
|
+
require_approved_peer,
|
|
38
|
+
validate_public_https_url,
|
|
39
|
+
)
|
|
40
|
+
from mostlyright.data_harness.canonical import canonical_sha256, sha256_bytes
|
|
41
|
+
from mostlyright.data_harness.formats import (
|
|
42
|
+
DIRECT_FETCH_MEDIA_TYPES,
|
|
43
|
+
SNAPSHOT_SUFFIXES,
|
|
44
|
+
)
|
|
45
|
+
from mostlyright.data_harness.readers.registry import TOOLBOX, ReaderRegistry
|
|
46
|
+
|
|
47
|
+
MAX_HEADER_COUNT = 128
|
|
48
|
+
MAX_HEADER_BYTES = 32_768
|
|
49
|
+
REDIRECT_STATUSES = frozenset({301, 302, 303, 307, 308})
|
|
50
|
+
_REDIRECT_HEADER = "location"
|
|
51
|
+
_ALLOWED_RESPONSE_ENCODINGS = frozenset({"", "identity"})
|
|
52
|
+
DATAGOV_V4_HOST = "api.gsa.gov"
|
|
53
|
+
DATAGOV_V4_PATH = "/technology/datagov/v4/search"
|
|
54
|
+
DATAGOV_V4_AUTHENTICATION_KIND = "api_data_gov_x_api_key_file"
|
|
55
|
+
MAX_DATAGOV_KEY_BYTES = 4_096
|
|
56
|
+
MAX_DATAGOV_RATE_SECONDS = 86_400
|
|
57
|
+
|
|
58
|
+
# The default media-type allowlist, derived from the format table rather than restated as a
|
|
59
|
+
# literal. This is a live enforcement site, not documentation: a response whose content
|
|
60
|
+
# type is outside it is refused as ``MEDIA_TYPE``, and the outgoing ``Accept`` header is
|
|
61
|
+
# built from it. Restating the union here meant a format added to ``formats.py`` was
|
|
62
|
+
# admitted by the recipe, the parser, the sandbox policy and the deletion sweep while every
|
|
63
|
+
# HTTPS fetch of it was refused and it was never even requested -- and the anti-duplication
|
|
64
|
+
# gate could not see the drift, because these are media types rather than format names.
|
|
65
|
+
# Sorted so the header is deterministic; a coordinator may still narrow the tuple.
|
|
66
|
+
#
|
|
67
|
+
# Derived from the direct-fetch map rather than the sealing table: a source with no Reader pin
|
|
68
|
+
# also admits the weak label a publisher sends when it declines to describe its file, and
|
|
69
|
+
# ``formats.DIRECT_FETCH_MEDIA_TYPES`` is where that is decided and reasoned about. A pinned
|
|
70
|
+
# fetch does not read this tuple at all -- ``reader_retrieval_limits`` below replaces it with
|
|
71
|
+
# the pinned family's own declaration.
|
|
72
|
+
#
|
|
73
|
+
# Deduplicated, because one label may name more than one format: ``text/plain`` is the weak
|
|
74
|
+
# label for csv, json and ndjson alike. Without the set the union repeats it once per format,
|
|
75
|
+
# and this tuple builds the outgoing ``Accept`` header as well as gating the response.
|
|
76
|
+
_TABLE_MEDIA_TYPES: tuple[str, ...] = tuple(
|
|
77
|
+
sorted(
|
|
78
|
+
{media_type for accepted in DIRECT_FETCH_MEDIA_TYPES.values() for media_type in accepted}
|
|
79
|
+
)
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
# The metadata channel: media types a per-fetch narrowing may name even when the parent retriever's
|
|
83
|
+
# own allowlist does not carry them.
|
|
84
|
+
#
|
|
85
|
+
# ``allowed_media_types`` defaults to the *payload* format table, which is a closed vocabulary of
|
|
86
|
+
# encodings a Reader can open. A catalogue's structure message is not a payload and never will be
|
|
87
|
+
# in that table -- an SDMX 2.1 dataflow message describes dataflows, and nothing downstream parses
|
|
88
|
+
# it as data. Without this channel the two contracts contradict each other: a coordinator building
|
|
89
|
+
# a retriever with ``RetrievalLimits()`` defaults could not narrow to the SDMX harvester's declared
|
|
90
|
+
# type at all, so ``narrowed`` refused the fetch before a request was made, and the only way out
|
|
91
|
+
# was for the coordinator to widen its *payload* allowlist for every other fetch -- the opposite of
|
|
92
|
+
# what this seam exists for.
|
|
93
|
+
#
|
|
94
|
+
# Two properties keep this from being a hole. It is a closed vocabulary declared here, in the
|
|
95
|
+
# transport, so a caller cannot name an arbitrary type through it -- ``text/html`` is not a member,
|
|
96
|
+
# which is what keeps the bot-wall refusal intact. And membership is decided by one rule: a member
|
|
97
|
+
# describes a *catalogue's own description of itself*, never bytes any Reader opens. A type that
|
|
98
|
+
# fails that rule belongs in ``formats.py`` or nowhere.
|
|
99
|
+
METADATA_MEDIA_TYPES: tuple[str, ...] = ("application/vnd.sdmx.structure+xml",)
|
|
100
|
+
|
|
101
|
+
# Spellings of a media type that mean the media type beside them, canonicalised by
|
|
102
|
+
# ``_media_type`` the moment a ``Content-Type`` header is read. ``binary/octet-stream`` is
|
|
103
|
+
# S3's own default for an object it was told nothing about, and it is what the NOAA GFS
|
|
104
|
+
# mirror this boundary exists for answers with -- for the sidecar and for the object both.
|
|
105
|
+
# It is not a second admitted type: it is the same "the server declined to say anything"
|
|
106
|
+
# signal under another name, and reading it as one keeps a real source from being refused
|
|
107
|
+
# for a spelling while leaving every allowlist in this package written once.
|
|
108
|
+
#
|
|
109
|
+
# Anything not in this table passes through untouched. A table of equivalences is not a
|
|
110
|
+
# widening: no type is admitted here that an allowlist does not already admit under its
|
|
111
|
+
# canonical name, and the structural checks behind the sandbox boundary remain the only
|
|
112
|
+
# thing that decides whether bytes are the bytes that were ordered.
|
|
113
|
+
MEDIA_TYPE_ALIASES: dict[str, str] = {"binary/octet-stream": "application/octet-stream"}
|
|
114
|
+
|
|
115
|
+
# The two media types a slice fetch meets: ``application/octet-stream`` for the object's
|
|
116
|
+
# bytes and ``text/plain`` for the ``.idx`` sidecar that says where in the object to look.
|
|
117
|
+
#
|
|
118
|
+
# They are deliberately not in ``formats.py``. That table is one closed vocabulary read by
|
|
119
|
+
# the recipe, the parser, the sandbox policy and the deletion sweep; adding a media type
|
|
120
|
+
# there admits it for every format at every one of those layers, and neither of these is a
|
|
121
|
+
# format -- no parser accepts them and no plan may name them. ``allowed_media_types`` is
|
|
122
|
+
# already a per-``RetrievalLimits`` field and narrowing it is an explicitly preserved right,
|
|
123
|
+
# so the slice-fetch caller passes this tuple for its own calls and no other call is widened.
|
|
124
|
+
#
|
|
125
|
+
# ``application/octet-stream`` is a weak signal and is treated as one: it says only that a
|
|
126
|
+
# server declined to say anything. It is not evidence that these are the ordered bytes. The
|
|
127
|
+
# control for that is the structural check on the slice itself, behind the sandbox boundary;
|
|
128
|
+
# this tuple only keeps a response from being refused for the one honest thing it said.
|
|
129
|
+
#
|
|
130
|
+
# The pair is named in halves because a call narrows to the half it is fetching. The sidecar
|
|
131
|
+
# half admits ``text/plain`` and the byte-stream spelling both, because a real object store
|
|
132
|
+
# answers the ``.idx`` as an opaque byte stream: the NOAA GFS S3 mirror this boundary was built
|
|
133
|
+
# against answers ``binary/octet-stream`` for the sidecar and for the object alike, verified
|
|
134
|
+
# against the live source on 2026-08-08. Admitting only ``text/plain`` there refused the
|
|
135
|
+
# sidecar before a single slice could be planned, and the survey half reported the very same
|
|
136
|
+
# response as present -- so look-then-lock locked a run the fetch could never take a byte of.
|
|
137
|
+
# The union below is derived from the halves rather than restated, so narrowing per call cannot
|
|
138
|
+
# drift from the pair this boundary admits.
|
|
139
|
+
#
|
|
140
|
+
# Sorted, like the tuple above, so the outgoing ``Accept`` header is deterministic.
|
|
141
|
+
SIDECAR_MEDIA_TYPES: tuple[str, ...] = ("application/octet-stream", "text/plain")
|
|
142
|
+
SLICE_OBJECT_MEDIA_TYPES: tuple[str, ...] = ("application/octet-stream",)
|
|
143
|
+
SLICE_FETCH_MEDIA_TYPES: tuple[str, ...] = tuple(
|
|
144
|
+
sorted(set(SIDECAR_MEDIA_TYPES + SLICE_OBJECT_MEDIA_TYPES))
|
|
145
|
+
)
|
|
146
|
+
|
|
147
|
+
# Byte offsets stay inside the exactly-representable integer range, because a range rides the
|
|
148
|
+
# sealed hop evidence and must survive every reader of that evidence unchanged. The span
|
|
149
|
+
# ceiling matches the ceiling ``max_response_bytes`` is itself bounded by: no single ordered
|
|
150
|
+
# slice may be larger than the largest response this boundary would ever accept.
|
|
151
|
+
MAX_BYTE_RANGE_OFFSET = (1 << 53) - 1
|
|
152
|
+
MAX_BYTE_RANGE_LENGTH = 1 << 34
|
|
153
|
+
|
|
154
|
+
# The largest range plan one acquisition may order. A slice fetch is many small requests
|
|
155
|
+
# against one object, so an unbounded plan is a denial-of-service lever on both this process
|
|
156
|
+
# and the source; 64 covers the 10-to-50 slice acquisitions this boundary exists for.
|
|
157
|
+
MAX_RANGE_PLAN = 64
|
|
158
|
+
|
|
159
|
+
NOT_FOUND_STATUS = 404
|
|
160
|
+
PARTIAL_CONTENT_STATUS = 206
|
|
161
|
+
# The typed refusal code that means "this address is not published". Named here rather than
|
|
162
|
+
# left as a literal at the raise site because the availability survey decides absence by
|
|
163
|
+
# comparing against this exact value: a consumer that restated the spelling would keep
|
|
164
|
+
# compiling and quietly stop recognising absence the day this one changed, and the failure
|
|
165
|
+
# would look like a source that is permanently broken rather than like a mismatch.
|
|
166
|
+
HTTP_NOT_FOUND = "HTTP_NOT_FOUND"
|
|
167
|
+
_MULTIPART_BYTERANGES = "multipart/byteranges"
|
|
168
|
+
# ``bytes <first>-<last>/<complete>``: exactly one space after the unit, decimal integers in
|
|
169
|
+
# canonical form, and either a complete length or ``*`` for an unknown one. Every other shape
|
|
170
|
+
# a server might send -- another unit, a list of spans, extra whitespace, a ``*`` span, a
|
|
171
|
+
# request-form ``bytes=`` -- is refused rather than interpreted, because a span that is read
|
|
172
|
+
# loosely cannot be compared exactly against the span that was ordered. Leading zeros are
|
|
173
|
+
# refused so one delivered span has one header spelling in the sealed evidence, and the digit
|
|
174
|
+
# bound keeps a header from carrying an offset no object could have.
|
|
175
|
+
_DECIMAL = r"(?:0|[1-9][0-9]{0,15})"
|
|
176
|
+
_CONTENT_RANGE = re.compile(rf"bytes ({_DECIMAL})-({_DECIMAL})/({_DECIMAL}|\*)")
|
|
177
|
+
# The same spelling on its own, for ``Content-Length``. Built from ``_DECIMAL`` rather than
|
|
178
|
+
# written out again, so one edit moves both and the two headers cannot drift apart.
|
|
179
|
+
_DECIMAL_ONLY = re.compile(_DECIMAL)
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
@dataclass(frozen=True)
|
|
183
|
+
class DatagovRateObservation:
|
|
184
|
+
"""Validated, secret-free API-gateway rate metadata for one v4 response."""
|
|
185
|
+
|
|
186
|
+
limit: int
|
|
187
|
+
remaining: int
|
|
188
|
+
retry_after_seconds: int | None
|
|
189
|
+
reset_epoch_seconds: int | None
|
|
190
|
+
|
|
191
|
+
def __post_init__(self) -> None:
|
|
192
|
+
if type(self.limit) is not int or not 1 <= self.limit <= 1_000_000_000:
|
|
193
|
+
raise AcquisitionSecurityError("DATAGOV_RATE", "rate limit is invalid")
|
|
194
|
+
if type(self.remaining) is not int or not 0 <= self.remaining <= self.limit:
|
|
195
|
+
raise AcquisitionSecurityError("DATAGOV_RATE", "rate remaining is invalid")
|
|
196
|
+
if self.retry_after_seconds is not None and (
|
|
197
|
+
type(self.retry_after_seconds) is not int
|
|
198
|
+
or not 0 <= self.retry_after_seconds <= MAX_DATAGOV_RATE_SECONDS
|
|
199
|
+
):
|
|
200
|
+
raise AcquisitionSecurityError("DATAGOV_RATE", "Retry-After is invalid")
|
|
201
|
+
if self.reset_epoch_seconds is not None and (
|
|
202
|
+
type(self.reset_epoch_seconds) is not int
|
|
203
|
+
or not 0 <= self.reset_epoch_seconds <= (1 << 53) - 1
|
|
204
|
+
):
|
|
205
|
+
raise AcquisitionSecurityError("DATAGOV_RATE", "rate reset is invalid")
|
|
206
|
+
|
|
207
|
+
def to_dict(self) -> dict[str, int | None]:
|
|
208
|
+
return {
|
|
209
|
+
"limit": self.limit,
|
|
210
|
+
"remaining": self.remaining,
|
|
211
|
+
"retry_after_seconds": self.retry_after_seconds,
|
|
212
|
+
"reset_epoch_seconds": self.reset_epoch_seconds,
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
@dataclass(frozen=True, repr=False)
|
|
217
|
+
class DatagovV4Authorization:
|
|
218
|
+
"""One in-memory Data.gov key whose only rendering is the exact v4 header."""
|
|
219
|
+
|
|
220
|
+
_key: str
|
|
221
|
+
authentication_kind: str = DATAGOV_V4_AUTHENTICATION_KIND
|
|
222
|
+
|
|
223
|
+
def __post_init__(self) -> None:
|
|
224
|
+
if self.authentication_kind != DATAGOV_V4_AUTHENTICATION_KIND:
|
|
225
|
+
raise AcquisitionSecurityError(
|
|
226
|
+
"DATAGOV_AUTHORIZATION", "unsupported Data.gov authentication kind"
|
|
227
|
+
)
|
|
228
|
+
if not isinstance(self._key, str) or not self._key:
|
|
229
|
+
raise AcquisitionSecurityError("DATAGOV_AUTHORIZATION", "Data.gov key is empty")
|
|
230
|
+
|
|
231
|
+
@classmethod
|
|
232
|
+
def from_key_file(cls, path: Path) -> DatagovV4Authorization:
|
|
233
|
+
"""Read one key through a retained no-follow parent chain and stable descriptor."""
|
|
234
|
+
|
|
235
|
+
raw_path = os.fspath(path)
|
|
236
|
+
if not raw_path or "\x00" in raw_path:
|
|
237
|
+
raise AcquisitionSecurityError("DATAGOV_KEY_FILE", "key file path is invalid")
|
|
238
|
+
absolute = Path(os.path.abspath(raw_path))
|
|
239
|
+
if absolute.name in {"", ".", ".."}:
|
|
240
|
+
raise AcquisitionSecurityError("DATAGOV_KEY_FILE", "key file path is invalid")
|
|
241
|
+
directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0)
|
|
242
|
+
owned: list[int] = []
|
|
243
|
+
descriptor = -1
|
|
244
|
+
try:
|
|
245
|
+
parent = os.open("/", directory_flags)
|
|
246
|
+
owned.append(parent)
|
|
247
|
+
for component in absolute.parts[1:-1]:
|
|
248
|
+
parent = os.open(component, directory_flags, dir_fd=parent)
|
|
249
|
+
owned.append(parent)
|
|
250
|
+
flags = (
|
|
251
|
+
os.O_RDONLY
|
|
252
|
+
| getattr(os, "O_NOFOLLOW", 0)
|
|
253
|
+
| getattr(os, "O_CLOEXEC", 0)
|
|
254
|
+
| getattr(os, "O_NONBLOCK", 0)
|
|
255
|
+
)
|
|
256
|
+
descriptor = os.open(absolute.name, flags, dir_fd=parent)
|
|
257
|
+
before = os.fstat(descriptor)
|
|
258
|
+
_require_datagov_key_stat(before)
|
|
259
|
+
raw = _read_datagov_key(descriptor, expected_size=before.st_size)
|
|
260
|
+
after = os.fstat(descriptor)
|
|
261
|
+
if _stable_file_signature(before) != _stable_file_signature(after):
|
|
262
|
+
raise AcquisitionSecurityError(
|
|
263
|
+
"DATAGOV_KEY_RACE", "key file changed during its bounded read"
|
|
264
|
+
)
|
|
265
|
+
except AcquisitionSecurityError:
|
|
266
|
+
raise
|
|
267
|
+
except OSError:
|
|
268
|
+
raise AcquisitionSecurityError(
|
|
269
|
+
"DATAGOV_KEY_FILE", "key file must be a retained no-follow regular file"
|
|
270
|
+
) from None
|
|
271
|
+
finally:
|
|
272
|
+
if descriptor >= 0:
|
|
273
|
+
os.close(descriptor)
|
|
274
|
+
for opened in reversed(owned):
|
|
275
|
+
os.close(opened)
|
|
276
|
+
try:
|
|
277
|
+
key = raw.decode("ascii", errors="strict")
|
|
278
|
+
except UnicodeDecodeError:
|
|
279
|
+
raise AcquisitionSecurityError(
|
|
280
|
+
"DATAGOV_KEY_FILE", "key file must contain one ASCII line"
|
|
281
|
+
) from None
|
|
282
|
+
if key.endswith("\n"):
|
|
283
|
+
key = key[:-1]
|
|
284
|
+
if (
|
|
285
|
+
not key
|
|
286
|
+
or "\n" in key
|
|
287
|
+
or "\r" in key
|
|
288
|
+
or any(not 0x21 <= ord(char) <= 0x7E for char in key)
|
|
289
|
+
):
|
|
290
|
+
raise AcquisitionSecurityError(
|
|
291
|
+
"DATAGOV_KEY_FILE", "key file must contain one nonempty printable ASCII line"
|
|
292
|
+
)
|
|
293
|
+
return cls(key)
|
|
294
|
+
|
|
295
|
+
def __repr__(self) -> str:
|
|
296
|
+
return (
|
|
297
|
+
"DatagovV4Authorization("
|
|
298
|
+
f"authentication_kind={self.authentication_kind!r}, key=<redacted>)"
|
|
299
|
+
)
|
|
300
|
+
|
|
301
|
+
def header_for(self, url: str) -> tuple[str, str]:
|
|
302
|
+
parsed = urlsplit(url)
|
|
303
|
+
if (
|
|
304
|
+
parsed.scheme != "https"
|
|
305
|
+
or parsed.hostname != DATAGOV_V4_HOST
|
|
306
|
+
or parsed.port not in {None, 443}
|
|
307
|
+
or parsed.username is not None
|
|
308
|
+
or parsed.password is not None
|
|
309
|
+
or parsed.path != DATAGOV_V4_PATH
|
|
310
|
+
or parsed.fragment
|
|
311
|
+
):
|
|
312
|
+
raise AcquisitionSecurityError(
|
|
313
|
+
"DATAGOV_AUTHORIZATION",
|
|
314
|
+
"Data.gov authorization is scoped to the exact v4 search endpoint",
|
|
315
|
+
)
|
|
316
|
+
return "X-Api-Key", self._key
|
|
317
|
+
|
|
318
|
+
def reject_echoed_response(self, payload: bytes) -> None:
|
|
319
|
+
"""Reject raw or JSON-equivalent key echoes while the key remains ephemeral."""
|
|
320
|
+
|
|
321
|
+
key_bytes = self._key.encode("ascii", errors="strict")
|
|
322
|
+
if key_bytes in payload:
|
|
323
|
+
raise AcquisitionSecurityError(
|
|
324
|
+
"DATAGOV_SECRET_ECHO",
|
|
325
|
+
"Data.gov response echoed authorization text and was not retained",
|
|
326
|
+
)
|
|
327
|
+
try:
|
|
328
|
+
document = json.loads(
|
|
329
|
+
payload.decode("utf-8", errors="strict"),
|
|
330
|
+
object_pairs_hook=_DatagovEchoObject,
|
|
331
|
+
parse_constant=lambda _value: (_ for _ in ()).throw(ValueError()),
|
|
332
|
+
)
|
|
333
|
+
except (UnicodeDecodeError, json.JSONDecodeError, RecursionError, ValueError):
|
|
334
|
+
raise AcquisitionSecurityError(
|
|
335
|
+
"DATAGOV_RESPONSE_UNSCANNABLE",
|
|
336
|
+
"Data.gov response could not be fully inspected before retention",
|
|
337
|
+
) from None
|
|
338
|
+
pending = [document]
|
|
339
|
+
visited = 0
|
|
340
|
+
while pending:
|
|
341
|
+
value = pending.pop()
|
|
342
|
+
visited += 1
|
|
343
|
+
if visited > 1_000_000:
|
|
344
|
+
raise AcquisitionSecurityError(
|
|
345
|
+
"DATAGOV_RESPONSE_STRUCTURE",
|
|
346
|
+
"Data.gov response exceeds the decoded traversal bound",
|
|
347
|
+
)
|
|
348
|
+
if isinstance(value, str):
|
|
349
|
+
if self._key in value:
|
|
350
|
+
raise AcquisitionSecurityError(
|
|
351
|
+
"DATAGOV_SECRET_ECHO",
|
|
352
|
+
"Data.gov response echoed authorization text and was not retained",
|
|
353
|
+
)
|
|
354
|
+
elif isinstance(value, _DatagovEchoObject):
|
|
355
|
+
for key, member in value:
|
|
356
|
+
pending.append(key)
|
|
357
|
+
pending.append(member)
|
|
358
|
+
elif isinstance(value, list):
|
|
359
|
+
pending.extend(value)
|
|
360
|
+
|
|
361
|
+
def reject_echoed_headers(self, headers: tuple[tuple[str, str], ...]) -> None:
|
|
362
|
+
"""Reject the key in any response-header name or value before hashing or parsing."""
|
|
363
|
+
|
|
364
|
+
for name, value in headers:
|
|
365
|
+
if self._key in name or self._key in value:
|
|
366
|
+
raise AcquisitionSecurityError(
|
|
367
|
+
"DATAGOV_SECRET_ECHO",
|
|
368
|
+
"Data.gov response echoed authorization text and was not retained",
|
|
369
|
+
)
|
|
370
|
+
|
|
371
|
+
|
|
372
|
+
class _DatagovEchoObject(list[tuple[str, object]]):
|
|
373
|
+
"""JSON object retaining duplicate keys for authorization echo inspection."""
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
def _require_datagov_key_stat(info: os.stat_result) -> None:
|
|
377
|
+
if (
|
|
378
|
+
not stat.S_ISREG(info.st_mode)
|
|
379
|
+
or info.st_uid != os.geteuid()
|
|
380
|
+
or info.st_nlink != 1
|
|
381
|
+
or stat.S_IMODE(info.st_mode) & 0o077
|
|
382
|
+
or not stat.S_IMODE(info.st_mode) & stat.S_IRUSR
|
|
383
|
+
or not 1 <= info.st_size <= MAX_DATAGOV_KEY_BYTES
|
|
384
|
+
):
|
|
385
|
+
raise AcquisitionSecurityError(
|
|
386
|
+
"DATAGOV_KEY_FILE",
|
|
387
|
+
"key file must be owner-readable, owner-owned, private, bounded, and single-link",
|
|
388
|
+
)
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
def _read_datagov_key(descriptor: int, *, expected_size: int) -> bytes:
|
|
392
|
+
chunks: list[bytes] = []
|
|
393
|
+
observed = 0
|
|
394
|
+
while observed <= MAX_DATAGOV_KEY_BYTES:
|
|
395
|
+
chunk = os.read(descriptor, min(512, MAX_DATAGOV_KEY_BYTES + 1 - observed))
|
|
396
|
+
if not chunk:
|
|
397
|
+
break
|
|
398
|
+
observed += len(chunk)
|
|
399
|
+
chunks.append(chunk)
|
|
400
|
+
if observed != expected_size or observed > MAX_DATAGOV_KEY_BYTES:
|
|
401
|
+
raise AcquisitionSecurityError("DATAGOV_KEY_RACE", "key file size changed during read")
|
|
402
|
+
return b"".join(chunks)
|
|
403
|
+
|
|
404
|
+
|
|
405
|
+
def _stable_file_signature(info: os.stat_result) -> tuple[int, ...]:
|
|
406
|
+
return (
|
|
407
|
+
info.st_dev,
|
|
408
|
+
info.st_ino,
|
|
409
|
+
info.st_uid,
|
|
410
|
+
info.st_gid,
|
|
411
|
+
info.st_mode,
|
|
412
|
+
info.st_nlink,
|
|
413
|
+
info.st_size,
|
|
414
|
+
info.st_mtime_ns,
|
|
415
|
+
info.st_ctime_ns,
|
|
416
|
+
)
|
|
417
|
+
|
|
418
|
+
|
|
419
|
+
@dataclass(frozen=True)
|
|
420
|
+
class ByteRange:
|
|
421
|
+
"""One closed byte span, inclusive on both ends, exactly as RFC 9110 numbers them.
|
|
422
|
+
|
|
423
|
+
There is deliberately no open-ended (``bytes=N-``) or suffix (``bytes=-N``) form. With
|
|
424
|
+
either of those, "the span that was ordered" is not a fact the client holds before the
|
|
425
|
+
response arrives, so the delivery check could not compare what came back against what was
|
|
426
|
+
asked for -- which is the entire purpose of ordering a slice.
|
|
427
|
+
"""
|
|
428
|
+
|
|
429
|
+
first_byte: int
|
|
430
|
+
last_byte: int
|
|
431
|
+
|
|
432
|
+
def __post_init__(self) -> None:
|
|
433
|
+
for name, value in (("first_byte", self.first_byte), ("last_byte", self.last_byte)):
|
|
434
|
+
if type(value) is not int:
|
|
435
|
+
raise AcquisitionSecurityError("BYTE_RANGE", f"{name} must be an integer")
|
|
436
|
+
if not 0 <= value <= MAX_BYTE_RANGE_OFFSET:
|
|
437
|
+
raise AcquisitionSecurityError(
|
|
438
|
+
"BYTE_RANGE",
|
|
439
|
+
f"{name} must be in [0, {MAX_BYTE_RANGE_OFFSET}]",
|
|
440
|
+
)
|
|
441
|
+
if self.last_byte < self.first_byte:
|
|
442
|
+
raise AcquisitionSecurityError(
|
|
443
|
+
"BYTE_RANGE",
|
|
444
|
+
"byte range must end at or after it begins",
|
|
445
|
+
)
|
|
446
|
+
if self.last_byte - self.first_byte + 1 > MAX_BYTE_RANGE_LENGTH:
|
|
447
|
+
raise AcquisitionSecurityError(
|
|
448
|
+
"BYTE_RANGE",
|
|
449
|
+
f"byte range length must not exceed {MAX_BYTE_RANGE_LENGTH}",
|
|
450
|
+
)
|
|
451
|
+
|
|
452
|
+
@property
|
|
453
|
+
def length(self) -> int:
|
|
454
|
+
return self.last_byte - self.first_byte + 1
|
|
455
|
+
|
|
456
|
+
@property
|
|
457
|
+
def header_value(self) -> str:
|
|
458
|
+
return f"bytes={self.first_byte}-{self.last_byte}"
|
|
459
|
+
|
|
460
|
+
|
|
461
|
+
# The span a presence probe orders: the first byte and no more. A probe that read further
|
|
462
|
+
# would be an acquisition nobody ordered, and the whole point of the look half is that it
|
|
463
|
+
# costs the source next to nothing. Held here rather than at the call site so no caller can
|
|
464
|
+
# widen it.
|
|
465
|
+
_PRESENCE_PROBE_RANGE = ByteRange(first_byte=0, last_byte=0)
|
|
466
|
+
|
|
467
|
+
|
|
468
|
+
@dataclass(frozen=True)
|
|
469
|
+
class RetrievalLimits:
|
|
470
|
+
max_response_bytes: int = 16 * 1024 * 1024
|
|
471
|
+
max_aggregate_response_bytes: int = 1 << 34
|
|
472
|
+
max_redirects: int = 5
|
|
473
|
+
max_requests: int = 6
|
|
474
|
+
# How many ranged requests one acquisition may plan. This is deliberately a separate
|
|
475
|
+
# budget from ``max_requests``: raising that one to 50 so a 50-slice acquisition could
|
|
476
|
+
# afford its requests would also loosen the ``max_requests >= max_redirects + 1``
|
|
477
|
+
# relation below, which is a redirect control and has nothing to do with slices. The
|
|
478
|
+
# redirect budget therefore stays exactly where it is.
|
|
479
|
+
max_ranges: int = MAX_RANGE_PLAN
|
|
480
|
+
connect_timeout_seconds: float = 10.0
|
|
481
|
+
read_timeout_seconds: float = 30.0
|
|
482
|
+
total_timeout_seconds: float = 45.0
|
|
483
|
+
max_concurrency: int = 4
|
|
484
|
+
min_interval_seconds: float = 0.0
|
|
485
|
+
allowed_media_types: tuple[str, ...] = _TABLE_MEDIA_TYPES
|
|
486
|
+
user_agent: str = "MostlyRightDataHarness/1.0"
|
|
487
|
+
|
|
488
|
+
def __post_init__(self) -> None:
|
|
489
|
+
if type(self.max_response_bytes) is not int or not 1 <= self.max_response_bytes <= 1 << 34:
|
|
490
|
+
raise AcquisitionSecurityError(
|
|
491
|
+
"RESPONSE_LIMIT",
|
|
492
|
+
"max_response_bytes must be a positive bounded integer",
|
|
493
|
+
)
|
|
494
|
+
if (
|
|
495
|
+
type(self.max_aggregate_response_bytes) is not int
|
|
496
|
+
or not 1 <= self.max_aggregate_response_bytes <= 1 << 40
|
|
497
|
+
):
|
|
498
|
+
raise AcquisitionSecurityError(
|
|
499
|
+
"AGGREGATE_RESPONSE_LIMIT",
|
|
500
|
+
"max_aggregate_response_bytes must be a positive bounded integer",
|
|
501
|
+
)
|
|
502
|
+
if type(self.max_redirects) is not int or not 0 <= self.max_redirects <= 10:
|
|
503
|
+
raise AcquisitionSecurityError(
|
|
504
|
+
"REDIRECT_LIMIT",
|
|
505
|
+
"max_redirects must be an integer in [0, 10]",
|
|
506
|
+
)
|
|
507
|
+
if type(self.max_requests) is not int or not 1 <= self.max_requests <= 16:
|
|
508
|
+
raise AcquisitionSecurityError(
|
|
509
|
+
"REQUEST_LIMIT",
|
|
510
|
+
"max_requests must be an integer in [1, 16]",
|
|
511
|
+
)
|
|
512
|
+
if self.max_requests < self.max_redirects + 1:
|
|
513
|
+
raise AcquisitionSecurityError(
|
|
514
|
+
"REQUEST_LIMIT",
|
|
515
|
+
"max_requests must allow the configured redirect chain",
|
|
516
|
+
)
|
|
517
|
+
if type(self.max_ranges) is not int or not 1 <= self.max_ranges <= MAX_RANGE_PLAN:
|
|
518
|
+
raise AcquisitionSecurityError(
|
|
519
|
+
"RANGE_LIMIT",
|
|
520
|
+
f"max_ranges must be an integer in [1, {MAX_RANGE_PLAN}]",
|
|
521
|
+
)
|
|
522
|
+
for name, value, maximum in (
|
|
523
|
+
("connect_timeout_seconds", self.connect_timeout_seconds, 60.0),
|
|
524
|
+
("read_timeout_seconds", self.read_timeout_seconds, 300.0),
|
|
525
|
+
("total_timeout_seconds", self.total_timeout_seconds, 600.0),
|
|
526
|
+
):
|
|
527
|
+
if type(value) not in {int, float} or not 0 < value <= maximum:
|
|
528
|
+
raise AcquisitionSecurityError(
|
|
529
|
+
"TIMEOUT",
|
|
530
|
+
f"{name} must be in (0, {maximum}]",
|
|
531
|
+
)
|
|
532
|
+
if type(self.max_concurrency) is not int or not 1 <= self.max_concurrency <= 64:
|
|
533
|
+
raise AcquisitionSecurityError(
|
|
534
|
+
"CONCURRENCY_LIMIT",
|
|
535
|
+
"max_concurrency must be an integer in [1, 64]",
|
|
536
|
+
)
|
|
537
|
+
if (
|
|
538
|
+
type(self.min_interval_seconds) not in {int, float}
|
|
539
|
+
or not 0 <= self.min_interval_seconds <= 3_600
|
|
540
|
+
):
|
|
541
|
+
raise AcquisitionSecurityError(
|
|
542
|
+
"RATE_LIMIT",
|
|
543
|
+
"min_interval_seconds must be in [0, 3600]",
|
|
544
|
+
)
|
|
545
|
+
if not self.allowed_media_types or len(set(self.allowed_media_types)) != len(
|
|
546
|
+
self.allowed_media_types
|
|
547
|
+
):
|
|
548
|
+
raise AcquisitionSecurityError(
|
|
549
|
+
"MEDIA_ALLOWLIST",
|
|
550
|
+
"allowed media types must be non-empty and unique",
|
|
551
|
+
)
|
|
552
|
+
# An allowlist is compared against what ``_media_type`` returns, and that is always
|
|
553
|
+
# the canonical spelling. An alias written here would therefore match nothing and
|
|
554
|
+
# refuse every response it was added to admit -- a silent hole rather than a loud
|
|
555
|
+
# one -- so it is refused at construction, where the caller can still be told.
|
|
556
|
+
aliased = [
|
|
557
|
+
media_type
|
|
558
|
+
for media_type in self.allowed_media_types
|
|
559
|
+
if media_type in MEDIA_TYPE_ALIASES
|
|
560
|
+
]
|
|
561
|
+
if aliased:
|
|
562
|
+
raise AcquisitionSecurityError(
|
|
563
|
+
"MEDIA_ALLOWLIST",
|
|
564
|
+
f"{aliased[0]!r} is an alias of {MEDIA_TYPE_ALIASES[aliased[0]]!r}; "
|
|
565
|
+
f"allowlist the canonical spelling, which admits both",
|
|
566
|
+
)
|
|
567
|
+
if not self.user_agent or "\r" in self.user_agent or "\n" in self.user_agent:
|
|
568
|
+
raise AcquisitionSecurityError("USER_AGENT", "user agent is invalid")
|
|
569
|
+
|
|
570
|
+
|
|
571
|
+
def reader_retrieval_limits(
|
|
572
|
+
limits: RetrievalLimits,
|
|
573
|
+
*,
|
|
574
|
+
family_id: str,
|
|
575
|
+
family_version: str,
|
|
576
|
+
registry: ReaderRegistry | None = None,
|
|
577
|
+
) -> RetrievalLimits:
|
|
578
|
+
"""The limits a fetch for a Reader-pinned source runs under.
|
|
579
|
+
|
|
580
|
+
The table-derived default above is right for a direct fetch and wrong for a pinned one: a
|
|
581
|
+
zipped source arrives as an archive media type that no wire format claims, so the fetch
|
|
582
|
+
would be refused before the Reader could ever see it. The admitted set for a pinned fetch
|
|
583
|
+
is therefore the pinned family's own ``accepted_media_types``.
|
|
584
|
+
|
|
585
|
+
Taken from the family rather than added to ``FORMAT_MEDIA_TYPES``, for two reasons that
|
|
586
|
+
point the same way. The format table answers "what encoding may be sealed", and a fetched
|
|
587
|
+
container is never sealed. And widening that table would widen the allowlist for every
|
|
588
|
+
*unpinned* fetch as well, which is the opposite of what is wanted -- a direct fetch must go
|
|
589
|
+
on admitting exactly what it admits today.
|
|
590
|
+
|
|
591
|
+
Narrowing to the one family rather than to ``registry.accepted_media_types()`` is
|
|
592
|
+
deliberate: the union is the Toolbox's outer bound, and a source pinned to a zip Reader has
|
|
593
|
+
no business admitting a spreadsheet. Every other bound on ``limits`` is carried through
|
|
594
|
+
untouched, so a coordinator's own narrowing still holds.
|
|
595
|
+
|
|
596
|
+
A family the closed table does not hold is refused here, by name, before any connection is
|
|
597
|
+
attempted.
|
|
598
|
+
|
|
599
|
+
``registry`` defaults to ``None`` and is resolved to the shipped Toolbox at call time
|
|
600
|
+
rather than being bound as a default argument at import time, so which table answers is a
|
|
601
|
+
property of the running process and not of import order.
|
|
602
|
+
"""
|
|
603
|
+
|
|
604
|
+
family = (registry if registry is not None else TOOLBOX).resolve(family_id, family_version)
|
|
605
|
+
return replace(limits, allowed_media_types=tuple(family.accepted_media_types))
|
|
606
|
+
|
|
607
|
+
|
|
608
|
+
def require_range_plan_within_budget(planned_ranges: int, *, max_ranges: int) -> None:
|
|
609
|
+
"""Refuse a range plan that orders more slices than the acquisition's budget allows.
|
|
610
|
+
|
|
611
|
+
``retrieve`` fetches one range per call, so the budget belongs to whoever plans a range
|
|
612
|
+
set rather than to a single request. The bound lives here, beside the limits it comes
|
|
613
|
+
from, so the range planner and the sandbox operation that calls it do not each restate it.
|
|
614
|
+
"""
|
|
615
|
+
|
|
616
|
+
if type(max_ranges) is not int or not 1 <= max_ranges <= MAX_RANGE_PLAN:
|
|
617
|
+
raise AcquisitionSecurityError(
|
|
618
|
+
"RANGE_LIMIT",
|
|
619
|
+
f"range budget must be an integer in [1, {MAX_RANGE_PLAN}]",
|
|
620
|
+
)
|
|
621
|
+
if type(planned_ranges) is not int or planned_ranges < 1:
|
|
622
|
+
raise AcquisitionSecurityError(
|
|
623
|
+
"RANGE_LIMIT",
|
|
624
|
+
"a range plan must order at least one range",
|
|
625
|
+
)
|
|
626
|
+
if planned_ranges > max_ranges:
|
|
627
|
+
raise AcquisitionSecurityError(
|
|
628
|
+
"RANGE_LIMIT",
|
|
629
|
+
f"range plan of {planned_ranges} exceeds the budget of {max_ranges}",
|
|
630
|
+
)
|
|
631
|
+
|
|
632
|
+
|
|
633
|
+
@dataclass(frozen=True)
|
|
634
|
+
class TransportResponse:
|
|
635
|
+
status: int
|
|
636
|
+
headers: tuple[tuple[str, str], ...]
|
|
637
|
+
body: bytes
|
|
638
|
+
connected_peer: str
|
|
639
|
+
tls_verified: bool
|
|
640
|
+
hostname_verified: bool
|
|
641
|
+
|
|
642
|
+
def header(self, name: str) -> str | None:
|
|
643
|
+
wanted = name.lower()
|
|
644
|
+
values = [value for key, value in self.headers if key.lower() == wanted]
|
|
645
|
+
if len(values) > 1 and wanted in {
|
|
646
|
+
_REDIRECT_HEADER,
|
|
647
|
+
"content-length",
|
|
648
|
+
"content-type",
|
|
649
|
+
"content-range",
|
|
650
|
+
"retry-after",
|
|
651
|
+
"x-ratelimit-limit",
|
|
652
|
+
"x-ratelimit-remaining",
|
|
653
|
+
"x-ratelimit-reset",
|
|
654
|
+
# ``content-encoding`` belongs here for the same reason the other four do, and was
|
|
655
|
+
# missing. Returning the first of two would let a source send ``identity`` then
|
|
656
|
+
# ``gzip``: the encoding check reads the first and passes while the bytes are the
|
|
657
|
+
# second, and every offset in a range plan stops meaning anything. Nothing in
|
|
658
|
+
# this build decompresses, so the practical reach was small -- but "small because
|
|
659
|
+
# of something elsewhere" is not the property this set exists to give.
|
|
660
|
+
"content-encoding",
|
|
661
|
+
}:
|
|
662
|
+
raise AcquisitionSecurityError(
|
|
663
|
+
"RESPONSE_HEADER_DUPLICATE",
|
|
664
|
+
f"response repeated security-sensitive header {wanted}",
|
|
665
|
+
)
|
|
666
|
+
return values[0] if values else None
|
|
667
|
+
|
|
668
|
+
|
|
669
|
+
class PinnedTransport(Protocol):
|
|
670
|
+
def request(
|
|
671
|
+
self,
|
|
672
|
+
target: ValidatedTarget,
|
|
673
|
+
*,
|
|
674
|
+
approved_ip: str,
|
|
675
|
+
headers: Mapping[str, str],
|
|
676
|
+
limits: RetrievalLimits,
|
|
677
|
+
byte_range: ByteRange | None = None,
|
|
678
|
+
) -> TransportResponse:
|
|
679
|
+
"""Connect only to approved_ip while verifying target.hostname in TLS.
|
|
680
|
+
|
|
681
|
+
A slice is ordered through the typed ``byte_range`` parameter rather than through a
|
|
682
|
+
caller-supplied header mapping. A general ``extra_headers`` argument would put a
|
|
683
|
+
header-injection surface at the most security-sensitive seam in this package; a
|
|
684
|
+
``ByteRange`` validates itself and renders to exactly one header value that no caller
|
|
685
|
+
can influence character by character.
|
|
686
|
+
"""
|
|
687
|
+
|
|
688
|
+
|
|
689
|
+
class TransportFailureError(AcquisitionSecurityError):
|
|
690
|
+
"""Retryable transport failure with exact response-body bytes already consumed."""
|
|
691
|
+
|
|
692
|
+
def __init__(self, detail: str, *, response_body_size_bytes: int) -> None:
|
|
693
|
+
if (
|
|
694
|
+
type(response_body_size_bytes) is not int
|
|
695
|
+
or not 0 <= response_body_size_bytes <= (1 << 34) + 1
|
|
696
|
+
):
|
|
697
|
+
raise AcquisitionSecurityError(
|
|
698
|
+
"TRANSPORT_FAILURE",
|
|
699
|
+
"transport failure byte evidence is outside the admitted range",
|
|
700
|
+
)
|
|
701
|
+
self.response_body_size_bytes = response_body_size_bytes
|
|
702
|
+
super().__init__("TRANSPORT_FAILURE", detail)
|
|
703
|
+
|
|
704
|
+
|
|
705
|
+
@dataclass(frozen=True)
|
|
706
|
+
class PeerAttempt:
|
|
707
|
+
"""Canonical evidence for one address from a resolver-approved peer set."""
|
|
708
|
+
|
|
709
|
+
approved_ip: str
|
|
710
|
+
outcome: str
|
|
711
|
+
failure_code: str | None
|
|
712
|
+
response_body_size_bytes: int = 0
|
|
713
|
+
|
|
714
|
+
def __post_init__(self) -> None:
|
|
715
|
+
if (
|
|
716
|
+
type(self.response_body_size_bytes) is not int
|
|
717
|
+
or not 0 <= self.response_body_size_bytes <= (1 << 34) + 1
|
|
718
|
+
):
|
|
719
|
+
raise AcquisitionSecurityError(
|
|
720
|
+
"TRANSPORT_EVIDENCE",
|
|
721
|
+
"peer-attempt response-body bytes are outside the admitted range",
|
|
722
|
+
)
|
|
723
|
+
|
|
724
|
+
def to_dict(self) -> dict[str, object]:
|
|
725
|
+
return {
|
|
726
|
+
"approved_ip": self.approved_ip,
|
|
727
|
+
"outcome": self.outcome,
|
|
728
|
+
"failure_code": self.failure_code,
|
|
729
|
+
"response_body_size_bytes": self.response_body_size_bytes,
|
|
730
|
+
}
|
|
731
|
+
|
|
732
|
+
|
|
733
|
+
class PeerAttemptError(AcquisitionSecurityError):
|
|
734
|
+
"""A typed retrieval refusal carrying redacted, ordered peer-attempt evidence.
|
|
735
|
+
|
|
736
|
+
``hops`` is the exact sealed-evidence hop list the refused retrieval had already
|
|
737
|
+
measured when it refused -- every completed hop, including the one whose response was
|
|
738
|
+
consumed in full and then refused (a 200 answering a ranged probe, for instance). It
|
|
739
|
+
rides the refusal so a caller that legitimately continues after one (the ranged bulk
|
|
740
|
+
fetch falling back to its single-shot path) can seal what the refused attempt consumed
|
|
741
|
+
instead of letting those response bytes vanish from the evidence. It is the same
|
|
742
|
+
``RetrievalHop`` type ``RetrievedBytes`` seals; nothing new is measured here.
|
|
743
|
+
"""
|
|
744
|
+
|
|
745
|
+
def __init__(
|
|
746
|
+
self,
|
|
747
|
+
code: str,
|
|
748
|
+
detail: str,
|
|
749
|
+
peer_attempts: tuple[PeerAttempt, ...],
|
|
750
|
+
*,
|
|
751
|
+
hops: tuple[RetrievalHop, ...] = (),
|
|
752
|
+
) -> None:
|
|
753
|
+
if any(not isinstance(attempt, PeerAttempt) for attempt in peer_attempts):
|
|
754
|
+
raise AcquisitionSecurityError(
|
|
755
|
+
"TRANSPORT_EVIDENCE",
|
|
756
|
+
"peer-attempt evidence contains an invalid entry",
|
|
757
|
+
)
|
|
758
|
+
if any(not isinstance(hop, RetrievalHop) for hop in hops):
|
|
759
|
+
raise AcquisitionSecurityError(
|
|
760
|
+
"TRANSPORT_EVIDENCE",
|
|
761
|
+
"peer-attempt hop evidence contains an invalid entry",
|
|
762
|
+
)
|
|
763
|
+
self.peer_attempts = peer_attempts
|
|
764
|
+
self.hops = hops
|
|
765
|
+
self.total_response_body_size_bytes = sum(
|
|
766
|
+
attempt.response_body_size_bytes for attempt in peer_attempts
|
|
767
|
+
)
|
|
768
|
+
super().__init__(code, detail)
|
|
769
|
+
|
|
770
|
+
|
|
771
|
+
class DatagovResponseRefused(AcquisitionSecurityError):
|
|
772
|
+
"""Authenticated response refused before any secret-derived digest was constructed."""
|
|
773
|
+
|
|
774
|
+
def __init__(self, code: str, detail: str, *, status: int, network_bytes: int) -> None:
|
|
775
|
+
self.status = status
|
|
776
|
+
self.network_bytes = network_bytes
|
|
777
|
+
super().__init__(code, detail)
|
|
778
|
+
|
|
779
|
+
|
|
780
|
+
@dataclass(frozen=True)
|
|
781
|
+
class RetrievalHop:
|
|
782
|
+
url: str
|
|
783
|
+
resolution_digest: str
|
|
784
|
+
connected_peer: str
|
|
785
|
+
status: int
|
|
786
|
+
response_headers_digest: str
|
|
787
|
+
response_body_size_bytes: int
|
|
788
|
+
response_body_sha256: str
|
|
789
|
+
requested_range: str | None = None
|
|
790
|
+
observed_content_range: str | None = None
|
|
791
|
+
etag: str | None = None
|
|
792
|
+
last_modified: str | None = None
|
|
793
|
+
peer_attempts: tuple[PeerAttempt, ...] = ()
|
|
794
|
+
|
|
795
|
+
def to_dict(self) -> dict[str, object]:
|
|
796
|
+
# Both range keys are always present, ``None`` when the hop ordered no slice, so a hop
|
|
797
|
+
# has exactly one shape rather than two and the digest below covers the same fields on
|
|
798
|
+
# every acquisition.
|
|
799
|
+
return {
|
|
800
|
+
"url": self.url,
|
|
801
|
+
"resolution_digest": self.resolution_digest,
|
|
802
|
+
"connected_peer": self.connected_peer,
|
|
803
|
+
"status": self.status,
|
|
804
|
+
"response_headers_digest": self.response_headers_digest,
|
|
805
|
+
"response_body_size_bytes": self.response_body_size_bytes,
|
|
806
|
+
"response_body_sha256": self.response_body_sha256,
|
|
807
|
+
"requested_range": self.requested_range,
|
|
808
|
+
"observed_content_range": self.observed_content_range,
|
|
809
|
+
"etag": self.etag,
|
|
810
|
+
"last_modified": self.last_modified,
|
|
811
|
+
"peer_attempts": [attempt.to_dict() for attempt in self.peer_attempts],
|
|
812
|
+
}
|
|
813
|
+
|
|
814
|
+
|
|
815
|
+
@dataclass(frozen=True)
|
|
816
|
+
class ProbeTransport:
|
|
817
|
+
"""The sanitized transport facts one whole-document retrieval proves about its response.
|
|
818
|
+
|
|
819
|
+
A cadence observation records the response status and the two validator headers, because a
|
|
820
|
+
publisher that answers 200 with an unchanged ``ETag`` is stating something about its own
|
|
821
|
+
editions. Neither validator leaves the confinement as text. Both are untrusted publisher
|
|
822
|
+
bytes of the publisher's own choosing, and the observation contract stores them hashed for
|
|
823
|
+
exactly that reason, so they are hashed here -- inside the Clean room, beside the response
|
|
824
|
+
they came from -- and only the digest crosses the boundary. The status is an integer from a
|
|
825
|
+
closed range and carries nothing a publisher composed.
|
|
826
|
+
|
|
827
|
+
This is the terminal response of the exchange. Redirect hops keep their own evidence in the
|
|
828
|
+
transport hop list; what an observation states about the publisher is what the address it
|
|
829
|
+
settled on answered.
|
|
830
|
+
"""
|
|
831
|
+
|
|
832
|
+
http_status_code: int
|
|
833
|
+
etag_digest: str | None
|
|
834
|
+
last_modified_digest: str | None
|
|
835
|
+
|
|
836
|
+
def __post_init__(self) -> None:
|
|
837
|
+
if type(self.http_status_code) is not int or not 100 <= self.http_status_code <= 599:
|
|
838
|
+
raise AcquisitionSecurityError(
|
|
839
|
+
"PROBE_TRANSPORT",
|
|
840
|
+
"probe transport status is not a real HTTP status code",
|
|
841
|
+
)
|
|
842
|
+
for name, value in (
|
|
843
|
+
("etag_digest", self.etag_digest),
|
|
844
|
+
("last_modified_digest", self.last_modified_digest),
|
|
845
|
+
):
|
|
846
|
+
if value is not None and (
|
|
847
|
+
not isinstance(value, str) or re.fullmatch(r"[0-9a-f]{64}", value) is None
|
|
848
|
+
):
|
|
849
|
+
raise AcquisitionSecurityError(
|
|
850
|
+
"PROBE_TRANSPORT",
|
|
851
|
+
f"probe transport {name} is not a lowercase SHA-256 digest",
|
|
852
|
+
)
|
|
853
|
+
|
|
854
|
+
@classmethod
|
|
855
|
+
def from_terminal_hop(cls, hop: RetrievalHop) -> ProbeTransport:
|
|
856
|
+
"""Seal the terminal hop's status and hashed validators."""
|
|
857
|
+
|
|
858
|
+
return cls(
|
|
859
|
+
http_status_code=hop.status,
|
|
860
|
+
etag_digest=None if hop.etag is None else sha256_bytes(hop.etag.encode("utf-8")),
|
|
861
|
+
last_modified_digest=(
|
|
862
|
+
None
|
|
863
|
+
if hop.last_modified is None
|
|
864
|
+
else sha256_bytes(hop.last_modified.encode("utf-8"))
|
|
865
|
+
),
|
|
866
|
+
)
|
|
867
|
+
|
|
868
|
+
def to_dict(self) -> dict[str, object]:
|
|
869
|
+
return {
|
|
870
|
+
"http_status_code": self.http_status_code,
|
|
871
|
+
"etag_digest": self.etag_digest,
|
|
872
|
+
"last_modified_digest": self.last_modified_digest,
|
|
873
|
+
}
|
|
874
|
+
|
|
875
|
+
|
|
876
|
+
@dataclass(frozen=True)
|
|
877
|
+
class RetrievedBytes:
|
|
878
|
+
source_url: str
|
|
879
|
+
final_url: str
|
|
880
|
+
media_type: str
|
|
881
|
+
content: bytes
|
|
882
|
+
content_sha256: str
|
|
883
|
+
hops: tuple[RetrievalHop, ...]
|
|
884
|
+
transport_evidence_digest: str
|
|
885
|
+
complete_size_bytes: int | None = None
|
|
886
|
+
rate_observation: DatagovRateObservation | None = None
|
|
887
|
+
|
|
888
|
+
@property
|
|
889
|
+
def total_response_body_size_bytes(self) -> int:
|
|
890
|
+
"""Every response-body byte consumed in the exact transport hop evidence.
|
|
891
|
+
|
|
892
|
+
``len(content)`` is only the terminal response. Redirect response bodies consumed the
|
|
893
|
+
same transport budget, as do partial bodies from failed peer attempts, so both belong
|
|
894
|
+
in cumulative acquisition accounting. Deriving the total from the authenticated hop
|
|
895
|
+
tuple avoids a second summary fact that could disagree with its underlying evidence.
|
|
896
|
+
"""
|
|
897
|
+
|
|
898
|
+
return sum(
|
|
899
|
+
hop.response_body_size_bytes
|
|
900
|
+
+ sum(
|
|
901
|
+
attempt.response_body_size_bytes
|
|
902
|
+
for attempt in hop.peer_attempts[:-1]
|
|
903
|
+
if attempt.outcome == "transport_failure"
|
|
904
|
+
)
|
|
905
|
+
for hop in self.hops
|
|
906
|
+
)
|
|
907
|
+
|
|
908
|
+
def __post_init__(self) -> None:
|
|
909
|
+
if sha256_bytes(self.content) != self.content_sha256:
|
|
910
|
+
raise AcquisitionSecurityError(
|
|
911
|
+
"CONTENT_DIGEST",
|
|
912
|
+
"retrieved content digest does not match exact bytes",
|
|
913
|
+
)
|
|
914
|
+
expected = canonical_sha256([hop.to_dict() for hop in self.hops])
|
|
915
|
+
if expected != self.transport_evidence_digest:
|
|
916
|
+
raise AcquisitionSecurityError(
|
|
917
|
+
"TRANSPORT_EVIDENCE_DIGEST",
|
|
918
|
+
"transport evidence digest does not match the exact hop list",
|
|
919
|
+
)
|
|
920
|
+
if self.complete_size_bytes is not None and (
|
|
921
|
+
type(self.complete_size_bytes) is not int
|
|
922
|
+
or self.complete_size_bytes < len(self.content)
|
|
923
|
+
or self.complete_size_bytes > MAX_BYTE_RANGE_OFFSET + 1
|
|
924
|
+
):
|
|
925
|
+
raise AcquisitionSecurityError(
|
|
926
|
+
"CONTENT_RANGE",
|
|
927
|
+
"complete object size is outside the admitted range",
|
|
928
|
+
)
|
|
929
|
+
if self.rate_observation is not None and not isinstance(
|
|
930
|
+
self.rate_observation, DatagovRateObservation
|
|
931
|
+
):
|
|
932
|
+
raise AcquisitionSecurityError(
|
|
933
|
+
"DATAGOV_RATE", "rate observation must use the strict contract"
|
|
934
|
+
)
|
|
935
|
+
|
|
936
|
+
|
|
937
|
+
def probe_transport_for(retrieved: RetrievedBytes) -> ProbeTransport:
|
|
938
|
+
"""Seal what the terminal response of one retrieval stated.
|
|
939
|
+
|
|
940
|
+
Taken from the retained hop evidence rather than from a summary field, so what an
|
|
941
|
+
observation says the publisher answered is the same response the transport digest covers.
|
|
942
|
+
"""
|
|
943
|
+
|
|
944
|
+
if not retrieved.hops:
|
|
945
|
+
raise AcquisitionSecurityError(
|
|
946
|
+
"PROBE_TRANSPORT",
|
|
947
|
+
"a retrieval with no hop evidence has no terminal response",
|
|
948
|
+
)
|
|
949
|
+
return ProbeTransport.from_terminal_hop(retrieved.hops[-1])
|
|
950
|
+
|
|
951
|
+
|
|
952
|
+
class AcquisitionLimiter:
|
|
953
|
+
"""Bounded concurrency, plus a per-host minimum interval that is waited out."""
|
|
954
|
+
|
|
955
|
+
def __init__(
|
|
956
|
+
self,
|
|
957
|
+
*,
|
|
958
|
+
max_concurrency: int,
|
|
959
|
+
min_interval_seconds: float,
|
|
960
|
+
clock: Callable[[], float] = time.monotonic,
|
|
961
|
+
sleep: Callable[[float], None] = time.sleep,
|
|
962
|
+
) -> None:
|
|
963
|
+
# The interval is validated here as well as on ``RetrievalLimits``, because a limiter
|
|
964
|
+
# may be constructed directly and the interval is now slept on rather than raised on:
|
|
965
|
+
# an unbounded or non-numeric interval would be a hang inside our own worker rather
|
|
966
|
+
# than a refusal a caller sees. ``NaN`` fails this comparison and is refused with it.
|
|
967
|
+
if type(min_interval_seconds) not in {int, float} or not 0 <= min_interval_seconds <= 3_600:
|
|
968
|
+
raise AcquisitionSecurityError(
|
|
969
|
+
"RATE_LIMIT",
|
|
970
|
+
"min_interval_seconds must be in [0, 3600]",
|
|
971
|
+
)
|
|
972
|
+
self._semaphore = threading.BoundedSemaphore(max_concurrency)
|
|
973
|
+
self._min_interval_seconds = float(min_interval_seconds)
|
|
974
|
+
self._clock = clock
|
|
975
|
+
self._sleep = sleep
|
|
976
|
+
self._lock = threading.Lock()
|
|
977
|
+
self._last_request: dict[str, float] = {}
|
|
978
|
+
|
|
979
|
+
@property
|
|
980
|
+
def clock(self) -> Callable[[], float]:
|
|
981
|
+
"""The clock every deadline handed to :meth:`acquire` must be an instant on."""
|
|
982
|
+
|
|
983
|
+
return self._clock
|
|
984
|
+
|
|
985
|
+
@contextlib.contextmanager
|
|
986
|
+
def acquire(self, hostname: str, *, deadline: float) -> Iterator[None]:
|
|
987
|
+
"""Hold a concurrency slot, waiting out this host's minimum interval first.
|
|
988
|
+
|
|
989
|
+
Two limits, two dispositions, deliberately. An exhausted concurrency semaphore
|
|
990
|
+
fails fast: more work is in flight than this process admitted, which is a bug on our
|
|
991
|
+
side, and waiting would hide it. The per-host interval waits: it is politeness
|
|
992
|
+
towards a source that is behaving exactly as documented, and raising there turns a
|
|
993
|
+
healthy source into a failed run.
|
|
994
|
+
|
|
995
|
+
``deadline`` is an absolute instant on this limiter's own clock -- the retrieval
|
|
996
|
+
passes its start plus the ``total_timeout_seconds`` it already measures itself
|
|
997
|
+
against. The bound is not optional: ``min_interval_seconds`` is admitted up to an
|
|
998
|
+
hour and a slice acquisition may order 64 ranges, so an unbounded wait is a
|
|
999
|
+
multi-hour hang in our own worker. A wait that would cross the deadline is refused
|
|
1000
|
+
up front as ``TOTAL_TIMEOUT`` rather than slept past and regretted afterwards.
|
|
1001
|
+
"""
|
|
1002
|
+
|
|
1003
|
+
if type(deadline) not in {int, float} or deadline != deadline: # NaN is not a bound
|
|
1004
|
+
raise AcquisitionSecurityError(
|
|
1005
|
+
"TOTAL_TIMEOUT",
|
|
1006
|
+
"a wait must be bounded by a deadline on the retrieval's own clock",
|
|
1007
|
+
)
|
|
1008
|
+
if not self._semaphore.acquire(blocking=False):
|
|
1009
|
+
raise AcquisitionSecurityError(
|
|
1010
|
+
"CONCURRENCY_LIMIT",
|
|
1011
|
+
"acquisition concurrency limit is exhausted",
|
|
1012
|
+
)
|
|
1013
|
+
try:
|
|
1014
|
+
now = float(self._clock())
|
|
1015
|
+
with self._lock:
|
|
1016
|
+
last = self._last_request.get(hostname)
|
|
1017
|
+
scheduled = now if last is None else max(now, last + self._min_interval_seconds)
|
|
1018
|
+
if scheduled > float(deadline):
|
|
1019
|
+
raise AcquisitionSecurityError(
|
|
1020
|
+
"TOTAL_TIMEOUT",
|
|
1021
|
+
"the wait this source requires would outlive the retrieval's budget",
|
|
1022
|
+
)
|
|
1023
|
+
# Reserve the instant the request will actually be made, before the lock is
|
|
1024
|
+
# released: a second caller for the same host then queues behind this one
|
|
1025
|
+
# instead of racing it. The refusal above reserves nothing, so a retrieval
|
|
1026
|
+
# that gives up does not push the host's next slot further out. The lock is
|
|
1027
|
+
# never held across the wait, which would stall every other host.
|
|
1028
|
+
self._last_request[hostname] = scheduled
|
|
1029
|
+
if scheduled > now:
|
|
1030
|
+
self._sleep(scheduled - now)
|
|
1031
|
+
yield
|
|
1032
|
+
finally:
|
|
1033
|
+
self._semaphore.release()
|
|
1034
|
+
|
|
1035
|
+
|
|
1036
|
+
class PinnedHttpsRetriever:
|
|
1037
|
+
"""Redirect-aware retriever; every hop gets a new DNS and connection epoch."""
|
|
1038
|
+
|
|
1039
|
+
def __init__(
|
|
1040
|
+
self,
|
|
1041
|
+
*,
|
|
1042
|
+
resolver: Resolver,
|
|
1043
|
+
egress_policy: EgressPolicy,
|
|
1044
|
+
transport: PinnedTransport,
|
|
1045
|
+
limits: RetrievalLimits,
|
|
1046
|
+
limiter: AcquisitionLimiter | None = None,
|
|
1047
|
+
clock: Callable[[], float] = time.monotonic,
|
|
1048
|
+
sleep: Callable[[float], None] = time.sleep,
|
|
1049
|
+
) -> None:
|
|
1050
|
+
# A deadline is an instant rather than a duration, so a limiter reading a different
|
|
1051
|
+
# clock would compare it against a number from another timeline: the wait bound would
|
|
1052
|
+
# either fire immediately on every call or never fire at all. A shared limiter is
|
|
1053
|
+
# still allowed -- it must simply share this retrieval's clock, which the default
|
|
1054
|
+
# monotonic pairing does.
|
|
1055
|
+
if limiter is not None and limiter.clock is not clock:
|
|
1056
|
+
raise AcquisitionSecurityError(
|
|
1057
|
+
"LIMITER_CLOCK",
|
|
1058
|
+
"an injected limiter must observe the retrieval's own clock",
|
|
1059
|
+
)
|
|
1060
|
+
self._resolver = resolver
|
|
1061
|
+
self._egress_policy = egress_policy
|
|
1062
|
+
self._transport = transport
|
|
1063
|
+
self._limits = limits
|
|
1064
|
+
self._limiter = limiter or AcquisitionLimiter(
|
|
1065
|
+
max_concurrency=limits.max_concurrency,
|
|
1066
|
+
min_interval_seconds=limits.min_interval_seconds,
|
|
1067
|
+
clock=clock,
|
|
1068
|
+
sleep=sleep,
|
|
1069
|
+
)
|
|
1070
|
+
self._clock = clock
|
|
1071
|
+
|
|
1072
|
+
@property
|
|
1073
|
+
def limits(self) -> RetrievalLimits:
|
|
1074
|
+
"""The caps this retriever actually enforces. Read-only: a caller narrows, never mutates."""
|
|
1075
|
+
|
|
1076
|
+
return self._limits
|
|
1077
|
+
|
|
1078
|
+
def narrowed(self, limits: RetrievalLimits) -> PinnedHttpsRetriever:
|
|
1079
|
+
"""A retriever over the same transport whose transport-shape caps are ``limits``.
|
|
1080
|
+
|
|
1081
|
+
A caller that declares tighter caps for one fetch -- a harvester saying it accepts only
|
|
1082
|
+
``application/json`` under a 2 MiB cap, a health probe saying only a 1 MiB citation target
|
|
1083
|
+
-- has to be able to make those caps the ones the transport enforces. Passing them
|
|
1084
|
+
alongside a retriever that was built with looser ones leaves them inert, which is worse
|
|
1085
|
+
than not declaring them: the declaration reads as a control and enforces nothing.
|
|
1086
|
+
|
|
1087
|
+
Only the five transport-shape caps are taken from ``limits``: ``max_response_bytes``,
|
|
1088
|
+
``max_redirects``, ``max_requests``, ``max_ranges`` and ``allowed_media_types``. Timeouts,
|
|
1089
|
+
concurrency, rate interval and user agent stay this retriever's, because those are the
|
|
1090
|
+
coordinator's operational policy and not a per-fetch shape. The four numeric caps may not
|
|
1091
|
+
be widened, so this can only tighten what the coordinator configured, never escape it.
|
|
1092
|
+
|
|
1093
|
+
The allowlist term is the one place the rule is not plain subset, and the superset it is
|
|
1094
|
+
checked against is stated rather than implied: this retriever's own types **plus**
|
|
1095
|
+
:data:`METADATA_MEDIA_TYPES`. ``allowed_media_types`` defaults to the payload format table,
|
|
1096
|
+
which by construction can never carry a catalogue's structure message -- so a plain subset
|
|
1097
|
+
rule made the SDMX harvester's declared type unnarrowable from a default-configured
|
|
1098
|
+
retriever and refused every SDMX fetch and every non-JSON citation probe before a request
|
|
1099
|
+
was made. The metadata vocabulary is closed and lives in this module, so this admits exactly
|
|
1100
|
+
the types the transport itself names and nothing a caller supplies: ``text/html`` is still a
|
|
1101
|
+
widening, which is what keeps the bot-wall refusal a refusal.
|
|
1102
|
+
|
|
1103
|
+
The rate limiter is shared rather than rebuilt, so a narrowed retriever does not get a
|
|
1104
|
+
second concurrency budget for the same host.
|
|
1105
|
+
"""
|
|
1106
|
+
|
|
1107
|
+
if not isinstance(limits, RetrievalLimits):
|
|
1108
|
+
raise AcquisitionSecurityError(
|
|
1109
|
+
"RETRIEVAL_LIMITS",
|
|
1110
|
+
"limits must be a RetrievalLimits",
|
|
1111
|
+
)
|
|
1112
|
+
widened = []
|
|
1113
|
+
for name in (
|
|
1114
|
+
"max_response_bytes",
|
|
1115
|
+
"max_aggregate_response_bytes",
|
|
1116
|
+
"max_redirects",
|
|
1117
|
+
"max_requests",
|
|
1118
|
+
"max_ranges",
|
|
1119
|
+
):
|
|
1120
|
+
if getattr(limits, name) > getattr(self._limits, name):
|
|
1121
|
+
widened.append(name)
|
|
1122
|
+
narrowable = set(self._limits.allowed_media_types) | set(METADATA_MEDIA_TYPES)
|
|
1123
|
+
if not set(limits.allowed_media_types) <= narrowable:
|
|
1124
|
+
widened.append("allowed_media_types")
|
|
1125
|
+
if widened:
|
|
1126
|
+
raise AcquisitionSecurityError(
|
|
1127
|
+
"RETRIEVAL_LIMITS",
|
|
1128
|
+
f"narrowing may not widen {sorted(widened)}",
|
|
1129
|
+
)
|
|
1130
|
+
if limits == self._limits:
|
|
1131
|
+
return self
|
|
1132
|
+
effective = RetrievalLimits(
|
|
1133
|
+
max_response_bytes=limits.max_response_bytes,
|
|
1134
|
+
max_aggregate_response_bytes=limits.max_aggregate_response_bytes,
|
|
1135
|
+
max_redirects=limits.max_redirects,
|
|
1136
|
+
max_requests=limits.max_requests,
|
|
1137
|
+
max_ranges=limits.max_ranges,
|
|
1138
|
+
connect_timeout_seconds=self._limits.connect_timeout_seconds,
|
|
1139
|
+
read_timeout_seconds=self._limits.read_timeout_seconds,
|
|
1140
|
+
total_timeout_seconds=self._limits.total_timeout_seconds,
|
|
1141
|
+
max_concurrency=self._limits.max_concurrency,
|
|
1142
|
+
min_interval_seconds=self._limits.min_interval_seconds,
|
|
1143
|
+
allowed_media_types=limits.allowed_media_types,
|
|
1144
|
+
user_agent=self._limits.user_agent,
|
|
1145
|
+
)
|
|
1146
|
+
return PinnedHttpsRetriever(
|
|
1147
|
+
resolver=self._resolver,
|
|
1148
|
+
egress_policy=self._egress_policy,
|
|
1149
|
+
transport=self._transport,
|
|
1150
|
+
limits=effective,
|
|
1151
|
+
limiter=self._limiter,
|
|
1152
|
+
clock=self._clock,
|
|
1153
|
+
)
|
|
1154
|
+
|
|
1155
|
+
def retrieve(self, url: str, *, byte_range: ByteRange | None = None) -> RetrievedBytes:
|
|
1156
|
+
if byte_range is not None and not isinstance(byte_range, ByteRange):
|
|
1157
|
+
raise AcquisitionSecurityError(
|
|
1158
|
+
"BYTE_RANGE",
|
|
1159
|
+
"a slice is ordered only through a validated byte range",
|
|
1160
|
+
)
|
|
1161
|
+
return self._fetch(
|
|
1162
|
+
url,
|
|
1163
|
+
byte_range=byte_range,
|
|
1164
|
+
presence_probe=False,
|
|
1165
|
+
datagov_authorization=None,
|
|
1166
|
+
)
|
|
1167
|
+
|
|
1168
|
+
def retrieve_datagov_v4(
|
|
1169
|
+
self,
|
|
1170
|
+
url: str,
|
|
1171
|
+
authorization: DatagovV4Authorization,
|
|
1172
|
+
*,
|
|
1173
|
+
on_response_observed: (
|
|
1174
|
+
Callable[[int, int, DatagovRateObservation | None], None] | None
|
|
1175
|
+
) = None,
|
|
1176
|
+
) -> RetrievedBytes:
|
|
1177
|
+
"""Retrieve only the exact authenticated v4 endpoint with zero redirects."""
|
|
1178
|
+
|
|
1179
|
+
if not isinstance(authorization, DatagovV4Authorization):
|
|
1180
|
+
raise AcquisitionSecurityError(
|
|
1181
|
+
"DATAGOV_AUTHORIZATION", "strict Data.gov authorization is required"
|
|
1182
|
+
)
|
|
1183
|
+
authorization.header_for(url)
|
|
1184
|
+
return self._fetch(
|
|
1185
|
+
url,
|
|
1186
|
+
byte_range=None,
|
|
1187
|
+
presence_probe=False,
|
|
1188
|
+
datagov_authorization=authorization,
|
|
1189
|
+
datagov_response_observed=on_response_observed,
|
|
1190
|
+
)
|
|
1191
|
+
|
|
1192
|
+
def probe_presence(self, url: str) -> bool:
|
|
1193
|
+
"""Ask one address whether it is published, keep nothing, and answer yes or no.
|
|
1194
|
+
|
|
1195
|
+
This is the look half of look-then-lock. It orders exactly one byte, discards the
|
|
1196
|
+
response, and returns presence. Nothing is parsed, nothing is sealed, no digest is
|
|
1197
|
+
taken and no media type is reported: the only thing that leaves this call is a
|
|
1198
|
+
boolean.
|
|
1199
|
+
|
|
1200
|
+
Absence is read from the typed ``HTTP_NOT_FOUND`` code and from nothing else. A
|
|
1201
|
+
string match on the message would silently start reporting "not published yet" the
|
|
1202
|
+
first time an unrelated message was reworded to contain the same words, and that is
|
|
1203
|
+
the failure mode that turns a broken source into a permanently quiet one: the dataset
|
|
1204
|
+
stops updating, every run reports a clean no-new-run, and nobody is told. Every other
|
|
1205
|
+
refusal -- a server error, a TLS failure, a peer outside the pin, a source that
|
|
1206
|
+
ignored the range -- propagates, because a broken source must stay loud.
|
|
1207
|
+
|
|
1208
|
+
Presence probes admit any media type. GRIB objects commonly return
|
|
1209
|
+
``application/octet-stream``, which the normal retrieval allowlist rejects. Only this
|
|
1210
|
+
method can request the exemption; data retrieval still uses ``retrieve``.
|
|
1211
|
+
"""
|
|
1212
|
+
|
|
1213
|
+
try:
|
|
1214
|
+
self._fetch(
|
|
1215
|
+
url,
|
|
1216
|
+
byte_range=_PRESENCE_PROBE_RANGE,
|
|
1217
|
+
presence_probe=True,
|
|
1218
|
+
datagov_authorization=None,
|
|
1219
|
+
)
|
|
1220
|
+
except AcquisitionSecurityError as exc:
|
|
1221
|
+
if exc.code != HTTP_NOT_FOUND:
|
|
1222
|
+
raise
|
|
1223
|
+
return False
|
|
1224
|
+
return True
|
|
1225
|
+
|
|
1226
|
+
def retrieve_presence_evidence(self, url: str) -> RetrievedBytes:
|
|
1227
|
+
"""Probe one byte while retaining the exact measured response evidence.
|
|
1228
|
+
|
|
1229
|
+
Range/Reader composition seals this evidence and accounts every response body. The
|
|
1230
|
+
ordinary survey intentionally returns only booleans, so it continues to use
|
|
1231
|
+
:meth:`probe_presence`; this method exists for an acquisition that has already locked
|
|
1232
|
+
the candidate and therefore must retain, rather than discard, what it measured.
|
|
1233
|
+
"""
|
|
1234
|
+
|
|
1235
|
+
return self._fetch(
|
|
1236
|
+
url,
|
|
1237
|
+
byte_range=_PRESENCE_PROBE_RANGE,
|
|
1238
|
+
presence_probe=True,
|
|
1239
|
+
datagov_authorization=None,
|
|
1240
|
+
)
|
|
1241
|
+
|
|
1242
|
+
def _fetch(
|
|
1243
|
+
self,
|
|
1244
|
+
url: str,
|
|
1245
|
+
*,
|
|
1246
|
+
byte_range: ByteRange | None,
|
|
1247
|
+
presence_probe: bool,
|
|
1248
|
+
datagov_authorization: DatagovV4Authorization | None,
|
|
1249
|
+
datagov_response_observed: (
|
|
1250
|
+
Callable[[int, int, DatagovRateObservation | None], None] | None
|
|
1251
|
+
) = None,
|
|
1252
|
+
) -> RetrievedBytes:
|
|
1253
|
+
started = float(self._clock())
|
|
1254
|
+
current_url = url
|
|
1255
|
+
hops: list[RetrievalHop] = []
|
|
1256
|
+
aggregate_response_bytes = 0
|
|
1257
|
+
requests = 0
|
|
1258
|
+
deadline = started + self._limits.total_timeout_seconds
|
|
1259
|
+
redirect_budget = 0 if datagov_authorization is not None else self._limits.max_redirects
|
|
1260
|
+
for redirect_count in range(redirect_budget + 1):
|
|
1261
|
+
self._check_total_timeout(started)
|
|
1262
|
+
target = validate_public_https_url(
|
|
1263
|
+
current_url,
|
|
1264
|
+
resolver=self._resolver,
|
|
1265
|
+
egress_policy=self._egress_policy,
|
|
1266
|
+
)
|
|
1267
|
+
headers = {
|
|
1268
|
+
# A presence probe asks for anything, because it admits anything: sending the
|
|
1269
|
+
# acquisition allowlist would invite a 406 for a run that is published and
|
|
1270
|
+
# fine, which is the same false failure the admission exemption removes.
|
|
1271
|
+
"Accept": "*/*" if presence_probe else _accept_header(self._limits),
|
|
1272
|
+
"Accept-Encoding": "identity",
|
|
1273
|
+
"Connection": "close",
|
|
1274
|
+
"Host": urlsplit(target.normalized_url).netloc,
|
|
1275
|
+
"User-Agent": self._limits.user_agent,
|
|
1276
|
+
}
|
|
1277
|
+
if datagov_authorization is not None:
|
|
1278
|
+
name, value = datagov_authorization.header_for(target.normalized_url)
|
|
1279
|
+
headers[name] = value
|
|
1280
|
+
peer_attempts: list[PeerAttempt] = []
|
|
1281
|
+
response: TransportResponse | None = None
|
|
1282
|
+
for approved_ip in target.approved_ips:
|
|
1283
|
+
if requests >= self._limits.max_requests:
|
|
1284
|
+
raise PeerAttemptError(
|
|
1285
|
+
"REQUEST_LIMIT",
|
|
1286
|
+
"retrieval exceeded its request budget",
|
|
1287
|
+
tuple(peer_attempts),
|
|
1288
|
+
hops=tuple(hops),
|
|
1289
|
+
)
|
|
1290
|
+
requests += 1
|
|
1291
|
+
remaining_aggregate_bytes = (
|
|
1292
|
+
self._limits.max_aggregate_response_bytes - aggregate_response_bytes
|
|
1293
|
+
)
|
|
1294
|
+
try:
|
|
1295
|
+
attempt_limits = self._remaining_attempt_limits(
|
|
1296
|
+
deadline,
|
|
1297
|
+
remaining_aggregate_bytes=remaining_aggregate_bytes,
|
|
1298
|
+
)
|
|
1299
|
+
except AcquisitionSecurityError as exc:
|
|
1300
|
+
raise PeerAttemptError(
|
|
1301
|
+
exc.code,
|
|
1302
|
+
exc.detail,
|
|
1303
|
+
tuple(peer_attempts),
|
|
1304
|
+
hops=tuple(hops),
|
|
1305
|
+
) from None
|
|
1306
|
+
try:
|
|
1307
|
+
# One budget, one clock: every peer and limiter wait is bounded by the
|
|
1308
|
+
# same absolute instant. A fallback never starts a second DNS epoch.
|
|
1309
|
+
with self._limiter.acquire(target.hostname, deadline=deadline):
|
|
1310
|
+
response = self._transport.request(
|
|
1311
|
+
target,
|
|
1312
|
+
approved_ip=approved_ip,
|
|
1313
|
+
headers=headers,
|
|
1314
|
+
limits=attempt_limits,
|
|
1315
|
+
byte_range=byte_range,
|
|
1316
|
+
)
|
|
1317
|
+
except AcquisitionSecurityError as exc:
|
|
1318
|
+
if exc.code == "TRANSPORT_FAILURE" and not isinstance(
|
|
1319
|
+
exc, TransportFailureError
|
|
1320
|
+
):
|
|
1321
|
+
raise PeerAttemptError(
|
|
1322
|
+
"TRANSPORT_EVIDENCE",
|
|
1323
|
+
"transport failure omitted exact consumed response-body bytes",
|
|
1324
|
+
tuple(peer_attempts),
|
|
1325
|
+
hops=tuple(hops),
|
|
1326
|
+
) from None
|
|
1327
|
+
consumed_body_bytes = (
|
|
1328
|
+
exc.response_body_size_bytes
|
|
1329
|
+
if isinstance(exc, TransportFailureError)
|
|
1330
|
+
else 0
|
|
1331
|
+
)
|
|
1332
|
+
aggregate_response_bytes += consumed_body_bytes
|
|
1333
|
+
failure_code = exc.code
|
|
1334
|
+
failure_detail = exc.detail
|
|
1335
|
+
if (
|
|
1336
|
+
exc.code == "RESPONSE_LIMIT"
|
|
1337
|
+
and remaining_aggregate_bytes < self._limits.max_response_bytes
|
|
1338
|
+
):
|
|
1339
|
+
failure_code = "AGGREGATE_RESPONSE_LIMIT"
|
|
1340
|
+
failure_detail = "retrieval exceeded its aggregate response-body budget"
|
|
1341
|
+
peer_attempts.append(
|
|
1342
|
+
PeerAttempt(
|
|
1343
|
+
approved_ip=approved_ip,
|
|
1344
|
+
outcome="transport_failure",
|
|
1345
|
+
failure_code=failure_code,
|
|
1346
|
+
response_body_size_bytes=consumed_body_bytes,
|
|
1347
|
+
)
|
|
1348
|
+
)
|
|
1349
|
+
aggregate_exhausted = (
|
|
1350
|
+
aggregate_response_bytes >= self._limits.max_aggregate_response_bytes
|
|
1351
|
+
)
|
|
1352
|
+
if (
|
|
1353
|
+
failure_code == "TRANSPORT_FAILURE"
|
|
1354
|
+
and not aggregate_exhausted
|
|
1355
|
+
and approved_ip != target.approved_ips[-1]
|
|
1356
|
+
):
|
|
1357
|
+
continue
|
|
1358
|
+
if failure_code == "TRANSPORT_FAILURE" and aggregate_exhausted:
|
|
1359
|
+
raise PeerAttemptError(
|
|
1360
|
+
"AGGREGATE_RESPONSE_LIMIT",
|
|
1361
|
+
"failed response bodies exhausted the aggregate byte limit",
|
|
1362
|
+
tuple(peer_attempts),
|
|
1363
|
+
hops=tuple(hops),
|
|
1364
|
+
) from None
|
|
1365
|
+
detail = (
|
|
1366
|
+
"all resolver-approved peers failed before an HTTP response"
|
|
1367
|
+
if failure_code == "TRANSPORT_FAILURE"
|
|
1368
|
+
else failure_detail
|
|
1369
|
+
)
|
|
1370
|
+
raise PeerAttemptError(
|
|
1371
|
+
failure_code, detail, tuple(peer_attempts), hops=tuple(hops)
|
|
1372
|
+
) from None
|
|
1373
|
+
peer_attempts.append(
|
|
1374
|
+
PeerAttempt(
|
|
1375
|
+
approved_ip=approved_ip,
|
|
1376
|
+
outcome="response",
|
|
1377
|
+
failure_code=None,
|
|
1378
|
+
response_body_size_bytes=len(response.body),
|
|
1379
|
+
)
|
|
1380
|
+
)
|
|
1381
|
+
break
|
|
1382
|
+
if response is None: # pragma: no cover - ValidatedTarget forbids an empty peer set
|
|
1383
|
+
raise PeerAttemptError(
|
|
1384
|
+
"TRANSPORT_FAILURE",
|
|
1385
|
+
"all resolver-approved peers failed before an HTTP response",
|
|
1386
|
+
tuple(peer_attempts),
|
|
1387
|
+
hops=tuple(hops),
|
|
1388
|
+
)
|
|
1389
|
+
if datagov_response_observed is not None:
|
|
1390
|
+
datagov_response_observed(
|
|
1391
|
+
response.status,
|
|
1392
|
+
aggregate_response_bytes + len(response.body),
|
|
1393
|
+
None,
|
|
1394
|
+
)
|
|
1395
|
+
if (
|
|
1396
|
+
aggregate_response_bytes + len(response.body)
|
|
1397
|
+
> self._limits.max_aggregate_response_bytes
|
|
1398
|
+
):
|
|
1399
|
+
self._raise_peer_failure(
|
|
1400
|
+
"AGGREGATE_RESPONSE_LIMIT",
|
|
1401
|
+
"redirect and final response bodies exceeded the aggregate byte limit",
|
|
1402
|
+
peer_attempts,
|
|
1403
|
+
hops,
|
|
1404
|
+
)
|
|
1405
|
+
aggregate_response_bytes += len(response.body)
|
|
1406
|
+
try:
|
|
1407
|
+
self._check_total_timeout(started)
|
|
1408
|
+
except AcquisitionSecurityError as exc:
|
|
1409
|
+
self._raise_peer_failure(exc.code, exc.detail, peer_attempts, hops)
|
|
1410
|
+
if not response.tls_verified or not response.hostname_verified:
|
|
1411
|
+
self._raise_peer_failure(
|
|
1412
|
+
"TLS_VERIFICATION",
|
|
1413
|
+
"transport did not prove certificate and hostname verification",
|
|
1414
|
+
peer_attempts,
|
|
1415
|
+
hops,
|
|
1416
|
+
)
|
|
1417
|
+
try:
|
|
1418
|
+
peer = require_approved_peer(target, response.connected_peer)
|
|
1419
|
+
normalized_headers = _validate_response_headers(response.headers)
|
|
1420
|
+
except AcquisitionSecurityError as exc:
|
|
1421
|
+
self._raise_peer_failure(exc.code, exc.detail, peer_attempts, hops)
|
|
1422
|
+
if datagov_authorization is not None:
|
|
1423
|
+
try:
|
|
1424
|
+
datagov_authorization.reject_echoed_headers(normalized_headers)
|
|
1425
|
+
if response.status in REDIRECT_STATUSES:
|
|
1426
|
+
self._raise_peer_failure(
|
|
1427
|
+
"REDIRECT_LIMIT",
|
|
1428
|
+
"response exceeded the redirect limit",
|
|
1429
|
+
peer_attempts,
|
|
1430
|
+
hops,
|
|
1431
|
+
)
|
|
1432
|
+
datagov_authorization.reject_echoed_response(response.body)
|
|
1433
|
+
except AcquisitionSecurityError as exc:
|
|
1434
|
+
raise DatagovResponseRefused(
|
|
1435
|
+
exc.code,
|
|
1436
|
+
exc.detail,
|
|
1437
|
+
status=response.status,
|
|
1438
|
+
network_bytes=aggregate_response_bytes,
|
|
1439
|
+
) from None
|
|
1440
|
+
try:
|
|
1441
|
+
datagov_rate_observation = _datagov_rate_observation(response)
|
|
1442
|
+
except AcquisitionSecurityError:
|
|
1443
|
+
raise
|
|
1444
|
+
if datagov_response_observed is not None:
|
|
1445
|
+
datagov_response_observed(
|
|
1446
|
+
response.status,
|
|
1447
|
+
aggregate_response_bytes,
|
|
1448
|
+
datagov_rate_observation,
|
|
1449
|
+
)
|
|
1450
|
+
else:
|
|
1451
|
+
datagov_rate_observation = None
|
|
1452
|
+
if len(response.body) > attempt_limits.max_response_bytes:
|
|
1453
|
+
limit_code = (
|
|
1454
|
+
"AGGREGATE_RESPONSE_LIMIT"
|
|
1455
|
+
if remaining_aggregate_bytes < self._limits.max_response_bytes
|
|
1456
|
+
else "RESPONSE_LIMIT"
|
|
1457
|
+
)
|
|
1458
|
+
self._raise_peer_failure(
|
|
1459
|
+
limit_code,
|
|
1460
|
+
"response bodies exceeded the active cumulative byte limit",
|
|
1461
|
+
peer_attempts,
|
|
1462
|
+
hops,
|
|
1463
|
+
)
|
|
1464
|
+
content_length = self._response_header(response, "content-length", peer_attempts, hops)
|
|
1465
|
+
declared_length: int | None = None
|
|
1466
|
+
if content_length is not None:
|
|
1467
|
+
# The same canonical spelling ``Content-Range`` is held to, and for the same
|
|
1468
|
+
# reason: ``int`` accepts ``+10`` and ``10_000`` and a leading zero, none of
|
|
1469
|
+
# which is a Content-Length, and one declared length should have one spelling
|
|
1470
|
+
# in the sealed evidence. Nothing downstream trusted the parsed value over
|
|
1471
|
+
# ``len(response.body)``, so this closes an inconsistency rather than a hole --
|
|
1472
|
+
# but the inconsistency is on the surface where exactness is the whole control.
|
|
1473
|
+
if not _DECIMAL_ONLY.fullmatch(content_length):
|
|
1474
|
+
self._raise_peer_failure(
|
|
1475
|
+
"CONTENT_LENGTH",
|
|
1476
|
+
"response content length is invalid",
|
|
1477
|
+
peer_attempts,
|
|
1478
|
+
hops,
|
|
1479
|
+
)
|
|
1480
|
+
declared_length = int(content_length)
|
|
1481
|
+
if declared_length < 0 or declared_length > attempt_limits.max_response_bytes:
|
|
1482
|
+
self._raise_peer_failure(
|
|
1483
|
+
"RESPONSE_LIMIT",
|
|
1484
|
+
"declared response length exceeds the remaining cumulative limit",
|
|
1485
|
+
peer_attempts,
|
|
1486
|
+
hops,
|
|
1487
|
+
)
|
|
1488
|
+
if declared_length != len(response.body):
|
|
1489
|
+
self._raise_peer_failure(
|
|
1490
|
+
"CONTENT_LENGTH",
|
|
1491
|
+
"response bytes do not match declared content length",
|
|
1492
|
+
peer_attempts,
|
|
1493
|
+
hops,
|
|
1494
|
+
)
|
|
1495
|
+
content_encoding = (
|
|
1496
|
+
(self._response_header(response, "content-encoding", peer_attempts, hops) or "")
|
|
1497
|
+
.lower()
|
|
1498
|
+
.strip()
|
|
1499
|
+
)
|
|
1500
|
+
if content_encoding not in _ALLOWED_RESPONSE_ENCODINGS:
|
|
1501
|
+
self._raise_peer_failure(
|
|
1502
|
+
"CONTENT_ENCODING",
|
|
1503
|
+
"compressed network responses are disabled at this boundary",
|
|
1504
|
+
peer_attempts,
|
|
1505
|
+
hops,
|
|
1506
|
+
)
|
|
1507
|
+
observed_content_range = self._response_header(
|
|
1508
|
+
response, "content-range", peer_attempts, hops
|
|
1509
|
+
)
|
|
1510
|
+
hop = RetrievalHop(
|
|
1511
|
+
url=target.normalized_url,
|
|
1512
|
+
resolution_digest=target.resolution_digest,
|
|
1513
|
+
connected_peer=peer,
|
|
1514
|
+
status=response.status,
|
|
1515
|
+
response_headers_digest=canonical_sha256(normalized_headers),
|
|
1516
|
+
response_body_size_bytes=len(response.body),
|
|
1517
|
+
response_body_sha256=sha256_bytes(response.body),
|
|
1518
|
+
requested_range=byte_range.header_value if byte_range is not None else None,
|
|
1519
|
+
observed_content_range=observed_content_range,
|
|
1520
|
+
etag=self._response_header(response, "etag", peer_attempts, hops),
|
|
1521
|
+
last_modified=self._response_header(response, "last-modified", peer_attempts, hops),
|
|
1522
|
+
peer_attempts=tuple(peer_attempts),
|
|
1523
|
+
)
|
|
1524
|
+
hops.append(hop)
|
|
1525
|
+
# The delivery check. A slice is delivered only when the server answered 206,
|
|
1526
|
+
# named exactly the span that was ordered, and shipped exactly that many bytes.
|
|
1527
|
+
# It sits here, with the other response-integrity controls (status, declared
|
|
1528
|
+
# length, encoding, media type, peer), because a response must clear all of them
|
|
1529
|
+
# before anything downstream is allowed to treat it as content -- and it runs
|
|
1530
|
+
# before the redirect branch below, because a ranged request is never followed.
|
|
1531
|
+
#
|
|
1532
|
+
# What this check cannot do: the content range can match, the three lengths can
|
|
1533
|
+
# agree, and the bytes can still be the wrong bytes -- a stale cache, a re-issued
|
|
1534
|
+
# object, a mirror that skewed. HTTP carries no evidence that would detect that,
|
|
1535
|
+
# and nothing here closes that case. Two controls behind the sandbox boundary
|
|
1536
|
+
# divide what is left, and neither is the other. ``check_grib_message`` proves
|
|
1537
|
+
# the framing, which is what a window at the wrong offset breaks.
|
|
1538
|
+
# ``check_grib_reference_time`` proves the delivered message came from the model
|
|
1539
|
+
# run the order named, which is what a re-issued object or a lagging mirror
|
|
1540
|
+
# breaks; it runs only when the order named a run. A different object of the
|
|
1541
|
+
# *same* run is covered by none of the three, and ``acquisition.ranges`` says so
|
|
1542
|
+
# where those two are defined.
|
|
1543
|
+
complete_size_bytes: int | None = None
|
|
1544
|
+
if byte_range is not None:
|
|
1545
|
+
if response.status in REDIRECT_STATUSES:
|
|
1546
|
+
self._raise_peer_failure(
|
|
1547
|
+
"REDIRECT_ON_RANGE",
|
|
1548
|
+
"a ranged request is never followed through a redirect",
|
|
1549
|
+
peer_attempts,
|
|
1550
|
+
hops,
|
|
1551
|
+
)
|
|
1552
|
+
if 200 <= response.status <= 299 and response.status != PARTIAL_CONTENT_STATUS:
|
|
1553
|
+
self._raise_peer_failure(
|
|
1554
|
+
"RANGE_IGNORED",
|
|
1555
|
+
f"source answered a ranged request with status {response.status}",
|
|
1556
|
+
peer_attempts,
|
|
1557
|
+
hops,
|
|
1558
|
+
)
|
|
1559
|
+
if response.status == PARTIAL_CONTENT_STATUS:
|
|
1560
|
+
content_type = self._response_header(
|
|
1561
|
+
response, "content-type", peer_attempts, hops
|
|
1562
|
+
)
|
|
1563
|
+
try:
|
|
1564
|
+
parsed_media_type = _media_type(content_type)
|
|
1565
|
+
except AcquisitionSecurityError as exc:
|
|
1566
|
+
self._raise_peer_failure(exc.code, exc.detail, peer_attempts, hops)
|
|
1567
|
+
if parsed_media_type == _MULTIPART_BYTERANGES:
|
|
1568
|
+
self._raise_peer_failure(
|
|
1569
|
+
"MULTIPART_RANGE",
|
|
1570
|
+
"multipart byte-range responses are refused at this boundary",
|
|
1571
|
+
peer_attempts,
|
|
1572
|
+
hops,
|
|
1573
|
+
)
|
|
1574
|
+
try:
|
|
1575
|
+
first_byte, last_byte, complete_size_bytes = _parse_content_range(
|
|
1576
|
+
observed_content_range
|
|
1577
|
+
)
|
|
1578
|
+
except AcquisitionSecurityError as exc:
|
|
1579
|
+
self._raise_peer_failure(exc.code, exc.detail, peer_attempts, hops)
|
|
1580
|
+
if first_byte != byte_range.first_byte or last_byte != byte_range.last_byte:
|
|
1581
|
+
self._raise_peer_failure(
|
|
1582
|
+
"RANGE_MISMATCH",
|
|
1583
|
+
"response content range is not the range that was ordered",
|
|
1584
|
+
peer_attempts,
|
|
1585
|
+
hops,
|
|
1586
|
+
)
|
|
1587
|
+
delivered_length = last_byte - first_byte + 1
|
|
1588
|
+
# Three lengths must agree: the span the server named, the length it
|
|
1589
|
+
# declared, and the bytes that arrived. The declared-versus-body leg is
|
|
1590
|
+
# already enforced above; it is restated against the span here so the
|
|
1591
|
+
# whole agreement is stated in one place rather than inferred across two.
|
|
1592
|
+
if delivered_length != len(response.body) or (
|
|
1593
|
+
declared_length is not None and declared_length != delivered_length
|
|
1594
|
+
):
|
|
1595
|
+
self._raise_peer_failure(
|
|
1596
|
+
"RANGE_LENGTH",
|
|
1597
|
+
"response bytes do not match the content range they claim",
|
|
1598
|
+
peer_attempts,
|
|
1599
|
+
hops,
|
|
1600
|
+
)
|
|
1601
|
+
elif response.status == PARTIAL_CONTENT_STATUS:
|
|
1602
|
+
self._raise_peer_failure(
|
|
1603
|
+
"UNSOLICITED_RANGE",
|
|
1604
|
+
"source returned partial content for a request that ordered no range",
|
|
1605
|
+
peer_attempts,
|
|
1606
|
+
hops,
|
|
1607
|
+
)
|
|
1608
|
+
if response.status in REDIRECT_STATUSES:
|
|
1609
|
+
location = self._response_header(response, _REDIRECT_HEADER, peer_attempts, hops)
|
|
1610
|
+
if not location:
|
|
1611
|
+
self._raise_peer_failure(
|
|
1612
|
+
"REDIRECT_LOCATION",
|
|
1613
|
+
"redirect response has no location",
|
|
1614
|
+
peer_attempts,
|
|
1615
|
+
hops,
|
|
1616
|
+
)
|
|
1617
|
+
if redirect_count >= redirect_budget:
|
|
1618
|
+
self._raise_peer_failure(
|
|
1619
|
+
"REDIRECT_LIMIT",
|
|
1620
|
+
"response exceeded the redirect limit",
|
|
1621
|
+
peer_attempts,
|
|
1622
|
+
hops,
|
|
1623
|
+
)
|
|
1624
|
+
current_url = urljoin(target.normalized_url, location)
|
|
1625
|
+
continue
|
|
1626
|
+
# A 404 gets its own code. This is not a softening -- the retrieval still fails
|
|
1627
|
+
# closed here -- but an availability survey has to tell "this cycle is not
|
|
1628
|
+
# published yet" from "this source is broken", and it must do that from a typed
|
|
1629
|
+
# code rather than by matching an error message. Every other status stays
|
|
1630
|
+
# generic.
|
|
1631
|
+
datagov_non_success = datagov_authorization is not None and response.status in {
|
|
1632
|
+
403,
|
|
1633
|
+
429,
|
|
1634
|
+
}
|
|
1635
|
+
if response.status == NOT_FOUND_STATUS:
|
|
1636
|
+
self._raise_peer_failure(
|
|
1637
|
+
HTTP_NOT_FOUND,
|
|
1638
|
+
f"source returned HTTP status {response.status}",
|
|
1639
|
+
peer_attempts,
|
|
1640
|
+
hops,
|
|
1641
|
+
)
|
|
1642
|
+
if not 200 <= response.status <= 299 and not datagov_non_success:
|
|
1643
|
+
self._raise_peer_failure(
|
|
1644
|
+
"HTTP_STATUS",
|
|
1645
|
+
f"source returned HTTP status {response.status}",
|
|
1646
|
+
peer_attempts,
|
|
1647
|
+
hops,
|
|
1648
|
+
)
|
|
1649
|
+
content_type = self._response_header(response, "content-type", peer_attempts, hops)
|
|
1650
|
+
try:
|
|
1651
|
+
media_type = _media_type(content_type)
|
|
1652
|
+
except AcquisitionSecurityError as exc:
|
|
1653
|
+
self._raise_peer_failure(exc.code, exc.detail, peer_attempts, hops)
|
|
1654
|
+
# Exempted for a presence probe only, and only because a probe keeps nothing: see
|
|
1655
|
+
# ``probe_presence``. Every acquiring call reaches here through ``retrieve``,
|
|
1656
|
+
# where ``presence_probe`` is a literal ``False`` rather than an argument.
|
|
1657
|
+
if not presence_probe and media_type not in self._limits.allowed_media_types:
|
|
1658
|
+
self._raise_peer_failure(
|
|
1659
|
+
"MEDIA_TYPE",
|
|
1660
|
+
f"response media type {media_type!r} is not allowlisted",
|
|
1661
|
+
peer_attempts,
|
|
1662
|
+
hops,
|
|
1663
|
+
)
|
|
1664
|
+
evidence_digest = canonical_sha256([item.to_dict() for item in hops])
|
|
1665
|
+
return RetrievedBytes(
|
|
1666
|
+
source_url=url,
|
|
1667
|
+
final_url=target.normalized_url,
|
|
1668
|
+
media_type=media_type,
|
|
1669
|
+
content=response.body,
|
|
1670
|
+
content_sha256=sha256_bytes(response.body),
|
|
1671
|
+
hops=tuple(hops),
|
|
1672
|
+
transport_evidence_digest=evidence_digest,
|
|
1673
|
+
complete_size_bytes=complete_size_bytes,
|
|
1674
|
+
rate_observation=datagov_rate_observation,
|
|
1675
|
+
)
|
|
1676
|
+
raise AcquisitionSecurityError("REDIRECT_LIMIT", "redirect loop did not terminate")
|
|
1677
|
+
|
|
1678
|
+
def _check_total_timeout(self, started: float) -> None:
|
|
1679
|
+
if float(self._clock()) - started > self._limits.total_timeout_seconds:
|
|
1680
|
+
raise AcquisitionSecurityError(
|
|
1681
|
+
"TOTAL_TIMEOUT",
|
|
1682
|
+
"retrieval exceeded its total time budget",
|
|
1683
|
+
)
|
|
1684
|
+
|
|
1685
|
+
def _remaining_attempt_limits(
|
|
1686
|
+
self,
|
|
1687
|
+
deadline: float,
|
|
1688
|
+
*,
|
|
1689
|
+
remaining_aggregate_bytes: int,
|
|
1690
|
+
) -> RetrievalLimits:
|
|
1691
|
+
remaining = deadline - float(self._clock())
|
|
1692
|
+
if remaining <= 0:
|
|
1693
|
+
raise AcquisitionSecurityError(
|
|
1694
|
+
"TOTAL_TIMEOUT",
|
|
1695
|
+
"retrieval exceeded its total time budget",
|
|
1696
|
+
)
|
|
1697
|
+
if remaining_aggregate_bytes <= 0:
|
|
1698
|
+
raise AcquisitionSecurityError(
|
|
1699
|
+
"AGGREGATE_RESPONSE_LIMIT",
|
|
1700
|
+
"retrieval exhausted its aggregate response-body budget",
|
|
1701
|
+
)
|
|
1702
|
+
return replace(
|
|
1703
|
+
self._limits,
|
|
1704
|
+
max_response_bytes=min(self._limits.max_response_bytes, remaining_aggregate_bytes),
|
|
1705
|
+
max_aggregate_response_bytes=remaining_aggregate_bytes,
|
|
1706
|
+
connect_timeout_seconds=min(self._limits.connect_timeout_seconds, remaining),
|
|
1707
|
+
read_timeout_seconds=min(self._limits.read_timeout_seconds, remaining),
|
|
1708
|
+
total_timeout_seconds=remaining,
|
|
1709
|
+
)
|
|
1710
|
+
|
|
1711
|
+
@staticmethod
|
|
1712
|
+
def _raise_peer_failure(
|
|
1713
|
+
code: str,
|
|
1714
|
+
detail: str,
|
|
1715
|
+
peer_attempts: list[PeerAttempt],
|
|
1716
|
+
hops: list[RetrievalHop],
|
|
1717
|
+
) -> None:
|
|
1718
|
+
latest = peer_attempts[-1]
|
|
1719
|
+
peer_attempts[-1] = PeerAttempt(
|
|
1720
|
+
approved_ip=latest.approved_ip,
|
|
1721
|
+
outcome="response_failure",
|
|
1722
|
+
failure_code=code,
|
|
1723
|
+
response_body_size_bytes=latest.response_body_size_bytes,
|
|
1724
|
+
)
|
|
1725
|
+
raise PeerAttemptError(code, detail, tuple(peer_attempts), hops=tuple(hops))
|
|
1726
|
+
|
|
1727
|
+
def _response_header(
|
|
1728
|
+
self,
|
|
1729
|
+
response: TransportResponse,
|
|
1730
|
+
name: str,
|
|
1731
|
+
peer_attempts: list[PeerAttempt],
|
|
1732
|
+
hops: list[RetrievalHop],
|
|
1733
|
+
) -> str | None:
|
|
1734
|
+
try:
|
|
1735
|
+
return response.header(name)
|
|
1736
|
+
except AcquisitionSecurityError as exc:
|
|
1737
|
+
self._raise_peer_failure(exc.code, exc.detail, peer_attempts, hops)
|
|
1738
|
+
|
|
1739
|
+
|
|
1740
|
+
class _PinnedHTTPSConnection(http.client.HTTPSConnection):
|
|
1741
|
+
"""One non-pooled connection to a validated IP with hostname TLS verification."""
|
|
1742
|
+
|
|
1743
|
+
def __init__(
|
|
1744
|
+
self,
|
|
1745
|
+
*,
|
|
1746
|
+
hostname: str,
|
|
1747
|
+
approved_ip: str,
|
|
1748
|
+
port: int,
|
|
1749
|
+
timeout: float,
|
|
1750
|
+
context: ssl.SSLContext,
|
|
1751
|
+
) -> None:
|
|
1752
|
+
super().__init__(host=hostname, port=port, timeout=timeout, context=context)
|
|
1753
|
+
self._approved_ip = approved_ip
|
|
1754
|
+
self.connected_peer: str | None = None
|
|
1755
|
+
|
|
1756
|
+
def connect(self) -> None:
|
|
1757
|
+
raw_socket = socket.create_connection(
|
|
1758
|
+
(self._approved_ip, self.port),
|
|
1759
|
+
timeout=self.timeout,
|
|
1760
|
+
)
|
|
1761
|
+
try:
|
|
1762
|
+
peer = raw_socket.getpeername()[0]
|
|
1763
|
+
self.connected_peer = require_approved_peer(
|
|
1764
|
+
ValidatedTarget(
|
|
1765
|
+
original_url=f"https://{self.host}/",
|
|
1766
|
+
normalized_url=f"https://{self.host}/",
|
|
1767
|
+
hostname=self.host,
|
|
1768
|
+
port=self.port,
|
|
1769
|
+
approved_ips=(self._approved_ip,),
|
|
1770
|
+
resolution_digest=canonical_sha256(
|
|
1771
|
+
{
|
|
1772
|
+
"hostname": self.host,
|
|
1773
|
+
"port": self.port,
|
|
1774
|
+
"approved_ips": [self._approved_ip],
|
|
1775
|
+
}
|
|
1776
|
+
),
|
|
1777
|
+
),
|
|
1778
|
+
peer,
|
|
1779
|
+
)
|
|
1780
|
+
self.sock = self._context.wrap_socket(raw_socket, server_hostname=self.host)
|
|
1781
|
+
except BaseException:
|
|
1782
|
+
raw_socket.close()
|
|
1783
|
+
raise
|
|
1784
|
+
|
|
1785
|
+
|
|
1786
|
+
class StdlibPinnedTransport:
|
|
1787
|
+
"""Production transport with no environment-proxy integration or connection pooling."""
|
|
1788
|
+
|
|
1789
|
+
def __init__(self, *, ssl_context: ssl.SSLContext | None = None) -> None:
|
|
1790
|
+
self._context = ssl_context or ssl.create_default_context()
|
|
1791
|
+
if (
|
|
1792
|
+
self._context.check_hostname is not True
|
|
1793
|
+
or self._context.verify_mode != ssl.CERT_REQUIRED
|
|
1794
|
+
):
|
|
1795
|
+
raise AcquisitionSecurityError(
|
|
1796
|
+
"TLS_CONTEXT",
|
|
1797
|
+
"TLS context must require certificate and hostname verification",
|
|
1798
|
+
)
|
|
1799
|
+
|
|
1800
|
+
def request(
|
|
1801
|
+
self,
|
|
1802
|
+
target: ValidatedTarget,
|
|
1803
|
+
*,
|
|
1804
|
+
approved_ip: str,
|
|
1805
|
+
headers: Mapping[str, str],
|
|
1806
|
+
limits: RetrievalLimits,
|
|
1807
|
+
byte_range: ByteRange | None = None,
|
|
1808
|
+
) -> TransportResponse:
|
|
1809
|
+
outgoing = dict(headers)
|
|
1810
|
+
if byte_range is not None:
|
|
1811
|
+
outgoing["Range"] = byte_range.header_value
|
|
1812
|
+
attempt_deadline = time.monotonic() + limits.total_timeout_seconds
|
|
1813
|
+
connection = _PinnedHTTPSConnection(
|
|
1814
|
+
hostname=target.hostname,
|
|
1815
|
+
approved_ip=approved_ip,
|
|
1816
|
+
port=target.port,
|
|
1817
|
+
timeout=min(limits.connect_timeout_seconds, limits.total_timeout_seconds),
|
|
1818
|
+
context=self._context,
|
|
1819
|
+
)
|
|
1820
|
+
total = 0
|
|
1821
|
+
try:
|
|
1822
|
+
connection.request("GET", target.request_target, headers=outgoing)
|
|
1823
|
+
if connection.sock is not None:
|
|
1824
|
+
connection.sock.settimeout(
|
|
1825
|
+
_remaining_socket_timeout(
|
|
1826
|
+
attempt_deadline,
|
|
1827
|
+
configured=limits.read_timeout_seconds,
|
|
1828
|
+
)
|
|
1829
|
+
)
|
|
1830
|
+
response = connection.getresponse()
|
|
1831
|
+
raw_headers = tuple(
|
|
1832
|
+
(key.lower(), value.strip()) for key, value in response.getheaders()
|
|
1833
|
+
)
|
|
1834
|
+
_validate_response_headers(raw_headers)
|
|
1835
|
+
content_length = next(
|
|
1836
|
+
(value for key, value in raw_headers if key == "content-length"),
|
|
1837
|
+
None,
|
|
1838
|
+
)
|
|
1839
|
+
if content_length is not None:
|
|
1840
|
+
try:
|
|
1841
|
+
if int(content_length) > limits.max_response_bytes:
|
|
1842
|
+
raise AcquisitionSecurityError(
|
|
1843
|
+
"RESPONSE_LIMIT",
|
|
1844
|
+
"declared response length exceeds the configured limit",
|
|
1845
|
+
)
|
|
1846
|
+
except ValueError:
|
|
1847
|
+
raise AcquisitionSecurityError(
|
|
1848
|
+
"CONTENT_LENGTH",
|
|
1849
|
+
"response content length is invalid",
|
|
1850
|
+
) from None
|
|
1851
|
+
chunks: list[bytes] = []
|
|
1852
|
+
while True:
|
|
1853
|
+
if connection.sock is not None:
|
|
1854
|
+
connection.sock.settimeout(
|
|
1855
|
+
_remaining_socket_timeout(
|
|
1856
|
+
attempt_deadline,
|
|
1857
|
+
configured=limits.read_timeout_seconds,
|
|
1858
|
+
)
|
|
1859
|
+
)
|
|
1860
|
+
chunk = response.read(min(65_536, limits.max_response_bytes + 1 - total))
|
|
1861
|
+
if not chunk:
|
|
1862
|
+
break
|
|
1863
|
+
total += len(chunk)
|
|
1864
|
+
if total > limits.max_response_bytes:
|
|
1865
|
+
raise AcquisitionSecurityError(
|
|
1866
|
+
"RESPONSE_LIMIT",
|
|
1867
|
+
"response exceeded the configured byte limit",
|
|
1868
|
+
)
|
|
1869
|
+
chunks.append(chunk)
|
|
1870
|
+
peer = connection.connected_peer
|
|
1871
|
+
if peer is None:
|
|
1872
|
+
raise AcquisitionSecurityError(
|
|
1873
|
+
"PEER_ADDRESS",
|
|
1874
|
+
"transport did not record its connected peer",
|
|
1875
|
+
)
|
|
1876
|
+
return TransportResponse(
|
|
1877
|
+
status=response.status,
|
|
1878
|
+
headers=raw_headers,
|
|
1879
|
+
body=b"".join(chunks),
|
|
1880
|
+
connected_peer=peer,
|
|
1881
|
+
tls_verified=True,
|
|
1882
|
+
hostname_verified=True,
|
|
1883
|
+
)
|
|
1884
|
+
except ssl.SSLError as exc:
|
|
1885
|
+
raise AcquisitionSecurityError(
|
|
1886
|
+
"TLS_VERIFICATION",
|
|
1887
|
+
f"HTTPS TLS verification failed: {type(exc).__name__}",
|
|
1888
|
+
) from None
|
|
1889
|
+
except http.client.IncompleteRead as exc:
|
|
1890
|
+
partial = exc.partial if isinstance(exc.partial, bytes) else bytes(exc.partial)
|
|
1891
|
+
raise TransportFailureError(
|
|
1892
|
+
f"HTTPS peer disconnected during a response: {type(exc).__name__}",
|
|
1893
|
+
response_body_size_bytes=total + len(partial),
|
|
1894
|
+
) from None
|
|
1895
|
+
except http.client.RemoteDisconnected as exc:
|
|
1896
|
+
raise TransportFailureError(
|
|
1897
|
+
f"HTTPS peer disconnected before a response: {type(exc).__name__}",
|
|
1898
|
+
response_body_size_bytes=total,
|
|
1899
|
+
) from None
|
|
1900
|
+
except http.client.HTTPException as exc:
|
|
1901
|
+
raise AcquisitionSecurityError(
|
|
1902
|
+
"RESPONSE_HEADERS",
|
|
1903
|
+
f"HTTPS response protocol failed: {type(exc).__name__}",
|
|
1904
|
+
) from None
|
|
1905
|
+
except OSError as exc:
|
|
1906
|
+
raise TransportFailureError(
|
|
1907
|
+
f"HTTPS transport failed: {type(exc).__name__}",
|
|
1908
|
+
response_body_size_bytes=total,
|
|
1909
|
+
) from None
|
|
1910
|
+
finally:
|
|
1911
|
+
connection.close()
|
|
1912
|
+
|
|
1913
|
+
|
|
1914
|
+
def _remaining_socket_timeout(deadline: float, *, configured: float) -> float:
|
|
1915
|
+
remaining = deadline - time.monotonic()
|
|
1916
|
+
if remaining <= 0:
|
|
1917
|
+
raise AcquisitionSecurityError(
|
|
1918
|
+
"TOTAL_TIMEOUT",
|
|
1919
|
+
"retrieval exceeded its total time budget",
|
|
1920
|
+
)
|
|
1921
|
+
return min(configured, remaining)
|
|
1922
|
+
|
|
1923
|
+
|
|
1924
|
+
def _datagov_rate_observation(response: TransportResponse) -> DatagovRateObservation | None:
|
|
1925
|
+
"""Parse only the bounded numeric Data.gov rate headers; retain no arbitrary headers."""
|
|
1926
|
+
|
|
1927
|
+
limit = response.header("x-ratelimit-limit")
|
|
1928
|
+
remaining = response.header("x-ratelimit-remaining")
|
|
1929
|
+
retry_after = response.header("retry-after")
|
|
1930
|
+
reset = response.header("x-ratelimit-reset")
|
|
1931
|
+
if limit is None and remaining is None and retry_after is None and reset is None:
|
|
1932
|
+
return None
|
|
1933
|
+
if limit is None or remaining is None:
|
|
1934
|
+
raise AcquisitionSecurityError(
|
|
1935
|
+
"DATAGOV_RATE", "Data.gov rate limit and remaining headers must appear together"
|
|
1936
|
+
)
|
|
1937
|
+
|
|
1938
|
+
def bounded_decimal(value: str, *, name: str, maximum: int) -> int:
|
|
1939
|
+
if not _DECIMAL_ONLY.fullmatch(value):
|
|
1940
|
+
raise AcquisitionSecurityError("DATAGOV_RATE", f"{name} is not a bounded integer")
|
|
1941
|
+
parsed = int(value)
|
|
1942
|
+
if parsed > maximum:
|
|
1943
|
+
raise AcquisitionSecurityError("DATAGOV_RATE", f"{name} is outside its bound")
|
|
1944
|
+
return parsed
|
|
1945
|
+
|
|
1946
|
+
return DatagovRateObservation(
|
|
1947
|
+
limit=bounded_decimal(limit, name="X-RateLimit-Limit", maximum=1_000_000_000),
|
|
1948
|
+
remaining=bounded_decimal(remaining, name="X-RateLimit-Remaining", maximum=1_000_000_000),
|
|
1949
|
+
retry_after_seconds=(
|
|
1950
|
+
None
|
|
1951
|
+
if retry_after is None
|
|
1952
|
+
else bounded_decimal(
|
|
1953
|
+
retry_after,
|
|
1954
|
+
name="Retry-After",
|
|
1955
|
+
maximum=MAX_DATAGOV_RATE_SECONDS,
|
|
1956
|
+
)
|
|
1957
|
+
),
|
|
1958
|
+
reset_epoch_seconds=(
|
|
1959
|
+
None
|
|
1960
|
+
if reset is None
|
|
1961
|
+
else bounded_decimal(reset, name="X-RateLimit-Reset", maximum=(1 << 53) - 1)
|
|
1962
|
+
),
|
|
1963
|
+
)
|
|
1964
|
+
|
|
1965
|
+
|
|
1966
|
+
def seal_content_addressed_snapshot(
|
|
1967
|
+
root: Path,
|
|
1968
|
+
*,
|
|
1969
|
+
content: bytes,
|
|
1970
|
+
suffix: str,
|
|
1971
|
+
) -> Path:
|
|
1972
|
+
"""Seal exact bytes, refusing a digest withdrawn by governed deletion."""
|
|
1973
|
+
|
|
1974
|
+
if not root.is_absolute():
|
|
1975
|
+
raise AcquisitionSecurityError("SNAPSHOT_ROOT", "snapshot root must be absolute")
|
|
1976
|
+
if not _is_allowlisted_snapshot_suffix(suffix):
|
|
1977
|
+
raise AcquisitionSecurityError("SNAPSHOT_SUFFIX", "snapshot suffix is not allowlisted")
|
|
1978
|
+
digest = sha256_bytes(content)
|
|
1979
|
+
final_name = f"{digest}.{suffix}"
|
|
1980
|
+
directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0)
|
|
1981
|
+
try:
|
|
1982
|
+
root_fd = os.open(root, directory_flags)
|
|
1983
|
+
except OSError:
|
|
1984
|
+
raise AcquisitionSecurityError(
|
|
1985
|
+
"SNAPSHOT_ROOT",
|
|
1986
|
+
"snapshot root must be an existing non-symlink directory",
|
|
1987
|
+
) from None
|
|
1988
|
+
temporary_name = f".stage-{os.getpid()}-{threading.get_ident()}-{time.monotonic_ns()}"
|
|
1989
|
+
file_flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0)
|
|
1990
|
+
temporary_fd: int | None = None
|
|
1991
|
+
try:
|
|
1992
|
+
root_info = os.fstat(root_fd)
|
|
1993
|
+
ownership_policy = (root_info.st_uid, root_info.st_gid)
|
|
1994
|
+
with _serialized_snapshot_seal(
|
|
1995
|
+
root_fd,
|
|
1996
|
+
root_info.st_dev,
|
|
1997
|
+
root_info.st_ino,
|
|
1998
|
+
final_name,
|
|
1999
|
+
):
|
|
2000
|
+
evidence_root = configured_retention_evidence_root(root)
|
|
2001
|
+
if evidence_root is not None:
|
|
2002
|
+
require_snapshot_admission(
|
|
2003
|
+
evidence_root=evidence_root,
|
|
2004
|
+
content_sha256=digest,
|
|
2005
|
+
)
|
|
2006
|
+
try:
|
|
2007
|
+
try:
|
|
2008
|
+
_read_existing_digest_snapshot(
|
|
2009
|
+
root_fd,
|
|
2010
|
+
final_name,
|
|
2011
|
+
expected_content=content,
|
|
2012
|
+
expected_digest=digest,
|
|
2013
|
+
ownership_policy=ownership_policy,
|
|
2014
|
+
)
|
|
2015
|
+
except FileNotFoundError:
|
|
2016
|
+
pass
|
|
2017
|
+
else:
|
|
2018
|
+
return root / final_name
|
|
2019
|
+
temporary_fd = os.open(
|
|
2020
|
+
temporary_name,
|
|
2021
|
+
file_flags,
|
|
2022
|
+
0o600,
|
|
2023
|
+
dir_fd=root_fd,
|
|
2024
|
+
)
|
|
2025
|
+
view = memoryview(content)
|
|
2026
|
+
written = 0
|
|
2027
|
+
while written < len(view):
|
|
2028
|
+
count = os.write(temporary_fd, view[written:])
|
|
2029
|
+
if count <= 0:
|
|
2030
|
+
raise AcquisitionSecurityError(
|
|
2031
|
+
"SNAPSHOT_WRITE",
|
|
2032
|
+
"snapshot write made no progress",
|
|
2033
|
+
)
|
|
2034
|
+
written += count
|
|
2035
|
+
os.fchmod(temporary_fd, 0o444)
|
|
2036
|
+
os.fsync(temporary_fd)
|
|
2037
|
+
os.close(temporary_fd)
|
|
2038
|
+
temporary_fd = None
|
|
2039
|
+
try:
|
|
2040
|
+
os.link(
|
|
2041
|
+
temporary_name,
|
|
2042
|
+
final_name,
|
|
2043
|
+
src_dir_fd=root_fd,
|
|
2044
|
+
dst_dir_fd=root_fd,
|
|
2045
|
+
follow_symlinks=False,
|
|
2046
|
+
)
|
|
2047
|
+
except FileExistsError:
|
|
2048
|
+
_read_existing_digest_snapshot(
|
|
2049
|
+
root_fd,
|
|
2050
|
+
final_name,
|
|
2051
|
+
expected_content=content,
|
|
2052
|
+
expected_digest=digest,
|
|
2053
|
+
ownership_policy=ownership_policy,
|
|
2054
|
+
)
|
|
2055
|
+
os.unlink(temporary_name, dir_fd=root_fd)
|
|
2056
|
+
# The staging name is deliberately visible for the shortest possible interval,
|
|
2057
|
+
# but a same-UID directory racer can still link that inode while it is visible.
|
|
2058
|
+
# Authenticate the published name only after our staging link is gone: nlink must
|
|
2059
|
+
# now be exactly one, and the stable-descriptor check below also rebinds kind,
|
|
2060
|
+
# mode, owner, size, name and digest.
|
|
2061
|
+
_read_existing_digest_snapshot(
|
|
2062
|
+
root_fd,
|
|
2063
|
+
final_name,
|
|
2064
|
+
expected_content=content,
|
|
2065
|
+
expected_digest=digest,
|
|
2066
|
+
ownership_policy=ownership_policy,
|
|
2067
|
+
)
|
|
2068
|
+
os.fsync(root_fd)
|
|
2069
|
+
return root / final_name
|
|
2070
|
+
finally:
|
|
2071
|
+
# Inside the per-target lock: the staging entry must be gone before the next
|
|
2072
|
+
# sealer of this same digest observes the published name, or that sealer sees
|
|
2073
|
+
# our second link and refuses a file we are in the middle of publishing.
|
|
2074
|
+
if temporary_fd is not None:
|
|
2075
|
+
os.close(temporary_fd)
|
|
2076
|
+
with contextlib.suppress(FileNotFoundError):
|
|
2077
|
+
os.unlink(temporary_name, dir_fd=root_fd)
|
|
2078
|
+
finally:
|
|
2079
|
+
os.close(root_fd)
|
|
2080
|
+
|
|
2081
|
+
|
|
2082
|
+
# The cross-process half's lock file sits beside the staging entries, under a prefix that no
|
|
2083
|
+
# content-addressed name can take: a snapshot is `<64 hex>.<suffix>`, and `sources.deletion`
|
|
2084
|
+
# matches removable names against exactly that. It is created while a name is held and unlinked
|
|
2085
|
+
# when it is released, so a settled root holds snapshots and nothing else.
|
|
2086
|
+
SNAPSHOT_LOCK_PREFIX = ".seal-lock-"
|
|
2087
|
+
# The backoff that keeps re-taking a moved lock file from being a spin. Every release destroys the
|
|
2088
|
+
# file, so one release wakes every waiter at once and all but one of them find a name that has
|
|
2089
|
+
# moved -- a herd that re-races itself. Measured at a deliberately lowered ceiling of 32, sixteen
|
|
2090
|
+
# processes holding for 0.3 ms over 400 rounds each: 1117-1270 losses that deep without a backoff,
|
|
2091
|
+
# 139-152 with one, an 8x reduction. There is deliberately no ceiling above it; see
|
|
2092
|
+
# :func:`_hold_snapshot_lock`.
|
|
2093
|
+
SNAPSHOT_LOCK_BACKOFF_SECONDS = 0.0002
|
|
2094
|
+
MAX_SNAPSHOT_LOCK_BACKOFF = 0.005
|
|
2095
|
+
# The one thing the unbounded retry still has to give up on: a root directory removed out from
|
|
2096
|
+
# under the descriptor answers every ``O_CREAT`` open with the same ``ENOENT``, forever. Only
|
|
2097
|
+
# CONSECUTIVE losses on that branch are counted, and the deepest streak observed under real
|
|
2098
|
+
# contention was four, so this is sixty-four times the worst measured case.
|
|
2099
|
+
MAX_CONSECUTIVE_VANISHED_LOCK_OPENS = 256
|
|
2100
|
+
|
|
2101
|
+
# One lock per (snapshot root inode, final name), created on demand and dropped when the last
|
|
2102
|
+
# holder leaves, so the registry cannot grow with the number of digests a long-lived process
|
|
2103
|
+
# seals. Keying on the root's device and inode rather than its pathname means two spellings of
|
|
2104
|
+
# one directory serialize together, which is exactly the contention that matters.
|
|
2105
|
+
_SNAPSHOT_SEAL_REGISTRY_GUARD = threading.Lock()
|
|
2106
|
+
_SNAPSHOT_SEAL_LOCKS: dict[tuple[int, int, str], tuple[threading.Lock, int]] = {}
|
|
2107
|
+
|
|
2108
|
+
|
|
2109
|
+
@contextlib.contextmanager
|
|
2110
|
+
def _serialized_snapshot_seal(
|
|
2111
|
+
root_fd: int,
|
|
2112
|
+
root_device: int,
|
|
2113
|
+
root_inode: int,
|
|
2114
|
+
final_name: str,
|
|
2115
|
+
) -> Iterator[None]:
|
|
2116
|
+
"""Hold one content-addressed name against every other writer of it, in and out of process.
|
|
2117
|
+
|
|
2118
|
+
Publishing is ``link(stage, final)`` followed by ``unlink(stage)``, and non-overwriting
|
|
2119
|
+
publication has no primitive that skips that pair: ``rename`` would clobber whatever already
|
|
2120
|
+
holds the name, which is the one thing this sealer must never do. So between those two calls
|
|
2121
|
+
the published inode legitimately carries two links, and a second sealer of the *same* bytes --
|
|
2122
|
+
same content, therefore same digest, therefore same final name -- that reads the name in that
|
|
2123
|
+
window sees ``st_nlink == 2`` and refuses a snapshot that is merely mid-publication. A
|
|
2124
|
+
remover of that name is the mirror of it: an ``unlink`` inside the read window moves
|
|
2125
|
+
``st_nlink`` from 1 to 0 under the open descriptor, which is the ``changed during validation``
|
|
2126
|
+
refusal issue #209 reported.
|
|
2127
|
+
|
|
2128
|
+
Neither refusal is relaxed here, and neither may be: a second link on the published inode is
|
|
2129
|
+
exactly how a same-UID racer keeps a writable alias to a sealed snapshot, and a name that
|
|
2130
|
+
vanishes mid-read is exactly what a rebind looks like from the inside. What is removed is the
|
|
2131
|
+
benign racer. Writers of one name take turns, so the next one arrives after the last has
|
|
2132
|
+
unlinked its stage and finds a settled, single-linked file through the ordinary already-exists
|
|
2133
|
+
path. Serializing also stops N sealers from each writing N full copies of the same
|
|
2134
|
+
multi-gigabyte snapshot.
|
|
2135
|
+
|
|
2136
|
+
**Two locks, because there are two kinds of racer.** The ``threading.Lock`` below excludes
|
|
2137
|
+
other threads of this interpreter. It cannot exclude other PROCESSES, and those are real: the
|
|
2138
|
+
hosted crawler and the CLI seal into one snapshot root, and #229 shipped the thread half alone
|
|
2139
|
+
and named this as the residual it left open. Measured on macOS/APFS, six processes sealing
|
|
2140
|
+
identical bytes into one root over 24 rounds refused 5 to 7 of 144 attempts, every one of them
|
|
2141
|
+
``st_nlink 2 != 1``. The second half is an ``flock`` on a per-name lock file beside the
|
|
2142
|
+
staging area, taken by :func:`_hold_snapshot_lock`.
|
|
2143
|
+
|
|
2144
|
+
Order is fixed -- threads first, then the file -- and only ever one name is held, so nothing
|
|
2145
|
+
can wait on a cycle. ``sources.deletion`` takes the same pair through
|
|
2146
|
+
:func:`exclusive_content_addressed_name` before it removes a snapshot, which is what stops the
|
|
2147
|
+
harness racing its own retention sweep; while it holds one it may take a *second* name under
|
|
2148
|
+
its evidence root to seal a tombstone, and that direction is the only nesting in the tree.
|
|
2149
|
+
|
|
2150
|
+
Neither wait is bounded. A holder finishes on its own -- the kernel drops an ``flock`` when
|
|
2151
|
+
its holder dies -- but "finishes" is measured in the payload: a seal or a governed deletion of
|
|
2152
|
+
a multi-gigabyte snapshot holds its name for the full write or re-hash plus its fsyncs, which
|
|
2153
|
+
is seconds on local disk and longer on slow storage, and every same-name waiter waits it out.
|
|
2154
|
+
A deadline here would have to choose between refusing an acquisition that is merely queued and
|
|
2155
|
+
admitting a second writer of a name that is mid-publication, so there is none.
|
|
2156
|
+
"""
|
|
2157
|
+
|
|
2158
|
+
key = (root_device, root_inode, final_name)
|
|
2159
|
+
with _SNAPSHOT_SEAL_REGISTRY_GUARD:
|
|
2160
|
+
lock, holders = _SNAPSHOT_SEAL_LOCKS.get(key, (threading.Lock(), 0))
|
|
2161
|
+
_SNAPSHOT_SEAL_LOCKS[key] = (lock, holders + 1)
|
|
2162
|
+
try:
|
|
2163
|
+
# Acquiring inside this ``try``, and releasing through ``with``, is what keeps the
|
|
2164
|
+
# registry honest. A signal delivered during the blocking acquire would otherwise skip
|
|
2165
|
+
# the bookkeeping below and strand this key's entry -- held, in the worst case -- for the
|
|
2166
|
+
# life of the process.
|
|
2167
|
+
lock_name = f"{SNAPSHOT_LOCK_PREFIX}{final_name}"
|
|
2168
|
+
with lock:
|
|
2169
|
+
# The same shape as the comment above, one level down and with a worse consequence.
|
|
2170
|
+
# ``descriptor`` is bound inside this ``try`` so that a signal delivered between the
|
|
2171
|
+
# acquire returning and the release being armed does not strand an ``flock`` -- which,
|
|
2172
|
+
# unlike a thread lock, no other process can break for the life of this one. This
|
|
2173
|
+
# narrows that window to the bytecode between the call returning and the store; it
|
|
2174
|
+
# does not close it, because the eval breaker is still checked there.
|
|
2175
|
+
descriptor: int | None = None
|
|
2176
|
+
try:
|
|
2177
|
+
descriptor = _hold_snapshot_lock(root_fd, lock_name)
|
|
2178
|
+
yield
|
|
2179
|
+
finally:
|
|
2180
|
+
if descriptor is not None:
|
|
2181
|
+
_release_snapshot_lock(root_fd, lock_name, descriptor)
|
|
2182
|
+
finally:
|
|
2183
|
+
with _SNAPSHOT_SEAL_REGISTRY_GUARD:
|
|
2184
|
+
held, holders = _SNAPSHOT_SEAL_LOCKS[key]
|
|
2185
|
+
if holders <= 1:
|
|
2186
|
+
del _SNAPSHOT_SEAL_LOCKS[key]
|
|
2187
|
+
else:
|
|
2188
|
+
_SNAPSHOT_SEAL_LOCKS[key] = (held, holders - 1)
|
|
2189
|
+
|
|
2190
|
+
|
|
2191
|
+
@contextlib.contextmanager
|
|
2192
|
+
def exclusive_content_addressed_name(root: Path, name: str) -> Iterator[None]:
|
|
2193
|
+
"""Hold ``name`` under ``root`` against every sealer and remover of it, in and out of process.
|
|
2194
|
+
|
|
2195
|
+
The seam :mod:`mostlyright.data_harness.sources.deletion` reaches for. Removing a published
|
|
2196
|
+
snapshot and validating one are the same window from opposite ends -- an ``unlink`` between a
|
|
2197
|
+
validator's two observations of its descriptor is the ``st_nlink 1 -> 0`` drift of #209 -- and
|
|
2198
|
+
a remover that takes this cannot land inside a sealer's read, or a sealer inside its removal.
|
|
2199
|
+
It is the same pair of locks :func:`_serialized_snapshot_seal` takes, keyed the same way, so
|
|
2200
|
+
the two paths exclude each other rather than each other's copy.
|
|
2201
|
+
|
|
2202
|
+
The key is the root's device and inode, read from this function's own descriptor. A caller
|
|
2203
|
+
that re-opens the root by pathname afterwards is holding a lock on the directory this saw,
|
|
2204
|
+
which is the directory that name resolved to when the lock was taken; if the pathname is
|
|
2205
|
+
rebound in between, the caller validates a different directory than the one locked. Nothing
|
|
2206
|
+
in this tree does that on purpose -- the caller that exists,
|
|
2207
|
+
:meth:`~mostlyright.data_harness.sources.deletion.DeletionCoordinator.delete`, re-validates
|
|
2208
|
+
its target by identity and refuses anything that moved -- and a caller that wants the
|
|
2209
|
+
guarantee outright should hand its own descriptor down rather than a path.
|
|
2210
|
+
"""
|
|
2211
|
+
|
|
2212
|
+
if not root.is_absolute():
|
|
2213
|
+
raise AcquisitionSecurityError("SNAPSHOT_ROOT", "snapshot root must be absolute")
|
|
2214
|
+
if name in {"", ".", ".."} or name != PurePosixPath(name).name:
|
|
2215
|
+
raise AcquisitionSecurityError(
|
|
2216
|
+
"SNAPSHOT_NAME",
|
|
2217
|
+
"content-addressed name must be one path component",
|
|
2218
|
+
)
|
|
2219
|
+
directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0)
|
|
2220
|
+
try:
|
|
2221
|
+
root_fd = os.open(root, directory_flags)
|
|
2222
|
+
except OSError:
|
|
2223
|
+
raise AcquisitionSecurityError(
|
|
2224
|
+
"SNAPSHOT_ROOT",
|
|
2225
|
+
"snapshot root must be an existing non-symlink directory",
|
|
2226
|
+
) from None
|
|
2227
|
+
try:
|
|
2228
|
+
root_info = os.fstat(root_fd)
|
|
2229
|
+
with _serialized_snapshot_seal(root_fd, root_info.st_dev, root_info.st_ino, name):
|
|
2230
|
+
yield
|
|
2231
|
+
finally:
|
|
2232
|
+
os.close(root_fd)
|
|
2233
|
+
|
|
2234
|
+
|
|
2235
|
+
def _hold_snapshot_lock(root_fd: int, lock_name: str) -> int:
|
|
2236
|
+
"""Take the cross-process half of the per-name lock, and prove it is still that name's.
|
|
2237
|
+
|
|
2238
|
+
``flock`` binds an inode, not a name, so a lock file that is unlinked while a waiter is
|
|
2239
|
+
blocked on it leaves that waiter holding an inode no longer reachable through the name --
|
|
2240
|
+
while a newcomer creates a fresh file at the name and takes it uncontended. Two holders, no
|
|
2241
|
+
exclusion. The published fix for that is here: unlink only while holding the lock (see
|
|
2242
|
+
:func:`_release_snapshot_lock`), and after acquiring, re-read the name and accept only if it
|
|
2243
|
+
still resolves to the inode just locked. Anything else means the file was replaced under the
|
|
2244
|
+
wait, so drop it and take the new one.
|
|
2245
|
+
|
|
2246
|
+
``ENOENT`` from an ``O_CREAT`` open is part of that same race rather than a fault. Measured
|
|
2247
|
+
on macOS/APFS, six processes opening and unlinking one name 4,000 times each took a transient
|
|
2248
|
+
``ENOENT`` on 17 to 26 percent of opens -- the entry is found and then removed before the open
|
|
2249
|
+
completes -- so it is retried, not refused. Every other errno is refused: an unreadable entry
|
|
2250
|
+
at this name is not contention, and ``O_NOFOLLOW`` makes a symlink here ``ELOOP`` rather than
|
|
2251
|
+
something to follow.
|
|
2252
|
+
|
|
2253
|
+
**Queueing is not bounded, and a ceiling on it would be a bug.** Losing the name is
|
|
2254
|
+
queueing, not tampering, and every loss means some other holder had it -- so a ceiling refuses
|
|
2255
|
+
a byte-correct snapshot for being popular, which is the class of false refusal #209 is about.
|
|
2256
|
+
An earlier revision capped total attempts at 1024 and an independent review measured the cap
|
|
2257
|
+
being spent by ordinary contention. What a ceiling would otherwise guard against is a hot
|
|
2258
|
+
loop, and :func:`_snapshot_lock_backoff` rules that out: every retry sleeps, so this waits at
|
|
2259
|
+
a bounded rate exactly as the ``flock`` above waits without one. A same-UID racer rewriting
|
|
2260
|
+
this name in a loop can stall a caller here, which is strictly less than the stall it can
|
|
2261
|
+
already impose by holding the lock and never releasing it.
|
|
2262
|
+
|
|
2263
|
+
**One state is not queueing, and it does end the wait.** ``openat`` answers ``ENOENT`` both
|
|
2264
|
+
when a holder unlinked as this open resolved the name and when the directory ``root_fd``
|
|
2265
|
+
names has been REMOVED -- and the second is permanent for the life of that descriptor, so
|
|
2266
|
+
retrying it never terminates. Nothing hostile is needed to reach it: a caller queued behind
|
|
2267
|
+
a multi-gigabyte seal holds a ``root_fd`` that a sweep of the acquisition boundary
|
|
2268
|
+
invalidates, and neither ``st_nlink`` nor ``stat(".", dir_fd=...)`` tells the two apart on
|
|
2269
|
+
APFS. So consecutive losses on that branch alone are counted against
|
|
2270
|
+
:data:`MAX_CONSECUTIVE_VANISHED_LOCK_OPENS`, and any other outcome resets the count: a queued
|
|
2271
|
+
caller keeps an unbounded wait, and a vanished root gets a prompt, typed refusal.
|
|
2272
|
+
|
|
2273
|
+
``O_NONBLOCK`` is not optional and is here for the reason it is on the snapshot read at
|
|
2274
|
+
:func:`_read_existing_digest_snapshot`: opening a FIFO read-only BLOCKS in the kernel until a
|
|
2275
|
+
writer arrives, and nothing on this path holds a deadline. Without it a same-UID racer could
|
|
2276
|
+
leave one ``mkfifo`` at this name and wedge every sealer and every governed deletion of that
|
|
2277
|
+
digest for the life of the process -- an artifact that outlives the racer, that
|
|
2278
|
+
``sources.deletion`` cannot remove because it matches only ``<64 hex>.<suffix>``, and that no
|
|
2279
|
+
kind check can reach because the open never returns.
|
|
2280
|
+
|
|
2281
|
+
The kind is bound too, and it is bound before the wait: a FIFO opens without blocking under
|
|
2282
|
+
``O_NONBLOCK``, and an exclusive ``flock`` already held on a hostile FIFO would otherwise
|
|
2283
|
+
queue this caller behind a live racer instead of refusing -- the kind check exists to refuse
|
|
2284
|
+
exactly that, so it runs on the descriptor before the lock is asked for. A directory at this
|
|
2285
|
+
name refuses at the open itself, because the descriptor is opened for writing.
|
|
2286
|
+
|
|
2287
|
+
It is opened for writing -- ``O_RDWR``, never ``O_RDONLY`` -- even though nothing is ever
|
|
2288
|
+
written through it, because an exclusive ``flock`` is emulated as a whole-file write lock on
|
|
2289
|
+
NFS and refuses a read-only descriptor with ``EBADF``. Every other regular-file lock in this
|
|
2290
|
+
tree already opens ``O_RDWR`` for this reason, ``review``'s publication lock and
|
|
2291
|
+
``FileCapStore.lock`` among them.
|
|
2292
|
+
|
|
2293
|
+
None of this is a security control. The stat binding in
|
|
2294
|
+
:func:`_read_existing_digest_snapshot` is what refuses a hostile link, and it is untouched by
|
|
2295
|
+
anything here; a same-UID racer squatting this name can withhold the lock, which costs
|
|
2296
|
+
availability that a same-UID racer in the snapshot root already has by other means.
|
|
2297
|
+
|
|
2298
|
+
**What the storage has to be.** ``flock`` is a POSIX advisory lock, so the snapshot root has
|
|
2299
|
+
to sit on a filesystem that honours it, exactly as ``FileCapStore`` requires of the volume it
|
|
2300
|
+
is pointed at. A Cloud Storage bucket mounted with gcsfuse is not such a volume. Nothing
|
|
2301
|
+
here can detect the difference in advance, so a filesystem that refuses the lock refuses the
|
|
2302
|
+
seal, with the errno in the message: sealing without exclusion is the state #209 was filed
|
|
2303
|
+
about, and choosing it silently is not this function's call to make.
|
|
2304
|
+
|
|
2305
|
+
That is a widened requirement and it is worth stating plainly. The lock file has to be
|
|
2306
|
+
creatable, so sealing now needs write access to the snapshot root even on the path where the
|
|
2307
|
+
snapshot is already published and nothing would be written -- a read-only root, or one whose
|
|
2308
|
+
permissions were narrowed after the fact, refuses with ``EROFS`` or ``EACCES`` where it used
|
|
2309
|
+
to return the existing path.
|
|
2310
|
+
|
|
2311
|
+
A refusal raised after the lock file is created leaves that file behind, because a caller that
|
|
2312
|
+
could not establish it holds the name must not unlink a name it cannot prove is its own. On a
|
|
2313
|
+
volume that honours the lock the entry is inert and self-healing -- the next successful hold
|
|
2314
|
+
of that digest adopts and removes it -- and one stray zero-byte file is the cheaper of the two
|
|
2315
|
+
mistakes. On a volume that refuses the lock outright no later hold ever succeeds, so a stray
|
|
2316
|
+
left there is permanent until an operator removes it; the refusal that created it is the
|
|
2317
|
+
signal that the volume cannot host this root at all.
|
|
2318
|
+
"""
|
|
2319
|
+
|
|
2320
|
+
if fcntl is None: # pragma: no cover - POSIX-only path; sealing needs dir_fd links anyway
|
|
2321
|
+
raise AcquisitionSecurityError(
|
|
2322
|
+
"SNAPSHOT_LOCK",
|
|
2323
|
+
"content-addressed sealing requires POSIX advisory locking",
|
|
2324
|
+
)
|
|
2325
|
+
flags = (
|
|
2326
|
+
os.O_RDWR
|
|
2327
|
+
| os.O_CREAT
|
|
2328
|
+
| getattr(os, "O_NOFOLLOW", 0)
|
|
2329
|
+
| getattr(os, "O_NONBLOCK", 0)
|
|
2330
|
+
| getattr(os, "O_CLOEXEC", 0)
|
|
2331
|
+
)
|
|
2332
|
+
attempt = 0
|
|
2333
|
+
vanished = 0
|
|
2334
|
+
while True:
|
|
2335
|
+
if attempt:
|
|
2336
|
+
time.sleep(_snapshot_lock_backoff(attempt))
|
|
2337
|
+
attempt += 1
|
|
2338
|
+
try:
|
|
2339
|
+
descriptor = os.open(lock_name, flags, 0o600, dir_fd=root_fd)
|
|
2340
|
+
except FileNotFoundError:
|
|
2341
|
+
# Two different states arrive here. A holder unlinking as this open resolved the
|
|
2342
|
+
# name is transient and must be retried; a root directory REMOVED out from under this
|
|
2343
|
+
# descriptor gives the same errno forever, and retrying that is a spin with no end.
|
|
2344
|
+
# Only consecutive losses are counted, so a caller merely queueing -- which never
|
|
2345
|
+
# loses on this branch, it loses on the identity re-check below -- keeps its
|
|
2346
|
+
# unbounded wait. Sixteen and twenty-four processes contending on one name never ran
|
|
2347
|
+
# a streak longer than four.
|
|
2348
|
+
vanished += 1
|
|
2349
|
+
if vanished > MAX_CONSECUTIVE_VANISHED_LOCK_OPENS:
|
|
2350
|
+
raise AcquisitionSecurityError(
|
|
2351
|
+
"SNAPSHOT_LOCK",
|
|
2352
|
+
"snapshot name lock could not be created"
|
|
2353
|
+
" (the snapshot root no longer holds a name to lock)",
|
|
2354
|
+
) from None
|
|
2355
|
+
continue
|
|
2356
|
+
except OSError as error:
|
|
2357
|
+
raise AcquisitionSecurityError(
|
|
2358
|
+
"SNAPSHOT_LOCK",
|
|
2359
|
+
"snapshot name lock is not a safe lock file"
|
|
2360
|
+
f" (open refused: {_describe_os_error(error)})",
|
|
2361
|
+
) from None
|
|
2362
|
+
try:
|
|
2363
|
+
# Kind first, wait second: the descriptor's kind is a property of its inode and
|
|
2364
|
+
# cannot change, and binding it before the flock means a hostile FIFO or directory
|
|
2365
|
+
# already holding an exclusive lock is refused here rather than queued on -- the
|
|
2366
|
+
# same order _require_readable_evidence_kind states for the evidence reads.
|
|
2367
|
+
held = os.fstat(descriptor)
|
|
2368
|
+
if not stat.S_ISREG(held.st_mode):
|
|
2369
|
+
raise AcquisitionSecurityError(
|
|
2370
|
+
"SNAPSHOT_LOCK",
|
|
2371
|
+
"snapshot name lock is not a safe lock file"
|
|
2372
|
+
f" (kind {stat.S_IFMT(held.st_mode):#o} is not a regular file)",
|
|
2373
|
+
)
|
|
2374
|
+
fcntl.flock(descriptor, fcntl.LOCK_EX)
|
|
2375
|
+
# An adopted stray keeps whatever mode its creator gave it; bind it to the mode a
|
|
2376
|
+
# fresh create gets, the way the review publication lock does after its own flock.
|
|
2377
|
+
os.fchmod(descriptor, 0o600)
|
|
2378
|
+
try:
|
|
2379
|
+
named: os.stat_result | None = os.stat(
|
|
2380
|
+
lock_name,
|
|
2381
|
+
dir_fd=root_fd,
|
|
2382
|
+
follow_symlinks=False,
|
|
2383
|
+
)
|
|
2384
|
+
except FileNotFoundError:
|
|
2385
|
+
named = None
|
|
2386
|
+
except OSError as error:
|
|
2387
|
+
os.close(descriptor)
|
|
2388
|
+
raise AcquisitionSecurityError(
|
|
2389
|
+
"SNAPSHOT_LOCK",
|
|
2390
|
+
f"snapshot name lock could not be taken ({_describe_os_error(error)})",
|
|
2391
|
+
) from None
|
|
2392
|
+
except BaseException:
|
|
2393
|
+
os.close(descriptor)
|
|
2394
|
+
raise
|
|
2395
|
+
vanished = 0
|
|
2396
|
+
if named is not None and (named.st_dev, named.st_ino) == (held.st_dev, held.st_ino):
|
|
2397
|
+
return descriptor
|
|
2398
|
+
os.close(descriptor)
|
|
2399
|
+
|
|
2400
|
+
|
|
2401
|
+
def _snapshot_lock_backoff(attempt: int) -> float:
|
|
2402
|
+
"""Grow the wait between attempts, and spread it so the woken herd does not re-race in step.
|
|
2403
|
+
|
|
2404
|
+
The spread is derived from the caller's process and thread rather than from :mod:`random`,
|
|
2405
|
+
which nothing in this package imports: what has to differ between two waiters is their phase,
|
|
2406
|
+
not the unpredictability of it, and a deterministic function of identifiers that already
|
|
2407
|
+
differ gives exactly that while leaving the sealer's behaviour reproducible. Only timing
|
|
2408
|
+
depends on it -- never which caller wins, and never what is accepted.
|
|
2409
|
+
|
|
2410
|
+
Between PROCESSES the pid does all of the work. ``threading.get_ident()`` was measured
|
|
2411
|
+
identical in eight separate interpreters on macOS, so the thread half separates threads and
|
|
2412
|
+
nothing else -- which is the half that matters least, because threads of one process are
|
|
2413
|
+
already serialized on this name before they reach here. Over sixteen consecutive pids the
|
|
2414
|
+
sixteen spreads are distinct at every attempt, and the delay stays inside its declared
|
|
2415
|
+
bounds: 120 us to 4998 us against a 5000 us ceiling over the first hundred thousand attempts.
|
|
2416
|
+
"""
|
|
2417
|
+
|
|
2418
|
+
spread = (((os.getpid() ^ threading.get_ident()) * 2_654_435_761) + attempt * 40_503) % 1024
|
|
2419
|
+
ceiling = min(attempt * SNAPSHOT_LOCK_BACKOFF_SECONDS, MAX_SNAPSHOT_LOCK_BACKOFF)
|
|
2420
|
+
return ceiling * (0.5 + spread / 2048)
|
|
2421
|
+
|
|
2422
|
+
|
|
2423
|
+
def _release_snapshot_lock(root_fd: int, lock_name: str, descriptor: int) -> None:
|
|
2424
|
+
"""Drop the lock file and then the lock, in that order.
|
|
2425
|
+
|
|
2426
|
+
Unlinking first is what makes the identity re-check in :func:`_hold_snapshot_lock` sufficient:
|
|
2427
|
+
the name can only stop resolving to this inode while this process still holds it, so no waiter
|
|
2428
|
+
can be admitted on a stale inode. Closing is what releases the ``flock``, so it comes second,
|
|
2429
|
+
and that order is pinned by
|
|
2430
|
+
``tests/h3/test_http.py::test_the_name_lock_is_unlinked_before_the_lock_on_it_is_dropped``
|
|
2431
|
+
rather than left to whoever edits these four lines next. A settled root is left holding
|
|
2432
|
+
snapshots and nothing else, which is the invariant the staging name already keeps.
|
|
2433
|
+
|
|
2434
|
+
Create-and-unlink is the harder of the two shapes a lock file can take, and it is chosen
|
|
2435
|
+
deliberately. ``FileCapStore.lock`` next door keeps one sidecar that is created once and
|
|
2436
|
+
never replaced, which needs no protocol at all -- but there is one cap store and there is one
|
|
2437
|
+
lock file per digest, and ``sources.deletion`` matches removable names against
|
|
2438
|
+
``<64 hex>.<suffix>`` alone, so a permanent sidecar per digest is an entry no governed path
|
|
2439
|
+
could ever remove. The protocol above is the price of not leaving those behind.
|
|
2440
|
+
|
|
2441
|
+
This runs from a ``finally``, so a failure to unlink is swallowed rather than allowed to
|
|
2442
|
+
replace whatever the sealer was already reporting. A lock file left behind costs one stray
|
|
2443
|
+
entry; a raise here would turn a published snapshot into a refusal, or bury the refusal that
|
|
2444
|
+
matters under a second one about a lock. The close is what has to happen, and it does.
|
|
2445
|
+
"""
|
|
2446
|
+
|
|
2447
|
+
try:
|
|
2448
|
+
with contextlib.suppress(OSError):
|
|
2449
|
+
os.unlink(lock_name, dir_fd=root_fd)
|
|
2450
|
+
finally:
|
|
2451
|
+
os.close(descriptor)
|
|
2452
|
+
|
|
2453
|
+
|
|
2454
|
+
def _read_existing_digest_snapshot(
|
|
2455
|
+
root_fd: int,
|
|
2456
|
+
name: str,
|
|
2457
|
+
*,
|
|
2458
|
+
expected_content: bytes,
|
|
2459
|
+
expected_digest: str,
|
|
2460
|
+
ownership_policy: tuple[int, int],
|
|
2461
|
+
) -> None:
|
|
2462
|
+
"""Authenticate one already-named immutable snapshot without trusting its pathname."""
|
|
2463
|
+
|
|
2464
|
+
flags = (
|
|
2465
|
+
os.O_RDONLY
|
|
2466
|
+
| getattr(os, "O_NOFOLLOW", 0)
|
|
2467
|
+
| getattr(os, "O_NONBLOCK", 0)
|
|
2468
|
+
| getattr(os, "O_CLOEXEC", 0)
|
|
2469
|
+
)
|
|
2470
|
+
try:
|
|
2471
|
+
descriptor = os.open(name, flags, dir_fd=root_fd)
|
|
2472
|
+
except FileNotFoundError:
|
|
2473
|
+
raise
|
|
2474
|
+
except OSError as error:
|
|
2475
|
+
raise AcquisitionSecurityError(
|
|
2476
|
+
"SNAPSHOT_COLLISION",
|
|
2477
|
+
"existing content-addressed snapshot is not a safe immutable file"
|
|
2478
|
+
f" (open refused: {_describe_os_error(error)})",
|
|
2479
|
+
) from None
|
|
2480
|
+
try:
|
|
2481
|
+
opened = os.fstat(descriptor)
|
|
2482
|
+
_require_existing_snapshot_stat(
|
|
2483
|
+
opened,
|
|
2484
|
+
expected_size=len(expected_content),
|
|
2485
|
+
ownership_policy=ownership_policy,
|
|
2486
|
+
)
|
|
2487
|
+
try:
|
|
2488
|
+
named = os.stat(name, dir_fd=root_fd, follow_symlinks=False)
|
|
2489
|
+
except OSError as error:
|
|
2490
|
+
raise AcquisitionSecurityError(
|
|
2491
|
+
"SNAPSHOT_COLLISION",
|
|
2492
|
+
"existing content-addressed snapshot name changed during validation"
|
|
2493
|
+
f" (before the read: {_describe_os_error(error)})",
|
|
2494
|
+
) from None
|
|
2495
|
+
_require_stable_snapshot_stat(opened, named, observed_between=_OPEN_VERSUS_NAME)
|
|
2496
|
+
_require_snapshot_fd_bytes(descriptor, expected_content, expected_digest)
|
|
2497
|
+
_require_stable_snapshot_stat(
|
|
2498
|
+
opened,
|
|
2499
|
+
os.fstat(descriptor),
|
|
2500
|
+
observed_between=_OPEN_VERSUS_READ,
|
|
2501
|
+
)
|
|
2502
|
+
finally:
|
|
2503
|
+
os.close(descriptor)
|
|
2504
|
+
try:
|
|
2505
|
+
after = os.stat(name, dir_fd=root_fd, follow_symlinks=False)
|
|
2506
|
+
except OSError as error:
|
|
2507
|
+
raise AcquisitionSecurityError(
|
|
2508
|
+
"SNAPSHOT_COLLISION",
|
|
2509
|
+
"existing content-addressed snapshot name changed during validation"
|
|
2510
|
+
f" (after the read: {_describe_os_error(error)})",
|
|
2511
|
+
) from None
|
|
2512
|
+
_require_stable_snapshot_stat(named, after, observed_between=_NAME_VERSUS_READ)
|
|
2513
|
+
|
|
2514
|
+
|
|
2515
|
+
def _describe_os_error(error: OSError) -> str:
|
|
2516
|
+
"""Name an errno without putting a path or any other caller string in the message."""
|
|
2517
|
+
|
|
2518
|
+
return f"{errno.errorcode.get(error.errno or 0, 'errno')} {error.strerror or ''}".strip()
|
|
2519
|
+
|
|
2520
|
+
|
|
2521
|
+
def _require_existing_snapshot_stat(
|
|
2522
|
+
info: os.stat_result,
|
|
2523
|
+
*,
|
|
2524
|
+
expected_size: int,
|
|
2525
|
+
ownership_policy: tuple[int, int],
|
|
2526
|
+
) -> None:
|
|
2527
|
+
"""Refuse an existing entry that is not one immutable, singly-linked, owned regular file.
|
|
2528
|
+
|
|
2529
|
+
Every field is evaluated rather than short-circuited so the refusal can name each one that
|
|
2530
|
+
failed with its observed and required value. A ``SNAPSHOT_COLLISION`` reaching an operator
|
|
2531
|
+
from a machine nobody can attach a debugger to is otherwise indistinguishable from every
|
|
2532
|
+
other reason this entry could be unsafe, and those reasons used to share one sentence.
|
|
2533
|
+
``st_nlink`` is the one that a concurrent same-digest sealer can move.
|
|
2534
|
+
"""
|
|
2535
|
+
|
|
2536
|
+
faults: list[str] = []
|
|
2537
|
+
if not stat.S_ISREG(info.st_mode):
|
|
2538
|
+
faults.append(f"kind {stat.S_IFMT(info.st_mode):#o} is not a regular file")
|
|
2539
|
+
if stat.S_IMODE(info.st_mode) != 0o444:
|
|
2540
|
+
faults.append(f"st_mode {stat.S_IMODE(info.st_mode):#o} != 0o444")
|
|
2541
|
+
if info.st_nlink != 1:
|
|
2542
|
+
faults.append(f"st_nlink {info.st_nlink} != 1")
|
|
2543
|
+
if info.st_size != expected_size:
|
|
2544
|
+
faults.append(f"st_size {info.st_size} != {expected_size}")
|
|
2545
|
+
if info.st_uid != ownership_policy[0]:
|
|
2546
|
+
faults.append(f"st_uid {info.st_uid} != {ownership_policy[0]}")
|
|
2547
|
+
if info.st_gid != ownership_policy[1]:
|
|
2548
|
+
faults.append(f"st_gid {info.st_gid} != {ownership_policy[1]}")
|
|
2549
|
+
if faults:
|
|
2550
|
+
raise AcquisitionSecurityError(
|
|
2551
|
+
"SNAPSHOT_COLLISION",
|
|
2552
|
+
"existing content-addressed snapshot is not a safe immutable file"
|
|
2553
|
+
f" ({', '.join(faults)})",
|
|
2554
|
+
)
|
|
2555
|
+
|
|
2556
|
+
|
|
2557
|
+
# The three points at which the bound stat signature is re-observed. They are named in the
|
|
2558
|
+
# refusal so a field report says which observation drifted rather than only that one did.
|
|
2559
|
+
_OPEN_VERSUS_NAME = "the opened descriptor and the named entry"
|
|
2560
|
+
_OPEN_VERSUS_READ = "the opened descriptor before and after the read"
|
|
2561
|
+
_NAME_VERSUS_READ = "the named entry before and after the read"
|
|
2562
|
+
|
|
2563
|
+
# The one drift with a single cause, said in words rather than left to be read off a field.
|
|
2564
|
+
_LAST_NAME_REMOVED = "; the last name for it was removed while it was open"
|
|
2565
|
+
|
|
2566
|
+
_SNAPSHOT_STAT_FIELDS = (
|
|
2567
|
+
"st_dev",
|
|
2568
|
+
"st_ino",
|
|
2569
|
+
"st_uid",
|
|
2570
|
+
"st_gid",
|
|
2571
|
+
"st_mode",
|
|
2572
|
+
"st_nlink",
|
|
2573
|
+
"st_size",
|
|
2574
|
+
"st_mtime_ns",
|
|
2575
|
+
"st_ctime_ns",
|
|
2576
|
+
)
|
|
2577
|
+
|
|
2578
|
+
|
|
2579
|
+
def _require_stable_snapshot_stat(
|
|
2580
|
+
before: os.stat_result,
|
|
2581
|
+
after: os.stat_result,
|
|
2582
|
+
*,
|
|
2583
|
+
observed_between: str,
|
|
2584
|
+
) -> None:
|
|
2585
|
+
"""Refuse a snapshot whose bound identity moved, naming where it moved and by what.
|
|
2586
|
+
|
|
2587
|
+
The signature is a deliberate anti-TOCTOU binding, so it stays exactly as wide as it was.
|
|
2588
|
+
What changes is only what the refusal says: which of the three observation pairs disagreed,
|
|
2589
|
+
and every field that differs with its old and new value. ``st_nlink`` moving means another
|
|
2590
|
+
name was attached to or detached from this inode; ``st_ino`` or ``st_dev`` moving means the
|
|
2591
|
+
name was rebound to a different file; ``st_ctime_ns`` alone means metadata was touched.
|
|
2592
|
+
Those are different incidents and a field recurrence has to be able to tell them apart.
|
|
2593
|
+
|
|
2594
|
+
``st_nlink`` reaching zero is the one of those with only one cause, so it is named rather than
|
|
2595
|
+
left to be inferred: the last name for this inode was removed while this descriptor held it
|
|
2596
|
+
open, which is a deletion of the snapshot and not a change to it. That is the refusal #209
|
|
2597
|
+
reported from the field, and it is the reason a remover inside this tree now takes the same
|
|
2598
|
+
per-name lock a sealer does -- so a refusal carrying this sentence is a remover the harness
|
|
2599
|
+
does not own.
|
|
2600
|
+
"""
|
|
2601
|
+
|
|
2602
|
+
moved = tuple(
|
|
2603
|
+
f"{field} {getattr(before, field)} -> {getattr(after, field)}"
|
|
2604
|
+
for field in _SNAPSHOT_STAT_FIELDS
|
|
2605
|
+
if getattr(before, field) != getattr(after, field)
|
|
2606
|
+
)
|
|
2607
|
+
if moved:
|
|
2608
|
+
raise AcquisitionSecurityError(
|
|
2609
|
+
"SNAPSHOT_COLLISION",
|
|
2610
|
+
"existing content-addressed snapshot changed during validation"
|
|
2611
|
+
f" (between {observed_between}: {', '.join(moved)})"
|
|
2612
|
+
f"{_LAST_NAME_REMOVED if before.st_nlink and not after.st_nlink else ''}",
|
|
2613
|
+
)
|
|
2614
|
+
|
|
2615
|
+
|
|
2616
|
+
def _is_allowlisted_snapshot_suffix(suffix: str) -> bool:
|
|
2617
|
+
"""Report whether a snapshot filename suffix is allowlisted.
|
|
2618
|
+
|
|
2619
|
+
Read from the single closed set rather than restated or unioned here. What this admits
|
|
2620
|
+
for writing is exactly what ``SNAPSHOT_SUFFIX_PATTERN`` -- and therefore the governed
|
|
2621
|
+
deletion path -- will match for removal: a suffix that could be sealed but not matched
|
|
2622
|
+
would strand snapshots no retention sweep can find.
|
|
2623
|
+
"""
|
|
2624
|
+
|
|
2625
|
+
return suffix in SNAPSHOT_SUFFIXES
|
|
2626
|
+
|
|
2627
|
+
|
|
2628
|
+
def _validate_response_headers(
|
|
2629
|
+
headers: tuple[tuple[str, str], ...],
|
|
2630
|
+
) -> tuple[tuple[str, str], ...]:
|
|
2631
|
+
if len(headers) > MAX_HEADER_COUNT:
|
|
2632
|
+
raise AcquisitionSecurityError(
|
|
2633
|
+
"RESPONSE_HEADERS",
|
|
2634
|
+
f"response has more than {MAX_HEADER_COUNT} headers",
|
|
2635
|
+
)
|
|
2636
|
+
total = 0
|
|
2637
|
+
normalized: list[tuple[str, str]] = []
|
|
2638
|
+
for raw_name, raw_value in headers:
|
|
2639
|
+
name = raw_name.lower().strip()
|
|
2640
|
+
value = raw_value.strip()
|
|
2641
|
+
if (
|
|
2642
|
+
not name
|
|
2643
|
+
or not all(char.isalnum() or char == "-" for char in name)
|
|
2644
|
+
or "\r" in value
|
|
2645
|
+
or "\n" in value
|
|
2646
|
+
):
|
|
2647
|
+
raise AcquisitionSecurityError(
|
|
2648
|
+
"RESPONSE_HEADER",
|
|
2649
|
+
"response contains an invalid header",
|
|
2650
|
+
)
|
|
2651
|
+
total += len(name.encode("ascii", errors="strict")) + len(
|
|
2652
|
+
value.encode("utf-8", errors="strict")
|
|
2653
|
+
)
|
|
2654
|
+
normalized.append((name, value))
|
|
2655
|
+
if total > MAX_HEADER_BYTES:
|
|
2656
|
+
raise AcquisitionSecurityError(
|
|
2657
|
+
"RESPONSE_HEADERS",
|
|
2658
|
+
f"response headers exceed {MAX_HEADER_BYTES} bytes",
|
|
2659
|
+
)
|
|
2660
|
+
return tuple(normalized)
|
|
2661
|
+
|
|
2662
|
+
|
|
2663
|
+
def _parse_content_range(value: str | None) -> tuple[int, int, int | None]:
|
|
2664
|
+
"""Parse a ``Content-Range`` strictly, or refuse it."""
|
|
2665
|
+
|
|
2666
|
+
if value is None:
|
|
2667
|
+
raise AcquisitionSecurityError(
|
|
2668
|
+
"CONTENT_RANGE",
|
|
2669
|
+
"partial response carries no content range",
|
|
2670
|
+
)
|
|
2671
|
+
match = _CONTENT_RANGE.fullmatch(value)
|
|
2672
|
+
if match is None:
|
|
2673
|
+
raise AcquisitionSecurityError(
|
|
2674
|
+
"CONTENT_RANGE",
|
|
2675
|
+
"response content range is not a single canonical byte span",
|
|
2676
|
+
)
|
|
2677
|
+
first_byte = int(match.group(1))
|
|
2678
|
+
last_byte = int(match.group(2))
|
|
2679
|
+
complete = match.group(3)
|
|
2680
|
+
if last_byte < first_byte:
|
|
2681
|
+
raise AcquisitionSecurityError(
|
|
2682
|
+
"CONTENT_RANGE",
|
|
2683
|
+
"response content range ends before it begins",
|
|
2684
|
+
)
|
|
2685
|
+
complete_size = None if complete == "*" else int(complete)
|
|
2686
|
+
if complete_size is not None and complete_size < last_byte + 1:
|
|
2687
|
+
raise AcquisitionSecurityError(
|
|
2688
|
+
"CONTENT_RANGE",
|
|
2689
|
+
"response content range runs past the object it names",
|
|
2690
|
+
)
|
|
2691
|
+
return first_byte, last_byte, complete_size
|
|
2692
|
+
|
|
2693
|
+
|
|
2694
|
+
def _accept_header(limits: RetrievalLimits) -> str:
|
|
2695
|
+
"""What this retrieval will admit, spelled every way it would admit it.
|
|
2696
|
+
|
|
2697
|
+
Derived from the allowlist and the alias table rather than restated, so what is asked
|
|
2698
|
+
for cannot drift from what is accepted: a source negotiating on ``Accept`` must not be
|
|
2699
|
+
told a spelling is unwelcome that the response check would then have admitted. Sorted,
|
|
2700
|
+
so one allowlist has one header.
|
|
2701
|
+
"""
|
|
2702
|
+
|
|
2703
|
+
admitted = set(limits.allowed_media_types)
|
|
2704
|
+
admitted.update(
|
|
2705
|
+
alias for alias, canonical in MEDIA_TYPE_ALIASES.items() if canonical in admitted
|
|
2706
|
+
)
|
|
2707
|
+
return ", ".join(sorted(admitted))
|
|
2708
|
+
|
|
2709
|
+
|
|
2710
|
+
def _media_type(value: str | None) -> str:
|
|
2711
|
+
"""The one reader of a ``Content-Type`` header, and the one place a spelling is settled.
|
|
2712
|
+
|
|
2713
|
+
Parameters are dropped, case is folded, and a non-standard spelling of a type is
|
|
2714
|
+
canonicalised to the spelling every allowlist in this package is written in. The
|
|
2715
|
+
canonicalisation belongs HERE rather than beside each allowlist: an alias added to one
|
|
2716
|
+
tuple and missed on the other is precisely how the slice path came to admit the object
|
|
2717
|
+
but refuse the sidecar of the same S3 response. One normaliser means the allowlists, the
|
|
2718
|
+
``multipart/byteranges`` refusal, the outgoing ``Accept`` header and the media type this
|
|
2719
|
+
boundary reports all speak the same vocabulary, and a new alias is one line.
|
|
2720
|
+
"""
|
|
2721
|
+
|
|
2722
|
+
if value is None:
|
|
2723
|
+
raise AcquisitionSecurityError("MEDIA_TYPE", "response has no content type")
|
|
2724
|
+
media_type = value.split(";", 1)[0].strip().lower()
|
|
2725
|
+
if not media_type or "/" not in media_type:
|
|
2726
|
+
raise AcquisitionSecurityError("MEDIA_TYPE", "response content type is invalid")
|
|
2727
|
+
return MEDIA_TYPE_ALIASES.get(media_type, media_type)
|
|
2728
|
+
|
|
2729
|
+
|
|
2730
|
+
def _require_snapshot_fd_bytes(
|
|
2731
|
+
file_descriptor: int,
|
|
2732
|
+
expected_content: bytes,
|
|
2733
|
+
expected_digest: str,
|
|
2734
|
+
) -> None:
|
|
2735
|
+
"""Verify a descriptor's bytes against the expected snapshot without materializing them.
|
|
2736
|
+
|
|
2737
|
+
The verdict is identical to reading the whole descriptor and then comparing: the same
|
|
2738
|
+
``expected_length + 1`` over-read still detects a longer file, the length is still decided
|
|
2739
|
+
before the contents are, and acceptance still requires *both* the expected SHA-256 and exact
|
|
2740
|
+
equality with the expected bytes. Only the residency changes. Accumulating the file into a
|
|
2741
|
+
chunk list and joining it cost a second and a third full copy alongside the caller's own
|
|
2742
|
+
``content``, so sealing the 2.12 GB EPA snapshot peaked near 6.4 GB; comparing each 64 KiB
|
|
2743
|
+
chunk against the corresponding zero-copy slice of ``expected_content`` peaks at one copy
|
|
2744
|
+
plus a chunk.
|
|
2745
|
+
|
|
2746
|
+
A mismatched chunk stops further comparison but not the read: the length verdict has to come
|
|
2747
|
+
first, exactly as it did when the whole file was read before anything was compared, so a
|
|
2748
|
+
short *and* different file still reports its length rather than its contents.
|
|
2749
|
+
"""
|
|
2750
|
+
|
|
2751
|
+
expected_length = len(expected_content)
|
|
2752
|
+
expected = memoryview(expected_content)
|
|
2753
|
+
digest = hashlib.sha256()
|
|
2754
|
+
identical = True
|
|
2755
|
+
total = 0
|
|
2756
|
+
while total <= expected_length:
|
|
2757
|
+
chunk = os.read(file_descriptor, min(65_536, expected_length + 1 - total))
|
|
2758
|
+
if not chunk:
|
|
2759
|
+
break
|
|
2760
|
+
digest.update(chunk)
|
|
2761
|
+
if identical:
|
|
2762
|
+
identical = expected[total : total + len(chunk)] == chunk
|
|
2763
|
+
total += len(chunk)
|
|
2764
|
+
if total != expected_length:
|
|
2765
|
+
raise AcquisitionSecurityError(
|
|
2766
|
+
"SNAPSHOT_COLLISION",
|
|
2767
|
+
f"existing snapshot length differs (read {total} bytes, expected {expected_length})",
|
|
2768
|
+
)
|
|
2769
|
+
if digest.hexdigest() != expected_digest or not identical:
|
|
2770
|
+
raise AcquisitionSecurityError(
|
|
2771
|
+
"SNAPSHOT_COLLISION",
|
|
2772
|
+
"existing content-addressed snapshot bytes differ",
|
|
2773
|
+
)
|