mostlyright-data 0.9.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mostlyright/data_harness/__init__.py +158 -0
- mostlyright/data_harness/acquisition/__init__.py +55 -0
- mostlyright/data_harness/acquisition/http.py +2773 -0
- mostlyright/data_harness/acquisition/parsing.py +809 -0
- mostlyright/data_harness/acquisition/ranges.py +495 -0
- mostlyright/data_harness/acquisition/result_download.py +360 -0
- mostlyright/data_harness/acquisition/retention_admission.py +248 -0
- mostlyright/data_harness/acquisition/sandbox.py +4888 -0
- mostlyright/data_harness/acquisition/url_policy.py +530 -0
- mostlyright/data_harness/agent_runtime.py +2743 -0
- mostlyright/data_harness/assets/logo-ink.svg +31 -0
- mostlyright/data_harness/backends/__init__.py +28 -0
- mostlyright/data_harness/backends/pandas_backend.py +350 -0
- mostlyright/data_harness/backends/polars_backend.py +366 -0
- mostlyright/data_harness/backends/protocol.py +124 -0
- mostlyright/data_harness/backends/reference.py +83 -0
- mostlyright/data_harness/backends/registry.py +55 -0
- mostlyright/data_harness/backends/restrictions.py +126 -0
- mostlyright/data_harness/canonical.py +333 -0
- mostlyright/data_harness/catalog_job.py +625 -0
- mostlyright/data_harness/cli.py +5398 -0
- mostlyright/data_harness/contracts.py +53 -0
- mostlyright/data_harness/coordinator.py +1307 -0
- mostlyright/data_harness/deploy.py +924 -0
- mostlyright/data_harness/deploy_target.py +312 -0
- mostlyright/data_harness/deployment_evidence.py +1067 -0
- mostlyright/data_harness/event_presentation.py +576 -0
- mostlyright/data_harness/events.py +2152 -0
- mostlyright/data_harness/fast_delimited.py +239 -0
- mostlyright/data_harness/fleet.py +237 -0
- mostlyright/data_harness/formats.py +236 -0
- mostlyright/data_harness/governors.py +1163 -0
- mostlyright/data_harness/hosted_bootstrap.py +972 -0
- mostlyright/data_harness/hosted_crawler.py +1115 -0
- mostlyright/data_harness/hosted_crawler_container_smoke.py +351 -0
- mostlyright/data_harness/hosted_crawler_fetch.py +423 -0
- mostlyright/data_harness/hosted_crawler_job.py +1277 -0
- mostlyright/data_harness/hosted_crawler_protocol.py +676 -0
- mostlyright/data_harness/hosted_dataset.py +1500 -0
- mostlyright/data_harness/hosted_deploy.py +3037 -0
- mostlyright/data_harness/hosted_handoff.py +62 -0
- mostlyright/data_harness/hosted_ingestion_contract.py +504 -0
- mostlyright/data_harness/hosted_ingestion_job.py +356 -0
- mostlyright/data_harness/hosted_ingestion_job_smoke.py +40 -0
- mostlyright/data_harness/hosted_session_container_smoke.py +194 -0
- mostlyright/data_harness/hosted_session_worker.py +3554 -0
- mostlyright/data_harness/hosted_session_worker_job_smoke.py +46 -0
- mostlyright/data_harness/hosted_worker.py +6784 -0
- mostlyright/data_harness/ingestion/__init__.py +56 -0
- mostlyright/data_harness/ingestion/contracts.py +461 -0
- mostlyright/data_harness/ingestion/faults.py +42 -0
- mostlyright/data_harness/ingestion/gcs_store.py +1162 -0
- mostlyright/data_harness/ingestion/spool.py +130 -0
- mostlyright/data_harness/ingestion/store.py +885 -0
- mostlyright/data_harness/key_seam.py +434 -0
- mostlyright/data_harness/linux_process_boundary.py +262 -0
- mostlyright/data_harness/local_contracts.py +2880 -0
- mostlyright/data_harness/local_search/__init__.py +5 -0
- mostlyright/data_harness/local_search/build_index.py +1087 -0
- mostlyright/data_harness/local_search/contracts.py +920 -0
- mostlyright/data_harness/local_search/query_trace.py +266 -0
- mostlyright/data_harness/local_search/retrieval.py +700 -0
- mostlyright/data_harness/local_search/sealed.py +474 -0
- mostlyright/data_harness/local_search/service.py +784 -0
- mostlyright/data_harness/nbrender/CONTRACT.md +212 -0
- mostlyright/data_harness/nbrender/__init__.py +12 -0
- mostlyright/data_harness/nbrender/chrome.py +359 -0
- mostlyright/data_harness/nbrender/code_body.py +266 -0
- mostlyright/data_harness/nbrender/document.py +407 -0
- mostlyright/data_harness/nbrender/frame.py +275 -0
- mostlyright/data_harness/nbrender/interactive.py +337 -0
- mostlyright/data_harness/nbrender/markdown_body.py +477 -0
- mostlyright/data_harness/nbrender/mr_components.py +134 -0
- mostlyright/data_harness/nbrender/outputs_data.py +595 -0
- mostlyright/data_harness/nbrender/outputs_rich.py +906 -0
- mostlyright/data_harness/nbrender/outputs_source.py +260 -0
- mostlyright/data_harness/nbrender/outputs_stage.py +176 -0
- mostlyright/data_harness/nbrender/outputs_text.py +400 -0
- mostlyright/data_harness/nbrender/parse.py +394 -0
- mostlyright/data_harness/nbrender/status.py +40 -0
- mostlyright/data_harness/nbrender/tokens.py +1295 -0
- mostlyright/data_harness/notebook.py +1710 -0
- mostlyright/data_harness/offline.py +2049 -0
- mostlyright/data_harness/operation_registry.py +1007 -0
- mostlyright/data_harness/operator_setup.py +239 -0
- mostlyright/data_harness/pipeline.py +6428 -0
- mostlyright/data_harness/plan_graph.py +2026 -0
- mostlyright/data_harness/preparation/__init__.py +104 -0
- mostlyright/data_harness/preparation/contracts.py +1017 -0
- mostlyright/data_harness/preparation/engine.py +221 -0
- mostlyright/data_harness/preparation/errors.py +14 -0
- mostlyright/data_harness/preparation/gates.py +751 -0
- mostlyright/data_harness/preparation/joins.py +574 -0
- mostlyright/data_harness/preparation/profile.py +384 -0
- mostlyright/data_harness/preparation/table.py +217 -0
- mostlyright/data_harness/preparation/transforms.py +568 -0
- mostlyright/data_harness/progress_events.py +534 -0
- mostlyright/data_harness/readers/__init__.py +46 -0
- mostlyright/data_harness/readers/containers.py +963 -0
- mostlyright/data_harness/readers/contracts.py +542 -0
- mostlyright/data_harness/readers/delimited.py +257 -0
- mostlyright/data_harness/readers/grib2/__init__.py +33 -0
- mostlyright/data_harness/readers/grib2/admission.py +722 -0
- mostlyright/data_harness/readers/grib2/decode.py +1009 -0
- mostlyright/data_harness/readers/grib2/geometry.py +1133 -0
- mostlyright/data_harness/readers/grib2/portable_math.py +501 -0
- mostlyright/data_harness/readers/json_tabular.py +485 -0
- mostlyright/data_harness/readers/registry.py +514 -0
- mostlyright/data_harness/readers/samples/README.md +110 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/cities.csv.gz +0 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/cities.csv.gz +0 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/cities.tar +0 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/cities.tar +0 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/cities.zip +0 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/expected.json +25 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/expected.json +25 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/expected.json +25 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/cities.csv +3 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/expected.json +23 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/cities.tsv +5 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/expected.json +25 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/expected.json +30 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/observations.csv +5 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/expected.json +44 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/stations.json +1 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/expected.json +48 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/observations.ndjson +4 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/cities.xlsx +0 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/expected.json +24 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/expected.json +27 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/expected.json +27 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/README.md +20 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/expected.json +55 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/gfs-2m-temperature.grib2 +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/expected.json +54 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/expected.json +54 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/hrrr-categorical-rain.grib2 +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/expected.json +54 -0
- mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
- mostlyright/data_harness/readers/samples.py +582 -0
- mostlyright/data_harness/readers/spreadsheet.py +803 -0
- mostlyright/data_harness/readers/tabular.py +510 -0
- mostlyright/data_harness/recipe.py +5321 -0
- mostlyright/data_harness/repair/__init__.py +78 -0
- mostlyright/data_harness/repair/adapters.py +274 -0
- mostlyright/data_harness/repair/contracts.py +872 -0
- mostlyright/data_harness/repair/coordinator.py +1099 -0
- mostlyright/data_harness/repair/errors.py +16 -0
- mostlyright/data_harness/review.py +2533 -0
- mostlyright/data_harness/rowset.py +283 -0
- mostlyright/data_harness/serving.py +1975 -0
- mostlyright/data_harness/serving_edge.py +590 -0
- mostlyright/data_harness/serving_http.py +1031 -0
- mostlyright/data_harness/session_probes.py +759 -0
- mostlyright/data_harness/signing.py +101 -0
- mostlyright/data_harness/source_discovery.py +898 -0
- mostlyright/data_harness/sources/__init__.py +209 -0
- mostlyright/data_harness/sources/_adapter_steps.py +213 -0
- mostlyright/data_harness/sources/adapters.py +1214 -0
- mostlyright/data_harness/sources/cadence.py +1428 -0
- mostlyright/data_harness/sources/cadence_emission.py +453 -0
- mostlyright/data_harness/sources/cadence_history.py +546 -0
- mostlyright/data_harness/sources/catalog/__init__.py +17 -0
- mostlyright/data_harness/sources/catalog/admission.py +477 -0
- mostlyright/data_harness/sources/catalog/authoring.py +1701 -0
- mostlyright/data_harness/sources/catalog/authoring_policy.py +701 -0
- mostlyright/data_harness/sources/catalog/authoring_shards.py +1217 -0
- mostlyright/data_harness/sources/catalog/bounded_io.py +231 -0
- mostlyright/data_harness/sources/catalog/channel.py +523 -0
- mostlyright/data_harness/sources/catalog/channel_client.py +296 -0
- mostlyright/data_harness/sources/catalog/contracts.py +825 -0
- mostlyright/data_harness/sources/catalog/coverage.py +137 -0
- mostlyright/data_harness/sources/catalog/delta.py +1340 -0
- mostlyright/data_harness/sources/catalog/embedding.py +532 -0
- mostlyright/data_harness/sources/catalog/entry_v2.py +1182 -0
- mostlyright/data_harness/sources/catalog/fill.py +3889 -0
- mostlyright/data_harness/sources/catalog/fill_partitions.py +459 -0
- mostlyright/data_harness/sources/catalog/fill_staging.py +1105 -0
- mostlyright/data_harness/sources/catalog/gating.py +374 -0
- mostlyright/data_harness/sources/catalog/generation_receipt.py +1607 -0
- mostlyright/data_harness/sources/catalog/harvest/__init__.py +7 -0
- mostlyright/data_harness/sources/catalog/harvest/ckan.py +384 -0
- mostlyright/data_harness/sources/catalog/harvest/datagov_v4.py +798 -0
- mostlyright/data_harness/sources/catalog/harvest/protocol.py +964 -0
- mostlyright/data_harness/sources/catalog/harvest/sdmx.py +445 -0
- mostlyright/data_harness/sources/catalog/harvest/stac.py +384 -0
- mostlyright/data_harness/sources/catalog/health.py +447 -0
- mostlyright/data_harness/sources/catalog/hosted_catalog.py +105 -0
- mostlyright/data_harness/sources/catalog/identity_history.py +1549 -0
- mostlyright/data_harness/sources/catalog/neural.py +1618 -0
- mostlyright/data_harness/sources/catalog/packed_catalog.py +2345 -0
- mostlyright/data_harness/sources/catalog/packed_retrieval.py +1517 -0
- mostlyright/data_harness/sources/catalog/packed_writer.py +2802 -0
- mostlyright/data_harness/sources/catalog/query_trace.py +1037 -0
- mostlyright/data_harness/sources/catalog/recommend.py +171 -0
- mostlyright/data_harness/sources/catalog/retrieval.py +230 -0
- mostlyright/data_harness/sources/catalog/retrieval_manifest.py +995 -0
- mostlyright/data_harness/sources/catalog/rights_decisions.py +254 -0
- mostlyright/data_harness/sources/catalog/sealed.py +560 -0
- mostlyright/data_harness/sources/catalog/search.py +230 -0
- mostlyright/data_harness/sources/catalog/streaming_delta.py +1097 -0
- mostlyright/data_harness/sources/catalog/update.py +891 -0
- mostlyright/data_harness/sources/collections.py +815 -0
- mostlyright/data_harness/sources/contracts.py +2223 -0
- mostlyright/data_harness/sources/deletion.py +761 -0
- mostlyright/data_harness/sources/fitness.py +162 -0
- mostlyright/data_harness/sources/governance.py +163 -0
- mostlyright/data_harness/sources/hosted.py +173 -0
- mostlyright/data_harness/sources/integration.py +218 -0
- mostlyright/data_harness/sources/range_reader.py +418 -0
- mostlyright/data_harness/sources/registry.py +514 -0
- mostlyright/data_harness/sources/rights_rule.py +59 -0
- mostlyright/data_harness/sources/source_cadence_vectors.v1.json +1 -0
- mostlyright/data_harness/sources/sports.py +521 -0
- mostlyright/data_harness/sources/stream.py +524 -0
- mostlyright/data_harness/sources/stream_connector.py +418 -0
- mostlyright/data_harness/sources/stream_recorder.py +1404 -0
- mostlyright/data_harness/studio_boundary.py +2019 -0
- mostlyright/data_harness/thin/__init__.py +37 -0
- mostlyright/data_harness/thin/acquire.py +1137 -0
- mostlyright/data_harness/thin/acquire_cancel.py +579 -0
- mostlyright/data_harness/thin/approvals.py +617 -0
- mostlyright/data_harness/thin/commands.py +406 -0
- mostlyright/data_harness/thin/download.py +194 -0
- mostlyright/data_harness/thin/narrative.py +589 -0
- mostlyright/data_harness/thin/parity.py +1070 -0
- mostlyright/data_harness/thin/propose.py +2759 -0
- mostlyright/data_harness/thin/research.py +1663 -0
- mostlyright/data_harness/thin/router.py +924 -0
- mostlyright/data_harness/thin/runs.py +519 -0
- mostlyright/data_harness/thin/session.py +281 -0
- mostlyright/data_harness/thin/stream.py +501 -0
- mostlyright/data_harness/thin/transport.py +187 -0
- mostlyright/data_harness/thin/vocabulary.py +368 -0
- mostlyright/data_harness/thin/workers.py +164 -0
- mostlyright/data_harness/ucum/TABLE-PIN.json +40 -0
- mostlyright/data_harness/ucum/ucum-subset.v1.json +632 -0
- mostlyright/data_harness/unit_flow.py +927 -0
- mostlyright/data_harness/units.py +572 -0
- mostlyright/data_harness/ux/__init__.py +9 -0
- mostlyright/data_harness/ux/approve.py +485 -0
- mostlyright/data_harness/ux/author_yaml.py +597 -0
- mostlyright/data_harness/ux/cloud_auth.py +447 -0
- mostlyright/data_harness/ux/commands/__init__.py +260 -0
- mostlyright/data_harness/ux/commands/approve.py +136 -0
- mostlyright/data_harness/ux/commands/auth.py +744 -0
- mostlyright/data_harness/ux/commands/author.py +79 -0
- mostlyright/data_harness/ux/commands/catalog_author.py +403 -0
- mostlyright/data_harness/ux/commands/catalog_fill.py +523 -0
- mostlyright/data_harness/ux/commands/catalog_harvest.py +545 -0
- mostlyright/data_harness/ux/commands/catalog_publish.py +1838 -0
- mostlyright/data_harness/ux/commands/catalog_search.py +71 -0
- mostlyright/data_harness/ux/commands/catalog_update.py +437 -0
- mostlyright/data_harness/ux/commands/deploy.py +134 -0
- mostlyright/data_harness/ux/commands/deploy_dataset.py +98 -0
- mostlyright/data_harness/ux/commands/deploy_plan.py +105 -0
- mostlyright/data_harness/ux/commands/deploy_status.py +104 -0
- mostlyright/data_harness/ux/commands/diff.py +74 -0
- mostlyright/data_harness/ux/commands/index.py +84 -0
- mostlyright/data_harness/ux/commands/inventory.py +47 -0
- mostlyright/data_harness/ux/commands/list_builds.py +143 -0
- mostlyright/data_harness/ux/commands/login.py +63 -0
- mostlyright/data_harness/ux/commands/peek.py +236 -0
- mostlyright/data_harness/ux/commands/plan_check.py +90 -0
- mostlyright/data_harness/ux/commands/preflight.py +97 -0
- mostlyright/data_harness/ux/commands/record.py +107 -0
- mostlyright/data_harness/ux/commands/review_setup.py +47 -0
- mostlyright/data_harness/ux/commands/search.py +440 -0
- mostlyright/data_harness/ux/commands/show.py +61 -0
- mostlyright/data_harness/ux/commands/whoami.py +37 -0
- mostlyright/data_harness/ux/credential_native.py +551 -0
- mostlyright/data_harness/ux/credential_store.py +1055 -0
- mostlyright/data_harness/ux/credentials.py +631 -0
- mostlyright/data_harness/ux/diffing.py +444 -0
- mostlyright/data_harness/ux/headline.py +671 -0
- mostlyright/data_harness/ux/hosted_acquisition.py +974 -0
- mostlyright/data_harness/ux/hosted_run_status.py +619 -0
- mostlyright/data_harness/ux/inventory.py +427 -0
- mostlyright/data_harness/ux/local_review.py +375 -0
- mostlyright/data_harness/ux/login.py +691 -0
- mostlyright/data_harness/ux/path_kind.py +147 -0
- mostlyright/data_harness/ux/peek.py +1000 -0
- mostlyright/data_harness/ux/plain_file.py +178 -0
- mostlyright/data_harness/ux/plan_check.py +311 -0
- mostlyright/data_harness/ux/preflight.py +918 -0
- mostlyright/data_harness/ux/readers.py +1124 -0
- mostlyright/data_harness/ux/remediation.py +2195 -0
- mostlyright/data_harness/ux/render.py +657 -0
- mostlyright/data_harness/ux/workload.py +1077 -0
- mostlyright/data_harness/viewer.py +3713 -0
- mostlyright/data_harness/visual_run/__init__.py +83 -0
- mostlyright/data_harness/visual_run/authoring.py +235 -0
- mostlyright/data_harness/visual_run/contracts.py +673 -0
- mostlyright/data_harness/visual_run/materialize.py +486 -0
- mostlyright/data_harness/visual_run/observations.py +874 -0
- mostlyright/data_harness/visual_run/query.py +259 -0
- mostlyright/data_harness/visual_run/reducer.py +280 -0
- mostlyright/data_harness/visual_run/sdk.py +892 -0
- mostlyright/data_harness/visual_run/store.py +584 -0
- mostlyright/data_harness/visual_run/transport.py +239 -0
- mostlyright/data_harness/watch.py +2999 -0
- mostlyright_data-0.9.0.dist-info/METADATA +607 -0
- mostlyright_data-0.9.0.dist-info/RECORD +314 -0
- mostlyright_data-0.9.0.dist-info/WHEEL +4 -0
- mostlyright_data-0.9.0.dist-info/entry_points.txt +12 -0
|
@@ -0,0 +1,4888 @@
|
|
|
1
|
+
"""Credential-free subprocess boundary for retrieval and untrusted parsing."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import base64
|
|
6
|
+
import contextlib
|
|
7
|
+
import ipaddress
|
|
8
|
+
import json
|
|
9
|
+
import os
|
|
10
|
+
import platform
|
|
11
|
+
import resource
|
|
12
|
+
import secrets
|
|
13
|
+
import selectors
|
|
14
|
+
import signal
|
|
15
|
+
import socket
|
|
16
|
+
import stat
|
|
17
|
+
import subprocess
|
|
18
|
+
import sys
|
|
19
|
+
import threading
|
|
20
|
+
import time
|
|
21
|
+
import unicodedata
|
|
22
|
+
from collections.abc import Iterator, Mapping
|
|
23
|
+
from dataclasses import dataclass, replace
|
|
24
|
+
from pathlib import Path
|
|
25
|
+
from typing import Any, Protocol
|
|
26
|
+
from urllib.parse import urlsplit
|
|
27
|
+
|
|
28
|
+
from mostlyright.data_harness.acquisition.http import (
|
|
29
|
+
MAX_BYTE_RANGE_OFFSET,
|
|
30
|
+
MAX_RANGE_PLAN,
|
|
31
|
+
SIDECAR_MEDIA_TYPES,
|
|
32
|
+
SLICE_OBJECT_MEDIA_TYPES,
|
|
33
|
+
AcquisitionLimiter,
|
|
34
|
+
PeerAttempt,
|
|
35
|
+
PinnedHttpsRetriever,
|
|
36
|
+
ProbeTransport,
|
|
37
|
+
RetrievalHop,
|
|
38
|
+
RetrievalLimits,
|
|
39
|
+
StdlibPinnedTransport,
|
|
40
|
+
probe_transport_for,
|
|
41
|
+
require_range_plan_within_budget,
|
|
42
|
+
)
|
|
43
|
+
from mostlyright.data_harness.acquisition.parsing import (
|
|
44
|
+
ParsedTable,
|
|
45
|
+
ParseLimits,
|
|
46
|
+
parse_tabular_bytes,
|
|
47
|
+
prime_parquet_support,
|
|
48
|
+
)
|
|
49
|
+
from mostlyright.data_harness.acquisition.ranges import (
|
|
50
|
+
PlannedRange,
|
|
51
|
+
SidecarLimits,
|
|
52
|
+
check_grib_message,
|
|
53
|
+
check_grib_reference_time,
|
|
54
|
+
parse_index_sidecar,
|
|
55
|
+
plan_ranges,
|
|
56
|
+
require_reference_time_spelling,
|
|
57
|
+
)
|
|
58
|
+
from mostlyright.data_harness.acquisition.url_policy import (
|
|
59
|
+
AcquisitionSecurityError,
|
|
60
|
+
EgressPolicy,
|
|
61
|
+
SystemResolver,
|
|
62
|
+
validate_public_https_url,
|
|
63
|
+
)
|
|
64
|
+
from mostlyright.data_harness.canonical import canonical_sha256, sha256_bytes
|
|
65
|
+
from mostlyright.data_harness.formats import (
|
|
66
|
+
PARSER_FORMATS_BY_OPERATION,
|
|
67
|
+
READER_CONTRACT_VERSION,
|
|
68
|
+
READER_WORKER_MEMORY_BYTES,
|
|
69
|
+
)
|
|
70
|
+
from mostlyright.data_harness.linux_process_boundary import (
|
|
71
|
+
_disable_process_dumpability,
|
|
72
|
+
_install_networkless_seccomp,
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
# The dependency edge this module owns: ``acquisition`` consumes ``readers``, never the
|
|
76
|
+
# reverse. ``readers/`` is a leaf beneath this one, and anything both layers need is declared
|
|
77
|
+
# in ``formats.py`` where neither has to import the other. A back-edge would close an import
|
|
78
|
+
# cycle whose failure depends on which module a process imports first, so it would break the
|
|
79
|
+
# Reader tests while the application, which imports the other way round, stayed green.
|
|
80
|
+
from mostlyright.data_harness.readers.contracts import (
|
|
81
|
+
DECODE_FLAGS,
|
|
82
|
+
READER_ERROR_CODES,
|
|
83
|
+
ReaderPin,
|
|
84
|
+
)
|
|
85
|
+
from mostlyright.data_harness.readers.registry import TOOLBOX
|
|
86
|
+
|
|
87
|
+
# V2 adds the exact fetched byte count to Reader retrieval responses. The response shape is strict,
|
|
88
|
+
# so adding that measured fact is a protocol change rather than an optional field on V1.
|
|
89
|
+
SANDBOX_PROTOCOL_VERSION = "harness-crawler-sandbox.v1"
|
|
90
|
+
|
|
91
|
+
# Every operation this boundary admits, named once and read by both allowlists -- the
|
|
92
|
+
# coordinator-side one in ``sandbox_policy_digest`` and the worker-side one in
|
|
93
|
+
# ``_execute_worker_request``. Two literal sets would be two places to remember: registered
|
|
94
|
+
# on the coordinator side alone an operation computes a digest no worker will execute, and on
|
|
95
|
+
# the worker side alone it executes under a digest no coordinator can bind.
|
|
96
|
+
SANDBOX_OPERATIONS = frozenset(
|
|
97
|
+
{
|
|
98
|
+
"parse",
|
|
99
|
+
"probe",
|
|
100
|
+
"retrieve_and_parse",
|
|
101
|
+
"decode_and_parse",
|
|
102
|
+
"retrieve_decode_and_parse",
|
|
103
|
+
"fetch_ranges",
|
|
104
|
+
"survey_cycles",
|
|
105
|
+
}
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
# The operations that actually open a socket. This set is the *single* source of socket
|
|
109
|
+
# authority: ``_policy_document`` derives the attested ``network`` clause from it, and
|
|
110
|
+
# ``_invoke`` derives from it whether an invocation may resolve approved endpoints at all.
|
|
111
|
+
# Neither side is written by hand at a call site, because a hand-written flag beside a
|
|
112
|
+
# derived clause is two independent literals and the failure mode is silent and severe: an
|
|
113
|
+
# operation invoked with socket authority while the clause still named only
|
|
114
|
+
# ``retrieve_and_parse`` would attest ``"os-denied"`` while holding a live TCP/443 rule.
|
|
115
|
+
# The attestation would describe a posture the sandbox does not have, which is worse
|
|
116
|
+
# than no attestation at all. With one source, granting the rule and attesting it are one
|
|
117
|
+
# edit, and an operation left out of this set fails closed -- no socket -- rather than open.
|
|
118
|
+
NETWORK_PERFORMING_OPERATIONS = frozenset(
|
|
119
|
+
{"retrieve_and_parse", "retrieve_decode_and_parse", "fetch_ranges", "survey_cycles"}
|
|
120
|
+
)
|
|
121
|
+
|
|
122
|
+
# Every operation that executes a certified Reader must have the Reader worker's enforceable
|
|
123
|
+
# memory boundary. Keep this classification independent of socket authority: the composed
|
|
124
|
+
# HTTPS operation both opens a socket and executes a Reader, while local decode only does the
|
|
125
|
+
# latter. Deriving the off-Linux refusal from this set prevents a newly composed Reader path
|
|
126
|
+
# from accidentally bypassing the clean-room memory invariant.
|
|
127
|
+
READER_PERFORMING_OPERATIONS = frozenset({"decode_and_parse", "retrieve_decode_and_parse"})
|
|
128
|
+
# Every operation whose terminal response *is* the document a cadence probe observed. A range
|
|
129
|
+
# acquisition and a cycle survey are excluded on purpose: neither settles on one whole document,
|
|
130
|
+
# and both already retain per-response status and validator evidence in ``fetched_members``.
|
|
131
|
+
WHOLE_DOCUMENT_RETRIEVAL_OPERATIONS = frozenset({"retrieve_and_parse", "retrieve_decode_and_parse"})
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
class ConfinedWorkerRefusal(AcquisitionSecurityError):
|
|
135
|
+
"""One refusal raised inside the confinement, carrying which operation raised it.
|
|
136
|
+
|
|
137
|
+
Every worker refusal reaches a caller as ``SANDBOX_FAILURE``, because a confinement that
|
|
138
|
+
described itself in its own refusals would be describing the confinement. That is right
|
|
139
|
+
for what a person is told and wrong for one caller: an empirical cadence observation may
|
|
140
|
+
only be recorded for a refusal that provably reached the source, and "reached the source"
|
|
141
|
+
is a question about *which operation* was running and *which code* it raised, neither of
|
|
142
|
+
which survives the collapse.
|
|
143
|
+
|
|
144
|
+
So both facts ride the refusal as structured values while ``code`` stays ``SANDBOX_FAILURE``
|
|
145
|
+
and ``detail`` stays exactly what it was. Every existing catch site is unchanged: this is
|
|
146
|
+
an ``AcquisitionSecurityError`` with the same code and the same sentence. ``worker_code``
|
|
147
|
+
is ``None`` when the worker's stderr yielded no code at all, which is the redacted case and
|
|
148
|
+
is evidence of nothing.
|
|
149
|
+
"""
|
|
150
|
+
|
|
151
|
+
def __init__(
|
|
152
|
+
self,
|
|
153
|
+
code: str,
|
|
154
|
+
detail: str,
|
|
155
|
+
*,
|
|
156
|
+
worker_code: str | None,
|
|
157
|
+
operation: str,
|
|
158
|
+
) -> None:
|
|
159
|
+
super().__init__(code, detail)
|
|
160
|
+
self.worker_code = worker_code
|
|
161
|
+
self.operation = operation
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
@dataclass(frozen=True)
|
|
165
|
+
class _WorkerRefusal:
|
|
166
|
+
"""What the worker's stderr yielded: its typed code, and the sentence a person is told."""
|
|
167
|
+
|
|
168
|
+
code: str | None
|
|
169
|
+
message: str
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
# The one line a pre-warmed child writes before it reads its request, and the only difference
|
|
173
|
+
# between a warm child's stream and a cold child's. It is written after the child has finished
|
|
174
|
+
# importing and before it has read a byte of stdin, so a coordinator that has read it knows the
|
|
175
|
+
# expensive half of a cold start is already paid. Cold children never write it and the argv that
|
|
176
|
+
# asks for it is the argv that produces it, so the two streams can never be confused.
|
|
177
|
+
WORKER_READY_MARKER = b"mostlyright-sandbox-worker-ready\n"
|
|
178
|
+
WORKER_ARGV = ("--worker",)
|
|
179
|
+
WORKER_PREWARM_ARGV = ("--worker", "--prewarm")
|
|
180
|
+
MAX_REQUEST_BYTES = 24 * 1024 * 1024
|
|
181
|
+
MAX_RESPONSE_BYTES = 48 * 1024 * 1024
|
|
182
|
+
MAX_WORKER_STDERR_BYTES = 64 * 1024
|
|
183
|
+
# The worker imports PyArrow and may decode a 128 MiB admitted member. The 512 MiB process limit
|
|
184
|
+
# includes interpreter and decoder overhead.
|
|
185
|
+
MAX_WORKER_MEMORY_BYTES = READER_WORKER_MEMORY_BYTES
|
|
186
|
+
MAX_WORKER_PIDS = 64
|
|
187
|
+
# The hosted research-session service is deliberately one session per 4 GiB Cloud Run instance.
|
|
188
|
+
# This is a separate boundary from the 512 MiB operator-delegated cgroup root above: the container
|
|
189
|
+
# is the hard aggregate memory ceiling for the coordinator, parser child, and every descendant,
|
|
190
|
+
# while an exec-inherited seccomp filter denies networking inside the parser tree. The exact value
|
|
191
|
+
# is checked against the live controller before every spawn; a resource-class change therefore
|
|
192
|
+
# refuses until Harness and Studio move together instead of quietly widening the Clean room.
|
|
193
|
+
HOSTED_SESSION_CONTAINER_MEMORY_BYTES = 4 * 1024 * 1024 * 1024
|
|
194
|
+
_HOSTED_SESSION_CGROUP_ROOT = Path("/sys/fs/cgroup")
|
|
195
|
+
# The most idle children one coordinator may hold. Each one is a live process, a cgroup, and
|
|
196
|
+
# three pipes against an RLIMIT_NOFILE the coordinator also has to live inside, so the pool is
|
|
197
|
+
# bounded for the same reason every other budget in this module is: an unbounded one is a
|
|
198
|
+
# resource-exhaustion lever pointed at the host that granted it.
|
|
199
|
+
MAX_WARM_POOL_SIZE = 32
|
|
200
|
+
# And a coordinator holding a bounded cgroup root may hold no warm children at all. Not because
|
|
201
|
+
# of the descendant ceiling -- that would admit one -- but because of the stricter precondition
|
|
202
|
+
# beside it: ``_require_bounded_cgroup_root`` refuses a root whose ``nr_descendants`` is anything
|
|
203
|
+
# other than 1, meaning the operator-owned coordinator child and nothing else, and it is re-audited
|
|
204
|
+
# before *every* Linux spawn rather than once at construction. An idle warm child holds a job leaf,
|
|
205
|
+
# which makes that count 2, so a single warm child would turn every subsequent cold spawn -- every
|
|
206
|
+
# retrieval, and every request whose confinement does not match the pool -- into a
|
|
207
|
+
# ``SANDBOX_MEMORY_BOUNDARY`` refusal. A pool that breaks the path it is supposed to fall back to
|
|
208
|
+
# is not a pool, so under that root the answer is zero and the coordinator keeps the behaviour it
|
|
209
|
+
# has always had.
|
|
210
|
+
#
|
|
211
|
+
# Making Linux poolable means teaching that audit to distinguish a coordinator-owned warm leaf from
|
|
212
|
+
# a leaf an attacker pre-created. That is a change to a reviewed property of the memory boundary,
|
|
213
|
+
# which is the owner's decision and not a latency one, so it is surfaced here rather than taken.
|
|
214
|
+
BOUNDED_CGROUP_WARM_POOL_SIZE = 0
|
|
215
|
+
# The whole of one ``start_warm_pool`` call, not a budget per child. Serial per-child readiness
|
|
216
|
+
# waits multiply, and a coordinator asking for a full pool against a broken interpreter should
|
|
217
|
+
# find out in a minute rather than in the sixteen that thirty seconds times thirty-two would cost.
|
|
218
|
+
WARM_POOL_FILL_TIMEOUT_SECONDS = 60.0
|
|
219
|
+
_CGROUP_CLEANUP_TIMEOUT_SECONDS = 2.0
|
|
220
|
+
# Exact environment handed to the worker process; nothing is inherited. The key set is
|
|
221
|
+
# bound into the policy digest and is what the worker's reported keys are checked
|
|
222
|
+
# against, so adding or removing a key changes that digest. Values are not digest-covered.
|
|
223
|
+
_ALLOWED_ENVIRONMENT = {
|
|
224
|
+
# macOS Core Foundation text-encoding hint. It predates ADR 0021, which removed
|
|
225
|
+
# the Seatbelt profile that made reading it from the home directory a denial rather than a
|
|
226
|
+
# fallback, and it stays because this key set is bound into the policy digest: dropping it
|
|
227
|
+
# would rotate every operation's attestation to delete a string nothing reads back.
|
|
228
|
+
"__CF_USER_TEXT_ENCODING": "0x1F5:0x0:0x0",
|
|
229
|
+
"LANG": "C.UTF-8",
|
|
230
|
+
"LC_ALL": "C.UTF-8",
|
|
231
|
+
"PATH": os.defpath,
|
|
232
|
+
"PYTHONDONTWRITEBYTECODE": "1",
|
|
233
|
+
"PYTHONIOENCODING": "utf-8",
|
|
234
|
+
"PYTHONNOUSERSITE": "1",
|
|
235
|
+
}
|
|
236
|
+
_FORBIDDEN_EGRESS_HOSTS = frozenset(
|
|
237
|
+
{
|
|
238
|
+
"api.mostlyright.md",
|
|
239
|
+
"studio.mostlyright.md",
|
|
240
|
+
"metadata.google.internal",
|
|
241
|
+
"metadata.goog",
|
|
242
|
+
"localhost",
|
|
243
|
+
}
|
|
244
|
+
)
|
|
245
|
+
# The longest a single sidecar descriptor may be. ``SidecarLimits.max_line_bytes`` bounds the
|
|
246
|
+
# line a descriptor is read from, so a selector longer than a whole line could match nothing
|
|
247
|
+
# and is refused before it is carried anywhere.
|
|
248
|
+
MAX_SELECTOR_BYTES = 512
|
|
249
|
+
# The most slice bytes one invocation may carry back. See ``_require_slice_bytes_within_budget``
|
|
250
|
+
# for the base64 arithmetic that puts this comfortably inside ``MAX_RESPONSE_BYTES``.
|
|
251
|
+
MAX_TOTAL_SLICE_BYTES = 24 * 1024 * 1024
|
|
252
|
+
# The exact, closed shape of a range plan. Both addresses are coordinator-supplied: the
|
|
253
|
+
# worker never derives one from the other, because an address a worker constructs is an
|
|
254
|
+
# address no coordinator resolved and pinned.
|
|
255
|
+
_RANGE_PLAN_KEYS = frozenset(
|
|
256
|
+
{"sidecar_url", "object_url", "selectors", "max_ranges", "expected_reference_time"}
|
|
257
|
+
)
|
|
258
|
+
_FETCHED_SLICE_KEYS = frozenset(
|
|
259
|
+
{"index", "descriptor", "first_byte", "last_byte", "sha256", "content_base64"}
|
|
260
|
+
)
|
|
261
|
+
_FETCHED_MEMBER_KEYS = frozenset(
|
|
262
|
+
{
|
|
263
|
+
"role",
|
|
264
|
+
"sequence",
|
|
265
|
+
"url",
|
|
266
|
+
"resolution_digest",
|
|
267
|
+
"connected_peer",
|
|
268
|
+
"status",
|
|
269
|
+
"response_headers_digest",
|
|
270
|
+
"response_body_size_bytes",
|
|
271
|
+
"response_body_sha256",
|
|
272
|
+
"requested_range",
|
|
273
|
+
"observed_content_range",
|
|
274
|
+
"etag",
|
|
275
|
+
"last_modified",
|
|
276
|
+
"peer_attempts",
|
|
277
|
+
}
|
|
278
|
+
)
|
|
279
|
+
_PEER_ATTEMPT_KEYS = frozenset(
|
|
280
|
+
{"approved_ip", "outcome", "failure_code", "response_body_size_bytes"}
|
|
281
|
+
)
|
|
282
|
+
_PEER_ATTEMPT_OUTCOMES = frozenset({"transport_failure", "response", "response_failure"})
|
|
283
|
+
_FETCHED_MEMBER_ROLES = frozenset(
|
|
284
|
+
{"cycle_probe", "sidecar_initial", "selected_slice", "sidecar_final"}
|
|
285
|
+
)
|
|
286
|
+
# The most candidate runs one availability survey may probe. A survey costs one request per
|
|
287
|
+
# candidate against a source that has done nothing wrong, so an unbounded candidate list is a
|
|
288
|
+
# denial-of-service lever pointed at somebody else; 32 covers every cycle cadence this exists
|
|
289
|
+
# for, including a full day of quarter-hourly runs.
|
|
290
|
+
MAX_CYCLE_CANDIDATES = 32
|
|
291
|
+
_CYCLE_SURVEY_KEYS = frozenset({"candidates"})
|
|
292
|
+
_CYCLE_CANDIDATE_KEYS = frozenset({"label", "url"})
|
|
293
|
+
_CYCLE_AVAILABILITY_KEYS = frozenset({"label", "present"})
|
|
294
|
+
# The span a candidate is probed with, and the mapping from a typed refusal to absence, both
|
|
295
|
+
# live in ``PinnedHttpsRetriever.probe_presence`` rather than here. A survey asks whether a run
|
|
296
|
+
# exists, so reading more than the one byte that settles the question would be an acquisition
|
|
297
|
+
# nobody ordered, on an address no coordinator has locked yet -- and the boundary that owns the
|
|
298
|
+
# request is the boundary that should own that bound.
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
@dataclass(frozen=True)
|
|
302
|
+
class CycleCandidate:
|
|
303
|
+
"""One candidate run a coordinator asks the worker to look for.
|
|
304
|
+
|
|
305
|
+
The label, not the URL, is what a result is matched back to. The worker never parses
|
|
306
|
+
meaning out of an address -- which run this is, what hour it covers, whether it is newer
|
|
307
|
+
than the last one -- because a worker that read a cycle out of a URL would be choosing,
|
|
308
|
+
and choosing is the coordinator's half of look-then-lock.
|
|
309
|
+
"""
|
|
310
|
+
|
|
311
|
+
label: str
|
|
312
|
+
url: str
|
|
313
|
+
|
|
314
|
+
def __post_init__(self) -> None:
|
|
315
|
+
_bounded_identifier(self.label)
|
|
316
|
+
_https_origin(_bounded_text(self.url, maximum=2_048), code="SANDBOX_CYCLE_SURVEY")
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
@dataclass(frozen=True)
|
|
320
|
+
class CycleAvailability:
|
|
321
|
+
"""What the worker reports for one candidate: it is there, or it is not there yet."""
|
|
322
|
+
|
|
323
|
+
label: str
|
|
324
|
+
present: bool
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
@dataclass(frozen=True)
|
|
328
|
+
class FetchedSlice:
|
|
329
|
+
"""One delivered message: which one it is, exactly which bytes, and its own digest.
|
|
330
|
+
|
|
331
|
+
The digest is per slice rather than per acquisition so a caller can bind an individual
|
|
332
|
+
message. A single digest over the concatenation would make one message's provenance
|
|
333
|
+
unstatable without restating the whole plan.
|
|
334
|
+
|
|
335
|
+
``descriptor`` is the selector this slice answers, and it is carried because delivery
|
|
336
|
+
order is not selection order: ``plan_ranges`` emits in message-index order so the sealed
|
|
337
|
+
digest is a function of the recipe rather than of network timing. Without it a caller
|
|
338
|
+
holds N slices labelled only by a sidecar message number -- a number only the worker ever
|
|
339
|
+
saw the sidecar for -- and wiring "the first field I named" to the first slice silently
|
|
340
|
+
reads a different field. Matching is exact equality in ``plan_ranges``, so the descriptor
|
|
341
|
+
a slice reports is one of the selectors that were ordered, and the coordinator checks it
|
|
342
|
+
against its own order rather than taking the worker's word for the pairing.
|
|
343
|
+
"""
|
|
344
|
+
|
|
345
|
+
index: int
|
|
346
|
+
descriptor: str
|
|
347
|
+
first_byte: int
|
|
348
|
+
last_byte: int
|
|
349
|
+
sha256: str
|
|
350
|
+
content: bytes
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
@dataclass(frozen=True)
|
|
354
|
+
class FetchedMember:
|
|
355
|
+
"""One measured HTTP response, including redirects, in acquisition order."""
|
|
356
|
+
|
|
357
|
+
role: str
|
|
358
|
+
sequence: int
|
|
359
|
+
url: str
|
|
360
|
+
resolution_digest: str
|
|
361
|
+
connected_peer: str
|
|
362
|
+
status: int
|
|
363
|
+
response_headers_digest: str
|
|
364
|
+
response_body_size_bytes: int
|
|
365
|
+
response_body_sha256: str
|
|
366
|
+
requested_range: str | None
|
|
367
|
+
observed_content_range: str | None
|
|
368
|
+
etag: str | None
|
|
369
|
+
last_modified: str | None
|
|
370
|
+
peer_attempts: tuple[PeerAttempt, ...]
|
|
371
|
+
|
|
372
|
+
@property
|
|
373
|
+
def total_response_body_size_bytes(self) -> int:
|
|
374
|
+
"""All response-body bytes consumed while obtaining this retained response."""
|
|
375
|
+
|
|
376
|
+
return sum(attempt.response_body_size_bytes for attempt in self.peer_attempts)
|
|
377
|
+
|
|
378
|
+
def to_dict(self) -> dict[str, Any]:
|
|
379
|
+
return {
|
|
380
|
+
"role": self.role,
|
|
381
|
+
"sequence": self.sequence,
|
|
382
|
+
"url": self.url,
|
|
383
|
+
"resolution_digest": self.resolution_digest,
|
|
384
|
+
"connected_peer": self.connected_peer,
|
|
385
|
+
"status": self.status,
|
|
386
|
+
"response_headers_digest": self.response_headers_digest,
|
|
387
|
+
"response_body_size_bytes": self.response_body_size_bytes,
|
|
388
|
+
"response_body_sha256": self.response_body_sha256,
|
|
389
|
+
"requested_range": self.requested_range,
|
|
390
|
+
"observed_content_range": self.observed_content_range,
|
|
391
|
+
"etag": self.etag,
|
|
392
|
+
"last_modified": self.last_modified,
|
|
393
|
+
"peer_attempts": [attempt.to_dict() for attempt in self.peer_attempts],
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
@dataclass(frozen=True)
|
|
398
|
+
class SandboxResult:
|
|
399
|
+
request_id: str
|
|
400
|
+
operation: str
|
|
401
|
+
policy_digest: str
|
|
402
|
+
parsed: ParsedTable | None
|
|
403
|
+
content: bytes | None
|
|
404
|
+
media_type: str | None
|
|
405
|
+
final_url: str | None
|
|
406
|
+
transport_evidence_digest: str | None
|
|
407
|
+
visible_environment_keys: tuple[str, ...]
|
|
408
|
+
# The terminal response's status and hashed validators, sealed inside the Clean room by the
|
|
409
|
+
# operation that retrieved one whole document. ``None`` for every other operation, which is
|
|
410
|
+
# how "this probe observed no single response" stays distinguishable from "it observed one
|
|
411
|
+
# that stated no validator".
|
|
412
|
+
probe_transport: ProbeTransport | None = None
|
|
413
|
+
home_directory_readable: bool | None = None
|
|
414
|
+
network_probe_errors: tuple[tuple[str, int], ...] | None = None
|
|
415
|
+
external_network_policy_attestation: str | None = None
|
|
416
|
+
# The facts a Reader-decoded acquisition writes onto its receipt: which certified code ran,
|
|
417
|
+
# with which sealed settings, and what it reported about the decode and charged geometry.
|
|
418
|
+
# They cross the IPC boundary as data rather than being recomputed here, because the decode
|
|
419
|
+
# did not happen here. All five are ``None`` for an operation that decoded nothing, which
|
|
420
|
+
# is how "no Reader ran" stays distinguishable from "a Reader ran and declared nothing".
|
|
421
|
+
decode_family_id: str | None = None
|
|
422
|
+
decode_family_version: str | None = None
|
|
423
|
+
decode_options_digest: str | None = None
|
|
424
|
+
decode_flags: tuple[str, ...] | None = None
|
|
425
|
+
# Exact family-owned geometry charged against ``max_declared_cells``. This can exceed
|
|
426
|
+
# parsed rows times columns (for example, an XLSX header and declared used area).
|
|
427
|
+
decode_declared_cell_count: int | None = None
|
|
428
|
+
fetched_content_sha256: str | None = None
|
|
429
|
+
# Exact terminal source-body bytes decoded by ``retrieve_decode_and_parse``. It is paired with
|
|
430
|
+
# ``fetched_content_sha256`` and remains the per-member V3 content evidence; normalized output
|
|
431
|
+
# size remains ``len(content)`` and is not substituted for it.
|
|
432
|
+
fetched_content_size_bytes: int | None = None
|
|
433
|
+
# Exact sum of response-body bytes across the retained redirect hop evidence. The terminal
|
|
434
|
+
# body remains ``fetched_content_size_bytes`` because it is the object the Reader decoded;
|
|
435
|
+
# collection resource accounting uses this wider measured transport total.
|
|
436
|
+
fetched_total_response_body_size_bytes: int | None = None
|
|
437
|
+
# Exact peer-attempt count, including failed approved-IP fallbacks and redirects. Kept beside
|
|
438
|
+
# the byte count so a collection coordinator can spend one request budget across several
|
|
439
|
+
# confined invocations.
|
|
440
|
+
fetched_request_count: int | None = None
|
|
441
|
+
ranges: tuple[FetchedSlice, ...] | None = None
|
|
442
|
+
cycle_availability: tuple[CycleAvailability, ...] | None = None
|
|
443
|
+
sidecar_content_sha256: str | None = None
|
|
444
|
+
sidecar_size_bytes: int | None = None
|
|
445
|
+
full_object_size_bytes: int | None = None
|
|
446
|
+
fetched_members: tuple[FetchedMember, ...] | None = None
|
|
447
|
+
|
|
448
|
+
|
|
449
|
+
class CrawlerSandbox:
|
|
450
|
+
"""Trusted coordinator for one clean-environment sandbox process per request."""
|
|
451
|
+
|
|
452
|
+
def __init__(
|
|
453
|
+
self,
|
|
454
|
+
*,
|
|
455
|
+
staging_root: Path,
|
|
456
|
+
timeout_seconds: float = 45.0,
|
|
457
|
+
python_executable: str | None = None,
|
|
458
|
+
external_network_policy_attestation: str | None = None,
|
|
459
|
+
memory_cgroup_root_fd: int | None = None,
|
|
460
|
+
hosted_session_container_memory_bytes: int | None = None,
|
|
461
|
+
) -> None:
|
|
462
|
+
if not staging_root.is_absolute() or staging_root.is_symlink() or not staging_root.is_dir():
|
|
463
|
+
raise AcquisitionSecurityError(
|
|
464
|
+
"SANDBOX_ROOT",
|
|
465
|
+
"sandbox staging root must be an existing absolute non-symlink directory",
|
|
466
|
+
)
|
|
467
|
+
if type(timeout_seconds) not in {int, float} or not 0 < timeout_seconds <= 300:
|
|
468
|
+
raise AcquisitionSecurityError(
|
|
469
|
+
"SANDBOX_TIMEOUT",
|
|
470
|
+
"sandbox timeout must be in (0, 300] seconds",
|
|
471
|
+
)
|
|
472
|
+
self._staging_root = staging_root
|
|
473
|
+
self._timeout_seconds = float(timeout_seconds)
|
|
474
|
+
self._python = python_executable or sys.executable
|
|
475
|
+
if external_network_policy_attestation is not None:
|
|
476
|
+
_require_digest(external_network_policy_attestation, "external network policy")
|
|
477
|
+
self._external_network_policy_attestation = external_network_policy_attestation
|
|
478
|
+
if memory_cgroup_root_fd is not None and hosted_session_container_memory_bytes is not None:
|
|
479
|
+
raise AcquisitionSecurityError(
|
|
480
|
+
"SANDBOX_MEMORY_BOUNDARY",
|
|
481
|
+
"delegated and hosted-session memory boundaries are mutually exclusive",
|
|
482
|
+
)
|
|
483
|
+
if memory_cgroup_root_fd is not None:
|
|
484
|
+
_require_bounded_cgroup_root(memory_cgroup_root_fd)
|
|
485
|
+
if hosted_session_container_memory_bytes is not None:
|
|
486
|
+
try:
|
|
487
|
+
_disable_process_dumpability()
|
|
488
|
+
except OSError as error:
|
|
489
|
+
raise AcquisitionSecurityError(
|
|
490
|
+
"SANDBOX_OS_BOUNDARY",
|
|
491
|
+
"hosted-session coordinator process hardening is unavailable",
|
|
492
|
+
) from error
|
|
493
|
+
_require_hosted_session_container_boundary(hosted_session_container_memory_bytes)
|
|
494
|
+
self._memory_cgroup_root_fd = memory_cgroup_root_fd
|
|
495
|
+
self._hosted_session_container_memory_bytes = hosted_session_container_memory_bytes
|
|
496
|
+
# Off unless a caller asks for it. Default behaviour is one fresh spawn per request,
|
|
497
|
+
# exactly as before; the pool is a lifecycle a long-lived session owner opts into and
|
|
498
|
+
# is responsible for closing.
|
|
499
|
+
self._warm_pool: _WarmSandboxPool | None = None
|
|
500
|
+
|
|
501
|
+
# ----------------------------------------------------------------------------------------
|
|
502
|
+
# Warm pool: an opt-in lifecycle for a coordinator that will serve many probes
|
|
503
|
+
# ----------------------------------------------------------------------------------------
|
|
504
|
+
|
|
505
|
+
def start_warm_pool(self, *, size: int) -> int:
|
|
506
|
+
"""Pre-spawn up to ``size`` idle children for invocations that attest no network.
|
|
507
|
+
|
|
508
|
+
Returns the number now ready, which is the number to believe rather than the number
|
|
509
|
+
asked for: dead children are reaped rather than counted, and where the boundary admits
|
|
510
|
+
fewer than ``size`` the answer says so. Calling it again tops the pool back up, so a
|
|
511
|
+
session owner that has just spent a child can refill without tracking what it holds.
|
|
512
|
+
|
|
513
|
+
**A coordinator holding a bounded cgroup root gets zero, and must read the return value
|
|
514
|
+
to find that out.** That is the Linux boundary, and the reason is recorded beside
|
|
515
|
+
``BOUNDED_CGROUP_WARM_POOL_SIZE``: the root audit re-run before every Linux spawn
|
|
516
|
+
refuses a root that contains anything but the coordinator's own child, so one idle warm
|
|
517
|
+
child would turn every cold spawn into a refusal. Zero rather than a typed refusal
|
|
518
|
+
because a caller should not have to branch on the platform to ask for a pool it may not
|
|
519
|
+
get; what it must do is believe the count.
|
|
520
|
+
|
|
521
|
+
Warm is pre-spawned and pre-imported, never reused: each child serves exactly one
|
|
522
|
+
request and exits, and a request that finds no matching child spawns its own. The pool
|
|
523
|
+
is therefore a latency optimisation with no failure mode of its own -- an empty pool,
|
|
524
|
+
a pool of the wrong shape, and a pool that was never started all produce the identical
|
|
525
|
+
cold path.
|
|
526
|
+
|
|
527
|
+
Only the no-network confinement is pooled. A retrieving invocation's approved
|
|
528
|
+
endpoints are derived from the request being made, and those do not exist before the
|
|
529
|
+
request does; a child spawned earlier would either carry socket authority nobody
|
|
530
|
+
validated for the request that binds to it, or authority that differs from what that
|
|
531
|
+
request would have been given. Retrieval therefore spawns cold.
|
|
532
|
+
"""
|
|
533
|
+
|
|
534
|
+
if type(size) is not int or not 0 <= size <= MAX_WARM_POOL_SIZE:
|
|
535
|
+
raise AcquisitionSecurityError(
|
|
536
|
+
"SANDBOX_WARM_POOL",
|
|
537
|
+
f"warm pool size must be an integer in [0, {MAX_WARM_POOL_SIZE}]",
|
|
538
|
+
)
|
|
539
|
+
if self._memory_cgroup_root_fd is not None:
|
|
540
|
+
size = min(size, BOUNDED_CGROUP_WARM_POOL_SIZE)
|
|
541
|
+
if size == 0:
|
|
542
|
+
self.close_warm_pool()
|
|
543
|
+
return 0
|
|
544
|
+
launch = self._worker_launch(prewarm=True)
|
|
545
|
+
if self._warm_pool is None:
|
|
546
|
+
self._warm_pool = _WarmSandboxPool(
|
|
547
|
+
timeout_seconds=self._timeout_seconds,
|
|
548
|
+
memory_cgroup_root_fd=self._memory_cgroup_root_fd,
|
|
549
|
+
hosted_session_container_memory_bytes=(self._hosted_session_container_memory_bytes),
|
|
550
|
+
)
|
|
551
|
+
return self._warm_pool.fill(launch, size=size)
|
|
552
|
+
|
|
553
|
+
def warm_pool_ready_count(self) -> int:
|
|
554
|
+
"""How many pre-spawned children are idle and unbound right now."""
|
|
555
|
+
|
|
556
|
+
return 0 if self._warm_pool is None else self._warm_pool.ready_count()
|
|
557
|
+
|
|
558
|
+
def close_warm_pool(self) -> None:
|
|
559
|
+
"""Kill and reap every idle child. Idempotent; safe to call without a pool."""
|
|
560
|
+
|
|
561
|
+
pool, self._warm_pool = self._warm_pool, None
|
|
562
|
+
if pool is not None:
|
|
563
|
+
pool.close()
|
|
564
|
+
|
|
565
|
+
@contextlib.contextmanager
|
|
566
|
+
def warm_pool(self, *, size: int) -> Iterator[CrawlerSandbox]:
|
|
567
|
+
"""Own a warm pool for the duration of a block, closing it on the way out."""
|
|
568
|
+
|
|
569
|
+
# Inside the ``try``: ``fill`` keeps every child it has already started when a later one
|
|
570
|
+
# fails, so a partial fill that raises still has children to close.
|
|
571
|
+
try:
|
|
572
|
+
self.start_warm_pool(size=size)
|
|
573
|
+
yield self
|
|
574
|
+
finally:
|
|
575
|
+
self.close_warm_pool()
|
|
576
|
+
|
|
577
|
+
def parse(
|
|
578
|
+
self,
|
|
579
|
+
*,
|
|
580
|
+
request_id: str,
|
|
581
|
+
content: bytes,
|
|
582
|
+
data_format: str,
|
|
583
|
+
media_type: str,
|
|
584
|
+
filename: str,
|
|
585
|
+
limits: ParseLimits,
|
|
586
|
+
) -> SandboxResult:
|
|
587
|
+
return self._invoke(
|
|
588
|
+
{
|
|
589
|
+
"schema_version": SANDBOX_PROTOCOL_VERSION,
|
|
590
|
+
"request_id": request_id,
|
|
591
|
+
"operation": "parse",
|
|
592
|
+
"content_base64": base64.b64encode(content).decode("ascii"),
|
|
593
|
+
"data_format": data_format,
|
|
594
|
+
"media_type": media_type,
|
|
595
|
+
"filename": filename,
|
|
596
|
+
"parse_limits": _parse_limits_dict(limits),
|
|
597
|
+
"url": None,
|
|
598
|
+
"allowed_hostnames": [],
|
|
599
|
+
"approved_endpoints": [],
|
|
600
|
+
"retrieval_limits": None,
|
|
601
|
+
"network_probe_targets": [],
|
|
602
|
+
"reader_pin": None,
|
|
603
|
+
"reader_budgets": None,
|
|
604
|
+
"range_plan": None,
|
|
605
|
+
"cycle_survey": None,
|
|
606
|
+
},
|
|
607
|
+
approved_endpoints=(),
|
|
608
|
+
)
|
|
609
|
+
|
|
610
|
+
def decode_and_parse(
|
|
611
|
+
self,
|
|
612
|
+
*,
|
|
613
|
+
request_id: str,
|
|
614
|
+
content: bytes,
|
|
615
|
+
reader_pin: Mapping[str, Any],
|
|
616
|
+
output_format: str,
|
|
617
|
+
limits: ParseLimits,
|
|
618
|
+
reader_budgets: Mapping[str, Any] | None = None,
|
|
619
|
+
) -> SandboxResult:
|
|
620
|
+
"""Decode untrusted bytes with the pinned Reader, inside the confinement.
|
|
621
|
+
|
|
622
|
+
A fourth operation rather than a widened ``parse``. ``parse`` returns a table and a
|
|
623
|
+
Reader must hand back **bytes**, because those bytes become the sealed snapshot.
|
|
624
|
+
Widening ``parse`` would silently grow what a ``parse``-attested worker may do, which
|
|
625
|
+
is a security regression disguised as a smaller diff; a separate operation carries its
|
|
626
|
+
own policy digest and leaves the other three unrotated.
|
|
627
|
+
|
|
628
|
+
``output_format`` is the encoding the coordinator will accept back and is what the
|
|
629
|
+
operation's ``parser_formats`` clause is checked against, so a decode that would seal
|
|
630
|
+
an unattested encoding is refused before the family runs. ``reader_budgets`` may only
|
|
631
|
+
narrow the family's own defaults; it can never buy more room than the family allows.
|
|
632
|
+
"""
|
|
633
|
+
|
|
634
|
+
return self._invoke(
|
|
635
|
+
{
|
|
636
|
+
"schema_version": SANDBOX_PROTOCOL_VERSION,
|
|
637
|
+
"request_id": request_id,
|
|
638
|
+
"operation": "decode_and_parse",
|
|
639
|
+
"content_base64": base64.b64encode(content).decode("ascii"),
|
|
640
|
+
"data_format": output_format,
|
|
641
|
+
# The media type and the filename of the parsed artifact are the Reader
|
|
642
|
+
# family's declarations, not the coordinator's choice, so they travel back on
|
|
643
|
+
# the response rather than out on the request.
|
|
644
|
+
"media_type": None,
|
|
645
|
+
"filename": None,
|
|
646
|
+
"parse_limits": _parse_limits_dict(limits),
|
|
647
|
+
"url": None,
|
|
648
|
+
"allowed_hostnames": [],
|
|
649
|
+
"approved_endpoints": [],
|
|
650
|
+
"retrieval_limits": None,
|
|
651
|
+
"network_probe_targets": [],
|
|
652
|
+
"reader_pin": dict(reader_pin),
|
|
653
|
+
"reader_budgets": None if reader_budgets is None else dict(reader_budgets),
|
|
654
|
+
"range_plan": None,
|
|
655
|
+
"cycle_survey": None,
|
|
656
|
+
},
|
|
657
|
+
approved_endpoints=(),
|
|
658
|
+
)
|
|
659
|
+
|
|
660
|
+
def retrieve_and_parse(
|
|
661
|
+
self,
|
|
662
|
+
*,
|
|
663
|
+
request_id: str,
|
|
664
|
+
url: str,
|
|
665
|
+
allowed_hostnames: tuple[str, ...],
|
|
666
|
+
data_format: str,
|
|
667
|
+
filename: str,
|
|
668
|
+
parse_limits: ParseLimits,
|
|
669
|
+
retrieval_limits: RetrievalLimits,
|
|
670
|
+
) -> SandboxResult:
|
|
671
|
+
_validate_egress_hosts(allowed_hostnames)
|
|
672
|
+
if self._external_network_policy_attestation is None:
|
|
673
|
+
raise AcquisitionSecurityError(
|
|
674
|
+
"SANDBOX_OS_BOUNDARY",
|
|
675
|
+
(
|
|
676
|
+
"live retrieval requires an explicit externally enforced "
|
|
677
|
+
"deny-default public-IP network policy attestation"
|
|
678
|
+
),
|
|
679
|
+
)
|
|
680
|
+
approved_endpoints = _resolve_approved_endpoints(url, allowed_hostnames)
|
|
681
|
+
return self._invoke(
|
|
682
|
+
{
|
|
683
|
+
"schema_version": SANDBOX_PROTOCOL_VERSION,
|
|
684
|
+
"request_id": request_id,
|
|
685
|
+
"operation": "retrieve_and_parse",
|
|
686
|
+
"content_base64": None,
|
|
687
|
+
"data_format": data_format,
|
|
688
|
+
"media_type": None,
|
|
689
|
+
"filename": filename,
|
|
690
|
+
"parse_limits": _parse_limits_dict(parse_limits),
|
|
691
|
+
"url": url,
|
|
692
|
+
"allowed_hostnames": list(allowed_hostnames),
|
|
693
|
+
"approved_endpoints": [
|
|
694
|
+
{
|
|
695
|
+
"hostname": hostname,
|
|
696
|
+
"port": port,
|
|
697
|
+
"approved_ips": list(addresses),
|
|
698
|
+
}
|
|
699
|
+
for hostname, port, addresses in approved_endpoints
|
|
700
|
+
],
|
|
701
|
+
"retrieval_limits": _retrieval_limits_dict(retrieval_limits),
|
|
702
|
+
"network_probe_targets": [],
|
|
703
|
+
"reader_pin": None,
|
|
704
|
+
"reader_budgets": None,
|
|
705
|
+
"range_plan": None,
|
|
706
|
+
"cycle_survey": None,
|
|
707
|
+
},
|
|
708
|
+
approved_endpoints=approved_endpoints,
|
|
709
|
+
)
|
|
710
|
+
|
|
711
|
+
def retrieve_decode_and_parse(
|
|
712
|
+
self,
|
|
713
|
+
*,
|
|
714
|
+
request_id: str,
|
|
715
|
+
url: str,
|
|
716
|
+
allowed_hostnames: tuple[str, ...],
|
|
717
|
+
reader_pin: Mapping[str, Any],
|
|
718
|
+
output_format: str,
|
|
719
|
+
parse_limits: ParseLimits,
|
|
720
|
+
retrieval_limits: RetrievalLimits,
|
|
721
|
+
reader_budgets: Mapping[str, Any] | None = None,
|
|
722
|
+
) -> SandboxResult:
|
|
723
|
+
"""Retrieve, decode, and parse one Reader-pinned public HTTPS source.
|
|
724
|
+
|
|
725
|
+
This is one operation because fetched container bytes must stay inside the clean room.
|
|
726
|
+
Returning them to the coordinator between network and decode operations would create an
|
|
727
|
+
unconfined materialization path and no single attestation would describe the composition.
|
|
728
|
+
"""
|
|
729
|
+
|
|
730
|
+
_validate_egress_hosts(allowed_hostnames)
|
|
731
|
+
if self._external_network_policy_attestation is None:
|
|
732
|
+
raise AcquisitionSecurityError(
|
|
733
|
+
"SANDBOX_OS_BOUNDARY",
|
|
734
|
+
(
|
|
735
|
+
"live Reader retrieval requires an explicit externally enforced "
|
|
736
|
+
"deny-default public-IP network policy attestation"
|
|
737
|
+
),
|
|
738
|
+
)
|
|
739
|
+
approved_endpoints = _resolve_approved_endpoints(url, allowed_hostnames)
|
|
740
|
+
return self._invoke(
|
|
741
|
+
{
|
|
742
|
+
"schema_version": SANDBOX_PROTOCOL_VERSION,
|
|
743
|
+
"request_id": request_id,
|
|
744
|
+
"operation": "retrieve_decode_and_parse",
|
|
745
|
+
"content_base64": None,
|
|
746
|
+
"data_format": output_format,
|
|
747
|
+
"media_type": None,
|
|
748
|
+
"filename": None,
|
|
749
|
+
"parse_limits": _parse_limits_dict(parse_limits),
|
|
750
|
+
"url": url,
|
|
751
|
+
"allowed_hostnames": list(allowed_hostnames),
|
|
752
|
+
"approved_endpoints": [
|
|
753
|
+
{
|
|
754
|
+
"hostname": hostname,
|
|
755
|
+
"port": port,
|
|
756
|
+
"approved_ips": list(addresses),
|
|
757
|
+
}
|
|
758
|
+
for hostname, port, addresses in approved_endpoints
|
|
759
|
+
],
|
|
760
|
+
"retrieval_limits": _retrieval_limits_dict(retrieval_limits),
|
|
761
|
+
"network_probe_targets": [],
|
|
762
|
+
"reader_pin": dict(reader_pin),
|
|
763
|
+
"reader_budgets": None if reader_budgets is None else dict(reader_budgets),
|
|
764
|
+
"range_plan": None,
|
|
765
|
+
"cycle_survey": None,
|
|
766
|
+
},
|
|
767
|
+
approved_endpoints=approved_endpoints,
|
|
768
|
+
)
|
|
769
|
+
|
|
770
|
+
def fetch_ranges(
|
|
771
|
+
self,
|
|
772
|
+
*,
|
|
773
|
+
request_id: str,
|
|
774
|
+
sidecar_url: str,
|
|
775
|
+
object_url: str,
|
|
776
|
+
selectors: tuple[str, ...],
|
|
777
|
+
allowed_hostnames: tuple[str, ...],
|
|
778
|
+
retrieval_limits: RetrievalLimits,
|
|
779
|
+
expected_reference_time: str | None = None,
|
|
780
|
+
) -> SandboxResult:
|
|
781
|
+
"""Fetch an index sidecar and every slice it names, in one clean-room invocation.
|
|
782
|
+
|
|
783
|
+
One invocation is not an optimisation. ``_resolve_approved_endpoints`` runs once per
|
|
784
|
+
invocation, so one invocation is the only way N ranges share one DNS epoch and one
|
|
785
|
+
retrieval budget; a second invocation would be a second chance to be rebound between
|
|
786
|
+
the sidecar that says where to look and the slice taken from there.
|
|
787
|
+
|
|
788
|
+
Both addresses are supplied here and travel together. The worker is never asked to
|
|
789
|
+
derive the sidecar's address from the object's, because an address a worker builds is
|
|
790
|
+
an address no coordinator resolved and pinned.
|
|
791
|
+
|
|
792
|
+
``expected_reference_time`` is the model run this order is for, spelled
|
|
793
|
+
``YYYY-MM-DDTHH:MM:SSZ``. Supplied, it binds every delivered message to that run in the
|
|
794
|
+
worker and again here -- the control the span check and the structural check cannot be,
|
|
795
|
+
because both are properties of a message alone and a re-issued run answers the ordered
|
|
796
|
+
span with a well-formed message of the same length. Omitted, the run binding is not in
|
|
797
|
+
force and the result says so rather than implying otherwise.
|
|
798
|
+
"""
|
|
799
|
+
|
|
800
|
+
if expected_reference_time is not None:
|
|
801
|
+
expected_reference_time = require_reference_time_spelling(expected_reference_time)
|
|
802
|
+
|
|
803
|
+
_validate_egress_hosts(allowed_hostnames)
|
|
804
|
+
if self._external_network_policy_attestation is None:
|
|
805
|
+
raise AcquisitionSecurityError(
|
|
806
|
+
"SANDBOX_OS_BOUNDARY",
|
|
807
|
+
(
|
|
808
|
+
"slice fetching requires an explicit externally enforced "
|
|
809
|
+
"deny-default public-IP network policy attestation"
|
|
810
|
+
),
|
|
811
|
+
)
|
|
812
|
+
approved_endpoints = _resolve_approved_endpoints(sidecar_url, allowed_hostnames)
|
|
813
|
+
result = self._invoke(
|
|
814
|
+
{
|
|
815
|
+
"schema_version": SANDBOX_PROTOCOL_VERSION,
|
|
816
|
+
"request_id": request_id,
|
|
817
|
+
"operation": "fetch_ranges",
|
|
818
|
+
"content_base64": None,
|
|
819
|
+
"data_format": None,
|
|
820
|
+
"media_type": None,
|
|
821
|
+
"filename": None,
|
|
822
|
+
"parse_limits": None,
|
|
823
|
+
"url": None,
|
|
824
|
+
"allowed_hostnames": list(allowed_hostnames),
|
|
825
|
+
"approved_endpoints": [
|
|
826
|
+
{
|
|
827
|
+
"hostname": hostname,
|
|
828
|
+
"port": port,
|
|
829
|
+
"approved_ips": list(addresses),
|
|
830
|
+
}
|
|
831
|
+
for hostname, port, addresses in approved_endpoints
|
|
832
|
+
],
|
|
833
|
+
"retrieval_limits": _retrieval_limits_dict(retrieval_limits),
|
|
834
|
+
"network_probe_targets": [],
|
|
835
|
+
"reader_pin": None,
|
|
836
|
+
"reader_budgets": None,
|
|
837
|
+
"range_plan": {
|
|
838
|
+
"sidecar_url": sidecar_url,
|
|
839
|
+
"object_url": object_url,
|
|
840
|
+
"selectors": list(selectors),
|
|
841
|
+
"max_ranges": retrieval_limits.max_ranges,
|
|
842
|
+
"expected_reference_time": expected_reference_time,
|
|
843
|
+
},
|
|
844
|
+
"cycle_survey": None,
|
|
845
|
+
},
|
|
846
|
+
approved_endpoints=approved_endpoints,
|
|
847
|
+
)
|
|
848
|
+
# The response is evidence, not testimony -- the same posture the slices themselves are
|
|
849
|
+
# held to. The worker refuses a sidecar answered off the object's origin; the
|
|
850
|
+
# coordinator re-makes that check on the address the worker reports, so a response that
|
|
851
|
+
# named no delivered sidecar address at all, or named one somewhere else, is refused on
|
|
852
|
+
# this side too rather than being taken on the worker's word.
|
|
853
|
+
if result.final_url is None or _https_origin(
|
|
854
|
+
result.final_url, code="SANDBOX_SIDECAR_ORIGIN"
|
|
855
|
+
) != _https_origin(object_url, code="SANDBOX_SIDECAR_ORIGIN"):
|
|
856
|
+
raise AcquisitionSecurityError(
|
|
857
|
+
"SANDBOX_SIDECAR_ORIGIN",
|
|
858
|
+
"the clean room's response does not bind the index sidecar to the object's origin",
|
|
859
|
+
)
|
|
860
|
+
return result
|
|
861
|
+
|
|
862
|
+
def survey_cycles(
|
|
863
|
+
self,
|
|
864
|
+
*,
|
|
865
|
+
request_id: str,
|
|
866
|
+
candidates: tuple[CycleCandidate, ...],
|
|
867
|
+
allowed_hostnames: tuple[str, ...],
|
|
868
|
+
retrieval_limits: RetrievalLimits,
|
|
869
|
+
) -> SandboxResult:
|
|
870
|
+
"""Report which of these candidate runs exist. Report only -- never choose one.
|
|
871
|
+
|
|
872
|
+
This is the *look* half of look-then-lock. The candidate list is supplied here, by the
|
|
873
|
+
side that already holds wall-clock authority and the watermark; the worker probes each
|
|
874
|
+
address and says present or absent, and the coordinator locks one and orders exactly
|
|
875
|
+
that. Discovery deliberately does not move into the worker: a candidate a worker
|
|
876
|
+
derived from its own clock at run time is a candidate no coordinator pinned, which is
|
|
877
|
+
the execution binding this boundary exists to preserve.
|
|
878
|
+
|
|
879
|
+
A candidate that is not published yet is reported absent and the invocation succeeds.
|
|
880
|
+
Every other failure -- a server error, a TLS failure, a peer outside the pin, a source
|
|
881
|
+
that ignored the range -- fails the whole invocation, because a broken source reported
|
|
882
|
+
as "nothing new yet" is a dataset that stops updating and never says so.
|
|
883
|
+
"""
|
|
884
|
+
|
|
885
|
+
_validate_egress_hosts(allowed_hostnames)
|
|
886
|
+
if self._external_network_policy_attestation is None:
|
|
887
|
+
raise AcquisitionSecurityError(
|
|
888
|
+
"SANDBOX_OS_BOUNDARY",
|
|
889
|
+
(
|
|
890
|
+
"an availability survey requires an explicit externally enforced "
|
|
891
|
+
"deny-default public-IP network policy attestation"
|
|
892
|
+
),
|
|
893
|
+
)
|
|
894
|
+
# Before ``_resolve_approved_endpoints``, which performs real DNS. A candidate set that
|
|
895
|
+
# was never going to be admitted must cost no lookup, on this host or on the source's.
|
|
896
|
+
_require_surveyable_candidates(candidates, allowed_hostnames=allowed_hostnames)
|
|
897
|
+
approved_endpoints = _resolve_approved_endpoints(candidates[0].url, allowed_hostnames)
|
|
898
|
+
return self._invoke(
|
|
899
|
+
{
|
|
900
|
+
"schema_version": SANDBOX_PROTOCOL_VERSION,
|
|
901
|
+
"request_id": request_id,
|
|
902
|
+
"operation": "survey_cycles",
|
|
903
|
+
"content_base64": None,
|
|
904
|
+
"data_format": None,
|
|
905
|
+
"media_type": None,
|
|
906
|
+
"filename": None,
|
|
907
|
+
"parse_limits": None,
|
|
908
|
+
"url": None,
|
|
909
|
+
"allowed_hostnames": list(allowed_hostnames),
|
|
910
|
+
"approved_endpoints": [
|
|
911
|
+
{
|
|
912
|
+
"hostname": hostname,
|
|
913
|
+
"port": port,
|
|
914
|
+
"approved_ips": list(addresses),
|
|
915
|
+
}
|
|
916
|
+
for hostname, port, addresses in approved_endpoints
|
|
917
|
+
],
|
|
918
|
+
"retrieval_limits": _retrieval_limits_dict(retrieval_limits),
|
|
919
|
+
"network_probe_targets": [],
|
|
920
|
+
"reader_pin": None,
|
|
921
|
+
"reader_budgets": None,
|
|
922
|
+
"range_plan": None,
|
|
923
|
+
"cycle_survey": {
|
|
924
|
+
"candidates": [
|
|
925
|
+
{"label": candidate.label, "url": candidate.url} for candidate in candidates
|
|
926
|
+
]
|
|
927
|
+
},
|
|
928
|
+
},
|
|
929
|
+
approved_endpoints=approved_endpoints,
|
|
930
|
+
)
|
|
931
|
+
|
|
932
|
+
def probe_isolation(
|
|
933
|
+
self,
|
|
934
|
+
*,
|
|
935
|
+
request_id: str,
|
|
936
|
+
loopback_probe_port: int = 9,
|
|
937
|
+
) -> SandboxResult:
|
|
938
|
+
if type(loopback_probe_port) is not int or not 1 <= loopback_probe_port <= 65_535:
|
|
939
|
+
raise AcquisitionSecurityError(
|
|
940
|
+
"SANDBOX_PROBE",
|
|
941
|
+
"loopback probe port must be a valid integer",
|
|
942
|
+
)
|
|
943
|
+
return self._invoke(
|
|
944
|
+
{
|
|
945
|
+
"schema_version": SANDBOX_PROTOCOL_VERSION,
|
|
946
|
+
"request_id": request_id,
|
|
947
|
+
"operation": "probe",
|
|
948
|
+
"content_base64": None,
|
|
949
|
+
"data_format": None,
|
|
950
|
+
"media_type": None,
|
|
951
|
+
"filename": None,
|
|
952
|
+
"parse_limits": None,
|
|
953
|
+
"url": None,
|
|
954
|
+
"allowed_hostnames": [],
|
|
955
|
+
"approved_endpoints": [],
|
|
956
|
+
"retrieval_limits": None,
|
|
957
|
+
"network_probe_targets": [
|
|
958
|
+
{"label": "loopback", "ip": "127.0.0.1", "port": loopback_probe_port},
|
|
959
|
+
{"label": "private", "ip": "10.0.0.1", "port": 443},
|
|
960
|
+
{"label": "metadata", "ip": "169.254.169.254", "port": 80},
|
|
961
|
+
],
|
|
962
|
+
"reader_pin": None,
|
|
963
|
+
"reader_budgets": None,
|
|
964
|
+
"range_plan": None,
|
|
965
|
+
"cycle_survey": None,
|
|
966
|
+
},
|
|
967
|
+
approved_endpoints=(),
|
|
968
|
+
)
|
|
969
|
+
|
|
970
|
+
def _worker_launch(self, *, prewarm: bool) -> _WorkerLaunch:
|
|
971
|
+
"""Build the exact confinement one child is started in.
|
|
972
|
+
|
|
973
|
+
One builder for both paths. A warm child and the cold child it stands in for are
|
|
974
|
+
started from the same specification produced by the same code, so "is a warm child
|
|
975
|
+
indistinguishable from a fresh spawn" is answered by construction rather than by two
|
|
976
|
+
launch sites that have to be kept in agreement.
|
|
977
|
+
"""
|
|
978
|
+
|
|
979
|
+
command = [
|
|
980
|
+
self._python,
|
|
981
|
+
"-I",
|
|
982
|
+
"-m",
|
|
983
|
+
"mostlyright.data_harness.acquisition.sandbox",
|
|
984
|
+
"--worker",
|
|
985
|
+
]
|
|
986
|
+
# One rule, no platform arm. Before ADR 0021 there were two boundaries here: an
|
|
987
|
+
# external deny-default one that every host had to attest, and macOS Seatbelt, which
|
|
988
|
+
# this process composed for itself and which therefore excused a darwin host from the
|
|
989
|
+
# attestation for non-networking operations. ADR 0021 deletes the Seatbelt backend --
|
|
990
|
+
# "The Linux cgroups-v2 sandbox remains and becomes the only boundary implementation,
|
|
991
|
+
# protecting our workers. The Seatbelt backend and every user-machine enforcement path
|
|
992
|
+
# leave the product surface." What is left is the rule that always applied to every
|
|
993
|
+
# other host: no attested boundary, no clean room. A host that cannot present one fails
|
|
994
|
+
# closed rather than running unconfined, which is the same answer this code has always
|
|
995
|
+
# given on Windows.
|
|
996
|
+
if (
|
|
997
|
+
self._external_network_policy_attestation is None
|
|
998
|
+
and self._hosted_session_container_memory_bytes is None
|
|
999
|
+
):
|
|
1000
|
+
raise AcquisitionSecurityError(
|
|
1001
|
+
"SANDBOX_OS_BOUNDARY",
|
|
1002
|
+
(
|
|
1003
|
+
"crawler execution requires an explicit externally enforced "
|
|
1004
|
+
"deny-default network policy attestation"
|
|
1005
|
+
),
|
|
1006
|
+
)
|
|
1007
|
+
return _WorkerLaunch(
|
|
1008
|
+
command=tuple(command),
|
|
1009
|
+
cwd=self._staging_root,
|
|
1010
|
+
env=tuple(sorted(_ALLOWED_ENVIRONMENT.items())),
|
|
1011
|
+
prewarm=prewarm,
|
|
1012
|
+
)
|
|
1013
|
+
|
|
1014
|
+
def _invoke(
|
|
1015
|
+
self,
|
|
1016
|
+
request: dict[str, Any],
|
|
1017
|
+
*,
|
|
1018
|
+
approved_endpoints: tuple[tuple[str, int, tuple[str, ...]], ...],
|
|
1019
|
+
) -> SandboxResult:
|
|
1020
|
+
# Socket authority is derived from the operation being invoked, never handed in
|
|
1021
|
+
# beside it. A ``network_allowed`` argument written at each call site was a second
|
|
1022
|
+
# literal that could disagree with the ``network`` clause this same operation
|
|
1023
|
+
# attests, and the disagreement was invisible: the profile would carry a live
|
|
1024
|
+
# TCP/443 rule while the digest-bound document said ``"os-denied"``. Reading the
|
|
1025
|
+
# authority out of ``request["operation"]`` makes the grant and the attestation the
|
|
1026
|
+
# same decision, taken once, from one set.
|
|
1027
|
+
operation = request["operation"]
|
|
1028
|
+
if operation not in SANDBOX_OPERATIONS:
|
|
1029
|
+
raise AcquisitionSecurityError(
|
|
1030
|
+
"SANDBOX_OPERATION",
|
|
1031
|
+
"sandbox operation is not allowlisted",
|
|
1032
|
+
)
|
|
1033
|
+
network_allowed = operation in NETWORK_PERFORMING_OPERATIONS
|
|
1034
|
+
if self._hosted_session_container_memory_bytes is not None and operation != "parse":
|
|
1035
|
+
raise AcquisitionSecurityError(
|
|
1036
|
+
"SANDBOX_OS_BOUNDARY",
|
|
1037
|
+
"hosted-session container confinement admits only no-network parsing",
|
|
1038
|
+
)
|
|
1039
|
+
# Was darwin-only, and is now every host that is not Linux. The refusal never really
|
|
1040
|
+
# was about macOS: it was about there being no cgroup to bound a Reader's memory with,
|
|
1041
|
+
# which is true of every host without cgroups v2. Seatbelt used to make macOS the one
|
|
1042
|
+
# non-Linux host that got this far with a boundary of its own; with it gone (ADR 0021
|
|
1043
|
+
# ADR 0021) there is no reason to keep naming one platform. The typed code is unchanged,
|
|
1044
|
+
# because ``SANDBOX_MEMORY_BOUNDARY`` is what callers and receipts already read.
|
|
1045
|
+
if operation in READER_PERFORMING_OPERATIONS and sys.platform != "linux":
|
|
1046
|
+
raise AcquisitionSecurityError(
|
|
1047
|
+
"SANDBOX_MEMORY_BOUNDARY",
|
|
1048
|
+
"Clean room Reader execution has no enforceable memory boundary off Linux",
|
|
1049
|
+
)
|
|
1050
|
+
if approved_endpoints and not network_allowed:
|
|
1051
|
+
# Endpoints for an operation that attests no network is a caller confusion, not
|
|
1052
|
+
# something to quietly discard: the request it built is not the request its
|
|
1053
|
+
# policy document describes.
|
|
1054
|
+
raise AcquisitionSecurityError(
|
|
1055
|
+
"SANDBOX_OPERATION",
|
|
1056
|
+
"an operation that attests no network was given approved endpoints",
|
|
1057
|
+
)
|
|
1058
|
+
request = {
|
|
1059
|
+
**request,
|
|
1060
|
+
"external_network_policy_attestation": (self._external_network_policy_attestation),
|
|
1061
|
+
}
|
|
1062
|
+
request_bytes = _ipc_json_bytes(request)
|
|
1063
|
+
if len(request_bytes) > MAX_REQUEST_BYTES:
|
|
1064
|
+
raise AcquisitionSecurityError(
|
|
1065
|
+
"SANDBOX_IPC_LIMIT",
|
|
1066
|
+
"sandbox request exceeds the IPC byte budget",
|
|
1067
|
+
)
|
|
1068
|
+
if (
|
|
1069
|
+
sys.platform == "linux"
|
|
1070
|
+
and self._memory_cgroup_root_fd is None
|
|
1071
|
+
and self._hosted_session_container_memory_bytes is None
|
|
1072
|
+
):
|
|
1073
|
+
raise AcquisitionSecurityError(
|
|
1074
|
+
"SANDBOX_MEMORY_BOUNDARY",
|
|
1075
|
+
"Linux Clean room execution requires a trusted bounded cgroup root",
|
|
1076
|
+
)
|
|
1077
|
+
launch = self._worker_launch(prewarm=False)
|
|
1078
|
+
# A warm child is only ever taken for a confinement identical to the one this
|
|
1079
|
+
# invocation would have spawned for itself, and only when no socket is in play. Taking
|
|
1080
|
+
# it removes it from the pool: a child is handed out once, serves this one request,
|
|
1081
|
+
# and exits.
|
|
1082
|
+
prepared: _PreparedWorker | None = None
|
|
1083
|
+
if self._warm_pool is not None and not network_allowed:
|
|
1084
|
+
prepared = self._warm_pool.take(launch)
|
|
1085
|
+
if prepared is None:
|
|
1086
|
+
# No warm child for this confinement: the untouched cold path, spawn and all.
|
|
1087
|
+
stdout, stderr, returncode = _run_worker_command(
|
|
1088
|
+
launch.argv(),
|
|
1089
|
+
cwd=launch.cwd,
|
|
1090
|
+
env=dict(launch.env),
|
|
1091
|
+
request_bytes=request_bytes,
|
|
1092
|
+
timeout_seconds=self._timeout_seconds,
|
|
1093
|
+
memory_cgroup_root_fd=self._memory_cgroup_root_fd,
|
|
1094
|
+
hosted_session_container_memory_bytes=(self._hosted_session_container_memory_bytes),
|
|
1095
|
+
)
|
|
1096
|
+
else:
|
|
1097
|
+
stdout, stderr, returncode = _complete_worker(
|
|
1098
|
+
prepared,
|
|
1099
|
+
request_bytes=request_bytes,
|
|
1100
|
+
timeout_seconds=self._timeout_seconds,
|
|
1101
|
+
terminate_process_group=(self._hosted_session_container_memory_bytes is not None),
|
|
1102
|
+
)
|
|
1103
|
+
if returncode != 0:
|
|
1104
|
+
refusal = _bounded_worker_error(stderr, returncode=returncode)
|
|
1105
|
+
raise ConfinedWorkerRefusal(
|
|
1106
|
+
"SANDBOX_FAILURE",
|
|
1107
|
+
f"sandbox failed closed: {refusal.message}",
|
|
1108
|
+
worker_code=refusal.code,
|
|
1109
|
+
operation=operation,
|
|
1110
|
+
)
|
|
1111
|
+
response = _strict_json_object(stdout, "sandbox.response")
|
|
1112
|
+
survey = request["cycle_survey"]
|
|
1113
|
+
range_plan = request["range_plan"]
|
|
1114
|
+
return _parse_response(
|
|
1115
|
+
response,
|
|
1116
|
+
expected_request_id=request["request_id"],
|
|
1117
|
+
# The operation this invocation ordered. The policy attestation below is only an
|
|
1118
|
+
# attestation if the operation it is derived from is the coordinator's, not the
|
|
1119
|
+
# responder's.
|
|
1120
|
+
expected_operation=request["operation"],
|
|
1121
|
+
expected_external_network_policy_attestation=(
|
|
1122
|
+
self._external_network_policy_attestation
|
|
1123
|
+
),
|
|
1124
|
+
# The labels this invocation ordered, in the order it ordered them. A survey result
|
|
1125
|
+
# is matched back to a run by label, so the coordinator checks the answer set
|
|
1126
|
+
# against its own question rather than trusting the response to have kept both.
|
|
1127
|
+
expected_cycle_labels=(
|
|
1128
|
+
None
|
|
1129
|
+
if survey is None
|
|
1130
|
+
else tuple(candidate["label"] for candidate in survey["candidates"])
|
|
1131
|
+
),
|
|
1132
|
+
# The descriptors this invocation ordered. One selector is one message, so the set
|
|
1133
|
+
# is known from the request alone: a short answer is a partial acquisition rather
|
|
1134
|
+
# than a smaller success, and a slice that answers a descriptor nobody named is
|
|
1135
|
+
# refused rather than carried back under a label the coordinator never asked for.
|
|
1136
|
+
expected_selectors=(None if range_plan is None else tuple(range_plan["selectors"])),
|
|
1137
|
+
# The model run this invocation ordered, when it ordered one. Read from the
|
|
1138
|
+
# coordinator's own request rather than from the response, for the same reason the
|
|
1139
|
+
# count above is: a binding a response supplies for itself binds nothing.
|
|
1140
|
+
expected_reference_time=(
|
|
1141
|
+
None if range_plan is None else range_plan["expected_reference_time"]
|
|
1142
|
+
),
|
|
1143
|
+
)
|
|
1144
|
+
|
|
1145
|
+
|
|
1146
|
+
def sandbox_policy_digest(operation: str) -> str:
|
|
1147
|
+
"""Return the exact policy digest the coordinator must bind into its context."""
|
|
1148
|
+
|
|
1149
|
+
# The coordinator and worker read the same operation allowlist.
|
|
1150
|
+
if operation not in SANDBOX_OPERATIONS:
|
|
1151
|
+
raise AcquisitionSecurityError(
|
|
1152
|
+
"SANDBOX_OPERATION",
|
|
1153
|
+
"sandbox operation is not allowlisted",
|
|
1154
|
+
)
|
|
1155
|
+
return _policy_digest(operation)
|
|
1156
|
+
|
|
1157
|
+
|
|
1158
|
+
def worker_main(*, prewarm: bool = False) -> int:
|
|
1159
|
+
"""Read one bounded request from stdin and emit one canonical response.
|
|
1160
|
+
|
|
1161
|
+
One request, one response, one exit -- and ``prewarm`` does not change that. It only moves
|
|
1162
|
+
two things earlier: the deferred parser imports, and a marker line saying they are done.
|
|
1163
|
+
Both happen before stdin is read, so a pre-warmed child announces readiness while it still
|
|
1164
|
+
holds nothing a caller supplied, and then behaves exactly as a cold child does.
|
|
1165
|
+
|
|
1166
|
+
Two handlers, and the second one is the point. The typed handler carries the refusal's
|
|
1167
|
+
own code out to the coordinator, which is how an operator is told what to fix. The broad
|
|
1168
|
+
handler underneath it exists because this worker runs on bytes a stranger supplied: a
|
|
1169
|
+
decoder that reports its own failures with an exception type nobody mapped would otherwise
|
|
1170
|
+
unwind out of the process, and the operator would be told only that the sandbox failed
|
|
1171
|
+
closed, with no code and a Python traceback on the stream the coordinator reads. Both
|
|
1172
|
+
outcomes halt the run -- neither handler can make a failed job look like a finished one --
|
|
1173
|
+
but only one of them says what happened.
|
|
1174
|
+
|
|
1175
|
+
The broad handler reports the exception's type and nothing else. An unmapped failure has
|
|
1176
|
+
no vetted message, and the text of an arbitrary exception may carry a path from inside the
|
|
1177
|
+
confinement; the type name is enough to route a bug report and carries nothing a caller
|
|
1178
|
+
supplied. A failure arriving here is a defect in this codebase rather than a refusal a
|
|
1179
|
+
recipe can repair, and the code says so.
|
|
1180
|
+
"""
|
|
1181
|
+
|
|
1182
|
+
if prewarm:
|
|
1183
|
+
# Before the request, never after it: the point of priming is that the import cost is
|
|
1184
|
+
# paid by an idle child rather than by a bound one. A failure here is a startup
|
|
1185
|
+
# failure and is reported on the same channel every other worker refusal uses.
|
|
1186
|
+
try:
|
|
1187
|
+
prime_parquet_support()
|
|
1188
|
+
except Exception as exc:
|
|
1189
|
+
return _worker_error(
|
|
1190
|
+
"SANDBOX_UNEXPECTED",
|
|
1191
|
+
f"pre-warm raised {type(exc).__name__}, which no refusal maps",
|
|
1192
|
+
)
|
|
1193
|
+
sys.stdout.buffer.write(WORKER_READY_MARKER)
|
|
1194
|
+
sys.stdout.buffer.flush()
|
|
1195
|
+
raw = sys.stdin.buffer.read(MAX_REQUEST_BYTES + 1)
|
|
1196
|
+
if len(raw) > MAX_REQUEST_BYTES:
|
|
1197
|
+
return _worker_error("SANDBOX_IPC_LIMIT", "request exceeds the IPC byte budget")
|
|
1198
|
+
try:
|
|
1199
|
+
request = _strict_json_object(raw, "sandbox.request")
|
|
1200
|
+
response = _execute_worker_request(request)
|
|
1201
|
+
encoded = _ipc_json_bytes(response)
|
|
1202
|
+
if len(encoded) > MAX_RESPONSE_BYTES:
|
|
1203
|
+
raise AcquisitionSecurityError(
|
|
1204
|
+
"SANDBOX_IPC_LIMIT",
|
|
1205
|
+
"response exceeds the IPC byte budget",
|
|
1206
|
+
)
|
|
1207
|
+
except (AcquisitionSecurityError, ValueError, TypeError) as exc:
|
|
1208
|
+
code = getattr(exc, "code", "SANDBOX_REQUEST")
|
|
1209
|
+
return _worker_error(code, str(exc))
|
|
1210
|
+
except Exception as exc:
|
|
1211
|
+
return _worker_error(
|
|
1212
|
+
"SANDBOX_UNEXPECTED",
|
|
1213
|
+
f"the worker raised {type(exc).__name__}, which no refusal maps",
|
|
1214
|
+
)
|
|
1215
|
+
sys.stdout.buffer.write(encoded)
|
|
1216
|
+
sys.stdout.buffer.flush()
|
|
1217
|
+
return 0
|
|
1218
|
+
|
|
1219
|
+
|
|
1220
|
+
def _execute_worker_request(request: dict[str, Any]) -> dict[str, Any]:
|
|
1221
|
+
expected = {
|
|
1222
|
+
"schema_version",
|
|
1223
|
+
"request_id",
|
|
1224
|
+
"operation",
|
|
1225
|
+
"content_base64",
|
|
1226
|
+
"data_format",
|
|
1227
|
+
"media_type",
|
|
1228
|
+
"filename",
|
|
1229
|
+
"parse_limits",
|
|
1230
|
+
"url",
|
|
1231
|
+
"allowed_hostnames",
|
|
1232
|
+
"approved_endpoints",
|
|
1233
|
+
"retrieval_limits",
|
|
1234
|
+
"network_probe_targets",
|
|
1235
|
+
# Reader authority. Every operation other than the decode sends both null and refuses
|
|
1236
|
+
# them non-null, exactly as ``parse`` already refuses ``url``. This literal is not
|
|
1237
|
+
# part of the policy document, so adding to it rotates no digest.
|
|
1238
|
+
"reader_pin",
|
|
1239
|
+
"reader_budgets",
|
|
1240
|
+
"range_plan",
|
|
1241
|
+
"cycle_survey",
|
|
1242
|
+
"external_network_policy_attestation",
|
|
1243
|
+
}
|
|
1244
|
+
if set(request) != expected:
|
|
1245
|
+
raise AcquisitionSecurityError(
|
|
1246
|
+
"SANDBOX_FIELDS",
|
|
1247
|
+
"sandbox request contains missing or extra fields",
|
|
1248
|
+
)
|
|
1249
|
+
if request["schema_version"] != SANDBOX_PROTOCOL_VERSION:
|
|
1250
|
+
raise AcquisitionSecurityError("SANDBOX_VERSION", "sandbox protocol version is invalid")
|
|
1251
|
+
request_id = _bounded_identifier(request["request_id"])
|
|
1252
|
+
operation = request["operation"]
|
|
1253
|
+
if operation not in SANDBOX_OPERATIONS:
|
|
1254
|
+
raise AcquisitionSecurityError("SANDBOX_OPERATION", "sandbox operation is not allowlisted")
|
|
1255
|
+
policy_digest = _policy_digest(operation)
|
|
1256
|
+
external_network_policy_attestation = _optional_digest(
|
|
1257
|
+
request["external_network_policy_attestation"]
|
|
1258
|
+
)
|
|
1259
|
+
visible_keys = tuple(sorted(os.environ))
|
|
1260
|
+
base: dict[str, Any] = {
|
|
1261
|
+
"schema_version": SANDBOX_PROTOCOL_VERSION,
|
|
1262
|
+
"request_id": request_id,
|
|
1263
|
+
"operation": operation,
|
|
1264
|
+
"policy_digest": policy_digest,
|
|
1265
|
+
"visible_environment_keys": list(visible_keys),
|
|
1266
|
+
"parsed": None,
|
|
1267
|
+
"content_base64": None,
|
|
1268
|
+
"media_type": None,
|
|
1269
|
+
"final_url": None,
|
|
1270
|
+
"transport_evidence_digest": None,
|
|
1271
|
+
"probe_transport": None,
|
|
1272
|
+
"home_directory_readable": None,
|
|
1273
|
+
"network_probe_errors": None,
|
|
1274
|
+
"decode_family_id": None,
|
|
1275
|
+
"decode_family_version": None,
|
|
1276
|
+
"decode_options_digest": None,
|
|
1277
|
+
"decode_flags": None,
|
|
1278
|
+
"decode_declared_cell_count": None,
|
|
1279
|
+
"fetched_content_sha256": None,
|
|
1280
|
+
"fetched_content_size_bytes": None,
|
|
1281
|
+
"fetched_total_response_body_size_bytes": None,
|
|
1282
|
+
"fetched_request_count": None,
|
|
1283
|
+
"ranges": None,
|
|
1284
|
+
"cycle_availability": None,
|
|
1285
|
+
"sidecar_content_sha256": None,
|
|
1286
|
+
"sidecar_size_bytes": None,
|
|
1287
|
+
"full_object_size_bytes": None,
|
|
1288
|
+
"fetched_members": None,
|
|
1289
|
+
"external_network_policy_attestation": external_network_policy_attestation,
|
|
1290
|
+
}
|
|
1291
|
+
if operation == "probe":
|
|
1292
|
+
_require_null_request_fields(request)
|
|
1293
|
+
probe_targets = _parse_network_probe_targets(request)
|
|
1294
|
+
base["home_directory_readable"] = _home_directory_readable()
|
|
1295
|
+
base["network_probe_errors"] = [
|
|
1296
|
+
{"label": label, "errno": error}
|
|
1297
|
+
for label, error in _probe_network_denials(probe_targets)
|
|
1298
|
+
]
|
|
1299
|
+
return base
|
|
1300
|
+
|
|
1301
|
+
if operation == "fetch_ranges":
|
|
1302
|
+
_require_no_reader_authority(request)
|
|
1303
|
+
return _execute_range_plan(request, base)
|
|
1304
|
+
|
|
1305
|
+
if operation == "survey_cycles":
|
|
1306
|
+
_require_no_reader_authority(request)
|
|
1307
|
+
return _execute_cycle_survey(request, base)
|
|
1308
|
+
|
|
1309
|
+
if request["range_plan"] is not None or request["cycle_survey"] is not None:
|
|
1310
|
+
raise AcquisitionSecurityError(
|
|
1311
|
+
"SANDBOX_FIELDS",
|
|
1312
|
+
"a parsing request must not carry a range plan or a candidate survey",
|
|
1313
|
+
)
|
|
1314
|
+
limits = _parse_parse_limits(request["parse_limits"])
|
|
1315
|
+
data_format = _bounded_text(request["data_format"], maximum=16)
|
|
1316
|
+
# The ``parser_formats`` clause of this operation's policy document is enforced here,
|
|
1317
|
+
# before any bytes reach a parser. Without this check the clause would be attested and
|
|
1318
|
+
# digest-bound but never honoured: ``parse_tabular_bytes`` admits from the global format
|
|
1319
|
+
# tables and never sees ``operation``, so a worker bound to a csv-only attestation would
|
|
1320
|
+
# still decode parquet. An operation absent from the registry declares no parser formats
|
|
1321
|
+
# at all and therefore may parse nothing -- ``probe`` returns above and never reaches here.
|
|
1322
|
+
#
|
|
1323
|
+
# It runs before the filename is read rather than after, which is a move earlier and never
|
|
1324
|
+
# later: no bytes reach a parser that would not have reached one before, and the decode
|
|
1325
|
+
# operation declares no filename of its own because the Reader family declares it.
|
|
1326
|
+
admitted_formats = PARSER_FORMATS_BY_OPERATION.get(operation)
|
|
1327
|
+
if admitted_formats is None or data_format not in admitted_formats:
|
|
1328
|
+
raise AcquisitionSecurityError(
|
|
1329
|
+
"SANDBOX_FORMAT",
|
|
1330
|
+
"sandbox operation is not attested to parse this data format",
|
|
1331
|
+
)
|
|
1332
|
+
if operation == "decode_and_parse":
|
|
1333
|
+
return _decode_in_worker(request, base, output_format=data_format, limits=limits)
|
|
1334
|
+
if operation == "retrieve_decode_and_parse":
|
|
1335
|
+
return _retrieve_decode_in_worker(request, base, output_format=data_format, limits=limits)
|
|
1336
|
+
|
|
1337
|
+
_require_no_reader_authority(request)
|
|
1338
|
+
filename = _bounded_text(request["filename"], maximum=255)
|
|
1339
|
+
if operation == "parse":
|
|
1340
|
+
if request["url"] is not None:
|
|
1341
|
+
raise AcquisitionSecurityError(
|
|
1342
|
+
"SANDBOX_FIELDS",
|
|
1343
|
+
"parse request contains retrieval-only fields",
|
|
1344
|
+
)
|
|
1345
|
+
if (
|
|
1346
|
+
request["allowed_hostnames"] != []
|
|
1347
|
+
or request["approved_endpoints"] != []
|
|
1348
|
+
or request["retrieval_limits"] is not None
|
|
1349
|
+
or request["network_probe_targets"] != []
|
|
1350
|
+
):
|
|
1351
|
+
raise AcquisitionSecurityError(
|
|
1352
|
+
"SANDBOX_FIELDS",
|
|
1353
|
+
"parse request contains retrieval-only authority",
|
|
1354
|
+
)
|
|
1355
|
+
media_type = _bounded_text(request["media_type"], maximum=255)
|
|
1356
|
+
content = _request_content(request)
|
|
1357
|
+
parsed = parse_tabular_bytes(
|
|
1358
|
+
content,
|
|
1359
|
+
data_format=data_format,
|
|
1360
|
+
media_type=media_type,
|
|
1361
|
+
filename=filename,
|
|
1362
|
+
limits=limits,
|
|
1363
|
+
)
|
|
1364
|
+
base["parsed"] = _parsed_dict(parsed)
|
|
1365
|
+
return base
|
|
1366
|
+
|
|
1367
|
+
if request["content_base64"] is not None or request["media_type"] is not None:
|
|
1368
|
+
raise AcquisitionSecurityError(
|
|
1369
|
+
"SANDBOX_FIELDS",
|
|
1370
|
+
"retrieve request cannot supply source bytes or choose response media type",
|
|
1371
|
+
)
|
|
1372
|
+
url = _bounded_text(request["url"], maximum=2_048)
|
|
1373
|
+
allowed_hostnames = _string_tuple(
|
|
1374
|
+
request["allowed_hostnames"],
|
|
1375
|
+
maximum_items=32,
|
|
1376
|
+
item_maximum=253,
|
|
1377
|
+
)
|
|
1378
|
+
_validate_egress_hosts(allowed_hostnames)
|
|
1379
|
+
if request["network_probe_targets"] != []:
|
|
1380
|
+
raise AcquisitionSecurityError(
|
|
1381
|
+
"SANDBOX_FIELDS",
|
|
1382
|
+
"retrieve request cannot carry network probes",
|
|
1383
|
+
)
|
|
1384
|
+
approved_resolver = _parse_approved_endpoints(
|
|
1385
|
+
request["approved_endpoints"],
|
|
1386
|
+
allowed_hostnames=allowed_hostnames,
|
|
1387
|
+
)
|
|
1388
|
+
retrieval_limits = _parse_retrieval_limits(request["retrieval_limits"])
|
|
1389
|
+
retriever = PinnedHttpsRetriever(
|
|
1390
|
+
resolver=approved_resolver,
|
|
1391
|
+
egress_policy=EgressPolicy(allowed_hostnames=allowed_hostnames),
|
|
1392
|
+
transport=StdlibPinnedTransport(),
|
|
1393
|
+
limits=retrieval_limits,
|
|
1394
|
+
)
|
|
1395
|
+
retrieved = retriever.retrieve(url)
|
|
1396
|
+
parsed = parse_tabular_bytes(
|
|
1397
|
+
retrieved.content,
|
|
1398
|
+
data_format=data_format,
|
|
1399
|
+
media_type=retrieved.media_type,
|
|
1400
|
+
filename=filename,
|
|
1401
|
+
limits=limits,
|
|
1402
|
+
)
|
|
1403
|
+
base.update(
|
|
1404
|
+
{
|
|
1405
|
+
"parsed": _parsed_dict(parsed),
|
|
1406
|
+
"content_base64": base64.b64encode(retrieved.content).decode("ascii"),
|
|
1407
|
+
"media_type": retrieved.media_type,
|
|
1408
|
+
"final_url": retrieved.final_url,
|
|
1409
|
+
"transport_evidence_digest": retrieved.transport_evidence_digest,
|
|
1410
|
+
"probe_transport": probe_transport_for(retrieved).to_dict(),
|
|
1411
|
+
}
|
|
1412
|
+
)
|
|
1413
|
+
return base
|
|
1414
|
+
|
|
1415
|
+
|
|
1416
|
+
def _request_content(request: dict[str, Any]) -> bytes:
|
|
1417
|
+
"""Decode the request's carried bytes, refusing anything but canonical base64."""
|
|
1418
|
+
|
|
1419
|
+
try:
|
|
1420
|
+
return base64.b64decode(
|
|
1421
|
+
_bounded_text(request["content_base64"], maximum=MAX_REQUEST_BYTES),
|
|
1422
|
+
validate=True,
|
|
1423
|
+
)
|
|
1424
|
+
except (ValueError, TypeError):
|
|
1425
|
+
raise AcquisitionSecurityError(
|
|
1426
|
+
"SANDBOX_BASE64",
|
|
1427
|
+
"sandbox content is not canonical base64",
|
|
1428
|
+
) from None
|
|
1429
|
+
|
|
1430
|
+
|
|
1431
|
+
def _require_no_reader_authority(request: dict[str, Any]) -> None:
|
|
1432
|
+
"""Refuse Reader authority on an operation that is not the decode.
|
|
1433
|
+
|
|
1434
|
+
The mirror of the retrieval-only refusals: an operation that may not run a Reader must
|
|
1435
|
+
refuse the fields that would name one, rather than ignoring them. An ignored pin is a
|
|
1436
|
+
decoder a caller believes it selected.
|
|
1437
|
+
"""
|
|
1438
|
+
|
|
1439
|
+
if request["reader_pin"] is not None or request["reader_budgets"] is not None:
|
|
1440
|
+
raise AcquisitionSecurityError(
|
|
1441
|
+
"SANDBOX_FIELDS",
|
|
1442
|
+
"only the decode operation may carry Reader authority",
|
|
1443
|
+
)
|
|
1444
|
+
|
|
1445
|
+
|
|
1446
|
+
def _decode_in_worker(
|
|
1447
|
+
request: dict[str, Any],
|
|
1448
|
+
base: dict[str, Any],
|
|
1449
|
+
*,
|
|
1450
|
+
output_format: str,
|
|
1451
|
+
limits: ParseLimits,
|
|
1452
|
+
) -> dict[str, Any]:
|
|
1453
|
+
"""Run one pinned Reader on untrusted bytes and parse what it produced.
|
|
1454
|
+
|
|
1455
|
+
Two rules live here as comments rather than in a document, because they are the security
|
|
1456
|
+
argument for admitting a decoder at all and not implementation detail.
|
|
1457
|
+
|
|
1458
|
+
* **The Reader never constructs a ``ParsedTable``.** It returns bytes, and the one
|
|
1459
|
+
allowlisted parser produces the table and its ``schema_digest`` from those bytes. A
|
|
1460
|
+
Reader therefore cannot attest a schema it did not derive.
|
|
1461
|
+
* **The normalized bytes are parsed inside this same confined process.** Untrusted bytes
|
|
1462
|
+
never cross to the coordinator undecoded; what crosses is the Reader's own canonical
|
|
1463
|
+
output and a table derived from it here, under the policy this operation is bound to.
|
|
1464
|
+
"""
|
|
1465
|
+
|
|
1466
|
+
if (
|
|
1467
|
+
request["url"] is not None
|
|
1468
|
+
or request["allowed_hostnames"] != []
|
|
1469
|
+
or request["approved_endpoints"] != []
|
|
1470
|
+
or request["retrieval_limits"] is not None
|
|
1471
|
+
or request["network_probe_targets"] != []
|
|
1472
|
+
):
|
|
1473
|
+
raise AcquisitionSecurityError(
|
|
1474
|
+
"SANDBOX_FIELDS",
|
|
1475
|
+
"decode request contains retrieval-only authority",
|
|
1476
|
+
)
|
|
1477
|
+
if request["media_type"] is not None or request["filename"] is not None:
|
|
1478
|
+
raise AcquisitionSecurityError(
|
|
1479
|
+
"SANDBOX_FIELDS",
|
|
1480
|
+
"decode request cannot choose the media type or filename the Reader declares",
|
|
1481
|
+
)
|
|
1482
|
+
content = _request_content(request)
|
|
1483
|
+
return _decode_content_in_worker(
|
|
1484
|
+
request,
|
|
1485
|
+
base,
|
|
1486
|
+
output_format=output_format,
|
|
1487
|
+
limits=limits,
|
|
1488
|
+
content=content,
|
|
1489
|
+
)
|
|
1490
|
+
|
|
1491
|
+
|
|
1492
|
+
def _retrieve_decode_in_worker(
|
|
1493
|
+
request: dict[str, Any],
|
|
1494
|
+
base: dict[str, Any],
|
|
1495
|
+
*,
|
|
1496
|
+
output_format: str,
|
|
1497
|
+
limits: ParseLimits,
|
|
1498
|
+
) -> dict[str, Any]:
|
|
1499
|
+
"""Fetch Reader input and normalize it without exporting container bytes."""
|
|
1500
|
+
|
|
1501
|
+
if request["content_base64"] is not None or request["media_type"] is not None:
|
|
1502
|
+
raise AcquisitionSecurityError(
|
|
1503
|
+
"SANDBOX_FIELDS",
|
|
1504
|
+
"Reader retrieval cannot supply source bytes or choose a response media type",
|
|
1505
|
+
)
|
|
1506
|
+
if request["filename"] is not None or request["network_probe_targets"] != []:
|
|
1507
|
+
raise AcquisitionSecurityError(
|
|
1508
|
+
"SANDBOX_FIELDS",
|
|
1509
|
+
"Reader retrieval cannot choose a filename or carry network probes",
|
|
1510
|
+
)
|
|
1511
|
+
url = _bounded_text(request["url"], maximum=2_048)
|
|
1512
|
+
allowed_hostnames = _string_tuple(
|
|
1513
|
+
request["allowed_hostnames"], maximum_items=32, item_maximum=253
|
|
1514
|
+
)
|
|
1515
|
+
_validate_egress_hosts(allowed_hostnames)
|
|
1516
|
+
approved_resolver = _parse_approved_endpoints(
|
|
1517
|
+
request["approved_endpoints"], allowed_hostnames=allowed_hostnames
|
|
1518
|
+
)
|
|
1519
|
+
retrieval_limits = _parse_retrieval_limits(request["retrieval_limits"])
|
|
1520
|
+
retrieved = PinnedHttpsRetriever(
|
|
1521
|
+
resolver=approved_resolver,
|
|
1522
|
+
egress_policy=EgressPolicy(allowed_hostnames=allowed_hostnames),
|
|
1523
|
+
transport=StdlibPinnedTransport(),
|
|
1524
|
+
limits=retrieval_limits,
|
|
1525
|
+
).retrieve(url)
|
|
1526
|
+
base.update(
|
|
1527
|
+
{
|
|
1528
|
+
"final_url": retrieved.final_url,
|
|
1529
|
+
"transport_evidence_digest": retrieved.transport_evidence_digest,
|
|
1530
|
+
"probe_transport": probe_transport_for(retrieved).to_dict(),
|
|
1531
|
+
"fetched_content_sha256": retrieved.content_sha256,
|
|
1532
|
+
"fetched_content_size_bytes": len(retrieved.content),
|
|
1533
|
+
"fetched_total_response_body_size_bytes": (retrieved.total_response_body_size_bytes),
|
|
1534
|
+
# One redirect hop can contain several approved-IP transport attempts. The
|
|
1535
|
+
# retriever charges each of those attempts against ``max_requests``, so collection
|
|
1536
|
+
# evidence must count the same measured events rather than merely the number of
|
|
1537
|
+
# response hops.
|
|
1538
|
+
"fetched_request_count": sum(len(hop.peer_attempts) for hop in retrieved.hops),
|
|
1539
|
+
}
|
|
1540
|
+
)
|
|
1541
|
+
return _decode_content_in_worker(
|
|
1542
|
+
request,
|
|
1543
|
+
base,
|
|
1544
|
+
output_format=output_format,
|
|
1545
|
+
limits=limits,
|
|
1546
|
+
content=retrieved.content,
|
|
1547
|
+
)
|
|
1548
|
+
|
|
1549
|
+
|
|
1550
|
+
def _decode_content_in_worker(
|
|
1551
|
+
request: dict[str, Any],
|
|
1552
|
+
base: dict[str, Any],
|
|
1553
|
+
*,
|
|
1554
|
+
output_format: str,
|
|
1555
|
+
limits: ParseLimits,
|
|
1556
|
+
content: bytes,
|
|
1557
|
+
) -> dict[str, Any]:
|
|
1558
|
+
"""Run the exact pinned Reader and parse its normalized bytes."""
|
|
1559
|
+
|
|
1560
|
+
pin = _reader_pin_from_request(request["reader_pin"])
|
|
1561
|
+
caps = request["reader_budgets"]
|
|
1562
|
+
if caps is not None and not isinstance(caps, dict):
|
|
1563
|
+
raise AcquisitionSecurityError("SANDBOX_FIELDS", "decode request budgets are invalid")
|
|
1564
|
+
|
|
1565
|
+
# Resolution is against the closed Toolbox on an exact coordinate, with no fallback and
|
|
1566
|
+
# no discovery path, and it happens here inside the confinement. An unattested decoder
|
|
1567
|
+
# cannot run even if a request naming one reached the worker.
|
|
1568
|
+
family = TOOLBOX.resolve(pin.family_id, pin.family_version)
|
|
1569
|
+
# Admission before decode: the family's declared output encoding is checked against the
|
|
1570
|
+
# encoding this operation is attested to parse, so a family that would seal an unattested
|
|
1571
|
+
# encoding is refused before it sees a byte.
|
|
1572
|
+
if family.output_format != output_format:
|
|
1573
|
+
raise AcquisitionSecurityError(
|
|
1574
|
+
"SANDBOX_FORMAT",
|
|
1575
|
+
"the pinned Reader family does not emit the format this request is attested to seal",
|
|
1576
|
+
)
|
|
1577
|
+
budgets = family.default_budgets.narrowed_by(caps)
|
|
1578
|
+
if len(content) > budgets.max_input_bytes:
|
|
1579
|
+
raise AcquisitionSecurityError(
|
|
1580
|
+
"SANDBOX_IPC_LIMIT",
|
|
1581
|
+
"decode input exceeds the Reader input byte budget",
|
|
1582
|
+
)
|
|
1583
|
+
result = family.decode(content, pin, budgets)
|
|
1584
|
+
result.validate_for(family.output_format, budgets)
|
|
1585
|
+
parsed = parse_tabular_bytes(
|
|
1586
|
+
result.content,
|
|
1587
|
+
data_format=result.data_format,
|
|
1588
|
+
media_type=result.media_type,
|
|
1589
|
+
filename=result.filename,
|
|
1590
|
+
limits=limits,
|
|
1591
|
+
)
|
|
1592
|
+
base.update(
|
|
1593
|
+
{
|
|
1594
|
+
"parsed": _parsed_dict(parsed),
|
|
1595
|
+
"content_base64": base64.b64encode(result.content).decode("ascii"),
|
|
1596
|
+
"media_type": result.media_type,
|
|
1597
|
+
"decode_family_id": family.family_id,
|
|
1598
|
+
"decode_family_version": family.family_version,
|
|
1599
|
+
"decode_options_digest": pin.options_digest,
|
|
1600
|
+
# Already sorted and de-duplicated by ``ReaderResult``; carried as data because
|
|
1601
|
+
# the coordinator cannot recompute a fact about a decode it did not run.
|
|
1602
|
+
"decode_flags": list(result.flags),
|
|
1603
|
+
"decode_declared_cell_count": result.declared_cell_count,
|
|
1604
|
+
}
|
|
1605
|
+
)
|
|
1606
|
+
return base
|
|
1607
|
+
|
|
1608
|
+
|
|
1609
|
+
def _reader_pin_from_request(value: Any) -> ReaderPin:
|
|
1610
|
+
"""Admit the request's Reader pin as the contract's own three-key object."""
|
|
1611
|
+
|
|
1612
|
+
if not isinstance(value, dict) or set(value) != {
|
|
1613
|
+
"family_id",
|
|
1614
|
+
"family_version",
|
|
1615
|
+
"decode_options",
|
|
1616
|
+
}:
|
|
1617
|
+
raise AcquisitionSecurityError(
|
|
1618
|
+
"SANDBOX_FIELDS",
|
|
1619
|
+
"decode request must carry exactly a three-key Reader pin",
|
|
1620
|
+
)
|
|
1621
|
+
return ReaderPin(value["family_id"], value["family_version"], value["decode_options"])
|
|
1622
|
+
|
|
1623
|
+
|
|
1624
|
+
def _execute_range_plan(request: dict[str, Any], base: dict[str, Any]) -> dict[str, Any]:
|
|
1625
|
+
"""Validate a slice-fetch request's authority and its plan, then run it.
|
|
1626
|
+
|
|
1627
|
+
Ordered so that every refusal that can be made from the request alone is made before any
|
|
1628
|
+
address is looked at: parsing inputs first, then probe authority, then egress, then the
|
|
1629
|
+
pinned endpoint set, then the retrieval budget, and only then the plan those bounds apply
|
|
1630
|
+
to. A plan checked against a budget that had not been validated would be checked against
|
|
1631
|
+
nothing.
|
|
1632
|
+
"""
|
|
1633
|
+
|
|
1634
|
+
for key in ("content_base64", "data_format", "media_type", "filename", "parse_limits", "url"):
|
|
1635
|
+
if request[key] is not None:
|
|
1636
|
+
raise AcquisitionSecurityError(
|
|
1637
|
+
"SANDBOX_FIELDS",
|
|
1638
|
+
"a slice fetch carries no parsing inputs and no single-object URL",
|
|
1639
|
+
)
|
|
1640
|
+
if request["network_probe_targets"] != [] or request["cycle_survey"] is not None:
|
|
1641
|
+
raise AcquisitionSecurityError(
|
|
1642
|
+
"SANDBOX_FIELDS",
|
|
1643
|
+
"a slice fetch cannot carry network probes or a candidate survey",
|
|
1644
|
+
)
|
|
1645
|
+
allowed_hostnames = _string_tuple(
|
|
1646
|
+
request["allowed_hostnames"],
|
|
1647
|
+
maximum_items=32,
|
|
1648
|
+
item_maximum=253,
|
|
1649
|
+
)
|
|
1650
|
+
_validate_egress_hosts(allowed_hostnames)
|
|
1651
|
+
approved_resolver = _parse_approved_endpoints(
|
|
1652
|
+
request["approved_endpoints"],
|
|
1653
|
+
allowed_hostnames=allowed_hostnames,
|
|
1654
|
+
)
|
|
1655
|
+
retrieval_limits = _parse_retrieval_limits(request["retrieval_limits"])
|
|
1656
|
+
plan = _parse_range_plan(request["range_plan"], budget_ceiling=retrieval_limits.max_ranges)
|
|
1657
|
+
|
|
1658
|
+
# One resolver, one transport and one limiter for the whole plan. The resolver is the
|
|
1659
|
+
# coordinator's single DNS epoch, already pinned to exact public addresses; reusing it is
|
|
1660
|
+
# what stops a rebind between the sidecar that says where to look and the slice taken
|
|
1661
|
+
# from there. The limiter is shared for the other half of the same property: N sequential
|
|
1662
|
+
# fetches against one source must be polite as one acquisition rather than as N bursts.
|
|
1663
|
+
#
|
|
1664
|
+
# Two retrievers rather than one, differing only in their media allowlist, because a
|
|
1665
|
+
# sidecar and a slice are different responses and each call should admit only the one it
|
|
1666
|
+
# ordered. Both narrow within whatever the coordinator already allowed and neither can
|
|
1667
|
+
# widen it.
|
|
1668
|
+
egress_policy = EgressPolicy(allowed_hostnames=allowed_hostnames)
|
|
1669
|
+
transport = StdlibPinnedTransport()
|
|
1670
|
+
limiter = AcquisitionLimiter(
|
|
1671
|
+
max_concurrency=retrieval_limits.max_concurrency,
|
|
1672
|
+
min_interval_seconds=retrieval_limits.min_interval_seconds,
|
|
1673
|
+
)
|
|
1674
|
+
|
|
1675
|
+
def retriever(media_types: tuple[str, ...]) -> PinnedHttpsRetriever:
|
|
1676
|
+
return PinnedHttpsRetriever(
|
|
1677
|
+
resolver=approved_resolver,
|
|
1678
|
+
egress_policy=egress_policy,
|
|
1679
|
+
transport=transport,
|
|
1680
|
+
limits=_narrowed_to_media_types(retrieval_limits, media_types),
|
|
1681
|
+
limiter=limiter,
|
|
1682
|
+
)
|
|
1683
|
+
|
|
1684
|
+
# Construct both narrowed retrievers before any request. A coordinator allowlist that
|
|
1685
|
+
# admits neither half is therefore still a request-free refusal even though the retained
|
|
1686
|
+
# presence probe itself admits any media type.
|
|
1687
|
+
sidecar_retriever = retriever(SIDECAR_MEDIA_TYPES)
|
|
1688
|
+
slice_retriever = retriever(SLICE_OBJECT_MEDIA_TYPES)
|
|
1689
|
+
member_documents: list[dict[str, Any]] = []
|
|
1690
|
+
|
|
1691
|
+
def retain_members(role: str, hops: tuple[RetrievalHop, ...]) -> None:
|
|
1692
|
+
for hop in hops:
|
|
1693
|
+
member_documents.append(
|
|
1694
|
+
{
|
|
1695
|
+
"role": role,
|
|
1696
|
+
"sequence": len(member_documents),
|
|
1697
|
+
**hop.to_dict(),
|
|
1698
|
+
}
|
|
1699
|
+
)
|
|
1700
|
+
|
|
1701
|
+
sidecar = sidecar_retriever.retrieve(plan.sidecar_url)
|
|
1702
|
+
retain_members("sidecar_initial", sidecar.hops)
|
|
1703
|
+
# ``_parse_range_plan`` bound the sidecar and the object to one origin, but it bound the
|
|
1704
|
+
# address that was *ordered*. An unranged retrieval follows redirects, so without this the
|
|
1705
|
+
# ordered host can hand the sidecar fetch to any other allowlisted host, and that host then
|
|
1706
|
+
# chooses which bytes of the first host's object are delivered: it renames the offsets of a
|
|
1707
|
+
# message the operator did not order to the descriptor the operator did order, and every
|
|
1708
|
+
# control downstream still passes -- exactly 206, a matching ``Content-Range``, three
|
|
1709
|
+
# lengths in agreement, and a structurally valid GRIB message, just the wrong field. The
|
|
1710
|
+
# ranged fetches are already closed to this by ``REDIRECT_ON_RANGE``; this is the same
|
|
1711
|
+
# property for the half of the order that says where to look. Compared against the object's
|
|
1712
|
+
# origin rather than the ordered sidecar's, because those two are equal by the check above
|
|
1713
|
+
# and the object is the thing whose bytes the index is allowed to speak for.
|
|
1714
|
+
if _https_origin(sidecar.final_url, code="SANDBOX_SIDECAR_ORIGIN") != _https_origin(
|
|
1715
|
+
plan.object_url, code="SANDBOX_SIDECAR_ORIGIN"
|
|
1716
|
+
):
|
|
1717
|
+
raise AcquisitionSecurityError(
|
|
1718
|
+
"SANDBOX_SIDECAR_ORIGIN",
|
|
1719
|
+
"the index sidecar was answered by an origin other than the object's",
|
|
1720
|
+
)
|
|
1721
|
+
# The address that actually answered, so the receipt can state it rather than restating the
|
|
1722
|
+
# address that was ordered. The hop list is digested, not carried, so this is the only field
|
|
1723
|
+
# in which a redirect that was allowed is legible after the fact.
|
|
1724
|
+
base["final_url"] = sidecar.final_url
|
|
1725
|
+
# The sidecar is untrusted input from here on. Its numbers may become a ``ByteRange`` and
|
|
1726
|
+
# nothing else: never an address, never an allocation size, never a length trusted without
|
|
1727
|
+
# the bytes that arrive being measured against it.
|
|
1728
|
+
messages = parse_index_sidecar(sidecar.content, limits=SidecarLimits())
|
|
1729
|
+
planned = plan_ranges(messages, selectors=plan.selectors, max_ranges=plan.max_ranges)
|
|
1730
|
+
_require_slice_bytes_within_budget(planned, limits=retrieval_limits)
|
|
1731
|
+
|
|
1732
|
+
# The selected cycle is probed again inside the acquisition's DNS epoch. Unlike
|
|
1733
|
+
# ``survey_cycles`` this is retained evidence: the object has already been chosen, so the
|
|
1734
|
+
# exact one-byte response and every redirect body belong in the receipt and efficiency
|
|
1735
|
+
# numerator rather than being accepted as a caller-supplied count.
|
|
1736
|
+
probe_retriever = PinnedHttpsRetriever(
|
|
1737
|
+
resolver=approved_resolver,
|
|
1738
|
+
egress_policy=egress_policy,
|
|
1739
|
+
transport=transport,
|
|
1740
|
+
limits=retrieval_limits,
|
|
1741
|
+
limiter=limiter,
|
|
1742
|
+
)
|
|
1743
|
+
probe = probe_retriever.retrieve_presence_evidence(plan.object_url)
|
|
1744
|
+
retain_members("cycle_probe", probe.hops)
|
|
1745
|
+
|
|
1746
|
+
hops = [*sidecar.hops, *probe.hops]
|
|
1747
|
+
slices: list[dict[str, Any]] = []
|
|
1748
|
+
complete_sizes: set[int] = set()
|
|
1749
|
+
for entry in planned:
|
|
1750
|
+
# ``retrieve`` runs the delivery check: exactly 206, a ``Content-Range`` equal to the
|
|
1751
|
+
# span that was ordered, three lengths in agreement. That proves the span. It cannot
|
|
1752
|
+
# prove the framing, so the structural check below runs before this slice is kept, and
|
|
1753
|
+
# neither proves which run the bytes came from, so the run binding below runs too when
|
|
1754
|
+
# the order named a run. Any of the three refusing fails the whole invocation. A
|
|
1755
|
+
# partial slice set is not a smaller success; it is an acquisition that did not happen.
|
|
1756
|
+
parcel = slice_retriever.retrieve(plan.object_url, byte_range=entry.byte_range)
|
|
1757
|
+
hops.extend(parcel.hops)
|
|
1758
|
+
retain_members("selected_slice", parcel.hops)
|
|
1759
|
+
check_grib_message(parcel.content)
|
|
1760
|
+
# The run binding, when the order named a run. Ordered after the structural check
|
|
1761
|
+
# because that check is what makes the section offsets below meaningful at all.
|
|
1762
|
+
if plan.expected_reference_time is not None:
|
|
1763
|
+
check_grib_reference_time(parcel.content, expected=plan.expected_reference_time)
|
|
1764
|
+
if parcel.complete_size_bytes is None:
|
|
1765
|
+
raise AcquisitionSecurityError(
|
|
1766
|
+
"CONTENT_RANGE",
|
|
1767
|
+
"a slice response must name the complete object size",
|
|
1768
|
+
)
|
|
1769
|
+
complete_sizes.add(parcel.complete_size_bytes)
|
|
1770
|
+
slices.append(
|
|
1771
|
+
{
|
|
1772
|
+
"index": entry.index,
|
|
1773
|
+
# The selector this span was planned for. The plan is emitted in message-index
|
|
1774
|
+
# order, so without it the response says which message it delivered and never
|
|
1775
|
+
# which of the ordered descriptors that message answers.
|
|
1776
|
+
"descriptor": entry.descriptor,
|
|
1777
|
+
"first_byte": entry.byte_range.first_byte,
|
|
1778
|
+
"last_byte": entry.byte_range.last_byte,
|
|
1779
|
+
"sha256": parcel.content_sha256,
|
|
1780
|
+
"content_base64": base64.b64encode(parcel.content).decode("ascii"),
|
|
1781
|
+
}
|
|
1782
|
+
)
|
|
1783
|
+
# Re-read the complete sidecar only after every selected member has arrived. The source
|
|
1784
|
+
# revision is admitted only when the terminal bytes are identical to the bytes that chose
|
|
1785
|
+
# the ranges; an update during the acquisition is a refusal, never a mixed revision.
|
|
1786
|
+
terminal_sidecar = sidecar_retriever.retrieve(plan.sidecar_url)
|
|
1787
|
+
retain_members("sidecar_final", terminal_sidecar.hops)
|
|
1788
|
+
hops.extend(terminal_sidecar.hops)
|
|
1789
|
+
if (
|
|
1790
|
+
terminal_sidecar.content_sha256 != sidecar.content_sha256
|
|
1791
|
+
or terminal_sidecar.content != sidecar.content
|
|
1792
|
+
or terminal_sidecar.final_url != sidecar.final_url
|
|
1793
|
+
):
|
|
1794
|
+
raise AcquisitionSecurityError(
|
|
1795
|
+
"SANDBOX_SOURCE_REVISION",
|
|
1796
|
+
"the index sidecar changed while selected messages were being fetched",
|
|
1797
|
+
)
|
|
1798
|
+
base["final_url"] = terminal_sidecar.final_url
|
|
1799
|
+
base["ranges"] = slices
|
|
1800
|
+
if probe.complete_size_bytes is None:
|
|
1801
|
+
raise AcquisitionSecurityError(
|
|
1802
|
+
"CONTENT_RANGE",
|
|
1803
|
+
"the retained cycle probe must name the complete object size",
|
|
1804
|
+
)
|
|
1805
|
+
complete_sizes.add(probe.complete_size_bytes)
|
|
1806
|
+
if len(complete_sizes) != 1:
|
|
1807
|
+
raise AcquisitionSecurityError(
|
|
1808
|
+
"CONTENT_RANGE",
|
|
1809
|
+
"all slice responses must name one complete object size",
|
|
1810
|
+
)
|
|
1811
|
+
base["sidecar_content_sha256"] = sidecar.content_sha256
|
|
1812
|
+
base["sidecar_size_bytes"] = len(sidecar.content)
|
|
1813
|
+
base["full_object_size_bytes"] = complete_sizes.pop()
|
|
1814
|
+
base["fetched_members"] = member_documents
|
|
1815
|
+
# One acquisition, one transport evidence digest: the sidecar hop followed by every range
|
|
1816
|
+
# hop in fetch order, flattened, digested exactly as a single retrieval's hop list is.
|
|
1817
|
+
base["transport_evidence_digest"] = canonical_sha256([hop.to_dict() for hop in hops])
|
|
1818
|
+
return base
|
|
1819
|
+
|
|
1820
|
+
|
|
1821
|
+
def _execute_cycle_survey(request: dict[str, Any], base: dict[str, Any]) -> dict[str, Any]:
|
|
1822
|
+
"""Probe each coordinator-supplied candidate once and report presence, in their order.
|
|
1823
|
+
|
|
1824
|
+
Ordered exactly as ``_execute_range_plan`` is: every refusal that can be made from the
|
|
1825
|
+
request alone is made before any address is looked at, so a candidate set that was never
|
|
1826
|
+
going to be admitted costs no request against a source that has done nothing wrong.
|
|
1827
|
+
"""
|
|
1828
|
+
|
|
1829
|
+
for key in ("content_base64", "data_format", "media_type", "filename", "parse_limits", "url"):
|
|
1830
|
+
if request[key] is not None:
|
|
1831
|
+
raise AcquisitionSecurityError(
|
|
1832
|
+
"SANDBOX_FIELDS",
|
|
1833
|
+
"an availability survey carries no parsing inputs and no single-object URL",
|
|
1834
|
+
)
|
|
1835
|
+
if request["network_probe_targets"] != [] or request["range_plan"] is not None:
|
|
1836
|
+
raise AcquisitionSecurityError(
|
|
1837
|
+
"SANDBOX_FIELDS",
|
|
1838
|
+
"an availability survey cannot carry network probes or a range plan",
|
|
1839
|
+
)
|
|
1840
|
+
allowed_hostnames = _string_tuple(
|
|
1841
|
+
request["allowed_hostnames"],
|
|
1842
|
+
maximum_items=32,
|
|
1843
|
+
item_maximum=253,
|
|
1844
|
+
)
|
|
1845
|
+
_validate_egress_hosts(allowed_hostnames)
|
|
1846
|
+
approved_resolver = _parse_approved_endpoints(
|
|
1847
|
+
request["approved_endpoints"],
|
|
1848
|
+
allowed_hostnames=allowed_hostnames,
|
|
1849
|
+
)
|
|
1850
|
+
retrieval_limits = _parse_retrieval_limits(request["retrieval_limits"])
|
|
1851
|
+
candidates = _parse_cycle_survey(
|
|
1852
|
+
request["cycle_survey"],
|
|
1853
|
+
allowed_hostnames=allowed_hostnames,
|
|
1854
|
+
)
|
|
1855
|
+
|
|
1856
|
+
# One retriever for the candidate set, so one resolver, one transport and one limiter cover
|
|
1857
|
+
# it: the survey is polite to a source as one look rather than as N bursts, and every probe
|
|
1858
|
+
# is answered inside the single DNS epoch the coordinator pinned.
|
|
1859
|
+
#
|
|
1860
|
+
# Every probe goes through ``probe_presence``, which admits any media type. The survey does
|
|
1861
|
+
# not know what a candidate is -- a cycle-addressed CSV run and a GRIB object are both
|
|
1862
|
+
# legitimate candidates, and the GRIB case answers ``application/octet-stream``, which the
|
|
1863
|
+
# default acquisition allowlist deliberately excludes. Judging presence by content type
|
|
1864
|
+
# would report a healthy published run as a hard failure on every poll. The coordinator's
|
|
1865
|
+
# ``retrieval_limits`` still govern this call in every other respect -- budgets, timeouts,
|
|
1866
|
+
# redirects, response ceiling and rate limit -- and its media allowlist governs every call
|
|
1867
|
+
# that actually acquires bytes.
|
|
1868
|
+
retriever = PinnedHttpsRetriever(
|
|
1869
|
+
resolver=approved_resolver,
|
|
1870
|
+
egress_policy=EgressPolicy(allowed_hostnames=allowed_hostnames),
|
|
1871
|
+
transport=StdlibPinnedTransport(),
|
|
1872
|
+
limits=retrieval_limits,
|
|
1873
|
+
)
|
|
1874
|
+
availability: list[dict[str, Any]] = []
|
|
1875
|
+
for candidate in candidates:
|
|
1876
|
+
# Presence and nothing else. No bytes, no digest, no status code: the survey answers one
|
|
1877
|
+
# question, and an address the coordinator has not locked yet is not an acquisition.
|
|
1878
|
+
# Absence is decided inside ``probe_presence`` from the typed ``HTTP_NOT_FOUND`` code
|
|
1879
|
+
# and never from message text; every other refusal propagates and fails the invocation.
|
|
1880
|
+
present = retriever.probe_presence(candidate.url)
|
|
1881
|
+
availability.append({"label": candidate.label, "present": present})
|
|
1882
|
+
base["cycle_availability"] = availability
|
|
1883
|
+
# No transport digest is emitted because not-found responses carry no hop list. A partial
|
|
1884
|
+
# digest would omit the absent candidates being reported.
|
|
1885
|
+
return base
|
|
1886
|
+
|
|
1887
|
+
|
|
1888
|
+
def _parse_cycle_survey(
|
|
1889
|
+
value: Any,
|
|
1890
|
+
*,
|
|
1891
|
+
allowed_hostnames: tuple[str, ...],
|
|
1892
|
+
) -> tuple[CycleCandidate, ...]:
|
|
1893
|
+
"""Parse the closed candidate list, or refuse it.
|
|
1894
|
+
|
|
1895
|
+
No refusal here quotes an address or a label. The refusal crosses the sandbox boundary and
|
|
1896
|
+
both are text that came from outside this process.
|
|
1897
|
+
"""
|
|
1898
|
+
|
|
1899
|
+
if not isinstance(value, dict) or set(value) != _CYCLE_SURVEY_KEYS:
|
|
1900
|
+
raise AcquisitionSecurityError(
|
|
1901
|
+
"SANDBOX_CYCLE_SURVEY",
|
|
1902
|
+
"an availability survey must be the exact closed object of one field",
|
|
1903
|
+
)
|
|
1904
|
+
candidates = value["candidates"]
|
|
1905
|
+
if not isinstance(candidates, list) or not 1 <= len(candidates) <= MAX_CYCLE_CANDIDATES:
|
|
1906
|
+
raise AcquisitionSecurityError(
|
|
1907
|
+
"SANDBOX_CYCLE_SURVEY",
|
|
1908
|
+
f"a survey must name between one and {MAX_CYCLE_CANDIDATES} candidates",
|
|
1909
|
+
)
|
|
1910
|
+
parsed: list[CycleCandidate] = []
|
|
1911
|
+
labels: set[str] = set()
|
|
1912
|
+
for item in candidates:
|
|
1913
|
+
if not isinstance(item, dict) or set(item) != _CYCLE_CANDIDATE_KEYS:
|
|
1914
|
+
raise AcquisitionSecurityError(
|
|
1915
|
+
"SANDBOX_CYCLE_SURVEY",
|
|
1916
|
+
"a survey candidate is exactly a label and an address",
|
|
1917
|
+
)
|
|
1918
|
+
candidate = CycleCandidate(label=item["label"], url=item["url"])
|
|
1919
|
+
# Unique, because the label is the only key a coordinator matches a result back to.
|
|
1920
|
+
# Two candidates sharing one label make an answer set that cannot be read.
|
|
1921
|
+
if candidate.label in labels:
|
|
1922
|
+
raise AcquisitionSecurityError(
|
|
1923
|
+
"SANDBOX_CYCLE_SURVEY",
|
|
1924
|
+
"survey candidate labels must be unique within one request",
|
|
1925
|
+
)
|
|
1926
|
+
labels.add(candidate.label)
|
|
1927
|
+
# Checked here as well as by the egress policy at fetch time, so a candidate outside
|
|
1928
|
+
# the coordinator's allowlist costs no request at all -- including no request for the
|
|
1929
|
+
# legitimate candidates that would otherwise have been probed before reaching it.
|
|
1930
|
+
hostname = urlsplit(candidate.url).hostname
|
|
1931
|
+
if hostname is None or hostname.lower() not in allowed_hostnames:
|
|
1932
|
+
raise AcquisitionSecurityError(
|
|
1933
|
+
"SANDBOX_EGRESS",
|
|
1934
|
+
"a survey candidate names a host outside the egress allowlist",
|
|
1935
|
+
)
|
|
1936
|
+
parsed.append(candidate)
|
|
1937
|
+
return tuple(parsed)
|
|
1938
|
+
|
|
1939
|
+
|
|
1940
|
+
def _require_surveyable_candidates(
|
|
1941
|
+
candidates: tuple[CycleCandidate, ...],
|
|
1942
|
+
*,
|
|
1943
|
+
allowed_hostnames: tuple[str, ...],
|
|
1944
|
+
) -> None:
|
|
1945
|
+
"""Refuse a candidate set on the trusted side, before a single name is resolved."""
|
|
1946
|
+
|
|
1947
|
+
if not isinstance(candidates, tuple) or not 1 <= len(candidates) <= MAX_CYCLE_CANDIDATES:
|
|
1948
|
+
raise AcquisitionSecurityError(
|
|
1949
|
+
"SANDBOX_CYCLE_SURVEY",
|
|
1950
|
+
f"a survey must name between one and {MAX_CYCLE_CANDIDATES} candidates",
|
|
1951
|
+
)
|
|
1952
|
+
labels = [candidate.label for candidate in candidates]
|
|
1953
|
+
if len(set(labels)) != len(labels):
|
|
1954
|
+
raise AcquisitionSecurityError(
|
|
1955
|
+
"SANDBOX_CYCLE_SURVEY",
|
|
1956
|
+
"survey candidate labels must be unique within one request",
|
|
1957
|
+
)
|
|
1958
|
+
for candidate in candidates:
|
|
1959
|
+
if not isinstance(candidate, CycleCandidate):
|
|
1960
|
+
raise AcquisitionSecurityError(
|
|
1961
|
+
"SANDBOX_CYCLE_SURVEY",
|
|
1962
|
+
"a survey candidate must be a validated candidate value",
|
|
1963
|
+
)
|
|
1964
|
+
hostname = urlsplit(candidate.url).hostname
|
|
1965
|
+
if hostname is None or hostname.lower() not in allowed_hostnames:
|
|
1966
|
+
raise AcquisitionSecurityError(
|
|
1967
|
+
"SANDBOX_EGRESS",
|
|
1968
|
+
"a survey candidate names a host outside the egress allowlist",
|
|
1969
|
+
)
|
|
1970
|
+
|
|
1971
|
+
|
|
1972
|
+
def _narrowed_to_media_types(
|
|
1973
|
+
limits: RetrievalLimits,
|
|
1974
|
+
media_types: tuple[str, ...],
|
|
1975
|
+
) -> RetrievalLimits:
|
|
1976
|
+
"""Narrow a retrieval's media allowlist to one call's half of it, never widening it."""
|
|
1977
|
+
|
|
1978
|
+
admitted = tuple(
|
|
1979
|
+
media_type for media_type in media_types if media_type in limits.allowed_media_types
|
|
1980
|
+
)
|
|
1981
|
+
if not admitted:
|
|
1982
|
+
raise AcquisitionSecurityError(
|
|
1983
|
+
"SANDBOX_LIMITS",
|
|
1984
|
+
"the retrieval's media allowlist admits nothing this fetch would accept",
|
|
1985
|
+
)
|
|
1986
|
+
return replace(limits, allowed_media_types=admitted)
|
|
1987
|
+
|
|
1988
|
+
|
|
1989
|
+
def _require_slice_bytes_within_budget(
|
|
1990
|
+
planned: tuple[PlannedRange, ...],
|
|
1991
|
+
*,
|
|
1992
|
+
limits: RetrievalLimits,
|
|
1993
|
+
) -> None:
|
|
1994
|
+
"""Refuse a plan whose slices would not fit, before a single byte of them is fetched.
|
|
1995
|
+
|
|
1996
|
+
Two ceilings, because they bound different things. One slice may not exceed the response
|
|
1997
|
+
limit the retrieval already enforces per fetch -- checked here as well so an over-large
|
|
1998
|
+
span is refused before the request rather than after the bytes arrive.
|
|
1999
|
+
|
|
2000
|
+
The total is bounded by the IPC response cap it has to survive. Base64 expands by four
|
|
2001
|
+
thirds, so ``MAX_TOTAL_SLICE_BYTES`` of slices becomes 32 MiB of response text against a
|
|
2002
|
+
48 MiB cap, leaving 16 MiB for a response whose other fields do not reach a kilobyte. The
|
|
2003
|
+
arithmetic is stated rather than left for a reader to rediscover, and the plan is refused
|
|
2004
|
+
rather than truncated: a shortened slice set is a different acquisition, silently.
|
|
2005
|
+
"""
|
|
2006
|
+
|
|
2007
|
+
total = 0
|
|
2008
|
+
for entry in planned:
|
|
2009
|
+
length = entry.byte_range.length
|
|
2010
|
+
if length > limits.max_response_bytes:
|
|
2011
|
+
raise AcquisitionSecurityError(
|
|
2012
|
+
"SANDBOX_SLICE_BUDGET",
|
|
2013
|
+
"a planned slice is larger than the retrieval's own response limit",
|
|
2014
|
+
)
|
|
2015
|
+
total += length
|
|
2016
|
+
if total > MAX_TOTAL_SLICE_BYTES:
|
|
2017
|
+
raise AcquisitionSecurityError(
|
|
2018
|
+
"SANDBOX_SLICE_BUDGET",
|
|
2019
|
+
f"the planned slices total more than the {MAX_TOTAL_SLICE_BYTES} byte ceiling",
|
|
2020
|
+
)
|
|
2021
|
+
|
|
2022
|
+
|
|
2023
|
+
@dataclass(frozen=True)
|
|
2024
|
+
class _RangePlan:
|
|
2025
|
+
"""A coordinator's order: which object, which sidecar describes it, and what to take."""
|
|
2026
|
+
|
|
2027
|
+
sidecar_url: str
|
|
2028
|
+
object_url: str
|
|
2029
|
+
selectors: tuple[str, ...]
|
|
2030
|
+
max_ranges: int
|
|
2031
|
+
# The model run this order is for, or ``None`` when the order names none. When it is set,
|
|
2032
|
+
# every delivered message must say in its own Section 1 that it came from that run; when it
|
|
2033
|
+
# is ``None`` the run binding is simply not in force, which is a smaller claim, not a
|
|
2034
|
+
# silent one -- ``mr-data acquire-slices`` prints which of the two it ran under.
|
|
2035
|
+
expected_reference_time: str | None
|
|
2036
|
+
|
|
2037
|
+
|
|
2038
|
+
def _parse_range_plan(value: Any, *, budget_ceiling: int) -> _RangePlan:
|
|
2039
|
+
"""Parse the closed range-plan object, or refuse it.
|
|
2040
|
+
|
|
2041
|
+
No refusal here quotes an address or a selector. The refusal crosses the sandbox boundary
|
|
2042
|
+
and both are text that came from outside this process.
|
|
2043
|
+
"""
|
|
2044
|
+
|
|
2045
|
+
if not isinstance(value, dict) or set(value) != _RANGE_PLAN_KEYS:
|
|
2046
|
+
raise AcquisitionSecurityError(
|
|
2047
|
+
"SANDBOX_RANGE_PLAN",
|
|
2048
|
+
"a range plan must be the exact closed object of five fields",
|
|
2049
|
+
)
|
|
2050
|
+
sidecar_url = _bounded_text(value["sidecar_url"], maximum=2_048)
|
|
2051
|
+
object_url = _bounded_text(value["object_url"], maximum=2_048)
|
|
2052
|
+
# Checked, never derived. Appending ``.idx`` to the object's address here would make the
|
|
2053
|
+
# sidecar an address this worker invented rather than one the coordinator resolved and
|
|
2054
|
+
# pinned, and the whole value of the pin is that every address fetched came from outside.
|
|
2055
|
+
if _https_origin(sidecar_url) != _https_origin(object_url):
|
|
2056
|
+
raise AcquisitionSecurityError(
|
|
2057
|
+
"SANDBOX_RANGE_PLAN",
|
|
2058
|
+
"a range plan's sidecar and object must share one https origin",
|
|
2059
|
+
)
|
|
2060
|
+
selectors = _string_tuple(
|
|
2061
|
+
value["selectors"],
|
|
2062
|
+
maximum_items=MAX_RANGE_PLAN,
|
|
2063
|
+
item_maximum=MAX_SELECTOR_BYTES,
|
|
2064
|
+
)
|
|
2065
|
+
if not selectors:
|
|
2066
|
+
raise AcquisitionSecurityError(
|
|
2067
|
+
"SANDBOX_RANGE_PLAN",
|
|
2068
|
+
"a range plan must order at least one message",
|
|
2069
|
+
)
|
|
2070
|
+
max_ranges = value["max_ranges"]
|
|
2071
|
+
# Bounded by the retrieval's own budget rather than only by the transport ceiling: a plan
|
|
2072
|
+
# that could name its own larger budget would make the limits it travels beside advisory.
|
|
2073
|
+
if type(max_ranges) is not int or not 1 <= max_ranges <= budget_ceiling:
|
|
2074
|
+
raise AcquisitionSecurityError(
|
|
2075
|
+
"SANDBOX_RANGE_PLAN",
|
|
2076
|
+
"a range plan's budget must lie within the retrieval's own range budget",
|
|
2077
|
+
)
|
|
2078
|
+
# How many slices are ordered is known from the request alone, so the budget is applied
|
|
2079
|
+
# here rather than after the sidecar fetch. ``plan_ranges`` applies the same bound again
|
|
2080
|
+
# to the plan it builds; this one exists so an over-long order costs no request at all.
|
|
2081
|
+
require_range_plan_within_budget(len(selectors), max_ranges=max_ranges)
|
|
2082
|
+
# Parsed here rather than trusted as it arrives, and parsed by the same function the
|
|
2083
|
+
# coordinator used to build it, so the two sides of the boundary cannot come to hold two
|
|
2084
|
+
# spellings of one run.
|
|
2085
|
+
expected_reference_time = value["expected_reference_time"]
|
|
2086
|
+
if expected_reference_time is not None:
|
|
2087
|
+
expected_reference_time = require_reference_time_spelling(expected_reference_time)
|
|
2088
|
+
return _RangePlan(
|
|
2089
|
+
sidecar_url=sidecar_url,
|
|
2090
|
+
object_url=object_url,
|
|
2091
|
+
selectors=selectors,
|
|
2092
|
+
max_ranges=max_ranges,
|
|
2093
|
+
expected_reference_time=expected_reference_time,
|
|
2094
|
+
)
|
|
2095
|
+
|
|
2096
|
+
|
|
2097
|
+
def _https_origin(url: str, *, code: str = "SANDBOX_RANGE_PLAN") -> tuple[str, str]:
|
|
2098
|
+
"""Return an address's origin, refusing anything that is not an absolute https URL.
|
|
2099
|
+
|
|
2100
|
+
The refusal code is the caller's, because the two callers refuse different things and a
|
|
2101
|
+
survey that reported a malformed candidate as a range-plan failure would send a reader to
|
|
2102
|
+
the wrong half of the boundary.
|
|
2103
|
+
"""
|
|
2104
|
+
|
|
2105
|
+
split = urlsplit(url)
|
|
2106
|
+
if split.scheme != "https" or not split.netloc:
|
|
2107
|
+
raise AcquisitionSecurityError(
|
|
2108
|
+
code,
|
|
2109
|
+
"a clean-room address must be an absolute https URL",
|
|
2110
|
+
)
|
|
2111
|
+
return split.scheme, split.netloc.lower()
|
|
2112
|
+
|
|
2113
|
+
|
|
2114
|
+
def _parse_response(
|
|
2115
|
+
response: dict[str, Any],
|
|
2116
|
+
*,
|
|
2117
|
+
expected_request_id: str,
|
|
2118
|
+
expected_operation: str,
|
|
2119
|
+
expected_external_network_policy_attestation: str | None,
|
|
2120
|
+
expected_cycle_labels: tuple[str, ...] | None = None,
|
|
2121
|
+
expected_selectors: tuple[str, ...] | None = None,
|
|
2122
|
+
expected_reference_time: str | None = None,
|
|
2123
|
+
) -> SandboxResult:
|
|
2124
|
+
expected = {
|
|
2125
|
+
"schema_version",
|
|
2126
|
+
"request_id",
|
|
2127
|
+
"operation",
|
|
2128
|
+
"policy_digest",
|
|
2129
|
+
"visible_environment_keys",
|
|
2130
|
+
"parsed",
|
|
2131
|
+
"content_base64",
|
|
2132
|
+
"media_type",
|
|
2133
|
+
"final_url",
|
|
2134
|
+
"transport_evidence_digest",
|
|
2135
|
+
"probe_transport",
|
|
2136
|
+
"home_directory_readable",
|
|
2137
|
+
"network_probe_errors",
|
|
2138
|
+
"decode_family_id",
|
|
2139
|
+
"decode_family_version",
|
|
2140
|
+
"decode_options_digest",
|
|
2141
|
+
"decode_flags",
|
|
2142
|
+
"decode_declared_cell_count",
|
|
2143
|
+
"fetched_content_sha256",
|
|
2144
|
+
"fetched_content_size_bytes",
|
|
2145
|
+
"fetched_total_response_body_size_bytes",
|
|
2146
|
+
"fetched_request_count",
|
|
2147
|
+
"ranges",
|
|
2148
|
+
"cycle_availability",
|
|
2149
|
+
"sidecar_content_sha256",
|
|
2150
|
+
"sidecar_size_bytes",
|
|
2151
|
+
"full_object_size_bytes",
|
|
2152
|
+
"fetched_members",
|
|
2153
|
+
"external_network_policy_attestation",
|
|
2154
|
+
}
|
|
2155
|
+
if set(response) != expected or response["schema_version"] != SANDBOX_PROTOCOL_VERSION:
|
|
2156
|
+
raise AcquisitionSecurityError(
|
|
2157
|
+
"SANDBOX_RESPONSE",
|
|
2158
|
+
"sandbox response shape or version is invalid",
|
|
2159
|
+
)
|
|
2160
|
+
request_id = _bounded_identifier(response["request_id"])
|
|
2161
|
+
if request_id != expected_request_id:
|
|
2162
|
+
raise AcquisitionSecurityError(
|
|
2163
|
+
"SANDBOX_RESPONSE_BINDING",
|
|
2164
|
+
"sandbox response does not bind the exact request",
|
|
2165
|
+
)
|
|
2166
|
+
operation = _bounded_text(response["operation"], maximum=32)
|
|
2167
|
+
# The operation is bound to the one that was ordered, exactly as ``request_id`` is above.
|
|
2168
|
+
# Without this the response names its own operation and ``_policy_digest`` below then
|
|
2169
|
+
# validates it against itself: a clean room invoked for ``fetch_ranges`` -- which resolves
|
|
2170
|
+
# endpoints, opens the attested network clause and requires the external
|
|
2171
|
+
# network attestation -- could answer with ``parse``'s digest, and the sealed evidence
|
|
2172
|
+
# would record a stricter policy than the one that actually ran. A self-declared
|
|
2173
|
+
# attestation is not an attestation.
|
|
2174
|
+
if operation != expected_operation:
|
|
2175
|
+
raise AcquisitionSecurityError(
|
|
2176
|
+
"SANDBOX_RESPONSE_BINDING",
|
|
2177
|
+
"the clean room's response does not bind the operation that was requested",
|
|
2178
|
+
)
|
|
2179
|
+
expected_policy = _policy_digest(operation)
|
|
2180
|
+
if response["policy_digest"] != expected_policy:
|
|
2181
|
+
raise AcquisitionSecurityError(
|
|
2182
|
+
"SANDBOX_POLICY_BINDING",
|
|
2183
|
+
"sandbox response policy attestation is invalid",
|
|
2184
|
+
)
|
|
2185
|
+
external_network_policy_attestation = _optional_digest(
|
|
2186
|
+
response["external_network_policy_attestation"]
|
|
2187
|
+
)
|
|
2188
|
+
if external_network_policy_attestation != expected_external_network_policy_attestation:
|
|
2189
|
+
raise AcquisitionSecurityError(
|
|
2190
|
+
"SANDBOX_POLICY_BINDING",
|
|
2191
|
+
"sandbox response does not bind the external network policy attestation",
|
|
2192
|
+
)
|
|
2193
|
+
visible = _string_tuple(
|
|
2194
|
+
response["visible_environment_keys"],
|
|
2195
|
+
maximum_items=32,
|
|
2196
|
+
item_maximum=128,
|
|
2197
|
+
)
|
|
2198
|
+
if set(visible) - set(_ALLOWED_ENVIRONMENT):
|
|
2199
|
+
raise AcquisitionSecurityError(
|
|
2200
|
+
"SANDBOX_ENVIRONMENT",
|
|
2201
|
+
"sandbox inherited non-allowlisted environment keys",
|
|
2202
|
+
)
|
|
2203
|
+
home_directory_readable = response["home_directory_readable"]
|
|
2204
|
+
if home_directory_readable is not None and type(home_directory_readable) is not bool:
|
|
2205
|
+
raise AcquisitionSecurityError(
|
|
2206
|
+
"SANDBOX_RESPONSE",
|
|
2207
|
+
"sandbox home-directory probe is invalid",
|
|
2208
|
+
)
|
|
2209
|
+
# The home-directory answer is still reported, and is still bound into the response the
|
|
2210
|
+
# policy digest covers, but nothing enforces it here any more. It was enforced on darwin
|
|
2211
|
+
# alone, because the Seatbelt profile ADR 0021 deleted was the thing that made the
|
|
2212
|
+
# home directory unreadable; on every other host the boundary that owns the filesystem is
|
|
2213
|
+
# the external container the attestation names, and this process is in no position to
|
|
2214
|
+
# second-guess it. Keeping the field is what lets a container's own conformance check read
|
|
2215
|
+
# it.
|
|
2216
|
+
network_probe_errors = _parse_network_probe_response(
|
|
2217
|
+
response["network_probe_errors"],
|
|
2218
|
+
required=operation == "probe",
|
|
2219
|
+
)
|
|
2220
|
+
ranges = _parse_fetched_slices(
|
|
2221
|
+
response["ranges"],
|
|
2222
|
+
required=operation == "fetch_ranges",
|
|
2223
|
+
expected_selectors=expected_selectors,
|
|
2224
|
+
expected_reference_time=expected_reference_time,
|
|
2225
|
+
)
|
|
2226
|
+
cycle_availability = _parse_cycle_availability(
|
|
2227
|
+
response["cycle_availability"],
|
|
2228
|
+
required=operation == "survey_cycles",
|
|
2229
|
+
expected_labels=expected_cycle_labels,
|
|
2230
|
+
)
|
|
2231
|
+
fetched_members = _parse_fetched_members(
|
|
2232
|
+
response["fetched_members"],
|
|
2233
|
+
required=operation == "fetch_ranges",
|
|
2234
|
+
)
|
|
2235
|
+
sidecar_content_sha256 = _optional_digest(response["sidecar_content_sha256"])
|
|
2236
|
+
sidecar_size_bytes = response["sidecar_size_bytes"]
|
|
2237
|
+
full_object_size_bytes = response["full_object_size_bytes"]
|
|
2238
|
+
if operation == "fetch_ranges":
|
|
2239
|
+
if sidecar_content_sha256 is None:
|
|
2240
|
+
raise AcquisitionSecurityError(
|
|
2241
|
+
"SANDBOX_RESPONSE", "range response omits sidecar identity"
|
|
2242
|
+
)
|
|
2243
|
+
for name, value in (
|
|
2244
|
+
("sidecar_size_bytes", sidecar_size_bytes),
|
|
2245
|
+
("full_object_size_bytes", full_object_size_bytes),
|
|
2246
|
+
):
|
|
2247
|
+
if type(value) is not int or not 1 <= value <= MAX_BYTE_RANGE_OFFSET + 1:
|
|
2248
|
+
raise AcquisitionSecurityError(
|
|
2249
|
+
"SANDBOX_RESPONSE", f"range response {name} is invalid"
|
|
2250
|
+
)
|
|
2251
|
+
elif (
|
|
2252
|
+
sidecar_content_sha256 is not None
|
|
2253
|
+
or sidecar_size_bytes is not None
|
|
2254
|
+
or full_object_size_bytes is not None
|
|
2255
|
+
):
|
|
2256
|
+
raise AcquisitionSecurityError(
|
|
2257
|
+
"SANDBOX_RESPONSE", "non-range response carries range identity"
|
|
2258
|
+
)
|
|
2259
|
+
transport_evidence_digest = _optional_digest(response["transport_evidence_digest"])
|
|
2260
|
+
probe_transport = _parse_probe_transport(
|
|
2261
|
+
response["probe_transport"],
|
|
2262
|
+
required=operation in WHOLE_DOCUMENT_RETRIEVAL_OPERATIONS,
|
|
2263
|
+
)
|
|
2264
|
+
if operation == "fetch_ranges":
|
|
2265
|
+
assert fetched_members is not None
|
|
2266
|
+
expected_transport_digest = canonical_sha256(
|
|
2267
|
+
[
|
|
2268
|
+
{
|
|
2269
|
+
key: value
|
|
2270
|
+
for key, value in member.to_dict().items()
|
|
2271
|
+
if key not in {"role", "sequence"}
|
|
2272
|
+
}
|
|
2273
|
+
for member in fetched_members
|
|
2274
|
+
]
|
|
2275
|
+
)
|
|
2276
|
+
if transport_evidence_digest != expected_transport_digest:
|
|
2277
|
+
raise AcquisitionSecurityError(
|
|
2278
|
+
"SANDBOX_RESPONSE",
|
|
2279
|
+
"range response transport digest does not bind every measured response",
|
|
2280
|
+
)
|
|
2281
|
+
parsed_value = response["parsed"]
|
|
2282
|
+
parsed = _parsed_from_dict(parsed_value) if parsed_value is not None else None
|
|
2283
|
+
content_value = response["content_base64"]
|
|
2284
|
+
content: bytes | None
|
|
2285
|
+
if content_value is None:
|
|
2286
|
+
content = None
|
|
2287
|
+
else:
|
|
2288
|
+
try:
|
|
2289
|
+
content = base64.b64decode(
|
|
2290
|
+
_bounded_text(content_value, maximum=MAX_RESPONSE_BYTES),
|
|
2291
|
+
validate=True,
|
|
2292
|
+
)
|
|
2293
|
+
except (ValueError, TypeError):
|
|
2294
|
+
raise AcquisitionSecurityError(
|
|
2295
|
+
"SANDBOX_RESPONSE",
|
|
2296
|
+
"sandbox response content is not canonical base64",
|
|
2297
|
+
) from None
|
|
2298
|
+
decode_facts = _parse_decode_facts(response, operation=operation)
|
|
2299
|
+
if decode_facts is not None:
|
|
2300
|
+
# The returned table must describe the returned bytes. The worker parses what it
|
|
2301
|
+
# decoded, so this can only fail for a worker that did something else -- which is
|
|
2302
|
+
# exactly the case a boundary check exists for.
|
|
2303
|
+
if parsed is None or content is None or parsed.input_sha256 != sha256_bytes(content):
|
|
2304
|
+
raise AcquisitionSecurityError(
|
|
2305
|
+
"SANDBOX_DECODE_BINDING",
|
|
2306
|
+
"decode response table does not describe the bytes the decode returned",
|
|
2307
|
+
)
|
|
2308
|
+
family_id, family_version, options_digest, flags, declared_cell_count = decode_facts or (
|
|
2309
|
+
None,
|
|
2310
|
+
None,
|
|
2311
|
+
None,
|
|
2312
|
+
None,
|
|
2313
|
+
None,
|
|
2314
|
+
)
|
|
2315
|
+
fetched_content_sha256 = _optional_digest(response["fetched_content_sha256"])
|
|
2316
|
+
fetched_content_size_bytes = response["fetched_content_size_bytes"]
|
|
2317
|
+
fetched_total_response_body_size_bytes = response["fetched_total_response_body_size_bytes"]
|
|
2318
|
+
fetched_request_count = response["fetched_request_count"]
|
|
2319
|
+
if operation == "retrieve_decode_and_parse":
|
|
2320
|
+
if fetched_content_sha256 is None:
|
|
2321
|
+
raise AcquisitionSecurityError(
|
|
2322
|
+
"SANDBOX_RESPONSE", "Reader retrieval response omits fetched content identity"
|
|
2323
|
+
)
|
|
2324
|
+
if (
|
|
2325
|
+
type(fetched_content_size_bytes) is not int
|
|
2326
|
+
or not 1 <= fetched_content_size_bytes <= 1 << 34
|
|
2327
|
+
):
|
|
2328
|
+
raise AcquisitionSecurityError(
|
|
2329
|
+
"SANDBOX_RESPONSE", "Reader retrieval response omits fetched content size"
|
|
2330
|
+
)
|
|
2331
|
+
if (
|
|
2332
|
+
type(fetched_total_response_body_size_bytes) is not int
|
|
2333
|
+
or not fetched_content_size_bytes <= fetched_total_response_body_size_bytes <= 1 << 34
|
|
2334
|
+
):
|
|
2335
|
+
raise AcquisitionSecurityError(
|
|
2336
|
+
"SANDBOX_RESPONSE",
|
|
2337
|
+
"Reader retrieval response omits cumulative response-body size",
|
|
2338
|
+
)
|
|
2339
|
+
if type(fetched_request_count) is not int or not 1 <= fetched_request_count <= 256:
|
|
2340
|
+
raise AcquisitionSecurityError(
|
|
2341
|
+
"SANDBOX_RESPONSE", "Reader retrieval response omits fetched request count"
|
|
2342
|
+
)
|
|
2343
|
+
elif (
|
|
2344
|
+
fetched_content_sha256 is not None
|
|
2345
|
+
or fetched_content_size_bytes is not None
|
|
2346
|
+
or fetched_total_response_body_size_bytes is not None
|
|
2347
|
+
or fetched_request_count is not None
|
|
2348
|
+
):
|
|
2349
|
+
raise AcquisitionSecurityError(
|
|
2350
|
+
"SANDBOX_RESPONSE",
|
|
2351
|
+
"a non-Reader-retrieval response carries fetched content identity",
|
|
2352
|
+
)
|
|
2353
|
+
return SandboxResult(
|
|
2354
|
+
request_id=request_id,
|
|
2355
|
+
operation=operation,
|
|
2356
|
+
policy_digest=expected_policy,
|
|
2357
|
+
parsed=parsed,
|
|
2358
|
+
content=content,
|
|
2359
|
+
media_type=_optional_text(response["media_type"], 255),
|
|
2360
|
+
final_url=_optional_text(response["final_url"], 2_048),
|
|
2361
|
+
transport_evidence_digest=transport_evidence_digest,
|
|
2362
|
+
visible_environment_keys=visible,
|
|
2363
|
+
probe_transport=probe_transport,
|
|
2364
|
+
home_directory_readable=home_directory_readable,
|
|
2365
|
+
network_probe_errors=network_probe_errors,
|
|
2366
|
+
external_network_policy_attestation=external_network_policy_attestation,
|
|
2367
|
+
decode_family_id=family_id,
|
|
2368
|
+
decode_family_version=family_version,
|
|
2369
|
+
decode_options_digest=options_digest,
|
|
2370
|
+
decode_flags=flags,
|
|
2371
|
+
decode_declared_cell_count=declared_cell_count,
|
|
2372
|
+
fetched_content_sha256=fetched_content_sha256,
|
|
2373
|
+
fetched_content_size_bytes=fetched_content_size_bytes,
|
|
2374
|
+
fetched_total_response_body_size_bytes=fetched_total_response_body_size_bytes,
|
|
2375
|
+
fetched_request_count=fetched_request_count,
|
|
2376
|
+
ranges=ranges,
|
|
2377
|
+
cycle_availability=cycle_availability,
|
|
2378
|
+
sidecar_content_sha256=sidecar_content_sha256,
|
|
2379
|
+
sidecar_size_bytes=sidecar_size_bytes,
|
|
2380
|
+
full_object_size_bytes=full_object_size_bytes,
|
|
2381
|
+
fetched_members=fetched_members,
|
|
2382
|
+
)
|
|
2383
|
+
|
|
2384
|
+
|
|
2385
|
+
def _parse_decode_facts(
|
|
2386
|
+
response: dict[str, Any],
|
|
2387
|
+
*,
|
|
2388
|
+
operation: str,
|
|
2389
|
+
) -> tuple[str, str, str, tuple[str, ...], int] | None:
|
|
2390
|
+
"""Admit the five decode facts, or refuse a response that carries them out of place.
|
|
2391
|
+
|
|
2392
|
+
The flags are checked against the closed ``DECODE_FLAGS`` vocabulary here, at the
|
|
2393
|
+
boundary, rather than trusted onward. A family that could name its own flag could write
|
|
2394
|
+
an unreviewed field into a receipt, so the vocabulary is enforced where the untrusted
|
|
2395
|
+
process's answer arrives and not only where it was produced.
|
|
2396
|
+
"""
|
|
2397
|
+
|
|
2398
|
+
stated = (
|
|
2399
|
+
response["decode_family_id"],
|
|
2400
|
+
response["decode_family_version"],
|
|
2401
|
+
response["decode_options_digest"],
|
|
2402
|
+
response["decode_flags"],
|
|
2403
|
+
response["decode_declared_cell_count"],
|
|
2404
|
+
)
|
|
2405
|
+
if operation not in READER_PERFORMING_OPERATIONS:
|
|
2406
|
+
if any(item is not None for item in stated):
|
|
2407
|
+
raise AcquisitionSecurityError(
|
|
2408
|
+
"SANDBOX_RESPONSE",
|
|
2409
|
+
"a response for an operation that decodes nothing carries decode facts",
|
|
2410
|
+
)
|
|
2411
|
+
return None
|
|
2412
|
+
if any(item is None for item in stated):
|
|
2413
|
+
raise AcquisitionSecurityError(
|
|
2414
|
+
"SANDBOX_RESPONSE",
|
|
2415
|
+
"decode response does not carry the decode facts the receipt needs",
|
|
2416
|
+
)
|
|
2417
|
+
family_id = _bounded_text(stated[0], maximum=128)
|
|
2418
|
+
family_version = _bounded_text(stated[1], maximum=32)
|
|
2419
|
+
options_digest = _digest(stated[2])
|
|
2420
|
+
flags = _string_tuple(stated[3], maximum_items=32, item_maximum=64)
|
|
2421
|
+
unknown = sorted(flag for flag in flags if flag not in DECODE_FLAGS)
|
|
2422
|
+
if unknown:
|
|
2423
|
+
raise AcquisitionSecurityError(
|
|
2424
|
+
"SANDBOX_DECODE_FLAGS",
|
|
2425
|
+
f"decode response names no such decode flag: {', '.join(unknown)}",
|
|
2426
|
+
)
|
|
2427
|
+
if tuple(sorted(flags)) != flags:
|
|
2428
|
+
raise AcquisitionSecurityError(
|
|
2429
|
+
"SANDBOX_DECODE_FLAGS",
|
|
2430
|
+
"decode flags must arrive sorted, as the Reader contract produces them",
|
|
2431
|
+
)
|
|
2432
|
+
declared_cell_count = stated[4]
|
|
2433
|
+
if type(declared_cell_count) is not int or not 0 <= declared_cell_count <= (1 << 63) - 1:
|
|
2434
|
+
raise AcquisitionSecurityError(
|
|
2435
|
+
"SANDBOX_RESPONSE",
|
|
2436
|
+
"decode response declared-cell count is invalid",
|
|
2437
|
+
)
|
|
2438
|
+
return family_id, family_version, options_digest, flags, declared_cell_count
|
|
2439
|
+
|
|
2440
|
+
|
|
2441
|
+
def _parse_cycle_availability(
|
|
2442
|
+
value: Any,
|
|
2443
|
+
*,
|
|
2444
|
+
required: bool,
|
|
2445
|
+
expected_labels: tuple[str, ...] | None,
|
|
2446
|
+
) -> tuple[CycleAvailability, ...] | None:
|
|
2447
|
+
"""Check a survey's answers against the exact question the coordinator asked, or refuse.
|
|
2448
|
+
|
|
2449
|
+
Compare labels positionally so availability cannot be attached to a different run.
|
|
2450
|
+
"""
|
|
2451
|
+
|
|
2452
|
+
if not required:
|
|
2453
|
+
if value is not None:
|
|
2454
|
+
raise AcquisitionSecurityError(
|
|
2455
|
+
"SANDBOX_RESPONSE",
|
|
2456
|
+
"non-survey response contains availability evidence",
|
|
2457
|
+
)
|
|
2458
|
+
return None
|
|
2459
|
+
if expected_labels is None:
|
|
2460
|
+
raise AcquisitionSecurityError(
|
|
2461
|
+
"SANDBOX_RESPONSE",
|
|
2462
|
+
"a survey response can only be read against the labels that were ordered",
|
|
2463
|
+
)
|
|
2464
|
+
if not isinstance(value, list) or len(value) != len(expected_labels):
|
|
2465
|
+
raise AcquisitionSecurityError(
|
|
2466
|
+
"SANDBOX_RESPONSE",
|
|
2467
|
+
"survey response does not answer the exact candidate set that was ordered",
|
|
2468
|
+
)
|
|
2469
|
+
results: list[CycleAvailability] = []
|
|
2470
|
+
for item, label in zip(value, expected_labels, strict=True):
|
|
2471
|
+
if not isinstance(item, dict) or set(item) != _CYCLE_AVAILABILITY_KEYS:
|
|
2472
|
+
raise AcquisitionSecurityError("SANDBOX_RESPONSE", "survey result shape is invalid")
|
|
2473
|
+
if item["label"] != label:
|
|
2474
|
+
raise AcquisitionSecurityError(
|
|
2475
|
+
"SANDBOX_RESPONSE",
|
|
2476
|
+
"survey results are not the candidates that were ordered, in that order",
|
|
2477
|
+
)
|
|
2478
|
+
present = item["present"]
|
|
2479
|
+
if type(present) is not bool:
|
|
2480
|
+
raise AcquisitionSecurityError(
|
|
2481
|
+
"SANDBOX_RESPONSE",
|
|
2482
|
+
"survey presence must be exactly true or false",
|
|
2483
|
+
)
|
|
2484
|
+
results.append(CycleAvailability(label=label, present=present))
|
|
2485
|
+
return tuple(results)
|
|
2486
|
+
|
|
2487
|
+
|
|
2488
|
+
_PROBE_TRANSPORT_KEYS = frozenset({"http_status_code", "etag_digest", "last_modified_digest"})
|
|
2489
|
+
|
|
2490
|
+
|
|
2491
|
+
def _parse_probe_transport(value: Any, *, required: bool) -> ProbeTransport | None:
|
|
2492
|
+
"""Re-admit the terminal-response facts a whole-document retrieval sealed.
|
|
2493
|
+
|
|
2494
|
+
The worker computed both validator digests inside the confinement, so the coordinator never
|
|
2495
|
+
sees a publisher's header text and cannot be handed one here: a value that is not a bare
|
|
2496
|
+
lowercase SHA-256 is refused rather than carried. An operation that settled on no single
|
|
2497
|
+
response must report none, because a status attached to the wrong exchange would be recorded
|
|
2498
|
+
as an observation of the source.
|
|
2499
|
+
"""
|
|
2500
|
+
|
|
2501
|
+
if not required:
|
|
2502
|
+
if value is not None:
|
|
2503
|
+
raise AcquisitionSecurityError(
|
|
2504
|
+
"SANDBOX_RESPONSE",
|
|
2505
|
+
"an operation that retrieves no whole document carries probe transport evidence",
|
|
2506
|
+
)
|
|
2507
|
+
return None
|
|
2508
|
+
if not isinstance(value, dict) or set(value) != _PROBE_TRANSPORT_KEYS:
|
|
2509
|
+
raise AcquisitionSecurityError(
|
|
2510
|
+
"SANDBOX_RESPONSE",
|
|
2511
|
+
"whole-document retrieval response omits its exact probe transport evidence",
|
|
2512
|
+
)
|
|
2513
|
+
try:
|
|
2514
|
+
transport = ProbeTransport(
|
|
2515
|
+
http_status_code=value["http_status_code"],
|
|
2516
|
+
etag_digest=_optional_digest(value["etag_digest"]),
|
|
2517
|
+
last_modified_digest=_optional_digest(value["last_modified_digest"]),
|
|
2518
|
+
)
|
|
2519
|
+
except AcquisitionSecurityError as error:
|
|
2520
|
+
raise AcquisitionSecurityError(
|
|
2521
|
+
"SANDBOX_RESPONSE",
|
|
2522
|
+
"probe transport evidence is not the exact sealed shape",
|
|
2523
|
+
) from error
|
|
2524
|
+
# A retrieval that returned a document returned it with a successful status; the retriever
|
|
2525
|
+
# refuses every other class before a byte is admitted. Pinning the range here rather than
|
|
2526
|
+
# trusting the worker keeps a response that states 404 beside acquired bytes from becoming
|
|
2527
|
+
# an observation whose status contradicts its own content. The hosted Courier pins the
|
|
2528
|
+
# same range on its own acquired path.
|
|
2529
|
+
if not 200 <= transport.http_status_code <= 299:
|
|
2530
|
+
raise AcquisitionSecurityError(
|
|
2531
|
+
"SANDBOX_RESPONSE",
|
|
2532
|
+
"a whole-document retrieval reported a response it could not have admitted",
|
|
2533
|
+
)
|
|
2534
|
+
return transport
|
|
2535
|
+
|
|
2536
|
+
|
|
2537
|
+
def _parse_fetched_members(value: Any, *, required: bool) -> tuple[FetchedMember, ...] | None:
|
|
2538
|
+
"""Admit the exact ordered HTTP responses retained by a range acquisition."""
|
|
2539
|
+
|
|
2540
|
+
if not required:
|
|
2541
|
+
if value is not None:
|
|
2542
|
+
raise AcquisitionSecurityError(
|
|
2543
|
+
"SANDBOX_RESPONSE",
|
|
2544
|
+
"non-range response contains fetched-member evidence",
|
|
2545
|
+
)
|
|
2546
|
+
return None
|
|
2547
|
+
if not isinstance(value, list) or not 4 <= len(value) <= 256:
|
|
2548
|
+
raise AcquisitionSecurityError(
|
|
2549
|
+
"SANDBOX_RESPONSE",
|
|
2550
|
+
"range response must retain a bounded complete fetched-member list",
|
|
2551
|
+
)
|
|
2552
|
+
members: list[FetchedMember] = []
|
|
2553
|
+
roles: list[str] = []
|
|
2554
|
+
for sequence, item in enumerate(value):
|
|
2555
|
+
if not isinstance(item, dict) or set(item) != _FETCHED_MEMBER_KEYS:
|
|
2556
|
+
raise AcquisitionSecurityError("SANDBOX_RESPONSE", "fetched-member shape is invalid")
|
|
2557
|
+
if item["sequence"] != sequence:
|
|
2558
|
+
raise AcquisitionSecurityError(
|
|
2559
|
+
"SANDBOX_RESPONSE", "fetched members are not in exact response order"
|
|
2560
|
+
)
|
|
2561
|
+
role = item["role"]
|
|
2562
|
+
if role not in _FETCHED_MEMBER_ROLES:
|
|
2563
|
+
raise AcquisitionSecurityError("SANDBOX_RESPONSE", "fetched-member role is invalid")
|
|
2564
|
+
status = item["status"]
|
|
2565
|
+
body_size = item["response_body_size_bytes"]
|
|
2566
|
+
if type(status) is not int or not 100 <= status <= 599:
|
|
2567
|
+
raise AcquisitionSecurityError("SANDBOX_RESPONSE", "fetched-member status is invalid")
|
|
2568
|
+
if type(body_size) is not int or not 0 <= body_size <= MAX_RESPONSE_BYTES:
|
|
2569
|
+
raise AcquisitionSecurityError(
|
|
2570
|
+
"SANDBOX_RESPONSE", "fetched-member body size is invalid"
|
|
2571
|
+
)
|
|
2572
|
+
member = FetchedMember(
|
|
2573
|
+
role=role,
|
|
2574
|
+
sequence=sequence,
|
|
2575
|
+
url=_bounded_text(item["url"], maximum=2_048),
|
|
2576
|
+
resolution_digest=_digest(item["resolution_digest"]),
|
|
2577
|
+
connected_peer=_bounded_text(item["connected_peer"], maximum=64),
|
|
2578
|
+
status=status,
|
|
2579
|
+
response_headers_digest=_digest(item["response_headers_digest"]),
|
|
2580
|
+
response_body_size_bytes=body_size,
|
|
2581
|
+
response_body_sha256=_digest(item["response_body_sha256"]),
|
|
2582
|
+
requested_range=_optional_text(item["requested_range"], 128),
|
|
2583
|
+
observed_content_range=_optional_text(item["observed_content_range"], 128),
|
|
2584
|
+
etag=_optional_text(item["etag"], 1_024),
|
|
2585
|
+
last_modified=_optional_text(item["last_modified"], 1_024),
|
|
2586
|
+
peer_attempts=_parse_peer_attempts(item["peer_attempts"]),
|
|
2587
|
+
)
|
|
2588
|
+
if member.peer_attempts[-1].response_body_size_bytes != body_size:
|
|
2589
|
+
raise AcquisitionSecurityError(
|
|
2590
|
+
"SANDBOX_RESPONSE",
|
|
2591
|
+
"terminal peer-attempt bytes differ from the retained response body",
|
|
2592
|
+
)
|
|
2593
|
+
members.append(member)
|
|
2594
|
+
roles.append(role)
|
|
2595
|
+
# Redirects may repeat a role; the four acquisition stages remain ordered and complete.
|
|
2596
|
+
collapsed = tuple(
|
|
2597
|
+
role for index, role in enumerate(roles) if index == 0 or role != roles[index - 1]
|
|
2598
|
+
)
|
|
2599
|
+
if collapsed != ("sidecar_initial", "cycle_probe", "selected_slice", "sidecar_final"):
|
|
2600
|
+
raise AcquisitionSecurityError(
|
|
2601
|
+
"SANDBOX_RESPONSE",
|
|
2602
|
+
"fetched members do not contain the complete ordered range acquisition",
|
|
2603
|
+
)
|
|
2604
|
+
return tuple(members)
|
|
2605
|
+
|
|
2606
|
+
|
|
2607
|
+
def _parse_peer_attempts(value: Any) -> tuple[PeerAttempt, ...]:
|
|
2608
|
+
"""Re-admit the closed, ordered peer evidence carried by one retrieval hop."""
|
|
2609
|
+
|
|
2610
|
+
if not isinstance(value, list) or not 1 <= len(value) <= 32:
|
|
2611
|
+
raise AcquisitionSecurityError(
|
|
2612
|
+
"SANDBOX_RESPONSE",
|
|
2613
|
+
"fetched-member peer attempts must be a non-empty bounded list",
|
|
2614
|
+
)
|
|
2615
|
+
attempts: list[PeerAttempt] = []
|
|
2616
|
+
saw_response = False
|
|
2617
|
+
for index, item in enumerate(value):
|
|
2618
|
+
if not isinstance(item, dict) or set(item) != _PEER_ATTEMPT_KEYS:
|
|
2619
|
+
raise AcquisitionSecurityError(
|
|
2620
|
+
"SANDBOX_RESPONSE", "fetched-member peer-attempt shape is invalid"
|
|
2621
|
+
)
|
|
2622
|
+
approved_ip = _bounded_text(item["approved_ip"], maximum=64)
|
|
2623
|
+
try:
|
|
2624
|
+
address = ipaddress.ip_address(approved_ip)
|
|
2625
|
+
except ValueError as error:
|
|
2626
|
+
raise AcquisitionSecurityError(
|
|
2627
|
+
"SANDBOX_RESPONSE", "fetched-member peer-attempt address is invalid"
|
|
2628
|
+
) from error
|
|
2629
|
+
if str(address) != approved_ip or not address.is_global:
|
|
2630
|
+
raise AcquisitionSecurityError(
|
|
2631
|
+
"SANDBOX_RESPONSE", "fetched-member peer-attempt address is not canonical public IP"
|
|
2632
|
+
)
|
|
2633
|
+
outcome = item["outcome"]
|
|
2634
|
+
failure_code = item["failure_code"]
|
|
2635
|
+
response_body_size_bytes = item["response_body_size_bytes"]
|
|
2636
|
+
if (
|
|
2637
|
+
type(response_body_size_bytes) is not int
|
|
2638
|
+
or not 0 <= response_body_size_bytes <= (1 << 34) + 1
|
|
2639
|
+
):
|
|
2640
|
+
raise AcquisitionSecurityError(
|
|
2641
|
+
"SANDBOX_RESPONSE",
|
|
2642
|
+
"fetched-member peer-attempt byte evidence is invalid",
|
|
2643
|
+
)
|
|
2644
|
+
if outcome not in _PEER_ATTEMPT_OUTCOMES:
|
|
2645
|
+
raise AcquisitionSecurityError(
|
|
2646
|
+
"SANDBOX_RESPONSE", "fetched-member peer-attempt outcome is invalid"
|
|
2647
|
+
)
|
|
2648
|
+
if outcome in {"transport_failure", "response_failure"}:
|
|
2649
|
+
failure_code = _bounded_text(failure_code, maximum=128)
|
|
2650
|
+
if saw_response:
|
|
2651
|
+
raise AcquisitionSecurityError(
|
|
2652
|
+
"SANDBOX_RESPONSE", "peer attempts continue after a successful response"
|
|
2653
|
+
)
|
|
2654
|
+
if outcome == "response_failure" and index != len(value) - 1:
|
|
2655
|
+
raise AcquisitionSecurityError(
|
|
2656
|
+
"SANDBOX_RESPONSE", "a response failure must terminate peer attempts"
|
|
2657
|
+
)
|
|
2658
|
+
elif failure_code is not None or index != len(value) - 1:
|
|
2659
|
+
raise AcquisitionSecurityError(
|
|
2660
|
+
"SANDBOX_RESPONSE", "successful peer attempt must terminate the attempt list"
|
|
2661
|
+
)
|
|
2662
|
+
else:
|
|
2663
|
+
saw_response = True
|
|
2664
|
+
attempts.append(
|
|
2665
|
+
PeerAttempt(
|
|
2666
|
+
approved_ip=approved_ip,
|
|
2667
|
+
outcome=outcome,
|
|
2668
|
+
failure_code=failure_code,
|
|
2669
|
+
response_body_size_bytes=response_body_size_bytes,
|
|
2670
|
+
)
|
|
2671
|
+
)
|
|
2672
|
+
if not saw_response:
|
|
2673
|
+
raise AcquisitionSecurityError(
|
|
2674
|
+
"SANDBOX_RESPONSE", "successful fetched member has no successful peer attempt"
|
|
2675
|
+
)
|
|
2676
|
+
return tuple(attempts)
|
|
2677
|
+
|
|
2678
|
+
|
|
2679
|
+
def _parse_fetched_slices(
|
|
2680
|
+
value: Any,
|
|
2681
|
+
*,
|
|
2682
|
+
required: bool,
|
|
2683
|
+
expected_selectors: tuple[str, ...] | None = None,
|
|
2684
|
+
expected_reference_time: str | None = None,
|
|
2685
|
+
) -> tuple[FetchedSlice, ...] | None:
|
|
2686
|
+
"""Re-check every delivered slice on the trusted side of the boundary, or refuse it.
|
|
2687
|
+
|
|
2688
|
+
The worker already proved each slice's span and structure. This is not that check again:
|
|
2689
|
+
it is the coordinator refusing to build a result out of anything the response says that
|
|
2690
|
+
the response's own bytes do not support. Each slice's digest is recomputed, each span is
|
|
2691
|
+
compared against the bytes that claim it, the sequence must ascend, and every slice must
|
|
2692
|
+
still be a well-formed message -- so the ordering and structural properties this operation
|
|
2693
|
+
promises are facts the coordinator verified rather than ones the worker asserted.
|
|
2694
|
+
|
|
2695
|
+
``expected_selectors`` is the exact set of descriptors the invocation ordered, which the
|
|
2696
|
+
coordinator knows from its own request. One selector yields exactly one slice, so the
|
|
2697
|
+
response must answer every selector exactly once: a response carrying fewer is a partial
|
|
2698
|
+
acquisition, one carrying more is a set nobody ordered, and one answering the same
|
|
2699
|
+
descriptor twice is not the order that was placed. This is the count check it replaces
|
|
2700
|
+
and more -- the count was the weakest fact the coordinator held, and holding the answer
|
|
2701
|
+
to the question instead makes each delivered slice say which descriptor it answers.
|
|
2702
|
+
``plan_ranges`` matches by exact equality, so a descriptor that is not one of these is a
|
|
2703
|
+
descriptor no selector named. The exact spans still cannot be checked here -- they come
|
|
2704
|
+
from the sidecar, which only the worker fetched -- and neither can the descriptor-to-span
|
|
2705
|
+
mapping, which is the sidecar's and is the documented known gap.
|
|
2706
|
+
|
|
2707
|
+
``expected_reference_time`` is the model run the invocation ordered, when it ordered one.
|
|
2708
|
+
It is re-checked here for the same reason the structural check is: the worker is the process
|
|
2709
|
+
that handled the untrusted bytes, so a run binding that only ever ran there is a binding the
|
|
2710
|
+
coordinator is taking on trust. When the order named no run, no run binding is claimed.
|
|
2711
|
+
"""
|
|
2712
|
+
|
|
2713
|
+
if not required:
|
|
2714
|
+
if value is not None:
|
|
2715
|
+
raise AcquisitionSecurityError(
|
|
2716
|
+
"SANDBOX_RESPONSE",
|
|
2717
|
+
"non-slice response contains slice evidence",
|
|
2718
|
+
)
|
|
2719
|
+
return None
|
|
2720
|
+
if expected_selectors is None:
|
|
2721
|
+
raise AcquisitionSecurityError(
|
|
2722
|
+
"SANDBOX_RESPONSE",
|
|
2723
|
+
"a slice response can only be read against the plan that was ordered",
|
|
2724
|
+
)
|
|
2725
|
+
if not isinstance(value, list) or not 1 <= len(value) <= MAX_RANGE_PLAN:
|
|
2726
|
+
raise AcquisitionSecurityError(
|
|
2727
|
+
"SANDBOX_RESPONSE",
|
|
2728
|
+
"slice response is not a bounded non-empty list",
|
|
2729
|
+
)
|
|
2730
|
+
if len(value) != len(expected_selectors):
|
|
2731
|
+
raise AcquisitionSecurityError(
|
|
2732
|
+
"SANDBOX_RESPONSE",
|
|
2733
|
+
"slice response does not answer the exact plan that was ordered",
|
|
2734
|
+
)
|
|
2735
|
+
unanswered = list(expected_selectors)
|
|
2736
|
+
slices: list[FetchedSlice] = []
|
|
2737
|
+
for item in value:
|
|
2738
|
+
if not isinstance(item, dict) or set(item) != _FETCHED_SLICE_KEYS:
|
|
2739
|
+
raise AcquisitionSecurityError("SANDBOX_RESPONSE", "slice shape is invalid")
|
|
2740
|
+
index = item["index"]
|
|
2741
|
+
first_byte = item["first_byte"]
|
|
2742
|
+
last_byte = item["last_byte"]
|
|
2743
|
+
if type(index) is not int or index < 1:
|
|
2744
|
+
raise AcquisitionSecurityError("SANDBOX_RESPONSE", "slice index is invalid")
|
|
2745
|
+
descriptor = _bounded_text(item["descriptor"], maximum=MAX_SELECTOR_BYTES)
|
|
2746
|
+
if descriptor not in unanswered:
|
|
2747
|
+
# Either a descriptor no selector named, or one named once and answered twice.
|
|
2748
|
+
# Neither is the order that was placed, and both would put a slice on the receipt
|
|
2749
|
+
# under a label the coordinator never asked for.
|
|
2750
|
+
raise AcquisitionSecurityError(
|
|
2751
|
+
"SANDBOX_RESPONSE",
|
|
2752
|
+
"a delivered slice does not answer an unanswered selector that was ordered",
|
|
2753
|
+
)
|
|
2754
|
+
unanswered.remove(descriptor)
|
|
2755
|
+
if (
|
|
2756
|
+
type(first_byte) is not int
|
|
2757
|
+
or type(last_byte) is not int
|
|
2758
|
+
or not 0 <= first_byte <= last_byte
|
|
2759
|
+
):
|
|
2760
|
+
raise AcquisitionSecurityError("SANDBOX_RESPONSE", "slice span is invalid")
|
|
2761
|
+
digest = _digest(item["sha256"])
|
|
2762
|
+
try:
|
|
2763
|
+
content = base64.b64decode(
|
|
2764
|
+
_bounded_text(item["content_base64"], maximum=MAX_RESPONSE_BYTES),
|
|
2765
|
+
validate=True,
|
|
2766
|
+
)
|
|
2767
|
+
except (ValueError, TypeError):
|
|
2768
|
+
raise AcquisitionSecurityError(
|
|
2769
|
+
"SANDBOX_RESPONSE",
|
|
2770
|
+
"slice content is not canonical base64",
|
|
2771
|
+
) from None
|
|
2772
|
+
if len(content) != last_byte - first_byte + 1:
|
|
2773
|
+
raise AcquisitionSecurityError(
|
|
2774
|
+
"SANDBOX_RESPONSE",
|
|
2775
|
+
"slice bytes do not fill the span they name",
|
|
2776
|
+
)
|
|
2777
|
+
if sha256_bytes(content) != digest:
|
|
2778
|
+
raise AcquisitionSecurityError(
|
|
2779
|
+
"SANDBOX_RESPONSE",
|
|
2780
|
+
"slice digest does not match its own bytes",
|
|
2781
|
+
)
|
|
2782
|
+
# The structural check again, on this side of the boundary. The worker runs it before
|
|
2783
|
+
# it keeps a slice, but the worker is the process that handled the untrusted bytes; a
|
|
2784
|
+
# check that only ever ran there is a check the coordinator is taking on trust. It is
|
|
2785
|
+
# pure, it is cheap, and re-running it is the difference between "the worker says these
|
|
2786
|
+
# are messages" and "these are messages".
|
|
2787
|
+
try:
|
|
2788
|
+
check_grib_message(content)
|
|
2789
|
+
except AcquisitionSecurityError as exc:
|
|
2790
|
+
raise AcquisitionSecurityError(
|
|
2791
|
+
"SANDBOX_RESPONSE",
|
|
2792
|
+
f"slice bytes are not a well-formed message ({exc.code})",
|
|
2793
|
+
) from None
|
|
2794
|
+
if expected_reference_time is not None:
|
|
2795
|
+
try:
|
|
2796
|
+
check_grib_reference_time(content, expected=expected_reference_time)
|
|
2797
|
+
except AcquisitionSecurityError as exc:
|
|
2798
|
+
raise AcquisitionSecurityError(
|
|
2799
|
+
"SANDBOX_RESPONSE",
|
|
2800
|
+
f"slice bytes are not from the model run that was ordered ({exc.code})",
|
|
2801
|
+
) from None
|
|
2802
|
+
if slices and index <= slices[-1].index:
|
|
2803
|
+
raise AcquisitionSecurityError(
|
|
2804
|
+
"SANDBOX_RESPONSE",
|
|
2805
|
+
"slices are not emitted in ascending message-index order",
|
|
2806
|
+
)
|
|
2807
|
+
slices.append(
|
|
2808
|
+
FetchedSlice(
|
|
2809
|
+
index=index,
|
|
2810
|
+
descriptor=descriptor,
|
|
2811
|
+
first_byte=first_byte,
|
|
2812
|
+
last_byte=last_byte,
|
|
2813
|
+
sha256=digest,
|
|
2814
|
+
content=content,
|
|
2815
|
+
)
|
|
2816
|
+
)
|
|
2817
|
+
return tuple(slices)
|
|
2818
|
+
|
|
2819
|
+
|
|
2820
|
+
def _parsed_dict(parsed: ParsedTable) -> dict[str, Any]:
|
|
2821
|
+
return {
|
|
2822
|
+
"data_format": parsed.data_format,
|
|
2823
|
+
"input_sha256": parsed.input_sha256,
|
|
2824
|
+
"columns": list(parsed.columns),
|
|
2825
|
+
"rows": [list(row) for row in parsed.rows],
|
|
2826
|
+
"schema_digest": parsed.schema_digest,
|
|
2827
|
+
}
|
|
2828
|
+
|
|
2829
|
+
|
|
2830
|
+
def _parsed_from_dict(value: Any) -> ParsedTable:
|
|
2831
|
+
if not isinstance(value, dict) or set(value) != {
|
|
2832
|
+
"data_format",
|
|
2833
|
+
"input_sha256",
|
|
2834
|
+
"columns",
|
|
2835
|
+
"rows",
|
|
2836
|
+
"schema_digest",
|
|
2837
|
+
}:
|
|
2838
|
+
raise AcquisitionSecurityError("SANDBOX_RESPONSE", "parsed response shape is invalid")
|
|
2839
|
+
columns = _string_tuple(value["columns"], maximum_items=10_000, item_maximum=256)
|
|
2840
|
+
if not isinstance(value["rows"], list) or len(value["rows"]) > 10_000_000:
|
|
2841
|
+
raise AcquisitionSecurityError("SANDBOX_RESPONSE", "parsed response rows are invalid")
|
|
2842
|
+
rows: list[tuple[Any, ...]] = []
|
|
2843
|
+
for row in value["rows"]:
|
|
2844
|
+
if not isinstance(row, list):
|
|
2845
|
+
raise AcquisitionSecurityError("SANDBOX_RESPONSE", "parsed row is not an array")
|
|
2846
|
+
rows.append(tuple(row))
|
|
2847
|
+
return ParsedTable(
|
|
2848
|
+
data_format=_bounded_text(value["data_format"], maximum=16),
|
|
2849
|
+
input_sha256=_digest(value["input_sha256"]),
|
|
2850
|
+
columns=columns,
|
|
2851
|
+
rows=tuple(rows),
|
|
2852
|
+
schema_digest=_digest(value["schema_digest"]),
|
|
2853
|
+
)
|
|
2854
|
+
|
|
2855
|
+
|
|
2856
|
+
def _strict_json_object(raw: bytes, path: str) -> dict[str, Any]:
|
|
2857
|
+
try:
|
|
2858
|
+
value = json.loads(raw, object_pairs_hook=_unique_object, parse_constant=_reject_constant)
|
|
2859
|
+
except (UnicodeDecodeError, json.JSONDecodeError):
|
|
2860
|
+
raise AcquisitionSecurityError("SANDBOX_JSON", f"{path} is not strict JSON") from None
|
|
2861
|
+
if not isinstance(value, dict):
|
|
2862
|
+
raise AcquisitionSecurityError("SANDBOX_JSON", f"{path} must be an object")
|
|
2863
|
+
return value
|
|
2864
|
+
|
|
2865
|
+
|
|
2866
|
+
def _unique_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
|
|
2867
|
+
result: dict[str, Any] = {}
|
|
2868
|
+
for key, value in pairs:
|
|
2869
|
+
if key in result:
|
|
2870
|
+
raise AcquisitionSecurityError("SANDBOX_JSON", "JSON keys must be unique")
|
|
2871
|
+
result[key] = value
|
|
2872
|
+
return result
|
|
2873
|
+
|
|
2874
|
+
|
|
2875
|
+
def _reject_constant(value: str) -> Any:
|
|
2876
|
+
raise AcquisitionSecurityError("SANDBOX_JSON", f"non-finite number {value!r} is forbidden")
|
|
2877
|
+
|
|
2878
|
+
|
|
2879
|
+
def _parse_limits_dict(limits: ParseLimits) -> dict[str, int]:
|
|
2880
|
+
return {
|
|
2881
|
+
"max_input_bytes": limits.max_input_bytes,
|
|
2882
|
+
"max_uncompressed_bytes": limits.max_uncompressed_bytes,
|
|
2883
|
+
"max_expansion_ratio": limits.max_expansion_ratio,
|
|
2884
|
+
"max_rows": limits.max_rows,
|
|
2885
|
+
"max_columns": limits.max_columns,
|
|
2886
|
+
"max_field_bytes": limits.max_field_bytes,
|
|
2887
|
+
"max_json_depth": limits.max_json_depth,
|
|
2888
|
+
"max_total_cells": limits.max_total_cells,
|
|
2889
|
+
}
|
|
2890
|
+
|
|
2891
|
+
|
|
2892
|
+
def _parse_parse_limits(value: Any) -> ParseLimits:
|
|
2893
|
+
if not isinstance(value, dict) or set(value) != set(_parse_limits_dict(ParseLimits())):
|
|
2894
|
+
raise AcquisitionSecurityError("SANDBOX_LIMITS", "parse limits shape is invalid")
|
|
2895
|
+
return ParseLimits(**value)
|
|
2896
|
+
|
|
2897
|
+
|
|
2898
|
+
def _retrieval_limits_dict(limits: RetrievalLimits) -> dict[str, Any]:
|
|
2899
|
+
return {
|
|
2900
|
+
"max_response_bytes": limits.max_response_bytes,
|
|
2901
|
+
"max_aggregate_response_bytes": limits.max_aggregate_response_bytes,
|
|
2902
|
+
"max_redirects": limits.max_redirects,
|
|
2903
|
+
"max_requests": limits.max_requests,
|
|
2904
|
+
# Carried across the boundary like every other bound. Left out, a coordinator that
|
|
2905
|
+
# narrowed the range budget would have the worker silently rebuild the default, and
|
|
2906
|
+
# the plan's own budget would be checked against a ceiling nobody chose.
|
|
2907
|
+
"max_ranges": limits.max_ranges,
|
|
2908
|
+
"connect_timeout_seconds": limits.connect_timeout_seconds,
|
|
2909
|
+
"read_timeout_seconds": limits.read_timeout_seconds,
|
|
2910
|
+
"total_timeout_seconds": limits.total_timeout_seconds,
|
|
2911
|
+
"max_concurrency": limits.max_concurrency,
|
|
2912
|
+
"min_interval_seconds": limits.min_interval_seconds,
|
|
2913
|
+
"allowed_media_types": list(limits.allowed_media_types),
|
|
2914
|
+
"user_agent": limits.user_agent,
|
|
2915
|
+
}
|
|
2916
|
+
|
|
2917
|
+
|
|
2918
|
+
def _parse_retrieval_limits(value: Any) -> RetrievalLimits:
|
|
2919
|
+
if not isinstance(value, dict) or set(value) != set(_retrieval_limits_dict(RetrievalLimits())):
|
|
2920
|
+
raise AcquisitionSecurityError("SANDBOX_LIMITS", "retrieval limits shape is invalid")
|
|
2921
|
+
copied = dict(value)
|
|
2922
|
+
copied["allowed_media_types"] = _string_tuple(
|
|
2923
|
+
copied["allowed_media_types"],
|
|
2924
|
+
maximum_items=16,
|
|
2925
|
+
item_maximum=255,
|
|
2926
|
+
)
|
|
2927
|
+
return RetrievalLimits(**copied)
|
|
2928
|
+
|
|
2929
|
+
|
|
2930
|
+
@dataclass(frozen=True)
|
|
2931
|
+
class _ApprovedEndpointResolver:
|
|
2932
|
+
endpoints: dict[tuple[str, int], tuple[str, ...]]
|
|
2933
|
+
|
|
2934
|
+
def resolve(self, hostname: str, port: int) -> tuple[str, ...]:
|
|
2935
|
+
try:
|
|
2936
|
+
return self.endpoints[(hostname, port)]
|
|
2937
|
+
except KeyError as exc:
|
|
2938
|
+
raise AcquisitionSecurityError(
|
|
2939
|
+
"SANDBOX_EGRESS",
|
|
2940
|
+
"worker attempted an endpoint outside the coordinator DNS pin",
|
|
2941
|
+
) from exc
|
|
2942
|
+
|
|
2943
|
+
|
|
2944
|
+
def _resolve_approved_endpoints(
|
|
2945
|
+
url: str,
|
|
2946
|
+
allowed_hostnames: tuple[str, ...],
|
|
2947
|
+
) -> tuple[tuple[str, int, tuple[str, ...]], ...]:
|
|
2948
|
+
resolver = SystemResolver()
|
|
2949
|
+
policy = EgressPolicy(allowed_hostnames=allowed_hostnames)
|
|
2950
|
+
initial = validate_public_https_url(url, resolver=resolver, egress_policy=policy)
|
|
2951
|
+
resolved: dict[str, tuple[str, ...]] = {initial.hostname: initial.approved_ips}
|
|
2952
|
+
for hostname in allowed_hostnames:
|
|
2953
|
+
if hostname not in resolved:
|
|
2954
|
+
target = validate_public_https_url(
|
|
2955
|
+
_https_root_url(hostname),
|
|
2956
|
+
resolver=resolver,
|
|
2957
|
+
egress_policy=policy,
|
|
2958
|
+
)
|
|
2959
|
+
resolved[hostname] = target.approved_ips
|
|
2960
|
+
return tuple((hostname, 443, resolved[hostname]) for hostname in sorted(resolved))
|
|
2961
|
+
|
|
2962
|
+
|
|
2963
|
+
def _parse_approved_endpoints(
|
|
2964
|
+
value: Any,
|
|
2965
|
+
*,
|
|
2966
|
+
allowed_hostnames: tuple[str, ...],
|
|
2967
|
+
) -> _ApprovedEndpointResolver:
|
|
2968
|
+
if not isinstance(value, list) or len(value) != len(allowed_hostnames):
|
|
2969
|
+
raise AcquisitionSecurityError(
|
|
2970
|
+
"SANDBOX_EGRESS",
|
|
2971
|
+
"approved endpoint set does not cover the exact hostname allowlist",
|
|
2972
|
+
)
|
|
2973
|
+
endpoints: dict[tuple[str, int], tuple[str, ...]] = {}
|
|
2974
|
+
for item in value:
|
|
2975
|
+
if not isinstance(item, dict) or set(item) != {
|
|
2976
|
+
"hostname",
|
|
2977
|
+
"port",
|
|
2978
|
+
"approved_ips",
|
|
2979
|
+
}:
|
|
2980
|
+
raise AcquisitionSecurityError(
|
|
2981
|
+
"SANDBOX_EGRESS",
|
|
2982
|
+
"approved endpoint shape is invalid",
|
|
2983
|
+
)
|
|
2984
|
+
hostname = _bounded_text(item["hostname"], maximum=253)
|
|
2985
|
+
port = item["port"]
|
|
2986
|
+
if hostname not in allowed_hostnames or type(port) is not int or port != 443:
|
|
2987
|
+
raise AcquisitionSecurityError(
|
|
2988
|
+
"SANDBOX_EGRESS",
|
|
2989
|
+
"approved endpoint is outside the hostname/port authority",
|
|
2990
|
+
)
|
|
2991
|
+
addresses = _string_tuple(
|
|
2992
|
+
item["approved_ips"],
|
|
2993
|
+
maximum_items=32,
|
|
2994
|
+
item_maximum=64,
|
|
2995
|
+
)
|
|
2996
|
+
if not addresses:
|
|
2997
|
+
raise AcquisitionSecurityError("SANDBOX_EGRESS", "approved IP set is empty")
|
|
2998
|
+
normalized: list[str] = []
|
|
2999
|
+
for raw in addresses:
|
|
3000
|
+
try:
|
|
3001
|
+
address = ipaddress.ip_address(raw)
|
|
3002
|
+
except ValueError:
|
|
3003
|
+
raise AcquisitionSecurityError(
|
|
3004
|
+
"SANDBOX_EGRESS",
|
|
3005
|
+
"approved endpoint contains a non-IP address",
|
|
3006
|
+
) from None
|
|
3007
|
+
if not address.is_global:
|
|
3008
|
+
raise AcquisitionSecurityError(
|
|
3009
|
+
"SANDBOX_EGRESS",
|
|
3010
|
+
"approved endpoint contains a non-public address",
|
|
3011
|
+
)
|
|
3012
|
+
normalized.append(address.compressed)
|
|
3013
|
+
key = (hostname, port)
|
|
3014
|
+
if key in endpoints or tuple(sorted(normalized)) != addresses:
|
|
3015
|
+
raise AcquisitionSecurityError(
|
|
3016
|
+
"SANDBOX_EGRESS",
|
|
3017
|
+
"approved endpoint set is duplicate or non-canonical",
|
|
3018
|
+
)
|
|
3019
|
+
endpoints[key] = addresses
|
|
3020
|
+
if {hostname for hostname, _port in endpoints} != set(allowed_hostnames):
|
|
3021
|
+
raise AcquisitionSecurityError(
|
|
3022
|
+
"SANDBOX_EGRESS",
|
|
3023
|
+
"approved endpoint set differs from the hostname allowlist",
|
|
3024
|
+
)
|
|
3025
|
+
return _ApprovedEndpointResolver(endpoints)
|
|
3026
|
+
|
|
3027
|
+
|
|
3028
|
+
def _https_root_url(hostname: str) -> str:
|
|
3029
|
+
try:
|
|
3030
|
+
address = ipaddress.ip_address(hostname)
|
|
3031
|
+
except ValueError:
|
|
3032
|
+
address = None
|
|
3033
|
+
host = f"[{hostname}]" if isinstance(address, ipaddress.IPv6Address) else hostname
|
|
3034
|
+
return f"https://{host}/"
|
|
3035
|
+
|
|
3036
|
+
|
|
3037
|
+
def _parse_network_probe_targets(
|
|
3038
|
+
request: dict[str, Any],
|
|
3039
|
+
) -> tuple[tuple[str, str, int], ...]:
|
|
3040
|
+
value = request["network_probe_targets"]
|
|
3041
|
+
if not isinstance(value, list) or len(value) != 3:
|
|
3042
|
+
raise AcquisitionSecurityError(
|
|
3043
|
+
"SANDBOX_PROBE",
|
|
3044
|
+
"probe must contain the exact three network-denial targets",
|
|
3045
|
+
)
|
|
3046
|
+
parsed: dict[str, tuple[str, int]] = {}
|
|
3047
|
+
for item in value:
|
|
3048
|
+
if not isinstance(item, dict) or set(item) != {"label", "ip", "port"}:
|
|
3049
|
+
raise AcquisitionSecurityError("SANDBOX_PROBE", "network probe shape is invalid")
|
|
3050
|
+
label = _bounded_text(item["label"], maximum=16)
|
|
3051
|
+
ip = _bounded_text(item["ip"], maximum=64)
|
|
3052
|
+
port = item["port"]
|
|
3053
|
+
if type(port) is not int or not 1 <= port <= 65_535 or label in parsed:
|
|
3054
|
+
raise AcquisitionSecurityError("SANDBOX_PROBE", "network probe target is invalid")
|
|
3055
|
+
parsed[label] = (ip, port)
|
|
3056
|
+
if set(parsed) != {"loopback", "private", "metadata"}:
|
|
3057
|
+
raise AcquisitionSecurityError(
|
|
3058
|
+
"SANDBOX_PROBE",
|
|
3059
|
+
"network probe labels are invalid",
|
|
3060
|
+
)
|
|
3061
|
+
if (
|
|
3062
|
+
parsed["loopback"][0] != "127.0.0.1"
|
|
3063
|
+
or parsed["private"] != ("10.0.0.1", 443)
|
|
3064
|
+
or parsed["metadata"] != ("169.254.169.254", 80)
|
|
3065
|
+
):
|
|
3066
|
+
raise AcquisitionSecurityError(
|
|
3067
|
+
"SANDBOX_PROBE",
|
|
3068
|
+
"network probe destinations are invalid",
|
|
3069
|
+
)
|
|
3070
|
+
return tuple((label, *parsed[label]) for label in ("loopback", "private", "metadata"))
|
|
3071
|
+
|
|
3072
|
+
|
|
3073
|
+
def _probe_network_denials(
|
|
3074
|
+
targets: tuple[tuple[str, str, int], ...],
|
|
3075
|
+
) -> tuple[tuple[str, int], ...]:
|
|
3076
|
+
results: list[tuple[str, int]] = []
|
|
3077
|
+
for label, ip, port in targets:
|
|
3078
|
+
family = socket.AF_INET6 if ":" in ip else socket.AF_INET
|
|
3079
|
+
probe = socket.socket(family, socket.SOCK_STREAM)
|
|
3080
|
+
try:
|
|
3081
|
+
probe.settimeout(0.25)
|
|
3082
|
+
error = probe.connect_ex((ip, port))
|
|
3083
|
+
except OSError as exc:
|
|
3084
|
+
error = exc.errno or 1
|
|
3085
|
+
finally:
|
|
3086
|
+
probe.close()
|
|
3087
|
+
results.append((label, error))
|
|
3088
|
+
return tuple(results)
|
|
3089
|
+
|
|
3090
|
+
|
|
3091
|
+
def _parse_network_probe_response(
|
|
3092
|
+
value: Any,
|
|
3093
|
+
*,
|
|
3094
|
+
required: bool,
|
|
3095
|
+
) -> tuple[tuple[str, int], ...] | None:
|
|
3096
|
+
if not required:
|
|
3097
|
+
if value is not None:
|
|
3098
|
+
raise AcquisitionSecurityError(
|
|
3099
|
+
"SANDBOX_RESPONSE",
|
|
3100
|
+
"non-probe response contains network probe evidence",
|
|
3101
|
+
)
|
|
3102
|
+
return None
|
|
3103
|
+
if not isinstance(value, list) or len(value) != 3:
|
|
3104
|
+
raise AcquisitionSecurityError(
|
|
3105
|
+
"SANDBOX_OS_BOUNDARY",
|
|
3106
|
+
"sandbox did not return complete network-denial evidence",
|
|
3107
|
+
)
|
|
3108
|
+
result: list[tuple[str, int]] = []
|
|
3109
|
+
for item in value:
|
|
3110
|
+
if not isinstance(item, dict) or set(item) != {"label", "errno"}:
|
|
3111
|
+
raise AcquisitionSecurityError(
|
|
3112
|
+
"SANDBOX_OS_BOUNDARY",
|
|
3113
|
+
"sandbox network-denial evidence is invalid",
|
|
3114
|
+
)
|
|
3115
|
+
label = item["label"]
|
|
3116
|
+
error = item["errno"]
|
|
3117
|
+
if label not in {"loopback", "private", "metadata"} or type(error) is not int or error <= 0:
|
|
3118
|
+
raise AcquisitionSecurityError(
|
|
3119
|
+
"SANDBOX_OS_BOUNDARY",
|
|
3120
|
+
"sandbox completed a forbidden direct socket attempt",
|
|
3121
|
+
)
|
|
3122
|
+
result.append((label, error))
|
|
3123
|
+
if {label for label, _error in result} != {"loopback", "private", "metadata"}:
|
|
3124
|
+
raise AcquisitionSecurityError(
|
|
3125
|
+
"SANDBOX_OS_BOUNDARY",
|
|
3126
|
+
"sandbox network-denial evidence is incomplete",
|
|
3127
|
+
)
|
|
3128
|
+
return tuple(result)
|
|
3129
|
+
|
|
3130
|
+
|
|
3131
|
+
def _require_null_request_fields(request: dict[str, Any]) -> None:
|
|
3132
|
+
for key in (
|
|
3133
|
+
"content_base64",
|
|
3134
|
+
"data_format",
|
|
3135
|
+
"media_type",
|
|
3136
|
+
"filename",
|
|
3137
|
+
"parse_limits",
|
|
3138
|
+
"url",
|
|
3139
|
+
"approved_endpoints",
|
|
3140
|
+
"retrieval_limits",
|
|
3141
|
+
"reader_pin",
|
|
3142
|
+
"reader_budgets",
|
|
3143
|
+
"range_plan",
|
|
3144
|
+
"cycle_survey",
|
|
3145
|
+
):
|
|
3146
|
+
expected = [] if key == "approved_endpoints" else None
|
|
3147
|
+
if request[key] != expected:
|
|
3148
|
+
raise AcquisitionSecurityError(
|
|
3149
|
+
"SANDBOX_FIELDS",
|
|
3150
|
+
"probe request must not carry acquisition inputs",
|
|
3151
|
+
)
|
|
3152
|
+
if request["allowed_hostnames"] != []:
|
|
3153
|
+
raise AcquisitionSecurityError(
|
|
3154
|
+
"SANDBOX_FIELDS",
|
|
3155
|
+
"probe request must not carry egress authority",
|
|
3156
|
+
)
|
|
3157
|
+
|
|
3158
|
+
|
|
3159
|
+
def _validate_egress_hosts(hostnames: tuple[str, ...]) -> None:
|
|
3160
|
+
if not hostnames or len(hostnames) > 32 or len(set(hostnames)) != len(hostnames):
|
|
3161
|
+
raise AcquisitionSecurityError(
|
|
3162
|
+
"SANDBOX_EGRESS",
|
|
3163
|
+
"sandbox egress hostnames must be non-empty, unique, and bounded",
|
|
3164
|
+
)
|
|
3165
|
+
for hostname in hostnames:
|
|
3166
|
+
normalized = hostname.rstrip(".").lower()
|
|
3167
|
+
if (
|
|
3168
|
+
normalized != hostname
|
|
3169
|
+
or normalized in _FORBIDDEN_EGRESS_HOSTS
|
|
3170
|
+
or normalized.endswith(".internal")
|
|
3171
|
+
):
|
|
3172
|
+
raise AcquisitionSecurityError(
|
|
3173
|
+
"SANDBOX_EGRESS",
|
|
3174
|
+
"sandbox egress includes a forbidden production or metadata host",
|
|
3175
|
+
)
|
|
3176
|
+
|
|
3177
|
+
|
|
3178
|
+
def _policy_document(operation: str) -> dict[str, Any]:
|
|
3179
|
+
"""Build the exact policy mapping whose digest binds a worker to its sandbox policy.
|
|
3180
|
+
|
|
3181
|
+
Split out from ``_policy_digest`` so a test can assert the document's shape directly --
|
|
3182
|
+
in particular that a non-parsing operation carries no ``parser_formats`` key -- rather
|
|
3183
|
+
than inferring shape from an opaque digest.
|
|
3184
|
+
"""
|
|
3185
|
+
|
|
3186
|
+
policy: dict[str, Any] = {
|
|
3187
|
+
"schema_version": SANDBOX_PROTOCOL_VERSION,
|
|
3188
|
+
"operation": operation,
|
|
3189
|
+
"environment_keys": sorted(_ALLOWED_ENVIRONMENT),
|
|
3190
|
+
"ambient_credentials": False,
|
|
3191
|
+
"ambient_proxy": False,
|
|
3192
|
+
"studio_clients": False,
|
|
3193
|
+
# One value since ADR 0021. The other arm named the Seatbelt profile, which no
|
|
3194
|
+
# longer exists; this is the clause every non-darwin host already attested, so no
|
|
3195
|
+
# digest a Linux worker ever produced moves.
|
|
3196
|
+
"filesystem": "external-container-boundary-required",
|
|
3197
|
+
# Driven by the named set rather than by a comparison against one operation name, so
|
|
3198
|
+
# an operation cannot be given live socket authority in ``_invoke`` while its own
|
|
3199
|
+
# attestation keeps claiming the network is denied to it.
|
|
3200
|
+
# The word "seatbelt" in this constant outlived the Seatbelt backend on purpose. This
|
|
3201
|
+
# string is not a description that can be freshened -- it is digest input, and every
|
|
3202
|
+
# sealed attestation, golden, and coordinator pin in the repository and in already
|
|
3203
|
+
# released evidence binds the digest it produces. Renaming it would rotate four
|
|
3204
|
+
# operation digests to improve a label. It reads today as the historical name of the
|
|
3205
|
+
# approved-public-IP-plus-TCP/443 posture, which is still exactly the posture.
|
|
3206
|
+
"network": (
|
|
3207
|
+
"external-deny-default-approved-public-ip-plus-seatbelt-tcp-443"
|
|
3208
|
+
if operation in NETWORK_PERFORMING_OPERATIONS
|
|
3209
|
+
else "os-denied"
|
|
3210
|
+
),
|
|
3211
|
+
}
|
|
3212
|
+
# Scoped per operation, following the pattern ``filesystem`` and ``network`` already use
|
|
3213
|
+
# above. An operation absent from the registry gets no key at all rather than an empty
|
|
3214
|
+
# list: absent parsing support produces no key. ``canonical_sha256`` sorts keys, so insertion
|
|
3215
|
+
# conditionally cannot perturb the digest by ordering.
|
|
3216
|
+
parser_formats = PARSER_FORMATS_BY_OPERATION.get(operation)
|
|
3217
|
+
if parser_formats is not None:
|
|
3218
|
+
policy["parser_formats"] = list(parser_formats)
|
|
3219
|
+
# The Reader contract label, present only on the operation that may run a Reader, in the
|
|
3220
|
+
# same conditional register the two clauses above use.
|
|
3221
|
+
#
|
|
3222
|
+
# It is the contract label and deliberately not the list of admitted families. Embedding
|
|
3223
|
+
# the family ids would make every family registration rotate this digest and require a
|
|
3224
|
+
# coordinator-configuration change -- for no security gain, because the family that ran is
|
|
3225
|
+
# already bound per source by three independent checks: the pin inside ``request.digest``,
|
|
3226
|
+
# the adapter-side cross-check in ``sources/_adapter_steps.py``, and
|
|
3227
|
+
# ``recipe._validate_reader_pin_binding``. What the label actually versions is the shape
|
|
3228
|
+
# of that authority, which is the thing an attestation should be bound to.
|
|
3229
|
+
if operation in READER_PERFORMING_OPERATIONS:
|
|
3230
|
+
policy["reader_contract"] = READER_CONTRACT_VERSION
|
|
3231
|
+
return policy
|
|
3232
|
+
|
|
3233
|
+
|
|
3234
|
+
def _policy_digest(operation: str) -> str:
|
|
3235
|
+
return canonical_sha256(_policy_document(operation))
|
|
3236
|
+
|
|
3237
|
+
|
|
3238
|
+
def _home_directory_readable() -> bool:
|
|
3239
|
+
try:
|
|
3240
|
+
next(Path.home().iterdir(), None)
|
|
3241
|
+
except OSError:
|
|
3242
|
+
return False
|
|
3243
|
+
return True
|
|
3244
|
+
|
|
3245
|
+
|
|
3246
|
+
class _WorkerProcess(Protocol):
|
|
3247
|
+
"""The small process surface needed by the bounded IPC pump."""
|
|
3248
|
+
|
|
3249
|
+
pid: int
|
|
3250
|
+
stdin_fd: int
|
|
3251
|
+
stdout_fd: int
|
|
3252
|
+
stderr_fd: int
|
|
3253
|
+
|
|
3254
|
+
def close_stdin(self) -> None: ...
|
|
3255
|
+
|
|
3256
|
+
def close_output(self, descriptor: int) -> None: ...
|
|
3257
|
+
|
|
3258
|
+
def wait(self, timeout: float) -> int: ...
|
|
3259
|
+
|
|
3260
|
+
def kill_group(self) -> None: ...
|
|
3261
|
+
|
|
3262
|
+
|
|
3263
|
+
class _PopenWorker:
|
|
3264
|
+
def __init__(self, process: subprocess.Popen[bytes]) -> None:
|
|
3265
|
+
if process.stdin is None or process.stdout is None or process.stderr is None:
|
|
3266
|
+
raise RuntimeError("Clean room process pipes were not created")
|
|
3267
|
+
self._process = process
|
|
3268
|
+
self.pid = process.pid
|
|
3269
|
+
self.stdin_fd = process.stdin.fileno()
|
|
3270
|
+
self.stdout_fd = process.stdout.fileno()
|
|
3271
|
+
self.stderr_fd = process.stderr.fileno()
|
|
3272
|
+
|
|
3273
|
+
def close_stdin(self) -> None:
|
|
3274
|
+
if self._process.stdin is not None and not self._process.stdin.closed:
|
|
3275
|
+
self._process.stdin.close()
|
|
3276
|
+
|
|
3277
|
+
def wait(self, timeout: float) -> int:
|
|
3278
|
+
return self._process.wait(timeout=max(timeout, 0.0))
|
|
3279
|
+
|
|
3280
|
+
def close_output(self, descriptor: int) -> None:
|
|
3281
|
+
for stream in (self._process.stdout, self._process.stderr):
|
|
3282
|
+
if stream is not None and not stream.closed and stream.fileno() == descriptor:
|
|
3283
|
+
stream.close()
|
|
3284
|
+
|
|
3285
|
+
def kill_group(self) -> None:
|
|
3286
|
+
try:
|
|
3287
|
+
os.killpg(self.pid, signal.SIGKILL)
|
|
3288
|
+
except ProcessLookupError:
|
|
3289
|
+
pass
|
|
3290
|
+
|
|
3291
|
+
|
|
3292
|
+
class _ForkedWorker:
|
|
3293
|
+
def __init__(
|
|
3294
|
+
self,
|
|
3295
|
+
*,
|
|
3296
|
+
pid: int,
|
|
3297
|
+
stdin_fd: int,
|
|
3298
|
+
stdout_fd: int,
|
|
3299
|
+
stderr_fd: int,
|
|
3300
|
+
) -> None:
|
|
3301
|
+
self.pid = pid
|
|
3302
|
+
self.stdin_fd = stdin_fd
|
|
3303
|
+
self.stdout_fd = stdout_fd
|
|
3304
|
+
self.stderr_fd = stderr_fd
|
|
3305
|
+
self._returncode: int | None = None
|
|
3306
|
+
|
|
3307
|
+
def close_stdin(self) -> None:
|
|
3308
|
+
if self.stdin_fd >= 0:
|
|
3309
|
+
try:
|
|
3310
|
+
os.close(self.stdin_fd)
|
|
3311
|
+
except OSError:
|
|
3312
|
+
pass
|
|
3313
|
+
self.stdin_fd = -1
|
|
3314
|
+
|
|
3315
|
+
def wait(self, timeout: float) -> int:
|
|
3316
|
+
if self._returncode is not None:
|
|
3317
|
+
return self._returncode
|
|
3318
|
+
deadline = time.monotonic() + max(timeout, 0.0)
|
|
3319
|
+
while True:
|
|
3320
|
+
pid, status = os.waitpid(self.pid, os.WNOHANG)
|
|
3321
|
+
if pid == self.pid:
|
|
3322
|
+
self._returncode = os.waitstatus_to_exitcode(status)
|
|
3323
|
+
return self._returncode
|
|
3324
|
+
if time.monotonic() >= deadline:
|
|
3325
|
+
raise subprocess.TimeoutExpired(("sandbox-worker",), timeout)
|
|
3326
|
+
time.sleep(0.005)
|
|
3327
|
+
|
|
3328
|
+
def close_output(self, descriptor: int) -> None:
|
|
3329
|
+
if descriptor == self.stdout_fd:
|
|
3330
|
+
self.stdout_fd = -1
|
|
3331
|
+
elif descriptor == self.stderr_fd:
|
|
3332
|
+
self.stderr_fd = -1
|
|
3333
|
+
else:
|
|
3334
|
+
return
|
|
3335
|
+
try:
|
|
3336
|
+
os.close(descriptor)
|
|
3337
|
+
except OSError:
|
|
3338
|
+
pass
|
|
3339
|
+
|
|
3340
|
+
def kill_group(self) -> None:
|
|
3341
|
+
try:
|
|
3342
|
+
os.killpg(self.pid, signal.SIGKILL)
|
|
3343
|
+
except ProcessLookupError:
|
|
3344
|
+
pass
|
|
3345
|
+
|
|
3346
|
+
|
|
3347
|
+
def _start_popen_worker(
|
|
3348
|
+
command: list[str],
|
|
3349
|
+
*,
|
|
3350
|
+
cwd: Path,
|
|
3351
|
+
env: dict[str, str],
|
|
3352
|
+
) -> _WorkerProcess:
|
|
3353
|
+
"""Start a session-isolated worker for platforms without the Linux barrier."""
|
|
3354
|
+
|
|
3355
|
+
nproc_limit = _nproc_limit()
|
|
3356
|
+
|
|
3357
|
+
def apply_limits() -> None:
|
|
3358
|
+
_apply_resource_limits(nproc_limit)
|
|
3359
|
+
|
|
3360
|
+
return _PopenWorker(
|
|
3361
|
+
subprocess.Popen(
|
|
3362
|
+
command,
|
|
3363
|
+
cwd=cwd,
|
|
3364
|
+
env=env,
|
|
3365
|
+
stdin=subprocess.PIPE,
|
|
3366
|
+
stdout=subprocess.PIPE,
|
|
3367
|
+
stderr=subprocess.PIPE,
|
|
3368
|
+
start_new_session=True,
|
|
3369
|
+
preexec_fn=apply_limits,
|
|
3370
|
+
)
|
|
3371
|
+
)
|
|
3372
|
+
|
|
3373
|
+
|
|
3374
|
+
def _stop_worker(process: _WorkerProcess) -> None:
|
|
3375
|
+
failure: BaseException | None = None
|
|
3376
|
+
|
|
3377
|
+
def attempt(operation: Any) -> None:
|
|
3378
|
+
nonlocal failure
|
|
3379
|
+
try:
|
|
3380
|
+
operation()
|
|
3381
|
+
except BaseException as error:
|
|
3382
|
+
if failure is None:
|
|
3383
|
+
failure = error
|
|
3384
|
+
|
|
3385
|
+
stdout_fd = process.stdout_fd
|
|
3386
|
+
stderr_fd = process.stderr_fd
|
|
3387
|
+
attempt(process.kill_group)
|
|
3388
|
+
attempt(process.close_stdin)
|
|
3389
|
+
try:
|
|
3390
|
+
process.wait(1.0)
|
|
3391
|
+
except subprocess.TimeoutExpired:
|
|
3392
|
+
# A timeout is only a failed first reap attempt. Preserve failures from the second
|
|
3393
|
+
# kill/reap, but do not report this timeout when the retry succeeds.
|
|
3394
|
+
attempt(process.kill_group)
|
|
3395
|
+
attempt(lambda: process.wait(1.0))
|
|
3396
|
+
except BaseException as error:
|
|
3397
|
+
if failure is None:
|
|
3398
|
+
failure = error
|
|
3399
|
+
attempt(process.close_stdin)
|
|
3400
|
+
attempt(lambda: process.close_output(stdout_fd))
|
|
3401
|
+
attempt(lambda: process.close_output(stderr_fd))
|
|
3402
|
+
if failure is not None:
|
|
3403
|
+
raise failure
|
|
3404
|
+
|
|
3405
|
+
|
|
3406
|
+
def _reap_owned_process_group_children(process_group: int, deadline: float) -> None:
|
|
3407
|
+
"""Reap adopted hosted descendants without touching another warm worker's group."""
|
|
3408
|
+
|
|
3409
|
+
while True:
|
|
3410
|
+
try:
|
|
3411
|
+
pid, _status = os.waitpid(-process_group, os.WNOHANG)
|
|
3412
|
+
except ChildProcessError:
|
|
3413
|
+
return
|
|
3414
|
+
if pid > 0:
|
|
3415
|
+
continue
|
|
3416
|
+
if time.monotonic() >= deadline:
|
|
3417
|
+
raise OSError("hosted-session process-group descendants did not become reapable")
|
|
3418
|
+
time.sleep(0.005)
|
|
3419
|
+
|
|
3420
|
+
|
|
3421
|
+
def _drain_worker_ipc(
|
|
3422
|
+
process: _WorkerProcess,
|
|
3423
|
+
*,
|
|
3424
|
+
request_bytes: bytes,
|
|
3425
|
+
timeout_seconds: float,
|
|
3426
|
+
terminate_process_group: bool = False,
|
|
3427
|
+
) -> tuple[bytes, bytes, int]:
|
|
3428
|
+
"""Pump all three pipes concurrently and enforce byte caps during streaming."""
|
|
3429
|
+
|
|
3430
|
+
selector: selectors.BaseSelector | None = None
|
|
3431
|
+
stdout = bytearray()
|
|
3432
|
+
stderr = bytearray()
|
|
3433
|
+
request_offset = 0
|
|
3434
|
+
streams = {
|
|
3435
|
+
process.stdout_fd: (stdout, MAX_RESPONSE_BYTES, "stdout"),
|
|
3436
|
+
process.stderr_fd: (stderr, MAX_WORKER_STDERR_BYTES, "stderr"),
|
|
3437
|
+
}
|
|
3438
|
+
result: tuple[bytes, bytes, int] | None = None
|
|
3439
|
+
failure: BaseException | None = None
|
|
3440
|
+
|
|
3441
|
+
def remember(error: BaseException) -> None:
|
|
3442
|
+
nonlocal failure
|
|
3443
|
+
if failure is None:
|
|
3444
|
+
failure = error
|
|
3445
|
+
|
|
3446
|
+
try:
|
|
3447
|
+
selector = selectors.DefaultSelector()
|
|
3448
|
+
deadline = time.monotonic() + timeout_seconds
|
|
3449
|
+
for descriptor in (process.stdin_fd, process.stdout_fd, process.stderr_fd):
|
|
3450
|
+
os.set_blocking(descriptor, False)
|
|
3451
|
+
if request_bytes:
|
|
3452
|
+
selector.register(process.stdin_fd, selectors.EVENT_WRITE, "stdin")
|
|
3453
|
+
else:
|
|
3454
|
+
process.close_stdin()
|
|
3455
|
+
selector.register(process.stdout_fd, selectors.EVENT_READ, "stdout")
|
|
3456
|
+
selector.register(process.stderr_fd, selectors.EVENT_READ, "stderr")
|
|
3457
|
+
|
|
3458
|
+
while streams:
|
|
3459
|
+
remaining = deadline - time.monotonic()
|
|
3460
|
+
if remaining <= 0:
|
|
3461
|
+
raise AcquisitionSecurityError(
|
|
3462
|
+
"SANDBOX_TIMEOUT",
|
|
3463
|
+
"The Clean room process did not finish in time",
|
|
3464
|
+
)
|
|
3465
|
+
events = selector.select(remaining)
|
|
3466
|
+
if not events:
|
|
3467
|
+
continue
|
|
3468
|
+
for key, mask in events:
|
|
3469
|
+
descriptor = key.fd
|
|
3470
|
+
if key.data == "stdin" and mask & selectors.EVENT_WRITE:
|
|
3471
|
+
try:
|
|
3472
|
+
written = os.write(descriptor, request_bytes[request_offset:])
|
|
3473
|
+
except BrokenPipeError:
|
|
3474
|
+
written = 0
|
|
3475
|
+
request_offset += written
|
|
3476
|
+
if written == 0 or request_offset == len(request_bytes):
|
|
3477
|
+
selector.unregister(descriptor)
|
|
3478
|
+
process.close_stdin()
|
|
3479
|
+
continue
|
|
3480
|
+
if not mask & selectors.EVENT_READ:
|
|
3481
|
+
continue
|
|
3482
|
+
buffer, cap, _label = streams[descriptor]
|
|
3483
|
+
try:
|
|
3484
|
+
chunk = os.read(descriptor, min(65_536, cap - len(buffer) + 1))
|
|
3485
|
+
except BlockingIOError:
|
|
3486
|
+
continue
|
|
3487
|
+
if not chunk:
|
|
3488
|
+
selector.unregister(descriptor)
|
|
3489
|
+
process.close_output(descriptor)
|
|
3490
|
+
del streams[descriptor]
|
|
3491
|
+
continue
|
|
3492
|
+
buffer.extend(chunk)
|
|
3493
|
+
if len(buffer) > cap:
|
|
3494
|
+
raise AcquisitionSecurityError(
|
|
3495
|
+
"SANDBOX_IPC_LIMIT",
|
|
3496
|
+
"The Clean room process returned more data than this job allows",
|
|
3497
|
+
)
|
|
3498
|
+
|
|
3499
|
+
# Hosted-session children may fork bounded parser helpers, but no helper may outlive the
|
|
3500
|
+
# one request. Signal the still-owned process group before reaping its leader: this covers
|
|
3501
|
+
# the success path, catches descendants that deliberately closed all inherited pipes, and
|
|
3502
|
+
# avoids signalling a numeric group id after the leader pid could have been reused.
|
|
3503
|
+
if terminate_process_group:
|
|
3504
|
+
process.kill_group()
|
|
3505
|
+
remaining = deadline - time.monotonic()
|
|
3506
|
+
if remaining <= 0:
|
|
3507
|
+
raise AcquisitionSecurityError(
|
|
3508
|
+
"SANDBOX_TIMEOUT",
|
|
3509
|
+
"The Clean room process did not finish in time",
|
|
3510
|
+
)
|
|
3511
|
+
try:
|
|
3512
|
+
returncode = process.wait(remaining)
|
|
3513
|
+
except subprocess.TimeoutExpired as error:
|
|
3514
|
+
raise AcquisitionSecurityError(
|
|
3515
|
+
"SANDBOX_TIMEOUT",
|
|
3516
|
+
"The Clean room process did not finish in time",
|
|
3517
|
+
) from error
|
|
3518
|
+
if terminate_process_group:
|
|
3519
|
+
_reap_owned_process_group_children(process.pid, deadline)
|
|
3520
|
+
result = bytes(stdout), bytes(stderr), returncode
|
|
3521
|
+
except BaseException as error:
|
|
3522
|
+
remember(error)
|
|
3523
|
+
try:
|
|
3524
|
+
_stop_worker(process)
|
|
3525
|
+
except BaseException as cleanup_error:
|
|
3526
|
+
# Cleanup is subordinate to the exception that initiated it.
|
|
3527
|
+
remember(cleanup_error)
|
|
3528
|
+
finally:
|
|
3529
|
+
cleanup_steps = [process.close_stdin]
|
|
3530
|
+
if selector is not None:
|
|
3531
|
+
cleanup_steps.insert(0, selector.close)
|
|
3532
|
+
for descriptor in tuple(streams):
|
|
3533
|
+
cleanup_steps.append(lambda descriptor=descriptor: process.close_output(descriptor))
|
|
3534
|
+
for cleanup_step in cleanup_steps:
|
|
3535
|
+
try:
|
|
3536
|
+
cleanup_step()
|
|
3537
|
+
except BaseException as cleanup_error:
|
|
3538
|
+
remember(cleanup_error)
|
|
3539
|
+
if failure is not None:
|
|
3540
|
+
raise failure
|
|
3541
|
+
assert result is not None
|
|
3542
|
+
return result
|
|
3543
|
+
|
|
3544
|
+
|
|
3545
|
+
def _read_control(directory_fd: int, name: str) -> str:
|
|
3546
|
+
descriptor = os.open(name, os.O_RDONLY | os.O_CLOEXEC | os.O_NOFOLLOW, dir_fd=directory_fd)
|
|
3547
|
+
try:
|
|
3548
|
+
chunks: list[bytes] = []
|
|
3549
|
+
while True:
|
|
3550
|
+
chunk = os.read(descriptor, 4096)
|
|
3551
|
+
if not chunk:
|
|
3552
|
+
return b"".join(chunks).decode("ascii", "strict")
|
|
3553
|
+
chunks.append(chunk)
|
|
3554
|
+
finally:
|
|
3555
|
+
os.close(descriptor)
|
|
3556
|
+
|
|
3557
|
+
|
|
3558
|
+
def _write_control(directory_fd: int, name: str, value: str) -> None:
|
|
3559
|
+
descriptor = os.open(name, os.O_WRONLY | os.O_CLOEXEC | os.O_NOFOLLOW, dir_fd=directory_fd)
|
|
3560
|
+
try:
|
|
3561
|
+
payload = value.encode("ascii", "strict")
|
|
3562
|
+
if os.write(descriptor, payload) != len(payload):
|
|
3563
|
+
raise OSError("short write to cgroup control")
|
|
3564
|
+
finally:
|
|
3565
|
+
os.close(descriptor)
|
|
3566
|
+
|
|
3567
|
+
|
|
3568
|
+
def _control_can_be_opened_for_write(directory_fd: int, name: str) -> bool:
|
|
3569
|
+
try:
|
|
3570
|
+
descriptor = os.open(
|
|
3571
|
+
name,
|
|
3572
|
+
os.O_WRONLY | os.O_CLOEXEC | os.O_NOFOLLOW,
|
|
3573
|
+
dir_fd=directory_fd,
|
|
3574
|
+
)
|
|
3575
|
+
except OSError:
|
|
3576
|
+
return False
|
|
3577
|
+
os.close(descriptor)
|
|
3578
|
+
return True
|
|
3579
|
+
|
|
3580
|
+
|
|
3581
|
+
def _require_hosted_session_container_boundary(expected_memory_bytes: int) -> None:
|
|
3582
|
+
"""Validate the immutable whole-container boundary used only by research sessions.
|
|
3583
|
+
|
|
3584
|
+
Cloud Run does not delegate a writable cgroup subtree to the service. Instead, one session is
|
|
3585
|
+
the sole tenant of one instance and the platform's cgroup is the hard aggregate ceiling for
|
|
3586
|
+
the coordinator, parser, and every descendant. The same credentials that reach the parser
|
|
3587
|
+
must be unable to change that ceiling, create a child cgroup, or migrate a process elsewhere.
|
|
3588
|
+
A networkless seccomp filter is installed separately before the parser is released to exec.
|
|
3589
|
+
"""
|
|
3590
|
+
|
|
3591
|
+
if (
|
|
3592
|
+
type(expected_memory_bytes) is not int
|
|
3593
|
+
or expected_memory_bytes != HOSTED_SESSION_CONTAINER_MEMORY_BYTES
|
|
3594
|
+
):
|
|
3595
|
+
raise AcquisitionSecurityError(
|
|
3596
|
+
"SANDBOX_MEMORY_BOUNDARY",
|
|
3597
|
+
"hosted-session memory authority must name the exact 4 GiB service ceiling",
|
|
3598
|
+
)
|
|
3599
|
+
if sys.platform != "linux" or platform.machine().lower() not in {"x86_64", "amd64"}:
|
|
3600
|
+
raise AcquisitionSecurityError(
|
|
3601
|
+
"SANDBOX_MEMORY_BOUNDARY",
|
|
3602
|
+
"hosted-session container confinement requires Linux/amd64",
|
|
3603
|
+
)
|
|
3604
|
+
selected: (
|
|
3605
|
+
tuple[
|
|
3606
|
+
int,
|
|
3607
|
+
os.stat_result,
|
|
3608
|
+
str,
|
|
3609
|
+
str,
|
|
3610
|
+
tuple[str, ...],
|
|
3611
|
+
tuple[str, ...],
|
|
3612
|
+
]
|
|
3613
|
+
| None
|
|
3614
|
+
) = None
|
|
3615
|
+
last_error: BaseException | None = None
|
|
3616
|
+
layouts = (
|
|
3617
|
+
(_HOSTED_SESSION_CGROUP_ROOT, "memory.max", ("cgroup.procs",), ("cgroup.subtree_control",)),
|
|
3618
|
+
(
|
|
3619
|
+
_HOSTED_SESSION_CGROUP_ROOT / "memory",
|
|
3620
|
+
"memory.limit_in_bytes",
|
|
3621
|
+
("cgroup.procs", "tasks"),
|
|
3622
|
+
(),
|
|
3623
|
+
),
|
|
3624
|
+
)
|
|
3625
|
+
for root, limit_name, member_names, extra_immutable in layouts:
|
|
3626
|
+
root_fd = -1
|
|
3627
|
+
try:
|
|
3628
|
+
root_fd = os.open(
|
|
3629
|
+
root,
|
|
3630
|
+
os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC | os.O_NOFOLLOW,
|
|
3631
|
+
)
|
|
3632
|
+
metadata = os.fstat(root_fd)
|
|
3633
|
+
raw_limit = _read_control(root_fd, limit_name).strip()
|
|
3634
|
+
except (OSError, UnicodeError) as error:
|
|
3635
|
+
last_error = error
|
|
3636
|
+
if root_fd >= 0:
|
|
3637
|
+
os.close(root_fd)
|
|
3638
|
+
continue
|
|
3639
|
+
selected = (
|
|
3640
|
+
root_fd,
|
|
3641
|
+
metadata,
|
|
3642
|
+
raw_limit,
|
|
3643
|
+
limit_name,
|
|
3644
|
+
member_names,
|
|
3645
|
+
extra_immutable,
|
|
3646
|
+
)
|
|
3647
|
+
break
|
|
3648
|
+
if selected is None:
|
|
3649
|
+
raise AcquisitionSecurityError(
|
|
3650
|
+
"SANDBOX_MEMORY_BOUNDARY",
|
|
3651
|
+
"hosted-session container cgroup is unavailable or unreadable",
|
|
3652
|
+
) from last_error
|
|
3653
|
+
root_fd, metadata, raw_limit, limit_name, member_names, extra_immutable = selected
|
|
3654
|
+
try:
|
|
3655
|
+
member_name: str | None = None
|
|
3656
|
+
members: set[int] | None = None
|
|
3657
|
+
for candidate in member_names:
|
|
3658
|
+
try:
|
|
3659
|
+
raw_members = _read_control(root_fd, candidate)
|
|
3660
|
+
except (OSError, UnicodeError):
|
|
3661
|
+
continue
|
|
3662
|
+
try:
|
|
3663
|
+
members = {int(value) for value in raw_members.split()}
|
|
3664
|
+
except ValueError as error:
|
|
3665
|
+
raise AcquisitionSecurityError(
|
|
3666
|
+
"SANDBOX_MEMORY_BOUNDARY",
|
|
3667
|
+
"hosted-session container process membership is malformed",
|
|
3668
|
+
) from error
|
|
3669
|
+
member_name = candidate
|
|
3670
|
+
break
|
|
3671
|
+
if member_name is None or members is None:
|
|
3672
|
+
raise AcquisitionSecurityError(
|
|
3673
|
+
"SANDBOX_MEMORY_BOUNDARY",
|
|
3674
|
+
"hosted-session container process membership is unavailable",
|
|
3675
|
+
)
|
|
3676
|
+
if not stat.S_ISDIR(metadata.st_mode):
|
|
3677
|
+
raise AcquisitionSecurityError(
|
|
3678
|
+
"SANDBOX_MEMORY_BOUNDARY",
|
|
3679
|
+
"hosted-session memory authority is not a cgroup directory",
|
|
3680
|
+
)
|
|
3681
|
+
if raw_limit == "max":
|
|
3682
|
+
raise AcquisitionSecurityError(
|
|
3683
|
+
"SANDBOX_MEMORY_BOUNDARY",
|
|
3684
|
+
"hosted-session container cgroup has no finite memory ceiling",
|
|
3685
|
+
)
|
|
3686
|
+
if not raw_limit.isascii() or not raw_limit.isdecimal():
|
|
3687
|
+
raise AcquisitionSecurityError(
|
|
3688
|
+
"SANDBOX_MEMORY_BOUNDARY",
|
|
3689
|
+
"hosted-session container memory ceiling is not an unsigned byte count",
|
|
3690
|
+
)
|
|
3691
|
+
if int(raw_limit) != expected_memory_bytes:
|
|
3692
|
+
raise AcquisitionSecurityError(
|
|
3693
|
+
"SANDBOX_MEMORY_BOUNDARY",
|
|
3694
|
+
"hosted-session container memory ceiling does not match the exact service class",
|
|
3695
|
+
)
|
|
3696
|
+
if os.getpid() not in members:
|
|
3697
|
+
raise AcquisitionSecurityError(
|
|
3698
|
+
"SANDBOX_MEMORY_BOUNDARY",
|
|
3699
|
+
"hosted-session coordinator is not in the bounded container cgroup",
|
|
3700
|
+
)
|
|
3701
|
+
descriptor_path = f"/proc/self/fd/{root_fd}"
|
|
3702
|
+
if os.access(descriptor_path, os.W_OK, effective_ids=True) or any(
|
|
3703
|
+
_control_can_be_opened_for_write(root_fd, control)
|
|
3704
|
+
for control in (
|
|
3705
|
+
limit_name,
|
|
3706
|
+
*member_names,
|
|
3707
|
+
*extra_immutable,
|
|
3708
|
+
)
|
|
3709
|
+
):
|
|
3710
|
+
raise AcquisitionSecurityError(
|
|
3711
|
+
"SANDBOX_MEMORY_BOUNDARY",
|
|
3712
|
+
"hosted-session container cgroup must be immutable to worker credentials",
|
|
3713
|
+
)
|
|
3714
|
+
finally:
|
|
3715
|
+
os.close(root_fd)
|
|
3716
|
+
|
|
3717
|
+
|
|
3718
|
+
def _require_bounded_cgroup_root(root_fd: int) -> None:
|
|
3719
|
+
"""Validate one empty, operation-scoped, hierarchically bounded cgroup-v2 root.
|
|
3720
|
+
|
|
3721
|
+
The coordinator and worker share Unix credentials, so the worker can exercise the root's
|
|
3722
|
+
process-migration and child-creation delegation too. Containment therefore lives on the root
|
|
3723
|
+
itself: immutable memory/pid ceilings apply to every descendant, depth and descendant counts
|
|
3724
|
+
admit one protected coordinator plus exactly one job child, the no-internal-process rule keeps
|
|
3725
|
+
the worker out of the root, and the non-writable parent prevents escape above it.
|
|
3726
|
+
"""
|
|
3727
|
+
|
|
3728
|
+
if type(root_fd) is not int or root_fd < 0:
|
|
3729
|
+
raise AcquisitionSecurityError(
|
|
3730
|
+
"SANDBOX_MEMORY_BOUNDARY",
|
|
3731
|
+
"Linux memory authority must be an open bounded cgroup root descriptor",
|
|
3732
|
+
)
|
|
3733
|
+
try:
|
|
3734
|
+
metadata = os.fstat(root_fd)
|
|
3735
|
+
controllers = set(_read_control(root_fd, "cgroup.controllers").split())
|
|
3736
|
+
enabled = set(_read_control(root_fd, "cgroup.subtree_control").split())
|
|
3737
|
+
limits = {
|
|
3738
|
+
"memory.max": _read_control(root_fd, "memory.max").strip(),
|
|
3739
|
+
"memory.swap.max": _read_control(root_fd, "memory.swap.max").strip(),
|
|
3740
|
+
"pids.max": _read_control(root_fd, "pids.max").strip(),
|
|
3741
|
+
"cgroup.max.depth": _read_control(root_fd, "cgroup.max.depth").strip(),
|
|
3742
|
+
"cgroup.max.descendants": _read_control(root_fd, "cgroup.max.descendants").strip(),
|
|
3743
|
+
}
|
|
3744
|
+
cgroup_stat = dict(
|
|
3745
|
+
line.split() for line in _read_control(root_fd, "cgroup.stat").splitlines()
|
|
3746
|
+
)
|
|
3747
|
+
except (OSError, UnicodeError) as error:
|
|
3748
|
+
raise AcquisitionSecurityError(
|
|
3749
|
+
"SANDBOX_MEMORY_BOUNDARY",
|
|
3750
|
+
"Linux memory authority is not a readable bounded cgroup-v2 root",
|
|
3751
|
+
) from error
|
|
3752
|
+
expected = {
|
|
3753
|
+
"memory.max": str(MAX_WORKER_MEMORY_BYTES),
|
|
3754
|
+
"memory.swap.max": "0",
|
|
3755
|
+
"pids.max": str(MAX_WORKER_PIDS),
|
|
3756
|
+
"cgroup.max.depth": "1",
|
|
3757
|
+
"cgroup.max.descendants": "2",
|
|
3758
|
+
}
|
|
3759
|
+
if not stat.S_ISDIR(metadata.st_mode) or not {"memory", "pids"} <= controllers | enabled:
|
|
3760
|
+
raise AcquisitionSecurityError(
|
|
3761
|
+
"SANDBOX_MEMORY_BOUNDARY",
|
|
3762
|
+
"bounded cgroup root must expose the memory and pids controllers",
|
|
3763
|
+
)
|
|
3764
|
+
if limits != expected or not {"memory", "pids"} <= enabled:
|
|
3765
|
+
raise AcquisitionSecurityError(
|
|
3766
|
+
"SANDBOX_MEMORY_BOUNDARY",
|
|
3767
|
+
"bounded cgroup root must carry immutable hierarchy, memory, swap, and process limits",
|
|
3768
|
+
)
|
|
3769
|
+
descriptor_path = f"/proc/self/fd/{root_fd}"
|
|
3770
|
+
if not os.access(descriptor_path, os.W_OK, effective_ids=True):
|
|
3771
|
+
raise AcquisitionSecurityError(
|
|
3772
|
+
"SANDBOX_MEMORY_BOUNDARY",
|
|
3773
|
+
"bounded cgroup root must grant child-creation authority",
|
|
3774
|
+
)
|
|
3775
|
+
immutable_controls = (*expected, "cgroup.subtree_control")
|
|
3776
|
+
if any(_control_can_be_opened_for_write(root_fd, name) for name in immutable_controls):
|
|
3777
|
+
raise AcquisitionSecurityError(
|
|
3778
|
+
"SANDBOX_MEMORY_BOUNDARY",
|
|
3779
|
+
"bounded cgroup root limits must not be writable by worker credentials",
|
|
3780
|
+
)
|
|
3781
|
+
if not _control_can_be_opened_for_write(root_fd, "cgroup.procs"):
|
|
3782
|
+
raise AcquisitionSecurityError(
|
|
3783
|
+
"SANDBOX_MEMORY_BOUNDARY",
|
|
3784
|
+
"bounded cgroup root must grant process-migration authority",
|
|
3785
|
+
)
|
|
3786
|
+
if _control_can_be_opened_for_write(root_fd, "../cgroup.procs"):
|
|
3787
|
+
raise AcquisitionSecurityError(
|
|
3788
|
+
"SANDBOX_MEMORY_BOUNDARY",
|
|
3789
|
+
"worker credentials must not be able to leave the bounded cgroup root",
|
|
3790
|
+
)
|
|
3791
|
+
if _read_control(root_fd, "cgroup.procs").split() or cgroup_stat.get("nr_descendants") != "1":
|
|
3792
|
+
raise AcquisitionSecurityError(
|
|
3793
|
+
"SANDBOX_MEMORY_BOUNDARY",
|
|
3794
|
+
"bounded cgroup root must contain only its operator-owned coordinator child",
|
|
3795
|
+
)
|
|
3796
|
+
coordinator_fd = -1
|
|
3797
|
+
try:
|
|
3798
|
+
coordinator_fd = os.open(
|
|
3799
|
+
"coordinator",
|
|
3800
|
+
os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC | os.O_NOFOLLOW,
|
|
3801
|
+
dir_fd=root_fd,
|
|
3802
|
+
)
|
|
3803
|
+
coordinator_members = {
|
|
3804
|
+
int(value) for value in _read_control(coordinator_fd, "cgroup.procs").split()
|
|
3805
|
+
}
|
|
3806
|
+
if os.getpid() not in coordinator_members or _control_can_be_opened_for_write(
|
|
3807
|
+
coordinator_fd, "cgroup.procs"
|
|
3808
|
+
):
|
|
3809
|
+
raise AcquisitionSecurityError(
|
|
3810
|
+
"SANDBOX_MEMORY_BOUNDARY",
|
|
3811
|
+
"the trusted coordinator must run in a non-writable sibling cgroup",
|
|
3812
|
+
)
|
|
3813
|
+
except OSError as error:
|
|
3814
|
+
raise AcquisitionSecurityError(
|
|
3815
|
+
"SANDBOX_MEMORY_BOUNDARY",
|
|
3816
|
+
"bounded cgroup root has no protected coordinator child",
|
|
3817
|
+
) from error
|
|
3818
|
+
finally:
|
|
3819
|
+
if coordinator_fd >= 0:
|
|
3820
|
+
os.close(coordinator_fd)
|
|
3821
|
+
|
|
3822
|
+
|
|
3823
|
+
class _CgroupJob:
|
|
3824
|
+
def __init__(self, *, root_fd: int, name: str, job_fd: int) -> None:
|
|
3825
|
+
self.root_fd = root_fd
|
|
3826
|
+
self.name = name
|
|
3827
|
+
self.job_fd = job_fd
|
|
3828
|
+
self._root_oom_before = self._oom_events(root_fd)
|
|
3829
|
+
|
|
3830
|
+
@classmethod
|
|
3831
|
+
def create(cls, root_fd: int, pid: int) -> _CgroupJob:
|
|
3832
|
+
name = f"mostlyright-sandbox-{pid}-{secrets.token_hex(8)}"
|
|
3833
|
+
duplicated_root = os.dup(root_fd)
|
|
3834
|
+
job_fd = -1
|
|
3835
|
+
leaf_created = False
|
|
3836
|
+
job: _CgroupJob | None = None
|
|
3837
|
+
try:
|
|
3838
|
+
os.mkdir(name, mode=0o700, dir_fd=duplicated_root)
|
|
3839
|
+
leaf_created = True
|
|
3840
|
+
job_fd = os.open(
|
|
3841
|
+
name,
|
|
3842
|
+
os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC | os.O_NOFOLLOW,
|
|
3843
|
+
dir_fd=duplicated_root,
|
|
3844
|
+
)
|
|
3845
|
+
# Construction reads root telemetry. It belongs inside the ownership scope too: an
|
|
3846
|
+
# OSError -- or a process-level BaseException -- after the leaf and its descriptor
|
|
3847
|
+
# exist must not strand either merely because no `_CgroupJob` was returned to the
|
|
3848
|
+
# caller yet.
|
|
3849
|
+
job = cls(root_fd=duplicated_root, name=name, job_fd=job_fd)
|
|
3850
|
+
job._configure(pid)
|
|
3851
|
+
except BaseException:
|
|
3852
|
+
if job is not None:
|
|
3853
|
+
job._best_effort_remove()
|
|
3854
|
+
else:
|
|
3855
|
+
# These are independent owned resources. A failing close or removal must not
|
|
3856
|
+
# prevent the later cleanup attempts, and none may replace the construction
|
|
3857
|
+
# failure being re-raised below.
|
|
3858
|
+
if job_fd >= 0:
|
|
3859
|
+
try:
|
|
3860
|
+
os.close(job_fd)
|
|
3861
|
+
except BaseException:
|
|
3862
|
+
pass
|
|
3863
|
+
if leaf_created:
|
|
3864
|
+
try:
|
|
3865
|
+
os.rmdir(name, dir_fd=duplicated_root)
|
|
3866
|
+
except BaseException:
|
|
3867
|
+
pass
|
|
3868
|
+
try:
|
|
3869
|
+
os.close(duplicated_root)
|
|
3870
|
+
except BaseException:
|
|
3871
|
+
pass
|
|
3872
|
+
raise
|
|
3873
|
+
assert job is not None
|
|
3874
|
+
return job
|
|
3875
|
+
|
|
3876
|
+
def _configure(self, pid: int) -> None:
|
|
3877
|
+
expected = {
|
|
3878
|
+
"memory.max": str(MAX_WORKER_MEMORY_BYTES),
|
|
3879
|
+
"memory.swap.max": "0",
|
|
3880
|
+
"pids.max": str(MAX_WORKER_PIDS),
|
|
3881
|
+
}
|
|
3882
|
+
for control, value in expected.items():
|
|
3883
|
+
_write_control(self.job_fd, control, value)
|
|
3884
|
+
if _read_control(self.job_fd, control).strip() != value:
|
|
3885
|
+
raise OSError(f"cgroup refused {control}")
|
|
3886
|
+
_write_control(self.job_fd, "cgroup.procs", str(pid))
|
|
3887
|
+
members = {int(value) for value in _read_control(self.job_fd, "cgroup.procs").split()}
|
|
3888
|
+
if pid not in members:
|
|
3889
|
+
raise OSError("worker was not attached to its cgroup before release")
|
|
3890
|
+
|
|
3891
|
+
@staticmethod
|
|
3892
|
+
def _oom_events(directory_fd: int) -> int:
|
|
3893
|
+
events = {
|
|
3894
|
+
key: int(value)
|
|
3895
|
+
for key, value in (
|
|
3896
|
+
line.split() for line in _read_control(directory_fd, "memory.events").splitlines()
|
|
3897
|
+
)
|
|
3898
|
+
}
|
|
3899
|
+
return max(events.get("oom", 0), events.get("oom_kill", 0))
|
|
3900
|
+
|
|
3901
|
+
def memory_limit_hit(self) -> bool:
|
|
3902
|
+
return (
|
|
3903
|
+
self._oom_events(self.job_fd) > 0
|
|
3904
|
+
or self._oom_events(self.root_fd) > self._root_oom_before
|
|
3905
|
+
)
|
|
3906
|
+
|
|
3907
|
+
def cleanup(self) -> None:
|
|
3908
|
+
failure: BaseException | None = None
|
|
3909
|
+
|
|
3910
|
+
def remember(error: BaseException) -> None:
|
|
3911
|
+
nonlocal failure
|
|
3912
|
+
if failure is None:
|
|
3913
|
+
failure = error
|
|
3914
|
+
|
|
3915
|
+
def kill_members() -> None:
|
|
3916
|
+
try:
|
|
3917
|
+
members = _read_control(self.job_fd, "cgroup.procs").split()
|
|
3918
|
+
except BaseException as error:
|
|
3919
|
+
remember(error)
|
|
3920
|
+
return
|
|
3921
|
+
for pid_text in members:
|
|
3922
|
+
try:
|
|
3923
|
+
os.kill(int(pid_text), signal.SIGKILL)
|
|
3924
|
+
except ProcessLookupError:
|
|
3925
|
+
pass
|
|
3926
|
+
except BaseException as error:
|
|
3927
|
+
remember(error)
|
|
3928
|
+
|
|
3929
|
+
populated: bool | None = None
|
|
3930
|
+
try:
|
|
3931
|
+
populated = bool(
|
|
3932
|
+
"populated 0" not in _read_control(self.job_fd, "cgroup.events").splitlines()
|
|
3933
|
+
)
|
|
3934
|
+
except BaseException as error:
|
|
3935
|
+
remember(error)
|
|
3936
|
+
|
|
3937
|
+
# Unknown telemetry is not evidence that the leaf is empty. Both kill mechanisms are
|
|
3938
|
+
# attempted independently: cgroup.kill may be absent on an older kernel, while reading
|
|
3939
|
+
# cgroup.procs or killing one member may itself be the operation that was interrupted.
|
|
3940
|
+
if populated is not False:
|
|
3941
|
+
deadline: float | None = None
|
|
3942
|
+
try:
|
|
3943
|
+
deadline = time.monotonic() + _CGROUP_CLEANUP_TIMEOUT_SECONDS
|
|
3944
|
+
except BaseException as error:
|
|
3945
|
+
remember(error)
|
|
3946
|
+
# The attempt cap is an independent bound when clock access itself is interrupted.
|
|
3947
|
+
# When the clock works, the earlier of the wall-clock deadline and this cap wins.
|
|
3948
|
+
attempts_left = max(1, int(_CGROUP_CLEANUP_TIMEOUT_SECONDS / 0.01) + 1)
|
|
3949
|
+
while attempts_left > 0:
|
|
3950
|
+
attempts_left -= 1
|
|
3951
|
+
# Repeat both kill mechanisms: a descendant can appear after one procs
|
|
3952
|
+
# snapshot, and cgroup.kill is optional on older cgroup-v2 kernels.
|
|
3953
|
+
try:
|
|
3954
|
+
_write_control(self.job_fd, "cgroup.kill", "1")
|
|
3955
|
+
except OSError:
|
|
3956
|
+
pass
|
|
3957
|
+
except BaseException as error:
|
|
3958
|
+
remember(error)
|
|
3959
|
+
kill_members()
|
|
3960
|
+
try:
|
|
3961
|
+
populated = (
|
|
3962
|
+
"populated 0"
|
|
3963
|
+
not in _read_control(self.job_fd, "cgroup.events").splitlines()
|
|
3964
|
+
)
|
|
3965
|
+
except BaseException as error:
|
|
3966
|
+
remember(error)
|
|
3967
|
+
populated = None
|
|
3968
|
+
if populated is False:
|
|
3969
|
+
break
|
|
3970
|
+
timed_out = attempts_left == 0
|
|
3971
|
+
if deadline is not None:
|
|
3972
|
+
try:
|
|
3973
|
+
timed_out = timed_out or time.monotonic() >= deadline
|
|
3974
|
+
except BaseException as error:
|
|
3975
|
+
remember(error)
|
|
3976
|
+
if timed_out:
|
|
3977
|
+
remember(OSError("cgroup descendants survived cleanup"))
|
|
3978
|
+
break
|
|
3979
|
+
try:
|
|
3980
|
+
time.sleep(0.01)
|
|
3981
|
+
except BaseException as error:
|
|
3982
|
+
remember(error)
|
|
3983
|
+
continue
|
|
3984
|
+
try:
|
|
3985
|
+
os.close(self.job_fd)
|
|
3986
|
+
except BaseException as error:
|
|
3987
|
+
remember(error)
|
|
3988
|
+
try:
|
|
3989
|
+
os.rmdir(self.name, dir_fd=self.root_fd)
|
|
3990
|
+
except BaseException as error:
|
|
3991
|
+
remember(error)
|
|
3992
|
+
try:
|
|
3993
|
+
os.close(self.root_fd)
|
|
3994
|
+
except BaseException as error:
|
|
3995
|
+
remember(error)
|
|
3996
|
+
if failure is not None:
|
|
3997
|
+
raise failure
|
|
3998
|
+
|
|
3999
|
+
def _best_effort_remove(self) -> None:
|
|
4000
|
+
try:
|
|
4001
|
+
self.cleanup()
|
|
4002
|
+
except BaseException:
|
|
4003
|
+
pass
|
|
4004
|
+
|
|
4005
|
+
|
|
4006
|
+
def _close_child_descriptors_except(keep: set[int]) -> None:
|
|
4007
|
+
try:
|
|
4008
|
+
descriptors = [int(name) for name in os.listdir("/proc/self/fd") if name.isdigit()]
|
|
4009
|
+
except OSError:
|
|
4010
|
+
soft_limit, _hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE)
|
|
4011
|
+
descriptors = list(range(3, min(int(soft_limit), 65_536)))
|
|
4012
|
+
for descriptor in descriptors:
|
|
4013
|
+
if descriptor not in keep:
|
|
4014
|
+
try:
|
|
4015
|
+
os.close(descriptor)
|
|
4016
|
+
except OSError:
|
|
4017
|
+
pass
|
|
4018
|
+
|
|
4019
|
+
|
|
4020
|
+
def _start_linux_barrier_worker(
|
|
4021
|
+
command: list[str],
|
|
4022
|
+
*,
|
|
4023
|
+
cwd: Path,
|
|
4024
|
+
env: dict[str, str],
|
|
4025
|
+
hosted_session_networkless: bool = False,
|
|
4026
|
+
) -> tuple[_ForkedWorker, int, int]:
|
|
4027
|
+
"""Fork a stopped-before-exec child; return it plus ready/release pipe ends."""
|
|
4028
|
+
|
|
4029
|
+
stdin_read, stdin_write = os.pipe2(os.O_CLOEXEC)
|
|
4030
|
+
stdout_read, stdout_write = os.pipe2(os.O_CLOEXEC)
|
|
4031
|
+
stderr_read, stderr_write = os.pipe2(os.O_CLOEXEC)
|
|
4032
|
+
ready_read, ready_write = os.pipe2(os.O_CLOEXEC)
|
|
4033
|
+
release_read, release_write = os.pipe2(os.O_CLOEXEC)
|
|
4034
|
+
nproc_limit = _nproc_limit()
|
|
4035
|
+
try:
|
|
4036
|
+
pid = os.fork()
|
|
4037
|
+
except BaseException:
|
|
4038
|
+
for descriptor in (
|
|
4039
|
+
stdin_read,
|
|
4040
|
+
stdin_write,
|
|
4041
|
+
stdout_read,
|
|
4042
|
+
stdout_write,
|
|
4043
|
+
stderr_read,
|
|
4044
|
+
stderr_write,
|
|
4045
|
+
ready_read,
|
|
4046
|
+
ready_write,
|
|
4047
|
+
release_read,
|
|
4048
|
+
release_write,
|
|
4049
|
+
):
|
|
4050
|
+
os.close(descriptor)
|
|
4051
|
+
raise
|
|
4052
|
+
if pid == 0:
|
|
4053
|
+
try:
|
|
4054
|
+
os.close(stdin_write)
|
|
4055
|
+
os.close(stdout_read)
|
|
4056
|
+
os.close(stderr_read)
|
|
4057
|
+
os.close(ready_read)
|
|
4058
|
+
os.close(release_write)
|
|
4059
|
+
os.setsid()
|
|
4060
|
+
os.dup2(stdin_read, 0)
|
|
4061
|
+
os.dup2(stdout_write, 1)
|
|
4062
|
+
os.dup2(stderr_write, 2)
|
|
4063
|
+
_close_child_descriptors_except({0, 1, 2, ready_write, release_read})
|
|
4064
|
+
_apply_resource_limits(nproc_limit)
|
|
4065
|
+
if hosted_session_networkless:
|
|
4066
|
+
_install_networkless_seccomp(allow_descendants=True)
|
|
4067
|
+
os.write(ready_write, b"R")
|
|
4068
|
+
if os.read(release_read, 1) != b"G":
|
|
4069
|
+
os._exit(126)
|
|
4070
|
+
os.close(ready_write)
|
|
4071
|
+
os.close(release_read)
|
|
4072
|
+
os.chdir(cwd)
|
|
4073
|
+
os.execve(command[0], command, env)
|
|
4074
|
+
except BaseException:
|
|
4075
|
+
try:
|
|
4076
|
+
os.write(2, b"sandbox worker failed before exec\n")
|
|
4077
|
+
except OSError:
|
|
4078
|
+
pass
|
|
4079
|
+
os._exit(127)
|
|
4080
|
+
os.close(stdin_read)
|
|
4081
|
+
os.close(stdout_write)
|
|
4082
|
+
os.close(stderr_write)
|
|
4083
|
+
os.close(ready_write)
|
|
4084
|
+
os.close(release_read)
|
|
4085
|
+
return (
|
|
4086
|
+
_ForkedWorker(
|
|
4087
|
+
pid=pid,
|
|
4088
|
+
stdin_fd=stdin_write,
|
|
4089
|
+
stdout_fd=stdout_read,
|
|
4090
|
+
stderr_fd=stderr_read,
|
|
4091
|
+
),
|
|
4092
|
+
ready_read,
|
|
4093
|
+
release_write,
|
|
4094
|
+
)
|
|
4095
|
+
|
|
4096
|
+
|
|
4097
|
+
def _wait_for_barrier(descriptor: int, timeout_seconds: float) -> None:
|
|
4098
|
+
selector = selectors.DefaultSelector()
|
|
4099
|
+
try:
|
|
4100
|
+
selector.register(descriptor, selectors.EVENT_READ)
|
|
4101
|
+
if not selector.select(timeout_seconds) or os.read(descriptor, 1) != b"R":
|
|
4102
|
+
raise OSError("The Clean room process did not stop before starting its job")
|
|
4103
|
+
finally:
|
|
4104
|
+
selector.close()
|
|
4105
|
+
os.close(descriptor)
|
|
4106
|
+
|
|
4107
|
+
|
|
4108
|
+
@dataclass
|
|
4109
|
+
class _PreparedWorker:
|
|
4110
|
+
"""One started, resource-limited child that has not been given a request yet.
|
|
4111
|
+
|
|
4112
|
+
The split between preparing a child and completing a request against it is what makes a
|
|
4113
|
+
warm pool expressible without a second launch path. Both paths build the identical
|
|
4114
|
+
``_PreparedWorker``; the only difference is when it is built. A cold invocation builds one
|
|
4115
|
+
and binds it in the same call, a warm one built it earlier and idle. Because there is one
|
|
4116
|
+
constructor, a warm child cannot drift away from a cold child's environment, descriptors,
|
|
4117
|
+
session, or resource limits -- there is no second place for it to drift in.
|
|
4118
|
+
|
|
4119
|
+
``deadline`` is the wall-clock budget a cold invocation already started spending on the
|
|
4120
|
+
barrier handshake, carried forward so the split does not silently widen the timeout. A
|
|
4121
|
+
warm child carries ``None``: its request budget starts when the request binds, not when
|
|
4122
|
+
the child was spawned, because the seconds it spent idle are not seconds the request had.
|
|
4123
|
+
"""
|
|
4124
|
+
|
|
4125
|
+
process: _WorkerProcess
|
|
4126
|
+
job: _CgroupJob | None
|
|
4127
|
+
deadline: float | None
|
|
4128
|
+
launch: _WorkerLaunch
|
|
4129
|
+
|
|
4130
|
+
|
|
4131
|
+
@dataclass(frozen=True)
|
|
4132
|
+
class _WorkerLaunch:
|
|
4133
|
+
"""The exact launch specification a child was started from.
|
|
4134
|
+
|
|
4135
|
+
A warm child may only be bound to a request whose launch specification is byte-identical
|
|
4136
|
+
to the one the child was started from. The boundary is the container and the cgroup, which
|
|
4137
|
+
are the same for every child of one coordinator, so the specification is what is left to
|
|
4138
|
+
compare. Comparing the whole of it rather than an operation name keeps the argument
|
|
4139
|
+
mechanical: nothing about a pooled child's confinement can differ from the confinement the
|
|
4140
|
+
binding request would have been given had it spawned its own. (Until ADR 0021 there
|
|
4141
|
+
was a second reason -- on darwin the Seatbelt profile was an argv element, so equal commands
|
|
4142
|
+
were equal kernel authority. That backend is gone; the comparison it motivated is not, because
|
|
4143
|
+
it is what makes a warm child indistinguishable from a fresh spawn.)
|
|
4144
|
+
"""
|
|
4145
|
+
|
|
4146
|
+
command: tuple[str, ...]
|
|
4147
|
+
cwd: Path
|
|
4148
|
+
env: tuple[tuple[str, str], ...]
|
|
4149
|
+
prewarm: bool
|
|
4150
|
+
|
|
4151
|
+
@property
|
|
4152
|
+
def confinement(self) -> tuple[tuple[str, ...], Path, tuple[tuple[str, str], ...]]:
|
|
4153
|
+
"""Everything about a child that a request may rely on: argv, directory, environment."""
|
|
4154
|
+
|
|
4155
|
+
return (self.command, self.cwd, self.env)
|
|
4156
|
+
|
|
4157
|
+
def argv(self) -> list[str]:
|
|
4158
|
+
"""The exact argv this child is exec'd with.
|
|
4159
|
+
|
|
4160
|
+
``--prewarm`` is appended rather than folded into ``command`` so the confinement a
|
|
4161
|
+
warm child was given and the confinement a binding request asks for stay comparable.
|
|
4162
|
+
It is the one enumerated difference between a warm child and a cold one, it is
|
|
4163
|
+
readable in ``ps``, and it grants nothing: the flag chooses when the parser imports
|
|
4164
|
+
happen and whether a readiness line is written, and the child's authority is decided
|
|
4165
|
+
entirely by the container boundary, the cgroup, the closed environment, and the request.
|
|
4166
|
+
"""
|
|
4167
|
+
|
|
4168
|
+
return [*self.command, "--prewarm"] if self.prewarm else list(self.command)
|
|
4169
|
+
|
|
4170
|
+
|
|
4171
|
+
def _prepare_worker(
|
|
4172
|
+
launch: _WorkerLaunch,
|
|
4173
|
+
*,
|
|
4174
|
+
timeout_seconds: float,
|
|
4175
|
+
memory_cgroup_root_fd: int | None,
|
|
4176
|
+
hosted_session_container_memory_bytes: int | None = None,
|
|
4177
|
+
track_deadline: bool,
|
|
4178
|
+
ready_timeout_seconds: float | None = None,
|
|
4179
|
+
) -> _PreparedWorker:
|
|
4180
|
+
"""Start one confined child and bring it to the point of reading its request.
|
|
4181
|
+
|
|
4182
|
+
``ready_timeout_seconds`` bounds only the pre-warm readiness wait, and a filler passes what
|
|
4183
|
+
is left of one budget for the whole fill rather than a fresh budget per child.
|
|
4184
|
+
"""
|
|
4185
|
+
|
|
4186
|
+
command = launch.argv()
|
|
4187
|
+
cwd = launch.cwd
|
|
4188
|
+
env = dict(launch.env)
|
|
4189
|
+
deadline = time.monotonic() + timeout_seconds
|
|
4190
|
+
if ready_timeout_seconds is None:
|
|
4191
|
+
ready_timeout_seconds = min(timeout_seconds, 30.0)
|
|
4192
|
+
if sys.platform != "linux":
|
|
4193
|
+
# No deadline is carried here, and that is the pre-existing behaviour rather than an
|
|
4194
|
+
# oversight: this platform has no barrier handshake, so nothing was spent before the
|
|
4195
|
+
# request, and the drain gets the whole timeout exactly as it did before the split.
|
|
4196
|
+
return _become_ready_if_asked(
|
|
4197
|
+
_PreparedWorker(
|
|
4198
|
+
process=_start_popen_worker(command, cwd=cwd, env=env),
|
|
4199
|
+
job=None,
|
|
4200
|
+
deadline=None,
|
|
4201
|
+
launch=launch,
|
|
4202
|
+
),
|
|
4203
|
+
timeout_seconds=ready_timeout_seconds,
|
|
4204
|
+
)
|
|
4205
|
+
if memory_cgroup_root_fd is not None and hosted_session_container_memory_bytes is not None:
|
|
4206
|
+
raise AcquisitionSecurityError(
|
|
4207
|
+
"SANDBOX_MEMORY_BOUNDARY",
|
|
4208
|
+
"delegated and hosted-session memory boundaries are mutually exclusive",
|
|
4209
|
+
)
|
|
4210
|
+
if memory_cgroup_root_fd is None and hosted_session_container_memory_bytes is None:
|
|
4211
|
+
raise AcquisitionSecurityError(
|
|
4212
|
+
"SANDBOX_MEMORY_BOUNDARY",
|
|
4213
|
+
"Linux Clean room execution requires a trusted bounded cgroup root",
|
|
4214
|
+
)
|
|
4215
|
+
if memory_cgroup_root_fd is not None:
|
|
4216
|
+
_require_bounded_cgroup_root(memory_cgroup_root_fd)
|
|
4217
|
+
else:
|
|
4218
|
+
assert hosted_session_container_memory_bytes is not None
|
|
4219
|
+
_require_hosted_session_container_boundary(hosted_session_container_memory_bytes)
|
|
4220
|
+
process, ready_fd, release_fd = _start_linux_barrier_worker(
|
|
4221
|
+
command,
|
|
4222
|
+
cwd=cwd,
|
|
4223
|
+
env=env,
|
|
4224
|
+
hosted_session_networkless=hosted_session_container_memory_bytes is not None,
|
|
4225
|
+
)
|
|
4226
|
+
job: _CgroupJob | None = None
|
|
4227
|
+
try:
|
|
4228
|
+
_wait_for_barrier(ready_fd, min(max(deadline - time.monotonic(), 0.0), 5.0))
|
|
4229
|
+
if memory_cgroup_root_fd is not None:
|
|
4230
|
+
job = _CgroupJob.create(memory_cgroup_root_fd, process.pid)
|
|
4231
|
+
if os.write(release_fd, b"G") != 1:
|
|
4232
|
+
raise OSError("The Clean room process could not be released to start its job")
|
|
4233
|
+
os.close(release_fd)
|
|
4234
|
+
release_fd = -1
|
|
4235
|
+
except BaseException as error:
|
|
4236
|
+
if release_fd >= 0:
|
|
4237
|
+
try:
|
|
4238
|
+
os.close(release_fd)
|
|
4239
|
+
except BaseException:
|
|
4240
|
+
pass
|
|
4241
|
+
try:
|
|
4242
|
+
_stop_worker(process)
|
|
4243
|
+
except BaseException:
|
|
4244
|
+
pass
|
|
4245
|
+
if job is not None:
|
|
4246
|
+
job._best_effort_remove()
|
|
4247
|
+
if not isinstance(error, Exception):
|
|
4248
|
+
raise error
|
|
4249
|
+
if isinstance(error, AcquisitionSecurityError):
|
|
4250
|
+
raise
|
|
4251
|
+
if hosted_session_container_memory_bytes is not None:
|
|
4252
|
+
raise AcquisitionSecurityError(
|
|
4253
|
+
"SANDBOX_OS_BOUNDARY",
|
|
4254
|
+
"hosted-session networkless process boundary could not be established before exec",
|
|
4255
|
+
) from error
|
|
4256
|
+
raise AcquisitionSecurityError(
|
|
4257
|
+
"SANDBOX_MEMORY_BOUNDARY",
|
|
4258
|
+
"Linux cgroup memory boundary could not be established before exec",
|
|
4259
|
+
) from error
|
|
4260
|
+
|
|
4261
|
+
return _become_ready_if_asked(
|
|
4262
|
+
_PreparedWorker(
|
|
4263
|
+
process=process,
|
|
4264
|
+
job=job,
|
|
4265
|
+
deadline=deadline if track_deadline else None,
|
|
4266
|
+
launch=launch,
|
|
4267
|
+
),
|
|
4268
|
+
timeout_seconds=ready_timeout_seconds,
|
|
4269
|
+
)
|
|
4270
|
+
|
|
4271
|
+
|
|
4272
|
+
def _become_ready_if_asked(
|
|
4273
|
+
prepared: _PreparedWorker,
|
|
4274
|
+
*,
|
|
4275
|
+
timeout_seconds: float,
|
|
4276
|
+
) -> _PreparedWorker:
|
|
4277
|
+
"""Wait for a pre-warm child's readiness line, discarding it whole if it never comes.
|
|
4278
|
+
|
|
4279
|
+
After exec and inside whatever boundary this platform establishes, so the child that
|
|
4280
|
+
announces readiness is the confined child and not something on its way to becoming one. A
|
|
4281
|
+
child that fails here is torn down completely -- process group and cgroup both -- because
|
|
4282
|
+
it was never handed out and nothing is waiting on its response.
|
|
4283
|
+
"""
|
|
4284
|
+
|
|
4285
|
+
if not prepared.launch.prewarm:
|
|
4286
|
+
return prepared
|
|
4287
|
+
try:
|
|
4288
|
+
_await_worker_ready(prepared.process, timeout_seconds=timeout_seconds)
|
|
4289
|
+
except BaseException:
|
|
4290
|
+
_discard_prepared_worker(prepared)
|
|
4291
|
+
raise
|
|
4292
|
+
return prepared
|
|
4293
|
+
|
|
4294
|
+
|
|
4295
|
+
def _worker_is_running(process: _WorkerProcess) -> bool:
|
|
4296
|
+
"""Whether a child is still alive, reaping it if it is not.
|
|
4297
|
+
|
|
4298
|
+
``wait`` with a zero budget is the one liveness question both worker shapes already answer:
|
|
4299
|
+
a returncode means it exited and has now been reaped, and a timeout means it is running.
|
|
4300
|
+
|
|
4301
|
+
Anything else answers neither, and the two possible guesses are not symmetric. Guessing
|
|
4302
|
+
"exited" sends the child down the teardown that does not signal it, which would leak a live
|
|
4303
|
+
process holding a cgroup. Guessing "running" sends it down the teardown that kills and
|
|
4304
|
+
reaps, which is correct for a live child and harmless for a dead one. So an unrecognised
|
|
4305
|
+
failure reports running. Interruptions are not answers at all and are re-raised.
|
|
4306
|
+
"""
|
|
4307
|
+
|
|
4308
|
+
try:
|
|
4309
|
+
process.wait(0.0)
|
|
4310
|
+
except subprocess.TimeoutExpired:
|
|
4311
|
+
return True
|
|
4312
|
+
except (KeyboardInterrupt, SystemExit):
|
|
4313
|
+
raise
|
|
4314
|
+
except BaseException:
|
|
4315
|
+
return True
|
|
4316
|
+
return False
|
|
4317
|
+
|
|
4318
|
+
|
|
4319
|
+
def _discard_prepared_worker(
|
|
4320
|
+
prepared: _PreparedWorker,
|
|
4321
|
+
*,
|
|
4322
|
+
already_exited: bool = False,
|
|
4323
|
+
) -> None:
|
|
4324
|
+
"""Tear down a prepared child that will never be given a request.
|
|
4325
|
+
|
|
4326
|
+
``already_exited`` changes the teardown rather than shortening it. A reaped child's pid is
|
|
4327
|
+
free for the kernel to hand to somebody else, so signalling its process group could reach
|
|
4328
|
+
whatever now holds that number; a child known to have exited therefore gets its
|
|
4329
|
+
descriptors closed and its cgroup removed, and nothing is signalled. Cgroup teardown is
|
|
4330
|
+
unaffected either way: it only ever reaches members of the leaf it owns.
|
|
4331
|
+
"""
|
|
4332
|
+
|
|
4333
|
+
process = prepared.process
|
|
4334
|
+
stdout_fd = process.stdout_fd
|
|
4335
|
+
stderr_fd = process.stderr_fd
|
|
4336
|
+
steps: tuple[Any, ...] = (
|
|
4337
|
+
(
|
|
4338
|
+
process.close_stdin,
|
|
4339
|
+
lambda: process.close_output(stdout_fd),
|
|
4340
|
+
lambda: process.close_output(stderr_fd),
|
|
4341
|
+
)
|
|
4342
|
+
if already_exited
|
|
4343
|
+
else (lambda: _stop_worker(process),)
|
|
4344
|
+
)
|
|
4345
|
+
for step in steps:
|
|
4346
|
+
try:
|
|
4347
|
+
step()
|
|
4348
|
+
except BaseException:
|
|
4349
|
+
pass
|
|
4350
|
+
if prepared.job is not None:
|
|
4351
|
+
try:
|
|
4352
|
+
prepared.job.cleanup()
|
|
4353
|
+
except BaseException:
|
|
4354
|
+
prepared.job._best_effort_remove()
|
|
4355
|
+
|
|
4356
|
+
|
|
4357
|
+
def _await_worker_ready(process: _WorkerProcess, *, timeout_seconds: float) -> None:
|
|
4358
|
+
"""Read exactly the readiness marker a pre-warmed child writes before its response.
|
|
4359
|
+
|
|
4360
|
+
Never more than the marker: the read length is capped at the bytes still outstanding, so
|
|
4361
|
+
this cannot consume a byte of the response the request will later produce, and the pump
|
|
4362
|
+
that drains that response sees the same stream it would have seen from a cold child.
|
|
4363
|
+
"""
|
|
4364
|
+
|
|
4365
|
+
selector = selectors.DefaultSelector()
|
|
4366
|
+
seen = bytearray()
|
|
4367
|
+
try:
|
|
4368
|
+
os.set_blocking(process.stdout_fd, False)
|
|
4369
|
+
selector.register(process.stdout_fd, selectors.EVENT_READ)
|
|
4370
|
+
deadline = time.monotonic() + timeout_seconds
|
|
4371
|
+
while len(seen) < len(WORKER_READY_MARKER):
|
|
4372
|
+
remaining = deadline - time.monotonic()
|
|
4373
|
+
if remaining <= 0 or not selector.select(remaining):
|
|
4374
|
+
raise AcquisitionSecurityError(
|
|
4375
|
+
"SANDBOX_WARM_POOL",
|
|
4376
|
+
"a warm Clean room process did not become ready in time",
|
|
4377
|
+
)
|
|
4378
|
+
try:
|
|
4379
|
+
chunk = os.read(process.stdout_fd, len(WORKER_READY_MARKER) - len(seen))
|
|
4380
|
+
except BlockingIOError:
|
|
4381
|
+
continue
|
|
4382
|
+
if not chunk:
|
|
4383
|
+
raise AcquisitionSecurityError(
|
|
4384
|
+
"SANDBOX_WARM_POOL",
|
|
4385
|
+
"a warm Clean room process ended before it became ready",
|
|
4386
|
+
)
|
|
4387
|
+
seen.extend(chunk)
|
|
4388
|
+
if bytes(seen) != WORKER_READY_MARKER:
|
|
4389
|
+
raise AcquisitionSecurityError(
|
|
4390
|
+
"SANDBOX_WARM_POOL",
|
|
4391
|
+
"a warm Clean room process did not announce readiness",
|
|
4392
|
+
)
|
|
4393
|
+
finally:
|
|
4394
|
+
selector.close()
|
|
4395
|
+
# Left exactly as a cold child's descriptor is found. The pump sets all three
|
|
4396
|
+
# non-blocking itself, so this changes no behaviour -- but "a warm child is
|
|
4397
|
+
# indistinguishable from a fresh spawn" should not rest on the next reader repeating
|
|
4398
|
+
# the state this one happened to leave behind.
|
|
4399
|
+
try:
|
|
4400
|
+
os.set_blocking(process.stdout_fd, True)
|
|
4401
|
+
except OSError:
|
|
4402
|
+
pass
|
|
4403
|
+
|
|
4404
|
+
|
|
4405
|
+
def _complete_worker(
|
|
4406
|
+
prepared: _PreparedWorker,
|
|
4407
|
+
*,
|
|
4408
|
+
request_bytes: bytes,
|
|
4409
|
+
timeout_seconds: float,
|
|
4410
|
+
terminate_process_group: bool = False,
|
|
4411
|
+
) -> tuple[bytes, bytes, int]:
|
|
4412
|
+
"""Give one prepared child its one request and collect its one response."""
|
|
4413
|
+
|
|
4414
|
+
process = prepared.process
|
|
4415
|
+
job = prepared.job
|
|
4416
|
+
deadline = prepared.deadline
|
|
4417
|
+
if deadline is None:
|
|
4418
|
+
deadline = time.monotonic() + timeout_seconds
|
|
4419
|
+
if job is None:
|
|
4420
|
+
return _drain_worker_ipc(
|
|
4421
|
+
process,
|
|
4422
|
+
request_bytes=request_bytes,
|
|
4423
|
+
timeout_seconds=max(deadline - time.monotonic(), 0.0),
|
|
4424
|
+
terminate_process_group=terminate_process_group,
|
|
4425
|
+
)
|
|
4426
|
+
result: tuple[bytes, bytes, int] | None = None
|
|
4427
|
+
primary_error: BaseException | None = None
|
|
4428
|
+
primary_stage: str | None = None
|
|
4429
|
+
memory_limit_hit = False
|
|
4430
|
+
try:
|
|
4431
|
+
try:
|
|
4432
|
+
result = _drain_worker_ipc(
|
|
4433
|
+
process,
|
|
4434
|
+
request_bytes=request_bytes,
|
|
4435
|
+
timeout_seconds=max(deadline - time.monotonic(), 0.0),
|
|
4436
|
+
terminate_process_group=terminate_process_group,
|
|
4437
|
+
)
|
|
4438
|
+
except BaseException as error:
|
|
4439
|
+
primary_error = error
|
|
4440
|
+
primary_stage = "drain"
|
|
4441
|
+
try:
|
|
4442
|
+
memory_limit_hit = job.memory_limit_hit()
|
|
4443
|
+
except BaseException as error:
|
|
4444
|
+
if primary_error is None:
|
|
4445
|
+
primary_error = error
|
|
4446
|
+
primary_stage = "telemetry"
|
|
4447
|
+
finally:
|
|
4448
|
+
try:
|
|
4449
|
+
job.cleanup()
|
|
4450
|
+
except BaseException as error:
|
|
4451
|
+
if primary_error is None:
|
|
4452
|
+
primary_error = error
|
|
4453
|
+
primary_stage = "cleanup"
|
|
4454
|
+
|
|
4455
|
+
if primary_error is not None:
|
|
4456
|
+
# Interruptions are control flow, not worker failures. Teardown above still owns every
|
|
4457
|
+
# cgroup resource, but the exact process-level exception must reach the caller unchanged.
|
|
4458
|
+
if not isinstance(primary_error, Exception):
|
|
4459
|
+
raise primary_error
|
|
4460
|
+
if primary_stage == "drain" and isinstance(primary_error, AcquisitionSecurityError):
|
|
4461
|
+
raise primary_error
|
|
4462
|
+
if primary_stage == "telemetry":
|
|
4463
|
+
raise AcquisitionSecurityError(
|
|
4464
|
+
"SANDBOX_MEMORY_BOUNDARY",
|
|
4465
|
+
"Linux cgroup memory telemetry could not be corroborated",
|
|
4466
|
+
) from primary_error
|
|
4467
|
+
if primary_stage == "cleanup":
|
|
4468
|
+
raise AcquisitionSecurityError(
|
|
4469
|
+
"SANDBOX_MEMORY_BOUNDARY",
|
|
4470
|
+
"Linux cgroup descendants could not be killed and removed",
|
|
4471
|
+
) from primary_error
|
|
4472
|
+
raise AcquisitionSecurityError(
|
|
4473
|
+
"SANDBOX_FAILURE",
|
|
4474
|
+
"The Clean room process could not be watched safely",
|
|
4475
|
+
) from primary_error
|
|
4476
|
+
if memory_limit_hit:
|
|
4477
|
+
raise AcquisitionSecurityError(
|
|
4478
|
+
"SANDBOX_MEMORY_LIMIT",
|
|
4479
|
+
"The Clean room process used more memory than this job allows",
|
|
4480
|
+
)
|
|
4481
|
+
if result is None:
|
|
4482
|
+
raise AcquisitionSecurityError(
|
|
4483
|
+
"SANDBOX_FAILURE", "The Clean room process returned no result"
|
|
4484
|
+
)
|
|
4485
|
+
return result
|
|
4486
|
+
|
|
4487
|
+
|
|
4488
|
+
def _run_worker_command(
|
|
4489
|
+
command: list[str],
|
|
4490
|
+
*,
|
|
4491
|
+
cwd: Path,
|
|
4492
|
+
env: dict[str, str],
|
|
4493
|
+
request_bytes: bytes,
|
|
4494
|
+
timeout_seconds: float,
|
|
4495
|
+
memory_cgroup_root_fd: int | None,
|
|
4496
|
+
hosted_session_container_memory_bytes: int | None = None,
|
|
4497
|
+
) -> tuple[bytes, bytes, int]:
|
|
4498
|
+
"""Spawn one child and run one request against it: the cold path, start to finish."""
|
|
4499
|
+
|
|
4500
|
+
launch = _WorkerLaunch(
|
|
4501
|
+
command=tuple(command),
|
|
4502
|
+
cwd=cwd,
|
|
4503
|
+
env=tuple(sorted(env.items())),
|
|
4504
|
+
prewarm=False,
|
|
4505
|
+
)
|
|
4506
|
+
prepared = _prepare_worker(
|
|
4507
|
+
launch,
|
|
4508
|
+
timeout_seconds=timeout_seconds,
|
|
4509
|
+
memory_cgroup_root_fd=memory_cgroup_root_fd,
|
|
4510
|
+
hosted_session_container_memory_bytes=hosted_session_container_memory_bytes,
|
|
4511
|
+
track_deadline=True,
|
|
4512
|
+
)
|
|
4513
|
+
return _complete_worker(
|
|
4514
|
+
prepared,
|
|
4515
|
+
request_bytes=request_bytes,
|
|
4516
|
+
timeout_seconds=timeout_seconds,
|
|
4517
|
+
terminate_process_group=hosted_session_container_memory_bytes is not None,
|
|
4518
|
+
)
|
|
4519
|
+
|
|
4520
|
+
|
|
4521
|
+
class _WarmSandboxPool:
|
|
4522
|
+
"""Pre-spawned, pre-imported children that have not been bound to a request.
|
|
4523
|
+
|
|
4524
|
+
Warm means spawned early and finished importing -- it never means reused. Every child
|
|
4525
|
+
here is handed out at most once, serves exactly the one request it is handed, and exits;
|
|
4526
|
+
:func:`worker_main` reads stdin once and returns, so there is no code path by which a
|
|
4527
|
+
second request could reach a child that has already served one. What pooling removes is
|
|
4528
|
+
the interpreter start and the module imports, which are the same work for every child and
|
|
4529
|
+
depend on nothing a request supplies.
|
|
4530
|
+
|
|
4531
|
+
A child sitting here holds no request, no credential, and no coordinator state: it was
|
|
4532
|
+
exec'd with the same closed environment and closed descriptor set as a cold child and is
|
|
4533
|
+
blocked reading an empty stdin. The one thing a warm child has that a cold child does not
|
|
4534
|
+
is time already spent, and time is not authority.
|
|
4535
|
+
|
|
4536
|
+
Every method that reads or writes ``_ready`` holds ``_lock``. Handing one child to two
|
|
4537
|
+
callers is the one way this class could break the invariant it exists to keep, and it is
|
|
4538
|
+
reachable without it: the liveness check releases the GIL, so two threads could select the
|
|
4539
|
+
same child between the scan and the reassignment. No caller is threaded today. The lock is
|
|
4540
|
+
here because the invariant is stated unconditionally, and an invariant that depends on the
|
|
4541
|
+
caller staying single-threaded is a comment rather than a guarantee.
|
|
4542
|
+
"""
|
|
4543
|
+
|
|
4544
|
+
def __init__(
|
|
4545
|
+
self,
|
|
4546
|
+
*,
|
|
4547
|
+
timeout_seconds: float,
|
|
4548
|
+
memory_cgroup_root_fd: int | None,
|
|
4549
|
+
hosted_session_container_memory_bytes: int | None = None,
|
|
4550
|
+
) -> None:
|
|
4551
|
+
self._timeout_seconds = timeout_seconds
|
|
4552
|
+
self._memory_cgroup_root_fd = memory_cgroup_root_fd
|
|
4553
|
+
self._hosted_session_container_memory_bytes = hosted_session_container_memory_bytes
|
|
4554
|
+
self._ready: list[_PreparedWorker] = []
|
|
4555
|
+
self._lock = threading.Lock()
|
|
4556
|
+
|
|
4557
|
+
def fill(self, launch: _WorkerLaunch, *, size: int) -> int:
|
|
4558
|
+
"""Bring the pool up to ``size`` live children ready for exactly ``launch``.
|
|
4559
|
+
|
|
4560
|
+
One readiness budget covers the whole call. A child that fails to start leaves the
|
|
4561
|
+
ones already started in the pool rather than unwinding them, so a caller that wraps
|
|
4562
|
+
this in a lifecycle still has something to close.
|
|
4563
|
+
"""
|
|
4564
|
+
|
|
4565
|
+
with self._lock:
|
|
4566
|
+
self._prune(launch)
|
|
4567
|
+
deadline = time.monotonic() + WARM_POOL_FILL_TIMEOUT_SECONDS
|
|
4568
|
+
while len(self._ready) < size:
|
|
4569
|
+
self._ready.append(
|
|
4570
|
+
_prepare_worker(
|
|
4571
|
+
launch,
|
|
4572
|
+
timeout_seconds=self._timeout_seconds,
|
|
4573
|
+
memory_cgroup_root_fd=self._memory_cgroup_root_fd,
|
|
4574
|
+
hosted_session_container_memory_bytes=(
|
|
4575
|
+
self._hosted_session_container_memory_bytes
|
|
4576
|
+
),
|
|
4577
|
+
track_deadline=False,
|
|
4578
|
+
ready_timeout_seconds=max(deadline - time.monotonic(), 0.0),
|
|
4579
|
+
)
|
|
4580
|
+
)
|
|
4581
|
+
return len(self._ready)
|
|
4582
|
+
|
|
4583
|
+
def take(self, launch: _WorkerLaunch) -> _PreparedWorker | None:
|
|
4584
|
+
"""Hand out one live child started from exactly ``launch``, or nothing.
|
|
4585
|
+
|
|
4586
|
+
Nothing is a complete answer: the caller spawns its own child, which is what it would
|
|
4587
|
+
have done anyway. A near-match is never a match, because the launch specification is
|
|
4588
|
+
the whole of a child's confinement.
|
|
4589
|
+
|
|
4590
|
+
A child that died while it was idle -- an operator kill, a host-level OOM -- is reaped
|
|
4591
|
+
here and skipped rather than handed out, so the pool cannot turn a dead process into a
|
|
4592
|
+
failed probe. That is the property the pool is meant to have: every way it can be
|
|
4593
|
+
unhelpful degrades to the cold path, and none of them degrade to an error.
|
|
4594
|
+
"""
|
|
4595
|
+
|
|
4596
|
+
with self._lock:
|
|
4597
|
+
remaining: list[_PreparedWorker] = []
|
|
4598
|
+
taken: _PreparedWorker | None = None
|
|
4599
|
+
for prepared in self._ready:
|
|
4600
|
+
if taken is not None or prepared.launch.confinement != launch.confinement:
|
|
4601
|
+
remaining.append(prepared)
|
|
4602
|
+
continue
|
|
4603
|
+
if _worker_is_running(prepared.process):
|
|
4604
|
+
taken = prepared
|
|
4605
|
+
continue
|
|
4606
|
+
_discard_prepared_worker(prepared, already_exited=True)
|
|
4607
|
+
self._ready = remaining
|
|
4608
|
+
return taken
|
|
4609
|
+
|
|
4610
|
+
def _prune(self, launch: _WorkerLaunch | None) -> None:
|
|
4611
|
+
"""Drop every child that is not live, or not launched from exactly ``launch``.
|
|
4612
|
+
|
|
4613
|
+
A refill has to count live children rather than remembered ones. Without this the
|
|
4614
|
+
pool would report a child an operator killed hours ago as ready, and topping up to
|
|
4615
|
+
``size`` would keep the corpse and add nothing. ``launch`` of ``None`` prunes on
|
|
4616
|
+
liveness alone, which is what a caller asking only for a count wants.
|
|
4617
|
+
|
|
4618
|
+
Callers hold ``_lock``.
|
|
4619
|
+
"""
|
|
4620
|
+
|
|
4621
|
+
keep: list[_PreparedWorker] = []
|
|
4622
|
+
for prepared in self._ready:
|
|
4623
|
+
if launch is not None and prepared.launch.confinement != launch.confinement:
|
|
4624
|
+
_discard_prepared_worker(prepared)
|
|
4625
|
+
elif _worker_is_running(prepared.process):
|
|
4626
|
+
keep.append(prepared)
|
|
4627
|
+
else:
|
|
4628
|
+
_discard_prepared_worker(prepared, already_exited=True)
|
|
4629
|
+
self._ready = keep
|
|
4630
|
+
|
|
4631
|
+
def ready_count(self) -> int:
|
|
4632
|
+
"""How many children are idle *and* still alive.
|
|
4633
|
+
|
|
4634
|
+
The liveness scan is the whole point: a count that includes children the host killed
|
|
4635
|
+
while they were idle is the misreport this method exists to avoid, and a caller sizing
|
|
4636
|
+
a refill from it would under-provision by exactly the number of corpses.
|
|
4637
|
+
"""
|
|
4638
|
+
|
|
4639
|
+
with self._lock:
|
|
4640
|
+
self._prune(None)
|
|
4641
|
+
return len(self._ready)
|
|
4642
|
+
|
|
4643
|
+
def close(self) -> None:
|
|
4644
|
+
with self._lock:
|
|
4645
|
+
ready, self._ready = self._ready, []
|
|
4646
|
+
for prepared in ready:
|
|
4647
|
+
_discard_prepared_worker(prepared)
|
|
4648
|
+
|
|
4649
|
+
|
|
4650
|
+
def _apply_resource_limits(nproc_limit: int) -> None:
|
|
4651
|
+
resource.setrlimit(resource.RLIMIT_CPU, (30, 30))
|
|
4652
|
+
resource.setrlimit(resource.RLIMIT_NOFILE, (64, 64))
|
|
4653
|
+
resource.setrlimit(resource.RLIMIT_FSIZE, (64 * 1024 * 1024, 64 * 1024 * 1024))
|
|
4654
|
+
if hasattr(resource, "RLIMIT_NPROC"):
|
|
4655
|
+
resource.setrlimit(resource.RLIMIT_NPROC, (nproc_limit, nproc_limit))
|
|
4656
|
+
|
|
4657
|
+
|
|
4658
|
+
def _nproc_limit() -> int:
|
|
4659
|
+
"""Pick the RLIMIT_NPROC cap in the coordinator before forking the worker.
|
|
4660
|
+
|
|
4661
|
+
macOS charges RLIMIT_NPROC only on fork, so an absolute cap of 32 blocks the
|
|
4662
|
+
worker from spawning processes without limiting its threads. Linux charges
|
|
4663
|
+
every task (process or thread) of the real UID system-wide against the cap,
|
|
4664
|
+
so the same absolute 32 makes the worker's first pthread_create fail on any
|
|
4665
|
+
host whose UID already runs that many tasks (CI runners do) and its BLAS
|
|
4666
|
+
runtime aborts. Linux therefore gets the UID's current tasks plus bounded
|
|
4667
|
+
headroom for the worker and its BLAS/Arrow thread pools: runaway task
|
|
4668
|
+
creation still fails closed, at the same order of magnitude as before.
|
|
4669
|
+
"""
|
|
4670
|
+
|
|
4671
|
+
if sys.platform != "linux":
|
|
4672
|
+
return 32
|
|
4673
|
+
limit = _linux_uid_task_count() + 32 + 4 * (os.cpu_count() or 1)
|
|
4674
|
+
_soft, hard = resource.getrlimit(resource.RLIMIT_NPROC)
|
|
4675
|
+
if hard != resource.RLIM_INFINITY:
|
|
4676
|
+
limit = min(limit, hard)
|
|
4677
|
+
return limit
|
|
4678
|
+
|
|
4679
|
+
|
|
4680
|
+
def _linux_uid_task_count() -> int:
|
|
4681
|
+
"""Count the real UID's current tasks via /proc.
|
|
4682
|
+
|
|
4683
|
+
Entries hidden by hidepid, owned by another UID, or gone mid-scan are
|
|
4684
|
+
skipped; undercounting only lowers the limit, which fails closed.
|
|
4685
|
+
"""
|
|
4686
|
+
|
|
4687
|
+
uid = os.getuid()
|
|
4688
|
+
tasks = 0
|
|
4689
|
+
try:
|
|
4690
|
+
with os.scandir("/proc") as entries:
|
|
4691
|
+
for entry in entries:
|
|
4692
|
+
if not entry.name.isdigit():
|
|
4693
|
+
continue
|
|
4694
|
+
try:
|
|
4695
|
+
if entry.stat(follow_symlinks=False).st_uid != uid:
|
|
4696
|
+
continue
|
|
4697
|
+
tasks += len(os.listdir(os.path.join(entry.path, "task")))
|
|
4698
|
+
except OSError:
|
|
4699
|
+
continue
|
|
4700
|
+
except OSError:
|
|
4701
|
+
return 0
|
|
4702
|
+
return tasks
|
|
4703
|
+
|
|
4704
|
+
|
|
4705
|
+
def _worker_error(code: str, detail: str) -> int:
|
|
4706
|
+
value = {"status": "error", "code": code, "detail": detail[:MAX_CARRIED_DETAIL]}
|
|
4707
|
+
sys.stderr.buffer.write(_ipc_json_bytes(value))
|
|
4708
|
+
sys.stderr.buffer.flush()
|
|
4709
|
+
return 1
|
|
4710
|
+
|
|
4711
|
+
|
|
4712
|
+
def _bounded_worker_error(raw: bytes, *, returncode: int) -> _WorkerRefusal:
|
|
4713
|
+
"""Recover the worker's refusal from a stderr stream it does not own alone.
|
|
4714
|
+
|
|
4715
|
+
The worker writes exactly one single-line JSON object, but the interpreter may have
|
|
4716
|
+
written to the same stream first: running ``-m`` on a module the package's ``__init__``
|
|
4717
|
+
has already imported emits a runpy ``RuntimeWarning``, and that one line was enough to
|
|
4718
|
+
make every fail-closed refusal arrive at the coordinator as "redacted", losing the code a
|
|
4719
|
+
person needs. The whole buffer is tried first and the last non-empty line second; a
|
|
4720
|
+
stream that yields neither is still redacted rather than guessed at.
|
|
4721
|
+
|
|
4722
|
+
**The sentence crosses too, and only for a Reader refusal.** A code alone was not enough:
|
|
4723
|
+
the weather family composes the pair of templates it refused, the constant to edit and the
|
|
4724
|
+
three-step fix entirely inside ``ReaderError.detail``, and none of those facts is
|
|
4725
|
+
recoverable from a static code-to-fix map, because the code is only ``READER_GRID_UNKNOWN``.
|
|
4726
|
+
Two documents promised the operator that sentence, so it is carried -- under the narrowest
|
|
4727
|
+
rule that delivers it:
|
|
4728
|
+
|
|
4729
|
+
* only when the code is in ``READER_ERROR_CODES``, the closed table a Reader may raise from.
|
|
4730
|
+
Every other code -- ``SANDBOX_*`` and anything a caller invented -- carries nothing, so
|
|
4731
|
+
no failure inside the confinement can describe the confinement;
|
|
4732
|
+
* only after ``_readable_detail`` accepts it, which bounds the length and admits printable
|
|
4733
|
+
single-line text and nothing else.
|
|
4734
|
+
|
|
4735
|
+
A Reader is handed bytes and never a path -- the container module asserts structurally that
|
|
4736
|
+
no filesystem-taking name appears in it -- so a Reader refusal has no staging path, no
|
|
4737
|
+
temporary directory and no environment to name. That is why this is the code set where
|
|
4738
|
+
carrying the sentence is safe, and it is asserted rather than assumed in
|
|
4739
|
+
``tests/h3/test_sandbox.py``.
|
|
4740
|
+
|
|
4741
|
+
When no candidate yields a code at all the refusal is still redacted, but it carries the
|
|
4742
|
+
fixed-shape ``_worker_exit_diagnostic``: trusted kernel exit status plus a one-way digest
|
|
4743
|
+
of the first stderr line. A worker killed by a signal, or one whose stderr never became
|
|
4744
|
+
JSON, is then distinguishable in logs without anything the worker wrote reaching a person
|
|
4745
|
+
verbatim.
|
|
4746
|
+
"""
|
|
4747
|
+
|
|
4748
|
+
lines = [line for line in raw.splitlines() if line.strip()]
|
|
4749
|
+
for candidate in (raw, lines[-1] if lines else b""):
|
|
4750
|
+
try:
|
|
4751
|
+
value = _strict_json_object(candidate, "sandbox.error")
|
|
4752
|
+
code = _bounded_text(value.get("code"), maximum=128)
|
|
4753
|
+
except AcquisitionSecurityError:
|
|
4754
|
+
continue
|
|
4755
|
+
detail = _readable_detail(value.get("detail")) if code in READER_ERROR_CODES else None
|
|
4756
|
+
return _WorkerRefusal(code, code if detail is None else f"{code}: {detail}")
|
|
4757
|
+
return _WorkerRefusal(
|
|
4758
|
+
None,
|
|
4759
|
+
f"worker returned a redacted failure ({_worker_exit_diagnostic(raw, returncode)})",
|
|
4760
|
+
)
|
|
4761
|
+
|
|
4762
|
+
|
|
4763
|
+
def _worker_exit_diagnostic(raw: bytes, returncode: int) -> str:
|
|
4764
|
+
"""Fixed-shape diagnostic from the kernel exit status and a stderr digest.
|
|
4765
|
+
|
|
4766
|
+
Only trusted kernel metadata and a one-way SHA-256 prefix of the first
|
|
4767
|
+
stderr line appear; nothing the untrusted worker wrote reaches the error
|
|
4768
|
+
verbatim, so signal kills and known failure signatures are distinguishable
|
|
4769
|
+
in logs without weakening redaction.
|
|
4770
|
+
"""
|
|
4771
|
+
|
|
4772
|
+
if returncode < 0:
|
|
4773
|
+
try:
|
|
4774
|
+
status = f"signal={signal.Signals(-returncode).name}"
|
|
4775
|
+
except ValueError:
|
|
4776
|
+
status = f"signal={-returncode}"
|
|
4777
|
+
else:
|
|
4778
|
+
status = f"exit={returncode}"
|
|
4779
|
+
if not raw:
|
|
4780
|
+
return f"{status}, stderr=empty"
|
|
4781
|
+
first_line = raw.split(b"\n", 1)[0].rstrip(b"\r")
|
|
4782
|
+
return f"{status}, stderr={len(raw)}B, first-line-sha256={sha256_bytes(first_line)[:16]}"
|
|
4783
|
+
|
|
4784
|
+
|
|
4785
|
+
# How much of a Reader's own sentence crosses the boundary. The worker truncates at the same
|
|
4786
|
+
# number when it writes one, so the two are the same bound stated once on each side rather than
|
|
4787
|
+
# two bounds that can drift. The longest refusal any shipped family composes -- the weather
|
|
4788
|
+
# family's grid-template halt, which carries the pair, the constant and the three-step edit --
|
|
4789
|
+
# is comfortably inside it.
|
|
4790
|
+
MAX_CARRIED_DETAIL = 1_000
|
|
4791
|
+
|
|
4792
|
+
|
|
4793
|
+
def _readable_detail(value: Any) -> str | None:
|
|
4794
|
+
"""The worker's sentence if it is plain readable text, and ``None`` otherwise.
|
|
4795
|
+
|
|
4796
|
+
Deliberately strict and deliberately silent: a detail this refuses is dropped rather than
|
|
4797
|
+
escalated, because the code has already been recovered and losing the sentence is strictly
|
|
4798
|
+
better than failing the refusal itself. Printable single-line text only -- no control
|
|
4799
|
+
characters, so nothing can rewrite a terminal or forge a second log line, and no surrogate,
|
|
4800
|
+
so the string survives every encoder it will pass through on the way to a person.
|
|
4801
|
+
"""
|
|
4802
|
+
|
|
4803
|
+
if not isinstance(value, str) or not value.strip() or len(value) > MAX_CARRIED_DETAIL:
|
|
4804
|
+
return None
|
|
4805
|
+
if any(unicodedata.category(char) in {"Cc", "Cf", "Cs", "Co", "Cn"} for char in value):
|
|
4806
|
+
return None
|
|
4807
|
+
return value.strip()
|
|
4808
|
+
|
|
4809
|
+
|
|
4810
|
+
def _ipc_json_bytes(value: Any) -> bytes:
|
|
4811
|
+
try:
|
|
4812
|
+
return json.dumps(
|
|
4813
|
+
value,
|
|
4814
|
+
ensure_ascii=False,
|
|
4815
|
+
allow_nan=False,
|
|
4816
|
+
sort_keys=True,
|
|
4817
|
+
separators=(",", ":"),
|
|
4818
|
+
).encode("utf-8", errors="strict")
|
|
4819
|
+
except (TypeError, ValueError, UnicodeEncodeError):
|
|
4820
|
+
raise AcquisitionSecurityError(
|
|
4821
|
+
"SANDBOX_JSON",
|
|
4822
|
+
"sandbox message is not strict JSON",
|
|
4823
|
+
) from None
|
|
4824
|
+
|
|
4825
|
+
|
|
4826
|
+
def _bounded_identifier(value: Any) -> str:
|
|
4827
|
+
text = _bounded_text(value, maximum=128)
|
|
4828
|
+
if not text[0].islower() or not all(
|
|
4829
|
+
char.islower() or char.isdigit() or char in "._-" for char in text
|
|
4830
|
+
):
|
|
4831
|
+
raise AcquisitionSecurityError(
|
|
4832
|
+
"SANDBOX_IDENTIFIER",
|
|
4833
|
+
"request ID must be a lowercase bounded identifier",
|
|
4834
|
+
)
|
|
4835
|
+
return text
|
|
4836
|
+
|
|
4837
|
+
|
|
4838
|
+
def _bounded_text(value: Any, *, maximum: int) -> str:
|
|
4839
|
+
if not isinstance(value, str) or not value or len(value) > maximum:
|
|
4840
|
+
raise AcquisitionSecurityError("SANDBOX_TEXT", "sandbox text field is invalid")
|
|
4841
|
+
if any(0xD800 <= ord(char) <= 0xDFFF for char in value):
|
|
4842
|
+
raise AcquisitionSecurityError("SANDBOX_TEXT", "sandbox text contains invalid Unicode")
|
|
4843
|
+
return value
|
|
4844
|
+
|
|
4845
|
+
|
|
4846
|
+
def _optional_text(value: Any, maximum: int) -> str | None:
|
|
4847
|
+
return None if value is None else _bounded_text(value, maximum=maximum)
|
|
4848
|
+
|
|
4849
|
+
|
|
4850
|
+
def _string_tuple(value: Any, *, maximum_items: int, item_maximum: int) -> tuple[str, ...]:
|
|
4851
|
+
if not isinstance(value, list) or len(value) > maximum_items:
|
|
4852
|
+
raise AcquisitionSecurityError("SANDBOX_ARRAY", "sandbox string array is invalid")
|
|
4853
|
+
result = tuple(_bounded_text(item, maximum=item_maximum) for item in value)
|
|
4854
|
+
if len(set(result)) != len(result):
|
|
4855
|
+
raise AcquisitionSecurityError("SANDBOX_ARRAY", "sandbox string array has duplicates")
|
|
4856
|
+
return result
|
|
4857
|
+
|
|
4858
|
+
|
|
4859
|
+
def _digest(value: Any) -> str:
|
|
4860
|
+
text = _bounded_text(value, maximum=64)
|
|
4861
|
+
if len(text) != 64 or any(char not in "0123456789abcdef" for char in text):
|
|
4862
|
+
raise AcquisitionSecurityError("SANDBOX_DIGEST", "sandbox digest is invalid")
|
|
4863
|
+
return text
|
|
4864
|
+
|
|
4865
|
+
|
|
4866
|
+
def _require_digest(value: Any, label: str) -> str:
|
|
4867
|
+
try:
|
|
4868
|
+
return _digest(value)
|
|
4869
|
+
except AcquisitionSecurityError as exc:
|
|
4870
|
+
raise AcquisitionSecurityError(
|
|
4871
|
+
"SANDBOX_OS_BOUNDARY",
|
|
4872
|
+
f"{label} attestation must be a lowercase SHA-256 digest",
|
|
4873
|
+
) from exc
|
|
4874
|
+
|
|
4875
|
+
|
|
4876
|
+
def _optional_digest(value: Any) -> str | None:
|
|
4877
|
+
return None if value is None else _digest(value)
|
|
4878
|
+
|
|
4879
|
+
|
|
4880
|
+
if __name__ == "__main__":
|
|
4881
|
+
# Two argv shapes and no parser: the set is closed, and anything outside it exits before a
|
|
4882
|
+
# byte of stdin is read. ``--prewarm`` selects when the imports happen, never what the
|
|
4883
|
+
# child is allowed to do.
|
|
4884
|
+
if tuple(sys.argv[1:]) == WORKER_ARGV:
|
|
4885
|
+
raise SystemExit(worker_main())
|
|
4886
|
+
if tuple(sys.argv[1:]) == WORKER_PREWARM_ARGV:
|
|
4887
|
+
raise SystemExit(worker_main(prewarm=True))
|
|
4888
|
+
raise SystemExit(2)
|