mostlyright-data 0.9.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mostlyright/data_harness/__init__.py +158 -0
- mostlyright/data_harness/acquisition/__init__.py +55 -0
- mostlyright/data_harness/acquisition/http.py +2773 -0
- mostlyright/data_harness/acquisition/parsing.py +809 -0
- mostlyright/data_harness/acquisition/ranges.py +495 -0
- mostlyright/data_harness/acquisition/result_download.py +360 -0
- mostlyright/data_harness/acquisition/retention_admission.py +248 -0
- mostlyright/data_harness/acquisition/sandbox.py +4888 -0
- mostlyright/data_harness/acquisition/url_policy.py +530 -0
- mostlyright/data_harness/agent_runtime.py +2743 -0
- mostlyright/data_harness/assets/logo-ink.svg +31 -0
- mostlyright/data_harness/backends/__init__.py +28 -0
- mostlyright/data_harness/backends/pandas_backend.py +350 -0
- mostlyright/data_harness/backends/polars_backend.py +366 -0
- mostlyright/data_harness/backends/protocol.py +124 -0
- mostlyright/data_harness/backends/reference.py +83 -0
- mostlyright/data_harness/backends/registry.py +55 -0
- mostlyright/data_harness/backends/restrictions.py +126 -0
- mostlyright/data_harness/canonical.py +333 -0
- mostlyright/data_harness/catalog_job.py +625 -0
- mostlyright/data_harness/cli.py +5398 -0
- mostlyright/data_harness/contracts.py +53 -0
- mostlyright/data_harness/coordinator.py +1307 -0
- mostlyright/data_harness/deploy.py +924 -0
- mostlyright/data_harness/deploy_target.py +312 -0
- mostlyright/data_harness/deployment_evidence.py +1067 -0
- mostlyright/data_harness/event_presentation.py +576 -0
- mostlyright/data_harness/events.py +2152 -0
- mostlyright/data_harness/fast_delimited.py +239 -0
- mostlyright/data_harness/fleet.py +237 -0
- mostlyright/data_harness/formats.py +236 -0
- mostlyright/data_harness/governors.py +1163 -0
- mostlyright/data_harness/hosted_bootstrap.py +972 -0
- mostlyright/data_harness/hosted_crawler.py +1115 -0
- mostlyright/data_harness/hosted_crawler_container_smoke.py +351 -0
- mostlyright/data_harness/hosted_crawler_fetch.py +423 -0
- mostlyright/data_harness/hosted_crawler_job.py +1277 -0
- mostlyright/data_harness/hosted_crawler_protocol.py +676 -0
- mostlyright/data_harness/hosted_dataset.py +1500 -0
- mostlyright/data_harness/hosted_deploy.py +3037 -0
- mostlyright/data_harness/hosted_handoff.py +62 -0
- mostlyright/data_harness/hosted_ingestion_contract.py +504 -0
- mostlyright/data_harness/hosted_ingestion_job.py +356 -0
- mostlyright/data_harness/hosted_ingestion_job_smoke.py +40 -0
- mostlyright/data_harness/hosted_session_container_smoke.py +194 -0
- mostlyright/data_harness/hosted_session_worker.py +3554 -0
- mostlyright/data_harness/hosted_session_worker_job_smoke.py +46 -0
- mostlyright/data_harness/hosted_worker.py +6784 -0
- mostlyright/data_harness/ingestion/__init__.py +56 -0
- mostlyright/data_harness/ingestion/contracts.py +461 -0
- mostlyright/data_harness/ingestion/faults.py +42 -0
- mostlyright/data_harness/ingestion/gcs_store.py +1162 -0
- mostlyright/data_harness/ingestion/spool.py +130 -0
- mostlyright/data_harness/ingestion/store.py +885 -0
- mostlyright/data_harness/key_seam.py +434 -0
- mostlyright/data_harness/linux_process_boundary.py +262 -0
- mostlyright/data_harness/local_contracts.py +2880 -0
- mostlyright/data_harness/local_search/__init__.py +5 -0
- mostlyright/data_harness/local_search/build_index.py +1087 -0
- mostlyright/data_harness/local_search/contracts.py +920 -0
- mostlyright/data_harness/local_search/query_trace.py +266 -0
- mostlyright/data_harness/local_search/retrieval.py +700 -0
- mostlyright/data_harness/local_search/sealed.py +474 -0
- mostlyright/data_harness/local_search/service.py +784 -0
- mostlyright/data_harness/nbrender/CONTRACT.md +212 -0
- mostlyright/data_harness/nbrender/__init__.py +12 -0
- mostlyright/data_harness/nbrender/chrome.py +359 -0
- mostlyright/data_harness/nbrender/code_body.py +266 -0
- mostlyright/data_harness/nbrender/document.py +407 -0
- mostlyright/data_harness/nbrender/frame.py +275 -0
- mostlyright/data_harness/nbrender/interactive.py +337 -0
- mostlyright/data_harness/nbrender/markdown_body.py +477 -0
- mostlyright/data_harness/nbrender/mr_components.py +134 -0
- mostlyright/data_harness/nbrender/outputs_data.py +595 -0
- mostlyright/data_harness/nbrender/outputs_rich.py +906 -0
- mostlyright/data_harness/nbrender/outputs_source.py +260 -0
- mostlyright/data_harness/nbrender/outputs_stage.py +176 -0
- mostlyright/data_harness/nbrender/outputs_text.py +400 -0
- mostlyright/data_harness/nbrender/parse.py +394 -0
- mostlyright/data_harness/nbrender/status.py +40 -0
- mostlyright/data_harness/nbrender/tokens.py +1295 -0
- mostlyright/data_harness/notebook.py +1710 -0
- mostlyright/data_harness/offline.py +2049 -0
- mostlyright/data_harness/operation_registry.py +1007 -0
- mostlyright/data_harness/operator_setup.py +239 -0
- mostlyright/data_harness/pipeline.py +6428 -0
- mostlyright/data_harness/plan_graph.py +2026 -0
- mostlyright/data_harness/preparation/__init__.py +104 -0
- mostlyright/data_harness/preparation/contracts.py +1017 -0
- mostlyright/data_harness/preparation/engine.py +221 -0
- mostlyright/data_harness/preparation/errors.py +14 -0
- mostlyright/data_harness/preparation/gates.py +751 -0
- mostlyright/data_harness/preparation/joins.py +574 -0
- mostlyright/data_harness/preparation/profile.py +384 -0
- mostlyright/data_harness/preparation/table.py +217 -0
- mostlyright/data_harness/preparation/transforms.py +568 -0
- mostlyright/data_harness/progress_events.py +534 -0
- mostlyright/data_harness/readers/__init__.py +46 -0
- mostlyright/data_harness/readers/containers.py +963 -0
- mostlyright/data_harness/readers/contracts.py +542 -0
- mostlyright/data_harness/readers/delimited.py +257 -0
- mostlyright/data_harness/readers/grib2/__init__.py +33 -0
- mostlyright/data_harness/readers/grib2/admission.py +722 -0
- mostlyright/data_harness/readers/grib2/decode.py +1009 -0
- mostlyright/data_harness/readers/grib2/geometry.py +1133 -0
- mostlyright/data_harness/readers/grib2/portable_math.py +501 -0
- mostlyright/data_harness/readers/json_tabular.py +485 -0
- mostlyright/data_harness/readers/registry.py +514 -0
- mostlyright/data_harness/readers/samples/README.md +110 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/cities.csv.gz +0 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/cities.csv.gz +0 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/cities.tar +0 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/cities.tar +0 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/cities.zip +0 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/expected.json +25 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/expected.json +25 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/expected.json +25 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/cities.csv +3 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/expected.json +23 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/cities.tsv +5 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/expected.json +25 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/expected.json +30 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/observations.csv +5 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/expected.json +44 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/stations.json +1 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/expected.json +48 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/observations.ndjson +4 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/cities.xlsx +0 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/expected.json +24 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/expected.json +27 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/expected.json +27 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/README.md +20 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/expected.json +55 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/gfs-2m-temperature.grib2 +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/expected.json +54 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/expected.json +54 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/hrrr-categorical-rain.grib2 +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/expected.json +54 -0
- mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
- mostlyright/data_harness/readers/samples.py +582 -0
- mostlyright/data_harness/readers/spreadsheet.py +803 -0
- mostlyright/data_harness/readers/tabular.py +510 -0
- mostlyright/data_harness/recipe.py +5321 -0
- mostlyright/data_harness/repair/__init__.py +78 -0
- mostlyright/data_harness/repair/adapters.py +274 -0
- mostlyright/data_harness/repair/contracts.py +872 -0
- mostlyright/data_harness/repair/coordinator.py +1099 -0
- mostlyright/data_harness/repair/errors.py +16 -0
- mostlyright/data_harness/review.py +2533 -0
- mostlyright/data_harness/rowset.py +283 -0
- mostlyright/data_harness/serving.py +1975 -0
- mostlyright/data_harness/serving_edge.py +590 -0
- mostlyright/data_harness/serving_http.py +1031 -0
- mostlyright/data_harness/session_probes.py +759 -0
- mostlyright/data_harness/signing.py +101 -0
- mostlyright/data_harness/source_discovery.py +898 -0
- mostlyright/data_harness/sources/__init__.py +209 -0
- mostlyright/data_harness/sources/_adapter_steps.py +213 -0
- mostlyright/data_harness/sources/adapters.py +1214 -0
- mostlyright/data_harness/sources/cadence.py +1428 -0
- mostlyright/data_harness/sources/cadence_emission.py +453 -0
- mostlyright/data_harness/sources/cadence_history.py +546 -0
- mostlyright/data_harness/sources/catalog/__init__.py +17 -0
- mostlyright/data_harness/sources/catalog/admission.py +477 -0
- mostlyright/data_harness/sources/catalog/authoring.py +1701 -0
- mostlyright/data_harness/sources/catalog/authoring_policy.py +701 -0
- mostlyright/data_harness/sources/catalog/authoring_shards.py +1217 -0
- mostlyright/data_harness/sources/catalog/bounded_io.py +231 -0
- mostlyright/data_harness/sources/catalog/channel.py +523 -0
- mostlyright/data_harness/sources/catalog/channel_client.py +296 -0
- mostlyright/data_harness/sources/catalog/contracts.py +825 -0
- mostlyright/data_harness/sources/catalog/coverage.py +137 -0
- mostlyright/data_harness/sources/catalog/delta.py +1340 -0
- mostlyright/data_harness/sources/catalog/embedding.py +532 -0
- mostlyright/data_harness/sources/catalog/entry_v2.py +1182 -0
- mostlyright/data_harness/sources/catalog/fill.py +3889 -0
- mostlyright/data_harness/sources/catalog/fill_partitions.py +459 -0
- mostlyright/data_harness/sources/catalog/fill_staging.py +1105 -0
- mostlyright/data_harness/sources/catalog/gating.py +374 -0
- mostlyright/data_harness/sources/catalog/generation_receipt.py +1607 -0
- mostlyright/data_harness/sources/catalog/harvest/__init__.py +7 -0
- mostlyright/data_harness/sources/catalog/harvest/ckan.py +384 -0
- mostlyright/data_harness/sources/catalog/harvest/datagov_v4.py +798 -0
- mostlyright/data_harness/sources/catalog/harvest/protocol.py +964 -0
- mostlyright/data_harness/sources/catalog/harvest/sdmx.py +445 -0
- mostlyright/data_harness/sources/catalog/harvest/stac.py +384 -0
- mostlyright/data_harness/sources/catalog/health.py +447 -0
- mostlyright/data_harness/sources/catalog/hosted_catalog.py +105 -0
- mostlyright/data_harness/sources/catalog/identity_history.py +1549 -0
- mostlyright/data_harness/sources/catalog/neural.py +1618 -0
- mostlyright/data_harness/sources/catalog/packed_catalog.py +2345 -0
- mostlyright/data_harness/sources/catalog/packed_retrieval.py +1517 -0
- mostlyright/data_harness/sources/catalog/packed_writer.py +2802 -0
- mostlyright/data_harness/sources/catalog/query_trace.py +1037 -0
- mostlyright/data_harness/sources/catalog/recommend.py +171 -0
- mostlyright/data_harness/sources/catalog/retrieval.py +230 -0
- mostlyright/data_harness/sources/catalog/retrieval_manifest.py +995 -0
- mostlyright/data_harness/sources/catalog/rights_decisions.py +254 -0
- mostlyright/data_harness/sources/catalog/sealed.py +560 -0
- mostlyright/data_harness/sources/catalog/search.py +230 -0
- mostlyright/data_harness/sources/catalog/streaming_delta.py +1097 -0
- mostlyright/data_harness/sources/catalog/update.py +891 -0
- mostlyright/data_harness/sources/collections.py +815 -0
- mostlyright/data_harness/sources/contracts.py +2223 -0
- mostlyright/data_harness/sources/deletion.py +761 -0
- mostlyright/data_harness/sources/fitness.py +162 -0
- mostlyright/data_harness/sources/governance.py +163 -0
- mostlyright/data_harness/sources/hosted.py +173 -0
- mostlyright/data_harness/sources/integration.py +218 -0
- mostlyright/data_harness/sources/range_reader.py +418 -0
- mostlyright/data_harness/sources/registry.py +514 -0
- mostlyright/data_harness/sources/rights_rule.py +59 -0
- mostlyright/data_harness/sources/source_cadence_vectors.v1.json +1 -0
- mostlyright/data_harness/sources/sports.py +521 -0
- mostlyright/data_harness/sources/stream.py +524 -0
- mostlyright/data_harness/sources/stream_connector.py +418 -0
- mostlyright/data_harness/sources/stream_recorder.py +1404 -0
- mostlyright/data_harness/studio_boundary.py +2019 -0
- mostlyright/data_harness/thin/__init__.py +37 -0
- mostlyright/data_harness/thin/acquire.py +1137 -0
- mostlyright/data_harness/thin/acquire_cancel.py +579 -0
- mostlyright/data_harness/thin/approvals.py +617 -0
- mostlyright/data_harness/thin/commands.py +406 -0
- mostlyright/data_harness/thin/download.py +194 -0
- mostlyright/data_harness/thin/narrative.py +589 -0
- mostlyright/data_harness/thin/parity.py +1070 -0
- mostlyright/data_harness/thin/propose.py +2759 -0
- mostlyright/data_harness/thin/research.py +1663 -0
- mostlyright/data_harness/thin/router.py +924 -0
- mostlyright/data_harness/thin/runs.py +519 -0
- mostlyright/data_harness/thin/session.py +281 -0
- mostlyright/data_harness/thin/stream.py +501 -0
- mostlyright/data_harness/thin/transport.py +187 -0
- mostlyright/data_harness/thin/vocabulary.py +368 -0
- mostlyright/data_harness/thin/workers.py +164 -0
- mostlyright/data_harness/ucum/TABLE-PIN.json +40 -0
- mostlyright/data_harness/ucum/ucum-subset.v1.json +632 -0
- mostlyright/data_harness/unit_flow.py +927 -0
- mostlyright/data_harness/units.py +572 -0
- mostlyright/data_harness/ux/__init__.py +9 -0
- mostlyright/data_harness/ux/approve.py +485 -0
- mostlyright/data_harness/ux/author_yaml.py +597 -0
- mostlyright/data_harness/ux/cloud_auth.py +447 -0
- mostlyright/data_harness/ux/commands/__init__.py +260 -0
- mostlyright/data_harness/ux/commands/approve.py +136 -0
- mostlyright/data_harness/ux/commands/auth.py +744 -0
- mostlyright/data_harness/ux/commands/author.py +79 -0
- mostlyright/data_harness/ux/commands/catalog_author.py +403 -0
- mostlyright/data_harness/ux/commands/catalog_fill.py +523 -0
- mostlyright/data_harness/ux/commands/catalog_harvest.py +545 -0
- mostlyright/data_harness/ux/commands/catalog_publish.py +1838 -0
- mostlyright/data_harness/ux/commands/catalog_search.py +71 -0
- mostlyright/data_harness/ux/commands/catalog_update.py +437 -0
- mostlyright/data_harness/ux/commands/deploy.py +134 -0
- mostlyright/data_harness/ux/commands/deploy_dataset.py +98 -0
- mostlyright/data_harness/ux/commands/deploy_plan.py +105 -0
- mostlyright/data_harness/ux/commands/deploy_status.py +104 -0
- mostlyright/data_harness/ux/commands/diff.py +74 -0
- mostlyright/data_harness/ux/commands/index.py +84 -0
- mostlyright/data_harness/ux/commands/inventory.py +47 -0
- mostlyright/data_harness/ux/commands/list_builds.py +143 -0
- mostlyright/data_harness/ux/commands/login.py +63 -0
- mostlyright/data_harness/ux/commands/peek.py +236 -0
- mostlyright/data_harness/ux/commands/plan_check.py +90 -0
- mostlyright/data_harness/ux/commands/preflight.py +97 -0
- mostlyright/data_harness/ux/commands/record.py +107 -0
- mostlyright/data_harness/ux/commands/review_setup.py +47 -0
- mostlyright/data_harness/ux/commands/search.py +440 -0
- mostlyright/data_harness/ux/commands/show.py +61 -0
- mostlyright/data_harness/ux/commands/whoami.py +37 -0
- mostlyright/data_harness/ux/credential_native.py +551 -0
- mostlyright/data_harness/ux/credential_store.py +1055 -0
- mostlyright/data_harness/ux/credentials.py +631 -0
- mostlyright/data_harness/ux/diffing.py +444 -0
- mostlyright/data_harness/ux/headline.py +671 -0
- mostlyright/data_harness/ux/hosted_acquisition.py +974 -0
- mostlyright/data_harness/ux/hosted_run_status.py +619 -0
- mostlyright/data_harness/ux/inventory.py +427 -0
- mostlyright/data_harness/ux/local_review.py +375 -0
- mostlyright/data_harness/ux/login.py +691 -0
- mostlyright/data_harness/ux/path_kind.py +147 -0
- mostlyright/data_harness/ux/peek.py +1000 -0
- mostlyright/data_harness/ux/plain_file.py +178 -0
- mostlyright/data_harness/ux/plan_check.py +311 -0
- mostlyright/data_harness/ux/preflight.py +918 -0
- mostlyright/data_harness/ux/readers.py +1124 -0
- mostlyright/data_harness/ux/remediation.py +2195 -0
- mostlyright/data_harness/ux/render.py +657 -0
- mostlyright/data_harness/ux/workload.py +1077 -0
- mostlyright/data_harness/viewer.py +3713 -0
- mostlyright/data_harness/visual_run/__init__.py +83 -0
- mostlyright/data_harness/visual_run/authoring.py +235 -0
- mostlyright/data_harness/visual_run/contracts.py +673 -0
- mostlyright/data_harness/visual_run/materialize.py +486 -0
- mostlyright/data_harness/visual_run/observations.py +874 -0
- mostlyright/data_harness/visual_run/query.py +259 -0
- mostlyright/data_harness/visual_run/reducer.py +280 -0
- mostlyright/data_harness/visual_run/sdk.py +892 -0
- mostlyright/data_harness/visual_run/store.py +584 -0
- mostlyright/data_harness/visual_run/transport.py +239 -0
- mostlyright/data_harness/watch.py +2999 -0
- mostlyright_data-0.9.0.dist-info/METADATA +607 -0
- mostlyright_data-0.9.0.dist-info/RECORD +314 -0
- mostlyright_data-0.9.0.dist-info/WHEEL +4 -0
- mostlyright_data-0.9.0.dist-info/entry_points.txt +12 -0
|
@@ -0,0 +1,434 @@
|
|
|
1
|
+
"""The one seam between the two planes: cloud owns identity, the harness verifies against it.
|
|
2
|
+
|
|
3
|
+
Cloud and the harness are separate repositories with separate owners, and exactly one contract
|
|
4
|
+
joins them: an API key presented by a caller is verified against cloud's verify surface. The
|
|
5
|
+
authority for that contract is ``mostlyright-cloud/manifest/api-key.ts``. This module is the
|
|
6
|
+
harness side of it and must stay byte-compatible with that file.
|
|
7
|
+
|
|
8
|
+
Three rules are copied here verbatim from api-key.ts's header comment, because every one of them
|
|
9
|
+
is security-load-bearing and every one of them looks like dead weight to a later reader:
|
|
10
|
+
|
|
11
|
+
* Hash both sides to fixed-length digests before comparing. The comparison then runs over two
|
|
12
|
+
32-byte values whatever the caller presented.
|
|
13
|
+
* **Never pre-check length.** A length pre-check answers "how long is the expected key" in the
|
|
14
|
+
time it takes to return, which is the question an attacker is asking.
|
|
15
|
+
* **Never wrap the compare in try/except.** Throwing is measurably faster than a full compare, so
|
|
16
|
+
a raising path reintroduces exactly the timing channel that hashing exists to close. Digesting
|
|
17
|
+
first is what makes the compare total, so there is nothing left to catch.
|
|
18
|
+
|
|
19
|
+
Fail-closed is the documented default for everything in this module. A key is authorized only by
|
|
20
|
+
an explicit, well-formed allow from the verify surface reached over https. Any other outcome — a
|
|
21
|
+
non-200 status, a redirect, a transport fault, a timeout, a body that is not strict JSON, a body
|
|
22
|
+
over the cap, a body that names a different key — denies.
|
|
23
|
+
|
|
24
|
+
The raw key never leaves this module. It goes out in one place, the ``x-api-key`` request header,
|
|
25
|
+
and it appears in no return value, no :class:`KeyVerdict`, no serialized dict, no reason string,
|
|
26
|
+
and no exception message. Only :func:`hash_key_id` output is safe to log.
|
|
27
|
+
|
|
28
|
+
**Where this seam is enforced, and why not in the worker.** The enforcement point is named in
|
|
29
|
+
:data:`CONSUMER_CALL_SITE`, and it is deliberately not this repository's hosted worker. Three
|
|
30
|
+
measured facts establish that, and they are recorded here so a later reader does not "fix" an
|
|
31
|
+
omission that is not one:
|
|
32
|
+
|
|
33
|
+
1. ``hosted_bootstrap.launch`` hands the worker an environment of exactly three variables — PATH,
|
|
34
|
+
PYTHONDONTWRITEBYTECODE, PYTHONUNBUFFERED (the environment mapping at hosted_bootstrap.py:475).
|
|
35
|
+
No API key can reach the worker process through it. Passing one would mean deliberately
|
|
36
|
+
widening that barrier to carry a customer credential into the least-trusted process in the
|
|
37
|
+
system, which inverts the design the barrier exists to hold.
|
|
38
|
+
2. ``mostlyright-studio``'s ``infra/tofu`` declares no ``MOSTLYRIGHT_API_KEY`` on the worker job,
|
|
39
|
+
so the credential is not present in that container today.
|
|
40
|
+
3. The worker authenticates with an attempt-scoped capability token minted by Studio during the
|
|
41
|
+
bootstrap exchange, and its tenant coordinate is ``workspace_id``. The customer API key is
|
|
42
|
+
verified upstream, where the request enters the platform, before any job exists to run.
|
|
43
|
+
|
|
44
|
+
So this module is the harness-side reference implementation and parity anchor for a check that
|
|
45
|
+
runs at the API edge, not a library the worker forgot to call. It is intentionally not imported by
|
|
46
|
+
``producer_main`` or ``verifier_main``; wiring it in there would punch a credential through the
|
|
47
|
+
sanitization barrier to no benefit. ``tests/test_key_seam_consumer_contract.py`` pins the two
|
|
48
|
+
implementations against each other.
|
|
49
|
+
"""
|
|
50
|
+
|
|
51
|
+
from __future__ import annotations
|
|
52
|
+
|
|
53
|
+
import hashlib
|
|
54
|
+
import hmac
|
|
55
|
+
import ssl
|
|
56
|
+
from collections.abc import Mapping
|
|
57
|
+
from dataclasses import dataclass
|
|
58
|
+
from typing import Any, Protocol
|
|
59
|
+
from urllib.error import HTTPError
|
|
60
|
+
from urllib.parse import urlsplit
|
|
61
|
+
from urllib.request import (
|
|
62
|
+
HTTPRedirectHandler,
|
|
63
|
+
HTTPSHandler,
|
|
64
|
+
ProxyHandler,
|
|
65
|
+
Request,
|
|
66
|
+
build_opener,
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
from mostlyright.data_harness.canonical import CanonicalJSONError, parse_json
|
|
70
|
+
|
|
71
|
+
#: The environment variable holding the configured legacy v1 shared key. Matches
|
|
72
|
+
#: ``LEGACY_API_KEY_ENV`` at api-key.ts:27 — one shared secret, one env var name, on both the
|
|
73
|
+
#: sending and the receiving side.
|
|
74
|
+
LEGACY_API_KEY_ENV = "MOSTLYRIGHT_API_KEY"
|
|
75
|
+
|
|
76
|
+
#: The only header the credential is ever read from or sent in. Matches ``API_KEY_HEADER`` at
|
|
77
|
+
#: manifest-handler.ts:25. Never a query string, never a URL path segment: those are logged by
|
|
78
|
+
#: every proxy, load balancer, and browser history on the route.
|
|
79
|
+
API_KEY_HEADER = "x-api-key"
|
|
80
|
+
|
|
81
|
+
V2_LIVE_PREFIX = "mr_live_"
|
|
82
|
+
V2_TEST_PREFIX = "mr_test_"
|
|
83
|
+
|
|
84
|
+
#: Where a presented API key is actually refused or allowed, recorded in code because it is not in
|
|
85
|
+
#: this repository and an unrecorded consumer reads as a missing one. Named consumers: the Studio
|
|
86
|
+
#: API edge that authenticates a customer request before any run exists, and the worker-bootstrap
|
|
87
|
+
#: exchange that mints the attempt-scoped capability the worker carries instead of a key
|
|
88
|
+
#: (``mostlyright-cloud/manifest/manifest-handler.ts`` is the same check on the manifest surface).
|
|
89
|
+
#: This module is the harness-side reference implementation and parity anchor for that check; see
|
|
90
|
+
#: the module docstring for the three facts establishing why the worker must never hold a key.
|
|
91
|
+
CONSUMER_CALL_SITE = (
|
|
92
|
+
"mostlyright-studio API edge and worker-bootstrap exchange; "
|
|
93
|
+
"mostlyright-cloud/manifest/manifest-handler.ts on the manifest surface"
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
#: A verify response is a verdict, not a payload. Anything larger than this is not a verdict, so
|
|
97
|
+
#: the cap is small on purpose — it bounds memory and it bounds the time an attacker can hold the
|
|
98
|
+
#: connection open. Compare ``MAX_BOOTSTRAP_RESPONSE_BYTES`` (hosted_bootstrap.py:50), which is
|
|
99
|
+
#: large because a bootstrap envelope legitimately is.
|
|
100
|
+
MAX_VERIFY_RESPONSE_BYTES = 8 * 1024
|
|
101
|
+
|
|
102
|
+
#: A bounded wait. A verify surface that has not answered by now has failed, and a hang is a
|
|
103
|
+
#: denial of service against every caller waiting behind it.
|
|
104
|
+
VERIFY_TIMEOUT_SECONDS = 10.0
|
|
105
|
+
|
|
106
|
+
MAX_URL_LENGTH = 2048
|
|
107
|
+
|
|
108
|
+
#: The allow marker. A 200 alone is not an allow: an intercepting proxy, a captive portal, and a
|
|
109
|
+
#: misrouted request all return 200 with a body that means nothing. The body must be a JSON object
|
|
110
|
+
#: that says so explicitly.
|
|
111
|
+
ALLOW_MARKER = "ok"
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
class KeySeamError(RuntimeError):
|
|
115
|
+
"""A typed key-seam failure.
|
|
116
|
+
|
|
117
|
+
Raised only for a structurally invalid call — a base URL that is not https, a key that is not
|
|
118
|
+
text. A rejected key is never an exception; it is a denying :class:`KeyVerdict`.
|
|
119
|
+
|
|
120
|
+
Like everything else here, the message never contains the raw key.
|
|
121
|
+
"""
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def hash_key_id(key: str) -> str:
|
|
125
|
+
"""Return a stable, irreversible identifier for ``key``.
|
|
126
|
+
|
|
127
|
+
Exactly cloud's ``hashKeyId`` (api-key.ts:49): the first 8 bytes of the SHA-256 digest of the
|
|
128
|
+
key's UTF-8 encoding, hex-encoded — 16 lowercase hex characters.
|
|
129
|
+
|
|
130
|
+
This is the only value derived from a key that is safe to log, record, or return. It is not
|
|
131
|
+
reversible and it is not a credential.
|
|
132
|
+
|
|
133
|
+
The coupling is load-bearing and invisible from either side alone: cloud's
|
|
134
|
+
``dashboard/lib/usage.ts`` stores this same value in ``usage_events.key_id``. If either
|
|
135
|
+
implementation changes its truncation length or its string encoding, nothing raises anywhere —
|
|
136
|
+
usage counting for every key simply goes to zero, and attribution for a key becomes
|
|
137
|
+
unresolvable between the two planes. ``tests/test_key_seam.py`` pins the derivation with
|
|
138
|
+
literal vectors produced by running both implementations.
|
|
139
|
+
"""
|
|
140
|
+
|
|
141
|
+
if not isinstance(key, str):
|
|
142
|
+
raise KeySeamError("an API key must be text")
|
|
143
|
+
return hashlib.sha256(key.encode("utf-8")).hexdigest()[:16]
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def constant_time_equal(a: str, b: str) -> bool:
|
|
147
|
+
"""Return whether ``a`` and ``b`` are equal, in time independent of either value.
|
|
148
|
+
|
|
149
|
+
Mirrors ``constantTimeEqual`` (api-key.ts:38). Both operands are digested to fixed 32-byte
|
|
150
|
+
SHA-256 values first, and only the digests reach :func:`hmac.compare_digest`.
|
|
151
|
+
|
|
152
|
+
The digest step is not decoration. It is what makes this function *total* — every pairing of
|
|
153
|
+
lengths, including empty and non-ASCII inputs, returns a bool and none raises — and it is what
|
|
154
|
+
makes the running time independent of the presented value's length. There is deliberately no
|
|
155
|
+
length pre-check (it would leak the expected key's length) and deliberately no try/except
|
|
156
|
+
around the compare (an early raise is faster than a full compare, which is the timing channel
|
|
157
|
+
this design closes).
|
|
158
|
+
"""
|
|
159
|
+
|
|
160
|
+
if not isinstance(a, str) or not isinstance(b, str):
|
|
161
|
+
raise KeySeamError("a credential comparison takes two text values")
|
|
162
|
+
digest_a = hashlib.sha256(a.encode("utf-8")).digest()
|
|
163
|
+
digest_b = hashlib.sha256(b.encode("utf-8")).digest()
|
|
164
|
+
return hmac.compare_digest(digest_a, digest_b)
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def detect_key_format(key: str) -> str:
|
|
168
|
+
"""Classify a presented credential's shape as ``v2-live``, ``v2-test``, or ``legacy``.
|
|
169
|
+
|
|
170
|
+
Mirrors ``detectKeyFormat`` (api-key.ts:87). **Purely descriptive: it authorizes nothing.**
|
|
171
|
+
|
|
172
|
+
A ``v2-live`` classification says only that a string starts with eight specified characters.
|
|
173
|
+
Every format still resolves through the
|
|
174
|
+
same verification, exactly as cloud's v0 does. This module must never mint a v2 key and must
|
|
175
|
+
never independently validate one; ``key_format`` on a :class:`KeyVerdict` is a label for
|
|
176
|
+
humans, never an input to the decision.
|
|
177
|
+
"""
|
|
178
|
+
|
|
179
|
+
if not isinstance(key, str):
|
|
180
|
+
raise KeySeamError("an API key must be text")
|
|
181
|
+
if key.startswith(V2_LIVE_PREFIX):
|
|
182
|
+
return "v2-live"
|
|
183
|
+
if key.startswith(V2_TEST_PREFIX):
|
|
184
|
+
return "v2-test"
|
|
185
|
+
return "legacy"
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
@dataclass(frozen=True)
|
|
189
|
+
class KeyVerdict:
|
|
190
|
+
"""The outcome of verifying one presented key.
|
|
191
|
+
|
|
192
|
+
Invariant: a verdict never carries the raw key. It carries ``key_id``, the irreversible
|
|
193
|
+
identifier from :func:`hash_key_id`, which is the value cloud records too — so the same key is
|
|
194
|
+
attributed identically on both sides of the seam.
|
|
195
|
+
|
|
196
|
+
``reason`` is a plain-language phrase safe to show a user and safe to write to a log. It names
|
|
197
|
+
the class of fault only. It never contains, quotes, truncates, or measures the presented key,
|
|
198
|
+
and it never reports the expected key's length.
|
|
199
|
+
|
|
200
|
+
``key_format`` is descriptive (see :func:`detect_key_format`) and never explains ``ok``.
|
|
201
|
+
"""
|
|
202
|
+
|
|
203
|
+
ok: bool
|
|
204
|
+
key_id: str
|
|
205
|
+
key_format: str
|
|
206
|
+
reason: str
|
|
207
|
+
|
|
208
|
+
def __post_init__(self) -> None:
|
|
209
|
+
if type(self.ok) is not bool:
|
|
210
|
+
raise KeySeamError("a verdict must be allowed or denied, not something else")
|
|
211
|
+
for name in ("key_id", "key_format", "reason"):
|
|
212
|
+
value = getattr(self, name)
|
|
213
|
+
if type(value) is not str or not value:
|
|
214
|
+
raise KeySeamError(f"{name} must be a non-empty text value")
|
|
215
|
+
if len(self.key_id) != 16:
|
|
216
|
+
raise KeySeamError("key_id must be the 16-character identifier from hash_key_id")
|
|
217
|
+
|
|
218
|
+
def to_dict(self) -> dict[str, Any]:
|
|
219
|
+
"""Return a plain JSON-safe mapping, carrying the key id and never the key."""
|
|
220
|
+
|
|
221
|
+
return {
|
|
222
|
+
"ok": self.ok,
|
|
223
|
+
"key_id": self.key_id,
|
|
224
|
+
"key_format": self.key_format,
|
|
225
|
+
"reason": self.reason,
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
class VerifyTransport(Protocol):
|
|
230
|
+
"""Minimal redirect-free, proxy-free verify transport.
|
|
231
|
+
|
|
232
|
+
Deliberately the same shape as ``BootstrapTransport`` (hosted_bootstrap.py:57) so this
|
|
233
|
+
codebase has one transport idiom rather than two: a bootstrap transport satisfies this
|
|
234
|
+
protocol structurally, and a test stub written for one reads the same as a stub for the other.
|
|
235
|
+
"""
|
|
236
|
+
|
|
237
|
+
def request(
|
|
238
|
+
self,
|
|
239
|
+
method: str,
|
|
240
|
+
url: str,
|
|
241
|
+
headers: Mapping[str, str],
|
|
242
|
+
body: bytes | None,
|
|
243
|
+
maximum: int,
|
|
244
|
+
) -> tuple[int, bytes]: ...
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
class _NoRedirect(HTTPRedirectHandler):
|
|
248
|
+
"""Refuse every redirect.
|
|
249
|
+
|
|
250
|
+
Following a redirect would re-send the ``x-api-key`` header to a host named by the response
|
|
251
|
+
rather than by the caller, which is a credential handed to whoever answered. Returning ``None``
|
|
252
|
+
here makes urllib surface the 3xx as an ``HTTPError``, so the status reaches the caller as a
|
|
253
|
+
plain non-200 and denies. Mirrors hosted_bootstrap.py:70.
|
|
254
|
+
"""
|
|
255
|
+
|
|
256
|
+
def redirect_request(
|
|
257
|
+
self,
|
|
258
|
+
request: Request,
|
|
259
|
+
file_pointer: Any,
|
|
260
|
+
code: int,
|
|
261
|
+
message: str,
|
|
262
|
+
headers: Any,
|
|
263
|
+
new_url: str,
|
|
264
|
+
) -> None:
|
|
265
|
+
return None
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
class UrlLibVerifyTransport:
|
|
269
|
+
"""Small stdlib transport with ambient proxies and redirects disabled.
|
|
270
|
+
|
|
271
|
+
Ambient proxies are disabled because a proxy configured by the environment is another party
|
|
272
|
+
that would see the credential.
|
|
273
|
+
"""
|
|
274
|
+
|
|
275
|
+
def __init__(self) -> None:
|
|
276
|
+
self._opener = build_opener(
|
|
277
|
+
ProxyHandler({}),
|
|
278
|
+
HTTPSHandler(context=ssl.create_default_context()),
|
|
279
|
+
_NoRedirect(),
|
|
280
|
+
)
|
|
281
|
+
|
|
282
|
+
def request(
|
|
283
|
+
self,
|
|
284
|
+
method: str,
|
|
285
|
+
url: str,
|
|
286
|
+
headers: Mapping[str, str],
|
|
287
|
+
body: bytes | None,
|
|
288
|
+
maximum: int,
|
|
289
|
+
) -> tuple[int, bytes]:
|
|
290
|
+
request = Request(url, data=body, headers=dict(headers), method=method)
|
|
291
|
+
try:
|
|
292
|
+
response = self._opener.open(request, timeout=VERIFY_TIMEOUT_SECONDS)
|
|
293
|
+
except HTTPError as error:
|
|
294
|
+
response = error
|
|
295
|
+
with response:
|
|
296
|
+
status = int(response.status)
|
|
297
|
+
content_length = response.headers.get("Content-Length")
|
|
298
|
+
if content_length is not None:
|
|
299
|
+
try:
|
|
300
|
+
declared = int(content_length)
|
|
301
|
+
except ValueError as error:
|
|
302
|
+
raise KeySeamError(
|
|
303
|
+
"the verify response has an invalid content length"
|
|
304
|
+
) from error
|
|
305
|
+
if declared < 0 or declared > maximum:
|
|
306
|
+
raise KeySeamError("the verify response exceeds its byte limit")
|
|
307
|
+
raw = response.read(maximum + 1)
|
|
308
|
+
if len(raw) > maximum:
|
|
309
|
+
raise KeySeamError("the verify response exceeds its byte limit")
|
|
310
|
+
return status, raw
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
def _https_url(value: str, label: str) -> str:
|
|
314
|
+
"""Return ``value`` when it is a plain absolute https URL, otherwise refuse it.
|
|
315
|
+
|
|
316
|
+
Mirrors ``hosted_bootstrap._https_url`` (hosted_bootstrap.py:147). It is copied rather than
|
|
317
|
+
imported for the reason recorded in ``deploy_target._https_url``: importing that module pulls
|
|
318
|
+
the whole hosted-worker stack in behind it and raises a bootstrap-specific error code. The
|
|
319
|
+
policy itself must stay identical — https only, absolute, no user information, no fragment, no
|
|
320
|
+
trailing slash.
|
|
321
|
+
|
|
322
|
+
Plain http is refused *before any request is attempted*, because the failure mode is not a bad
|
|
323
|
+
response, it is the credential already having been sent in the clear.
|
|
324
|
+
"""
|
|
325
|
+
|
|
326
|
+
if not isinstance(value, str) or not 1 <= len(value) <= MAX_URL_LENGTH:
|
|
327
|
+
raise KeySeamError(f"{label} is not a usable URL")
|
|
328
|
+
if value.endswith("/"):
|
|
329
|
+
raise KeySeamError(f"{label} must not end with a slash")
|
|
330
|
+
parsed = urlsplit(value)
|
|
331
|
+
if (
|
|
332
|
+
parsed.scheme != "https"
|
|
333
|
+
or not parsed.hostname
|
|
334
|
+
or parsed.username is not None
|
|
335
|
+
or parsed.password is not None
|
|
336
|
+
or parsed.fragment
|
|
337
|
+
):
|
|
338
|
+
raise KeySeamError(
|
|
339
|
+
f"{label} must be an absolute https URL without user information or fragment"
|
|
340
|
+
)
|
|
341
|
+
return value
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
def _denied(key_id: str, key_format: str, reason: str) -> KeyVerdict:
|
|
345
|
+
return KeyVerdict(ok=False, key_id=key_id, key_format=key_format, reason=reason)
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
def _allow_body_is_valid(raw: bytes, key_id: str) -> tuple[bool, str]:
|
|
349
|
+
"""Return whether ``raw`` is a well-formed allow body for ``key_id``, and why not if it is not.
|
|
350
|
+
|
|
351
|
+
Strict parse only: duplicate keys, byte-order marks, non-UTF-8 bytes, and non-integer numbers
|
|
352
|
+
are all rejected by :func:`canonical.parse_json`, so a body that two parsers would read
|
|
353
|
+
differently is never acted on.
|
|
354
|
+
"""
|
|
355
|
+
|
|
356
|
+
try:
|
|
357
|
+
parsed = parse_json(raw)
|
|
358
|
+
except CanonicalJSONError:
|
|
359
|
+
return False, "the verify surface returned a body that is not strict JSON"
|
|
360
|
+
if not isinstance(parsed, dict):
|
|
361
|
+
return False, "the verify surface returned a body that is not a verdict object"
|
|
362
|
+
if parsed.get(ALLOW_MARKER) is not True:
|
|
363
|
+
return False, "the verify surface did not allow this key"
|
|
364
|
+
named = parsed.get("key_id")
|
|
365
|
+
if named is not None and named != key_id:
|
|
366
|
+
# The response is about some other key. Accepting it would let a captured allow for one
|
|
367
|
+
# key authorize another, and would attribute usage to the wrong key id on this side.
|
|
368
|
+
return False, "the verify surface answered about a different key"
|
|
369
|
+
return True, "the verify surface allowed this key"
|
|
370
|
+
|
|
371
|
+
|
|
372
|
+
def verify_api_key(
|
|
373
|
+
key: str,
|
|
374
|
+
*,
|
|
375
|
+
base_url: str,
|
|
376
|
+
transport: VerifyTransport | None = None,
|
|
377
|
+
) -> KeyVerdict:
|
|
378
|
+
"""Verify one presented key against cloud's verify surface, failing closed.
|
|
379
|
+
|
|
380
|
+
The key is sent in the ``x-api-key`` header and nowhere else: never in the URL, never in a
|
|
381
|
+
query string, never in a log line, never in the returned verdict.
|
|
382
|
+
|
|
383
|
+
``key_id`` and ``key_format`` are computed locally *before* any request, so a denial still
|
|
384
|
+
hands back a usable non-secret identifier — the caller can report and count a rejected key
|
|
385
|
+
without ever holding the key again.
|
|
386
|
+
|
|
387
|
+
Fail closed is the default and is exhaustive. Every one of the following denies:
|
|
388
|
+
|
|
389
|
+
* any status other than 200, including 401, 403, 429, 5xx, and any 3xx redirect;
|
|
390
|
+
* any transport exception at all — connection refused, TLS failure, timeout, DNS failure;
|
|
391
|
+
* a body that is not strict canonical JSON, or is not an object;
|
|
392
|
+
* a body larger than :data:`MAX_VERIFY_RESPONSE_BYTES`;
|
|
393
|
+
* a body without the explicit allow marker, or naming a different key id.
|
|
394
|
+
|
|
395
|
+
A non-https ``base_url`` raises :class:`KeySeamError` before any request is attempted, because
|
|
396
|
+
that is a caller configuration bug and there is no safe way to proceed with it.
|
|
397
|
+
|
|
398
|
+
Raises:
|
|
399
|
+
KeySeamError: when ``key`` is not text or ``base_url`` is not an absolute https URL.
|
|
400
|
+
"""
|
|
401
|
+
|
|
402
|
+
if not isinstance(key, str):
|
|
403
|
+
raise KeySeamError("an API key must be text")
|
|
404
|
+
url = _https_url(base_url, "the verify URL")
|
|
405
|
+
key_id = hash_key_id(key)
|
|
406
|
+
key_format = detect_key_format(key)
|
|
407
|
+
client = UrlLibVerifyTransport() if transport is None else transport
|
|
408
|
+
|
|
409
|
+
# One broad boundary catch, on purpose. Anything that goes wrong past this point becomes a
|
|
410
|
+
# denying verdict with a reason naming the class of fault, never a propagating exception whose
|
|
411
|
+
# message, traceback frame, or repr could carry the key that was in scope when it was raised.
|
|
412
|
+
try:
|
|
413
|
+
status, raw = client.request(
|
|
414
|
+
"POST",
|
|
415
|
+
url,
|
|
416
|
+
{API_KEY_HEADER: key, "Accept": "application/json"},
|
|
417
|
+
b"",
|
|
418
|
+
MAX_VERIFY_RESPONSE_BYTES,
|
|
419
|
+
)
|
|
420
|
+
except Exception:
|
|
421
|
+
return _denied(key_id, key_format, "the verify surface could not be reached")
|
|
422
|
+
|
|
423
|
+
if not isinstance(status, int) or status != 200:
|
|
424
|
+
if isinstance(status, int) and 300 <= status < 400:
|
|
425
|
+
return _denied(key_id, key_format, "the verify surface redirected, which is refused")
|
|
426
|
+
return _denied(key_id, key_format, "the verify surface did not confirm this key")
|
|
427
|
+
|
|
428
|
+
if not isinstance(raw, (bytes, bytearray)):
|
|
429
|
+
return _denied(key_id, key_format, "the verify surface returned a body that is not bytes")
|
|
430
|
+
if len(raw) > MAX_VERIFY_RESPONSE_BYTES:
|
|
431
|
+
return _denied(key_id, key_format, "the verify surface returned an oversized body")
|
|
432
|
+
|
|
433
|
+
allowed, reason = _allow_body_is_valid(bytes(raw), key_id)
|
|
434
|
+
return KeyVerdict(ok=allowed, key_id=key_id, key_format=key_format, reason=reason)
|
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
"""Shared Linux/amd64 process hardening for hosted untrusted-data children."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import ctypes
|
|
6
|
+
import errno
|
|
7
|
+
import platform
|
|
8
|
+
import resource
|
|
9
|
+
import sys
|
|
10
|
+
|
|
11
|
+
_AUDIT_ARCH_X86_64 = 0xC000003E
|
|
12
|
+
_PR_SET_DUMPABLE = 4
|
|
13
|
+
_PR_GET_CHILD_SUBREAPER = 37
|
|
14
|
+
_PR_SET_CHILD_SUBREAPER = 36
|
|
15
|
+
_PR_SET_SECCOMP = 22
|
|
16
|
+
_PR_SET_NO_NEW_PRIVS = 38
|
|
17
|
+
_SECCOMP_MODE_FILTER = 2
|
|
18
|
+
_SECCOMP_RET_KILL_PROCESS = 0x80000000
|
|
19
|
+
_SECCOMP_RET_ERRNO = 0x00050000
|
|
20
|
+
_SECCOMP_RET_ALLOW = 0x7FFF0000
|
|
21
|
+
_BPF_LD_W_ABS = 0x20
|
|
22
|
+
_BPF_JMP_JEQ_K = 0x15
|
|
23
|
+
_BPF_JMP_JSET_K = 0x45
|
|
24
|
+
_BPF_RET_K = 0x06
|
|
25
|
+
_X32_SYSCALL_BIT = 0x40000000
|
|
26
|
+
_NETWORK_SYSCALLS_X86_64 = (
|
|
27
|
+
41, # socket
|
|
28
|
+
42, # connect
|
|
29
|
+
43, # accept
|
|
30
|
+
44, # sendto
|
|
31
|
+
45, # recvfrom
|
|
32
|
+
46, # sendmsg
|
|
33
|
+
47, # recvmsg
|
|
34
|
+
48, # shutdown
|
|
35
|
+
49, # bind
|
|
36
|
+
50, # listen
|
|
37
|
+
51, # getsockname
|
|
38
|
+
52, # getpeername
|
|
39
|
+
53, # socketpair
|
|
40
|
+
54, # setsockopt
|
|
41
|
+
55, # getsockopt
|
|
42
|
+
288, # accept4
|
|
43
|
+
299, # recvmmsg
|
|
44
|
+
307, # sendmmsg
|
|
45
|
+
425, # io_uring_setup
|
|
46
|
+
426, # io_uring_enter
|
|
47
|
+
427, # io_uring_register
|
|
48
|
+
)
|
|
49
|
+
_PROCESS_CREATION_SYSCALLS_X86_64 = (
|
|
50
|
+
56, # clone
|
|
51
|
+
57, # fork
|
|
52
|
+
58, # vfork
|
|
53
|
+
435, # clone3
|
|
54
|
+
)
|
|
55
|
+
_PROCESS_HANDLE_SYSCALLS_X86_64 = (
|
|
56
|
+
101, # ptrace
|
|
57
|
+
310, # process_vm_readv
|
|
58
|
+
311, # process_vm_writev
|
|
59
|
+
312, # kcmp
|
|
60
|
+
434, # pidfd_open
|
|
61
|
+
438, # pidfd_getfd
|
|
62
|
+
)
|
|
63
|
+
_PROCESS_CONTAINMENT_SYSCALLS_X86_64 = (
|
|
64
|
+
109, # setpgid
|
|
65
|
+
112, # setsid
|
|
66
|
+
272, # unshare
|
|
67
|
+
308, # setns
|
|
68
|
+
)
|
|
69
|
+
_CLONE_NAMESPACE_FLAGS = (
|
|
70
|
+
0x00000080 # CLONE_NEWTIME
|
|
71
|
+
| 0x00020000 # CLONE_NEWNS
|
|
72
|
+
| 0x02000000 # CLONE_NEWCGROUP
|
|
73
|
+
| 0x04000000 # CLONE_NEWUTS
|
|
74
|
+
| 0x08000000 # CLONE_NEWIPC
|
|
75
|
+
| 0x10000000 # CLONE_NEWUSER
|
|
76
|
+
| 0x20000000 # CLONE_NEWPID
|
|
77
|
+
| 0x40000000 # CLONE_NEWNET
|
|
78
|
+
)
|
|
79
|
+
_PROCESS_ESCAPE_SYSCALLS_X86_64 = tuple(
|
|
80
|
+
sorted(
|
|
81
|
+
(
|
|
82
|
+
*_PROCESS_CREATION_SYSCALLS_X86_64,
|
|
83
|
+
*_PROCESS_HANDLE_SYSCALLS_X86_64,
|
|
84
|
+
*_PROCESS_CONTAINMENT_SYSCALLS_X86_64,
|
|
85
|
+
)
|
|
86
|
+
)
|
|
87
|
+
)
|
|
88
|
+
_DENIED_CHILD_SYSCALLS_X86_64 = tuple(
|
|
89
|
+
sorted((*_NETWORK_SYSCALLS_X86_64, *_PROCESS_ESCAPE_SYSCALLS_X86_64))
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
class _SockFilter(ctypes.Structure):
|
|
94
|
+
_fields_ = (
|
|
95
|
+
("code", ctypes.c_ushort),
|
|
96
|
+
("jt", ctypes.c_ubyte),
|
|
97
|
+
("jf", ctypes.c_ubyte),
|
|
98
|
+
("k", ctypes.c_uint32),
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
class _SockFprog(ctypes.Structure):
|
|
103
|
+
_fields_ = (
|
|
104
|
+
("length", ctypes.c_ushort),
|
|
105
|
+
("filter", ctypes.POINTER(_SockFilter)),
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _child_subreaper_enabled() -> bool:
|
|
110
|
+
"""Return this Linux process's child-subreaper state.
|
|
111
|
+
|
|
112
|
+
A short-lived, capability-free session monitor becomes a subreaper before it creates the
|
|
113
|
+
capability child. If that child is force-killed, its sandbox process groups are reparented to
|
|
114
|
+
the monitor instead of the long-lived Service or container init, where the monitor can kill and
|
|
115
|
+
reap only those newly adopted descendants. Other platforms have no equivalent primitive and
|
|
116
|
+
do not run hosted session isolation.
|
|
117
|
+
"""
|
|
118
|
+
|
|
119
|
+
if sys.platform != "linux":
|
|
120
|
+
return False
|
|
121
|
+
value = ctypes.c_int()
|
|
122
|
+
libc = ctypes.CDLL(None, use_errno=True)
|
|
123
|
+
libc.prctl.restype = ctypes.c_int
|
|
124
|
+
libc.prctl.argtypes = (
|
|
125
|
+
ctypes.c_int,
|
|
126
|
+
ctypes.c_ulong,
|
|
127
|
+
ctypes.c_ulong,
|
|
128
|
+
ctypes.c_ulong,
|
|
129
|
+
ctypes.c_ulong,
|
|
130
|
+
)
|
|
131
|
+
if libc.prctl(_PR_GET_CHILD_SUBREAPER, ctypes.addressof(value), 0, 0, 0) != 0:
|
|
132
|
+
failure = ctypes.get_errno()
|
|
133
|
+
raise OSError(failure, "unable to inspect hosted session child-subreaper state")
|
|
134
|
+
return bool(value.value)
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def _set_child_subreaper(enabled: bool) -> None:
|
|
138
|
+
"""Set the Linux child-subreaper state used by the session process monitor."""
|
|
139
|
+
|
|
140
|
+
if sys.platform != "linux":
|
|
141
|
+
return
|
|
142
|
+
libc = ctypes.CDLL(None, use_errno=True)
|
|
143
|
+
libc.prctl.restype = ctypes.c_int
|
|
144
|
+
libc.prctl.argtypes = (
|
|
145
|
+
ctypes.c_int,
|
|
146
|
+
ctypes.c_ulong,
|
|
147
|
+
ctypes.c_ulong,
|
|
148
|
+
ctypes.c_ulong,
|
|
149
|
+
ctypes.c_ulong,
|
|
150
|
+
)
|
|
151
|
+
if libc.prctl(_PR_SET_CHILD_SUBREAPER, int(enabled), 0, 0, 0) != 0:
|
|
152
|
+
failure = ctypes.get_errno()
|
|
153
|
+
raise OSError(failure, "unable to configure hosted session child-subreaper")
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def _disable_process_dumpability() -> None:
|
|
157
|
+
"""Prevent same-UID subprocesses from reading a coordinator's live authority."""
|
|
158
|
+
|
|
159
|
+
if sys.platform != "linux":
|
|
160
|
+
return
|
|
161
|
+
libc = ctypes.CDLL(None, use_errno=True)
|
|
162
|
+
libc.prctl.restype = ctypes.c_int
|
|
163
|
+
libc.prctl.argtypes = (
|
|
164
|
+
ctypes.c_int,
|
|
165
|
+
ctypes.c_ulong,
|
|
166
|
+
ctypes.c_ulong,
|
|
167
|
+
ctypes.c_ulong,
|
|
168
|
+
ctypes.c_ulong,
|
|
169
|
+
)
|
|
170
|
+
if libc.prctl(_PR_SET_DUMPABLE, 0, 0, 0, 0) != 0:
|
|
171
|
+
failure = ctypes.get_errno()
|
|
172
|
+
raise OSError(failure, "unable to disable hosted coordinator dumpability")
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def _install_networkless_seccomp(*, allow_descendants: bool = False) -> None:
|
|
176
|
+
"""Install an exec-inherited Linux/amd64 filter denying network creation and I/O.
|
|
177
|
+
|
|
178
|
+
The one-shot Courier child also denies process creation because it has no reason to fork.
|
|
179
|
+
Long-lived research-session parser children may retain their already-bounded thread/process
|
|
180
|
+
allowance; their descendants inherit this filter and the outer container cgroup, so the
|
|
181
|
+
network denial cannot be escaped by forking. Allowing descendants never restores ptrace,
|
|
182
|
+
cross-process memory access, or pidfd handles against the same-UID trusted coordinator.
|
|
183
|
+
"""
|
|
184
|
+
|
|
185
|
+
if sys.platform != "linux" or platform.machine().lower() not in {"x86_64", "amd64"}:
|
|
186
|
+
raise OSError(errno.ENOTSUP, "hosted seccomp requires Linux/amd64")
|
|
187
|
+
instructions: list[tuple[int, int, int, int]] = [
|
|
188
|
+
(_BPF_LD_W_ABS, 0, 0, 4),
|
|
189
|
+
(_BPF_JMP_JEQ_K, 1, 0, _AUDIT_ARCH_X86_64),
|
|
190
|
+
(_BPF_RET_K, 0, 0, _SECCOMP_RET_KILL_PROCESS),
|
|
191
|
+
(_BPF_LD_W_ABS, 0, 0, 0),
|
|
192
|
+
(_BPF_JMP_JSET_K, 0, 1, _X32_SYSCALL_BIT),
|
|
193
|
+
(_BPF_RET_K, 0, 0, _SECCOMP_RET_ERRNO | errno.EPERM),
|
|
194
|
+
]
|
|
195
|
+
denied_syscalls = _DENIED_CHILD_SYSCALLS_X86_64
|
|
196
|
+
if allow_descendants:
|
|
197
|
+
denied_syscalls = tuple(
|
|
198
|
+
sorted(
|
|
199
|
+
(
|
|
200
|
+
*_NETWORK_SYSCALLS_X86_64,
|
|
201
|
+
*_PROCESS_HANDLE_SYSCALLS_X86_64,
|
|
202
|
+
*_PROCESS_CONTAINMENT_SYSCALLS_X86_64,
|
|
203
|
+
)
|
|
204
|
+
)
|
|
205
|
+
)
|
|
206
|
+
# Native runtimes need ordinary clone/fork for threads and bounded parser helpers. Keep
|
|
207
|
+
# them in the process group and namespace the trusted launcher created: classic clone is
|
|
208
|
+
# admitted only without namespace flags, while clone3 is reported unavailable so libc
|
|
209
|
+
# falls back to inspectable classic clone rather than passing an opaque struct pointer the
|
|
210
|
+
# classic seccomp filter cannot safely dereference.
|
|
211
|
+
instructions.extend(
|
|
212
|
+
(
|
|
213
|
+
(_BPF_JMP_JEQ_K, 0, 3, 56), # clone
|
|
214
|
+
(_BPF_LD_W_ABS, 0, 0, 16), # seccomp_data.args[0], low 32 bits
|
|
215
|
+
(_BPF_JMP_JSET_K, 0, 1, _CLONE_NAMESPACE_FLAGS),
|
|
216
|
+
(_BPF_RET_K, 0, 0, _SECCOMP_RET_ERRNO | errno.EPERM),
|
|
217
|
+
(_BPF_LD_W_ABS, 0, 0, 0),
|
|
218
|
+
(_BPF_JMP_JEQ_K, 0, 1, 435), # clone3
|
|
219
|
+
(_BPF_RET_K, 0, 0, _SECCOMP_RET_ERRNO | errno.ENOSYS),
|
|
220
|
+
)
|
|
221
|
+
)
|
|
222
|
+
for syscall_number in denied_syscalls:
|
|
223
|
+
instructions.extend(
|
|
224
|
+
(
|
|
225
|
+
(_BPF_JMP_JEQ_K, 0, 1, syscall_number),
|
|
226
|
+
(_BPF_RET_K, 0, 0, _SECCOMP_RET_ERRNO | errno.EPERM),
|
|
227
|
+
)
|
|
228
|
+
)
|
|
229
|
+
instructions.append((_BPF_RET_K, 0, 0, _SECCOMP_RET_ALLOW))
|
|
230
|
+
filters = (_SockFilter * len(instructions))(
|
|
231
|
+
*(_SockFilter(*instruction) for instruction in instructions)
|
|
232
|
+
)
|
|
233
|
+
program = _SockFprog(
|
|
234
|
+
length=len(filters),
|
|
235
|
+
filter=ctypes.cast(filters, ctypes.POINTER(_SockFilter)),
|
|
236
|
+
)
|
|
237
|
+
libc = ctypes.CDLL(None, use_errno=True)
|
|
238
|
+
libc.prctl.restype = ctypes.c_int
|
|
239
|
+
libc.prctl.argtypes = (
|
|
240
|
+
ctypes.c_int,
|
|
241
|
+
ctypes.c_ulong,
|
|
242
|
+
ctypes.c_ulong,
|
|
243
|
+
ctypes.c_ulong,
|
|
244
|
+
ctypes.c_ulong,
|
|
245
|
+
)
|
|
246
|
+
resource.setrlimit(resource.RLIMIT_CORE, (0, 0))
|
|
247
|
+
_disable_process_dumpability()
|
|
248
|
+
if libc.prctl(_PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) != 0:
|
|
249
|
+
failure = ctypes.get_errno()
|
|
250
|
+
raise OSError(failure, "unable to set hosted child no_new_privs")
|
|
251
|
+
if (
|
|
252
|
+
libc.prctl(
|
|
253
|
+
_PR_SET_SECCOMP,
|
|
254
|
+
_SECCOMP_MODE_FILTER,
|
|
255
|
+
ctypes.addressof(program),
|
|
256
|
+
0,
|
|
257
|
+
0,
|
|
258
|
+
)
|
|
259
|
+
!= 0
|
|
260
|
+
):
|
|
261
|
+
failure = ctypes.get_errno()
|
|
262
|
+
raise OSError(failure, "unable to install hosted child seccomp")
|