mostlyright-data 0.9.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mostlyright-data might be problematic. Click here for more details.
- mostlyright/data_harness/__init__.py +158 -0
- mostlyright/data_harness/acquisition/__init__.py +55 -0
- mostlyright/data_harness/acquisition/http.py +2773 -0
- mostlyright/data_harness/acquisition/parsing.py +809 -0
- mostlyright/data_harness/acquisition/ranges.py +495 -0
- mostlyright/data_harness/acquisition/result_download.py +360 -0
- mostlyright/data_harness/acquisition/retention_admission.py +248 -0
- mostlyright/data_harness/acquisition/sandbox.py +4888 -0
- mostlyright/data_harness/acquisition/url_policy.py +530 -0
- mostlyright/data_harness/agent_runtime.py +2743 -0
- mostlyright/data_harness/assets/logo-ink.svg +31 -0
- mostlyright/data_harness/backends/__init__.py +28 -0
- mostlyright/data_harness/backends/pandas_backend.py +350 -0
- mostlyright/data_harness/backends/polars_backend.py +366 -0
- mostlyright/data_harness/backends/protocol.py +124 -0
- mostlyright/data_harness/backends/reference.py +83 -0
- mostlyright/data_harness/backends/registry.py +55 -0
- mostlyright/data_harness/backends/restrictions.py +126 -0
- mostlyright/data_harness/canonical.py +333 -0
- mostlyright/data_harness/catalog_job.py +625 -0
- mostlyright/data_harness/cli.py +5398 -0
- mostlyright/data_harness/contracts.py +53 -0
- mostlyright/data_harness/coordinator.py +1307 -0
- mostlyright/data_harness/deploy.py +924 -0
- mostlyright/data_harness/deploy_target.py +312 -0
- mostlyright/data_harness/deployment_evidence.py +1067 -0
- mostlyright/data_harness/event_presentation.py +576 -0
- mostlyright/data_harness/events.py +2152 -0
- mostlyright/data_harness/fast_delimited.py +239 -0
- mostlyright/data_harness/fleet.py +237 -0
- mostlyright/data_harness/formats.py +236 -0
- mostlyright/data_harness/governors.py +1163 -0
- mostlyright/data_harness/hosted_bootstrap.py +972 -0
- mostlyright/data_harness/hosted_crawler.py +1115 -0
- mostlyright/data_harness/hosted_crawler_container_smoke.py +351 -0
- mostlyright/data_harness/hosted_crawler_fetch.py +423 -0
- mostlyright/data_harness/hosted_crawler_job.py +1277 -0
- mostlyright/data_harness/hosted_crawler_protocol.py +676 -0
- mostlyright/data_harness/hosted_dataset.py +1500 -0
- mostlyright/data_harness/hosted_deploy.py +3037 -0
- mostlyright/data_harness/hosted_handoff.py +62 -0
- mostlyright/data_harness/hosted_ingestion_contract.py +504 -0
- mostlyright/data_harness/hosted_ingestion_job.py +356 -0
- mostlyright/data_harness/hosted_ingestion_job_smoke.py +40 -0
- mostlyright/data_harness/hosted_session_container_smoke.py +194 -0
- mostlyright/data_harness/hosted_session_worker.py +3554 -0
- mostlyright/data_harness/hosted_session_worker_job_smoke.py +46 -0
- mostlyright/data_harness/hosted_worker.py +6784 -0
- mostlyright/data_harness/ingestion/__init__.py +56 -0
- mostlyright/data_harness/ingestion/contracts.py +461 -0
- mostlyright/data_harness/ingestion/faults.py +42 -0
- mostlyright/data_harness/ingestion/gcs_store.py +1162 -0
- mostlyright/data_harness/ingestion/spool.py +130 -0
- mostlyright/data_harness/ingestion/store.py +885 -0
- mostlyright/data_harness/key_seam.py +434 -0
- mostlyright/data_harness/linux_process_boundary.py +262 -0
- mostlyright/data_harness/local_contracts.py +2880 -0
- mostlyright/data_harness/local_search/__init__.py +5 -0
- mostlyright/data_harness/local_search/build_index.py +1087 -0
- mostlyright/data_harness/local_search/contracts.py +920 -0
- mostlyright/data_harness/local_search/query_trace.py +266 -0
- mostlyright/data_harness/local_search/retrieval.py +700 -0
- mostlyright/data_harness/local_search/sealed.py +474 -0
- mostlyright/data_harness/local_search/service.py +784 -0
- mostlyright/data_harness/nbrender/CONTRACT.md +212 -0
- mostlyright/data_harness/nbrender/__init__.py +12 -0
- mostlyright/data_harness/nbrender/chrome.py +359 -0
- mostlyright/data_harness/nbrender/code_body.py +266 -0
- mostlyright/data_harness/nbrender/document.py +407 -0
- mostlyright/data_harness/nbrender/frame.py +275 -0
- mostlyright/data_harness/nbrender/interactive.py +337 -0
- mostlyright/data_harness/nbrender/markdown_body.py +477 -0
- mostlyright/data_harness/nbrender/mr_components.py +134 -0
- mostlyright/data_harness/nbrender/outputs_data.py +595 -0
- mostlyright/data_harness/nbrender/outputs_rich.py +906 -0
- mostlyright/data_harness/nbrender/outputs_source.py +260 -0
- mostlyright/data_harness/nbrender/outputs_stage.py +176 -0
- mostlyright/data_harness/nbrender/outputs_text.py +400 -0
- mostlyright/data_harness/nbrender/parse.py +394 -0
- mostlyright/data_harness/nbrender/status.py +40 -0
- mostlyright/data_harness/nbrender/tokens.py +1295 -0
- mostlyright/data_harness/notebook.py +1710 -0
- mostlyright/data_harness/offline.py +2049 -0
- mostlyright/data_harness/operation_registry.py +1007 -0
- mostlyright/data_harness/operator_setup.py +239 -0
- mostlyright/data_harness/pipeline.py +6428 -0
- mostlyright/data_harness/plan_graph.py +2026 -0
- mostlyright/data_harness/preparation/__init__.py +104 -0
- mostlyright/data_harness/preparation/contracts.py +1017 -0
- mostlyright/data_harness/preparation/engine.py +221 -0
- mostlyright/data_harness/preparation/errors.py +14 -0
- mostlyright/data_harness/preparation/gates.py +751 -0
- mostlyright/data_harness/preparation/joins.py +574 -0
- mostlyright/data_harness/preparation/profile.py +384 -0
- mostlyright/data_harness/preparation/table.py +217 -0
- mostlyright/data_harness/preparation/transforms.py +568 -0
- mostlyright/data_harness/progress_events.py +534 -0
- mostlyright/data_harness/readers/__init__.py +46 -0
- mostlyright/data_harness/readers/containers.py +963 -0
- mostlyright/data_harness/readers/contracts.py +542 -0
- mostlyright/data_harness/readers/delimited.py +257 -0
- mostlyright/data_harness/readers/grib2/__init__.py +33 -0
- mostlyright/data_harness/readers/grib2/admission.py +722 -0
- mostlyright/data_harness/readers/grib2/decode.py +1009 -0
- mostlyright/data_harness/readers/grib2/geometry.py +1133 -0
- mostlyright/data_harness/readers/grib2/portable_math.py +501 -0
- mostlyright/data_harness/readers/json_tabular.py +485 -0
- mostlyright/data_harness/readers/registry.py +514 -0
- mostlyright/data_harness/readers/samples/README.md +110 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/cities.csv.gz +0 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/cities.csv.gz +0 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/cities.tar +0 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/cities.tar +0 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/cities.zip +0 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/expected.json +25 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/expected.json +25 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/expected.json +25 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/cities.csv +3 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/expected.json +23 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/cities.tsv +5 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/expected.json +25 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/expected.json +30 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/observations.csv +5 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/expected.json +44 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/stations.json +1 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/expected.json +48 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/observations.ndjson +4 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/cities.xlsx +0 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/expected.json +24 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/expected.json +27 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/expected.json +27 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/README.md +20 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/expected.json +55 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/gfs-2m-temperature.grib2 +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/expected.json +54 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/expected.json +54 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/hrrr-categorical-rain.grib2 +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/expected.json +54 -0
- mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
- mostlyright/data_harness/readers/samples.py +582 -0
- mostlyright/data_harness/readers/spreadsheet.py +803 -0
- mostlyright/data_harness/readers/tabular.py +510 -0
- mostlyright/data_harness/recipe.py +5321 -0
- mostlyright/data_harness/repair/__init__.py +78 -0
- mostlyright/data_harness/repair/adapters.py +274 -0
- mostlyright/data_harness/repair/contracts.py +872 -0
- mostlyright/data_harness/repair/coordinator.py +1099 -0
- mostlyright/data_harness/repair/errors.py +16 -0
- mostlyright/data_harness/review.py +2533 -0
- mostlyright/data_harness/rowset.py +283 -0
- mostlyright/data_harness/serving.py +1975 -0
- mostlyright/data_harness/serving_edge.py +590 -0
- mostlyright/data_harness/serving_http.py +1031 -0
- mostlyright/data_harness/session_probes.py +759 -0
- mostlyright/data_harness/signing.py +101 -0
- mostlyright/data_harness/source_discovery.py +898 -0
- mostlyright/data_harness/sources/__init__.py +209 -0
- mostlyright/data_harness/sources/_adapter_steps.py +213 -0
- mostlyright/data_harness/sources/adapters.py +1214 -0
- mostlyright/data_harness/sources/cadence.py +1428 -0
- mostlyright/data_harness/sources/cadence_emission.py +453 -0
- mostlyright/data_harness/sources/cadence_history.py +546 -0
- mostlyright/data_harness/sources/catalog/__init__.py +17 -0
- mostlyright/data_harness/sources/catalog/admission.py +477 -0
- mostlyright/data_harness/sources/catalog/authoring.py +1701 -0
- mostlyright/data_harness/sources/catalog/authoring_policy.py +701 -0
- mostlyright/data_harness/sources/catalog/authoring_shards.py +1217 -0
- mostlyright/data_harness/sources/catalog/bounded_io.py +231 -0
- mostlyright/data_harness/sources/catalog/channel.py +523 -0
- mostlyright/data_harness/sources/catalog/channel_client.py +296 -0
- mostlyright/data_harness/sources/catalog/contracts.py +825 -0
- mostlyright/data_harness/sources/catalog/coverage.py +137 -0
- mostlyright/data_harness/sources/catalog/delta.py +1340 -0
- mostlyright/data_harness/sources/catalog/embedding.py +532 -0
- mostlyright/data_harness/sources/catalog/entry_v2.py +1182 -0
- mostlyright/data_harness/sources/catalog/fill.py +3889 -0
- mostlyright/data_harness/sources/catalog/fill_partitions.py +459 -0
- mostlyright/data_harness/sources/catalog/fill_staging.py +1105 -0
- mostlyright/data_harness/sources/catalog/gating.py +374 -0
- mostlyright/data_harness/sources/catalog/generation_receipt.py +1607 -0
- mostlyright/data_harness/sources/catalog/harvest/__init__.py +7 -0
- mostlyright/data_harness/sources/catalog/harvest/ckan.py +384 -0
- mostlyright/data_harness/sources/catalog/harvest/datagov_v4.py +798 -0
- mostlyright/data_harness/sources/catalog/harvest/protocol.py +964 -0
- mostlyright/data_harness/sources/catalog/harvest/sdmx.py +445 -0
- mostlyright/data_harness/sources/catalog/harvest/stac.py +384 -0
- mostlyright/data_harness/sources/catalog/health.py +447 -0
- mostlyright/data_harness/sources/catalog/hosted_catalog.py +105 -0
- mostlyright/data_harness/sources/catalog/identity_history.py +1549 -0
- mostlyright/data_harness/sources/catalog/neural.py +1618 -0
- mostlyright/data_harness/sources/catalog/packed_catalog.py +2345 -0
- mostlyright/data_harness/sources/catalog/packed_retrieval.py +1517 -0
- mostlyright/data_harness/sources/catalog/packed_writer.py +2802 -0
- mostlyright/data_harness/sources/catalog/query_trace.py +1037 -0
- mostlyright/data_harness/sources/catalog/recommend.py +171 -0
- mostlyright/data_harness/sources/catalog/retrieval.py +230 -0
- mostlyright/data_harness/sources/catalog/retrieval_manifest.py +995 -0
- mostlyright/data_harness/sources/catalog/rights_decisions.py +254 -0
- mostlyright/data_harness/sources/catalog/sealed.py +560 -0
- mostlyright/data_harness/sources/catalog/search.py +230 -0
- mostlyright/data_harness/sources/catalog/streaming_delta.py +1097 -0
- mostlyright/data_harness/sources/catalog/update.py +891 -0
- mostlyright/data_harness/sources/collections.py +815 -0
- mostlyright/data_harness/sources/contracts.py +2223 -0
- mostlyright/data_harness/sources/deletion.py +761 -0
- mostlyright/data_harness/sources/fitness.py +162 -0
- mostlyright/data_harness/sources/governance.py +163 -0
- mostlyright/data_harness/sources/hosted.py +173 -0
- mostlyright/data_harness/sources/integration.py +218 -0
- mostlyright/data_harness/sources/range_reader.py +418 -0
- mostlyright/data_harness/sources/registry.py +514 -0
- mostlyright/data_harness/sources/rights_rule.py +59 -0
- mostlyright/data_harness/sources/source_cadence_vectors.v1.json +1 -0
- mostlyright/data_harness/sources/sports.py +521 -0
- mostlyright/data_harness/sources/stream.py +524 -0
- mostlyright/data_harness/sources/stream_connector.py +418 -0
- mostlyright/data_harness/sources/stream_recorder.py +1404 -0
- mostlyright/data_harness/studio_boundary.py +2019 -0
- mostlyright/data_harness/thin/__init__.py +37 -0
- mostlyright/data_harness/thin/acquire.py +1137 -0
- mostlyright/data_harness/thin/acquire_cancel.py +579 -0
- mostlyright/data_harness/thin/approvals.py +617 -0
- mostlyright/data_harness/thin/commands.py +406 -0
- mostlyright/data_harness/thin/download.py +194 -0
- mostlyright/data_harness/thin/narrative.py +589 -0
- mostlyright/data_harness/thin/parity.py +1070 -0
- mostlyright/data_harness/thin/propose.py +2759 -0
- mostlyright/data_harness/thin/research.py +1663 -0
- mostlyright/data_harness/thin/router.py +924 -0
- mostlyright/data_harness/thin/runs.py +519 -0
- mostlyright/data_harness/thin/session.py +281 -0
- mostlyright/data_harness/thin/stream.py +501 -0
- mostlyright/data_harness/thin/transport.py +187 -0
- mostlyright/data_harness/thin/vocabulary.py +368 -0
- mostlyright/data_harness/thin/workers.py +164 -0
- mostlyright/data_harness/ucum/TABLE-PIN.json +40 -0
- mostlyright/data_harness/ucum/ucum-subset.v1.json +632 -0
- mostlyright/data_harness/unit_flow.py +927 -0
- mostlyright/data_harness/units.py +572 -0
- mostlyright/data_harness/ux/__init__.py +9 -0
- mostlyright/data_harness/ux/approve.py +485 -0
- mostlyright/data_harness/ux/author_yaml.py +597 -0
- mostlyright/data_harness/ux/cloud_auth.py +447 -0
- mostlyright/data_harness/ux/commands/__init__.py +260 -0
- mostlyright/data_harness/ux/commands/approve.py +136 -0
- mostlyright/data_harness/ux/commands/auth.py +744 -0
- mostlyright/data_harness/ux/commands/author.py +79 -0
- mostlyright/data_harness/ux/commands/catalog_author.py +403 -0
- mostlyright/data_harness/ux/commands/catalog_fill.py +523 -0
- mostlyright/data_harness/ux/commands/catalog_harvest.py +545 -0
- mostlyright/data_harness/ux/commands/catalog_publish.py +1838 -0
- mostlyright/data_harness/ux/commands/catalog_search.py +71 -0
- mostlyright/data_harness/ux/commands/catalog_update.py +437 -0
- mostlyright/data_harness/ux/commands/deploy.py +134 -0
- mostlyright/data_harness/ux/commands/deploy_dataset.py +98 -0
- mostlyright/data_harness/ux/commands/deploy_plan.py +105 -0
- mostlyright/data_harness/ux/commands/deploy_status.py +104 -0
- mostlyright/data_harness/ux/commands/diff.py +74 -0
- mostlyright/data_harness/ux/commands/index.py +84 -0
- mostlyright/data_harness/ux/commands/inventory.py +47 -0
- mostlyright/data_harness/ux/commands/list_builds.py +143 -0
- mostlyright/data_harness/ux/commands/login.py +63 -0
- mostlyright/data_harness/ux/commands/peek.py +236 -0
- mostlyright/data_harness/ux/commands/plan_check.py +90 -0
- mostlyright/data_harness/ux/commands/preflight.py +97 -0
- mostlyright/data_harness/ux/commands/record.py +107 -0
- mostlyright/data_harness/ux/commands/review_setup.py +47 -0
- mostlyright/data_harness/ux/commands/search.py +440 -0
- mostlyright/data_harness/ux/commands/show.py +61 -0
- mostlyright/data_harness/ux/commands/whoami.py +37 -0
- mostlyright/data_harness/ux/credential_native.py +551 -0
- mostlyright/data_harness/ux/credential_store.py +1055 -0
- mostlyright/data_harness/ux/credentials.py +631 -0
- mostlyright/data_harness/ux/diffing.py +444 -0
- mostlyright/data_harness/ux/headline.py +671 -0
- mostlyright/data_harness/ux/hosted_acquisition.py +974 -0
- mostlyright/data_harness/ux/hosted_run_status.py +619 -0
- mostlyright/data_harness/ux/inventory.py +427 -0
- mostlyright/data_harness/ux/local_review.py +375 -0
- mostlyright/data_harness/ux/login.py +691 -0
- mostlyright/data_harness/ux/path_kind.py +147 -0
- mostlyright/data_harness/ux/peek.py +1000 -0
- mostlyright/data_harness/ux/plain_file.py +178 -0
- mostlyright/data_harness/ux/plan_check.py +311 -0
- mostlyright/data_harness/ux/preflight.py +918 -0
- mostlyright/data_harness/ux/readers.py +1124 -0
- mostlyright/data_harness/ux/remediation.py +2195 -0
- mostlyright/data_harness/ux/render.py +657 -0
- mostlyright/data_harness/ux/workload.py +1077 -0
- mostlyright/data_harness/viewer.py +3713 -0
- mostlyright/data_harness/visual_run/__init__.py +83 -0
- mostlyright/data_harness/visual_run/authoring.py +235 -0
- mostlyright/data_harness/visual_run/contracts.py +673 -0
- mostlyright/data_harness/visual_run/materialize.py +486 -0
- mostlyright/data_harness/visual_run/observations.py +874 -0
- mostlyright/data_harness/visual_run/query.py +259 -0
- mostlyright/data_harness/visual_run/reducer.py +280 -0
- mostlyright/data_harness/visual_run/sdk.py +892 -0
- mostlyright/data_harness/visual_run/store.py +584 -0
- mostlyright/data_harness/visual_run/transport.py +239 -0
- mostlyright/data_harness/watch.py +2999 -0
- mostlyright_data-0.9.0.dist-info/METADATA +607 -0
- mostlyright_data-0.9.0.dist-info/RECORD +314 -0
- mostlyright_data-0.9.0.dist-info/WHEEL +4 -0
- mostlyright_data-0.9.0.dist-info/entry_points.txt +12 -0
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
"""``mr-data catalog-search`` over one exact verified sealed catalog."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
from mostlyright.data_harness.sources.catalog.gating import CatalogRequirement
|
|
10
|
+
from mostlyright.data_harness.sources.catalog.search import search_catalog
|
|
11
|
+
from mostlyright.data_harness.ux.commands import CommandSpec
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def add_arguments(parser: argparse.ArgumentParser) -> None:
|
|
15
|
+
parser.add_argument(
|
|
16
|
+
"--catalog", dest="dir", type=Path, required=True, help="the sealed catalog folder"
|
|
17
|
+
)
|
|
18
|
+
parser.add_argument("--catalog-sha256", required=True, help="the exact SHA-256 of catalog.json")
|
|
19
|
+
parser.add_argument("--question", required=True, help="the source question to answer")
|
|
20
|
+
parser.add_argument(
|
|
21
|
+
"--limit",
|
|
22
|
+
type=int,
|
|
23
|
+
choices=range(1, 21),
|
|
24
|
+
default=5,
|
|
25
|
+
help="the maximum number of ranked sources to inspect",
|
|
26
|
+
)
|
|
27
|
+
parser.add_argument(
|
|
28
|
+
"--format",
|
|
29
|
+
dest="formats",
|
|
30
|
+
action="append",
|
|
31
|
+
default=None,
|
|
32
|
+
help="a required data format; repeat to allow more than one",
|
|
33
|
+
)
|
|
34
|
+
parser.add_argument(
|
|
35
|
+
"--max-update-seconds",
|
|
36
|
+
type=int,
|
|
37
|
+
default=31_536_000,
|
|
38
|
+
help="the slowest acceptable source update interval in seconds",
|
|
39
|
+
)
|
|
40
|
+
parser.add_argument(
|
|
41
|
+
"--intended-use", default="analysis", help="how the selected source will be used"
|
|
42
|
+
)
|
|
43
|
+
parser.add_argument(
|
|
44
|
+
"--lifecycle", default="candidate", help="the lifecycle stage required of the source"
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def run(args: argparse.Namespace) -> dict[str, Any]:
|
|
49
|
+
requirement = CatalogRequirement(
|
|
50
|
+
intended_use=args.intended_use,
|
|
51
|
+
lifecycle=args.lifecycle,
|
|
52
|
+
required_data_formats=tuple(args.formats or ("csv",)),
|
|
53
|
+
max_update_frequency_seconds=args.max_update_seconds,
|
|
54
|
+
require_open_access=True,
|
|
55
|
+
require_live=True,
|
|
56
|
+
)
|
|
57
|
+
return search_catalog(
|
|
58
|
+
catalog_root=args.dir,
|
|
59
|
+
catalog_manifest_sha256=args.catalog_sha256,
|
|
60
|
+
question=args.question,
|
|
61
|
+
requirement=requirement,
|
|
62
|
+
limit=args.limit,
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
SPEC = CommandSpec(
|
|
67
|
+
name="catalog-search",
|
|
68
|
+
help="search one exact verified sealed source catalog",
|
|
69
|
+
add_arguments=add_arguments,
|
|
70
|
+
run=run,
|
|
71
|
+
)
|
|
@@ -0,0 +1,437 @@
|
|
|
1
|
+
"""``mr-data catalog-update`` installs one published catalog release on this computer.
|
|
2
|
+
|
|
3
|
+
The release is fetched from the address given -- or read from a file already here -- and nothing
|
|
4
|
+
about it is trusted until its bytes match the published SHA-256 the person typed. After that the
|
|
5
|
+
archive is screened, the generation is written next to the ones already installed, the real store
|
|
6
|
+
verification runs over it before it is made visible, and the recorded state advances only when
|
|
7
|
+
the release descends from the generation this computer already has. When the recorded state
|
|
8
|
+
already names the typed digest, the installed folder is re-verified and nothing is fetched.
|
|
9
|
+
|
|
10
|
+
THE THIRD WAY IN: ``--channel-url`` with ``--channel-manifest-sha256``. Typing an archive URL and
|
|
11
|
+
its digest is what one person does once. Tracking a channel is what everybody else does, and it is
|
|
12
|
+
what the hosted workers do (see ``sources.catalog.channel``): the pinned manifest names the
|
|
13
|
+
current generation, and the digest the person types is the manifest's rather than the archive's.
|
|
14
|
+
The trust story is unchanged -- one digest typed by a person, everything else derived from it --
|
|
15
|
+
and the installation itself is the identical code path, so a channel install and a hand-typed
|
|
16
|
+
install produce the same generation folder and the same recorded state.
|
|
17
|
+
|
|
18
|
+
This command needs the local execution lane, and so does ``catalog-search``: verifying an
|
|
19
|
+
installed generation *is* loading it with the real store reader. A thin install therefore refuses
|
|
20
|
+
this command by name and says what to install, which is the correct answer -- there is nothing for
|
|
21
|
+
a catalogue to do on a machine that cannot search it. See ``docs/CATALOG-CHANNEL.md``.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
import argparse
|
|
27
|
+
import os
|
|
28
|
+
from collections.abc import Mapping
|
|
29
|
+
from pathlib import Path
|
|
30
|
+
from typing import Any
|
|
31
|
+
from urllib.parse import urlsplit
|
|
32
|
+
|
|
33
|
+
from mostlyright.data_harness.sources.catalog.update import (
|
|
34
|
+
MAX_RELEASE_ARCHIVE_BYTES,
|
|
35
|
+
confirm_already_current,
|
|
36
|
+
install_release,
|
|
37
|
+
)
|
|
38
|
+
from mostlyright.data_harness.sources.contracts import SourceContractError
|
|
39
|
+
from mostlyright.data_harness.ux.commands import CommandSpec
|
|
40
|
+
from mostlyright.data_harness.ux.credentials import CREDENTIALS_DIR_NAME
|
|
41
|
+
from mostlyright.data_harness.ux.plain_file import PlainFileRefusal, open_plain_file, read_bounded
|
|
42
|
+
|
|
43
|
+
# GitHub serves release assets through one redirect onto its object hosts, so an address on the
|
|
44
|
+
# release page implies those hostnames. Any other address stands alone: the allowlist is the
|
|
45
|
+
# host the person named, plus every host they explicitly allowed with --allow-host.
|
|
46
|
+
_GITHUB_HOSTNAMES = frozenset({"github.com", "api.github.com", "www.github.com"})
|
|
47
|
+
_GITHUB_ASSET_HOSTNAMES = (
|
|
48
|
+
"objects.githubusercontent.com",
|
|
49
|
+
"release-assets.githubusercontent.com",
|
|
50
|
+
)
|
|
51
|
+
_RELEASE_MEDIA_TYPES = ("application/zip", "application/octet-stream")
|
|
52
|
+
|
|
53
|
+
#: A bearer token is a few kilobytes at most; anything larger is not a token.
|
|
54
|
+
MAX_CHANNEL_TOKEN_BYTES = 8192
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def add_arguments(parser: argparse.ArgumentParser) -> None:
|
|
58
|
+
parser.add_argument(
|
|
59
|
+
"--catalog",
|
|
60
|
+
dest="dir",
|
|
61
|
+
type=Path,
|
|
62
|
+
default=Path.home() / CREDENTIALS_DIR_NAME / "catalog",
|
|
63
|
+
help="the folder this computer keeps its installed catalog in",
|
|
64
|
+
)
|
|
65
|
+
parser.add_argument(
|
|
66
|
+
"--archive",
|
|
67
|
+
type=Path,
|
|
68
|
+
default=None,
|
|
69
|
+
help="a release archive already on this computer",
|
|
70
|
+
)
|
|
71
|
+
parser.add_argument(
|
|
72
|
+
"--release-url",
|
|
73
|
+
default=None,
|
|
74
|
+
help="the https address of one published release archive",
|
|
75
|
+
)
|
|
76
|
+
parser.add_argument(
|
|
77
|
+
"--channel-url",
|
|
78
|
+
default=None,
|
|
79
|
+
help="the https address of one release channel's channel.json",
|
|
80
|
+
)
|
|
81
|
+
parser.add_argument(
|
|
82
|
+
"--channel-manifest-sha256",
|
|
83
|
+
default=None,
|
|
84
|
+
help="the published SHA-256 of that channel manifest",
|
|
85
|
+
)
|
|
86
|
+
parser.add_argument(
|
|
87
|
+
"--channel-token-file",
|
|
88
|
+
type=Path,
|
|
89
|
+
default=None,
|
|
90
|
+
help="a file holding one bearer token, for a channel that is not world-readable",
|
|
91
|
+
)
|
|
92
|
+
parser.add_argument(
|
|
93
|
+
"--archive-sha256",
|
|
94
|
+
default=None,
|
|
95
|
+
help="the published SHA-256 of the release archive; not used with --channel-url",
|
|
96
|
+
)
|
|
97
|
+
parser.add_argument(
|
|
98
|
+
"--allow-host",
|
|
99
|
+
dest="allow_hosts",
|
|
100
|
+
action="append",
|
|
101
|
+
default=None,
|
|
102
|
+
help="one more hostname the download may reach; repeat to allow more than one",
|
|
103
|
+
)
|
|
104
|
+
parser.add_argument(
|
|
105
|
+
"--replace",
|
|
106
|
+
action="store_true",
|
|
107
|
+
help="accept a release that does not descend from the installed generation",
|
|
108
|
+
)
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _allowed_hostnames(url: str, extra: tuple[str, ...]) -> tuple[str, ...]:
|
|
112
|
+
"""The closed set of hosts one release download may reach, from the address and the caller."""
|
|
113
|
+
|
|
114
|
+
hostname = urlsplit(url).hostname
|
|
115
|
+
if not hostname:
|
|
116
|
+
raise SourceContractError(
|
|
117
|
+
"CATALOG_UPDATE_URL", url, "must be one https address with a hostname"
|
|
118
|
+
)
|
|
119
|
+
allowed = (hostname,)
|
|
120
|
+
if hostname in _GITHUB_HOSTNAMES:
|
|
121
|
+
allowed = allowed + _GITHUB_ASSET_HOSTNAMES
|
|
122
|
+
for host in extra:
|
|
123
|
+
if host not in allowed:
|
|
124
|
+
allowed = (*allowed, host)
|
|
125
|
+
return allowed
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _fetch(url: str, allow_hosts: tuple[str, ...]) -> bytes:
|
|
129
|
+
from mostlyright.data_harness.acquisition.http import (
|
|
130
|
+
PinnedHttpsRetriever,
|
|
131
|
+
RetrievalLimits,
|
|
132
|
+
StdlibPinnedTransport,
|
|
133
|
+
)
|
|
134
|
+
from mostlyright.data_harness.acquisition.url_policy import EgressPolicy, SystemResolver
|
|
135
|
+
|
|
136
|
+
retriever = PinnedHttpsRetriever(
|
|
137
|
+
resolver=SystemResolver(),
|
|
138
|
+
egress_policy=EgressPolicy(allowed_hostnames=_allowed_hostnames(url, allow_hosts)),
|
|
139
|
+
transport=StdlibPinnedTransport(),
|
|
140
|
+
limits=RetrievalLimits(
|
|
141
|
+
max_response_bytes=MAX_RELEASE_ARCHIVE_BYTES,
|
|
142
|
+
max_redirects=3,
|
|
143
|
+
max_requests=4,
|
|
144
|
+
allowed_media_types=_RELEASE_MEDIA_TYPES,
|
|
145
|
+
# The default transport budget suits metadata pages; a release archive is up to a
|
|
146
|
+
# gibibyte, so the read and total budgets are raised to their ceilings.
|
|
147
|
+
read_timeout_seconds=300.0,
|
|
148
|
+
total_timeout_seconds=600.0,
|
|
149
|
+
),
|
|
150
|
+
)
|
|
151
|
+
return retriever.retrieve(url).content
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
class PolicyPinnedChannelTransport:
|
|
155
|
+
"""The channel's two GETs, over the egress hardening ``--release-url`` already got.
|
|
156
|
+
|
|
157
|
+
WHY THIS EXISTS. One command had two fetch paths with materially different egress properties.
|
|
158
|
+
``--release-url`` went through :class:`PinnedHttpsRetriever`: DNS resolved once into an
|
|
159
|
+
approved peer set, the connection made to one of those addresses with the hostname verified in
|
|
160
|
+
TLS, metadata and internal hosts denied by name, obfuscated numeric hosts denied, and a closed
|
|
161
|
+
host allowlist. ``--channel-url`` went through the standard library with none of it. Digest
|
|
162
|
+
pinning made the *content* untrustable-until-verified either way, which is why nothing had
|
|
163
|
+
gone wrong -- but "the digest would have caught it" is a statement about the bytes that
|
|
164
|
+
arrived, not about which host this computer was persuaded to talk to, and the channel path
|
|
165
|
+
could be steered at a link-local metadata address by an address a person typed.
|
|
166
|
+
|
|
167
|
+
So the channel transport is the pinned one, and the seam is the ``ChannelTransport`` protocol
|
|
168
|
+
that ``sources.catalog.channel_client`` already accepts. Nothing in that module changes: the
|
|
169
|
+
worker keeps the standard-library transport it needs to read its own instance metadata, which
|
|
170
|
+
this policy would deny by name -- and denying it is the correct behaviour for a person's
|
|
171
|
+
laptop and the wrong behaviour for the worker, which is precisely why the choice is made here,
|
|
172
|
+
at the command, rather than there.
|
|
173
|
+
|
|
174
|
+
WHAT IS NOT CLAIMED, STATED PRECISELY. This is the pinned TRANSPORT, not
|
|
175
|
+
:class:`PinnedHttpsRetriever`. The retriever is what enforces the redirect budget, the request
|
|
176
|
+
budget and the media-type allowlist, and none of the three is reimplemented here. Two of them
|
|
177
|
+
are unnecessary: ``StdlibPinnedTransport`` speaks ``http.client`` and cannot follow a redirect
|
|
178
|
+
at all, so a 302 comes back AS a 302 and ``channel_client._get`` refuses it on the status --
|
|
179
|
+
which is a stronger guarantee than a budget of zero and does not depend on one. The third is
|
|
180
|
+
genuinely absent: nothing here checks ``Content-Type``, and it would be the wrong check anyway,
|
|
181
|
+
because the two objects on this route are a JSON manifest and a zip archive. What stands in its
|
|
182
|
+
place is what always stood there -- both are refused unless their bytes hash to a digest that
|
|
183
|
+
was pinned before the request was made.
|
|
184
|
+
|
|
185
|
+
So the properties this transport does add, and the only ones it adds, are the egress ones: one
|
|
186
|
+
DNS epoch bound into an approved peer set, the connection made to one of those addresses with
|
|
187
|
+
the hostname verified in TLS, metadata and internal hosts denied by name, obfuscated numeric
|
|
188
|
+
hosts denied, and a closed allowlist. Those are exactly the ones the channel path did not have.
|
|
189
|
+
"""
|
|
190
|
+
|
|
191
|
+
def __init__(self, allowed_hostnames: tuple[str, ...]) -> None:
|
|
192
|
+
from mostlyright.data_harness.acquisition.http import StdlibPinnedTransport
|
|
193
|
+
from mostlyright.data_harness.acquisition.url_policy import EgressPolicy, SystemResolver
|
|
194
|
+
|
|
195
|
+
self._resolver = SystemResolver()
|
|
196
|
+
self._egress_policy = EgressPolicy(allowed_hostnames=allowed_hostnames)
|
|
197
|
+
self._transport = StdlibPinnedTransport()
|
|
198
|
+
|
|
199
|
+
def request(
|
|
200
|
+
self,
|
|
201
|
+
method: str,
|
|
202
|
+
url: str,
|
|
203
|
+
headers: Mapping[str, str],
|
|
204
|
+
body: bytes | None,
|
|
205
|
+
maximum: int,
|
|
206
|
+
) -> tuple[int, bytes, Mapping[str, str]]:
|
|
207
|
+
from mostlyright.data_harness.acquisition.http import (
|
|
208
|
+
RetrievalLimits,
|
|
209
|
+
TransportFailureError,
|
|
210
|
+
)
|
|
211
|
+
from mostlyright.data_harness.acquisition.url_policy import validate_public_https_url
|
|
212
|
+
|
|
213
|
+
if method != "GET" or body is not None:
|
|
214
|
+
raise SourceContractError(
|
|
215
|
+
"CATALOG_UPDATE_URL", url, "a channel is read with one GET and no request body"
|
|
216
|
+
)
|
|
217
|
+
target = validate_public_https_url(
|
|
218
|
+
url, resolver=self._resolver, egress_policy=self._egress_policy
|
|
219
|
+
)
|
|
220
|
+
# ⚠ ONLY THE FIELDS THIS TRANSPORT ACTUALLY READS. `StdlibPinnedTransport.request` reads
|
|
221
|
+
# `max_response_bytes` and the three timeouts and nothing else -- the redirect, request and
|
|
222
|
+
# media-type budgets belong to the retriever, which is not on this path. Setting them here
|
|
223
|
+
# would be configuration that looks enforced and is not, which is worse than a gap because
|
|
224
|
+
# the next reader stops looking for one. The byte ceiling is the caller's own per-route
|
|
225
|
+
# bound; the read and total budgets are the ones the hand-typed release download uses,
|
|
226
|
+
# because an archive is up to a gibibyte and the metadata defaults are for a JSON page.
|
|
227
|
+
limits = RetrievalLimits(
|
|
228
|
+
max_response_bytes=maximum,
|
|
229
|
+
read_timeout_seconds=300.0,
|
|
230
|
+
total_timeout_seconds=600.0,
|
|
231
|
+
)
|
|
232
|
+
# Every approved address from the one DNS epoch, in turn. A peer that fails before it
|
|
233
|
+
# answers is a peer, not a verdict; anything else -- a refused hostname, a response over
|
|
234
|
+
# the ceiling -- is the answer and stops here.
|
|
235
|
+
failures: list[str] = []
|
|
236
|
+
for approved_ip in target.approved_ips:
|
|
237
|
+
try:
|
|
238
|
+
response = self._transport.request(
|
|
239
|
+
target, approved_ip=approved_ip, headers=dict(headers), limits=limits
|
|
240
|
+
)
|
|
241
|
+
except TransportFailureError as error:
|
|
242
|
+
# Each peer's own reason, kept. "No address answered" on its own cannot tell a
|
|
243
|
+
# refused connection from a failed handshake from a body cut mid-transfer, and
|
|
244
|
+
# those are three different things to go and look at.
|
|
245
|
+
failures.append(f"{approved_ip}: {error.detail}")
|
|
246
|
+
continue
|
|
247
|
+
return response.status, response.body, dict(response.headers)
|
|
248
|
+
raise SourceContractError(
|
|
249
|
+
"CATALOG_CHANNEL_FETCH",
|
|
250
|
+
url,
|
|
251
|
+
"no approved address for this channel host answered -- " + "; ".join(failures),
|
|
252
|
+
)
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def _read_local(path: Path) -> bytes:
|
|
256
|
+
"""Read the release archive the person named, through the one plain-file rule.
|
|
257
|
+
|
|
258
|
+
A second hard link to the archive is fine: nothing about the file is trusted anyway until
|
|
259
|
+
its bytes match the digest the person typed next to it.
|
|
260
|
+
"""
|
|
261
|
+
|
|
262
|
+
try:
|
|
263
|
+
descriptor, size = open_plain_file(path)
|
|
264
|
+
except PlainFileRefusal as refusal:
|
|
265
|
+
raise SourceContractError(
|
|
266
|
+
"CATALOG_UPDATE_ARCHIVE", str(path), "release archive could not be read as one file"
|
|
267
|
+
) from refusal
|
|
268
|
+
try:
|
|
269
|
+
if size > MAX_RELEASE_ARCHIVE_BYTES:
|
|
270
|
+
raise SourceContractError(
|
|
271
|
+
"CATALOG_UPDATE_ARCHIVE", str(path), "release archive is larger than the bound"
|
|
272
|
+
)
|
|
273
|
+
raw = read_bounded(descriptor, max_bytes=MAX_RELEASE_ARCHIVE_BYTES)
|
|
274
|
+
finally:
|
|
275
|
+
os.close(descriptor)
|
|
276
|
+
if len(raw) > MAX_RELEASE_ARCHIVE_BYTES:
|
|
277
|
+
raise SourceContractError(
|
|
278
|
+
"CATALOG_UPDATE_ARCHIVE", str(path), "release archive is larger than the bound"
|
|
279
|
+
)
|
|
280
|
+
return raw
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
def _channel_authorization(path: Path | None) -> str | None:
|
|
284
|
+
"""Read one bearer token out of a file, for a channel whose bucket is not world-readable.
|
|
285
|
+
|
|
286
|
+
A file rather than an argument or an environment variable, for the reason every other
|
|
287
|
+
credential in this product is a file: an argument is in the process table and the shell
|
|
288
|
+
history, and a token that reaches object storage is worth exactly as much as the objects it
|
|
289
|
+
reads. The token itself never appears in a refusal, a receipt, or a URL.
|
|
290
|
+
"""
|
|
291
|
+
|
|
292
|
+
if path is None:
|
|
293
|
+
return None
|
|
294
|
+
try:
|
|
295
|
+
descriptor, size = open_plain_file(path)
|
|
296
|
+
except PlainFileRefusal as refusal:
|
|
297
|
+
raise SourceContractError(
|
|
298
|
+
"CATALOG_UPDATE_TOKEN", str(path), "channel token could not be read as one file"
|
|
299
|
+
) from refusal
|
|
300
|
+
try:
|
|
301
|
+
if size > MAX_CHANNEL_TOKEN_BYTES:
|
|
302
|
+
raise SourceContractError(
|
|
303
|
+
"CATALOG_UPDATE_TOKEN", str(path), "channel token is larger than the bound"
|
|
304
|
+
)
|
|
305
|
+
raw = read_bounded(descriptor, max_bytes=MAX_CHANNEL_TOKEN_BYTES)
|
|
306
|
+
finally:
|
|
307
|
+
os.close(descriptor)
|
|
308
|
+
token = raw.decode("ascii", errors="replace").strip()
|
|
309
|
+
if not token or not all(0x21 <= ord(character) <= 0x7E for character in token):
|
|
310
|
+
raise SourceContractError(
|
|
311
|
+
"CATALOG_UPDATE_TOKEN", str(path), "channel token must be one printable ASCII line"
|
|
312
|
+
)
|
|
313
|
+
return token if token.lower().startswith("bearer ") else "Bearer " + token
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
def channel_transport(channel_url: str) -> PolicyPinnedChannelTransport:
|
|
317
|
+
"""The hardened transport one channel install fetches through.
|
|
318
|
+
|
|
319
|
+
A named function rather than an inline construction, so a test can substitute the seam the way
|
|
320
|
+
it already substitutes the standard-library one, and so the allowlist this path uses is
|
|
321
|
+
visible in one place: the pinned channel's own host, and nothing else at all.
|
|
322
|
+
|
|
323
|
+
⚠ NOT ``_allowed_hostnames``. That builder widens the set to GitHub's asset hosts when the
|
|
324
|
+
address is a GitHub one, which is right for a hand-typed release: a GitHub release URL is
|
|
325
|
+
served by one redirect onto those hosts, so an allowlist that omitted them would refuse the
|
|
326
|
+
ordinary case. This path follows no redirect and derives the archive address from the pinned
|
|
327
|
+
channel address, so a second host is unreachable from here -- and an allowlist naming a host
|
|
328
|
+
that cannot be reached is an allowlist whose written promise is not the one it keeps.
|
|
329
|
+
"""
|
|
330
|
+
|
|
331
|
+
hostname = urlsplit(channel_url).hostname
|
|
332
|
+
if not hostname:
|
|
333
|
+
raise SourceContractError(
|
|
334
|
+
"CATALOG_UPDATE_URL", channel_url, "must be one https address with a hostname"
|
|
335
|
+
)
|
|
336
|
+
return PolicyPinnedChannelTransport((hostname,))
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
def _run_channel(args: argparse.Namespace) -> dict[str, Any]:
|
|
340
|
+
"""Install whatever generation the pinned channel manifest currently names."""
|
|
341
|
+
|
|
342
|
+
from mostlyright.data_harness.sources.catalog.channel import build_catalog_channel_pin
|
|
343
|
+
from mostlyright.data_harness.sources.catalog.channel_client import install_from_channel
|
|
344
|
+
|
|
345
|
+
if args.archive is not None or args.release_url is not None or args.archive_sha256 is not None:
|
|
346
|
+
raise SourceContractError(
|
|
347
|
+
"CATALOG_UPDATE_URL",
|
|
348
|
+
"arguments",
|
|
349
|
+
"--channel-url names the release on its own; drop --archive, --release-url "
|
|
350
|
+
"and --archive-sha256",
|
|
351
|
+
)
|
|
352
|
+
if args.allow_hosts:
|
|
353
|
+
# ⚠ REFUSED, NOT HONOURED, AND NOT IGNORED EITHER. On the hand-typed path --allow-host
|
|
354
|
+
# widens a set that a redirect can actually reach. A channel reaches exactly two addresses
|
|
355
|
+
# and derives both from the pinned channel URL -- the manifest's own address, and the
|
|
356
|
+
# archive one object over in the same folder -- and it follows no redirect, so a further
|
|
357
|
+
# host is a host this command will never open a connection to. Honouring the flag would
|
|
358
|
+
# widen a policy to admit something unreachable, which reads to whoever typed it as though
|
|
359
|
+
# the host had been allowed. Saying so is the only answer that is true.
|
|
360
|
+
raise SourceContractError(
|
|
361
|
+
"CATALOG_UPDATE_URL",
|
|
362
|
+
"arguments",
|
|
363
|
+
"--allow-host means nothing with --channel-url: the manifest and the archive are both "
|
|
364
|
+
"fetched from the host the pinned channel address names, and no redirect is followed",
|
|
365
|
+
)
|
|
366
|
+
if args.channel_manifest_sha256 is None:
|
|
367
|
+
raise SourceContractError(
|
|
368
|
+
"CATALOG_UPDATE_URL",
|
|
369
|
+
"arguments",
|
|
370
|
+
"--channel-url needs --channel-manifest-sha256, the digest the channel published",
|
|
371
|
+
)
|
|
372
|
+
pin = build_catalog_channel_pin(
|
|
373
|
+
channel_url=args.channel_url, manifest_sha256=args.channel_manifest_sha256
|
|
374
|
+
)
|
|
375
|
+
report = install_from_channel(
|
|
376
|
+
Path(args.dir),
|
|
377
|
+
pin,
|
|
378
|
+
transport=channel_transport(pin.channel_url),
|
|
379
|
+
authorization=_channel_authorization(args.channel_token_file),
|
|
380
|
+
replace=args.replace,
|
|
381
|
+
)
|
|
382
|
+
payload = report.to_dict()
|
|
383
|
+
payload["channel"] = {
|
|
384
|
+
"channel_url": pin.channel_url,
|
|
385
|
+
"manifest_sha256": pin.manifest_sha256,
|
|
386
|
+
"pin_selection_id": pin.selection_id,
|
|
387
|
+
}
|
|
388
|
+
return payload
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
def run(args: argparse.Namespace) -> dict[str, Any]:
|
|
392
|
+
if args.channel_url is not None:
|
|
393
|
+
return _run_channel(args)
|
|
394
|
+
for name, flag in (
|
|
395
|
+
("channel_manifest_sha256", "--channel-manifest-sha256"),
|
|
396
|
+
("channel_token_file", "--channel-token-file"),
|
|
397
|
+
):
|
|
398
|
+
if getattr(args, name) is not None:
|
|
399
|
+
raise SourceContractError(
|
|
400
|
+
"CATALOG_UPDATE_URL",
|
|
401
|
+
"arguments",
|
|
402
|
+
f"{flag} only means something with --channel-url",
|
|
403
|
+
)
|
|
404
|
+
if args.archive_sha256 is None:
|
|
405
|
+
raise SourceContractError(
|
|
406
|
+
"CATALOG_UPDATE_DIGEST_SPELLING",
|
|
407
|
+
"arguments",
|
|
408
|
+
"--archive-sha256 is required unless the release comes from --channel-url",
|
|
409
|
+
)
|
|
410
|
+
if (args.archive is None) == (args.release_url is None):
|
|
411
|
+
raise SourceContractError(
|
|
412
|
+
"CATALOG_UPDATE_URL",
|
|
413
|
+
"arguments",
|
|
414
|
+
"exactly one of --archive and --release-url names the release",
|
|
415
|
+
)
|
|
416
|
+
current = confirm_already_current(Path(args.dir), expected_archive_sha256=args.archive_sha256)
|
|
417
|
+
if current is not None:
|
|
418
|
+
return current.to_dict()
|
|
419
|
+
if args.archive is not None:
|
|
420
|
+
raw = _read_local(args.archive)
|
|
421
|
+
else:
|
|
422
|
+
raw = _fetch(args.release_url, tuple(args.allow_hosts or ()))
|
|
423
|
+
report = install_release(
|
|
424
|
+
Path(args.dir),
|
|
425
|
+
raw,
|
|
426
|
+
expected_archive_sha256=args.archive_sha256,
|
|
427
|
+
replace=args.replace,
|
|
428
|
+
)
|
|
429
|
+
return report.to_dict()
|
|
430
|
+
|
|
431
|
+
|
|
432
|
+
SPEC = CommandSpec(
|
|
433
|
+
name="catalog-update",
|
|
434
|
+
help="install one published catalog release after checking its exact fingerprint",
|
|
435
|
+
add_arguments=add_arguments,
|
|
436
|
+
run=run,
|
|
437
|
+
)
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
"""``mr-data deploy`` -- stage and activate one exact verified Build."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
from collections.abc import Mapping
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
from mostlyright.data_harness.ux.commands import CommandSpec
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def add_arguments(parser: argparse.ArgumentParser) -> None:
|
|
14
|
+
parser.description = (
|
|
15
|
+
"Stage one exact verified Build in Studio and activate it as the authenticated Editor."
|
|
16
|
+
)
|
|
17
|
+
parser.add_argument("run_dir", type=Path, help="the folder holding the sealed Build")
|
|
18
|
+
parser.add_argument(
|
|
19
|
+
"--previous-run-dir",
|
|
20
|
+
type=Path,
|
|
21
|
+
help="required to update a Dataset that is already live: the run directory whose "
|
|
22
|
+
"deployment published it, and whose Build this version follows",
|
|
23
|
+
)
|
|
24
|
+
parser.add_argument("--dataset-name", required=True, help="what to call the managed Dataset")
|
|
25
|
+
parser.add_argument(
|
|
26
|
+
"--cron",
|
|
27
|
+
default="0 0 * * *",
|
|
28
|
+
help="UTC refresh schedule; Studio pauses it unless every source is refreshable",
|
|
29
|
+
)
|
|
30
|
+
parser.add_argument(
|
|
31
|
+
"--resume",
|
|
32
|
+
action="store_true",
|
|
33
|
+
help="recover the exact journaled deployment after an interruption or lost response",
|
|
34
|
+
)
|
|
35
|
+
parser.add_argument("--max-runs-per-month", type=int, help="tighten the monthly run allowance")
|
|
36
|
+
parser.add_argument(
|
|
37
|
+
"--max-concurrent-runs", type=int, help="tighten how many runs may happen at once"
|
|
38
|
+
)
|
|
39
|
+
parser.add_argument(
|
|
40
|
+
"--max-monthly-cost-units",
|
|
41
|
+
type=int,
|
|
42
|
+
help="tighten the recorded monthly spend ceiling, in cost units",
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def run(args: argparse.Namespace) -> dict[str, Any]:
|
|
47
|
+
# Authentication is resolved before the Build path is read. An absent/empty credential cannot
|
|
48
|
+
# be mistaken for permission to inspect or mutate a deployment.
|
|
49
|
+
from mostlyright.data_harness import cli
|
|
50
|
+
from mostlyright.data_harness.deploy import (
|
|
51
|
+
Cadence,
|
|
52
|
+
DeployRefused,
|
|
53
|
+
build_deployment_request,
|
|
54
|
+
)
|
|
55
|
+
from mostlyright.data_harness.governors import CapPolicy
|
|
56
|
+
from mostlyright.data_harness.hosted_deploy import (
|
|
57
|
+
HostedDeployError,
|
|
58
|
+
deploy_managed_dataset,
|
|
59
|
+
exchange_studio_token,
|
|
60
|
+
)
|
|
61
|
+
from mostlyright.data_harness.ux.credentials import LoginError, resolve_cloud_credentials
|
|
62
|
+
|
|
63
|
+
try:
|
|
64
|
+
credentials = resolve_cloud_credentials()
|
|
65
|
+
except LoginError as error:
|
|
66
|
+
raise cli._CliUsageError(f"{error.code}: {error.detail}") from error
|
|
67
|
+
try:
|
|
68
|
+
token = exchange_studio_token(credentials)
|
|
69
|
+
except HostedDeployError as error:
|
|
70
|
+
raise cli._CliUsageError(f"{error.code}: {error.detail}") from error
|
|
71
|
+
try:
|
|
72
|
+
caps = CapPolicy().tighten(
|
|
73
|
+
max_runs_per_month=args.max_runs_per_month,
|
|
74
|
+
max_concurrent_runs=args.max_concurrent_runs,
|
|
75
|
+
max_monthly_cost_units=args.max_monthly_cost_units,
|
|
76
|
+
)
|
|
77
|
+
except ValueError as error:
|
|
78
|
+
raise cli._CliUsageError(str(error)) from error
|
|
79
|
+
try:
|
|
80
|
+
# An ordinary Editor deploy is one authenticated Studio transaction chain. It always
|
|
81
|
+
# re-verifies the Build locally, but it neither looks for a coordinator review session nor
|
|
82
|
+
# asks a second person to approve the proposal. Studio records the request and the Editor's
|
|
83
|
+
# decision durably before activation; this command only recovers an interrupted chain.
|
|
84
|
+
# One predecessor, named once, threaded to both layers that need it. The Recipe layer
|
|
85
|
+
# authenticates the version chain out of it whenever this Build's execution mode is not
|
|
86
|
+
# `initial`; the deployment layer reads the Studio project, plan, and Dataset that its
|
|
87
|
+
# deployment published, so a corrected Build updates that Dataset instead of founding a
|
|
88
|
+
# second one. `hosted_deploy` settles which of the two shapes this invocation is before it
|
|
89
|
+
# writes anything -- see `_deployment_predecessor`.
|
|
90
|
+
request = build_deployment_request(
|
|
91
|
+
candidate_run_dir=args.run_dir,
|
|
92
|
+
previous_run_dir=args.previous_run_dir,
|
|
93
|
+
dataset_name=args.dataset_name,
|
|
94
|
+
target=None,
|
|
95
|
+
caps=caps,
|
|
96
|
+
cadence=Cadence(label="Studio UTC cron", cycle_template=args.cron),
|
|
97
|
+
review_authority_provider=None,
|
|
98
|
+
require_reviewed_build=False,
|
|
99
|
+
)
|
|
100
|
+
return deploy_managed_dataset(
|
|
101
|
+
request,
|
|
102
|
+
run_dir=args.run_dir,
|
|
103
|
+
previous_run_dir=args.previous_run_dir,
|
|
104
|
+
token=token,
|
|
105
|
+
credentials=credentials,
|
|
106
|
+
resume=args.resume,
|
|
107
|
+
cron=args.cron,
|
|
108
|
+
max_concurrent_runs=caps.max_concurrent_runs,
|
|
109
|
+
)
|
|
110
|
+
except (DeployRefused, HostedDeployError) as error:
|
|
111
|
+
if isinstance(error, HostedDeployError):
|
|
112
|
+
message = f"{error.code}: {error.detail}"
|
|
113
|
+
else:
|
|
114
|
+
message = str(error)
|
|
115
|
+
raise cli._CliUsageError(message) from error
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def exit_code(payload: Mapping[str, Any]) -> int:
|
|
119
|
+
return 0 if payload.get("status") == "dataset_activation_queued" else 1
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _mapping(value: Any, label: str, usage_error: type[Exception]) -> Mapping[str, object]:
|
|
123
|
+
if not isinstance(value, dict):
|
|
124
|
+
raise usage_error(f"the {label} document must be one object")
|
|
125
|
+
return value
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
SPEC = CommandSpec(
|
|
129
|
+
name="deploy",
|
|
130
|
+
help="stage one verified Build and activate it as the authenticated Editor",
|
|
131
|
+
add_arguments=add_arguments,
|
|
132
|
+
run=run,
|
|
133
|
+
exit_code=exit_code,
|
|
134
|
+
)
|