mostlyright-data 0.9.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mostlyright/data_harness/__init__.py +158 -0
- mostlyright/data_harness/acquisition/__init__.py +55 -0
- mostlyright/data_harness/acquisition/http.py +2773 -0
- mostlyright/data_harness/acquisition/parsing.py +809 -0
- mostlyright/data_harness/acquisition/ranges.py +495 -0
- mostlyright/data_harness/acquisition/result_download.py +360 -0
- mostlyright/data_harness/acquisition/retention_admission.py +248 -0
- mostlyright/data_harness/acquisition/sandbox.py +4888 -0
- mostlyright/data_harness/acquisition/url_policy.py +530 -0
- mostlyright/data_harness/agent_runtime.py +2743 -0
- mostlyright/data_harness/assets/logo-ink.svg +31 -0
- mostlyright/data_harness/backends/__init__.py +28 -0
- mostlyright/data_harness/backends/pandas_backend.py +350 -0
- mostlyright/data_harness/backends/polars_backend.py +366 -0
- mostlyright/data_harness/backends/protocol.py +124 -0
- mostlyright/data_harness/backends/reference.py +83 -0
- mostlyright/data_harness/backends/registry.py +55 -0
- mostlyright/data_harness/backends/restrictions.py +126 -0
- mostlyright/data_harness/canonical.py +333 -0
- mostlyright/data_harness/catalog_job.py +625 -0
- mostlyright/data_harness/cli.py +5398 -0
- mostlyright/data_harness/contracts.py +53 -0
- mostlyright/data_harness/coordinator.py +1307 -0
- mostlyright/data_harness/deploy.py +924 -0
- mostlyright/data_harness/deploy_target.py +312 -0
- mostlyright/data_harness/deployment_evidence.py +1067 -0
- mostlyright/data_harness/event_presentation.py +576 -0
- mostlyright/data_harness/events.py +2152 -0
- mostlyright/data_harness/fast_delimited.py +239 -0
- mostlyright/data_harness/fleet.py +237 -0
- mostlyright/data_harness/formats.py +236 -0
- mostlyright/data_harness/governors.py +1163 -0
- mostlyright/data_harness/hosted_bootstrap.py +972 -0
- mostlyright/data_harness/hosted_crawler.py +1115 -0
- mostlyright/data_harness/hosted_crawler_container_smoke.py +351 -0
- mostlyright/data_harness/hosted_crawler_fetch.py +423 -0
- mostlyright/data_harness/hosted_crawler_job.py +1277 -0
- mostlyright/data_harness/hosted_crawler_protocol.py +676 -0
- mostlyright/data_harness/hosted_dataset.py +1500 -0
- mostlyright/data_harness/hosted_deploy.py +3037 -0
- mostlyright/data_harness/hosted_handoff.py +62 -0
- mostlyright/data_harness/hosted_ingestion_contract.py +504 -0
- mostlyright/data_harness/hosted_ingestion_job.py +356 -0
- mostlyright/data_harness/hosted_ingestion_job_smoke.py +40 -0
- mostlyright/data_harness/hosted_session_container_smoke.py +194 -0
- mostlyright/data_harness/hosted_session_worker.py +3554 -0
- mostlyright/data_harness/hosted_session_worker_job_smoke.py +46 -0
- mostlyright/data_harness/hosted_worker.py +6784 -0
- mostlyright/data_harness/ingestion/__init__.py +56 -0
- mostlyright/data_harness/ingestion/contracts.py +461 -0
- mostlyright/data_harness/ingestion/faults.py +42 -0
- mostlyright/data_harness/ingestion/gcs_store.py +1162 -0
- mostlyright/data_harness/ingestion/spool.py +130 -0
- mostlyright/data_harness/ingestion/store.py +885 -0
- mostlyright/data_harness/key_seam.py +434 -0
- mostlyright/data_harness/linux_process_boundary.py +262 -0
- mostlyright/data_harness/local_contracts.py +2880 -0
- mostlyright/data_harness/local_search/__init__.py +5 -0
- mostlyright/data_harness/local_search/build_index.py +1087 -0
- mostlyright/data_harness/local_search/contracts.py +920 -0
- mostlyright/data_harness/local_search/query_trace.py +266 -0
- mostlyright/data_harness/local_search/retrieval.py +700 -0
- mostlyright/data_harness/local_search/sealed.py +474 -0
- mostlyright/data_harness/local_search/service.py +784 -0
- mostlyright/data_harness/nbrender/CONTRACT.md +212 -0
- mostlyright/data_harness/nbrender/__init__.py +12 -0
- mostlyright/data_harness/nbrender/chrome.py +359 -0
- mostlyright/data_harness/nbrender/code_body.py +266 -0
- mostlyright/data_harness/nbrender/document.py +407 -0
- mostlyright/data_harness/nbrender/frame.py +275 -0
- mostlyright/data_harness/nbrender/interactive.py +337 -0
- mostlyright/data_harness/nbrender/markdown_body.py +477 -0
- mostlyright/data_harness/nbrender/mr_components.py +134 -0
- mostlyright/data_harness/nbrender/outputs_data.py +595 -0
- mostlyright/data_harness/nbrender/outputs_rich.py +906 -0
- mostlyright/data_harness/nbrender/outputs_source.py +260 -0
- mostlyright/data_harness/nbrender/outputs_stage.py +176 -0
- mostlyright/data_harness/nbrender/outputs_text.py +400 -0
- mostlyright/data_harness/nbrender/parse.py +394 -0
- mostlyright/data_harness/nbrender/status.py +40 -0
- mostlyright/data_harness/nbrender/tokens.py +1295 -0
- mostlyright/data_harness/notebook.py +1710 -0
- mostlyright/data_harness/offline.py +2049 -0
- mostlyright/data_harness/operation_registry.py +1007 -0
- mostlyright/data_harness/operator_setup.py +239 -0
- mostlyright/data_harness/pipeline.py +6428 -0
- mostlyright/data_harness/plan_graph.py +2026 -0
- mostlyright/data_harness/preparation/__init__.py +104 -0
- mostlyright/data_harness/preparation/contracts.py +1017 -0
- mostlyright/data_harness/preparation/engine.py +221 -0
- mostlyright/data_harness/preparation/errors.py +14 -0
- mostlyright/data_harness/preparation/gates.py +751 -0
- mostlyright/data_harness/preparation/joins.py +574 -0
- mostlyright/data_harness/preparation/profile.py +384 -0
- mostlyright/data_harness/preparation/table.py +217 -0
- mostlyright/data_harness/preparation/transforms.py +568 -0
- mostlyright/data_harness/progress_events.py +534 -0
- mostlyright/data_harness/readers/__init__.py +46 -0
- mostlyright/data_harness/readers/containers.py +963 -0
- mostlyright/data_harness/readers/contracts.py +542 -0
- mostlyright/data_harness/readers/delimited.py +257 -0
- mostlyright/data_harness/readers/grib2/__init__.py +33 -0
- mostlyright/data_harness/readers/grib2/admission.py +722 -0
- mostlyright/data_harness/readers/grib2/decode.py +1009 -0
- mostlyright/data_harness/readers/grib2/geometry.py +1133 -0
- mostlyright/data_harness/readers/grib2/portable_math.py +501 -0
- mostlyright/data_harness/readers/json_tabular.py +485 -0
- mostlyright/data_harness/readers/registry.py +514 -0
- mostlyright/data_harness/readers/samples/README.md +110 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/cities.csv.gz +0 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/cities.csv.gz +0 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/cities.tar +0 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/cities.tar +0 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/cities.zip +0 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/expected.json +25 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/expected.json +25 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/expected.json +25 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/cities.csv +3 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/expected.json +23 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/cities.tsv +5 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/expected.json +25 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/expected.json +30 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/observations.csv +5 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/expected.json +44 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/stations.json +1 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/expected.json +48 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/observations.ndjson +4 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/cities.xlsx +0 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/expected.json +24 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/expected.json +27 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/expected.json +27 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/README.md +20 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/expected.json +55 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/gfs-2m-temperature.grib2 +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/expected.json +54 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/expected.json +54 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/hrrr-categorical-rain.grib2 +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/expected.json +54 -0
- mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
- mostlyright/data_harness/readers/samples.py +582 -0
- mostlyright/data_harness/readers/spreadsheet.py +803 -0
- mostlyright/data_harness/readers/tabular.py +510 -0
- mostlyright/data_harness/recipe.py +5321 -0
- mostlyright/data_harness/repair/__init__.py +78 -0
- mostlyright/data_harness/repair/adapters.py +274 -0
- mostlyright/data_harness/repair/contracts.py +872 -0
- mostlyright/data_harness/repair/coordinator.py +1099 -0
- mostlyright/data_harness/repair/errors.py +16 -0
- mostlyright/data_harness/review.py +2533 -0
- mostlyright/data_harness/rowset.py +283 -0
- mostlyright/data_harness/serving.py +1975 -0
- mostlyright/data_harness/serving_edge.py +590 -0
- mostlyright/data_harness/serving_http.py +1031 -0
- mostlyright/data_harness/session_probes.py +759 -0
- mostlyright/data_harness/signing.py +101 -0
- mostlyright/data_harness/source_discovery.py +898 -0
- mostlyright/data_harness/sources/__init__.py +209 -0
- mostlyright/data_harness/sources/_adapter_steps.py +213 -0
- mostlyright/data_harness/sources/adapters.py +1214 -0
- mostlyright/data_harness/sources/cadence.py +1428 -0
- mostlyright/data_harness/sources/cadence_emission.py +453 -0
- mostlyright/data_harness/sources/cadence_history.py +546 -0
- mostlyright/data_harness/sources/catalog/__init__.py +17 -0
- mostlyright/data_harness/sources/catalog/admission.py +477 -0
- mostlyright/data_harness/sources/catalog/authoring.py +1701 -0
- mostlyright/data_harness/sources/catalog/authoring_policy.py +701 -0
- mostlyright/data_harness/sources/catalog/authoring_shards.py +1217 -0
- mostlyright/data_harness/sources/catalog/bounded_io.py +231 -0
- mostlyright/data_harness/sources/catalog/channel.py +523 -0
- mostlyright/data_harness/sources/catalog/channel_client.py +296 -0
- mostlyright/data_harness/sources/catalog/contracts.py +825 -0
- mostlyright/data_harness/sources/catalog/coverage.py +137 -0
- mostlyright/data_harness/sources/catalog/delta.py +1340 -0
- mostlyright/data_harness/sources/catalog/embedding.py +532 -0
- mostlyright/data_harness/sources/catalog/entry_v2.py +1182 -0
- mostlyright/data_harness/sources/catalog/fill.py +3889 -0
- mostlyright/data_harness/sources/catalog/fill_partitions.py +459 -0
- mostlyright/data_harness/sources/catalog/fill_staging.py +1105 -0
- mostlyright/data_harness/sources/catalog/gating.py +374 -0
- mostlyright/data_harness/sources/catalog/generation_receipt.py +1607 -0
- mostlyright/data_harness/sources/catalog/harvest/__init__.py +7 -0
- mostlyright/data_harness/sources/catalog/harvest/ckan.py +384 -0
- mostlyright/data_harness/sources/catalog/harvest/datagov_v4.py +798 -0
- mostlyright/data_harness/sources/catalog/harvest/protocol.py +964 -0
- mostlyright/data_harness/sources/catalog/harvest/sdmx.py +445 -0
- mostlyright/data_harness/sources/catalog/harvest/stac.py +384 -0
- mostlyright/data_harness/sources/catalog/health.py +447 -0
- mostlyright/data_harness/sources/catalog/hosted_catalog.py +105 -0
- mostlyright/data_harness/sources/catalog/identity_history.py +1549 -0
- mostlyright/data_harness/sources/catalog/neural.py +1618 -0
- mostlyright/data_harness/sources/catalog/packed_catalog.py +2345 -0
- mostlyright/data_harness/sources/catalog/packed_retrieval.py +1517 -0
- mostlyright/data_harness/sources/catalog/packed_writer.py +2802 -0
- mostlyright/data_harness/sources/catalog/query_trace.py +1037 -0
- mostlyright/data_harness/sources/catalog/recommend.py +171 -0
- mostlyright/data_harness/sources/catalog/retrieval.py +230 -0
- mostlyright/data_harness/sources/catalog/retrieval_manifest.py +995 -0
- mostlyright/data_harness/sources/catalog/rights_decisions.py +254 -0
- mostlyright/data_harness/sources/catalog/sealed.py +560 -0
- mostlyright/data_harness/sources/catalog/search.py +230 -0
- mostlyright/data_harness/sources/catalog/streaming_delta.py +1097 -0
- mostlyright/data_harness/sources/catalog/update.py +891 -0
- mostlyright/data_harness/sources/collections.py +815 -0
- mostlyright/data_harness/sources/contracts.py +2223 -0
- mostlyright/data_harness/sources/deletion.py +761 -0
- mostlyright/data_harness/sources/fitness.py +162 -0
- mostlyright/data_harness/sources/governance.py +163 -0
- mostlyright/data_harness/sources/hosted.py +173 -0
- mostlyright/data_harness/sources/integration.py +218 -0
- mostlyright/data_harness/sources/range_reader.py +418 -0
- mostlyright/data_harness/sources/registry.py +514 -0
- mostlyright/data_harness/sources/rights_rule.py +59 -0
- mostlyright/data_harness/sources/source_cadence_vectors.v1.json +1 -0
- mostlyright/data_harness/sources/sports.py +521 -0
- mostlyright/data_harness/sources/stream.py +524 -0
- mostlyright/data_harness/sources/stream_connector.py +418 -0
- mostlyright/data_harness/sources/stream_recorder.py +1404 -0
- mostlyright/data_harness/studio_boundary.py +2019 -0
- mostlyright/data_harness/thin/__init__.py +37 -0
- mostlyright/data_harness/thin/acquire.py +1137 -0
- mostlyright/data_harness/thin/acquire_cancel.py +579 -0
- mostlyright/data_harness/thin/approvals.py +617 -0
- mostlyright/data_harness/thin/commands.py +406 -0
- mostlyright/data_harness/thin/download.py +194 -0
- mostlyright/data_harness/thin/narrative.py +589 -0
- mostlyright/data_harness/thin/parity.py +1070 -0
- mostlyright/data_harness/thin/propose.py +2759 -0
- mostlyright/data_harness/thin/research.py +1663 -0
- mostlyright/data_harness/thin/router.py +924 -0
- mostlyright/data_harness/thin/runs.py +519 -0
- mostlyright/data_harness/thin/session.py +281 -0
- mostlyright/data_harness/thin/stream.py +501 -0
- mostlyright/data_harness/thin/transport.py +187 -0
- mostlyright/data_harness/thin/vocabulary.py +368 -0
- mostlyright/data_harness/thin/workers.py +164 -0
- mostlyright/data_harness/ucum/TABLE-PIN.json +40 -0
- mostlyright/data_harness/ucum/ucum-subset.v1.json +632 -0
- mostlyright/data_harness/unit_flow.py +927 -0
- mostlyright/data_harness/units.py +572 -0
- mostlyright/data_harness/ux/__init__.py +9 -0
- mostlyright/data_harness/ux/approve.py +485 -0
- mostlyright/data_harness/ux/author_yaml.py +597 -0
- mostlyright/data_harness/ux/cloud_auth.py +447 -0
- mostlyright/data_harness/ux/commands/__init__.py +260 -0
- mostlyright/data_harness/ux/commands/approve.py +136 -0
- mostlyright/data_harness/ux/commands/auth.py +744 -0
- mostlyright/data_harness/ux/commands/author.py +79 -0
- mostlyright/data_harness/ux/commands/catalog_author.py +403 -0
- mostlyright/data_harness/ux/commands/catalog_fill.py +523 -0
- mostlyright/data_harness/ux/commands/catalog_harvest.py +545 -0
- mostlyright/data_harness/ux/commands/catalog_publish.py +1838 -0
- mostlyright/data_harness/ux/commands/catalog_search.py +71 -0
- mostlyright/data_harness/ux/commands/catalog_update.py +437 -0
- mostlyright/data_harness/ux/commands/deploy.py +134 -0
- mostlyright/data_harness/ux/commands/deploy_dataset.py +98 -0
- mostlyright/data_harness/ux/commands/deploy_plan.py +105 -0
- mostlyright/data_harness/ux/commands/deploy_status.py +104 -0
- mostlyright/data_harness/ux/commands/diff.py +74 -0
- mostlyright/data_harness/ux/commands/index.py +84 -0
- mostlyright/data_harness/ux/commands/inventory.py +47 -0
- mostlyright/data_harness/ux/commands/list_builds.py +143 -0
- mostlyright/data_harness/ux/commands/login.py +63 -0
- mostlyright/data_harness/ux/commands/peek.py +236 -0
- mostlyright/data_harness/ux/commands/plan_check.py +90 -0
- mostlyright/data_harness/ux/commands/preflight.py +97 -0
- mostlyright/data_harness/ux/commands/record.py +107 -0
- mostlyright/data_harness/ux/commands/review_setup.py +47 -0
- mostlyright/data_harness/ux/commands/search.py +440 -0
- mostlyright/data_harness/ux/commands/show.py +61 -0
- mostlyright/data_harness/ux/commands/whoami.py +37 -0
- mostlyright/data_harness/ux/credential_native.py +551 -0
- mostlyright/data_harness/ux/credential_store.py +1055 -0
- mostlyright/data_harness/ux/credentials.py +631 -0
- mostlyright/data_harness/ux/diffing.py +444 -0
- mostlyright/data_harness/ux/headline.py +671 -0
- mostlyright/data_harness/ux/hosted_acquisition.py +974 -0
- mostlyright/data_harness/ux/hosted_run_status.py +619 -0
- mostlyright/data_harness/ux/inventory.py +427 -0
- mostlyright/data_harness/ux/local_review.py +375 -0
- mostlyright/data_harness/ux/login.py +691 -0
- mostlyright/data_harness/ux/path_kind.py +147 -0
- mostlyright/data_harness/ux/peek.py +1000 -0
- mostlyright/data_harness/ux/plain_file.py +178 -0
- mostlyright/data_harness/ux/plan_check.py +311 -0
- mostlyright/data_harness/ux/preflight.py +918 -0
- mostlyright/data_harness/ux/readers.py +1124 -0
- mostlyright/data_harness/ux/remediation.py +2195 -0
- mostlyright/data_harness/ux/render.py +657 -0
- mostlyright/data_harness/ux/workload.py +1077 -0
- mostlyright/data_harness/viewer.py +3713 -0
- mostlyright/data_harness/visual_run/__init__.py +83 -0
- mostlyright/data_harness/visual_run/authoring.py +235 -0
- mostlyright/data_harness/visual_run/contracts.py +673 -0
- mostlyright/data_harness/visual_run/materialize.py +486 -0
- mostlyright/data_harness/visual_run/observations.py +874 -0
- mostlyright/data_harness/visual_run/query.py +259 -0
- mostlyright/data_harness/visual_run/reducer.py +280 -0
- mostlyright/data_harness/visual_run/sdk.py +892 -0
- mostlyright/data_harness/visual_run/store.py +584 -0
- mostlyright/data_harness/visual_run/transport.py +239 -0
- mostlyright/data_harness/watch.py +2999 -0
- mostlyright_data-0.9.0.dist-info/METADATA +607 -0
- mostlyright_data-0.9.0.dist-info/RECORD +314 -0
- mostlyright_data-0.9.0.dist-info/WHEEL +4 -0
- mostlyright_data-0.9.0.dist-info/entry_points.txt +12 -0
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
"""``mr-data deploy-dataset`` -- bring one released hosted Build here and write its notebook."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
from collections.abc import Mapping
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
from mostlyright.data_harness.ux.commands import CommandSpec
|
|
11
|
+
|
|
12
|
+
# Where the released Build lands when the caller does not say. It is a fixed name under the folder
|
|
13
|
+
# the deployment was run from, and being fixed is the point: a viewer can be opened on it before
|
|
14
|
+
# the hosted work has finished -- a folder that is not there yet is a target the viewer already
|
|
15
|
+
# accepts -- so the Dataset tab arrives without anyone going and looking for a path first. The
|
|
16
|
+
# deployment folder itself is not usable as the destination: it already holds the Build that was
|
|
17
|
+
# deployed, and a folder holds one Build.
|
|
18
|
+
DEFAULT_DESTINATION = "hosted"
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def add_arguments(parser: argparse.ArgumentParser) -> None:
|
|
22
|
+
parser.description = (
|
|
23
|
+
"Bring the Build this deployment's run released back to this machine and write its "
|
|
24
|
+
"dataset notebook. The rows are transferred through one short-lived Mostly Right cloud "
|
|
25
|
+
"transfer, checked against the fingerprints the release sealed, and checked again here by "
|
|
26
|
+
"the same checks a Build made on this machine passes. Nothing is written until all of "
|
|
27
|
+
"that has passed, and the notebook is derived here rather than downloaded. It lands in "
|
|
28
|
+
"the 'hosted' folder beside the deployment record unless you say otherwise, and a "
|
|
29
|
+
"workbench window already open on that folder shows its Dataset tab by itself."
|
|
30
|
+
)
|
|
31
|
+
parser.add_argument(
|
|
32
|
+
"run_dir",
|
|
33
|
+
type=Path,
|
|
34
|
+
help="the folder the deployment was run from, which holds its record",
|
|
35
|
+
)
|
|
36
|
+
parser.add_argument(
|
|
37
|
+
"--into",
|
|
38
|
+
type=Path,
|
|
39
|
+
help=(
|
|
40
|
+
"where to write the Build and its notebook; made if it is not there yet, "
|
|
41
|
+
"and never written over. Defaults to the 'hosted' folder beside the deployment record"
|
|
42
|
+
),
|
|
43
|
+
)
|
|
44
|
+
parser.add_argument(
|
|
45
|
+
"--research-dir",
|
|
46
|
+
type=Path,
|
|
47
|
+
help="the open workbench folder to tell about the dataset when it lands",
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def run(args: argparse.Namespace) -> dict[str, Any]:
|
|
52
|
+
# Authentication is resolved before anything local is read, so an absent or empty credential
|
|
53
|
+
# can never be mistaken for permission to look at a deployment. Typed refusals are raised, not
|
|
54
|
+
# rewritten: this command's failures each carry a code with a fix written against it, and
|
|
55
|
+
# flattening them into a usage error would tell a person their command line was wrong when
|
|
56
|
+
# what actually happened was a transfer that did not match its fingerprint.
|
|
57
|
+
from mostlyright.data_harness import cli
|
|
58
|
+
from mostlyright.data_harness.hosted_dataset import materialize_hosted_dataset
|
|
59
|
+
from mostlyright.data_harness.hosted_deploy import HostedDeployError, exchange_studio_token
|
|
60
|
+
from mostlyright.data_harness.ux.credentials import resolve_cloud_credentials
|
|
61
|
+
from mostlyright.data_harness.ux.hosted_run_status import journaled_activation
|
|
62
|
+
|
|
63
|
+
credentials = resolve_cloud_credentials()
|
|
64
|
+
token = exchange_studio_token(credentials)
|
|
65
|
+
# The same reader `mr-data deploy-status` uses. One deployment journal, one function that
|
|
66
|
+
# knows its shape, and one set of sentences for a journal that is not ready yet.
|
|
67
|
+
activated = journaled_activation(args.run_dir)
|
|
68
|
+
if token.workspace_id != activated.workspace_id:
|
|
69
|
+
raise HostedDeployError(
|
|
70
|
+
"DEPLOY_STATE_MISMATCH",
|
|
71
|
+
"this deployment belongs to a different workspace from the one you are signed in to",
|
|
72
|
+
)
|
|
73
|
+
destination = args.into if args.into is not None else args.run_dir / DEFAULT_DESTINATION
|
|
74
|
+
visual_store = cli._visual_store_for_research(
|
|
75
|
+
args.research_dir, run_id=f"hosted-{activated.run_id}"
|
|
76
|
+
)
|
|
77
|
+
handoff = materialize_hosted_dataset(
|
|
78
|
+
destination,
|
|
79
|
+
dataset_id=activated.dataset_id,
|
|
80
|
+
table_id=activated.table_id,
|
|
81
|
+
hosted_run_id=activated.run_id,
|
|
82
|
+
token=token,
|
|
83
|
+
visual_store=visual_store,
|
|
84
|
+
)
|
|
85
|
+
return {"status": "hosted_dataset_ready", **handoff.to_dict()}
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def exit_code(payload: Mapping[str, Any]) -> int:
|
|
89
|
+
return 0 if payload.get("status") == "hosted_dataset_ready" else 1
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
SPEC = CommandSpec(
|
|
93
|
+
name="deploy-dataset",
|
|
94
|
+
help="bring the Build a deployment released back here and write its dataset notebook",
|
|
95
|
+
add_arguments=add_arguments,
|
|
96
|
+
run=run,
|
|
97
|
+
exit_code=exit_code,
|
|
98
|
+
)
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
"""``mr-data deploy-plan`` -- build a deployment request without deploying anything."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
from collections.abc import Mapping
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
from mostlyright.data_harness.ux.commands import CommandSpec
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def add_arguments(parser: argparse.ArgumentParser) -> None:
|
|
14
|
+
parser.description = (
|
|
15
|
+
"Plan a live dataset request from one reviewed Build. This command contacts nothing, "
|
|
16
|
+
"deploys nothing, and reports whether the recorded caps are enforced here."
|
|
17
|
+
)
|
|
18
|
+
parser.add_argument("run_dir", type=Path, help="the folder holding the reviewed Build")
|
|
19
|
+
parser.add_argument(
|
|
20
|
+
"--previous-run-dir",
|
|
21
|
+
type=Path,
|
|
22
|
+
help="required for a refresh or backfill: the reviewed Build this version follows",
|
|
23
|
+
)
|
|
24
|
+
parser.add_argument("--dataset-name", required=True, help="what to call the planned dataset")
|
|
25
|
+
parser.add_argument(
|
|
26
|
+
"--target",
|
|
27
|
+
type=Path,
|
|
28
|
+
required=True,
|
|
29
|
+
help="the file naming the intended project, region, Studio account, and job",
|
|
30
|
+
)
|
|
31
|
+
parser.add_argument(
|
|
32
|
+
"--corroboration",
|
|
33
|
+
type=Path,
|
|
34
|
+
required=True,
|
|
35
|
+
help="the file containing an independent reading of the intended target project",
|
|
36
|
+
)
|
|
37
|
+
parser.add_argument("--cadence", default="daily", help="how often it should look for new data")
|
|
38
|
+
parser.add_argument("--cycle-template", help="the cycle variable the source publishes on")
|
|
39
|
+
parser.add_argument("--max-runs-per-month", type=int, help="tighten the monthly run allowance")
|
|
40
|
+
parser.add_argument(
|
|
41
|
+
"--max-concurrent-runs", type=int, help="tighten how many runs may happen at once"
|
|
42
|
+
)
|
|
43
|
+
parser.add_argument(
|
|
44
|
+
"--max-monthly-cost-units",
|
|
45
|
+
type=int,
|
|
46
|
+
help="tighten the planned monthly spend ceiling, in cost units",
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def run(args: argparse.Namespace) -> dict[str, Any]:
|
|
51
|
+
# Function-local imports are the point: parser construction and command inventory do not load
|
|
52
|
+
# deployment verification, recipe execution, or cap-governor implementation modules.
|
|
53
|
+
from mostlyright.data_harness import cli
|
|
54
|
+
from mostlyright.data_harness.deploy import Cadence, DeployRefused, plan_deployment
|
|
55
|
+
from mostlyright.data_harness.deploy_target import DeployTargetError
|
|
56
|
+
from mostlyright.data_harness.governors import CapPolicy
|
|
57
|
+
|
|
58
|
+
try:
|
|
59
|
+
caps = CapPolicy().tighten(
|
|
60
|
+
max_runs_per_month=args.max_runs_per_month,
|
|
61
|
+
max_concurrent_runs=args.max_concurrent_runs,
|
|
62
|
+
max_monthly_cost_units=args.max_monthly_cost_units,
|
|
63
|
+
)
|
|
64
|
+
except ValueError as error:
|
|
65
|
+
raise cli._CliUsageError(str(error)) from error
|
|
66
|
+
|
|
67
|
+
try:
|
|
68
|
+
result = plan_deployment(
|
|
69
|
+
descriptor=_mapping(cli._read_json(args.target), "target", cli._CliUsageError),
|
|
70
|
+
corroboration=_mapping(
|
|
71
|
+
cli._read_json(args.corroboration), "corroboration", cli._CliUsageError
|
|
72
|
+
),
|
|
73
|
+
candidate_run_dir=args.run_dir,
|
|
74
|
+
previous_run_dir=args.previous_run_dir,
|
|
75
|
+
dataset_name=args.dataset_name,
|
|
76
|
+
caps=caps,
|
|
77
|
+
cadence=Cadence(label=args.cadence, cycle_template=args.cycle_template),
|
|
78
|
+
review_authority_provider=cli._runtime_review_authority_provider(),
|
|
79
|
+
)
|
|
80
|
+
except (DeployRefused, DeployTargetError) as error:
|
|
81
|
+
raise cli._CliUsageError(str(error)) from error
|
|
82
|
+
return result.to_dict()
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def exit_code(payload: Mapping[str, Any]) -> int:
|
|
86
|
+
return 0 if payload.get("status") == "deployment_plan_ready" else 1
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _mapping(
|
|
90
|
+
value: Any,
|
|
91
|
+
label: str,
|
|
92
|
+
usage_error: type[Exception],
|
|
93
|
+
) -> Mapping[str, object]:
|
|
94
|
+
if not isinstance(value, dict):
|
|
95
|
+
raise usage_error(f"the {label} document must be one object")
|
|
96
|
+
return value
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
SPEC = CommandSpec(
|
|
100
|
+
name="deploy-plan",
|
|
101
|
+
help="plan a live dataset request without deploying it",
|
|
102
|
+
add_arguments=add_arguments,
|
|
103
|
+
run=run,
|
|
104
|
+
exit_code=exit_code,
|
|
105
|
+
)
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
"""``mr-data deploy-status`` -- what Studio says about the run this deployment queued.
|
|
2
|
+
|
|
3
|
+
`mr-data deploy --resume` hands back a `run_id` and returns. This command is how that identifier
|
|
4
|
+
becomes an answer: it reads the run's coordinates out of the deployment journal the deploy already
|
|
5
|
+
wrote, asks Studio for that exact run, and prints the state Studio holds -- one of thirteen, each
|
|
6
|
+
with what it means and what to do about it.
|
|
7
|
+
|
|
8
|
+
It is a read. It sends no body, carries no idempotency key, and reaches exactly one Studio
|
|
9
|
+
operation, so running it on a finished deployment, on a sleeping laptop's return, or from a
|
|
10
|
+
different terminal an hour later cannot change anything about the run it is reporting on, or
|
|
11
|
+
anything else Studio holds.
|
|
12
|
+
|
|
13
|
+
With no `--watch-seconds` it makes exactly one request and reports what came back. With one, it
|
|
14
|
+
reads on a bounded backoff until the state stops changing on its own and lists each distinct state
|
|
15
|
+
it saw; if the bound arrives first it says so, and says which state the run was actually in.
|
|
16
|
+
|
|
17
|
+
The exit code is 1 when the run needs a person, when it ended without a version, or when a watch
|
|
18
|
+
ran out before it settled -- so `mr-data deploy-status RUN_DIR --watch-seconds 900` reads as a gate
|
|
19
|
+
in a script. A run still working, and a run whose version is live, are both 0.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import argparse
|
|
25
|
+
from collections.abc import Mapping
|
|
26
|
+
from pathlib import Path
|
|
27
|
+
from typing import Any
|
|
28
|
+
|
|
29
|
+
from mostlyright.data_harness.ux.commands import CommandSpec
|
|
30
|
+
from mostlyright.data_harness.ux.hosted_run_status import (
|
|
31
|
+
MAX_WATCH_SECONDS,
|
|
32
|
+
RELEASED,
|
|
33
|
+
WATCHING,
|
|
34
|
+
report_run_status,
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
DESCRIPTION = (
|
|
38
|
+
"Report the real state of the run one deployment queued, read from Mostly Right Studio's own "
|
|
39
|
+
"durable record of it. The run is named by the deployment journal inside RUN_DIR, so a "
|
|
40
|
+
"process that was killed, a laptop that slept, or a fresh terminal picks the same run back "
|
|
41
|
+
"up. It sends no request body and no idempotency key, nothing Studio holds is changed, and "
|
|
42
|
+
"no state is shown that Studio did not report."
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def add_arguments(parser: argparse.ArgumentParser) -> None:
|
|
47
|
+
parser.description = DESCRIPTION
|
|
48
|
+
parser.add_argument(
|
|
49
|
+
"run_dir", type=Path, help="the folder whose deployment queued the run to report on"
|
|
50
|
+
)
|
|
51
|
+
parser.add_argument(
|
|
52
|
+
"--watch-seconds",
|
|
53
|
+
type=int,
|
|
54
|
+
help=(
|
|
55
|
+
"keep reading until the run settles, waiting at most this many seconds "
|
|
56
|
+
f"(1-{MAX_WATCH_SECONDS}); without it this makes exactly one request"
|
|
57
|
+
),
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def run(args: argparse.Namespace) -> dict[str, Any]:
|
|
62
|
+
# Authentication is resolved before the journal is read, exactly as `deploy` resolves it before
|
|
63
|
+
# the Build path is read: an absent or empty credential cannot be mistaken for permission to
|
|
64
|
+
# read a deployment's state.
|
|
65
|
+
from mostlyright.data_harness import cli
|
|
66
|
+
from mostlyright.data_harness.hosted_deploy import HostedDeployError, exchange_studio_token
|
|
67
|
+
from mostlyright.data_harness.ux.credentials import LoginError, resolve_cloud_credentials
|
|
68
|
+
|
|
69
|
+
try:
|
|
70
|
+
credentials = resolve_cloud_credentials()
|
|
71
|
+
except LoginError as error:
|
|
72
|
+
raise cli._CliUsageError(f"{error.code}: {error.detail}") from error
|
|
73
|
+
try:
|
|
74
|
+
token = exchange_studio_token(credentials)
|
|
75
|
+
return report_run_status(
|
|
76
|
+
args.run_dir,
|
|
77
|
+
token=token,
|
|
78
|
+
credentials=credentials,
|
|
79
|
+
watch_seconds=args.watch_seconds,
|
|
80
|
+
)
|
|
81
|
+
except HostedDeployError as error:
|
|
82
|
+
raise cli._CliUsageError(f"{error.code}: {error.detail}") from error
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def exit_code(payload: Mapping[str, Any]) -> int:
|
|
86
|
+
"""1 when somebody has to do something, so this reads as a gate in a script.
|
|
87
|
+
|
|
88
|
+
A run that is still working is not a failure and a released one is the point, so both are 0.
|
|
89
|
+
Everything else -- the two states waiting on somebody, the three that ended without a version,
|
|
90
|
+
and a watch that ran out before the run settled -- is 1.
|
|
91
|
+
"""
|
|
92
|
+
|
|
93
|
+
if payload.get("status") != "run_status_reported":
|
|
94
|
+
return 1
|
|
95
|
+
return 0 if payload.get("run_disposition") in {WATCHING, RELEASED} else 1
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
SPEC = CommandSpec(
|
|
99
|
+
name="deploy-status",
|
|
100
|
+
help="report what Studio says about the run one deployment queued",
|
|
101
|
+
add_arguments=add_arguments,
|
|
102
|
+
run=run,
|
|
103
|
+
exit_code=exit_code,
|
|
104
|
+
)
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
"""``mr-data diff`` -- point at two builds and ask what is different.
|
|
2
|
+
|
|
3
|
+
This module is the command and nothing else: its arguments, and the one call that turns them into
|
|
4
|
+
the payload both renderings are built from. The comparison itself lives in
|
|
5
|
+
:mod:`mostlyright.data_harness.ux.diffing`, so the same answer is available to a browser page
|
|
6
|
+
without going through argparse.
|
|
7
|
+
|
|
8
|
+
Nothing here prints, and nothing here catches: a refusal is a typed error, and the command line
|
|
9
|
+
renders it with the sentences the remediation map supplies.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import argparse
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from typing import Any
|
|
17
|
+
|
|
18
|
+
from mostlyright.data_harness.ux.commands import CommandSpec
|
|
19
|
+
from mostlyright.data_harness.ux.diffing import compare_builds
|
|
20
|
+
|
|
21
|
+
# Said in the command's own help, not only in a design document: a difference is an answer, and
|
|
22
|
+
# scripting against a nonzero exit code for one would be scripting against something this command
|
|
23
|
+
# never promised.
|
|
24
|
+
DESCRIPTION = (
|
|
25
|
+
"Compare two builds and say what changed: which columns were added, removed or retyped, "
|
|
26
|
+
"whether the row count moved, and whether the data itself is different. Both builds are "
|
|
27
|
+
"replayed and verified before anything is compared. "
|
|
28
|
+
"Finding a difference is not a failure, so the exit code is 0 whether the two builds match "
|
|
29
|
+
"or not; read the status field from --json (builds_identical or builds_differ) if you want "
|
|
30
|
+
"to act on the answer."
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def add_arguments(parser: argparse.ArgumentParser) -> None:
|
|
35
|
+
parser.description = DESCRIPTION
|
|
36
|
+
parser.add_argument(
|
|
37
|
+
"left",
|
|
38
|
+
type=Path,
|
|
39
|
+
help="the first build folder, or the Workbench folder that holds one",
|
|
40
|
+
)
|
|
41
|
+
parser.add_argument(
|
|
42
|
+
"right",
|
|
43
|
+
type=Path,
|
|
44
|
+
help="the second build folder, or the Workbench folder that holds one",
|
|
45
|
+
)
|
|
46
|
+
parser.add_argument(
|
|
47
|
+
"--columns-only",
|
|
48
|
+
action="store_true",
|
|
49
|
+
help=(
|
|
50
|
+
"list only the changes to the columns; the count and the status still cover "
|
|
51
|
+
"every change"
|
|
52
|
+
),
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def run(args: argparse.Namespace) -> dict[str, Any]:
|
|
57
|
+
# Imported here, not at the top: this package is loaded while the command line is still being
|
|
58
|
+
# set up. Calling the command line's own helper rather than restating its rule is what stops
|
|
59
|
+
# diff and verify ever disagreeing about which folder a Workbench folder's run lives in.
|
|
60
|
+
from mostlyright.data_harness.cli import _candidate_run_target
|
|
61
|
+
|
|
62
|
+
comparison = compare_builds(
|
|
63
|
+
_candidate_run_target(args.left),
|
|
64
|
+
_candidate_run_target(args.right),
|
|
65
|
+
)
|
|
66
|
+
return comparison.to_dict(columns_only=args.columns_only)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
SPEC = CommandSpec(
|
|
70
|
+
name="diff",
|
|
71
|
+
help="compare two builds and say what changed",
|
|
72
|
+
add_arguments=add_arguments,
|
|
73
|
+
run=run,
|
|
74
|
+
)
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
"""``mr-data index``: explicitly rebuild the private local Build index."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
from mostlyright.data_harness.local_search.build_index import (
|
|
10
|
+
BuildDiscoveryLimits,
|
|
11
|
+
LocalSearchIndexError,
|
|
12
|
+
build_private_index,
|
|
13
|
+
)
|
|
14
|
+
from mostlyright.data_harness.sources.contracts import SourceContractError
|
|
15
|
+
from mostlyright.data_harness.ux.commands import CommandSpec
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def add_arguments(parser: argparse.ArgumentParser) -> None:
|
|
19
|
+
parser.add_argument("--build-root", type=Path, required=True, help="the explicit Build root")
|
|
20
|
+
parser.add_argument(
|
|
21
|
+
"--index-root", type=Path, required=True, help="the explicit private index output root"
|
|
22
|
+
)
|
|
23
|
+
parser.add_argument("--max-depth", type=int, required=True, help="maximum discovery depth")
|
|
24
|
+
parser.add_argument(
|
|
25
|
+
"--max-entries", type=int, required=True, help="maximum directory entries inspected"
|
|
26
|
+
)
|
|
27
|
+
parser.add_argument("--max-builds", type=int, required=True, help="maximum Builds inspected")
|
|
28
|
+
parser.add_argument(
|
|
29
|
+
"--max-receipt-bytes", type=int, required=True, help="maximum bytes in one Build receipt"
|
|
30
|
+
)
|
|
31
|
+
parser.add_argument(
|
|
32
|
+
"--max-member-bytes", type=int, required=True, help="maximum bytes in one Build member"
|
|
33
|
+
)
|
|
34
|
+
parser.add_argument(
|
|
35
|
+
"--max-aggregate-bytes",
|
|
36
|
+
type=int,
|
|
37
|
+
required=True,
|
|
38
|
+
help="maximum aggregate verified Build bytes",
|
|
39
|
+
)
|
|
40
|
+
parser.add_argument(
|
|
41
|
+
"--max-elapsed-seconds",
|
|
42
|
+
type=float,
|
|
43
|
+
required=True,
|
|
44
|
+
help="maximum elapsed discovery and verification time",
|
|
45
|
+
)
|
|
46
|
+
parser.add_argument(
|
|
47
|
+
"--max-diagnostics", type=int, required=True, help="maximum retained diagnostics"
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def run(args: argparse.Namespace) -> dict[str, Any]:
|
|
52
|
+
try:
|
|
53
|
+
limits = BuildDiscoveryLimits(
|
|
54
|
+
max_depth=args.max_depth,
|
|
55
|
+
max_entries=args.max_entries,
|
|
56
|
+
max_builds=args.max_builds,
|
|
57
|
+
max_receipt_bytes=args.max_receipt_bytes,
|
|
58
|
+
max_member_bytes=args.max_member_bytes,
|
|
59
|
+
max_aggregate_bytes=args.max_aggregate_bytes,
|
|
60
|
+
max_elapsed_seconds=args.max_elapsed_seconds,
|
|
61
|
+
max_diagnostics=args.max_diagnostics,
|
|
62
|
+
)
|
|
63
|
+
published = build_private_index(args.build_root, args.index_root, limits=limits)
|
|
64
|
+
except LocalSearchIndexError as error:
|
|
65
|
+
raise SourceContractError(error.code, error.path, error.detail) from error
|
|
66
|
+
manifest = published.manifest
|
|
67
|
+
return {
|
|
68
|
+
"schema_version": "mr-data-private-index.v1",
|
|
69
|
+
"status": "build_index_published",
|
|
70
|
+
"manifest_sha256": published.manifest_sha256,
|
|
71
|
+
"aggregate_sha256": manifest.aggregate_sha256,
|
|
72
|
+
"entry_count": manifest.entry_count,
|
|
73
|
+
"locator_count": manifest.locator_count,
|
|
74
|
+
"diagnostic_count": manifest.diagnostic_count,
|
|
75
|
+
"total_bytes": manifest.total_bytes,
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
SPEC = CommandSpec(
|
|
80
|
+
name="index",
|
|
81
|
+
help="build one bounded private index from exact-verified local Builds",
|
|
82
|
+
add_arguments=add_arguments,
|
|
83
|
+
run=run,
|
|
84
|
+
)
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
"""``mr-data inventory`` -- the closed sets this build is held to, with their caveats.
|
|
2
|
+
|
|
3
|
+
This module is the command and nothing else: its one argument and the one call that turns it into
|
|
4
|
+
the payload both renderings are built from. The lists live in
|
|
5
|
+
:mod:`mostlyright.data_harness.ux.inventory`, derived from the frozensets the contract checker
|
|
6
|
+
uses, so the same answer is available to a browser page without going through argparse.
|
|
7
|
+
|
|
8
|
+
``--section`` offers exactly the sections that exist, taken from the inventory itself, so asking
|
|
9
|
+
for one that is not there is answered by the command line with the list of the ones that are.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import argparse
|
|
15
|
+
from typing import Any
|
|
16
|
+
|
|
17
|
+
from mostlyright.data_harness.ux.commands import CommandSpec
|
|
18
|
+
from mostlyright.data_harness.ux.inventory import SECTION_NAMES, build_inventory
|
|
19
|
+
|
|
20
|
+
DESCRIPTION = (
|
|
21
|
+
"List what this build can do: the cleaning steps, the types a cast may target, the join "
|
|
22
|
+
"relationships, the engines, the checks every build must pass, and the words a source option "
|
|
23
|
+
"is written in. Every list is read out of the code at the moment you ask, so it cannot fall "
|
|
24
|
+
"behind what the code does. Each one carries the sentence you need in order not to misread "
|
|
25
|
+
"it, on the same screen rather than in a footnote."
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def add_arguments(parser: argparse.ArgumentParser) -> None:
|
|
30
|
+
parser.description = DESCRIPTION
|
|
31
|
+
parser.add_argument(
|
|
32
|
+
"--section",
|
|
33
|
+
choices=SECTION_NAMES,
|
|
34
|
+
help="list one section on its own instead of all of them",
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def run(args: argparse.Namespace) -> dict[str, Any]:
|
|
39
|
+
return build_inventory().to_dict(section=args.section)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
SPEC = CommandSpec(
|
|
43
|
+
name="inventory",
|
|
44
|
+
help="list what this build can do: operations, types, joins, engines, and checks",
|
|
45
|
+
add_arguments=add_arguments,
|
|
46
|
+
run=run,
|
|
47
|
+
)
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
"""``mr-data list`` -- every build under a folder, and how much was actually read about each.
|
|
2
|
+
|
|
3
|
+
The module is named for what it does rather than for the command, because ``list`` is a name
|
|
4
|
+
Python already uses; ``SPEC.name`` is what reaches the command line.
|
|
5
|
+
|
|
6
|
+
The one thing this command must never do is imply a check it did not perform. Listing reads each
|
|
7
|
+
Receipt and stops there, so every row and the closing note say ``listed, not verified`` until
|
|
8
|
+
``--verify`` replays each build. Both output modes carry the same words, from the same facts.
|
|
9
|
+
|
|
10
|
+
That holds for the listing as a whole as well as for each row in it. The listing-level word comes
|
|
11
|
+
from :func:`readers.listing_verification`, which reads the entries; it is never the flag the caller
|
|
12
|
+
passed, because ``--verify`` is a request and "every build passed" is an outcome.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import argparse
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
from typing import Any
|
|
20
|
+
|
|
21
|
+
from mostlyright.data_harness.ux.commands import CommandSpec
|
|
22
|
+
from mostlyright.data_harness.ux.readers import (
|
|
23
|
+
DEFAULT_LIST_DEPTH,
|
|
24
|
+
DEFAULT_LIST_ROOT,
|
|
25
|
+
MAX_LIST_DEPTH,
|
|
26
|
+
BuildEntry,
|
|
27
|
+
list_builds,
|
|
28
|
+
listing_verification,
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _depth(value: str) -> int:
|
|
33
|
+
"""Bound how far down to look at the command line, before anything is opened."""
|
|
34
|
+
|
|
35
|
+
try:
|
|
36
|
+
depth = int(value)
|
|
37
|
+
except ValueError:
|
|
38
|
+
raise argparse.ArgumentTypeError(f"{value!r} is not a whole number of folders") from None
|
|
39
|
+
if not 0 <= depth <= MAX_LIST_DEPTH:
|
|
40
|
+
raise argparse.ArgumentTypeError(f"how far down to look must be from 0 to {MAX_LIST_DEPTH}")
|
|
41
|
+
return depth
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def add_arguments(parser: argparse.ArgumentParser) -> None:
|
|
45
|
+
parser.add_argument(
|
|
46
|
+
"root",
|
|
47
|
+
type=Path,
|
|
48
|
+
nargs="?",
|
|
49
|
+
default=DEFAULT_LIST_ROOT,
|
|
50
|
+
help=f"the folder to look in (default {DEFAULT_LIST_ROOT})",
|
|
51
|
+
)
|
|
52
|
+
parser.add_argument(
|
|
53
|
+
"--verify",
|
|
54
|
+
action="store_true",
|
|
55
|
+
help="replay every build found and check it against its sealed bytes",
|
|
56
|
+
)
|
|
57
|
+
parser.add_argument(
|
|
58
|
+
"--depth",
|
|
59
|
+
type=_depth,
|
|
60
|
+
default=DEFAULT_LIST_DEPTH,
|
|
61
|
+
help=f"how many folders down to look (default {DEFAULT_LIST_DEPTH})",
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def run(args: argparse.Namespace) -> dict[str, Any]:
|
|
66
|
+
entries = list_builds(args.root, verify=args.verify, max_depth=args.depth)
|
|
67
|
+
return {
|
|
68
|
+
"status": "builds_listed",
|
|
69
|
+
"root": str(Path(args.root)),
|
|
70
|
+
"depth": args.depth,
|
|
71
|
+
"verification": listing_verification(entries, verify=args.verify),
|
|
72
|
+
"note": _note(entries, root=Path(args.root), depth=args.depth, verify=args.verify),
|
|
73
|
+
"builds": _builds_payload(entries),
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _builds_payload(entries: tuple[BuildEntry, ...]) -> dict[str, Any]:
|
|
78
|
+
"""Facts about a build go in values; the keys only count. See the note in readers.py."""
|
|
79
|
+
|
|
80
|
+
width = len(str(max(len(entries), 1)))
|
|
81
|
+
return {
|
|
82
|
+
f"build {index:0{width}d}": entry.to_dict() for index, entry in enumerate(entries, start=1)
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _note(
|
|
87
|
+
entries: tuple[BuildEntry, ...],
|
|
88
|
+
*,
|
|
89
|
+
root: Path,
|
|
90
|
+
depth: int,
|
|
91
|
+
verify: bool,
|
|
92
|
+
) -> str:
|
|
93
|
+
"""One sentence that says what was done, and never more than was done.
|
|
94
|
+
|
|
95
|
+
The counts are of what was actually read, not of what the walk stopped at. A folder holding no
|
|
96
|
+
Build was not read from a Receipt and was not replayed, so it is counted separately rather than
|
|
97
|
+
folded into a claim the listing cannot make about it.
|
|
98
|
+
"""
|
|
99
|
+
|
|
100
|
+
if not entries:
|
|
101
|
+
return (
|
|
102
|
+
f"No builds under {root}, looking {depth} folders down. Run mr-data build to make one."
|
|
103
|
+
)
|
|
104
|
+
described = [entry for entry in entries if entry.reason is None]
|
|
105
|
+
if not described:
|
|
106
|
+
return f"{_leftover(entries)} {_ASK_FOR_MORE}"
|
|
107
|
+
if verify:
|
|
108
|
+
head = f"Each of these {len(described)} was replayed and matched its sealed bytes."
|
|
109
|
+
else:
|
|
110
|
+
head = (
|
|
111
|
+
f"These {len(described)} were read from their own Receipts and nothing more. "
|
|
112
|
+
"Add --verify to replay each one against its sealed bytes."
|
|
113
|
+
)
|
|
114
|
+
leftover = _leftover(entries)
|
|
115
|
+
return head if leftover is None else f"{head} {leftover} {_ASK_FOR_MORE}"
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
# Where the exact wording went. The listing carries one plain sentence per unreadable entry and no
|
|
119
|
+
# second engineering one, so it says which command does give the full answer.
|
|
120
|
+
_ASK_FOR_MORE = "Run mr-data show on one of those folders for the full answer."
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _leftover(entries: tuple[BuildEntry, ...]) -> str | None:
|
|
124
|
+
"""What the listing found but could not describe, counted by which of the two it was."""
|
|
125
|
+
|
|
126
|
+
unbuilt = sum(1 for entry in entries if entry.nothing_built)
|
|
127
|
+
unreadable = sum(1 for entry in entries if entry.reason is not None and not entry.nothing_built)
|
|
128
|
+
parts = []
|
|
129
|
+
if unbuilt:
|
|
130
|
+
parts.append(f"{unbuilt} {'folder' if unbuilt == 1 else 'folders'} here holds no build yet")
|
|
131
|
+
if unreadable:
|
|
132
|
+
parts.append(f"{unreadable} could not be read")
|
|
133
|
+
if not parts:
|
|
134
|
+
return None
|
|
135
|
+
return f"{' and '.join(parts)}; each says so below."
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
SPEC = CommandSpec(
|
|
139
|
+
name="list",
|
|
140
|
+
help="list the builds under a folder, and say whether they were checked",
|
|
141
|
+
add_arguments=add_arguments,
|
|
142
|
+
run=run,
|
|
143
|
+
)
|