mostlyright-data 0.9.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mostlyright/data_harness/__init__.py +158 -0
- mostlyright/data_harness/acquisition/__init__.py +55 -0
- mostlyright/data_harness/acquisition/http.py +2773 -0
- mostlyright/data_harness/acquisition/parsing.py +809 -0
- mostlyright/data_harness/acquisition/ranges.py +495 -0
- mostlyright/data_harness/acquisition/result_download.py +360 -0
- mostlyright/data_harness/acquisition/retention_admission.py +248 -0
- mostlyright/data_harness/acquisition/sandbox.py +4888 -0
- mostlyright/data_harness/acquisition/url_policy.py +530 -0
- mostlyright/data_harness/agent_runtime.py +2743 -0
- mostlyright/data_harness/assets/logo-ink.svg +31 -0
- mostlyright/data_harness/backends/__init__.py +28 -0
- mostlyright/data_harness/backends/pandas_backend.py +350 -0
- mostlyright/data_harness/backends/polars_backend.py +366 -0
- mostlyright/data_harness/backends/protocol.py +124 -0
- mostlyright/data_harness/backends/reference.py +83 -0
- mostlyright/data_harness/backends/registry.py +55 -0
- mostlyright/data_harness/backends/restrictions.py +126 -0
- mostlyright/data_harness/canonical.py +333 -0
- mostlyright/data_harness/catalog_job.py +625 -0
- mostlyright/data_harness/cli.py +5398 -0
- mostlyright/data_harness/contracts.py +53 -0
- mostlyright/data_harness/coordinator.py +1307 -0
- mostlyright/data_harness/deploy.py +924 -0
- mostlyright/data_harness/deploy_target.py +312 -0
- mostlyright/data_harness/deployment_evidence.py +1067 -0
- mostlyright/data_harness/event_presentation.py +576 -0
- mostlyright/data_harness/events.py +2152 -0
- mostlyright/data_harness/fast_delimited.py +239 -0
- mostlyright/data_harness/fleet.py +237 -0
- mostlyright/data_harness/formats.py +236 -0
- mostlyright/data_harness/governors.py +1163 -0
- mostlyright/data_harness/hosted_bootstrap.py +972 -0
- mostlyright/data_harness/hosted_crawler.py +1115 -0
- mostlyright/data_harness/hosted_crawler_container_smoke.py +351 -0
- mostlyright/data_harness/hosted_crawler_fetch.py +423 -0
- mostlyright/data_harness/hosted_crawler_job.py +1277 -0
- mostlyright/data_harness/hosted_crawler_protocol.py +676 -0
- mostlyright/data_harness/hosted_dataset.py +1500 -0
- mostlyright/data_harness/hosted_deploy.py +3037 -0
- mostlyright/data_harness/hosted_handoff.py +62 -0
- mostlyright/data_harness/hosted_ingestion_contract.py +504 -0
- mostlyright/data_harness/hosted_ingestion_job.py +356 -0
- mostlyright/data_harness/hosted_ingestion_job_smoke.py +40 -0
- mostlyright/data_harness/hosted_session_container_smoke.py +194 -0
- mostlyright/data_harness/hosted_session_worker.py +3554 -0
- mostlyright/data_harness/hosted_session_worker_job_smoke.py +46 -0
- mostlyright/data_harness/hosted_worker.py +6784 -0
- mostlyright/data_harness/ingestion/__init__.py +56 -0
- mostlyright/data_harness/ingestion/contracts.py +461 -0
- mostlyright/data_harness/ingestion/faults.py +42 -0
- mostlyright/data_harness/ingestion/gcs_store.py +1162 -0
- mostlyright/data_harness/ingestion/spool.py +130 -0
- mostlyright/data_harness/ingestion/store.py +885 -0
- mostlyright/data_harness/key_seam.py +434 -0
- mostlyright/data_harness/linux_process_boundary.py +262 -0
- mostlyright/data_harness/local_contracts.py +2880 -0
- mostlyright/data_harness/local_search/__init__.py +5 -0
- mostlyright/data_harness/local_search/build_index.py +1087 -0
- mostlyright/data_harness/local_search/contracts.py +920 -0
- mostlyright/data_harness/local_search/query_trace.py +266 -0
- mostlyright/data_harness/local_search/retrieval.py +700 -0
- mostlyright/data_harness/local_search/sealed.py +474 -0
- mostlyright/data_harness/local_search/service.py +784 -0
- mostlyright/data_harness/nbrender/CONTRACT.md +212 -0
- mostlyright/data_harness/nbrender/__init__.py +12 -0
- mostlyright/data_harness/nbrender/chrome.py +359 -0
- mostlyright/data_harness/nbrender/code_body.py +266 -0
- mostlyright/data_harness/nbrender/document.py +407 -0
- mostlyright/data_harness/nbrender/frame.py +275 -0
- mostlyright/data_harness/nbrender/interactive.py +337 -0
- mostlyright/data_harness/nbrender/markdown_body.py +477 -0
- mostlyright/data_harness/nbrender/mr_components.py +134 -0
- mostlyright/data_harness/nbrender/outputs_data.py +595 -0
- mostlyright/data_harness/nbrender/outputs_rich.py +906 -0
- mostlyright/data_harness/nbrender/outputs_source.py +260 -0
- mostlyright/data_harness/nbrender/outputs_stage.py +176 -0
- mostlyright/data_harness/nbrender/outputs_text.py +400 -0
- mostlyright/data_harness/nbrender/parse.py +394 -0
- mostlyright/data_harness/nbrender/status.py +40 -0
- mostlyright/data_harness/nbrender/tokens.py +1295 -0
- mostlyright/data_harness/notebook.py +1710 -0
- mostlyright/data_harness/offline.py +2049 -0
- mostlyright/data_harness/operation_registry.py +1007 -0
- mostlyright/data_harness/operator_setup.py +239 -0
- mostlyright/data_harness/pipeline.py +6428 -0
- mostlyright/data_harness/plan_graph.py +2026 -0
- mostlyright/data_harness/preparation/__init__.py +104 -0
- mostlyright/data_harness/preparation/contracts.py +1017 -0
- mostlyright/data_harness/preparation/engine.py +221 -0
- mostlyright/data_harness/preparation/errors.py +14 -0
- mostlyright/data_harness/preparation/gates.py +751 -0
- mostlyright/data_harness/preparation/joins.py +574 -0
- mostlyright/data_harness/preparation/profile.py +384 -0
- mostlyright/data_harness/preparation/table.py +217 -0
- mostlyright/data_harness/preparation/transforms.py +568 -0
- mostlyright/data_harness/progress_events.py +534 -0
- mostlyright/data_harness/readers/__init__.py +46 -0
- mostlyright/data_harness/readers/containers.py +963 -0
- mostlyright/data_harness/readers/contracts.py +542 -0
- mostlyright/data_harness/readers/delimited.py +257 -0
- mostlyright/data_harness/readers/grib2/__init__.py +33 -0
- mostlyright/data_harness/readers/grib2/admission.py +722 -0
- mostlyright/data_harness/readers/grib2/decode.py +1009 -0
- mostlyright/data_harness/readers/grib2/geometry.py +1133 -0
- mostlyright/data_harness/readers/grib2/portable_math.py +501 -0
- mostlyright/data_harness/readers/json_tabular.py +485 -0
- mostlyright/data_harness/readers/registry.py +514 -0
- mostlyright/data_harness/readers/samples/README.md +110 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/cities.csv.gz +0 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/cities.csv.gz +0 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/cities.tar +0 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/cities.tar +0 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/cities.zip +0 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/expected.json +25 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/expected.json +25 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/expected.json +25 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/cities.csv +3 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/expected.json +23 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/cities.tsv +5 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/expected.json +25 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/expected.json +30 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/observations.csv +5 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/expected.json +44 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/stations.json +1 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/expected.json +48 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/observations.ndjson +4 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/cities.xlsx +0 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/expected.json +24 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/expected.json +27 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/expected.json +27 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/README.md +20 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/expected.json +55 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/gfs-2m-temperature.grib2 +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/expected.json +54 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/expected.json +54 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/hrrr-categorical-rain.grib2 +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/expected.json +54 -0
- mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
- mostlyright/data_harness/readers/samples.py +582 -0
- mostlyright/data_harness/readers/spreadsheet.py +803 -0
- mostlyright/data_harness/readers/tabular.py +510 -0
- mostlyright/data_harness/recipe.py +5321 -0
- mostlyright/data_harness/repair/__init__.py +78 -0
- mostlyright/data_harness/repair/adapters.py +274 -0
- mostlyright/data_harness/repair/contracts.py +872 -0
- mostlyright/data_harness/repair/coordinator.py +1099 -0
- mostlyright/data_harness/repair/errors.py +16 -0
- mostlyright/data_harness/review.py +2533 -0
- mostlyright/data_harness/rowset.py +283 -0
- mostlyright/data_harness/serving.py +1975 -0
- mostlyright/data_harness/serving_edge.py +590 -0
- mostlyright/data_harness/serving_http.py +1031 -0
- mostlyright/data_harness/session_probes.py +759 -0
- mostlyright/data_harness/signing.py +101 -0
- mostlyright/data_harness/source_discovery.py +898 -0
- mostlyright/data_harness/sources/__init__.py +209 -0
- mostlyright/data_harness/sources/_adapter_steps.py +213 -0
- mostlyright/data_harness/sources/adapters.py +1214 -0
- mostlyright/data_harness/sources/cadence.py +1428 -0
- mostlyright/data_harness/sources/cadence_emission.py +453 -0
- mostlyright/data_harness/sources/cadence_history.py +546 -0
- mostlyright/data_harness/sources/catalog/__init__.py +17 -0
- mostlyright/data_harness/sources/catalog/admission.py +477 -0
- mostlyright/data_harness/sources/catalog/authoring.py +1701 -0
- mostlyright/data_harness/sources/catalog/authoring_policy.py +701 -0
- mostlyright/data_harness/sources/catalog/authoring_shards.py +1217 -0
- mostlyright/data_harness/sources/catalog/bounded_io.py +231 -0
- mostlyright/data_harness/sources/catalog/channel.py +523 -0
- mostlyright/data_harness/sources/catalog/channel_client.py +296 -0
- mostlyright/data_harness/sources/catalog/contracts.py +825 -0
- mostlyright/data_harness/sources/catalog/coverage.py +137 -0
- mostlyright/data_harness/sources/catalog/delta.py +1340 -0
- mostlyright/data_harness/sources/catalog/embedding.py +532 -0
- mostlyright/data_harness/sources/catalog/entry_v2.py +1182 -0
- mostlyright/data_harness/sources/catalog/fill.py +3889 -0
- mostlyright/data_harness/sources/catalog/fill_partitions.py +459 -0
- mostlyright/data_harness/sources/catalog/fill_staging.py +1105 -0
- mostlyright/data_harness/sources/catalog/gating.py +374 -0
- mostlyright/data_harness/sources/catalog/generation_receipt.py +1607 -0
- mostlyright/data_harness/sources/catalog/harvest/__init__.py +7 -0
- mostlyright/data_harness/sources/catalog/harvest/ckan.py +384 -0
- mostlyright/data_harness/sources/catalog/harvest/datagov_v4.py +798 -0
- mostlyright/data_harness/sources/catalog/harvest/protocol.py +964 -0
- mostlyright/data_harness/sources/catalog/harvest/sdmx.py +445 -0
- mostlyright/data_harness/sources/catalog/harvest/stac.py +384 -0
- mostlyright/data_harness/sources/catalog/health.py +447 -0
- mostlyright/data_harness/sources/catalog/hosted_catalog.py +105 -0
- mostlyright/data_harness/sources/catalog/identity_history.py +1549 -0
- mostlyright/data_harness/sources/catalog/neural.py +1618 -0
- mostlyright/data_harness/sources/catalog/packed_catalog.py +2345 -0
- mostlyright/data_harness/sources/catalog/packed_retrieval.py +1517 -0
- mostlyright/data_harness/sources/catalog/packed_writer.py +2802 -0
- mostlyright/data_harness/sources/catalog/query_trace.py +1037 -0
- mostlyright/data_harness/sources/catalog/recommend.py +171 -0
- mostlyright/data_harness/sources/catalog/retrieval.py +230 -0
- mostlyright/data_harness/sources/catalog/retrieval_manifest.py +995 -0
- mostlyright/data_harness/sources/catalog/rights_decisions.py +254 -0
- mostlyright/data_harness/sources/catalog/sealed.py +560 -0
- mostlyright/data_harness/sources/catalog/search.py +230 -0
- mostlyright/data_harness/sources/catalog/streaming_delta.py +1097 -0
- mostlyright/data_harness/sources/catalog/update.py +891 -0
- mostlyright/data_harness/sources/collections.py +815 -0
- mostlyright/data_harness/sources/contracts.py +2223 -0
- mostlyright/data_harness/sources/deletion.py +761 -0
- mostlyright/data_harness/sources/fitness.py +162 -0
- mostlyright/data_harness/sources/governance.py +163 -0
- mostlyright/data_harness/sources/hosted.py +173 -0
- mostlyright/data_harness/sources/integration.py +218 -0
- mostlyright/data_harness/sources/range_reader.py +418 -0
- mostlyright/data_harness/sources/registry.py +514 -0
- mostlyright/data_harness/sources/rights_rule.py +59 -0
- mostlyright/data_harness/sources/source_cadence_vectors.v1.json +1 -0
- mostlyright/data_harness/sources/sports.py +521 -0
- mostlyright/data_harness/sources/stream.py +524 -0
- mostlyright/data_harness/sources/stream_connector.py +418 -0
- mostlyright/data_harness/sources/stream_recorder.py +1404 -0
- mostlyright/data_harness/studio_boundary.py +2019 -0
- mostlyright/data_harness/thin/__init__.py +37 -0
- mostlyright/data_harness/thin/acquire.py +1137 -0
- mostlyright/data_harness/thin/acquire_cancel.py +579 -0
- mostlyright/data_harness/thin/approvals.py +617 -0
- mostlyright/data_harness/thin/commands.py +406 -0
- mostlyright/data_harness/thin/download.py +194 -0
- mostlyright/data_harness/thin/narrative.py +589 -0
- mostlyright/data_harness/thin/parity.py +1070 -0
- mostlyright/data_harness/thin/propose.py +2759 -0
- mostlyright/data_harness/thin/research.py +1663 -0
- mostlyright/data_harness/thin/router.py +924 -0
- mostlyright/data_harness/thin/runs.py +519 -0
- mostlyright/data_harness/thin/session.py +281 -0
- mostlyright/data_harness/thin/stream.py +501 -0
- mostlyright/data_harness/thin/transport.py +187 -0
- mostlyright/data_harness/thin/vocabulary.py +368 -0
- mostlyright/data_harness/thin/workers.py +164 -0
- mostlyright/data_harness/ucum/TABLE-PIN.json +40 -0
- mostlyright/data_harness/ucum/ucum-subset.v1.json +632 -0
- mostlyright/data_harness/unit_flow.py +927 -0
- mostlyright/data_harness/units.py +572 -0
- mostlyright/data_harness/ux/__init__.py +9 -0
- mostlyright/data_harness/ux/approve.py +485 -0
- mostlyright/data_harness/ux/author_yaml.py +597 -0
- mostlyright/data_harness/ux/cloud_auth.py +447 -0
- mostlyright/data_harness/ux/commands/__init__.py +260 -0
- mostlyright/data_harness/ux/commands/approve.py +136 -0
- mostlyright/data_harness/ux/commands/auth.py +744 -0
- mostlyright/data_harness/ux/commands/author.py +79 -0
- mostlyright/data_harness/ux/commands/catalog_author.py +403 -0
- mostlyright/data_harness/ux/commands/catalog_fill.py +523 -0
- mostlyright/data_harness/ux/commands/catalog_harvest.py +545 -0
- mostlyright/data_harness/ux/commands/catalog_publish.py +1838 -0
- mostlyright/data_harness/ux/commands/catalog_search.py +71 -0
- mostlyright/data_harness/ux/commands/catalog_update.py +437 -0
- mostlyright/data_harness/ux/commands/deploy.py +134 -0
- mostlyright/data_harness/ux/commands/deploy_dataset.py +98 -0
- mostlyright/data_harness/ux/commands/deploy_plan.py +105 -0
- mostlyright/data_harness/ux/commands/deploy_status.py +104 -0
- mostlyright/data_harness/ux/commands/diff.py +74 -0
- mostlyright/data_harness/ux/commands/index.py +84 -0
- mostlyright/data_harness/ux/commands/inventory.py +47 -0
- mostlyright/data_harness/ux/commands/list_builds.py +143 -0
- mostlyright/data_harness/ux/commands/login.py +63 -0
- mostlyright/data_harness/ux/commands/peek.py +236 -0
- mostlyright/data_harness/ux/commands/plan_check.py +90 -0
- mostlyright/data_harness/ux/commands/preflight.py +97 -0
- mostlyright/data_harness/ux/commands/record.py +107 -0
- mostlyright/data_harness/ux/commands/review_setup.py +47 -0
- mostlyright/data_harness/ux/commands/search.py +440 -0
- mostlyright/data_harness/ux/commands/show.py +61 -0
- mostlyright/data_harness/ux/commands/whoami.py +37 -0
- mostlyright/data_harness/ux/credential_native.py +551 -0
- mostlyright/data_harness/ux/credential_store.py +1055 -0
- mostlyright/data_harness/ux/credentials.py +631 -0
- mostlyright/data_harness/ux/diffing.py +444 -0
- mostlyright/data_harness/ux/headline.py +671 -0
- mostlyright/data_harness/ux/hosted_acquisition.py +974 -0
- mostlyright/data_harness/ux/hosted_run_status.py +619 -0
- mostlyright/data_harness/ux/inventory.py +427 -0
- mostlyright/data_harness/ux/local_review.py +375 -0
- mostlyright/data_harness/ux/login.py +691 -0
- mostlyright/data_harness/ux/path_kind.py +147 -0
- mostlyright/data_harness/ux/peek.py +1000 -0
- mostlyright/data_harness/ux/plain_file.py +178 -0
- mostlyright/data_harness/ux/plan_check.py +311 -0
- mostlyright/data_harness/ux/preflight.py +918 -0
- mostlyright/data_harness/ux/readers.py +1124 -0
- mostlyright/data_harness/ux/remediation.py +2195 -0
- mostlyright/data_harness/ux/render.py +657 -0
- mostlyright/data_harness/ux/workload.py +1077 -0
- mostlyright/data_harness/viewer.py +3713 -0
- mostlyright/data_harness/visual_run/__init__.py +83 -0
- mostlyright/data_harness/visual_run/authoring.py +235 -0
- mostlyright/data_harness/visual_run/contracts.py +673 -0
- mostlyright/data_harness/visual_run/materialize.py +486 -0
- mostlyright/data_harness/visual_run/observations.py +874 -0
- mostlyright/data_harness/visual_run/query.py +259 -0
- mostlyright/data_harness/visual_run/reducer.py +280 -0
- mostlyright/data_harness/visual_run/sdk.py +892 -0
- mostlyright/data_harness/visual_run/store.py +584 -0
- mostlyright/data_harness/visual_run/transport.py +239 -0
- mostlyright/data_harness/watch.py +2999 -0
- mostlyright_data-0.9.0.dist-info/METADATA +607 -0
- mostlyright_data-0.9.0.dist-info/RECORD +314 -0
- mostlyright_data-0.9.0.dist-info/WHEEL +4 -0
- mostlyright_data-0.9.0.dist-info/entry_points.txt +12 -0
|
@@ -0,0 +1,1009 @@
|
|
|
1
|
+
"""``weather.grib2``: named places, their values, and the arithmetic written down.
|
|
2
|
+
|
|
3
|
+
This module is the join. Everything it needs already exists and is certified on its own:
|
|
4
|
+
``admission`` refuses an unknown grid or packing from the headers, before a value is read;
|
|
5
|
+
``geometry`` computes coordinates in closed form and grades them against the publisher's own
|
|
6
|
+
declaration on every message; the vendored binding unpacks the packed integers and cross-checks
|
|
7
|
+
the template numbers it was told to expect. What is added here is the composition, the recipe's
|
|
8
|
+
own settings, and the one piece of arithmetic nobody else owns: turning a packed integer into a
|
|
9
|
+
physical value.
|
|
10
|
+
|
|
11
|
+
Mostly wiring, deliberately
|
|
12
|
+
---------------------------
|
|
13
|
+
If this module grows admission logic, projection maths, or bit unpacking, that logic has been
|
|
14
|
+
duplicated out of the module that already owns it and the two copies will disagree. What lives
|
|
15
|
+
here is what lives nowhere else: the option schema, the variable and time the product section
|
|
16
|
+
declares, the scaling rule, and the emitted table.
|
|
17
|
+
|
|
18
|
+
Admission is structural rather than remembered
|
|
19
|
+
----------------------------------------------
|
|
20
|
+
``decode_admitted`` takes a ``MessageAdmission`` and there is no function here that takes message
|
|
21
|
+
bytes without one -- the single exception is ``WeatherGrib2Reader.decode``, which is the Reader
|
|
22
|
+
contract's own entry point, is handed bytes by the contract, and admits them before it does
|
|
23
|
+
anything else. ``tests/h3/test_readers_grib2.py`` parses this file and asserts both halves, so a
|
|
24
|
+
second door has to be added to a named tuple in a reviewed edit rather than appearing quietly.
|
|
25
|
+
The binding's own template cross-check is the second layer, for a call site that forgot.
|
|
26
|
+
|
|
27
|
+
The recipe pins the allowlist it was written against
|
|
28
|
+
----------------------------------------------------
|
|
29
|
+
``admission_allowlist_version`` is a required setting. The refusals in ``admission`` handle a
|
|
30
|
+
message this product has never checked; this handles the other direction, where the Toolbox has
|
|
31
|
+
learned a new template since the recipe was frozen and the same recipe would now decode a file it
|
|
32
|
+
would previously have refused. Both directions are the same property: a recipe's meaning must not
|
|
33
|
+
drift while the recipe does not change.
|
|
34
|
+
|
|
35
|
+
Row order is the recipe's
|
|
36
|
+
-------------------------
|
|
37
|
+
Rows come out in the order the recipe lists its points, not the order the grid stores them. Grid
|
|
38
|
+
order is an implementation detail of the file -- a model that reorganises its grid changes it
|
|
39
|
+
without announcing anything -- and the recipe's order is what the author asked for and is stable
|
|
40
|
+
across that.
|
|
41
|
+
|
|
42
|
+
A missing value is a missing value
|
|
43
|
+
----------------------------------
|
|
44
|
+
A weather message may carry a bitmap marking points that have no value. Such a point keeps its
|
|
45
|
+
row and gets an empty cell. Never a sentinel, which would be indistinguishable from a real
|
|
46
|
+
reading, and never a non-finite value, which the sandbox IPC refuses outright -- the boundary
|
|
47
|
+
telling us the right answer. A decode that met one reports ``contains_masked_points``, so the
|
|
48
|
+
empty cell is a fact the receipt carries rather than a gap a reader has to interpret.
|
|
49
|
+
|
|
50
|
+
A coordinate is decimal text, and that is not a style choice
|
|
51
|
+
------------------------------------------------------------
|
|
52
|
+
Decode options are sealed through canonical JSON, which admits **no non-integer number at all**
|
|
53
|
+
(``canonical.py``, ``JSON_NONINTEGER_NUMBER``): a sealed document whose numbers depend on a JSON
|
|
54
|
+
writer's float formatting is a document that does not have one canonical form. So a recipe
|
|
55
|
+
states a latitude and a longitude as decimal text -- ``"39.7392"`` -- and this module converts it
|
|
56
|
+
with the one correctly-rounded decimal-to-binary64 conversion the language has. What is sealed
|
|
57
|
+
is then the digits the author wrote, and it is the same digits on every host. Exponent notation
|
|
58
|
+
is refused: two spellings of one coordinate would seal to two digests for one recipe.
|
|
59
|
+
|
|
60
|
+
Import discipline: ``formats``, ``readers.contracts``, ``readers.tabular``, ``readers.grib2``,
|
|
61
|
+
the vendored binding, and the standard library. Nothing from ``acquisition``, ``sources``,
|
|
62
|
+
``recipe``, or ``pipeline``.
|
|
63
|
+
"""
|
|
64
|
+
|
|
65
|
+
from __future__ import annotations
|
|
66
|
+
|
|
67
|
+
import datetime
|
|
68
|
+
import math
|
|
69
|
+
import re
|
|
70
|
+
from collections.abc import Mapping, Sequence
|
|
71
|
+
from dataclasses import dataclass, field
|
|
72
|
+
from fractions import Fraction
|
|
73
|
+
from typing import Any, NoReturn
|
|
74
|
+
|
|
75
|
+
import mostlyright_grib
|
|
76
|
+
|
|
77
|
+
from mostlyright.data_harness.canonical import sha256_bytes
|
|
78
|
+
from mostlyright.data_harness.formats import (
|
|
79
|
+
FORMAT_MEDIA_TYPES,
|
|
80
|
+
FORMAT_SUFFIXES,
|
|
81
|
+
READER_CONTRACT_VERSION,
|
|
82
|
+
READER_OUTPUT_FORMATS,
|
|
83
|
+
)
|
|
84
|
+
from mostlyright.data_harness.readers.contracts import (
|
|
85
|
+
DECODE_FLAGS,
|
|
86
|
+
ReaderBudgets,
|
|
87
|
+
ReaderError,
|
|
88
|
+
ReaderPin,
|
|
89
|
+
ReaderResult,
|
|
90
|
+
)
|
|
91
|
+
from mostlyright.data_harness.readers.grib2.admission import (
|
|
92
|
+
ADMISSION_ALLOWLIST_VERSION,
|
|
93
|
+
ALLOWLIST_VERSION_CONSTANT,
|
|
94
|
+
MessageAdmission,
|
|
95
|
+
admit,
|
|
96
|
+
)
|
|
97
|
+
from mostlyright.data_harness.readers.grib2.geometry import (
|
|
98
|
+
build_grid,
|
|
99
|
+
nearest_grid_index,
|
|
100
|
+
)
|
|
101
|
+
from mostlyright.data_harness.readers.tabular import (
|
|
102
|
+
check_declared_size,
|
|
103
|
+
encode_canonical_csv,
|
|
104
|
+
sealed_filename,
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
_MISSING_BINDING_MEMBERS = tuple(
|
|
108
|
+
name for name in ("GribBindingError", "unpack") if not hasattr(mostlyright_grib, name)
|
|
109
|
+
)
|
|
110
|
+
if _MISSING_BINDING_MEMBERS:
|
|
111
|
+
raise ImportError(
|
|
112
|
+
"mostlyright-grib is incompatible: missing " + ", ".join(_MISSING_BINDING_MEMBERS)
|
|
113
|
+
)
|
|
114
|
+
if not callable(mostlyright_grib.unpack) or not isinstance(mostlyright_grib.GribBindingError, type):
|
|
115
|
+
raise ImportError("mostlyright-grib is incompatible: binding API members have invalid types")
|
|
116
|
+
|
|
117
|
+
__all__ = [
|
|
118
|
+
"ADMITTED_PRODUCT_TEMPLATES",
|
|
119
|
+
"ADMITTED_TIME_SIGNIFICANCE",
|
|
120
|
+
"ADMITTED_TIME_UNIT_SECONDS",
|
|
121
|
+
"COLUMN_NAMES",
|
|
122
|
+
"FLOAT_RULE",
|
|
123
|
+
"MASKED_POINT_FLAG",
|
|
124
|
+
"MAX_BINARY_SCALE_MAGNITUDE",
|
|
125
|
+
"MAX_DECIMAL_SCALE_MAGNITUDE",
|
|
126
|
+
"V2_COLUMN_NAMES",
|
|
127
|
+
"NamedPoint",
|
|
128
|
+
"Variable",
|
|
129
|
+
"WeatherGrib2Reader",
|
|
130
|
+
"WeatherGrib2ReaderV2",
|
|
131
|
+
"decode_admitted",
|
|
132
|
+
"decode_admitted_v2",
|
|
133
|
+
"physical_value",
|
|
134
|
+
"settings_from_options",
|
|
135
|
+
]
|
|
136
|
+
|
|
137
|
+
# The one encoding a Reader may seal, unpacked rather than restated.
|
|
138
|
+
(_OUTPUT_FORMAT,) = READER_OUTPUT_FORMATS
|
|
139
|
+
_OUTPUT_MEDIA_TYPE = sorted(FORMAT_MEDIA_TYPES[_OUTPUT_FORMAT])[0]
|
|
140
|
+
_OUTPUT_SUFFIX = sorted(FORMAT_SUFFIXES[_OUTPUT_FORMAT])[0]
|
|
141
|
+
|
|
142
|
+
# The fact a decode reports when the message marked one of the requested points absent. Declared
|
|
143
|
+
# in the closed vocabulary in ``readers/contracts.py`` and nowhere else; this line fails at import
|
|
144
|
+
# if the two ever disagree, which is the loud failure rather than a flag the result would refuse.
|
|
145
|
+
MASKED_POINT_FLAG = "contains_masked_points"
|
|
146
|
+
if MASKED_POINT_FLAG not in DECODE_FLAGS: # pragma: no cover - a contradiction caught at import
|
|
147
|
+
raise RuntimeError(f"{MASKED_POINT_FLAG} is not in the closed decode-flag vocabulary")
|
|
148
|
+
|
|
149
|
+
# The emitted table, in this order, for every weather dataset this product builds.
|
|
150
|
+
#
|
|
151
|
+
# ``grid_latitude`` and ``grid_longitude`` are the coordinates of the cell that answered, and they
|
|
152
|
+
# are carried because the attribution of a value to a place is either right or confidently wrong.
|
|
153
|
+
# A reader of the dataset can see how far the answering point is from the place that was asked
|
|
154
|
+
# for, without re-running anything; a table carrying only the request cannot be checked at all.
|
|
155
|
+
COLUMN_NAMES: tuple[str, ...] = (
|
|
156
|
+
"point_id",
|
|
157
|
+
"requested_latitude",
|
|
158
|
+
"requested_longitude",
|
|
159
|
+
"grid_row",
|
|
160
|
+
"grid_column",
|
|
161
|
+
"grid_latitude",
|
|
162
|
+
"grid_longitude",
|
|
163
|
+
"valid_time",
|
|
164
|
+
"variable",
|
|
165
|
+
"value",
|
|
166
|
+
)
|
|
167
|
+
|
|
168
|
+
# Version 2 is the evidence-bearing row contract used when a range-selected message must remain
|
|
169
|
+
# attributable after it becomes a table. It is deliberately a second family version rather than
|
|
170
|
+
# an edit to ``COLUMN_NAMES``: recipes that pin 1.0.0 keep producing byte-identical tables.
|
|
171
|
+
V2_COLUMN_NAMES: tuple[str, ...] = (
|
|
172
|
+
"point_id",
|
|
173
|
+
"variable",
|
|
174
|
+
"level",
|
|
175
|
+
"run_time",
|
|
176
|
+
"valid_time",
|
|
177
|
+
"value",
|
|
178
|
+
"matched_grid_lat",
|
|
179
|
+
"matched_grid_lon",
|
|
180
|
+
"message_sha256",
|
|
181
|
+
)
|
|
182
|
+
|
|
183
|
+
# The product templates this family reads. 4.0 is an instantaneous value at a single level, which
|
|
184
|
+
# is what every message on the admission allowlist declares. A statistically processed product --
|
|
185
|
+
# an accumulation or an average over a period -- is 4.8 and is refused, because its value is not
|
|
186
|
+
# valid *at* a time but *over* one, and a table with a single valid-time column would state
|
|
187
|
+
# something the message does not. Admitting one is a reviewed edit here plus a column that says
|
|
188
|
+
# what period the value covers.
|
|
189
|
+
ADMITTED_PRODUCT_TEMPLATES: frozenset[int] = frozenset({0})
|
|
190
|
+
|
|
191
|
+
# What octet 12 of the identification section may say the reference time means. Analysis and
|
|
192
|
+
# start-of-forecast are the two under which "valid time is the reference time plus the forecast
|
|
193
|
+
# offset" is true. Under verifying time or observation time the reference time already *is* the
|
|
194
|
+
# valid time and adding the offset would move every stamp, so those are refused rather than
|
|
195
|
+
# guessed at.
|
|
196
|
+
ADMITTED_TIME_SIGNIFICANCE: frozenset[int] = frozenset({0, 1})
|
|
197
|
+
|
|
198
|
+
# Code table 4.4, restricted to the units that are a fixed number of seconds. A month, a year, a
|
|
199
|
+
# decade, a normal, and a century are not durations -- they depend on which month and which year --
|
|
200
|
+
# so a forecast measured in them has no valid time this Reader can compute, and it is refused
|
|
201
|
+
# rather than approximated at 30 days.
|
|
202
|
+
ADMITTED_TIME_UNIT_SECONDS: Mapping[int, int] = {
|
|
203
|
+
0: 60,
|
|
204
|
+
1: 3600,
|
|
205
|
+
2: 86400,
|
|
206
|
+
10: 3 * 3600,
|
|
207
|
+
11: 6 * 3600,
|
|
208
|
+
12: 12 * 3600,
|
|
209
|
+
13: 1,
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
# How large a scale exponent this Reader will do arithmetic with. The bound is a resource bound
|
|
213
|
+
# and it is stated as one: the rule below is exact integer arithmetic, so an unbounded exponent
|
|
214
|
+
# means an unbounded integer, and an exponent of thirty thousand -- which the format's two-octet
|
|
215
|
+
# sign-magnitude field can express -- would be a denial of service written in a header. Both
|
|
216
|
+
# bounds sit far outside anything published: the vendored messages carry decimal exponents of 0
|
|
217
|
+
# and 1, and the largest reported in the wild is 27. A message beyond them is refused by name.
|
|
218
|
+
MAX_BINARY_SCALE_MAGNITUDE = 1100
|
|
219
|
+
MAX_DECIMAL_SCALE_MAGNITUDE = 350
|
|
220
|
+
|
|
221
|
+
# The magnitude of the surface scale factor this Reader reads, bounded for the same reason.
|
|
222
|
+
_MAX_SURFACE_SCALE_MAGNITUDE = 20
|
|
223
|
+
|
|
224
|
+
# The octet a missing one-byte field carries, and the value a missing four-byte one carries.
|
|
225
|
+
_MISSING_OCTET = 0xFF
|
|
226
|
+
_MISSING_SURFACE = 255
|
|
227
|
+
|
|
228
|
+
# The shortest identification and product sections this Reader reads a field out of.
|
|
229
|
+
_IDENTIFICATION_OCTETS = 21
|
|
230
|
+
_PRODUCT_OCTETS = 34
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
FLOAT_RULE = """Weather values follow these numeric and output invariants.
|
|
234
|
+
|
|
235
|
+
The decoded value is the binary64 nearest to the exact rational
|
|
236
|
+
|
|
237
|
+
(R + X * 2^E) / 10^D
|
|
238
|
+
|
|
239
|
+
where X is the packed integer, R is the binary32 reference value widened exactly to binary64, and
|
|
240
|
+
E and D are the scale exponents. Integer arithmetic constructs the rational and rounds once at the
|
|
241
|
+
end. Direct binary64 division or multiplication is rejected because it rounds intermediate values.
|
|
242
|
+
No floating-point multiply-add exists on this path, so fused multiply-add contraction is impossible.
|
|
243
|
+
|
|
244
|
+
Decoding is single-threaded. Scaling runs only for requested output points. Text output uses the
|
|
245
|
+
shortest round-trip decimal representation. A missing point produces an empty cell; sentinels and
|
|
246
|
+
non-finite values are not emitted.
|
|
247
|
+
"""
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
@dataclass(frozen=True)
|
|
251
|
+
class NamedPoint:
|
|
252
|
+
"""One place a recipe asked for: what to call it, and where it is."""
|
|
253
|
+
|
|
254
|
+
identity: str
|
|
255
|
+
latitude: float
|
|
256
|
+
longitude: float
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
@dataclass(frozen=True)
|
|
260
|
+
class Variable:
|
|
261
|
+
"""The field a recipe wants, stated so the message can be held to it.
|
|
262
|
+
|
|
263
|
+
The name is the label that reaches the table. The five numbers are the message's own
|
|
264
|
+
coordinates for a field -- discipline, parameter category, parameter number, and the surface
|
|
265
|
+
the value sits on -- and they are stated by the recipe rather than looked up from a name
|
|
266
|
+
table on purpose. Variable names drift across decoder table versions even when the numbers
|
|
267
|
+
do not, so a name table would make a dataset's meaning depend on which tables were installed.
|
|
268
|
+
"""
|
|
269
|
+
|
|
270
|
+
name: str
|
|
271
|
+
discipline: int
|
|
272
|
+
parameter_category: int
|
|
273
|
+
parameter_number: int
|
|
274
|
+
surface_type: int
|
|
275
|
+
surface_value: Fraction
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
@dataclass(frozen=True)
|
|
279
|
+
class DecodeSettings:
|
|
280
|
+
"""Everything the recipe decides, in one typed record."""
|
|
281
|
+
|
|
282
|
+
variable: Variable
|
|
283
|
+
points: tuple[NamedPoint, ...]
|
|
284
|
+
allowlist_version: int
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
@dataclass(frozen=True)
|
|
288
|
+
class _Product:
|
|
289
|
+
"""What the identification and product sections declare about the field and its time."""
|
|
290
|
+
|
|
291
|
+
template: int
|
|
292
|
+
discipline: int
|
|
293
|
+
parameter_category: int
|
|
294
|
+
parameter_number: int
|
|
295
|
+
surface_type: int
|
|
296
|
+
surface_value: Fraction
|
|
297
|
+
run_time: datetime.datetime
|
|
298
|
+
valid_time: datetime.datetime
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
# --- The options, which are the whole of what a recipe decides ------------------------------
|
|
302
|
+
|
|
303
|
+
_OPTION_KEYS = ("variable", "points", "admission_allowlist_version")
|
|
304
|
+
_VARIABLE_KEYS = (
|
|
305
|
+
"name",
|
|
306
|
+
"discipline",
|
|
307
|
+
"parameter_category",
|
|
308
|
+
"parameter_number",
|
|
309
|
+
"surface_type",
|
|
310
|
+
"surface_value",
|
|
311
|
+
)
|
|
312
|
+
_POINT_KEYS = ("id", "latitude", "longitude")
|
|
313
|
+
|
|
314
|
+
# A coordinate as a recipe states it: plain decimal text, bounded, no exponent and no leading
|
|
315
|
+
# plus. Twelve fractional digits is four orders of magnitude finer than the millionths of a
|
|
316
|
+
# degree a message's own fields are quantised to, so nothing a publisher can express is lost.
|
|
317
|
+
_COORDINATE = re.compile(r"^-?(0|[1-9][0-9]{0,3})(\.[0-9]{1,12})?$")
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
def validate_options(options: Any, *, subject: str = "reader.weather.grib2.decode_options") -> dict:
|
|
321
|
+
"""Admit this family's closed option set, refusing an unknown key by name.
|
|
322
|
+
|
|
323
|
+
Strict on purpose. The options digest is sealed into the recipe forever, so a key that was
|
|
324
|
+
accepted and ignored would be an unreviewable difference between two recipes that read
|
|
325
|
+
identically to everyone who looks at them.
|
|
326
|
+
"""
|
|
327
|
+
|
|
328
|
+
if not isinstance(options, Mapping):
|
|
329
|
+
_refuse("READER_OPTIONS", subject, "must be an object")
|
|
330
|
+
_closed(options, _OPTION_KEYS, subject, "this family takes")
|
|
331
|
+
variable = _variable_options(options.get("variable"), f"{subject}.variable")
|
|
332
|
+
points = _point_options(options.get("points"), f"{subject}.points")
|
|
333
|
+
version = options.get("admission_allowlist_version")
|
|
334
|
+
if type(version) is not int or version < 1:
|
|
335
|
+
_refuse(
|
|
336
|
+
"READER_OPTIONS",
|
|
337
|
+
f"{subject}.admission_allowlist_version",
|
|
338
|
+
f"states {version!r}, and it must be the positive integer version of the admission "
|
|
339
|
+
f"allowlist this recipe was written against. It is required because a recipe that did "
|
|
340
|
+
f"not state one would change meaning the day the Toolbox learned a template. This "
|
|
341
|
+
f"build admits version {ADMISSION_ALLOWLIST_VERSION}",
|
|
342
|
+
)
|
|
343
|
+
return {"variable": variable, "points": points, "admission_allowlist_version": version}
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
def settings_from_options(
|
|
347
|
+
options: Any,
|
|
348
|
+
*,
|
|
349
|
+
subject: str = "reader.weather.grib2.decode_options",
|
|
350
|
+
) -> DecodeSettings:
|
|
351
|
+
"""The typed record the decode runs on, built from settings that have been admitted."""
|
|
352
|
+
|
|
353
|
+
admitted = validate_options(options, subject=subject)
|
|
354
|
+
variable = admitted["variable"]
|
|
355
|
+
return DecodeSettings(
|
|
356
|
+
variable=Variable(
|
|
357
|
+
name=str(variable["name"]),
|
|
358
|
+
discipline=int(variable["discipline"]),
|
|
359
|
+
parameter_category=int(variable["parameter_category"]),
|
|
360
|
+
parameter_number=int(variable["parameter_number"]),
|
|
361
|
+
surface_type=int(variable["surface_type"]),
|
|
362
|
+
surface_value=Fraction(int(variable["surface_value"])),
|
|
363
|
+
),
|
|
364
|
+
points=tuple(
|
|
365
|
+
NamedPoint(str(point["id"]), float(point["latitude"]), float(point["longitude"]))
|
|
366
|
+
for point in admitted["points"]
|
|
367
|
+
),
|
|
368
|
+
allowlist_version=int(admitted["admission_allowlist_version"]),
|
|
369
|
+
)
|
|
370
|
+
|
|
371
|
+
|
|
372
|
+
def _closed(stated: Mapping[str, Any], keys: Sequence[str], subject: str, lead: str) -> None:
|
|
373
|
+
unknown = sorted(str(key) for key in stated if key not in keys)
|
|
374
|
+
if unknown:
|
|
375
|
+
_refuse(
|
|
376
|
+
"READER_OPTIONS",
|
|
377
|
+
subject,
|
|
378
|
+
f"names no such setting: {', '.join(unknown)}; {lead} {', '.join(sorted(keys))}",
|
|
379
|
+
)
|
|
380
|
+
missing = sorted(key for key in keys if key not in stated)
|
|
381
|
+
if missing:
|
|
382
|
+
_refuse("READER_OPTIONS", subject, f"does not state: {', '.join(missing)}")
|
|
383
|
+
|
|
384
|
+
|
|
385
|
+
def _variable_options(stated: Any, subject: str) -> dict:
|
|
386
|
+
if not isinstance(stated, Mapping):
|
|
387
|
+
_refuse("READER_OPTIONS", subject, "must be an object naming the field this recipe wants")
|
|
388
|
+
_closed(stated, _VARIABLE_KEYS, subject, "a variable is stated as")
|
|
389
|
+
name = stated["name"]
|
|
390
|
+
if not isinstance(name, str) or not name.strip() or len(name) > 128:
|
|
391
|
+
_refuse("READER_OPTIONS", f"{subject}.name", "must be the label this field carries")
|
|
392
|
+
for key in ("discipline", "parameter_category", "parameter_number", "surface_type"):
|
|
393
|
+
value = stated[key]
|
|
394
|
+
if type(value) is not int or not 0 <= value <= 255:
|
|
395
|
+
_refuse(
|
|
396
|
+
"READER_OPTIONS",
|
|
397
|
+
f"{subject}.{key}",
|
|
398
|
+
"must be the octet the message states for it, between 0 and 255",
|
|
399
|
+
)
|
|
400
|
+
surface = stated["surface_value"]
|
|
401
|
+
if type(surface) is not int:
|
|
402
|
+
_refuse(
|
|
403
|
+
"READER_OPTIONS",
|
|
404
|
+
f"{subject}.surface_value",
|
|
405
|
+
"must be the whole number the level is stated at -- 2 for two metres above ground, "
|
|
406
|
+
"85000 for the 850 hPa level in pascals -- because a sealed decode option is "
|
|
407
|
+
"canonical JSON and canonical JSON carries no fractional number",
|
|
408
|
+
)
|
|
409
|
+
return dict(stated)
|
|
410
|
+
|
|
411
|
+
|
|
412
|
+
def _point_options(stated: Any, subject: str) -> tuple[dict, ...]:
|
|
413
|
+
if isinstance(stated, (str, bytes)) or not isinstance(stated, Sequence) or not stated:
|
|
414
|
+
_refuse(
|
|
415
|
+
"READER_OPTIONS",
|
|
416
|
+
subject,
|
|
417
|
+
"must be a nonempty list of the places this dataset is built at; a weather file is "
|
|
418
|
+
"a grid and a dataset is the places you named, so naming none asks for nothing",
|
|
419
|
+
)
|
|
420
|
+
admitted: list[dict] = []
|
|
421
|
+
seen: set[str] = set()
|
|
422
|
+
for index, point in enumerate(stated):
|
|
423
|
+
where = f"{subject}[{index}]"
|
|
424
|
+
if not isinstance(point, Mapping):
|
|
425
|
+
_refuse("READER_OPTIONS", where, "must be an object")
|
|
426
|
+
_closed(point, _POINT_KEYS, where, "a place is stated as")
|
|
427
|
+
identity = point["id"]
|
|
428
|
+
if not isinstance(identity, str) or not identity.strip() or len(identity) > 128:
|
|
429
|
+
_refuse("READER_OPTIONS", f"{where}.id", "must be the name this place carries")
|
|
430
|
+
if identity in seen:
|
|
431
|
+
_refuse(
|
|
432
|
+
"READER_OPTIONS",
|
|
433
|
+
f"{where}.id",
|
|
434
|
+
f"repeats {identity}; two rows under one name is a table nobody can join on",
|
|
435
|
+
)
|
|
436
|
+
seen.add(identity)
|
|
437
|
+
for key in ("latitude", "longitude"):
|
|
438
|
+
value = point[key]
|
|
439
|
+
if not isinstance(value, str) or _COORDINATE.fullmatch(value) is None:
|
|
440
|
+
_refuse(
|
|
441
|
+
"READER_OPTIONS",
|
|
442
|
+
f"{where}.{key}",
|
|
443
|
+
'must be plain decimal text such as "39.7392", with no exponent and no '
|
|
444
|
+
"leading plus. It is text rather than a number because a sealed decode "
|
|
445
|
+
"option is canonical JSON, which carries no fractional number at all -- so "
|
|
446
|
+
"what is sealed is the digits written here, identically on every host",
|
|
447
|
+
)
|
|
448
|
+
admitted.append(dict(point))
|
|
449
|
+
return tuple(admitted)
|
|
450
|
+
|
|
451
|
+
|
|
452
|
+
# --- The arithmetic ---------------------------------------------------------------------------
|
|
453
|
+
|
|
454
|
+
|
|
455
|
+
def physical_value(
|
|
456
|
+
level: float,
|
|
457
|
+
*,
|
|
458
|
+
reference_value: float,
|
|
459
|
+
binary_scale: int,
|
|
460
|
+
decimal_scale: int,
|
|
461
|
+
) -> float:
|
|
462
|
+
"""Recover one physical value from one packed integer, under the rule in ``FLOAT_RULE``.
|
|
463
|
+
|
|
464
|
+
The whole rule is in that constant and this function is its one implementation. The exact
|
|
465
|
+
rational is built with integer arithmetic and rounded to binary64 exactly once, by the
|
|
466
|
+
division on the last line -- ``Fraction.__float__`` divides two integers, which CPython
|
|
467
|
+
rounds correctly. Nothing else on this path rounds.
|
|
468
|
+
"""
|
|
469
|
+
|
|
470
|
+
if not isinstance(level, float) or not math.isfinite(level):
|
|
471
|
+
_refuse(
|
|
472
|
+
"READER_OUTPUT",
|
|
473
|
+
"reader.weather.grib2.level",
|
|
474
|
+
"a point with no value is an empty cell rather than a number, so it never reaches "
|
|
475
|
+
"the scaling rule",
|
|
476
|
+
)
|
|
477
|
+
if abs(binary_scale) > MAX_BINARY_SCALE_MAGNITUDE:
|
|
478
|
+
_refuse(
|
|
479
|
+
"READER_ADMISSION",
|
|
480
|
+
"reader.weather.grib2.binary_scale",
|
|
481
|
+
f"declares a binary scale exponent of {binary_scale}, beyond the "
|
|
482
|
+
f"{MAX_BINARY_SCALE_MAGNITUDE} this Reader computes with; the rule is exact integer "
|
|
483
|
+
f"arithmetic, so an unbounded exponent is an unbounded integer",
|
|
484
|
+
)
|
|
485
|
+
if abs(decimal_scale) > MAX_DECIMAL_SCALE_MAGNITUDE:
|
|
486
|
+
_refuse(
|
|
487
|
+
"READER_ADMISSION",
|
|
488
|
+
"reader.weather.grib2.decimal_scale",
|
|
489
|
+
f"declares a decimal scale exponent of {decimal_scale}, beyond the "
|
|
490
|
+
f"{MAX_DECIMAL_SCALE_MAGNITUDE} this Reader computes with; the rule is exact integer "
|
|
491
|
+
f"arithmetic, so an unbounded exponent is an unbounded integer",
|
|
492
|
+
)
|
|
493
|
+
exact = (Fraction(reference_value) + Fraction(level) * Fraction(2) ** binary_scale) / Fraction(
|
|
494
|
+
10
|
|
495
|
+
) ** decimal_scale
|
|
496
|
+
value = float(exact)
|
|
497
|
+
if not math.isfinite(value):
|
|
498
|
+
_refuse(
|
|
499
|
+
"READER_OUTPUT",
|
|
500
|
+
"reader.weather.grib2.value",
|
|
501
|
+
f"scales to {exact.numerator}/{exact.denominator}, which is outside the range a "
|
|
502
|
+
f"binary64 value can carry; a value that cannot be represented is not rendered as "
|
|
503
|
+
f"one that can",
|
|
504
|
+
)
|
|
505
|
+
return value
|
|
506
|
+
|
|
507
|
+
|
|
508
|
+
# --- What the message says about its field and its time ---------------------------------------
|
|
509
|
+
|
|
510
|
+
|
|
511
|
+
def _product(admission: MessageAdmission, content: bytes) -> _Product:
|
|
512
|
+
"""Read the field and the valid time out of the sections admission located."""
|
|
513
|
+
|
|
514
|
+
if admission.identification_length < _IDENTIFICATION_OCTETS:
|
|
515
|
+
_refuse(
|
|
516
|
+
"READER_ADMISSION",
|
|
517
|
+
"reader.weather.grib2.identification",
|
|
518
|
+
f"declares {admission.identification_length} bytes, too few to state the reference "
|
|
519
|
+
f"time a valid time is computed from",
|
|
520
|
+
)
|
|
521
|
+
if admission.product_length < _PRODUCT_OCTETS:
|
|
522
|
+
_refuse(
|
|
523
|
+
"READER_ADMISSION",
|
|
524
|
+
"reader.weather.grib2.product",
|
|
525
|
+
f"declares {admission.product_length} bytes, too few to state which field it holds",
|
|
526
|
+
)
|
|
527
|
+
|
|
528
|
+
template = int.from_bytes(
|
|
529
|
+
content[admission.product_start + 7 : admission.product_start + 9], "big"
|
|
530
|
+
)
|
|
531
|
+
if template not in ADMITTED_PRODUCT_TEMPLATES:
|
|
532
|
+
_refuse(
|
|
533
|
+
"READER_ADMISSION",
|
|
534
|
+
"reader.weather.grib2.product_template",
|
|
535
|
+
f"describes its field with product template 4.{template}, and this Reader reads "
|
|
536
|
+
f"4.{', 4.'.join(str(number) for number in sorted(ADMITTED_PRODUCT_TEMPLATES))}, an "
|
|
537
|
+
f"instantaneous value at one level. A product averaged or accumulated over a period "
|
|
538
|
+
f"is not valid at a time, so a table with one valid-time column would state something "
|
|
539
|
+
f"the message does not; admitting one means a reviewed edit plus a column saying what "
|
|
540
|
+
f"period each value covers",
|
|
541
|
+
)
|
|
542
|
+
|
|
543
|
+
surface_type = content[admission.product_start + 22]
|
|
544
|
+
if content[admission.product_start + 28] != _MISSING_SURFACE:
|
|
545
|
+
_refuse(
|
|
546
|
+
"READER_ADMISSION",
|
|
547
|
+
"reader.weather.grib2.product",
|
|
548
|
+
f"states a second fixed surface of type {content[admission.product_start + 28]}, so "
|
|
549
|
+
f"its values describe a layer between two surfaces rather than one surface; this "
|
|
550
|
+
f"Reader reads single-surface fields, and reading a layer as a surface would attach a "
|
|
551
|
+
f"value to a level it does not describe",
|
|
552
|
+
)
|
|
553
|
+
|
|
554
|
+
run_time = _reference_time(admission, content)
|
|
555
|
+
return _Product(
|
|
556
|
+
template=template,
|
|
557
|
+
discipline=admission.discipline,
|
|
558
|
+
parameter_category=content[admission.product_start + 9],
|
|
559
|
+
parameter_number=content[admission.product_start + 10],
|
|
560
|
+
surface_type=surface_type,
|
|
561
|
+
surface_value=_surface_value(admission, content),
|
|
562
|
+
run_time=run_time,
|
|
563
|
+
valid_time=_valid_time(admission, content, reference=run_time),
|
|
564
|
+
)
|
|
565
|
+
|
|
566
|
+
|
|
567
|
+
def _surface_value(admission: MessageAdmission, content: bytes) -> Fraction:
|
|
568
|
+
"""The level the values sit at, as the exact rational the message states."""
|
|
569
|
+
|
|
570
|
+
scale_octet = content[admission.product_start + 23]
|
|
571
|
+
scaled = _sign_magnitude(content[admission.product_start + 24 : admission.product_start + 28])
|
|
572
|
+
if scale_octet == _MISSING_OCTET:
|
|
573
|
+
return Fraction(0)
|
|
574
|
+
scale = _sign_magnitude(bytes([scale_octet]))
|
|
575
|
+
if abs(scale) > _MAX_SURFACE_SCALE_MAGNITUDE:
|
|
576
|
+
_refuse(
|
|
577
|
+
"READER_ADMISSION",
|
|
578
|
+
"reader.weather.grib2.surface",
|
|
579
|
+
f"scales its level by ten to the {scale}, which is not a level anything publishes",
|
|
580
|
+
)
|
|
581
|
+
return Fraction(scaled) / Fraction(10) ** scale
|
|
582
|
+
|
|
583
|
+
|
|
584
|
+
def _reference_time(admission: MessageAdmission, content: bytes) -> datetime.datetime:
|
|
585
|
+
"""Return the admitted UTC model-run coordinate declared by the message."""
|
|
586
|
+
|
|
587
|
+
start = admission.identification_start
|
|
588
|
+
significance = content[start + 11]
|
|
589
|
+
if significance not in ADMITTED_TIME_SIGNIFICANCE:
|
|
590
|
+
_refuse(
|
|
591
|
+
"READER_ADMISSION",
|
|
592
|
+
"reader.weather.grib2.reference_time",
|
|
593
|
+
f"says its reference time means significance {significance} rather than an analysis "
|
|
594
|
+
f"or the start of a forecast; under those two the valid time is the reference time "
|
|
595
|
+
f"plus the forecast offset, and under the others adding the offset would move every "
|
|
596
|
+
f"stamp in the table",
|
|
597
|
+
)
|
|
598
|
+
year = int.from_bytes(content[start + 12 : start + 14], "big")
|
|
599
|
+
try:
|
|
600
|
+
return datetime.datetime(
|
|
601
|
+
year,
|
|
602
|
+
content[start + 14],
|
|
603
|
+
content[start + 15],
|
|
604
|
+
content[start + 16],
|
|
605
|
+
content[start + 17],
|
|
606
|
+
min(content[start + 18], 59),
|
|
607
|
+
tzinfo=datetime.UTC,
|
|
608
|
+
)
|
|
609
|
+
except ValueError as error:
|
|
610
|
+
_refuse(
|
|
611
|
+
"READER_ADMISSION",
|
|
612
|
+
"reader.weather.grib2.reference_time",
|
|
613
|
+
f"states a reference time that is not a date: {error}",
|
|
614
|
+
)
|
|
615
|
+
|
|
616
|
+
|
|
617
|
+
def _valid_time(
|
|
618
|
+
admission: MessageAdmission,
|
|
619
|
+
content: bytes,
|
|
620
|
+
*,
|
|
621
|
+
reference: datetime.datetime | None = None,
|
|
622
|
+
) -> datetime.datetime:
|
|
623
|
+
"""The reference time plus the forecast offset, in UTC, or a refusal naming what stopped it."""
|
|
624
|
+
|
|
625
|
+
if reference is None:
|
|
626
|
+
reference = _reference_time(admission, content)
|
|
627
|
+
|
|
628
|
+
unit = content[admission.product_start + 17]
|
|
629
|
+
if unit not in ADMITTED_TIME_UNIT_SECONDS:
|
|
630
|
+
_refuse(
|
|
631
|
+
"READER_ADMISSION",
|
|
632
|
+
"reader.weather.grib2.forecast_time",
|
|
633
|
+
f"measures its forecast offset in unit {unit} of code table 4.4, which is not a fixed "
|
|
634
|
+
f"number of seconds; a month and a year depend on which month and which year, so a "
|
|
635
|
+
f"valid time computed from one would be an approximation presented as a fact",
|
|
636
|
+
)
|
|
637
|
+
offset = _sign_magnitude(content[admission.product_start + 18 : admission.product_start + 22])
|
|
638
|
+
if offset < 0:
|
|
639
|
+
_refuse(
|
|
640
|
+
"READER_ADMISSION",
|
|
641
|
+
"reader.weather.grib2.forecast_time",
|
|
642
|
+
f"states a forecast offset of {offset}, which runs backwards from its own reference "
|
|
643
|
+
f"time",
|
|
644
|
+
)
|
|
645
|
+
seconds = offset * ADMITTED_TIME_UNIT_SECONDS[unit]
|
|
646
|
+
try:
|
|
647
|
+
return reference + datetime.timedelta(seconds=seconds)
|
|
648
|
+
except OverflowError:
|
|
649
|
+
_refuse(
|
|
650
|
+
"READER_ADMISSION",
|
|
651
|
+
"reader.weather.grib2.forecast_time",
|
|
652
|
+
f"states a forecast offset of {seconds} seconds, which is not a time",
|
|
653
|
+
)
|
|
654
|
+
|
|
655
|
+
|
|
656
|
+
def _require_the_variable_the_recipe_asked_for(product: _Product, wanted: Variable) -> None:
|
|
657
|
+
"""Hold the message to the field the recipe named, refusing a mismatch on any coordinate.
|
|
658
|
+
|
|
659
|
+
Pointing a recipe at the wrong file is the ordinary mistake here, and it is the one that
|
|
660
|
+
verifies clean: categorical rain and a temperature are both real numbers on the same grid at
|
|
661
|
+
the same time, and a table labelled with the recipe's variable name would carry the other
|
|
662
|
+
field's values under it without a mark.
|
|
663
|
+
"""
|
|
664
|
+
|
|
665
|
+
stated = (
|
|
666
|
+
wanted.discipline,
|
|
667
|
+
wanted.parameter_category,
|
|
668
|
+
wanted.parameter_number,
|
|
669
|
+
wanted.surface_type,
|
|
670
|
+
wanted.surface_value,
|
|
671
|
+
)
|
|
672
|
+
declared = (
|
|
673
|
+
product.discipline,
|
|
674
|
+
product.parameter_category,
|
|
675
|
+
product.parameter_number,
|
|
676
|
+
product.surface_type,
|
|
677
|
+
product.surface_value,
|
|
678
|
+
)
|
|
679
|
+
if stated == declared:
|
|
680
|
+
return
|
|
681
|
+
_refuse(
|
|
682
|
+
"READER_ADMISSION",
|
|
683
|
+
"reader.weather.grib2.variable",
|
|
684
|
+
f"holds discipline {product.discipline}, parameter {product.parameter_category}."
|
|
685
|
+
f"{product.parameter_number} on surface type {product.surface_type} at "
|
|
686
|
+
f"{product.surface_value}, and the recipe asks for {wanted.name}: discipline "
|
|
687
|
+
f"{wanted.discipline}, parameter {wanted.parameter_category}.{wanted.parameter_number} on "
|
|
688
|
+
f"surface type {wanted.surface_type} at {wanted.surface_value}. A message carries one "
|
|
689
|
+
f"field, so this file is not the one this recipe describes; either the source points at "
|
|
690
|
+
f"the wrong record or the recipe names the wrong field",
|
|
691
|
+
)
|
|
692
|
+
|
|
693
|
+
|
|
694
|
+
# --- The decode -------------------------------------------------------------------------------
|
|
695
|
+
|
|
696
|
+
|
|
697
|
+
def _unpack(admission: MessageAdmission, content: bytes) -> Any:
|
|
698
|
+
"""The one call into the vendored binding, which cannot be made without an admission result.
|
|
699
|
+
|
|
700
|
+
The template numbers handed over are the ones admission established, and the binding refuses
|
|
701
|
+
to decode a message declaring anything else -- the second layer, for a call site that skipped
|
|
702
|
+
the first. The parameters it read out of Section 5 are then held against the ones admission
|
|
703
|
+
read out of the same octets, because two readings of one field that disagree mean one of them
|
|
704
|
+
is reading the wrong bytes and the values would be scaled by whichever won.
|
|
705
|
+
|
|
706
|
+
Translate ``GribBindingError`` to the typed Reader error used for corrupt or truncated packed
|
|
707
|
+
values. Raw runtime errors must not escape the Reader boundary.
|
|
708
|
+
"""
|
|
709
|
+
|
|
710
|
+
try:
|
|
711
|
+
unpacked = mostlyright_grib.unpack(
|
|
712
|
+
content,
|
|
713
|
+
grid_template=admission.grid_template,
|
|
714
|
+
packing_template=admission.packing_template,
|
|
715
|
+
)
|
|
716
|
+
except mostlyright_grib.GribBindingError as error:
|
|
717
|
+
_refuse(
|
|
718
|
+
"READER_DECODE",
|
|
719
|
+
"reader.weather.grib2.message",
|
|
720
|
+
f"was admitted and then did not decode: {error.text} (decoder code {error.code}). "
|
|
721
|
+
f"The grid and packing this message declares are on the approved list, so the "
|
|
722
|
+
f"refusal is about the packed values themselves rather than about the format: the "
|
|
723
|
+
f"file is truncated or damaged. Fetch it again and compare the hash the receipt "
|
|
724
|
+
f"recorded; nothing was published and the version you already have is untouched",
|
|
725
|
+
)
|
|
726
|
+
declared = (admission.reference_value, admission.binary_scale, admission.decimal_scale)
|
|
727
|
+
read = (unpacked.reference_value, unpacked.binary_scale, unpacked.decimal_scale)
|
|
728
|
+
if declared != read:
|
|
729
|
+
_refuse(
|
|
730
|
+
"READER_ADMISSION",
|
|
731
|
+
"reader.weather.grib2.packing",
|
|
732
|
+
f"is read as {declared} by admission and as {read} by the decoder; two readings of "
|
|
733
|
+
f"one field that disagree mean one of them is reading the wrong bytes, and the values "
|
|
734
|
+
f"would be scaled by whichever answered",
|
|
735
|
+
)
|
|
736
|
+
if len(unpacked.levels) != admission.declared_points:
|
|
737
|
+
_refuse(
|
|
738
|
+
"READER_ADMISSION",
|
|
739
|
+
"reader.weather.grib2.packing",
|
|
740
|
+
f"unpacks {len(unpacked.levels)} values for a grid of {admission.declared_points} "
|
|
741
|
+
f"points",
|
|
742
|
+
)
|
|
743
|
+
return unpacked
|
|
744
|
+
|
|
745
|
+
|
|
746
|
+
def decode_admitted(
|
|
747
|
+
admission: MessageAdmission,
|
|
748
|
+
content: bytes,
|
|
749
|
+
settings: DecodeSettings,
|
|
750
|
+
budgets: ReaderBudgets,
|
|
751
|
+
) -> ReaderResult:
|
|
752
|
+
"""Turn one admitted message into the rows the recipe asked for.
|
|
753
|
+
|
|
754
|
+
Takes an admission result, which is the only way this Reader decodes anything: a message
|
|
755
|
+
nobody admitted has nothing to pass here. That is the same rule ``geometry.build_grid``
|
|
756
|
+
applies, for the same reason, and it is asserted structurally rather than described.
|
|
757
|
+
"""
|
|
758
|
+
|
|
759
|
+
if not isinstance(admission, MessageAdmission):
|
|
760
|
+
_refuse(
|
|
761
|
+
"READER_CONTRACT",
|
|
762
|
+
"reader.weather.grib2.admission",
|
|
763
|
+
"a message is decoded from an admission result, so an unadmitted message has nothing "
|
|
764
|
+
"to pass; this is the rule that makes decoding an unadmitted message inexpressible "
|
|
765
|
+
"rather than merely undone",
|
|
766
|
+
)
|
|
767
|
+
if not isinstance(settings, DecodeSettings):
|
|
768
|
+
_refuse(
|
|
769
|
+
"READER_CONTRACT",
|
|
770
|
+
"reader.weather.grib2.settings",
|
|
771
|
+
"a decode runs on settings that have been admitted, not on a mapping",
|
|
772
|
+
)
|
|
773
|
+
if settings.allowlist_version != ADMISSION_ALLOWLIST_VERSION:
|
|
774
|
+
_refuse(
|
|
775
|
+
"READER_ADMISSION",
|
|
776
|
+
"reader.weather.grib2.admission_allowlist_version",
|
|
777
|
+
f"was written against allowlist version {settings.allowlist_version} and this build "
|
|
778
|
+
f"admits version {ADMISSION_ALLOWLIST_VERSION}. The list of grid and packing "
|
|
779
|
+
f"combinations this product opens has changed since the recipe was frozen, so the "
|
|
780
|
+
f"same recipe would now mean something different -- it would open files it used to "
|
|
781
|
+
f"refuse, or refuse files it used to open. Nothing is decoded until a person decides "
|
|
782
|
+
f"which meaning is wanted: re-approve the recipe against the current list, or build "
|
|
783
|
+
f"with the release that carried version {settings.allowlist_version}. The current "
|
|
784
|
+
f"version is {ALLOWLIST_VERSION_CONSTANT}",
|
|
785
|
+
)
|
|
786
|
+
|
|
787
|
+
check_declared_size(
|
|
788
|
+
row_count=len(settings.points),
|
|
789
|
+
column_count=len(COLUMN_NAMES),
|
|
790
|
+
budgets=budgets,
|
|
791
|
+
subject="reader.weather.grib2.table",
|
|
792
|
+
)
|
|
793
|
+
|
|
794
|
+
product = _product(admission, content)
|
|
795
|
+
_require_the_variable_the_recipe_asked_for(product, settings.variable)
|
|
796
|
+
grid = build_grid(admission, content)
|
|
797
|
+
unpacked = _unpack(admission, content)
|
|
798
|
+
|
|
799
|
+
stamp = product.valid_time.strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
800
|
+
rows: list[tuple[Any, ...]] = []
|
|
801
|
+
masked = False
|
|
802
|
+
for point in settings.points:
|
|
803
|
+
row, column = nearest_grid_index(grid, point.latitude, point.longitude, name=point.identity)
|
|
804
|
+
latitude, longitude = grid.coordinate_at(row, column)
|
|
805
|
+
level = unpacked.levels[grid.data_index(row, column)]
|
|
806
|
+
if math.isfinite(level):
|
|
807
|
+
value: float | None = physical_value(
|
|
808
|
+
level,
|
|
809
|
+
reference_value=admission.reference_value,
|
|
810
|
+
binary_scale=admission.binary_scale,
|
|
811
|
+
decimal_scale=admission.decimal_scale,
|
|
812
|
+
)
|
|
813
|
+
else:
|
|
814
|
+
value = None
|
|
815
|
+
masked = True
|
|
816
|
+
rows.append(
|
|
817
|
+
(
|
|
818
|
+
point.identity,
|
|
819
|
+
point.latitude,
|
|
820
|
+
point.longitude,
|
|
821
|
+
row,
|
|
822
|
+
column,
|
|
823
|
+
latitude,
|
|
824
|
+
longitude,
|
|
825
|
+
stamp,
|
|
826
|
+
settings.variable.name,
|
|
827
|
+
value,
|
|
828
|
+
)
|
|
829
|
+
)
|
|
830
|
+
|
|
831
|
+
return ReaderResult(
|
|
832
|
+
content=encode_canonical_csv(COLUMN_NAMES, rows, budgets=budgets),
|
|
833
|
+
data_format=_OUTPUT_FORMAT,
|
|
834
|
+
media_type=_OUTPUT_MEDIA_TYPE,
|
|
835
|
+
# The recipe's own name for the field, which is the one piece of text on this path a
|
|
836
|
+
# person wrote down deliberately. A weather message carries no filename of its own.
|
|
837
|
+
filename=sealed_filename(settings.variable.name, suffix=_OUTPUT_SUFFIX),
|
|
838
|
+
row_count=len(rows),
|
|
839
|
+
column_names=COLUMN_NAMES,
|
|
840
|
+
declared_cell_count=max(admission.declared_points, len(rows) * len(COLUMN_NAMES)),
|
|
841
|
+
flags=(MASKED_POINT_FLAG,) if masked else (),
|
|
842
|
+
)
|
|
843
|
+
|
|
844
|
+
|
|
845
|
+
def _level_coordinate(product: _Product) -> str:
|
|
846
|
+
"""Spell the GRIB fixed-surface coordinate without a lossy unit-name lookup."""
|
|
847
|
+
|
|
848
|
+
value = product.surface_value
|
|
849
|
+
amount = str(value.numerator)
|
|
850
|
+
if value.denominator != 1:
|
|
851
|
+
amount = f"{amount}/{value.denominator}"
|
|
852
|
+
return f"{product.surface_type}:{amount}"
|
|
853
|
+
|
|
854
|
+
|
|
855
|
+
def decode_admitted_v2(
|
|
856
|
+
admission: MessageAdmission,
|
|
857
|
+
content: bytes,
|
|
858
|
+
settings: DecodeSettings,
|
|
859
|
+
budgets: ReaderBudgets,
|
|
860
|
+
) -> ReaderResult:
|
|
861
|
+
"""Decode one admitted message into the evidence-bearing 2.0 row contract."""
|
|
862
|
+
|
|
863
|
+
if not isinstance(admission, MessageAdmission):
|
|
864
|
+
_refuse(
|
|
865
|
+
"READER_CONTRACT",
|
|
866
|
+
"reader.weather.grib2.admission",
|
|
867
|
+
"a message is decoded from an admission result",
|
|
868
|
+
)
|
|
869
|
+
if not isinstance(settings, DecodeSettings):
|
|
870
|
+
_refuse(
|
|
871
|
+
"READER_CONTRACT",
|
|
872
|
+
"reader.weather.grib2.settings",
|
|
873
|
+
"a decode runs on admitted settings",
|
|
874
|
+
)
|
|
875
|
+
if settings.allowlist_version != ADMISSION_ALLOWLIST_VERSION:
|
|
876
|
+
_refuse(
|
|
877
|
+
"READER_ADMISSION",
|
|
878
|
+
"reader.weather.grib2.admission_allowlist_version",
|
|
879
|
+
f"was written against allowlist version {settings.allowlist_version}, not "
|
|
880
|
+
f"{ADMISSION_ALLOWLIST_VERSION}",
|
|
881
|
+
)
|
|
882
|
+
|
|
883
|
+
check_declared_size(
|
|
884
|
+
row_count=len(settings.points),
|
|
885
|
+
column_count=len(V2_COLUMN_NAMES),
|
|
886
|
+
budgets=budgets,
|
|
887
|
+
subject="reader.weather.grib2.v2.table",
|
|
888
|
+
)
|
|
889
|
+
product = _product(admission, content)
|
|
890
|
+
_require_the_variable_the_recipe_asked_for(product, settings.variable)
|
|
891
|
+
grid = build_grid(admission, content)
|
|
892
|
+
unpacked = _unpack(admission, content)
|
|
893
|
+
message_digest = sha256_bytes(content)
|
|
894
|
+
run_time = product.run_time.strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
895
|
+
valid_time = product.valid_time.strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
896
|
+
level_coordinate = _level_coordinate(product)
|
|
897
|
+
rows: list[tuple[Any, ...]] = []
|
|
898
|
+
masked = False
|
|
899
|
+
for point in settings.points:
|
|
900
|
+
row, column = nearest_grid_index(grid, point.latitude, point.longitude, name=point.identity)
|
|
901
|
+
latitude, longitude = grid.coordinate_at(row, column)
|
|
902
|
+
level = unpacked.levels[grid.data_index(row, column)]
|
|
903
|
+
if math.isfinite(level):
|
|
904
|
+
value: float | None = physical_value(
|
|
905
|
+
level,
|
|
906
|
+
reference_value=admission.reference_value,
|
|
907
|
+
binary_scale=admission.binary_scale,
|
|
908
|
+
decimal_scale=admission.decimal_scale,
|
|
909
|
+
)
|
|
910
|
+
else:
|
|
911
|
+
value = None
|
|
912
|
+
masked = True
|
|
913
|
+
rows.append(
|
|
914
|
+
(
|
|
915
|
+
point.identity,
|
|
916
|
+
settings.variable.name,
|
|
917
|
+
level_coordinate,
|
|
918
|
+
run_time,
|
|
919
|
+
valid_time,
|
|
920
|
+
value,
|
|
921
|
+
latitude,
|
|
922
|
+
longitude,
|
|
923
|
+
message_digest,
|
|
924
|
+
)
|
|
925
|
+
)
|
|
926
|
+
return ReaderResult(
|
|
927
|
+
content=encode_canonical_csv(V2_COLUMN_NAMES, rows, budgets=budgets),
|
|
928
|
+
data_format=_OUTPUT_FORMAT,
|
|
929
|
+
media_type=_OUTPUT_MEDIA_TYPE,
|
|
930
|
+
filename=sealed_filename(settings.variable.name, suffix=_OUTPUT_SUFFIX),
|
|
931
|
+
row_count=len(rows),
|
|
932
|
+
column_names=V2_COLUMN_NAMES,
|
|
933
|
+
declared_cell_count=max(admission.declared_points, len(rows) * len(V2_COLUMN_NAMES)),
|
|
934
|
+
flags=(MASKED_POINT_FLAG,) if masked else (),
|
|
935
|
+
)
|
|
936
|
+
|
|
937
|
+
|
|
938
|
+
@dataclass(frozen=True)
|
|
939
|
+
class WeatherGrib2Reader:
|
|
940
|
+
"""``weather.grib2``: a weather-model message read as the places a recipe named.
|
|
941
|
+
|
|
942
|
+
``accepted_media_types`` admits what the mirrors actually serve. That is an input-side
|
|
943
|
+
declaration and it is not a format table: what may be *sealed* is still canonical CSV alone.
|
|
944
|
+
"""
|
|
945
|
+
|
|
946
|
+
family_id: str = "weather.grib2"
|
|
947
|
+
family_version: str = "1.0.0"
|
|
948
|
+
contract_version: str = READER_CONTRACT_VERSION
|
|
949
|
+
output_format: str = _OUTPUT_FORMAT
|
|
950
|
+
accepted_media_types: tuple[str, ...] = (
|
|
951
|
+
"application/octet-stream",
|
|
952
|
+
"application/wmo-grib2",
|
|
953
|
+
)
|
|
954
|
+
default_budgets: ReaderBudgets = field(default_factory=ReaderBudgets)
|
|
955
|
+
|
|
956
|
+
def validate_options(self, options: Mapping[str, Any]) -> Mapping[str, Any]:
|
|
957
|
+
return validate_options(options)
|
|
958
|
+
|
|
959
|
+
def decode(self, content: bytes, pin: ReaderPin, budgets: ReaderBudgets) -> ReaderResult:
|
|
960
|
+
"""The Reader contract's entry point: bytes in, sealed bytes out, admission first.
|
|
961
|
+
|
|
962
|
+
This is the one function here that receives message bytes without an admission result,
|
|
963
|
+
because the contract hands it bytes and this is the thing that admits them. It admits
|
|
964
|
+
before it does anything else, and the test module asserts that from the syntax tree.
|
|
965
|
+
"""
|
|
966
|
+
|
|
967
|
+
settings = settings_from_options(pin.decode_options)
|
|
968
|
+
admission = admit(content, budgets)
|
|
969
|
+
return decode_admitted(admission, content, settings, budgets)
|
|
970
|
+
|
|
971
|
+
|
|
972
|
+
@dataclass(frozen=True)
|
|
973
|
+
class WeatherGrib2ReaderV2:
|
|
974
|
+
"""Evidence-bearing weather rows; co-installed beside byte-stable 1.0.0."""
|
|
975
|
+
|
|
976
|
+
family_id: str = "weather.grib2"
|
|
977
|
+
family_version: str = "2.0.0"
|
|
978
|
+
contract_version: str = READER_CONTRACT_VERSION
|
|
979
|
+
output_format: str = _OUTPUT_FORMAT
|
|
980
|
+
accepted_media_types: tuple[str, ...] = (
|
|
981
|
+
"application/octet-stream",
|
|
982
|
+
"application/wmo-grib2",
|
|
983
|
+
)
|
|
984
|
+
default_budgets: ReaderBudgets = field(default_factory=ReaderBudgets)
|
|
985
|
+
|
|
986
|
+
def validate_options(self, options: Mapping[str, Any]) -> Mapping[str, Any]:
|
|
987
|
+
return validate_options(options)
|
|
988
|
+
|
|
989
|
+
def decode(self, content: bytes, pin: ReaderPin, budgets: ReaderBudgets) -> ReaderResult:
|
|
990
|
+
settings = settings_from_options(pin.decode_options)
|
|
991
|
+
admission = admit(content, budgets)
|
|
992
|
+
return decode_admitted_v2(admission, content, settings, budgets)
|
|
993
|
+
|
|
994
|
+
|
|
995
|
+
def _sign_magnitude(raw: bytes) -> int:
|
|
996
|
+
"""A GRIB2 signed integer: a sign bit and a magnitude, never two's complement.
|
|
997
|
+
|
|
998
|
+
The same rule admission and geometry each read their signed fields with, written out here
|
|
999
|
+
rather than imported across a module boundary for the reason recorded in ``geometry``, and
|
|
1000
|
+
held to agree with both by test.
|
|
1001
|
+
"""
|
|
1002
|
+
|
|
1003
|
+
value = int.from_bytes(raw, "big")
|
|
1004
|
+
sign = 1 << (8 * len(raw) - 1)
|
|
1005
|
+
return -(value & ~sign) if value & sign else value
|
|
1006
|
+
|
|
1007
|
+
|
|
1008
|
+
def _refuse(code: str, subject: str, detail: str) -> NoReturn:
|
|
1009
|
+
raise ReaderError(code, subject, detail)
|