mostlyright-data 0.9.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (314) hide show
  1. mostlyright/data_harness/__init__.py +158 -0
  2. mostlyright/data_harness/acquisition/__init__.py +55 -0
  3. mostlyright/data_harness/acquisition/http.py +2773 -0
  4. mostlyright/data_harness/acquisition/parsing.py +809 -0
  5. mostlyright/data_harness/acquisition/ranges.py +495 -0
  6. mostlyright/data_harness/acquisition/result_download.py +360 -0
  7. mostlyright/data_harness/acquisition/retention_admission.py +248 -0
  8. mostlyright/data_harness/acquisition/sandbox.py +4888 -0
  9. mostlyright/data_harness/acquisition/url_policy.py +530 -0
  10. mostlyright/data_harness/agent_runtime.py +2743 -0
  11. mostlyright/data_harness/assets/logo-ink.svg +31 -0
  12. mostlyright/data_harness/backends/__init__.py +28 -0
  13. mostlyright/data_harness/backends/pandas_backend.py +350 -0
  14. mostlyright/data_harness/backends/polars_backend.py +366 -0
  15. mostlyright/data_harness/backends/protocol.py +124 -0
  16. mostlyright/data_harness/backends/reference.py +83 -0
  17. mostlyright/data_harness/backends/registry.py +55 -0
  18. mostlyright/data_harness/backends/restrictions.py +126 -0
  19. mostlyright/data_harness/canonical.py +333 -0
  20. mostlyright/data_harness/catalog_job.py +625 -0
  21. mostlyright/data_harness/cli.py +5398 -0
  22. mostlyright/data_harness/contracts.py +53 -0
  23. mostlyright/data_harness/coordinator.py +1307 -0
  24. mostlyright/data_harness/deploy.py +924 -0
  25. mostlyright/data_harness/deploy_target.py +312 -0
  26. mostlyright/data_harness/deployment_evidence.py +1067 -0
  27. mostlyright/data_harness/event_presentation.py +576 -0
  28. mostlyright/data_harness/events.py +2152 -0
  29. mostlyright/data_harness/fast_delimited.py +239 -0
  30. mostlyright/data_harness/fleet.py +237 -0
  31. mostlyright/data_harness/formats.py +236 -0
  32. mostlyright/data_harness/governors.py +1163 -0
  33. mostlyright/data_harness/hosted_bootstrap.py +972 -0
  34. mostlyright/data_harness/hosted_crawler.py +1115 -0
  35. mostlyright/data_harness/hosted_crawler_container_smoke.py +351 -0
  36. mostlyright/data_harness/hosted_crawler_fetch.py +423 -0
  37. mostlyright/data_harness/hosted_crawler_job.py +1277 -0
  38. mostlyright/data_harness/hosted_crawler_protocol.py +676 -0
  39. mostlyright/data_harness/hosted_dataset.py +1500 -0
  40. mostlyright/data_harness/hosted_deploy.py +3037 -0
  41. mostlyright/data_harness/hosted_handoff.py +62 -0
  42. mostlyright/data_harness/hosted_ingestion_contract.py +504 -0
  43. mostlyright/data_harness/hosted_ingestion_job.py +356 -0
  44. mostlyright/data_harness/hosted_ingestion_job_smoke.py +40 -0
  45. mostlyright/data_harness/hosted_session_container_smoke.py +194 -0
  46. mostlyright/data_harness/hosted_session_worker.py +3554 -0
  47. mostlyright/data_harness/hosted_session_worker_job_smoke.py +46 -0
  48. mostlyright/data_harness/hosted_worker.py +6784 -0
  49. mostlyright/data_harness/ingestion/__init__.py +56 -0
  50. mostlyright/data_harness/ingestion/contracts.py +461 -0
  51. mostlyright/data_harness/ingestion/faults.py +42 -0
  52. mostlyright/data_harness/ingestion/gcs_store.py +1162 -0
  53. mostlyright/data_harness/ingestion/spool.py +130 -0
  54. mostlyright/data_harness/ingestion/store.py +885 -0
  55. mostlyright/data_harness/key_seam.py +434 -0
  56. mostlyright/data_harness/linux_process_boundary.py +262 -0
  57. mostlyright/data_harness/local_contracts.py +2880 -0
  58. mostlyright/data_harness/local_search/__init__.py +5 -0
  59. mostlyright/data_harness/local_search/build_index.py +1087 -0
  60. mostlyright/data_harness/local_search/contracts.py +920 -0
  61. mostlyright/data_harness/local_search/query_trace.py +266 -0
  62. mostlyright/data_harness/local_search/retrieval.py +700 -0
  63. mostlyright/data_harness/local_search/sealed.py +474 -0
  64. mostlyright/data_harness/local_search/service.py +784 -0
  65. mostlyright/data_harness/nbrender/CONTRACT.md +212 -0
  66. mostlyright/data_harness/nbrender/__init__.py +12 -0
  67. mostlyright/data_harness/nbrender/chrome.py +359 -0
  68. mostlyright/data_harness/nbrender/code_body.py +266 -0
  69. mostlyright/data_harness/nbrender/document.py +407 -0
  70. mostlyright/data_harness/nbrender/frame.py +275 -0
  71. mostlyright/data_harness/nbrender/interactive.py +337 -0
  72. mostlyright/data_harness/nbrender/markdown_body.py +477 -0
  73. mostlyright/data_harness/nbrender/mr_components.py +134 -0
  74. mostlyright/data_harness/nbrender/outputs_data.py +595 -0
  75. mostlyright/data_harness/nbrender/outputs_rich.py +906 -0
  76. mostlyright/data_harness/nbrender/outputs_source.py +260 -0
  77. mostlyright/data_harness/nbrender/outputs_stage.py +176 -0
  78. mostlyright/data_harness/nbrender/outputs_text.py +400 -0
  79. mostlyright/data_harness/nbrender/parse.py +394 -0
  80. mostlyright/data_harness/nbrender/status.py +40 -0
  81. mostlyright/data_harness/nbrender/tokens.py +1295 -0
  82. mostlyright/data_harness/notebook.py +1710 -0
  83. mostlyright/data_harness/offline.py +2049 -0
  84. mostlyright/data_harness/operation_registry.py +1007 -0
  85. mostlyright/data_harness/operator_setup.py +239 -0
  86. mostlyright/data_harness/pipeline.py +6428 -0
  87. mostlyright/data_harness/plan_graph.py +2026 -0
  88. mostlyright/data_harness/preparation/__init__.py +104 -0
  89. mostlyright/data_harness/preparation/contracts.py +1017 -0
  90. mostlyright/data_harness/preparation/engine.py +221 -0
  91. mostlyright/data_harness/preparation/errors.py +14 -0
  92. mostlyright/data_harness/preparation/gates.py +751 -0
  93. mostlyright/data_harness/preparation/joins.py +574 -0
  94. mostlyright/data_harness/preparation/profile.py +384 -0
  95. mostlyright/data_harness/preparation/table.py +217 -0
  96. mostlyright/data_harness/preparation/transforms.py +568 -0
  97. mostlyright/data_harness/progress_events.py +534 -0
  98. mostlyright/data_harness/readers/__init__.py +46 -0
  99. mostlyright/data_harness/readers/containers.py +963 -0
  100. mostlyright/data_harness/readers/contracts.py +542 -0
  101. mostlyright/data_harness/readers/delimited.py +257 -0
  102. mostlyright/data_harness/readers/grib2/__init__.py +33 -0
  103. mostlyright/data_harness/readers/grib2/admission.py +722 -0
  104. mostlyright/data_harness/readers/grib2/decode.py +1009 -0
  105. mostlyright/data_harness/readers/grib2/geometry.py +1133 -0
  106. mostlyright/data_harness/readers/grib2/portable_math.py +501 -0
  107. mostlyright/data_harness/readers/json_tabular.py +485 -0
  108. mostlyright/data_harness/readers/registry.py +514 -0
  109. mostlyright/data_harness/readers/samples/README.md +110 -0
  110. mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/cities.csv.gz +0 -0
  111. mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/expected.json +24 -0
  112. mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/cities.csv.gz +0 -0
  113. mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/expected.json +24 -0
  114. mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/cities.tar +0 -0
  115. mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/expected.json +24 -0
  116. mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/cities.tar +0 -0
  117. mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/expected.json +24 -0
  118. mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/cities.zip +0 -0
  119. mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/expected.json +25 -0
  120. mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
  121. mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/expected.json +25 -0
  122. mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
  123. mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/expected.json +25 -0
  124. mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/cities.csv +3 -0
  125. mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/expected.json +23 -0
  126. mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/cities.tsv +5 -0
  127. mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/expected.json +25 -0
  128. mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/expected.json +30 -0
  129. mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/observations.csv +5 -0
  130. mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/expected.json +44 -0
  131. mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/stations.json +1 -0
  132. mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/expected.json +48 -0
  133. mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/observations.ndjson +4 -0
  134. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/cities.xlsx +0 -0
  135. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/expected.json +24 -0
  136. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/expected.json +27 -0
  137. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
  138. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/expected.json +27 -0
  139. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
  140. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/README.md +20 -0
  141. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/expected.json +55 -0
  142. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/gfs-2m-temperature.grib2 +0 -0
  143. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/expected.json +54 -0
  144. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
  145. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/expected.json +54 -0
  146. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/hrrr-categorical-rain.grib2 +0 -0
  147. mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/expected.json +54 -0
  148. mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
  149. mostlyright/data_harness/readers/samples.py +582 -0
  150. mostlyright/data_harness/readers/spreadsheet.py +803 -0
  151. mostlyright/data_harness/readers/tabular.py +510 -0
  152. mostlyright/data_harness/recipe.py +5321 -0
  153. mostlyright/data_harness/repair/__init__.py +78 -0
  154. mostlyright/data_harness/repair/adapters.py +274 -0
  155. mostlyright/data_harness/repair/contracts.py +872 -0
  156. mostlyright/data_harness/repair/coordinator.py +1099 -0
  157. mostlyright/data_harness/repair/errors.py +16 -0
  158. mostlyright/data_harness/review.py +2533 -0
  159. mostlyright/data_harness/rowset.py +283 -0
  160. mostlyright/data_harness/serving.py +1975 -0
  161. mostlyright/data_harness/serving_edge.py +590 -0
  162. mostlyright/data_harness/serving_http.py +1031 -0
  163. mostlyright/data_harness/session_probes.py +759 -0
  164. mostlyright/data_harness/signing.py +101 -0
  165. mostlyright/data_harness/source_discovery.py +898 -0
  166. mostlyright/data_harness/sources/__init__.py +209 -0
  167. mostlyright/data_harness/sources/_adapter_steps.py +213 -0
  168. mostlyright/data_harness/sources/adapters.py +1214 -0
  169. mostlyright/data_harness/sources/cadence.py +1428 -0
  170. mostlyright/data_harness/sources/cadence_emission.py +453 -0
  171. mostlyright/data_harness/sources/cadence_history.py +546 -0
  172. mostlyright/data_harness/sources/catalog/__init__.py +17 -0
  173. mostlyright/data_harness/sources/catalog/admission.py +477 -0
  174. mostlyright/data_harness/sources/catalog/authoring.py +1701 -0
  175. mostlyright/data_harness/sources/catalog/authoring_policy.py +701 -0
  176. mostlyright/data_harness/sources/catalog/authoring_shards.py +1217 -0
  177. mostlyright/data_harness/sources/catalog/bounded_io.py +231 -0
  178. mostlyright/data_harness/sources/catalog/channel.py +523 -0
  179. mostlyright/data_harness/sources/catalog/channel_client.py +296 -0
  180. mostlyright/data_harness/sources/catalog/contracts.py +825 -0
  181. mostlyright/data_harness/sources/catalog/coverage.py +137 -0
  182. mostlyright/data_harness/sources/catalog/delta.py +1340 -0
  183. mostlyright/data_harness/sources/catalog/embedding.py +532 -0
  184. mostlyright/data_harness/sources/catalog/entry_v2.py +1182 -0
  185. mostlyright/data_harness/sources/catalog/fill.py +3889 -0
  186. mostlyright/data_harness/sources/catalog/fill_partitions.py +459 -0
  187. mostlyright/data_harness/sources/catalog/fill_staging.py +1105 -0
  188. mostlyright/data_harness/sources/catalog/gating.py +374 -0
  189. mostlyright/data_harness/sources/catalog/generation_receipt.py +1607 -0
  190. mostlyright/data_harness/sources/catalog/harvest/__init__.py +7 -0
  191. mostlyright/data_harness/sources/catalog/harvest/ckan.py +384 -0
  192. mostlyright/data_harness/sources/catalog/harvest/datagov_v4.py +798 -0
  193. mostlyright/data_harness/sources/catalog/harvest/protocol.py +964 -0
  194. mostlyright/data_harness/sources/catalog/harvest/sdmx.py +445 -0
  195. mostlyright/data_harness/sources/catalog/harvest/stac.py +384 -0
  196. mostlyright/data_harness/sources/catalog/health.py +447 -0
  197. mostlyright/data_harness/sources/catalog/hosted_catalog.py +105 -0
  198. mostlyright/data_harness/sources/catalog/identity_history.py +1549 -0
  199. mostlyright/data_harness/sources/catalog/neural.py +1618 -0
  200. mostlyright/data_harness/sources/catalog/packed_catalog.py +2345 -0
  201. mostlyright/data_harness/sources/catalog/packed_retrieval.py +1517 -0
  202. mostlyright/data_harness/sources/catalog/packed_writer.py +2802 -0
  203. mostlyright/data_harness/sources/catalog/query_trace.py +1037 -0
  204. mostlyright/data_harness/sources/catalog/recommend.py +171 -0
  205. mostlyright/data_harness/sources/catalog/retrieval.py +230 -0
  206. mostlyright/data_harness/sources/catalog/retrieval_manifest.py +995 -0
  207. mostlyright/data_harness/sources/catalog/rights_decisions.py +254 -0
  208. mostlyright/data_harness/sources/catalog/sealed.py +560 -0
  209. mostlyright/data_harness/sources/catalog/search.py +230 -0
  210. mostlyright/data_harness/sources/catalog/streaming_delta.py +1097 -0
  211. mostlyright/data_harness/sources/catalog/update.py +891 -0
  212. mostlyright/data_harness/sources/collections.py +815 -0
  213. mostlyright/data_harness/sources/contracts.py +2223 -0
  214. mostlyright/data_harness/sources/deletion.py +761 -0
  215. mostlyright/data_harness/sources/fitness.py +162 -0
  216. mostlyright/data_harness/sources/governance.py +163 -0
  217. mostlyright/data_harness/sources/hosted.py +173 -0
  218. mostlyright/data_harness/sources/integration.py +218 -0
  219. mostlyright/data_harness/sources/range_reader.py +418 -0
  220. mostlyright/data_harness/sources/registry.py +514 -0
  221. mostlyright/data_harness/sources/rights_rule.py +59 -0
  222. mostlyright/data_harness/sources/source_cadence_vectors.v1.json +1 -0
  223. mostlyright/data_harness/sources/sports.py +521 -0
  224. mostlyright/data_harness/sources/stream.py +524 -0
  225. mostlyright/data_harness/sources/stream_connector.py +418 -0
  226. mostlyright/data_harness/sources/stream_recorder.py +1404 -0
  227. mostlyright/data_harness/studio_boundary.py +2019 -0
  228. mostlyright/data_harness/thin/__init__.py +37 -0
  229. mostlyright/data_harness/thin/acquire.py +1137 -0
  230. mostlyright/data_harness/thin/acquire_cancel.py +579 -0
  231. mostlyright/data_harness/thin/approvals.py +617 -0
  232. mostlyright/data_harness/thin/commands.py +406 -0
  233. mostlyright/data_harness/thin/download.py +194 -0
  234. mostlyright/data_harness/thin/narrative.py +589 -0
  235. mostlyright/data_harness/thin/parity.py +1070 -0
  236. mostlyright/data_harness/thin/propose.py +2759 -0
  237. mostlyright/data_harness/thin/research.py +1663 -0
  238. mostlyright/data_harness/thin/router.py +924 -0
  239. mostlyright/data_harness/thin/runs.py +519 -0
  240. mostlyright/data_harness/thin/session.py +281 -0
  241. mostlyright/data_harness/thin/stream.py +501 -0
  242. mostlyright/data_harness/thin/transport.py +187 -0
  243. mostlyright/data_harness/thin/vocabulary.py +368 -0
  244. mostlyright/data_harness/thin/workers.py +164 -0
  245. mostlyright/data_harness/ucum/TABLE-PIN.json +40 -0
  246. mostlyright/data_harness/ucum/ucum-subset.v1.json +632 -0
  247. mostlyright/data_harness/unit_flow.py +927 -0
  248. mostlyright/data_harness/units.py +572 -0
  249. mostlyright/data_harness/ux/__init__.py +9 -0
  250. mostlyright/data_harness/ux/approve.py +485 -0
  251. mostlyright/data_harness/ux/author_yaml.py +597 -0
  252. mostlyright/data_harness/ux/cloud_auth.py +447 -0
  253. mostlyright/data_harness/ux/commands/__init__.py +260 -0
  254. mostlyright/data_harness/ux/commands/approve.py +136 -0
  255. mostlyright/data_harness/ux/commands/auth.py +744 -0
  256. mostlyright/data_harness/ux/commands/author.py +79 -0
  257. mostlyright/data_harness/ux/commands/catalog_author.py +403 -0
  258. mostlyright/data_harness/ux/commands/catalog_fill.py +523 -0
  259. mostlyright/data_harness/ux/commands/catalog_harvest.py +545 -0
  260. mostlyright/data_harness/ux/commands/catalog_publish.py +1838 -0
  261. mostlyright/data_harness/ux/commands/catalog_search.py +71 -0
  262. mostlyright/data_harness/ux/commands/catalog_update.py +437 -0
  263. mostlyright/data_harness/ux/commands/deploy.py +134 -0
  264. mostlyright/data_harness/ux/commands/deploy_dataset.py +98 -0
  265. mostlyright/data_harness/ux/commands/deploy_plan.py +105 -0
  266. mostlyright/data_harness/ux/commands/deploy_status.py +104 -0
  267. mostlyright/data_harness/ux/commands/diff.py +74 -0
  268. mostlyright/data_harness/ux/commands/index.py +84 -0
  269. mostlyright/data_harness/ux/commands/inventory.py +47 -0
  270. mostlyright/data_harness/ux/commands/list_builds.py +143 -0
  271. mostlyright/data_harness/ux/commands/login.py +63 -0
  272. mostlyright/data_harness/ux/commands/peek.py +236 -0
  273. mostlyright/data_harness/ux/commands/plan_check.py +90 -0
  274. mostlyright/data_harness/ux/commands/preflight.py +97 -0
  275. mostlyright/data_harness/ux/commands/record.py +107 -0
  276. mostlyright/data_harness/ux/commands/review_setup.py +47 -0
  277. mostlyright/data_harness/ux/commands/search.py +440 -0
  278. mostlyright/data_harness/ux/commands/show.py +61 -0
  279. mostlyright/data_harness/ux/commands/whoami.py +37 -0
  280. mostlyright/data_harness/ux/credential_native.py +551 -0
  281. mostlyright/data_harness/ux/credential_store.py +1055 -0
  282. mostlyright/data_harness/ux/credentials.py +631 -0
  283. mostlyright/data_harness/ux/diffing.py +444 -0
  284. mostlyright/data_harness/ux/headline.py +671 -0
  285. mostlyright/data_harness/ux/hosted_acquisition.py +974 -0
  286. mostlyright/data_harness/ux/hosted_run_status.py +619 -0
  287. mostlyright/data_harness/ux/inventory.py +427 -0
  288. mostlyright/data_harness/ux/local_review.py +375 -0
  289. mostlyright/data_harness/ux/login.py +691 -0
  290. mostlyright/data_harness/ux/path_kind.py +147 -0
  291. mostlyright/data_harness/ux/peek.py +1000 -0
  292. mostlyright/data_harness/ux/plain_file.py +178 -0
  293. mostlyright/data_harness/ux/plan_check.py +311 -0
  294. mostlyright/data_harness/ux/preflight.py +918 -0
  295. mostlyright/data_harness/ux/readers.py +1124 -0
  296. mostlyright/data_harness/ux/remediation.py +2195 -0
  297. mostlyright/data_harness/ux/render.py +657 -0
  298. mostlyright/data_harness/ux/workload.py +1077 -0
  299. mostlyright/data_harness/viewer.py +3713 -0
  300. mostlyright/data_harness/visual_run/__init__.py +83 -0
  301. mostlyright/data_harness/visual_run/authoring.py +235 -0
  302. mostlyright/data_harness/visual_run/contracts.py +673 -0
  303. mostlyright/data_harness/visual_run/materialize.py +486 -0
  304. mostlyright/data_harness/visual_run/observations.py +874 -0
  305. mostlyright/data_harness/visual_run/query.py +259 -0
  306. mostlyright/data_harness/visual_run/reducer.py +280 -0
  307. mostlyright/data_harness/visual_run/sdk.py +892 -0
  308. mostlyright/data_harness/visual_run/store.py +584 -0
  309. mostlyright/data_harness/visual_run/transport.py +239 -0
  310. mostlyright/data_harness/watch.py +2999 -0
  311. mostlyright_data-0.9.0.dist-info/METADATA +607 -0
  312. mostlyright_data-0.9.0.dist-info/RECORD +314 -0
  313. mostlyright_data-0.9.0.dist-info/WHEEL +4 -0
  314. mostlyright_data-0.9.0.dist-info/entry_points.txt +12 -0
@@ -0,0 +1,1009 @@
1
+ """``weather.grib2``: named places, their values, and the arithmetic written down.
2
+
3
+ This module is the join. Everything it needs already exists and is certified on its own:
4
+ ``admission`` refuses an unknown grid or packing from the headers, before a value is read;
5
+ ``geometry`` computes coordinates in closed form and grades them against the publisher's own
6
+ declaration on every message; the vendored binding unpacks the packed integers and cross-checks
7
+ the template numbers it was told to expect. What is added here is the composition, the recipe's
8
+ own settings, and the one piece of arithmetic nobody else owns: turning a packed integer into a
9
+ physical value.
10
+
11
+ Mostly wiring, deliberately
12
+ ---------------------------
13
+ If this module grows admission logic, projection maths, or bit unpacking, that logic has been
14
+ duplicated out of the module that already owns it and the two copies will disagree. What lives
15
+ here is what lives nowhere else: the option schema, the variable and time the product section
16
+ declares, the scaling rule, and the emitted table.
17
+
18
+ Admission is structural rather than remembered
19
+ ----------------------------------------------
20
+ ``decode_admitted`` takes a ``MessageAdmission`` and there is no function here that takes message
21
+ bytes without one -- the single exception is ``WeatherGrib2Reader.decode``, which is the Reader
22
+ contract's own entry point, is handed bytes by the contract, and admits them before it does
23
+ anything else. ``tests/h3/test_readers_grib2.py`` parses this file and asserts both halves, so a
24
+ second door has to be added to a named tuple in a reviewed edit rather than appearing quietly.
25
+ The binding's own template cross-check is the second layer, for a call site that forgot.
26
+
27
+ The recipe pins the allowlist it was written against
28
+ ----------------------------------------------------
29
+ ``admission_allowlist_version`` is a required setting. The refusals in ``admission`` handle a
30
+ message this product has never checked; this handles the other direction, where the Toolbox has
31
+ learned a new template since the recipe was frozen and the same recipe would now decode a file it
32
+ would previously have refused. Both directions are the same property: a recipe's meaning must not
33
+ drift while the recipe does not change.
34
+
35
+ Row order is the recipe's
36
+ -------------------------
37
+ Rows come out in the order the recipe lists its points, not the order the grid stores them. Grid
38
+ order is an implementation detail of the file -- a model that reorganises its grid changes it
39
+ without announcing anything -- and the recipe's order is what the author asked for and is stable
40
+ across that.
41
+
42
+ A missing value is a missing value
43
+ ----------------------------------
44
+ A weather message may carry a bitmap marking points that have no value. Such a point keeps its
45
+ row and gets an empty cell. Never a sentinel, which would be indistinguishable from a real
46
+ reading, and never a non-finite value, which the sandbox IPC refuses outright -- the boundary
47
+ telling us the right answer. A decode that met one reports ``contains_masked_points``, so the
48
+ empty cell is a fact the receipt carries rather than a gap a reader has to interpret.
49
+
50
+ A coordinate is decimal text, and that is not a style choice
51
+ ------------------------------------------------------------
52
+ Decode options are sealed through canonical JSON, which admits **no non-integer number at all**
53
+ (``canonical.py``, ``JSON_NONINTEGER_NUMBER``): a sealed document whose numbers depend on a JSON
54
+ writer's float formatting is a document that does not have one canonical form. So a recipe
55
+ states a latitude and a longitude as decimal text -- ``"39.7392"`` -- and this module converts it
56
+ with the one correctly-rounded decimal-to-binary64 conversion the language has. What is sealed
57
+ is then the digits the author wrote, and it is the same digits on every host. Exponent notation
58
+ is refused: two spellings of one coordinate would seal to two digests for one recipe.
59
+
60
+ Import discipline: ``formats``, ``readers.contracts``, ``readers.tabular``, ``readers.grib2``,
61
+ the vendored binding, and the standard library. Nothing from ``acquisition``, ``sources``,
62
+ ``recipe``, or ``pipeline``.
63
+ """
64
+
65
+ from __future__ import annotations
66
+
67
+ import datetime
68
+ import math
69
+ import re
70
+ from collections.abc import Mapping, Sequence
71
+ from dataclasses import dataclass, field
72
+ from fractions import Fraction
73
+ from typing import Any, NoReturn
74
+
75
+ import mostlyright_grib
76
+
77
+ from mostlyright.data_harness.canonical import sha256_bytes
78
+ from mostlyright.data_harness.formats import (
79
+ FORMAT_MEDIA_TYPES,
80
+ FORMAT_SUFFIXES,
81
+ READER_CONTRACT_VERSION,
82
+ READER_OUTPUT_FORMATS,
83
+ )
84
+ from mostlyright.data_harness.readers.contracts import (
85
+ DECODE_FLAGS,
86
+ ReaderBudgets,
87
+ ReaderError,
88
+ ReaderPin,
89
+ ReaderResult,
90
+ )
91
+ from mostlyright.data_harness.readers.grib2.admission import (
92
+ ADMISSION_ALLOWLIST_VERSION,
93
+ ALLOWLIST_VERSION_CONSTANT,
94
+ MessageAdmission,
95
+ admit,
96
+ )
97
+ from mostlyright.data_harness.readers.grib2.geometry import (
98
+ build_grid,
99
+ nearest_grid_index,
100
+ )
101
+ from mostlyright.data_harness.readers.tabular import (
102
+ check_declared_size,
103
+ encode_canonical_csv,
104
+ sealed_filename,
105
+ )
106
+
107
+ _MISSING_BINDING_MEMBERS = tuple(
108
+ name for name in ("GribBindingError", "unpack") if not hasattr(mostlyright_grib, name)
109
+ )
110
+ if _MISSING_BINDING_MEMBERS:
111
+ raise ImportError(
112
+ "mostlyright-grib is incompatible: missing " + ", ".join(_MISSING_BINDING_MEMBERS)
113
+ )
114
+ if not callable(mostlyright_grib.unpack) or not isinstance(mostlyright_grib.GribBindingError, type):
115
+ raise ImportError("mostlyright-grib is incompatible: binding API members have invalid types")
116
+
117
+ __all__ = [
118
+ "ADMITTED_PRODUCT_TEMPLATES",
119
+ "ADMITTED_TIME_SIGNIFICANCE",
120
+ "ADMITTED_TIME_UNIT_SECONDS",
121
+ "COLUMN_NAMES",
122
+ "FLOAT_RULE",
123
+ "MASKED_POINT_FLAG",
124
+ "MAX_BINARY_SCALE_MAGNITUDE",
125
+ "MAX_DECIMAL_SCALE_MAGNITUDE",
126
+ "V2_COLUMN_NAMES",
127
+ "NamedPoint",
128
+ "Variable",
129
+ "WeatherGrib2Reader",
130
+ "WeatherGrib2ReaderV2",
131
+ "decode_admitted",
132
+ "decode_admitted_v2",
133
+ "physical_value",
134
+ "settings_from_options",
135
+ ]
136
+
137
+ # The one encoding a Reader may seal, unpacked rather than restated.
138
+ (_OUTPUT_FORMAT,) = READER_OUTPUT_FORMATS
139
+ _OUTPUT_MEDIA_TYPE = sorted(FORMAT_MEDIA_TYPES[_OUTPUT_FORMAT])[0]
140
+ _OUTPUT_SUFFIX = sorted(FORMAT_SUFFIXES[_OUTPUT_FORMAT])[0]
141
+
142
+ # The fact a decode reports when the message marked one of the requested points absent. Declared
143
+ # in the closed vocabulary in ``readers/contracts.py`` and nowhere else; this line fails at import
144
+ # if the two ever disagree, which is the loud failure rather than a flag the result would refuse.
145
+ MASKED_POINT_FLAG = "contains_masked_points"
146
+ if MASKED_POINT_FLAG not in DECODE_FLAGS: # pragma: no cover - a contradiction caught at import
147
+ raise RuntimeError(f"{MASKED_POINT_FLAG} is not in the closed decode-flag vocabulary")
148
+
149
+ # The emitted table, in this order, for every weather dataset this product builds.
150
+ #
151
+ # ``grid_latitude`` and ``grid_longitude`` are the coordinates of the cell that answered, and they
152
+ # are carried because the attribution of a value to a place is either right or confidently wrong.
153
+ # A reader of the dataset can see how far the answering point is from the place that was asked
154
+ # for, without re-running anything; a table carrying only the request cannot be checked at all.
155
+ COLUMN_NAMES: tuple[str, ...] = (
156
+ "point_id",
157
+ "requested_latitude",
158
+ "requested_longitude",
159
+ "grid_row",
160
+ "grid_column",
161
+ "grid_latitude",
162
+ "grid_longitude",
163
+ "valid_time",
164
+ "variable",
165
+ "value",
166
+ )
167
+
168
+ # Version 2 is the evidence-bearing row contract used when a range-selected message must remain
169
+ # attributable after it becomes a table. It is deliberately a second family version rather than
170
+ # an edit to ``COLUMN_NAMES``: recipes that pin 1.0.0 keep producing byte-identical tables.
171
+ V2_COLUMN_NAMES: tuple[str, ...] = (
172
+ "point_id",
173
+ "variable",
174
+ "level",
175
+ "run_time",
176
+ "valid_time",
177
+ "value",
178
+ "matched_grid_lat",
179
+ "matched_grid_lon",
180
+ "message_sha256",
181
+ )
182
+
183
+ # The product templates this family reads. 4.0 is an instantaneous value at a single level, which
184
+ # is what every message on the admission allowlist declares. A statistically processed product --
185
+ # an accumulation or an average over a period -- is 4.8 and is refused, because its value is not
186
+ # valid *at* a time but *over* one, and a table with a single valid-time column would state
187
+ # something the message does not. Admitting one is a reviewed edit here plus a column that says
188
+ # what period the value covers.
189
+ ADMITTED_PRODUCT_TEMPLATES: frozenset[int] = frozenset({0})
190
+
191
+ # What octet 12 of the identification section may say the reference time means. Analysis and
192
+ # start-of-forecast are the two under which "valid time is the reference time plus the forecast
193
+ # offset" is true. Under verifying time or observation time the reference time already *is* the
194
+ # valid time and adding the offset would move every stamp, so those are refused rather than
195
+ # guessed at.
196
+ ADMITTED_TIME_SIGNIFICANCE: frozenset[int] = frozenset({0, 1})
197
+
198
+ # Code table 4.4, restricted to the units that are a fixed number of seconds. A month, a year, a
199
+ # decade, a normal, and a century are not durations -- they depend on which month and which year --
200
+ # so a forecast measured in them has no valid time this Reader can compute, and it is refused
201
+ # rather than approximated at 30 days.
202
+ ADMITTED_TIME_UNIT_SECONDS: Mapping[int, int] = {
203
+ 0: 60,
204
+ 1: 3600,
205
+ 2: 86400,
206
+ 10: 3 * 3600,
207
+ 11: 6 * 3600,
208
+ 12: 12 * 3600,
209
+ 13: 1,
210
+ }
211
+
212
+ # How large a scale exponent this Reader will do arithmetic with. The bound is a resource bound
213
+ # and it is stated as one: the rule below is exact integer arithmetic, so an unbounded exponent
214
+ # means an unbounded integer, and an exponent of thirty thousand -- which the format's two-octet
215
+ # sign-magnitude field can express -- would be a denial of service written in a header. Both
216
+ # bounds sit far outside anything published: the vendored messages carry decimal exponents of 0
217
+ # and 1, and the largest reported in the wild is 27. A message beyond them is refused by name.
218
+ MAX_BINARY_SCALE_MAGNITUDE = 1100
219
+ MAX_DECIMAL_SCALE_MAGNITUDE = 350
220
+
221
+ # The magnitude of the surface scale factor this Reader reads, bounded for the same reason.
222
+ _MAX_SURFACE_SCALE_MAGNITUDE = 20
223
+
224
+ # The octet a missing one-byte field carries, and the value a missing four-byte one carries.
225
+ _MISSING_OCTET = 0xFF
226
+ _MISSING_SURFACE = 255
227
+
228
+ # The shortest identification and product sections this Reader reads a field out of.
229
+ _IDENTIFICATION_OCTETS = 21
230
+ _PRODUCT_OCTETS = 34
231
+
232
+
233
+ FLOAT_RULE = """Weather values follow these numeric and output invariants.
234
+
235
+ The decoded value is the binary64 nearest to the exact rational
236
+
237
+ (R + X * 2^E) / 10^D
238
+
239
+ where X is the packed integer, R is the binary32 reference value widened exactly to binary64, and
240
+ E and D are the scale exponents. Integer arithmetic constructs the rational and rounds once at the
241
+ end. Direct binary64 division or multiplication is rejected because it rounds intermediate values.
242
+ No floating-point multiply-add exists on this path, so fused multiply-add contraction is impossible.
243
+
244
+ Decoding is single-threaded. Scaling runs only for requested output points. Text output uses the
245
+ shortest round-trip decimal representation. A missing point produces an empty cell; sentinels and
246
+ non-finite values are not emitted.
247
+ """
248
+
249
+
250
+ @dataclass(frozen=True)
251
+ class NamedPoint:
252
+ """One place a recipe asked for: what to call it, and where it is."""
253
+
254
+ identity: str
255
+ latitude: float
256
+ longitude: float
257
+
258
+
259
+ @dataclass(frozen=True)
260
+ class Variable:
261
+ """The field a recipe wants, stated so the message can be held to it.
262
+
263
+ The name is the label that reaches the table. The five numbers are the message's own
264
+ coordinates for a field -- discipline, parameter category, parameter number, and the surface
265
+ the value sits on -- and they are stated by the recipe rather than looked up from a name
266
+ table on purpose. Variable names drift across decoder table versions even when the numbers
267
+ do not, so a name table would make a dataset's meaning depend on which tables were installed.
268
+ """
269
+
270
+ name: str
271
+ discipline: int
272
+ parameter_category: int
273
+ parameter_number: int
274
+ surface_type: int
275
+ surface_value: Fraction
276
+
277
+
278
+ @dataclass(frozen=True)
279
+ class DecodeSettings:
280
+ """Everything the recipe decides, in one typed record."""
281
+
282
+ variable: Variable
283
+ points: tuple[NamedPoint, ...]
284
+ allowlist_version: int
285
+
286
+
287
+ @dataclass(frozen=True)
288
+ class _Product:
289
+ """What the identification and product sections declare about the field and its time."""
290
+
291
+ template: int
292
+ discipline: int
293
+ parameter_category: int
294
+ parameter_number: int
295
+ surface_type: int
296
+ surface_value: Fraction
297
+ run_time: datetime.datetime
298
+ valid_time: datetime.datetime
299
+
300
+
301
+ # --- The options, which are the whole of what a recipe decides ------------------------------
302
+
303
+ _OPTION_KEYS = ("variable", "points", "admission_allowlist_version")
304
+ _VARIABLE_KEYS = (
305
+ "name",
306
+ "discipline",
307
+ "parameter_category",
308
+ "parameter_number",
309
+ "surface_type",
310
+ "surface_value",
311
+ )
312
+ _POINT_KEYS = ("id", "latitude", "longitude")
313
+
314
+ # A coordinate as a recipe states it: plain decimal text, bounded, no exponent and no leading
315
+ # plus. Twelve fractional digits is four orders of magnitude finer than the millionths of a
316
+ # degree a message's own fields are quantised to, so nothing a publisher can express is lost.
317
+ _COORDINATE = re.compile(r"^-?(0|[1-9][0-9]{0,3})(\.[0-9]{1,12})?$")
318
+
319
+
320
+ def validate_options(options: Any, *, subject: str = "reader.weather.grib2.decode_options") -> dict:
321
+ """Admit this family's closed option set, refusing an unknown key by name.
322
+
323
+ Strict on purpose. The options digest is sealed into the recipe forever, so a key that was
324
+ accepted and ignored would be an unreviewable difference between two recipes that read
325
+ identically to everyone who looks at them.
326
+ """
327
+
328
+ if not isinstance(options, Mapping):
329
+ _refuse("READER_OPTIONS", subject, "must be an object")
330
+ _closed(options, _OPTION_KEYS, subject, "this family takes")
331
+ variable = _variable_options(options.get("variable"), f"{subject}.variable")
332
+ points = _point_options(options.get("points"), f"{subject}.points")
333
+ version = options.get("admission_allowlist_version")
334
+ if type(version) is not int or version < 1:
335
+ _refuse(
336
+ "READER_OPTIONS",
337
+ f"{subject}.admission_allowlist_version",
338
+ f"states {version!r}, and it must be the positive integer version of the admission "
339
+ f"allowlist this recipe was written against. It is required because a recipe that did "
340
+ f"not state one would change meaning the day the Toolbox learned a template. This "
341
+ f"build admits version {ADMISSION_ALLOWLIST_VERSION}",
342
+ )
343
+ return {"variable": variable, "points": points, "admission_allowlist_version": version}
344
+
345
+
346
+ def settings_from_options(
347
+ options: Any,
348
+ *,
349
+ subject: str = "reader.weather.grib2.decode_options",
350
+ ) -> DecodeSettings:
351
+ """The typed record the decode runs on, built from settings that have been admitted."""
352
+
353
+ admitted = validate_options(options, subject=subject)
354
+ variable = admitted["variable"]
355
+ return DecodeSettings(
356
+ variable=Variable(
357
+ name=str(variable["name"]),
358
+ discipline=int(variable["discipline"]),
359
+ parameter_category=int(variable["parameter_category"]),
360
+ parameter_number=int(variable["parameter_number"]),
361
+ surface_type=int(variable["surface_type"]),
362
+ surface_value=Fraction(int(variable["surface_value"])),
363
+ ),
364
+ points=tuple(
365
+ NamedPoint(str(point["id"]), float(point["latitude"]), float(point["longitude"]))
366
+ for point in admitted["points"]
367
+ ),
368
+ allowlist_version=int(admitted["admission_allowlist_version"]),
369
+ )
370
+
371
+
372
+ def _closed(stated: Mapping[str, Any], keys: Sequence[str], subject: str, lead: str) -> None:
373
+ unknown = sorted(str(key) for key in stated if key not in keys)
374
+ if unknown:
375
+ _refuse(
376
+ "READER_OPTIONS",
377
+ subject,
378
+ f"names no such setting: {', '.join(unknown)}; {lead} {', '.join(sorted(keys))}",
379
+ )
380
+ missing = sorted(key for key in keys if key not in stated)
381
+ if missing:
382
+ _refuse("READER_OPTIONS", subject, f"does not state: {', '.join(missing)}")
383
+
384
+
385
+ def _variable_options(stated: Any, subject: str) -> dict:
386
+ if not isinstance(stated, Mapping):
387
+ _refuse("READER_OPTIONS", subject, "must be an object naming the field this recipe wants")
388
+ _closed(stated, _VARIABLE_KEYS, subject, "a variable is stated as")
389
+ name = stated["name"]
390
+ if not isinstance(name, str) or not name.strip() or len(name) > 128:
391
+ _refuse("READER_OPTIONS", f"{subject}.name", "must be the label this field carries")
392
+ for key in ("discipline", "parameter_category", "parameter_number", "surface_type"):
393
+ value = stated[key]
394
+ if type(value) is not int or not 0 <= value <= 255:
395
+ _refuse(
396
+ "READER_OPTIONS",
397
+ f"{subject}.{key}",
398
+ "must be the octet the message states for it, between 0 and 255",
399
+ )
400
+ surface = stated["surface_value"]
401
+ if type(surface) is not int:
402
+ _refuse(
403
+ "READER_OPTIONS",
404
+ f"{subject}.surface_value",
405
+ "must be the whole number the level is stated at -- 2 for two metres above ground, "
406
+ "85000 for the 850 hPa level in pascals -- because a sealed decode option is "
407
+ "canonical JSON and canonical JSON carries no fractional number",
408
+ )
409
+ return dict(stated)
410
+
411
+
412
+ def _point_options(stated: Any, subject: str) -> tuple[dict, ...]:
413
+ if isinstance(stated, (str, bytes)) or not isinstance(stated, Sequence) or not stated:
414
+ _refuse(
415
+ "READER_OPTIONS",
416
+ subject,
417
+ "must be a nonempty list of the places this dataset is built at; a weather file is "
418
+ "a grid and a dataset is the places you named, so naming none asks for nothing",
419
+ )
420
+ admitted: list[dict] = []
421
+ seen: set[str] = set()
422
+ for index, point in enumerate(stated):
423
+ where = f"{subject}[{index}]"
424
+ if not isinstance(point, Mapping):
425
+ _refuse("READER_OPTIONS", where, "must be an object")
426
+ _closed(point, _POINT_KEYS, where, "a place is stated as")
427
+ identity = point["id"]
428
+ if not isinstance(identity, str) or not identity.strip() or len(identity) > 128:
429
+ _refuse("READER_OPTIONS", f"{where}.id", "must be the name this place carries")
430
+ if identity in seen:
431
+ _refuse(
432
+ "READER_OPTIONS",
433
+ f"{where}.id",
434
+ f"repeats {identity}; two rows under one name is a table nobody can join on",
435
+ )
436
+ seen.add(identity)
437
+ for key in ("latitude", "longitude"):
438
+ value = point[key]
439
+ if not isinstance(value, str) or _COORDINATE.fullmatch(value) is None:
440
+ _refuse(
441
+ "READER_OPTIONS",
442
+ f"{where}.{key}",
443
+ 'must be plain decimal text such as "39.7392", with no exponent and no '
444
+ "leading plus. It is text rather than a number because a sealed decode "
445
+ "option is canonical JSON, which carries no fractional number at all -- so "
446
+ "what is sealed is the digits written here, identically on every host",
447
+ )
448
+ admitted.append(dict(point))
449
+ return tuple(admitted)
450
+
451
+
452
+ # --- The arithmetic ---------------------------------------------------------------------------
453
+
454
+
455
+ def physical_value(
456
+ level: float,
457
+ *,
458
+ reference_value: float,
459
+ binary_scale: int,
460
+ decimal_scale: int,
461
+ ) -> float:
462
+ """Recover one physical value from one packed integer, under the rule in ``FLOAT_RULE``.
463
+
464
+ The whole rule is in that constant and this function is its one implementation. The exact
465
+ rational is built with integer arithmetic and rounded to binary64 exactly once, by the
466
+ division on the last line -- ``Fraction.__float__`` divides two integers, which CPython
467
+ rounds correctly. Nothing else on this path rounds.
468
+ """
469
+
470
+ if not isinstance(level, float) or not math.isfinite(level):
471
+ _refuse(
472
+ "READER_OUTPUT",
473
+ "reader.weather.grib2.level",
474
+ "a point with no value is an empty cell rather than a number, so it never reaches "
475
+ "the scaling rule",
476
+ )
477
+ if abs(binary_scale) > MAX_BINARY_SCALE_MAGNITUDE:
478
+ _refuse(
479
+ "READER_ADMISSION",
480
+ "reader.weather.grib2.binary_scale",
481
+ f"declares a binary scale exponent of {binary_scale}, beyond the "
482
+ f"{MAX_BINARY_SCALE_MAGNITUDE} this Reader computes with; the rule is exact integer "
483
+ f"arithmetic, so an unbounded exponent is an unbounded integer",
484
+ )
485
+ if abs(decimal_scale) > MAX_DECIMAL_SCALE_MAGNITUDE:
486
+ _refuse(
487
+ "READER_ADMISSION",
488
+ "reader.weather.grib2.decimal_scale",
489
+ f"declares a decimal scale exponent of {decimal_scale}, beyond the "
490
+ f"{MAX_DECIMAL_SCALE_MAGNITUDE} this Reader computes with; the rule is exact integer "
491
+ f"arithmetic, so an unbounded exponent is an unbounded integer",
492
+ )
493
+ exact = (Fraction(reference_value) + Fraction(level) * Fraction(2) ** binary_scale) / Fraction(
494
+ 10
495
+ ) ** decimal_scale
496
+ value = float(exact)
497
+ if not math.isfinite(value):
498
+ _refuse(
499
+ "READER_OUTPUT",
500
+ "reader.weather.grib2.value",
501
+ f"scales to {exact.numerator}/{exact.denominator}, which is outside the range a "
502
+ f"binary64 value can carry; a value that cannot be represented is not rendered as "
503
+ f"one that can",
504
+ )
505
+ return value
506
+
507
+
508
+ # --- What the message says about its field and its time ---------------------------------------
509
+
510
+
511
+ def _product(admission: MessageAdmission, content: bytes) -> _Product:
512
+ """Read the field and the valid time out of the sections admission located."""
513
+
514
+ if admission.identification_length < _IDENTIFICATION_OCTETS:
515
+ _refuse(
516
+ "READER_ADMISSION",
517
+ "reader.weather.grib2.identification",
518
+ f"declares {admission.identification_length} bytes, too few to state the reference "
519
+ f"time a valid time is computed from",
520
+ )
521
+ if admission.product_length < _PRODUCT_OCTETS:
522
+ _refuse(
523
+ "READER_ADMISSION",
524
+ "reader.weather.grib2.product",
525
+ f"declares {admission.product_length} bytes, too few to state which field it holds",
526
+ )
527
+
528
+ template = int.from_bytes(
529
+ content[admission.product_start + 7 : admission.product_start + 9], "big"
530
+ )
531
+ if template not in ADMITTED_PRODUCT_TEMPLATES:
532
+ _refuse(
533
+ "READER_ADMISSION",
534
+ "reader.weather.grib2.product_template",
535
+ f"describes its field with product template 4.{template}, and this Reader reads "
536
+ f"4.{', 4.'.join(str(number) for number in sorted(ADMITTED_PRODUCT_TEMPLATES))}, an "
537
+ f"instantaneous value at one level. A product averaged or accumulated over a period "
538
+ f"is not valid at a time, so a table with one valid-time column would state something "
539
+ f"the message does not; admitting one means a reviewed edit plus a column saying what "
540
+ f"period each value covers",
541
+ )
542
+
543
+ surface_type = content[admission.product_start + 22]
544
+ if content[admission.product_start + 28] != _MISSING_SURFACE:
545
+ _refuse(
546
+ "READER_ADMISSION",
547
+ "reader.weather.grib2.product",
548
+ f"states a second fixed surface of type {content[admission.product_start + 28]}, so "
549
+ f"its values describe a layer between two surfaces rather than one surface; this "
550
+ f"Reader reads single-surface fields, and reading a layer as a surface would attach a "
551
+ f"value to a level it does not describe",
552
+ )
553
+
554
+ run_time = _reference_time(admission, content)
555
+ return _Product(
556
+ template=template,
557
+ discipline=admission.discipline,
558
+ parameter_category=content[admission.product_start + 9],
559
+ parameter_number=content[admission.product_start + 10],
560
+ surface_type=surface_type,
561
+ surface_value=_surface_value(admission, content),
562
+ run_time=run_time,
563
+ valid_time=_valid_time(admission, content, reference=run_time),
564
+ )
565
+
566
+
567
+ def _surface_value(admission: MessageAdmission, content: bytes) -> Fraction:
568
+ """The level the values sit at, as the exact rational the message states."""
569
+
570
+ scale_octet = content[admission.product_start + 23]
571
+ scaled = _sign_magnitude(content[admission.product_start + 24 : admission.product_start + 28])
572
+ if scale_octet == _MISSING_OCTET:
573
+ return Fraction(0)
574
+ scale = _sign_magnitude(bytes([scale_octet]))
575
+ if abs(scale) > _MAX_SURFACE_SCALE_MAGNITUDE:
576
+ _refuse(
577
+ "READER_ADMISSION",
578
+ "reader.weather.grib2.surface",
579
+ f"scales its level by ten to the {scale}, which is not a level anything publishes",
580
+ )
581
+ return Fraction(scaled) / Fraction(10) ** scale
582
+
583
+
584
+ def _reference_time(admission: MessageAdmission, content: bytes) -> datetime.datetime:
585
+ """Return the admitted UTC model-run coordinate declared by the message."""
586
+
587
+ start = admission.identification_start
588
+ significance = content[start + 11]
589
+ if significance not in ADMITTED_TIME_SIGNIFICANCE:
590
+ _refuse(
591
+ "READER_ADMISSION",
592
+ "reader.weather.grib2.reference_time",
593
+ f"says its reference time means significance {significance} rather than an analysis "
594
+ f"or the start of a forecast; under those two the valid time is the reference time "
595
+ f"plus the forecast offset, and under the others adding the offset would move every "
596
+ f"stamp in the table",
597
+ )
598
+ year = int.from_bytes(content[start + 12 : start + 14], "big")
599
+ try:
600
+ return datetime.datetime(
601
+ year,
602
+ content[start + 14],
603
+ content[start + 15],
604
+ content[start + 16],
605
+ content[start + 17],
606
+ min(content[start + 18], 59),
607
+ tzinfo=datetime.UTC,
608
+ )
609
+ except ValueError as error:
610
+ _refuse(
611
+ "READER_ADMISSION",
612
+ "reader.weather.grib2.reference_time",
613
+ f"states a reference time that is not a date: {error}",
614
+ )
615
+
616
+
617
+ def _valid_time(
618
+ admission: MessageAdmission,
619
+ content: bytes,
620
+ *,
621
+ reference: datetime.datetime | None = None,
622
+ ) -> datetime.datetime:
623
+ """The reference time plus the forecast offset, in UTC, or a refusal naming what stopped it."""
624
+
625
+ if reference is None:
626
+ reference = _reference_time(admission, content)
627
+
628
+ unit = content[admission.product_start + 17]
629
+ if unit not in ADMITTED_TIME_UNIT_SECONDS:
630
+ _refuse(
631
+ "READER_ADMISSION",
632
+ "reader.weather.grib2.forecast_time",
633
+ f"measures its forecast offset in unit {unit} of code table 4.4, which is not a fixed "
634
+ f"number of seconds; a month and a year depend on which month and which year, so a "
635
+ f"valid time computed from one would be an approximation presented as a fact",
636
+ )
637
+ offset = _sign_magnitude(content[admission.product_start + 18 : admission.product_start + 22])
638
+ if offset < 0:
639
+ _refuse(
640
+ "READER_ADMISSION",
641
+ "reader.weather.grib2.forecast_time",
642
+ f"states a forecast offset of {offset}, which runs backwards from its own reference "
643
+ f"time",
644
+ )
645
+ seconds = offset * ADMITTED_TIME_UNIT_SECONDS[unit]
646
+ try:
647
+ return reference + datetime.timedelta(seconds=seconds)
648
+ except OverflowError:
649
+ _refuse(
650
+ "READER_ADMISSION",
651
+ "reader.weather.grib2.forecast_time",
652
+ f"states a forecast offset of {seconds} seconds, which is not a time",
653
+ )
654
+
655
+
656
+ def _require_the_variable_the_recipe_asked_for(product: _Product, wanted: Variable) -> None:
657
+ """Hold the message to the field the recipe named, refusing a mismatch on any coordinate.
658
+
659
+ Pointing a recipe at the wrong file is the ordinary mistake here, and it is the one that
660
+ verifies clean: categorical rain and a temperature are both real numbers on the same grid at
661
+ the same time, and a table labelled with the recipe's variable name would carry the other
662
+ field's values under it without a mark.
663
+ """
664
+
665
+ stated = (
666
+ wanted.discipline,
667
+ wanted.parameter_category,
668
+ wanted.parameter_number,
669
+ wanted.surface_type,
670
+ wanted.surface_value,
671
+ )
672
+ declared = (
673
+ product.discipline,
674
+ product.parameter_category,
675
+ product.parameter_number,
676
+ product.surface_type,
677
+ product.surface_value,
678
+ )
679
+ if stated == declared:
680
+ return
681
+ _refuse(
682
+ "READER_ADMISSION",
683
+ "reader.weather.grib2.variable",
684
+ f"holds discipline {product.discipline}, parameter {product.parameter_category}."
685
+ f"{product.parameter_number} on surface type {product.surface_type} at "
686
+ f"{product.surface_value}, and the recipe asks for {wanted.name}: discipline "
687
+ f"{wanted.discipline}, parameter {wanted.parameter_category}.{wanted.parameter_number} on "
688
+ f"surface type {wanted.surface_type} at {wanted.surface_value}. A message carries one "
689
+ f"field, so this file is not the one this recipe describes; either the source points at "
690
+ f"the wrong record or the recipe names the wrong field",
691
+ )
692
+
693
+
694
+ # --- The decode -------------------------------------------------------------------------------
695
+
696
+
697
+ def _unpack(admission: MessageAdmission, content: bytes) -> Any:
698
+ """The one call into the vendored binding, which cannot be made without an admission result.
699
+
700
+ The template numbers handed over are the ones admission established, and the binding refuses
701
+ to decode a message declaring anything else -- the second layer, for a call site that skipped
702
+ the first. The parameters it read out of Section 5 are then held against the ones admission
703
+ read out of the same octets, because two readings of one field that disagree mean one of them
704
+ is reading the wrong bytes and the values would be scaled by whichever won.
705
+
706
+ Translate ``GribBindingError`` to the typed Reader error used for corrupt or truncated packed
707
+ values. Raw runtime errors must not escape the Reader boundary.
708
+ """
709
+
710
+ try:
711
+ unpacked = mostlyright_grib.unpack(
712
+ content,
713
+ grid_template=admission.grid_template,
714
+ packing_template=admission.packing_template,
715
+ )
716
+ except mostlyright_grib.GribBindingError as error:
717
+ _refuse(
718
+ "READER_DECODE",
719
+ "reader.weather.grib2.message",
720
+ f"was admitted and then did not decode: {error.text} (decoder code {error.code}). "
721
+ f"The grid and packing this message declares are on the approved list, so the "
722
+ f"refusal is about the packed values themselves rather than about the format: the "
723
+ f"file is truncated or damaged. Fetch it again and compare the hash the receipt "
724
+ f"recorded; nothing was published and the version you already have is untouched",
725
+ )
726
+ declared = (admission.reference_value, admission.binary_scale, admission.decimal_scale)
727
+ read = (unpacked.reference_value, unpacked.binary_scale, unpacked.decimal_scale)
728
+ if declared != read:
729
+ _refuse(
730
+ "READER_ADMISSION",
731
+ "reader.weather.grib2.packing",
732
+ f"is read as {declared} by admission and as {read} by the decoder; two readings of "
733
+ f"one field that disagree mean one of them is reading the wrong bytes, and the values "
734
+ f"would be scaled by whichever answered",
735
+ )
736
+ if len(unpacked.levels) != admission.declared_points:
737
+ _refuse(
738
+ "READER_ADMISSION",
739
+ "reader.weather.grib2.packing",
740
+ f"unpacks {len(unpacked.levels)} values for a grid of {admission.declared_points} "
741
+ f"points",
742
+ )
743
+ return unpacked
744
+
745
+
746
+ def decode_admitted(
747
+ admission: MessageAdmission,
748
+ content: bytes,
749
+ settings: DecodeSettings,
750
+ budgets: ReaderBudgets,
751
+ ) -> ReaderResult:
752
+ """Turn one admitted message into the rows the recipe asked for.
753
+
754
+ Takes an admission result, which is the only way this Reader decodes anything: a message
755
+ nobody admitted has nothing to pass here. That is the same rule ``geometry.build_grid``
756
+ applies, for the same reason, and it is asserted structurally rather than described.
757
+ """
758
+
759
+ if not isinstance(admission, MessageAdmission):
760
+ _refuse(
761
+ "READER_CONTRACT",
762
+ "reader.weather.grib2.admission",
763
+ "a message is decoded from an admission result, so an unadmitted message has nothing "
764
+ "to pass; this is the rule that makes decoding an unadmitted message inexpressible "
765
+ "rather than merely undone",
766
+ )
767
+ if not isinstance(settings, DecodeSettings):
768
+ _refuse(
769
+ "READER_CONTRACT",
770
+ "reader.weather.grib2.settings",
771
+ "a decode runs on settings that have been admitted, not on a mapping",
772
+ )
773
+ if settings.allowlist_version != ADMISSION_ALLOWLIST_VERSION:
774
+ _refuse(
775
+ "READER_ADMISSION",
776
+ "reader.weather.grib2.admission_allowlist_version",
777
+ f"was written against allowlist version {settings.allowlist_version} and this build "
778
+ f"admits version {ADMISSION_ALLOWLIST_VERSION}. The list of grid and packing "
779
+ f"combinations this product opens has changed since the recipe was frozen, so the "
780
+ f"same recipe would now mean something different -- it would open files it used to "
781
+ f"refuse, or refuse files it used to open. Nothing is decoded until a person decides "
782
+ f"which meaning is wanted: re-approve the recipe against the current list, or build "
783
+ f"with the release that carried version {settings.allowlist_version}. The current "
784
+ f"version is {ALLOWLIST_VERSION_CONSTANT}",
785
+ )
786
+
787
+ check_declared_size(
788
+ row_count=len(settings.points),
789
+ column_count=len(COLUMN_NAMES),
790
+ budgets=budgets,
791
+ subject="reader.weather.grib2.table",
792
+ )
793
+
794
+ product = _product(admission, content)
795
+ _require_the_variable_the_recipe_asked_for(product, settings.variable)
796
+ grid = build_grid(admission, content)
797
+ unpacked = _unpack(admission, content)
798
+
799
+ stamp = product.valid_time.strftime("%Y-%m-%dT%H:%M:%SZ")
800
+ rows: list[tuple[Any, ...]] = []
801
+ masked = False
802
+ for point in settings.points:
803
+ row, column = nearest_grid_index(grid, point.latitude, point.longitude, name=point.identity)
804
+ latitude, longitude = grid.coordinate_at(row, column)
805
+ level = unpacked.levels[grid.data_index(row, column)]
806
+ if math.isfinite(level):
807
+ value: float | None = physical_value(
808
+ level,
809
+ reference_value=admission.reference_value,
810
+ binary_scale=admission.binary_scale,
811
+ decimal_scale=admission.decimal_scale,
812
+ )
813
+ else:
814
+ value = None
815
+ masked = True
816
+ rows.append(
817
+ (
818
+ point.identity,
819
+ point.latitude,
820
+ point.longitude,
821
+ row,
822
+ column,
823
+ latitude,
824
+ longitude,
825
+ stamp,
826
+ settings.variable.name,
827
+ value,
828
+ )
829
+ )
830
+
831
+ return ReaderResult(
832
+ content=encode_canonical_csv(COLUMN_NAMES, rows, budgets=budgets),
833
+ data_format=_OUTPUT_FORMAT,
834
+ media_type=_OUTPUT_MEDIA_TYPE,
835
+ # The recipe's own name for the field, which is the one piece of text on this path a
836
+ # person wrote down deliberately. A weather message carries no filename of its own.
837
+ filename=sealed_filename(settings.variable.name, suffix=_OUTPUT_SUFFIX),
838
+ row_count=len(rows),
839
+ column_names=COLUMN_NAMES,
840
+ declared_cell_count=max(admission.declared_points, len(rows) * len(COLUMN_NAMES)),
841
+ flags=(MASKED_POINT_FLAG,) if masked else (),
842
+ )
843
+
844
+
845
+ def _level_coordinate(product: _Product) -> str:
846
+ """Spell the GRIB fixed-surface coordinate without a lossy unit-name lookup."""
847
+
848
+ value = product.surface_value
849
+ amount = str(value.numerator)
850
+ if value.denominator != 1:
851
+ amount = f"{amount}/{value.denominator}"
852
+ return f"{product.surface_type}:{amount}"
853
+
854
+
855
+ def decode_admitted_v2(
856
+ admission: MessageAdmission,
857
+ content: bytes,
858
+ settings: DecodeSettings,
859
+ budgets: ReaderBudgets,
860
+ ) -> ReaderResult:
861
+ """Decode one admitted message into the evidence-bearing 2.0 row contract."""
862
+
863
+ if not isinstance(admission, MessageAdmission):
864
+ _refuse(
865
+ "READER_CONTRACT",
866
+ "reader.weather.grib2.admission",
867
+ "a message is decoded from an admission result",
868
+ )
869
+ if not isinstance(settings, DecodeSettings):
870
+ _refuse(
871
+ "READER_CONTRACT",
872
+ "reader.weather.grib2.settings",
873
+ "a decode runs on admitted settings",
874
+ )
875
+ if settings.allowlist_version != ADMISSION_ALLOWLIST_VERSION:
876
+ _refuse(
877
+ "READER_ADMISSION",
878
+ "reader.weather.grib2.admission_allowlist_version",
879
+ f"was written against allowlist version {settings.allowlist_version}, not "
880
+ f"{ADMISSION_ALLOWLIST_VERSION}",
881
+ )
882
+
883
+ check_declared_size(
884
+ row_count=len(settings.points),
885
+ column_count=len(V2_COLUMN_NAMES),
886
+ budgets=budgets,
887
+ subject="reader.weather.grib2.v2.table",
888
+ )
889
+ product = _product(admission, content)
890
+ _require_the_variable_the_recipe_asked_for(product, settings.variable)
891
+ grid = build_grid(admission, content)
892
+ unpacked = _unpack(admission, content)
893
+ message_digest = sha256_bytes(content)
894
+ run_time = product.run_time.strftime("%Y-%m-%dT%H:%M:%SZ")
895
+ valid_time = product.valid_time.strftime("%Y-%m-%dT%H:%M:%SZ")
896
+ level_coordinate = _level_coordinate(product)
897
+ rows: list[tuple[Any, ...]] = []
898
+ masked = False
899
+ for point in settings.points:
900
+ row, column = nearest_grid_index(grid, point.latitude, point.longitude, name=point.identity)
901
+ latitude, longitude = grid.coordinate_at(row, column)
902
+ level = unpacked.levels[grid.data_index(row, column)]
903
+ if math.isfinite(level):
904
+ value: float | None = physical_value(
905
+ level,
906
+ reference_value=admission.reference_value,
907
+ binary_scale=admission.binary_scale,
908
+ decimal_scale=admission.decimal_scale,
909
+ )
910
+ else:
911
+ value = None
912
+ masked = True
913
+ rows.append(
914
+ (
915
+ point.identity,
916
+ settings.variable.name,
917
+ level_coordinate,
918
+ run_time,
919
+ valid_time,
920
+ value,
921
+ latitude,
922
+ longitude,
923
+ message_digest,
924
+ )
925
+ )
926
+ return ReaderResult(
927
+ content=encode_canonical_csv(V2_COLUMN_NAMES, rows, budgets=budgets),
928
+ data_format=_OUTPUT_FORMAT,
929
+ media_type=_OUTPUT_MEDIA_TYPE,
930
+ filename=sealed_filename(settings.variable.name, suffix=_OUTPUT_SUFFIX),
931
+ row_count=len(rows),
932
+ column_names=V2_COLUMN_NAMES,
933
+ declared_cell_count=max(admission.declared_points, len(rows) * len(V2_COLUMN_NAMES)),
934
+ flags=(MASKED_POINT_FLAG,) if masked else (),
935
+ )
936
+
937
+
938
+ @dataclass(frozen=True)
939
+ class WeatherGrib2Reader:
940
+ """``weather.grib2``: a weather-model message read as the places a recipe named.
941
+
942
+ ``accepted_media_types`` admits what the mirrors actually serve. That is an input-side
943
+ declaration and it is not a format table: what may be *sealed* is still canonical CSV alone.
944
+ """
945
+
946
+ family_id: str = "weather.grib2"
947
+ family_version: str = "1.0.0"
948
+ contract_version: str = READER_CONTRACT_VERSION
949
+ output_format: str = _OUTPUT_FORMAT
950
+ accepted_media_types: tuple[str, ...] = (
951
+ "application/octet-stream",
952
+ "application/wmo-grib2",
953
+ )
954
+ default_budgets: ReaderBudgets = field(default_factory=ReaderBudgets)
955
+
956
+ def validate_options(self, options: Mapping[str, Any]) -> Mapping[str, Any]:
957
+ return validate_options(options)
958
+
959
+ def decode(self, content: bytes, pin: ReaderPin, budgets: ReaderBudgets) -> ReaderResult:
960
+ """The Reader contract's entry point: bytes in, sealed bytes out, admission first.
961
+
962
+ This is the one function here that receives message bytes without an admission result,
963
+ because the contract hands it bytes and this is the thing that admits them. It admits
964
+ before it does anything else, and the test module asserts that from the syntax tree.
965
+ """
966
+
967
+ settings = settings_from_options(pin.decode_options)
968
+ admission = admit(content, budgets)
969
+ return decode_admitted(admission, content, settings, budgets)
970
+
971
+
972
+ @dataclass(frozen=True)
973
+ class WeatherGrib2ReaderV2:
974
+ """Evidence-bearing weather rows; co-installed beside byte-stable 1.0.0."""
975
+
976
+ family_id: str = "weather.grib2"
977
+ family_version: str = "2.0.0"
978
+ contract_version: str = READER_CONTRACT_VERSION
979
+ output_format: str = _OUTPUT_FORMAT
980
+ accepted_media_types: tuple[str, ...] = (
981
+ "application/octet-stream",
982
+ "application/wmo-grib2",
983
+ )
984
+ default_budgets: ReaderBudgets = field(default_factory=ReaderBudgets)
985
+
986
+ def validate_options(self, options: Mapping[str, Any]) -> Mapping[str, Any]:
987
+ return validate_options(options)
988
+
989
+ def decode(self, content: bytes, pin: ReaderPin, budgets: ReaderBudgets) -> ReaderResult:
990
+ settings = settings_from_options(pin.decode_options)
991
+ admission = admit(content, budgets)
992
+ return decode_admitted_v2(admission, content, settings, budgets)
993
+
994
+
995
+ def _sign_magnitude(raw: bytes) -> int:
996
+ """A GRIB2 signed integer: a sign bit and a magnitude, never two's complement.
997
+
998
+ The same rule admission and geometry each read their signed fields with, written out here
999
+ rather than imported across a module boundary for the reason recorded in ``geometry``, and
1000
+ held to agree with both by test.
1001
+ """
1002
+
1003
+ value = int.from_bytes(raw, "big")
1004
+ sign = 1 << (8 * len(raw) - 1)
1005
+ return -(value & ~sign) if value & sign else value
1006
+
1007
+
1008
+ def _refuse(code: str, subject: str, detail: str) -> NoReturn:
1009
+ raise ReaderError(code, subject, detail)