mostlyright-data 0.9.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (314) hide show
  1. mostlyright/data_harness/__init__.py +158 -0
  2. mostlyright/data_harness/acquisition/__init__.py +55 -0
  3. mostlyright/data_harness/acquisition/http.py +2773 -0
  4. mostlyright/data_harness/acquisition/parsing.py +809 -0
  5. mostlyright/data_harness/acquisition/ranges.py +495 -0
  6. mostlyright/data_harness/acquisition/result_download.py +360 -0
  7. mostlyright/data_harness/acquisition/retention_admission.py +248 -0
  8. mostlyright/data_harness/acquisition/sandbox.py +4888 -0
  9. mostlyright/data_harness/acquisition/url_policy.py +530 -0
  10. mostlyright/data_harness/agent_runtime.py +2743 -0
  11. mostlyright/data_harness/assets/logo-ink.svg +31 -0
  12. mostlyright/data_harness/backends/__init__.py +28 -0
  13. mostlyright/data_harness/backends/pandas_backend.py +350 -0
  14. mostlyright/data_harness/backends/polars_backend.py +366 -0
  15. mostlyright/data_harness/backends/protocol.py +124 -0
  16. mostlyright/data_harness/backends/reference.py +83 -0
  17. mostlyright/data_harness/backends/registry.py +55 -0
  18. mostlyright/data_harness/backends/restrictions.py +126 -0
  19. mostlyright/data_harness/canonical.py +333 -0
  20. mostlyright/data_harness/catalog_job.py +625 -0
  21. mostlyright/data_harness/cli.py +5398 -0
  22. mostlyright/data_harness/contracts.py +53 -0
  23. mostlyright/data_harness/coordinator.py +1307 -0
  24. mostlyright/data_harness/deploy.py +924 -0
  25. mostlyright/data_harness/deploy_target.py +312 -0
  26. mostlyright/data_harness/deployment_evidence.py +1067 -0
  27. mostlyright/data_harness/event_presentation.py +576 -0
  28. mostlyright/data_harness/events.py +2152 -0
  29. mostlyright/data_harness/fast_delimited.py +239 -0
  30. mostlyright/data_harness/fleet.py +237 -0
  31. mostlyright/data_harness/formats.py +236 -0
  32. mostlyright/data_harness/governors.py +1163 -0
  33. mostlyright/data_harness/hosted_bootstrap.py +972 -0
  34. mostlyright/data_harness/hosted_crawler.py +1115 -0
  35. mostlyright/data_harness/hosted_crawler_container_smoke.py +351 -0
  36. mostlyright/data_harness/hosted_crawler_fetch.py +423 -0
  37. mostlyright/data_harness/hosted_crawler_job.py +1277 -0
  38. mostlyright/data_harness/hosted_crawler_protocol.py +676 -0
  39. mostlyright/data_harness/hosted_dataset.py +1500 -0
  40. mostlyright/data_harness/hosted_deploy.py +3037 -0
  41. mostlyright/data_harness/hosted_handoff.py +62 -0
  42. mostlyright/data_harness/hosted_ingestion_contract.py +504 -0
  43. mostlyright/data_harness/hosted_ingestion_job.py +356 -0
  44. mostlyright/data_harness/hosted_ingestion_job_smoke.py +40 -0
  45. mostlyright/data_harness/hosted_session_container_smoke.py +194 -0
  46. mostlyright/data_harness/hosted_session_worker.py +3554 -0
  47. mostlyright/data_harness/hosted_session_worker_job_smoke.py +46 -0
  48. mostlyright/data_harness/hosted_worker.py +6784 -0
  49. mostlyright/data_harness/ingestion/__init__.py +56 -0
  50. mostlyright/data_harness/ingestion/contracts.py +461 -0
  51. mostlyright/data_harness/ingestion/faults.py +42 -0
  52. mostlyright/data_harness/ingestion/gcs_store.py +1162 -0
  53. mostlyright/data_harness/ingestion/spool.py +130 -0
  54. mostlyright/data_harness/ingestion/store.py +885 -0
  55. mostlyright/data_harness/key_seam.py +434 -0
  56. mostlyright/data_harness/linux_process_boundary.py +262 -0
  57. mostlyright/data_harness/local_contracts.py +2880 -0
  58. mostlyright/data_harness/local_search/__init__.py +5 -0
  59. mostlyright/data_harness/local_search/build_index.py +1087 -0
  60. mostlyright/data_harness/local_search/contracts.py +920 -0
  61. mostlyright/data_harness/local_search/query_trace.py +266 -0
  62. mostlyright/data_harness/local_search/retrieval.py +700 -0
  63. mostlyright/data_harness/local_search/sealed.py +474 -0
  64. mostlyright/data_harness/local_search/service.py +784 -0
  65. mostlyright/data_harness/nbrender/CONTRACT.md +212 -0
  66. mostlyright/data_harness/nbrender/__init__.py +12 -0
  67. mostlyright/data_harness/nbrender/chrome.py +359 -0
  68. mostlyright/data_harness/nbrender/code_body.py +266 -0
  69. mostlyright/data_harness/nbrender/document.py +407 -0
  70. mostlyright/data_harness/nbrender/frame.py +275 -0
  71. mostlyright/data_harness/nbrender/interactive.py +337 -0
  72. mostlyright/data_harness/nbrender/markdown_body.py +477 -0
  73. mostlyright/data_harness/nbrender/mr_components.py +134 -0
  74. mostlyright/data_harness/nbrender/outputs_data.py +595 -0
  75. mostlyright/data_harness/nbrender/outputs_rich.py +906 -0
  76. mostlyright/data_harness/nbrender/outputs_source.py +260 -0
  77. mostlyright/data_harness/nbrender/outputs_stage.py +176 -0
  78. mostlyright/data_harness/nbrender/outputs_text.py +400 -0
  79. mostlyright/data_harness/nbrender/parse.py +394 -0
  80. mostlyright/data_harness/nbrender/status.py +40 -0
  81. mostlyright/data_harness/nbrender/tokens.py +1295 -0
  82. mostlyright/data_harness/notebook.py +1710 -0
  83. mostlyright/data_harness/offline.py +2049 -0
  84. mostlyright/data_harness/operation_registry.py +1007 -0
  85. mostlyright/data_harness/operator_setup.py +239 -0
  86. mostlyright/data_harness/pipeline.py +6428 -0
  87. mostlyright/data_harness/plan_graph.py +2026 -0
  88. mostlyright/data_harness/preparation/__init__.py +104 -0
  89. mostlyright/data_harness/preparation/contracts.py +1017 -0
  90. mostlyright/data_harness/preparation/engine.py +221 -0
  91. mostlyright/data_harness/preparation/errors.py +14 -0
  92. mostlyright/data_harness/preparation/gates.py +751 -0
  93. mostlyright/data_harness/preparation/joins.py +574 -0
  94. mostlyright/data_harness/preparation/profile.py +384 -0
  95. mostlyright/data_harness/preparation/table.py +217 -0
  96. mostlyright/data_harness/preparation/transforms.py +568 -0
  97. mostlyright/data_harness/progress_events.py +534 -0
  98. mostlyright/data_harness/readers/__init__.py +46 -0
  99. mostlyright/data_harness/readers/containers.py +963 -0
  100. mostlyright/data_harness/readers/contracts.py +542 -0
  101. mostlyright/data_harness/readers/delimited.py +257 -0
  102. mostlyright/data_harness/readers/grib2/__init__.py +33 -0
  103. mostlyright/data_harness/readers/grib2/admission.py +722 -0
  104. mostlyright/data_harness/readers/grib2/decode.py +1009 -0
  105. mostlyright/data_harness/readers/grib2/geometry.py +1133 -0
  106. mostlyright/data_harness/readers/grib2/portable_math.py +501 -0
  107. mostlyright/data_harness/readers/json_tabular.py +485 -0
  108. mostlyright/data_harness/readers/registry.py +514 -0
  109. mostlyright/data_harness/readers/samples/README.md +110 -0
  110. mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/cities.csv.gz +0 -0
  111. mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/expected.json +24 -0
  112. mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/cities.csv.gz +0 -0
  113. mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/expected.json +24 -0
  114. mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/cities.tar +0 -0
  115. mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/expected.json +24 -0
  116. mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/cities.tar +0 -0
  117. mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/expected.json +24 -0
  118. mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/cities.zip +0 -0
  119. mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/expected.json +25 -0
  120. mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
  121. mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/expected.json +25 -0
  122. mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
  123. mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/expected.json +25 -0
  124. mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/cities.csv +3 -0
  125. mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/expected.json +23 -0
  126. mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/cities.tsv +5 -0
  127. mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/expected.json +25 -0
  128. mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/expected.json +30 -0
  129. mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/observations.csv +5 -0
  130. mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/expected.json +44 -0
  131. mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/stations.json +1 -0
  132. mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/expected.json +48 -0
  133. mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/observations.ndjson +4 -0
  134. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/cities.xlsx +0 -0
  135. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/expected.json +24 -0
  136. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/expected.json +27 -0
  137. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
  138. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/expected.json +27 -0
  139. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
  140. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/README.md +20 -0
  141. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/expected.json +55 -0
  142. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/gfs-2m-temperature.grib2 +0 -0
  143. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/expected.json +54 -0
  144. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
  145. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/expected.json +54 -0
  146. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/hrrr-categorical-rain.grib2 +0 -0
  147. mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/expected.json +54 -0
  148. mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
  149. mostlyright/data_harness/readers/samples.py +582 -0
  150. mostlyright/data_harness/readers/spreadsheet.py +803 -0
  151. mostlyright/data_harness/readers/tabular.py +510 -0
  152. mostlyright/data_harness/recipe.py +5321 -0
  153. mostlyright/data_harness/repair/__init__.py +78 -0
  154. mostlyright/data_harness/repair/adapters.py +274 -0
  155. mostlyright/data_harness/repair/contracts.py +872 -0
  156. mostlyright/data_harness/repair/coordinator.py +1099 -0
  157. mostlyright/data_harness/repair/errors.py +16 -0
  158. mostlyright/data_harness/review.py +2533 -0
  159. mostlyright/data_harness/rowset.py +283 -0
  160. mostlyright/data_harness/serving.py +1975 -0
  161. mostlyright/data_harness/serving_edge.py +590 -0
  162. mostlyright/data_harness/serving_http.py +1031 -0
  163. mostlyright/data_harness/session_probes.py +759 -0
  164. mostlyright/data_harness/signing.py +101 -0
  165. mostlyright/data_harness/source_discovery.py +898 -0
  166. mostlyright/data_harness/sources/__init__.py +209 -0
  167. mostlyright/data_harness/sources/_adapter_steps.py +213 -0
  168. mostlyright/data_harness/sources/adapters.py +1214 -0
  169. mostlyright/data_harness/sources/cadence.py +1428 -0
  170. mostlyright/data_harness/sources/cadence_emission.py +453 -0
  171. mostlyright/data_harness/sources/cadence_history.py +546 -0
  172. mostlyright/data_harness/sources/catalog/__init__.py +17 -0
  173. mostlyright/data_harness/sources/catalog/admission.py +477 -0
  174. mostlyright/data_harness/sources/catalog/authoring.py +1701 -0
  175. mostlyright/data_harness/sources/catalog/authoring_policy.py +701 -0
  176. mostlyright/data_harness/sources/catalog/authoring_shards.py +1217 -0
  177. mostlyright/data_harness/sources/catalog/bounded_io.py +231 -0
  178. mostlyright/data_harness/sources/catalog/channel.py +523 -0
  179. mostlyright/data_harness/sources/catalog/channel_client.py +296 -0
  180. mostlyright/data_harness/sources/catalog/contracts.py +825 -0
  181. mostlyright/data_harness/sources/catalog/coverage.py +137 -0
  182. mostlyright/data_harness/sources/catalog/delta.py +1340 -0
  183. mostlyright/data_harness/sources/catalog/embedding.py +532 -0
  184. mostlyright/data_harness/sources/catalog/entry_v2.py +1182 -0
  185. mostlyright/data_harness/sources/catalog/fill.py +3889 -0
  186. mostlyright/data_harness/sources/catalog/fill_partitions.py +459 -0
  187. mostlyright/data_harness/sources/catalog/fill_staging.py +1105 -0
  188. mostlyright/data_harness/sources/catalog/gating.py +374 -0
  189. mostlyright/data_harness/sources/catalog/generation_receipt.py +1607 -0
  190. mostlyright/data_harness/sources/catalog/harvest/__init__.py +7 -0
  191. mostlyright/data_harness/sources/catalog/harvest/ckan.py +384 -0
  192. mostlyright/data_harness/sources/catalog/harvest/datagov_v4.py +798 -0
  193. mostlyright/data_harness/sources/catalog/harvest/protocol.py +964 -0
  194. mostlyright/data_harness/sources/catalog/harvest/sdmx.py +445 -0
  195. mostlyright/data_harness/sources/catalog/harvest/stac.py +384 -0
  196. mostlyright/data_harness/sources/catalog/health.py +447 -0
  197. mostlyright/data_harness/sources/catalog/hosted_catalog.py +105 -0
  198. mostlyright/data_harness/sources/catalog/identity_history.py +1549 -0
  199. mostlyright/data_harness/sources/catalog/neural.py +1618 -0
  200. mostlyright/data_harness/sources/catalog/packed_catalog.py +2345 -0
  201. mostlyright/data_harness/sources/catalog/packed_retrieval.py +1517 -0
  202. mostlyright/data_harness/sources/catalog/packed_writer.py +2802 -0
  203. mostlyright/data_harness/sources/catalog/query_trace.py +1037 -0
  204. mostlyright/data_harness/sources/catalog/recommend.py +171 -0
  205. mostlyright/data_harness/sources/catalog/retrieval.py +230 -0
  206. mostlyright/data_harness/sources/catalog/retrieval_manifest.py +995 -0
  207. mostlyright/data_harness/sources/catalog/rights_decisions.py +254 -0
  208. mostlyright/data_harness/sources/catalog/sealed.py +560 -0
  209. mostlyright/data_harness/sources/catalog/search.py +230 -0
  210. mostlyright/data_harness/sources/catalog/streaming_delta.py +1097 -0
  211. mostlyright/data_harness/sources/catalog/update.py +891 -0
  212. mostlyright/data_harness/sources/collections.py +815 -0
  213. mostlyright/data_harness/sources/contracts.py +2223 -0
  214. mostlyright/data_harness/sources/deletion.py +761 -0
  215. mostlyright/data_harness/sources/fitness.py +162 -0
  216. mostlyright/data_harness/sources/governance.py +163 -0
  217. mostlyright/data_harness/sources/hosted.py +173 -0
  218. mostlyright/data_harness/sources/integration.py +218 -0
  219. mostlyright/data_harness/sources/range_reader.py +418 -0
  220. mostlyright/data_harness/sources/registry.py +514 -0
  221. mostlyright/data_harness/sources/rights_rule.py +59 -0
  222. mostlyright/data_harness/sources/source_cadence_vectors.v1.json +1 -0
  223. mostlyright/data_harness/sources/sports.py +521 -0
  224. mostlyright/data_harness/sources/stream.py +524 -0
  225. mostlyright/data_harness/sources/stream_connector.py +418 -0
  226. mostlyright/data_harness/sources/stream_recorder.py +1404 -0
  227. mostlyright/data_harness/studio_boundary.py +2019 -0
  228. mostlyright/data_harness/thin/__init__.py +37 -0
  229. mostlyright/data_harness/thin/acquire.py +1137 -0
  230. mostlyright/data_harness/thin/acquire_cancel.py +579 -0
  231. mostlyright/data_harness/thin/approvals.py +617 -0
  232. mostlyright/data_harness/thin/commands.py +406 -0
  233. mostlyright/data_harness/thin/download.py +194 -0
  234. mostlyright/data_harness/thin/narrative.py +589 -0
  235. mostlyright/data_harness/thin/parity.py +1070 -0
  236. mostlyright/data_harness/thin/propose.py +2759 -0
  237. mostlyright/data_harness/thin/research.py +1663 -0
  238. mostlyright/data_harness/thin/router.py +924 -0
  239. mostlyright/data_harness/thin/runs.py +519 -0
  240. mostlyright/data_harness/thin/session.py +281 -0
  241. mostlyright/data_harness/thin/stream.py +501 -0
  242. mostlyright/data_harness/thin/transport.py +187 -0
  243. mostlyright/data_harness/thin/vocabulary.py +368 -0
  244. mostlyright/data_harness/thin/workers.py +164 -0
  245. mostlyright/data_harness/ucum/TABLE-PIN.json +40 -0
  246. mostlyright/data_harness/ucum/ucum-subset.v1.json +632 -0
  247. mostlyright/data_harness/unit_flow.py +927 -0
  248. mostlyright/data_harness/units.py +572 -0
  249. mostlyright/data_harness/ux/__init__.py +9 -0
  250. mostlyright/data_harness/ux/approve.py +485 -0
  251. mostlyright/data_harness/ux/author_yaml.py +597 -0
  252. mostlyright/data_harness/ux/cloud_auth.py +447 -0
  253. mostlyright/data_harness/ux/commands/__init__.py +260 -0
  254. mostlyright/data_harness/ux/commands/approve.py +136 -0
  255. mostlyright/data_harness/ux/commands/auth.py +744 -0
  256. mostlyright/data_harness/ux/commands/author.py +79 -0
  257. mostlyright/data_harness/ux/commands/catalog_author.py +403 -0
  258. mostlyright/data_harness/ux/commands/catalog_fill.py +523 -0
  259. mostlyright/data_harness/ux/commands/catalog_harvest.py +545 -0
  260. mostlyright/data_harness/ux/commands/catalog_publish.py +1838 -0
  261. mostlyright/data_harness/ux/commands/catalog_search.py +71 -0
  262. mostlyright/data_harness/ux/commands/catalog_update.py +437 -0
  263. mostlyright/data_harness/ux/commands/deploy.py +134 -0
  264. mostlyright/data_harness/ux/commands/deploy_dataset.py +98 -0
  265. mostlyright/data_harness/ux/commands/deploy_plan.py +105 -0
  266. mostlyright/data_harness/ux/commands/deploy_status.py +104 -0
  267. mostlyright/data_harness/ux/commands/diff.py +74 -0
  268. mostlyright/data_harness/ux/commands/index.py +84 -0
  269. mostlyright/data_harness/ux/commands/inventory.py +47 -0
  270. mostlyright/data_harness/ux/commands/list_builds.py +143 -0
  271. mostlyright/data_harness/ux/commands/login.py +63 -0
  272. mostlyright/data_harness/ux/commands/peek.py +236 -0
  273. mostlyright/data_harness/ux/commands/plan_check.py +90 -0
  274. mostlyright/data_harness/ux/commands/preflight.py +97 -0
  275. mostlyright/data_harness/ux/commands/record.py +107 -0
  276. mostlyright/data_harness/ux/commands/review_setup.py +47 -0
  277. mostlyright/data_harness/ux/commands/search.py +440 -0
  278. mostlyright/data_harness/ux/commands/show.py +61 -0
  279. mostlyright/data_harness/ux/commands/whoami.py +37 -0
  280. mostlyright/data_harness/ux/credential_native.py +551 -0
  281. mostlyright/data_harness/ux/credential_store.py +1055 -0
  282. mostlyright/data_harness/ux/credentials.py +631 -0
  283. mostlyright/data_harness/ux/diffing.py +444 -0
  284. mostlyright/data_harness/ux/headline.py +671 -0
  285. mostlyright/data_harness/ux/hosted_acquisition.py +974 -0
  286. mostlyright/data_harness/ux/hosted_run_status.py +619 -0
  287. mostlyright/data_harness/ux/inventory.py +427 -0
  288. mostlyright/data_harness/ux/local_review.py +375 -0
  289. mostlyright/data_harness/ux/login.py +691 -0
  290. mostlyright/data_harness/ux/path_kind.py +147 -0
  291. mostlyright/data_harness/ux/peek.py +1000 -0
  292. mostlyright/data_harness/ux/plain_file.py +178 -0
  293. mostlyright/data_harness/ux/plan_check.py +311 -0
  294. mostlyright/data_harness/ux/preflight.py +918 -0
  295. mostlyright/data_harness/ux/readers.py +1124 -0
  296. mostlyright/data_harness/ux/remediation.py +2195 -0
  297. mostlyright/data_harness/ux/render.py +657 -0
  298. mostlyright/data_harness/ux/workload.py +1077 -0
  299. mostlyright/data_harness/viewer.py +3713 -0
  300. mostlyright/data_harness/visual_run/__init__.py +83 -0
  301. mostlyright/data_harness/visual_run/authoring.py +235 -0
  302. mostlyright/data_harness/visual_run/contracts.py +673 -0
  303. mostlyright/data_harness/visual_run/materialize.py +486 -0
  304. mostlyright/data_harness/visual_run/observations.py +874 -0
  305. mostlyright/data_harness/visual_run/query.py +259 -0
  306. mostlyright/data_harness/visual_run/reducer.py +280 -0
  307. mostlyright/data_harness/visual_run/sdk.py +892 -0
  308. mostlyright/data_harness/visual_run/store.py +584 -0
  309. mostlyright/data_harness/visual_run/transport.py +239 -0
  310. mostlyright/data_harness/watch.py +2999 -0
  311. mostlyright_data-0.9.0.dist-info/METADATA +607 -0
  312. mostlyright_data-0.9.0.dist-info/RECORD +314 -0
  313. mostlyright_data-0.9.0.dist-info/WHEEL +4 -0
  314. mostlyright_data-0.9.0.dist-info/entry_points.txt +12 -0
@@ -0,0 +1,924 @@
1
+ """Build a deployment request from a reviewed recipe and sealed build.
2
+
3
+ A deployment request says WHAT should run live — which recipe, which sealed build of it, which
4
+ target project, which caps, and how often. It is the seam between the terminal, where a human
5
+ approves a recipe, and Studio, where the dataset keeps itself current.
6
+
7
+ Three boundaries this module holds on purpose:
8
+
9
+ * **It describes, it does not run.** Nothing here schedules, fetches, or refreshes. That is
10
+ why the request is a versioned, digestible artifact rather than a function call: the two sides
11
+ can be written, reviewed, and changed independently, and the digest says whether they are talking
12
+ about the same thing.
13
+ * **It records caps, it does not enforce them.** A :class:`governors.CapPolicy` is a required
14
+ component of every request, never optional, so no live dataset can exist without a stated ceiling.
15
+ Turning that ceiling into a refused run happens at the hosted entrypoint, not here. No string this
16
+ module emits may claim a limit was applied.
17
+
18
+ A recorded cap is advisory and is not currently read. The admission check
19
+ (:func:`hosted_bootstrap.admission_ledger`) builds its ledger with no policy, so every workspace
20
+ is admitted against :class:`governors.CapPolicy` defaults whatever the request says. A request
21
+ tightened to ten runs a month does not make the eleventh run refuse. The gap is not closable from
22
+ this side: the worker job is shared by every workspace, so job-level configuration cannot carry a
23
+ per-workspace policy either. What would close it is a policy delivered per run in the validated
24
+ bootstrap response.
25
+ * **It never touches a credential.** :func:`build_deployment_request` accepts no key argument at
26
+ all, so a request cannot carry one. That is a property of the signature, not of a filter.
27
+
28
+ Refusals are quiet and typed. Every path that cannot produce a deployment plan raises
29
+ :class:`DeployRefused` with named reasons, or returns a refusing :class:`DeployPlanResult` — so a
30
+ caller has exactly one thing to catch and exactly one shape to render.
31
+ """
32
+
33
+ from __future__ import annotations
34
+
35
+ import base64
36
+ import hashlib
37
+ import re
38
+ from collections.abc import Mapping
39
+ from dataclasses import dataclass
40
+ from pathlib import Path
41
+ from typing import Any
42
+
43
+ from mostlyright.data_harness import (
44
+ canonical,
45
+ deploy_target,
46
+ deployment_evidence,
47
+ package_version,
48
+ pipeline,
49
+ recipe,
50
+ review,
51
+ )
52
+ from mostlyright.data_harness.governors import CapPolicy
53
+ from mostlyright.data_harness.pipeline import canonical_json_line_bytes
54
+ from mostlyright.data_harness.signing import (
55
+ SignerProviderError,
56
+ TrustedReviewAuthorityProvider,
57
+ resolve_review_authority,
58
+ )
59
+
60
+ #: The wire name of the artifact this module produces, in the convention used by
61
+ #: ``hosted_bootstrap.WORKER_BOOTSTRAP_REQUEST_VERSION``: one product-scoped name and one integer
62
+ #: version, bumped whenever the fields change meaning.
63
+ #:
64
+ #: v5 is v4 with two fields that may now be absent, and it is a new integer rather than a widened
65
+ #: v4 because a reader of v4 was entitled to assume both were always there: a request built for a
66
+ #: dataset the caller publishes itself carries no reviewed-Build binding under
67
+ #: ``bootstrap_evidence`` and no ``review_decision_digest``. See
68
+ #: :func:`build_deployment_request` for which of the three authorities produced a given request.
69
+ DEPLOYMENT_REQUEST_SCHEMA = "mostlyright-deployment-request.v6"
70
+
71
+ #: The wire name of what the command line renders, in the ``mr-data-*.v1`` style every other
72
+ #: command already emits.
73
+ DEPLOY_PLAN_RESULT_SCHEMA = "mr-data-deploy-plan.v6"
74
+ # Compatibility names for library callers while the truthful command name moves to deploy-plan.
75
+ GO_LIVE_RESULT_SCHEMA = DEPLOY_PLAN_RESULT_SCHEMA
76
+
77
+ MAX_NAME_LENGTH = 128
78
+ MAX_TEMPLATE_LENGTH = 256
79
+
80
+ _DIGEST = re.compile(r"\A[0-9a-f]{64}\Z")
81
+
82
+ # One plain line: no ASCII control/DEL and no Unicode boundary that ``str.splitlines`` treats as a
83
+ # new rendered line. U+0085, U+2028 and U+2029 are outside the ASCII ranges but can otherwise
84
+ # smuggle an attacker-authored line into terminal output or a log record.
85
+ _PLAIN_LINE = re.compile(r"\A[^\x00-\x1f\x7f\x85\u2028\u2029]+\Z")
86
+
87
+
88
+ class DeployRefused(RuntimeError):
89
+ """Go live was declined, with every reason named. Never a partial deployment.
90
+
91
+ A refusal is an expected outcome, not a defect: an unreviewed build, an ambiguous target, or a
92
+ refresh whose predecessor was not named are all things the operator can act on. Every refusing
93
+ path in this module raises this one type, so the command line has exactly one thing to catch.
94
+ """
95
+
96
+ def __init__(self, *reasons: str) -> None:
97
+ cleaned = tuple(reason.strip() for reason in reasons if reason and reason.strip())
98
+ if not cleaned:
99
+ raise ValueError("a refusal must name at least one reason")
100
+ self.reasons: tuple[str, ...] = cleaned
101
+ super().__init__("; ".join(cleaned))
102
+
103
+
104
+ def _plain_name(value: Any, label: str) -> str:
105
+ if not isinstance(value, str) or not value.strip():
106
+ raise DeployRefused(f"the request does not name {label}")
107
+ text = value.strip()
108
+ if len(text) > MAX_NAME_LENGTH:
109
+ raise DeployRefused(f"{label} is longer than {MAX_NAME_LENGTH} characters")
110
+ if _PLAIN_LINE.fullmatch(text) is None:
111
+ raise DeployRefused(f"{label} must be one plain line of text")
112
+ return text
113
+
114
+
115
+ @dataclass(frozen=True)
116
+ class Cadence:
117
+ """How often the live dataset should look for new data, as a plain descriptor.
118
+
119
+ This module does not interpret either field. ``label`` is the operator's name for
120
+ the rhythm ("daily", "hourly", "each model run"); ``cycle_template`` is an optional template
121
+ string naming the cycle variable a source publishes on. This request records both as
122
+ uninterpreted descriptors.
123
+
124
+ ``FrozenRecipe.refresh_policy`` is different. That field holds integer drift
125
+ bounds (``min_row_count_percent``, ``max_row_count_percent``, ``allow_column_additions``); using
126
+ it as a schedule would put meaningless numbers in the request.
127
+ """
128
+
129
+ label: str
130
+ cycle_template: str | None = None
131
+
132
+ def __post_init__(self) -> None:
133
+ _plain_name(self.label, "a cadence")
134
+ if self.cycle_template is not None:
135
+ if not isinstance(self.cycle_template, str) or not self.cycle_template.strip():
136
+ raise DeployRefused("the cadence cycle template must be text, or absent")
137
+ if len(self.cycle_template) > MAX_TEMPLATE_LENGTH:
138
+ raise DeployRefused(
139
+ f"the cadence cycle template is longer than {MAX_TEMPLATE_LENGTH} characters"
140
+ )
141
+ if _PLAIN_LINE.fullmatch(self.cycle_template) is None:
142
+ raise DeployRefused("the cadence cycle template must be one plain line of text")
143
+
144
+ def to_dict(self) -> dict[str, Any]:
145
+ """Return the descriptor as plain data, uninterpreted."""
146
+
147
+ return {"label": self.label.strip(), "cycle_template": self.cycle_template}
148
+
149
+
150
+ @dataclass(frozen=True)
151
+ class DeploymentRequest:
152
+ """One reviewed recipe, one named target, one set of caps: what should run live.
153
+
154
+ The request carries the recipe's identity and the digest of the exact build that was reviewed —
155
+ never the recipe body. Two reasons: the body is large and already immutably stored under its
156
+ digest, and a request that repeated it could disagree with it. A digest cannot disagree with
157
+ what it names.
158
+
159
+ :meth:`digest` is a content digest over the whole request, so any alteration between building it
160
+ and acting on it is detectable by whoever acts on it.
161
+
162
+ ``review_decision_digest`` and ``bootstrap_evidence["reviewed_build"]`` are the two things a
163
+ request may not have. They are present together or absent together, and absent means what it
164
+ says: this deployment carries no external review attestation, and nothing signed it. Every
165
+ other field is present on every request, including on that one.
166
+ """
167
+
168
+ schema: str
169
+ recipe_id: str
170
+ recipe_version: int
171
+ recipe_digest: str
172
+ candidate_digest: str
173
+ review_decision_digest: str | None
174
+ acquisition_bundle_digest: str
175
+ execution_plan_digest: str
176
+ canonical_recipe_json: str
177
+ recipe_schema_version: str
178
+ bootstrap_evidence: dict[str, Any]
179
+ dataset_name: str
180
+ target: deploy_target.DeployTarget | None
181
+ caps: CapPolicy
182
+ cadence: Cadence
183
+
184
+ def __post_init__(self) -> None:
185
+ if self.schema != DEPLOYMENT_REQUEST_SCHEMA:
186
+ raise DeployRefused(f"a deployment request must be {DEPLOYMENT_REQUEST_SCHEMA}")
187
+ _plain_name(self.recipe_id, "a recipe")
188
+ _plain_name(self.dataset_name, "a live dataset")
189
+ if type(self.recipe_version) is not int or self.recipe_version < 1:
190
+ raise DeployRefused("the recipe version must be a whole number of at least 1")
191
+ for name in (
192
+ "recipe_digest",
193
+ "candidate_digest",
194
+ "acquisition_bundle_digest",
195
+ "execution_plan_digest",
196
+ ):
197
+ value = getattr(self, name)
198
+ if not isinstance(value, str) or _DIGEST.fullmatch(value) is None:
199
+ raise DeployRefused(f"{name} must be a 64-character lowercase hex digest")
200
+ # The one digest that may be absent, and only absent: a deployment that carries no
201
+ # reviewed-Build binding has no signed review decision to name. Absent is written as
202
+ # ``None`` rather than as an empty or zero digest, because a reader that treats a
203
+ # 64-character field as present would read either of those as a decision that exists.
204
+ if self.review_decision_digest is not None and (
205
+ not isinstance(self.review_decision_digest, str)
206
+ or _DIGEST.fullmatch(self.review_decision_digest) is None
207
+ ):
208
+ raise DeployRefused(
209
+ "review_decision_digest must be a 64-character lowercase hex digest, or absent"
210
+ )
211
+ if self.target is not None and not isinstance(self.target, deploy_target.DeployTarget):
212
+ raise DeployRefused("the target must be one fully resolved go-live target, or absent")
213
+ if not isinstance(self.caps, CapPolicy):
214
+ raise DeployRefused("the caps must be a cap policy; a live dataset is never uncapped")
215
+ if not isinstance(self.cadence, Cadence):
216
+ raise DeployRefused("the cadence must be a cadence descriptor")
217
+ try:
218
+ canonical_recipe = canonical.parse_canonical_json(
219
+ self.canonical_recipe_json.encode("utf-8")
220
+ )
221
+ except canonical.CanonicalJSONError as error:
222
+ raise DeployRefused("canonical_recipe_json is not exact canonical JSON") from error
223
+ if (
224
+ not isinstance(canonical_recipe, dict)
225
+ or canonical_recipe.get("schema_version") != self.recipe_schema_version
226
+ or canonical_recipe.get("recipe_id") != self.recipe_id
227
+ or canonical_recipe.get("recipe_version") != self.recipe_version
228
+ or hashlib.sha256(self.canonical_recipe_json.encode("utf-8")).hexdigest()
229
+ != self.recipe_digest
230
+ ):
231
+ raise DeployRefused("canonical_recipe_json differs from the reviewed Recipe identity")
232
+ if not isinstance(self.bootstrap_evidence, dict):
233
+ raise DeployRefused("bootstrap_evidence must bind the exact reviewed Build")
234
+
235
+ def to_dict(self) -> dict[str, Any]:
236
+ """Return the request as plain JSON-safe data. Carries digests and names, never a key."""
237
+
238
+ return {
239
+ "schema": self.schema,
240
+ "recipe_id": self.recipe_id,
241
+ "recipe_version": self.recipe_version,
242
+ "recipe_digest": self.recipe_digest,
243
+ "candidate_digest": self.candidate_digest,
244
+ "review_decision_digest": self.review_decision_digest,
245
+ "acquisition_bundle_digest": self.acquisition_bundle_digest,
246
+ "execution_plan_digest": self.execution_plan_digest,
247
+ "canonical_recipe_json": self.canonical_recipe_json,
248
+ "recipe_schema_version": self.recipe_schema_version,
249
+ "bootstrap_evidence": self.bootstrap_evidence,
250
+ "dataset_name": self.dataset_name,
251
+ "target": None if self.target is None else self.target.to_dict(),
252
+ "caps": {
253
+ "max_runs_per_month": self.caps.max_runs_per_month,
254
+ "max_concurrent_runs": self.caps.max_concurrent_runs,
255
+ "max_monthly_cost_units": self.caps.max_monthly_cost_units,
256
+ },
257
+ "cadence": self.cadence.to_dict(),
258
+ }
259
+
260
+ def digest(self) -> str:
261
+ """Return the canonical content digest of the whole request."""
262
+
263
+ return canonical.canonical_sha256(self.to_dict())
264
+
265
+ def to_summary_dict(self) -> dict[str, Any]:
266
+ """Return printable handoff coordinates without replaying executable Recipe bytes."""
267
+
268
+ bootstrap = self.bootstrap_evidence
269
+ return {
270
+ "schema": self.schema,
271
+ "recipe_id": self.recipe_id,
272
+ "recipe_version": self.recipe_version,
273
+ "recipe_digest": self.recipe_digest,
274
+ "recipe_schema_version": self.recipe_schema_version,
275
+ "candidate_digest": self.candidate_digest,
276
+ "review_decision_digest": self.review_decision_digest,
277
+ "acquisition_bundle_digest": self.acquisition_bundle_digest,
278
+ "execution_plan_digest": self.execution_plan_digest,
279
+ "bootstrap_table_digest": bootstrap["bootstrap_table_digest"],
280
+ "bootstrap_verification_digest": bootstrap["bootstrap_verification_digest"],
281
+ "artifact_manifest_digest": bootstrap["artifact_manifest_digest"],
282
+ "lineage_digest": bootstrap["lineage_digest"],
283
+ "dataset_name": self.dataset_name,
284
+ "target": None if self.target is None else self.target.to_dict(),
285
+ "caps": {
286
+ "max_runs_per_month": self.caps.max_runs_per_month,
287
+ "max_concurrent_runs": self.caps.max_concurrent_runs,
288
+ "max_monthly_cost_units": self.caps.max_monthly_cost_units,
289
+ },
290
+ "cadence": self.cadence.to_dict(),
291
+ }
292
+
293
+
294
+ def _refusal_for(error: recipe.RecipeError, candidate_run_dir: Path) -> str:
295
+ """Turn a typed recipe failure into one plain sentence the operator can act on."""
296
+
297
+ if error.code == "RECIPE_PREDECESSOR" and error.path == "previous_run_dir":
298
+ # A different next action from every other refusal: the build is fine, the operator simply
299
+ # has not said which live version this one follows. Paraphrasing it as a verification
300
+ # failure would send them off to rebuild something that is not broken.
301
+ return (
302
+ f"the run directory {candidate_run_dir} holds a refresh of a dataset that is already "
303
+ "live, so the previous run directory — the version this one follows — has to be "
304
+ "named: pass it with --previous-run-dir [RECIPE_PREDECESSOR]"
305
+ )
306
+ # The failing part and the code travel; the verifier's own free-text detail does not. That
307
+ # detail is written in the engineering vocabulary ("is not a recipe candidate"), and these
308
+ # reasons are printed straight to an operator. The code is the precise, greppable handle, and
309
+ # `mr-data recipe-verify` on the same directory prints the full engineering text.
310
+ return (
311
+ f"the run directory {candidate_run_dir} does not hold a reviewed recipe build that can go "
312
+ f"live: {error.path} [{error.code}]"
313
+ )
314
+
315
+
316
+ def _predecessor_forbidden_refusal(candidate_run_dir: Path) -> str:
317
+ """The other half of the predecessor rule, in the same voice as the half above.
318
+
319
+ :func:`recipe.execute_recipe` refuses a predecessor to an execution whose mode is ``initial``,
320
+ so a Build produced that way never read the predecessor's bytes and its version chain was never
321
+ authenticated against them. :func:`recipe.verify_recipe_candidate` enforces only the "is
322
+ required" direction and silently ignores a predecessor handed to an initial run -- which is
323
+ exactly how a Build with no authenticated predecessor could otherwise reach a deployment that
324
+ updates the predecessor's Dataset. This is that direction, stated where the request is built,
325
+ so both commands that build one get it and neither ignores what it was handed.
326
+ """
327
+
328
+ return (
329
+ f"the run directory {candidate_run_dir} holds an initial run, which starts a dataset "
330
+ "rather than following one, so there is no previous run directory for it to name: rerun "
331
+ "it without --previous-run-dir, or build the version that follows the live one as a "
332
+ "refresh of it [RECIPE_PREDECESSOR]"
333
+ )
334
+
335
+
336
+ def _require_passing_local_checks(candidate_run_dir: Path) -> None:
337
+ """Refuse a Build whose own deterministic checks do not pass, naming every one that failed.
338
+
339
+ This is the gate on the path that carries no reviewed-Build binding. The checks are exactly the
340
+ ones :func:`ux.local_review.run_local_review` reports -- the seal replay, the recorded quality
341
+ checks under the names the Build recorded them with, the join row multiplier, lineage
342
+ completeness, and the per-source rights findings -- so what refuses a deployment here is the
343
+ same set of facts ``mr-data review --local`` prints, read from the same single verified
344
+ reading. Nothing is written, and no decision is sealed: this module cannot approve a Build and
345
+ does not claim to.
346
+
347
+ Two things this deliberately does not do. It does not stand in for the external two-reviewer
348
+ attestation, which is a judgement no machine makes. And it does not replace anything the caller
349
+ already ran: :func:`build_deployment_request` has verified the sealed Build, its recorded drift
350
+ report, and its acquisition evidence before it gets here.
351
+
352
+ The import is deferred rather than module-level. ``ux`` is the layer above this one and imports
353
+ freely from it; today nothing under ``ux`` imports this module back, and taking the dependency
354
+ at call time rather than at import time keeps that from being a thing a later edit has to know.
355
+ """
356
+
357
+ from mostlyright.data_harness.ux.local_review import run_local_review
358
+
359
+ try:
360
+ local = run_local_review(Path(candidate_run_dir))
361
+ except pipeline.BuildError as error:
362
+ # Unreachable through `build_deployment_request`, which has already read this directory as
363
+ # a sealed recipe Build. Translated rather than left to escape, because a caller of this
364
+ # module catches `DeployRefused` and nothing else.
365
+ raise DeployRefused(
366
+ f"the deterministic checks could not be run on {candidate_run_dir} [{error.finding_id}]"
367
+ ) from error
368
+ if local.passed:
369
+ return
370
+ failed = ", ".join(check.name for check in local.checks if not check.passed)
371
+ raise DeployRefused(
372
+ f"the build in {candidate_run_dir} did not pass its own deterministic checks, so it "
373
+ f"cannot go live: {failed} [LOCAL_CHECKS_FAILED]"
374
+ )
375
+
376
+
377
+ #: The exact key set one reviewed-Build binding carries. A journal that offers anything else is
378
+ #: not the document this function wrote, so it is refused by name rather than digested silently.
379
+ _REVIEWED_BUILD_KEYS = frozenset(
380
+ {
381
+ "schema_version",
382
+ "candidate_digest",
383
+ "acquisition_bundle_digest",
384
+ "review_key_id",
385
+ "review_key_version",
386
+ "acquisition_bundle",
387
+ "decision",
388
+ "binding_digest",
389
+ "binding_signature",
390
+ }
391
+ )
392
+
393
+
394
+ def _adoptable_reviewed_build(
395
+ value: Mapping[str, Any],
396
+ candidate_run_dir: Path,
397
+ ) -> dict[str, Any]:
398
+ """Read one journaled reviewed-Build binding, or refuse it in the word that names the fault.
399
+
400
+ Only the shape is settled here. Whether the binding describes *this* run directory is settled
401
+ where the run directory's own recomputed digests are in hand, so a moved binding is refused as
402
+ a moved binding rather than as an unreadable one.
403
+ """
404
+
405
+ if not isinstance(value, Mapping) or set(value) != _REVIEWED_BUILD_KEYS:
406
+ raise DeployRefused(
407
+ f"the journaled reviewed Build for {candidate_run_dir} is not the agreed shape "
408
+ "[REVIEW_BINDING_UNREADABLE]"
409
+ )
410
+ decision = value["decision"]
411
+ if not isinstance(decision, Mapping):
412
+ raise DeployRefused(
413
+ f"the journaled reviewed Build for {candidate_run_dir} carries no signed decision "
414
+ "[REVIEW_BINDING_UNREADABLE]"
415
+ )
416
+ adopted = {
417
+ "candidate_digest": value["candidate_digest"],
418
+ "acquisition_bundle_digest": value["acquisition_bundle_digest"],
419
+ "review_key_id": value["review_key_id"],
420
+ "review_key_version": value["review_key_version"],
421
+ "decision": decision,
422
+ "decision_digest": decision.get("decision_digest"),
423
+ "binding_digest": value["binding_digest"],
424
+ "binding_signature": value["binding_signature"],
425
+ }
426
+ for name in (
427
+ "candidate_digest",
428
+ "acquisition_bundle_digest",
429
+ "decision_digest",
430
+ "binding_digest",
431
+ ):
432
+ if not isinstance(adopted[name], str) or _DIGEST.fullmatch(adopted[name]) is None:
433
+ raise DeployRefused(
434
+ f"the journaled reviewed Build for {candidate_run_dir} does not name a "
435
+ f"{name.replace('_', ' ')} [REVIEW_BINDING_UNREADABLE]"
436
+ )
437
+ if (
438
+ value["schema_version"] != "mostlyright-reviewed-build-bootstrap.v1"
439
+ or not isinstance(adopted["binding_signature"], str)
440
+ or not adopted["binding_signature"]
441
+ or decision.get("status") != "reviewed_candidate"
442
+ ):
443
+ raise DeployRefused(
444
+ f"the journaled reviewed Build for {candidate_run_dir} is not a signed reviewed "
445
+ "decision this Build can go live on [REVIEW_BINDING_UNREADABLE]"
446
+ )
447
+ return adopted
448
+
449
+
450
+ def _adopted_binding_signature(
451
+ adopted: Mapping[str, Any],
452
+ binding_payload: Mapping[str, Any],
453
+ binding_digest: str,
454
+ ) -> str:
455
+ """Return the journaled signature, once every value it covers was recomputed and agreed.
456
+
457
+ The signature is the one thing that cannot be recomputed without the review session that made
458
+ it. Everything it was made over is recomputed here from the run directory, and the binding
459
+ digest is recomputed from those recomputed values, so a signature is carried across only when
460
+ it still stands over exactly what this Build says today.
461
+ """
462
+
463
+ covered = (
464
+ "candidate_digest",
465
+ "acquisition_bundle_digest",
466
+ "review_key_id",
467
+ "review_key_version",
468
+ "decision_digest",
469
+ )
470
+ if any(adopted[name] != binding_payload[name] for name in covered) or (
471
+ adopted["binding_digest"] != binding_digest
472
+ ):
473
+ raise DeployRefused(
474
+ "the journaled reviewed Build binds different evidence than this Build produced "
475
+ "[REVIEW_BINDING_MISMATCH]"
476
+ )
477
+ signature: str = adopted["binding_signature"]
478
+ return signature
479
+
480
+
481
+ def build_deployment_request(
482
+ *,
483
+ candidate_run_dir: Path,
484
+ previous_run_dir: Path | None = None,
485
+ dataset_name: str,
486
+ target: deploy_target.DeployTarget | None,
487
+ caps: CapPolicy,
488
+ cadence: Cadence,
489
+ review_authority_provider: TrustedReviewAuthorityProvider | None,
490
+ journaled_reviewed_build: Mapping[str, Any] | None = None,
491
+ require_reviewed_build: bool = True,
492
+ ) -> DeploymentRequest:
493
+ """Build the request that turns one verified, sealed run directory into a live dataset.
494
+
495
+ The input is a sealed run directory, not a Recipe file. Deployment uses work that
496
+ has already been produced and reviewed, so the thing being promoted must be the exact reviewed
497
+ artifact. The protected review decision and :func:`recipe.verify_recipe_candidate` are both
498
+ verified and must name the same candidate. Nothing that skipped review can reach this
499
+ function's return statement. The resulting request also names the exact signed decision and
500
+ the manifest-last acquisition-evidence bundle derived under ``candidate_run_dir``.
501
+
502
+ ``previous_run_dir`` is required whenever the build is a refresh or a backfill of a dataset that
503
+ is already live, because the predecessor is authenticated as part of verification, and it is
504
+ refused for an initial run, which follows nothing. Both execution modes are supported; a missing
505
+ predecessor is refused as a missing predecessor, not as a refusal of refresh mode, and a
506
+ predecessor handed to an initial run is refused rather than ignored.
507
+
508
+ Three authorities can stand behind the resulting request, and they are tried in that order.
509
+ A **journaled binding** is adopted when one is passed. Otherwise a **coordinator review
510
+ session**, when ``review_authority_provider`` is not None, verifies the installed two-reviewer
511
+ decision and signs a fresh binding — the path every Mostly Right-published dataset takes, and
512
+ the preferred one whenever a session exists. Otherwise, and only when the caller has explicitly
513
+ passed ``require_reviewed_build=False``, the request is built with **no reviewed-Build binding
514
+ at all**: no signature, no signing key, and no ``reviewed_build`` under ``bootstrap_evidence``.
515
+
516
+ That third path is for the customer publishing their own dataset. Nothing about the Build is
517
+ taken on trust there: the sealed Build is verified, its own recorded checks must have passed,
518
+ its acquisition evidence is closed and rebound, the whole reading is repeated under a verified
519
+ snapshot, and :func:`ux.local_review.run_local_review` reruns the deterministic checks —
520
+ exactly what ``mr-data review --local`` reports — and refuses if any one of them fails. What is
521
+ absent is the external two-reviewer attestation, not the verification of the Build. The
522
+ authority that admits such a proposal is the authenticated workspace human on the far side, and
523
+ the in-product approval remains the gate before anything activates.
524
+
525
+ ``require_reviewed_build`` defaults to True, so a caller that simply has no provider to pass
526
+ still gets the refusal it has always got. Dropping the binding is something a caller says, not
527
+ something it can fall into.
528
+
529
+ ``journaled_reviewed_build`` is the reviewed-Build binding an earlier, session-backed build of
530
+ this same request already minted and journaled — the one a paused hosted deployment records
531
+ before it stops for human approval. Passing it **adopts** that binding instead of minting a new
532
+ one, which is what lets ``mr-data deploy --resume`` run from an ordinary terminal long after the
533
+ coordinator's review session has ended. Adoption is not trust: every field the binding covers is
534
+ recomputed from ``candidate_run_dir`` here and must agree with it, the binding digest is
535
+ recomputed from those recomputed fields, and the resulting request is byte-identical to the one
536
+ the session-backed build produced — so the deployment identity the caller checks the journal
537
+ against still names this exact handoff. Only the signature itself is carried over, because the
538
+ key that made it lives in the session that has closed. Without it, nothing changes: the binding
539
+ is minted from ``review_authority_provider`` exactly as before.
540
+
541
+ This function accepts no key, token, or credential of any kind, which is why a request cannot
542
+ carry one.
543
+
544
+ Raises:
545
+ DeployRefused: when the run directory is not a reviewed recipe build, when its predecessor
546
+ was not named, when the build did not pass its own checks, when a deterministic local
547
+ check fails on a request that carries no reviewed-Build binding, when an adopted
548
+ binding is not the agreed shape or names another Build, or when any named value is not
549
+ the agreed shape.
550
+ """
551
+
552
+ try:
553
+ inspection, embedded = recipe.verify_recipe_candidate(
554
+ Path(candidate_run_dir),
555
+ # Always passed explicitly: this module's boundary is where the default lives, so the
556
+ # verifier's own required keyword is never silently omitted.
557
+ previous_run_dir=None if previous_run_dir is None else Path(previous_run_dir),
558
+ )
559
+ except recipe.RecipeError as error:
560
+ raise DeployRefused(_refusal_for(error, candidate_run_dir)) from error
561
+ except pipeline.BuildError as error:
562
+ if error.finding_id == "VALIDATION_POLICY_MISMATCH":
563
+ # Recipe verification is still entirely local at this point: no Studio request,
564
+ # approval, or deployment journal has happened. Preserve the verifier's two digests
565
+ # and exact re-export remedy instead of collapsing them to a finding id after the
566
+ # human would otherwise be asked to approve an unschedulable Recipe (#235).
567
+ raise DeployRefused(
568
+ "this deployment was refused before Studio staging, approval, or activation: "
569
+ f"{error} [{error.finding_id}]"
570
+ ) from error
571
+ # A directory with no sealed build in it at all is refused one layer below the recipe
572
+ # rules, so the greppable code here is the build gate's finding id rather than a recipe
573
+ # code. The operator's next action is the same either way: point at a reviewed build.
574
+ raise DeployRefused(
575
+ f"the run directory {candidate_run_dir} does not hold a reviewed recipe build that "
576
+ f"can go live [{error.finding_id}]"
577
+ ) from error
578
+ except (canonical.CanonicalJSONError, OSError) as error:
579
+ raise DeployRefused(
580
+ f"the run directory {candidate_run_dir} could not be read as a reviewed recipe build "
581
+ f"[{type(error).__name__}]"
582
+ ) from error
583
+
584
+ drift_report = embedded["drift_report"]
585
+ if drift_report.status != "passed":
586
+ # Release eligibility is what ``mr-data recipe-verify`` reports, and a version that failed
587
+ # its own checks is not a version to make self-updating.
588
+ codes = ", ".join(sorted({item.code for item in drift_report.findings})) or drift_report
589
+ raise DeployRefused(
590
+ f"the build in {candidate_run_dir} did not pass its own checks, so it cannot go live: "
591
+ f"{codes}"
592
+ )
593
+
594
+ if previous_run_dir is not None and embedded["execution"].mode == "initial":
595
+ raise DeployRefused(_predecessor_forbidden_refusal(candidate_run_dir))
596
+
597
+ unattested = (
598
+ journaled_reviewed_build is None
599
+ and review_authority_provider is None
600
+ and not require_reviewed_build
601
+ )
602
+ if unattested:
603
+ # The customer's own dataset: no review session, no trust anchor, no key, no signature.
604
+ # Everything above this line has already verified the sealed Build; the deterministic
605
+ # checks a person would read out of `mr-data review --local` are rerun here so the
606
+ # cheapest way to reach this branch is not also the way to skip them.
607
+ _require_passing_local_checks(Path(candidate_run_dir))
608
+ adopted = None
609
+ review_candidate_digest = None
610
+ review_decision_digest = None
611
+ decision = None
612
+ elif journaled_reviewed_build is None:
613
+ try:
614
+ review_result = review.verify_enrolled_review_decision(
615
+ Path(candidate_run_dir),
616
+ review_authority_provider=review_authority_provider,
617
+ )
618
+ except review.ReviewError as error:
619
+ code = error.code or "REVIEW_INVALID"
620
+ raise DeployRefused(
621
+ f"the run directory {candidate_run_dir} does not have an exact protected review "
622
+ f"that can authorize this Build [{code}]"
623
+ ) from error
624
+ except SignerProviderError as error:
625
+ raise DeployRefused(
626
+ f"the protected review authority for {candidate_run_dir} is unavailable "
627
+ "[REVIEW_AUTHORITY_UNAVAILABLE]"
628
+ ) from error
629
+ if review_result.status != "reviewed_candidate":
630
+ raise DeployRefused(
631
+ f"the signed review for {candidate_run_dir} requires fixes, so this Build cannot "
632
+ "go live [REVIEW_FIXES_REQUIRED]"
633
+ )
634
+ adopted = None
635
+ review_candidate_digest = review_result.candidate_digest
636
+ review_decision_digest = review_result.decision_digest
637
+ decision = dict(review_result.decision_document)
638
+ else:
639
+ adopted = _adoptable_reviewed_build(journaled_reviewed_build, candidate_run_dir)
640
+ review_candidate_digest = adopted["candidate_digest"]
641
+ decision = dict(adopted["decision"])
642
+ review_decision_digest = adopted["decision_digest"]
643
+
644
+ frozen = embedded["recipe"]
645
+ execution = embedded["execution"]
646
+ if (
647
+ review_candidate_digest is not None
648
+ and review_candidate_digest != inspection.candidate_digest
649
+ ):
650
+ raise DeployRefused(
651
+ f"the signed review for {candidate_run_dir} names a different Build "
652
+ "[REVIEW_CANDIDATE_MISMATCH]"
653
+ )
654
+ try:
655
+ acquisition_bundle = deployment_evidence.persist_acquisition_bundle(
656
+ Path(candidate_run_dir),
657
+ previous_run_dir=None if previous_run_dir is None else Path(previous_run_dir),
658
+ )
659
+ except deployment_evidence.DeploymentEvidenceError as error:
660
+ raise DeployRefused(
661
+ f"the acquisition evidence for {candidate_run_dir} could not be closed [{error.code}]"
662
+ ) from error
663
+ except (
664
+ recipe.RecipeError,
665
+ pipeline.BuildError,
666
+ canonical.CanonicalJSONError,
667
+ OSError,
668
+ ) as error:
669
+ raise DeployRefused(
670
+ f"the acquisition evidence for {candidate_run_dir} could not be closed "
671
+ f"[{type(error).__name__}]"
672
+ ) from error
673
+ if (
674
+ acquisition_bundle.candidate_digest != inspection.candidate_digest
675
+ or acquisition_bundle.recipe_digest != execution.recipe_digest
676
+ or acquisition_bundle.execution_digest != execution.digest
677
+ ):
678
+ raise DeployRefused(
679
+ f"the acquisition evidence for {candidate_run_dir} names a different reviewed Build "
680
+ "[DEPLOYMENT_EVIDENCE_BINDING]"
681
+ )
682
+ try:
683
+ bundle_document = deployment_evidence.load_acquisition_bundle_document(
684
+ Path(candidate_run_dir),
685
+ acquisition_bundle.bundle_digest,
686
+ previous_run_dir=None if previous_run_dir is None else Path(previous_run_dir),
687
+ )
688
+ with pipeline.open_verified_snapshot(Path(candidate_run_dir)) as snapshot:
689
+ repeated_inspection, repeated_embedded = recipe.verify_recipe_candidate(
690
+ Path(candidate_run_dir),
691
+ previous_run_dir=None if previous_run_dir is None else Path(previous_run_dir),
692
+ _snapshot=snapshot,
693
+ )
694
+ manifest_raw = snapshot.member_bytes("manifest.json")
695
+ plan_raw = snapshot.member_bytes("plan.json")
696
+ lineage_raw = snapshot.member_bytes("evidence/lineage.json")
697
+ snapshot.validate()
698
+ except (
699
+ deployment_evidence.DeploymentEvidenceError,
700
+ recipe.RecipeError,
701
+ pipeline.BuildError,
702
+ ) as error:
703
+ raise DeployRefused(
704
+ f"the Studio handoff for {candidate_run_dir} could not authenticate its Build evidence "
705
+ f"[{type(error).__name__}]"
706
+ ) from error
707
+ if (
708
+ repeated_inspection.candidate_digest != inspection.candidate_digest
709
+ or repeated_embedded["recipe"].digest != frozen.digest
710
+ ):
711
+ raise DeployRefused(
712
+ f"the Studio handoff for {candidate_run_dir} changed during preparation "
713
+ "[DEPLOYMENT_EVIDENCE_RACE]"
714
+ )
715
+ reviewed_build: dict[str, Any] | None = None
716
+ if decision is not None:
717
+ try:
718
+ review_key_id = decision["verifier_principal"]
719
+ if not isinstance(review_key_id, str) or not review_key_id:
720
+ raise ValueError("review verifier principal is absent")
721
+ binding_payload = {
722
+ "schema_version": "mostlyright-reviewed-build-bootstrap.v1",
723
+ "candidate_digest": review_candidate_digest,
724
+ "acquisition_bundle_digest": acquisition_bundle.bundle_digest,
725
+ "review_key_id": review_key_id,
726
+ "review_key_version": 1,
727
+ "decision_digest": review_decision_digest,
728
+ }
729
+ binding_digest = hashlib.sha256(canonical_json_line_bytes(binding_payload)).hexdigest()
730
+ binding_body = {**binding_payload, "binding_digest": binding_digest}
731
+ if adopted is None:
732
+ authority = resolve_review_authority(review_authority_provider)
733
+ verifier_signer = authority.verifier_signer()
734
+ binding_signature = base64.b64encode(
735
+ verifier_signer.sign(canonical_json_line_bytes(binding_body))
736
+ ).decode("ascii")
737
+ else:
738
+ binding_signature = _adopted_binding_signature(
739
+ adopted, binding_payload, binding_digest
740
+ )
741
+ except (KeyError, TypeError, ValueError, SignerProviderError) as error:
742
+ raise DeployRefused(
743
+ f"the protected review authority for {candidate_run_dir} could not bind the Studio "
744
+ "handoff [REVIEW_AUTHORITY_UNAVAILABLE]"
745
+ ) from error
746
+ reviewed_build = {
747
+ "schema_version": binding_payload["schema_version"],
748
+ "candidate_digest": binding_payload["candidate_digest"],
749
+ "acquisition_bundle_digest": binding_payload["acquisition_bundle_digest"],
750
+ "review_key_id": binding_payload["review_key_id"],
751
+ "review_key_version": binding_payload["review_key_version"],
752
+ "acquisition_bundle": bundle_document,
753
+ "decision": decision,
754
+ "binding_digest": binding_digest,
755
+ "binding_signature": binding_signature,
756
+ }
757
+ canonical_recipe_json = canonical.canonical_json_bytes(frozen.to_dict()).decode("utf-8")
758
+ bootstrap_evidence = {
759
+ "bootstrap_table_digest": inspection.table_sha256,
760
+ "bootstrap_verification_digest": inspection.validation_policy_digest,
761
+ "artifact_manifest_digest": hashlib.sha256(manifest_raw).hexdigest(),
762
+ "lineage_digest": hashlib.sha256(lineage_raw).hexdigest(),
763
+ "harness_version": package_version(),
764
+ # Present exactly when a binding was adopted or minted. Absent, rather than present and
765
+ # empty: a caller reading a key that is there has no way to tell an unsigned placeholder
766
+ # from a signature it should have checked. `bundle_document` still travels to Studio inside
767
+ # it on the two attested paths; on this one the bundle is read back out of the run
768
+ # directory under `acquisition_bundle_digest`, which is a field of the request itself.
769
+ **({} if reviewed_build is None else {"reviewed_build": reviewed_build}),
770
+ }
771
+ return DeploymentRequest(
772
+ schema=DEPLOYMENT_REQUEST_SCHEMA,
773
+ recipe_id=frozen.recipe_id,
774
+ recipe_version=frozen.recipe_version,
775
+ recipe_digest=execution.recipe_digest,
776
+ candidate_digest=inspection.candidate_digest,
777
+ review_decision_digest=review_decision_digest,
778
+ acquisition_bundle_digest=acquisition_bundle.bundle_digest,
779
+ execution_plan_digest=hashlib.sha256(plan_raw).hexdigest(),
780
+ canonical_recipe_json=canonical_recipe_json,
781
+ recipe_schema_version=frozen.schema_version,
782
+ bootstrap_evidence=bootstrap_evidence,
783
+ dataset_name=_plain_name(dataset_name, "a live dataset"),
784
+ target=target,
785
+ caps=caps,
786
+ cadence=cadence,
787
+ )
788
+
789
+
790
+ @dataclass(frozen=True)
791
+ class DeployPlanResult:
792
+ """The verdict on one deployment-planning attempt: a request, or why there is none.
793
+
794
+ Invariant: ``ok`` is true if and only if ``refusals`` is empty and ``request`` is not None. Only
795
+ two shapes exist, exactly as in :class:`deploy_target.DeployPreflight`. A refused result never
796
+ hands back a request, because a request in hand is the thing a caller acts on.
797
+
798
+ ``caps`` is present in both shapes: the caps a live dataset would run under are worth reporting
799
+ even when the answer is no.
800
+ """
801
+
802
+ ok: bool
803
+ refusals: tuple[str, ...]
804
+ request: DeploymentRequest | None
805
+ caps: CapPolicy
806
+
807
+ def __post_init__(self) -> None:
808
+ if type(self.ok) is not bool:
809
+ raise DeployRefused("ok must be a boolean verdict")
810
+ if type(self.refusals) is not tuple or any(
811
+ type(item) is not str or not item.strip() for item in self.refusals
812
+ ):
813
+ raise DeployRefused("refusals must be a tuple of named reasons")
814
+ if self.request is not None and not isinstance(self.request, DeploymentRequest):
815
+ raise DeployRefused("request must be a deployment request or nothing")
816
+ if not isinstance(self.caps, CapPolicy):
817
+ raise DeployRefused("a deployment plan always carries the caps it was asked for")
818
+ clean = not self.refusals and self.request is not None
819
+ if self.ok is not clean:
820
+ raise DeployRefused(
821
+ "a deployment plan succeeds only when there are no refusals and a request was built"
822
+ )
823
+
824
+ @property
825
+ def request_digest(self) -> str | None:
826
+ """The content digest of the request, or nothing when there is no request."""
827
+
828
+ return None if self.request is None else self.request.digest()
829
+
830
+ def to_dict(self) -> dict[str, Any]:
831
+ """Return the one fact set both renderings are built from.
832
+
833
+ Every human line and every JSON field this product prints for a deployment plan comes from
834
+ here, so the two cannot report different things.
835
+ """
836
+
837
+ return {
838
+ "schema_version": DEPLOY_PLAN_RESULT_SCHEMA,
839
+ "status": "deployment_plan_ready" if self.ok else "deployment_plan_refused",
840
+ "ok": self.ok,
841
+ "refusals": list(self.refusals),
842
+ "request": None if self.request is None else self.request.to_summary_dict(),
843
+ "request_digest": self.request_digest,
844
+ "caps": {
845
+ "max_runs_per_month": self.caps.max_runs_per_month,
846
+ "max_concurrent_runs": self.caps.max_concurrent_runs,
847
+ "max_monthly_cost_units": self.caps.max_monthly_cost_units,
848
+ },
849
+ "caps_enforced_here": False,
850
+ }
851
+
852
+
853
+ def plan_deployment(
854
+ *,
855
+ descriptor: Mapping[str, object],
856
+ corroboration: Mapping[str, object],
857
+ candidate_run_dir: Path,
858
+ previous_run_dir: Path | None = None,
859
+ dataset_name: str,
860
+ caps: CapPolicy,
861
+ cadence: Cadence,
862
+ review_authority_provider: TrustedReviewAuthorityProvider | None,
863
+ ) -> DeployPlanResult:
864
+ """Plan whether this recipe could go live, without deploying or contacting anything.
865
+
866
+ Order is load-bearing and is checked first on purpose: **the target before anything else.**
867
+ Establishing where a dataset would be published is the cheapest check available and the most
868
+ expensive one to get wrong — it spends real money in a real project. So nothing is read,
869
+ fetched, sent, or written before we know where it would go. When the target is ambiguous this
870
+ function returns before the run directory is even opened.
871
+
872
+ No network call happens on any path through this function. Verification is local, and the
873
+ request is data.
874
+
875
+ Raises:
876
+ deploy_target.TargetAmbiguityError: only when ``descriptor`` or ``corroboration`` is not a
877
+ well-formed mapping — a structurally invalid call, not an ambiguous target.
878
+ """
879
+
880
+ if not isinstance(caps, CapPolicy):
881
+ raise DeployRefused("the caps must be a cap policy; a live dataset is never uncapped")
882
+
883
+ verdict = deploy_target.preflight(descriptor, corroboration=corroboration)
884
+ if not verdict.ok:
885
+ # The preflight's reasons are carried verbatim: it already names both projects, and
886
+ # rewording them here would create a second wording of the same fact.
887
+ return DeployPlanResult(ok=False, refusals=verdict.refusals, request=None, caps=caps)
888
+
889
+ assert verdict.target is not None # the preflight invariant, restated for the type checker
890
+ try:
891
+ request = build_deployment_request(
892
+ candidate_run_dir=candidate_run_dir,
893
+ previous_run_dir=previous_run_dir,
894
+ dataset_name=dataset_name,
895
+ target=verdict.target,
896
+ caps=caps,
897
+ cadence=cadence,
898
+ review_authority_provider=review_authority_provider,
899
+ )
900
+ except DeployRefused as refusal:
901
+ return DeployPlanResult(ok=False, refusals=refusal.reasons, request=None, caps=caps)
902
+
903
+ return DeployPlanResult(ok=True, refusals=(), request=request, caps=caps)
904
+
905
+
906
+ # Compatibility aliases for callers of the pre-35 library vocabulary. Neither alias grants a
907
+ # deployment acknowledgement or changes the planning-only semantics.
908
+ GoLiveResult = DeployPlanResult
909
+ plan_go_live = plan_deployment
910
+
911
+
912
+ __all__ = [
913
+ "DEPLOYMENT_REQUEST_SCHEMA",
914
+ "DEPLOY_PLAN_RESULT_SCHEMA",
915
+ "GO_LIVE_RESULT_SCHEMA",
916
+ "Cadence",
917
+ "DeployPlanResult",
918
+ "DeployRefused",
919
+ "DeploymentRequest",
920
+ "GoLiveResult",
921
+ "build_deployment_request",
922
+ "plan_deployment",
923
+ "plan_go_live",
924
+ ]