mostlyright-data 0.9.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (314) hide show
  1. mostlyright/data_harness/__init__.py +158 -0
  2. mostlyright/data_harness/acquisition/__init__.py +55 -0
  3. mostlyright/data_harness/acquisition/http.py +2773 -0
  4. mostlyright/data_harness/acquisition/parsing.py +809 -0
  5. mostlyright/data_harness/acquisition/ranges.py +495 -0
  6. mostlyright/data_harness/acquisition/result_download.py +360 -0
  7. mostlyright/data_harness/acquisition/retention_admission.py +248 -0
  8. mostlyright/data_harness/acquisition/sandbox.py +4888 -0
  9. mostlyright/data_harness/acquisition/url_policy.py +530 -0
  10. mostlyright/data_harness/agent_runtime.py +2743 -0
  11. mostlyright/data_harness/assets/logo-ink.svg +31 -0
  12. mostlyright/data_harness/backends/__init__.py +28 -0
  13. mostlyright/data_harness/backends/pandas_backend.py +350 -0
  14. mostlyright/data_harness/backends/polars_backend.py +366 -0
  15. mostlyright/data_harness/backends/protocol.py +124 -0
  16. mostlyright/data_harness/backends/reference.py +83 -0
  17. mostlyright/data_harness/backends/registry.py +55 -0
  18. mostlyright/data_harness/backends/restrictions.py +126 -0
  19. mostlyright/data_harness/canonical.py +333 -0
  20. mostlyright/data_harness/catalog_job.py +625 -0
  21. mostlyright/data_harness/cli.py +5398 -0
  22. mostlyright/data_harness/contracts.py +53 -0
  23. mostlyright/data_harness/coordinator.py +1307 -0
  24. mostlyright/data_harness/deploy.py +924 -0
  25. mostlyright/data_harness/deploy_target.py +312 -0
  26. mostlyright/data_harness/deployment_evidence.py +1067 -0
  27. mostlyright/data_harness/event_presentation.py +576 -0
  28. mostlyright/data_harness/events.py +2152 -0
  29. mostlyright/data_harness/fast_delimited.py +239 -0
  30. mostlyright/data_harness/fleet.py +237 -0
  31. mostlyright/data_harness/formats.py +236 -0
  32. mostlyright/data_harness/governors.py +1163 -0
  33. mostlyright/data_harness/hosted_bootstrap.py +972 -0
  34. mostlyright/data_harness/hosted_crawler.py +1115 -0
  35. mostlyright/data_harness/hosted_crawler_container_smoke.py +351 -0
  36. mostlyright/data_harness/hosted_crawler_fetch.py +423 -0
  37. mostlyright/data_harness/hosted_crawler_job.py +1277 -0
  38. mostlyright/data_harness/hosted_crawler_protocol.py +676 -0
  39. mostlyright/data_harness/hosted_dataset.py +1500 -0
  40. mostlyright/data_harness/hosted_deploy.py +3037 -0
  41. mostlyright/data_harness/hosted_handoff.py +62 -0
  42. mostlyright/data_harness/hosted_ingestion_contract.py +504 -0
  43. mostlyright/data_harness/hosted_ingestion_job.py +356 -0
  44. mostlyright/data_harness/hosted_ingestion_job_smoke.py +40 -0
  45. mostlyright/data_harness/hosted_session_container_smoke.py +194 -0
  46. mostlyright/data_harness/hosted_session_worker.py +3554 -0
  47. mostlyright/data_harness/hosted_session_worker_job_smoke.py +46 -0
  48. mostlyright/data_harness/hosted_worker.py +6784 -0
  49. mostlyright/data_harness/ingestion/__init__.py +56 -0
  50. mostlyright/data_harness/ingestion/contracts.py +461 -0
  51. mostlyright/data_harness/ingestion/faults.py +42 -0
  52. mostlyright/data_harness/ingestion/gcs_store.py +1162 -0
  53. mostlyright/data_harness/ingestion/spool.py +130 -0
  54. mostlyright/data_harness/ingestion/store.py +885 -0
  55. mostlyright/data_harness/key_seam.py +434 -0
  56. mostlyright/data_harness/linux_process_boundary.py +262 -0
  57. mostlyright/data_harness/local_contracts.py +2880 -0
  58. mostlyright/data_harness/local_search/__init__.py +5 -0
  59. mostlyright/data_harness/local_search/build_index.py +1087 -0
  60. mostlyright/data_harness/local_search/contracts.py +920 -0
  61. mostlyright/data_harness/local_search/query_trace.py +266 -0
  62. mostlyright/data_harness/local_search/retrieval.py +700 -0
  63. mostlyright/data_harness/local_search/sealed.py +474 -0
  64. mostlyright/data_harness/local_search/service.py +784 -0
  65. mostlyright/data_harness/nbrender/CONTRACT.md +212 -0
  66. mostlyright/data_harness/nbrender/__init__.py +12 -0
  67. mostlyright/data_harness/nbrender/chrome.py +359 -0
  68. mostlyright/data_harness/nbrender/code_body.py +266 -0
  69. mostlyright/data_harness/nbrender/document.py +407 -0
  70. mostlyright/data_harness/nbrender/frame.py +275 -0
  71. mostlyright/data_harness/nbrender/interactive.py +337 -0
  72. mostlyright/data_harness/nbrender/markdown_body.py +477 -0
  73. mostlyright/data_harness/nbrender/mr_components.py +134 -0
  74. mostlyright/data_harness/nbrender/outputs_data.py +595 -0
  75. mostlyright/data_harness/nbrender/outputs_rich.py +906 -0
  76. mostlyright/data_harness/nbrender/outputs_source.py +260 -0
  77. mostlyright/data_harness/nbrender/outputs_stage.py +176 -0
  78. mostlyright/data_harness/nbrender/outputs_text.py +400 -0
  79. mostlyright/data_harness/nbrender/parse.py +394 -0
  80. mostlyright/data_harness/nbrender/status.py +40 -0
  81. mostlyright/data_harness/nbrender/tokens.py +1295 -0
  82. mostlyright/data_harness/notebook.py +1710 -0
  83. mostlyright/data_harness/offline.py +2049 -0
  84. mostlyright/data_harness/operation_registry.py +1007 -0
  85. mostlyright/data_harness/operator_setup.py +239 -0
  86. mostlyright/data_harness/pipeline.py +6428 -0
  87. mostlyright/data_harness/plan_graph.py +2026 -0
  88. mostlyright/data_harness/preparation/__init__.py +104 -0
  89. mostlyright/data_harness/preparation/contracts.py +1017 -0
  90. mostlyright/data_harness/preparation/engine.py +221 -0
  91. mostlyright/data_harness/preparation/errors.py +14 -0
  92. mostlyright/data_harness/preparation/gates.py +751 -0
  93. mostlyright/data_harness/preparation/joins.py +574 -0
  94. mostlyright/data_harness/preparation/profile.py +384 -0
  95. mostlyright/data_harness/preparation/table.py +217 -0
  96. mostlyright/data_harness/preparation/transforms.py +568 -0
  97. mostlyright/data_harness/progress_events.py +534 -0
  98. mostlyright/data_harness/readers/__init__.py +46 -0
  99. mostlyright/data_harness/readers/containers.py +963 -0
  100. mostlyright/data_harness/readers/contracts.py +542 -0
  101. mostlyright/data_harness/readers/delimited.py +257 -0
  102. mostlyright/data_harness/readers/grib2/__init__.py +33 -0
  103. mostlyright/data_harness/readers/grib2/admission.py +722 -0
  104. mostlyright/data_harness/readers/grib2/decode.py +1009 -0
  105. mostlyright/data_harness/readers/grib2/geometry.py +1133 -0
  106. mostlyright/data_harness/readers/grib2/portable_math.py +501 -0
  107. mostlyright/data_harness/readers/json_tabular.py +485 -0
  108. mostlyright/data_harness/readers/registry.py +514 -0
  109. mostlyright/data_harness/readers/samples/README.md +110 -0
  110. mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/cities.csv.gz +0 -0
  111. mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/expected.json +24 -0
  112. mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/cities.csv.gz +0 -0
  113. mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/expected.json +24 -0
  114. mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/cities.tar +0 -0
  115. mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/expected.json +24 -0
  116. mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/cities.tar +0 -0
  117. mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/expected.json +24 -0
  118. mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/cities.zip +0 -0
  119. mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/expected.json +25 -0
  120. mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
  121. mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/expected.json +25 -0
  122. mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
  123. mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/expected.json +25 -0
  124. mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/cities.csv +3 -0
  125. mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/expected.json +23 -0
  126. mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/cities.tsv +5 -0
  127. mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/expected.json +25 -0
  128. mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/expected.json +30 -0
  129. mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/observations.csv +5 -0
  130. mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/expected.json +44 -0
  131. mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/stations.json +1 -0
  132. mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/expected.json +48 -0
  133. mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/observations.ndjson +4 -0
  134. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/cities.xlsx +0 -0
  135. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/expected.json +24 -0
  136. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/expected.json +27 -0
  137. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
  138. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/expected.json +27 -0
  139. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
  140. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/README.md +20 -0
  141. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/expected.json +55 -0
  142. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/gfs-2m-temperature.grib2 +0 -0
  143. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/expected.json +54 -0
  144. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
  145. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/expected.json +54 -0
  146. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/hrrr-categorical-rain.grib2 +0 -0
  147. mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/expected.json +54 -0
  148. mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
  149. mostlyright/data_harness/readers/samples.py +582 -0
  150. mostlyright/data_harness/readers/spreadsheet.py +803 -0
  151. mostlyright/data_harness/readers/tabular.py +510 -0
  152. mostlyright/data_harness/recipe.py +5321 -0
  153. mostlyright/data_harness/repair/__init__.py +78 -0
  154. mostlyright/data_harness/repair/adapters.py +274 -0
  155. mostlyright/data_harness/repair/contracts.py +872 -0
  156. mostlyright/data_harness/repair/coordinator.py +1099 -0
  157. mostlyright/data_harness/repair/errors.py +16 -0
  158. mostlyright/data_harness/review.py +2533 -0
  159. mostlyright/data_harness/rowset.py +283 -0
  160. mostlyright/data_harness/serving.py +1975 -0
  161. mostlyright/data_harness/serving_edge.py +590 -0
  162. mostlyright/data_harness/serving_http.py +1031 -0
  163. mostlyright/data_harness/session_probes.py +759 -0
  164. mostlyright/data_harness/signing.py +101 -0
  165. mostlyright/data_harness/source_discovery.py +898 -0
  166. mostlyright/data_harness/sources/__init__.py +209 -0
  167. mostlyright/data_harness/sources/_adapter_steps.py +213 -0
  168. mostlyright/data_harness/sources/adapters.py +1214 -0
  169. mostlyright/data_harness/sources/cadence.py +1428 -0
  170. mostlyright/data_harness/sources/cadence_emission.py +453 -0
  171. mostlyright/data_harness/sources/cadence_history.py +546 -0
  172. mostlyright/data_harness/sources/catalog/__init__.py +17 -0
  173. mostlyright/data_harness/sources/catalog/admission.py +477 -0
  174. mostlyright/data_harness/sources/catalog/authoring.py +1701 -0
  175. mostlyright/data_harness/sources/catalog/authoring_policy.py +701 -0
  176. mostlyright/data_harness/sources/catalog/authoring_shards.py +1217 -0
  177. mostlyright/data_harness/sources/catalog/bounded_io.py +231 -0
  178. mostlyright/data_harness/sources/catalog/channel.py +523 -0
  179. mostlyright/data_harness/sources/catalog/channel_client.py +296 -0
  180. mostlyright/data_harness/sources/catalog/contracts.py +825 -0
  181. mostlyright/data_harness/sources/catalog/coverage.py +137 -0
  182. mostlyright/data_harness/sources/catalog/delta.py +1340 -0
  183. mostlyright/data_harness/sources/catalog/embedding.py +532 -0
  184. mostlyright/data_harness/sources/catalog/entry_v2.py +1182 -0
  185. mostlyright/data_harness/sources/catalog/fill.py +3889 -0
  186. mostlyright/data_harness/sources/catalog/fill_partitions.py +459 -0
  187. mostlyright/data_harness/sources/catalog/fill_staging.py +1105 -0
  188. mostlyright/data_harness/sources/catalog/gating.py +374 -0
  189. mostlyright/data_harness/sources/catalog/generation_receipt.py +1607 -0
  190. mostlyright/data_harness/sources/catalog/harvest/__init__.py +7 -0
  191. mostlyright/data_harness/sources/catalog/harvest/ckan.py +384 -0
  192. mostlyright/data_harness/sources/catalog/harvest/datagov_v4.py +798 -0
  193. mostlyright/data_harness/sources/catalog/harvest/protocol.py +964 -0
  194. mostlyright/data_harness/sources/catalog/harvest/sdmx.py +445 -0
  195. mostlyright/data_harness/sources/catalog/harvest/stac.py +384 -0
  196. mostlyright/data_harness/sources/catalog/health.py +447 -0
  197. mostlyright/data_harness/sources/catalog/hosted_catalog.py +105 -0
  198. mostlyright/data_harness/sources/catalog/identity_history.py +1549 -0
  199. mostlyright/data_harness/sources/catalog/neural.py +1618 -0
  200. mostlyright/data_harness/sources/catalog/packed_catalog.py +2345 -0
  201. mostlyright/data_harness/sources/catalog/packed_retrieval.py +1517 -0
  202. mostlyright/data_harness/sources/catalog/packed_writer.py +2802 -0
  203. mostlyright/data_harness/sources/catalog/query_trace.py +1037 -0
  204. mostlyright/data_harness/sources/catalog/recommend.py +171 -0
  205. mostlyright/data_harness/sources/catalog/retrieval.py +230 -0
  206. mostlyright/data_harness/sources/catalog/retrieval_manifest.py +995 -0
  207. mostlyright/data_harness/sources/catalog/rights_decisions.py +254 -0
  208. mostlyright/data_harness/sources/catalog/sealed.py +560 -0
  209. mostlyright/data_harness/sources/catalog/search.py +230 -0
  210. mostlyright/data_harness/sources/catalog/streaming_delta.py +1097 -0
  211. mostlyright/data_harness/sources/catalog/update.py +891 -0
  212. mostlyright/data_harness/sources/collections.py +815 -0
  213. mostlyright/data_harness/sources/contracts.py +2223 -0
  214. mostlyright/data_harness/sources/deletion.py +761 -0
  215. mostlyright/data_harness/sources/fitness.py +162 -0
  216. mostlyright/data_harness/sources/governance.py +163 -0
  217. mostlyright/data_harness/sources/hosted.py +173 -0
  218. mostlyright/data_harness/sources/integration.py +218 -0
  219. mostlyright/data_harness/sources/range_reader.py +418 -0
  220. mostlyright/data_harness/sources/registry.py +514 -0
  221. mostlyright/data_harness/sources/rights_rule.py +59 -0
  222. mostlyright/data_harness/sources/source_cadence_vectors.v1.json +1 -0
  223. mostlyright/data_harness/sources/sports.py +521 -0
  224. mostlyright/data_harness/sources/stream.py +524 -0
  225. mostlyright/data_harness/sources/stream_connector.py +418 -0
  226. mostlyright/data_harness/sources/stream_recorder.py +1404 -0
  227. mostlyright/data_harness/studio_boundary.py +2019 -0
  228. mostlyright/data_harness/thin/__init__.py +37 -0
  229. mostlyright/data_harness/thin/acquire.py +1137 -0
  230. mostlyright/data_harness/thin/acquire_cancel.py +579 -0
  231. mostlyright/data_harness/thin/approvals.py +617 -0
  232. mostlyright/data_harness/thin/commands.py +406 -0
  233. mostlyright/data_harness/thin/download.py +194 -0
  234. mostlyright/data_harness/thin/narrative.py +589 -0
  235. mostlyright/data_harness/thin/parity.py +1070 -0
  236. mostlyright/data_harness/thin/propose.py +2759 -0
  237. mostlyright/data_harness/thin/research.py +1663 -0
  238. mostlyright/data_harness/thin/router.py +924 -0
  239. mostlyright/data_harness/thin/runs.py +519 -0
  240. mostlyright/data_harness/thin/session.py +281 -0
  241. mostlyright/data_harness/thin/stream.py +501 -0
  242. mostlyright/data_harness/thin/transport.py +187 -0
  243. mostlyright/data_harness/thin/vocabulary.py +368 -0
  244. mostlyright/data_harness/thin/workers.py +164 -0
  245. mostlyright/data_harness/ucum/TABLE-PIN.json +40 -0
  246. mostlyright/data_harness/ucum/ucum-subset.v1.json +632 -0
  247. mostlyright/data_harness/unit_flow.py +927 -0
  248. mostlyright/data_harness/units.py +572 -0
  249. mostlyright/data_harness/ux/__init__.py +9 -0
  250. mostlyright/data_harness/ux/approve.py +485 -0
  251. mostlyright/data_harness/ux/author_yaml.py +597 -0
  252. mostlyright/data_harness/ux/cloud_auth.py +447 -0
  253. mostlyright/data_harness/ux/commands/__init__.py +260 -0
  254. mostlyright/data_harness/ux/commands/approve.py +136 -0
  255. mostlyright/data_harness/ux/commands/auth.py +744 -0
  256. mostlyright/data_harness/ux/commands/author.py +79 -0
  257. mostlyright/data_harness/ux/commands/catalog_author.py +403 -0
  258. mostlyright/data_harness/ux/commands/catalog_fill.py +523 -0
  259. mostlyright/data_harness/ux/commands/catalog_harvest.py +545 -0
  260. mostlyright/data_harness/ux/commands/catalog_publish.py +1838 -0
  261. mostlyright/data_harness/ux/commands/catalog_search.py +71 -0
  262. mostlyright/data_harness/ux/commands/catalog_update.py +437 -0
  263. mostlyright/data_harness/ux/commands/deploy.py +134 -0
  264. mostlyright/data_harness/ux/commands/deploy_dataset.py +98 -0
  265. mostlyright/data_harness/ux/commands/deploy_plan.py +105 -0
  266. mostlyright/data_harness/ux/commands/deploy_status.py +104 -0
  267. mostlyright/data_harness/ux/commands/diff.py +74 -0
  268. mostlyright/data_harness/ux/commands/index.py +84 -0
  269. mostlyright/data_harness/ux/commands/inventory.py +47 -0
  270. mostlyright/data_harness/ux/commands/list_builds.py +143 -0
  271. mostlyright/data_harness/ux/commands/login.py +63 -0
  272. mostlyright/data_harness/ux/commands/peek.py +236 -0
  273. mostlyright/data_harness/ux/commands/plan_check.py +90 -0
  274. mostlyright/data_harness/ux/commands/preflight.py +97 -0
  275. mostlyright/data_harness/ux/commands/record.py +107 -0
  276. mostlyright/data_harness/ux/commands/review_setup.py +47 -0
  277. mostlyright/data_harness/ux/commands/search.py +440 -0
  278. mostlyright/data_harness/ux/commands/show.py +61 -0
  279. mostlyright/data_harness/ux/commands/whoami.py +37 -0
  280. mostlyright/data_harness/ux/credential_native.py +551 -0
  281. mostlyright/data_harness/ux/credential_store.py +1055 -0
  282. mostlyright/data_harness/ux/credentials.py +631 -0
  283. mostlyright/data_harness/ux/diffing.py +444 -0
  284. mostlyright/data_harness/ux/headline.py +671 -0
  285. mostlyright/data_harness/ux/hosted_acquisition.py +974 -0
  286. mostlyright/data_harness/ux/hosted_run_status.py +619 -0
  287. mostlyright/data_harness/ux/inventory.py +427 -0
  288. mostlyright/data_harness/ux/local_review.py +375 -0
  289. mostlyright/data_harness/ux/login.py +691 -0
  290. mostlyright/data_harness/ux/path_kind.py +147 -0
  291. mostlyright/data_harness/ux/peek.py +1000 -0
  292. mostlyright/data_harness/ux/plain_file.py +178 -0
  293. mostlyright/data_harness/ux/plan_check.py +311 -0
  294. mostlyright/data_harness/ux/preflight.py +918 -0
  295. mostlyright/data_harness/ux/readers.py +1124 -0
  296. mostlyright/data_harness/ux/remediation.py +2195 -0
  297. mostlyright/data_harness/ux/render.py +657 -0
  298. mostlyright/data_harness/ux/workload.py +1077 -0
  299. mostlyright/data_harness/viewer.py +3713 -0
  300. mostlyright/data_harness/visual_run/__init__.py +83 -0
  301. mostlyright/data_harness/visual_run/authoring.py +235 -0
  302. mostlyright/data_harness/visual_run/contracts.py +673 -0
  303. mostlyright/data_harness/visual_run/materialize.py +486 -0
  304. mostlyright/data_harness/visual_run/observations.py +874 -0
  305. mostlyright/data_harness/visual_run/query.py +259 -0
  306. mostlyright/data_harness/visual_run/reducer.py +280 -0
  307. mostlyright/data_harness/visual_run/sdk.py +892 -0
  308. mostlyright/data_harness/visual_run/store.py +584 -0
  309. mostlyright/data_harness/visual_run/transport.py +239 -0
  310. mostlyright/data_harness/watch.py +2999 -0
  311. mostlyright_data-0.9.0.dist-info/METADATA +607 -0
  312. mostlyright_data-0.9.0.dist-info/RECORD +314 -0
  313. mostlyright_data-0.9.0.dist-info/WHEEL +4 -0
  314. mostlyright_data-0.9.0.dist-info/entry_points.txt +12 -0
@@ -0,0 +1,918 @@
1
+ """What would stop a build on this machine, asked before the build rather than during it.
2
+
3
+ Three refusals are decided by the operating system: creating an output safely, re-checking a sealed
4
+ Build, and noticing a folder swapped underneath either operation. Preflight asks those platform
5
+ questions before expensive source reads and transformations begin.
6
+
7
+ The remaining checks cover absent or invalid Workbench and input folders, an output path that is
8
+ already taken, an engine this machine does not have installed, and -- the one that is about the
9
+ job rather than the machine -- whether *this workload* fits the ceilings the deterministic engine
10
+ holds whatever it is run on. That last one is answered from the plan and one ``lstat`` per source,
11
+ so a job that cannot run is refused here rather than inside a container after its sources have
12
+ been uploaded and its deployment approved.
13
+
14
+ **It writes nothing and looks at nothing it was not pointed at.** No environment variable is read,
15
+ no credential material is touched, and the only paths opened are the ones named in the call. The
16
+ stored stage of a workbench folder is read through the coordinator's own reader rather than by
17
+ opening ``state.json`` directly, so a folder that is mid-repair is reported the way every other
18
+ command reports it.
19
+
20
+ **The stored stage words never move.** ``initialized``, ``planned``, ``building``,
21
+ ``candidate_built``, ``review_recorded`` and ``failed`` are contract values living in folders
22
+ already on disk. :data:`PHASE_SENTENCES` maps them to plain sentences at display time only; nothing
23
+ here writes a stage, and nothing here renames one.
24
+
25
+ **No engine is imported unless one is named.** The engine table is deliberately lazy -- constructing
26
+ a backend does not import its library -- and a preflight that probed all three on every run would
27
+ undo that. The probe happens inside :func:`_engine_check`, which is reached only when ``engine`` is
28
+ passed.
29
+
30
+ Nothing here prints. :meth:`Preflight.to_dict` returns the one payload both renderings are built
31
+ from, so the same answer can be shown in a browser later without re-deriving it.
32
+ """
33
+
34
+ from __future__ import annotations
35
+
36
+ import os
37
+ import select
38
+ import sys
39
+ from collections.abc import Mapping, Sequence
40
+ from dataclasses import dataclass
41
+ from pathlib import Path
42
+ from typing import Any
43
+
44
+ from mostlyright.data_harness.offline import CONTROL_PATH
45
+ from mostlyright.data_harness.pipeline import _VERIFY_ALLOWLISTED_ENGINES
46
+ from mostlyright.data_harness.ux.headline import (
47
+ json_headline,
48
+ naming,
49
+ plain_headline,
50
+ system_headline,
51
+ without_repeated_code,
52
+ )
53
+ from mostlyright.data_harness.ux.path_kind import (
54
+ A_FOLDER,
55
+ A_LINK,
56
+ NOTHING_THERE,
57
+ UNKNOWN_KIND,
58
+ kind_at,
59
+ presence_at,
60
+ )
61
+ from mostlyright.data_harness.ux.remediation import remediation_for
62
+ from mostlyright.data_harness.ux.render import render_human
63
+ from mostlyright.data_harness.ux.workload import (
64
+ Workload,
65
+ WorkloadUnreadable,
66
+ read_workload,
67
+ )
68
+
69
+ # The one status this command reports. It reports it whether anything is blocking or not: "nothing
70
+ # would stop you" is an answer, and an answer is not an error.
71
+ PREFLIGHT_STATUS = "preflight_completed"
72
+
73
+ # The version of the payload below, carried the way every sibling command carries one -- `index`
74
+ # emits `mr-data-private-index.v1`, `whoami` emits `mr-data-whoami.v1`. This command emitted none
75
+ # while it answered one question about one machine; now that it also answers what a *workload*
76
+ # needs, a consumer has to be able to tell which shape of answer it is holding.
77
+ PREFLIGHT_SCHEMA_VERSION = "mr-data-preflight.v1"
78
+
79
+ # What one check can come back as. `not_applicable` is the honest word for a check that was never
80
+ # asked -- an omitted check reads as "unknown", and those are different answers.
81
+ OK = "ok"
82
+ BLOCKED = "blocked"
83
+ NOT_APPLICABLE = "not_applicable"
84
+
85
+ _MARKS = {OK: "[ok]", BLOCKED: "[stops you]", NOT_APPLICABLE: "[not asked]"}
86
+
87
+ # The eight checks, by the name a person reads. They are values in the payload, never keys, so a
88
+ # name is shown exactly as it is written here. The first seven are about this machine; the last is
89
+ # about the job, and is the only one that reads anything the job itself declares.
90
+ WRITING_CHECK = "writing a file safely"
91
+ VERIFYING_CHECK = "re-checking a build"
92
+ WATCHING_CHECK = "noticing a folder that changes underneath"
93
+ WORKBENCH_CHECK = "the workbench folder"
94
+ INPUT_CHECK = "the input folder"
95
+ OUTPUT_CHECK = "where it would write"
96
+ ENGINE_CHECK = "the engine"
97
+ WORKLOAD_CHECK = "the workload itself"
98
+
99
+ # Where the rendered lines and the closing advice sit in the payload. Both are keys a person reads.
100
+ CHECKS_KEY = "checks"
101
+ BLOCKED_KEY = "blocked"
102
+ WHAT_TO_DO_KEY = "what to do"
103
+ WORKLOAD_KEY = "the workload"
104
+
105
+ NOTHING_TO_DO = "Nothing here would stop a build."
106
+
107
+ # The error map ends its platform advice by naming this command. Printed inside this command that
108
+ # line answers a question the reader has already acted on, so it is the one line dropped.
109
+ _SELF_REFERENCE = "Run: mr-data preflight"
110
+
111
+ # The stored stage of a workbench folder, in plain words. The keys are persisted contract values
112
+ # and are never written or renamed here; only the sentence shown to a person is plain language.
113
+ PHASE_SENTENCES: dict[str, str] = {
114
+ "initialized": "This workbench folder is set up; the plan has not been checked yet.",
115
+ "planned": "The plan here is checked and recorded; nothing has been built yet.",
116
+ "building": "A build is running here, or one stopped part way through.",
117
+ "candidate_built": "A build is sealed here; run mr-data show to read it.",
118
+ "review_recorded": "A review decision is sealed here.",
119
+ "failed": "The last build here failed.",
120
+ }
121
+
122
+ # The three steps a workbench folder records, and the four words it records against them. Both sets
123
+ # are contract values on disk; both are mapped to plain words at display time only.
124
+ STEP_NAMES: dict[str, str] = {
125
+ "plan": "checking the plan",
126
+ "build": "building",
127
+ "review": "review",
128
+ }
129
+ STEP_STATES: dict[str, str] = {
130
+ "pending": "not started",
131
+ "running": "in progress",
132
+ "completed": "done",
133
+ "failed": "failed",
134
+ }
135
+
136
+ # What each engine needs before it can build. The reference engine is pure Python and needs
137
+ # nothing; the other two are asked for their own version, which is the call that imports them.
138
+ _BUILT_IN = "is built in here and needs nothing installed"
139
+
140
+ # Said about a path that is not a Workbench folder, whichever of the three ways it is not one: it
141
+ # is not there, it is not a folder at all, or it is a folder nothing has been set up in. The second
142
+ # line said "that folder" until the branch above it learned to tell those apart, which put the same
143
+ # false claim in the advice that the sentence had just stopped making.
144
+ _NOT_A_WORKBENCH = (
145
+ "Run mr-data init on a path that does not exist yet to set one up.",
146
+ "Nothing was read from that path, so nothing has to be undone.",
147
+ )
148
+
149
+
150
+ @dataclass(frozen=True)
151
+ class PreflightCheck:
152
+ """One thing that was checked, how it came out, and what to do when it did not.
153
+
154
+ ``sentence`` is written for a person who has not read the source: it says what was looked at
155
+ and what was found, never only that something failed. ``remediation`` comes from the error map
156
+ wherever the same failure has a typed code, so this command and the refusal it predicts say
157
+ one thing rather than two.
158
+ """
159
+
160
+ name: str
161
+ outcome: str
162
+ sentence: str
163
+ remediation: tuple[str, ...] = ()
164
+
165
+ @property
166
+ def blocking(self) -> bool:
167
+ return self.outcome == BLOCKED
168
+
169
+ @property
170
+ def line(self) -> str:
171
+ """This check as one plain line: outcome first, then what it was, then what was found."""
172
+
173
+ return f"{_MARKS[self.outcome]} {self.name}: {self.sentence}"
174
+
175
+
176
+ @dataclass(frozen=True)
177
+ class Preflight:
178
+ """Every check this machine can answer before a build, and whether any of them stops you.
179
+
180
+ ``workload`` is the bounded facts of the job itself when one was named -- what its sources
181
+ weigh, and how each of those weights compares with the ceiling the engine will judge it by.
182
+ It is carried as a value beside the checks rather than folded into one of them, because a
183
+ check is one line and a workload is a table; the line summarising it is still in ``checks``,
184
+ so a reader who only reads the lines is not told less than the payload knows.
185
+ """
186
+
187
+ checks: tuple[PreflightCheck, ...]
188
+ workload: Workload | None = None
189
+
190
+ @property
191
+ def blocked(self) -> bool:
192
+ return any(check.blocking for check in self.checks)
193
+
194
+ @property
195
+ def what_to_do(self) -> str:
196
+ """The advice for every blocking check, in check order, each line said once."""
197
+
198
+ lines: list[str] = []
199
+ for check in self.checks:
200
+ if not check.blocking:
201
+ continue
202
+ for line in check.remediation or (check.sentence,):
203
+ if line not in lines:
204
+ lines.append(line)
205
+ return "\n".join(lines) if lines else NOTHING_TO_DO
206
+
207
+ def to_dict(self) -> dict[str, Any]:
208
+ """The one payload both renderings are built from."""
209
+
210
+ width = len(str(max(len(self.checks), 1)))
211
+ payload: dict[str, Any] = {
212
+ "schema_version": PREFLIGHT_SCHEMA_VERSION,
213
+ "status": PREFLIGHT_STATUS,
214
+ BLOCKED_KEY: self.blocked,
215
+ WHAT_TO_DO_KEY: self.what_to_do,
216
+ CHECKS_KEY: {
217
+ f"check {index:0{width}d}": check.line
218
+ for index, check in enumerate(self.checks, start=1)
219
+ },
220
+ }
221
+ if self.workload is not None:
222
+ payload[WORKLOAD_KEY] = self.workload.to_block()
223
+ return payload
224
+
225
+ def lines(self) -> list[str]:
226
+ """Exactly the plain lines the command line prints, from exactly the same payload."""
227
+
228
+ return render_human(self.to_dict()).splitlines()
229
+
230
+
231
+ def run_preflight(
232
+ *,
233
+ workspace: Path | str | None = None,
234
+ input_root: Path | str | None = None,
235
+ output: Path | str | None = None,
236
+ engine: str | None = None,
237
+ workload: Path | str | None = None,
238
+ ) -> Preflight:
239
+ """Answer what would stop a build here, without starting one and without writing anything.
240
+
241
+ The three platform checks are always run: they are what this command exists for, and they cost
242
+ nothing. The other five are run only when the caller names the thing to check, and are reported
243
+ as not asked otherwise -- an omitted check would read as an unknown one.
244
+
245
+ ``workload`` is the job: a workbench folder, a plan, or a frozen recipe. It is the only
246
+ argument whose answer is about the *data* rather than about this machine, and it is the one a
247
+ person about to deploy actually needs, because a job the engine will refuse is refused here
248
+ before a byte of it is uploaded.
249
+ """
250
+
251
+ facts = _workload_facts(workload, input_root=input_root)
252
+ checks = [
253
+ *_platform_checks(),
254
+ _workbench_check(workspace),
255
+ _input_check(input_root),
256
+ _output_check(output),
257
+ _engine_check(engine),
258
+ facts.check,
259
+ ]
260
+ return Preflight(checks=tuple(checks), workload=facts.workload)
261
+
262
+
263
+ # ------------------------------------------------------------------------------------------------
264
+ # What the operating system will and will not do
265
+ # ------------------------------------------------------------------------------------------------
266
+
267
+
268
+ def _platform_checks() -> list[PreflightCheck]:
269
+ """The three refusals this machine decides on its own, asked before any work is started."""
270
+
271
+ # Imported here, not at the top: the command line imports this package while it is still being
272
+ # set up. Asking the command line's own booleans rather than restating them is what keeps this
273
+ # answer and the refusal it predicts the same answer.
274
+ from mostlyright.data_harness import cli
275
+
276
+ return [
277
+ _platform_check(
278
+ WRITING_CHECK,
279
+ supported=cli._secure_cli_output_supported(),
280
+ yes="this machine can create a file exactly once, without following a link into it",
281
+ no="this machine cannot create a file safely, so nothing can be written here",
282
+ code="PLATFORM_OUTPUT_UNSUPPORTED",
283
+ ),
284
+ _platform_check(
285
+ VERIFYING_CHECK,
286
+ supported=cli._secure_cli_verification_supported(),
287
+ yes="this machine can re-read a sealed build without the folders moving underneath",
288
+ no="this machine cannot re-check a sealed build, so a build made here cannot be "
289
+ "re-checked here",
290
+ code="PLATFORM_VERIFY_UNSUPPORTED",
291
+ ),
292
+ _platform_check(
293
+ WATCHING_CHECK,
294
+ supported=_watching_supported(),
295
+ yes="this machine reports a folder that is moved or replaced while it is in use",
296
+ no="this machine cannot report a folder that is moved while it is in use, and both "
297
+ "writing and re-checking need that",
298
+ code=None,
299
+ ),
300
+ ]
301
+
302
+
303
+ def _platform_check(
304
+ name: str,
305
+ *,
306
+ supported: bool,
307
+ yes: str,
308
+ no: str,
309
+ code: str | None,
310
+ ) -> PreflightCheck:
311
+ if supported:
312
+ return PreflightCheck(name=name, outcome=OK, sentence=yes)
313
+ return PreflightCheck(name=name, outcome=BLOCKED, sentence=no, remediation=_advice(code))
314
+
315
+
316
+ def _watching_supported() -> bool:
317
+ """Whether a folder-watch can be opened at all, in the same order the command line tries.
318
+
319
+ This mirrors ``cli._create_cli_ancestor_monitor``, which takes descriptors and would therefore
320
+ have to be handed a folder to answer for. A test opens a real folder, asks that function, and
321
+ asserts the two agree, so the mirror cannot drift into a comfortable lie.
322
+ """
323
+
324
+ return hasattr(select, "kqueue") or sys.platform.startswith("linux")
325
+
326
+
327
+ def _advice(code: str | None, **subject: Any) -> tuple[str, ...]:
328
+ """The error map's own sentences, minus the one that names the command already running."""
329
+
330
+ return tuple(line for line in remediation_for(code, **subject) if line != _SELF_REFERENCE)
331
+
332
+
333
+ def _plainly(error: BaseException) -> str:
334
+ """A caught refusal's own wording, in plain words, ready to sit inside a check's sentence.
335
+
336
+ A check that stops you is a *successful* answer to the question this command asks, so its
337
+ sentence is carried on a payload that never passes through the command line's error boundary.
338
+ That boundary is where the one-direction translation happens, and a raise-site message
339
+ interpolated into a success payload would go around it -- which is exactly how ``durable state
340
+ references a missing candidate`` reached a person's screen with an internal word in it.
341
+
342
+ So the same boundary is applied here, from the same functions, in the same order the command
343
+ line applies them: the strict-JSON assembly, then a refusal the operating system raised, then
344
+ the term translation. A message that is already plain comes through untouched, as it does
345
+ everywhere else. No detail is lost: the trailing name a refusal ends in is carried onto the
346
+ plain sentence by ``plain_headline``, and the exact wording is still one ``--json`` away on
347
+ every command that raises rather than reports.
348
+
349
+ The closing full stop is dropped because the caller is embedding this after a colon inside a
350
+ longer sentence, not printing it as a headline of its own.
351
+ """
352
+
353
+ code = getattr(error, "code", None)
354
+ message = without_repeated_code(str(error), code if isinstance(code, str) else None)
355
+ plain = (
356
+ json_headline(error)
357
+ or system_headline(error)
358
+ or plain_headline(
359
+ code if isinstance(code, str) else None,
360
+ message,
361
+ raised_as=naming(error),
362
+ )
363
+ )
364
+ return (plain if plain is not None else message).rstrip(".")
365
+
366
+
367
+ # ------------------------------------------------------------------------------------------------
368
+ # The workbench folder
369
+ # ------------------------------------------------------------------------------------------------
370
+
371
+
372
+ def _workbench_check(workspace: Path | str | None) -> PreflightCheck:
373
+ """What stage the folder is at, what its three steps say, and whether it already holds a build.
374
+
375
+ A path that is not there is named as a path that is not there. The reader's own refusal for
376
+ that case is about the safety of the folders leading to it, which is true but unhelpful for an
377
+ absent path that is usually a typo.
378
+
379
+ "Not a folder" and "a folder that is not a workbench" are asked as two questions, in that
380
+ order, the way ``_input_check`` and ``peek`` ask them. Deciding from the control tree inside the
381
+ path cannot distinguish the two and would call a plain file, pipe, socket, or device "a folder,
382
+ but not a workbench folder". The noun therefore comes from a single ``lstat`` shared with the
383
+ writers rather than a second classification rule written here.
384
+
385
+ The ``lstat`` is taken once at the top, and its answer decides both the branch and the sentence.
386
+ ``Path.exists`` and ``Path.is_dir`` cannot participate because both follow links. At a *link to
387
+ a Workbench folder*, a following check says yes while the command refuses: ``offline`` opens
388
+ the folder chain with ``O_NOFOLLOW`` and stops at the link. The preview would otherwise report
389
+ a folder that is ready even though the Build cannot start. ``Path.is_dir`` also raises under a
390
+ parent this account may not search. :func:`kind_at` is the same look the
391
+ sentence already uses and answers ``None`` for a failed observation, allowing preflight to
392
+ report the unknown kind rather than emit a traceback.
393
+ """
394
+
395
+ if workspace is None:
396
+ return _not_asked(WORKBENCH_CHECK, "no workbench folder was named, so none was looked at")
397
+ path = Path(workspace)
398
+ found = presence_at(path)
399
+ if found == NOTHING_THERE:
400
+ return PreflightCheck(
401
+ name=WORKBENCH_CHECK,
402
+ outcome=BLOCKED,
403
+ # `presence_at` answers existence and kind from the same non-following lookup. A
404
+ # dangling link is therefore not described as an absent path simply because its target
405
+ # is absent.
406
+ sentence=f"there is {found} at {path}, "
407
+ "and a workbench has to be a folder this harness set up",
408
+ remediation=_NOT_A_WORKBENCH,
409
+ )
410
+ if found != A_FOLDER:
411
+ return PreflightCheck(
412
+ name=WORKBENCH_CHECK,
413
+ outcome=BLOCKED,
414
+ # "has to be" rather than "is": the phrase "is a folder" is exactly what this branch
415
+ # was found saying about a file, and a sentence that clears the finding while leaving
416
+ # the words that carried it in place is a sentence a reader still has to parse twice.
417
+ sentence=f"there is {found or UNKNOWN_KIND} at {path}, "
418
+ "and a workbench has to be a folder",
419
+ remediation=_NOT_A_WORKBENCH,
420
+ )
421
+ if kind_at(path / CONTROL_PATH) != A_FOLDER:
422
+ return PreflightCheck(
423
+ name=WORKBENCH_CHECK,
424
+ outcome=BLOCKED,
425
+ sentence=f"there is {found or UNKNOWN_KIND} at {path}, "
426
+ "but it is not a workbench folder",
427
+ remediation=_NOT_A_WORKBENCH,
428
+ )
429
+ # Imported here for the same reason the platform booleans are: this package is loaded while the
430
+ # command line is still being set up.
431
+ from mostlyright.data_harness.offline import OfflineRunError, status_workspace
432
+
433
+ try:
434
+ # Never waited for. A build holds the folder's durable lock for its whole length, and this
435
+ # is the command a person and a script are told to run *before* a build -- so a preflight
436
+ # that queued behind a running one would print nothing at all for as long as that build
437
+ # ran. Asked this way it says the folder is busy, which is an answer and is the answer.
438
+ recorded = status_workspace(path, wait=False)
439
+ except (OfflineRunError, OSError) as error:
440
+ code = getattr(error, "code", None)
441
+ return PreflightCheck(
442
+ name=WORKBENCH_CHECK,
443
+ outcome=BLOCKED,
444
+ sentence=f"the workbench folder at {path} could not be read: {_plainly(error)}",
445
+ remediation=_advice(code, path=str(path), path_exists=True),
446
+ )
447
+ return PreflightCheck(
448
+ name=WORKBENCH_CHECK,
449
+ outcome=OK,
450
+ sentence=_workbench_sentence(recorded),
451
+ )
452
+
453
+
454
+ def _workbench_sentence(recorded: Mapping[str, Any]) -> str:
455
+ """The stage, the three steps, and whether there is already a build, in one plain line."""
456
+
457
+ phase = str(recorded.get("status", ""))
458
+ stage = PHASE_SENTENCES.get(phase, f"this folder records the stage {phase}.")
459
+ steps = ", ".join(_step_phrases(recorded.get("tasks", ())))
460
+ built = (
461
+ # `idempotent: true` on a second run means exactly "this build already existed and nothing
462
+ # was rebuilt" (recorded in 29-03's A7), so saying so here is safe.
463
+ "There is already a build in this folder, and building again would report that one "
464
+ "rather than making another."
465
+ if recorded.get("candidate") is not None
466
+ else "Nothing has been built here yet."
467
+ )
468
+ return f"{stage} Steps: {steps}. {built}"
469
+
470
+
471
+ def _step_phrases(tasks: Any) -> list[str]:
472
+ phrases: list[str] = []
473
+ for task in tasks if isinstance(tasks, Sequence) else ():
474
+ if not isinstance(task, Mapping):
475
+ continue
476
+ task_id = str(task.get("task_id", ""))
477
+ status = str(task.get("status", ""))
478
+ phrases.append(
479
+ f"{STEP_NAMES.get(task_id, task_id)} {STEP_STATES.get(status, status)}",
480
+ )
481
+ return phrases
482
+
483
+
484
+ # ------------------------------------------------------------------------------------------------
485
+ # The folder read from, and the path written to
486
+ # ------------------------------------------------------------------------------------------------
487
+
488
+
489
+ def _input_check(input_root: Path | str | None) -> PreflightCheck:
490
+ """Whether there is anything to read, and whether this account may read it.
491
+
492
+ Anything that is not a folder is named from the same single ``lstat`` the rest of this module
493
+ and the writers share. A pipe, bound socket, or ``/dev/zero`` must not be called a file.
494
+
495
+ The link rule is applied here because the build applies it here.
496
+ ``pipeline._open_source_root`` opens every component of the input root with ``O_NOFOLLOW`` and
497
+ refuses the whole build on the first link, exactly as the output side does -- and this check
498
+ never asked. On this platform ``/tmp`` is a link, so *every* ``/tmp`` input root was previewed
499
+ as "nothing here would stop a build" and then refused with ``SOURCE_SYMLINK``, which is the
500
+ commonest path anyone points a first build at. The rule is asked first because the writer meets
501
+ it first: it walks down from the anchor, so a link above an input folder that is not there is
502
+ what stops the build, not the folder that is not there.
503
+
504
+ Unlike the output side, the root **itself** is included in the walk. The writer stats every
505
+ component of the input root, the last one included, and a root that is a link is refused on that
506
+ same test; an output's own name is not walked, because the writer creates it.
507
+ """
508
+
509
+ if input_root is None:
510
+ return _not_asked(INPUT_CHECK, "no input folder was named, so none was looked at")
511
+ root = Path(input_root)
512
+ linked = _first_link_in(Path(os.fspath(root)).absolute())
513
+ if linked is not None:
514
+ return _blocked(
515
+ INPUT_CHECK,
516
+ f"{linked} is {kind_at(linked) or UNKNOWN_KIND}, and a build refuses to read "
517
+ "through one",
518
+ _advice("SOURCE_SYMLINK", path=str(linked), path_exists=True),
519
+ )
520
+ found = presence_at(root)
521
+ # The same non-following observation drives both the message and remediation. A dangling link
522
+ # is an entry in its own right and must not be treated as an absent root.
523
+ advice = _advice(
524
+ "SOURCE_ROOT_INVALID",
525
+ path=str(root),
526
+ # Three answers rather than two: a look that failed is not a look that found nothing, and
527
+ # the map has a kind-neutral wording for exactly that.
528
+ path_exists=None if found is None else found != NOTHING_THERE,
529
+ )
530
+ if found == NOTHING_THERE:
531
+ return _blocked(
532
+ INPUT_CHECK,
533
+ f"there is {found} at {root} to read your sources from",
534
+ advice,
535
+ )
536
+ if found != A_FOLDER:
537
+ # Use the shared observation rather than a second `is_dir` lookup. The latter raises under
538
+ # an unsearchable parent, while the shared look reports an unknown kind without a traceback.
539
+ return _blocked(
540
+ INPUT_CHECK,
541
+ f"there is {found or UNKNOWN_KIND} at {root}, and sources are read from a folder",
542
+ advice,
543
+ )
544
+ if not os.access(root, os.R_OK | os.X_OK):
545
+ return _blocked(INPUT_CHECK, f"{root} cannot be opened by this account", advice)
546
+ try:
547
+ count = _readable_file_count(root)
548
+ except OSError as error:
549
+ return _blocked(INPUT_CHECK, f"{root} could not be listed: {_plainly(error)}", advice)
550
+ if not count:
551
+ return _blocked(INPUT_CHECK, f"there are no files in {root} to read", advice)
552
+ if count >= _INPUT_SCAN_CEILING:
553
+ return PreflightCheck(
554
+ name=INPUT_CHECK,
555
+ outcome=OK,
556
+ sentence=f"{root} holds {_INPUT_SCAN_CEILING} files or more",
557
+ )
558
+ noun = "file" if count == 1 else "files"
559
+ return PreflightCheck(name=INPUT_CHECK, outcome=OK, sentence=f"{root} holds {count} {noun}")
560
+
561
+
562
+ # How far down to look for something a build could read, and how many files are worth counting
563
+ # before the answer stops changing. A source path in a plan is a relative path that may name a
564
+ # subfolder -- `raw/temps.csv` is legal, and the contract that validates source paths accepts any
565
+ # relative path that does not climb out of the input folder -- so counting only the top level told
566
+ # input folders that hold their sources one level down that they held nothing, and stopped a build
567
+ # that would have worked. Both bounds are here so a yes/no question never becomes a scan of
568
+ # somebody's home directory.
569
+ _INPUT_SCAN_DEPTH = 6
570
+ _INPUT_SCAN_CEILING = 1000
571
+
572
+
573
+ def _readable_file_count(root: Path) -> int:
574
+ """How many plain files a build could read under ``root``, looking a bounded way down.
575
+
576
+ Links are not counted and not followed, which is the same rule the build itself applies to a
577
+ source path. A subfolder this account may not open is skipped rather than treated as an empty
578
+ input folder; the folder the caller actually named is the one whose refusal reaches them.
579
+ """
580
+
581
+ found = 0
582
+ pending: list[tuple[Path, int]] = [(root, 0)]
583
+ while pending and found < _INPUT_SCAN_CEILING:
584
+ directory, depth = pending.pop()
585
+ try:
586
+ names = sorted(os.listdir(directory))
587
+ except OSError:
588
+ if directory == root:
589
+ raise
590
+ continue
591
+ for name in names:
592
+ child = directory / name
593
+ if child.is_symlink():
594
+ continue
595
+ if child.is_file():
596
+ found += 1
597
+ if found >= _INPUT_SCAN_CEILING:
598
+ break
599
+ elif depth < _INPUT_SCAN_DEPTH and child.is_dir():
600
+ pending.append((child, depth + 1))
601
+ return found
602
+
603
+
604
+ def _output_check(output: Path | str | None) -> PreflightCheck:
605
+ """Whether the path is free, has no link above it, and sits in a folder that can be written to.
606
+
607
+ Nothing is ever overwritten, so a path that is already taken stops a build. Meeting that after
608
+ the build rather than before it wastes the completed work, so this asks the same refusal first.
609
+
610
+ The link check is the same rule stated the same way: sealing opens every component of the
611
+ output's parent chain without following links and refuses the whole build the moment one of
612
+ them is a link. On this platform ``/tmp`` is such a link, and ``/tmp`` is the commonest place
613
+ to write a first build to, so a preflight that omitted the rule reported "nothing here would
614
+ stop a build" about exactly the case it exists for.
615
+ """
616
+
617
+ if output is None:
618
+ return _not_asked(OUTPUT_CHECK, "no path to write to was named, so none was looked at")
619
+ destination = Path(output)
620
+ parent = destination.parent
621
+ linked = _linked_ancestor(destination)
622
+ if linked is not None:
623
+ return _blocked(
624
+ OUTPUT_CHECK,
625
+ # Derive the noun from the same observation `_linked_ancestor` selected on. Importing a
626
+ # constant into the sentence would assert the kind without binding it to that lookup.
627
+ f"{linked} is {kind_at(linked) or UNKNOWN_KIND}, and a build refuses to write "
628
+ "through one",
629
+ _advice("OUTPUT_SYMLINK", path=str(linked), path_exists=True),
630
+ )
631
+ if os.path.lexists(destination):
632
+ return _blocked(
633
+ OUTPUT_CHECK,
634
+ f"{destination} already holds something, and nothing is ever overwritten",
635
+ # This command looked, so it says what it found. `--output` is documented as an output
636
+ # path and README's own author and approve steps point it at Recipe files, so what is
637
+ # in the way here is regularly a file rather than a Build -- and the wording that calls
638
+ # it an overwritten build, and ends by naming `mr-data show`, is then two false
639
+ # sentences and a command that exits 1. `holds_a_build` is the same question the
640
+ # command line's error boundary asks about the same paths.
641
+ _advice(
642
+ "OUTPUT_EXISTS",
643
+ path=str(destination),
644
+ path_exists=True,
645
+ build_present=_holds_a_build(destination),
646
+ ),
647
+ )
648
+ above = presence_at(parent)
649
+ if above == NOTHING_THERE:
650
+ # `--output` is the path a Build would be written to, and
651
+ # `pipeline._open_output_parent` *creates* every missing component of it -- so
652
+ # `mr-data preflight --output .work/new/run` said you were blocked and
653
+ # `mr-data build --output .work/new/run` then built, which is the expensive direction to
654
+ # be wrong in: it stops work that would have finished. The refusal being quoted,
655
+ # `OUTPUT_PARENT_ABSENT`, belongs to `cli._write_output_exclusive` -- author, approve and
656
+ # export-hosted-candidate, the writers that really do make nothing on the way -- and its
657
+ # third sentence stated that as a property of every output, which the build does not have.
658
+ # A rewording would have left the two answers contradicting each other, so what changed is
659
+ # which answer this gives.
660
+ return PreflightCheck(
661
+ name=OUTPUT_CHECK,
662
+ outcome=OK,
663
+ sentence=f"{destination} is free, and there is {above} at {parent} yet; "
664
+ "a build creates the folders on the way to its output",
665
+ )
666
+ if above != A_FOLDER:
667
+ return _blocked(
668
+ OUTPUT_CHECK,
669
+ f"there is {above or UNKNOWN_KIND} at {parent}, so nothing can be written inside it",
670
+ # The writer's own code for this exact path, not the missing-folder one: `mkdir -p` on
671
+ # a path whose parent is a file fails with "File exists", so the warning surface used
672
+ # to name a fix that cannot be carried out and disagreed with the refusal it predicts
673
+ # (`cli._write_output_exclusive` raises `OUTPUT_PARENT_INVALID` here).
674
+ #
675
+ # The subject is the output rather than its parent, because the refusal's subject is
676
+ # the output and the two surfaces are meant to print one wording. The observation is
677
+ # the parent's, which is what the sentence names.
678
+ _advice(
679
+ "OUTPUT_PARENT_INVALID",
680
+ path=str(destination),
681
+ path_exists=False,
682
+ path_kind=above,
683
+ ),
684
+ )
685
+ if not os.access(parent, os.W_OK | os.X_OK):
686
+ return _blocked(
687
+ OUTPUT_CHECK,
688
+ f"{parent} cannot be written to by this account",
689
+ _advice("PERMISSION_DENIED", path=str(parent), path_exists=True),
690
+ )
691
+ return PreflightCheck(
692
+ name=OUTPUT_CHECK,
693
+ outcome=OK,
694
+ sentence=f"{destination} is free, and {parent} can be written to",
695
+ )
696
+
697
+
698
+ def _holds_a_build(destination: Path) -> bool | None:
699
+ """Whether what is already at that path is a Build, or ``None`` when it cannot be told.
700
+
701
+ The reader the command line's error boundary uses, asked here for the same reason: a sentence
702
+ about an overwritten build, and a command that opens one, are answers only where a Build is
703
+ what is in the way.
704
+ """
705
+
706
+ from mostlyright.data_harness.ux.readers import holds_a_build
707
+
708
+ try:
709
+ return holds_a_build(destination)
710
+ except OSError:
711
+ return None
712
+
713
+
714
+ def _linked_ancestor(destination: Path) -> Path | None:
715
+ """The first component of the output's parent chain that is a link, if any.
716
+
717
+ The walk is the writer's own walk, deliberately: the path is made absolute the way the writer
718
+ makes it absolute -- ``Path.absolute``, which prepends the working directory and collapses
719
+ nothing -- and each component above the output is then checked without following anything,
720
+ which is what ``pipeline._open_output_parent`` and ``cli._open_cli_directory_chain`` do before
721
+ they raise ``OUTPUT_SYMLINK``. A component that is not there yet cannot be a link and is
722
+ created by the writer, so it is passed over rather than reported.
723
+
724
+ ``os.path.abspath`` was used here and is the one thing this walk may not do: it collapses
725
+ ``..`` *lexically*, so ``/tmp/link/../out`` became ``/tmp/out`` and the link component the
726
+ writer meets on the way down was deleted from the path before it was ever looked at. Preflight
727
+ then reported ``[ok]`` about a path the write refuses as a blocker.
728
+
729
+ The output's own name is not walked, because the writer creates it. That is the one difference
730
+ from the input side, which walks its root's own name too -- see :func:`_input_check` -- and it
731
+ is why the shared walk below takes the path to walk rather than deciding for itself.
732
+ """
733
+
734
+ return _first_link_in(Path(os.fspath(destination)).absolute().parent)
735
+
736
+
737
+ def _first_link_in(walked_to: Path) -> Path | None:
738
+ """The first component of ``walked_to``, its own name included, that is a link.
739
+
740
+ One walk serves both input and output checks because ``pipeline._open_source_root`` and the
741
+ output writer apply the same non-following ancestor rule.
742
+ """
743
+
744
+ walked = Path(walked_to.anchor)
745
+ for part in walked_to.parts[1:]:
746
+ walked = walked / part
747
+ # Use `ux.path_kind` for the decision as well as the rendered noun so the test and sentence
748
+ # cannot drift into different answers about the same entry.
749
+ if kind_at(walked) == A_LINK:
750
+ return walked
751
+ return None
752
+
753
+
754
+ # ------------------------------------------------------------------------------------------------
755
+ # The engine, asked for only when one is named
756
+ # ------------------------------------------------------------------------------------------------
757
+
758
+
759
+ def _engine_check(engine: str | None) -> PreflightCheck:
760
+ """Whether this build has that engine, and whether its library is installed here.
761
+
762
+ The library import happens through the backend's own version call, which is the same call a
763
+ build makes. Nothing on the default path reaches this function, so asking for a preflight does
764
+ not import an engine the caller never mentioned.
765
+ """
766
+
767
+ if engine is None:
768
+ return _not_asked(ENGINE_CHECK, "no engine was named, so the default one is assumed")
769
+ offered = ", ".join(sorted(_VERIFY_ALLOWLISTED_ENGINES))
770
+ if engine not in _VERIFY_ALLOWLISTED_ENGINES:
771
+ return _blocked(
772
+ ENGINE_CHECK,
773
+ f"there is no {engine} engine in this build; it has: {offered}",
774
+ (f"Ask for one of: {offered}.",),
775
+ )
776
+ from mostlyright.data_harness.backends import registry
777
+
778
+ try:
779
+ versions = registry.get_backend(engine).versions()
780
+ except Exception as error:
781
+ # Deliberately broad: an engine whose library will not load is an answer this command was
782
+ # asked for, and every way a library fails to load is that same answer.
783
+ return _blocked(
784
+ ENGINE_CHECK,
785
+ f"the {engine} engine is in this build, but its library will not load here: "
786
+ f"{_plainly(error)}",
787
+ (f"Install {engine} in the environment you are running mr-data from.",),
788
+ )
789
+ if not versions:
790
+ return PreflightCheck(
791
+ name=ENGINE_CHECK,
792
+ outcome=OK,
793
+ sentence=f"{engine} {_BUILT_IN}",
794
+ )
795
+ installed = ", ".join(f"{name} {value}" for name, value in sorted(versions.items()))
796
+ return PreflightCheck(
797
+ name=ENGINE_CHECK,
798
+ outcome=OK,
799
+ sentence=f"{engine} is installed here: {installed}",
800
+ )
801
+
802
+
803
+ # ------------------------------------------------------------------------------------------------
804
+ # The workload, asked for only when one is named
805
+ # ------------------------------------------------------------------------------------------------
806
+
807
+
808
+ @dataclass(frozen=True)
809
+ class _WorkloadAnswer:
810
+ """One line for the check list, and the table behind it when there was one."""
811
+
812
+ check: PreflightCheck
813
+ workload: Workload | None
814
+
815
+
816
+ def _workload_facts(
817
+ workload: Path | str | None,
818
+ *,
819
+ input_root: Path | str | None,
820
+ ) -> _WorkloadAnswer:
821
+ """What this job weighs, against what the engine allows, before any of it is uploaded.
822
+
823
+ This is the one check that reads the data rather than the machine, and it is deliberately the
824
+ cheapest reading of it there is: the plan, and one ``lstat`` per source. No source is opened,
825
+ no graph is executed, and nothing is fetched, so the answer costs about what the six checks
826
+ above it cost and can honestly be the first thing anybody runs.
827
+
828
+ A workload that cannot be read is reported as a blocking check rather than raised, for the
829
+ same reason every other branch here is: "that is not a plan I can read" is an answer to the
830
+ question this command asks. The reason travels through the same plain-language boundary the
831
+ caught refusals above go through, so a raise-site message never reaches a person untranslated.
832
+ """
833
+
834
+ if workload is None:
835
+ return _WorkloadAnswer(
836
+ check=_not_asked(
837
+ WORKLOAD_CHECK,
838
+ "no workload was named, so nothing was measured about the job itself; without "
839
+ "one this command answers only for this machine",
840
+ ),
841
+ workload=None,
842
+ )
843
+ try:
844
+ facts = read_workload(workload, input_root=input_root)
845
+ except (WorkloadUnreadable, OSError) as error:
846
+ # The refusal underneath is preferred where the reader that raised it carried a code of
847
+ # its own: `plan.schema_version is unsupported` has a sentence written for it, and
848
+ # "the workload could not be read" is the class of failure rather than the failure.
849
+ code = getattr(error, "refused_as", None) or getattr(error, "code", None)
850
+ return _WorkloadAnswer(
851
+ check=PreflightCheck(
852
+ name=WORKLOAD_CHECK,
853
+ outcome=BLOCKED,
854
+ sentence=f"the workload at {workload} could not be read: {_plainly(error)}",
855
+ remediation=_advice(
856
+ code if isinstance(code, str) else None,
857
+ path=str(workload),
858
+ ),
859
+ ),
860
+ workload=None,
861
+ )
862
+ return _WorkloadAnswer(
863
+ check=PreflightCheck(
864
+ name=WORKLOAD_CHECK,
865
+ outcome=BLOCKED if facts.blocked else OK,
866
+ sentence=facts.sentence,
867
+ remediation=facts.remediation,
868
+ ),
869
+ workload=facts,
870
+ )
871
+
872
+
873
+ # ------------------------------------------------------------------------------------------------
874
+ # Small shared shapes
875
+ # ------------------------------------------------------------------------------------------------
876
+
877
+
878
+ def _not_asked(name: str, sentence: str) -> PreflightCheck:
879
+ return PreflightCheck(name=name, outcome=NOT_APPLICABLE, sentence=sentence)
880
+
881
+
882
+ def _blocked(name: str, sentence: str, remediation: Sequence[str]) -> PreflightCheck:
883
+ return PreflightCheck(
884
+ name=name,
885
+ outcome=BLOCKED,
886
+ sentence=sentence,
887
+ remediation=tuple(remediation),
888
+ )
889
+
890
+
891
+ # The stage sentences are held to cover every stored stage by a test rather than by an assertion
892
+ # here, so a new stage cannot arrive without a plain sentence to show for it.
893
+ __all__ = [
894
+ "BLOCKED",
895
+ "BLOCKED_KEY",
896
+ "CHECKS_KEY",
897
+ "ENGINE_CHECK",
898
+ "INPUT_CHECK",
899
+ "NOTHING_TO_DO",
900
+ "NOT_APPLICABLE",
901
+ "OK",
902
+ "OUTPUT_CHECK",
903
+ "PHASE_SENTENCES",
904
+ "PREFLIGHT_SCHEMA_VERSION",
905
+ "PREFLIGHT_STATUS",
906
+ "STEP_NAMES",
907
+ "STEP_STATES",
908
+ "VERIFYING_CHECK",
909
+ "WATCHING_CHECK",
910
+ "WHAT_TO_DO_KEY",
911
+ "WORKBENCH_CHECK",
912
+ "WORKLOAD_CHECK",
913
+ "WORKLOAD_KEY",
914
+ "WRITING_CHECK",
915
+ "Preflight",
916
+ "PreflightCheck",
917
+ "run_preflight",
918
+ ]