mostlyright-data 0.9.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (314) hide show
  1. mostlyright/data_harness/__init__.py +158 -0
  2. mostlyright/data_harness/acquisition/__init__.py +55 -0
  3. mostlyright/data_harness/acquisition/http.py +2773 -0
  4. mostlyright/data_harness/acquisition/parsing.py +809 -0
  5. mostlyright/data_harness/acquisition/ranges.py +495 -0
  6. mostlyright/data_harness/acquisition/result_download.py +360 -0
  7. mostlyright/data_harness/acquisition/retention_admission.py +248 -0
  8. mostlyright/data_harness/acquisition/sandbox.py +4888 -0
  9. mostlyright/data_harness/acquisition/url_policy.py +530 -0
  10. mostlyright/data_harness/agent_runtime.py +2743 -0
  11. mostlyright/data_harness/assets/logo-ink.svg +31 -0
  12. mostlyright/data_harness/backends/__init__.py +28 -0
  13. mostlyright/data_harness/backends/pandas_backend.py +350 -0
  14. mostlyright/data_harness/backends/polars_backend.py +366 -0
  15. mostlyright/data_harness/backends/protocol.py +124 -0
  16. mostlyright/data_harness/backends/reference.py +83 -0
  17. mostlyright/data_harness/backends/registry.py +55 -0
  18. mostlyright/data_harness/backends/restrictions.py +126 -0
  19. mostlyright/data_harness/canonical.py +333 -0
  20. mostlyright/data_harness/catalog_job.py +625 -0
  21. mostlyright/data_harness/cli.py +5398 -0
  22. mostlyright/data_harness/contracts.py +53 -0
  23. mostlyright/data_harness/coordinator.py +1307 -0
  24. mostlyright/data_harness/deploy.py +924 -0
  25. mostlyright/data_harness/deploy_target.py +312 -0
  26. mostlyright/data_harness/deployment_evidence.py +1067 -0
  27. mostlyright/data_harness/event_presentation.py +576 -0
  28. mostlyright/data_harness/events.py +2152 -0
  29. mostlyright/data_harness/fast_delimited.py +239 -0
  30. mostlyright/data_harness/fleet.py +237 -0
  31. mostlyright/data_harness/formats.py +236 -0
  32. mostlyright/data_harness/governors.py +1163 -0
  33. mostlyright/data_harness/hosted_bootstrap.py +972 -0
  34. mostlyright/data_harness/hosted_crawler.py +1115 -0
  35. mostlyright/data_harness/hosted_crawler_container_smoke.py +351 -0
  36. mostlyright/data_harness/hosted_crawler_fetch.py +423 -0
  37. mostlyright/data_harness/hosted_crawler_job.py +1277 -0
  38. mostlyright/data_harness/hosted_crawler_protocol.py +676 -0
  39. mostlyright/data_harness/hosted_dataset.py +1500 -0
  40. mostlyright/data_harness/hosted_deploy.py +3037 -0
  41. mostlyright/data_harness/hosted_handoff.py +62 -0
  42. mostlyright/data_harness/hosted_ingestion_contract.py +504 -0
  43. mostlyright/data_harness/hosted_ingestion_job.py +356 -0
  44. mostlyright/data_harness/hosted_ingestion_job_smoke.py +40 -0
  45. mostlyright/data_harness/hosted_session_container_smoke.py +194 -0
  46. mostlyright/data_harness/hosted_session_worker.py +3554 -0
  47. mostlyright/data_harness/hosted_session_worker_job_smoke.py +46 -0
  48. mostlyright/data_harness/hosted_worker.py +6784 -0
  49. mostlyright/data_harness/ingestion/__init__.py +56 -0
  50. mostlyright/data_harness/ingestion/contracts.py +461 -0
  51. mostlyright/data_harness/ingestion/faults.py +42 -0
  52. mostlyright/data_harness/ingestion/gcs_store.py +1162 -0
  53. mostlyright/data_harness/ingestion/spool.py +130 -0
  54. mostlyright/data_harness/ingestion/store.py +885 -0
  55. mostlyright/data_harness/key_seam.py +434 -0
  56. mostlyright/data_harness/linux_process_boundary.py +262 -0
  57. mostlyright/data_harness/local_contracts.py +2880 -0
  58. mostlyright/data_harness/local_search/__init__.py +5 -0
  59. mostlyright/data_harness/local_search/build_index.py +1087 -0
  60. mostlyright/data_harness/local_search/contracts.py +920 -0
  61. mostlyright/data_harness/local_search/query_trace.py +266 -0
  62. mostlyright/data_harness/local_search/retrieval.py +700 -0
  63. mostlyright/data_harness/local_search/sealed.py +474 -0
  64. mostlyright/data_harness/local_search/service.py +784 -0
  65. mostlyright/data_harness/nbrender/CONTRACT.md +212 -0
  66. mostlyright/data_harness/nbrender/__init__.py +12 -0
  67. mostlyright/data_harness/nbrender/chrome.py +359 -0
  68. mostlyright/data_harness/nbrender/code_body.py +266 -0
  69. mostlyright/data_harness/nbrender/document.py +407 -0
  70. mostlyright/data_harness/nbrender/frame.py +275 -0
  71. mostlyright/data_harness/nbrender/interactive.py +337 -0
  72. mostlyright/data_harness/nbrender/markdown_body.py +477 -0
  73. mostlyright/data_harness/nbrender/mr_components.py +134 -0
  74. mostlyright/data_harness/nbrender/outputs_data.py +595 -0
  75. mostlyright/data_harness/nbrender/outputs_rich.py +906 -0
  76. mostlyright/data_harness/nbrender/outputs_source.py +260 -0
  77. mostlyright/data_harness/nbrender/outputs_stage.py +176 -0
  78. mostlyright/data_harness/nbrender/outputs_text.py +400 -0
  79. mostlyright/data_harness/nbrender/parse.py +394 -0
  80. mostlyright/data_harness/nbrender/status.py +40 -0
  81. mostlyright/data_harness/nbrender/tokens.py +1295 -0
  82. mostlyright/data_harness/notebook.py +1710 -0
  83. mostlyright/data_harness/offline.py +2049 -0
  84. mostlyright/data_harness/operation_registry.py +1007 -0
  85. mostlyright/data_harness/operator_setup.py +239 -0
  86. mostlyright/data_harness/pipeline.py +6428 -0
  87. mostlyright/data_harness/plan_graph.py +2026 -0
  88. mostlyright/data_harness/preparation/__init__.py +104 -0
  89. mostlyright/data_harness/preparation/contracts.py +1017 -0
  90. mostlyright/data_harness/preparation/engine.py +221 -0
  91. mostlyright/data_harness/preparation/errors.py +14 -0
  92. mostlyright/data_harness/preparation/gates.py +751 -0
  93. mostlyright/data_harness/preparation/joins.py +574 -0
  94. mostlyright/data_harness/preparation/profile.py +384 -0
  95. mostlyright/data_harness/preparation/table.py +217 -0
  96. mostlyright/data_harness/preparation/transforms.py +568 -0
  97. mostlyright/data_harness/progress_events.py +534 -0
  98. mostlyright/data_harness/readers/__init__.py +46 -0
  99. mostlyright/data_harness/readers/containers.py +963 -0
  100. mostlyright/data_harness/readers/contracts.py +542 -0
  101. mostlyright/data_harness/readers/delimited.py +257 -0
  102. mostlyright/data_harness/readers/grib2/__init__.py +33 -0
  103. mostlyright/data_harness/readers/grib2/admission.py +722 -0
  104. mostlyright/data_harness/readers/grib2/decode.py +1009 -0
  105. mostlyright/data_harness/readers/grib2/geometry.py +1133 -0
  106. mostlyright/data_harness/readers/grib2/portable_math.py +501 -0
  107. mostlyright/data_harness/readers/json_tabular.py +485 -0
  108. mostlyright/data_harness/readers/registry.py +514 -0
  109. mostlyright/data_harness/readers/samples/README.md +110 -0
  110. mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/cities.csv.gz +0 -0
  111. mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/expected.json +24 -0
  112. mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/cities.csv.gz +0 -0
  113. mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/expected.json +24 -0
  114. mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/cities.tar +0 -0
  115. mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/expected.json +24 -0
  116. mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/cities.tar +0 -0
  117. mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/expected.json +24 -0
  118. mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/cities.zip +0 -0
  119. mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/expected.json +25 -0
  120. mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
  121. mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/expected.json +25 -0
  122. mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
  123. mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/expected.json +25 -0
  124. mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/cities.csv +3 -0
  125. mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/expected.json +23 -0
  126. mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/cities.tsv +5 -0
  127. mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/expected.json +25 -0
  128. mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/expected.json +30 -0
  129. mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/observations.csv +5 -0
  130. mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/expected.json +44 -0
  131. mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/stations.json +1 -0
  132. mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/expected.json +48 -0
  133. mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/observations.ndjson +4 -0
  134. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/cities.xlsx +0 -0
  135. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/expected.json +24 -0
  136. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/expected.json +27 -0
  137. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
  138. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/expected.json +27 -0
  139. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
  140. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/README.md +20 -0
  141. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/expected.json +55 -0
  142. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/gfs-2m-temperature.grib2 +0 -0
  143. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/expected.json +54 -0
  144. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
  145. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/expected.json +54 -0
  146. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/hrrr-categorical-rain.grib2 +0 -0
  147. mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/expected.json +54 -0
  148. mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
  149. mostlyright/data_harness/readers/samples.py +582 -0
  150. mostlyright/data_harness/readers/spreadsheet.py +803 -0
  151. mostlyright/data_harness/readers/tabular.py +510 -0
  152. mostlyright/data_harness/recipe.py +5321 -0
  153. mostlyright/data_harness/repair/__init__.py +78 -0
  154. mostlyright/data_harness/repair/adapters.py +274 -0
  155. mostlyright/data_harness/repair/contracts.py +872 -0
  156. mostlyright/data_harness/repair/coordinator.py +1099 -0
  157. mostlyright/data_harness/repair/errors.py +16 -0
  158. mostlyright/data_harness/review.py +2533 -0
  159. mostlyright/data_harness/rowset.py +283 -0
  160. mostlyright/data_harness/serving.py +1975 -0
  161. mostlyright/data_harness/serving_edge.py +590 -0
  162. mostlyright/data_harness/serving_http.py +1031 -0
  163. mostlyright/data_harness/session_probes.py +759 -0
  164. mostlyright/data_harness/signing.py +101 -0
  165. mostlyright/data_harness/source_discovery.py +898 -0
  166. mostlyright/data_harness/sources/__init__.py +209 -0
  167. mostlyright/data_harness/sources/_adapter_steps.py +213 -0
  168. mostlyright/data_harness/sources/adapters.py +1214 -0
  169. mostlyright/data_harness/sources/cadence.py +1428 -0
  170. mostlyright/data_harness/sources/cadence_emission.py +453 -0
  171. mostlyright/data_harness/sources/cadence_history.py +546 -0
  172. mostlyright/data_harness/sources/catalog/__init__.py +17 -0
  173. mostlyright/data_harness/sources/catalog/admission.py +477 -0
  174. mostlyright/data_harness/sources/catalog/authoring.py +1701 -0
  175. mostlyright/data_harness/sources/catalog/authoring_policy.py +701 -0
  176. mostlyright/data_harness/sources/catalog/authoring_shards.py +1217 -0
  177. mostlyright/data_harness/sources/catalog/bounded_io.py +231 -0
  178. mostlyright/data_harness/sources/catalog/channel.py +523 -0
  179. mostlyright/data_harness/sources/catalog/channel_client.py +296 -0
  180. mostlyright/data_harness/sources/catalog/contracts.py +825 -0
  181. mostlyright/data_harness/sources/catalog/coverage.py +137 -0
  182. mostlyright/data_harness/sources/catalog/delta.py +1340 -0
  183. mostlyright/data_harness/sources/catalog/embedding.py +532 -0
  184. mostlyright/data_harness/sources/catalog/entry_v2.py +1182 -0
  185. mostlyright/data_harness/sources/catalog/fill.py +3889 -0
  186. mostlyright/data_harness/sources/catalog/fill_partitions.py +459 -0
  187. mostlyright/data_harness/sources/catalog/fill_staging.py +1105 -0
  188. mostlyright/data_harness/sources/catalog/gating.py +374 -0
  189. mostlyright/data_harness/sources/catalog/generation_receipt.py +1607 -0
  190. mostlyright/data_harness/sources/catalog/harvest/__init__.py +7 -0
  191. mostlyright/data_harness/sources/catalog/harvest/ckan.py +384 -0
  192. mostlyright/data_harness/sources/catalog/harvest/datagov_v4.py +798 -0
  193. mostlyright/data_harness/sources/catalog/harvest/protocol.py +964 -0
  194. mostlyright/data_harness/sources/catalog/harvest/sdmx.py +445 -0
  195. mostlyright/data_harness/sources/catalog/harvest/stac.py +384 -0
  196. mostlyright/data_harness/sources/catalog/health.py +447 -0
  197. mostlyright/data_harness/sources/catalog/hosted_catalog.py +105 -0
  198. mostlyright/data_harness/sources/catalog/identity_history.py +1549 -0
  199. mostlyright/data_harness/sources/catalog/neural.py +1618 -0
  200. mostlyright/data_harness/sources/catalog/packed_catalog.py +2345 -0
  201. mostlyright/data_harness/sources/catalog/packed_retrieval.py +1517 -0
  202. mostlyright/data_harness/sources/catalog/packed_writer.py +2802 -0
  203. mostlyright/data_harness/sources/catalog/query_trace.py +1037 -0
  204. mostlyright/data_harness/sources/catalog/recommend.py +171 -0
  205. mostlyright/data_harness/sources/catalog/retrieval.py +230 -0
  206. mostlyright/data_harness/sources/catalog/retrieval_manifest.py +995 -0
  207. mostlyright/data_harness/sources/catalog/rights_decisions.py +254 -0
  208. mostlyright/data_harness/sources/catalog/sealed.py +560 -0
  209. mostlyright/data_harness/sources/catalog/search.py +230 -0
  210. mostlyright/data_harness/sources/catalog/streaming_delta.py +1097 -0
  211. mostlyright/data_harness/sources/catalog/update.py +891 -0
  212. mostlyright/data_harness/sources/collections.py +815 -0
  213. mostlyright/data_harness/sources/contracts.py +2223 -0
  214. mostlyright/data_harness/sources/deletion.py +761 -0
  215. mostlyright/data_harness/sources/fitness.py +162 -0
  216. mostlyright/data_harness/sources/governance.py +163 -0
  217. mostlyright/data_harness/sources/hosted.py +173 -0
  218. mostlyright/data_harness/sources/integration.py +218 -0
  219. mostlyright/data_harness/sources/range_reader.py +418 -0
  220. mostlyright/data_harness/sources/registry.py +514 -0
  221. mostlyright/data_harness/sources/rights_rule.py +59 -0
  222. mostlyright/data_harness/sources/source_cadence_vectors.v1.json +1 -0
  223. mostlyright/data_harness/sources/sports.py +521 -0
  224. mostlyright/data_harness/sources/stream.py +524 -0
  225. mostlyright/data_harness/sources/stream_connector.py +418 -0
  226. mostlyright/data_harness/sources/stream_recorder.py +1404 -0
  227. mostlyright/data_harness/studio_boundary.py +2019 -0
  228. mostlyright/data_harness/thin/__init__.py +37 -0
  229. mostlyright/data_harness/thin/acquire.py +1137 -0
  230. mostlyright/data_harness/thin/acquire_cancel.py +579 -0
  231. mostlyright/data_harness/thin/approvals.py +617 -0
  232. mostlyright/data_harness/thin/commands.py +406 -0
  233. mostlyright/data_harness/thin/download.py +194 -0
  234. mostlyright/data_harness/thin/narrative.py +589 -0
  235. mostlyright/data_harness/thin/parity.py +1070 -0
  236. mostlyright/data_harness/thin/propose.py +2759 -0
  237. mostlyright/data_harness/thin/research.py +1663 -0
  238. mostlyright/data_harness/thin/router.py +924 -0
  239. mostlyright/data_harness/thin/runs.py +519 -0
  240. mostlyright/data_harness/thin/session.py +281 -0
  241. mostlyright/data_harness/thin/stream.py +501 -0
  242. mostlyright/data_harness/thin/transport.py +187 -0
  243. mostlyright/data_harness/thin/vocabulary.py +368 -0
  244. mostlyright/data_harness/thin/workers.py +164 -0
  245. mostlyright/data_harness/ucum/TABLE-PIN.json +40 -0
  246. mostlyright/data_harness/ucum/ucum-subset.v1.json +632 -0
  247. mostlyright/data_harness/unit_flow.py +927 -0
  248. mostlyright/data_harness/units.py +572 -0
  249. mostlyright/data_harness/ux/__init__.py +9 -0
  250. mostlyright/data_harness/ux/approve.py +485 -0
  251. mostlyright/data_harness/ux/author_yaml.py +597 -0
  252. mostlyright/data_harness/ux/cloud_auth.py +447 -0
  253. mostlyright/data_harness/ux/commands/__init__.py +260 -0
  254. mostlyright/data_harness/ux/commands/approve.py +136 -0
  255. mostlyright/data_harness/ux/commands/auth.py +744 -0
  256. mostlyright/data_harness/ux/commands/author.py +79 -0
  257. mostlyright/data_harness/ux/commands/catalog_author.py +403 -0
  258. mostlyright/data_harness/ux/commands/catalog_fill.py +523 -0
  259. mostlyright/data_harness/ux/commands/catalog_harvest.py +545 -0
  260. mostlyright/data_harness/ux/commands/catalog_publish.py +1838 -0
  261. mostlyright/data_harness/ux/commands/catalog_search.py +71 -0
  262. mostlyright/data_harness/ux/commands/catalog_update.py +437 -0
  263. mostlyright/data_harness/ux/commands/deploy.py +134 -0
  264. mostlyright/data_harness/ux/commands/deploy_dataset.py +98 -0
  265. mostlyright/data_harness/ux/commands/deploy_plan.py +105 -0
  266. mostlyright/data_harness/ux/commands/deploy_status.py +104 -0
  267. mostlyright/data_harness/ux/commands/diff.py +74 -0
  268. mostlyright/data_harness/ux/commands/index.py +84 -0
  269. mostlyright/data_harness/ux/commands/inventory.py +47 -0
  270. mostlyright/data_harness/ux/commands/list_builds.py +143 -0
  271. mostlyright/data_harness/ux/commands/login.py +63 -0
  272. mostlyright/data_harness/ux/commands/peek.py +236 -0
  273. mostlyright/data_harness/ux/commands/plan_check.py +90 -0
  274. mostlyright/data_harness/ux/commands/preflight.py +97 -0
  275. mostlyright/data_harness/ux/commands/record.py +107 -0
  276. mostlyright/data_harness/ux/commands/review_setup.py +47 -0
  277. mostlyright/data_harness/ux/commands/search.py +440 -0
  278. mostlyright/data_harness/ux/commands/show.py +61 -0
  279. mostlyright/data_harness/ux/commands/whoami.py +37 -0
  280. mostlyright/data_harness/ux/credential_native.py +551 -0
  281. mostlyright/data_harness/ux/credential_store.py +1055 -0
  282. mostlyright/data_harness/ux/credentials.py +631 -0
  283. mostlyright/data_harness/ux/diffing.py +444 -0
  284. mostlyright/data_harness/ux/headline.py +671 -0
  285. mostlyright/data_harness/ux/hosted_acquisition.py +974 -0
  286. mostlyright/data_harness/ux/hosted_run_status.py +619 -0
  287. mostlyright/data_harness/ux/inventory.py +427 -0
  288. mostlyright/data_harness/ux/local_review.py +375 -0
  289. mostlyright/data_harness/ux/login.py +691 -0
  290. mostlyright/data_harness/ux/path_kind.py +147 -0
  291. mostlyright/data_harness/ux/peek.py +1000 -0
  292. mostlyright/data_harness/ux/plain_file.py +178 -0
  293. mostlyright/data_harness/ux/plan_check.py +311 -0
  294. mostlyright/data_harness/ux/preflight.py +918 -0
  295. mostlyright/data_harness/ux/readers.py +1124 -0
  296. mostlyright/data_harness/ux/remediation.py +2195 -0
  297. mostlyright/data_harness/ux/render.py +657 -0
  298. mostlyright/data_harness/ux/workload.py +1077 -0
  299. mostlyright/data_harness/viewer.py +3713 -0
  300. mostlyright/data_harness/visual_run/__init__.py +83 -0
  301. mostlyright/data_harness/visual_run/authoring.py +235 -0
  302. mostlyright/data_harness/visual_run/contracts.py +673 -0
  303. mostlyright/data_harness/visual_run/materialize.py +486 -0
  304. mostlyright/data_harness/visual_run/observations.py +874 -0
  305. mostlyright/data_harness/visual_run/query.py +259 -0
  306. mostlyright/data_harness/visual_run/reducer.py +280 -0
  307. mostlyright/data_harness/visual_run/sdk.py +892 -0
  308. mostlyright/data_harness/visual_run/store.py +584 -0
  309. mostlyright/data_harness/visual_run/transport.py +239 -0
  310. mostlyright/data_harness/watch.py +2999 -0
  311. mostlyright_data-0.9.0.dist-info/METADATA +607 -0
  312. mostlyright_data-0.9.0.dist-info/RECORD +314 -0
  313. mostlyright_data-0.9.0.dist-info/WHEEL +4 -0
  314. mostlyright_data-0.9.0.dist-info/entry_points.txt +12 -0
@@ -0,0 +1,2049 @@
1
+ """Durable local coordinator for the H2 offline vertical slice."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import errno
6
+ import os
7
+ import re
8
+ import stat
9
+ import uuid
10
+ from collections.abc import Callable
11
+ from copy import deepcopy
12
+ from dataclasses import dataclass
13
+ from pathlib import Path, PurePosixPath
14
+ from typing import TYPE_CHECKING, Any
15
+
16
+ from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey
17
+
18
+ try:
19
+ import fcntl
20
+ except ImportError: # pragma: no cover - H2 already fails closed on Windows
21
+ fcntl = None # type: ignore[assignment]
22
+
23
+ from mostlyright.data_harness.canonical import (
24
+ CanonicalJSONError,
25
+ canonical_json_bytes,
26
+ parse_canonical_json,
27
+ parse_json,
28
+ sha256_bytes,
29
+ )
30
+ from mostlyright.data_harness.contracts import ContractError, TablePlan, parse_plan
31
+ from mostlyright.data_harness.local_contracts import (
32
+ GraphTablePlan,
33
+ LocalQuestion,
34
+ LocalRequirements,
35
+ SourceProposal,
36
+ TableSemantics,
37
+ parse_question,
38
+ parse_requirements,
39
+ parse_source_proposal,
40
+ parse_table_semantics,
41
+ )
42
+
43
+ # The underscore-prefixed names below are private to pipeline.py by convention only; this
44
+ # module imports and calls each of them directly, so their signatures have to change
45
+ # together with the call sites here.
46
+ from mostlyright.data_harness.pipeline import (
47
+ OFFLINE_MANIFEST_VERSION,
48
+ OFFLINE_SEMANTICS_MANIFEST_VERSION,
49
+ SOURCE_PROPOSAL_SET_VERSION,
50
+ BuildError,
51
+ CandidateInspection,
52
+ OfflinePackageContext,
53
+ ResourceLimits,
54
+ _durability_barrier,
55
+ _entry_identity,
56
+ _open_candidate_run_handle,
57
+ _open_output_parent,
58
+ _open_source_root,
59
+ _read_confined_source_from_handle,
60
+ _remove_tree,
61
+ _rename_no_replace,
62
+ _validate_candidate_run_handle,
63
+ _validate_output_parent_handle,
64
+ _validate_root_separation,
65
+ build_candidate,
66
+ classify_plan_sources_before_persistence,
67
+ inspect_candidate,
68
+ )
69
+ from mostlyright.data_harness.review import (
70
+ ReviewResult,
71
+ _finalize_review,
72
+ _review_authority_material,
73
+ _verify_review_decision,
74
+ )
75
+ from mostlyright.data_harness.signing import (
76
+ TrustedReviewAuthorityProvider,
77
+ TrustedSigner,
78
+ )
79
+ from mostlyright.data_harness.ux.path_kind import UNKNOWN_KIND, name_of_kind, presence_at
80
+ from mostlyright.data_harness.ux.plain_file import (
81
+ DANGLING,
82
+ NOT_PLAIN,
83
+ TOO_LARGE,
84
+ PlainFileRefusal,
85
+ read_plain_file,
86
+ )
87
+
88
+ if TYPE_CHECKING:
89
+ from mostlyright.data_harness.visual_run.store import VisualRunStore
90
+
91
+ _OFFLINE_MANIFEST_VERSIONS = frozenset(
92
+ {OFFLINE_MANIFEST_VERSION, OFFLINE_SEMANTICS_MANIFEST_VERSION}
93
+ )
94
+
95
+ RUN_STATE_VERSION = "offline-run-state.v1"
96
+ RUN_RESULT_PATH = "result"
97
+ INPUT_ROOT_PATH = "inputs"
98
+ DEFINITION_PATH = "definitions"
99
+ CONTROL_PATH = ".mr-data"
100
+ WORKSPACE_LOCK_NAME = ".lock"
101
+ # Retain the old private spelling for callers that reached into this module before the workspace
102
+ # control contract had a public lock name. New code must use ``WORKSPACE_LOCK_NAME``.
103
+ _LOCK_NAME = WORKSPACE_LOCK_NAME
104
+ _STATE_NAME = "state.json"
105
+ _DEFINITION_FILES = {
106
+ "question": "question.json",
107
+ "requirements": "requirements.json",
108
+ "source_proposals": "source-proposals.json",
109
+ "plan": "plan.json",
110
+ }
111
+ _SEMANTICS_DEFINITION = {"semantics": "semantics.json"}
112
+ _TASK_IDS = ("plan", "build", "review")
113
+ _PHASES = frozenset(
114
+ {
115
+ "initialized",
116
+ "planned",
117
+ "building",
118
+ "candidate_built",
119
+ "review_recorded",
120
+ "failed",
121
+ }
122
+ )
123
+ _TASK_STATUSES = frozenset({"pending", "running", "completed", "failed"})
124
+ _IDENTIFIER = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$")
125
+ _DIGEST = re.compile(r"^[0-9a-f]{64}$")
126
+ _ORPHAN_STAGING = re.compile(r"^\.result\.tmp-[0-9a-f]{32}$")
127
+ _MAX_STATE_BYTES = 8 * 1024 * 1024
128
+ _MAX_DEFINITION_BYTES = 4 * 1024 * 1024
129
+ _H2_PRODUCER_PRINCIPAL = "h2offlinecoordinatorproducer"
130
+ _H2_PRODUCER_ATTEMPT = "h2offlineinitialattempt"
131
+ _PHASE_TASK_STATUSES = {
132
+ "initialized": ("pending", "pending", "pending"),
133
+ "planned": ("completed", "pending", "pending"),
134
+ "building": ("completed", "running", "pending"),
135
+ "candidate_built": ("completed", "completed", "pending"),
136
+ "review_recorded": ("completed", "completed", "completed"),
137
+ "failed": ("completed", "failed", "pending"),
138
+ }
139
+
140
+
141
+ def _h2_producer_identity() -> dict[str, str]:
142
+ return {
143
+ "principal_id": _H2_PRODUCER_PRINCIPAL,
144
+ "attempt_id": _H2_PRODUCER_ATTEMPT,
145
+ }
146
+
147
+
148
+ class OfflineRunError(RuntimeError):
149
+ """A local workspace failed a closed coordinator invariant."""
150
+
151
+ def __init__(self, code: str, detail: str) -> None:
152
+ self.code = code
153
+ self.detail = detail
154
+ super().__init__(detail)
155
+
156
+
157
+ @dataclass(frozen=True)
158
+ class OfflineInputs:
159
+ question: LocalQuestion
160
+ requirements: LocalRequirements
161
+ source_proposals: tuple[SourceProposal, ...]
162
+ plan: TablePlan
163
+ semantics: TableSemantics | None = None
164
+
165
+ @property
166
+ def package_context(self) -> OfflinePackageContext:
167
+ return OfflinePackageContext(
168
+ question=self.question,
169
+ requirements=self.requirements,
170
+ source_proposals=self.source_proposals,
171
+ semantics=self.semantics,
172
+ )
173
+
174
+ def definition_values(self) -> dict[str, Any]:
175
+ values = {
176
+ "question": self.question.to_dict(),
177
+ "requirements": self.requirements.to_dict(),
178
+ "source_proposals": {
179
+ "schema_version": SOURCE_PROPOSAL_SET_VERSION,
180
+ "source_proposals": [
181
+ item.to_dict()
182
+ for item in sorted(self.source_proposals, key=lambda item: item.source_id)
183
+ ],
184
+ },
185
+ "plan": self.plan.to_dict(),
186
+ }
187
+ if self.semantics is not None:
188
+ values["semantics"] = self.semantics.to_dict()
189
+ return values
190
+
191
+
192
+ @dataclass(frozen=True)
193
+ class OfflineExecutionResult:
194
+ status: str
195
+ run_id: str
196
+ candidate: CandidateInspection
197
+ workspace: Path
198
+ recovered: bool
199
+ idempotent: bool
200
+
201
+ def to_dict(self) -> dict[str, Any]:
202
+ return {
203
+ "status": self.status,
204
+ "run_id": self.run_id,
205
+ "workspace": str(self.workspace),
206
+ "recovered": self.recovered,
207
+ "idempotent": self.idempotent,
208
+ "candidate": self.candidate.to_dict(),
209
+ }
210
+
211
+
212
+ @dataclass
213
+ class _Workspace:
214
+ path: Path
215
+ run_handle: Any
216
+ control_fd: int
217
+ control_identity: tuple[int, int, int]
218
+ lock_fd: int
219
+ lock_acquired: bool
220
+
221
+ @property
222
+ def root_fd(self) -> int:
223
+ return self.run_handle.run_fd
224
+
225
+ def validate(self) -> None:
226
+ _validate_candidate_run_handle(self.run_handle)
227
+ named = os.stat(CONTROL_PATH, dir_fd=self.root_fd, follow_symlinks=False)
228
+ opened = os.fstat(self.control_fd)
229
+ if (
230
+ not stat.S_ISDIR(named.st_mode)
231
+ or _entry_identity(named) != self.control_identity
232
+ or _entry_identity(opened) != self.control_identity
233
+ ):
234
+ raise OfflineRunError(
235
+ "WORKSPACE_RACE",
236
+ "workspace control directory changed identity",
237
+ )
238
+ lock_named = os.stat(WORKSPACE_LOCK_NAME, dir_fd=self.control_fd, follow_symlinks=False)
239
+ lock_opened = os.fstat(self.lock_fd)
240
+ if (
241
+ not stat.S_ISREG(lock_named.st_mode)
242
+ or lock_named.st_nlink != 1
243
+ or _stat_signature(lock_named) != _stat_signature(lock_opened)
244
+ ):
245
+ raise OfflineRunError("WORKSPACE_RACE", "workspace lock changed identity")
246
+
247
+ def close(self) -> None:
248
+ if self.lock_acquired and fcntl is not None:
249
+ fcntl.flock(self.lock_fd, fcntl.LOCK_UN)
250
+ os.close(self.lock_fd)
251
+ os.close(self.control_fd)
252
+ self.run_handle.close()
253
+
254
+
255
+ def load_offline_inputs(
256
+ *,
257
+ question_path: Path,
258
+ requirements_path: Path,
259
+ source_proposals_path: Path,
260
+ plan_path: Path,
261
+ semantics_path: Path | None = None,
262
+ ) -> OfflineInputs:
263
+ """Load strict JSON definitions and verify their complete H2 relationship."""
264
+
265
+ question = parse_question(_read_external_json(question_path, "question"))
266
+ requirements = parse_requirements(_read_external_json(requirements_path, "requirements"))
267
+ proposal_set = _read_external_json(source_proposals_path, "source proposals")
268
+ if not isinstance(proposal_set, dict) or set(proposal_set) != {
269
+ "schema_version",
270
+ "source_proposals",
271
+ }:
272
+ raise OfflineRunError(
273
+ "SOURCE_PROPOSAL_SET_INVALID",
274
+ "source proposal set fields are invalid",
275
+ )
276
+ if proposal_set["schema_version"] != SOURCE_PROPOSAL_SET_VERSION:
277
+ raise OfflineRunError(
278
+ "SOURCE_PROPOSAL_SET_INVALID",
279
+ "source proposal set version is unsupported",
280
+ )
281
+ proposal_values = proposal_set["source_proposals"]
282
+ if not isinstance(proposal_values, list):
283
+ raise OfflineRunError(
284
+ "SOURCE_PROPOSAL_SET_INVALID",
285
+ "source_proposals must be an array",
286
+ )
287
+ proposals = tuple(parse_source_proposal(item) for item in proposal_values)
288
+ plan = parse_plan(_read_external_json(plan_path, "plan"))
289
+ if isinstance(plan, GraphTablePlan):
290
+ raise OfflineRunError(
291
+ "GRAPH_OFFLINE_UNSUPPORTED",
292
+ "local-graph-table-plan.v1 must be executed through an approved Recipe or "
293
+ "direct plan-check",
294
+ )
295
+ semantics = (
296
+ parse_table_semantics(_read_external_json(semantics_path, "semantics"))
297
+ if semantics_path is not None
298
+ else None
299
+ )
300
+ inputs = OfflineInputs(
301
+ question=question,
302
+ requirements=requirements,
303
+ source_proposals=proposals,
304
+ plan=plan,
305
+ semantics=semantics,
306
+ )
307
+ inputs.package_context.validate_for_plan(plan)
308
+ return inputs
309
+
310
+
311
+ def init_workspace(
312
+ workspace_path: Path,
313
+ *,
314
+ run_id: str,
315
+ input_root: Path,
316
+ inputs: OfflineInputs,
317
+ limits: ResourceLimits | None = None,
318
+ ) -> dict[str, Any]:
319
+ """Snapshot two sources and atomically install one resumable local workspace."""
320
+
321
+ if isinstance(inputs.plan, GraphTablePlan):
322
+ raise OfflineRunError(
323
+ "GRAPH_OFFLINE_UNSUPPORTED",
324
+ "local-graph-table-plan.v1 must be executed through an approved Recipe or "
325
+ "direct plan-check",
326
+ )
327
+ run_id = _identifier(run_id, "run_id")
328
+ inputs.package_context.validate_for_plan(inputs.plan)
329
+ limits = limits or ResourceLimits()
330
+ workspace = Path(os.path.abspath(os.fspath(workspace_path)))
331
+ source_handle = _open_source_root(input_root)
332
+ try:
333
+ _validate_root_separation(source_handle.path, workspace)
334
+ source_bytes = {
335
+ source.source_id: _read_confined_source_from_handle(
336
+ source_handle,
337
+ source.path,
338
+ limits,
339
+ )
340
+ for source in inputs.plan.sources
341
+ }
342
+ finally:
343
+ source_handle.close()
344
+ classify_plan_sources_before_persistence(inputs.plan, source_bytes)
345
+
346
+ definition_values = inputs.definition_values()
347
+ definition_bytes = {
348
+ name: canonical_json_bytes(value) for name, value in definition_values.items()
349
+ }
350
+ definition_sha256s = {name: sha256_bytes(raw) for name, raw in definition_bytes.items()}
351
+ state = _initial_state(run_id, definition_sha256s)
352
+ output_parent = _open_output_parent(workspace)
353
+ staging = workspace.parent / f".{workspace.name}.tmp-{uuid.uuid4().hex}"
354
+ staging_identity: tuple[int, int, int] | None = None
355
+ staging_fd = -1
356
+ installed = False
357
+ try:
358
+ _validate_output_parent_handle(output_parent)
359
+ os.mkdir(staging.name, 0o700, dir_fd=output_parent.parent_fd)
360
+ staging_named = os.stat(
361
+ staging.name,
362
+ dir_fd=output_parent.parent_fd,
363
+ follow_symlinks=False,
364
+ )
365
+ if not stat.S_ISDIR(staging_named.st_mode):
366
+ raise OfflineRunError("WORKSPACE_INIT_RACE", "workspace staging is not a directory")
367
+ staging_identity = _entry_identity(staging_named)
368
+ staging_fd = os.open(
369
+ staging.name,
370
+ os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW,
371
+ dir_fd=output_parent.parent_fd,
372
+ )
373
+ if _entry_identity(os.fstat(staging_fd)) != staging_identity:
374
+ raise OfflineRunError(
375
+ "WORKSPACE_INIT_RACE",
376
+ "workspace staging changed identity",
377
+ )
378
+ _populate_workspace(
379
+ staging_fd,
380
+ definition_bytes=definition_bytes,
381
+ source_bytes=source_bytes,
382
+ plan=inputs.plan,
383
+ state=state,
384
+ )
385
+ _validate_output_parent_handle(output_parent)
386
+ _rename_no_replace(
387
+ staging,
388
+ workspace,
389
+ expected_source_identity=staging_identity,
390
+ parent_fd=output_parent.parent_fd,
391
+ )
392
+ installed = True
393
+ os.fsync(output_parent.parent_fd)
394
+ _validate_output_parent_handle(output_parent)
395
+ except Exception:
396
+ if staging_identity is not None:
397
+ _remove_tree(
398
+ workspace if installed else staging,
399
+ expected_identity=staging_identity,
400
+ parent_fd=output_parent.parent_fd,
401
+ )
402
+ raise
403
+ finally:
404
+ if staging_fd >= 0:
405
+ os.close(staging_fd)
406
+ output_parent.close()
407
+
408
+ with _open_workspace(workspace) as opened:
409
+ observed = _read_state(opened)
410
+ _load_definitions(opened, observed)
411
+ opened.validate()
412
+ return _status_payload(observed, workspace)
413
+
414
+
415
+ def plan_workspace(workspace_path: Path) -> dict[str, Any]:
416
+ """Validate the immutable definitions and durably complete the plan task.
417
+
418
+ Nothing here authors or edits a plan. The plan JSON is an input that was already sealed
419
+ into the workspace; this re-verifies it and records the plan task as completed.
420
+ """
421
+
422
+ workspace_path = Path(os.path.abspath(os.fspath(workspace_path)))
423
+ with _open_workspace(workspace_path) as workspace:
424
+ state = _read_state(workspace)
425
+ inputs = _load_definitions(workspace, state)
426
+ if _task(state, "plan")["status"] != "completed":
427
+ next_state = deepcopy(state)
428
+ plan_task = _task(next_state, "plan")
429
+ plan_task.update(
430
+ {
431
+ "status": "completed",
432
+ "attempts": plan_task["attempts"] + 1,
433
+ "result_digest": sha256_bytes(canonical_json_bytes(inputs.plan.to_dict())),
434
+ }
435
+ )
436
+ next_state["phase"] = "planned"
437
+ _append_event(next_state, "plan_validated", "plan")
438
+ _write_state(workspace, next_state)
439
+ state = next_state
440
+ return {
441
+ "status": "plan_validated",
442
+ "run_id": state["run_id"],
443
+ "workspace": str(workspace_path),
444
+ "question": inputs.question.to_dict(),
445
+ "requirements": inputs.requirements.to_dict(),
446
+ "source_proposals": [
447
+ item.to_dict()
448
+ for item in sorted(inputs.source_proposals, key=lambda item: item.source_id)
449
+ ],
450
+ "plan": inputs.plan.to_dict(),
451
+ "plan_digest": _task(state, "plan")["result_digest"],
452
+ }
453
+
454
+
455
+ def run_workspace(
456
+ workspace_path: Path,
457
+ *,
458
+ fault_hook: Callable[[str], None] | None = None,
459
+ visual_store: VisualRunStore | None = None,
460
+ ) -> OfflineExecutionResult:
461
+ """Build using the fixed H2 coordinator-owned producer enrollment."""
462
+
463
+ return _execute_workspace_visualized(
464
+ workspace_path, fault_hook=fault_hook, is_resume=False, visual_store=visual_store
465
+ )
466
+
467
+
468
+ def resume_workspace(
469
+ workspace_path: Path,
470
+ *,
471
+ fault_hook: Callable[[str], None] | None = None,
472
+ visual_store: VisualRunStore | None = None,
473
+ ) -> OfflineExecutionResult:
474
+ """Resume only with the fixed coordinator-owned H2 producer identity."""
475
+
476
+ workspace_path = Path(os.path.abspath(os.fspath(workspace_path)))
477
+ with _open_workspace(workspace_path) as workspace:
478
+ state = _read_state(workspace)
479
+ producer = state["producer"]
480
+ if producer is None:
481
+ raise OfflineRunError(
482
+ "RESUME_NOT_STARTED",
483
+ "run has no persisted producer identity; invoke run first",
484
+ )
485
+ return _execute_workspace_visualized(
486
+ workspace_path, fault_hook=fault_hook, is_resume=True, visual_store=visual_store
487
+ )
488
+
489
+
490
+ def _execute_workspace_visualized(
491
+ workspace_path: Path,
492
+ *,
493
+ fault_hook: Callable[[str], None] | None,
494
+ is_resume: bool,
495
+ visual_store: VisualRunStore | None,
496
+ ) -> OfflineExecutionResult:
497
+ """Include coordinator validation/recovery in one retryable visual attempt."""
498
+
499
+ work = None
500
+ attempt = _H2_PRODUCER_ATTEMPT
501
+ execution_visual_store = visual_store
502
+ if visual_store is not None:
503
+ from mostlyright.data_harness.visual_run.reducer import reduce_events
504
+ from mostlyright.data_harness.visual_run.sdk import (
505
+ VisualWork,
506
+ attempt_event_key,
507
+ next_attempt_coordinate,
508
+ )
509
+
510
+ work = VisualWork(visual_store, actor="worker")
511
+ if not visual_store.events():
512
+ work.start_run(key="offline:run-started")
513
+ visual_state = reduce_events(visual_store.events())
514
+ if visual_state.terminal:
515
+ if visual_state.lifecycle != "completed":
516
+ raise OfflineRunError(
517
+ "VISUAL_RUN_TERMINAL",
518
+ "a terminal failed visual run cannot accept a resume",
519
+ )
520
+ # The coordinator's existing-result path remains read-only and deterministic. Its
521
+ # already-completed visual log is evidence, not a new invocation sink.
522
+ work = None
523
+ execution_visual_store = None
524
+ else:
525
+ attempt = next_attempt_coordinate(visual_store, "offline", attempt)
526
+ work.attempt_started(attempt, key=attempt_event_key("offline", attempt, "started"))
527
+ try:
528
+ result = _execute_workspace(
529
+ workspace_path,
530
+ producer_principal=_H2_PRODUCER_PRINCIPAL,
531
+ producer_attempt=_H2_PRODUCER_ATTEMPT,
532
+ fault_hook=fault_hook,
533
+ is_resume=is_resume,
534
+ visual_store=execution_visual_store,
535
+ )
536
+ except BaseException as exc:
537
+ if work is not None:
538
+ from mostlyright.data_harness.visual_run.contracts import TerminalError
539
+
540
+ candidate = getattr(exc, "code", "OFFLINE_EXECUTION_FAILED")
541
+ code = (
542
+ candidate
543
+ if isinstance(candidate, str) and re.fullmatch(r"[A-Z][A-Z0-9_]{1,63}", candidate)
544
+ else "OFFLINE_EXECUTION_FAILED"
545
+ )
546
+ work.attempt_failed(
547
+ attempt,
548
+ TerminalError(code, f"Offline execution stopped with {code}.", True),
549
+ key=attempt_event_key("offline", attempt, "failed"),
550
+ )
551
+ raise
552
+ if work is not None:
553
+ work.attempt_completed(attempt, key=attempt_event_key("offline", attempt, "completed"))
554
+ return result
555
+
556
+
557
+ def status_workspace(
558
+ workspace_path: Path,
559
+ *,
560
+ wait: bool = True,
561
+ ) -> dict[str, Any]:
562
+ """Return coordinator state without accepting caller-asserted review authority.
563
+
564
+ ``wait=False`` gives the same reading without queueing behind a build that is holding the
565
+ folder: it refuses with ``WORKSPACE_BUSY`` instead. That is what a command which only reports
566
+ wants -- `mr-data preflight` says "check this before you build", and a check that never returns
567
+ while a build is running is not one. Nothing about the reading is weakened; only the waiting is.
568
+ """
569
+
570
+ return _status_workspace(workspace_path, wait=wait)
571
+
572
+
573
+ def _status_enrolled_workspace(
574
+ workspace_path: Path,
575
+ *,
576
+ review_authority_provider: TrustedReviewAuthorityProvider | None,
577
+ ) -> dict[str, Any]:
578
+ """Return reviewed state only through an inherited coordinator session."""
579
+
580
+ _authority, assignment, verifier = _review_authority_material(review_authority_provider)
581
+ return _status_workspace(
582
+ workspace_path,
583
+ trusted_verifier_public_key=verifier.public_key(),
584
+ trusted_assignment_value=assignment,
585
+ )
586
+
587
+
588
+ def _status_workspace(
589
+ workspace_path: Path,
590
+ *,
591
+ trusted_verifier_public_key: Ed25519PublicKey | None = None,
592
+ trusted_assignment_value: Any | None = None,
593
+ wait: bool = True,
594
+ ) -> dict[str, Any]:
595
+ """Return the durable coordinator record without overstating review authority."""
596
+
597
+ workspace_path = Path(os.path.abspath(os.fspath(workspace_path)))
598
+ with _open_workspace(workspace_path, wait=wait) as workspace:
599
+ state = _read_state(workspace)
600
+ _load_definitions(workspace, state)
601
+ if state["candidate"] is not None:
602
+ if not _named_child_exists(workspace.root_fd, RUN_RESULT_PATH):
603
+ raise OfflineRunError(
604
+ "CANDIDATE_MISSING",
605
+ "durable state references a missing candidate",
606
+ )
607
+ if state["producer"] is None: # pragma: no cover - state validator enforces this
608
+ raise OfflineRunError(
609
+ "RUN_STATE_INVALID",
610
+ "candidate state is missing its producer fence",
611
+ )
612
+ _inspect_and_match_state(
613
+ workspace_path / RUN_RESULT_PATH,
614
+ state,
615
+ state["producer"],
616
+ workspace=workspace,
617
+ )
618
+ payload = _status_payload(state, workspace_path)
619
+ if trusted_verifier_public_key is not None:
620
+ verified_review = _verify_workspace_review(
621
+ workspace,
622
+ state,
623
+ trusted_verifier_public_key=trusted_verifier_public_key,
624
+ trusted_assignment_value=trusted_assignment_value,
625
+ )
626
+ payload["verified_review"] = {
627
+ "status": verified_review.status,
628
+ "candidate_digest": verified_review.candidate_digest,
629
+ "decision_digest": verified_review.decision_digest,
630
+ }
631
+ payload["status"] = verified_review.status
632
+ payload["review_authority"] = "cryptographically_verified_exact_review"
633
+ return payload
634
+
635
+
636
+ def inspect_workspace(
637
+ workspace_path: Path,
638
+ ) -> dict[str, Any]:
639
+ """Inspect a candidate without accepting caller-asserted review authority."""
640
+
641
+ return _inspect_workspace(workspace_path)
642
+
643
+
644
+ def _inspect_enrolled_workspace(
645
+ workspace_path: Path,
646
+ *,
647
+ review_authority_provider: TrustedReviewAuthorityProvider | None,
648
+ ) -> dict[str, Any]:
649
+ """Inspect reviewed state only through an inherited coordinator session."""
650
+
651
+ _authority, assignment, verifier = _review_authority_material(review_authority_provider)
652
+ return _inspect_workspace(
653
+ workspace_path,
654
+ trusted_verifier_public_key=verifier.public_key(),
655
+ trusted_assignment_value=assignment,
656
+ )
657
+
658
+
659
+ def _inspect_workspace(
660
+ workspace_path: Path,
661
+ *,
662
+ trusted_verifier_public_key: Ed25519PublicKey | None = None,
663
+ trusted_assignment_value: Any | None = None,
664
+ ) -> dict[str, Any]:
665
+ """Return exact replay-verified candidate evidence plus coordinator state."""
666
+
667
+ workspace_path = Path(os.path.abspath(os.fspath(workspace_path)))
668
+ with _open_workspace(workspace_path) as workspace:
669
+ state = _read_state(workspace)
670
+ _load_definitions(workspace, state)
671
+ result_path = workspace_path / state["paths"]["result"]
672
+ producer = state["producer"]
673
+ if state["candidate"] is None or producer is None:
674
+ raise OfflineRunError(
675
+ "CANDIDATE_NOT_COMMITTED",
676
+ "workspace has no durably committed candidate to inspect",
677
+ )
678
+ inspection, lease = _inspect_and_retain_matched_state(
679
+ result_path,
680
+ state,
681
+ producer,
682
+ workspace=workspace,
683
+ )
684
+ try:
685
+ payload = {
686
+ "status": "candidate_inspected",
687
+ "workspace": str(workspace_path),
688
+ "run_id": state["run_id"],
689
+ "state_phase": state["phase"],
690
+ "candidate": inspection.to_dict(),
691
+ "review_authority": "not_verified_without_enrolled_verifier_key",
692
+ "hosted_release": False,
693
+ }
694
+ if trusted_verifier_public_key is not None:
695
+ verified_review = _verify_workspace_review(
696
+ workspace,
697
+ state,
698
+ trusted_verifier_public_key=trusted_verifier_public_key,
699
+ trusted_assignment_value=trusted_assignment_value,
700
+ )
701
+ if verified_review.candidate_digest != inspection.candidate_digest:
702
+ raise OfflineRunError(
703
+ "REVIEW_CANDIDATE_MISMATCH",
704
+ "verified review does not match the retained candidate snapshot",
705
+ )
706
+ payload["verified_review"] = {
707
+ "status": verified_review.status,
708
+ "candidate_digest": verified_review.candidate_digest,
709
+ "decision_digest": verified_review.decision_digest,
710
+ }
711
+ payload["status"] = verified_review.status
712
+ payload["review_authority"] = "cryptographically_verified_exact_review"
713
+ lease.validate()
714
+ _match_candidate_inspection(inspection, state, producer)
715
+ workspace.validate()
716
+ lease.validate()
717
+ return payload
718
+ finally:
719
+ lease.close()
720
+
721
+
722
+ def workspace_result_path(workspace_path: Path) -> Path:
723
+ """Resolve the fixed descriptor-validated result child for review/verification."""
724
+
725
+ workspace_path = Path(os.path.abspath(os.fspath(workspace_path)))
726
+ with _open_workspace(workspace_path) as workspace:
727
+ state = _read_state(workspace)
728
+ _load_definitions(workspace, state)
729
+ return workspace_path / state["paths"]["result"]
730
+
731
+
732
+ def review_target_path(workspace_path: Path) -> Path:
733
+ """Resolve the run directory review enrollment acts on: a workspace or a sealed run dir.
734
+
735
+ A workspace is marked by its ``.mr-data`` control directory and keeps its run under the
736
+ recorded result child, resolved through the locked, replayed reader above. Any other path is
737
+ returned absolutized and unjudged, mirroring the CLI's own run-dir-or-workspace dispatch:
738
+ every caller hands the result to ``verify_candidate``, which is the gate that decides whether
739
+ a sealed Build is actually there. Recipe executions seal run directories without any
740
+ workspace, and review enrollment accepts them exactly the way ``mr-data verify`` and
741
+ ``mr-data review --local`` already do.
742
+ """
743
+
744
+ absolute = Path(os.path.abspath(os.fspath(workspace_path)))
745
+ try:
746
+ control = os.lstat(absolute / CONTROL_PATH)
747
+ except (FileNotFoundError, NotADirectoryError):
748
+ return absolute
749
+ except OSError:
750
+ return absolute
751
+ if not stat.S_ISDIR(control.st_mode):
752
+ return absolute
753
+ return workspace_result_path(absolute)
754
+
755
+
756
+ def recorded_result_path(workspace_path: Path) -> Path:
757
+ """Where a workspace records its run, for a reading that verifies nothing and must not wait.
758
+
759
+ This is not the resolver a command that acts on a run uses. That one is
760
+ :func:`workspace_result_path`, which takes the durable lock, *waits* for it, and replays the
761
+ sealed definitions before it answers: exactly right for a command that is about to open,
762
+ resume, review, or verify the run, and exactly wrong for a listing. A listing walks folders
763
+ nobody named, says in every row that it verified nothing, and must not stall behind a build
764
+ that is running in one of them.
765
+
766
+ So this one never waits. It takes a shared lock without blocking and refuses with
767
+ ``WORKSPACE_BUSY`` when a build holds the exclusive one, and it reads the recorded result path
768
+ out of the run state and nothing else. Nothing is weakened by the shorter reading, because
769
+ nothing is trusted from it: the value it returns is validated by :func:`_validate_state` to be
770
+ exactly ``RUN_RESULT_PATH``, and whatever a caller then does with that path goes through the
771
+ same verified reader every other caller uses.
772
+ """
773
+
774
+ if os.name == "nt" or fcntl is None:
775
+ raise OfflineRunError(
776
+ "WORKSPACE_PLATFORM_UNSUPPORTED",
777
+ "descriptor-relative durable workspaces are unavailable on this platform",
778
+ )
779
+ workspace_path = Path(os.path.abspath(os.fspath(workspace_path)))
780
+ try:
781
+ control_fd = os.open(
782
+ workspace_path / CONTROL_PATH,
783
+ os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW,
784
+ )
785
+ except FileNotFoundError as exc:
786
+ raise OfflineRunError(
787
+ "WORKSPACE_ABSENT",
788
+ f"there is {presence_at(workspace_path) or UNKNOWN_KIND} at {workspace_path}, "
789
+ "and a workbench has to be a folder this harness set up",
790
+ ) from exc
791
+ except NotADirectoryError as exc:
792
+ raise OfflineRunError("WORKSPACE_INVALID", "control path is not a directory") from exc
793
+ try:
794
+ state = _read_state_without_waiting(control_fd, workspace_path)
795
+ finally:
796
+ os.close(control_fd)
797
+ return workspace_path / state["paths"]["result"]
798
+
799
+
800
+ def _read_state_without_waiting(control_fd: int, workspace_path: Path) -> dict[str, Any]:
801
+ """The run state, read under a shared lock that is never waited for.
802
+
803
+ The shared lock is taken for one reason: a build holds the exclusive one for its whole length,
804
+ so failing to take this one immediately is how a reader learns a build is running without
805
+ joining the queue behind it. It is released before anything is parsed, and a build that starts
806
+ a microsecond later waits on nothing.
807
+ """
808
+
809
+ try:
810
+ lock_fd = os.open(
811
+ WORKSPACE_LOCK_NAME,
812
+ os.O_RDONLY | os.O_NOFOLLOW | getattr(os, "O_CLOEXEC", 0),
813
+ dir_fd=control_fd,
814
+ )
815
+ except FileNotFoundError as exc:
816
+ raise OfflineRunError(
817
+ "WORKSPACE_ABSENT",
818
+ f"there is {presence_at(workspace_path) or UNKNOWN_KIND} at {workspace_path}, "
819
+ "and a workbench has to be a folder this harness set up",
820
+ ) from exc
821
+ try:
822
+ try:
823
+ fcntl.flock(lock_fd, fcntl.LOCK_SH | fcntl.LOCK_NB)
824
+ except OSError as exc:
825
+ if exc.errno not in (errno.EWOULDBLOCK, errno.EAGAIN, errno.EACCES):
826
+ raise
827
+ raise OfflineRunError(
828
+ "WORKSPACE_BUSY",
829
+ f"a build is running in {workspace_path}",
830
+ ) from exc
831
+ try:
832
+ raw = _read_regular_file_at(
833
+ control_fd,
834
+ _STATE_NAME,
835
+ maximum=_MAX_STATE_BYTES,
836
+ required_mode=0o600,
837
+ )
838
+ except FileNotFoundError as exc:
839
+ # The folder was set up but no run was ever recorded in it. That is not a damaged
840
+ # record; it is the absence of one, and it names a different fix.
841
+ raise OfflineRunError(
842
+ "WORKSPACE_ABSENT",
843
+ f"there is {presence_at(workspace_path) or UNKNOWN_KIND} at {workspace_path}, "
844
+ "and a workbench has to be a folder this harness set up",
845
+ ) from exc
846
+ finally:
847
+ os.close(lock_fd)
848
+ try:
849
+ value = parse_canonical_json(raw)
850
+ except ValueError as exc:
851
+ raise OfflineRunError("RUN_STATE_INVALID", "run state is not canonical JSON") from exc
852
+ return _validate_state(value)
853
+
854
+
855
+ def _record_review(
856
+ workspace_path: Path,
857
+ *,
858
+ status: str,
859
+ candidate_digest: str,
860
+ decision_digest: str,
861
+ ) -> dict[str, Any]:
862
+ """Record a verified review receipt without making it the verification authority."""
863
+
864
+ if status not in {"reviewed_candidate", "fixes_required"}:
865
+ raise OfflineRunError("REVIEW_STATUS_INVALID", "review status is unsupported in H2")
866
+ candidate_digest = _digest(candidate_digest, "candidate_digest")
867
+ decision_digest = _digest(decision_digest, "decision_digest")
868
+ workspace_path = Path(os.path.abspath(os.fspath(workspace_path)))
869
+ with _open_workspace(workspace_path) as workspace:
870
+ state = _read_state(workspace)
871
+ next_state = _record_review_in_open_workspace(
872
+ workspace,
873
+ state,
874
+ status=status,
875
+ candidate_digest=candidate_digest,
876
+ decision_digest=decision_digest,
877
+ )
878
+ return _status_payload(next_state, workspace_path)
879
+
880
+
881
+ def _finalize_enrolled_workspace_review(
882
+ workspace_path: Path,
883
+ *,
884
+ report_values: list[Any],
885
+ review_authority_provider: TrustedReviewAuthorityProvider | None,
886
+ ) -> ReviewResult:
887
+ """Finalize workspace review only through an inherited coordinator session."""
888
+
889
+ _authority, assignment, verifier = _review_authority_material(review_authority_provider)
890
+ return _finalize_workspace_review(
891
+ workspace_path,
892
+ assignment_value=assignment,
893
+ report_values=report_values,
894
+ verifier_private_key=verifier,
895
+ )
896
+
897
+
898
+ def _finalize_workspace_review(
899
+ workspace_path: Path,
900
+ *,
901
+ assignment_value: Any,
902
+ report_values: list[Any],
903
+ verifier_private_key: TrustedSigner,
904
+ ) -> ReviewResult:
905
+ """Finalize and record one review against a single retained workspace snapshot."""
906
+
907
+ workspace_path = Path(os.path.abspath(os.fspath(workspace_path)))
908
+ with _open_workspace(workspace_path) as workspace:
909
+ state = _read_state(workspace)
910
+ _load_definitions(workspace, state)
911
+ if state["candidate"] is None:
912
+ raise OfflineRunError(
913
+ "CANDIDATE_NOT_COMMITTED",
914
+ "workspace has no durably committed candidate to review",
915
+ )
916
+ result_path = workspace_path / state["paths"]["result"]
917
+ result = _finalize_review(
918
+ run_dir=result_path,
919
+ assignment_value=assignment_value,
920
+ report_values=report_values,
921
+ verifier_private_key=verifier_private_key,
922
+ _run_parent_fd=workspace.root_fd,
923
+ _transaction_validator=workspace.validate,
924
+ )
925
+ if result.candidate_digest != state["candidate"]["candidate_digest"]:
926
+ raise OfflineRunError(
927
+ "REVIEW_CANDIDATE_MISMATCH",
928
+ "review candidate does not match durable run state",
929
+ )
930
+ _record_review_in_open_workspace(
931
+ workspace,
932
+ state,
933
+ status=result.status,
934
+ candidate_digest=result.candidate_digest,
935
+ decision_digest=result.decision_digest,
936
+ )
937
+ workspace.validate()
938
+ return result
939
+
940
+
941
+ def _record_review_in_open_workspace(
942
+ workspace: _Workspace,
943
+ state: dict[str, Any],
944
+ *,
945
+ status: str,
946
+ candidate_digest: str,
947
+ decision_digest: str,
948
+ ) -> dict[str, Any]:
949
+ if status not in {"reviewed_candidate", "fixes_required"}:
950
+ raise OfflineRunError("REVIEW_STATUS_INVALID", "review status is unsupported in H2")
951
+ candidate_digest = _digest(candidate_digest, "candidate_digest")
952
+ decision_digest = _digest(decision_digest, "decision_digest")
953
+ if state["candidate"] is None or state["candidate"]["candidate_digest"] != candidate_digest:
954
+ raise OfflineRunError(
955
+ "REVIEW_CANDIDATE_MISMATCH",
956
+ "review candidate does not match durable run state",
957
+ )
958
+ review_task = _task(state, "review")
959
+ if (
960
+ review_task["status"] == "completed"
961
+ and state["review"] is not None
962
+ and state["review"]["decision_digest"] == decision_digest
963
+ ):
964
+ return state
965
+ next_state = deepcopy(state)
966
+ next_review_task = _task(next_state, "review")
967
+ next_review_task.update(
968
+ {
969
+ "status": "completed",
970
+ "attempts": next_review_task["attempts"] + 1,
971
+ "result_digest": decision_digest,
972
+ }
973
+ )
974
+ next_state["review"] = {
975
+ "candidate_digest": candidate_digest,
976
+ "decision_digest": decision_digest,
977
+ "verified_status": status,
978
+ }
979
+ next_state["phase"] = "review_recorded"
980
+ _append_event(next_state, "exact_review_recorded", "review")
981
+ _write_state(workspace, next_state)
982
+ return next_state
983
+
984
+
985
+ def _verify_workspace_review(
986
+ workspace: _Workspace,
987
+ state: dict[str, Any],
988
+ *,
989
+ trusted_verifier_public_key: Ed25519PublicKey,
990
+ trusted_assignment_value: Any | None,
991
+ ) -> ReviewResult:
992
+ if state["candidate"] is None:
993
+ raise OfflineRunError(
994
+ "CANDIDATE_NOT_COMMITTED",
995
+ "workspace has no durably committed candidate to verify",
996
+ )
997
+ result_path = workspace.path / state["paths"]["result"]
998
+ result = _verify_review_decision(
999
+ result_path,
1000
+ trusted_verifier_public_key=trusted_verifier_public_key,
1001
+ trusted_assignment_value=trusted_assignment_value,
1002
+ _run_parent_fd=workspace.root_fd,
1003
+ _ancestor_validator=workspace.validate,
1004
+ )
1005
+ if result.candidate_digest != state["candidate"]["candidate_digest"]:
1006
+ raise OfflineRunError(
1007
+ "REVIEW_CANDIDATE_MISMATCH",
1008
+ "verified review does not match durable run state",
1009
+ )
1010
+ workspace.validate()
1011
+ return result
1012
+
1013
+
1014
+ def _execute_workspace(
1015
+ workspace_path: Path,
1016
+ *,
1017
+ producer_principal: str,
1018
+ producer_attempt: str,
1019
+ fault_hook: Callable[[str], None] | None,
1020
+ is_resume: bool,
1021
+ visual_store: VisualRunStore | None = None,
1022
+ ) -> OfflineExecutionResult:
1023
+ workspace_path = Path(os.path.abspath(os.fspath(workspace_path)))
1024
+ trusted_producer = _h2_producer_identity()
1025
+ if {
1026
+ "principal_id": producer_principal,
1027
+ "attempt_id": producer_attempt,
1028
+ } != trusted_producer:
1029
+ raise OfflineRunError(
1030
+ "PRODUCER_MISMATCH",
1031
+ "offline execution requires the fixed coordinator-owned H2 producer",
1032
+ )
1033
+ with _open_workspace(workspace_path) as workspace:
1034
+ state = _read_state(workspace)
1035
+ inputs = _load_definitions(workspace, state)
1036
+ producer = trusted_producer
1037
+ if state["producer"] is not None and state["producer"] != producer:
1038
+ raise OfflineRunError(
1039
+ "PRODUCER_MISMATCH",
1040
+ "resume/retry must use the persisted producer principal and attempt",
1041
+ )
1042
+ result_path = workspace_path / state["paths"]["result"]
1043
+ if _named_child_exists(workspace.root_fd, RUN_RESULT_PATH):
1044
+ # A process may have died after rename but before the output-parent
1045
+ # barrier. Retry it before adopting or committing an existing result.
1046
+ _durability_barrier(workspace.root_fd, "output-parent")
1047
+ inspection = _inspect_and_match_state(
1048
+ result_path,
1049
+ state,
1050
+ producer,
1051
+ workspace=workspace,
1052
+ )
1053
+ if _task(state, "build")["status"] == "completed":
1054
+ return OfflineExecutionResult(
1055
+ status="candidate_built",
1056
+ run_id=state["run_id"],
1057
+ candidate=inspection,
1058
+ workspace=workspace_path,
1059
+ recovered=False,
1060
+ idempotent=True,
1061
+ )
1062
+ next_state = _candidate_state(
1063
+ state,
1064
+ inspection,
1065
+ producer=producer,
1066
+ event="candidate_recovered",
1067
+ )
1068
+ _write_state(workspace, next_state)
1069
+ return OfflineExecutionResult(
1070
+ status="candidate_built",
1071
+ run_id=next_state["run_id"],
1072
+ candidate=inspection,
1073
+ workspace=workspace_path,
1074
+ recovered=True,
1075
+ idempotent=False,
1076
+ )
1077
+
1078
+ _remove_orphan_staging(workspace)
1079
+ next_state = deepcopy(state)
1080
+ plan_task = _task(next_state, "plan")
1081
+ if plan_task["status"] != "completed":
1082
+ plan_task.update(
1083
+ {
1084
+ "status": "completed",
1085
+ "attempts": plan_task["attempts"] + 1,
1086
+ "result_digest": sha256_bytes(canonical_json_bytes(inputs.plan.to_dict())),
1087
+ }
1088
+ )
1089
+ _append_event(next_state, "plan_validated", "plan")
1090
+ build_task = _task(next_state, "build")
1091
+ build_task.update(
1092
+ {
1093
+ "status": "running",
1094
+ "attempts": build_task["attempts"] + 1,
1095
+ "result_digest": None,
1096
+ }
1097
+ )
1098
+ next_state["phase"] = "building"
1099
+ next_state["producer"] = producer
1100
+ _append_event(next_state, "build_resumed" if is_resume else "build_started", "build")
1101
+ _write_state(workspace, next_state)
1102
+ state = next_state
1103
+ try:
1104
+ build_candidate(
1105
+ inputs.plan,
1106
+ input_root=workspace_path / state["paths"]["input_root"],
1107
+ output_dir=result_path,
1108
+ producer_principal=producer_principal,
1109
+ producer_attempt=producer_attempt,
1110
+ package_context=inputs.package_context,
1111
+ visual_store=visual_store,
1112
+ )
1113
+ except (BuildError, ContractError):
1114
+ failed = deepcopy(state)
1115
+ _task(failed, "build")["status"] = "failed"
1116
+ failed["phase"] = "failed"
1117
+ _append_event(failed, "build_failed", "build")
1118
+ _write_state(workspace, failed)
1119
+ raise
1120
+ if fault_hook is not None:
1121
+ fault_hook("after_candidate_sealed")
1122
+ inspection = _inspect_and_match_state(
1123
+ result_path,
1124
+ state,
1125
+ producer,
1126
+ workspace=workspace,
1127
+ )
1128
+ completed = _candidate_state(
1129
+ state,
1130
+ inspection,
1131
+ producer=producer,
1132
+ event="candidate_built",
1133
+ )
1134
+ _write_state(workspace, completed)
1135
+ return OfflineExecutionResult(
1136
+ status="candidate_built",
1137
+ run_id=completed["run_id"],
1138
+ candidate=inspection,
1139
+ workspace=workspace_path,
1140
+ recovered=False,
1141
+ idempotent=False,
1142
+ )
1143
+
1144
+
1145
+ def _candidate_state(
1146
+ state: dict[str, Any],
1147
+ inspection: CandidateInspection,
1148
+ *,
1149
+ producer: dict[str, str],
1150
+ event: str,
1151
+ ) -> dict[str, Any]:
1152
+ if inspection.manifest_version not in _OFFLINE_MANIFEST_VERSIONS:
1153
+ raise OfflineRunError(
1154
+ "CANDIDATE_VERSION_INVALID",
1155
+ "workspace result is not a complete H2 offline package",
1156
+ )
1157
+ next_state = deepcopy(state)
1158
+ next_state["producer"] = producer
1159
+ next_state["candidate"] = {
1160
+ "candidate_digest": inspection.candidate_digest,
1161
+ "table_sha256": inspection.table_sha256,
1162
+ "manifest_version": inspection.manifest_version,
1163
+ "row_count": inspection.row_count,
1164
+ }
1165
+ next_state["phase"] = "candidate_built"
1166
+ _task(next_state, "build").update(
1167
+ {
1168
+ "status": "completed",
1169
+ "result_digest": inspection.candidate_digest,
1170
+ }
1171
+ )
1172
+ _append_event(next_state, event, "build")
1173
+ return next_state
1174
+
1175
+
1176
+ def _inspect_and_match_state(
1177
+ result_path: Path,
1178
+ state: dict[str, Any],
1179
+ producer: dict[str, str],
1180
+ *,
1181
+ workspace: _Workspace | None = None,
1182
+ ) -> CandidateInspection:
1183
+ inspection, lease = _inspect_and_retain_matched_state(
1184
+ result_path,
1185
+ state,
1186
+ producer,
1187
+ workspace=workspace,
1188
+ )
1189
+ try:
1190
+ return inspection
1191
+ finally:
1192
+ lease.close()
1193
+
1194
+
1195
+ def _inspect_and_retain_matched_state(
1196
+ result_path: Path,
1197
+ state: dict[str, Any],
1198
+ producer: dict[str, str],
1199
+ *,
1200
+ workspace: _Workspace | None = None,
1201
+ ) -> tuple[CandidateInspection, Any]:
1202
+ retained_snapshots: list[Any] = []
1203
+ inspection = inspect_candidate(
1204
+ result_path,
1205
+ _run_parent_fd=workspace.root_fd if workspace is not None else None,
1206
+ _ancestor_validator=workspace.validate if workspace is not None else None,
1207
+ _retained_snapshots=retained_snapshots,
1208
+ )
1209
+ if len(retained_snapshots) != 1:
1210
+ for snapshot in retained_snapshots:
1211
+ snapshot.close()
1212
+ raise OfflineRunError(
1213
+ "CANDIDATE_SNAPSHOT_INVALID",
1214
+ "candidate inspection did not retain its exact bound snapshot",
1215
+ )
1216
+ verified_snapshot = retained_snapshots[0]
1217
+ try:
1218
+ # Deliberate repeat, not a duplicate. The comparison runs once before the workspace
1219
+ # and lease identity checks and once between two rounds of them, so the accepted
1220
+ # comparison is bracketed by identity checks on both sides: a workspace or candidate
1221
+ # root swapped in during either pass fails the surrounding check.
1222
+ _match_candidate_inspection(inspection, state, producer)
1223
+ if workspace is not None:
1224
+ workspace.validate()
1225
+ verified_snapshot.validate()
1226
+ _match_candidate_inspection(inspection, state, producer)
1227
+ if workspace is not None:
1228
+ workspace.validate()
1229
+ verified_snapshot.validate()
1230
+ return inspection, verified_snapshot
1231
+ except Exception:
1232
+ verified_snapshot.close()
1233
+ raise
1234
+
1235
+
1236
+ def _match_candidate_inspection(
1237
+ inspection: CandidateInspection,
1238
+ state: dict[str, Any],
1239
+ producer: dict[str, str],
1240
+ ) -> None:
1241
+ if (
1242
+ inspection.producer_principal != producer["principal_id"]
1243
+ or inspection.producer_attempt != producer["attempt_id"]
1244
+ or inspection.manifest_version not in _OFFLINE_MANIFEST_VERSIONS
1245
+ ):
1246
+ raise OfflineRunError(
1247
+ "CANDIDATE_PRODUCER_MISMATCH",
1248
+ "existing candidate is not the expected H2 producer result",
1249
+ )
1250
+ recorded = state["candidate"]
1251
+ if recorded is not None and (
1252
+ recorded["candidate_digest"] != inspection.candidate_digest
1253
+ or recorded["table_sha256"] != inspection.table_sha256
1254
+ or recorded["row_count"] != inspection.row_count
1255
+ ):
1256
+ raise OfflineRunError(
1257
+ "CANDIDATE_STATE_MISMATCH",
1258
+ "verified candidate does not match durable run state",
1259
+ )
1260
+
1261
+
1262
+ def _populate_workspace(
1263
+ staging_fd: int,
1264
+ *,
1265
+ definition_bytes: dict[str, bytes],
1266
+ source_bytes: dict[str, bytes],
1267
+ plan: TablePlan,
1268
+ state: dict[str, Any],
1269
+ ) -> None:
1270
+ directories: dict[str, int] = {"": staging_fd}
1271
+ opened_fds: list[int] = []
1272
+ try:
1273
+ for path in (CONTROL_PATH, DEFINITION_PATH, INPUT_ROOT_PATH):
1274
+ descriptor = _create_directory_at(staging_fd, path)
1275
+ directories[path] = descriptor
1276
+ opened_fds.append(descriptor)
1277
+ _create_file_at(
1278
+ directories[CONTROL_PATH],
1279
+ WORKSPACE_LOCK_NAME,
1280
+ b"",
1281
+ mode=0o600,
1282
+ )
1283
+ _create_file_at(
1284
+ directories[CONTROL_PATH],
1285
+ _STATE_NAME,
1286
+ canonical_json_bytes(state),
1287
+ mode=0o600,
1288
+ )
1289
+ definition_files = dict(_DEFINITION_FILES)
1290
+ if "semantics" in definition_bytes:
1291
+ definition_files.update(_SEMANTICS_DEFINITION)
1292
+ for logical_name, file_name in definition_files.items():
1293
+ _create_file_at(
1294
+ directories[DEFINITION_PATH],
1295
+ file_name,
1296
+ definition_bytes[logical_name],
1297
+ mode=0o444,
1298
+ )
1299
+ source_by_id = {source.source_id: source for source in plan.sources}
1300
+ for source_id in sorted(source_bytes):
1301
+ source = source_by_id[source_id]
1302
+ parent_fd, newly_opened = _create_relative_parents(
1303
+ directories[INPUT_ROOT_PATH],
1304
+ PurePosixPath(source.path).parts[:-1],
1305
+ )
1306
+ opened_fds.extend(newly_opened)
1307
+ _create_file_at(
1308
+ parent_fd,
1309
+ PurePosixPath(source.path).name,
1310
+ source_bytes[source_id],
1311
+ mode=0o444,
1312
+ )
1313
+ for descriptor in reversed(opened_fds):
1314
+ if descriptor not in {
1315
+ directories[CONTROL_PATH],
1316
+ directories[DEFINITION_PATH],
1317
+ directories[INPUT_ROOT_PATH],
1318
+ }:
1319
+ os.fchmod(descriptor, 0o555)
1320
+ os.fsync(descriptor)
1321
+ os.fchmod(directories[DEFINITION_PATH], 0o555)
1322
+ os.fchmod(directories[INPUT_ROOT_PATH], 0o555)
1323
+ for descriptor in {
1324
+ staging_fd,
1325
+ directories[CONTROL_PATH],
1326
+ directories[DEFINITION_PATH],
1327
+ directories[INPUT_ROOT_PATH],
1328
+ }:
1329
+ os.fsync(descriptor)
1330
+ finally:
1331
+ for descriptor in reversed(opened_fds):
1332
+ os.close(descriptor)
1333
+
1334
+
1335
+ def _create_relative_parents(
1336
+ root_fd: int,
1337
+ parts: tuple[str, ...],
1338
+ ) -> tuple[int, list[int]]:
1339
+ parent_fd = root_fd
1340
+ opened: list[int] = []
1341
+ for part in parts:
1342
+ if part in {"", ".", ".."}:
1343
+ raise OfflineRunError("SOURCE_PATH_INVALID", "source path is not canonical")
1344
+ try:
1345
+ os.mkdir(part, 0o700, dir_fd=parent_fd)
1346
+ except FileExistsError:
1347
+ pass
1348
+ descriptor = os.open(
1349
+ part,
1350
+ os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW,
1351
+ dir_fd=parent_fd,
1352
+ )
1353
+ opened.append(descriptor)
1354
+ parent_fd = descriptor
1355
+ return parent_fd, opened
1356
+
1357
+
1358
+ def _create_directory_at(parent_fd: int, name: str) -> int:
1359
+ os.mkdir(name, 0o700, dir_fd=parent_fd)
1360
+ named = os.stat(name, dir_fd=parent_fd, follow_symlinks=False)
1361
+ descriptor = os.open(
1362
+ name,
1363
+ os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW,
1364
+ dir_fd=parent_fd,
1365
+ )
1366
+ opened = os.fstat(descriptor)
1367
+ if not stat.S_ISDIR(named.st_mode) or _entry_identity(named) != _entry_identity(opened):
1368
+ os.close(descriptor)
1369
+ raise OfflineRunError("WORKSPACE_INIT_RACE", f"directory changed identity: {name}")
1370
+ return descriptor
1371
+
1372
+
1373
+ def _create_file_at(parent_fd: int, name: str, raw: bytes, *, mode: int) -> None:
1374
+ descriptor = os.open(
1375
+ name,
1376
+ os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW,
1377
+ 0o600,
1378
+ dir_fd=parent_fd,
1379
+ )
1380
+ try:
1381
+ offset = 0
1382
+ while offset < len(raw):
1383
+ written = os.write(descriptor, raw[offset:])
1384
+ if written < 1:
1385
+ raise OfflineRunError("WORKSPACE_WRITE_FAILED", f"write stalled: {name}")
1386
+ offset += written
1387
+ os.fsync(descriptor)
1388
+ os.fchmod(descriptor, mode)
1389
+ os.fsync(descriptor)
1390
+ opened = os.fstat(descriptor)
1391
+ named = os.stat(name, dir_fd=parent_fd, follow_symlinks=False)
1392
+ if (
1393
+ not stat.S_ISREG(opened.st_mode)
1394
+ or opened.st_nlink != 1
1395
+ or _stat_signature(opened) != _stat_signature(named)
1396
+ ):
1397
+ raise OfflineRunError("WORKSPACE_INIT_RACE", f"file changed identity: {name}")
1398
+ finally:
1399
+ os.close(descriptor)
1400
+
1401
+
1402
+ class _WorkspaceContext:
1403
+ def __init__(self, workspace: _Workspace) -> None:
1404
+ self.workspace = workspace
1405
+
1406
+ def __enter__(self) -> _Workspace:
1407
+ return self.workspace
1408
+
1409
+ def __exit__(self, _exc_type: Any, _exc: Any, _traceback: Any) -> None:
1410
+ self.workspace.close()
1411
+
1412
+
1413
+ def validate_workspace_structure(path: Path) -> None:
1414
+ """Validate a Harness workspace control tree without acquiring its execution lock.
1415
+
1416
+ A successful return proves that the lexical path and its ancestors stayed descriptor-bound,
1417
+ the control directory and single-link ``0600`` lock kept their identities, and one bounded,
1418
+ single-link ``0600`` state file was stable canonical JSON satisfying the current run-state
1419
+ schema. It does not claim the state is current after this function returns, replay immutable
1420
+ definitions, or verify a Build. Callers that need those guarantees must use a locked offline
1421
+ operation instead.
1422
+
1423
+ ``OfflineRunError`` or ``BuildError`` reports a refusal. The function never waits for, takes,
1424
+ or releases the workspace execution lock, so a viewer may use it while a build is running.
1425
+ """
1426
+
1427
+ workspace_path = Path(os.path.abspath(os.fspath(path)))
1428
+ with _open_workspace(workspace_path, acquire_lock=False) as workspace:
1429
+ _read_state(workspace)
1430
+
1431
+
1432
+ def _open_workspace(
1433
+ path: Path,
1434
+ *,
1435
+ wait: bool = True,
1436
+ acquire_lock: bool = True,
1437
+ ) -> _WorkspaceContext:
1438
+ """Open and validate a Workbench folder, optionally under its durable lock.
1439
+
1440
+ ``wait`` is about the lock and nothing else. Every command that acts on a run waits, because
1441
+ joining the queue behind a build is exactly right for a command that is about to change what
1442
+ the build is producing. A command that only *reports* must not wait: a build holds this lock
1443
+ for its whole length, and a reporting command that queues behind one returns no answer at all
1444
+ for as long as the build runs. Nothing else about the reading changes -- the same validation,
1445
+ the same definitions replay, the same candidate match -- so a non-waiting caller is never told
1446
+ less than a waiting one, only told sooner that it could not look.
1447
+
1448
+ ``acquire_lock=False`` is the read-only structural-validation mode used only by
1449
+ :func:`validate_workspace_structure`; in that mode ``wait`` has no effect.
1450
+ """
1451
+
1452
+ if os.name == "nt" or fcntl is None:
1453
+ raise OfflineRunError(
1454
+ "WORKSPACE_PLATFORM_UNSUPPORTED",
1455
+ "descriptor-relative durable workspaces are unavailable on this platform",
1456
+ )
1457
+ run_handle = _open_candidate_run_handle(path)
1458
+ control_fd = -1
1459
+ lock_fd = -1
1460
+ try:
1461
+ try:
1462
+ control_named = os.stat(
1463
+ CONTROL_PATH,
1464
+ dir_fd=run_handle.run_fd,
1465
+ follow_symlinks=False,
1466
+ )
1467
+ except FileNotFoundError as exc:
1468
+ # Pointing a command at a folder that was never initialized is the commonest way to
1469
+ # reach this function, and it is not a fault in the durable record: there is no record.
1470
+ # Letting the raw `FileNotFoundError` out named the control directory -- a path the
1471
+ # caller never typed -- and carried no typed code, so nothing downstream could say what
1472
+ # to do about it.
1473
+ raise OfflineRunError(
1474
+ "WORKSPACE_ABSENT",
1475
+ f"there is {presence_at(path) or UNKNOWN_KIND} at {path}, "
1476
+ "and a workbench has to be a folder this harness set up",
1477
+ ) from exc
1478
+ if not stat.S_ISDIR(control_named.st_mode):
1479
+ raise OfflineRunError("WORKSPACE_INVALID", "control path is not a directory")
1480
+ control_fd = os.open(
1481
+ CONTROL_PATH,
1482
+ os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | getattr(os, "O_CLOEXEC", 0),
1483
+ dir_fd=run_handle.run_fd,
1484
+ )
1485
+ control_identity = _entry_identity(os.fstat(control_fd))
1486
+ if control_identity != _entry_identity(control_named):
1487
+ raise OfflineRunError("WORKSPACE_RACE", "control path changed identity")
1488
+ lock_fd = os.open(
1489
+ WORKSPACE_LOCK_NAME,
1490
+ os.O_RDWR | os.O_NOFOLLOW | getattr(os, "O_CLOEXEC", 0),
1491
+ dir_fd=control_fd,
1492
+ )
1493
+ lock_info = os.fstat(lock_fd)
1494
+ if (
1495
+ not stat.S_ISREG(lock_info.st_mode)
1496
+ or lock_info.st_nlink != 1
1497
+ or stat.S_IMODE(lock_info.st_mode) != 0o600
1498
+ ):
1499
+ raise OfflineRunError("WORKSPACE_INVALID", "workspace lock is invalid")
1500
+ if acquire_lock:
1501
+ if wait:
1502
+ fcntl.flock(lock_fd, fcntl.LOCK_EX)
1503
+ else:
1504
+ try:
1505
+ fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
1506
+ except OSError as exc:
1507
+ if exc.errno not in (errno.EWOULDBLOCK, errno.EAGAIN, errno.EACCES):
1508
+ raise
1509
+ raise OfflineRunError(
1510
+ "WORKSPACE_BUSY",
1511
+ f"a build is running in {path}",
1512
+ ) from exc
1513
+ workspace = _Workspace(
1514
+ path=path,
1515
+ run_handle=run_handle,
1516
+ control_fd=control_fd,
1517
+ control_identity=control_identity,
1518
+ lock_fd=lock_fd,
1519
+ lock_acquired=acquire_lock,
1520
+ )
1521
+ workspace.validate()
1522
+ return _WorkspaceContext(workspace)
1523
+ except Exception:
1524
+ if lock_fd >= 0:
1525
+ os.close(lock_fd)
1526
+ if control_fd >= 0:
1527
+ os.close(control_fd)
1528
+ run_handle.close()
1529
+ raise
1530
+
1531
+
1532
+ def _read_state(workspace: _Workspace) -> dict[str, Any]:
1533
+ workspace.validate()
1534
+ raw = _read_regular_file_at(
1535
+ workspace.control_fd,
1536
+ _STATE_NAME,
1537
+ maximum=_MAX_STATE_BYTES,
1538
+ required_mode=0o600,
1539
+ )
1540
+ try:
1541
+ value = parse_canonical_json(raw)
1542
+ except ValueError as exc:
1543
+ raise OfflineRunError("RUN_STATE_INVALID", "run state is not canonical JSON") from exc
1544
+ state = _validate_state(value)
1545
+ workspace.validate()
1546
+ return state
1547
+
1548
+
1549
+ def _write_state(workspace: _Workspace, state: dict[str, Any]) -> None:
1550
+ state["revision"] += 1
1551
+ _validate_state(state)
1552
+ raw = canonical_json_bytes(state)
1553
+ temporary = f".state.tmp-{uuid.uuid4().hex}"
1554
+ workspace.validate()
1555
+ try:
1556
+ _create_file_at(workspace.control_fd, temporary, raw, mode=0o600)
1557
+ os.replace(
1558
+ temporary,
1559
+ _STATE_NAME,
1560
+ src_dir_fd=workspace.control_fd,
1561
+ dst_dir_fd=workspace.control_fd,
1562
+ )
1563
+ os.fsync(workspace.control_fd)
1564
+ workspace.validate()
1565
+ observed = _read_state(workspace)
1566
+ if observed != state:
1567
+ raise OfflineRunError(
1568
+ "RUN_STATE_DURABILITY",
1569
+ "atomic run state does not match the committed successor",
1570
+ )
1571
+ except Exception:
1572
+ try:
1573
+ os.unlink(temporary, dir_fd=workspace.control_fd)
1574
+ except FileNotFoundError:
1575
+ pass
1576
+ raise
1577
+
1578
+
1579
+ def _load_definitions(workspace: _Workspace, state: dict[str, Any]) -> OfflineInputs:
1580
+ workspace.validate()
1581
+ definitions_fd = os.open(
1582
+ DEFINITION_PATH,
1583
+ os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW,
1584
+ dir_fd=workspace.root_fd,
1585
+ )
1586
+ try:
1587
+ info = os.fstat(definitions_fd)
1588
+ if not stat.S_ISDIR(info.st_mode) or stat.S_IMODE(info.st_mode) != 0o555:
1589
+ raise OfflineRunError("DEFINITIONS_INVALID", "definitions directory is not sealed")
1590
+ definition_files = dict(_DEFINITION_FILES)
1591
+ if "semantics" in state["definition_sha256s"]:
1592
+ definition_files.update(_SEMANTICS_DEFINITION)
1593
+ if set(os.listdir(definitions_fd)) != set(definition_files.values()):
1594
+ raise OfflineRunError("DEFINITIONS_INVALID", "definition member set changed")
1595
+ values: dict[str, Any] = {}
1596
+ for logical_name, file_name in definition_files.items():
1597
+ raw = _read_regular_file_at(
1598
+ definitions_fd,
1599
+ file_name,
1600
+ maximum=_MAX_DEFINITION_BYTES,
1601
+ required_mode=0o444,
1602
+ )
1603
+ if sha256_bytes(raw) != state["definition_sha256s"][logical_name]:
1604
+ raise OfflineRunError(
1605
+ "DEFINITION_DIGEST_MISMATCH",
1606
+ f"definition changed: {logical_name}",
1607
+ )
1608
+ try:
1609
+ values[logical_name] = parse_canonical_json(raw)
1610
+ except ValueError as exc:
1611
+ raise OfflineRunError(
1612
+ "DEFINITIONS_INVALID",
1613
+ f"definition is not canonical: {logical_name}",
1614
+ ) from exc
1615
+ finally:
1616
+ os.close(definitions_fd)
1617
+ proposal_set = values["source_proposals"]
1618
+ if not isinstance(proposal_set, dict) or set(proposal_set) != {
1619
+ "schema_version",
1620
+ "source_proposals",
1621
+ }:
1622
+ raise OfflineRunError("DEFINITIONS_INVALID", "source proposal set is invalid")
1623
+ if proposal_set["schema_version"] != SOURCE_PROPOSAL_SET_VERSION:
1624
+ raise OfflineRunError("DEFINITIONS_INVALID", "source proposal set version is invalid")
1625
+ inputs = OfflineInputs(
1626
+ question=parse_question(values["question"]),
1627
+ requirements=parse_requirements(values["requirements"]),
1628
+ source_proposals=tuple(
1629
+ parse_source_proposal(item) for item in proposal_set["source_proposals"]
1630
+ ),
1631
+ plan=parse_plan(values["plan"]),
1632
+ semantics=(parse_table_semantics(values["semantics"]) if "semantics" in values else None),
1633
+ )
1634
+ inputs.package_context.validate_for_plan(inputs.plan)
1635
+ workspace.validate()
1636
+ return inputs
1637
+
1638
+
1639
+ def _read_regular_file_at(
1640
+ parent_fd: int,
1641
+ name: str,
1642
+ *,
1643
+ maximum: int,
1644
+ required_mode: int,
1645
+ ) -> bytes:
1646
+ named = os.stat(name, dir_fd=parent_fd, follow_symlinks=False)
1647
+ if (
1648
+ not stat.S_ISREG(named.st_mode)
1649
+ or named.st_nlink != 1
1650
+ or stat.S_IMODE(named.st_mode) != required_mode
1651
+ or named.st_size > maximum
1652
+ ):
1653
+ raise OfflineRunError("WORKSPACE_FILE_INVALID", f"workspace file is invalid: {name}")
1654
+ descriptor = os.open(
1655
+ name,
1656
+ os.O_RDONLY | os.O_NOFOLLOW | getattr(os, "O_CLOEXEC", 0),
1657
+ dir_fd=parent_fd,
1658
+ )
1659
+ try:
1660
+ opened = os.fstat(descriptor)
1661
+ if _stat_signature(opened) != _stat_signature(named):
1662
+ raise OfflineRunError("WORKSPACE_RACE", f"workspace file changed: {name}")
1663
+ raw = bytearray()
1664
+ while True:
1665
+ chunk = os.read(descriptor, min(1024 * 1024, maximum + 1 - len(raw)))
1666
+ if not chunk:
1667
+ break
1668
+ raw.extend(chunk)
1669
+ if len(raw) > maximum:
1670
+ raise OfflineRunError("WORKSPACE_FILE_INVALID", f"workspace file too large: {name}")
1671
+ if len(raw) != opened.st_size or _stat_signature(os.fstat(descriptor)) != _stat_signature(
1672
+ opened
1673
+ ):
1674
+ raise OfflineRunError("WORKSPACE_RACE", f"workspace file changed: {name}")
1675
+ finally:
1676
+ os.close(descriptor)
1677
+ after = os.stat(name, dir_fd=parent_fd, follow_symlinks=False)
1678
+ if _stat_signature(after) != _stat_signature(named):
1679
+ raise OfflineRunError("WORKSPACE_RACE", f"workspace file changed: {name}")
1680
+ return bytes(raw)
1681
+
1682
+
1683
+ def _initial_state(run_id: str, definition_sha256s: dict[str, str]) -> dict[str, Any]:
1684
+ state = {
1685
+ "schema_version": RUN_STATE_VERSION,
1686
+ "run_id": run_id,
1687
+ "revision": 0,
1688
+ "phase": "initialized",
1689
+ "paths": {
1690
+ "input_root": INPUT_ROOT_PATH,
1691
+ "result": RUN_RESULT_PATH,
1692
+ },
1693
+ "definition_sha256s": definition_sha256s,
1694
+ "producer": None,
1695
+ "candidate": None,
1696
+ "review": None,
1697
+ "tasks": [
1698
+ {
1699
+ "task_id": task_id,
1700
+ "status": "pending",
1701
+ "attempts": 0,
1702
+ "result_digest": None,
1703
+ }
1704
+ for task_id in _TASK_IDS
1705
+ ],
1706
+ "events": [],
1707
+ }
1708
+ _append_event(state, "run_initialized", None)
1709
+ return _validate_state(state)
1710
+
1711
+
1712
+ def _validate_state(value: Any) -> dict[str, Any]:
1713
+ expected = {
1714
+ "schema_version",
1715
+ "run_id",
1716
+ "revision",
1717
+ "phase",
1718
+ "paths",
1719
+ "definition_sha256s",
1720
+ "producer",
1721
+ "candidate",
1722
+ "review",
1723
+ "tasks",
1724
+ "events",
1725
+ }
1726
+ if not isinstance(value, dict) or set(value) != expected:
1727
+ raise OfflineRunError("RUN_STATE_INVALID", "run state fields are invalid")
1728
+ if value["schema_version"] != RUN_STATE_VERSION:
1729
+ raise OfflineRunError("RUN_STATE_INVALID", "run state version is unsupported")
1730
+ _identifier(value["run_id"], "run_id")
1731
+ if type(value["revision"]) is not int or value["revision"] < 0:
1732
+ raise OfflineRunError("RUN_STATE_INVALID", "run revision is invalid")
1733
+ if value["phase"] not in _PHASES:
1734
+ raise OfflineRunError("RUN_STATE_INVALID", "run phase is invalid")
1735
+ if value["paths"] != {"input_root": INPUT_ROOT_PATH, "result": RUN_RESULT_PATH}:
1736
+ raise OfflineRunError("RUN_STATE_INVALID", "run paths are not the closed H2 paths")
1737
+ digests = value["definition_sha256s"]
1738
+ valid_definition_sets = {
1739
+ frozenset(_DEFINITION_FILES),
1740
+ frozenset({*_DEFINITION_FILES, *_SEMANTICS_DEFINITION}),
1741
+ }
1742
+ if not isinstance(digests, dict) or frozenset(digests) not in valid_definition_sets:
1743
+ raise OfflineRunError("RUN_STATE_INVALID", "definition digests are invalid")
1744
+ for name, digest in digests.items():
1745
+ _digest(digest, f"definition_sha256s.{name}")
1746
+ producer = value["producer"]
1747
+ if producer is not None:
1748
+ if not isinstance(producer, dict) or set(producer) != {"principal_id", "attempt_id"}:
1749
+ raise OfflineRunError("RUN_STATE_INVALID", "producer record is invalid")
1750
+ _identifier(producer["principal_id"], "producer.principal_id")
1751
+ _identifier(producer["attempt_id"], "producer.attempt_id")
1752
+ if producer != _h2_producer_identity():
1753
+ raise OfflineRunError(
1754
+ "RUN_STATE_INVALID",
1755
+ "run producer is not the fixed coordinator-owned H2 identity",
1756
+ )
1757
+ candidate = value["candidate"]
1758
+ if candidate is not None:
1759
+ if not isinstance(candidate, dict) or set(candidate) != {
1760
+ "candidate_digest",
1761
+ "table_sha256",
1762
+ "manifest_version",
1763
+ "row_count",
1764
+ }:
1765
+ raise OfflineRunError("RUN_STATE_INVALID", "candidate record is invalid")
1766
+ _digest(candidate["candidate_digest"], "candidate.candidate_digest")
1767
+ _digest(candidate["table_sha256"], "candidate.table_sha256")
1768
+ if candidate["manifest_version"] not in _OFFLINE_MANIFEST_VERSIONS:
1769
+ raise OfflineRunError("RUN_STATE_INVALID", "candidate manifest version is invalid")
1770
+ if type(candidate["row_count"]) is not int or candidate["row_count"] < 1:
1771
+ raise OfflineRunError("RUN_STATE_INVALID", "candidate row count is invalid")
1772
+ review = value["review"]
1773
+ if review is not None:
1774
+ if not isinstance(review, dict) or set(review) != {
1775
+ "candidate_digest",
1776
+ "decision_digest",
1777
+ "verified_status",
1778
+ }:
1779
+ raise OfflineRunError("RUN_STATE_INVALID", "review record is invalid")
1780
+ _digest(review["candidate_digest"], "review.candidate_digest")
1781
+ _digest(review["decision_digest"], "review.decision_digest")
1782
+ if review["verified_status"] not in {"reviewed_candidate", "fixes_required"}:
1783
+ raise OfflineRunError("RUN_STATE_INVALID", "review status is invalid")
1784
+ tasks = value["tasks"]
1785
+ if not isinstance(tasks, list) or len(tasks) != len(_TASK_IDS):
1786
+ raise OfflineRunError("RUN_STATE_INVALID", "run tasks are invalid")
1787
+ for index, task in enumerate(tasks):
1788
+ if not isinstance(task, dict) or set(task) != {
1789
+ "task_id",
1790
+ "status",
1791
+ "attempts",
1792
+ "result_digest",
1793
+ }:
1794
+ raise OfflineRunError("RUN_STATE_INVALID", f"task {index} is invalid")
1795
+ if task["task_id"] != _TASK_IDS[index] or task["status"] not in _TASK_STATUSES:
1796
+ raise OfflineRunError("RUN_STATE_INVALID", f"task {index} identity/status is invalid")
1797
+ if type(task["attempts"]) is not int or task["attempts"] < 0:
1798
+ raise OfflineRunError("RUN_STATE_INVALID", f"task {index} attempts are invalid")
1799
+ if task["result_digest"] is not None:
1800
+ _digest(task["result_digest"], f"tasks[{index}].result_digest")
1801
+ if task["status"] == "pending" and (
1802
+ task["attempts"] != 0 or task["result_digest"] is not None
1803
+ ):
1804
+ raise OfflineRunError(
1805
+ "RUN_STATE_INVALID",
1806
+ f"pending task {index} has execution state",
1807
+ )
1808
+ if task["status"] in {"running", "failed"} and (
1809
+ task["attempts"] < 1 or task["result_digest"] is not None
1810
+ ):
1811
+ raise OfflineRunError(
1812
+ "RUN_STATE_INVALID",
1813
+ f"active task {index} has invalid execution state",
1814
+ )
1815
+ if task["status"] == "completed" and (
1816
+ task["attempts"] < 1 or task["result_digest"] is None
1817
+ ):
1818
+ raise OfflineRunError(
1819
+ "RUN_STATE_INVALID",
1820
+ f"completed task {index} lacks its result",
1821
+ )
1822
+ task_statuses = tuple(task["status"] for task in tasks)
1823
+ if task_statuses != _PHASE_TASK_STATUSES[value["phase"]]:
1824
+ raise OfflineRunError(
1825
+ "RUN_STATE_INVALID",
1826
+ "run phase and task statuses are inconsistent",
1827
+ )
1828
+ events = value["events"]
1829
+ if not isinstance(events, list):
1830
+ raise OfflineRunError("RUN_STATE_INVALID", "events must be an array")
1831
+ for index, event in enumerate(events, start=1):
1832
+ if not isinstance(event, dict) or set(event) != {
1833
+ "sequence",
1834
+ "event",
1835
+ "task_id",
1836
+ "phase",
1837
+ }:
1838
+ raise OfflineRunError("RUN_STATE_INVALID", f"event {index} is invalid")
1839
+ if event["sequence"] != index or event["phase"] not in _PHASES:
1840
+ raise OfflineRunError("RUN_STATE_INVALID", f"event {index} ordering is invalid")
1841
+ _identifier(event["event"], f"events[{index}].event")
1842
+ if event["task_id"] is not None and event["task_id"] not in _TASK_IDS:
1843
+ raise OfflineRunError("RUN_STATE_INVALID", f"event {index} task is invalid")
1844
+ build_task = tasks[_TASK_IDS.index("build")]
1845
+ review_task = tasks[_TASK_IDS.index("review")]
1846
+ producer_required = value["phase"] not in {"initialized", "planned"}
1847
+ if (producer is not None) != producer_required:
1848
+ raise OfflineRunError(
1849
+ "RUN_STATE_INVALID",
1850
+ "run phase and producer enrollment are inconsistent",
1851
+ )
1852
+ candidate_required = value["phase"] in {"candidate_built", "review_recorded"}
1853
+ if (candidate is not None) != candidate_required:
1854
+ raise OfflineRunError(
1855
+ "RUN_STATE_INVALID",
1856
+ "run phase and candidate state are inconsistent",
1857
+ )
1858
+ review_required = value["phase"] == "review_recorded"
1859
+ if (review is not None) != review_required:
1860
+ raise OfflineRunError(
1861
+ "RUN_STATE_INVALID",
1862
+ "run phase and review state are inconsistent",
1863
+ )
1864
+ if candidate is not None and producer is None:
1865
+ raise OfflineRunError(
1866
+ "RUN_STATE_INVALID",
1867
+ "candidate state requires a producer fence",
1868
+ )
1869
+ if build_task["status"] == "completed" and candidate is None:
1870
+ raise OfflineRunError(
1871
+ "RUN_STATE_INVALID",
1872
+ "completed build task requires candidate state",
1873
+ )
1874
+ if value["phase"] in {"candidate_built", "review_recorded"} and candidate is None:
1875
+ raise OfflineRunError(
1876
+ "RUN_STATE_INVALID",
1877
+ "terminal candidate phase requires candidate state",
1878
+ )
1879
+ if candidate is not None and build_task["result_digest"] != candidate["candidate_digest"]:
1880
+ raise OfflineRunError(
1881
+ "RUN_STATE_INVALID",
1882
+ "completed build result does not match candidate state",
1883
+ )
1884
+ if review is not None:
1885
+ if candidate is None or review["candidate_digest"] != candidate["candidate_digest"]:
1886
+ raise OfflineRunError(
1887
+ "RUN_STATE_INVALID",
1888
+ "review record must reference the durable candidate",
1889
+ )
1890
+ if review_task["status"] != "completed":
1891
+ raise OfflineRunError(
1892
+ "RUN_STATE_INVALID",
1893
+ "review record requires a completed review task",
1894
+ )
1895
+ if value["phase"] == "review_recorded" and review is None:
1896
+ raise OfflineRunError(
1897
+ "RUN_STATE_INVALID",
1898
+ "review_recorded phase requires a review record",
1899
+ )
1900
+ if review is not None and review_task["result_digest"] != review["decision_digest"]:
1901
+ raise OfflineRunError(
1902
+ "RUN_STATE_INVALID",
1903
+ "completed review result does not match review state",
1904
+ )
1905
+ return value
1906
+
1907
+
1908
+ def _append_event(state: dict[str, Any], event: str, task_id: str | None) -> None:
1909
+ state["events"].append(
1910
+ {
1911
+ "sequence": len(state["events"]) + 1,
1912
+ "event": _identifier(event, "event"),
1913
+ "task_id": task_id,
1914
+ "phase": state["phase"],
1915
+ }
1916
+ )
1917
+
1918
+
1919
+ def _task(state: dict[str, Any], task_id: str) -> dict[str, Any]:
1920
+ return state["tasks"][_TASK_IDS.index(task_id)]
1921
+
1922
+
1923
+ def _status_payload(state: dict[str, Any], workspace: Path) -> dict[str, Any]:
1924
+ review = state["review"]
1925
+ return {
1926
+ "status": state["phase"],
1927
+ "run_id": state["run_id"],
1928
+ "workspace": str(workspace),
1929
+ "revision": state["revision"],
1930
+ "tasks": deepcopy(state["tasks"]),
1931
+ "events": deepcopy(state["events"]),
1932
+ "candidate": deepcopy(state["candidate"]),
1933
+ "review_record": deepcopy(review),
1934
+ "review_authority": (
1935
+ "record_only_reverify_with_enrolled_public_key"
1936
+ if review is not None
1937
+ else "not_reviewed"
1938
+ ),
1939
+ # The local coordinator cannot authorize hosted release or verify a repaired successor.
1940
+ "hosted_release": False,
1941
+ "successor_repair_closure": False,
1942
+ }
1943
+
1944
+
1945
+ def _remove_orphan_staging(workspace: _Workspace) -> None:
1946
+ workspace.validate()
1947
+ for child in os.listdir(workspace.root_fd):
1948
+ if not _ORPHAN_STAGING.fullmatch(child):
1949
+ continue
1950
+ named = os.stat(child, dir_fd=workspace.root_fd, follow_symlinks=False)
1951
+ if stat.S_ISDIR(named.st_mode):
1952
+ _remove_tree(
1953
+ workspace.path / child,
1954
+ expected_identity=_entry_identity(named),
1955
+ parent_fd=workspace.root_fd,
1956
+ )
1957
+ workspace.validate()
1958
+
1959
+
1960
+ def _named_child_exists(parent_fd: int, name: str) -> bool:
1961
+ try:
1962
+ os.stat(name, dir_fd=parent_fd, follow_symlinks=False)
1963
+ except FileNotFoundError:
1964
+ return False
1965
+ return True
1966
+
1967
+
1968
+ def _read_external_json(path: Path, label: str) -> Any:
1969
+ """One of the four files a job is described in, read strictly.
1970
+
1971
+ Each refusal names the file twice over: by what it is for (`question`, `plan`) and by where it
1972
+ is, because a caller passes four paths on one command line and "one of them is not valid JSON"
1973
+ is not an answer. The strict reader's own detail -- the line and column -- is carried through
1974
+ for the same reason: it is the half that says where to look.
1975
+
1976
+ The read goes through :mod:`ux.plain_file` -- open once, ask the descriptor what it is, read
1977
+ that descriptor under a bound -- because ``path.read_bytes()`` on a path somebody typed is the
1978
+ recurring defect class shared with the authoring, preview, and command-line document readers.
1979
+ Every one of the four paths ``mr-data init`` takes comes through this function, so an unguarded
1980
+ ``--question <a fifo>`` would open the pipe
1981
+ and waited for a writer that never came: no output, no exit, and nothing created. The bound was
1982
+ real but arrived too late -- ``read_bytes`` had already read the whole file before anything
1983
+ measured it -- so a file far past the limit was held in memory to be refused.
1984
+ """
1985
+
1986
+ try:
1987
+ raw = read_plain_file(path, max_bytes=_MAX_DEFINITION_BYTES)
1988
+ except PlainFileRefusal as refusal:
1989
+ if refusal.reason == TOO_LARGE:
1990
+ raise OfflineRunError(
1991
+ "DEFINITION_TOO_LARGE", f"{label} exceeds the byte limit: {path}"
1992
+ ) from None
1993
+ # A pipe, a socket or a device is the one refusal `DEFINITION_UNREADABLE` has no words for:
1994
+ # its remediation says to check the file is there, is a plain file rather than a folder,
1995
+ # and is readable, and all three of those are true of a FIFO nobody is writing to. A folder
1996
+ # and a missing path keep that code, because that is the sentence written for them.
1997
+ if refusal.reason == DANGLING:
1998
+ # A dangling link needs its own refusal: generic readability advice would claim the
1999
+ # named entry is absent or unreadable without identifying the link. The noun comes
2000
+ # from the mode this rule observed.
2001
+ raise OfflineRunError(
2002
+ "DEFINITION_TARGET_DANGLING",
2003
+ f"{label} is at {refusal.path}, which is "
2004
+ f"{name_of_kind(refusal.mode) or UNKNOWN_KIND} "
2005
+ "that leads nowhere",
2006
+ ) from None
2007
+ if refusal.reason == NOT_PLAIN and not stat.S_ISDIR(refusal.mode or 0):
2008
+ raise OfflineRunError(
2009
+ "DEFINITION_TARGET_NOT_A_FILE",
2010
+ f"{label} is not a file: {path}",
2011
+ ) from None
2012
+ raise OfflineRunError("DEFINITION_UNREADABLE", f"{label} cannot be read: {path}") from None
2013
+ except OSError as exc:
2014
+ raise OfflineRunError("DEFINITION_UNREADABLE", f"{label} cannot be read: {path}") from exc
2015
+ try:
2016
+ return parse_json(raw)
2017
+ except CanonicalJSONError as exc:
2018
+ raise OfflineRunError(
2019
+ "DEFINITION_INVALID", f"{label} is not strict JSON ({exc.detail}): {path}"
2020
+ ) from exc
2021
+ except ValueError as exc:
2022
+ raise OfflineRunError("DEFINITION_INVALID", f"{label} is not strict JSON: {path}") from exc
2023
+
2024
+
2025
+ def _identifier(value: Any, name: str) -> str:
2026
+ if not isinstance(value, str) or _IDENTIFIER.fullmatch(value) is None:
2027
+ raise OfflineRunError(
2028
+ "IDENTIFIER_INVALID",
2029
+ f"{name} must be a bounded alphanumeric, '-' or '_' identifier",
2030
+ )
2031
+ return value
2032
+
2033
+
2034
+ def _digest(value: Any, name: str) -> str:
2035
+ if not isinstance(value, str) or _DIGEST.fullmatch(value) is None:
2036
+ raise OfflineRunError("DIGEST_INVALID", f"{name} must be a lowercase SHA-256")
2037
+ return value
2038
+
2039
+
2040
+ def _stat_signature(info: os.stat_result) -> tuple[int, ...]:
2041
+ return (
2042
+ info.st_dev,
2043
+ info.st_ino,
2044
+ info.st_mode,
2045
+ info.st_nlink,
2046
+ info.st_size,
2047
+ info.st_mtime_ns,
2048
+ info.st_ctime_ns,
2049
+ )