mostlyright-data 0.9.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (314) hide show
  1. mostlyright/data_harness/__init__.py +158 -0
  2. mostlyright/data_harness/acquisition/__init__.py +55 -0
  3. mostlyright/data_harness/acquisition/http.py +2773 -0
  4. mostlyright/data_harness/acquisition/parsing.py +809 -0
  5. mostlyright/data_harness/acquisition/ranges.py +495 -0
  6. mostlyright/data_harness/acquisition/result_download.py +360 -0
  7. mostlyright/data_harness/acquisition/retention_admission.py +248 -0
  8. mostlyright/data_harness/acquisition/sandbox.py +4888 -0
  9. mostlyright/data_harness/acquisition/url_policy.py +530 -0
  10. mostlyright/data_harness/agent_runtime.py +2743 -0
  11. mostlyright/data_harness/assets/logo-ink.svg +31 -0
  12. mostlyright/data_harness/backends/__init__.py +28 -0
  13. mostlyright/data_harness/backends/pandas_backend.py +350 -0
  14. mostlyright/data_harness/backends/polars_backend.py +366 -0
  15. mostlyright/data_harness/backends/protocol.py +124 -0
  16. mostlyright/data_harness/backends/reference.py +83 -0
  17. mostlyright/data_harness/backends/registry.py +55 -0
  18. mostlyright/data_harness/backends/restrictions.py +126 -0
  19. mostlyright/data_harness/canonical.py +333 -0
  20. mostlyright/data_harness/catalog_job.py +625 -0
  21. mostlyright/data_harness/cli.py +5398 -0
  22. mostlyright/data_harness/contracts.py +53 -0
  23. mostlyright/data_harness/coordinator.py +1307 -0
  24. mostlyright/data_harness/deploy.py +924 -0
  25. mostlyright/data_harness/deploy_target.py +312 -0
  26. mostlyright/data_harness/deployment_evidence.py +1067 -0
  27. mostlyright/data_harness/event_presentation.py +576 -0
  28. mostlyright/data_harness/events.py +2152 -0
  29. mostlyright/data_harness/fast_delimited.py +239 -0
  30. mostlyright/data_harness/fleet.py +237 -0
  31. mostlyright/data_harness/formats.py +236 -0
  32. mostlyright/data_harness/governors.py +1163 -0
  33. mostlyright/data_harness/hosted_bootstrap.py +972 -0
  34. mostlyright/data_harness/hosted_crawler.py +1115 -0
  35. mostlyright/data_harness/hosted_crawler_container_smoke.py +351 -0
  36. mostlyright/data_harness/hosted_crawler_fetch.py +423 -0
  37. mostlyright/data_harness/hosted_crawler_job.py +1277 -0
  38. mostlyright/data_harness/hosted_crawler_protocol.py +676 -0
  39. mostlyright/data_harness/hosted_dataset.py +1500 -0
  40. mostlyright/data_harness/hosted_deploy.py +3037 -0
  41. mostlyright/data_harness/hosted_handoff.py +62 -0
  42. mostlyright/data_harness/hosted_ingestion_contract.py +504 -0
  43. mostlyright/data_harness/hosted_ingestion_job.py +356 -0
  44. mostlyright/data_harness/hosted_ingestion_job_smoke.py +40 -0
  45. mostlyright/data_harness/hosted_session_container_smoke.py +194 -0
  46. mostlyright/data_harness/hosted_session_worker.py +3554 -0
  47. mostlyright/data_harness/hosted_session_worker_job_smoke.py +46 -0
  48. mostlyright/data_harness/hosted_worker.py +6784 -0
  49. mostlyright/data_harness/ingestion/__init__.py +56 -0
  50. mostlyright/data_harness/ingestion/contracts.py +461 -0
  51. mostlyright/data_harness/ingestion/faults.py +42 -0
  52. mostlyright/data_harness/ingestion/gcs_store.py +1162 -0
  53. mostlyright/data_harness/ingestion/spool.py +130 -0
  54. mostlyright/data_harness/ingestion/store.py +885 -0
  55. mostlyright/data_harness/key_seam.py +434 -0
  56. mostlyright/data_harness/linux_process_boundary.py +262 -0
  57. mostlyright/data_harness/local_contracts.py +2880 -0
  58. mostlyright/data_harness/local_search/__init__.py +5 -0
  59. mostlyright/data_harness/local_search/build_index.py +1087 -0
  60. mostlyright/data_harness/local_search/contracts.py +920 -0
  61. mostlyright/data_harness/local_search/query_trace.py +266 -0
  62. mostlyright/data_harness/local_search/retrieval.py +700 -0
  63. mostlyright/data_harness/local_search/sealed.py +474 -0
  64. mostlyright/data_harness/local_search/service.py +784 -0
  65. mostlyright/data_harness/nbrender/CONTRACT.md +212 -0
  66. mostlyright/data_harness/nbrender/__init__.py +12 -0
  67. mostlyright/data_harness/nbrender/chrome.py +359 -0
  68. mostlyright/data_harness/nbrender/code_body.py +266 -0
  69. mostlyright/data_harness/nbrender/document.py +407 -0
  70. mostlyright/data_harness/nbrender/frame.py +275 -0
  71. mostlyright/data_harness/nbrender/interactive.py +337 -0
  72. mostlyright/data_harness/nbrender/markdown_body.py +477 -0
  73. mostlyright/data_harness/nbrender/mr_components.py +134 -0
  74. mostlyright/data_harness/nbrender/outputs_data.py +595 -0
  75. mostlyright/data_harness/nbrender/outputs_rich.py +906 -0
  76. mostlyright/data_harness/nbrender/outputs_source.py +260 -0
  77. mostlyright/data_harness/nbrender/outputs_stage.py +176 -0
  78. mostlyright/data_harness/nbrender/outputs_text.py +400 -0
  79. mostlyright/data_harness/nbrender/parse.py +394 -0
  80. mostlyright/data_harness/nbrender/status.py +40 -0
  81. mostlyright/data_harness/nbrender/tokens.py +1295 -0
  82. mostlyright/data_harness/notebook.py +1710 -0
  83. mostlyright/data_harness/offline.py +2049 -0
  84. mostlyright/data_harness/operation_registry.py +1007 -0
  85. mostlyright/data_harness/operator_setup.py +239 -0
  86. mostlyright/data_harness/pipeline.py +6428 -0
  87. mostlyright/data_harness/plan_graph.py +2026 -0
  88. mostlyright/data_harness/preparation/__init__.py +104 -0
  89. mostlyright/data_harness/preparation/contracts.py +1017 -0
  90. mostlyright/data_harness/preparation/engine.py +221 -0
  91. mostlyright/data_harness/preparation/errors.py +14 -0
  92. mostlyright/data_harness/preparation/gates.py +751 -0
  93. mostlyright/data_harness/preparation/joins.py +574 -0
  94. mostlyright/data_harness/preparation/profile.py +384 -0
  95. mostlyright/data_harness/preparation/table.py +217 -0
  96. mostlyright/data_harness/preparation/transforms.py +568 -0
  97. mostlyright/data_harness/progress_events.py +534 -0
  98. mostlyright/data_harness/readers/__init__.py +46 -0
  99. mostlyright/data_harness/readers/containers.py +963 -0
  100. mostlyright/data_harness/readers/contracts.py +542 -0
  101. mostlyright/data_harness/readers/delimited.py +257 -0
  102. mostlyright/data_harness/readers/grib2/__init__.py +33 -0
  103. mostlyright/data_harness/readers/grib2/admission.py +722 -0
  104. mostlyright/data_harness/readers/grib2/decode.py +1009 -0
  105. mostlyright/data_harness/readers/grib2/geometry.py +1133 -0
  106. mostlyright/data_harness/readers/grib2/portable_math.py +501 -0
  107. mostlyright/data_harness/readers/json_tabular.py +485 -0
  108. mostlyright/data_harness/readers/registry.py +514 -0
  109. mostlyright/data_harness/readers/samples/README.md +110 -0
  110. mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/cities.csv.gz +0 -0
  111. mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/expected.json +24 -0
  112. mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/cities.csv.gz +0 -0
  113. mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/expected.json +24 -0
  114. mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/cities.tar +0 -0
  115. mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/expected.json +24 -0
  116. mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/cities.tar +0 -0
  117. mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/expected.json +24 -0
  118. mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/cities.zip +0 -0
  119. mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/expected.json +25 -0
  120. mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
  121. mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/expected.json +25 -0
  122. mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
  123. mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/expected.json +25 -0
  124. mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/cities.csv +3 -0
  125. mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/expected.json +23 -0
  126. mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/cities.tsv +5 -0
  127. mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/expected.json +25 -0
  128. mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/expected.json +30 -0
  129. mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/observations.csv +5 -0
  130. mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/expected.json +44 -0
  131. mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/stations.json +1 -0
  132. mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/expected.json +48 -0
  133. mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/observations.ndjson +4 -0
  134. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/cities.xlsx +0 -0
  135. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/expected.json +24 -0
  136. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/expected.json +27 -0
  137. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
  138. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/expected.json +27 -0
  139. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
  140. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/README.md +20 -0
  141. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/expected.json +55 -0
  142. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/gfs-2m-temperature.grib2 +0 -0
  143. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/expected.json +54 -0
  144. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
  145. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/expected.json +54 -0
  146. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/hrrr-categorical-rain.grib2 +0 -0
  147. mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/expected.json +54 -0
  148. mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
  149. mostlyright/data_harness/readers/samples.py +582 -0
  150. mostlyright/data_harness/readers/spreadsheet.py +803 -0
  151. mostlyright/data_harness/readers/tabular.py +510 -0
  152. mostlyright/data_harness/recipe.py +5321 -0
  153. mostlyright/data_harness/repair/__init__.py +78 -0
  154. mostlyright/data_harness/repair/adapters.py +274 -0
  155. mostlyright/data_harness/repair/contracts.py +872 -0
  156. mostlyright/data_harness/repair/coordinator.py +1099 -0
  157. mostlyright/data_harness/repair/errors.py +16 -0
  158. mostlyright/data_harness/review.py +2533 -0
  159. mostlyright/data_harness/rowset.py +283 -0
  160. mostlyright/data_harness/serving.py +1975 -0
  161. mostlyright/data_harness/serving_edge.py +590 -0
  162. mostlyright/data_harness/serving_http.py +1031 -0
  163. mostlyright/data_harness/session_probes.py +759 -0
  164. mostlyright/data_harness/signing.py +101 -0
  165. mostlyright/data_harness/source_discovery.py +898 -0
  166. mostlyright/data_harness/sources/__init__.py +209 -0
  167. mostlyright/data_harness/sources/_adapter_steps.py +213 -0
  168. mostlyright/data_harness/sources/adapters.py +1214 -0
  169. mostlyright/data_harness/sources/cadence.py +1428 -0
  170. mostlyright/data_harness/sources/cadence_emission.py +453 -0
  171. mostlyright/data_harness/sources/cadence_history.py +546 -0
  172. mostlyright/data_harness/sources/catalog/__init__.py +17 -0
  173. mostlyright/data_harness/sources/catalog/admission.py +477 -0
  174. mostlyright/data_harness/sources/catalog/authoring.py +1701 -0
  175. mostlyright/data_harness/sources/catalog/authoring_policy.py +701 -0
  176. mostlyright/data_harness/sources/catalog/authoring_shards.py +1217 -0
  177. mostlyright/data_harness/sources/catalog/bounded_io.py +231 -0
  178. mostlyright/data_harness/sources/catalog/channel.py +523 -0
  179. mostlyright/data_harness/sources/catalog/channel_client.py +296 -0
  180. mostlyright/data_harness/sources/catalog/contracts.py +825 -0
  181. mostlyright/data_harness/sources/catalog/coverage.py +137 -0
  182. mostlyright/data_harness/sources/catalog/delta.py +1340 -0
  183. mostlyright/data_harness/sources/catalog/embedding.py +532 -0
  184. mostlyright/data_harness/sources/catalog/entry_v2.py +1182 -0
  185. mostlyright/data_harness/sources/catalog/fill.py +3889 -0
  186. mostlyright/data_harness/sources/catalog/fill_partitions.py +459 -0
  187. mostlyright/data_harness/sources/catalog/fill_staging.py +1105 -0
  188. mostlyright/data_harness/sources/catalog/gating.py +374 -0
  189. mostlyright/data_harness/sources/catalog/generation_receipt.py +1607 -0
  190. mostlyright/data_harness/sources/catalog/harvest/__init__.py +7 -0
  191. mostlyright/data_harness/sources/catalog/harvest/ckan.py +384 -0
  192. mostlyright/data_harness/sources/catalog/harvest/datagov_v4.py +798 -0
  193. mostlyright/data_harness/sources/catalog/harvest/protocol.py +964 -0
  194. mostlyright/data_harness/sources/catalog/harvest/sdmx.py +445 -0
  195. mostlyright/data_harness/sources/catalog/harvest/stac.py +384 -0
  196. mostlyright/data_harness/sources/catalog/health.py +447 -0
  197. mostlyright/data_harness/sources/catalog/hosted_catalog.py +105 -0
  198. mostlyright/data_harness/sources/catalog/identity_history.py +1549 -0
  199. mostlyright/data_harness/sources/catalog/neural.py +1618 -0
  200. mostlyright/data_harness/sources/catalog/packed_catalog.py +2345 -0
  201. mostlyright/data_harness/sources/catalog/packed_retrieval.py +1517 -0
  202. mostlyright/data_harness/sources/catalog/packed_writer.py +2802 -0
  203. mostlyright/data_harness/sources/catalog/query_trace.py +1037 -0
  204. mostlyright/data_harness/sources/catalog/recommend.py +171 -0
  205. mostlyright/data_harness/sources/catalog/retrieval.py +230 -0
  206. mostlyright/data_harness/sources/catalog/retrieval_manifest.py +995 -0
  207. mostlyright/data_harness/sources/catalog/rights_decisions.py +254 -0
  208. mostlyright/data_harness/sources/catalog/sealed.py +560 -0
  209. mostlyright/data_harness/sources/catalog/search.py +230 -0
  210. mostlyright/data_harness/sources/catalog/streaming_delta.py +1097 -0
  211. mostlyright/data_harness/sources/catalog/update.py +891 -0
  212. mostlyright/data_harness/sources/collections.py +815 -0
  213. mostlyright/data_harness/sources/contracts.py +2223 -0
  214. mostlyright/data_harness/sources/deletion.py +761 -0
  215. mostlyright/data_harness/sources/fitness.py +162 -0
  216. mostlyright/data_harness/sources/governance.py +163 -0
  217. mostlyright/data_harness/sources/hosted.py +173 -0
  218. mostlyright/data_harness/sources/integration.py +218 -0
  219. mostlyright/data_harness/sources/range_reader.py +418 -0
  220. mostlyright/data_harness/sources/registry.py +514 -0
  221. mostlyright/data_harness/sources/rights_rule.py +59 -0
  222. mostlyright/data_harness/sources/source_cadence_vectors.v1.json +1 -0
  223. mostlyright/data_harness/sources/sports.py +521 -0
  224. mostlyright/data_harness/sources/stream.py +524 -0
  225. mostlyright/data_harness/sources/stream_connector.py +418 -0
  226. mostlyright/data_harness/sources/stream_recorder.py +1404 -0
  227. mostlyright/data_harness/studio_boundary.py +2019 -0
  228. mostlyright/data_harness/thin/__init__.py +37 -0
  229. mostlyright/data_harness/thin/acquire.py +1137 -0
  230. mostlyright/data_harness/thin/acquire_cancel.py +579 -0
  231. mostlyright/data_harness/thin/approvals.py +617 -0
  232. mostlyright/data_harness/thin/commands.py +406 -0
  233. mostlyright/data_harness/thin/download.py +194 -0
  234. mostlyright/data_harness/thin/narrative.py +589 -0
  235. mostlyright/data_harness/thin/parity.py +1070 -0
  236. mostlyright/data_harness/thin/propose.py +2759 -0
  237. mostlyright/data_harness/thin/research.py +1663 -0
  238. mostlyright/data_harness/thin/router.py +924 -0
  239. mostlyright/data_harness/thin/runs.py +519 -0
  240. mostlyright/data_harness/thin/session.py +281 -0
  241. mostlyright/data_harness/thin/stream.py +501 -0
  242. mostlyright/data_harness/thin/transport.py +187 -0
  243. mostlyright/data_harness/thin/vocabulary.py +368 -0
  244. mostlyright/data_harness/thin/workers.py +164 -0
  245. mostlyright/data_harness/ucum/TABLE-PIN.json +40 -0
  246. mostlyright/data_harness/ucum/ucum-subset.v1.json +632 -0
  247. mostlyright/data_harness/unit_flow.py +927 -0
  248. mostlyright/data_harness/units.py +572 -0
  249. mostlyright/data_harness/ux/__init__.py +9 -0
  250. mostlyright/data_harness/ux/approve.py +485 -0
  251. mostlyright/data_harness/ux/author_yaml.py +597 -0
  252. mostlyright/data_harness/ux/cloud_auth.py +447 -0
  253. mostlyright/data_harness/ux/commands/__init__.py +260 -0
  254. mostlyright/data_harness/ux/commands/approve.py +136 -0
  255. mostlyright/data_harness/ux/commands/auth.py +744 -0
  256. mostlyright/data_harness/ux/commands/author.py +79 -0
  257. mostlyright/data_harness/ux/commands/catalog_author.py +403 -0
  258. mostlyright/data_harness/ux/commands/catalog_fill.py +523 -0
  259. mostlyright/data_harness/ux/commands/catalog_harvest.py +545 -0
  260. mostlyright/data_harness/ux/commands/catalog_publish.py +1838 -0
  261. mostlyright/data_harness/ux/commands/catalog_search.py +71 -0
  262. mostlyright/data_harness/ux/commands/catalog_update.py +437 -0
  263. mostlyright/data_harness/ux/commands/deploy.py +134 -0
  264. mostlyright/data_harness/ux/commands/deploy_dataset.py +98 -0
  265. mostlyright/data_harness/ux/commands/deploy_plan.py +105 -0
  266. mostlyright/data_harness/ux/commands/deploy_status.py +104 -0
  267. mostlyright/data_harness/ux/commands/diff.py +74 -0
  268. mostlyright/data_harness/ux/commands/index.py +84 -0
  269. mostlyright/data_harness/ux/commands/inventory.py +47 -0
  270. mostlyright/data_harness/ux/commands/list_builds.py +143 -0
  271. mostlyright/data_harness/ux/commands/login.py +63 -0
  272. mostlyright/data_harness/ux/commands/peek.py +236 -0
  273. mostlyright/data_harness/ux/commands/plan_check.py +90 -0
  274. mostlyright/data_harness/ux/commands/preflight.py +97 -0
  275. mostlyright/data_harness/ux/commands/record.py +107 -0
  276. mostlyright/data_harness/ux/commands/review_setup.py +47 -0
  277. mostlyright/data_harness/ux/commands/search.py +440 -0
  278. mostlyright/data_harness/ux/commands/show.py +61 -0
  279. mostlyright/data_harness/ux/commands/whoami.py +37 -0
  280. mostlyright/data_harness/ux/credential_native.py +551 -0
  281. mostlyright/data_harness/ux/credential_store.py +1055 -0
  282. mostlyright/data_harness/ux/credentials.py +631 -0
  283. mostlyright/data_harness/ux/diffing.py +444 -0
  284. mostlyright/data_harness/ux/headline.py +671 -0
  285. mostlyright/data_harness/ux/hosted_acquisition.py +974 -0
  286. mostlyright/data_harness/ux/hosted_run_status.py +619 -0
  287. mostlyright/data_harness/ux/inventory.py +427 -0
  288. mostlyright/data_harness/ux/local_review.py +375 -0
  289. mostlyright/data_harness/ux/login.py +691 -0
  290. mostlyright/data_harness/ux/path_kind.py +147 -0
  291. mostlyright/data_harness/ux/peek.py +1000 -0
  292. mostlyright/data_harness/ux/plain_file.py +178 -0
  293. mostlyright/data_harness/ux/plan_check.py +311 -0
  294. mostlyright/data_harness/ux/preflight.py +918 -0
  295. mostlyright/data_harness/ux/readers.py +1124 -0
  296. mostlyright/data_harness/ux/remediation.py +2195 -0
  297. mostlyright/data_harness/ux/render.py +657 -0
  298. mostlyright/data_harness/ux/workload.py +1077 -0
  299. mostlyright/data_harness/viewer.py +3713 -0
  300. mostlyright/data_harness/visual_run/__init__.py +83 -0
  301. mostlyright/data_harness/visual_run/authoring.py +235 -0
  302. mostlyright/data_harness/visual_run/contracts.py +673 -0
  303. mostlyright/data_harness/visual_run/materialize.py +486 -0
  304. mostlyright/data_harness/visual_run/observations.py +874 -0
  305. mostlyright/data_harness/visual_run/query.py +259 -0
  306. mostlyright/data_harness/visual_run/reducer.py +280 -0
  307. mostlyright/data_harness/visual_run/sdk.py +892 -0
  308. mostlyright/data_harness/visual_run/store.py +584 -0
  309. mostlyright/data_harness/visual_run/transport.py +239 -0
  310. mostlyright/data_harness/watch.py +2999 -0
  311. mostlyright_data-0.9.0.dist-info/METADATA +607 -0
  312. mostlyright_data-0.9.0.dist-info/RECORD +314 -0
  313. mostlyright_data-0.9.0.dist-info/WHEEL +4 -0
  314. mostlyright_data-0.9.0.dist-info/entry_points.txt +12 -0
@@ -0,0 +1,2999 @@
1
+ """The live run view for ``mr-data watch``: a localhost page that tails one run's event feed.
2
+
3
+ Watch is the live run view; the notebook sidecar (``mr-data view``) is the finished dataset view.
4
+ They share HTTP serving conventions but no files. Watch uses ``ThreadingHTTPServer``, an ephemeral
5
+ loopback port, a 0.25 s SSE poll, and a disabled access log.
6
+
7
+ Watch is a spectator. No route on this server mutates anything, and there is no POST handler at
8
+ all: v1 approval is display-only, and the ABSENCE of a mutating route is the mitigation for the
9
+ approval-authority threat. Nothing here writes to the run directory, to ``state.json``, or to the
10
+ feed; deleting the feed loses nothing but the show.
11
+
12
+ One thing is fixed here that the merged viewer still gets wrong: rendering on the request thread.
13
+ Watch is ALWAYS reading a file another process is appending to, so the watcher thread reads, parses
14
+ and renders, and the handler only writes bytes it was handed and cannot fail on bad feed bytes.
15
+ """
16
+
17
+ from __future__ import annotations
18
+
19
+ import errno
20
+ import hashlib
21
+ import hmac
22
+ import ipaddress
23
+ import json
24
+ import os
25
+ import re
26
+ import socket
27
+ import threading
28
+ import time
29
+ import urllib.parse
30
+ import webbrowser
31
+ from collections.abc import Callable, Mapping, Sequence
32
+ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
33
+ from pathlib import Path
34
+ from typing import Any, NamedTuple
35
+
36
+ from mostlyright.data_harness import events
37
+ from mostlyright.data_harness.event_presentation import (
38
+ EVENT_SENTENCES as _EVENT_SENTENCES,
39
+ )
40
+ from mostlyright.data_harness.event_presentation import (
41
+ STAGE_RAIL,
42
+ STOPPED_EVENTS,
43
+ sentence,
44
+ )
45
+ from mostlyright.data_harness.nbrender.parse import esc
46
+
47
+ EVENT_SENTENCES = _EVENT_SENTENCES
48
+
49
+ # Keep the live-run palette independent of notebook renderer imports.
50
+ PAPER = "#F5F2EB" # page ground
51
+ SUB_PAPER = "#FAF8F3" # card fill
52
+ SURFACE = "#EDE9DF" # rail fill, command block fill
53
+ BORDER = "#E2DDD0" # every 1px border
54
+ INK = "#17160F" # headings, reached labels
55
+ BODY = "#3B382F" # body text
56
+ MUTED = "#57534A" # secondary text
57
+ FAINT = "#6E6959" # tertiary text, unreached labels
58
+ DIM = "#837D6E" # the "receipt appears later" note
59
+ COBALT = "#2B5FE3" # links, reached marks, the copy button
60
+ ORANGE = "#F15B22" # a build that stopped
61
+ FONT_SANS = '"Space Grotesk", "Helvetica Neue", sans-serif'
62
+ FONT_MONO = '"JetBrains Mono", Menlo, monospace'
63
+
64
+ # The watcher and SSE loop use the same interval.
65
+ _POLL_SECONDS = 0.25
66
+
67
+ # The manifest is a file another local user may have written. Read it with a ceiling rather than
68
+ # allocating whatever is on disk.
69
+ _MAX_MANIFEST_BYTES = 4 * 1024 * 1024
70
+
71
+ # These errnos mean the candidate path is absent, not a directory, or a symlink. Other errors fail
72
+ # the current poll without invalidating an already pinned descriptor.
73
+ _NAME_HOLDS_NO_CANDIDATE = frozenset({errno.ENOENT, errno.ENOTDIR, errno.ELOOP})
74
+
75
+ # `manifest.json` is a real file inside the run but is NOT one of `manifest["members"]` -- a
76
+ # manifest cannot carry its own digest -- so the member set carries it explicitly.
77
+ _MANIFEST_NAME = "manifest.json"
78
+
79
+ # The ceiling on a single served receipt. Every member of a real run is far below it; the cap
80
+ # exists because the run directory is a place another local user can write (T-30-24).
81
+ MAX_RECEIPT_BYTES = 2 * 1024 * 1024
82
+
83
+ # Proving a member is the file the run recorded means hashing it, and a dataset is the one member
84
+ # that can be large. It is read in chunks so the cost is time and not memory.
85
+ _MEMBER_HASH_CHUNK_BYTES = 1024 * 1024
86
+
87
+ # Bound the total bytes and member count verified in one pass. Members are considered in manifest
88
+ # order; a member larger than the remaining byte budget is skipped.
89
+ MAX_VERIFIED_TOTAL_BYTES = 256 * 1024 * 1024
90
+ MAX_MANIFEST_MEMBERS = 4096
91
+
92
+ # A settled run must still notice a member changed without turning the maximum accepted receipt
93
+ # into a permanent syscall budget. Page-authoritative members are checked every tick; the rest are
94
+ # walked as a rotating slice, so every accepted member is revisited within a bounded number of
95
+ # ticks while one tick never opens all 4,096. Receipt requests remain stronger still: they hash the
96
+ # requested bytes synchronously and never serve from this change-detection cache.
97
+ MAX_SETTLED_MEMBER_PROBES_PER_POLL = 256
98
+
99
+ # These members can change the main page's cards, approval decision, rail, or fixed event receipts.
100
+ # They stay outside the rotating delay. The event mapping is the normative source for build-step
101
+ # evidence; the five authored/governance records are the pre-build surface declared by STAGE_RAIL.
102
+ _PAGE_AUTHORITY_MEMBERS = frozenset(events.EVIDENCE_MEMBER.values()) | frozenset(
103
+ {
104
+ "question.json",
105
+ "requirements.json",
106
+ "source-proposals.json",
107
+ "recipe.json",
108
+ "recipe-approval.json",
109
+ }
110
+ )
111
+
112
+ # The pacing clamp for `--replay`: a run that sat idle overnight must not make a demo sit idle
113
+ # overnight, so no released gap is ever longer than this before the speed divisor is applied.
114
+ MAX_REPLAY_GAP_SECONDS = 5.0
115
+
116
+ # A rederived feed has no clock -- every projected record carries `at: 0.0` -- so a rederived
117
+ # replay chooses a readable rhythm rather than inventing timings it does not have.
118
+ REDERIVED_REPLAY_STEP_SECONDS = 0.35
119
+
120
+ # The four refusal sentences. Plain language, no internal vocabulary, and -- for the two tamper
121
+ # cases -- deliberately identical, so a refusal never tells a prober WHICH check it failed.
122
+ _NOT_FINISHED = "This build has not finished yet, so there is no receipt to show."
123
+ _NOT_A_RECEIPT = "This build has no receipt by that name."
124
+ _NOT_THE_RECORDED_FILE = "The file on disk is not the file this build recorded, so it is not shown."
125
+ _TOO_LARGE = "This receipt is too large to show here."
126
+
127
+ # The member-path character class, copied deliberately from `events._MEMBER_PATH_RE` rather than
128
+ # imported: a manifest is a file on disk that another local user may have written, so the member
129
+ # names it offers are untrusted strings even though they arrive through the run's own receipt.
130
+ _MEMBER_PATH_RE = re.compile(r"\A[A-Za-z0-9._/-]{1,128}\Z")
131
+
132
+ # The shape a recorded SHA-256 digest has: 64 lowercase hex characters, which is what
133
+ # `hashlib.sha256(...).hexdigest()` and `canonical.sha256_bytes` produce and therefore the only
134
+ # string a recorded digest can be.
135
+ _HEX_DIGEST_RE = re.compile(r"\A[0-9a-f]{64}\Z")
136
+
137
+
138
+ def _is_member_path(value: Any) -> bool:
139
+ """True when ``value`` is a candidate-relative member path and nothing else."""
140
+
141
+ if not isinstance(value, str) or not _MEMBER_PATH_RE.match(value):
142
+ return False
143
+ if value.startswith("/") or ".." in value.split("/"):
144
+ return False
145
+ return True
146
+
147
+
148
+ def _is_recorded_digest(value: Any) -> bool:
149
+ """True when ``value`` is the shape a recorded SHA-256 digest has, and nothing else."""
150
+
151
+ return isinstance(value, str) and _HEX_DIGEST_RE.match(value) is not None
152
+
153
+
154
+ def _matches_digest(found: Any, recorded: Any) -> bool:
155
+ """Return whether both values are lowercase SHA-256 digests with equal bytes.
156
+
157
+ Validate both operands first because ``hmac.compare_digest`` raises for non-ASCII strings.
158
+ Invalid values are verification failures.
159
+ """
160
+
161
+ return (
162
+ _is_recorded_digest(found)
163
+ and _is_recorded_digest(recorded)
164
+ and hmac.compare_digest(found, recorded)
165
+ )
166
+
167
+
168
+ # Shared nonblocking, no-follow regular-file opener.
169
+ _open_regular_member = events.open_regular_file
170
+
171
+
172
+ def _writable_path_text(value: str | os.PathLike[str]) -> str:
173
+ """Return a path, or one component of it, as text a served page can actually encode.
174
+
175
+ The run-directory name is also a display-string boundary. ``cli`` hands
176
+ ``resolve_target`` whatever ``os.fsdecode`` made of ``argv``, POSIX decodes an argument that is
177
+ not valid UTF-8 through ``surrogateescape``, and ``mr-data watch /data/runs/caf<0xe9>-run`` puts
178
+ ``'caf\\udce9-run'`` into ``_render_header``'s "Build:" line on EVERY page and into
179
+ ``approve_command``'s copyable line -- with no member, no feed and no adversary. The directory
180
+ need not exist when the watcher starts.
181
+
182
+ ``render_page`` does not encode the result. Invalid text would let polling succeed and then make
183
+ ``_write_html`` fail during
184
+ ``markup.encode("utf-8")`` BEFORE ``send_response``, inside the handler thread where
185
+ ``BaseHTTPRequestHandler`` has no ``except``; ``/events`` keeps writing version frames and the
186
+ tab's ``fetch('/')`` rejects into the live script's empty ``.catch``. ``/`` is dead for the life
187
+ of the server and the page says nothing.
188
+
189
+ Apply ``events.writable_text`` at this boundary, not in ``_write_html``. It changes display
190
+ text, never the ``Path``: every filesystem
191
+ call in this module goes through a descriptor opened from the real name, so a directory whose
192
+ name really does carry those bytes is still read, and only what a page writes is bounded.
193
+ """
194
+
195
+ # This is display-only path algebra. ``os.fspath`` made the source gate classify the
196
+ # conversion itself as a filesystem lookup even though no name is resolved here.
197
+ return events.writable_text(str(value))
198
+
199
+
200
+ def _file_identity(info: os.stat_result) -> tuple[int, int, int, int, int]:
201
+ """Bind a digest proof to the file snapshot it measured.
202
+
203
+ Size and mtime are writer-controlled and therefore only hints. POSIX ctime is the invalidation
204
+ token here: ordinary file APIs advance it for a content, mode, link or timestamp change and do
205
+ not let that writer restore its old value. The digest remains the proof of bytes; this tuple
206
+ says when that proof must be paid again.
207
+ """
208
+
209
+ return (info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns, info.st_ctime_ns)
210
+
211
+
212
+ def _feed_rejection_proof(
213
+ path: Path,
214
+ ) -> tuple[str, tuple[int, int, int, int, int], str] | None:
215
+ """Return stable identity plus bounded content evidence for one rejected feed snapshot.
216
+
217
+ Size alone cannot identify bytes: an atomic replacement may carry a valid feed of the exact
218
+ same length. The cache therefore binds both descriptor identity and a digest of the bounded
219
+ bytes read from that descriptor. A file that changes during the read earns no reusable proof.
220
+ """
221
+
222
+ try:
223
+ descriptor = events.open_regular_file(path)
224
+ except OSError:
225
+ return None
226
+ try:
227
+ identity_before = _file_identity(os.fstat(descriptor))
228
+ remaining = events.MAX_FEED_BYTES + 1
229
+ digest = hashlib.sha256()
230
+ while remaining:
231
+ chunk = os.read(descriptor, min(64 * 1024, remaining))
232
+ if not chunk:
233
+ break
234
+ digest.update(chunk)
235
+ remaining -= len(chunk)
236
+ identity_after = _file_identity(os.fstat(descriptor))
237
+ except OSError:
238
+ return None
239
+ finally:
240
+ os.close(descriptor)
241
+ if identity_after != identity_before:
242
+ return None
243
+ # Display/cache-key conversion only. The descriptor open above is the sole name resolution.
244
+ return str(path), identity_after, digest.hexdigest()
245
+
246
+
247
+ class ReceiptRefused(Exception):
248
+ """A receipt could not be served. Carries the status and the one plain sentence to show.
249
+
250
+ Every refusal renders a page with the sentence and NOTHING of the file's content: a member
251
+ whose bytes do not match the digest the run recorded has already failed the only test that
252
+ makes it a receipt, and showing it anyway would be showing an unlabelled claim.
253
+ """
254
+
255
+ def __init__(self, status: int, sentence: str) -> None:
256
+ super().__init__(sentence)
257
+ self.status = status
258
+ self.sentence = sentence
259
+
260
+
261
+ class _Target(NamedTuple):
262
+ """The two directories a watcher needs: the run it describes and the feed it tails."""
263
+
264
+ run_dir: Path
265
+ feed_dir: Path
266
+
267
+
268
+ def resolve_target(run_dir: str | os.PathLike[str]) -> _Target:
269
+ """Return ``(run_dir, feed_dir)`` for a run directory.
270
+
271
+ The feed lives in the runs PARENT (``events.run_feed_dir`` puts it there), because a failed
272
+ build has no run directory at all and a file inside the run directory would be a new file
273
+ inside a closed member set. It is scoped by the run's own name, so a sibling run's feed is not
274
+ in this directory and cannot be mistaken for this run's. Neither directory need exist: the
275
+ primary scenario is a watcher started BEFORE the build, when neither does.
276
+ """
277
+
278
+ resolved = Path(run_dir).absolute()
279
+ return _Target(resolved, events.run_feed_dir(resolved))
280
+
281
+
282
+ class _WatchState:
283
+ """Everything the request path must not do: read the feed, parse it, and render the page.
284
+
285
+ The member set is an INPUT to each render, not a property of the server, because the run it
286
+ describes does not exist yet when the server starts. ``poll`` passes the current value on every
287
+ render and never a value captured at construction.
288
+ """
289
+
290
+ def __init__(
291
+ self,
292
+ target: _Target | str | os.PathLike[str],
293
+ *,
294
+ replay: float | None = None,
295
+ clock: Callable[[], float] = time.monotonic,
296
+ ) -> None:
297
+ self.target = target if isinstance(target, _Target) else resolve_target(target)
298
+ self.run_dir = self.target.run_dir
299
+ self.feed_dir = self.target.feed_dir
300
+
301
+ self._lock = threading.Lock()
302
+ self._version = 0
303
+ self._last_seq = 0
304
+ # A page before any I/O has happened: `render_page` reads no file of its own, so this
305
+ # cannot raise, and a request that arrives before the first poll gets a document rather
306
+ # than an empty body.
307
+ self._html = render_page(
308
+ [], run_dir=self.run_dir, members_present=frozenset(), now=time.time()
309
+ )
310
+ # The last render that SUCCEEDED, with no stall note spliced into it. `_html` is what a
311
+ # request is served and may carry the note; this is what the note is spliced onto, so a
312
+ # stall that lasts a thousand polls still yields one note and not a thousand.
313
+ self._good_html = self._html
314
+ self._offset = 0 # bytes of the last successfully parsed feed read
315
+ self._fingerprint: tuple[Any, ...] | None = None
316
+ self._rejected_feed: tuple[str, tuple[int, int, int, int, int], str] | None = None
317
+ self._failed_polls = 0
318
+ # The class name of the last failure, or None while the last poll cycle completed. Only the
319
+ # NAME: an exception's message can carry bytes off disk, and this one is rendered.
320
+ self._failure_reason: str | None = None
321
+
322
+ self._manifest: Mapping[str, Any] | None = None
323
+ self._members_present: frozenset[str] = frozenset()
324
+ self._member_index: dict[str, Mapping[str, Any]] = {}
325
+ self._candidate_fd: int | None = None
326
+ self._candidate_identity: tuple[int, int] | None = None
327
+ self._manifest_file_identity: tuple[int, int, int, int, int] | None = None
328
+ # True when a manifest is on disk and does NOT carry its own fingerprint. Distinct from
329
+ # "no manifest yet", which is every run before it exists. It is a verdict about the bytes
330
+ # THIS poll read, so it is re-derived like every other one and never held past a poll that
331
+ # found the file intact again.
332
+ self._manifest_disowned = False
333
+ # The members this reader has PROVED under the currently pinned candidate, and the ones it
334
+ # has read whole and found not to be what the run recorded. Between them and the manifest's
335
+ # own index they say whether the answer is complete; see `_refresh_run`.
336
+ self._verified: dict[str, tuple[int, int, int, int, int]] = {}
337
+ self._measured_wrong: dict[str, tuple[int, int, int, int, int] | None] = {}
338
+ self._settled_probe_cursor = 0
339
+
340
+ # Replay state. `replay` is None for a live watch and a positive speed multiplier for a
341
+ # paced one; the released count is the only thing pacing changes, so live and paced share
342
+ # one renderer, one channel and one code path.
343
+ self._replay = replay
344
+ self._clock = clock
345
+ self._released = 0
346
+ self._released_at: float | None = None
347
+ # The records the last successful rederivation returned, retained only with the complete
348
+ # descriptor-relative proof beside them. A changed candidate, manifest, member decision,
349
+ # or directory namespace makes the pair unusable; refusals are never retained.
350
+ self._rederived: list[dict[str, Any]] | None = None
351
+ self._rederived_proof: tuple[Any, ...] | None = None
352
+ self._records_source = "none"
353
+
354
+ try:
355
+ self.poll()
356
+ except BaseException:
357
+ # `poll` may have adopted the candidate directory before a process-level exception.
358
+ # The caller cannot close an object whose constructor never returned, so ownership
359
+ # has to unwind here. Cleanup is secondary and must not replace the initiating error.
360
+ try:
361
+ self.close()
362
+ except BaseException:
363
+ pass
364
+ raise
365
+
366
+ # -- reads -----------------------------------------------------------------------------------
367
+
368
+ def snapshot(self) -> tuple[int, int, str]:
369
+ """Return one consistent ``(version, last_seq, html)`` snapshot.
370
+
371
+ Every ``_html`` update increments ``_version`` so open clients observe page changes.
372
+ """
373
+
374
+ with self._lock:
375
+ return self._version, self._last_seq, self._html
376
+
377
+ def evidence_index(self) -> tuple[Mapping[str, Any] | None, frozenset[str]]:
378
+ """Return the memoized manifest and member set as a CONSISTENT pair, under the lock."""
379
+
380
+ with self._lock:
381
+ return self._manifest, self._members_present
382
+
383
+ def members_present(self) -> frozenset[str]:
384
+ """The member paths this run carries, or an empty set before the manifest exists."""
385
+
386
+ return self.evidence_index()[1]
387
+
388
+ def member_index(self) -> dict[str, Mapping[str, Any]]:
389
+ """The run's own allowlist: member path to the entry the run recorded for it.
390
+
391
+ This is an ALLOWLIST, never a path sanitizer. A sanitizer is a list of attacks someone
392
+ thought of; an allowlist is the set of files this run actually recorded. A name that is not
393
+ a key here never reaches the filesystem at all (T-30-21).
394
+ """
395
+
396
+ with self._lock:
397
+ return dict(self._member_index)
398
+
399
+ def recorded_digests(self) -> dict[str, str]:
400
+ """Return each member path and the SHA-256 recorded in the manifest."""
401
+
402
+ return {
403
+ member: entry["sha256"]
404
+ for member, entry in self.member_index().items()
405
+ if _is_recorded_digest(entry.get("sha256"))
406
+ }
407
+
408
+ def _duplicate_candidate_fd(self) -> int | None:
409
+ """Own a stable reference to the pinned candidate, or return ``None``.
410
+
411
+ Reading the integer under the lock is not ownership. Another thread may drop the run,
412
+ close that descriptor and let the kernel reuse the same number before an ``openat`` uses
413
+ it. Duplication happens while the retained descriptor is protected by the same lock that
414
+ detaches it in ``_drop_run``; the caller then owns an independent reference and must close
415
+ it after its complete descriptor-rooted operation.
416
+ """
417
+
418
+ with self._lock:
419
+ if self._candidate_fd is None:
420
+ return None
421
+ try:
422
+ return events.duplicate_directory(self._candidate_fd)
423
+ except OSError:
424
+ # A descriptor-pressure failure is a fact about this read, not about the run.
425
+ return None
426
+
427
+ # -- the verified read ------------------------------------------------------------------------
428
+
429
+ def read_member(self, member: str, cited: str | None = None) -> bytes:
430
+ """Return one member's bytes, or raise ``ReceiptRefused`` with the sentence to show.
431
+
432
+ The order is the whole security argument and it does not commute: resolve the name against
433
+ the run's own allowlist FIRST, then open relative to the retained directory descriptor,
434
+ then hash, then compare. A name that fails the allowlist causes no filesystem call, and a
435
+ file whose bytes fail their fingerprint renders nothing.
436
+ """
437
+
438
+ manifest, _members = self.evidence_index()
439
+ if manifest is None:
440
+ with self._lock:
441
+ disowned = self._manifest_disowned
442
+ # Two different facts, and they must not be told as one. A run that does not exist has
443
+ # no receipt YET; a run whose receipt is not its own receipt has failed the only test
444
+ # that makes it a receipt, and that reads exactly like a member whose bytes changed --
445
+ # same status, same sentence -- so a refusal never tells a prober which check it failed.
446
+ if disowned:
447
+ raise ReceiptRefused(409, _NOT_THE_RECORDED_FILE)
448
+ raise ReceiptRefused(404, _NOT_FINISHED)
449
+
450
+ if member == _MANIFEST_NAME:
451
+ # ONE named branch, taken before the allowlist lookup, and not a special case a later
452
+ # reader may tidy away. `manifest.json` is NOT an entry in `manifest["members"]` and
453
+ # structurally cannot be: `members` is built from the expected file set and the
454
+ # manifest is written afterwards, so a file would have to contain its own fingerprint.
455
+ # Looking it up in the allowlist therefore returns nothing, and comparing it against
456
+ # "the recorded fingerprint" compares against a value that does not exist.
457
+ return self._verify_manifest_self(self._read_member_bytes(_MANIFEST_NAME), cited)
458
+
459
+ entry = self.member_index().get(member)
460
+ if entry is None:
461
+ raise ReceiptRefused(404, _NOT_A_RECEIPT)
462
+ recorded = entry.get("sha256")
463
+ if not _is_recorded_digest(recorded):
464
+ # Checked BEFORE the read so a receipt carrying a value that is not a digest costs no
465
+ # filesystem call and no hash, and reported as tamper because that is what it is.
466
+ raise ReceiptRefused(409, _NOT_THE_RECORDED_FILE)
467
+
468
+ raw = self._read_member_bytes(member)
469
+ found = hashlib.sha256(raw).hexdigest()
470
+ if not _matches_digest(found, recorded):
471
+ raise ReceiptRefused(409, _NOT_THE_RECORDED_FILE)
472
+ if cited is not None and not _matches_digest(found, cited):
473
+ # `cited` came off the query string, so it is the least trusted value on this path.
474
+ raise ReceiptRefused(409, _NOT_THE_RECORDED_FILE)
475
+ return raw
476
+
477
+ def _read_member_bytes(self, member: str) -> bytes:
478
+ """Open ``member`` RELATIVE to the retained descriptor, never by string concatenation."""
479
+
480
+ fd = self._duplicate_candidate_fd()
481
+ if fd is None:
482
+ raise ReceiptRefused(404, _NOT_FINISHED)
483
+ try:
484
+ try:
485
+ handle_fd = _open_regular_member(member, dir_fd=fd)
486
+ with os.fdopen(handle_fd, "rb") as handle:
487
+ raw = handle.read(MAX_RECEIPT_BYTES + 1)
488
+ except OSError as exc:
489
+ raise ReceiptRefused(404, _NOT_A_RECEIPT) from exc
490
+ finally:
491
+ os.close(fd)
492
+ if len(raw) > MAX_RECEIPT_BYTES:
493
+ raise ReceiptRefused(413, _TOO_LARGE)
494
+ return raw
495
+
496
+ def _durable_candidate_digest(self) -> str | None:
497
+ """The candidate digest the WORKSPACE durably recorded, or ``None`` when there is none.
498
+
499
+ The second authority, and the only one on this disk that is not the manifest talking about
500
+ itself. ``offline`` writes ``state['candidate']['candidate_digest']`` and re-checks it at
501
+ three linearization points, and an enrolled review binds it under a signature. ``watch``
502
+ consulted neither, which is why ``_verify_manifest_self`` proves coherence and not
503
+ authenticity.
504
+
505
+ Read through the hardened reader, relative to a descriptor for the workspace, so
506
+ ``.mr-data`` cannot be a symlink out of it. A run directory that is not a workspace's
507
+ ``result/`` simply has no such file, and this returns ``None``: absence of a second
508
+ authority is reported as absence, never as agreement.
509
+ """
510
+
511
+ _status, digest = self._durable_candidate_digest_status()
512
+ return digest
513
+
514
+ def _durable_candidate_digest_status(self) -> tuple[str, str | None]:
515
+ """Return ``(absent|unreadable|present, digest)`` for the workspace authority.
516
+
517
+ ``absent`` is deliberately not an approval verdict. It only distinguishes a workspace
518
+ without a durable authority from one whose authority is present but unreadable; callers
519
+ that need corroboration must require ``present`` and an agreeing digest. Otherwise deleting
520
+ the whole control directory -- including across a Watch restart -- manufactures approval.
521
+ """
522
+
523
+ try:
524
+ workspace_fd = events.open_directory(self.run_dir.parent)
525
+ except OSError:
526
+ return "unreadable", None
527
+ try:
528
+ try:
529
+ state_fd = events.open_directory(".mr-data", dir_fd=workspace_fd)
530
+ except OSError as exc:
531
+ if exc.errno == errno.ENOENT:
532
+ return "absent", None
533
+ return "unreadable", None
534
+ try:
535
+ raw = events.read_regular_bytes(
536
+ "state.json",
537
+ dir_fd=state_fd,
538
+ max_bytes=_MAX_MANIFEST_BYTES,
539
+ )
540
+ finally:
541
+ os.close(state_fd)
542
+ candidate = json.loads(raw.decode("utf-8")).get("candidate")
543
+ except (OSError, ValueError, AttributeError, RecursionError):
544
+ return "unreadable", None
545
+ finally:
546
+ os.close(workspace_fd)
547
+ digest = candidate.get("candidate_digest") if isinstance(candidate, Mapping) else None
548
+ if not _is_recorded_digest(digest):
549
+ return "unreadable", None
550
+ return "present", digest
551
+
552
+ def _approval_is_corroborated(self) -> bool:
553
+ """True only when the second authority corroborates the approval summary's receipt.
554
+
555
+ The summary is the one part of this page that talks a person into an action (T-30-27), and
556
+ the receipt underneath it proves only coherence: whoever writes ``manifest.json`` writes
557
+ its ``candidate_digest`` too, so an adversary who reseals is not detected by recomputing
558
+ it. This is the cross-check that is available -- the workspace's durable
559
+ ``state.json`` -- and its verdict is deliberately fail closed:
560
+
561
+ * the durable digest AGREES with the manifest's: render;
562
+ * the durable digest DISAGREES: render nothing, because two authorities on this disk
563
+ disagree about which candidate this is and a summary is a claim about a specific one;
564
+ * there is NO readable durable digest (a bare run directory or a workspace mid-write):
565
+ render no approval summary. Candidate-local bytes prove only their own coherence, and
566
+ process-local memory cannot distinguish a genuinely bare run from an authority an
567
+ attacker deleted before this Watch process started.
568
+
569
+ Nothing here is memoized, and the ANSWER is part of the poll's fingerprint. Those two are
570
+ one sentence, not two: a disagreement while the seal is mid-flight -- the candidate
571
+ directory renamed into place before ``state.json`` is republished -- must cost that render
572
+ and not that candidate, so the next poll asks again AND acts on a different answer. Asked
573
+ below the fingerprint gate, as an argument to ``render_page``, the next poll returned
574
+ before asking; see ``_poll_once``.
575
+ """
576
+
577
+ manifest, _members = self.evidence_index()
578
+ if manifest is None:
579
+ return False
580
+ durable_status, durable = self._durable_candidate_digest_status()
581
+ if durable_status != "present" or durable is None:
582
+ return False
583
+ # No shape check here: the comparator owns it, on both operands. A proof two methods away
584
+ # -- `_verify_manifest_self` refuses a manifest whose `candidate_digest` is not a digest
585
+ # before `self._manifest` is ever set -- is not a guard at the comparison.
586
+ return _matches_digest(manifest.get("candidate_digest"), durable)
587
+
588
+ def _verify_manifest_self(self, raw: bytes, cited: str | None) -> bytes:
589
+ """Verify that the run's receipt is COHERENT, by recomputing its fingerprint over itself.
590
+
591
+ COHERENCE, NOT AUTHENTICITY, and the difference is the whole reading of this method. The
592
+ seal computes the fingerprint over the receipt minus that one field, so verifying it means
593
+ doing exactly that again and comparing -- against a value in the same file, written by
594
+ whoever wrote the file. It detects an edit that forgot to reseal. It detects nothing at all
595
+ from an adversary who reseals, and the adversary this module names throughout is a local
596
+ user who can write the run directory (T-30-24), which is to say one who can reseal.
597
+
598
+ There is no other file inside the candidate that records this file's fingerprint, so this
599
+ is the strongest statement available from the candidate alone. The authority that is NOT
600
+ inside the candidate is the workspace's durable ``state.json``; see
601
+ ``_durable_candidate_digest`` and ``_approval_is_corroborated``, which is where the one
602
+ claim on this page that asks a person to act is gated on it.
603
+ """
604
+
605
+ # Deferred: `watch` must stay importable without pulling `pipeline` in at module load.
606
+ from mostlyright.data_harness.canonical import sha256_bytes
607
+ from mostlyright.data_harness.pipeline import canonical_json_line_bytes
608
+
609
+ try:
610
+ payload = json.loads(raw.decode("utf-8"))
611
+ recorded = payload.pop("candidate_digest")
612
+ except (ValueError, TypeError, KeyError, AttributeError, RecursionError) as exc:
613
+ raise ReceiptRefused(409, _NOT_THE_RECORDED_FILE) from exc
614
+ if not _is_recorded_digest(recorded):
615
+ raise ReceiptRefused(409, _NOT_THE_RECORDED_FILE)
616
+ try:
617
+ recomputed = sha256_bytes(canonical_json_line_bytes(payload))
618
+ except (TypeError, ValueError) as exc:
619
+ raise ReceiptRefused(409, _NOT_THE_RECORDED_FILE) from exc
620
+ if not _matches_digest(recomputed, recorded):
621
+ raise ReceiptRefused(409, _NOT_THE_RECORDED_FILE)
622
+ if cited is not None and not _matches_digest(recorded, cited):
623
+ raise ReceiptRefused(409, _NOT_THE_RECORDED_FILE)
624
+ return raw
625
+
626
+ # -- the poll --------------------------------------------------------------------------------
627
+
628
+ def poll(self) -> None:
629
+ """Refresh run state, feed records, and rendered HTML when inputs change.
630
+
631
+ An exception increments failed-poll state and preserves the previous valid page. The page
632
+ reports the exception class and failure count.
633
+ """
634
+
635
+ try:
636
+ self._poll_once()
637
+ except Exception as exc:
638
+ self._record_failed_poll(exc)
639
+ else:
640
+ self._clear_failed_poll()
641
+
642
+ def _record_failed_poll(self, exc: BaseException) -> None:
643
+ """Count a cycle that could not complete, and say so on the page that stopped moving.
644
+
645
+ The version moves with the page. Splicing the note in without it left the note reachable
646
+ only by a hard reload: ``/events`` writes a ``data:`` frame when the version changes and a
647
+ comment ping otherwise, and a comment fires nothing in a browser, so the tab ``mr-data
648
+ watch`` opened -- the only view this command produces -- sat on a frozen page and was told
649
+ nothing. A note about a stall that the stalled page cannot deliver is not a note.
650
+ """
651
+
652
+ with self._lock:
653
+ self._failed_polls += 1
654
+ self._failure_reason = type(exc).__name__
655
+ self._html = _with_stall_note(self._good_html, self._failed_polls, self._failure_reason)
656
+ self._version += 1
657
+
658
+ def _clear_failed_poll(self) -> None:
659
+ """Take the note off once a cycle completes again.
660
+
661
+ Cleared on a COMPLETED cycle, not on a fresh render: most polls change nothing and return
662
+ early by design, and a page that only recovered when something new happened would keep
663
+ telling a reader it was stuck long after it had started reading again.
664
+
665
+ The count is not reset. It is the accounting of how many reads of this run have failed, and
666
+ that is a fact about the run, not about the current streak.
667
+
668
+ The version moves here for the same reason it moves when the note goes on: a tab that was
669
+ told the page had stopped must be told it started again, or the note outlives the stall.
670
+ """
671
+
672
+ with self._lock:
673
+ if self._failure_reason is None:
674
+ return
675
+ self._failure_reason = None
676
+ self._html = self._good_html
677
+ self._version += 1
678
+
679
+ def _poll_once(self) -> None:
680
+ """One poll cycle. Every raise it can produce is caught by ``poll``."""
681
+
682
+ self._refresh_run()
683
+ # This verdict is asked on every cycle, before any feed memo can return. A rejected feed
684
+ # may suppress another read of the same bad bytes, but it cannot freeze the authority that
685
+ # decides whether the approval summary is safe to show.
686
+ corroborated = self._approval_is_corroborated()
687
+
688
+ path, size, records = self._current_feed()
689
+ key = None if path is None else (str(path), size)
690
+ rejection_proof = _feed_rejection_proof(path) if path is not None and not records else None
691
+ with self._lock:
692
+ rejected = self._rejected_feed
693
+ fingerprint_before = self._fingerprint
694
+ members = self._members_present
695
+ identity = self._candidate_identity
696
+ recorded_digests = tuple(
697
+ sorted(
698
+ (member, entry["sha256"])
699
+ for member, entry in self._member_index.items()
700
+ if _is_recorded_digest(entry.get("sha256"))
701
+ )
702
+ )
703
+ feed_rejected = rejection_proof is not None and rejection_proof == rejected
704
+ if feed_rejected:
705
+ # These exact bytes did not parse last tick and have not changed since. Re-reading them
706
+ # every 0.25 s would be work with a known answer.
707
+ records = []
708
+
709
+ if (
710
+ not feed_rejected
711
+ and path is not None
712
+ and size > 0
713
+ and not records
714
+ and not events.feed_is_readable(path)
715
+ ):
716
+ # These bytes are not a feed at all: a foreign header, a truncation, or a half-written
717
+ # first line. Memoize and keep serving the last good render.
718
+ #
719
+ # READ AS NOTHING is not the same state as NARRATED NOTHING, and only the first belongs
720
+ # here. An attempt file that parses cleanly and carries no record is routine -- an
721
+ # idempotent re-run arms a feed and writes only its header -- and it never changes size,
722
+ # so memoizing it against `(path, size)` blocked the receipts fallback below for the
723
+ # life of the server and rendered a sealed, verifying run as "Nothing has happened yet."
724
+ proof_after = _feed_rejection_proof(path)
725
+ with self._lock:
726
+ self._rejected_feed = (
727
+ rejection_proof
728
+ if rejection_proof is not None and rejection_proof == proof_after
729
+ else None
730
+ )
731
+ feed_rejected = True
732
+
733
+ if not feed_rejected:
734
+ with self._lock:
735
+ self._rejected_feed = None
736
+
737
+ if feed_rejected and identity is None:
738
+ # A live build has no receipts to fall back to. The corroboration verdict above was
739
+ # still re-asked, but there is no page input it can safely replace the last good feed
740
+ # render with, so keep serving it until readable bytes or a candidate exists.
741
+ return
742
+
743
+ source = "feed" if records else "none"
744
+ if not records:
745
+ rederived = self._rederived_records(identity)
746
+ if rederived:
747
+ records, source = rederived, "receipts"
748
+ self._records_source = source
749
+
750
+ visible = self._release(records, source)
751
+
752
+ # Include every page input in the fingerprint so a changed verification result rerenders.
753
+ fingerprint = (
754
+ key,
755
+ source,
756
+ len(visible),
757
+ visible[-1]["seq"] if visible else 0,
758
+ visible[-1]["at"] if visible else 0.0,
759
+ identity,
760
+ members,
761
+ recorded_digests,
762
+ corroborated,
763
+ )
764
+ if fingerprint == fingerprint_before:
765
+ return
766
+
767
+ # `poll` records render failures and preserves the previous valid page.
768
+ markup = render_page(
769
+ visible,
770
+ run_dir=self.run_dir,
771
+ members_present=members,
772
+ now=time.time(),
773
+ read_member=self.read_member,
774
+ recorded_digests=dict(recorded_digests),
775
+ approval_corroborated=corroborated,
776
+ )
777
+
778
+ with self._lock:
779
+ self._offset = size if path is not None else 0
780
+ self._fingerprint = fingerprint
781
+ self._version += 1
782
+ self._last_seq = int(visible[-1]["seq"]) if visible else 0
783
+ self._html = markup
784
+ self._good_html = markup
785
+
786
+ # -- the record source and the paced release --------------------------------------------------
787
+
788
+ def _rederived_records(self, identity: tuple[int, int] | None) -> list[dict[str, Any]]:
789
+ """Project this run's feed from its own receipts when no feed file exists.
790
+
791
+ The retained candidate descriptor, namespace proof, and member identities must remain
792
+ unchanged before cached records are reused. Actual member bytes and member count are bounded
793
+ by the same limits as direct verification. Failed verification is not cached.
794
+ """
795
+
796
+ if identity is None:
797
+ return []
798
+ fd = self._duplicate_candidate_fd()
799
+ with self._lock:
800
+ manifest = self._manifest
801
+ if fd is None or manifest is None:
802
+ # Nothing pinned this instant, or no member set established, so there is nothing to ask
803
+ # about and nothing to bound the asking with. The next poll decides.
804
+ if fd is not None:
805
+ os.close(fd)
806
+ return []
807
+ try:
808
+ if not self._rederivation_fits_the_budget(fd):
809
+ return []
810
+ proof_before = self._rederivation_proof(fd)
811
+ with self._lock:
812
+ cached = None if self._rederived is None else list(self._rederived)
813
+ cached_proof = self._rederived_proof
814
+ if proof_before is not None and cached is not None and proof_before == cached_proof:
815
+ return cached
816
+ try:
817
+ rederived = events.rederive_feed(self.run_dir, pinned_candidate_fd=fd)
818
+ except Exception:
819
+ # Retry failed verification on the next poll.
820
+ rederived = []
821
+ proof_after = self._rederivation_proof(fd)
822
+ if proof_before is None or proof_after != proof_before:
823
+ rederived = []
824
+ proof_after = None
825
+ with self._lock:
826
+ self._rederived = list(rederived)
827
+ # A refusal is never memoized. The next poll must be able to observe a repair
828
+ # without depending on an incomplete proxy for the state that caused the refusal.
829
+ self._rederived_proof = proof_after if rederived else None
830
+ return rederived
831
+ finally:
832
+ os.close(fd)
833
+
834
+ def _rederivation_proof(self, fd: int) -> tuple[Any, ...] | None:
835
+ """Complete cheap identity paired with a successful receipts replay.
836
+
837
+ File contents retain the rotating descriptor probes in ``_run_is_settled``. Directory
838
+ namespace is cheaper and must be complete: adding an unrecorded member changes the child
839
+ set and invalidates the cached projection.
840
+ """
841
+
842
+ with self._lock:
843
+ if self._manifest is None or self._manifest_file_identity is None:
844
+ return None
845
+ candidate_identity = self._candidate_identity
846
+ manifest_identity = self._manifest_file_identity
847
+ index = dict(self._member_index)
848
+ verified = tuple(sorted(self._verified.items()))
849
+ wrong = tuple(
850
+ sorted(
851
+ (member, () if decided is None else decided)
852
+ for member, decided in self._measured_wrong.items()
853
+ )
854
+ )
855
+ try:
856
+ namespace = events.snapshot_candidate_namespace(fd, tuple(index))
857
+ except OSError:
858
+ return None
859
+ return candidate_identity, manifest_identity, verified, wrong, namespace
860
+
861
+ def _rederivation_fits_the_budget(self, fd: int) -> bool:
862
+ """Whether re-proving this candidate costs less than the ceiling the sibling reader uses.
863
+
864
+ Measured through the PINNED descriptor with open-then-fstat, so the size is the object's and
865
+ not the name's, and so the answer cannot be lied to by the manifest that is about to be
866
+ read: the pipeline's own pre-read ceiling is asked of the DECLARED byte counts, and the
867
+ declared counts are exactly what an attacker writes. A member this cannot open is counted as
868
+ nothing, because the rederivation will not read it either -- it will refuse the whole feed.
869
+
870
+ The sum is over the member set the run declares and the projection reads it TWICE (once in
871
+ ``verify_candidate``'s snapshot, once to project), so a set that fits here is bounded at
872
+ both passes.
873
+ """
874
+
875
+ index = self.member_index()
876
+ if len(index) > MAX_MANIFEST_MEMBERS:
877
+ return False
878
+ total = 0
879
+ for member in index:
880
+ try:
881
+ handle_fd = _open_regular_member(member, dir_fd=fd)
882
+ except OSError:
883
+ continue
884
+ try:
885
+ total += os.fstat(handle_fd).st_size
886
+ finally:
887
+ os.close(handle_fd)
888
+ if total > MAX_VERIFIED_TOTAL_BYTES:
889
+ return False
890
+ return True
891
+
892
+ def _release(self, records: list[dict[str, Any]], source: str) -> list[dict[str, Any]]:
893
+ """Return the prefix of ``records`` the page may show right now.
894
+
895
+ With no ``replay`` speed this is every record, which is what a live watch wants. With a
896
+ speed it is the records whose original inter-arrival gap has elapsed, divided by the speed
897
+ and clamped, so the existing channel and the existing renderer drive the replay with no new
898
+ machinery. Reaching the end stops releasing and leaves the final state on screen; it does
899
+ not loop and does not clear.
900
+ """
901
+
902
+ if self._replay is None or not records:
903
+ return records
904
+
905
+ now = self._clock()
906
+ if self._released_at is None:
907
+ self._released = 1
908
+ self._released_at = now
909
+ return records[:1]
910
+
911
+ while self._released < len(records):
912
+ if source == "receipts":
913
+ gap = REDERIVED_REPLAY_STEP_SECONDS
914
+ else:
915
+ previous = float(records[self._released - 1]["at"])
916
+ following = float(records[self._released]["at"])
917
+ gap = min(max(following - previous, 0.0), MAX_REPLAY_GAP_SECONDS)
918
+ delay = gap / self._replay
919
+ if now - self._released_at < delay:
920
+ break
921
+ self._released += 1
922
+ self._released_at += delay
923
+ if self._released >= len(records):
924
+ # Caught up. A feed that is still being written continues from here in live mode --
925
+ # one code path, two modes -- so the release clock must not stay in the past.
926
+ self._released_at = max(self._released_at, now)
927
+ return records[: self._released]
928
+
929
+ def _current_feed(self) -> tuple[Path | None, int, list[dict[str, Any]]]:
930
+ """Return the attempt being watched: ``(path, size, records)``, or ``(None, 0, [])``.
931
+
932
+ Resolve the attempt on every poll. Selection uses the sealed candidate digest when present;
933
+ otherwise it uses the newest active attempt according to ``events.select_narrated_feed``.
934
+ """
935
+
936
+ with self._lock:
937
+ manifest = self._manifest
938
+ digest = manifest.get("candidate_digest") if isinstance(manifest, Mapping) else None
939
+ return events.select_narrated_feed(
940
+ self.feed_dir,
941
+ sealed_candidate_digest=digest if isinstance(digest, str) else None,
942
+ )
943
+
944
+ # -- the run directory -----------------------------------------------------------------------
945
+
946
+ def _refresh_run(self) -> None:
947
+ """Load, keep or drop the run's manifest and member set. Called before every render.
948
+
949
+ The run directory does NOT exist while the build runs: ``build_candidate`` stages into a
950
+ hidden temporary directory and renames into place only after sealing. A manifest memoized
951
+ at construction is therefore permanently empty for exactly the build the user is watching.
952
+
953
+ Open the directory on every poll and measure identity from that descriptor. An unchanged
954
+ directory closes the new descriptor and retains the existing pinned reference.
955
+
956
+ Manifest content is not cached against directory identity. Incomplete reads are retried:
957
+
958
+ * complete means a manifest that parses and verifies, and every member it names either
959
+ proved present or READ WHOLE and found not to be the bytes the run recorded;
960
+ * a manifest that will not parse or will not verify is not complete, so the next poll reads
961
+ the file again;
962
+ * a member whose open or read RAISED measured nothing, so it is not complete either. That
963
+ covers the transient errnos above and it covers a run still arriving on disk.
964
+
965
+ A completed digest mismatch remains valid only for the measured file identity.
966
+ """
967
+
968
+ candidate = self.run_dir / "candidate"
969
+ try:
970
+ opened = events.open_directory(candidate)
971
+ except OSError as exc:
972
+ if exc.errno in _NAME_HOLDS_NO_CANDIDATE:
973
+ # Absent, removed, or no longer a directory.
974
+ self._drop_run()
975
+ return
976
+
977
+ adopted = False
978
+ try:
979
+ info = os.fstat(opened)
980
+ identity = (info.st_dev, info.st_ino)
981
+ with self._lock:
982
+ retained_identity = self._candidate_identity
983
+ if retained_identity is not None and identity != retained_identity:
984
+ # The directory at this path was REPLACED -- a rebuild. Drop the descriptor, the
985
+ # manifest and the member set before loading anything, so a new run is never served
986
+ # through the previous run's allowlist.
987
+ self._drop_run()
988
+
989
+ with self._lock:
990
+ if self._candidate_fd is None:
991
+ # Pin the descriptor used to measure `identity`.
992
+ self._candidate_fd = opened
993
+ self._candidate_identity = identity
994
+ adopted = True
995
+ finally:
996
+ if not adopted:
997
+ os.close(opened)
998
+
999
+ owned = self._duplicate_candidate_fd()
1000
+ if owned is None:
1001
+ return
1002
+ try:
1003
+ self._refresh_pinned_run(owned)
1004
+ finally:
1005
+ os.close(owned)
1006
+
1007
+ def _refresh_pinned_run(self, pinned: int) -> None:
1008
+ """Refresh manifest state through one directory reference owned for the whole operation."""
1009
+
1010
+ if self._run_is_settled(pinned):
1011
+ return
1012
+
1013
+ try:
1014
+ raw, manifest_file_identity = self._read_manifest_snapshot(pinned)
1015
+ except OSError:
1016
+ # The seal is mid-flight, absent, or unreadable this instant. An earlier accepted
1017
+ # manifest cannot remain the page's authority while its own receipt route says 404,
1018
+ # so revoke every conclusion derived from it and retry next tick.
1019
+ self._invalidate_installed_manifest(disowned=False)
1020
+ return
1021
+
1022
+ try:
1023
+ manifest = json.loads(raw.decode("utf-8"))
1024
+ # The member names come out of a file another local user may have written, so they are
1025
+ # re-checked against the member-path shape before any of them can become an allowlist
1026
+ # key. A name that is not a member path is dropped, not repaired.
1027
+ index = {
1028
+ entry["path"]: dict(entry)
1029
+ for entry in manifest["members"]
1030
+ if isinstance(entry, Mapping) and _is_member_path(entry.get("path"))
1031
+ }
1032
+ except (ValueError, TypeError, KeyError, RecursionError):
1033
+ # A half-written file. NOT memoized: this is a verdict about bytes, and the candidate
1034
+ # directory's inode -- the only key this class has -- cannot see those bytes change.
1035
+ # It DOES revoke an earlier accepted manifest immediately. Keeping the installed
1036
+ # allowlist/page while the current receipt is invalid lets one page describe bytes
1037
+ # that its own receipt endpoint refuses.
1038
+ self._invalidate_installed_manifest(disowned=True)
1039
+ return
1040
+
1041
+ # A NAME IN A FILE IS NOT EVIDENCE. Everything downstream of `members_present` -- the rail,
1042
+ # the receipt links, the cards, the freeze summary -- reads that set as "this run really
1043
+ # carries this file", and the rail turns membership alone into a done mark. So the set is
1044
+ # built from what this reader has ESTABLISHED, in two steps that do not commute:
1045
+ #
1046
+ # 1. The receipt must be COHERENT. `manifest.json` records its fingerprint over itself,
1047
+ # so it is recomputed and compared before a single member name is believed. Appending
1048
+ # fabricated entries and leaving that fingerprint alone breaks it, and a manifest that
1049
+ # fails here contributes no members at all -- not even `manifest.json`.
1050
+ # 2. Each member must be the bytes THIS receipt recorded. A name whose file is absent,
1051
+ # truncated or altered is dropped from the set, so a partial copy narrates the members
1052
+ # it still has and claims nothing for the ones it lost.
1053
+ #
1054
+ # WHAT THESE TWO STEPS DO NOT DO, stated here because this comment used to claim it: they
1055
+ # do not establish that the receipt is the run's. Whoever writes the file writes its
1056
+ # fingerprint, so a local user who RESEALS passes step 1, and step 2 then measures their
1057
+ # members against their own manifest. A forged, fully self-consistent manifest naming
1058
+ # attacker-written `recipe.json` and `recipe-approval.json` members renders as a verified
1059
+ # member set. The claim on this page that a forgery could turn into an action -- the
1060
+ # approval summary -- is therefore gated separately on the workspace's durable digest; see
1061
+ # `_approval_is_corroborated`. Other content describes verified candidate members.
1062
+ try:
1063
+ self._verify_manifest_self(raw, None)
1064
+ except ReceiptRefused:
1065
+ # Also not memoized, and for the same reason. The flag says what THIS poll found, so a
1066
+ # manifest restored on disk stops being disowned on the next tick rather than never.
1067
+ self._invalidate_installed_manifest(disowned=True)
1068
+ return
1069
+
1070
+ with self._lock:
1071
+ if self._manifest_file_identity != manifest_file_identity:
1072
+ self._verified.clear()
1073
+ self._measured_wrong.clear()
1074
+ self._settled_probe_cursor = 0
1075
+ self._manifest_file_identity = manifest_file_identity
1076
+ self._verify_members(pinned, index)
1077
+
1078
+ with self._lock:
1079
+ self._manifest = manifest
1080
+ self._members_present = frozenset(self._verified) | {_MANIFEST_NAME}
1081
+ self._member_index = index
1082
+ self._manifest_disowned = False
1083
+
1084
+ def _run_is_settled(self, fd: int) -> bool:
1085
+ """Whether every installed digest decision still names the same kernel snapshot.
1086
+
1087
+ The digest is paid once for a snapshot. Each later poll opens and fstats the manifest, every
1088
+ page-authoritative member, and one rotating bounded slice of the remaining decisions. Any
1089
+ device, inode, size, mtime OR unforgeable ctime change invalidates the decision and routes
1090
+ the new snapshot through bounded hashing. Missing and unreadable proved objects become
1091
+ unsettled, so manifest revocation and copy-in-progress recovery keep their fail-closed
1092
+ behaviour. The rotation means every accepted member is revisited, without making 4,096
1093
+ opens the permanent price of watching an unchanged run.
1094
+ """
1095
+
1096
+ with self._lock:
1097
+ if self._manifest is None or self._manifest_file_identity is None:
1098
+ return False
1099
+ manifest_identity = self._manifest_file_identity
1100
+ index = dict(self._member_index)
1101
+ verified = dict(self._verified)
1102
+ wrong = dict(self._measured_wrong)
1103
+
1104
+ try:
1105
+ manifest_fd = _open_regular_member(_MANIFEST_NAME, dir_fd=fd)
1106
+ except OSError:
1107
+ return False
1108
+ try:
1109
+ if _file_identity(os.fstat(manifest_fd)) != manifest_identity:
1110
+ return False
1111
+ finally:
1112
+ os.close(manifest_fd)
1113
+
1114
+ complete = not (set(index) - set(verified) - set(wrong))
1115
+ decided = sorted(
1116
+ set(verified)
1117
+ | {member for member, decided_identity in wrong.items() if decided_identity is not None}
1118
+ )
1119
+ priority = [member for member in decided if member in _PAGE_AUTHORITY_MEMBERS]
1120
+ background = [member for member in decided if member not in _PAGE_AUTHORITY_MEMBERS]
1121
+ remaining = max(0, MAX_SETTLED_MEMBER_PROBES_PER_POLL - len(priority))
1122
+ if background and remaining:
1123
+ with self._lock:
1124
+ start = self._settled_probe_cursor % len(background)
1125
+ count = min(remaining, len(background))
1126
+ self._settled_probe_cursor = (start + count) % len(background)
1127
+ rotated = background[start:] + background[:start]
1128
+ probes = priority + rotated[:count]
1129
+ else:
1130
+ probes = priority
1131
+
1132
+ changed = False
1133
+ for member in probes:
1134
+ decided_identity = verified.get(member)
1135
+ was_verified = decided_identity is not None
1136
+ if decided_identity is None:
1137
+ decided_identity = wrong[member]
1138
+ try:
1139
+ member_fd = _open_regular_member(member, dir_fd=fd)
1140
+ except OSError:
1141
+ if was_verified:
1142
+ with self._lock:
1143
+ self._verified.pop(member, None)
1144
+ changed = True
1145
+ # An already-refused member becoming absent does not make the page less true. Keep
1146
+ # its refusal; when a file appears here again its identity is compared on its next
1147
+ # rotating probe and a changed snapshot is re-proved.
1148
+ continue
1149
+ try:
1150
+ unchanged = _file_identity(os.fstat(member_fd)) == decided_identity
1151
+ finally:
1152
+ os.close(member_fd)
1153
+ if not unchanged:
1154
+ with self._lock:
1155
+ if was_verified:
1156
+ self._verified.pop(member, None)
1157
+ else:
1158
+ self._measured_wrong.pop(member, None)
1159
+ changed = True
1160
+ return complete and not changed
1161
+
1162
+ def _verify_members(self, fd: int, index: Mapping[str, Mapping[str, Any]]) -> None:
1163
+ """Verify undecided members under ``fd`` within the per-pass limits.
1164
+
1165
+ Digests are streamed and reused only while file identity is unchanged. Open or read errors
1166
+ remain undecided and are retried on the next poll.
1167
+ """
1168
+
1169
+ budget = MAX_VERIFIED_TOTAL_BYTES
1170
+ for member, entry in list(index.items())[:MAX_MANIFEST_MEMBERS]:
1171
+ if member in self._verified or member in self._measured_wrong:
1172
+ continue
1173
+ recorded = entry.get("sha256")
1174
+ size = entry.get("bytes")
1175
+ if not _is_recorded_digest(recorded) or type(size) is not int or size < 0:
1176
+ # The ENTRY is unusable, and the entry is part of the manifest this pass already
1177
+ # proved coherent. Nothing about the file on disk can change that, so it is decided.
1178
+ self._measured_wrong[member] = None
1179
+ continue
1180
+ if size > budget:
1181
+ # Not called present and not called absent: this reader declines to hash it. The
1182
+ # rail still lights the step from the member's OWN event, which is evidence of the
1183
+ # same fact from the producer rather than from this reader. Undecided, so the next
1184
+ # pass -- which starts with a full budget -- offers it the remainder again.
1185
+ continue
1186
+ digest = hashlib.sha256()
1187
+ read = 0
1188
+ try:
1189
+ handle_fd = _open_regular_member(member, dir_fd=fd)
1190
+ with os.fdopen(handle_fd, "rb") as handle:
1191
+ identity_before = _file_identity(os.fstat(handle.fileno()))
1192
+ while True:
1193
+ chunk = handle.read(_MEMBER_HASH_CHUNK_BYTES)
1194
+ if not chunk:
1195
+ break
1196
+ read += len(chunk)
1197
+ if read > size:
1198
+ break
1199
+ digest.update(chunk)
1200
+ identity_after = _file_identity(os.fstat(handle.fileno()))
1201
+ except OSError:
1202
+ # Absent, unreadable, a symlink or a FIFO where a member should be. This poll could
1203
+ # not look; it is not a fact about the run, and the next poll asks again.
1204
+ budget -= read
1205
+ continue
1206
+ budget -= read
1207
+ if identity_before != identity_after:
1208
+ # The name changed while it was being measured. Neither the digest nor a refusal
1209
+ # is attached to that mixed read; the next poll opens the resulting snapshot.
1210
+ continue
1211
+ if read != size or not _matches_digest(digest.hexdigest(), recorded):
1212
+ self._measured_wrong[member] = identity_after
1213
+ continue
1214
+ self._measured_wrong.pop(member, None)
1215
+ self._verified[member] = identity_after
1216
+
1217
+ def _read_manifest_snapshot(self, fd: int) -> tuple[bytes, tuple[int, int, int, int, int]]:
1218
+ """Read ``manifest.json`` RELATIVE to the retained descriptor, never by path.
1219
+
1220
+ Holding the descriptor rather than re-opening by path per request is what makes a receipt
1221
+ immune to a path swapped underneath it. A missing or non-regular manifest is retried on the
1222
+ next poll.
1223
+ """
1224
+
1225
+ handle_fd = _open_regular_member(_MANIFEST_NAME, dir_fd=fd)
1226
+ with os.fdopen(handle_fd, "rb") as handle:
1227
+ identity_before = _file_identity(os.fstat(handle.fileno()))
1228
+ raw = handle.read(_MAX_MANIFEST_BYTES + 1)
1229
+ identity_after = _file_identity(os.fstat(handle.fileno()))
1230
+ if len(raw) > _MAX_MANIFEST_BYTES:
1231
+ raise OSError("manifest is larger than this reader accepts")
1232
+ if identity_before != identity_after:
1233
+ raise OSError("manifest changed while this reader measured it")
1234
+ return raw, identity_after
1235
+
1236
+ def _drop_run(self) -> None:
1237
+ # Detach the retained descriptor while holding the same lock used to duplicate it. A
1238
+ # reader therefore either owns its duplicate before this point or observes no candidate;
1239
+ # it can never carry a bare descriptor number across this close and into kernel reuse.
1240
+ with self._lock:
1241
+ candidate_fd = self._candidate_fd
1242
+ self._candidate_fd = None
1243
+ self._candidate_identity = None
1244
+ self._rederived = None
1245
+ self._rederived_proof = None
1246
+ self._manifest = None
1247
+ self._members_present = frozenset()
1248
+ self._member_index = {}
1249
+ self._manifest_disowned = False
1250
+ self._manifest_file_identity = None
1251
+ self._verified = {}
1252
+ self._measured_wrong = {}
1253
+ self._settled_probe_cursor = 0
1254
+ if candidate_fd is not None:
1255
+ try:
1256
+ os.close(candidate_fd)
1257
+ except OSError: # pragma: no cover - closing a descriptor twice is the only shape
1258
+ pass
1259
+
1260
+ def _invalidate_installed_manifest(self, *, disowned: bool) -> None:
1261
+ """Revoke every conclusion derived from manifest bytes this poll found invalid.
1262
+
1263
+ The candidate descriptor remains pinned so a restored receipt can recover on the next
1264
+ poll. Only the installed authority, allowlist and member decisions are cleared; retaining
1265
+ any one of them would keep the stale page or a stale receipt route alive.
1266
+ """
1267
+
1268
+ self._rederived = None
1269
+ self._rederived_proof = None
1270
+ with self._lock:
1271
+ self._manifest = None
1272
+ self._members_present = frozenset()
1273
+ self._member_index = {}
1274
+ self._manifest_disowned = disowned
1275
+ self._manifest_file_identity = None
1276
+ self._verified.clear()
1277
+ self._measured_wrong.clear()
1278
+ self._settled_probe_cursor = 0
1279
+
1280
+ def close(self) -> None:
1281
+ """Release the retained descriptor. Called on server shutdown."""
1282
+
1283
+ self._drop_run()
1284
+
1285
+
1286
+ class _WatchServer(ThreadingHTTPServer):
1287
+ daemon_threads = True
1288
+
1289
+ def __init__(self, address: tuple[str, int], handler: type, state: _WatchState) -> None:
1290
+ super().__init__(address, handler)
1291
+ self.watch_state = state
1292
+ self.watch_stopped = False
1293
+ self.watch_thread: threading.Thread | None = None
1294
+
1295
+
1296
+ class _WatchServerV6(_WatchServer):
1297
+ address_family = socket.AF_INET6
1298
+
1299
+
1300
+ class _WatchHandler(BaseHTTPRequestHandler):
1301
+ # The server exposes read-only GET routes. Approval remains a terminal operation.
1302
+
1303
+ def log_message(self, *_args: object) -> None: # silence default stderr access logging
1304
+ return
1305
+
1306
+ @property
1307
+ def _state(self) -> _WatchState:
1308
+ return self.server.watch_state # type: ignore[attr-defined]
1309
+
1310
+ def do_GET(self) -> None: # BaseHTTPRequestHandler's fixed dispatch name
1311
+ if not self._request_targets_this_loopback_server():
1312
+ self.send_error(421)
1313
+ return
1314
+ path = self.path.split("?", 1)[0]
1315
+ if path == "/":
1316
+ self._serve_index()
1317
+ elif path == "/events":
1318
+ self._serve_events()
1319
+ elif path == "/receipt":
1320
+ self._serve_receipt()
1321
+ else:
1322
+ self.send_error(404)
1323
+
1324
+ def _request_targets_this_loopback_server(self) -> bool:
1325
+ """Reject DNS-rebound and cross-origin reads before any run content is written."""
1326
+
1327
+ bound_port = int(self.server.server_address[1])
1328
+ try:
1329
+ bound_address = ipaddress.ip_address(str(self.server.server_address[0]))
1330
+ except ValueError:
1331
+ return False
1332
+
1333
+ def local_authority(value: str, *, origin: bool) -> bool:
1334
+ try:
1335
+ parsed = urllib.parse.urlsplit(value if origin else f"//{value}")
1336
+ hostname = parsed.hostname
1337
+ port = parsed.port
1338
+ except ValueError:
1339
+ return False
1340
+ if parsed.username is not None or parsed.password is not None:
1341
+ return False
1342
+ if hostname is None or port != bound_port:
1343
+ return False
1344
+ if hostname == "localhost":
1345
+ return True
1346
+ try:
1347
+ requested = ipaddress.ip_address(hostname)
1348
+ except ValueError:
1349
+ return False
1350
+ return requested.is_loopback and requested == bound_address
1351
+
1352
+ host = self.headers.get("Host")
1353
+ if host is None or not local_authority(host, origin=False):
1354
+ return False
1355
+ origin = self.headers.get("Origin")
1356
+ return origin is None or local_authority(origin, origin=True)
1357
+
1358
+ def _serve_index(self) -> None:
1359
+ """Write the memoized bytes. Performs no file I/O and cannot raise on bad feed bytes."""
1360
+
1361
+ _version, _last_seq, markup = self._state.snapshot()
1362
+ self._write_html(200, markup)
1363
+
1364
+ def _serve_receipt(self) -> None:
1365
+ """Serve one receipt, verified against the fingerprint the run recorded for it.
1366
+
1367
+ The manifest, the allowlist and the directory descriptor are read from ``_WatchState`` on
1368
+ every request and never captured at handler construction because they may change while the
1369
+ build runs.
1370
+ """
1371
+
1372
+ query = urllib.parse.parse_qs(urllib.parse.urlparse(self.path).query)
1373
+ member = (query.get("member") or [""])[0]
1374
+ cited_values = query.get("sha256") or []
1375
+ cited = cited_values[0] if cited_values else None
1376
+
1377
+ try:
1378
+ raw = self._state.read_member(member, cited)
1379
+ except ReceiptRefused as refusal:
1380
+ self._write_html(refusal.status, render_refusal(refusal.sentence))
1381
+ return
1382
+ except Exception: # pragma: no cover - every refusal shape above is typed
1383
+ self._write_html(404, render_refusal(_NOT_A_RECEIPT))
1384
+ return
1385
+
1386
+ try:
1387
+ markup = render_receipt(member, raw)
1388
+ except Exception: # pragma: no cover - render_receipt degrades rather than raising
1389
+ self._write_html(404, render_refusal(_NOT_A_RECEIPT))
1390
+ return
1391
+ self._write_html(200, markup)
1392
+
1393
+ def _write_html(self, status: int, markup: str) -> None:
1394
+ body = markup.encode("utf-8")
1395
+ self.send_response(status)
1396
+ self.send_header("Content-Type", "text/html; charset=utf-8")
1397
+ self.send_header("Content-Length", str(len(body)))
1398
+ self.end_headers()
1399
+ self.wfile.write(body)
1400
+
1401
+ def _serve_events(self) -> None:
1402
+ self.send_response(200)
1403
+ self.send_header("Content-Type", "text/event-stream")
1404
+ self.send_header("Cache-Control", "no-cache")
1405
+ self.send_header("Connection", "keep-alive")
1406
+ self.end_headers()
1407
+ last = self._state.snapshot()[0]
1408
+ try:
1409
+ while not getattr(self.server, "watch_stopped", False):
1410
+ version, last_seq, _markup = self._state.snapshot()
1411
+ if version != last:
1412
+ last = version
1413
+ # The version leads, and the sequence number rides along. The client treats a
1414
+ # frame it has already seen as nothing to do, so the value on the wire has to
1415
+ # change whenever the served page changes -- and the sequence number alone does
1416
+ # not: a failed poll puts a stall note on the page and adds no record, so a tab
1417
+ # deduping on the sequence threw away the one frame that mattered. It is still
1418
+ # a state identity and not a reload signal: a full reload on every event is a
1419
+ # flicker, and the page should feel like a record of events accumulating, not a
1420
+ # dashboard refreshing.
1421
+ self.wfile.write(f"data: {version}.{last_seq}\n\n".encode())
1422
+ else:
1423
+ self.wfile.write(b": ping\n\n")
1424
+ self.wfile.flush()
1425
+ time.sleep(_POLL_SECONDS)
1426
+ except (BrokenPipeError, ConnectionResetError, OSError):
1427
+ return
1428
+
1429
+
1430
+ def _watch_loop(server: _WatchServer) -> None:
1431
+ state = server.watch_state
1432
+ while not server.watch_stopped:
1433
+ state.poll()
1434
+ time.sleep(_POLL_SECONDS)
1435
+
1436
+
1437
+ def _shutdown_watch_server(server: _WatchServer) -> None:
1438
+ """Stop and join the sole poll thread before closing anything it can still reach."""
1439
+
1440
+ server.watch_stopped = True
1441
+ failure: BaseException | None = None
1442
+ thread = server.watch_thread
1443
+ if thread is not None and thread.ident is not None:
1444
+ try:
1445
+ thread.join()
1446
+ except BaseException as error:
1447
+ failure = error
1448
+ try:
1449
+ server.watch_state.close()
1450
+ except BaseException as error:
1451
+ if failure is None:
1452
+ failure = error
1453
+ try:
1454
+ server.server_close()
1455
+ except BaseException as error:
1456
+ if failure is None:
1457
+ failure = error
1458
+ if failure is not None:
1459
+ raise failure
1460
+
1461
+
1462
+ def _build_server(
1463
+ target: _Target | str | os.PathLike[str],
1464
+ host: str,
1465
+ port: int,
1466
+ *,
1467
+ replay: float | None = None,
1468
+ ) -> tuple[_WatchServer, str]:
1469
+ """Bind a loopback server over the run's feed and start its watcher thread."""
1470
+
1471
+ state = _WatchState(target, replay=replay)
1472
+ server: _WatchServer | None = None
1473
+ try:
1474
+ address = ipaddress.ip_address(host)
1475
+ server_type = _WatchServerV6 if address.version == 6 else _WatchServer
1476
+ server = server_type((address.compressed, port), _WatchHandler, state)
1477
+ bound_host = str(server.server_address[0])
1478
+ url_host = f"[{bound_host}]" if address.version == 6 else bound_host
1479
+ url = f"http://{url_host}:{server.server_address[1]}/"
1480
+ server.watch_thread = threading.Thread(target=_watch_loop, args=(server,), daemon=True)
1481
+ server.watch_thread.start()
1482
+ return server, url
1483
+ except BaseException:
1484
+ try:
1485
+ if server is not None:
1486
+ _shutdown_watch_server(server)
1487
+ else:
1488
+ state.close()
1489
+ except BaseException:
1490
+ # Construction/startup is the first failure. Cleanup still runs compoundly above,
1491
+ # but a secondary failure must not replace the one the caller needs to diagnose.
1492
+ pass
1493
+ raise
1494
+
1495
+
1496
+ def serve_watch(
1497
+ target: _Target | str | os.PathLike[str],
1498
+ *,
1499
+ host: str = "127.0.0.1",
1500
+ port: int = 0,
1501
+ open_browser: bool = False,
1502
+ replay: float | None = None,
1503
+ announce: Callable[[str], None] | None = None,
1504
+ ) -> None:
1505
+ """Serve the live run view on ``localhost``. ``KeyboardInterrupt`` (Ctrl-C) shuts it down.
1506
+
1507
+ The served-URL line is flushed immediately: the port is ephemeral, so a caller that starts this
1508
+ in the background has to read that line to learn the port, and a block-buffered pipe would
1509
+ withhold it for the entire life of the server.
1510
+
1511
+ ``replay`` paces a finished run's records on their original inter-arrival gaps divided by the
1512
+ given speed. Replay opens nothing for writing: it re-runs no build, appends to no feed and
1513
+ touches no byte of the run directory.
1514
+ """
1515
+
1516
+ server, url = _build_server(target, host, port, replay=replay)
1517
+ try:
1518
+ if announce is None:
1519
+ print(f"mr-data watch: serving {url} (Ctrl-C to stop)", flush=True)
1520
+ else:
1521
+ announce(url)
1522
+ if open_browser:
1523
+ webbrowser.open(url)
1524
+ try:
1525
+ server.serve_forever()
1526
+ except KeyboardInterrupt:
1527
+ pass
1528
+ finally:
1529
+ _shutdown_watch_server(server)
1530
+
1531
+
1532
+ # ------------------------------------------------------------------------------------------------
1533
+ # The page
1534
+ # ------------------------------------------------------------------------------------------------
1535
+
1536
+
1537
+ _STOPPED_EVENTS = STOPPED_EVENTS
1538
+
1539
+
1540
+ def _rail_positions() -> dict[str, tuple[int, int]]:
1541
+ """Map every event name onto its position in the rail. A stage header sits at step ``-1``."""
1542
+
1543
+ positions: dict[str, tuple[int, int]] = {}
1544
+ for stage_index, stage in enumerate(STAGE_RAIL):
1545
+ for name in stage["events"]:
1546
+ positions[name] = (stage_index, -1)
1547
+ for step_index, step in enumerate(stage["steps"]):
1548
+ for name in step["events"]:
1549
+ positions[name] = (stage_index, step_index)
1550
+ return positions
1551
+
1552
+
1553
+ _EVENT_POSITIONS = _rail_positions()
1554
+
1555
+
1556
+ def _rail_members(element: Mapping[str, Any]) -> tuple[str, ...]:
1557
+ """The files that prove this rail element happened, or an empty tuple when it has none.
1558
+
1559
+ A step declares no ``members`` key of its own: the member its event points at IS its proof, and
1560
+ ``_rail_member`` already resolves it from ``events.EVIDENCE_MEMBER``.
1561
+ """
1562
+
1563
+ declared = element.get("members")
1564
+ if declared:
1565
+ return tuple(declared)
1566
+ member = _rail_member(element["events"])
1567
+ return (member,) if member is not None else ()
1568
+
1569
+
1570
+ # Written with %-placeholders rather than an f-string so CSS braces stay unescaped and readable.
1571
+ # Every colour and type value is an imported token; nothing here invents a hex. Classes are
1572
+ # `mr-watch-` prefixed kebab-case with `is-<state>` for transient state, following the notebook
1573
+ # renderer's class convention without touching any of its files.
1574
+ _CSS = """
1575
+ body { margin: 0; padding: 32px 24px 56px; background: %(paper)s; color: %(ink)s;
1576
+ font-family: %(sans)s; }
1577
+ .mr-watch-page { max-width: 860px; margin: 0 auto; }
1578
+ .mr-watch-title { font-size: 25px; font-weight: 600; letter-spacing: -0.02em; margin: 0 0 6px;
1579
+ color: %(ink)s; }
1580
+ .mr-watch-run { font-family: %(mono)s; font-size: 11.5px; color: %(muted)s; margin: 0 0 4px; }
1581
+ .mr-watch-elapsed { font-family: %(mono)s; font-size: 11.5px; color: %(faint)s; margin: 0 0 26px; }
1582
+ .mr-watch-rail { list-style: none; margin: 0 0 28px; padding: 16px 20px; border-radius: 12px;
1583
+ border: 1px solid %(border)s; background: %(surface)s; }
1584
+ .mr-watch-stage { display: block; padding: 5px 0; }
1585
+ .mr-watch-stage-mark, .mr-watch-step-mark { display: inline-block; width: 8px; height: 8px;
1586
+ border-radius: 2px; margin-right: 10px; vertical-align: 1px; background: %(border)s; }
1587
+ .mr-watch-stage.is-done > .mr-watch-stage-mark { background: %(cobalt)s; }
1588
+ .mr-watch-step.is-done > .mr-watch-step-mark { background: %(cobalt)s; }
1589
+ .mr-watch-stage-label { font-size: 15px; font-weight: 600; color: %(faint)s; }
1590
+ .mr-watch-stage.is-done > .mr-watch-stage-label { color: %(ink)s; }
1591
+ .mr-watch-steps { list-style: none; margin: 6px 0 6px 18px; padding: 0; }
1592
+ .mr-watch-step { display: block; padding: 3px 0; }
1593
+ .mr-watch-step-label { font-family: %(mono)s; font-size: 12px; color: %(faint)s; }
1594
+ .mr-watch-step.is-done > .mr-watch-step-label { color: %(body)s; }
1595
+ .mr-watch-log { list-style: none; margin: 0; padding: 0; border-top: 1px solid %(border)s; }
1596
+ .mr-watch-event { display: flex; gap: 14px; align-items: baseline; padding: 7px 2px;
1597
+ border-bottom: 1px solid %(border)s; }
1598
+ .mr-watch-event.is-stopped > .mr-watch-text { color: %(orange)s; }
1599
+ .mr-watch-time { font-family: %(mono)s; font-size: 11px; color: %(faint)s; min-width: 56px;
1600
+ font-variant-numeric: tabular-nums; }
1601
+ .mr-watch-text { font-size: 14px; color: %(body)s; flex: 1; }
1602
+ .mr-watch-receipt { font-family: %(mono)s; font-size: 11px; color: %(muted)s; }
1603
+ .mr-watch-receipt-link { font-family: %(mono)s; font-size: 11px; color: %(cobalt)s; }
1604
+ .mr-watch-later { font-family: %(mono)s; font-size: 11px; color: %(dim)s; }
1605
+ .mr-watch-empty { font-family: %(mono)s; font-size: 12px; color: %(faint)s; }
1606
+ .mr-watch-stalled { font-size: 13.5px; color: %(orange)s; border: 1px solid %(orange)s;
1607
+ border-radius: 8px; background: %(sub_paper)s; padding: 10px 12px; margin: 0 0 20px; }
1608
+ .mr-watch-cards { margin: 0 0 28px; }
1609
+ .mr-watch-card { border: 1px solid %(border)s; border-radius: 12px; background: %(sub_paper)s;
1610
+ padding: 16px 20px; margin: 0 0 14px; }
1611
+ .mr-watch-card-title { font-size: 16px; font-weight: 600; margin: 0 0 8px; color: %(ink)s; }
1612
+ .mr-watch-card-lead { font-size: 14px; color: %(body)s; margin: 0 0 10px; }
1613
+ .mr-watch-list { margin: 0 0 8px; padding-left: 20px; }
1614
+ .mr-watch-list li { font-size: 13.5px; color: %(body)s; padding: 2px 0; }
1615
+ .mr-watch-sub { border-top: 1px solid %(border)s; margin-top: 10px; padding-top: 10px; }
1616
+ .mr-watch-sub-title { font-size: 14px; font-weight: 600; color: %(ink)s; margin: 0 0 4px; }
1617
+ .mr-watch-defs { margin: 0; font-size: 13.5px; color: %(body)s; }
1618
+ .mr-watch-defs dt { font-weight: 600; color: %(muted)s; font-size: 12px; margin-top: 6px; }
1619
+ .mr-watch-defs dd { margin: 1px 0 0; }
1620
+ .mr-watch-table { border-collapse: collapse; width: 100%%; font-size: 13px; margin: 6px 0 2px; }
1621
+ .mr-watch-table th { text-align: left; font-size: 11.5px; color: %(muted)s; font-weight: 600;
1622
+ border-bottom: 1px solid %(border)s; padding: 4px 10px 4px 0; }
1623
+ .mr-watch-table td { color: %(body)s; border-bottom: 1px solid %(border)s; padding: 4px 10px 4px 0;
1624
+ vertical-align: top; }
1625
+ .mr-watch-approve { border: 1px solid %(cobalt)s; border-radius: 12px; background: %(sub_paper)s;
1626
+ padding: 16px 20px; margin: 0 0 24px; }
1627
+ .mr-watch-note { font-size: 13px; color: %(muted)s; margin: 10px 0 8px; }
1628
+ .mr-watch-command { display: block; font-family: %(mono)s; font-size: 11.5px; color: %(ink)s;
1629
+ background: %(surface)s; border: 1px solid %(border)s; border-radius: 8px; padding: 10px 12px;
1630
+ overflow-x: auto; white-space: pre; }
1631
+ .mr-watch-copy { font-family: %(sans)s; font-size: 12.5px; color: %(paper)s; background: %(cobalt)s;
1632
+ border: 1px solid %(cobalt)s; border-radius: 8px; padding: 6px 14px; margin-top: 10px;
1633
+ cursor: pointer; }
1634
+ .mr-watch-back { font-size: 12.5px; color: %(cobalt)s; display: inline-block; margin-bottom: 18px; }
1635
+ .mr-watch-refusal { font-size: 15px; color: %(orange)s; margin: 0; }
1636
+ .mr-watch-pre { font-family: %(mono)s; font-size: 11.5px; color: %(body)s; white-space: pre-wrap;
1637
+ word-break: break-all; margin: 0; }
1638
+ """
1639
+
1640
+ # The whole client: one EventSource, one 1 Hz recompute of the elapsed text. No framework, no
1641
+ # bundler, no build step, no network. Wrapped in try/catch so live update is best-effort -- a page
1642
+ # whose script failed is still a correct record of everything that had happened when it was served.
1643
+ _LIVE_SCRIPT = (
1644
+ "<script>\n"
1645
+ "(function () {\n"
1646
+ " try {\n"
1647
+ " var seen = null;\n"
1648
+ " var source = new EventSource('/events');\n"
1649
+ " source.onmessage = function (event) {\n"
1650
+ " if (event.data === seen) { return; }\n"
1651
+ " seen = event.data;\n"
1652
+ " fetch('/').then(function (response) { return response.text(); })\n"
1653
+ " .then(function (text) {\n"
1654
+ " var parsed = new DOMParser().parseFromString(text, 'text/html');\n"
1655
+ " document.body.innerHTML = parsed.body.innerHTML;\n"
1656
+ " })\n"
1657
+ " .catch(function () { /* the next event tries again */ });\n"
1658
+ " };\n"
1659
+ " var retick = function () {\n"
1660
+ " var node = document.querySelector('.mr-watch-elapsed');\n"
1661
+ " if (!node) { return; }\n"
1662
+ " var at = parseFloat(node.getAttribute('data-at'));\n"
1663
+ " if (!(at > 0)) { return; }\n"
1664
+ " var seconds = Math.max(0, Math.round(Date.now() / 1000 - at));\n"
1665
+ " node.textContent = 'Last event ' + seconds + ' seconds ago';\n"
1666
+ " };\n"
1667
+ " window.setInterval(retick, 1000);\n"
1668
+ " document.addEventListener('click', function (clicked) {\n"
1669
+ " var button = clicked.target.closest('.mr-watch-copy');\n"
1670
+ " if (!button) { return; }\n"
1671
+ " var target = document.getElementById(button.getAttribute('data-copy-target'));\n"
1672
+ " if (!target) { return; }\n"
1673
+ " var text = target.textContent;\n"
1674
+ " var done = function () {\n"
1675
+ " button.textContent = 'Copied';\n"
1676
+ " window.setTimeout(function () { button.textContent = 'Copy'; }, 2000);\n"
1677
+ " };\n"
1678
+ " if (navigator.clipboard && navigator.clipboard.writeText) {\n"
1679
+ " navigator.clipboard.writeText(text).then(done, function () {});\n"
1680
+ " return;\n"
1681
+ " }\n"
1682
+ " var scratch = document.createElement('textarea');\n"
1683
+ " scratch.value = text;\n"
1684
+ " document.body.appendChild(scratch);\n"
1685
+ " scratch.select();\n"
1686
+ " try { document.execCommand('copy'); done(); } catch (error) { /* nothing to do */ }\n"
1687
+ " document.body.removeChild(scratch);\n"
1688
+ " });\n"
1689
+ " } catch (error) { /* live update is best-effort */ }\n"
1690
+ "})();\n"
1691
+ "</script>"
1692
+ )
1693
+
1694
+
1695
+ def stylesheet() -> str:
1696
+ return (
1697
+ "<style>"
1698
+ + _CSS
1699
+ % {
1700
+ "paper": PAPER,
1701
+ "ink": INK,
1702
+ "body": BODY,
1703
+ "muted": MUTED,
1704
+ "faint": FAINT,
1705
+ "dim": DIM,
1706
+ "border": BORDER,
1707
+ "surface": SURFACE,
1708
+ "sub_paper": SUB_PAPER,
1709
+ "cobalt": COBALT,
1710
+ "orange": ORANGE,
1711
+ "sans": FONT_SANS,
1712
+ "mono": FONT_MONO,
1713
+ }
1714
+ + "</style>"
1715
+ )
1716
+
1717
+
1718
+ def _seen_events(records: Sequence[Mapping[str, Any]]) -> frozenset[str]:
1719
+ """Return the rail event names present in the feed."""
1720
+
1721
+ return frozenset(
1722
+ str(record.get("event"))
1723
+ for record in records
1724
+ if str(record.get("event")) in _EVENT_POSITIONS
1725
+ )
1726
+
1727
+
1728
+ def _state_class(
1729
+ element: Mapping[str, Any],
1730
+ seen: frozenset[str],
1731
+ members_present: frozenset[str],
1732
+ ) -> str:
1733
+ """A rail item is done when its OWN evidence says so, and never on a later item's strength.
1734
+
1735
+ Two ways to be done, and no third: one of the element's own events is in the feed, or every
1736
+ file it names is in the sealed member set. Anything else waits. There is no animated state --
1737
+ progress on this page is events that happened, never a spinner.
1738
+ """
1739
+
1740
+ if any(name in seen for name in element["events"]):
1741
+ return "is-done"
1742
+ members = _rail_members(element)
1743
+ if members and all(member in members_present for member in members):
1744
+ return "is-done"
1745
+ return "is-waiting"
1746
+
1747
+
1748
+ def _sentence(event: str, facts: Mapping[str, Any]) -> str:
1749
+ return sentence(event, facts)
1750
+
1751
+
1752
+ def _pointer_of(record: Mapping[str, Any]) -> tuple[str | None, str | None]:
1753
+ """Return ``(member, sha256)`` for a record's evidence pointer, either half possibly ``None``.
1754
+
1755
+ A record sealed AFTER the fact carries both halves in ``evidence``; a record emitted DURING the
1756
+ build carries only the member name it will be provable against, in ``facts["evidence_member"]``.
1757
+ """
1758
+
1759
+ evidence = record.get("evidence")
1760
+ if isinstance(evidence, Mapping):
1761
+ member = evidence.get("member")
1762
+ sha256 = evidence.get("sha256")
1763
+ if isinstance(member, str):
1764
+ return member, sha256 if isinstance(sha256, str) else None
1765
+ facts = record.get("facts")
1766
+ member = facts.get("evidence_member") if isinstance(facts, Mapping) else None
1767
+ return (member if isinstance(member, str) else None), None
1768
+
1769
+
1770
+ def receipt_href(member: str, sha256: str | None = None) -> str:
1771
+ """Build the link to one receipt. Both values are percent-quoted before they reach the URL."""
1772
+
1773
+ query = {"member": member}
1774
+ if sha256:
1775
+ query["sha256"] = sha256
1776
+ return "/receipt?" + urllib.parse.urlencode(query, quote_via=urllib.parse.quote)
1777
+
1778
+
1779
+ def _receipt(
1780
+ record: Mapping[str, Any],
1781
+ members_present: frozenset[str],
1782
+ recorded_digests: Mapping[str, str] | None = None,
1783
+ ) -> str:
1784
+ """Return a receipt link when the member and manifest digest are verified.
1785
+
1786
+ A feed digest that disagrees with the manifest suppresses the link.
1787
+ """
1788
+
1789
+ member, sha256 = _pointer_of(record)
1790
+ if member is None:
1791
+ return ""
1792
+ if member not in members_present:
1793
+ return '<span class="mr-watch-later">receipt appears when the build finishes</span>'
1794
+ recorded = (recorded_digests or {}).get(member)
1795
+ if sha256 is not None and recorded is not None and not _matches_digest(sha256, recorded):
1796
+ return ""
1797
+ return (
1798
+ f'<a class="mr-watch-receipt-link" href="{esc(receipt_href(member, recorded))}"'
1799
+ f' data-member="{esc(member)}">Receipt: {esc(member)}</a>'
1800
+ )
1801
+
1802
+
1803
+ def render_stall_note(failures: int, reason: str) -> str:
1804
+ """The sentence a view that has stopped reading owes its reader.
1805
+
1806
+ Watch has no spinner: the page shows what happened, and where nothing has happened it says so.
1807
+ A page whose reader has stopped being able to read is the same
1808
+ obligation one step further on -- it is still showing a true record, but of a moment that has
1809
+ passed, and a reader has no way to tell that from a build that is simply taking its time.
1810
+
1811
+ The failure's own NAME is on the page and its message is not. A name comes from this codebase;
1812
+ a message can carry bytes out of the run directory, which is a place another local user writes.
1813
+ """
1814
+
1815
+ times = "once" if failures == 1 else f"{failures} times"
1816
+ return (
1817
+ f'<p class="mr-watch-stalled" data-failed-polls="{esc(failures)}"'
1818
+ f' data-reason="{esc(reason)}">'
1819
+ f"This page has stopped updating. Reading this build failed {esc(times)}, "
1820
+ f"the last time with {esc(reason)}. "
1821
+ "What you see below is the last thing this page could read."
1822
+ "</p>"
1823
+ )
1824
+
1825
+
1826
+ # The stall note is spliced in after the header rather than rendered with the page, because the page
1827
+ # it belongs on is by definition one this reader could NOT re-render. Every page this module builds
1828
+ # opens with exactly one header, so the first close tag is the anchor.
1829
+ _STALL_ANCHOR = "</header>"
1830
+
1831
+
1832
+ def _with_stall_note(markup: str, failures: int, reason: str) -> str:
1833
+ """Return the last good page with the stall note under its header."""
1834
+
1835
+ head, anchor, tail = markup.partition(_STALL_ANCHOR)
1836
+ note = render_stall_note(failures, reason)
1837
+ if not anchor: # pragma: no cover - every page this module renders carries a header
1838
+ return note + markup
1839
+ return head + anchor + note + tail
1840
+
1841
+
1842
+ def _render_header(run_dir: Path, records: Sequence[Mapping[str, Any]], now: float) -> str:
1843
+ last_at = float(records[-1]["at"]) if records else 0.0
1844
+ if records and last_at > 0:
1845
+ text = f"Last event {max(0, int(now - last_at))} seconds ago"
1846
+ elif records:
1847
+ # Rederived records have no source timestamp.
1848
+ text = "Rebuilt from the files this build recorded"
1849
+ else:
1850
+ text = "No events yet"
1851
+ return (
1852
+ '<header class="mr-watch-header">'
1853
+ '<h1 class="mr-watch-title">Watch</h1>'
1854
+ f'<p class="mr-watch-run">Build: {esc(_writable_path_text(run_dir.name))}</p>'
1855
+ f'<p class="mr-watch-elapsed" data-at="{esc(f"{last_at:.3f}")}">{esc(text)}</p>'
1856
+ "</header>"
1857
+ )
1858
+
1859
+
1860
+ def _rail_member(names: Sequence[str]) -> str | None:
1861
+ """The receipt a rail element stands for: the first of its events with a fixed pointer.
1862
+
1863
+ ``member_sealed`` and ``build_sealed`` have no fixed member -- the first varies per member and
1864
+ the second anchors on the run's own fingerprint -- so the "Receipt recorded" step falls through
1865
+ to the run's own receipt, which is exactly what a reader clicking it wants to see.
1866
+ """
1867
+
1868
+ for name in names:
1869
+ member = events.EVIDENCE_MEMBER.get(name)
1870
+ if member is not None:
1871
+ return member
1872
+ if "build_sealed" in names:
1873
+ return _MANIFEST_NAME
1874
+ return None
1875
+
1876
+
1877
+ def _rail_label(label: str, css: str, names: Sequence[str], members_present: frozenset[str]) -> str:
1878
+ """Render one rail element's label, linked to its receipt once that receipt exists."""
1879
+
1880
+ member = _rail_member(names)
1881
+ if member is None or member not in members_present:
1882
+ return f'<span class="{css}">{esc(label)}</span>'
1883
+ return (
1884
+ f'<a class="{css}" href="{esc(receipt_href(member))}"'
1885
+ f' data-member="{esc(member)}">{esc(label)}</a>'
1886
+ )
1887
+
1888
+
1889
+ def _render_rail(seen: frozenset[str], members_present: frozenset[str]) -> str:
1890
+ items: list[str] = []
1891
+ for stage in STAGE_RAIL:
1892
+ stage_state = _state_class(stage, seen, members_present)
1893
+ steps = "".join(
1894
+ f'<li class="mr-watch-step {_state_class(step, seen, members_present)}"'
1895
+ f' data-rail="{esc(stage["stage"])}.{esc(step["step"])}">'
1896
+ '<span class="mr-watch-step-mark"></span>'
1897
+ + _rail_label(step["label"], "mr-watch-step-label", step["events"], members_present)
1898
+ + "</li>"
1899
+ for step in stage["steps"]
1900
+ )
1901
+ items.append(
1902
+ f'<li class="mr-watch-stage {stage_state}" data-rail="{esc(stage["stage"])}">'
1903
+ '<span class="mr-watch-stage-mark"></span>'
1904
+ + _rail_label(stage["label"], "mr-watch-stage-label", stage["events"], members_present)
1905
+ + (f'<ol class="mr-watch-steps">{steps}</ol>' if steps else "")
1906
+ + "</li>"
1907
+ )
1908
+ return '<ol class="mr-watch-rail">' + "".join(items) + "</ol>"
1909
+
1910
+
1911
+ def _render_log(
1912
+ records: Sequence[Mapping[str, Any]],
1913
+ members_present: frozenset[str],
1914
+ recorded_digests: Mapping[str, str] | None = None,
1915
+ ) -> str:
1916
+ if not records:
1917
+ return '<p class="mr-watch-empty">Nothing has happened yet.</p>'
1918
+ started = float(records[0]["at"])
1919
+ items: list[str] = []
1920
+ for record in records:
1921
+ event = str(record.get("event", ""))
1922
+ facts = record.get("facts")
1923
+ offset = max(0, int(float(record.get("at", started)) - started))
1924
+ classes = "mr-watch-event is-stopped" if event in _STOPPED_EVENTS else "mr-watch-event"
1925
+ items.append(
1926
+ f'<li class="{classes}">'
1927
+ f'<span class="mr-watch-time">+{esc(offset)}s</span>'
1928
+ f'<span class="mr-watch-text">'
1929
+ f"{esc(_sentence(event, facts if isinstance(facts, Mapping) else {}))}</span>"
1930
+ f"{_receipt(record, members_present, recorded_digests)}"
1931
+ "</li>"
1932
+ )
1933
+ return '<ol class="mr-watch-log">' + "".join(items) + "</ol>"
1934
+
1935
+
1936
+ def render_page(
1937
+ records: Sequence[Mapping[str, Any]],
1938
+ *,
1939
+ run_dir: Path,
1940
+ members_present: frozenset[str],
1941
+ now: float,
1942
+ read_member: Callable[[str], bytes] | None = None,
1943
+ recorded_digests: Mapping[str, str] | None = None,
1944
+ approval_corroborated: bool = True,
1945
+ ) -> str:
1946
+ """Render the whole page. It opens no file of its own and takes every input as an argument.
1947
+
1948
+ ``read_member`` is the one exception and it is a PARAMETER, not a capability this function
1949
+ reaches for: the cards and the approval summary are claims about what the run recorded, so they
1950
+ are built from the same fingerprint-verified read ``/receipt`` uses and can never render bytes
1951
+ that failed their check. With no reader the page is the rail and the event log, which is exactly
1952
+ what a watcher started before the build should show.
1953
+
1954
+ Every interpolated value crosses ``esc`` because feed and evidence strings may contain markup.
1955
+
1956
+ ``recorded_digests`` is the run's OWN digest for each member -- ``member_index`` -- and it is
1957
+ what every receipt link uses.
1958
+ """
1959
+
1960
+ seen = _seen_events(records)
1961
+ approval = ""
1962
+ cards = ""
1963
+ if read_member is not None and approval_corroborated:
1964
+ approval = _safe(lambda: render_approval(run_dir, members_present, read_member))
1965
+ if read_member is not None:
1966
+ cards = _safe(lambda: render_cards(members_present, read_member))
1967
+ return (
1968
+ "<!doctype html>\n"
1969
+ '<html lang="en"><head><meta charset="utf-8">'
1970
+ '<meta name="viewport" content="width=device-width, initial-scale=1">'
1971
+ "<title>Watch</title>"
1972
+ f"{stylesheet()}"
1973
+ "</head><body>\n"
1974
+ '<div class="mr-watch-page">'
1975
+ f"{_render_header(run_dir, records, now)}"
1976
+ f"{_render_rail(seen, members_present)}"
1977
+ f"{approval}"
1978
+ f"{cards}"
1979
+ f"{_render_log(records, members_present, recorded_digests)}"
1980
+ "</div>\n"
1981
+ f"{_LIVE_SCRIPT}\n"
1982
+ "</body></html>\n"
1983
+ )
1984
+
1985
+
1986
+ def _safe(render: Callable[[], str]) -> str:
1987
+ """Render a section, or render nothing. A section is never allowed to blank the page.
1988
+
1989
+ The run directory is a place another local user can write, so a member can be any bytes at all.
1990
+ A card that raised on a shape it did not expect would take down the event log with it, and the
1991
+ event log is the part a reader most needs when something has gone wrong.
1992
+ """
1993
+
1994
+ try:
1995
+ return render()
1996
+ except Exception:
1997
+ return ""
1998
+
1999
+
2000
+ # ------------------------------------------------------------------------------------------------
2001
+ # Plain vocabulary
2002
+ # ------------------------------------------------------------------------------------------------
2003
+ #
2004
+ # Two vocabularies, one direction. The evidence a receipt renders is internal contract JSON, and its
2005
+ # field names are engineering-precise on purpose. None of them may reach a page. The rule this file
2006
+ # follows is to map every field to a plain phrase and omit fields without a plain rendering.
2007
+
2008
+ # A word that, appearing in a key, means that key has no plain rendering and must be dropped.
2009
+ _INTERNAL_WORDS = frozenset(
2010
+ {
2011
+ "acquisition",
2012
+ "adapter",
2013
+ "attempt",
2014
+ "candidate",
2015
+ "capability",
2016
+ "caps",
2017
+ "classification",
2018
+ "credential",
2019
+ "crawler",
2020
+ "decode",
2021
+ "digest",
2022
+ "digests",
2023
+ "golden",
2024
+ "governance",
2025
+ "id",
2026
+ "ids",
2027
+ "lawful",
2028
+ "manifest",
2029
+ "materialization",
2030
+ "normalizer",
2031
+ "policy",
2032
+ "predecessor",
2033
+ "principal",
2034
+ "producer",
2035
+ "promotion",
2036
+ "sandbox",
2037
+ "schema",
2038
+ "sealed",
2039
+ "semantics",
2040
+ "sha256",
2041
+ "sha256s",
2042
+ "template",
2043
+ "verifier",
2044
+ "version",
2045
+ "watermark",
2046
+ "workspace",
2047
+ }
2048
+ )
2049
+
2050
+ # Keys whose plain phrase is worth stating explicitly rather than deriving.
2051
+ _PLAIN_KEYS: Mapping[str, str] = {
2052
+ "bytes": "Size in bytes",
2053
+ "cardinality": "Shape of the match",
2054
+ "cleaning": "Cleaning",
2055
+ "columns": "Columns",
2056
+ "created_at": "Recorded at",
2057
+ "evidence": "What backs it",
2058
+ "expected": "Expected",
2059
+ "grain": "One row per",
2060
+ "join": "Join",
2061
+ "keys": "Joined on",
2062
+ "left_rows": "Rows on the left",
2063
+ "left_source": "Left side",
2064
+ "matched_left_rows": "Rows that matched",
2065
+ "null_counts": "Blanks per column",
2066
+ "observed": "Observed",
2067
+ "operations": "What was done to it",
2068
+ "origin": "Where it came from",
2069
+ "output_intent": "Meant for",
2070
+ "output_rows": "Rows out",
2071
+ "passed": "Result",
2072
+ "permissions": "You may",
2073
+ "population": "Who is in it",
2074
+ "quality": "Checks",
2075
+ "questions": "Questions",
2076
+ "relative_path": "File",
2077
+ "rights": "Rights",
2078
+ "right_rows": "Rows on the right",
2079
+ "right_source": "Right side",
2080
+ "row_count": "Rows",
2081
+ "row_multiplier": "Row multiplier",
2082
+ "rows": "Rows",
2083
+ "select": "Columns kept",
2084
+ "sources": "Sources",
2085
+ "stable_order": "Row order",
2086
+ "status": "State",
2087
+ "text": "Text",
2088
+ "types": "Types",
2089
+ "unmatched_left_rows": "Rows that did not match",
2090
+ }
2091
+
2092
+ # Contract values that have a plain reading. Anything not here is humanized, never dropped: a value
2093
+ # is data the run recorded, and dropping data would be editing the receipt.
2094
+ _RIGHTS_STATUS: Mapping[str, str] = {
2095
+ "project_owned": "owned by this project",
2096
+ "public_domain": "in the public domain",
2097
+ "licensed": "used under a licence",
2098
+ "permitted": "cleared for this use",
2099
+ "restricted": "restricted",
2100
+ "unknown": "not decided yet",
2101
+ }
2102
+
2103
+ _RIGHTS_DECISION: Mapping[str, str] = {
2104
+ "approved": "cleared for this use",
2105
+ "conditional": "conditions attached",
2106
+ "denied": "not usable",
2107
+ "blocked": "not usable",
2108
+ "pending": "not decided yet",
2109
+ }
2110
+
2111
+ _PERMISSIONS: Mapping[str, str] = {
2112
+ "local_use": "use it here",
2113
+ "redistribute": "share it on",
2114
+ "host": "publish it",
2115
+ }
2116
+
2117
+ _HOW_FETCHED: Mapping[str, str] = {
2118
+ "checked_in_fixture": "a file kept with the project",
2119
+ "user_file": "a file you supplied",
2120
+ "download": "downloaded over the network",
2121
+ "api": "read from an interface",
2122
+ }
2123
+
2124
+ _MATCH_SHAPE: Mapping[str, str] = {
2125
+ "one_to_one": "one row to one row",
2126
+ "many_to_one": "many rows to one row",
2127
+ "one_to_many": "one row to many rows",
2128
+ "many_to_many": "many rows to many rows",
2129
+ }
2130
+
2131
+ _JOIN_KIND: Mapping[str, str] = {
2132
+ "left": "kept every row on the left",
2133
+ "inner": "kept only rows on both sides",
2134
+ }
2135
+
2136
+ _OUTPUT_INTENT: Mapping[str, str] = {
2137
+ "local_use": "use here only",
2138
+ "redistribute": "sharing on",
2139
+ "host": "publishing",
2140
+ }
2141
+
2142
+ # The five checks a Recipe run must pass (`recipe.REQUIRED_CHECK_INVENTORY`) plus the three a plan
2143
+ # declares, each with the sentence a person reads. A check id is an internal name by design; this is
2144
+ # the map that keeps it from reaching a page.
2145
+ _PLAIN_CHECKS: Mapping[str, str] = {
2146
+ "EXACT_CANDIDATE_REPLAY": "The build reproduces exactly, byte for byte",
2147
+ "RECIPE_DRIFT": "Nothing has drifted from the Recipe",
2148
+ "SOURCE_AUTHORITY": "Every source is the one the Recipe names",
2149
+ "OUTPUT_SEMANTICS": "Every column still means what was approved",
2150
+ "TEMPORAL_INTEGRITY": "The time window still holds",
2151
+ }
2152
+
2153
+
2154
+ def _humanize(value: Any) -> str:
2155
+ """Render one contract value as words: underscores out, first letter up."""
2156
+
2157
+ text = str(value).replace("_", " ").replace("-", " ").strip()
2158
+ return text[:1].upper() + text[1:] if text else ""
2159
+
2160
+
2161
+ def _plain_label(key: Any) -> str | None:
2162
+ """Return the plain heading for one field name, or ``None`` when it has no plain reading."""
2163
+
2164
+ if not isinstance(key, str) or not key:
2165
+ return None
2166
+ known = _PLAIN_KEYS.get(key)
2167
+ if known is not None:
2168
+ return known
2169
+ words = key.replace("-", " ").replace("_", " ").split()
2170
+ if not words or any(word.lower() in _INTERNAL_WORDS for word in words):
2171
+ return None
2172
+ text = " ".join(words)
2173
+ return text[:1].upper() + text[1:]
2174
+
2175
+
2176
+ def _plain_from(table: Mapping[str, str], value: Any) -> str:
2177
+ known = table.get(str(value))
2178
+ return known if known is not None else _humanize(value)
2179
+
2180
+
2181
+ def _plain_operation(operation: Any) -> str:
2182
+ text = str(operation)
2183
+ if text.startswith("cast:"):
2184
+ return f"read as {text.split(':', 1)[1]}"
2185
+ if text == "trim":
2186
+ return "trimmed of stray spaces"
2187
+ return _humanize(text)
2188
+
2189
+
2190
+ def _plain_check(check_id: Any, expected: Any) -> str:
2191
+ """Name one check the way a person would say it.
2192
+
2193
+ Known internal check identifiers map to explicit operator-facing descriptions.
2194
+ """
2195
+
2196
+ text = str(check_id)
2197
+ known = _PLAIN_CHECKS.get(text)
2198
+ if known is not None:
2199
+ return known
2200
+ if text == "MIN_ROWS":
2201
+ return f"At least {str(expected).lstrip('>=')} rows"
2202
+ if text.startswith("NOT_NULL_"):
2203
+ return f"No blanks in {text[len('NOT_NULL_') :]}"
2204
+ if text == "UNIQUE_GRAIN":
2205
+ return "One row per key, with no repeats"
2206
+ words = text.replace("_", " ").replace("-", " ").split()
2207
+ if not words or any(word.lower() in _INTERNAL_WORDS for word in words):
2208
+ return "A check this build requires"
2209
+ return _humanize(text)
2210
+
2211
+
2212
+ def _joined(values: Any, empty: str = "none") -> str:
2213
+ if not isinstance(values, (list, tuple)) or not values:
2214
+ return empty
2215
+ return ", ".join(str(value) for value in values)
2216
+
2217
+
2218
+ # ------------------------------------------------------------------------------------------------
2219
+ # Receipts
2220
+ # ------------------------------------------------------------------------------------------------
2221
+
2222
+
2223
+ def _document(title: str, body: str, *, script: str = "") -> str:
2224
+ return (
2225
+ "<!doctype html>\n"
2226
+ '<html lang="en"><head><meta charset="utf-8">'
2227
+ '<meta name="viewport" content="width=device-width, initial-scale=1">'
2228
+ f"<title>{esc(title)}</title>"
2229
+ f"{stylesheet()}"
2230
+ "</head><body>\n"
2231
+ f'<div class="mr-watch-page">{body}</div>\n'
2232
+ f"{script}"
2233
+ "</body></html>\n"
2234
+ )
2235
+
2236
+
2237
+ def _back() -> str:
2238
+ return '<a class="mr-watch-back" href="/">Back to the build</a>'
2239
+
2240
+
2241
+ def render_refusal(sentence: str) -> str:
2242
+ """Render a refusal: the one plain sentence, the way back, and NONE of the file's content."""
2243
+
2244
+ return _document(
2245
+ "Receipt",
2246
+ _back() + f'<p class="mr-watch-refusal">{esc(sentence)}</p>',
2247
+ )
2248
+
2249
+
2250
+ def _definitions(pairs: Sequence[tuple[str, str]]) -> str:
2251
+ items = "".join(
2252
+ f"<dt>{esc(label)}</dt><dd>{esc(value)}</dd>" for label, value in pairs if label
2253
+ )
2254
+ return f'<dl class="mr-watch-defs">{items}</dl>' if items else ""
2255
+
2256
+
2257
+ def _table(headings: Sequence[str], rows: Sequence[Sequence[Any]]) -> str:
2258
+ head = "".join(f"<th>{esc(heading)}</th>" for heading in headings)
2259
+ body = "".join(
2260
+ "<tr>" + "".join(f"<td>{esc(cell)}</td>" for cell in row) + "</tr>" for row in rows
2261
+ )
2262
+ return (
2263
+ f'<table class="mr-watch-table"><thead><tr>{head}</tr></thead><tbody>{body}</tbody></table>'
2264
+ )
2265
+
2266
+
2267
+ def _lines(sentences: Sequence[str]) -> str:
2268
+ items = "".join(f"<li>{esc(sentence)}</li>" for sentence in sentences if sentence)
2269
+ return f'<ul class="mr-watch-list">{items}</ul>' if items else ""
2270
+
2271
+
2272
+ def _render_run_receipt(payload: Mapping[str, Any]) -> str:
2273
+ members = payload.get("members")
2274
+ members = members if isinstance(members, list) else []
2275
+ total = sum(entry.get("bytes", 0) for entry in members if isinstance(entry, Mapping))
2276
+ return "".join(
2277
+ [
2278
+ '<h1 class="mr-watch-card-title">The receipt for this whole build</h1>',
2279
+ '<p class="mr-watch-card-lead">This is what the build recorded about itself. '
2280
+ "Every file below was fingerprinted when the build finished, and the fingerprint of "
2281
+ "the whole set is the one line to quote when someone asks which build this was.</p>",
2282
+ _definitions(
2283
+ [
2284
+ ("Fingerprint of the whole build", str(payload.get("candidate_digest", ""))),
2285
+ ("Files recorded", str(len(members))),
2286
+ ("Total size in bytes", str(total)),
2287
+ ("Built with engine", str(payload.get("engine_version", ""))),
2288
+ ]
2289
+ ),
2290
+ _table(
2291
+ ["File", "Size in bytes", "Fingerprint"],
2292
+ [
2293
+ [entry.get("path", ""), entry.get("bytes", ""), entry.get("sha256", "")]
2294
+ for entry in members
2295
+ if isinstance(entry, Mapping)
2296
+ ],
2297
+ ),
2298
+ ]
2299
+ )
2300
+
2301
+
2302
+ def _render_join(payload: Mapping[str, Any]) -> str:
2303
+ matched = payload.get("matched_left_rows")
2304
+ left = payload.get("left_rows")
2305
+ unmatched = payload.get("unmatched_left_rows")
2306
+ keys = _joined(payload.get("keys"))
2307
+ return "".join(
2308
+ [
2309
+ '<h1 class="mr-watch-card-title">The join</h1>',
2310
+ f'<p class="mr-watch-card-lead">{esc(f"Joined on {keys}.")} '
2311
+ f"{esc(f'{matched} of {left} rows matched; {unmatched} did not.')}</p>",
2312
+ _definitions(
2313
+ [
2314
+ ("Joined on", keys),
2315
+ ("Left side", str(payload.get("left_source", ""))),
2316
+ ("Right side", str(payload.get("right_source", ""))),
2317
+ ("Rows on the left", str(left)),
2318
+ ("Rows on the right", str(payload.get("right_rows", ""))),
2319
+ ("Rows that matched", str(matched)),
2320
+ ("Rows that did not match", str(unmatched)),
2321
+ ("Rows out", str(payload.get("output_rows", ""))),
2322
+ ("Row multiplier", str(payload.get("row_multiplier", ""))),
2323
+ ("Shape of the match", _plain_from(_MATCH_SHAPE, payload.get("cardinality"))),
2324
+ ("Row order", _humanize(payload.get("stable_order", ""))),
2325
+ ]
2326
+ ),
2327
+ ]
2328
+ )
2329
+
2330
+
2331
+ def _render_quality(payload: Mapping[str, Any]) -> str:
2332
+ checks = payload.get("checks")
2333
+ checks = checks if isinstance(checks, list) else []
2334
+ passed = sum(
2335
+ 1 for check in checks if isinstance(check, Mapping) and check.get("passed") is True
2336
+ )
2337
+ rows = [
2338
+ [
2339
+ _plain_check(check.get("check_id"), check.get("expected")),
2340
+ "pass" if check.get("passed") is True else "fail",
2341
+ check.get("observed", ""),
2342
+ check.get("expected", ""),
2343
+ ]
2344
+ for check in checks
2345
+ if isinstance(check, Mapping)
2346
+ ]
2347
+ return "".join(
2348
+ [
2349
+ '<h1 class="mr-watch-card-title">The checks</h1>',
2350
+ f'<p class="mr-watch-card-lead">{esc(f"{passed} of {len(checks)} checks passed.")}</p>',
2351
+ _table(["Check", "Result", "Observed", "Expected"], rows),
2352
+ ]
2353
+ )
2354
+
2355
+
2356
+ def _render_sources(payload: Any) -> str:
2357
+ entries = payload if isinstance(payload, list) else []
2358
+ cards: list[str] = []
2359
+ for entry in entries:
2360
+ if not isinstance(entry, Mapping):
2361
+ continue
2362
+ rights = entry.get("rights")
2363
+ rights = rights if isinstance(rights, Mapping) else {}
2364
+ permissions = rights.get("permissions")
2365
+ cards.append(
2366
+ '<div class="mr-watch-sub">'
2367
+ f'<p class="mr-watch-sub-title">{esc(entry.get("source_id", ""))}</p>'
2368
+ + _definitions(
2369
+ [
2370
+ ("Where it came from", str(entry.get("origin", ""))),
2371
+ ("File", str(entry.get("relative_path", ""))),
2372
+ (
2373
+ "How it was fetched",
2374
+ _plain_from(_HOW_FETCHED, entry.get("acquisition_method")),
2375
+ ),
2376
+ ("Rights", _plain_from(_RIGHTS_STATUS, rights.get("status"))),
2377
+ (
2378
+ "You may",
2379
+ ", ".join(_plain_from(_PERMISSIONS, item) for item in permissions)
2380
+ if isinstance(permissions, list) and permissions
2381
+ else "nothing recorded",
2382
+ ),
2383
+ ("What backs that", str(rights.get("evidence", ""))),
2384
+ ("Size in bytes", str(entry.get("bytes", ""))),
2385
+ ("Fingerprint", str(entry.get("sha256", ""))),
2386
+ ]
2387
+ )
2388
+ + "</div>"
2389
+ )
2390
+ return (
2391
+ '<h1 class="mr-watch-card-title">The sources this build read</h1>'
2392
+ f'<p class="mr-watch-card-lead">{esc(f"{len(cards)} sources were read.")}</p>'
2393
+ + "".join(cards)
2394
+ )
2395
+
2396
+
2397
+ def _profile_table(profile: Mapping[str, Any]) -> str:
2398
+ columns = profile.get("columns")
2399
+ columns = columns if isinstance(columns, list) else []
2400
+ types = profile.get("types")
2401
+ types = types if isinstance(types, Mapping) else {}
2402
+ blanks = profile.get("null_counts")
2403
+ blanks = blanks if isinstance(blanks, Mapping) else {}
2404
+ return _table(
2405
+ ["Column", "Type", "Blanks"],
2406
+ [[name, types.get(name, ""), blanks.get(name, "")] for name in columns],
2407
+ )
2408
+
2409
+
2410
+ def _render_profile(payload: Mapping[str, Any]) -> str:
2411
+ return "".join(
2412
+ [
2413
+ '<h1 class="mr-watch-card-title">The dataset this build produced</h1>',
2414
+ f'<p class="mr-watch-card-lead">{esc(str(payload.get("row_count", 0)))} rows.</p>',
2415
+ _profile_table(payload),
2416
+ ]
2417
+ )
2418
+
2419
+
2420
+ def _render_source_profiles(payload: Mapping[str, Any]) -> str:
2421
+ parts = ['<h1 class="mr-watch-card-title">What each source looked like</h1>']
2422
+ for source_id in sorted(payload):
2423
+ shapes = payload[source_id]
2424
+ if not isinstance(shapes, Mapping):
2425
+ continue
2426
+ parts.append(
2427
+ f'<div class="mr-watch-sub"><p class="mr-watch-sub-title">{esc(source_id)}</p>'
2428
+ )
2429
+ for stage, label in (("raw", "As it was read"), ("cleaned", "After cleaning")):
2430
+ shape = shapes.get(stage)
2431
+ if isinstance(shape, Mapping):
2432
+ parts.append(f"<p><strong>{esc(label)}</strong></p>")
2433
+ parts.append(f'<p class="mr-watch-note">{esc(shape.get("row_count", 0))} rows</p>')
2434
+ parts.append(_profile_table(shape))
2435
+ parts.append("</div>")
2436
+ return "".join(parts)
2437
+
2438
+
2439
+ def _render_lineage(payload: Mapping[str, Any]) -> str:
2440
+ sentences = []
2441
+ for column in sorted(payload):
2442
+ entry = payload[column]
2443
+ if not isinstance(entry, Mapping):
2444
+ continue
2445
+ operations = entry.get("operations")
2446
+ applied = (
2447
+ ", ".join(_plain_operation(item) for item in operations)
2448
+ if isinstance(operations, list) and operations
2449
+ else "nothing"
2450
+ )
2451
+ sentences.append(
2452
+ f"{column} comes from {entry.get('source_column', '')} in "
2453
+ f"{entry.get('source_id', '')}; it was {applied}."
2454
+ )
2455
+ return (
2456
+ '<h1 class="mr-watch-card-title">Where every column came from</h1>'
2457
+ '<p class="mr-watch-card-lead">One line per column in the finished dataset.</p>'
2458
+ + _lines(sentences)
2459
+ )
2460
+
2461
+
2462
+ def _plan_summary(plan: Mapping[str, Any]) -> list[tuple[str, str]]:
2463
+ sources = plan.get("sources")
2464
+ sources = sources if isinstance(sources, list) else []
2465
+ join = plan.get("join")
2466
+ join = join if isinstance(join, Mapping) else {}
2467
+ quality = plan.get("quality")
2468
+ quality = quality if isinstance(quality, Mapping) else {}
2469
+ cleaning = plan.get("cleaning")
2470
+ cleaning = cleaning if isinstance(cleaning, list) else []
2471
+ return [
2472
+ (
2473
+ "Sources",
2474
+ ", ".join(
2475
+ str(source.get("id", "")) for source in sources if isinstance(source, Mapping)
2476
+ )
2477
+ or "none",
2478
+ ),
2479
+ ("One row per", _joined(plan.get("grain"))),
2480
+ ("Columns kept", _joined(plan.get("select"))),
2481
+ ("Cleaning", f"{len(cleaning)} steps"),
2482
+ (
2483
+ "Join",
2484
+ f"{join.get('left', '')} to {join.get('right', '')} on {_joined(join.get('on'))}, "
2485
+ f"{_plain_from(_JOIN_KIND, join.get('kind'))}, "
2486
+ f"{_plain_from(_MATCH_SHAPE, join.get('cardinality'))}",
2487
+ ),
2488
+ ("Meant for", _plain_from(_OUTPUT_INTENT, plan.get("output_intent", ""))),
2489
+ (
2490
+ "Checks",
2491
+ f"at least {quality.get('min_rows', 0)} rows; no blanks in "
2492
+ f"{_joined(quality.get('not_null'))}",
2493
+ ),
2494
+ ]
2495
+
2496
+
2497
+ def _cleaning_lines(plan: Mapping[str, Any]) -> list[str]:
2498
+ cleaning = plan.get("cleaning")
2499
+ cleaning = cleaning if isinstance(cleaning, list) else []
2500
+ lines: list[str] = []
2501
+ for step in cleaning:
2502
+ if not isinstance(step, Mapping):
2503
+ continue
2504
+ columns = step.get("columns")
2505
+ if isinstance(columns, Mapping):
2506
+ named = ", ".join(f"{key} as {value}" for key, value in columns.items())
2507
+ else:
2508
+ named = _joined(columns)
2509
+ lines.append(f"{_plain_operation(step.get('operation'))}: {named} in {step.get('source')}")
2510
+ return lines
2511
+
2512
+
2513
+ def _render_plan(payload: Mapping[str, Any]) -> str:
2514
+ return "".join(
2515
+ [
2516
+ '<h1 class="mr-watch-card-title">What this build was asked to do</h1>',
2517
+ f'<p class="mr-watch-card-lead">{esc(payload.get("question", ""))}</p>',
2518
+ _definitions(_plan_summary(payload)),
2519
+ "<p><strong>Cleaning, in order</strong></p>",
2520
+ _lines(_cleaning_lines(payload)),
2521
+ ]
2522
+ )
2523
+
2524
+
2525
+ def _render_generic(payload: Any) -> str:
2526
+ """The fallback for a file with no renderer of its own.
2527
+
2528
+ It renders only fields that HAVE a plain reading and drops the rest, because the alternative --
2529
+ dumping the field names the contract uses -- is the jargon leak this product refuses. A file
2530
+ whose every field is internal renders its size and fingerprint and nothing else, which is an
2531
+ honest thing for a page to say.
2532
+ """
2533
+
2534
+ if not isinstance(payload, Mapping):
2535
+ return '<p class="mr-watch-card-lead">This file was recorded for this build.</p>'
2536
+ pairs: list[tuple[str, str]] = []
2537
+ for key in payload:
2538
+ label = _plain_label(key)
2539
+ if label is None:
2540
+ continue
2541
+ value = payload[key]
2542
+ if isinstance(value, Mapping):
2543
+ rendered = f"{len(value)} entries"
2544
+ elif isinstance(value, list):
2545
+ rendered = (
2546
+ _joined(value)
2547
+ if all(not isinstance(v, (dict, list)) for v in value)
2548
+ else (f"{len(value)} entries")
2549
+ )
2550
+ else:
2551
+ rendered = str(value)
2552
+ pairs.append((label, rendered))
2553
+ return (
2554
+ '<p class="mr-watch-card-lead">This file was recorded for this build.</p>'
2555
+ + _definitions(pairs)
2556
+ )
2557
+
2558
+
2559
+ def _render_opaque(member: str, raw: bytes) -> str:
2560
+ """A data file's receipt is its size and its fingerprint, not its bytes.
2561
+
2562
+ Rendering the content of an arbitrary recorded file would put untrusted bytes of unknown
2563
+ encoding on a page for no gain: the thing that makes a data file trustworthy is that its
2564
+ fingerprint matches what the build recorded, and that is what this page states.
2565
+ """
2566
+
2567
+ return "".join(
2568
+ [
2569
+ '<h1 class="mr-watch-card-title">A data file from this build</h1>',
2570
+ '<p class="mr-watch-card-lead">This file holds data rather than a written record, '
2571
+ "so what is shown is what the build recorded about it.</p>",
2572
+ _definitions(
2573
+ [
2574
+ ("File", member),
2575
+ ("Size in bytes", str(len(raw))),
2576
+ ("Fingerprint", hashlib.sha256(raw).hexdigest()),
2577
+ ]
2578
+ ),
2579
+ ]
2580
+ )
2581
+
2582
+
2583
+ _RECEIPT_RENDERERS: Mapping[str, Callable[[Any], str]] = {
2584
+ "manifest.json": _render_run_receipt,
2585
+ "plan.json": _render_plan,
2586
+ "evidence/join.json": _render_join,
2587
+ "evidence/quality.json": _render_quality,
2588
+ "evidence/sources.json": _render_sources,
2589
+ "evidence/profile.json": _render_profile,
2590
+ "evidence/source_profiles.json": _render_source_profiles,
2591
+ "evidence/lineage.json": _render_lineage,
2592
+ }
2593
+
2594
+
2595
+ def render_receipt(member: str, raw: Any) -> str:
2596
+ """Render one verified receipt as a page a person can read.
2597
+
2598
+ ``raw`` is the member's bytes, already verified against the fingerprint the build recorded for
2599
+ it. A renderer that raises on a shape it did not expect degrades to the generic renderer; a
2600
+ malformed file must produce a plain page, never a traceback.
2601
+
2602
+ VERIFIED IS NOT WRITABLE. Everything that reaches the page here crosses ``_writable_payload``
2603
+ first -- the member name included, for the opaque path below -- because a member whose bytes
2604
+ match the digest the run recorded can still carry a lone surrogate that no surface can encode;
2605
+ see that function.
2606
+ """
2607
+
2608
+ member = events.writable_text(member)
2609
+ payload: Any
2610
+ if isinstance(raw, (bytes, bytearray)):
2611
+ try:
2612
+ payload = _writable_payload(json.loads(bytes(raw).decode("utf-8")))
2613
+ except (ValueError, UnicodeDecodeError, RecursionError):
2614
+ return _document("Receipt", _back() + _render_opaque(member, bytes(raw)))
2615
+ else:
2616
+ payload = _writable_payload(raw)
2617
+
2618
+ renderer = _RECEIPT_RENDERERS.get(member)
2619
+ body = ""
2620
+ if renderer is not None:
2621
+ try:
2622
+ body = renderer(payload)
2623
+ except Exception:
2624
+ body = ""
2625
+ if not body:
2626
+ body = _AUTHORED_RECEIPTS.get(member, _render_generic)(payload)
2627
+ return _document("Receipt", _back() + body)
2628
+
2629
+
2630
+ # ------------------------------------------------------------------------------------------------
2631
+ # Authored artifact cards
2632
+ # ------------------------------------------------------------------------------------------------
2633
+
2634
+
2635
+ def _card(title: str, lead: str, body: str) -> str:
2636
+ return (
2637
+ '<section class="mr-watch-card">'
2638
+ f'<h2 class="mr-watch-card-title">{esc(title)}</h2>'
2639
+ + (f'<p class="mr-watch-card-lead">{esc(lead)}</p>' if lead else "")
2640
+ + body
2641
+ + "</section>"
2642
+ )
2643
+
2644
+
2645
+ def _question_card(question: Any, requirements: Any = None) -> str:
2646
+ if not isinstance(question, Mapping):
2647
+ return ""
2648
+ criteria = []
2649
+ if isinstance(requirements, Mapping):
2650
+ listed = requirements.get("success_criteria")
2651
+ if isinstance(listed, list):
2652
+ criteria = [str(item) for item in listed]
2653
+ body = ("<p><strong>It is done when</strong></p>" + _lines(criteria)) if criteria else ""
2654
+ return _card("The question", str(question.get("text", "")), body)
2655
+
2656
+
2657
+ def _requirements_card(requirements: Any) -> str:
2658
+ if not isinstance(requirements, Mapping):
2659
+ return ""
2660
+ sentences: list[str] = []
2661
+ population = requirements.get("population")
2662
+ if population:
2663
+ sentences.append(str(population))
2664
+ window = requirements.get("time_range")
2665
+ if isinstance(window, Mapping):
2666
+ sentences.append(
2667
+ f"It covers {window.get('start_inclusive', '')} up to "
2668
+ f"{window.get('end_exclusive', '')}."
2669
+ )
2670
+ grain = requirements.get("output_grain")
2671
+ if isinstance(grain, Mapping) and grain.get("description"):
2672
+ sentences.append(str(grain["description"]))
2673
+ fields = requirements.get("required_fields")
2674
+ if isinstance(fields, list) and fields:
2675
+ sentences.append(f"Every row carries {_joined(fields)}.")
2676
+ feasibility = requirements.get("feasibility")
2677
+ if isinstance(feasibility, Mapping) and feasibility.get("narrative"):
2678
+ sentences.append(str(feasibility["narrative"]))
2679
+ if not sentences:
2680
+ return ""
2681
+ return _card("What it has to satisfy", "", _lines(sentences))
2682
+
2683
+
2684
+ def _proposals_card(proposals: Any) -> str:
2685
+ listed: Any = None
2686
+ if isinstance(proposals, Mapping):
2687
+ listed = proposals.get("source_proposals")
2688
+ elif isinstance(proposals, list):
2689
+ listed = proposals
2690
+ if not isinstance(listed, list) or not listed:
2691
+ return ""
2692
+ blocks: list[str] = []
2693
+ for proposal in listed:
2694
+ if not isinstance(proposal, Mapping):
2695
+ continue
2696
+ coverage = proposal.get("historical_coverage")
2697
+ evidence = proposal.get("evidence")
2698
+ argument = (
2699
+ "The agent chose this one."
2700
+ if proposal.get("fitness_decision") == "selected"
2701
+ else f"The agent's reading: {_humanize(proposal.get('fitness_decision'))}."
2702
+ )
2703
+ if isinstance(coverage, Mapping):
2704
+ argument += (
2705
+ f" It covers {coverage.get('start_inclusive', '')} up to "
2706
+ f"{coverage.get('end_exclusive', '')}."
2707
+ )
2708
+ if isinstance(evidence, list) and evidence:
2709
+ argument += f" {len(evidence)} pieces of backing were recorded."
2710
+ blocks.append(
2711
+ '<div class="mr-watch-sub">'
2712
+ f'<p class="mr-watch-sub-title">{esc(proposal.get("display_name", ""))}</p>'
2713
+ + _definitions(
2714
+ [
2715
+ ("What it is", _humanize(proposal.get("data_format", ""))),
2716
+ ("Where it came from", str(proposal.get("locator", ""))),
2717
+ ("Why it fits", argument),
2718
+ ("Rights", _plain_from(_RIGHTS_DECISION, proposal.get("rights_status"))),
2719
+ ]
2720
+ )
2721
+ + "</div>"
2722
+ )
2723
+ if not blocks:
2724
+ return ""
2725
+ return _card(
2726
+ "Sources the agent proposed",
2727
+ f"{len(blocks)} proposed.",
2728
+ "".join(blocks),
2729
+ )
2730
+
2731
+
2732
+ def _recipe_summary(recipe: Mapping[str, Any]) -> list[tuple[str, str]]:
2733
+ plan = recipe.get("transform_plan")
2734
+ plan = plan if isinstance(plan, Mapping) else {}
2735
+ validation = recipe.get("validation")
2736
+ validation = validation if isinstance(validation, Mapping) else {}
2737
+ pairs = _plan_summary(plan)
2738
+ required = validation.get("required_check_ids")
2739
+ if isinstance(required, list) and required:
2740
+ # Semicolons, not commas: every one of these is a sentence, and comma-joining sentences
2741
+ # produces a paragraph a reader cannot parse into its parts.
2742
+ pairs.append(
2743
+ (
2744
+ "Checks that must pass",
2745
+ "; ".join(_plain_check(item, "") for item in required),
2746
+ )
2747
+ )
2748
+ return pairs
2749
+
2750
+
2751
+ def _recipe_card(recipe: Any) -> str:
2752
+ if not isinstance(recipe, Mapping):
2753
+ return ""
2754
+ plan = recipe.get("transform_plan")
2755
+ plan = plan if isinstance(plan, Mapping) else {}
2756
+ return _card(
2757
+ "The Recipe",
2758
+ "These are the locked instructions this build followed. They do not change between runs.",
2759
+ _definitions(_recipe_summary(recipe))
2760
+ + "<p><strong>Cleaning, in order</strong></p>"
2761
+ + _lines(_cleaning_lines(plan)),
2762
+ )
2763
+
2764
+
2765
+ # The four authored artifacts render the SAME body whether they are reached as a card on the index
2766
+ # or as a receipt of their own, so a reader never meets two readings of one file.
2767
+ _AUTHORED_RECEIPTS: Mapping[str, Callable[[Any], str]] = {
2768
+ "question.json": _question_card,
2769
+ "requirements.json": _requirements_card,
2770
+ "source-proposals.json": _proposals_card,
2771
+ "recipe.json": _recipe_card,
2772
+ }
2773
+
2774
+ _CARD_MEMBERS = ("question.json", "requirements.json", "source-proposals.json", "recipe.json")
2775
+
2776
+
2777
+ def _writable_payload(value: Any) -> Any:
2778
+ """Normalize every string so the rendered payload is UTF-8 encodable."""
2779
+
2780
+ if isinstance(value, str):
2781
+ return events.writable_text(value)
2782
+ if isinstance(value, Mapping):
2783
+ return {_writable_payload(key): _writable_payload(item) for key, item in value.items()}
2784
+ if isinstance(value, list):
2785
+ return [_writable_payload(item) for item in value]
2786
+ return value
2787
+
2788
+
2789
+ def _member_payload(read_member: Callable[[str], bytes], member: str) -> Any:
2790
+ """Read and parse one member through the verified read, or return ``None``.
2791
+
2792
+ Verified is not writable: see ``_writable_payload`` for why a member that passed every check
2793
+ the run makes of it still cannot be put on a page unchanged.
2794
+ """
2795
+
2796
+ try:
2797
+ raw = read_member(member)
2798
+ except Exception:
2799
+ return None
2800
+ try:
2801
+ return _writable_payload(json.loads(raw.decode("utf-8")))
2802
+ except (ValueError, UnicodeDecodeError, RecursionError):
2803
+ return None
2804
+
2805
+
2806
+ def render_cards(members_present: frozenset[str], read_member: Callable[[str], bytes]) -> str:
2807
+ """Render each verified authored artifact that is currently present."""
2808
+
2809
+ loaded: dict[str, Any] = {}
2810
+ for member in _CARD_MEMBERS:
2811
+ if member in members_present:
2812
+ payload = _member_payload(read_member, member)
2813
+ if payload is not None:
2814
+ loaded[member] = payload
2815
+ if not loaded:
2816
+ return ""
2817
+
2818
+ cards: list[str] = []
2819
+ if "question.json" in loaded:
2820
+ cards.append(
2821
+ _safe(lambda: _question_card(loaded["question.json"], loaded.get("requirements.json")))
2822
+ )
2823
+ if "requirements.json" in loaded:
2824
+ cards.append(_safe(lambda: _requirements_card(loaded["requirements.json"])))
2825
+ if "source-proposals.json" in loaded:
2826
+ cards.append(_safe(lambda: _proposals_card(loaded["source-proposals.json"])))
2827
+ if "recipe.json" in loaded:
2828
+ cards.append(_safe(lambda: _recipe_card(loaded["recipe.json"])))
2829
+ rendered = "".join(card for card in cards if card)
2830
+ return f'<section class="mr-watch-cards">{rendered}</section>' if rendered else ""
2831
+
2832
+
2833
+ # ------------------------------------------------------------------------------------------------
2834
+ # The approval gate -- display only
2835
+ # ------------------------------------------------------------------------------------------------
2836
+ #
2837
+ # The displayed command uses the explicit recipe approval interface. Tests bind the command and
2838
+ # option names to the CLI parser.
2839
+ APPROVE_COMMAND_NAME = "recipe-approve"
2840
+ APPROVE_COMMAND_TEMPLATE = (
2841
+ "mr-data recipe-approve"
2842
+ " --recipe {recipe}"
2843
+ " --approval-id APPROVAL_ID"
2844
+ " --approved-by YOUR_NAME"
2845
+ " --approved-at TIMESTAMP"
2846
+ " --approval-policy-digest APPROVAL_RULES_FINGERPRINT"
2847
+ " --output {output}"
2848
+ )
2849
+
2850
+ _APPROVE_COMMAND_ID = "mr-watch-approve-command"
2851
+
2852
+
2853
+ def approve_command(run_dir: Path) -> str:
2854
+ """Return the approval command with normalized recipe and output paths."""
2855
+
2856
+ return APPROVE_COMMAND_TEMPLATE.format(
2857
+ recipe=_writable_path_text(run_dir / "candidate" / "recipe.json"),
2858
+ output=_writable_path_text(run_dir / "recipe-approval.json"),
2859
+ )
2860
+
2861
+
2862
+ # Relationship between an approval record and its recipe digest.
2863
+ _BINDS = "binds"
2864
+ _DISAGREES = "disagrees"
2865
+ _CLAIMS_NOTHING = "claims-nothing"
2866
+
2867
+
2868
+ def _recipe_digest(recipe: Mapping[str, Any]) -> str:
2869
+ """The Recipe's own canonical digest, computed exactly as ``RecipeApproval`` computes it."""
2870
+
2871
+ try:
2872
+ # Deferred: `watch` must stay importable without pulling `canonical` in at module load.
2873
+ from mostlyright.data_harness.canonical import canonical_json_bytes, sha256_bytes
2874
+
2875
+ return sha256_bytes(canonical_json_bytes(dict(recipe)))
2876
+ except Exception:
2877
+ return ""
2878
+
2879
+
2880
+ def _approval_record(
2881
+ read_member: Callable[[str], bytes], members_present: frozenset[str]
2882
+ ) -> Mapping[str, Any] | None:
2883
+ """The run's approval record, read once through the verified read the rest of the page uses."""
2884
+
2885
+ if "recipe-approval.json" not in members_present:
2886
+ return None
2887
+ record = _member_payload(read_member, "recipe-approval.json")
2888
+ return record if isinstance(record, Mapping) else None
2889
+
2890
+
2891
+ def _approval_binding(record: Mapping[str, Any] | None, computed: str) -> str:
2892
+ """Whether ``record`` binds an approval to the Recipe whose derived digest is ``computed``.
2893
+
2894
+ Through ``_matches_digest``, so a claim that is not a digest at all gets the same answer as a
2895
+ claim that is the wrong digest. ``computed`` is "" when the Recipe yields no canonical digest,
2896
+ and the comparator refuses that operand too: nothing can be proven to bind a Recipe whose own
2897
+ fingerprint cannot be derived.
2898
+ """
2899
+
2900
+ claimed = None if record is None else record.get("recipe_digest")
2901
+ if not isinstance(claimed, str) or not claimed:
2902
+ return _CLAIMS_NOTHING
2903
+ return _BINDS if _matches_digest(computed, claimed) else _DISAGREES
2904
+
2905
+
2906
+ def _recipe_fingerprint(
2907
+ read_member: Callable[[str], bytes],
2908
+ members_present: frozenset[str],
2909
+ recipe: Mapping[str, Any],
2910
+ ) -> str:
2911
+ """Return the canonical recipe digest unless the approval records a different digest."""
2912
+
2913
+ return _fingerprint_line(recipe, _approval_record(read_member, members_present))
2914
+
2915
+
2916
+ def _fingerprint_line(recipe: Mapping[str, Any], record: Mapping[str, Any] | None) -> str:
2917
+ """The value ``_recipe_fingerprint`` returns, from a record the caller has already read."""
2918
+
2919
+ computed = _recipe_digest(recipe)
2920
+ if not computed:
2921
+ return ""
2922
+ return "" if _approval_binding(record, computed) == _DISAGREES else computed
2923
+
2924
+
2925
+ def render_approval(
2926
+ run_dir: Path,
2927
+ members_present: frozenset[str],
2928
+ read_member: Callable[[str], bytes],
2929
+ ) -> str:
2930
+ """Render the verified recipe and its approval state, or return an empty string.
2931
+
2932
+ Approved output requires an approval digest that matches the canonical recipe digest. A
2933
+ mismatch is reported without an approval command. Approval itself remains a terminal action.
2934
+ """
2935
+
2936
+ if "recipe.json" not in members_present:
2937
+ return ""
2938
+ recipe = _member_payload(read_member, "recipe.json")
2939
+ if not isinstance(recipe, Mapping):
2940
+ return ""
2941
+
2942
+ # One binding result controls both the displayed digest and approval state.
2943
+ record = _approval_record(read_member, members_present)
2944
+ binding = _approval_binding(record, _recipe_digest(recipe))
2945
+
2946
+ pairs = _recipe_summary(recipe)
2947
+ fingerprint = _fingerprint_line(recipe, record)
2948
+ if fingerprint:
2949
+ pairs.append(("Fingerprint of the Recipe", fingerprint))
2950
+ receipt = _member_payload(read_member, _MANIFEST_NAME)
2951
+ if isinstance(receipt, Mapping) and receipt.get("candidate_digest"):
2952
+ # Kept, under a label that says whose fingerprint it is. It identifies the build, not the
2953
+ # Recipe, and a reader who quotes one for the other has quoted the wrong thing.
2954
+ pairs.append(("Fingerprint of this build", str(receipt["candidate_digest"])))
2955
+
2956
+ approval = (
2957
+ record if binding == _BINDS and record is not None and record.get("approved_by") else None
2958
+ )
2959
+
2960
+ if approval is not None:
2961
+ return (
2962
+ '<section class="mr-watch-approve">'
2963
+ '<h2 class="mr-watch-card-title">What this build was built from</h2>'
2964
+ '<p class="mr-watch-card-lead">This Recipe was approved before the build ran, and the '
2965
+ "build followed it exactly. Nothing on this page changes that.</p>"
2966
+ + _definitions(pairs)
2967
+ + f'<p class="mr-watch-note">Approved by {esc(approval.get("approved_by", ""))} '
2968
+ f"on {esc(approval.get('approved_at', ''))}.</p>" + "</section>"
2969
+ )
2970
+
2971
+ if binding == _DISAGREES:
2972
+ return (
2973
+ '<section class="mr-watch-approve">'
2974
+ '<h2 class="mr-watch-card-title">Approval does not match this Recipe</h2>'
2975
+ '<p class="mr-watch-card-lead">The approval on disk binds a different Recipe. '
2976
+ "This build cannot present it as approval for the Recipe shown here.</p>"
2977
+ + _definitions(pairs)
2978
+ + '<p class="mr-watch-note">No approval command is shown while these records '
2979
+ "disagree. Check the Recipe and approval receipts before taking any action.</p>"
2980
+ "</section>"
2981
+ )
2982
+
2983
+ command = approve_command(run_dir)
2984
+ return (
2985
+ '<section class="mr-watch-approve">'
2986
+ '<h2 class="mr-watch-card-title">What you are about to freeze</h2>'
2987
+ '<p class="mr-watch-card-lead">Freezing turns these instructions into a Recipe that runs '
2988
+ "the same way every time. Read what is below before you do it.</p>"
2989
+ + _definitions(pairs)
2990
+ + '<p class="mr-watch-note">Approving happens in your terminal. This page only shows you '
2991
+ "what you would be approving. Copy the line below and run it yourself.</p>"
2992
+ + '<p class="mr-watch-note">Four values in the line are yours to supply, and they are '
2993
+ "written in capitals: a name for this approval, your own name, the time you approved, and "
2994
+ "the fingerprint of the approval rules you followed.</p>"
2995
+ + f'<code class="mr-watch-command" id="{_APPROVE_COMMAND_ID}">{esc(command)}</code>'
2996
+ + f'<button type="button" class="mr-watch-copy" data-copy-target="{_APPROVE_COMMAND_ID}">'
2997
+ "Copy</button>"
2998
+ "</section>"
2999
+ )