mostlyright-data 0.9.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (314) hide show
  1. mostlyright/data_harness/__init__.py +158 -0
  2. mostlyright/data_harness/acquisition/__init__.py +55 -0
  3. mostlyright/data_harness/acquisition/http.py +2773 -0
  4. mostlyright/data_harness/acquisition/parsing.py +809 -0
  5. mostlyright/data_harness/acquisition/ranges.py +495 -0
  6. mostlyright/data_harness/acquisition/result_download.py +360 -0
  7. mostlyright/data_harness/acquisition/retention_admission.py +248 -0
  8. mostlyright/data_harness/acquisition/sandbox.py +4888 -0
  9. mostlyright/data_harness/acquisition/url_policy.py +530 -0
  10. mostlyright/data_harness/agent_runtime.py +2743 -0
  11. mostlyright/data_harness/assets/logo-ink.svg +31 -0
  12. mostlyright/data_harness/backends/__init__.py +28 -0
  13. mostlyright/data_harness/backends/pandas_backend.py +350 -0
  14. mostlyright/data_harness/backends/polars_backend.py +366 -0
  15. mostlyright/data_harness/backends/protocol.py +124 -0
  16. mostlyright/data_harness/backends/reference.py +83 -0
  17. mostlyright/data_harness/backends/registry.py +55 -0
  18. mostlyright/data_harness/backends/restrictions.py +126 -0
  19. mostlyright/data_harness/canonical.py +333 -0
  20. mostlyright/data_harness/catalog_job.py +625 -0
  21. mostlyright/data_harness/cli.py +5398 -0
  22. mostlyright/data_harness/contracts.py +53 -0
  23. mostlyright/data_harness/coordinator.py +1307 -0
  24. mostlyright/data_harness/deploy.py +924 -0
  25. mostlyright/data_harness/deploy_target.py +312 -0
  26. mostlyright/data_harness/deployment_evidence.py +1067 -0
  27. mostlyright/data_harness/event_presentation.py +576 -0
  28. mostlyright/data_harness/events.py +2152 -0
  29. mostlyright/data_harness/fast_delimited.py +239 -0
  30. mostlyright/data_harness/fleet.py +237 -0
  31. mostlyright/data_harness/formats.py +236 -0
  32. mostlyright/data_harness/governors.py +1163 -0
  33. mostlyright/data_harness/hosted_bootstrap.py +972 -0
  34. mostlyright/data_harness/hosted_crawler.py +1115 -0
  35. mostlyright/data_harness/hosted_crawler_container_smoke.py +351 -0
  36. mostlyright/data_harness/hosted_crawler_fetch.py +423 -0
  37. mostlyright/data_harness/hosted_crawler_job.py +1277 -0
  38. mostlyright/data_harness/hosted_crawler_protocol.py +676 -0
  39. mostlyright/data_harness/hosted_dataset.py +1500 -0
  40. mostlyright/data_harness/hosted_deploy.py +3037 -0
  41. mostlyright/data_harness/hosted_handoff.py +62 -0
  42. mostlyright/data_harness/hosted_ingestion_contract.py +504 -0
  43. mostlyright/data_harness/hosted_ingestion_job.py +356 -0
  44. mostlyright/data_harness/hosted_ingestion_job_smoke.py +40 -0
  45. mostlyright/data_harness/hosted_session_container_smoke.py +194 -0
  46. mostlyright/data_harness/hosted_session_worker.py +3554 -0
  47. mostlyright/data_harness/hosted_session_worker_job_smoke.py +46 -0
  48. mostlyright/data_harness/hosted_worker.py +6784 -0
  49. mostlyright/data_harness/ingestion/__init__.py +56 -0
  50. mostlyright/data_harness/ingestion/contracts.py +461 -0
  51. mostlyright/data_harness/ingestion/faults.py +42 -0
  52. mostlyright/data_harness/ingestion/gcs_store.py +1162 -0
  53. mostlyright/data_harness/ingestion/spool.py +130 -0
  54. mostlyright/data_harness/ingestion/store.py +885 -0
  55. mostlyright/data_harness/key_seam.py +434 -0
  56. mostlyright/data_harness/linux_process_boundary.py +262 -0
  57. mostlyright/data_harness/local_contracts.py +2880 -0
  58. mostlyright/data_harness/local_search/__init__.py +5 -0
  59. mostlyright/data_harness/local_search/build_index.py +1087 -0
  60. mostlyright/data_harness/local_search/contracts.py +920 -0
  61. mostlyright/data_harness/local_search/query_trace.py +266 -0
  62. mostlyright/data_harness/local_search/retrieval.py +700 -0
  63. mostlyright/data_harness/local_search/sealed.py +474 -0
  64. mostlyright/data_harness/local_search/service.py +784 -0
  65. mostlyright/data_harness/nbrender/CONTRACT.md +212 -0
  66. mostlyright/data_harness/nbrender/__init__.py +12 -0
  67. mostlyright/data_harness/nbrender/chrome.py +359 -0
  68. mostlyright/data_harness/nbrender/code_body.py +266 -0
  69. mostlyright/data_harness/nbrender/document.py +407 -0
  70. mostlyright/data_harness/nbrender/frame.py +275 -0
  71. mostlyright/data_harness/nbrender/interactive.py +337 -0
  72. mostlyright/data_harness/nbrender/markdown_body.py +477 -0
  73. mostlyright/data_harness/nbrender/mr_components.py +134 -0
  74. mostlyright/data_harness/nbrender/outputs_data.py +595 -0
  75. mostlyright/data_harness/nbrender/outputs_rich.py +906 -0
  76. mostlyright/data_harness/nbrender/outputs_source.py +260 -0
  77. mostlyright/data_harness/nbrender/outputs_stage.py +176 -0
  78. mostlyright/data_harness/nbrender/outputs_text.py +400 -0
  79. mostlyright/data_harness/nbrender/parse.py +394 -0
  80. mostlyright/data_harness/nbrender/status.py +40 -0
  81. mostlyright/data_harness/nbrender/tokens.py +1295 -0
  82. mostlyright/data_harness/notebook.py +1710 -0
  83. mostlyright/data_harness/offline.py +2049 -0
  84. mostlyright/data_harness/operation_registry.py +1007 -0
  85. mostlyright/data_harness/operator_setup.py +239 -0
  86. mostlyright/data_harness/pipeline.py +6428 -0
  87. mostlyright/data_harness/plan_graph.py +2026 -0
  88. mostlyright/data_harness/preparation/__init__.py +104 -0
  89. mostlyright/data_harness/preparation/contracts.py +1017 -0
  90. mostlyright/data_harness/preparation/engine.py +221 -0
  91. mostlyright/data_harness/preparation/errors.py +14 -0
  92. mostlyright/data_harness/preparation/gates.py +751 -0
  93. mostlyright/data_harness/preparation/joins.py +574 -0
  94. mostlyright/data_harness/preparation/profile.py +384 -0
  95. mostlyright/data_harness/preparation/table.py +217 -0
  96. mostlyright/data_harness/preparation/transforms.py +568 -0
  97. mostlyright/data_harness/progress_events.py +534 -0
  98. mostlyright/data_harness/readers/__init__.py +46 -0
  99. mostlyright/data_harness/readers/containers.py +963 -0
  100. mostlyright/data_harness/readers/contracts.py +542 -0
  101. mostlyright/data_harness/readers/delimited.py +257 -0
  102. mostlyright/data_harness/readers/grib2/__init__.py +33 -0
  103. mostlyright/data_harness/readers/grib2/admission.py +722 -0
  104. mostlyright/data_harness/readers/grib2/decode.py +1009 -0
  105. mostlyright/data_harness/readers/grib2/geometry.py +1133 -0
  106. mostlyright/data_harness/readers/grib2/portable_math.py +501 -0
  107. mostlyright/data_harness/readers/json_tabular.py +485 -0
  108. mostlyright/data_harness/readers/registry.py +514 -0
  109. mostlyright/data_harness/readers/samples/README.md +110 -0
  110. mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/cities.csv.gz +0 -0
  111. mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/expected.json +24 -0
  112. mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/cities.csv.gz +0 -0
  113. mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/expected.json +24 -0
  114. mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/cities.tar +0 -0
  115. mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/expected.json +24 -0
  116. mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/cities.tar +0 -0
  117. mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/expected.json +24 -0
  118. mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/cities.zip +0 -0
  119. mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/expected.json +25 -0
  120. mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
  121. mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/expected.json +25 -0
  122. mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
  123. mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/expected.json +25 -0
  124. mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/cities.csv +3 -0
  125. mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/expected.json +23 -0
  126. mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/cities.tsv +5 -0
  127. mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/expected.json +25 -0
  128. mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/expected.json +30 -0
  129. mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/observations.csv +5 -0
  130. mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/expected.json +44 -0
  131. mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/stations.json +1 -0
  132. mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/expected.json +48 -0
  133. mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/observations.ndjson +4 -0
  134. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/cities.xlsx +0 -0
  135. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/expected.json +24 -0
  136. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/expected.json +27 -0
  137. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
  138. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/expected.json +27 -0
  139. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
  140. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/README.md +20 -0
  141. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/expected.json +55 -0
  142. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/gfs-2m-temperature.grib2 +0 -0
  143. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/expected.json +54 -0
  144. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
  145. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/expected.json +54 -0
  146. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/hrrr-categorical-rain.grib2 +0 -0
  147. mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/expected.json +54 -0
  148. mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
  149. mostlyright/data_harness/readers/samples.py +582 -0
  150. mostlyright/data_harness/readers/spreadsheet.py +803 -0
  151. mostlyright/data_harness/readers/tabular.py +510 -0
  152. mostlyright/data_harness/recipe.py +5321 -0
  153. mostlyright/data_harness/repair/__init__.py +78 -0
  154. mostlyright/data_harness/repair/adapters.py +274 -0
  155. mostlyright/data_harness/repair/contracts.py +872 -0
  156. mostlyright/data_harness/repair/coordinator.py +1099 -0
  157. mostlyright/data_harness/repair/errors.py +16 -0
  158. mostlyright/data_harness/review.py +2533 -0
  159. mostlyright/data_harness/rowset.py +283 -0
  160. mostlyright/data_harness/serving.py +1975 -0
  161. mostlyright/data_harness/serving_edge.py +590 -0
  162. mostlyright/data_harness/serving_http.py +1031 -0
  163. mostlyright/data_harness/session_probes.py +759 -0
  164. mostlyright/data_harness/signing.py +101 -0
  165. mostlyright/data_harness/source_discovery.py +898 -0
  166. mostlyright/data_harness/sources/__init__.py +209 -0
  167. mostlyright/data_harness/sources/_adapter_steps.py +213 -0
  168. mostlyright/data_harness/sources/adapters.py +1214 -0
  169. mostlyright/data_harness/sources/cadence.py +1428 -0
  170. mostlyright/data_harness/sources/cadence_emission.py +453 -0
  171. mostlyright/data_harness/sources/cadence_history.py +546 -0
  172. mostlyright/data_harness/sources/catalog/__init__.py +17 -0
  173. mostlyright/data_harness/sources/catalog/admission.py +477 -0
  174. mostlyright/data_harness/sources/catalog/authoring.py +1701 -0
  175. mostlyright/data_harness/sources/catalog/authoring_policy.py +701 -0
  176. mostlyright/data_harness/sources/catalog/authoring_shards.py +1217 -0
  177. mostlyright/data_harness/sources/catalog/bounded_io.py +231 -0
  178. mostlyright/data_harness/sources/catalog/channel.py +523 -0
  179. mostlyright/data_harness/sources/catalog/channel_client.py +296 -0
  180. mostlyright/data_harness/sources/catalog/contracts.py +825 -0
  181. mostlyright/data_harness/sources/catalog/coverage.py +137 -0
  182. mostlyright/data_harness/sources/catalog/delta.py +1340 -0
  183. mostlyright/data_harness/sources/catalog/embedding.py +532 -0
  184. mostlyright/data_harness/sources/catalog/entry_v2.py +1182 -0
  185. mostlyright/data_harness/sources/catalog/fill.py +3889 -0
  186. mostlyright/data_harness/sources/catalog/fill_partitions.py +459 -0
  187. mostlyright/data_harness/sources/catalog/fill_staging.py +1105 -0
  188. mostlyright/data_harness/sources/catalog/gating.py +374 -0
  189. mostlyright/data_harness/sources/catalog/generation_receipt.py +1607 -0
  190. mostlyright/data_harness/sources/catalog/harvest/__init__.py +7 -0
  191. mostlyright/data_harness/sources/catalog/harvest/ckan.py +384 -0
  192. mostlyright/data_harness/sources/catalog/harvest/datagov_v4.py +798 -0
  193. mostlyright/data_harness/sources/catalog/harvest/protocol.py +964 -0
  194. mostlyright/data_harness/sources/catalog/harvest/sdmx.py +445 -0
  195. mostlyright/data_harness/sources/catalog/harvest/stac.py +384 -0
  196. mostlyright/data_harness/sources/catalog/health.py +447 -0
  197. mostlyright/data_harness/sources/catalog/hosted_catalog.py +105 -0
  198. mostlyright/data_harness/sources/catalog/identity_history.py +1549 -0
  199. mostlyright/data_harness/sources/catalog/neural.py +1618 -0
  200. mostlyright/data_harness/sources/catalog/packed_catalog.py +2345 -0
  201. mostlyright/data_harness/sources/catalog/packed_retrieval.py +1517 -0
  202. mostlyright/data_harness/sources/catalog/packed_writer.py +2802 -0
  203. mostlyright/data_harness/sources/catalog/query_trace.py +1037 -0
  204. mostlyright/data_harness/sources/catalog/recommend.py +171 -0
  205. mostlyright/data_harness/sources/catalog/retrieval.py +230 -0
  206. mostlyright/data_harness/sources/catalog/retrieval_manifest.py +995 -0
  207. mostlyright/data_harness/sources/catalog/rights_decisions.py +254 -0
  208. mostlyright/data_harness/sources/catalog/sealed.py +560 -0
  209. mostlyright/data_harness/sources/catalog/search.py +230 -0
  210. mostlyright/data_harness/sources/catalog/streaming_delta.py +1097 -0
  211. mostlyright/data_harness/sources/catalog/update.py +891 -0
  212. mostlyright/data_harness/sources/collections.py +815 -0
  213. mostlyright/data_harness/sources/contracts.py +2223 -0
  214. mostlyright/data_harness/sources/deletion.py +761 -0
  215. mostlyright/data_harness/sources/fitness.py +162 -0
  216. mostlyright/data_harness/sources/governance.py +163 -0
  217. mostlyright/data_harness/sources/hosted.py +173 -0
  218. mostlyright/data_harness/sources/integration.py +218 -0
  219. mostlyright/data_harness/sources/range_reader.py +418 -0
  220. mostlyright/data_harness/sources/registry.py +514 -0
  221. mostlyright/data_harness/sources/rights_rule.py +59 -0
  222. mostlyright/data_harness/sources/source_cadence_vectors.v1.json +1 -0
  223. mostlyright/data_harness/sources/sports.py +521 -0
  224. mostlyright/data_harness/sources/stream.py +524 -0
  225. mostlyright/data_harness/sources/stream_connector.py +418 -0
  226. mostlyright/data_harness/sources/stream_recorder.py +1404 -0
  227. mostlyright/data_harness/studio_boundary.py +2019 -0
  228. mostlyright/data_harness/thin/__init__.py +37 -0
  229. mostlyright/data_harness/thin/acquire.py +1137 -0
  230. mostlyright/data_harness/thin/acquire_cancel.py +579 -0
  231. mostlyright/data_harness/thin/approvals.py +617 -0
  232. mostlyright/data_harness/thin/commands.py +406 -0
  233. mostlyright/data_harness/thin/download.py +194 -0
  234. mostlyright/data_harness/thin/narrative.py +589 -0
  235. mostlyright/data_harness/thin/parity.py +1070 -0
  236. mostlyright/data_harness/thin/propose.py +2759 -0
  237. mostlyright/data_harness/thin/research.py +1663 -0
  238. mostlyright/data_harness/thin/router.py +924 -0
  239. mostlyright/data_harness/thin/runs.py +519 -0
  240. mostlyright/data_harness/thin/session.py +281 -0
  241. mostlyright/data_harness/thin/stream.py +501 -0
  242. mostlyright/data_harness/thin/transport.py +187 -0
  243. mostlyright/data_harness/thin/vocabulary.py +368 -0
  244. mostlyright/data_harness/thin/workers.py +164 -0
  245. mostlyright/data_harness/ucum/TABLE-PIN.json +40 -0
  246. mostlyright/data_harness/ucum/ucum-subset.v1.json +632 -0
  247. mostlyright/data_harness/unit_flow.py +927 -0
  248. mostlyright/data_harness/units.py +572 -0
  249. mostlyright/data_harness/ux/__init__.py +9 -0
  250. mostlyright/data_harness/ux/approve.py +485 -0
  251. mostlyright/data_harness/ux/author_yaml.py +597 -0
  252. mostlyright/data_harness/ux/cloud_auth.py +447 -0
  253. mostlyright/data_harness/ux/commands/__init__.py +260 -0
  254. mostlyright/data_harness/ux/commands/approve.py +136 -0
  255. mostlyright/data_harness/ux/commands/auth.py +744 -0
  256. mostlyright/data_harness/ux/commands/author.py +79 -0
  257. mostlyright/data_harness/ux/commands/catalog_author.py +403 -0
  258. mostlyright/data_harness/ux/commands/catalog_fill.py +523 -0
  259. mostlyright/data_harness/ux/commands/catalog_harvest.py +545 -0
  260. mostlyright/data_harness/ux/commands/catalog_publish.py +1838 -0
  261. mostlyright/data_harness/ux/commands/catalog_search.py +71 -0
  262. mostlyright/data_harness/ux/commands/catalog_update.py +437 -0
  263. mostlyright/data_harness/ux/commands/deploy.py +134 -0
  264. mostlyright/data_harness/ux/commands/deploy_dataset.py +98 -0
  265. mostlyright/data_harness/ux/commands/deploy_plan.py +105 -0
  266. mostlyright/data_harness/ux/commands/deploy_status.py +104 -0
  267. mostlyright/data_harness/ux/commands/diff.py +74 -0
  268. mostlyright/data_harness/ux/commands/index.py +84 -0
  269. mostlyright/data_harness/ux/commands/inventory.py +47 -0
  270. mostlyright/data_harness/ux/commands/list_builds.py +143 -0
  271. mostlyright/data_harness/ux/commands/login.py +63 -0
  272. mostlyright/data_harness/ux/commands/peek.py +236 -0
  273. mostlyright/data_harness/ux/commands/plan_check.py +90 -0
  274. mostlyright/data_harness/ux/commands/preflight.py +97 -0
  275. mostlyright/data_harness/ux/commands/record.py +107 -0
  276. mostlyright/data_harness/ux/commands/review_setup.py +47 -0
  277. mostlyright/data_harness/ux/commands/search.py +440 -0
  278. mostlyright/data_harness/ux/commands/show.py +61 -0
  279. mostlyright/data_harness/ux/commands/whoami.py +37 -0
  280. mostlyright/data_harness/ux/credential_native.py +551 -0
  281. mostlyright/data_harness/ux/credential_store.py +1055 -0
  282. mostlyright/data_harness/ux/credentials.py +631 -0
  283. mostlyright/data_harness/ux/diffing.py +444 -0
  284. mostlyright/data_harness/ux/headline.py +671 -0
  285. mostlyright/data_harness/ux/hosted_acquisition.py +974 -0
  286. mostlyright/data_harness/ux/hosted_run_status.py +619 -0
  287. mostlyright/data_harness/ux/inventory.py +427 -0
  288. mostlyright/data_harness/ux/local_review.py +375 -0
  289. mostlyright/data_harness/ux/login.py +691 -0
  290. mostlyright/data_harness/ux/path_kind.py +147 -0
  291. mostlyright/data_harness/ux/peek.py +1000 -0
  292. mostlyright/data_harness/ux/plain_file.py +178 -0
  293. mostlyright/data_harness/ux/plan_check.py +311 -0
  294. mostlyright/data_harness/ux/preflight.py +918 -0
  295. mostlyright/data_harness/ux/readers.py +1124 -0
  296. mostlyright/data_harness/ux/remediation.py +2195 -0
  297. mostlyright/data_harness/ux/render.py +657 -0
  298. mostlyright/data_harness/ux/workload.py +1077 -0
  299. mostlyright/data_harness/viewer.py +3713 -0
  300. mostlyright/data_harness/visual_run/__init__.py +83 -0
  301. mostlyright/data_harness/visual_run/authoring.py +235 -0
  302. mostlyright/data_harness/visual_run/contracts.py +673 -0
  303. mostlyright/data_harness/visual_run/materialize.py +486 -0
  304. mostlyright/data_harness/visual_run/observations.py +874 -0
  305. mostlyright/data_harness/visual_run/query.py +259 -0
  306. mostlyright/data_harness/visual_run/reducer.py +280 -0
  307. mostlyright/data_harness/visual_run/sdk.py +892 -0
  308. mostlyright/data_harness/visual_run/store.py +584 -0
  309. mostlyright/data_harness/visual_run/transport.py +239 -0
  310. mostlyright/data_harness/watch.py +2999 -0
  311. mostlyright_data-0.9.0.dist-info/METADATA +607 -0
  312. mostlyright_data-0.9.0.dist-info/RECORD +314 -0
  313. mostlyright_data-0.9.0.dist-info/WHEEL +4 -0
  314. mostlyright_data-0.9.0.dist-info/entry_points.txt +12 -0
@@ -0,0 +1,1607 @@
1
+ """The composite generation receipt, and the bounded reads every stage of it is built from.
2
+
3
+ One provider-scale generation is produced by three separate mutations -- harvest, author, publish
4
+ -- and the only durable evidence a downstream verifier may read is the composite receipt this module
5
+ composes. It is authoritative rather than descriptive: it carries the *exact bytes* of the harvest
6
+ and author receipts it was handed, together with their fingerprints, so nothing downstream has to
7
+ reconstruct a stage from counts somebody transcribed.
8
+
9
+ Every untrusted evidence document that reaches it is opened the same way. The parent chain is
10
+ walked with ``O_DIRECTORY|O_NOFOLLOW`` and retained, the final component is opened
11
+ ``O_NOFOLLOW``, the descriptor is confirmed to be a single-link regular file inside its byte cap,
12
+ the bytes are read through that one descriptor, and the descriptor's identity is confirmed
13
+ unchanged afterwards. A link, a swap during the read, a directory, a device, or one byte over the
14
+ cap is a refusal before anything is parsed. The sole narrower exception is a wheel-installed
15
+ distribution member: installers may hardlink an authenticated cache member into ``site-packages``,
16
+ so that reader permits a regular hardlink while retaining the same no-follow, bounded, stable
17
+ descriptor checks; the resulting complete distribution digest must still equal the exact wheel
18
+ coordinate.
19
+
20
+ The caps below are fixed. There is deliberately no argument anywhere on the three commands that
21
+ widens one: a caller who could raise the composite-receipt ceiling could publish evidence nobody
22
+ can read back inside the budget the reader states.
23
+ """
24
+
25
+ from __future__ import annotations
26
+
27
+ import ctypes
28
+ import errno
29
+ import fcntl
30
+ import hashlib
31
+ import importlib.metadata as metadata
32
+ import io
33
+ import os
34
+ import stat
35
+ import zipfile
36
+ from collections.abc import Callable, Iterator, Mapping, Sequence
37
+ from contextlib import contextmanager
38
+ from contextvars import ContextVar
39
+ from dataclasses import dataclass
40
+ from pathlib import Path
41
+ from typing import Any
42
+
43
+ from mostlyright.data_harness.canonical import (
44
+ CanonicalJSONError,
45
+ canonical_json_bytes,
46
+ canonical_sha256,
47
+ parse_canonical_json,
48
+ sha256_bytes,
49
+ )
50
+ from mostlyright.data_harness.sources.contracts import SourceContractError
51
+
52
+ #: The composite receipt this module composes, and the two stage receipts it binds.
53
+ GENERATION_RECEIPT_SCHEMA = "mr-data-catalog-generation.v1"
54
+ HARVEST_RECEIPT_SCHEMA = "mr-data-catalog-harvest.v1"
55
+ HARVEST_PREFLIGHT_SCHEMA = "mr-data-catalog-harvest-preflight.v1"
56
+
57
+ #: The committed tree one generation was produced from: wheel inputs, product source, tests,
58
+ #: verifier scripts, the Makefile and the lock. It deliberately names no narrative document.
59
+ EXECUTION_COORDINATE_SCHEMA = "mr-data-execution-coordinate.v1"
60
+
61
+ # --- the fixed caps, none of which any caller may widen ------------------------------------------
62
+
63
+ MAX_STAGE_RECEIPT_BYTES = 8_388_608
64
+ MAX_STAGE_RECEIPT_DEPTH = 16
65
+ MAX_STAGE_RECEIPT_MEMBERS = 512
66
+ MAX_STAGE_RECEIPT_ITEMS = 1024
67
+
68
+ MAX_EXECUTION_MANIFEST_BYTES = 16_777_216
69
+ MAX_EXECUTION_MANIFEST_DEPTH = 8
70
+ MAX_EXECUTION_MANIFEST_FILES = 20_000
71
+ MAX_EXECUTION_PATH_BYTES = 512
72
+ MAX_DISTRIBUTION_FILES = 10_000
73
+ MAX_DISTRIBUTION_MEMBER_BYTES = 33_554_432
74
+ MAX_DISTRIBUTION_TOTAL_BYTES = 33_554_432
75
+ MAX_DISTRIBUTION_COMPRESSED_BYTES = 67_108_864
76
+ # One expansion guard, shared with every other archive this package screens: an archive
77
+ # may not inflate past this floor unless it stays within this ratio of its packed size.
78
+ ARCHIVE_EXPANSION_FLOOR_BYTES = 4_194_304
79
+ ARCHIVE_EXPANSION_RATIO = 100
80
+ MAX_WHEEL_BYTES = 67_108_864
81
+
82
+ MAX_PREDECESSOR_HEAD_BYTES = 4_194_304
83
+ MAX_PREDECESSOR_HEAD_DEPTH = 16
84
+ MAX_PREDECESSOR_RANGE_DESCRIPTORS = 1024
85
+
86
+ MAX_RANGE_MANIFEST_BYTES = 65_536
87
+ MAX_RANGE_MANIFEST_MEMBERS = 24
88
+
89
+ MAX_COMPOSITE_RECEIPT_BYTES = 33_554_432
90
+ MAX_COMPOSITE_RECEIPT_DEPTH = 20
91
+ MAX_COMPOSITE_RANGE_DESCRIPTORS = 1024
92
+ MAX_COMPOSITE_ARRAY_ITEMS = 8192
93
+
94
+ _OPEN_MEMBER = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_CLOEXEC", 0)
95
+ _OPEN_DIRECTORY = (
96
+ os.O_RDONLY
97
+ | getattr(os, "O_DIRECTORY", 0)
98
+ | getattr(os, "O_NOFOLLOW", 0)
99
+ | getattr(os, "O_CLOEXEC", 0)
100
+ )
101
+ _OPEN_LOCK = os.O_RDWR | os.O_CREAT | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_CLOEXEC", 0)
102
+
103
+
104
+ @dataclass(frozen=True)
105
+ class ReceiptInstallation:
106
+ """The exact receipt parent retained for one locked mutation window."""
107
+
108
+ destination: Path
109
+ parent_descriptor: int
110
+ parent_identity: tuple[int, int]
111
+ name: str
112
+
113
+
114
+ _ACTIVE_RECEIPT_INSTALLATIONS: ContextVar[tuple[ReceiptInstallation, ...]] = ContextVar(
115
+ "active_receipt_installations", default=()
116
+ )
117
+
118
+
119
+ class GenerationReceiptRefused(SourceContractError):
120
+ """A stable refusal of a bounded input, an execution coordinate, or a composite receipt."""
121
+
122
+
123
+ def _absolute_path(path: Path) -> Path:
124
+ raw = os.fspath(path)
125
+ if not raw or "\x00" in raw:
126
+ raise OSError
127
+ return Path(os.path.abspath(raw))
128
+
129
+
130
+ def _open_directory_chain(path: Path, *, create: bool = False) -> list[int]:
131
+ """Open one absolute directory path without following any component."""
132
+
133
+ absolute = _absolute_path(path)
134
+ descriptors = [os.open("/", _OPEN_DIRECTORY)]
135
+ try:
136
+ for component in absolute.parts[1:]:
137
+ parent = descriptors[-1]
138
+ if create:
139
+ try:
140
+ os.mkdir(component, mode=0o700, dir_fd=parent)
141
+ except FileExistsError:
142
+ pass
143
+ descriptors.append(os.open(component, _OPEN_DIRECTORY, dir_fd=parent))
144
+ except BaseException:
145
+ for descriptor in reversed(descriptors):
146
+ os.close(descriptor)
147
+ raise
148
+ return descriptors
149
+
150
+
151
+ def _same_identity(left: os.stat_result, right: os.stat_result) -> bool:
152
+ return (left.st_dev, left.st_ino, left.st_size, left.st_mtime_ns, left.st_ctime_ns) == (
153
+ right.st_dev,
154
+ right.st_ino,
155
+ right.st_size,
156
+ right.st_mtime_ns,
157
+ right.st_ctime_ns,
158
+ )
159
+
160
+
161
+ @contextmanager
162
+ def publication_output_lock(
163
+ root: Path,
164
+ *,
165
+ label: str,
166
+ code_prefix: str,
167
+ refusal: type[SourceContractError] = GenerationReceiptRefused,
168
+ ) -> Iterator[int]:
169
+ """Retain and exclusively lock one private no-follow publication root."""
170
+
171
+ descriptors: list[int] = []
172
+ lock_descriptor: int | None = None
173
+ try:
174
+ absolute = _absolute_path(root)
175
+ if absolute == Path("/"):
176
+ raise OSError
177
+ descriptors = _open_directory_chain(absolute.parent, create=True)
178
+ parent_descriptor = descriptors[-1]
179
+ name = absolute.name
180
+ if name in {"", ".", ".."}:
181
+ raise OSError
182
+ # Lock a stable parent/name coordinate. Locking the root inode itself would let a rename
183
+ # plus replacement create a second lock domain at the same pathname.
184
+ lock_descriptor = os.open(
185
+ f".{name}.publication.lock", _OPEN_LOCK, 0o600, dir_fd=parent_descriptor
186
+ )
187
+ lock_info = os.fstat(lock_descriptor)
188
+ if not stat.S_ISREG(lock_info.st_mode) or lock_info.st_nlink != 1:
189
+ raise OSError
190
+ os.fchmod(lock_descriptor, 0o600)
191
+ fcntl.flock(lock_descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
192
+ try:
193
+ os.mkdir(name, mode=0o700, dir_fd=parent_descriptor)
194
+ except FileExistsError:
195
+ pass
196
+ root_descriptor = os.open(name, _OPEN_DIRECTORY, dir_fd=parent_descriptor)
197
+ descriptors.append(root_descriptor)
198
+ info = os.fstat(root_descriptor)
199
+ if not stat.S_ISDIR(info.st_mode):
200
+ raise OSError
201
+ os.fchmod(root_descriptor, 0o700)
202
+ except OSError as error:
203
+ if lock_descriptor is not None:
204
+ try:
205
+ fcntl.flock(lock_descriptor, fcntl.LOCK_UN)
206
+ except OSError:
207
+ pass
208
+ os.close(lock_descriptor)
209
+ for descriptor in reversed(descriptors):
210
+ os.close(descriptor)
211
+ code = (
212
+ f"{code_prefix}_OUTPUT_LOCKED"
213
+ if error.errno in {errno.EACCES, errno.EAGAIN}
214
+ else f"{code_prefix}_OUTPUT_PATH"
215
+ )
216
+ detail = (
217
+ "another invocation owns this output root"
218
+ if code.endswith("_LOCKED")
219
+ else "output root must be a retained no-follow directory path"
220
+ )
221
+ raise refusal(
222
+ code,
223
+ label,
224
+ detail,
225
+ ) from error
226
+ try:
227
+ require_output_root_identity(
228
+ absolute,
229
+ root_descriptor,
230
+ label=label,
231
+ code_prefix=code_prefix,
232
+ refusal=refusal,
233
+ )
234
+ yield root_descriptor
235
+ finally:
236
+ assert lock_descriptor is not None
237
+ fcntl.flock(lock_descriptor, fcntl.LOCK_UN)
238
+ os.close(lock_descriptor)
239
+ for descriptor in reversed(descriptors):
240
+ os.close(descriptor)
241
+
242
+
243
+ def require_output_root_identity(
244
+ root: Path,
245
+ descriptor: int,
246
+ *,
247
+ label: str,
248
+ code_prefix: str,
249
+ refusal: type[SourceContractError] = GenerationReceiptRefused,
250
+ ) -> None:
251
+ """Require the named root to remain the exact retained, no-follow directory."""
252
+
253
+ try:
254
+ named = os.stat(_absolute_path(root), follow_symlinks=False)
255
+ retained = os.fstat(descriptor)
256
+ except OSError as error:
257
+ raise refusal(
258
+ f"{code_prefix}_OUTPUT_PATH",
259
+ label,
260
+ "output root no longer names the retained directory",
261
+ ) from error
262
+ if (
263
+ not stat.S_ISDIR(named.st_mode)
264
+ or not stat.S_ISDIR(retained.st_mode)
265
+ or (named.st_dev, named.st_ino) != (retained.st_dev, retained.st_ino)
266
+ ):
267
+ raise refusal(
268
+ f"{code_prefix}_OUTPUT_PATH",
269
+ label,
270
+ "output root no longer names the retained directory",
271
+ )
272
+
273
+
274
+ @contextmanager
275
+ def retained_child_directory(
276
+ parent_descriptor: int,
277
+ name: str,
278
+ *,
279
+ create: bool,
280
+ label: str,
281
+ code_prefix: str,
282
+ refusal: type[SourceContractError] = GenerationReceiptRefused,
283
+ ) -> Iterator[int]:
284
+ """Open one private child relative to a retained parent, without reopening its pathname."""
285
+
286
+ descriptor = -1
287
+ try:
288
+ if not name or name in {".", ".."} or "/" in name or "\x00" in name:
289
+ raise OSError
290
+ if create:
291
+ try:
292
+ os.mkdir(name, mode=0o700, dir_fd=parent_descriptor)
293
+ except FileExistsError:
294
+ pass
295
+ descriptor = os.open(name, _OPEN_DIRECTORY, dir_fd=parent_descriptor)
296
+ info = os.fstat(descriptor)
297
+ if not stat.S_ISDIR(info.st_mode) or stat.S_IMODE(info.st_mode) & 0o077:
298
+ raise OSError
299
+ except OSError as error:
300
+ if descriptor >= 0:
301
+ os.close(descriptor)
302
+ raise refusal(
303
+ f"{code_prefix}_OUTPUT_PATH", label, "output child must be a private directory"
304
+ ) from error
305
+ try:
306
+ yield descriptor
307
+ finally:
308
+ os.close(descriptor)
309
+
310
+
311
+ @contextmanager
312
+ def retained_input_directory(
313
+ root: Path,
314
+ *,
315
+ label: str,
316
+ code_prefix: str,
317
+ refusal: type[SourceContractError] = GenerationReceiptRefused,
318
+ ) -> Iterator[int]:
319
+ """Retain one existing private input root without following any path component."""
320
+
321
+ descriptors: list[int] = []
322
+ try:
323
+ descriptors = _open_directory_chain(root)
324
+ descriptor = descriptors[-1]
325
+ info = os.fstat(descriptor)
326
+ if not stat.S_ISDIR(info.st_mode) or stat.S_IMODE(info.st_mode) & 0o077:
327
+ raise OSError
328
+ except OSError as error:
329
+ for opened in reversed(descriptors):
330
+ os.close(opened)
331
+ raise refusal(
332
+ f"{code_prefix}_INPUT_PATH",
333
+ label,
334
+ "input root must be an existing retained private no-follow directory",
335
+ ) from error
336
+ try:
337
+ yield descriptor
338
+ finally:
339
+ for opened in reversed(descriptors):
340
+ os.close(opened)
341
+
342
+
343
+ @contextmanager
344
+ def receipt_installation_lock(
345
+ path: Path,
346
+ *,
347
+ label: str,
348
+ code_prefix: str,
349
+ refusal: type[SourceContractError] = GenerationReceiptRefused,
350
+ ) -> Iterator[ReceiptInstallation]:
351
+ """Exclude another cooperative writer for the receipt's whole mutation window."""
352
+
353
+ descriptors: list[int] = []
354
+ try:
355
+ destination = _absolute_path(path)
356
+ descriptors = _open_directory_chain(destination.parent)
357
+ descriptor = descriptors[-1]
358
+ opened = os.fstat(descriptor)
359
+ if not stat.S_ISDIR(opened.st_mode):
360
+ raise OSError
361
+ name = destination.name
362
+ if name in {"", ".", ".."}:
363
+ raise OSError
364
+ fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
365
+ except OSError as error:
366
+ for opened_descriptor in reversed(descriptors):
367
+ os.close(opened_descriptor)
368
+ raise refusal(
369
+ f"{code_prefix}_RECEIPT_LOCKED",
370
+ label,
371
+ "another invocation owns this receipt destination",
372
+ ) from error
373
+ installation = ReceiptInstallation(
374
+ destination=destination,
375
+ parent_descriptor=descriptor,
376
+ parent_identity=(opened.st_dev, opened.st_ino),
377
+ name=name,
378
+ )
379
+ token = _ACTIVE_RECEIPT_INSTALLATIONS.set((*_ACTIVE_RECEIPT_INSTALLATIONS.get(), installation))
380
+ try:
381
+ _require_receipt_parent_identity(
382
+ installation, label=label, code_prefix=code_prefix, refusal=refusal
383
+ )
384
+ yield installation
385
+ finally:
386
+ _ACTIVE_RECEIPT_INSTALLATIONS.reset(token)
387
+ fcntl.flock(descriptor, fcntl.LOCK_UN)
388
+ for opened_descriptor in reversed(descriptors):
389
+ os.close(opened_descriptor)
390
+
391
+
392
+ def _require_receipt_parent_identity(
393
+ installation: ReceiptInstallation,
394
+ *,
395
+ label: str,
396
+ code_prefix: str,
397
+ refusal: type[SourceContractError],
398
+ ) -> None:
399
+ """Require the receipt pathname to retain the locked parent directory."""
400
+
401
+ try:
402
+ retained = os.fstat(installation.parent_descriptor)
403
+ # The named-parent lookup is deliberately last. Callers use this helper as their final
404
+ # filesystem operation before reporting success, so no retained-descriptor observation may
405
+ # follow and hide a pathname replacement that happened during it.
406
+ named = os.stat(installation.destination.parent, follow_symlinks=False)
407
+ except OSError as error:
408
+ raise refusal(
409
+ f"{code_prefix}_RECEIPT_DESTINATION",
410
+ label,
411
+ "receipt parent no longer names the retained locked directory",
412
+ ) from error
413
+ if (
414
+ not stat.S_ISDIR(named.st_mode)
415
+ or not stat.S_ISDIR(retained.st_mode)
416
+ or (named.st_dev, named.st_ino) != installation.parent_identity
417
+ or (retained.st_dev, retained.st_ino) != installation.parent_identity
418
+ ):
419
+ raise refusal(
420
+ f"{code_prefix}_RECEIPT_DESTINATION",
421
+ label,
422
+ "receipt parent no longer names the retained locked directory",
423
+ )
424
+
425
+
426
+ def is_digest(value: Any) -> bool:
427
+ """Whether this value is a lowercase SHA-256, which is the only fingerprint spelling here."""
428
+
429
+ return (
430
+ isinstance(value, str) and len(value) == 64 and all(c in "0123456789abcdef" for c in value)
431
+ )
432
+
433
+
434
+ def is_commit(value: Any) -> bool:
435
+ """Whether this value is a full lowercase 40-character commit identifier."""
436
+
437
+ return (
438
+ isinstance(value, str) and len(value) == 40 and all(c in "0123456789abcdef" for c in value)
439
+ )
440
+
441
+
442
+ # ------------------------------------------------------------------------------------------------
443
+ # Bounded reads
444
+ # ------------------------------------------------------------------------------------------------
445
+
446
+
447
+ def read_bounded_document(path: Path, *, maximum: int, label: str) -> bytes:
448
+ """Read one document through a retained no-follow parent chain and a stable descriptor.
449
+
450
+ The name is resolved component by component so that no directory in the chain may be a link
451
+ swapped underneath the read, and the file's own identity is compared before and after: a
452
+ document that moved while it was being read never becomes evidence.
453
+ """
454
+
455
+ raw_path = os.fspath(path)
456
+ if not raw_path or "\x00" in raw_path:
457
+ raise GenerationReceiptRefused("GENERATION_INPUT_PATH", label, "path is not a usable name")
458
+ absolute = Path(os.path.abspath(raw_path))
459
+ if absolute.name in {"", ".", ".."}:
460
+ raise GenerationReceiptRefused(
461
+ "GENERATION_INPUT_PATH", label, "path does not name one document"
462
+ )
463
+ owned: list[int] = []
464
+ descriptor = -1
465
+ try:
466
+ parent = os.open("/", _OPEN_DIRECTORY)
467
+ owned.append(parent)
468
+ for component in absolute.parts[1:-1]:
469
+ parent = os.open(component, _OPEN_DIRECTORY, dir_fd=parent)
470
+ owned.append(parent)
471
+ descriptor = os.open(absolute.name, _OPEN_MEMBER, dir_fd=parent)
472
+ except OSError as error:
473
+ for opened in reversed(owned):
474
+ os.close(opened)
475
+ raise GenerationReceiptRefused(
476
+ "GENERATION_INPUT_PATH",
477
+ label,
478
+ "must be reachable without following one link, from a retained parent",
479
+ ) from error
480
+ try:
481
+ before = os.fstat(descriptor)
482
+ if not stat.S_ISREG(before.st_mode) or before.st_nlink != 1:
483
+ raise GenerationReceiptRefused(
484
+ "GENERATION_INPUT_PATH", label, "must be a single-link regular file"
485
+ )
486
+ if before.st_size > maximum:
487
+ raise GenerationReceiptRefused(
488
+ "GENERATION_INPUT_LIMIT", label, f"must be at most {maximum} bytes"
489
+ )
490
+ chunks: list[bytes] = []
491
+ remaining = maximum + 1
492
+ while remaining > 0:
493
+ chunk = os.read(descriptor, min(remaining, 1 << 16))
494
+ if not chunk:
495
+ break
496
+ chunks.append(chunk)
497
+ remaining -= len(chunk)
498
+ after = os.fstat(descriptor)
499
+ if (before.st_dev, before.st_ino, before.st_size, before.st_mtime_ns) != (
500
+ after.st_dev,
501
+ after.st_ino,
502
+ after.st_size,
503
+ after.st_mtime_ns,
504
+ ):
505
+ raise GenerationReceiptRefused(
506
+ "GENERATION_INPUT_RACE", label, "document changed while it was being read"
507
+ )
508
+ finally:
509
+ os.close(descriptor)
510
+ for opened in reversed(owned):
511
+ os.close(opened)
512
+ raw = b"".join(chunks)
513
+ if len(raw) > maximum:
514
+ raise GenerationReceiptRefused(
515
+ "GENERATION_INPUT_LIMIT", label, f"must be at most {maximum} bytes"
516
+ )
517
+ return raw
518
+
519
+
520
+ def _read_installed_distribution_member(path: Path, *, maximum: int, label: str) -> bytes:
521
+ """Read one installer-owned member without rejecting a legitimate cache hardlink.
522
+
523
+ Unlike :func:`read_bounded_document`, this helper is used only for the closed
524
+ ``importlib.metadata`` distribution inventory that is subsequently compared with the exact
525
+ wheel coordinate. It deliberately permits ``st_nlink > 1`` because uv and other installers
526
+ may install wheel payloads by hardlink. It still refuses symlinks and special files, retains
527
+ every no-follow parent descriptor, enforces the byte cap, and requires the complete file
528
+ identity (including link count and ctime) and its current parent binding to remain stable
529
+ during the read.
530
+ """
531
+
532
+ raw_path = os.fspath(path)
533
+ if not raw_path or "\x00" in raw_path:
534
+ raise GenerationReceiptRefused("GENERATION_INPUT_PATH", label, "path is not a usable name")
535
+ absolute = Path(os.path.abspath(raw_path))
536
+ if absolute.name in {"", ".", ".."}:
537
+ raise GenerationReceiptRefused(
538
+ "GENERATION_INPUT_PATH", label, "path does not name one document"
539
+ )
540
+ owned: list[int] = []
541
+ descriptor = -1
542
+ try:
543
+ parent = os.open("/", _OPEN_DIRECTORY)
544
+ owned.append(parent)
545
+ for component in absolute.parts[1:-1]:
546
+ parent = os.open(component, _OPEN_DIRECTORY, dir_fd=parent)
547
+ owned.append(parent)
548
+ descriptor = os.open(
549
+ absolute.name,
550
+ _OPEN_MEMBER | getattr(os, "O_NONBLOCK", 0),
551
+ dir_fd=parent,
552
+ )
553
+ except OSError as error:
554
+ for opened in reversed(owned):
555
+ os.close(opened)
556
+ raise GenerationReceiptRefused(
557
+ "GENERATION_INPUT_PATH",
558
+ label,
559
+ "must be reachable without following one link, from a retained parent",
560
+ ) from error
561
+ try:
562
+ before = os.fstat(descriptor)
563
+ if not stat.S_ISREG(before.st_mode) or before.st_nlink < 1:
564
+ raise GenerationReceiptRefused(
565
+ "GENERATION_INPUT_PATH", label, "must be a regular installed file"
566
+ )
567
+ if before.st_size > maximum:
568
+ raise GenerationReceiptRefused(
569
+ "GENERATION_INPUT_LIMIT", label, f"must be at most {maximum} bytes"
570
+ )
571
+ chunks: list[bytes] = []
572
+ remaining = maximum + 1
573
+ while remaining > 0:
574
+ chunk = os.read(descriptor, min(remaining, 1 << 16))
575
+ if not chunk:
576
+ break
577
+ chunks.append(chunk)
578
+ remaining -= len(chunk)
579
+ after = os.fstat(descriptor)
580
+ before_identity = (
581
+ before.st_dev,
582
+ before.st_ino,
583
+ before.st_mode,
584
+ before.st_nlink,
585
+ before.st_size,
586
+ before.st_mtime_ns,
587
+ before.st_ctime_ns,
588
+ )
589
+ after_identity = (
590
+ after.st_dev,
591
+ after.st_ino,
592
+ after.st_mode,
593
+ after.st_nlink,
594
+ after.st_size,
595
+ after.st_mtime_ns,
596
+ after.st_ctime_ns,
597
+ )
598
+ raw = b"".join(chunks)
599
+ current_chain: list[int] = []
600
+ try:
601
+ current_chain = _open_directory_chain(absolute.parent)
602
+ current_parent = os.fstat(current_chain[-1])
603
+ captured_parent = os.fstat(owned[-1])
604
+ current_member = os.stat(
605
+ absolute.name,
606
+ dir_fd=current_chain[-1],
607
+ follow_symlinks=False,
608
+ )
609
+ except OSError as error:
610
+ raise GenerationReceiptRefused(
611
+ "GENERATION_INPUT_RACE", label, "installed file path changed while it was read"
612
+ ) from error
613
+ finally:
614
+ for opened in reversed(current_chain):
615
+ os.close(opened)
616
+ current_identity = (
617
+ current_member.st_dev,
618
+ current_member.st_ino,
619
+ current_member.st_mode,
620
+ current_member.st_nlink,
621
+ current_member.st_size,
622
+ current_member.st_mtime_ns,
623
+ current_member.st_ctime_ns,
624
+ )
625
+ if (
626
+ before_identity != after_identity
627
+ or current_identity != after_identity
628
+ or (current_parent.st_dev, current_parent.st_ino)
629
+ != (captured_parent.st_dev, captured_parent.st_ino)
630
+ or len(raw) != before.st_size
631
+ ):
632
+ raise GenerationReceiptRefused(
633
+ "GENERATION_INPUT_RACE", label, "installed file or path changed while it was read"
634
+ )
635
+ return raw
636
+ finally:
637
+ if descriptor >= 0:
638
+ os.close(descriptor)
639
+ for opened in reversed(owned):
640
+ os.close(opened)
641
+
642
+
643
+ def enforce_shape(
644
+ value: Any,
645
+ *,
646
+ label: str,
647
+ max_depth: int,
648
+ max_members: int,
649
+ max_items: int,
650
+ code: str = "GENERATION_INPUT_SHAPE",
651
+ ) -> None:
652
+ """Refuse a parsed document deeper, wider, or longer than its stated structural bounds."""
653
+
654
+ stack: list[tuple[Any, int]] = [(value, 1)]
655
+ while stack:
656
+ current, depth = stack.pop()
657
+ if depth > max_depth:
658
+ raise GenerationReceiptRefused(code, label, f"nesting exceeds {max_depth} levels")
659
+ if isinstance(current, Mapping):
660
+ if len(current) > max_members:
661
+ raise GenerationReceiptRefused(
662
+ code, label, f"an object exceeds {max_members} members"
663
+ )
664
+ stack.extend((member, depth + 1) for member in current.values())
665
+ elif isinstance(current, Sequence) and not isinstance(current, str | bytes | bytearray):
666
+ if len(current) > max_items:
667
+ raise GenerationReceiptRefused(code, label, f"an array exceeds {max_items} items")
668
+ stack.extend((member, depth + 1) for member in current)
669
+
670
+
671
+ def load_bounded_json(
672
+ path: Path,
673
+ *,
674
+ label: str,
675
+ maximum: int,
676
+ max_depth: int,
677
+ max_members: int,
678
+ max_items: int,
679
+ expected_sha256: str | None = None,
680
+ shape_code: str = "GENERATION_INPUT_SHAPE",
681
+ ) -> tuple[bytes, dict[str, Any]]:
682
+ """Read, fingerprint, parse, and shape-check one bounded canonical JSON document."""
683
+
684
+ if expected_sha256 is not None and not is_digest(expected_sha256):
685
+ raise GenerationReceiptRefused(
686
+ "GENERATION_INPUT_DIGEST", label, "must be a lowercase SHA-256"
687
+ )
688
+ raw = read_bounded_document(path, maximum=maximum, label=label)
689
+ if expected_sha256 is not None and sha256_bytes(raw) != expected_sha256:
690
+ raise GenerationReceiptRefused(
691
+ "GENERATION_INPUT_DIGEST", label, "document bytes differ from the required digest"
692
+ )
693
+ try:
694
+ parsed = parse_canonical_json(raw)
695
+ except CanonicalJSONError as error:
696
+ raise GenerationReceiptRefused(
697
+ shape_code, label, "document is not canonical JSON"
698
+ ) from error
699
+ if not isinstance(parsed, dict):
700
+ raise GenerationReceiptRefused(shape_code, label, "document must be one JSON object")
701
+ enforce_shape(
702
+ parsed,
703
+ label=label,
704
+ max_depth=max_depth,
705
+ max_members=max_members,
706
+ max_items=max_items,
707
+ code=shape_code,
708
+ )
709
+ return raw, parsed
710
+
711
+
712
+ def load_bounded_json_at(
713
+ parent_descriptor: int,
714
+ name: str,
715
+ *,
716
+ label: str,
717
+ maximum: int,
718
+ max_depth: int,
719
+ max_members: int,
720
+ max_items: int,
721
+ shape_code: str = "GENERATION_INPUT_SHAPE",
722
+ ) -> tuple[bytes, dict[str, Any]]:
723
+ """Read one bounded canonical JSON member relative to an already retained directory."""
724
+
725
+ try:
726
+ raw, _identity = _read_bounded_at(parent_descriptor, name, maximum=maximum)
727
+ except OSError as error:
728
+ raise GenerationReceiptRefused(
729
+ "GENERATION_INPUT_PATH", label, "member is not a bounded single-link regular file"
730
+ ) from error
731
+ try:
732
+ parsed = parse_canonical_json(raw)
733
+ except CanonicalJSONError as error:
734
+ raise GenerationReceiptRefused(
735
+ shape_code, label, "document is not canonical JSON"
736
+ ) from error
737
+ if not isinstance(parsed, dict):
738
+ raise GenerationReceiptRefused(shape_code, label, "document must be one JSON object")
739
+ enforce_shape(
740
+ parsed,
741
+ label=label,
742
+ max_depth=max_depth,
743
+ max_members=max_members,
744
+ max_items=max_items,
745
+ code=shape_code,
746
+ )
747
+ return raw, parsed
748
+
749
+
750
+ def load_stage_receipt(
751
+ path: Path,
752
+ expected_sha256: str,
753
+ *,
754
+ label: str,
755
+ schemas: frozenset[str],
756
+ ) -> tuple[bytes, dict[str, Any]]:
757
+ """One component receipt, read under the fixed stage caps and its caller-retained digest."""
758
+
759
+ raw, payload = load_bounded_json(
760
+ path,
761
+ label=label,
762
+ maximum=MAX_STAGE_RECEIPT_BYTES,
763
+ max_depth=MAX_STAGE_RECEIPT_DEPTH,
764
+ max_members=MAX_STAGE_RECEIPT_MEMBERS,
765
+ max_items=MAX_STAGE_RECEIPT_ITEMS,
766
+ expected_sha256=expected_sha256,
767
+ )
768
+ if payload.get("schema_version") not in schemas:
769
+ raise GenerationReceiptRefused(
770
+ "GENERATION_INPUT_SCHEMA", label, "receipt does not name one accepted stage schema"
771
+ )
772
+ return raw, payload
773
+
774
+
775
+ def load_execution_coordinate(path: Path, expected_commit: str) -> dict[str, Any]:
776
+ """The committed tree this generation was produced from, named by its own exact commit."""
777
+
778
+ if not is_commit(expected_commit):
779
+ raise GenerationReceiptRefused(
780
+ "GENERATION_EXECUTION_COMMIT",
781
+ "execution_commit",
782
+ "must be a full lowercase 40-character commit",
783
+ )
784
+ _raw, payload = load_bounded_json(
785
+ path,
786
+ label="execution_manifest",
787
+ maximum=MAX_EXECUTION_MANIFEST_BYTES,
788
+ max_depth=MAX_EXECUTION_MANIFEST_DEPTH,
789
+ max_members=MAX_EXECUTION_MANIFEST_FILES,
790
+ max_items=MAX_EXECUTION_MANIFEST_FILES,
791
+ shape_code="GENERATION_EXECUTION_SHAPE",
792
+ )
793
+ if payload.get("schema_version") != EXECUTION_COORDINATE_SCHEMA:
794
+ raise GenerationReceiptRefused(
795
+ "GENERATION_EXECUTION_SHAPE",
796
+ "execution_manifest",
797
+ "manifest does not name the execution coordinate schema",
798
+ )
799
+ files = payload.get("files")
800
+ if not isinstance(files, list) or not files or len(files) > MAX_EXECUTION_MANIFEST_FILES:
801
+ raise GenerationReceiptRefused(
802
+ "GENERATION_EXECUTION_SHAPE",
803
+ "execution_manifest.files",
804
+ f"must name 1 to {MAX_EXECUTION_MANIFEST_FILES} committed files",
805
+ )
806
+ seen: set[str] = set()
807
+ for index, item in enumerate(files):
808
+ member = f"execution_manifest.files[{index}]"
809
+ if not isinstance(item, dict) or set(item) != {"path", "sha256"}:
810
+ raise GenerationReceiptRefused(
811
+ "GENERATION_EXECUTION_SHAPE", member, "each file is one path and one fingerprint"
812
+ )
813
+ name = item["path"]
814
+ if (
815
+ not isinstance(name, str)
816
+ or not name
817
+ or len(name.encode("utf-8")) > MAX_EXECUTION_PATH_BYTES
818
+ or name in seen
819
+ ):
820
+ raise GenerationReceiptRefused(
821
+ "GENERATION_EXECUTION_SHAPE",
822
+ member,
823
+ f"each path is stated once and is at most {MAX_EXECUTION_PATH_BYTES} bytes",
824
+ )
825
+ seen.add(name)
826
+ if not is_digest(item["sha256"]):
827
+ raise GenerationReceiptRefused(
828
+ "GENERATION_EXECUTION_SHAPE", member, "each file states a lowercase SHA-256"
829
+ )
830
+ body = {key: value for key, value in payload.items() if key != "root_sha256"}
831
+ if payload.get("root_sha256") != canonical_sha256(body):
832
+ raise GenerationReceiptRefused(
833
+ "GENERATION_EXECUTION_SHAPE",
834
+ "execution_manifest.root_sha256",
835
+ "the manifest does not reproduce its own root fingerprint",
836
+ )
837
+ if payload.get("commit") != expected_commit:
838
+ raise GenerationReceiptRefused(
839
+ "GENERATION_EXECUTION_COMMIT",
840
+ "execution_manifest.commit",
841
+ "the manifest was written for another execution commit",
842
+ )
843
+ coordinate: dict[str, Any] = {
844
+ "commit": expected_commit,
845
+ "root_sha256": payload["root_sha256"],
846
+ "file_count": len(files),
847
+ }
848
+ installed = payload.get("installed_runtime")
849
+ if installed is not None:
850
+ if not isinstance(installed, dict) or set(installed) != {
851
+ "wheel_sha256",
852
+ "distribution_sha256",
853
+ "distribution_file_count",
854
+ }:
855
+ raise GenerationReceiptRefused(
856
+ "GENERATION_EXECUTION_SHAPE",
857
+ "execution_manifest.installed_runtime",
858
+ "installed runtime must be one exact wheel/distribution coordinate",
859
+ )
860
+ for member in ("wheel_sha256", "distribution_sha256"):
861
+ if not is_digest(installed.get(member)):
862
+ raise GenerationReceiptRefused(
863
+ "GENERATION_EXECUTION_SHAPE",
864
+ f"execution_manifest.installed_runtime.{member}",
865
+ "must be a lowercase SHA-256",
866
+ )
867
+ count = installed.get("distribution_file_count")
868
+ if type(count) is not int or count <= 0 or count > MAX_DISTRIBUTION_FILES:
869
+ raise GenerationReceiptRefused(
870
+ "GENERATION_EXECUTION_SHAPE",
871
+ "execution_manifest.installed_runtime.distribution_file_count",
872
+ "must be a positive bounded file count",
873
+ )
874
+ observed = installed_distribution_coordinate()
875
+ if observed != {
876
+ "distribution_sha256": installed["distribution_sha256"],
877
+ "distribution_file_count": count,
878
+ }:
879
+ raise GenerationReceiptRefused(
880
+ "GENERATION_EXECUTION_DISTRIBUTION",
881
+ "execution_manifest.installed_runtime",
882
+ "the executing installed distribution differs from the wheel coordinate",
883
+ )
884
+ coordinate["installed_runtime"] = dict(installed)
885
+ return coordinate
886
+
887
+
888
+ def _distribution_member(path: str, dist_info: str) -> bool:
889
+ if path.startswith("mostlyright/"):
890
+ return True
891
+ if not path.startswith(f"{dist_info}/"):
892
+ return False
893
+ relative = path[len(dist_info) + 1 :]
894
+ return relative not in {
895
+ "RECORD",
896
+ "INSTALLER",
897
+ "REQUESTED",
898
+ "direct_url.json",
899
+ "uv_cache.json",
900
+ }
901
+
902
+
903
+ def _distribution_coordinate(files: Sequence[Mapping[str, str]], version: str) -> dict[str, Any]:
904
+ ordered = sorted((dict(item) for item in files), key=lambda item: item["path"])
905
+ if not ordered or len(ordered) > MAX_DISTRIBUTION_FILES:
906
+ raise GenerationReceiptRefused(
907
+ "GENERATION_EXECUTION_DISTRIBUTION",
908
+ "installed_runtime.distribution",
909
+ "the installed distribution file inventory is absent or over its cap",
910
+ )
911
+ body = {"name": "mostlyright-data", "version": version, "files": ordered}
912
+ return {
913
+ "distribution_sha256": canonical_sha256(body),
914
+ "distribution_file_count": len(ordered),
915
+ }
916
+
917
+
918
+ def wheel_runtime_coordinate(
919
+ path: Path, *, expected_product_sha256s: Mapping[str, str] | None = None
920
+ ) -> dict[str, Any]:
921
+ """Fingerprint one wheel and the portable distribution bytes it must install."""
922
+
923
+ try:
924
+ wheel_raw = read_bounded_document(path, maximum=MAX_WHEEL_BYTES, label="installed_wheel")
925
+ with zipfile.ZipFile(io.BytesIO(wheel_raw)) as archive:
926
+ names = archive.namelist()
927
+ if len(names) != len(set(names)):
928
+ raise GenerationReceiptRefused(
929
+ "GENERATION_EXECUTION_DISTRIBUTION",
930
+ "installed_wheel",
931
+ "wheel carries duplicate member names",
932
+ )
933
+ records = [name for name in names if name.endswith(".dist-info/RECORD")]
934
+ if len(records) != 1 or len(names) > MAX_DISTRIBUTION_FILES:
935
+ raise GenerationReceiptRefused(
936
+ "GENERATION_EXECUTION_DISTRIBUTION",
937
+ "installed_wheel",
938
+ "wheel must carry one bounded distribution RECORD",
939
+ )
940
+ dist_info = records[0].split("/", 1)[0]
941
+ for info in archive.infolist():
942
+ name = info.filename
943
+ member_type = stat.S_IFMT(info.external_attr >> 16)
944
+ if (
945
+ "\\" in name
946
+ or name.startswith("/")
947
+ or "//" in name
948
+ or any(part in {".", ".."} for part in name.split("/"))
949
+ or not (name.startswith("mostlyright/") or name.startswith(f"{dist_info}/"))
950
+ or member_type not in {0, stat.S_IFREG, stat.S_IFDIR}
951
+ or bool(info.flag_bits & 0x1)
952
+ ):
953
+ raise GenerationReceiptRefused(
954
+ "GENERATION_EXECUTION_DISTRIBUTION",
955
+ "installed_wheel",
956
+ "wheel carries a payload outside its closed installed roots",
957
+ )
958
+ metadata_name = f"{dist_info}/METADATA"
959
+ version = ""
960
+ if metadata_name in names:
961
+ for line in archive.read(metadata_name).decode("utf-8").splitlines():
962
+ if line.startswith("Version: "):
963
+ version = line.removeprefix("Version: ").strip()
964
+ break
965
+ if not version:
966
+ raise GenerationReceiptRefused(
967
+ "GENERATION_EXECUTION_DISTRIBUTION",
968
+ "installed_wheel",
969
+ "wheel metadata states no distribution version",
970
+ )
971
+ selected_infos: list[zipfile.ZipInfo] = []
972
+ total_bytes = 0
973
+ total_compressed_bytes = 0
974
+ for name in names:
975
+ if not _distribution_member(name, dist_info):
976
+ continue
977
+ info = archive.getinfo(name)
978
+ if info.is_dir() or info.file_size > MAX_DISTRIBUTION_MEMBER_BYTES:
979
+ raise GenerationReceiptRefused(
980
+ "GENERATION_EXECUTION_DISTRIBUTION",
981
+ "installed_wheel",
982
+ "wheel distribution member is not a bounded regular payload",
983
+ )
984
+ total_bytes += info.file_size
985
+ total_compressed_bytes += info.compress_size
986
+ selected_infos.append(info)
987
+ if (
988
+ total_bytes > MAX_DISTRIBUTION_TOTAL_BYTES
989
+ or total_compressed_bytes > MAX_DISTRIBUTION_COMPRESSED_BYTES
990
+ or total_bytes
991
+ > max(
992
+ ARCHIVE_EXPANSION_FLOOR_BYTES,
993
+ total_compressed_bytes * ARCHIVE_EXPANSION_RATIO,
994
+ )
995
+ ):
996
+ raise GenerationReceiptRefused(
997
+ "GENERATION_EXECUTION_DISTRIBUTION",
998
+ "installed_wheel",
999
+ "wheel distribution exceeds its aggregate expansion budget",
1000
+ )
1001
+ selected = [
1002
+ {"path": info.filename, "sha256": sha256_bytes(archive.read(info))}
1003
+ for info in selected_infos
1004
+ ]
1005
+ if expected_product_sha256s is not None:
1006
+ actual_product_sha256s = {
1007
+ info.filename: sha256_bytes(archive.read(info))
1008
+ for info in selected_infos
1009
+ if info.filename.startswith("mostlyright/")
1010
+ }
1011
+ if not expected_product_sha256s or actual_product_sha256s != dict(
1012
+ expected_product_sha256s
1013
+ ):
1014
+ raise GenerationReceiptRefused(
1015
+ "GENERATION_EXECUTION_SOURCE",
1016
+ "installed_wheel",
1017
+ "wheel product bytes differ from the committed source coordinate",
1018
+ )
1019
+ except GenerationReceiptRefused:
1020
+ raise
1021
+ except (OSError, zipfile.BadZipFile, UnicodeError) as error:
1022
+ raise GenerationReceiptRefused(
1023
+ "GENERATION_EXECUTION_DISTRIBUTION",
1024
+ "installed_wheel",
1025
+ "wheel is not a readable installed distribution",
1026
+ ) from error
1027
+ return {
1028
+ "wheel_sha256": hashlib.sha256(wheel_raw).hexdigest(),
1029
+ **_distribution_coordinate(selected, version),
1030
+ }
1031
+
1032
+
1033
+ def installed_distribution_coordinate() -> dict[str, Any]:
1034
+ """Fingerprint the wheel-owned bytes and refuse an editable/source-tree import."""
1035
+
1036
+ try:
1037
+ distribution = metadata.distribution("mostlyright-data")
1038
+ root = Path(distribution.locate_file("")).resolve(strict=True)
1039
+ origin = Path(__file__).resolve(strict=True)
1040
+ origin.relative_to(root)
1041
+ if not ({"site-packages", "dist-packages"} & set(root.parts)):
1042
+ raise ValueError
1043
+ paths = [item.as_posix() for item in distribution.files or ()]
1044
+ records = [path for path in paths if path.endswith(".dist-info/RECORD")]
1045
+ if len(records) != 1:
1046
+ raise ValueError
1047
+ dist_info = records[0].split("/", 1)[0]
1048
+ selected: list[dict[str, str]] = []
1049
+ total_bytes = 0
1050
+ for name in paths:
1051
+ if not _distribution_member(name, dist_info):
1052
+ continue
1053
+ member = Path(distribution.locate_file(name))
1054
+ try:
1055
+ member.resolve(strict=True).relative_to(root)
1056
+ except (OSError, ValueError):
1057
+ raise GenerationReceiptRefused(
1058
+ "GENERATION_EXECUTION_DISTRIBUTION",
1059
+ "installed_runtime.distribution",
1060
+ "installed distribution member escapes its package root",
1061
+ ) from None
1062
+ raw = _read_installed_distribution_member(
1063
+ member,
1064
+ maximum=MAX_DISTRIBUTION_MEMBER_BYTES,
1065
+ label="installed_distribution.member",
1066
+ )
1067
+ total_bytes += len(raw)
1068
+ if total_bytes > MAX_DISTRIBUTION_TOTAL_BYTES:
1069
+ raise GenerationReceiptRefused(
1070
+ "GENERATION_EXECUTION_DISTRIBUTION",
1071
+ "installed_runtime.distribution",
1072
+ "installed distribution exceeds its aggregate byte budget",
1073
+ )
1074
+ selected.append({"path": name, "sha256": sha256_bytes(raw)})
1075
+ except (OSError, ValueError, metadata.PackageNotFoundError) as error:
1076
+ raise GenerationReceiptRefused(
1077
+ "GENERATION_EXECUTION_DISTRIBUTION",
1078
+ "installed_runtime.distribution",
1079
+ "catalog execution must originate from one installed wheel distribution",
1080
+ ) from error
1081
+ return _distribution_coordinate(selected, distribution.version)
1082
+
1083
+
1084
+ def root_coordinate(root: Path) -> str:
1085
+ """One output root's coordinate: the digest of its resolved name, never the name itself.
1086
+
1087
+ A receipt binds the root it describes so that a resume cannot be pointed at another one, and a
1088
+ digest binds it without writing an executor-local absolute path into durable evidence.
1089
+ """
1090
+
1091
+ return canonical_sha256({"output_root": str(_absolute_path(root))})
1092
+
1093
+
1094
+ # ------------------------------------------------------------------------------------------------
1095
+ # Installing a receipt, and replacing exactly one typed-incomplete predecessor
1096
+ # ------------------------------------------------------------------------------------------------
1097
+
1098
+
1099
+ def install_receipt(
1100
+ path: Path,
1101
+ payload: Mapping[str, Any],
1102
+ *,
1103
+ maximum: int,
1104
+ label: str,
1105
+ code_prefix: str,
1106
+ refusal: type[SourceContractError] = GenerationReceiptRefused,
1107
+ replaces: Mapping[str, Any] | None = None,
1108
+ installation: ReceiptInstallation | None = None,
1109
+ commit_guard: Callable[[], None] | None = None,
1110
+ ) -> bytes:
1111
+ """Install one complete receipt with append-only failure handling.
1112
+
1113
+ Every invocation first fsyncs the complete bytes under a unique private name. Publication is
1114
+ one atomic no-replace rename, so the authoritative name is absent or complete. Replacement
1115
+ first moves the exact authorized predecessor into immutable history. A failed post-install
1116
+ guard moves the visible bytes to a unique quarantine and restores the predecessor without
1117
+ overwriting or deleting any name. Private remnants are never authoritative and never block a
1118
+ retry. The final descriptor/name identity check is the last filesystem action before success.
1119
+ """
1120
+
1121
+ destination = _absolute_path(path)
1122
+ if installation is None:
1123
+ installation = next(
1124
+ (
1125
+ active
1126
+ for active in reversed(_ACTIVE_RECEIPT_INSTALLATIONS.get())
1127
+ if active.destination == destination
1128
+ ),
1129
+ None,
1130
+ )
1131
+ if installation is None:
1132
+ with receipt_installation_lock(
1133
+ destination, label=label, code_prefix=code_prefix, refusal=refusal
1134
+ ) as retained_parent:
1135
+ return install_receipt(
1136
+ destination,
1137
+ payload,
1138
+ maximum=maximum,
1139
+ label=label,
1140
+ code_prefix=code_prefix,
1141
+ refusal=refusal,
1142
+ replaces=replaces,
1143
+ installation=retained_parent,
1144
+ commit_guard=commit_guard,
1145
+ )
1146
+ if installation.destination != destination:
1147
+ raise refusal(
1148
+ f"{code_prefix}_RECEIPT_DESTINATION",
1149
+ label,
1150
+ "receipt installation does not own this destination",
1151
+ )
1152
+
1153
+ raw = canonical_json_bytes(payload)
1154
+ if len(raw) > maximum:
1155
+ raise refusal(f"{code_prefix}_RECEIPT_LIMIT", label, "receipt exceeds its byte bound")
1156
+ parent_descriptor = installation.parent_descriptor
1157
+ name = installation.name
1158
+ staged = _private_receipt_name(name, "staging", sha256_bytes(raw))
1159
+ retained_name: str | None = None
1160
+ predecessor_source: str | None = None
1161
+ predecessor_claimed = False
1162
+ published = False
1163
+ installed_identity: os.stat_result | None = None
1164
+ try:
1165
+ _require_receipt_parent_identity(
1166
+ installation, label=label, code_prefix=code_prefix, refusal=refusal
1167
+ )
1168
+ flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0)
1169
+ descriptor = os.open(staged, flags, 0o600, dir_fd=parent_descriptor)
1170
+ _write_all(descriptor, raw)
1171
+ staged_raw, staged_identity = _read_bounded_at(parent_descriptor, staged, maximum=maximum)
1172
+ staged_named_identity = os.stat(staged, dir_fd=parent_descriptor, follow_symlinks=False)
1173
+ if staged_raw != raw or not _same_identity(staged_identity, staged_named_identity):
1174
+ raise refusal(
1175
+ f"{code_prefix}_RECEIPT_READBACK",
1176
+ label,
1177
+ "private receipt staging changed before publication",
1178
+ )
1179
+ if commit_guard is not None:
1180
+ commit_guard()
1181
+ if replaces is not None:
1182
+ expected = replaces.get("receipt_sha256")
1183
+ if not is_digest(expected):
1184
+ raise refusal(
1185
+ f"{code_prefix}_RESUME_RECEIPT",
1186
+ label,
1187
+ "the retained receipt has no exact fingerprint",
1188
+ )
1189
+ retained_name = f".{name}.retained.{expected}"
1190
+ current, current_identity = _read_bounded_at(parent_descriptor, name, maximum=maximum)
1191
+ if sha256_bytes(current) != expected:
1192
+ raise refusal(
1193
+ f"{code_prefix}_RESUME_RECEIPT",
1194
+ label,
1195
+ "the retained receipt changed before replacement",
1196
+ )
1197
+ named_identity = os.stat(name, dir_fd=parent_descriptor, follow_symlinks=False)
1198
+ if not _same_identity(current_identity, named_identity):
1199
+ raise refusal(
1200
+ f"{code_prefix}_RESUME_RECEIPT",
1201
+ label,
1202
+ "the retained receipt changed before replacement",
1203
+ )
1204
+ _require_receipt_parent_identity(
1205
+ installation, label=label, code_prefix=code_prefix, refusal=refusal
1206
+ )
1207
+ claim_name = _private_receipt_name(name, "predecessor", expected)
1208
+ _rename_noreplace(parent_descriptor, name, claim_name)
1209
+ predecessor_claimed = True
1210
+ predecessor_source = claim_name
1211
+ os.fsync(parent_descriptor)
1212
+ claimed, claimed_identity = _read_bounded_at(
1213
+ parent_descriptor, claim_name, maximum=maximum
1214
+ )
1215
+ claimed_named_identity = os.stat(
1216
+ claim_name, dir_fd=parent_descriptor, follow_symlinks=False
1217
+ )
1218
+ if (
1219
+ claimed != current
1220
+ or sha256_bytes(claimed) != expected
1221
+ or (current_identity.st_dev, current_identity.st_ino)
1222
+ != (claimed_identity.st_dev, claimed_identity.st_ino)
1223
+ or not _same_identity(claimed_identity, claimed_named_identity)
1224
+ ):
1225
+ raise refusal(
1226
+ f"{code_prefix}_RESUME_RECEIPT",
1227
+ label,
1228
+ "the retained receipt changed before replacement",
1229
+ )
1230
+ try:
1231
+ _rename_noreplace(parent_descriptor, claim_name, retained_name)
1232
+ predecessor_source = retained_name
1233
+ os.fsync(parent_descriptor)
1234
+ except OSError as error:
1235
+ if error.errno != errno.EEXIST:
1236
+ raise
1237
+ archived, archived_identity = _read_bounded_at(
1238
+ parent_descriptor, retained_name, maximum=maximum
1239
+ )
1240
+ archived_named_identity = os.stat(
1241
+ retained_name, dir_fd=parent_descriptor, follow_symlinks=False
1242
+ )
1243
+ if (
1244
+ archived != current
1245
+ or sha256_bytes(archived) != expected
1246
+ or not _same_identity(archived_identity, archived_named_identity)
1247
+ ):
1248
+ raise refusal(
1249
+ f"{code_prefix}_RESUME_RECEIPT",
1250
+ label,
1251
+ "the retained receipt archive differs from the authorized predecessor",
1252
+ )
1253
+ _require_receipt_parent_identity(
1254
+ installation, label=label, code_prefix=code_prefix, refusal=refusal
1255
+ )
1256
+ _rename_noreplace(parent_descriptor, staged, name)
1257
+ published = True
1258
+ os.fsync(parent_descriptor)
1259
+ installed, installed_identity = _read_bounded_at(parent_descriptor, name, maximum=maximum)
1260
+ if installed != raw:
1261
+ raise refusal(
1262
+ f"{code_prefix}_RECEIPT_READBACK", label, "installed receipt readback differs"
1263
+ )
1264
+ _require_receipt_parent_identity(
1265
+ installation, label=label, code_prefix=code_prefix, refusal=refusal
1266
+ )
1267
+ if commit_guard is not None:
1268
+ commit_guard()
1269
+ final, final_identity = _read_bounded_at(parent_descriptor, name, maximum=maximum)
1270
+ if (
1271
+ final != raw
1272
+ or installed_identity is None
1273
+ or not _same_identity(installed_identity, final_identity)
1274
+ ):
1275
+ raise refusal(
1276
+ f"{code_prefix}_RECEIPT_READBACK",
1277
+ label,
1278
+ "installed receipt changed before commit completed",
1279
+ )
1280
+ final_named_identity = os.stat(name, dir_fd=parent_descriptor, follow_symlinks=False)
1281
+ if not _same_identity(final_identity, final_named_identity):
1282
+ raise refusal(
1283
+ f"{code_prefix}_RECEIPT_READBACK",
1284
+ label,
1285
+ "installed receipt changed before commit completed",
1286
+ )
1287
+ _require_receipt_parent_identity(
1288
+ installation, label=label, code_prefix=code_prefix, refusal=refusal
1289
+ )
1290
+ return raw
1291
+ except (OSError, SourceContractError) as error:
1292
+ try:
1293
+ if published:
1294
+ _quarantine_receipt(
1295
+ parent_descriptor,
1296
+ name,
1297
+ maximum=maximum,
1298
+ )
1299
+ if predecessor_claimed:
1300
+ assert predecessor_source is not None
1301
+ _restore_receipt_from_history(
1302
+ parent_descriptor,
1303
+ predecessor_source,
1304
+ name,
1305
+ maximum=maximum,
1306
+ )
1307
+ except OSError as recovery_error:
1308
+ code = (
1309
+ f"{code_prefix}_RESUME_RECEIPT"
1310
+ if replaces is not None
1311
+ else f"{code_prefix}_RECEIPT_DESTINATION"
1312
+ )
1313
+ raise refusal(
1314
+ code,
1315
+ label,
1316
+ "receipt publication failed and its prior visible state could not be restored",
1317
+ ) from recovery_error
1318
+ if isinstance(error, SourceContractError):
1319
+ raise
1320
+ code = (
1321
+ f"{code_prefix}_RESUME_RECEIPT"
1322
+ if replaces is not None
1323
+ else f"{code_prefix}_RECEIPT_DESTINATION"
1324
+ )
1325
+ detail = (
1326
+ "the retained receipt changed before replacement"
1327
+ if replaces is not None
1328
+ else "receipt destination must be an absent no-follow path"
1329
+ )
1330
+ raise refusal(code, label, detail) from error
1331
+
1332
+
1333
+ def _private_receipt_name(destination: str, role: str, digest: str) -> str:
1334
+ """Return an invocation-private, non-authoritative member name."""
1335
+
1336
+ return f".{destination}.{role}.{digest}.{os.urandom(16).hex()}"
1337
+
1338
+
1339
+ def _quarantine_receipt(parent: int, destination: str, *, maximum: int) -> str | None:
1340
+ """Move the current public name into append-only quarantine without deleting its bytes."""
1341
+
1342
+ quarantine = _private_receipt_name(destination, "quarantine", os.urandom(32).hex())
1343
+ try:
1344
+ _rename_noreplace(parent, destination, quarantine)
1345
+ except FileNotFoundError:
1346
+ return None
1347
+ os.fsync(parent)
1348
+ _raw, identity = _read_bounded_at(parent, quarantine, maximum=maximum)
1349
+ named_identity = os.stat(quarantine, dir_fd=parent, follow_symlinks=False)
1350
+ if not _same_identity(identity, named_identity):
1351
+ raise OSError(errno.EIO, "quarantined receipt changed")
1352
+ return quarantine
1353
+
1354
+
1355
+ def _restore_receipt_from_history(
1356
+ parent: int,
1357
+ source: str,
1358
+ destination: str,
1359
+ *,
1360
+ maximum: int,
1361
+ ) -> None:
1362
+ """Copy immutable predecessor bytes through a private stage and restore them no-replace."""
1363
+
1364
+ raw, source_identity = _read_bounded_at(parent, source, maximum=maximum)
1365
+ source_named_identity = os.stat(source, dir_fd=parent, follow_symlinks=False)
1366
+ if not _same_identity(source_identity, source_named_identity):
1367
+ raise OSError(errno.EIO, "receipt history changed")
1368
+ staged = _private_receipt_name(destination, "restoring", sha256_bytes(raw))
1369
+ flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0)
1370
+ descriptor = os.open(staged, flags, 0o600, dir_fd=parent)
1371
+ _write_all(descriptor, raw)
1372
+ staged_raw, staged_identity = _read_bounded_at(parent, staged, maximum=maximum)
1373
+ staged_named_identity = os.stat(staged, dir_fd=parent, follow_symlinks=False)
1374
+ if staged_raw != raw or not _same_identity(staged_identity, staged_named_identity):
1375
+ raise OSError(errno.EIO, "receipt restoration stage changed")
1376
+ _rename_noreplace(parent, staged, destination)
1377
+ os.fsync(parent)
1378
+ restored, restored_identity = _read_bounded_at(parent, destination, maximum=maximum)
1379
+ restored_named_identity = os.stat(destination, dir_fd=parent, follow_symlinks=False)
1380
+ if restored != raw or not _same_identity(restored_identity, restored_named_identity):
1381
+ raise OSError(errno.EIO, "restored receipt changed")
1382
+
1383
+
1384
+ def _rename_noreplace(parent: int, source: str, destination: str) -> None:
1385
+ """Rename one regular member only when the destination is absent."""
1386
+
1387
+ library = ctypes.CDLL(None, use_errno=True)
1388
+ source_bytes = os.fsencode(source)
1389
+ destination_bytes = os.fsencode(destination)
1390
+ if hasattr(library, "renameat2"):
1391
+ rename = library.renameat2
1392
+ rename.argtypes = [
1393
+ ctypes.c_int,
1394
+ ctypes.c_char_p,
1395
+ ctypes.c_int,
1396
+ ctypes.c_char_p,
1397
+ ctypes.c_uint,
1398
+ ]
1399
+ rename.restype = ctypes.c_int
1400
+ result = rename(parent, source_bytes, parent, destination_bytes, 1)
1401
+ elif hasattr(library, "renameatx_np"):
1402
+ rename = library.renameatx_np
1403
+ rename.argtypes = [
1404
+ ctypes.c_int,
1405
+ ctypes.c_char_p,
1406
+ ctypes.c_int,
1407
+ ctypes.c_char_p,
1408
+ ctypes.c_uint,
1409
+ ]
1410
+ rename.restype = ctypes.c_int
1411
+ result = rename(parent, source_bytes, parent, destination_bytes, 4)
1412
+ else:
1413
+ raise OSError(errno.ENOTSUP, "atomic no-replace rename is unavailable")
1414
+ if result != 0:
1415
+ value = ctypes.get_errno()
1416
+ raise OSError(value, os.strerror(value), destination)
1417
+
1418
+
1419
+ def _read_bounded_at(parent: int, name: str, *, maximum: int) -> tuple[bytes, os.stat_result]:
1420
+ descriptor = os.open(name, _OPEN_MEMBER, dir_fd=parent)
1421
+ try:
1422
+ before = os.fstat(descriptor)
1423
+ if not stat.S_ISREG(before.st_mode) or before.st_nlink != 1 or before.st_size > maximum:
1424
+ raise OSError
1425
+ chunks: list[bytes] = []
1426
+ remaining = maximum + 1
1427
+ while remaining > 0:
1428
+ chunk = os.read(descriptor, min(remaining, 1 << 16))
1429
+ if not chunk:
1430
+ break
1431
+ chunks.append(chunk)
1432
+ remaining -= len(chunk)
1433
+ after = os.fstat(descriptor)
1434
+ if not _same_identity(before, after):
1435
+ raise OSError
1436
+ raw = b"".join(chunks)
1437
+ if len(raw) > maximum:
1438
+ raise OSError
1439
+ return raw, after
1440
+ finally:
1441
+ os.close(descriptor)
1442
+
1443
+
1444
+ def _write_all(descriptor: int, raw: bytes) -> None:
1445
+ try:
1446
+ written = 0
1447
+ while written < len(raw):
1448
+ count = os.write(descriptor, raw[written:])
1449
+ if count <= 0:
1450
+ raise OSError(errno.EIO, "receipt write made no progress")
1451
+ written += count
1452
+ os.fsync(descriptor)
1453
+ finally:
1454
+ os.close(descriptor)
1455
+
1456
+
1457
+ def retained_incomplete_receipt(
1458
+ path: Path,
1459
+ *,
1460
+ maximum: int,
1461
+ label: str,
1462
+ schema: str,
1463
+ incomplete_status: str,
1464
+ output_root: Path,
1465
+ code_prefix: str,
1466
+ refusal: type[SourceContractError] = GenerationReceiptRefused,
1467
+ ) -> dict[str, Any] | None:
1468
+ """The typed-incomplete receipt a resume is allowed to replace, or ``None`` if there is none.
1469
+
1470
+ A resume that finds nothing at the destination is an ordinary first write. A resume that
1471
+ finds a receipt which is complete, written under another schema, or bound to another output
1472
+ root is refused rather than overwritten: replacing evidence is only ever allowed to continue
1473
+ the exact interrupted invocation that wrote it.
1474
+ """
1475
+
1476
+ try:
1477
+ raw = read_bounded_document(Path(path), maximum=maximum, label=label)
1478
+ except GenerationReceiptRefused as error:
1479
+ if error.code == "GENERATION_INPUT_PATH" and not Path(path).exists():
1480
+ return None
1481
+ raise
1482
+ try:
1483
+ payload = parse_canonical_json(raw)
1484
+ except CanonicalJSONError as error:
1485
+ raise refusal(
1486
+ f"{code_prefix}_RESUME_RECEIPT", label, "retained receipt is not canonical JSON"
1487
+ ) from error
1488
+ if (
1489
+ not isinstance(payload, dict)
1490
+ or payload.get("schema_version") != schema
1491
+ or payload.get("status") != incomplete_status
1492
+ ):
1493
+ raise refusal(
1494
+ f"{code_prefix}_RECEIPT_DESTINATION",
1495
+ label,
1496
+ "only this stage's own typed-incomplete receipt may be replaced",
1497
+ )
1498
+ if payload.get("output_root_sha256") != root_coordinate(output_root):
1499
+ raise refusal(
1500
+ f"{code_prefix}_RESUME_RECEIPT",
1501
+ label,
1502
+ "the retained typed-incomplete receipt describes another output root",
1503
+ )
1504
+ return {**payload, "receipt_sha256": sha256_bytes(raw)}
1505
+
1506
+
1507
+ # ------------------------------------------------------------------------------------------------
1508
+ # The composite receipt
1509
+ # ------------------------------------------------------------------------------------------------
1510
+
1511
+
1512
+ def compose_generation_receipt(body: Mapping[str, Any]) -> dict[str, Any]:
1513
+ """Seal one composite receipt: shape-check it, then bind it under its own coordinate."""
1514
+
1515
+ payload = {key: value for key, value in body.items() if key != "coordinate_sha256"}
1516
+ enforce_shape(
1517
+ payload,
1518
+ label="generation_receipt",
1519
+ max_depth=MAX_COMPOSITE_RECEIPT_DEPTH,
1520
+ max_members=MAX_COMPOSITE_ARRAY_ITEMS,
1521
+ max_items=MAX_COMPOSITE_ARRAY_ITEMS,
1522
+ code="GENERATION_RECEIPT_SHAPE",
1523
+ )
1524
+ receipt = {**payload, "coordinate_sha256": canonical_sha256(payload)}
1525
+ raw = canonical_json_bytes(receipt)
1526
+ if len(raw) > MAX_COMPOSITE_RECEIPT_BYTES:
1527
+ raise GenerationReceiptRefused(
1528
+ "GENERATION_RECEIPT_LIMIT",
1529
+ "generation_receipt",
1530
+ f"the composite receipt is at most {MAX_COMPOSITE_RECEIPT_BYTES} bytes",
1531
+ )
1532
+ return receipt
1533
+
1534
+
1535
+ def verify_generation_receipt(receipt: Mapping[str, Any]) -> None:
1536
+ """Re-derive one composite receipt against itself: schema, shape, and its own coordinate.
1537
+
1538
+ This is the check that costs no I/O. It says the document in front of you is a whole
1539
+ composite receipt that reproduces its own fingerprint -- not that the generation it describes
1540
+ is still installed, which is what the readback block inside it is for.
1541
+ """
1542
+
1543
+ if not isinstance(receipt, Mapping):
1544
+ raise GenerationReceiptRefused(
1545
+ "GENERATION_RECEIPT_SCHEMA", "generation_receipt", "must be one object"
1546
+ )
1547
+ if receipt.get("schema_version") != GENERATION_RECEIPT_SCHEMA:
1548
+ raise GenerationReceiptRefused(
1549
+ "GENERATION_RECEIPT_SCHEMA", "generation_receipt", "receipt names another schema"
1550
+ )
1551
+ for member in (
1552
+ "status",
1553
+ "components",
1554
+ "execution",
1555
+ "staging",
1556
+ "policy",
1557
+ "author",
1558
+ "classification",
1559
+ "publication",
1560
+ "readback",
1561
+ "limits",
1562
+ "coordinate_sha256",
1563
+ ):
1564
+ if member not in receipt:
1565
+ raise GenerationReceiptRefused(
1566
+ "GENERATION_RECEIPT_SCHEMA",
1567
+ f"generation_receipt.{member}",
1568
+ "a composite receipt states every one of its blocks",
1569
+ )
1570
+ body = {key: value for key, value in receipt.items() if key != "coordinate_sha256"}
1571
+ enforce_shape(
1572
+ body,
1573
+ label="generation_receipt",
1574
+ max_depth=MAX_COMPOSITE_RECEIPT_DEPTH,
1575
+ max_members=MAX_COMPOSITE_ARRAY_ITEMS,
1576
+ max_items=MAX_COMPOSITE_ARRAY_ITEMS,
1577
+ code="GENERATION_RECEIPT_SHAPE",
1578
+ )
1579
+ components = receipt["components"]
1580
+ for stage in ("harvest", "author"):
1581
+ embedded = components.get(f"{stage}_receipt")
1582
+ stated = components.get(f"{stage}_receipt_sha256")
1583
+ if not isinstance(embedded, Mapping) or not is_digest(stated):
1584
+ raise GenerationReceiptRefused(
1585
+ "GENERATION_RECEIPT_COMPONENT",
1586
+ f"generation_receipt.components.{stage}_receipt",
1587
+ "a composite receipt carries each component receipt and its exact fingerprint",
1588
+ )
1589
+ if sha256_bytes(canonical_json_bytes(embedded)) != stated:
1590
+ raise GenerationReceiptRefused(
1591
+ "GENERATION_RECEIPT_COMPONENT",
1592
+ f"generation_receipt.components.{stage}_receipt",
1593
+ "the carried receipt is not the bytes its fingerprint names",
1594
+ )
1595
+ ranges = receipt["publication"].get("range_manifest_sha256s", ())
1596
+ if not isinstance(ranges, list) or len(ranges) > MAX_COMPOSITE_RANGE_DESCRIPTORS:
1597
+ raise GenerationReceiptRefused(
1598
+ "GENERATION_RECEIPT_SHAPE",
1599
+ "generation_receipt.publication.range_manifest_sha256s",
1600
+ f"a generation states at most {MAX_COMPOSITE_RANGE_DESCRIPTORS} ranges",
1601
+ )
1602
+ if receipt["coordinate_sha256"] != canonical_sha256(body):
1603
+ raise GenerationReceiptRefused(
1604
+ "GENERATION_RECEIPT_COORDINATE",
1605
+ "generation_receipt.coordinate_sha256",
1606
+ "the receipt does not reproduce its own coordinate",
1607
+ )