mostlyright-data 0.9.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (314) hide show
  1. mostlyright/data_harness/__init__.py +158 -0
  2. mostlyright/data_harness/acquisition/__init__.py +55 -0
  3. mostlyright/data_harness/acquisition/http.py +2773 -0
  4. mostlyright/data_harness/acquisition/parsing.py +809 -0
  5. mostlyright/data_harness/acquisition/ranges.py +495 -0
  6. mostlyright/data_harness/acquisition/result_download.py +360 -0
  7. mostlyright/data_harness/acquisition/retention_admission.py +248 -0
  8. mostlyright/data_harness/acquisition/sandbox.py +4888 -0
  9. mostlyright/data_harness/acquisition/url_policy.py +530 -0
  10. mostlyright/data_harness/agent_runtime.py +2743 -0
  11. mostlyright/data_harness/assets/logo-ink.svg +31 -0
  12. mostlyright/data_harness/backends/__init__.py +28 -0
  13. mostlyright/data_harness/backends/pandas_backend.py +350 -0
  14. mostlyright/data_harness/backends/polars_backend.py +366 -0
  15. mostlyright/data_harness/backends/protocol.py +124 -0
  16. mostlyright/data_harness/backends/reference.py +83 -0
  17. mostlyright/data_harness/backends/registry.py +55 -0
  18. mostlyright/data_harness/backends/restrictions.py +126 -0
  19. mostlyright/data_harness/canonical.py +333 -0
  20. mostlyright/data_harness/catalog_job.py +625 -0
  21. mostlyright/data_harness/cli.py +5398 -0
  22. mostlyright/data_harness/contracts.py +53 -0
  23. mostlyright/data_harness/coordinator.py +1307 -0
  24. mostlyright/data_harness/deploy.py +924 -0
  25. mostlyright/data_harness/deploy_target.py +312 -0
  26. mostlyright/data_harness/deployment_evidence.py +1067 -0
  27. mostlyright/data_harness/event_presentation.py +576 -0
  28. mostlyright/data_harness/events.py +2152 -0
  29. mostlyright/data_harness/fast_delimited.py +239 -0
  30. mostlyright/data_harness/fleet.py +237 -0
  31. mostlyright/data_harness/formats.py +236 -0
  32. mostlyright/data_harness/governors.py +1163 -0
  33. mostlyright/data_harness/hosted_bootstrap.py +972 -0
  34. mostlyright/data_harness/hosted_crawler.py +1115 -0
  35. mostlyright/data_harness/hosted_crawler_container_smoke.py +351 -0
  36. mostlyright/data_harness/hosted_crawler_fetch.py +423 -0
  37. mostlyright/data_harness/hosted_crawler_job.py +1277 -0
  38. mostlyright/data_harness/hosted_crawler_protocol.py +676 -0
  39. mostlyright/data_harness/hosted_dataset.py +1500 -0
  40. mostlyright/data_harness/hosted_deploy.py +3037 -0
  41. mostlyright/data_harness/hosted_handoff.py +62 -0
  42. mostlyright/data_harness/hosted_ingestion_contract.py +504 -0
  43. mostlyright/data_harness/hosted_ingestion_job.py +356 -0
  44. mostlyright/data_harness/hosted_ingestion_job_smoke.py +40 -0
  45. mostlyright/data_harness/hosted_session_container_smoke.py +194 -0
  46. mostlyright/data_harness/hosted_session_worker.py +3554 -0
  47. mostlyright/data_harness/hosted_session_worker_job_smoke.py +46 -0
  48. mostlyright/data_harness/hosted_worker.py +6784 -0
  49. mostlyright/data_harness/ingestion/__init__.py +56 -0
  50. mostlyright/data_harness/ingestion/contracts.py +461 -0
  51. mostlyright/data_harness/ingestion/faults.py +42 -0
  52. mostlyright/data_harness/ingestion/gcs_store.py +1162 -0
  53. mostlyright/data_harness/ingestion/spool.py +130 -0
  54. mostlyright/data_harness/ingestion/store.py +885 -0
  55. mostlyright/data_harness/key_seam.py +434 -0
  56. mostlyright/data_harness/linux_process_boundary.py +262 -0
  57. mostlyright/data_harness/local_contracts.py +2880 -0
  58. mostlyright/data_harness/local_search/__init__.py +5 -0
  59. mostlyright/data_harness/local_search/build_index.py +1087 -0
  60. mostlyright/data_harness/local_search/contracts.py +920 -0
  61. mostlyright/data_harness/local_search/query_trace.py +266 -0
  62. mostlyright/data_harness/local_search/retrieval.py +700 -0
  63. mostlyright/data_harness/local_search/sealed.py +474 -0
  64. mostlyright/data_harness/local_search/service.py +784 -0
  65. mostlyright/data_harness/nbrender/CONTRACT.md +212 -0
  66. mostlyright/data_harness/nbrender/__init__.py +12 -0
  67. mostlyright/data_harness/nbrender/chrome.py +359 -0
  68. mostlyright/data_harness/nbrender/code_body.py +266 -0
  69. mostlyright/data_harness/nbrender/document.py +407 -0
  70. mostlyright/data_harness/nbrender/frame.py +275 -0
  71. mostlyright/data_harness/nbrender/interactive.py +337 -0
  72. mostlyright/data_harness/nbrender/markdown_body.py +477 -0
  73. mostlyright/data_harness/nbrender/mr_components.py +134 -0
  74. mostlyright/data_harness/nbrender/outputs_data.py +595 -0
  75. mostlyright/data_harness/nbrender/outputs_rich.py +906 -0
  76. mostlyright/data_harness/nbrender/outputs_source.py +260 -0
  77. mostlyright/data_harness/nbrender/outputs_stage.py +176 -0
  78. mostlyright/data_harness/nbrender/outputs_text.py +400 -0
  79. mostlyright/data_harness/nbrender/parse.py +394 -0
  80. mostlyright/data_harness/nbrender/status.py +40 -0
  81. mostlyright/data_harness/nbrender/tokens.py +1295 -0
  82. mostlyright/data_harness/notebook.py +1710 -0
  83. mostlyright/data_harness/offline.py +2049 -0
  84. mostlyright/data_harness/operation_registry.py +1007 -0
  85. mostlyright/data_harness/operator_setup.py +239 -0
  86. mostlyright/data_harness/pipeline.py +6428 -0
  87. mostlyright/data_harness/plan_graph.py +2026 -0
  88. mostlyright/data_harness/preparation/__init__.py +104 -0
  89. mostlyright/data_harness/preparation/contracts.py +1017 -0
  90. mostlyright/data_harness/preparation/engine.py +221 -0
  91. mostlyright/data_harness/preparation/errors.py +14 -0
  92. mostlyright/data_harness/preparation/gates.py +751 -0
  93. mostlyright/data_harness/preparation/joins.py +574 -0
  94. mostlyright/data_harness/preparation/profile.py +384 -0
  95. mostlyright/data_harness/preparation/table.py +217 -0
  96. mostlyright/data_harness/preparation/transforms.py +568 -0
  97. mostlyright/data_harness/progress_events.py +534 -0
  98. mostlyright/data_harness/readers/__init__.py +46 -0
  99. mostlyright/data_harness/readers/containers.py +963 -0
  100. mostlyright/data_harness/readers/contracts.py +542 -0
  101. mostlyright/data_harness/readers/delimited.py +257 -0
  102. mostlyright/data_harness/readers/grib2/__init__.py +33 -0
  103. mostlyright/data_harness/readers/grib2/admission.py +722 -0
  104. mostlyright/data_harness/readers/grib2/decode.py +1009 -0
  105. mostlyright/data_harness/readers/grib2/geometry.py +1133 -0
  106. mostlyright/data_harness/readers/grib2/portable_math.py +501 -0
  107. mostlyright/data_harness/readers/json_tabular.py +485 -0
  108. mostlyright/data_harness/readers/registry.py +514 -0
  109. mostlyright/data_harness/readers/samples/README.md +110 -0
  110. mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/cities.csv.gz +0 -0
  111. mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/expected.json +24 -0
  112. mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/cities.csv.gz +0 -0
  113. mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/expected.json +24 -0
  114. mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/cities.tar +0 -0
  115. mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/expected.json +24 -0
  116. mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/cities.tar +0 -0
  117. mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/expected.json +24 -0
  118. mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/cities.zip +0 -0
  119. mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/expected.json +25 -0
  120. mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
  121. mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/expected.json +25 -0
  122. mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
  123. mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/expected.json +25 -0
  124. mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/cities.csv +3 -0
  125. mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/expected.json +23 -0
  126. mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/cities.tsv +5 -0
  127. mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/expected.json +25 -0
  128. mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/expected.json +30 -0
  129. mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/observations.csv +5 -0
  130. mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/expected.json +44 -0
  131. mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/stations.json +1 -0
  132. mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/expected.json +48 -0
  133. mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/observations.ndjson +4 -0
  134. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/cities.xlsx +0 -0
  135. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/expected.json +24 -0
  136. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/expected.json +27 -0
  137. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
  138. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/expected.json +27 -0
  139. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
  140. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/README.md +20 -0
  141. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/expected.json +55 -0
  142. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/gfs-2m-temperature.grib2 +0 -0
  143. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/expected.json +54 -0
  144. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
  145. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/expected.json +54 -0
  146. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/hrrr-categorical-rain.grib2 +0 -0
  147. mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/expected.json +54 -0
  148. mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
  149. mostlyright/data_harness/readers/samples.py +582 -0
  150. mostlyright/data_harness/readers/spreadsheet.py +803 -0
  151. mostlyright/data_harness/readers/tabular.py +510 -0
  152. mostlyright/data_harness/recipe.py +5321 -0
  153. mostlyright/data_harness/repair/__init__.py +78 -0
  154. mostlyright/data_harness/repair/adapters.py +274 -0
  155. mostlyright/data_harness/repair/contracts.py +872 -0
  156. mostlyright/data_harness/repair/coordinator.py +1099 -0
  157. mostlyright/data_harness/repair/errors.py +16 -0
  158. mostlyright/data_harness/review.py +2533 -0
  159. mostlyright/data_harness/rowset.py +283 -0
  160. mostlyright/data_harness/serving.py +1975 -0
  161. mostlyright/data_harness/serving_edge.py +590 -0
  162. mostlyright/data_harness/serving_http.py +1031 -0
  163. mostlyright/data_harness/session_probes.py +759 -0
  164. mostlyright/data_harness/signing.py +101 -0
  165. mostlyright/data_harness/source_discovery.py +898 -0
  166. mostlyright/data_harness/sources/__init__.py +209 -0
  167. mostlyright/data_harness/sources/_adapter_steps.py +213 -0
  168. mostlyright/data_harness/sources/adapters.py +1214 -0
  169. mostlyright/data_harness/sources/cadence.py +1428 -0
  170. mostlyright/data_harness/sources/cadence_emission.py +453 -0
  171. mostlyright/data_harness/sources/cadence_history.py +546 -0
  172. mostlyright/data_harness/sources/catalog/__init__.py +17 -0
  173. mostlyright/data_harness/sources/catalog/admission.py +477 -0
  174. mostlyright/data_harness/sources/catalog/authoring.py +1701 -0
  175. mostlyright/data_harness/sources/catalog/authoring_policy.py +701 -0
  176. mostlyright/data_harness/sources/catalog/authoring_shards.py +1217 -0
  177. mostlyright/data_harness/sources/catalog/bounded_io.py +231 -0
  178. mostlyright/data_harness/sources/catalog/channel.py +523 -0
  179. mostlyright/data_harness/sources/catalog/channel_client.py +296 -0
  180. mostlyright/data_harness/sources/catalog/contracts.py +825 -0
  181. mostlyright/data_harness/sources/catalog/coverage.py +137 -0
  182. mostlyright/data_harness/sources/catalog/delta.py +1340 -0
  183. mostlyright/data_harness/sources/catalog/embedding.py +532 -0
  184. mostlyright/data_harness/sources/catalog/entry_v2.py +1182 -0
  185. mostlyright/data_harness/sources/catalog/fill.py +3889 -0
  186. mostlyright/data_harness/sources/catalog/fill_partitions.py +459 -0
  187. mostlyright/data_harness/sources/catalog/fill_staging.py +1105 -0
  188. mostlyright/data_harness/sources/catalog/gating.py +374 -0
  189. mostlyright/data_harness/sources/catalog/generation_receipt.py +1607 -0
  190. mostlyright/data_harness/sources/catalog/harvest/__init__.py +7 -0
  191. mostlyright/data_harness/sources/catalog/harvest/ckan.py +384 -0
  192. mostlyright/data_harness/sources/catalog/harvest/datagov_v4.py +798 -0
  193. mostlyright/data_harness/sources/catalog/harvest/protocol.py +964 -0
  194. mostlyright/data_harness/sources/catalog/harvest/sdmx.py +445 -0
  195. mostlyright/data_harness/sources/catalog/harvest/stac.py +384 -0
  196. mostlyright/data_harness/sources/catalog/health.py +447 -0
  197. mostlyright/data_harness/sources/catalog/hosted_catalog.py +105 -0
  198. mostlyright/data_harness/sources/catalog/identity_history.py +1549 -0
  199. mostlyright/data_harness/sources/catalog/neural.py +1618 -0
  200. mostlyright/data_harness/sources/catalog/packed_catalog.py +2345 -0
  201. mostlyright/data_harness/sources/catalog/packed_retrieval.py +1517 -0
  202. mostlyright/data_harness/sources/catalog/packed_writer.py +2802 -0
  203. mostlyright/data_harness/sources/catalog/query_trace.py +1037 -0
  204. mostlyright/data_harness/sources/catalog/recommend.py +171 -0
  205. mostlyright/data_harness/sources/catalog/retrieval.py +230 -0
  206. mostlyright/data_harness/sources/catalog/retrieval_manifest.py +995 -0
  207. mostlyright/data_harness/sources/catalog/rights_decisions.py +254 -0
  208. mostlyright/data_harness/sources/catalog/sealed.py +560 -0
  209. mostlyright/data_harness/sources/catalog/search.py +230 -0
  210. mostlyright/data_harness/sources/catalog/streaming_delta.py +1097 -0
  211. mostlyright/data_harness/sources/catalog/update.py +891 -0
  212. mostlyright/data_harness/sources/collections.py +815 -0
  213. mostlyright/data_harness/sources/contracts.py +2223 -0
  214. mostlyright/data_harness/sources/deletion.py +761 -0
  215. mostlyright/data_harness/sources/fitness.py +162 -0
  216. mostlyright/data_harness/sources/governance.py +163 -0
  217. mostlyright/data_harness/sources/hosted.py +173 -0
  218. mostlyright/data_harness/sources/integration.py +218 -0
  219. mostlyright/data_harness/sources/range_reader.py +418 -0
  220. mostlyright/data_harness/sources/registry.py +514 -0
  221. mostlyright/data_harness/sources/rights_rule.py +59 -0
  222. mostlyright/data_harness/sources/source_cadence_vectors.v1.json +1 -0
  223. mostlyright/data_harness/sources/sports.py +521 -0
  224. mostlyright/data_harness/sources/stream.py +524 -0
  225. mostlyright/data_harness/sources/stream_connector.py +418 -0
  226. mostlyright/data_harness/sources/stream_recorder.py +1404 -0
  227. mostlyright/data_harness/studio_boundary.py +2019 -0
  228. mostlyright/data_harness/thin/__init__.py +37 -0
  229. mostlyright/data_harness/thin/acquire.py +1137 -0
  230. mostlyright/data_harness/thin/acquire_cancel.py +579 -0
  231. mostlyright/data_harness/thin/approvals.py +617 -0
  232. mostlyright/data_harness/thin/commands.py +406 -0
  233. mostlyright/data_harness/thin/download.py +194 -0
  234. mostlyright/data_harness/thin/narrative.py +589 -0
  235. mostlyright/data_harness/thin/parity.py +1070 -0
  236. mostlyright/data_harness/thin/propose.py +2759 -0
  237. mostlyright/data_harness/thin/research.py +1663 -0
  238. mostlyright/data_harness/thin/router.py +924 -0
  239. mostlyright/data_harness/thin/runs.py +519 -0
  240. mostlyright/data_harness/thin/session.py +281 -0
  241. mostlyright/data_harness/thin/stream.py +501 -0
  242. mostlyright/data_harness/thin/transport.py +187 -0
  243. mostlyright/data_harness/thin/vocabulary.py +368 -0
  244. mostlyright/data_harness/thin/workers.py +164 -0
  245. mostlyright/data_harness/ucum/TABLE-PIN.json +40 -0
  246. mostlyright/data_harness/ucum/ucum-subset.v1.json +632 -0
  247. mostlyright/data_harness/unit_flow.py +927 -0
  248. mostlyright/data_harness/units.py +572 -0
  249. mostlyright/data_harness/ux/__init__.py +9 -0
  250. mostlyright/data_harness/ux/approve.py +485 -0
  251. mostlyright/data_harness/ux/author_yaml.py +597 -0
  252. mostlyright/data_harness/ux/cloud_auth.py +447 -0
  253. mostlyright/data_harness/ux/commands/__init__.py +260 -0
  254. mostlyright/data_harness/ux/commands/approve.py +136 -0
  255. mostlyright/data_harness/ux/commands/auth.py +744 -0
  256. mostlyright/data_harness/ux/commands/author.py +79 -0
  257. mostlyright/data_harness/ux/commands/catalog_author.py +403 -0
  258. mostlyright/data_harness/ux/commands/catalog_fill.py +523 -0
  259. mostlyright/data_harness/ux/commands/catalog_harvest.py +545 -0
  260. mostlyright/data_harness/ux/commands/catalog_publish.py +1838 -0
  261. mostlyright/data_harness/ux/commands/catalog_search.py +71 -0
  262. mostlyright/data_harness/ux/commands/catalog_update.py +437 -0
  263. mostlyright/data_harness/ux/commands/deploy.py +134 -0
  264. mostlyright/data_harness/ux/commands/deploy_dataset.py +98 -0
  265. mostlyright/data_harness/ux/commands/deploy_plan.py +105 -0
  266. mostlyright/data_harness/ux/commands/deploy_status.py +104 -0
  267. mostlyright/data_harness/ux/commands/diff.py +74 -0
  268. mostlyright/data_harness/ux/commands/index.py +84 -0
  269. mostlyright/data_harness/ux/commands/inventory.py +47 -0
  270. mostlyright/data_harness/ux/commands/list_builds.py +143 -0
  271. mostlyright/data_harness/ux/commands/login.py +63 -0
  272. mostlyright/data_harness/ux/commands/peek.py +236 -0
  273. mostlyright/data_harness/ux/commands/plan_check.py +90 -0
  274. mostlyright/data_harness/ux/commands/preflight.py +97 -0
  275. mostlyright/data_harness/ux/commands/record.py +107 -0
  276. mostlyright/data_harness/ux/commands/review_setup.py +47 -0
  277. mostlyright/data_harness/ux/commands/search.py +440 -0
  278. mostlyright/data_harness/ux/commands/show.py +61 -0
  279. mostlyright/data_harness/ux/commands/whoami.py +37 -0
  280. mostlyright/data_harness/ux/credential_native.py +551 -0
  281. mostlyright/data_harness/ux/credential_store.py +1055 -0
  282. mostlyright/data_harness/ux/credentials.py +631 -0
  283. mostlyright/data_harness/ux/diffing.py +444 -0
  284. mostlyright/data_harness/ux/headline.py +671 -0
  285. mostlyright/data_harness/ux/hosted_acquisition.py +974 -0
  286. mostlyright/data_harness/ux/hosted_run_status.py +619 -0
  287. mostlyright/data_harness/ux/inventory.py +427 -0
  288. mostlyright/data_harness/ux/local_review.py +375 -0
  289. mostlyright/data_harness/ux/login.py +691 -0
  290. mostlyright/data_harness/ux/path_kind.py +147 -0
  291. mostlyright/data_harness/ux/peek.py +1000 -0
  292. mostlyright/data_harness/ux/plain_file.py +178 -0
  293. mostlyright/data_harness/ux/plan_check.py +311 -0
  294. mostlyright/data_harness/ux/preflight.py +918 -0
  295. mostlyright/data_harness/ux/readers.py +1124 -0
  296. mostlyright/data_harness/ux/remediation.py +2195 -0
  297. mostlyright/data_harness/ux/render.py +657 -0
  298. mostlyright/data_harness/ux/workload.py +1077 -0
  299. mostlyright/data_harness/viewer.py +3713 -0
  300. mostlyright/data_harness/visual_run/__init__.py +83 -0
  301. mostlyright/data_harness/visual_run/authoring.py +235 -0
  302. mostlyright/data_harness/visual_run/contracts.py +673 -0
  303. mostlyright/data_harness/visual_run/materialize.py +486 -0
  304. mostlyright/data_harness/visual_run/observations.py +874 -0
  305. mostlyright/data_harness/visual_run/query.py +259 -0
  306. mostlyright/data_harness/visual_run/reducer.py +280 -0
  307. mostlyright/data_harness/visual_run/sdk.py +892 -0
  308. mostlyright/data_harness/visual_run/store.py +584 -0
  309. mostlyright/data_harness/visual_run/transport.py +239 -0
  310. mostlyright/data_harness/watch.py +2999 -0
  311. mostlyright_data-0.9.0.dist-info/METADATA +607 -0
  312. mostlyright_data-0.9.0.dist-info/RECORD +314 -0
  313. mostlyright_data-0.9.0.dist-info/WHEEL +4 -0
  314. mostlyright_data-0.9.0.dist-info/entry_points.txt +12 -0
@@ -0,0 +1,2773 @@
1
+ """Pinned-peer HTTPS retrieval with redirect and resource bounds."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import contextlib
6
+ import errno
7
+ import hashlib
8
+ import http.client
9
+ import json
10
+ import os
11
+ import re
12
+ import socket
13
+ import ssl
14
+ import stat
15
+ import threading
16
+ import time
17
+ from collections.abc import Callable, Iterator, Mapping
18
+ from dataclasses import dataclass, replace
19
+ from pathlib import Path, PurePosixPath
20
+ from typing import Protocol
21
+ from urllib.parse import urljoin, urlsplit
22
+
23
+ try:
24
+ import fcntl
25
+ except ImportError: # pragma: no cover - sealing already needs dir_fd links, which Windows lacks
26
+ fcntl = None # type: ignore[assignment]
27
+
28
+ from mostlyright.data_harness.acquisition.retention_admission import (
29
+ configured_retention_evidence_root,
30
+ require_snapshot_admission,
31
+ )
32
+ from mostlyright.data_harness.acquisition.url_policy import (
33
+ AcquisitionSecurityError,
34
+ EgressPolicy,
35
+ Resolver,
36
+ ValidatedTarget,
37
+ require_approved_peer,
38
+ validate_public_https_url,
39
+ )
40
+ from mostlyright.data_harness.canonical import canonical_sha256, sha256_bytes
41
+ from mostlyright.data_harness.formats import (
42
+ DIRECT_FETCH_MEDIA_TYPES,
43
+ SNAPSHOT_SUFFIXES,
44
+ )
45
+ from mostlyright.data_harness.readers.registry import TOOLBOX, ReaderRegistry
46
+
47
+ MAX_HEADER_COUNT = 128
48
+ MAX_HEADER_BYTES = 32_768
49
+ REDIRECT_STATUSES = frozenset({301, 302, 303, 307, 308})
50
+ _REDIRECT_HEADER = "location"
51
+ _ALLOWED_RESPONSE_ENCODINGS = frozenset({"", "identity"})
52
+ DATAGOV_V4_HOST = "api.gsa.gov"
53
+ DATAGOV_V4_PATH = "/technology/datagov/v4/search"
54
+ DATAGOV_V4_AUTHENTICATION_KIND = "api_data_gov_x_api_key_file"
55
+ MAX_DATAGOV_KEY_BYTES = 4_096
56
+ MAX_DATAGOV_RATE_SECONDS = 86_400
57
+
58
+ # The default media-type allowlist, derived from the format table rather than restated as a
59
+ # literal. This is a live enforcement site, not documentation: a response whose content
60
+ # type is outside it is refused as ``MEDIA_TYPE``, and the outgoing ``Accept`` header is
61
+ # built from it. Restating the union here meant a format added to ``formats.py`` was
62
+ # admitted by the recipe, the parser, the sandbox policy and the deletion sweep while every
63
+ # HTTPS fetch of it was refused and it was never even requested -- and the anti-duplication
64
+ # gate could not see the drift, because these are media types rather than format names.
65
+ # Sorted so the header is deterministic; a coordinator may still narrow the tuple.
66
+ #
67
+ # Derived from the direct-fetch map rather than the sealing table: a source with no Reader pin
68
+ # also admits the weak label a publisher sends when it declines to describe its file, and
69
+ # ``formats.DIRECT_FETCH_MEDIA_TYPES`` is where that is decided and reasoned about. A pinned
70
+ # fetch does not read this tuple at all -- ``reader_retrieval_limits`` below replaces it with
71
+ # the pinned family's own declaration.
72
+ #
73
+ # Deduplicated, because one label may name more than one format: ``text/plain`` is the weak
74
+ # label for csv, json and ndjson alike. Without the set the union repeats it once per format,
75
+ # and this tuple builds the outgoing ``Accept`` header as well as gating the response.
76
+ _TABLE_MEDIA_TYPES: tuple[str, ...] = tuple(
77
+ sorted(
78
+ {media_type for accepted in DIRECT_FETCH_MEDIA_TYPES.values() for media_type in accepted}
79
+ )
80
+ )
81
+
82
+ # The metadata channel: media types a per-fetch narrowing may name even when the parent retriever's
83
+ # own allowlist does not carry them.
84
+ #
85
+ # ``allowed_media_types`` defaults to the *payload* format table, which is a closed vocabulary of
86
+ # encodings a Reader can open. A catalogue's structure message is not a payload and never will be
87
+ # in that table -- an SDMX 2.1 dataflow message describes dataflows, and nothing downstream parses
88
+ # it as data. Without this channel the two contracts contradict each other: a coordinator building
89
+ # a retriever with ``RetrievalLimits()`` defaults could not narrow to the SDMX harvester's declared
90
+ # type at all, so ``narrowed`` refused the fetch before a request was made, and the only way out
91
+ # was for the coordinator to widen its *payload* allowlist for every other fetch -- the opposite of
92
+ # what this seam exists for.
93
+ #
94
+ # Two properties keep this from being a hole. It is a closed vocabulary declared here, in the
95
+ # transport, so a caller cannot name an arbitrary type through it -- ``text/html`` is not a member,
96
+ # which is what keeps the bot-wall refusal intact. And membership is decided by one rule: a member
97
+ # describes a *catalogue's own description of itself*, never bytes any Reader opens. A type that
98
+ # fails that rule belongs in ``formats.py`` or nowhere.
99
+ METADATA_MEDIA_TYPES: tuple[str, ...] = ("application/vnd.sdmx.structure+xml",)
100
+
101
+ # Spellings of a media type that mean the media type beside them, canonicalised by
102
+ # ``_media_type`` the moment a ``Content-Type`` header is read. ``binary/octet-stream`` is
103
+ # S3's own default for an object it was told nothing about, and it is what the NOAA GFS
104
+ # mirror this boundary exists for answers with -- for the sidecar and for the object both.
105
+ # It is not a second admitted type: it is the same "the server declined to say anything"
106
+ # signal under another name, and reading it as one keeps a real source from being refused
107
+ # for a spelling while leaving every allowlist in this package written once.
108
+ #
109
+ # Anything not in this table passes through untouched. A table of equivalences is not a
110
+ # widening: no type is admitted here that an allowlist does not already admit under its
111
+ # canonical name, and the structural checks behind the sandbox boundary remain the only
112
+ # thing that decides whether bytes are the bytes that were ordered.
113
+ MEDIA_TYPE_ALIASES: dict[str, str] = {"binary/octet-stream": "application/octet-stream"}
114
+
115
+ # The two media types a slice fetch meets: ``application/octet-stream`` for the object's
116
+ # bytes and ``text/plain`` for the ``.idx`` sidecar that says where in the object to look.
117
+ #
118
+ # They are deliberately not in ``formats.py``. That table is one closed vocabulary read by
119
+ # the recipe, the parser, the sandbox policy and the deletion sweep; adding a media type
120
+ # there admits it for every format at every one of those layers, and neither of these is a
121
+ # format -- no parser accepts them and no plan may name them. ``allowed_media_types`` is
122
+ # already a per-``RetrievalLimits`` field and narrowing it is an explicitly preserved right,
123
+ # so the slice-fetch caller passes this tuple for its own calls and no other call is widened.
124
+ #
125
+ # ``application/octet-stream`` is a weak signal and is treated as one: it says only that a
126
+ # server declined to say anything. It is not evidence that these are the ordered bytes. The
127
+ # control for that is the structural check on the slice itself, behind the sandbox boundary;
128
+ # this tuple only keeps a response from being refused for the one honest thing it said.
129
+ #
130
+ # The pair is named in halves because a call narrows to the half it is fetching. The sidecar
131
+ # half admits ``text/plain`` and the byte-stream spelling both, because a real object store
132
+ # answers the ``.idx`` as an opaque byte stream: the NOAA GFS S3 mirror this boundary was built
133
+ # against answers ``binary/octet-stream`` for the sidecar and for the object alike, verified
134
+ # against the live source on 2026-08-08. Admitting only ``text/plain`` there refused the
135
+ # sidecar before a single slice could be planned, and the survey half reported the very same
136
+ # response as present -- so look-then-lock locked a run the fetch could never take a byte of.
137
+ # The union below is derived from the halves rather than restated, so narrowing per call cannot
138
+ # drift from the pair this boundary admits.
139
+ #
140
+ # Sorted, like the tuple above, so the outgoing ``Accept`` header is deterministic.
141
+ SIDECAR_MEDIA_TYPES: tuple[str, ...] = ("application/octet-stream", "text/plain")
142
+ SLICE_OBJECT_MEDIA_TYPES: tuple[str, ...] = ("application/octet-stream",)
143
+ SLICE_FETCH_MEDIA_TYPES: tuple[str, ...] = tuple(
144
+ sorted(set(SIDECAR_MEDIA_TYPES + SLICE_OBJECT_MEDIA_TYPES))
145
+ )
146
+
147
+ # Byte offsets stay inside the exactly-representable integer range, because a range rides the
148
+ # sealed hop evidence and must survive every reader of that evidence unchanged. The span
149
+ # ceiling matches the ceiling ``max_response_bytes`` is itself bounded by: no single ordered
150
+ # slice may be larger than the largest response this boundary would ever accept.
151
+ MAX_BYTE_RANGE_OFFSET = (1 << 53) - 1
152
+ MAX_BYTE_RANGE_LENGTH = 1 << 34
153
+
154
+ # The largest range plan one acquisition may order. A slice fetch is many small requests
155
+ # against one object, so an unbounded plan is a denial-of-service lever on both this process
156
+ # and the source; 64 covers the 10-to-50 slice acquisitions this boundary exists for.
157
+ MAX_RANGE_PLAN = 64
158
+
159
+ NOT_FOUND_STATUS = 404
160
+ PARTIAL_CONTENT_STATUS = 206
161
+ # The typed refusal code that means "this address is not published". Named here rather than
162
+ # left as a literal at the raise site because the availability survey decides absence by
163
+ # comparing against this exact value: a consumer that restated the spelling would keep
164
+ # compiling and quietly stop recognising absence the day this one changed, and the failure
165
+ # would look like a source that is permanently broken rather than like a mismatch.
166
+ HTTP_NOT_FOUND = "HTTP_NOT_FOUND"
167
+ _MULTIPART_BYTERANGES = "multipart/byteranges"
168
+ # ``bytes <first>-<last>/<complete>``: exactly one space after the unit, decimal integers in
169
+ # canonical form, and either a complete length or ``*`` for an unknown one. Every other shape
170
+ # a server might send -- another unit, a list of spans, extra whitespace, a ``*`` span, a
171
+ # request-form ``bytes=`` -- is refused rather than interpreted, because a span that is read
172
+ # loosely cannot be compared exactly against the span that was ordered. Leading zeros are
173
+ # refused so one delivered span has one header spelling in the sealed evidence, and the digit
174
+ # bound keeps a header from carrying an offset no object could have.
175
+ _DECIMAL = r"(?:0|[1-9][0-9]{0,15})"
176
+ _CONTENT_RANGE = re.compile(rf"bytes ({_DECIMAL})-({_DECIMAL})/({_DECIMAL}|\*)")
177
+ # The same spelling on its own, for ``Content-Length``. Built from ``_DECIMAL`` rather than
178
+ # written out again, so one edit moves both and the two headers cannot drift apart.
179
+ _DECIMAL_ONLY = re.compile(_DECIMAL)
180
+
181
+
182
+ @dataclass(frozen=True)
183
+ class DatagovRateObservation:
184
+ """Validated, secret-free API-gateway rate metadata for one v4 response."""
185
+
186
+ limit: int
187
+ remaining: int
188
+ retry_after_seconds: int | None
189
+ reset_epoch_seconds: int | None
190
+
191
+ def __post_init__(self) -> None:
192
+ if type(self.limit) is not int or not 1 <= self.limit <= 1_000_000_000:
193
+ raise AcquisitionSecurityError("DATAGOV_RATE", "rate limit is invalid")
194
+ if type(self.remaining) is not int or not 0 <= self.remaining <= self.limit:
195
+ raise AcquisitionSecurityError("DATAGOV_RATE", "rate remaining is invalid")
196
+ if self.retry_after_seconds is not None and (
197
+ type(self.retry_after_seconds) is not int
198
+ or not 0 <= self.retry_after_seconds <= MAX_DATAGOV_RATE_SECONDS
199
+ ):
200
+ raise AcquisitionSecurityError("DATAGOV_RATE", "Retry-After is invalid")
201
+ if self.reset_epoch_seconds is not None and (
202
+ type(self.reset_epoch_seconds) is not int
203
+ or not 0 <= self.reset_epoch_seconds <= (1 << 53) - 1
204
+ ):
205
+ raise AcquisitionSecurityError("DATAGOV_RATE", "rate reset is invalid")
206
+
207
+ def to_dict(self) -> dict[str, int | None]:
208
+ return {
209
+ "limit": self.limit,
210
+ "remaining": self.remaining,
211
+ "retry_after_seconds": self.retry_after_seconds,
212
+ "reset_epoch_seconds": self.reset_epoch_seconds,
213
+ }
214
+
215
+
216
+ @dataclass(frozen=True, repr=False)
217
+ class DatagovV4Authorization:
218
+ """One in-memory Data.gov key whose only rendering is the exact v4 header."""
219
+
220
+ _key: str
221
+ authentication_kind: str = DATAGOV_V4_AUTHENTICATION_KIND
222
+
223
+ def __post_init__(self) -> None:
224
+ if self.authentication_kind != DATAGOV_V4_AUTHENTICATION_KIND:
225
+ raise AcquisitionSecurityError(
226
+ "DATAGOV_AUTHORIZATION", "unsupported Data.gov authentication kind"
227
+ )
228
+ if not isinstance(self._key, str) or not self._key:
229
+ raise AcquisitionSecurityError("DATAGOV_AUTHORIZATION", "Data.gov key is empty")
230
+
231
+ @classmethod
232
+ def from_key_file(cls, path: Path) -> DatagovV4Authorization:
233
+ """Read one key through a retained no-follow parent chain and stable descriptor."""
234
+
235
+ raw_path = os.fspath(path)
236
+ if not raw_path or "\x00" in raw_path:
237
+ raise AcquisitionSecurityError("DATAGOV_KEY_FILE", "key file path is invalid")
238
+ absolute = Path(os.path.abspath(raw_path))
239
+ if absolute.name in {"", ".", ".."}:
240
+ raise AcquisitionSecurityError("DATAGOV_KEY_FILE", "key file path is invalid")
241
+ directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0)
242
+ owned: list[int] = []
243
+ descriptor = -1
244
+ try:
245
+ parent = os.open("/", directory_flags)
246
+ owned.append(parent)
247
+ for component in absolute.parts[1:-1]:
248
+ parent = os.open(component, directory_flags, dir_fd=parent)
249
+ owned.append(parent)
250
+ flags = (
251
+ os.O_RDONLY
252
+ | getattr(os, "O_NOFOLLOW", 0)
253
+ | getattr(os, "O_CLOEXEC", 0)
254
+ | getattr(os, "O_NONBLOCK", 0)
255
+ )
256
+ descriptor = os.open(absolute.name, flags, dir_fd=parent)
257
+ before = os.fstat(descriptor)
258
+ _require_datagov_key_stat(before)
259
+ raw = _read_datagov_key(descriptor, expected_size=before.st_size)
260
+ after = os.fstat(descriptor)
261
+ if _stable_file_signature(before) != _stable_file_signature(after):
262
+ raise AcquisitionSecurityError(
263
+ "DATAGOV_KEY_RACE", "key file changed during its bounded read"
264
+ )
265
+ except AcquisitionSecurityError:
266
+ raise
267
+ except OSError:
268
+ raise AcquisitionSecurityError(
269
+ "DATAGOV_KEY_FILE", "key file must be a retained no-follow regular file"
270
+ ) from None
271
+ finally:
272
+ if descriptor >= 0:
273
+ os.close(descriptor)
274
+ for opened in reversed(owned):
275
+ os.close(opened)
276
+ try:
277
+ key = raw.decode("ascii", errors="strict")
278
+ except UnicodeDecodeError:
279
+ raise AcquisitionSecurityError(
280
+ "DATAGOV_KEY_FILE", "key file must contain one ASCII line"
281
+ ) from None
282
+ if key.endswith("\n"):
283
+ key = key[:-1]
284
+ if (
285
+ not key
286
+ or "\n" in key
287
+ or "\r" in key
288
+ or any(not 0x21 <= ord(char) <= 0x7E for char in key)
289
+ ):
290
+ raise AcquisitionSecurityError(
291
+ "DATAGOV_KEY_FILE", "key file must contain one nonempty printable ASCII line"
292
+ )
293
+ return cls(key)
294
+
295
+ def __repr__(self) -> str:
296
+ return (
297
+ "DatagovV4Authorization("
298
+ f"authentication_kind={self.authentication_kind!r}, key=<redacted>)"
299
+ )
300
+
301
+ def header_for(self, url: str) -> tuple[str, str]:
302
+ parsed = urlsplit(url)
303
+ if (
304
+ parsed.scheme != "https"
305
+ or parsed.hostname != DATAGOV_V4_HOST
306
+ or parsed.port not in {None, 443}
307
+ or parsed.username is not None
308
+ or parsed.password is not None
309
+ or parsed.path != DATAGOV_V4_PATH
310
+ or parsed.fragment
311
+ ):
312
+ raise AcquisitionSecurityError(
313
+ "DATAGOV_AUTHORIZATION",
314
+ "Data.gov authorization is scoped to the exact v4 search endpoint",
315
+ )
316
+ return "X-Api-Key", self._key
317
+
318
+ def reject_echoed_response(self, payload: bytes) -> None:
319
+ """Reject raw or JSON-equivalent key echoes while the key remains ephemeral."""
320
+
321
+ key_bytes = self._key.encode("ascii", errors="strict")
322
+ if key_bytes in payload:
323
+ raise AcquisitionSecurityError(
324
+ "DATAGOV_SECRET_ECHO",
325
+ "Data.gov response echoed authorization text and was not retained",
326
+ )
327
+ try:
328
+ document = json.loads(
329
+ payload.decode("utf-8", errors="strict"),
330
+ object_pairs_hook=_DatagovEchoObject,
331
+ parse_constant=lambda _value: (_ for _ in ()).throw(ValueError()),
332
+ )
333
+ except (UnicodeDecodeError, json.JSONDecodeError, RecursionError, ValueError):
334
+ raise AcquisitionSecurityError(
335
+ "DATAGOV_RESPONSE_UNSCANNABLE",
336
+ "Data.gov response could not be fully inspected before retention",
337
+ ) from None
338
+ pending = [document]
339
+ visited = 0
340
+ while pending:
341
+ value = pending.pop()
342
+ visited += 1
343
+ if visited > 1_000_000:
344
+ raise AcquisitionSecurityError(
345
+ "DATAGOV_RESPONSE_STRUCTURE",
346
+ "Data.gov response exceeds the decoded traversal bound",
347
+ )
348
+ if isinstance(value, str):
349
+ if self._key in value:
350
+ raise AcquisitionSecurityError(
351
+ "DATAGOV_SECRET_ECHO",
352
+ "Data.gov response echoed authorization text and was not retained",
353
+ )
354
+ elif isinstance(value, _DatagovEchoObject):
355
+ for key, member in value:
356
+ pending.append(key)
357
+ pending.append(member)
358
+ elif isinstance(value, list):
359
+ pending.extend(value)
360
+
361
+ def reject_echoed_headers(self, headers: tuple[tuple[str, str], ...]) -> None:
362
+ """Reject the key in any response-header name or value before hashing or parsing."""
363
+
364
+ for name, value in headers:
365
+ if self._key in name or self._key in value:
366
+ raise AcquisitionSecurityError(
367
+ "DATAGOV_SECRET_ECHO",
368
+ "Data.gov response echoed authorization text and was not retained",
369
+ )
370
+
371
+
372
+ class _DatagovEchoObject(list[tuple[str, object]]):
373
+ """JSON object retaining duplicate keys for authorization echo inspection."""
374
+
375
+
376
+ def _require_datagov_key_stat(info: os.stat_result) -> None:
377
+ if (
378
+ not stat.S_ISREG(info.st_mode)
379
+ or info.st_uid != os.geteuid()
380
+ or info.st_nlink != 1
381
+ or stat.S_IMODE(info.st_mode) & 0o077
382
+ or not stat.S_IMODE(info.st_mode) & stat.S_IRUSR
383
+ or not 1 <= info.st_size <= MAX_DATAGOV_KEY_BYTES
384
+ ):
385
+ raise AcquisitionSecurityError(
386
+ "DATAGOV_KEY_FILE",
387
+ "key file must be owner-readable, owner-owned, private, bounded, and single-link",
388
+ )
389
+
390
+
391
+ def _read_datagov_key(descriptor: int, *, expected_size: int) -> bytes:
392
+ chunks: list[bytes] = []
393
+ observed = 0
394
+ while observed <= MAX_DATAGOV_KEY_BYTES:
395
+ chunk = os.read(descriptor, min(512, MAX_DATAGOV_KEY_BYTES + 1 - observed))
396
+ if not chunk:
397
+ break
398
+ observed += len(chunk)
399
+ chunks.append(chunk)
400
+ if observed != expected_size or observed > MAX_DATAGOV_KEY_BYTES:
401
+ raise AcquisitionSecurityError("DATAGOV_KEY_RACE", "key file size changed during read")
402
+ return b"".join(chunks)
403
+
404
+
405
+ def _stable_file_signature(info: os.stat_result) -> tuple[int, ...]:
406
+ return (
407
+ info.st_dev,
408
+ info.st_ino,
409
+ info.st_uid,
410
+ info.st_gid,
411
+ info.st_mode,
412
+ info.st_nlink,
413
+ info.st_size,
414
+ info.st_mtime_ns,
415
+ info.st_ctime_ns,
416
+ )
417
+
418
+
419
+ @dataclass(frozen=True)
420
+ class ByteRange:
421
+ """One closed byte span, inclusive on both ends, exactly as RFC 9110 numbers them.
422
+
423
+ There is deliberately no open-ended (``bytes=N-``) or suffix (``bytes=-N``) form. With
424
+ either of those, "the span that was ordered" is not a fact the client holds before the
425
+ response arrives, so the delivery check could not compare what came back against what was
426
+ asked for -- which is the entire purpose of ordering a slice.
427
+ """
428
+
429
+ first_byte: int
430
+ last_byte: int
431
+
432
+ def __post_init__(self) -> None:
433
+ for name, value in (("first_byte", self.first_byte), ("last_byte", self.last_byte)):
434
+ if type(value) is not int:
435
+ raise AcquisitionSecurityError("BYTE_RANGE", f"{name} must be an integer")
436
+ if not 0 <= value <= MAX_BYTE_RANGE_OFFSET:
437
+ raise AcquisitionSecurityError(
438
+ "BYTE_RANGE",
439
+ f"{name} must be in [0, {MAX_BYTE_RANGE_OFFSET}]",
440
+ )
441
+ if self.last_byte < self.first_byte:
442
+ raise AcquisitionSecurityError(
443
+ "BYTE_RANGE",
444
+ "byte range must end at or after it begins",
445
+ )
446
+ if self.last_byte - self.first_byte + 1 > MAX_BYTE_RANGE_LENGTH:
447
+ raise AcquisitionSecurityError(
448
+ "BYTE_RANGE",
449
+ f"byte range length must not exceed {MAX_BYTE_RANGE_LENGTH}",
450
+ )
451
+
452
+ @property
453
+ def length(self) -> int:
454
+ return self.last_byte - self.first_byte + 1
455
+
456
+ @property
457
+ def header_value(self) -> str:
458
+ return f"bytes={self.first_byte}-{self.last_byte}"
459
+
460
+
461
+ # The span a presence probe orders: the first byte and no more. A probe that read further
462
+ # would be an acquisition nobody ordered, and the whole point of the look half is that it
463
+ # costs the source next to nothing. Held here rather than at the call site so no caller can
464
+ # widen it.
465
+ _PRESENCE_PROBE_RANGE = ByteRange(first_byte=0, last_byte=0)
466
+
467
+
468
+ @dataclass(frozen=True)
469
+ class RetrievalLimits:
470
+ max_response_bytes: int = 16 * 1024 * 1024
471
+ max_aggregate_response_bytes: int = 1 << 34
472
+ max_redirects: int = 5
473
+ max_requests: int = 6
474
+ # How many ranged requests one acquisition may plan. This is deliberately a separate
475
+ # budget from ``max_requests``: raising that one to 50 so a 50-slice acquisition could
476
+ # afford its requests would also loosen the ``max_requests >= max_redirects + 1``
477
+ # relation below, which is a redirect control and has nothing to do with slices. The
478
+ # redirect budget therefore stays exactly where it is.
479
+ max_ranges: int = MAX_RANGE_PLAN
480
+ connect_timeout_seconds: float = 10.0
481
+ read_timeout_seconds: float = 30.0
482
+ total_timeout_seconds: float = 45.0
483
+ max_concurrency: int = 4
484
+ min_interval_seconds: float = 0.0
485
+ allowed_media_types: tuple[str, ...] = _TABLE_MEDIA_TYPES
486
+ user_agent: str = "MostlyRightDataHarness/1.0"
487
+
488
+ def __post_init__(self) -> None:
489
+ if type(self.max_response_bytes) is not int or not 1 <= self.max_response_bytes <= 1 << 34:
490
+ raise AcquisitionSecurityError(
491
+ "RESPONSE_LIMIT",
492
+ "max_response_bytes must be a positive bounded integer",
493
+ )
494
+ if (
495
+ type(self.max_aggregate_response_bytes) is not int
496
+ or not 1 <= self.max_aggregate_response_bytes <= 1 << 40
497
+ ):
498
+ raise AcquisitionSecurityError(
499
+ "AGGREGATE_RESPONSE_LIMIT",
500
+ "max_aggregate_response_bytes must be a positive bounded integer",
501
+ )
502
+ if type(self.max_redirects) is not int or not 0 <= self.max_redirects <= 10:
503
+ raise AcquisitionSecurityError(
504
+ "REDIRECT_LIMIT",
505
+ "max_redirects must be an integer in [0, 10]",
506
+ )
507
+ if type(self.max_requests) is not int or not 1 <= self.max_requests <= 16:
508
+ raise AcquisitionSecurityError(
509
+ "REQUEST_LIMIT",
510
+ "max_requests must be an integer in [1, 16]",
511
+ )
512
+ if self.max_requests < self.max_redirects + 1:
513
+ raise AcquisitionSecurityError(
514
+ "REQUEST_LIMIT",
515
+ "max_requests must allow the configured redirect chain",
516
+ )
517
+ if type(self.max_ranges) is not int or not 1 <= self.max_ranges <= MAX_RANGE_PLAN:
518
+ raise AcquisitionSecurityError(
519
+ "RANGE_LIMIT",
520
+ f"max_ranges must be an integer in [1, {MAX_RANGE_PLAN}]",
521
+ )
522
+ for name, value, maximum in (
523
+ ("connect_timeout_seconds", self.connect_timeout_seconds, 60.0),
524
+ ("read_timeout_seconds", self.read_timeout_seconds, 300.0),
525
+ ("total_timeout_seconds", self.total_timeout_seconds, 600.0),
526
+ ):
527
+ if type(value) not in {int, float} or not 0 < value <= maximum:
528
+ raise AcquisitionSecurityError(
529
+ "TIMEOUT",
530
+ f"{name} must be in (0, {maximum}]",
531
+ )
532
+ if type(self.max_concurrency) is not int or not 1 <= self.max_concurrency <= 64:
533
+ raise AcquisitionSecurityError(
534
+ "CONCURRENCY_LIMIT",
535
+ "max_concurrency must be an integer in [1, 64]",
536
+ )
537
+ if (
538
+ type(self.min_interval_seconds) not in {int, float}
539
+ or not 0 <= self.min_interval_seconds <= 3_600
540
+ ):
541
+ raise AcquisitionSecurityError(
542
+ "RATE_LIMIT",
543
+ "min_interval_seconds must be in [0, 3600]",
544
+ )
545
+ if not self.allowed_media_types or len(set(self.allowed_media_types)) != len(
546
+ self.allowed_media_types
547
+ ):
548
+ raise AcquisitionSecurityError(
549
+ "MEDIA_ALLOWLIST",
550
+ "allowed media types must be non-empty and unique",
551
+ )
552
+ # An allowlist is compared against what ``_media_type`` returns, and that is always
553
+ # the canonical spelling. An alias written here would therefore match nothing and
554
+ # refuse every response it was added to admit -- a silent hole rather than a loud
555
+ # one -- so it is refused at construction, where the caller can still be told.
556
+ aliased = [
557
+ media_type
558
+ for media_type in self.allowed_media_types
559
+ if media_type in MEDIA_TYPE_ALIASES
560
+ ]
561
+ if aliased:
562
+ raise AcquisitionSecurityError(
563
+ "MEDIA_ALLOWLIST",
564
+ f"{aliased[0]!r} is an alias of {MEDIA_TYPE_ALIASES[aliased[0]]!r}; "
565
+ f"allowlist the canonical spelling, which admits both",
566
+ )
567
+ if not self.user_agent or "\r" in self.user_agent or "\n" in self.user_agent:
568
+ raise AcquisitionSecurityError("USER_AGENT", "user agent is invalid")
569
+
570
+
571
+ def reader_retrieval_limits(
572
+ limits: RetrievalLimits,
573
+ *,
574
+ family_id: str,
575
+ family_version: str,
576
+ registry: ReaderRegistry | None = None,
577
+ ) -> RetrievalLimits:
578
+ """The limits a fetch for a Reader-pinned source runs under.
579
+
580
+ The table-derived default above is right for a direct fetch and wrong for a pinned one: a
581
+ zipped source arrives as an archive media type that no wire format claims, so the fetch
582
+ would be refused before the Reader could ever see it. The admitted set for a pinned fetch
583
+ is therefore the pinned family's own ``accepted_media_types``.
584
+
585
+ Taken from the family rather than added to ``FORMAT_MEDIA_TYPES``, for two reasons that
586
+ point the same way. The format table answers "what encoding may be sealed", and a fetched
587
+ container is never sealed. And widening that table would widen the allowlist for every
588
+ *unpinned* fetch as well, which is the opposite of what is wanted -- a direct fetch must go
589
+ on admitting exactly what it admits today.
590
+
591
+ Narrowing to the one family rather than to ``registry.accepted_media_types()`` is
592
+ deliberate: the union is the Toolbox's outer bound, and a source pinned to a zip Reader has
593
+ no business admitting a spreadsheet. Every other bound on ``limits`` is carried through
594
+ untouched, so a coordinator's own narrowing still holds.
595
+
596
+ A family the closed table does not hold is refused here, by name, before any connection is
597
+ attempted.
598
+
599
+ ``registry`` defaults to ``None`` and is resolved to the shipped Toolbox at call time
600
+ rather than being bound as a default argument at import time, so which table answers is a
601
+ property of the running process and not of import order.
602
+ """
603
+
604
+ family = (registry if registry is not None else TOOLBOX).resolve(family_id, family_version)
605
+ return replace(limits, allowed_media_types=tuple(family.accepted_media_types))
606
+
607
+
608
+ def require_range_plan_within_budget(planned_ranges: int, *, max_ranges: int) -> None:
609
+ """Refuse a range plan that orders more slices than the acquisition's budget allows.
610
+
611
+ ``retrieve`` fetches one range per call, so the budget belongs to whoever plans a range
612
+ set rather than to a single request. The bound lives here, beside the limits it comes
613
+ from, so the range planner and the sandbox operation that calls it do not each restate it.
614
+ """
615
+
616
+ if type(max_ranges) is not int or not 1 <= max_ranges <= MAX_RANGE_PLAN:
617
+ raise AcquisitionSecurityError(
618
+ "RANGE_LIMIT",
619
+ f"range budget must be an integer in [1, {MAX_RANGE_PLAN}]",
620
+ )
621
+ if type(planned_ranges) is not int or planned_ranges < 1:
622
+ raise AcquisitionSecurityError(
623
+ "RANGE_LIMIT",
624
+ "a range plan must order at least one range",
625
+ )
626
+ if planned_ranges > max_ranges:
627
+ raise AcquisitionSecurityError(
628
+ "RANGE_LIMIT",
629
+ f"range plan of {planned_ranges} exceeds the budget of {max_ranges}",
630
+ )
631
+
632
+
633
+ @dataclass(frozen=True)
634
+ class TransportResponse:
635
+ status: int
636
+ headers: tuple[tuple[str, str], ...]
637
+ body: bytes
638
+ connected_peer: str
639
+ tls_verified: bool
640
+ hostname_verified: bool
641
+
642
+ def header(self, name: str) -> str | None:
643
+ wanted = name.lower()
644
+ values = [value for key, value in self.headers if key.lower() == wanted]
645
+ if len(values) > 1 and wanted in {
646
+ _REDIRECT_HEADER,
647
+ "content-length",
648
+ "content-type",
649
+ "content-range",
650
+ "retry-after",
651
+ "x-ratelimit-limit",
652
+ "x-ratelimit-remaining",
653
+ "x-ratelimit-reset",
654
+ # ``content-encoding`` belongs here for the same reason the other four do, and was
655
+ # missing. Returning the first of two would let a source send ``identity`` then
656
+ # ``gzip``: the encoding check reads the first and passes while the bytes are the
657
+ # second, and every offset in a range plan stops meaning anything. Nothing in
658
+ # this build decompresses, so the practical reach was small -- but "small because
659
+ # of something elsewhere" is not the property this set exists to give.
660
+ "content-encoding",
661
+ }:
662
+ raise AcquisitionSecurityError(
663
+ "RESPONSE_HEADER_DUPLICATE",
664
+ f"response repeated security-sensitive header {wanted}",
665
+ )
666
+ return values[0] if values else None
667
+
668
+
669
+ class PinnedTransport(Protocol):
670
+ def request(
671
+ self,
672
+ target: ValidatedTarget,
673
+ *,
674
+ approved_ip: str,
675
+ headers: Mapping[str, str],
676
+ limits: RetrievalLimits,
677
+ byte_range: ByteRange | None = None,
678
+ ) -> TransportResponse:
679
+ """Connect only to approved_ip while verifying target.hostname in TLS.
680
+
681
+ A slice is ordered through the typed ``byte_range`` parameter rather than through a
682
+ caller-supplied header mapping. A general ``extra_headers`` argument would put a
683
+ header-injection surface at the most security-sensitive seam in this package; a
684
+ ``ByteRange`` validates itself and renders to exactly one header value that no caller
685
+ can influence character by character.
686
+ """
687
+
688
+
689
+ class TransportFailureError(AcquisitionSecurityError):
690
+ """Retryable transport failure with exact response-body bytes already consumed."""
691
+
692
+ def __init__(self, detail: str, *, response_body_size_bytes: int) -> None:
693
+ if (
694
+ type(response_body_size_bytes) is not int
695
+ or not 0 <= response_body_size_bytes <= (1 << 34) + 1
696
+ ):
697
+ raise AcquisitionSecurityError(
698
+ "TRANSPORT_FAILURE",
699
+ "transport failure byte evidence is outside the admitted range",
700
+ )
701
+ self.response_body_size_bytes = response_body_size_bytes
702
+ super().__init__("TRANSPORT_FAILURE", detail)
703
+
704
+
705
+ @dataclass(frozen=True)
706
+ class PeerAttempt:
707
+ """Canonical evidence for one address from a resolver-approved peer set."""
708
+
709
+ approved_ip: str
710
+ outcome: str
711
+ failure_code: str | None
712
+ response_body_size_bytes: int = 0
713
+
714
+ def __post_init__(self) -> None:
715
+ if (
716
+ type(self.response_body_size_bytes) is not int
717
+ or not 0 <= self.response_body_size_bytes <= (1 << 34) + 1
718
+ ):
719
+ raise AcquisitionSecurityError(
720
+ "TRANSPORT_EVIDENCE",
721
+ "peer-attempt response-body bytes are outside the admitted range",
722
+ )
723
+
724
+ def to_dict(self) -> dict[str, object]:
725
+ return {
726
+ "approved_ip": self.approved_ip,
727
+ "outcome": self.outcome,
728
+ "failure_code": self.failure_code,
729
+ "response_body_size_bytes": self.response_body_size_bytes,
730
+ }
731
+
732
+
733
+ class PeerAttemptError(AcquisitionSecurityError):
734
+ """A typed retrieval refusal carrying redacted, ordered peer-attempt evidence.
735
+
736
+ ``hops`` is the exact sealed-evidence hop list the refused retrieval had already
737
+ measured when it refused -- every completed hop, including the one whose response was
738
+ consumed in full and then refused (a 200 answering a ranged probe, for instance). It
739
+ rides the refusal so a caller that legitimately continues after one (the ranged bulk
740
+ fetch falling back to its single-shot path) can seal what the refused attempt consumed
741
+ instead of letting those response bytes vanish from the evidence. It is the same
742
+ ``RetrievalHop`` type ``RetrievedBytes`` seals; nothing new is measured here.
743
+ """
744
+
745
+ def __init__(
746
+ self,
747
+ code: str,
748
+ detail: str,
749
+ peer_attempts: tuple[PeerAttempt, ...],
750
+ *,
751
+ hops: tuple[RetrievalHop, ...] = (),
752
+ ) -> None:
753
+ if any(not isinstance(attempt, PeerAttempt) for attempt in peer_attempts):
754
+ raise AcquisitionSecurityError(
755
+ "TRANSPORT_EVIDENCE",
756
+ "peer-attempt evidence contains an invalid entry",
757
+ )
758
+ if any(not isinstance(hop, RetrievalHop) for hop in hops):
759
+ raise AcquisitionSecurityError(
760
+ "TRANSPORT_EVIDENCE",
761
+ "peer-attempt hop evidence contains an invalid entry",
762
+ )
763
+ self.peer_attempts = peer_attempts
764
+ self.hops = hops
765
+ self.total_response_body_size_bytes = sum(
766
+ attempt.response_body_size_bytes for attempt in peer_attempts
767
+ )
768
+ super().__init__(code, detail)
769
+
770
+
771
+ class DatagovResponseRefused(AcquisitionSecurityError):
772
+ """Authenticated response refused before any secret-derived digest was constructed."""
773
+
774
+ def __init__(self, code: str, detail: str, *, status: int, network_bytes: int) -> None:
775
+ self.status = status
776
+ self.network_bytes = network_bytes
777
+ super().__init__(code, detail)
778
+
779
+
780
+ @dataclass(frozen=True)
781
+ class RetrievalHop:
782
+ url: str
783
+ resolution_digest: str
784
+ connected_peer: str
785
+ status: int
786
+ response_headers_digest: str
787
+ response_body_size_bytes: int
788
+ response_body_sha256: str
789
+ requested_range: str | None = None
790
+ observed_content_range: str | None = None
791
+ etag: str | None = None
792
+ last_modified: str | None = None
793
+ peer_attempts: tuple[PeerAttempt, ...] = ()
794
+
795
+ def to_dict(self) -> dict[str, object]:
796
+ # Both range keys are always present, ``None`` when the hop ordered no slice, so a hop
797
+ # has exactly one shape rather than two and the digest below covers the same fields on
798
+ # every acquisition.
799
+ return {
800
+ "url": self.url,
801
+ "resolution_digest": self.resolution_digest,
802
+ "connected_peer": self.connected_peer,
803
+ "status": self.status,
804
+ "response_headers_digest": self.response_headers_digest,
805
+ "response_body_size_bytes": self.response_body_size_bytes,
806
+ "response_body_sha256": self.response_body_sha256,
807
+ "requested_range": self.requested_range,
808
+ "observed_content_range": self.observed_content_range,
809
+ "etag": self.etag,
810
+ "last_modified": self.last_modified,
811
+ "peer_attempts": [attempt.to_dict() for attempt in self.peer_attempts],
812
+ }
813
+
814
+
815
+ @dataclass(frozen=True)
816
+ class ProbeTransport:
817
+ """The sanitized transport facts one whole-document retrieval proves about its response.
818
+
819
+ A cadence observation records the response status and the two validator headers, because a
820
+ publisher that answers 200 with an unchanged ``ETag`` is stating something about its own
821
+ editions. Neither validator leaves the confinement as text. Both are untrusted publisher
822
+ bytes of the publisher's own choosing, and the observation contract stores them hashed for
823
+ exactly that reason, so they are hashed here -- inside the Clean room, beside the response
824
+ they came from -- and only the digest crosses the boundary. The status is an integer from a
825
+ closed range and carries nothing a publisher composed.
826
+
827
+ This is the terminal response of the exchange. Redirect hops keep their own evidence in the
828
+ transport hop list; what an observation states about the publisher is what the address it
829
+ settled on answered.
830
+ """
831
+
832
+ http_status_code: int
833
+ etag_digest: str | None
834
+ last_modified_digest: str | None
835
+
836
+ def __post_init__(self) -> None:
837
+ if type(self.http_status_code) is not int or not 100 <= self.http_status_code <= 599:
838
+ raise AcquisitionSecurityError(
839
+ "PROBE_TRANSPORT",
840
+ "probe transport status is not a real HTTP status code",
841
+ )
842
+ for name, value in (
843
+ ("etag_digest", self.etag_digest),
844
+ ("last_modified_digest", self.last_modified_digest),
845
+ ):
846
+ if value is not None and (
847
+ not isinstance(value, str) or re.fullmatch(r"[0-9a-f]{64}", value) is None
848
+ ):
849
+ raise AcquisitionSecurityError(
850
+ "PROBE_TRANSPORT",
851
+ f"probe transport {name} is not a lowercase SHA-256 digest",
852
+ )
853
+
854
+ @classmethod
855
+ def from_terminal_hop(cls, hop: RetrievalHop) -> ProbeTransport:
856
+ """Seal the terminal hop's status and hashed validators."""
857
+
858
+ return cls(
859
+ http_status_code=hop.status,
860
+ etag_digest=None if hop.etag is None else sha256_bytes(hop.etag.encode("utf-8")),
861
+ last_modified_digest=(
862
+ None
863
+ if hop.last_modified is None
864
+ else sha256_bytes(hop.last_modified.encode("utf-8"))
865
+ ),
866
+ )
867
+
868
+ def to_dict(self) -> dict[str, object]:
869
+ return {
870
+ "http_status_code": self.http_status_code,
871
+ "etag_digest": self.etag_digest,
872
+ "last_modified_digest": self.last_modified_digest,
873
+ }
874
+
875
+
876
+ @dataclass(frozen=True)
877
+ class RetrievedBytes:
878
+ source_url: str
879
+ final_url: str
880
+ media_type: str
881
+ content: bytes
882
+ content_sha256: str
883
+ hops: tuple[RetrievalHop, ...]
884
+ transport_evidence_digest: str
885
+ complete_size_bytes: int | None = None
886
+ rate_observation: DatagovRateObservation | None = None
887
+
888
+ @property
889
+ def total_response_body_size_bytes(self) -> int:
890
+ """Every response-body byte consumed in the exact transport hop evidence.
891
+
892
+ ``len(content)`` is only the terminal response. Redirect response bodies consumed the
893
+ same transport budget, as do partial bodies from failed peer attempts, so both belong
894
+ in cumulative acquisition accounting. Deriving the total from the authenticated hop
895
+ tuple avoids a second summary fact that could disagree with its underlying evidence.
896
+ """
897
+
898
+ return sum(
899
+ hop.response_body_size_bytes
900
+ + sum(
901
+ attempt.response_body_size_bytes
902
+ for attempt in hop.peer_attempts[:-1]
903
+ if attempt.outcome == "transport_failure"
904
+ )
905
+ for hop in self.hops
906
+ )
907
+
908
+ def __post_init__(self) -> None:
909
+ if sha256_bytes(self.content) != self.content_sha256:
910
+ raise AcquisitionSecurityError(
911
+ "CONTENT_DIGEST",
912
+ "retrieved content digest does not match exact bytes",
913
+ )
914
+ expected = canonical_sha256([hop.to_dict() for hop in self.hops])
915
+ if expected != self.transport_evidence_digest:
916
+ raise AcquisitionSecurityError(
917
+ "TRANSPORT_EVIDENCE_DIGEST",
918
+ "transport evidence digest does not match the exact hop list",
919
+ )
920
+ if self.complete_size_bytes is not None and (
921
+ type(self.complete_size_bytes) is not int
922
+ or self.complete_size_bytes < len(self.content)
923
+ or self.complete_size_bytes > MAX_BYTE_RANGE_OFFSET + 1
924
+ ):
925
+ raise AcquisitionSecurityError(
926
+ "CONTENT_RANGE",
927
+ "complete object size is outside the admitted range",
928
+ )
929
+ if self.rate_observation is not None and not isinstance(
930
+ self.rate_observation, DatagovRateObservation
931
+ ):
932
+ raise AcquisitionSecurityError(
933
+ "DATAGOV_RATE", "rate observation must use the strict contract"
934
+ )
935
+
936
+
937
+ def probe_transport_for(retrieved: RetrievedBytes) -> ProbeTransport:
938
+ """Seal what the terminal response of one retrieval stated.
939
+
940
+ Taken from the retained hop evidence rather than from a summary field, so what an
941
+ observation says the publisher answered is the same response the transport digest covers.
942
+ """
943
+
944
+ if not retrieved.hops:
945
+ raise AcquisitionSecurityError(
946
+ "PROBE_TRANSPORT",
947
+ "a retrieval with no hop evidence has no terminal response",
948
+ )
949
+ return ProbeTransport.from_terminal_hop(retrieved.hops[-1])
950
+
951
+
952
+ class AcquisitionLimiter:
953
+ """Bounded concurrency, plus a per-host minimum interval that is waited out."""
954
+
955
+ def __init__(
956
+ self,
957
+ *,
958
+ max_concurrency: int,
959
+ min_interval_seconds: float,
960
+ clock: Callable[[], float] = time.monotonic,
961
+ sleep: Callable[[float], None] = time.sleep,
962
+ ) -> None:
963
+ # The interval is validated here as well as on ``RetrievalLimits``, because a limiter
964
+ # may be constructed directly and the interval is now slept on rather than raised on:
965
+ # an unbounded or non-numeric interval would be a hang inside our own worker rather
966
+ # than a refusal a caller sees. ``NaN`` fails this comparison and is refused with it.
967
+ if type(min_interval_seconds) not in {int, float} or not 0 <= min_interval_seconds <= 3_600:
968
+ raise AcquisitionSecurityError(
969
+ "RATE_LIMIT",
970
+ "min_interval_seconds must be in [0, 3600]",
971
+ )
972
+ self._semaphore = threading.BoundedSemaphore(max_concurrency)
973
+ self._min_interval_seconds = float(min_interval_seconds)
974
+ self._clock = clock
975
+ self._sleep = sleep
976
+ self._lock = threading.Lock()
977
+ self._last_request: dict[str, float] = {}
978
+
979
+ @property
980
+ def clock(self) -> Callable[[], float]:
981
+ """The clock every deadline handed to :meth:`acquire` must be an instant on."""
982
+
983
+ return self._clock
984
+
985
+ @contextlib.contextmanager
986
+ def acquire(self, hostname: str, *, deadline: float) -> Iterator[None]:
987
+ """Hold a concurrency slot, waiting out this host's minimum interval first.
988
+
989
+ Two limits, two dispositions, deliberately. An exhausted concurrency semaphore
990
+ fails fast: more work is in flight than this process admitted, which is a bug on our
991
+ side, and waiting would hide it. The per-host interval waits: it is politeness
992
+ towards a source that is behaving exactly as documented, and raising there turns a
993
+ healthy source into a failed run.
994
+
995
+ ``deadline`` is an absolute instant on this limiter's own clock -- the retrieval
996
+ passes its start plus the ``total_timeout_seconds`` it already measures itself
997
+ against. The bound is not optional: ``min_interval_seconds`` is admitted up to an
998
+ hour and a slice acquisition may order 64 ranges, so an unbounded wait is a
999
+ multi-hour hang in our own worker. A wait that would cross the deadline is refused
1000
+ up front as ``TOTAL_TIMEOUT`` rather than slept past and regretted afterwards.
1001
+ """
1002
+
1003
+ if type(deadline) not in {int, float} or deadline != deadline: # NaN is not a bound
1004
+ raise AcquisitionSecurityError(
1005
+ "TOTAL_TIMEOUT",
1006
+ "a wait must be bounded by a deadline on the retrieval's own clock",
1007
+ )
1008
+ if not self._semaphore.acquire(blocking=False):
1009
+ raise AcquisitionSecurityError(
1010
+ "CONCURRENCY_LIMIT",
1011
+ "acquisition concurrency limit is exhausted",
1012
+ )
1013
+ try:
1014
+ now = float(self._clock())
1015
+ with self._lock:
1016
+ last = self._last_request.get(hostname)
1017
+ scheduled = now if last is None else max(now, last + self._min_interval_seconds)
1018
+ if scheduled > float(deadline):
1019
+ raise AcquisitionSecurityError(
1020
+ "TOTAL_TIMEOUT",
1021
+ "the wait this source requires would outlive the retrieval's budget",
1022
+ )
1023
+ # Reserve the instant the request will actually be made, before the lock is
1024
+ # released: a second caller for the same host then queues behind this one
1025
+ # instead of racing it. The refusal above reserves nothing, so a retrieval
1026
+ # that gives up does not push the host's next slot further out. The lock is
1027
+ # never held across the wait, which would stall every other host.
1028
+ self._last_request[hostname] = scheduled
1029
+ if scheduled > now:
1030
+ self._sleep(scheduled - now)
1031
+ yield
1032
+ finally:
1033
+ self._semaphore.release()
1034
+
1035
+
1036
+ class PinnedHttpsRetriever:
1037
+ """Redirect-aware retriever; every hop gets a new DNS and connection epoch."""
1038
+
1039
+ def __init__(
1040
+ self,
1041
+ *,
1042
+ resolver: Resolver,
1043
+ egress_policy: EgressPolicy,
1044
+ transport: PinnedTransport,
1045
+ limits: RetrievalLimits,
1046
+ limiter: AcquisitionLimiter | None = None,
1047
+ clock: Callable[[], float] = time.monotonic,
1048
+ sleep: Callable[[float], None] = time.sleep,
1049
+ ) -> None:
1050
+ # A deadline is an instant rather than a duration, so a limiter reading a different
1051
+ # clock would compare it against a number from another timeline: the wait bound would
1052
+ # either fire immediately on every call or never fire at all. A shared limiter is
1053
+ # still allowed -- it must simply share this retrieval's clock, which the default
1054
+ # monotonic pairing does.
1055
+ if limiter is not None and limiter.clock is not clock:
1056
+ raise AcquisitionSecurityError(
1057
+ "LIMITER_CLOCK",
1058
+ "an injected limiter must observe the retrieval's own clock",
1059
+ )
1060
+ self._resolver = resolver
1061
+ self._egress_policy = egress_policy
1062
+ self._transport = transport
1063
+ self._limits = limits
1064
+ self._limiter = limiter or AcquisitionLimiter(
1065
+ max_concurrency=limits.max_concurrency,
1066
+ min_interval_seconds=limits.min_interval_seconds,
1067
+ clock=clock,
1068
+ sleep=sleep,
1069
+ )
1070
+ self._clock = clock
1071
+
1072
+ @property
1073
+ def limits(self) -> RetrievalLimits:
1074
+ """The caps this retriever actually enforces. Read-only: a caller narrows, never mutates."""
1075
+
1076
+ return self._limits
1077
+
1078
+ def narrowed(self, limits: RetrievalLimits) -> PinnedHttpsRetriever:
1079
+ """A retriever over the same transport whose transport-shape caps are ``limits``.
1080
+
1081
+ A caller that declares tighter caps for one fetch -- a harvester saying it accepts only
1082
+ ``application/json`` under a 2 MiB cap, a health probe saying only a 1 MiB citation target
1083
+ -- has to be able to make those caps the ones the transport enforces. Passing them
1084
+ alongside a retriever that was built with looser ones leaves them inert, which is worse
1085
+ than not declaring them: the declaration reads as a control and enforces nothing.
1086
+
1087
+ Only the five transport-shape caps are taken from ``limits``: ``max_response_bytes``,
1088
+ ``max_redirects``, ``max_requests``, ``max_ranges`` and ``allowed_media_types``. Timeouts,
1089
+ concurrency, rate interval and user agent stay this retriever's, because those are the
1090
+ coordinator's operational policy and not a per-fetch shape. The four numeric caps may not
1091
+ be widened, so this can only tighten what the coordinator configured, never escape it.
1092
+
1093
+ The allowlist term is the one place the rule is not plain subset, and the superset it is
1094
+ checked against is stated rather than implied: this retriever's own types **plus**
1095
+ :data:`METADATA_MEDIA_TYPES`. ``allowed_media_types`` defaults to the payload format table,
1096
+ which by construction can never carry a catalogue's structure message -- so a plain subset
1097
+ rule made the SDMX harvester's declared type unnarrowable from a default-configured
1098
+ retriever and refused every SDMX fetch and every non-JSON citation probe before a request
1099
+ was made. The metadata vocabulary is closed and lives in this module, so this admits exactly
1100
+ the types the transport itself names and nothing a caller supplies: ``text/html`` is still a
1101
+ widening, which is what keeps the bot-wall refusal a refusal.
1102
+
1103
+ The rate limiter is shared rather than rebuilt, so a narrowed retriever does not get a
1104
+ second concurrency budget for the same host.
1105
+ """
1106
+
1107
+ if not isinstance(limits, RetrievalLimits):
1108
+ raise AcquisitionSecurityError(
1109
+ "RETRIEVAL_LIMITS",
1110
+ "limits must be a RetrievalLimits",
1111
+ )
1112
+ widened = []
1113
+ for name in (
1114
+ "max_response_bytes",
1115
+ "max_aggregate_response_bytes",
1116
+ "max_redirects",
1117
+ "max_requests",
1118
+ "max_ranges",
1119
+ ):
1120
+ if getattr(limits, name) > getattr(self._limits, name):
1121
+ widened.append(name)
1122
+ narrowable = set(self._limits.allowed_media_types) | set(METADATA_MEDIA_TYPES)
1123
+ if not set(limits.allowed_media_types) <= narrowable:
1124
+ widened.append("allowed_media_types")
1125
+ if widened:
1126
+ raise AcquisitionSecurityError(
1127
+ "RETRIEVAL_LIMITS",
1128
+ f"narrowing may not widen {sorted(widened)}",
1129
+ )
1130
+ if limits == self._limits:
1131
+ return self
1132
+ effective = RetrievalLimits(
1133
+ max_response_bytes=limits.max_response_bytes,
1134
+ max_aggregate_response_bytes=limits.max_aggregate_response_bytes,
1135
+ max_redirects=limits.max_redirects,
1136
+ max_requests=limits.max_requests,
1137
+ max_ranges=limits.max_ranges,
1138
+ connect_timeout_seconds=self._limits.connect_timeout_seconds,
1139
+ read_timeout_seconds=self._limits.read_timeout_seconds,
1140
+ total_timeout_seconds=self._limits.total_timeout_seconds,
1141
+ max_concurrency=self._limits.max_concurrency,
1142
+ min_interval_seconds=self._limits.min_interval_seconds,
1143
+ allowed_media_types=limits.allowed_media_types,
1144
+ user_agent=self._limits.user_agent,
1145
+ )
1146
+ return PinnedHttpsRetriever(
1147
+ resolver=self._resolver,
1148
+ egress_policy=self._egress_policy,
1149
+ transport=self._transport,
1150
+ limits=effective,
1151
+ limiter=self._limiter,
1152
+ clock=self._clock,
1153
+ )
1154
+
1155
+ def retrieve(self, url: str, *, byte_range: ByteRange | None = None) -> RetrievedBytes:
1156
+ if byte_range is not None and not isinstance(byte_range, ByteRange):
1157
+ raise AcquisitionSecurityError(
1158
+ "BYTE_RANGE",
1159
+ "a slice is ordered only through a validated byte range",
1160
+ )
1161
+ return self._fetch(
1162
+ url,
1163
+ byte_range=byte_range,
1164
+ presence_probe=False,
1165
+ datagov_authorization=None,
1166
+ )
1167
+
1168
+ def retrieve_datagov_v4(
1169
+ self,
1170
+ url: str,
1171
+ authorization: DatagovV4Authorization,
1172
+ *,
1173
+ on_response_observed: (
1174
+ Callable[[int, int, DatagovRateObservation | None], None] | None
1175
+ ) = None,
1176
+ ) -> RetrievedBytes:
1177
+ """Retrieve only the exact authenticated v4 endpoint with zero redirects."""
1178
+
1179
+ if not isinstance(authorization, DatagovV4Authorization):
1180
+ raise AcquisitionSecurityError(
1181
+ "DATAGOV_AUTHORIZATION", "strict Data.gov authorization is required"
1182
+ )
1183
+ authorization.header_for(url)
1184
+ return self._fetch(
1185
+ url,
1186
+ byte_range=None,
1187
+ presence_probe=False,
1188
+ datagov_authorization=authorization,
1189
+ datagov_response_observed=on_response_observed,
1190
+ )
1191
+
1192
+ def probe_presence(self, url: str) -> bool:
1193
+ """Ask one address whether it is published, keep nothing, and answer yes or no.
1194
+
1195
+ This is the look half of look-then-lock. It orders exactly one byte, discards the
1196
+ response, and returns presence. Nothing is parsed, nothing is sealed, no digest is
1197
+ taken and no media type is reported: the only thing that leaves this call is a
1198
+ boolean.
1199
+
1200
+ Absence is read from the typed ``HTTP_NOT_FOUND`` code and from nothing else. A
1201
+ string match on the message would silently start reporting "not published yet" the
1202
+ first time an unrelated message was reworded to contain the same words, and that is
1203
+ the failure mode that turns a broken source into a permanently quiet one: the dataset
1204
+ stops updating, every run reports a clean no-new-run, and nobody is told. Every other
1205
+ refusal -- a server error, a TLS failure, a peer outside the pin, a source that
1206
+ ignored the range -- propagates, because a broken source must stay loud.
1207
+
1208
+ Presence probes admit any media type. GRIB objects commonly return
1209
+ ``application/octet-stream``, which the normal retrieval allowlist rejects. Only this
1210
+ method can request the exemption; data retrieval still uses ``retrieve``.
1211
+ """
1212
+
1213
+ try:
1214
+ self._fetch(
1215
+ url,
1216
+ byte_range=_PRESENCE_PROBE_RANGE,
1217
+ presence_probe=True,
1218
+ datagov_authorization=None,
1219
+ )
1220
+ except AcquisitionSecurityError as exc:
1221
+ if exc.code != HTTP_NOT_FOUND:
1222
+ raise
1223
+ return False
1224
+ return True
1225
+
1226
+ def retrieve_presence_evidence(self, url: str) -> RetrievedBytes:
1227
+ """Probe one byte while retaining the exact measured response evidence.
1228
+
1229
+ Range/Reader composition seals this evidence and accounts every response body. The
1230
+ ordinary survey intentionally returns only booleans, so it continues to use
1231
+ :meth:`probe_presence`; this method exists for an acquisition that has already locked
1232
+ the candidate and therefore must retain, rather than discard, what it measured.
1233
+ """
1234
+
1235
+ return self._fetch(
1236
+ url,
1237
+ byte_range=_PRESENCE_PROBE_RANGE,
1238
+ presence_probe=True,
1239
+ datagov_authorization=None,
1240
+ )
1241
+
1242
+ def _fetch(
1243
+ self,
1244
+ url: str,
1245
+ *,
1246
+ byte_range: ByteRange | None,
1247
+ presence_probe: bool,
1248
+ datagov_authorization: DatagovV4Authorization | None,
1249
+ datagov_response_observed: (
1250
+ Callable[[int, int, DatagovRateObservation | None], None] | None
1251
+ ) = None,
1252
+ ) -> RetrievedBytes:
1253
+ started = float(self._clock())
1254
+ current_url = url
1255
+ hops: list[RetrievalHop] = []
1256
+ aggregate_response_bytes = 0
1257
+ requests = 0
1258
+ deadline = started + self._limits.total_timeout_seconds
1259
+ redirect_budget = 0 if datagov_authorization is not None else self._limits.max_redirects
1260
+ for redirect_count in range(redirect_budget + 1):
1261
+ self._check_total_timeout(started)
1262
+ target = validate_public_https_url(
1263
+ current_url,
1264
+ resolver=self._resolver,
1265
+ egress_policy=self._egress_policy,
1266
+ )
1267
+ headers = {
1268
+ # A presence probe asks for anything, because it admits anything: sending the
1269
+ # acquisition allowlist would invite a 406 for a run that is published and
1270
+ # fine, which is the same false failure the admission exemption removes.
1271
+ "Accept": "*/*" if presence_probe else _accept_header(self._limits),
1272
+ "Accept-Encoding": "identity",
1273
+ "Connection": "close",
1274
+ "Host": urlsplit(target.normalized_url).netloc,
1275
+ "User-Agent": self._limits.user_agent,
1276
+ }
1277
+ if datagov_authorization is not None:
1278
+ name, value = datagov_authorization.header_for(target.normalized_url)
1279
+ headers[name] = value
1280
+ peer_attempts: list[PeerAttempt] = []
1281
+ response: TransportResponse | None = None
1282
+ for approved_ip in target.approved_ips:
1283
+ if requests >= self._limits.max_requests:
1284
+ raise PeerAttemptError(
1285
+ "REQUEST_LIMIT",
1286
+ "retrieval exceeded its request budget",
1287
+ tuple(peer_attempts),
1288
+ hops=tuple(hops),
1289
+ )
1290
+ requests += 1
1291
+ remaining_aggregate_bytes = (
1292
+ self._limits.max_aggregate_response_bytes - aggregate_response_bytes
1293
+ )
1294
+ try:
1295
+ attempt_limits = self._remaining_attempt_limits(
1296
+ deadline,
1297
+ remaining_aggregate_bytes=remaining_aggregate_bytes,
1298
+ )
1299
+ except AcquisitionSecurityError as exc:
1300
+ raise PeerAttemptError(
1301
+ exc.code,
1302
+ exc.detail,
1303
+ tuple(peer_attempts),
1304
+ hops=tuple(hops),
1305
+ ) from None
1306
+ try:
1307
+ # One budget, one clock: every peer and limiter wait is bounded by the
1308
+ # same absolute instant. A fallback never starts a second DNS epoch.
1309
+ with self._limiter.acquire(target.hostname, deadline=deadline):
1310
+ response = self._transport.request(
1311
+ target,
1312
+ approved_ip=approved_ip,
1313
+ headers=headers,
1314
+ limits=attempt_limits,
1315
+ byte_range=byte_range,
1316
+ )
1317
+ except AcquisitionSecurityError as exc:
1318
+ if exc.code == "TRANSPORT_FAILURE" and not isinstance(
1319
+ exc, TransportFailureError
1320
+ ):
1321
+ raise PeerAttemptError(
1322
+ "TRANSPORT_EVIDENCE",
1323
+ "transport failure omitted exact consumed response-body bytes",
1324
+ tuple(peer_attempts),
1325
+ hops=tuple(hops),
1326
+ ) from None
1327
+ consumed_body_bytes = (
1328
+ exc.response_body_size_bytes
1329
+ if isinstance(exc, TransportFailureError)
1330
+ else 0
1331
+ )
1332
+ aggregate_response_bytes += consumed_body_bytes
1333
+ failure_code = exc.code
1334
+ failure_detail = exc.detail
1335
+ if (
1336
+ exc.code == "RESPONSE_LIMIT"
1337
+ and remaining_aggregate_bytes < self._limits.max_response_bytes
1338
+ ):
1339
+ failure_code = "AGGREGATE_RESPONSE_LIMIT"
1340
+ failure_detail = "retrieval exceeded its aggregate response-body budget"
1341
+ peer_attempts.append(
1342
+ PeerAttempt(
1343
+ approved_ip=approved_ip,
1344
+ outcome="transport_failure",
1345
+ failure_code=failure_code,
1346
+ response_body_size_bytes=consumed_body_bytes,
1347
+ )
1348
+ )
1349
+ aggregate_exhausted = (
1350
+ aggregate_response_bytes >= self._limits.max_aggregate_response_bytes
1351
+ )
1352
+ if (
1353
+ failure_code == "TRANSPORT_FAILURE"
1354
+ and not aggregate_exhausted
1355
+ and approved_ip != target.approved_ips[-1]
1356
+ ):
1357
+ continue
1358
+ if failure_code == "TRANSPORT_FAILURE" and aggregate_exhausted:
1359
+ raise PeerAttemptError(
1360
+ "AGGREGATE_RESPONSE_LIMIT",
1361
+ "failed response bodies exhausted the aggregate byte limit",
1362
+ tuple(peer_attempts),
1363
+ hops=tuple(hops),
1364
+ ) from None
1365
+ detail = (
1366
+ "all resolver-approved peers failed before an HTTP response"
1367
+ if failure_code == "TRANSPORT_FAILURE"
1368
+ else failure_detail
1369
+ )
1370
+ raise PeerAttemptError(
1371
+ failure_code, detail, tuple(peer_attempts), hops=tuple(hops)
1372
+ ) from None
1373
+ peer_attempts.append(
1374
+ PeerAttempt(
1375
+ approved_ip=approved_ip,
1376
+ outcome="response",
1377
+ failure_code=None,
1378
+ response_body_size_bytes=len(response.body),
1379
+ )
1380
+ )
1381
+ break
1382
+ if response is None: # pragma: no cover - ValidatedTarget forbids an empty peer set
1383
+ raise PeerAttemptError(
1384
+ "TRANSPORT_FAILURE",
1385
+ "all resolver-approved peers failed before an HTTP response",
1386
+ tuple(peer_attempts),
1387
+ hops=tuple(hops),
1388
+ )
1389
+ if datagov_response_observed is not None:
1390
+ datagov_response_observed(
1391
+ response.status,
1392
+ aggregate_response_bytes + len(response.body),
1393
+ None,
1394
+ )
1395
+ if (
1396
+ aggregate_response_bytes + len(response.body)
1397
+ > self._limits.max_aggregate_response_bytes
1398
+ ):
1399
+ self._raise_peer_failure(
1400
+ "AGGREGATE_RESPONSE_LIMIT",
1401
+ "redirect and final response bodies exceeded the aggregate byte limit",
1402
+ peer_attempts,
1403
+ hops,
1404
+ )
1405
+ aggregate_response_bytes += len(response.body)
1406
+ try:
1407
+ self._check_total_timeout(started)
1408
+ except AcquisitionSecurityError as exc:
1409
+ self._raise_peer_failure(exc.code, exc.detail, peer_attempts, hops)
1410
+ if not response.tls_verified or not response.hostname_verified:
1411
+ self._raise_peer_failure(
1412
+ "TLS_VERIFICATION",
1413
+ "transport did not prove certificate and hostname verification",
1414
+ peer_attempts,
1415
+ hops,
1416
+ )
1417
+ try:
1418
+ peer = require_approved_peer(target, response.connected_peer)
1419
+ normalized_headers = _validate_response_headers(response.headers)
1420
+ except AcquisitionSecurityError as exc:
1421
+ self._raise_peer_failure(exc.code, exc.detail, peer_attempts, hops)
1422
+ if datagov_authorization is not None:
1423
+ try:
1424
+ datagov_authorization.reject_echoed_headers(normalized_headers)
1425
+ if response.status in REDIRECT_STATUSES:
1426
+ self._raise_peer_failure(
1427
+ "REDIRECT_LIMIT",
1428
+ "response exceeded the redirect limit",
1429
+ peer_attempts,
1430
+ hops,
1431
+ )
1432
+ datagov_authorization.reject_echoed_response(response.body)
1433
+ except AcquisitionSecurityError as exc:
1434
+ raise DatagovResponseRefused(
1435
+ exc.code,
1436
+ exc.detail,
1437
+ status=response.status,
1438
+ network_bytes=aggregate_response_bytes,
1439
+ ) from None
1440
+ try:
1441
+ datagov_rate_observation = _datagov_rate_observation(response)
1442
+ except AcquisitionSecurityError:
1443
+ raise
1444
+ if datagov_response_observed is not None:
1445
+ datagov_response_observed(
1446
+ response.status,
1447
+ aggregate_response_bytes,
1448
+ datagov_rate_observation,
1449
+ )
1450
+ else:
1451
+ datagov_rate_observation = None
1452
+ if len(response.body) > attempt_limits.max_response_bytes:
1453
+ limit_code = (
1454
+ "AGGREGATE_RESPONSE_LIMIT"
1455
+ if remaining_aggregate_bytes < self._limits.max_response_bytes
1456
+ else "RESPONSE_LIMIT"
1457
+ )
1458
+ self._raise_peer_failure(
1459
+ limit_code,
1460
+ "response bodies exceeded the active cumulative byte limit",
1461
+ peer_attempts,
1462
+ hops,
1463
+ )
1464
+ content_length = self._response_header(response, "content-length", peer_attempts, hops)
1465
+ declared_length: int | None = None
1466
+ if content_length is not None:
1467
+ # The same canonical spelling ``Content-Range`` is held to, and for the same
1468
+ # reason: ``int`` accepts ``+10`` and ``10_000`` and a leading zero, none of
1469
+ # which is a Content-Length, and one declared length should have one spelling
1470
+ # in the sealed evidence. Nothing downstream trusted the parsed value over
1471
+ # ``len(response.body)``, so this closes an inconsistency rather than a hole --
1472
+ # but the inconsistency is on the surface where exactness is the whole control.
1473
+ if not _DECIMAL_ONLY.fullmatch(content_length):
1474
+ self._raise_peer_failure(
1475
+ "CONTENT_LENGTH",
1476
+ "response content length is invalid",
1477
+ peer_attempts,
1478
+ hops,
1479
+ )
1480
+ declared_length = int(content_length)
1481
+ if declared_length < 0 or declared_length > attempt_limits.max_response_bytes:
1482
+ self._raise_peer_failure(
1483
+ "RESPONSE_LIMIT",
1484
+ "declared response length exceeds the remaining cumulative limit",
1485
+ peer_attempts,
1486
+ hops,
1487
+ )
1488
+ if declared_length != len(response.body):
1489
+ self._raise_peer_failure(
1490
+ "CONTENT_LENGTH",
1491
+ "response bytes do not match declared content length",
1492
+ peer_attempts,
1493
+ hops,
1494
+ )
1495
+ content_encoding = (
1496
+ (self._response_header(response, "content-encoding", peer_attempts, hops) or "")
1497
+ .lower()
1498
+ .strip()
1499
+ )
1500
+ if content_encoding not in _ALLOWED_RESPONSE_ENCODINGS:
1501
+ self._raise_peer_failure(
1502
+ "CONTENT_ENCODING",
1503
+ "compressed network responses are disabled at this boundary",
1504
+ peer_attempts,
1505
+ hops,
1506
+ )
1507
+ observed_content_range = self._response_header(
1508
+ response, "content-range", peer_attempts, hops
1509
+ )
1510
+ hop = RetrievalHop(
1511
+ url=target.normalized_url,
1512
+ resolution_digest=target.resolution_digest,
1513
+ connected_peer=peer,
1514
+ status=response.status,
1515
+ response_headers_digest=canonical_sha256(normalized_headers),
1516
+ response_body_size_bytes=len(response.body),
1517
+ response_body_sha256=sha256_bytes(response.body),
1518
+ requested_range=byte_range.header_value if byte_range is not None else None,
1519
+ observed_content_range=observed_content_range,
1520
+ etag=self._response_header(response, "etag", peer_attempts, hops),
1521
+ last_modified=self._response_header(response, "last-modified", peer_attempts, hops),
1522
+ peer_attempts=tuple(peer_attempts),
1523
+ )
1524
+ hops.append(hop)
1525
+ # The delivery check. A slice is delivered only when the server answered 206,
1526
+ # named exactly the span that was ordered, and shipped exactly that many bytes.
1527
+ # It sits here, with the other response-integrity controls (status, declared
1528
+ # length, encoding, media type, peer), because a response must clear all of them
1529
+ # before anything downstream is allowed to treat it as content -- and it runs
1530
+ # before the redirect branch below, because a ranged request is never followed.
1531
+ #
1532
+ # What this check cannot do: the content range can match, the three lengths can
1533
+ # agree, and the bytes can still be the wrong bytes -- a stale cache, a re-issued
1534
+ # object, a mirror that skewed. HTTP carries no evidence that would detect that,
1535
+ # and nothing here closes that case. Two controls behind the sandbox boundary
1536
+ # divide what is left, and neither is the other. ``check_grib_message`` proves
1537
+ # the framing, which is what a window at the wrong offset breaks.
1538
+ # ``check_grib_reference_time`` proves the delivered message came from the model
1539
+ # run the order named, which is what a re-issued object or a lagging mirror
1540
+ # breaks; it runs only when the order named a run. A different object of the
1541
+ # *same* run is covered by none of the three, and ``acquisition.ranges`` says so
1542
+ # where those two are defined.
1543
+ complete_size_bytes: int | None = None
1544
+ if byte_range is not None:
1545
+ if response.status in REDIRECT_STATUSES:
1546
+ self._raise_peer_failure(
1547
+ "REDIRECT_ON_RANGE",
1548
+ "a ranged request is never followed through a redirect",
1549
+ peer_attempts,
1550
+ hops,
1551
+ )
1552
+ if 200 <= response.status <= 299 and response.status != PARTIAL_CONTENT_STATUS:
1553
+ self._raise_peer_failure(
1554
+ "RANGE_IGNORED",
1555
+ f"source answered a ranged request with status {response.status}",
1556
+ peer_attempts,
1557
+ hops,
1558
+ )
1559
+ if response.status == PARTIAL_CONTENT_STATUS:
1560
+ content_type = self._response_header(
1561
+ response, "content-type", peer_attempts, hops
1562
+ )
1563
+ try:
1564
+ parsed_media_type = _media_type(content_type)
1565
+ except AcquisitionSecurityError as exc:
1566
+ self._raise_peer_failure(exc.code, exc.detail, peer_attempts, hops)
1567
+ if parsed_media_type == _MULTIPART_BYTERANGES:
1568
+ self._raise_peer_failure(
1569
+ "MULTIPART_RANGE",
1570
+ "multipart byte-range responses are refused at this boundary",
1571
+ peer_attempts,
1572
+ hops,
1573
+ )
1574
+ try:
1575
+ first_byte, last_byte, complete_size_bytes = _parse_content_range(
1576
+ observed_content_range
1577
+ )
1578
+ except AcquisitionSecurityError as exc:
1579
+ self._raise_peer_failure(exc.code, exc.detail, peer_attempts, hops)
1580
+ if first_byte != byte_range.first_byte or last_byte != byte_range.last_byte:
1581
+ self._raise_peer_failure(
1582
+ "RANGE_MISMATCH",
1583
+ "response content range is not the range that was ordered",
1584
+ peer_attempts,
1585
+ hops,
1586
+ )
1587
+ delivered_length = last_byte - first_byte + 1
1588
+ # Three lengths must agree: the span the server named, the length it
1589
+ # declared, and the bytes that arrived. The declared-versus-body leg is
1590
+ # already enforced above; it is restated against the span here so the
1591
+ # whole agreement is stated in one place rather than inferred across two.
1592
+ if delivered_length != len(response.body) or (
1593
+ declared_length is not None and declared_length != delivered_length
1594
+ ):
1595
+ self._raise_peer_failure(
1596
+ "RANGE_LENGTH",
1597
+ "response bytes do not match the content range they claim",
1598
+ peer_attempts,
1599
+ hops,
1600
+ )
1601
+ elif response.status == PARTIAL_CONTENT_STATUS:
1602
+ self._raise_peer_failure(
1603
+ "UNSOLICITED_RANGE",
1604
+ "source returned partial content for a request that ordered no range",
1605
+ peer_attempts,
1606
+ hops,
1607
+ )
1608
+ if response.status in REDIRECT_STATUSES:
1609
+ location = self._response_header(response, _REDIRECT_HEADER, peer_attempts, hops)
1610
+ if not location:
1611
+ self._raise_peer_failure(
1612
+ "REDIRECT_LOCATION",
1613
+ "redirect response has no location",
1614
+ peer_attempts,
1615
+ hops,
1616
+ )
1617
+ if redirect_count >= redirect_budget:
1618
+ self._raise_peer_failure(
1619
+ "REDIRECT_LIMIT",
1620
+ "response exceeded the redirect limit",
1621
+ peer_attempts,
1622
+ hops,
1623
+ )
1624
+ current_url = urljoin(target.normalized_url, location)
1625
+ continue
1626
+ # A 404 gets its own code. This is not a softening -- the retrieval still fails
1627
+ # closed here -- but an availability survey has to tell "this cycle is not
1628
+ # published yet" from "this source is broken", and it must do that from a typed
1629
+ # code rather than by matching an error message. Every other status stays
1630
+ # generic.
1631
+ datagov_non_success = datagov_authorization is not None and response.status in {
1632
+ 403,
1633
+ 429,
1634
+ }
1635
+ if response.status == NOT_FOUND_STATUS:
1636
+ self._raise_peer_failure(
1637
+ HTTP_NOT_FOUND,
1638
+ f"source returned HTTP status {response.status}",
1639
+ peer_attempts,
1640
+ hops,
1641
+ )
1642
+ if not 200 <= response.status <= 299 and not datagov_non_success:
1643
+ self._raise_peer_failure(
1644
+ "HTTP_STATUS",
1645
+ f"source returned HTTP status {response.status}",
1646
+ peer_attempts,
1647
+ hops,
1648
+ )
1649
+ content_type = self._response_header(response, "content-type", peer_attempts, hops)
1650
+ try:
1651
+ media_type = _media_type(content_type)
1652
+ except AcquisitionSecurityError as exc:
1653
+ self._raise_peer_failure(exc.code, exc.detail, peer_attempts, hops)
1654
+ # Exempted for a presence probe only, and only because a probe keeps nothing: see
1655
+ # ``probe_presence``. Every acquiring call reaches here through ``retrieve``,
1656
+ # where ``presence_probe`` is a literal ``False`` rather than an argument.
1657
+ if not presence_probe and media_type not in self._limits.allowed_media_types:
1658
+ self._raise_peer_failure(
1659
+ "MEDIA_TYPE",
1660
+ f"response media type {media_type!r} is not allowlisted",
1661
+ peer_attempts,
1662
+ hops,
1663
+ )
1664
+ evidence_digest = canonical_sha256([item.to_dict() for item in hops])
1665
+ return RetrievedBytes(
1666
+ source_url=url,
1667
+ final_url=target.normalized_url,
1668
+ media_type=media_type,
1669
+ content=response.body,
1670
+ content_sha256=sha256_bytes(response.body),
1671
+ hops=tuple(hops),
1672
+ transport_evidence_digest=evidence_digest,
1673
+ complete_size_bytes=complete_size_bytes,
1674
+ rate_observation=datagov_rate_observation,
1675
+ )
1676
+ raise AcquisitionSecurityError("REDIRECT_LIMIT", "redirect loop did not terminate")
1677
+
1678
+ def _check_total_timeout(self, started: float) -> None:
1679
+ if float(self._clock()) - started > self._limits.total_timeout_seconds:
1680
+ raise AcquisitionSecurityError(
1681
+ "TOTAL_TIMEOUT",
1682
+ "retrieval exceeded its total time budget",
1683
+ )
1684
+
1685
+ def _remaining_attempt_limits(
1686
+ self,
1687
+ deadline: float,
1688
+ *,
1689
+ remaining_aggregate_bytes: int,
1690
+ ) -> RetrievalLimits:
1691
+ remaining = deadline - float(self._clock())
1692
+ if remaining <= 0:
1693
+ raise AcquisitionSecurityError(
1694
+ "TOTAL_TIMEOUT",
1695
+ "retrieval exceeded its total time budget",
1696
+ )
1697
+ if remaining_aggregate_bytes <= 0:
1698
+ raise AcquisitionSecurityError(
1699
+ "AGGREGATE_RESPONSE_LIMIT",
1700
+ "retrieval exhausted its aggregate response-body budget",
1701
+ )
1702
+ return replace(
1703
+ self._limits,
1704
+ max_response_bytes=min(self._limits.max_response_bytes, remaining_aggregate_bytes),
1705
+ max_aggregate_response_bytes=remaining_aggregate_bytes,
1706
+ connect_timeout_seconds=min(self._limits.connect_timeout_seconds, remaining),
1707
+ read_timeout_seconds=min(self._limits.read_timeout_seconds, remaining),
1708
+ total_timeout_seconds=remaining,
1709
+ )
1710
+
1711
+ @staticmethod
1712
+ def _raise_peer_failure(
1713
+ code: str,
1714
+ detail: str,
1715
+ peer_attempts: list[PeerAttempt],
1716
+ hops: list[RetrievalHop],
1717
+ ) -> None:
1718
+ latest = peer_attempts[-1]
1719
+ peer_attempts[-1] = PeerAttempt(
1720
+ approved_ip=latest.approved_ip,
1721
+ outcome="response_failure",
1722
+ failure_code=code,
1723
+ response_body_size_bytes=latest.response_body_size_bytes,
1724
+ )
1725
+ raise PeerAttemptError(code, detail, tuple(peer_attempts), hops=tuple(hops))
1726
+
1727
+ def _response_header(
1728
+ self,
1729
+ response: TransportResponse,
1730
+ name: str,
1731
+ peer_attempts: list[PeerAttempt],
1732
+ hops: list[RetrievalHop],
1733
+ ) -> str | None:
1734
+ try:
1735
+ return response.header(name)
1736
+ except AcquisitionSecurityError as exc:
1737
+ self._raise_peer_failure(exc.code, exc.detail, peer_attempts, hops)
1738
+
1739
+
1740
+ class _PinnedHTTPSConnection(http.client.HTTPSConnection):
1741
+ """One non-pooled connection to a validated IP with hostname TLS verification."""
1742
+
1743
+ def __init__(
1744
+ self,
1745
+ *,
1746
+ hostname: str,
1747
+ approved_ip: str,
1748
+ port: int,
1749
+ timeout: float,
1750
+ context: ssl.SSLContext,
1751
+ ) -> None:
1752
+ super().__init__(host=hostname, port=port, timeout=timeout, context=context)
1753
+ self._approved_ip = approved_ip
1754
+ self.connected_peer: str | None = None
1755
+
1756
+ def connect(self) -> None:
1757
+ raw_socket = socket.create_connection(
1758
+ (self._approved_ip, self.port),
1759
+ timeout=self.timeout,
1760
+ )
1761
+ try:
1762
+ peer = raw_socket.getpeername()[0]
1763
+ self.connected_peer = require_approved_peer(
1764
+ ValidatedTarget(
1765
+ original_url=f"https://{self.host}/",
1766
+ normalized_url=f"https://{self.host}/",
1767
+ hostname=self.host,
1768
+ port=self.port,
1769
+ approved_ips=(self._approved_ip,),
1770
+ resolution_digest=canonical_sha256(
1771
+ {
1772
+ "hostname": self.host,
1773
+ "port": self.port,
1774
+ "approved_ips": [self._approved_ip],
1775
+ }
1776
+ ),
1777
+ ),
1778
+ peer,
1779
+ )
1780
+ self.sock = self._context.wrap_socket(raw_socket, server_hostname=self.host)
1781
+ except BaseException:
1782
+ raw_socket.close()
1783
+ raise
1784
+
1785
+
1786
+ class StdlibPinnedTransport:
1787
+ """Production transport with no environment-proxy integration or connection pooling."""
1788
+
1789
+ def __init__(self, *, ssl_context: ssl.SSLContext | None = None) -> None:
1790
+ self._context = ssl_context or ssl.create_default_context()
1791
+ if (
1792
+ self._context.check_hostname is not True
1793
+ or self._context.verify_mode != ssl.CERT_REQUIRED
1794
+ ):
1795
+ raise AcquisitionSecurityError(
1796
+ "TLS_CONTEXT",
1797
+ "TLS context must require certificate and hostname verification",
1798
+ )
1799
+
1800
+ def request(
1801
+ self,
1802
+ target: ValidatedTarget,
1803
+ *,
1804
+ approved_ip: str,
1805
+ headers: Mapping[str, str],
1806
+ limits: RetrievalLimits,
1807
+ byte_range: ByteRange | None = None,
1808
+ ) -> TransportResponse:
1809
+ outgoing = dict(headers)
1810
+ if byte_range is not None:
1811
+ outgoing["Range"] = byte_range.header_value
1812
+ attempt_deadline = time.monotonic() + limits.total_timeout_seconds
1813
+ connection = _PinnedHTTPSConnection(
1814
+ hostname=target.hostname,
1815
+ approved_ip=approved_ip,
1816
+ port=target.port,
1817
+ timeout=min(limits.connect_timeout_seconds, limits.total_timeout_seconds),
1818
+ context=self._context,
1819
+ )
1820
+ total = 0
1821
+ try:
1822
+ connection.request("GET", target.request_target, headers=outgoing)
1823
+ if connection.sock is not None:
1824
+ connection.sock.settimeout(
1825
+ _remaining_socket_timeout(
1826
+ attempt_deadline,
1827
+ configured=limits.read_timeout_seconds,
1828
+ )
1829
+ )
1830
+ response = connection.getresponse()
1831
+ raw_headers = tuple(
1832
+ (key.lower(), value.strip()) for key, value in response.getheaders()
1833
+ )
1834
+ _validate_response_headers(raw_headers)
1835
+ content_length = next(
1836
+ (value for key, value in raw_headers if key == "content-length"),
1837
+ None,
1838
+ )
1839
+ if content_length is not None:
1840
+ try:
1841
+ if int(content_length) > limits.max_response_bytes:
1842
+ raise AcquisitionSecurityError(
1843
+ "RESPONSE_LIMIT",
1844
+ "declared response length exceeds the configured limit",
1845
+ )
1846
+ except ValueError:
1847
+ raise AcquisitionSecurityError(
1848
+ "CONTENT_LENGTH",
1849
+ "response content length is invalid",
1850
+ ) from None
1851
+ chunks: list[bytes] = []
1852
+ while True:
1853
+ if connection.sock is not None:
1854
+ connection.sock.settimeout(
1855
+ _remaining_socket_timeout(
1856
+ attempt_deadline,
1857
+ configured=limits.read_timeout_seconds,
1858
+ )
1859
+ )
1860
+ chunk = response.read(min(65_536, limits.max_response_bytes + 1 - total))
1861
+ if not chunk:
1862
+ break
1863
+ total += len(chunk)
1864
+ if total > limits.max_response_bytes:
1865
+ raise AcquisitionSecurityError(
1866
+ "RESPONSE_LIMIT",
1867
+ "response exceeded the configured byte limit",
1868
+ )
1869
+ chunks.append(chunk)
1870
+ peer = connection.connected_peer
1871
+ if peer is None:
1872
+ raise AcquisitionSecurityError(
1873
+ "PEER_ADDRESS",
1874
+ "transport did not record its connected peer",
1875
+ )
1876
+ return TransportResponse(
1877
+ status=response.status,
1878
+ headers=raw_headers,
1879
+ body=b"".join(chunks),
1880
+ connected_peer=peer,
1881
+ tls_verified=True,
1882
+ hostname_verified=True,
1883
+ )
1884
+ except ssl.SSLError as exc:
1885
+ raise AcquisitionSecurityError(
1886
+ "TLS_VERIFICATION",
1887
+ f"HTTPS TLS verification failed: {type(exc).__name__}",
1888
+ ) from None
1889
+ except http.client.IncompleteRead as exc:
1890
+ partial = exc.partial if isinstance(exc.partial, bytes) else bytes(exc.partial)
1891
+ raise TransportFailureError(
1892
+ f"HTTPS peer disconnected during a response: {type(exc).__name__}",
1893
+ response_body_size_bytes=total + len(partial),
1894
+ ) from None
1895
+ except http.client.RemoteDisconnected as exc:
1896
+ raise TransportFailureError(
1897
+ f"HTTPS peer disconnected before a response: {type(exc).__name__}",
1898
+ response_body_size_bytes=total,
1899
+ ) from None
1900
+ except http.client.HTTPException as exc:
1901
+ raise AcquisitionSecurityError(
1902
+ "RESPONSE_HEADERS",
1903
+ f"HTTPS response protocol failed: {type(exc).__name__}",
1904
+ ) from None
1905
+ except OSError as exc:
1906
+ raise TransportFailureError(
1907
+ f"HTTPS transport failed: {type(exc).__name__}",
1908
+ response_body_size_bytes=total,
1909
+ ) from None
1910
+ finally:
1911
+ connection.close()
1912
+
1913
+
1914
+ def _remaining_socket_timeout(deadline: float, *, configured: float) -> float:
1915
+ remaining = deadline - time.monotonic()
1916
+ if remaining <= 0:
1917
+ raise AcquisitionSecurityError(
1918
+ "TOTAL_TIMEOUT",
1919
+ "retrieval exceeded its total time budget",
1920
+ )
1921
+ return min(configured, remaining)
1922
+
1923
+
1924
+ def _datagov_rate_observation(response: TransportResponse) -> DatagovRateObservation | None:
1925
+ """Parse only the bounded numeric Data.gov rate headers; retain no arbitrary headers."""
1926
+
1927
+ limit = response.header("x-ratelimit-limit")
1928
+ remaining = response.header("x-ratelimit-remaining")
1929
+ retry_after = response.header("retry-after")
1930
+ reset = response.header("x-ratelimit-reset")
1931
+ if limit is None and remaining is None and retry_after is None and reset is None:
1932
+ return None
1933
+ if limit is None or remaining is None:
1934
+ raise AcquisitionSecurityError(
1935
+ "DATAGOV_RATE", "Data.gov rate limit and remaining headers must appear together"
1936
+ )
1937
+
1938
+ def bounded_decimal(value: str, *, name: str, maximum: int) -> int:
1939
+ if not _DECIMAL_ONLY.fullmatch(value):
1940
+ raise AcquisitionSecurityError("DATAGOV_RATE", f"{name} is not a bounded integer")
1941
+ parsed = int(value)
1942
+ if parsed > maximum:
1943
+ raise AcquisitionSecurityError("DATAGOV_RATE", f"{name} is outside its bound")
1944
+ return parsed
1945
+
1946
+ return DatagovRateObservation(
1947
+ limit=bounded_decimal(limit, name="X-RateLimit-Limit", maximum=1_000_000_000),
1948
+ remaining=bounded_decimal(remaining, name="X-RateLimit-Remaining", maximum=1_000_000_000),
1949
+ retry_after_seconds=(
1950
+ None
1951
+ if retry_after is None
1952
+ else bounded_decimal(
1953
+ retry_after,
1954
+ name="Retry-After",
1955
+ maximum=MAX_DATAGOV_RATE_SECONDS,
1956
+ )
1957
+ ),
1958
+ reset_epoch_seconds=(
1959
+ None
1960
+ if reset is None
1961
+ else bounded_decimal(reset, name="X-RateLimit-Reset", maximum=(1 << 53) - 1)
1962
+ ),
1963
+ )
1964
+
1965
+
1966
+ def seal_content_addressed_snapshot(
1967
+ root: Path,
1968
+ *,
1969
+ content: bytes,
1970
+ suffix: str,
1971
+ ) -> Path:
1972
+ """Seal exact bytes, refusing a digest withdrawn by governed deletion."""
1973
+
1974
+ if not root.is_absolute():
1975
+ raise AcquisitionSecurityError("SNAPSHOT_ROOT", "snapshot root must be absolute")
1976
+ if not _is_allowlisted_snapshot_suffix(suffix):
1977
+ raise AcquisitionSecurityError("SNAPSHOT_SUFFIX", "snapshot suffix is not allowlisted")
1978
+ digest = sha256_bytes(content)
1979
+ final_name = f"{digest}.{suffix}"
1980
+ directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0)
1981
+ try:
1982
+ root_fd = os.open(root, directory_flags)
1983
+ except OSError:
1984
+ raise AcquisitionSecurityError(
1985
+ "SNAPSHOT_ROOT",
1986
+ "snapshot root must be an existing non-symlink directory",
1987
+ ) from None
1988
+ temporary_name = f".stage-{os.getpid()}-{threading.get_ident()}-{time.monotonic_ns()}"
1989
+ file_flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0)
1990
+ temporary_fd: int | None = None
1991
+ try:
1992
+ root_info = os.fstat(root_fd)
1993
+ ownership_policy = (root_info.st_uid, root_info.st_gid)
1994
+ with _serialized_snapshot_seal(
1995
+ root_fd,
1996
+ root_info.st_dev,
1997
+ root_info.st_ino,
1998
+ final_name,
1999
+ ):
2000
+ evidence_root = configured_retention_evidence_root(root)
2001
+ if evidence_root is not None:
2002
+ require_snapshot_admission(
2003
+ evidence_root=evidence_root,
2004
+ content_sha256=digest,
2005
+ )
2006
+ try:
2007
+ try:
2008
+ _read_existing_digest_snapshot(
2009
+ root_fd,
2010
+ final_name,
2011
+ expected_content=content,
2012
+ expected_digest=digest,
2013
+ ownership_policy=ownership_policy,
2014
+ )
2015
+ except FileNotFoundError:
2016
+ pass
2017
+ else:
2018
+ return root / final_name
2019
+ temporary_fd = os.open(
2020
+ temporary_name,
2021
+ file_flags,
2022
+ 0o600,
2023
+ dir_fd=root_fd,
2024
+ )
2025
+ view = memoryview(content)
2026
+ written = 0
2027
+ while written < len(view):
2028
+ count = os.write(temporary_fd, view[written:])
2029
+ if count <= 0:
2030
+ raise AcquisitionSecurityError(
2031
+ "SNAPSHOT_WRITE",
2032
+ "snapshot write made no progress",
2033
+ )
2034
+ written += count
2035
+ os.fchmod(temporary_fd, 0o444)
2036
+ os.fsync(temporary_fd)
2037
+ os.close(temporary_fd)
2038
+ temporary_fd = None
2039
+ try:
2040
+ os.link(
2041
+ temporary_name,
2042
+ final_name,
2043
+ src_dir_fd=root_fd,
2044
+ dst_dir_fd=root_fd,
2045
+ follow_symlinks=False,
2046
+ )
2047
+ except FileExistsError:
2048
+ _read_existing_digest_snapshot(
2049
+ root_fd,
2050
+ final_name,
2051
+ expected_content=content,
2052
+ expected_digest=digest,
2053
+ ownership_policy=ownership_policy,
2054
+ )
2055
+ os.unlink(temporary_name, dir_fd=root_fd)
2056
+ # The staging name is deliberately visible for the shortest possible interval,
2057
+ # but a same-UID directory racer can still link that inode while it is visible.
2058
+ # Authenticate the published name only after our staging link is gone: nlink must
2059
+ # now be exactly one, and the stable-descriptor check below also rebinds kind,
2060
+ # mode, owner, size, name and digest.
2061
+ _read_existing_digest_snapshot(
2062
+ root_fd,
2063
+ final_name,
2064
+ expected_content=content,
2065
+ expected_digest=digest,
2066
+ ownership_policy=ownership_policy,
2067
+ )
2068
+ os.fsync(root_fd)
2069
+ return root / final_name
2070
+ finally:
2071
+ # Inside the per-target lock: the staging entry must be gone before the next
2072
+ # sealer of this same digest observes the published name, or that sealer sees
2073
+ # our second link and refuses a file we are in the middle of publishing.
2074
+ if temporary_fd is not None:
2075
+ os.close(temporary_fd)
2076
+ with contextlib.suppress(FileNotFoundError):
2077
+ os.unlink(temporary_name, dir_fd=root_fd)
2078
+ finally:
2079
+ os.close(root_fd)
2080
+
2081
+
2082
+ # The cross-process half's lock file sits beside the staging entries, under a prefix that no
2083
+ # content-addressed name can take: a snapshot is `<64 hex>.<suffix>`, and `sources.deletion`
2084
+ # matches removable names against exactly that. It is created while a name is held and unlinked
2085
+ # when it is released, so a settled root holds snapshots and nothing else.
2086
+ SNAPSHOT_LOCK_PREFIX = ".seal-lock-"
2087
+ # The backoff that keeps re-taking a moved lock file from being a spin. Every release destroys the
2088
+ # file, so one release wakes every waiter at once and all but one of them find a name that has
2089
+ # moved -- a herd that re-races itself. Measured at a deliberately lowered ceiling of 32, sixteen
2090
+ # processes holding for 0.3 ms over 400 rounds each: 1117-1270 losses that deep without a backoff,
2091
+ # 139-152 with one, an 8x reduction. There is deliberately no ceiling above it; see
2092
+ # :func:`_hold_snapshot_lock`.
2093
+ SNAPSHOT_LOCK_BACKOFF_SECONDS = 0.0002
2094
+ MAX_SNAPSHOT_LOCK_BACKOFF = 0.005
2095
+ # The one thing the unbounded retry still has to give up on: a root directory removed out from
2096
+ # under the descriptor answers every ``O_CREAT`` open with the same ``ENOENT``, forever. Only
2097
+ # CONSECUTIVE losses on that branch are counted, and the deepest streak observed under real
2098
+ # contention was four, so this is sixty-four times the worst measured case.
2099
+ MAX_CONSECUTIVE_VANISHED_LOCK_OPENS = 256
2100
+
2101
+ # One lock per (snapshot root inode, final name), created on demand and dropped when the last
2102
+ # holder leaves, so the registry cannot grow with the number of digests a long-lived process
2103
+ # seals. Keying on the root's device and inode rather than its pathname means two spellings of
2104
+ # one directory serialize together, which is exactly the contention that matters.
2105
+ _SNAPSHOT_SEAL_REGISTRY_GUARD = threading.Lock()
2106
+ _SNAPSHOT_SEAL_LOCKS: dict[tuple[int, int, str], tuple[threading.Lock, int]] = {}
2107
+
2108
+
2109
+ @contextlib.contextmanager
2110
+ def _serialized_snapshot_seal(
2111
+ root_fd: int,
2112
+ root_device: int,
2113
+ root_inode: int,
2114
+ final_name: str,
2115
+ ) -> Iterator[None]:
2116
+ """Hold one content-addressed name against every other writer of it, in and out of process.
2117
+
2118
+ Publishing is ``link(stage, final)`` followed by ``unlink(stage)``, and non-overwriting
2119
+ publication has no primitive that skips that pair: ``rename`` would clobber whatever already
2120
+ holds the name, which is the one thing this sealer must never do. So between those two calls
2121
+ the published inode legitimately carries two links, and a second sealer of the *same* bytes --
2122
+ same content, therefore same digest, therefore same final name -- that reads the name in that
2123
+ window sees ``st_nlink == 2`` and refuses a snapshot that is merely mid-publication. A
2124
+ remover of that name is the mirror of it: an ``unlink`` inside the read window moves
2125
+ ``st_nlink`` from 1 to 0 under the open descriptor, which is the ``changed during validation``
2126
+ refusal issue #209 reported.
2127
+
2128
+ Neither refusal is relaxed here, and neither may be: a second link on the published inode is
2129
+ exactly how a same-UID racer keeps a writable alias to a sealed snapshot, and a name that
2130
+ vanishes mid-read is exactly what a rebind looks like from the inside. What is removed is the
2131
+ benign racer. Writers of one name take turns, so the next one arrives after the last has
2132
+ unlinked its stage and finds a settled, single-linked file through the ordinary already-exists
2133
+ path. Serializing also stops N sealers from each writing N full copies of the same
2134
+ multi-gigabyte snapshot.
2135
+
2136
+ **Two locks, because there are two kinds of racer.** The ``threading.Lock`` below excludes
2137
+ other threads of this interpreter. It cannot exclude other PROCESSES, and those are real: the
2138
+ hosted crawler and the CLI seal into one snapshot root, and #229 shipped the thread half alone
2139
+ and named this as the residual it left open. Measured on macOS/APFS, six processes sealing
2140
+ identical bytes into one root over 24 rounds refused 5 to 7 of 144 attempts, every one of them
2141
+ ``st_nlink 2 != 1``. The second half is an ``flock`` on a per-name lock file beside the
2142
+ staging area, taken by :func:`_hold_snapshot_lock`.
2143
+
2144
+ Order is fixed -- threads first, then the file -- and only ever one name is held, so nothing
2145
+ can wait on a cycle. ``sources.deletion`` takes the same pair through
2146
+ :func:`exclusive_content_addressed_name` before it removes a snapshot, which is what stops the
2147
+ harness racing its own retention sweep; while it holds one it may take a *second* name under
2148
+ its evidence root to seal a tombstone, and that direction is the only nesting in the tree.
2149
+
2150
+ Neither wait is bounded. A holder finishes on its own -- the kernel drops an ``flock`` when
2151
+ its holder dies -- but "finishes" is measured in the payload: a seal or a governed deletion of
2152
+ a multi-gigabyte snapshot holds its name for the full write or re-hash plus its fsyncs, which
2153
+ is seconds on local disk and longer on slow storage, and every same-name waiter waits it out.
2154
+ A deadline here would have to choose between refusing an acquisition that is merely queued and
2155
+ admitting a second writer of a name that is mid-publication, so there is none.
2156
+ """
2157
+
2158
+ key = (root_device, root_inode, final_name)
2159
+ with _SNAPSHOT_SEAL_REGISTRY_GUARD:
2160
+ lock, holders = _SNAPSHOT_SEAL_LOCKS.get(key, (threading.Lock(), 0))
2161
+ _SNAPSHOT_SEAL_LOCKS[key] = (lock, holders + 1)
2162
+ try:
2163
+ # Acquiring inside this ``try``, and releasing through ``with``, is what keeps the
2164
+ # registry honest. A signal delivered during the blocking acquire would otherwise skip
2165
+ # the bookkeeping below and strand this key's entry -- held, in the worst case -- for the
2166
+ # life of the process.
2167
+ lock_name = f"{SNAPSHOT_LOCK_PREFIX}{final_name}"
2168
+ with lock:
2169
+ # The same shape as the comment above, one level down and with a worse consequence.
2170
+ # ``descriptor`` is bound inside this ``try`` so that a signal delivered between the
2171
+ # acquire returning and the release being armed does not strand an ``flock`` -- which,
2172
+ # unlike a thread lock, no other process can break for the life of this one. This
2173
+ # narrows that window to the bytecode between the call returning and the store; it
2174
+ # does not close it, because the eval breaker is still checked there.
2175
+ descriptor: int | None = None
2176
+ try:
2177
+ descriptor = _hold_snapshot_lock(root_fd, lock_name)
2178
+ yield
2179
+ finally:
2180
+ if descriptor is not None:
2181
+ _release_snapshot_lock(root_fd, lock_name, descriptor)
2182
+ finally:
2183
+ with _SNAPSHOT_SEAL_REGISTRY_GUARD:
2184
+ held, holders = _SNAPSHOT_SEAL_LOCKS[key]
2185
+ if holders <= 1:
2186
+ del _SNAPSHOT_SEAL_LOCKS[key]
2187
+ else:
2188
+ _SNAPSHOT_SEAL_LOCKS[key] = (held, holders - 1)
2189
+
2190
+
2191
+ @contextlib.contextmanager
2192
+ def exclusive_content_addressed_name(root: Path, name: str) -> Iterator[None]:
2193
+ """Hold ``name`` under ``root`` against every sealer and remover of it, in and out of process.
2194
+
2195
+ The seam :mod:`mostlyright.data_harness.sources.deletion` reaches for. Removing a published
2196
+ snapshot and validating one are the same window from opposite ends -- an ``unlink`` between a
2197
+ validator's two observations of its descriptor is the ``st_nlink 1 -> 0`` drift of #209 -- and
2198
+ a remover that takes this cannot land inside a sealer's read, or a sealer inside its removal.
2199
+ It is the same pair of locks :func:`_serialized_snapshot_seal` takes, keyed the same way, so
2200
+ the two paths exclude each other rather than each other's copy.
2201
+
2202
+ The key is the root's device and inode, read from this function's own descriptor. A caller
2203
+ that re-opens the root by pathname afterwards is holding a lock on the directory this saw,
2204
+ which is the directory that name resolved to when the lock was taken; if the pathname is
2205
+ rebound in between, the caller validates a different directory than the one locked. Nothing
2206
+ in this tree does that on purpose -- the caller that exists,
2207
+ :meth:`~mostlyright.data_harness.sources.deletion.DeletionCoordinator.delete`, re-validates
2208
+ its target by identity and refuses anything that moved -- and a caller that wants the
2209
+ guarantee outright should hand its own descriptor down rather than a path.
2210
+ """
2211
+
2212
+ if not root.is_absolute():
2213
+ raise AcquisitionSecurityError("SNAPSHOT_ROOT", "snapshot root must be absolute")
2214
+ if name in {"", ".", ".."} or name != PurePosixPath(name).name:
2215
+ raise AcquisitionSecurityError(
2216
+ "SNAPSHOT_NAME",
2217
+ "content-addressed name must be one path component",
2218
+ )
2219
+ directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0)
2220
+ try:
2221
+ root_fd = os.open(root, directory_flags)
2222
+ except OSError:
2223
+ raise AcquisitionSecurityError(
2224
+ "SNAPSHOT_ROOT",
2225
+ "snapshot root must be an existing non-symlink directory",
2226
+ ) from None
2227
+ try:
2228
+ root_info = os.fstat(root_fd)
2229
+ with _serialized_snapshot_seal(root_fd, root_info.st_dev, root_info.st_ino, name):
2230
+ yield
2231
+ finally:
2232
+ os.close(root_fd)
2233
+
2234
+
2235
+ def _hold_snapshot_lock(root_fd: int, lock_name: str) -> int:
2236
+ """Take the cross-process half of the per-name lock, and prove it is still that name's.
2237
+
2238
+ ``flock`` binds an inode, not a name, so a lock file that is unlinked while a waiter is
2239
+ blocked on it leaves that waiter holding an inode no longer reachable through the name --
2240
+ while a newcomer creates a fresh file at the name and takes it uncontended. Two holders, no
2241
+ exclusion. The published fix for that is here: unlink only while holding the lock (see
2242
+ :func:`_release_snapshot_lock`), and after acquiring, re-read the name and accept only if it
2243
+ still resolves to the inode just locked. Anything else means the file was replaced under the
2244
+ wait, so drop it and take the new one.
2245
+
2246
+ ``ENOENT`` from an ``O_CREAT`` open is part of that same race rather than a fault. Measured
2247
+ on macOS/APFS, six processes opening and unlinking one name 4,000 times each took a transient
2248
+ ``ENOENT`` on 17 to 26 percent of opens -- the entry is found and then removed before the open
2249
+ completes -- so it is retried, not refused. Every other errno is refused: an unreadable entry
2250
+ at this name is not contention, and ``O_NOFOLLOW`` makes a symlink here ``ELOOP`` rather than
2251
+ something to follow.
2252
+
2253
+ **Queueing is not bounded, and a ceiling on it would be a bug.** Losing the name is
2254
+ queueing, not tampering, and every loss means some other holder had it -- so a ceiling refuses
2255
+ a byte-correct snapshot for being popular, which is the class of false refusal #209 is about.
2256
+ An earlier revision capped total attempts at 1024 and an independent review measured the cap
2257
+ being spent by ordinary contention. What a ceiling would otherwise guard against is a hot
2258
+ loop, and :func:`_snapshot_lock_backoff` rules that out: every retry sleeps, so this waits at
2259
+ a bounded rate exactly as the ``flock`` above waits without one. A same-UID racer rewriting
2260
+ this name in a loop can stall a caller here, which is strictly less than the stall it can
2261
+ already impose by holding the lock and never releasing it.
2262
+
2263
+ **One state is not queueing, and it does end the wait.** ``openat`` answers ``ENOENT`` both
2264
+ when a holder unlinked as this open resolved the name and when the directory ``root_fd``
2265
+ names has been REMOVED -- and the second is permanent for the life of that descriptor, so
2266
+ retrying it never terminates. Nothing hostile is needed to reach it: a caller queued behind
2267
+ a multi-gigabyte seal holds a ``root_fd`` that a sweep of the acquisition boundary
2268
+ invalidates, and neither ``st_nlink`` nor ``stat(".", dir_fd=...)`` tells the two apart on
2269
+ APFS. So consecutive losses on that branch alone are counted against
2270
+ :data:`MAX_CONSECUTIVE_VANISHED_LOCK_OPENS`, and any other outcome resets the count: a queued
2271
+ caller keeps an unbounded wait, and a vanished root gets a prompt, typed refusal.
2272
+
2273
+ ``O_NONBLOCK`` is not optional and is here for the reason it is on the snapshot read at
2274
+ :func:`_read_existing_digest_snapshot`: opening a FIFO read-only BLOCKS in the kernel until a
2275
+ writer arrives, and nothing on this path holds a deadline. Without it a same-UID racer could
2276
+ leave one ``mkfifo`` at this name and wedge every sealer and every governed deletion of that
2277
+ digest for the life of the process -- an artifact that outlives the racer, that
2278
+ ``sources.deletion`` cannot remove because it matches only ``<64 hex>.<suffix>``, and that no
2279
+ kind check can reach because the open never returns.
2280
+
2281
+ The kind is bound too, and it is bound before the wait: a FIFO opens without blocking under
2282
+ ``O_NONBLOCK``, and an exclusive ``flock`` already held on a hostile FIFO would otherwise
2283
+ queue this caller behind a live racer instead of refusing -- the kind check exists to refuse
2284
+ exactly that, so it runs on the descriptor before the lock is asked for. A directory at this
2285
+ name refuses at the open itself, because the descriptor is opened for writing.
2286
+
2287
+ It is opened for writing -- ``O_RDWR``, never ``O_RDONLY`` -- even though nothing is ever
2288
+ written through it, because an exclusive ``flock`` is emulated as a whole-file write lock on
2289
+ NFS and refuses a read-only descriptor with ``EBADF``. Every other regular-file lock in this
2290
+ tree already opens ``O_RDWR`` for this reason, ``review``'s publication lock and
2291
+ ``FileCapStore.lock`` among them.
2292
+
2293
+ None of this is a security control. The stat binding in
2294
+ :func:`_read_existing_digest_snapshot` is what refuses a hostile link, and it is untouched by
2295
+ anything here; a same-UID racer squatting this name can withhold the lock, which costs
2296
+ availability that a same-UID racer in the snapshot root already has by other means.
2297
+
2298
+ **What the storage has to be.** ``flock`` is a POSIX advisory lock, so the snapshot root has
2299
+ to sit on a filesystem that honours it, exactly as ``FileCapStore`` requires of the volume it
2300
+ is pointed at. A Cloud Storage bucket mounted with gcsfuse is not such a volume. Nothing
2301
+ here can detect the difference in advance, so a filesystem that refuses the lock refuses the
2302
+ seal, with the errno in the message: sealing without exclusion is the state #209 was filed
2303
+ about, and choosing it silently is not this function's call to make.
2304
+
2305
+ That is a widened requirement and it is worth stating plainly. The lock file has to be
2306
+ creatable, so sealing now needs write access to the snapshot root even on the path where the
2307
+ snapshot is already published and nothing would be written -- a read-only root, or one whose
2308
+ permissions were narrowed after the fact, refuses with ``EROFS`` or ``EACCES`` where it used
2309
+ to return the existing path.
2310
+
2311
+ A refusal raised after the lock file is created leaves that file behind, because a caller that
2312
+ could not establish it holds the name must not unlink a name it cannot prove is its own. On a
2313
+ volume that honours the lock the entry is inert and self-healing -- the next successful hold
2314
+ of that digest adopts and removes it -- and one stray zero-byte file is the cheaper of the two
2315
+ mistakes. On a volume that refuses the lock outright no later hold ever succeeds, so a stray
2316
+ left there is permanent until an operator removes it; the refusal that created it is the
2317
+ signal that the volume cannot host this root at all.
2318
+ """
2319
+
2320
+ if fcntl is None: # pragma: no cover - POSIX-only path; sealing needs dir_fd links anyway
2321
+ raise AcquisitionSecurityError(
2322
+ "SNAPSHOT_LOCK",
2323
+ "content-addressed sealing requires POSIX advisory locking",
2324
+ )
2325
+ flags = (
2326
+ os.O_RDWR
2327
+ | os.O_CREAT
2328
+ | getattr(os, "O_NOFOLLOW", 0)
2329
+ | getattr(os, "O_NONBLOCK", 0)
2330
+ | getattr(os, "O_CLOEXEC", 0)
2331
+ )
2332
+ attempt = 0
2333
+ vanished = 0
2334
+ while True:
2335
+ if attempt:
2336
+ time.sleep(_snapshot_lock_backoff(attempt))
2337
+ attempt += 1
2338
+ try:
2339
+ descriptor = os.open(lock_name, flags, 0o600, dir_fd=root_fd)
2340
+ except FileNotFoundError:
2341
+ # Two different states arrive here. A holder unlinking as this open resolved the
2342
+ # name is transient and must be retried; a root directory REMOVED out from under this
2343
+ # descriptor gives the same errno forever, and retrying that is a spin with no end.
2344
+ # Only consecutive losses are counted, so a caller merely queueing -- which never
2345
+ # loses on this branch, it loses on the identity re-check below -- keeps its
2346
+ # unbounded wait. Sixteen and twenty-four processes contending on one name never ran
2347
+ # a streak longer than four.
2348
+ vanished += 1
2349
+ if vanished > MAX_CONSECUTIVE_VANISHED_LOCK_OPENS:
2350
+ raise AcquisitionSecurityError(
2351
+ "SNAPSHOT_LOCK",
2352
+ "snapshot name lock could not be created"
2353
+ " (the snapshot root no longer holds a name to lock)",
2354
+ ) from None
2355
+ continue
2356
+ except OSError as error:
2357
+ raise AcquisitionSecurityError(
2358
+ "SNAPSHOT_LOCK",
2359
+ "snapshot name lock is not a safe lock file"
2360
+ f" (open refused: {_describe_os_error(error)})",
2361
+ ) from None
2362
+ try:
2363
+ # Kind first, wait second: the descriptor's kind is a property of its inode and
2364
+ # cannot change, and binding it before the flock means a hostile FIFO or directory
2365
+ # already holding an exclusive lock is refused here rather than queued on -- the
2366
+ # same order _require_readable_evidence_kind states for the evidence reads.
2367
+ held = os.fstat(descriptor)
2368
+ if not stat.S_ISREG(held.st_mode):
2369
+ raise AcquisitionSecurityError(
2370
+ "SNAPSHOT_LOCK",
2371
+ "snapshot name lock is not a safe lock file"
2372
+ f" (kind {stat.S_IFMT(held.st_mode):#o} is not a regular file)",
2373
+ )
2374
+ fcntl.flock(descriptor, fcntl.LOCK_EX)
2375
+ # An adopted stray keeps whatever mode its creator gave it; bind it to the mode a
2376
+ # fresh create gets, the way the review publication lock does after its own flock.
2377
+ os.fchmod(descriptor, 0o600)
2378
+ try:
2379
+ named: os.stat_result | None = os.stat(
2380
+ lock_name,
2381
+ dir_fd=root_fd,
2382
+ follow_symlinks=False,
2383
+ )
2384
+ except FileNotFoundError:
2385
+ named = None
2386
+ except OSError as error:
2387
+ os.close(descriptor)
2388
+ raise AcquisitionSecurityError(
2389
+ "SNAPSHOT_LOCK",
2390
+ f"snapshot name lock could not be taken ({_describe_os_error(error)})",
2391
+ ) from None
2392
+ except BaseException:
2393
+ os.close(descriptor)
2394
+ raise
2395
+ vanished = 0
2396
+ if named is not None and (named.st_dev, named.st_ino) == (held.st_dev, held.st_ino):
2397
+ return descriptor
2398
+ os.close(descriptor)
2399
+
2400
+
2401
+ def _snapshot_lock_backoff(attempt: int) -> float:
2402
+ """Grow the wait between attempts, and spread it so the woken herd does not re-race in step.
2403
+
2404
+ The spread is derived from the caller's process and thread rather than from :mod:`random`,
2405
+ which nothing in this package imports: what has to differ between two waiters is their phase,
2406
+ not the unpredictability of it, and a deterministic function of identifiers that already
2407
+ differ gives exactly that while leaving the sealer's behaviour reproducible. Only timing
2408
+ depends on it -- never which caller wins, and never what is accepted.
2409
+
2410
+ Between PROCESSES the pid does all of the work. ``threading.get_ident()`` was measured
2411
+ identical in eight separate interpreters on macOS, so the thread half separates threads and
2412
+ nothing else -- which is the half that matters least, because threads of one process are
2413
+ already serialized on this name before they reach here. Over sixteen consecutive pids the
2414
+ sixteen spreads are distinct at every attempt, and the delay stays inside its declared
2415
+ bounds: 120 us to 4998 us against a 5000 us ceiling over the first hundred thousand attempts.
2416
+ """
2417
+
2418
+ spread = (((os.getpid() ^ threading.get_ident()) * 2_654_435_761) + attempt * 40_503) % 1024
2419
+ ceiling = min(attempt * SNAPSHOT_LOCK_BACKOFF_SECONDS, MAX_SNAPSHOT_LOCK_BACKOFF)
2420
+ return ceiling * (0.5 + spread / 2048)
2421
+
2422
+
2423
+ def _release_snapshot_lock(root_fd: int, lock_name: str, descriptor: int) -> None:
2424
+ """Drop the lock file and then the lock, in that order.
2425
+
2426
+ Unlinking first is what makes the identity re-check in :func:`_hold_snapshot_lock` sufficient:
2427
+ the name can only stop resolving to this inode while this process still holds it, so no waiter
2428
+ can be admitted on a stale inode. Closing is what releases the ``flock``, so it comes second,
2429
+ and that order is pinned by
2430
+ ``tests/h3/test_http.py::test_the_name_lock_is_unlinked_before_the_lock_on_it_is_dropped``
2431
+ rather than left to whoever edits these four lines next. A settled root is left holding
2432
+ snapshots and nothing else, which is the invariant the staging name already keeps.
2433
+
2434
+ Create-and-unlink is the harder of the two shapes a lock file can take, and it is chosen
2435
+ deliberately. ``FileCapStore.lock`` next door keeps one sidecar that is created once and
2436
+ never replaced, which needs no protocol at all -- but there is one cap store and there is one
2437
+ lock file per digest, and ``sources.deletion`` matches removable names against
2438
+ ``<64 hex>.<suffix>`` alone, so a permanent sidecar per digest is an entry no governed path
2439
+ could ever remove. The protocol above is the price of not leaving those behind.
2440
+
2441
+ This runs from a ``finally``, so a failure to unlink is swallowed rather than allowed to
2442
+ replace whatever the sealer was already reporting. A lock file left behind costs one stray
2443
+ entry; a raise here would turn a published snapshot into a refusal, or bury the refusal that
2444
+ matters under a second one about a lock. The close is what has to happen, and it does.
2445
+ """
2446
+
2447
+ try:
2448
+ with contextlib.suppress(OSError):
2449
+ os.unlink(lock_name, dir_fd=root_fd)
2450
+ finally:
2451
+ os.close(descriptor)
2452
+
2453
+
2454
+ def _read_existing_digest_snapshot(
2455
+ root_fd: int,
2456
+ name: str,
2457
+ *,
2458
+ expected_content: bytes,
2459
+ expected_digest: str,
2460
+ ownership_policy: tuple[int, int],
2461
+ ) -> None:
2462
+ """Authenticate one already-named immutable snapshot without trusting its pathname."""
2463
+
2464
+ flags = (
2465
+ os.O_RDONLY
2466
+ | getattr(os, "O_NOFOLLOW", 0)
2467
+ | getattr(os, "O_NONBLOCK", 0)
2468
+ | getattr(os, "O_CLOEXEC", 0)
2469
+ )
2470
+ try:
2471
+ descriptor = os.open(name, flags, dir_fd=root_fd)
2472
+ except FileNotFoundError:
2473
+ raise
2474
+ except OSError as error:
2475
+ raise AcquisitionSecurityError(
2476
+ "SNAPSHOT_COLLISION",
2477
+ "existing content-addressed snapshot is not a safe immutable file"
2478
+ f" (open refused: {_describe_os_error(error)})",
2479
+ ) from None
2480
+ try:
2481
+ opened = os.fstat(descriptor)
2482
+ _require_existing_snapshot_stat(
2483
+ opened,
2484
+ expected_size=len(expected_content),
2485
+ ownership_policy=ownership_policy,
2486
+ )
2487
+ try:
2488
+ named = os.stat(name, dir_fd=root_fd, follow_symlinks=False)
2489
+ except OSError as error:
2490
+ raise AcquisitionSecurityError(
2491
+ "SNAPSHOT_COLLISION",
2492
+ "existing content-addressed snapshot name changed during validation"
2493
+ f" (before the read: {_describe_os_error(error)})",
2494
+ ) from None
2495
+ _require_stable_snapshot_stat(opened, named, observed_between=_OPEN_VERSUS_NAME)
2496
+ _require_snapshot_fd_bytes(descriptor, expected_content, expected_digest)
2497
+ _require_stable_snapshot_stat(
2498
+ opened,
2499
+ os.fstat(descriptor),
2500
+ observed_between=_OPEN_VERSUS_READ,
2501
+ )
2502
+ finally:
2503
+ os.close(descriptor)
2504
+ try:
2505
+ after = os.stat(name, dir_fd=root_fd, follow_symlinks=False)
2506
+ except OSError as error:
2507
+ raise AcquisitionSecurityError(
2508
+ "SNAPSHOT_COLLISION",
2509
+ "existing content-addressed snapshot name changed during validation"
2510
+ f" (after the read: {_describe_os_error(error)})",
2511
+ ) from None
2512
+ _require_stable_snapshot_stat(named, after, observed_between=_NAME_VERSUS_READ)
2513
+
2514
+
2515
+ def _describe_os_error(error: OSError) -> str:
2516
+ """Name an errno without putting a path or any other caller string in the message."""
2517
+
2518
+ return f"{errno.errorcode.get(error.errno or 0, 'errno')} {error.strerror or ''}".strip()
2519
+
2520
+
2521
+ def _require_existing_snapshot_stat(
2522
+ info: os.stat_result,
2523
+ *,
2524
+ expected_size: int,
2525
+ ownership_policy: tuple[int, int],
2526
+ ) -> None:
2527
+ """Refuse an existing entry that is not one immutable, singly-linked, owned regular file.
2528
+
2529
+ Every field is evaluated rather than short-circuited so the refusal can name each one that
2530
+ failed with its observed and required value. A ``SNAPSHOT_COLLISION`` reaching an operator
2531
+ from a machine nobody can attach a debugger to is otherwise indistinguishable from every
2532
+ other reason this entry could be unsafe, and those reasons used to share one sentence.
2533
+ ``st_nlink`` is the one that a concurrent same-digest sealer can move.
2534
+ """
2535
+
2536
+ faults: list[str] = []
2537
+ if not stat.S_ISREG(info.st_mode):
2538
+ faults.append(f"kind {stat.S_IFMT(info.st_mode):#o} is not a regular file")
2539
+ if stat.S_IMODE(info.st_mode) != 0o444:
2540
+ faults.append(f"st_mode {stat.S_IMODE(info.st_mode):#o} != 0o444")
2541
+ if info.st_nlink != 1:
2542
+ faults.append(f"st_nlink {info.st_nlink} != 1")
2543
+ if info.st_size != expected_size:
2544
+ faults.append(f"st_size {info.st_size} != {expected_size}")
2545
+ if info.st_uid != ownership_policy[0]:
2546
+ faults.append(f"st_uid {info.st_uid} != {ownership_policy[0]}")
2547
+ if info.st_gid != ownership_policy[1]:
2548
+ faults.append(f"st_gid {info.st_gid} != {ownership_policy[1]}")
2549
+ if faults:
2550
+ raise AcquisitionSecurityError(
2551
+ "SNAPSHOT_COLLISION",
2552
+ "existing content-addressed snapshot is not a safe immutable file"
2553
+ f" ({', '.join(faults)})",
2554
+ )
2555
+
2556
+
2557
+ # The three points at which the bound stat signature is re-observed. They are named in the
2558
+ # refusal so a field report says which observation drifted rather than only that one did.
2559
+ _OPEN_VERSUS_NAME = "the opened descriptor and the named entry"
2560
+ _OPEN_VERSUS_READ = "the opened descriptor before and after the read"
2561
+ _NAME_VERSUS_READ = "the named entry before and after the read"
2562
+
2563
+ # The one drift with a single cause, said in words rather than left to be read off a field.
2564
+ _LAST_NAME_REMOVED = "; the last name for it was removed while it was open"
2565
+
2566
+ _SNAPSHOT_STAT_FIELDS = (
2567
+ "st_dev",
2568
+ "st_ino",
2569
+ "st_uid",
2570
+ "st_gid",
2571
+ "st_mode",
2572
+ "st_nlink",
2573
+ "st_size",
2574
+ "st_mtime_ns",
2575
+ "st_ctime_ns",
2576
+ )
2577
+
2578
+
2579
+ def _require_stable_snapshot_stat(
2580
+ before: os.stat_result,
2581
+ after: os.stat_result,
2582
+ *,
2583
+ observed_between: str,
2584
+ ) -> None:
2585
+ """Refuse a snapshot whose bound identity moved, naming where it moved and by what.
2586
+
2587
+ The signature is a deliberate anti-TOCTOU binding, so it stays exactly as wide as it was.
2588
+ What changes is only what the refusal says: which of the three observation pairs disagreed,
2589
+ and every field that differs with its old and new value. ``st_nlink`` moving means another
2590
+ name was attached to or detached from this inode; ``st_ino`` or ``st_dev`` moving means the
2591
+ name was rebound to a different file; ``st_ctime_ns`` alone means metadata was touched.
2592
+ Those are different incidents and a field recurrence has to be able to tell them apart.
2593
+
2594
+ ``st_nlink`` reaching zero is the one of those with only one cause, so it is named rather than
2595
+ left to be inferred: the last name for this inode was removed while this descriptor held it
2596
+ open, which is a deletion of the snapshot and not a change to it. That is the refusal #209
2597
+ reported from the field, and it is the reason a remover inside this tree now takes the same
2598
+ per-name lock a sealer does -- so a refusal carrying this sentence is a remover the harness
2599
+ does not own.
2600
+ """
2601
+
2602
+ moved = tuple(
2603
+ f"{field} {getattr(before, field)} -> {getattr(after, field)}"
2604
+ for field in _SNAPSHOT_STAT_FIELDS
2605
+ if getattr(before, field) != getattr(after, field)
2606
+ )
2607
+ if moved:
2608
+ raise AcquisitionSecurityError(
2609
+ "SNAPSHOT_COLLISION",
2610
+ "existing content-addressed snapshot changed during validation"
2611
+ f" (between {observed_between}: {', '.join(moved)})"
2612
+ f"{_LAST_NAME_REMOVED if before.st_nlink and not after.st_nlink else ''}",
2613
+ )
2614
+
2615
+
2616
+ def _is_allowlisted_snapshot_suffix(suffix: str) -> bool:
2617
+ """Report whether a snapshot filename suffix is allowlisted.
2618
+
2619
+ Read from the single closed set rather than restated or unioned here. What this admits
2620
+ for writing is exactly what ``SNAPSHOT_SUFFIX_PATTERN`` -- and therefore the governed
2621
+ deletion path -- will match for removal: a suffix that could be sealed but not matched
2622
+ would strand snapshots no retention sweep can find.
2623
+ """
2624
+
2625
+ return suffix in SNAPSHOT_SUFFIXES
2626
+
2627
+
2628
+ def _validate_response_headers(
2629
+ headers: tuple[tuple[str, str], ...],
2630
+ ) -> tuple[tuple[str, str], ...]:
2631
+ if len(headers) > MAX_HEADER_COUNT:
2632
+ raise AcquisitionSecurityError(
2633
+ "RESPONSE_HEADERS",
2634
+ f"response has more than {MAX_HEADER_COUNT} headers",
2635
+ )
2636
+ total = 0
2637
+ normalized: list[tuple[str, str]] = []
2638
+ for raw_name, raw_value in headers:
2639
+ name = raw_name.lower().strip()
2640
+ value = raw_value.strip()
2641
+ if (
2642
+ not name
2643
+ or not all(char.isalnum() or char == "-" for char in name)
2644
+ or "\r" in value
2645
+ or "\n" in value
2646
+ ):
2647
+ raise AcquisitionSecurityError(
2648
+ "RESPONSE_HEADER",
2649
+ "response contains an invalid header",
2650
+ )
2651
+ total += len(name.encode("ascii", errors="strict")) + len(
2652
+ value.encode("utf-8", errors="strict")
2653
+ )
2654
+ normalized.append((name, value))
2655
+ if total > MAX_HEADER_BYTES:
2656
+ raise AcquisitionSecurityError(
2657
+ "RESPONSE_HEADERS",
2658
+ f"response headers exceed {MAX_HEADER_BYTES} bytes",
2659
+ )
2660
+ return tuple(normalized)
2661
+
2662
+
2663
+ def _parse_content_range(value: str | None) -> tuple[int, int, int | None]:
2664
+ """Parse a ``Content-Range`` strictly, or refuse it."""
2665
+
2666
+ if value is None:
2667
+ raise AcquisitionSecurityError(
2668
+ "CONTENT_RANGE",
2669
+ "partial response carries no content range",
2670
+ )
2671
+ match = _CONTENT_RANGE.fullmatch(value)
2672
+ if match is None:
2673
+ raise AcquisitionSecurityError(
2674
+ "CONTENT_RANGE",
2675
+ "response content range is not a single canonical byte span",
2676
+ )
2677
+ first_byte = int(match.group(1))
2678
+ last_byte = int(match.group(2))
2679
+ complete = match.group(3)
2680
+ if last_byte < first_byte:
2681
+ raise AcquisitionSecurityError(
2682
+ "CONTENT_RANGE",
2683
+ "response content range ends before it begins",
2684
+ )
2685
+ complete_size = None if complete == "*" else int(complete)
2686
+ if complete_size is not None and complete_size < last_byte + 1:
2687
+ raise AcquisitionSecurityError(
2688
+ "CONTENT_RANGE",
2689
+ "response content range runs past the object it names",
2690
+ )
2691
+ return first_byte, last_byte, complete_size
2692
+
2693
+
2694
+ def _accept_header(limits: RetrievalLimits) -> str:
2695
+ """What this retrieval will admit, spelled every way it would admit it.
2696
+
2697
+ Derived from the allowlist and the alias table rather than restated, so what is asked
2698
+ for cannot drift from what is accepted: a source negotiating on ``Accept`` must not be
2699
+ told a spelling is unwelcome that the response check would then have admitted. Sorted,
2700
+ so one allowlist has one header.
2701
+ """
2702
+
2703
+ admitted = set(limits.allowed_media_types)
2704
+ admitted.update(
2705
+ alias for alias, canonical in MEDIA_TYPE_ALIASES.items() if canonical in admitted
2706
+ )
2707
+ return ", ".join(sorted(admitted))
2708
+
2709
+
2710
+ def _media_type(value: str | None) -> str:
2711
+ """The one reader of a ``Content-Type`` header, and the one place a spelling is settled.
2712
+
2713
+ Parameters are dropped, case is folded, and a non-standard spelling of a type is
2714
+ canonicalised to the spelling every allowlist in this package is written in. The
2715
+ canonicalisation belongs HERE rather than beside each allowlist: an alias added to one
2716
+ tuple and missed on the other is precisely how the slice path came to admit the object
2717
+ but refuse the sidecar of the same S3 response. One normaliser means the allowlists, the
2718
+ ``multipart/byteranges`` refusal, the outgoing ``Accept`` header and the media type this
2719
+ boundary reports all speak the same vocabulary, and a new alias is one line.
2720
+ """
2721
+
2722
+ if value is None:
2723
+ raise AcquisitionSecurityError("MEDIA_TYPE", "response has no content type")
2724
+ media_type = value.split(";", 1)[0].strip().lower()
2725
+ if not media_type or "/" not in media_type:
2726
+ raise AcquisitionSecurityError("MEDIA_TYPE", "response content type is invalid")
2727
+ return MEDIA_TYPE_ALIASES.get(media_type, media_type)
2728
+
2729
+
2730
+ def _require_snapshot_fd_bytes(
2731
+ file_descriptor: int,
2732
+ expected_content: bytes,
2733
+ expected_digest: str,
2734
+ ) -> None:
2735
+ """Verify a descriptor's bytes against the expected snapshot without materializing them.
2736
+
2737
+ The verdict is identical to reading the whole descriptor and then comparing: the same
2738
+ ``expected_length + 1`` over-read still detects a longer file, the length is still decided
2739
+ before the contents are, and acceptance still requires *both* the expected SHA-256 and exact
2740
+ equality with the expected bytes. Only the residency changes. Accumulating the file into a
2741
+ chunk list and joining it cost a second and a third full copy alongside the caller's own
2742
+ ``content``, so sealing the 2.12 GB EPA snapshot peaked near 6.4 GB; comparing each 64 KiB
2743
+ chunk against the corresponding zero-copy slice of ``expected_content`` peaks at one copy
2744
+ plus a chunk.
2745
+
2746
+ A mismatched chunk stops further comparison but not the read: the length verdict has to come
2747
+ first, exactly as it did when the whole file was read before anything was compared, so a
2748
+ short *and* different file still reports its length rather than its contents.
2749
+ """
2750
+
2751
+ expected_length = len(expected_content)
2752
+ expected = memoryview(expected_content)
2753
+ digest = hashlib.sha256()
2754
+ identical = True
2755
+ total = 0
2756
+ while total <= expected_length:
2757
+ chunk = os.read(file_descriptor, min(65_536, expected_length + 1 - total))
2758
+ if not chunk:
2759
+ break
2760
+ digest.update(chunk)
2761
+ if identical:
2762
+ identical = expected[total : total + len(chunk)] == chunk
2763
+ total += len(chunk)
2764
+ if total != expected_length:
2765
+ raise AcquisitionSecurityError(
2766
+ "SNAPSHOT_COLLISION",
2767
+ f"existing snapshot length differs (read {total} bytes, expected {expected_length})",
2768
+ )
2769
+ if digest.hexdigest() != expected_digest or not identical:
2770
+ raise AcquisitionSecurityError(
2771
+ "SNAPSHOT_COLLISION",
2772
+ "existing content-addressed snapshot bytes differ",
2773
+ )