mostlyright-data 0.9.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (314) hide show
  1. mostlyright/data_harness/__init__.py +158 -0
  2. mostlyright/data_harness/acquisition/__init__.py +55 -0
  3. mostlyright/data_harness/acquisition/http.py +2773 -0
  4. mostlyright/data_harness/acquisition/parsing.py +809 -0
  5. mostlyright/data_harness/acquisition/ranges.py +495 -0
  6. mostlyright/data_harness/acquisition/result_download.py +360 -0
  7. mostlyright/data_harness/acquisition/retention_admission.py +248 -0
  8. mostlyright/data_harness/acquisition/sandbox.py +4888 -0
  9. mostlyright/data_harness/acquisition/url_policy.py +530 -0
  10. mostlyright/data_harness/agent_runtime.py +2743 -0
  11. mostlyright/data_harness/assets/logo-ink.svg +31 -0
  12. mostlyright/data_harness/backends/__init__.py +28 -0
  13. mostlyright/data_harness/backends/pandas_backend.py +350 -0
  14. mostlyright/data_harness/backends/polars_backend.py +366 -0
  15. mostlyright/data_harness/backends/protocol.py +124 -0
  16. mostlyright/data_harness/backends/reference.py +83 -0
  17. mostlyright/data_harness/backends/registry.py +55 -0
  18. mostlyright/data_harness/backends/restrictions.py +126 -0
  19. mostlyright/data_harness/canonical.py +333 -0
  20. mostlyright/data_harness/catalog_job.py +625 -0
  21. mostlyright/data_harness/cli.py +5398 -0
  22. mostlyright/data_harness/contracts.py +53 -0
  23. mostlyright/data_harness/coordinator.py +1307 -0
  24. mostlyright/data_harness/deploy.py +924 -0
  25. mostlyright/data_harness/deploy_target.py +312 -0
  26. mostlyright/data_harness/deployment_evidence.py +1067 -0
  27. mostlyright/data_harness/event_presentation.py +576 -0
  28. mostlyright/data_harness/events.py +2152 -0
  29. mostlyright/data_harness/fast_delimited.py +239 -0
  30. mostlyright/data_harness/fleet.py +237 -0
  31. mostlyright/data_harness/formats.py +236 -0
  32. mostlyright/data_harness/governors.py +1163 -0
  33. mostlyright/data_harness/hosted_bootstrap.py +972 -0
  34. mostlyright/data_harness/hosted_crawler.py +1115 -0
  35. mostlyright/data_harness/hosted_crawler_container_smoke.py +351 -0
  36. mostlyright/data_harness/hosted_crawler_fetch.py +423 -0
  37. mostlyright/data_harness/hosted_crawler_job.py +1277 -0
  38. mostlyright/data_harness/hosted_crawler_protocol.py +676 -0
  39. mostlyright/data_harness/hosted_dataset.py +1500 -0
  40. mostlyright/data_harness/hosted_deploy.py +3037 -0
  41. mostlyright/data_harness/hosted_handoff.py +62 -0
  42. mostlyright/data_harness/hosted_ingestion_contract.py +504 -0
  43. mostlyright/data_harness/hosted_ingestion_job.py +356 -0
  44. mostlyright/data_harness/hosted_ingestion_job_smoke.py +40 -0
  45. mostlyright/data_harness/hosted_session_container_smoke.py +194 -0
  46. mostlyright/data_harness/hosted_session_worker.py +3554 -0
  47. mostlyright/data_harness/hosted_session_worker_job_smoke.py +46 -0
  48. mostlyright/data_harness/hosted_worker.py +6784 -0
  49. mostlyright/data_harness/ingestion/__init__.py +56 -0
  50. mostlyright/data_harness/ingestion/contracts.py +461 -0
  51. mostlyright/data_harness/ingestion/faults.py +42 -0
  52. mostlyright/data_harness/ingestion/gcs_store.py +1162 -0
  53. mostlyright/data_harness/ingestion/spool.py +130 -0
  54. mostlyright/data_harness/ingestion/store.py +885 -0
  55. mostlyright/data_harness/key_seam.py +434 -0
  56. mostlyright/data_harness/linux_process_boundary.py +262 -0
  57. mostlyright/data_harness/local_contracts.py +2880 -0
  58. mostlyright/data_harness/local_search/__init__.py +5 -0
  59. mostlyright/data_harness/local_search/build_index.py +1087 -0
  60. mostlyright/data_harness/local_search/contracts.py +920 -0
  61. mostlyright/data_harness/local_search/query_trace.py +266 -0
  62. mostlyright/data_harness/local_search/retrieval.py +700 -0
  63. mostlyright/data_harness/local_search/sealed.py +474 -0
  64. mostlyright/data_harness/local_search/service.py +784 -0
  65. mostlyright/data_harness/nbrender/CONTRACT.md +212 -0
  66. mostlyright/data_harness/nbrender/__init__.py +12 -0
  67. mostlyright/data_harness/nbrender/chrome.py +359 -0
  68. mostlyright/data_harness/nbrender/code_body.py +266 -0
  69. mostlyright/data_harness/nbrender/document.py +407 -0
  70. mostlyright/data_harness/nbrender/frame.py +275 -0
  71. mostlyright/data_harness/nbrender/interactive.py +337 -0
  72. mostlyright/data_harness/nbrender/markdown_body.py +477 -0
  73. mostlyright/data_harness/nbrender/mr_components.py +134 -0
  74. mostlyright/data_harness/nbrender/outputs_data.py +595 -0
  75. mostlyright/data_harness/nbrender/outputs_rich.py +906 -0
  76. mostlyright/data_harness/nbrender/outputs_source.py +260 -0
  77. mostlyright/data_harness/nbrender/outputs_stage.py +176 -0
  78. mostlyright/data_harness/nbrender/outputs_text.py +400 -0
  79. mostlyright/data_harness/nbrender/parse.py +394 -0
  80. mostlyright/data_harness/nbrender/status.py +40 -0
  81. mostlyright/data_harness/nbrender/tokens.py +1295 -0
  82. mostlyright/data_harness/notebook.py +1710 -0
  83. mostlyright/data_harness/offline.py +2049 -0
  84. mostlyright/data_harness/operation_registry.py +1007 -0
  85. mostlyright/data_harness/operator_setup.py +239 -0
  86. mostlyright/data_harness/pipeline.py +6428 -0
  87. mostlyright/data_harness/plan_graph.py +2026 -0
  88. mostlyright/data_harness/preparation/__init__.py +104 -0
  89. mostlyright/data_harness/preparation/contracts.py +1017 -0
  90. mostlyright/data_harness/preparation/engine.py +221 -0
  91. mostlyright/data_harness/preparation/errors.py +14 -0
  92. mostlyright/data_harness/preparation/gates.py +751 -0
  93. mostlyright/data_harness/preparation/joins.py +574 -0
  94. mostlyright/data_harness/preparation/profile.py +384 -0
  95. mostlyright/data_harness/preparation/table.py +217 -0
  96. mostlyright/data_harness/preparation/transforms.py +568 -0
  97. mostlyright/data_harness/progress_events.py +534 -0
  98. mostlyright/data_harness/readers/__init__.py +46 -0
  99. mostlyright/data_harness/readers/containers.py +963 -0
  100. mostlyright/data_harness/readers/contracts.py +542 -0
  101. mostlyright/data_harness/readers/delimited.py +257 -0
  102. mostlyright/data_harness/readers/grib2/__init__.py +33 -0
  103. mostlyright/data_harness/readers/grib2/admission.py +722 -0
  104. mostlyright/data_harness/readers/grib2/decode.py +1009 -0
  105. mostlyright/data_harness/readers/grib2/geometry.py +1133 -0
  106. mostlyright/data_harness/readers/grib2/portable_math.py +501 -0
  107. mostlyright/data_harness/readers/json_tabular.py +485 -0
  108. mostlyright/data_harness/readers/registry.py +514 -0
  109. mostlyright/data_harness/readers/samples/README.md +110 -0
  110. mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/cities.csv.gz +0 -0
  111. mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/expected.json +24 -0
  112. mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/cities.csv.gz +0 -0
  113. mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/expected.json +24 -0
  114. mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/cities.tar +0 -0
  115. mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/expected.json +24 -0
  116. mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/cities.tar +0 -0
  117. mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/expected.json +24 -0
  118. mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/cities.zip +0 -0
  119. mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/expected.json +25 -0
  120. mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
  121. mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/expected.json +25 -0
  122. mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
  123. mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/expected.json +25 -0
  124. mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/cities.csv +3 -0
  125. mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/expected.json +23 -0
  126. mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/cities.tsv +5 -0
  127. mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/expected.json +25 -0
  128. mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/expected.json +30 -0
  129. mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/observations.csv +5 -0
  130. mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/expected.json +44 -0
  131. mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/stations.json +1 -0
  132. mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/expected.json +48 -0
  133. mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/observations.ndjson +4 -0
  134. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/cities.xlsx +0 -0
  135. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/expected.json +24 -0
  136. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/expected.json +27 -0
  137. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
  138. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/expected.json +27 -0
  139. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
  140. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/README.md +20 -0
  141. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/expected.json +55 -0
  142. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/gfs-2m-temperature.grib2 +0 -0
  143. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/expected.json +54 -0
  144. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
  145. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/expected.json +54 -0
  146. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/hrrr-categorical-rain.grib2 +0 -0
  147. mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/expected.json +54 -0
  148. mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
  149. mostlyright/data_harness/readers/samples.py +582 -0
  150. mostlyright/data_harness/readers/spreadsheet.py +803 -0
  151. mostlyright/data_harness/readers/tabular.py +510 -0
  152. mostlyright/data_harness/recipe.py +5321 -0
  153. mostlyright/data_harness/repair/__init__.py +78 -0
  154. mostlyright/data_harness/repair/adapters.py +274 -0
  155. mostlyright/data_harness/repair/contracts.py +872 -0
  156. mostlyright/data_harness/repair/coordinator.py +1099 -0
  157. mostlyright/data_harness/repair/errors.py +16 -0
  158. mostlyright/data_harness/review.py +2533 -0
  159. mostlyright/data_harness/rowset.py +283 -0
  160. mostlyright/data_harness/serving.py +1975 -0
  161. mostlyright/data_harness/serving_edge.py +590 -0
  162. mostlyright/data_harness/serving_http.py +1031 -0
  163. mostlyright/data_harness/session_probes.py +759 -0
  164. mostlyright/data_harness/signing.py +101 -0
  165. mostlyright/data_harness/source_discovery.py +898 -0
  166. mostlyright/data_harness/sources/__init__.py +209 -0
  167. mostlyright/data_harness/sources/_adapter_steps.py +213 -0
  168. mostlyright/data_harness/sources/adapters.py +1214 -0
  169. mostlyright/data_harness/sources/cadence.py +1428 -0
  170. mostlyright/data_harness/sources/cadence_emission.py +453 -0
  171. mostlyright/data_harness/sources/cadence_history.py +546 -0
  172. mostlyright/data_harness/sources/catalog/__init__.py +17 -0
  173. mostlyright/data_harness/sources/catalog/admission.py +477 -0
  174. mostlyright/data_harness/sources/catalog/authoring.py +1701 -0
  175. mostlyright/data_harness/sources/catalog/authoring_policy.py +701 -0
  176. mostlyright/data_harness/sources/catalog/authoring_shards.py +1217 -0
  177. mostlyright/data_harness/sources/catalog/bounded_io.py +231 -0
  178. mostlyright/data_harness/sources/catalog/channel.py +523 -0
  179. mostlyright/data_harness/sources/catalog/channel_client.py +296 -0
  180. mostlyright/data_harness/sources/catalog/contracts.py +825 -0
  181. mostlyright/data_harness/sources/catalog/coverage.py +137 -0
  182. mostlyright/data_harness/sources/catalog/delta.py +1340 -0
  183. mostlyright/data_harness/sources/catalog/embedding.py +532 -0
  184. mostlyright/data_harness/sources/catalog/entry_v2.py +1182 -0
  185. mostlyright/data_harness/sources/catalog/fill.py +3889 -0
  186. mostlyright/data_harness/sources/catalog/fill_partitions.py +459 -0
  187. mostlyright/data_harness/sources/catalog/fill_staging.py +1105 -0
  188. mostlyright/data_harness/sources/catalog/gating.py +374 -0
  189. mostlyright/data_harness/sources/catalog/generation_receipt.py +1607 -0
  190. mostlyright/data_harness/sources/catalog/harvest/__init__.py +7 -0
  191. mostlyright/data_harness/sources/catalog/harvest/ckan.py +384 -0
  192. mostlyright/data_harness/sources/catalog/harvest/datagov_v4.py +798 -0
  193. mostlyright/data_harness/sources/catalog/harvest/protocol.py +964 -0
  194. mostlyright/data_harness/sources/catalog/harvest/sdmx.py +445 -0
  195. mostlyright/data_harness/sources/catalog/harvest/stac.py +384 -0
  196. mostlyright/data_harness/sources/catalog/health.py +447 -0
  197. mostlyright/data_harness/sources/catalog/hosted_catalog.py +105 -0
  198. mostlyright/data_harness/sources/catalog/identity_history.py +1549 -0
  199. mostlyright/data_harness/sources/catalog/neural.py +1618 -0
  200. mostlyright/data_harness/sources/catalog/packed_catalog.py +2345 -0
  201. mostlyright/data_harness/sources/catalog/packed_retrieval.py +1517 -0
  202. mostlyright/data_harness/sources/catalog/packed_writer.py +2802 -0
  203. mostlyright/data_harness/sources/catalog/query_trace.py +1037 -0
  204. mostlyright/data_harness/sources/catalog/recommend.py +171 -0
  205. mostlyright/data_harness/sources/catalog/retrieval.py +230 -0
  206. mostlyright/data_harness/sources/catalog/retrieval_manifest.py +995 -0
  207. mostlyright/data_harness/sources/catalog/rights_decisions.py +254 -0
  208. mostlyright/data_harness/sources/catalog/sealed.py +560 -0
  209. mostlyright/data_harness/sources/catalog/search.py +230 -0
  210. mostlyright/data_harness/sources/catalog/streaming_delta.py +1097 -0
  211. mostlyright/data_harness/sources/catalog/update.py +891 -0
  212. mostlyright/data_harness/sources/collections.py +815 -0
  213. mostlyright/data_harness/sources/contracts.py +2223 -0
  214. mostlyright/data_harness/sources/deletion.py +761 -0
  215. mostlyright/data_harness/sources/fitness.py +162 -0
  216. mostlyright/data_harness/sources/governance.py +163 -0
  217. mostlyright/data_harness/sources/hosted.py +173 -0
  218. mostlyright/data_harness/sources/integration.py +218 -0
  219. mostlyright/data_harness/sources/range_reader.py +418 -0
  220. mostlyright/data_harness/sources/registry.py +514 -0
  221. mostlyright/data_harness/sources/rights_rule.py +59 -0
  222. mostlyright/data_harness/sources/source_cadence_vectors.v1.json +1 -0
  223. mostlyright/data_harness/sources/sports.py +521 -0
  224. mostlyright/data_harness/sources/stream.py +524 -0
  225. mostlyright/data_harness/sources/stream_connector.py +418 -0
  226. mostlyright/data_harness/sources/stream_recorder.py +1404 -0
  227. mostlyright/data_harness/studio_boundary.py +2019 -0
  228. mostlyright/data_harness/thin/__init__.py +37 -0
  229. mostlyright/data_harness/thin/acquire.py +1137 -0
  230. mostlyright/data_harness/thin/acquire_cancel.py +579 -0
  231. mostlyright/data_harness/thin/approvals.py +617 -0
  232. mostlyright/data_harness/thin/commands.py +406 -0
  233. mostlyright/data_harness/thin/download.py +194 -0
  234. mostlyright/data_harness/thin/narrative.py +589 -0
  235. mostlyright/data_harness/thin/parity.py +1070 -0
  236. mostlyright/data_harness/thin/propose.py +2759 -0
  237. mostlyright/data_harness/thin/research.py +1663 -0
  238. mostlyright/data_harness/thin/router.py +924 -0
  239. mostlyright/data_harness/thin/runs.py +519 -0
  240. mostlyright/data_harness/thin/session.py +281 -0
  241. mostlyright/data_harness/thin/stream.py +501 -0
  242. mostlyright/data_harness/thin/transport.py +187 -0
  243. mostlyright/data_harness/thin/vocabulary.py +368 -0
  244. mostlyright/data_harness/thin/workers.py +164 -0
  245. mostlyright/data_harness/ucum/TABLE-PIN.json +40 -0
  246. mostlyright/data_harness/ucum/ucum-subset.v1.json +632 -0
  247. mostlyright/data_harness/unit_flow.py +927 -0
  248. mostlyright/data_harness/units.py +572 -0
  249. mostlyright/data_harness/ux/__init__.py +9 -0
  250. mostlyright/data_harness/ux/approve.py +485 -0
  251. mostlyright/data_harness/ux/author_yaml.py +597 -0
  252. mostlyright/data_harness/ux/cloud_auth.py +447 -0
  253. mostlyright/data_harness/ux/commands/__init__.py +260 -0
  254. mostlyright/data_harness/ux/commands/approve.py +136 -0
  255. mostlyright/data_harness/ux/commands/auth.py +744 -0
  256. mostlyright/data_harness/ux/commands/author.py +79 -0
  257. mostlyright/data_harness/ux/commands/catalog_author.py +403 -0
  258. mostlyright/data_harness/ux/commands/catalog_fill.py +523 -0
  259. mostlyright/data_harness/ux/commands/catalog_harvest.py +545 -0
  260. mostlyright/data_harness/ux/commands/catalog_publish.py +1838 -0
  261. mostlyright/data_harness/ux/commands/catalog_search.py +71 -0
  262. mostlyright/data_harness/ux/commands/catalog_update.py +437 -0
  263. mostlyright/data_harness/ux/commands/deploy.py +134 -0
  264. mostlyright/data_harness/ux/commands/deploy_dataset.py +98 -0
  265. mostlyright/data_harness/ux/commands/deploy_plan.py +105 -0
  266. mostlyright/data_harness/ux/commands/deploy_status.py +104 -0
  267. mostlyright/data_harness/ux/commands/diff.py +74 -0
  268. mostlyright/data_harness/ux/commands/index.py +84 -0
  269. mostlyright/data_harness/ux/commands/inventory.py +47 -0
  270. mostlyright/data_harness/ux/commands/list_builds.py +143 -0
  271. mostlyright/data_harness/ux/commands/login.py +63 -0
  272. mostlyright/data_harness/ux/commands/peek.py +236 -0
  273. mostlyright/data_harness/ux/commands/plan_check.py +90 -0
  274. mostlyright/data_harness/ux/commands/preflight.py +97 -0
  275. mostlyright/data_harness/ux/commands/record.py +107 -0
  276. mostlyright/data_harness/ux/commands/review_setup.py +47 -0
  277. mostlyright/data_harness/ux/commands/search.py +440 -0
  278. mostlyright/data_harness/ux/commands/show.py +61 -0
  279. mostlyright/data_harness/ux/commands/whoami.py +37 -0
  280. mostlyright/data_harness/ux/credential_native.py +551 -0
  281. mostlyright/data_harness/ux/credential_store.py +1055 -0
  282. mostlyright/data_harness/ux/credentials.py +631 -0
  283. mostlyright/data_harness/ux/diffing.py +444 -0
  284. mostlyright/data_harness/ux/headline.py +671 -0
  285. mostlyright/data_harness/ux/hosted_acquisition.py +974 -0
  286. mostlyright/data_harness/ux/hosted_run_status.py +619 -0
  287. mostlyright/data_harness/ux/inventory.py +427 -0
  288. mostlyright/data_harness/ux/local_review.py +375 -0
  289. mostlyright/data_harness/ux/login.py +691 -0
  290. mostlyright/data_harness/ux/path_kind.py +147 -0
  291. mostlyright/data_harness/ux/peek.py +1000 -0
  292. mostlyright/data_harness/ux/plain_file.py +178 -0
  293. mostlyright/data_harness/ux/plan_check.py +311 -0
  294. mostlyright/data_harness/ux/preflight.py +918 -0
  295. mostlyright/data_harness/ux/readers.py +1124 -0
  296. mostlyright/data_harness/ux/remediation.py +2195 -0
  297. mostlyright/data_harness/ux/render.py +657 -0
  298. mostlyright/data_harness/ux/workload.py +1077 -0
  299. mostlyright/data_harness/viewer.py +3713 -0
  300. mostlyright/data_harness/visual_run/__init__.py +83 -0
  301. mostlyright/data_harness/visual_run/authoring.py +235 -0
  302. mostlyright/data_harness/visual_run/contracts.py +673 -0
  303. mostlyright/data_harness/visual_run/materialize.py +486 -0
  304. mostlyright/data_harness/visual_run/observations.py +874 -0
  305. mostlyright/data_harness/visual_run/query.py +259 -0
  306. mostlyright/data_harness/visual_run/reducer.py +280 -0
  307. mostlyright/data_harness/visual_run/sdk.py +892 -0
  308. mostlyright/data_harness/visual_run/store.py +584 -0
  309. mostlyright/data_harness/visual_run/transport.py +239 -0
  310. mostlyright/data_harness/watch.py +2999 -0
  311. mostlyright_data-0.9.0.dist-info/METADATA +607 -0
  312. mostlyright_data-0.9.0.dist-info/RECORD +314 -0
  313. mostlyright_data-0.9.0.dist-info/WHEEL +4 -0
  314. mostlyright_data-0.9.0.dist-info/entry_points.txt +12 -0
@@ -0,0 +1,4888 @@
1
+ """Credential-free subprocess boundary for retrieval and untrusted parsing."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import base64
6
+ import contextlib
7
+ import ipaddress
8
+ import json
9
+ import os
10
+ import platform
11
+ import resource
12
+ import secrets
13
+ import selectors
14
+ import signal
15
+ import socket
16
+ import stat
17
+ import subprocess
18
+ import sys
19
+ import threading
20
+ import time
21
+ import unicodedata
22
+ from collections.abc import Iterator, Mapping
23
+ from dataclasses import dataclass, replace
24
+ from pathlib import Path
25
+ from typing import Any, Protocol
26
+ from urllib.parse import urlsplit
27
+
28
+ from mostlyright.data_harness.acquisition.http import (
29
+ MAX_BYTE_RANGE_OFFSET,
30
+ MAX_RANGE_PLAN,
31
+ SIDECAR_MEDIA_TYPES,
32
+ SLICE_OBJECT_MEDIA_TYPES,
33
+ AcquisitionLimiter,
34
+ PeerAttempt,
35
+ PinnedHttpsRetriever,
36
+ ProbeTransport,
37
+ RetrievalHop,
38
+ RetrievalLimits,
39
+ StdlibPinnedTransport,
40
+ probe_transport_for,
41
+ require_range_plan_within_budget,
42
+ )
43
+ from mostlyright.data_harness.acquisition.parsing import (
44
+ ParsedTable,
45
+ ParseLimits,
46
+ parse_tabular_bytes,
47
+ prime_parquet_support,
48
+ )
49
+ from mostlyright.data_harness.acquisition.ranges import (
50
+ PlannedRange,
51
+ SidecarLimits,
52
+ check_grib_message,
53
+ check_grib_reference_time,
54
+ parse_index_sidecar,
55
+ plan_ranges,
56
+ require_reference_time_spelling,
57
+ )
58
+ from mostlyright.data_harness.acquisition.url_policy import (
59
+ AcquisitionSecurityError,
60
+ EgressPolicy,
61
+ SystemResolver,
62
+ validate_public_https_url,
63
+ )
64
+ from mostlyright.data_harness.canonical import canonical_sha256, sha256_bytes
65
+ from mostlyright.data_harness.formats import (
66
+ PARSER_FORMATS_BY_OPERATION,
67
+ READER_CONTRACT_VERSION,
68
+ READER_WORKER_MEMORY_BYTES,
69
+ )
70
+ from mostlyright.data_harness.linux_process_boundary import (
71
+ _disable_process_dumpability,
72
+ _install_networkless_seccomp,
73
+ )
74
+
75
+ # The dependency edge this module owns: ``acquisition`` consumes ``readers``, never the
76
+ # reverse. ``readers/`` is a leaf beneath this one, and anything both layers need is declared
77
+ # in ``formats.py`` where neither has to import the other. A back-edge would close an import
78
+ # cycle whose failure depends on which module a process imports first, so it would break the
79
+ # Reader tests while the application, which imports the other way round, stayed green.
80
+ from mostlyright.data_harness.readers.contracts import (
81
+ DECODE_FLAGS,
82
+ READER_ERROR_CODES,
83
+ ReaderPin,
84
+ )
85
+ from mostlyright.data_harness.readers.registry import TOOLBOX
86
+
87
+ # V2 adds the exact fetched byte count to Reader retrieval responses. The response shape is strict,
88
+ # so adding that measured fact is a protocol change rather than an optional field on V1.
89
+ SANDBOX_PROTOCOL_VERSION = "harness-crawler-sandbox.v1"
90
+
91
+ # Every operation this boundary admits, named once and read by both allowlists -- the
92
+ # coordinator-side one in ``sandbox_policy_digest`` and the worker-side one in
93
+ # ``_execute_worker_request``. Two literal sets would be two places to remember: registered
94
+ # on the coordinator side alone an operation computes a digest no worker will execute, and on
95
+ # the worker side alone it executes under a digest no coordinator can bind.
96
+ SANDBOX_OPERATIONS = frozenset(
97
+ {
98
+ "parse",
99
+ "probe",
100
+ "retrieve_and_parse",
101
+ "decode_and_parse",
102
+ "retrieve_decode_and_parse",
103
+ "fetch_ranges",
104
+ "survey_cycles",
105
+ }
106
+ )
107
+
108
+ # The operations that actually open a socket. This set is the *single* source of socket
109
+ # authority: ``_policy_document`` derives the attested ``network`` clause from it, and
110
+ # ``_invoke`` derives from it whether an invocation may resolve approved endpoints at all.
111
+ # Neither side is written by hand at a call site, because a hand-written flag beside a
112
+ # derived clause is two independent literals and the failure mode is silent and severe: an
113
+ # operation invoked with socket authority while the clause still named only
114
+ # ``retrieve_and_parse`` would attest ``"os-denied"`` while holding a live TCP/443 rule.
115
+ # The attestation would describe a posture the sandbox does not have, which is worse
116
+ # than no attestation at all. With one source, granting the rule and attesting it are one
117
+ # edit, and an operation left out of this set fails closed -- no socket -- rather than open.
118
+ NETWORK_PERFORMING_OPERATIONS = frozenset(
119
+ {"retrieve_and_parse", "retrieve_decode_and_parse", "fetch_ranges", "survey_cycles"}
120
+ )
121
+
122
+ # Every operation that executes a certified Reader must have the Reader worker's enforceable
123
+ # memory boundary. Keep this classification independent of socket authority: the composed
124
+ # HTTPS operation both opens a socket and executes a Reader, while local decode only does the
125
+ # latter. Deriving the off-Linux refusal from this set prevents a newly composed Reader path
126
+ # from accidentally bypassing the clean-room memory invariant.
127
+ READER_PERFORMING_OPERATIONS = frozenset({"decode_and_parse", "retrieve_decode_and_parse"})
128
+ # Every operation whose terminal response *is* the document a cadence probe observed. A range
129
+ # acquisition and a cycle survey are excluded on purpose: neither settles on one whole document,
130
+ # and both already retain per-response status and validator evidence in ``fetched_members``.
131
+ WHOLE_DOCUMENT_RETRIEVAL_OPERATIONS = frozenset({"retrieve_and_parse", "retrieve_decode_and_parse"})
132
+
133
+
134
+ class ConfinedWorkerRefusal(AcquisitionSecurityError):
135
+ """One refusal raised inside the confinement, carrying which operation raised it.
136
+
137
+ Every worker refusal reaches a caller as ``SANDBOX_FAILURE``, because a confinement that
138
+ described itself in its own refusals would be describing the confinement. That is right
139
+ for what a person is told and wrong for one caller: an empirical cadence observation may
140
+ only be recorded for a refusal that provably reached the source, and "reached the source"
141
+ is a question about *which operation* was running and *which code* it raised, neither of
142
+ which survives the collapse.
143
+
144
+ So both facts ride the refusal as structured values while ``code`` stays ``SANDBOX_FAILURE``
145
+ and ``detail`` stays exactly what it was. Every existing catch site is unchanged: this is
146
+ an ``AcquisitionSecurityError`` with the same code and the same sentence. ``worker_code``
147
+ is ``None`` when the worker's stderr yielded no code at all, which is the redacted case and
148
+ is evidence of nothing.
149
+ """
150
+
151
+ def __init__(
152
+ self,
153
+ code: str,
154
+ detail: str,
155
+ *,
156
+ worker_code: str | None,
157
+ operation: str,
158
+ ) -> None:
159
+ super().__init__(code, detail)
160
+ self.worker_code = worker_code
161
+ self.operation = operation
162
+
163
+
164
+ @dataclass(frozen=True)
165
+ class _WorkerRefusal:
166
+ """What the worker's stderr yielded: its typed code, and the sentence a person is told."""
167
+
168
+ code: str | None
169
+ message: str
170
+
171
+
172
+ # The one line a pre-warmed child writes before it reads its request, and the only difference
173
+ # between a warm child's stream and a cold child's. It is written after the child has finished
174
+ # importing and before it has read a byte of stdin, so a coordinator that has read it knows the
175
+ # expensive half of a cold start is already paid. Cold children never write it and the argv that
176
+ # asks for it is the argv that produces it, so the two streams can never be confused.
177
+ WORKER_READY_MARKER = b"mostlyright-sandbox-worker-ready\n"
178
+ WORKER_ARGV = ("--worker",)
179
+ WORKER_PREWARM_ARGV = ("--worker", "--prewarm")
180
+ MAX_REQUEST_BYTES = 24 * 1024 * 1024
181
+ MAX_RESPONSE_BYTES = 48 * 1024 * 1024
182
+ MAX_WORKER_STDERR_BYTES = 64 * 1024
183
+ # The worker imports PyArrow and may decode a 128 MiB admitted member. The 512 MiB process limit
184
+ # includes interpreter and decoder overhead.
185
+ MAX_WORKER_MEMORY_BYTES = READER_WORKER_MEMORY_BYTES
186
+ MAX_WORKER_PIDS = 64
187
+ # The hosted research-session service is deliberately one session per 4 GiB Cloud Run instance.
188
+ # This is a separate boundary from the 512 MiB operator-delegated cgroup root above: the container
189
+ # is the hard aggregate memory ceiling for the coordinator, parser child, and every descendant,
190
+ # while an exec-inherited seccomp filter denies networking inside the parser tree. The exact value
191
+ # is checked against the live controller before every spawn; a resource-class change therefore
192
+ # refuses until Harness and Studio move together instead of quietly widening the Clean room.
193
+ HOSTED_SESSION_CONTAINER_MEMORY_BYTES = 4 * 1024 * 1024 * 1024
194
+ _HOSTED_SESSION_CGROUP_ROOT = Path("/sys/fs/cgroup")
195
+ # The most idle children one coordinator may hold. Each one is a live process, a cgroup, and
196
+ # three pipes against an RLIMIT_NOFILE the coordinator also has to live inside, so the pool is
197
+ # bounded for the same reason every other budget in this module is: an unbounded one is a
198
+ # resource-exhaustion lever pointed at the host that granted it.
199
+ MAX_WARM_POOL_SIZE = 32
200
+ # And a coordinator holding a bounded cgroup root may hold no warm children at all. Not because
201
+ # of the descendant ceiling -- that would admit one -- but because of the stricter precondition
202
+ # beside it: ``_require_bounded_cgroup_root`` refuses a root whose ``nr_descendants`` is anything
203
+ # other than 1, meaning the operator-owned coordinator child and nothing else, and it is re-audited
204
+ # before *every* Linux spawn rather than once at construction. An idle warm child holds a job leaf,
205
+ # which makes that count 2, so a single warm child would turn every subsequent cold spawn -- every
206
+ # retrieval, and every request whose confinement does not match the pool -- into a
207
+ # ``SANDBOX_MEMORY_BOUNDARY`` refusal. A pool that breaks the path it is supposed to fall back to
208
+ # is not a pool, so under that root the answer is zero and the coordinator keeps the behaviour it
209
+ # has always had.
210
+ #
211
+ # Making Linux poolable means teaching that audit to distinguish a coordinator-owned warm leaf from
212
+ # a leaf an attacker pre-created. That is a change to a reviewed property of the memory boundary,
213
+ # which is the owner's decision and not a latency one, so it is surfaced here rather than taken.
214
+ BOUNDED_CGROUP_WARM_POOL_SIZE = 0
215
+ # The whole of one ``start_warm_pool`` call, not a budget per child. Serial per-child readiness
216
+ # waits multiply, and a coordinator asking for a full pool against a broken interpreter should
217
+ # find out in a minute rather than in the sixteen that thirty seconds times thirty-two would cost.
218
+ WARM_POOL_FILL_TIMEOUT_SECONDS = 60.0
219
+ _CGROUP_CLEANUP_TIMEOUT_SECONDS = 2.0
220
+ # Exact environment handed to the worker process; nothing is inherited. The key set is
221
+ # bound into the policy digest and is what the worker's reported keys are checked
222
+ # against, so adding or removing a key changes that digest. Values are not digest-covered.
223
+ _ALLOWED_ENVIRONMENT = {
224
+ # macOS Core Foundation text-encoding hint. It predates ADR 0021, which removed
225
+ # the Seatbelt profile that made reading it from the home directory a denial rather than a
226
+ # fallback, and it stays because this key set is bound into the policy digest: dropping it
227
+ # would rotate every operation's attestation to delete a string nothing reads back.
228
+ "__CF_USER_TEXT_ENCODING": "0x1F5:0x0:0x0",
229
+ "LANG": "C.UTF-8",
230
+ "LC_ALL": "C.UTF-8",
231
+ "PATH": os.defpath,
232
+ "PYTHONDONTWRITEBYTECODE": "1",
233
+ "PYTHONIOENCODING": "utf-8",
234
+ "PYTHONNOUSERSITE": "1",
235
+ }
236
+ _FORBIDDEN_EGRESS_HOSTS = frozenset(
237
+ {
238
+ "api.mostlyright.md",
239
+ "studio.mostlyright.md",
240
+ "metadata.google.internal",
241
+ "metadata.goog",
242
+ "localhost",
243
+ }
244
+ )
245
+ # The longest a single sidecar descriptor may be. ``SidecarLimits.max_line_bytes`` bounds the
246
+ # line a descriptor is read from, so a selector longer than a whole line could match nothing
247
+ # and is refused before it is carried anywhere.
248
+ MAX_SELECTOR_BYTES = 512
249
+ # The most slice bytes one invocation may carry back. See ``_require_slice_bytes_within_budget``
250
+ # for the base64 arithmetic that puts this comfortably inside ``MAX_RESPONSE_BYTES``.
251
+ MAX_TOTAL_SLICE_BYTES = 24 * 1024 * 1024
252
+ # The exact, closed shape of a range plan. Both addresses are coordinator-supplied: the
253
+ # worker never derives one from the other, because an address a worker constructs is an
254
+ # address no coordinator resolved and pinned.
255
+ _RANGE_PLAN_KEYS = frozenset(
256
+ {"sidecar_url", "object_url", "selectors", "max_ranges", "expected_reference_time"}
257
+ )
258
+ _FETCHED_SLICE_KEYS = frozenset(
259
+ {"index", "descriptor", "first_byte", "last_byte", "sha256", "content_base64"}
260
+ )
261
+ _FETCHED_MEMBER_KEYS = frozenset(
262
+ {
263
+ "role",
264
+ "sequence",
265
+ "url",
266
+ "resolution_digest",
267
+ "connected_peer",
268
+ "status",
269
+ "response_headers_digest",
270
+ "response_body_size_bytes",
271
+ "response_body_sha256",
272
+ "requested_range",
273
+ "observed_content_range",
274
+ "etag",
275
+ "last_modified",
276
+ "peer_attempts",
277
+ }
278
+ )
279
+ _PEER_ATTEMPT_KEYS = frozenset(
280
+ {"approved_ip", "outcome", "failure_code", "response_body_size_bytes"}
281
+ )
282
+ _PEER_ATTEMPT_OUTCOMES = frozenset({"transport_failure", "response", "response_failure"})
283
+ _FETCHED_MEMBER_ROLES = frozenset(
284
+ {"cycle_probe", "sidecar_initial", "selected_slice", "sidecar_final"}
285
+ )
286
+ # The most candidate runs one availability survey may probe. A survey costs one request per
287
+ # candidate against a source that has done nothing wrong, so an unbounded candidate list is a
288
+ # denial-of-service lever pointed at somebody else; 32 covers every cycle cadence this exists
289
+ # for, including a full day of quarter-hourly runs.
290
+ MAX_CYCLE_CANDIDATES = 32
291
+ _CYCLE_SURVEY_KEYS = frozenset({"candidates"})
292
+ _CYCLE_CANDIDATE_KEYS = frozenset({"label", "url"})
293
+ _CYCLE_AVAILABILITY_KEYS = frozenset({"label", "present"})
294
+ # The span a candidate is probed with, and the mapping from a typed refusal to absence, both
295
+ # live in ``PinnedHttpsRetriever.probe_presence`` rather than here. A survey asks whether a run
296
+ # exists, so reading more than the one byte that settles the question would be an acquisition
297
+ # nobody ordered, on an address no coordinator has locked yet -- and the boundary that owns the
298
+ # request is the boundary that should own that bound.
299
+
300
+
301
+ @dataclass(frozen=True)
302
+ class CycleCandidate:
303
+ """One candidate run a coordinator asks the worker to look for.
304
+
305
+ The label, not the URL, is what a result is matched back to. The worker never parses
306
+ meaning out of an address -- which run this is, what hour it covers, whether it is newer
307
+ than the last one -- because a worker that read a cycle out of a URL would be choosing,
308
+ and choosing is the coordinator's half of look-then-lock.
309
+ """
310
+
311
+ label: str
312
+ url: str
313
+
314
+ def __post_init__(self) -> None:
315
+ _bounded_identifier(self.label)
316
+ _https_origin(_bounded_text(self.url, maximum=2_048), code="SANDBOX_CYCLE_SURVEY")
317
+
318
+
319
+ @dataclass(frozen=True)
320
+ class CycleAvailability:
321
+ """What the worker reports for one candidate: it is there, or it is not there yet."""
322
+
323
+ label: str
324
+ present: bool
325
+
326
+
327
+ @dataclass(frozen=True)
328
+ class FetchedSlice:
329
+ """One delivered message: which one it is, exactly which bytes, and its own digest.
330
+
331
+ The digest is per slice rather than per acquisition so a caller can bind an individual
332
+ message. A single digest over the concatenation would make one message's provenance
333
+ unstatable without restating the whole plan.
334
+
335
+ ``descriptor`` is the selector this slice answers, and it is carried because delivery
336
+ order is not selection order: ``plan_ranges`` emits in message-index order so the sealed
337
+ digest is a function of the recipe rather than of network timing. Without it a caller
338
+ holds N slices labelled only by a sidecar message number -- a number only the worker ever
339
+ saw the sidecar for -- and wiring "the first field I named" to the first slice silently
340
+ reads a different field. Matching is exact equality in ``plan_ranges``, so the descriptor
341
+ a slice reports is one of the selectors that were ordered, and the coordinator checks it
342
+ against its own order rather than taking the worker's word for the pairing.
343
+ """
344
+
345
+ index: int
346
+ descriptor: str
347
+ first_byte: int
348
+ last_byte: int
349
+ sha256: str
350
+ content: bytes
351
+
352
+
353
+ @dataclass(frozen=True)
354
+ class FetchedMember:
355
+ """One measured HTTP response, including redirects, in acquisition order."""
356
+
357
+ role: str
358
+ sequence: int
359
+ url: str
360
+ resolution_digest: str
361
+ connected_peer: str
362
+ status: int
363
+ response_headers_digest: str
364
+ response_body_size_bytes: int
365
+ response_body_sha256: str
366
+ requested_range: str | None
367
+ observed_content_range: str | None
368
+ etag: str | None
369
+ last_modified: str | None
370
+ peer_attempts: tuple[PeerAttempt, ...]
371
+
372
+ @property
373
+ def total_response_body_size_bytes(self) -> int:
374
+ """All response-body bytes consumed while obtaining this retained response."""
375
+
376
+ return sum(attempt.response_body_size_bytes for attempt in self.peer_attempts)
377
+
378
+ def to_dict(self) -> dict[str, Any]:
379
+ return {
380
+ "role": self.role,
381
+ "sequence": self.sequence,
382
+ "url": self.url,
383
+ "resolution_digest": self.resolution_digest,
384
+ "connected_peer": self.connected_peer,
385
+ "status": self.status,
386
+ "response_headers_digest": self.response_headers_digest,
387
+ "response_body_size_bytes": self.response_body_size_bytes,
388
+ "response_body_sha256": self.response_body_sha256,
389
+ "requested_range": self.requested_range,
390
+ "observed_content_range": self.observed_content_range,
391
+ "etag": self.etag,
392
+ "last_modified": self.last_modified,
393
+ "peer_attempts": [attempt.to_dict() for attempt in self.peer_attempts],
394
+ }
395
+
396
+
397
+ @dataclass(frozen=True)
398
+ class SandboxResult:
399
+ request_id: str
400
+ operation: str
401
+ policy_digest: str
402
+ parsed: ParsedTable | None
403
+ content: bytes | None
404
+ media_type: str | None
405
+ final_url: str | None
406
+ transport_evidence_digest: str | None
407
+ visible_environment_keys: tuple[str, ...]
408
+ # The terminal response's status and hashed validators, sealed inside the Clean room by the
409
+ # operation that retrieved one whole document. ``None`` for every other operation, which is
410
+ # how "this probe observed no single response" stays distinguishable from "it observed one
411
+ # that stated no validator".
412
+ probe_transport: ProbeTransport | None = None
413
+ home_directory_readable: bool | None = None
414
+ network_probe_errors: tuple[tuple[str, int], ...] | None = None
415
+ external_network_policy_attestation: str | None = None
416
+ # The facts a Reader-decoded acquisition writes onto its receipt: which certified code ran,
417
+ # with which sealed settings, and what it reported about the decode and charged geometry.
418
+ # They cross the IPC boundary as data rather than being recomputed here, because the decode
419
+ # did not happen here. All five are ``None`` for an operation that decoded nothing, which
420
+ # is how "no Reader ran" stays distinguishable from "a Reader ran and declared nothing".
421
+ decode_family_id: str | None = None
422
+ decode_family_version: str | None = None
423
+ decode_options_digest: str | None = None
424
+ decode_flags: tuple[str, ...] | None = None
425
+ # Exact family-owned geometry charged against ``max_declared_cells``. This can exceed
426
+ # parsed rows times columns (for example, an XLSX header and declared used area).
427
+ decode_declared_cell_count: int | None = None
428
+ fetched_content_sha256: str | None = None
429
+ # Exact terminal source-body bytes decoded by ``retrieve_decode_and_parse``. It is paired with
430
+ # ``fetched_content_sha256`` and remains the per-member V3 content evidence; normalized output
431
+ # size remains ``len(content)`` and is not substituted for it.
432
+ fetched_content_size_bytes: int | None = None
433
+ # Exact sum of response-body bytes across the retained redirect hop evidence. The terminal
434
+ # body remains ``fetched_content_size_bytes`` because it is the object the Reader decoded;
435
+ # collection resource accounting uses this wider measured transport total.
436
+ fetched_total_response_body_size_bytes: int | None = None
437
+ # Exact peer-attempt count, including failed approved-IP fallbacks and redirects. Kept beside
438
+ # the byte count so a collection coordinator can spend one request budget across several
439
+ # confined invocations.
440
+ fetched_request_count: int | None = None
441
+ ranges: tuple[FetchedSlice, ...] | None = None
442
+ cycle_availability: tuple[CycleAvailability, ...] | None = None
443
+ sidecar_content_sha256: str | None = None
444
+ sidecar_size_bytes: int | None = None
445
+ full_object_size_bytes: int | None = None
446
+ fetched_members: tuple[FetchedMember, ...] | None = None
447
+
448
+
449
+ class CrawlerSandbox:
450
+ """Trusted coordinator for one clean-environment sandbox process per request."""
451
+
452
+ def __init__(
453
+ self,
454
+ *,
455
+ staging_root: Path,
456
+ timeout_seconds: float = 45.0,
457
+ python_executable: str | None = None,
458
+ external_network_policy_attestation: str | None = None,
459
+ memory_cgroup_root_fd: int | None = None,
460
+ hosted_session_container_memory_bytes: int | None = None,
461
+ ) -> None:
462
+ if not staging_root.is_absolute() or staging_root.is_symlink() or not staging_root.is_dir():
463
+ raise AcquisitionSecurityError(
464
+ "SANDBOX_ROOT",
465
+ "sandbox staging root must be an existing absolute non-symlink directory",
466
+ )
467
+ if type(timeout_seconds) not in {int, float} or not 0 < timeout_seconds <= 300:
468
+ raise AcquisitionSecurityError(
469
+ "SANDBOX_TIMEOUT",
470
+ "sandbox timeout must be in (0, 300] seconds",
471
+ )
472
+ self._staging_root = staging_root
473
+ self._timeout_seconds = float(timeout_seconds)
474
+ self._python = python_executable or sys.executable
475
+ if external_network_policy_attestation is not None:
476
+ _require_digest(external_network_policy_attestation, "external network policy")
477
+ self._external_network_policy_attestation = external_network_policy_attestation
478
+ if memory_cgroup_root_fd is not None and hosted_session_container_memory_bytes is not None:
479
+ raise AcquisitionSecurityError(
480
+ "SANDBOX_MEMORY_BOUNDARY",
481
+ "delegated and hosted-session memory boundaries are mutually exclusive",
482
+ )
483
+ if memory_cgroup_root_fd is not None:
484
+ _require_bounded_cgroup_root(memory_cgroup_root_fd)
485
+ if hosted_session_container_memory_bytes is not None:
486
+ try:
487
+ _disable_process_dumpability()
488
+ except OSError as error:
489
+ raise AcquisitionSecurityError(
490
+ "SANDBOX_OS_BOUNDARY",
491
+ "hosted-session coordinator process hardening is unavailable",
492
+ ) from error
493
+ _require_hosted_session_container_boundary(hosted_session_container_memory_bytes)
494
+ self._memory_cgroup_root_fd = memory_cgroup_root_fd
495
+ self._hosted_session_container_memory_bytes = hosted_session_container_memory_bytes
496
+ # Off unless a caller asks for it. Default behaviour is one fresh spawn per request,
497
+ # exactly as before; the pool is a lifecycle a long-lived session owner opts into and
498
+ # is responsible for closing.
499
+ self._warm_pool: _WarmSandboxPool | None = None
500
+
501
+ # ----------------------------------------------------------------------------------------
502
+ # Warm pool: an opt-in lifecycle for a coordinator that will serve many probes
503
+ # ----------------------------------------------------------------------------------------
504
+
505
+ def start_warm_pool(self, *, size: int) -> int:
506
+ """Pre-spawn up to ``size`` idle children for invocations that attest no network.
507
+
508
+ Returns the number now ready, which is the number to believe rather than the number
509
+ asked for: dead children are reaped rather than counted, and where the boundary admits
510
+ fewer than ``size`` the answer says so. Calling it again tops the pool back up, so a
511
+ session owner that has just spent a child can refill without tracking what it holds.
512
+
513
+ **A coordinator holding a bounded cgroup root gets zero, and must read the return value
514
+ to find that out.** That is the Linux boundary, and the reason is recorded beside
515
+ ``BOUNDED_CGROUP_WARM_POOL_SIZE``: the root audit re-run before every Linux spawn
516
+ refuses a root that contains anything but the coordinator's own child, so one idle warm
517
+ child would turn every cold spawn into a refusal. Zero rather than a typed refusal
518
+ because a caller should not have to branch on the platform to ask for a pool it may not
519
+ get; what it must do is believe the count.
520
+
521
+ Warm is pre-spawned and pre-imported, never reused: each child serves exactly one
522
+ request and exits, and a request that finds no matching child spawns its own. The pool
523
+ is therefore a latency optimisation with no failure mode of its own -- an empty pool,
524
+ a pool of the wrong shape, and a pool that was never started all produce the identical
525
+ cold path.
526
+
527
+ Only the no-network confinement is pooled. A retrieving invocation's approved
528
+ endpoints are derived from the request being made, and those do not exist before the
529
+ request does; a child spawned earlier would either carry socket authority nobody
530
+ validated for the request that binds to it, or authority that differs from what that
531
+ request would have been given. Retrieval therefore spawns cold.
532
+ """
533
+
534
+ if type(size) is not int or not 0 <= size <= MAX_WARM_POOL_SIZE:
535
+ raise AcquisitionSecurityError(
536
+ "SANDBOX_WARM_POOL",
537
+ f"warm pool size must be an integer in [0, {MAX_WARM_POOL_SIZE}]",
538
+ )
539
+ if self._memory_cgroup_root_fd is not None:
540
+ size = min(size, BOUNDED_CGROUP_WARM_POOL_SIZE)
541
+ if size == 0:
542
+ self.close_warm_pool()
543
+ return 0
544
+ launch = self._worker_launch(prewarm=True)
545
+ if self._warm_pool is None:
546
+ self._warm_pool = _WarmSandboxPool(
547
+ timeout_seconds=self._timeout_seconds,
548
+ memory_cgroup_root_fd=self._memory_cgroup_root_fd,
549
+ hosted_session_container_memory_bytes=(self._hosted_session_container_memory_bytes),
550
+ )
551
+ return self._warm_pool.fill(launch, size=size)
552
+
553
+ def warm_pool_ready_count(self) -> int:
554
+ """How many pre-spawned children are idle and unbound right now."""
555
+
556
+ return 0 if self._warm_pool is None else self._warm_pool.ready_count()
557
+
558
+ def close_warm_pool(self) -> None:
559
+ """Kill and reap every idle child. Idempotent; safe to call without a pool."""
560
+
561
+ pool, self._warm_pool = self._warm_pool, None
562
+ if pool is not None:
563
+ pool.close()
564
+
565
+ @contextlib.contextmanager
566
+ def warm_pool(self, *, size: int) -> Iterator[CrawlerSandbox]:
567
+ """Own a warm pool for the duration of a block, closing it on the way out."""
568
+
569
+ # Inside the ``try``: ``fill`` keeps every child it has already started when a later one
570
+ # fails, so a partial fill that raises still has children to close.
571
+ try:
572
+ self.start_warm_pool(size=size)
573
+ yield self
574
+ finally:
575
+ self.close_warm_pool()
576
+
577
+ def parse(
578
+ self,
579
+ *,
580
+ request_id: str,
581
+ content: bytes,
582
+ data_format: str,
583
+ media_type: str,
584
+ filename: str,
585
+ limits: ParseLimits,
586
+ ) -> SandboxResult:
587
+ return self._invoke(
588
+ {
589
+ "schema_version": SANDBOX_PROTOCOL_VERSION,
590
+ "request_id": request_id,
591
+ "operation": "parse",
592
+ "content_base64": base64.b64encode(content).decode("ascii"),
593
+ "data_format": data_format,
594
+ "media_type": media_type,
595
+ "filename": filename,
596
+ "parse_limits": _parse_limits_dict(limits),
597
+ "url": None,
598
+ "allowed_hostnames": [],
599
+ "approved_endpoints": [],
600
+ "retrieval_limits": None,
601
+ "network_probe_targets": [],
602
+ "reader_pin": None,
603
+ "reader_budgets": None,
604
+ "range_plan": None,
605
+ "cycle_survey": None,
606
+ },
607
+ approved_endpoints=(),
608
+ )
609
+
610
+ def decode_and_parse(
611
+ self,
612
+ *,
613
+ request_id: str,
614
+ content: bytes,
615
+ reader_pin: Mapping[str, Any],
616
+ output_format: str,
617
+ limits: ParseLimits,
618
+ reader_budgets: Mapping[str, Any] | None = None,
619
+ ) -> SandboxResult:
620
+ """Decode untrusted bytes with the pinned Reader, inside the confinement.
621
+
622
+ A fourth operation rather than a widened ``parse``. ``parse`` returns a table and a
623
+ Reader must hand back **bytes**, because those bytes become the sealed snapshot.
624
+ Widening ``parse`` would silently grow what a ``parse``-attested worker may do, which
625
+ is a security regression disguised as a smaller diff; a separate operation carries its
626
+ own policy digest and leaves the other three unrotated.
627
+
628
+ ``output_format`` is the encoding the coordinator will accept back and is what the
629
+ operation's ``parser_formats`` clause is checked against, so a decode that would seal
630
+ an unattested encoding is refused before the family runs. ``reader_budgets`` may only
631
+ narrow the family's own defaults; it can never buy more room than the family allows.
632
+ """
633
+
634
+ return self._invoke(
635
+ {
636
+ "schema_version": SANDBOX_PROTOCOL_VERSION,
637
+ "request_id": request_id,
638
+ "operation": "decode_and_parse",
639
+ "content_base64": base64.b64encode(content).decode("ascii"),
640
+ "data_format": output_format,
641
+ # The media type and the filename of the parsed artifact are the Reader
642
+ # family's declarations, not the coordinator's choice, so they travel back on
643
+ # the response rather than out on the request.
644
+ "media_type": None,
645
+ "filename": None,
646
+ "parse_limits": _parse_limits_dict(limits),
647
+ "url": None,
648
+ "allowed_hostnames": [],
649
+ "approved_endpoints": [],
650
+ "retrieval_limits": None,
651
+ "network_probe_targets": [],
652
+ "reader_pin": dict(reader_pin),
653
+ "reader_budgets": None if reader_budgets is None else dict(reader_budgets),
654
+ "range_plan": None,
655
+ "cycle_survey": None,
656
+ },
657
+ approved_endpoints=(),
658
+ )
659
+
660
+ def retrieve_and_parse(
661
+ self,
662
+ *,
663
+ request_id: str,
664
+ url: str,
665
+ allowed_hostnames: tuple[str, ...],
666
+ data_format: str,
667
+ filename: str,
668
+ parse_limits: ParseLimits,
669
+ retrieval_limits: RetrievalLimits,
670
+ ) -> SandboxResult:
671
+ _validate_egress_hosts(allowed_hostnames)
672
+ if self._external_network_policy_attestation is None:
673
+ raise AcquisitionSecurityError(
674
+ "SANDBOX_OS_BOUNDARY",
675
+ (
676
+ "live retrieval requires an explicit externally enforced "
677
+ "deny-default public-IP network policy attestation"
678
+ ),
679
+ )
680
+ approved_endpoints = _resolve_approved_endpoints(url, allowed_hostnames)
681
+ return self._invoke(
682
+ {
683
+ "schema_version": SANDBOX_PROTOCOL_VERSION,
684
+ "request_id": request_id,
685
+ "operation": "retrieve_and_parse",
686
+ "content_base64": None,
687
+ "data_format": data_format,
688
+ "media_type": None,
689
+ "filename": filename,
690
+ "parse_limits": _parse_limits_dict(parse_limits),
691
+ "url": url,
692
+ "allowed_hostnames": list(allowed_hostnames),
693
+ "approved_endpoints": [
694
+ {
695
+ "hostname": hostname,
696
+ "port": port,
697
+ "approved_ips": list(addresses),
698
+ }
699
+ for hostname, port, addresses in approved_endpoints
700
+ ],
701
+ "retrieval_limits": _retrieval_limits_dict(retrieval_limits),
702
+ "network_probe_targets": [],
703
+ "reader_pin": None,
704
+ "reader_budgets": None,
705
+ "range_plan": None,
706
+ "cycle_survey": None,
707
+ },
708
+ approved_endpoints=approved_endpoints,
709
+ )
710
+
711
+ def retrieve_decode_and_parse(
712
+ self,
713
+ *,
714
+ request_id: str,
715
+ url: str,
716
+ allowed_hostnames: tuple[str, ...],
717
+ reader_pin: Mapping[str, Any],
718
+ output_format: str,
719
+ parse_limits: ParseLimits,
720
+ retrieval_limits: RetrievalLimits,
721
+ reader_budgets: Mapping[str, Any] | None = None,
722
+ ) -> SandboxResult:
723
+ """Retrieve, decode, and parse one Reader-pinned public HTTPS source.
724
+
725
+ This is one operation because fetched container bytes must stay inside the clean room.
726
+ Returning them to the coordinator between network and decode operations would create an
727
+ unconfined materialization path and no single attestation would describe the composition.
728
+ """
729
+
730
+ _validate_egress_hosts(allowed_hostnames)
731
+ if self._external_network_policy_attestation is None:
732
+ raise AcquisitionSecurityError(
733
+ "SANDBOX_OS_BOUNDARY",
734
+ (
735
+ "live Reader retrieval requires an explicit externally enforced "
736
+ "deny-default public-IP network policy attestation"
737
+ ),
738
+ )
739
+ approved_endpoints = _resolve_approved_endpoints(url, allowed_hostnames)
740
+ return self._invoke(
741
+ {
742
+ "schema_version": SANDBOX_PROTOCOL_VERSION,
743
+ "request_id": request_id,
744
+ "operation": "retrieve_decode_and_parse",
745
+ "content_base64": None,
746
+ "data_format": output_format,
747
+ "media_type": None,
748
+ "filename": None,
749
+ "parse_limits": _parse_limits_dict(parse_limits),
750
+ "url": url,
751
+ "allowed_hostnames": list(allowed_hostnames),
752
+ "approved_endpoints": [
753
+ {
754
+ "hostname": hostname,
755
+ "port": port,
756
+ "approved_ips": list(addresses),
757
+ }
758
+ for hostname, port, addresses in approved_endpoints
759
+ ],
760
+ "retrieval_limits": _retrieval_limits_dict(retrieval_limits),
761
+ "network_probe_targets": [],
762
+ "reader_pin": dict(reader_pin),
763
+ "reader_budgets": None if reader_budgets is None else dict(reader_budgets),
764
+ "range_plan": None,
765
+ "cycle_survey": None,
766
+ },
767
+ approved_endpoints=approved_endpoints,
768
+ )
769
+
770
+ def fetch_ranges(
771
+ self,
772
+ *,
773
+ request_id: str,
774
+ sidecar_url: str,
775
+ object_url: str,
776
+ selectors: tuple[str, ...],
777
+ allowed_hostnames: tuple[str, ...],
778
+ retrieval_limits: RetrievalLimits,
779
+ expected_reference_time: str | None = None,
780
+ ) -> SandboxResult:
781
+ """Fetch an index sidecar and every slice it names, in one clean-room invocation.
782
+
783
+ One invocation is not an optimisation. ``_resolve_approved_endpoints`` runs once per
784
+ invocation, so one invocation is the only way N ranges share one DNS epoch and one
785
+ retrieval budget; a second invocation would be a second chance to be rebound between
786
+ the sidecar that says where to look and the slice taken from there.
787
+
788
+ Both addresses are supplied here and travel together. The worker is never asked to
789
+ derive the sidecar's address from the object's, because an address a worker builds is
790
+ an address no coordinator resolved and pinned.
791
+
792
+ ``expected_reference_time`` is the model run this order is for, spelled
793
+ ``YYYY-MM-DDTHH:MM:SSZ``. Supplied, it binds every delivered message to that run in the
794
+ worker and again here -- the control the span check and the structural check cannot be,
795
+ because both are properties of a message alone and a re-issued run answers the ordered
796
+ span with a well-formed message of the same length. Omitted, the run binding is not in
797
+ force and the result says so rather than implying otherwise.
798
+ """
799
+
800
+ if expected_reference_time is not None:
801
+ expected_reference_time = require_reference_time_spelling(expected_reference_time)
802
+
803
+ _validate_egress_hosts(allowed_hostnames)
804
+ if self._external_network_policy_attestation is None:
805
+ raise AcquisitionSecurityError(
806
+ "SANDBOX_OS_BOUNDARY",
807
+ (
808
+ "slice fetching requires an explicit externally enforced "
809
+ "deny-default public-IP network policy attestation"
810
+ ),
811
+ )
812
+ approved_endpoints = _resolve_approved_endpoints(sidecar_url, allowed_hostnames)
813
+ result = self._invoke(
814
+ {
815
+ "schema_version": SANDBOX_PROTOCOL_VERSION,
816
+ "request_id": request_id,
817
+ "operation": "fetch_ranges",
818
+ "content_base64": None,
819
+ "data_format": None,
820
+ "media_type": None,
821
+ "filename": None,
822
+ "parse_limits": None,
823
+ "url": None,
824
+ "allowed_hostnames": list(allowed_hostnames),
825
+ "approved_endpoints": [
826
+ {
827
+ "hostname": hostname,
828
+ "port": port,
829
+ "approved_ips": list(addresses),
830
+ }
831
+ for hostname, port, addresses in approved_endpoints
832
+ ],
833
+ "retrieval_limits": _retrieval_limits_dict(retrieval_limits),
834
+ "network_probe_targets": [],
835
+ "reader_pin": None,
836
+ "reader_budgets": None,
837
+ "range_plan": {
838
+ "sidecar_url": sidecar_url,
839
+ "object_url": object_url,
840
+ "selectors": list(selectors),
841
+ "max_ranges": retrieval_limits.max_ranges,
842
+ "expected_reference_time": expected_reference_time,
843
+ },
844
+ "cycle_survey": None,
845
+ },
846
+ approved_endpoints=approved_endpoints,
847
+ )
848
+ # The response is evidence, not testimony -- the same posture the slices themselves are
849
+ # held to. The worker refuses a sidecar answered off the object's origin; the
850
+ # coordinator re-makes that check on the address the worker reports, so a response that
851
+ # named no delivered sidecar address at all, or named one somewhere else, is refused on
852
+ # this side too rather than being taken on the worker's word.
853
+ if result.final_url is None or _https_origin(
854
+ result.final_url, code="SANDBOX_SIDECAR_ORIGIN"
855
+ ) != _https_origin(object_url, code="SANDBOX_SIDECAR_ORIGIN"):
856
+ raise AcquisitionSecurityError(
857
+ "SANDBOX_SIDECAR_ORIGIN",
858
+ "the clean room's response does not bind the index sidecar to the object's origin",
859
+ )
860
+ return result
861
+
862
+ def survey_cycles(
863
+ self,
864
+ *,
865
+ request_id: str,
866
+ candidates: tuple[CycleCandidate, ...],
867
+ allowed_hostnames: tuple[str, ...],
868
+ retrieval_limits: RetrievalLimits,
869
+ ) -> SandboxResult:
870
+ """Report which of these candidate runs exist. Report only -- never choose one.
871
+
872
+ This is the *look* half of look-then-lock. The candidate list is supplied here, by the
873
+ side that already holds wall-clock authority and the watermark; the worker probes each
874
+ address and says present or absent, and the coordinator locks one and orders exactly
875
+ that. Discovery deliberately does not move into the worker: a candidate a worker
876
+ derived from its own clock at run time is a candidate no coordinator pinned, which is
877
+ the execution binding this boundary exists to preserve.
878
+
879
+ A candidate that is not published yet is reported absent and the invocation succeeds.
880
+ Every other failure -- a server error, a TLS failure, a peer outside the pin, a source
881
+ that ignored the range -- fails the whole invocation, because a broken source reported
882
+ as "nothing new yet" is a dataset that stops updating and never says so.
883
+ """
884
+
885
+ _validate_egress_hosts(allowed_hostnames)
886
+ if self._external_network_policy_attestation is None:
887
+ raise AcquisitionSecurityError(
888
+ "SANDBOX_OS_BOUNDARY",
889
+ (
890
+ "an availability survey requires an explicit externally enforced "
891
+ "deny-default public-IP network policy attestation"
892
+ ),
893
+ )
894
+ # Before ``_resolve_approved_endpoints``, which performs real DNS. A candidate set that
895
+ # was never going to be admitted must cost no lookup, on this host or on the source's.
896
+ _require_surveyable_candidates(candidates, allowed_hostnames=allowed_hostnames)
897
+ approved_endpoints = _resolve_approved_endpoints(candidates[0].url, allowed_hostnames)
898
+ return self._invoke(
899
+ {
900
+ "schema_version": SANDBOX_PROTOCOL_VERSION,
901
+ "request_id": request_id,
902
+ "operation": "survey_cycles",
903
+ "content_base64": None,
904
+ "data_format": None,
905
+ "media_type": None,
906
+ "filename": None,
907
+ "parse_limits": None,
908
+ "url": None,
909
+ "allowed_hostnames": list(allowed_hostnames),
910
+ "approved_endpoints": [
911
+ {
912
+ "hostname": hostname,
913
+ "port": port,
914
+ "approved_ips": list(addresses),
915
+ }
916
+ for hostname, port, addresses in approved_endpoints
917
+ ],
918
+ "retrieval_limits": _retrieval_limits_dict(retrieval_limits),
919
+ "network_probe_targets": [],
920
+ "reader_pin": None,
921
+ "reader_budgets": None,
922
+ "range_plan": None,
923
+ "cycle_survey": {
924
+ "candidates": [
925
+ {"label": candidate.label, "url": candidate.url} for candidate in candidates
926
+ ]
927
+ },
928
+ },
929
+ approved_endpoints=approved_endpoints,
930
+ )
931
+
932
+ def probe_isolation(
933
+ self,
934
+ *,
935
+ request_id: str,
936
+ loopback_probe_port: int = 9,
937
+ ) -> SandboxResult:
938
+ if type(loopback_probe_port) is not int or not 1 <= loopback_probe_port <= 65_535:
939
+ raise AcquisitionSecurityError(
940
+ "SANDBOX_PROBE",
941
+ "loopback probe port must be a valid integer",
942
+ )
943
+ return self._invoke(
944
+ {
945
+ "schema_version": SANDBOX_PROTOCOL_VERSION,
946
+ "request_id": request_id,
947
+ "operation": "probe",
948
+ "content_base64": None,
949
+ "data_format": None,
950
+ "media_type": None,
951
+ "filename": None,
952
+ "parse_limits": None,
953
+ "url": None,
954
+ "allowed_hostnames": [],
955
+ "approved_endpoints": [],
956
+ "retrieval_limits": None,
957
+ "network_probe_targets": [
958
+ {"label": "loopback", "ip": "127.0.0.1", "port": loopback_probe_port},
959
+ {"label": "private", "ip": "10.0.0.1", "port": 443},
960
+ {"label": "metadata", "ip": "169.254.169.254", "port": 80},
961
+ ],
962
+ "reader_pin": None,
963
+ "reader_budgets": None,
964
+ "range_plan": None,
965
+ "cycle_survey": None,
966
+ },
967
+ approved_endpoints=(),
968
+ )
969
+
970
+ def _worker_launch(self, *, prewarm: bool) -> _WorkerLaunch:
971
+ """Build the exact confinement one child is started in.
972
+
973
+ One builder for both paths. A warm child and the cold child it stands in for are
974
+ started from the same specification produced by the same code, so "is a warm child
975
+ indistinguishable from a fresh spawn" is answered by construction rather than by two
976
+ launch sites that have to be kept in agreement.
977
+ """
978
+
979
+ command = [
980
+ self._python,
981
+ "-I",
982
+ "-m",
983
+ "mostlyright.data_harness.acquisition.sandbox",
984
+ "--worker",
985
+ ]
986
+ # One rule, no platform arm. Before ADR 0021 there were two boundaries here: an
987
+ # external deny-default one that every host had to attest, and macOS Seatbelt, which
988
+ # this process composed for itself and which therefore excused a darwin host from the
989
+ # attestation for non-networking operations. ADR 0021 deletes the Seatbelt backend --
990
+ # "The Linux cgroups-v2 sandbox remains and becomes the only boundary implementation,
991
+ # protecting our workers. The Seatbelt backend and every user-machine enforcement path
992
+ # leave the product surface." What is left is the rule that always applied to every
993
+ # other host: no attested boundary, no clean room. A host that cannot present one fails
994
+ # closed rather than running unconfined, which is the same answer this code has always
995
+ # given on Windows.
996
+ if (
997
+ self._external_network_policy_attestation is None
998
+ and self._hosted_session_container_memory_bytes is None
999
+ ):
1000
+ raise AcquisitionSecurityError(
1001
+ "SANDBOX_OS_BOUNDARY",
1002
+ (
1003
+ "crawler execution requires an explicit externally enforced "
1004
+ "deny-default network policy attestation"
1005
+ ),
1006
+ )
1007
+ return _WorkerLaunch(
1008
+ command=tuple(command),
1009
+ cwd=self._staging_root,
1010
+ env=tuple(sorted(_ALLOWED_ENVIRONMENT.items())),
1011
+ prewarm=prewarm,
1012
+ )
1013
+
1014
+ def _invoke(
1015
+ self,
1016
+ request: dict[str, Any],
1017
+ *,
1018
+ approved_endpoints: tuple[tuple[str, int, tuple[str, ...]], ...],
1019
+ ) -> SandboxResult:
1020
+ # Socket authority is derived from the operation being invoked, never handed in
1021
+ # beside it. A ``network_allowed`` argument written at each call site was a second
1022
+ # literal that could disagree with the ``network`` clause this same operation
1023
+ # attests, and the disagreement was invisible: the profile would carry a live
1024
+ # TCP/443 rule while the digest-bound document said ``"os-denied"``. Reading the
1025
+ # authority out of ``request["operation"]`` makes the grant and the attestation the
1026
+ # same decision, taken once, from one set.
1027
+ operation = request["operation"]
1028
+ if operation not in SANDBOX_OPERATIONS:
1029
+ raise AcquisitionSecurityError(
1030
+ "SANDBOX_OPERATION",
1031
+ "sandbox operation is not allowlisted",
1032
+ )
1033
+ network_allowed = operation in NETWORK_PERFORMING_OPERATIONS
1034
+ if self._hosted_session_container_memory_bytes is not None and operation != "parse":
1035
+ raise AcquisitionSecurityError(
1036
+ "SANDBOX_OS_BOUNDARY",
1037
+ "hosted-session container confinement admits only no-network parsing",
1038
+ )
1039
+ # Was darwin-only, and is now every host that is not Linux. The refusal never really
1040
+ # was about macOS: it was about there being no cgroup to bound a Reader's memory with,
1041
+ # which is true of every host without cgroups v2. Seatbelt used to make macOS the one
1042
+ # non-Linux host that got this far with a boundary of its own; with it gone (ADR 0021
1043
+ # ADR 0021) there is no reason to keep naming one platform. The typed code is unchanged,
1044
+ # because ``SANDBOX_MEMORY_BOUNDARY`` is what callers and receipts already read.
1045
+ if operation in READER_PERFORMING_OPERATIONS and sys.platform != "linux":
1046
+ raise AcquisitionSecurityError(
1047
+ "SANDBOX_MEMORY_BOUNDARY",
1048
+ "Clean room Reader execution has no enforceable memory boundary off Linux",
1049
+ )
1050
+ if approved_endpoints and not network_allowed:
1051
+ # Endpoints for an operation that attests no network is a caller confusion, not
1052
+ # something to quietly discard: the request it built is not the request its
1053
+ # policy document describes.
1054
+ raise AcquisitionSecurityError(
1055
+ "SANDBOX_OPERATION",
1056
+ "an operation that attests no network was given approved endpoints",
1057
+ )
1058
+ request = {
1059
+ **request,
1060
+ "external_network_policy_attestation": (self._external_network_policy_attestation),
1061
+ }
1062
+ request_bytes = _ipc_json_bytes(request)
1063
+ if len(request_bytes) > MAX_REQUEST_BYTES:
1064
+ raise AcquisitionSecurityError(
1065
+ "SANDBOX_IPC_LIMIT",
1066
+ "sandbox request exceeds the IPC byte budget",
1067
+ )
1068
+ if (
1069
+ sys.platform == "linux"
1070
+ and self._memory_cgroup_root_fd is None
1071
+ and self._hosted_session_container_memory_bytes is None
1072
+ ):
1073
+ raise AcquisitionSecurityError(
1074
+ "SANDBOX_MEMORY_BOUNDARY",
1075
+ "Linux Clean room execution requires a trusted bounded cgroup root",
1076
+ )
1077
+ launch = self._worker_launch(prewarm=False)
1078
+ # A warm child is only ever taken for a confinement identical to the one this
1079
+ # invocation would have spawned for itself, and only when no socket is in play. Taking
1080
+ # it removes it from the pool: a child is handed out once, serves this one request,
1081
+ # and exits.
1082
+ prepared: _PreparedWorker | None = None
1083
+ if self._warm_pool is not None and not network_allowed:
1084
+ prepared = self._warm_pool.take(launch)
1085
+ if prepared is None:
1086
+ # No warm child for this confinement: the untouched cold path, spawn and all.
1087
+ stdout, stderr, returncode = _run_worker_command(
1088
+ launch.argv(),
1089
+ cwd=launch.cwd,
1090
+ env=dict(launch.env),
1091
+ request_bytes=request_bytes,
1092
+ timeout_seconds=self._timeout_seconds,
1093
+ memory_cgroup_root_fd=self._memory_cgroup_root_fd,
1094
+ hosted_session_container_memory_bytes=(self._hosted_session_container_memory_bytes),
1095
+ )
1096
+ else:
1097
+ stdout, stderr, returncode = _complete_worker(
1098
+ prepared,
1099
+ request_bytes=request_bytes,
1100
+ timeout_seconds=self._timeout_seconds,
1101
+ terminate_process_group=(self._hosted_session_container_memory_bytes is not None),
1102
+ )
1103
+ if returncode != 0:
1104
+ refusal = _bounded_worker_error(stderr, returncode=returncode)
1105
+ raise ConfinedWorkerRefusal(
1106
+ "SANDBOX_FAILURE",
1107
+ f"sandbox failed closed: {refusal.message}",
1108
+ worker_code=refusal.code,
1109
+ operation=operation,
1110
+ )
1111
+ response = _strict_json_object(stdout, "sandbox.response")
1112
+ survey = request["cycle_survey"]
1113
+ range_plan = request["range_plan"]
1114
+ return _parse_response(
1115
+ response,
1116
+ expected_request_id=request["request_id"],
1117
+ # The operation this invocation ordered. The policy attestation below is only an
1118
+ # attestation if the operation it is derived from is the coordinator's, not the
1119
+ # responder's.
1120
+ expected_operation=request["operation"],
1121
+ expected_external_network_policy_attestation=(
1122
+ self._external_network_policy_attestation
1123
+ ),
1124
+ # The labels this invocation ordered, in the order it ordered them. A survey result
1125
+ # is matched back to a run by label, so the coordinator checks the answer set
1126
+ # against its own question rather than trusting the response to have kept both.
1127
+ expected_cycle_labels=(
1128
+ None
1129
+ if survey is None
1130
+ else tuple(candidate["label"] for candidate in survey["candidates"])
1131
+ ),
1132
+ # The descriptors this invocation ordered. One selector is one message, so the set
1133
+ # is known from the request alone: a short answer is a partial acquisition rather
1134
+ # than a smaller success, and a slice that answers a descriptor nobody named is
1135
+ # refused rather than carried back under a label the coordinator never asked for.
1136
+ expected_selectors=(None if range_plan is None else tuple(range_plan["selectors"])),
1137
+ # The model run this invocation ordered, when it ordered one. Read from the
1138
+ # coordinator's own request rather than from the response, for the same reason the
1139
+ # count above is: a binding a response supplies for itself binds nothing.
1140
+ expected_reference_time=(
1141
+ None if range_plan is None else range_plan["expected_reference_time"]
1142
+ ),
1143
+ )
1144
+
1145
+
1146
+ def sandbox_policy_digest(operation: str) -> str:
1147
+ """Return the exact policy digest the coordinator must bind into its context."""
1148
+
1149
+ # The coordinator and worker read the same operation allowlist.
1150
+ if operation not in SANDBOX_OPERATIONS:
1151
+ raise AcquisitionSecurityError(
1152
+ "SANDBOX_OPERATION",
1153
+ "sandbox operation is not allowlisted",
1154
+ )
1155
+ return _policy_digest(operation)
1156
+
1157
+
1158
+ def worker_main(*, prewarm: bool = False) -> int:
1159
+ """Read one bounded request from stdin and emit one canonical response.
1160
+
1161
+ One request, one response, one exit -- and ``prewarm`` does not change that. It only moves
1162
+ two things earlier: the deferred parser imports, and a marker line saying they are done.
1163
+ Both happen before stdin is read, so a pre-warmed child announces readiness while it still
1164
+ holds nothing a caller supplied, and then behaves exactly as a cold child does.
1165
+
1166
+ Two handlers, and the second one is the point. The typed handler carries the refusal's
1167
+ own code out to the coordinator, which is how an operator is told what to fix. The broad
1168
+ handler underneath it exists because this worker runs on bytes a stranger supplied: a
1169
+ decoder that reports its own failures with an exception type nobody mapped would otherwise
1170
+ unwind out of the process, and the operator would be told only that the sandbox failed
1171
+ closed, with no code and a Python traceback on the stream the coordinator reads. Both
1172
+ outcomes halt the run -- neither handler can make a failed job look like a finished one --
1173
+ but only one of them says what happened.
1174
+
1175
+ The broad handler reports the exception's type and nothing else. An unmapped failure has
1176
+ no vetted message, and the text of an arbitrary exception may carry a path from inside the
1177
+ confinement; the type name is enough to route a bug report and carries nothing a caller
1178
+ supplied. A failure arriving here is a defect in this codebase rather than a refusal a
1179
+ recipe can repair, and the code says so.
1180
+ """
1181
+
1182
+ if prewarm:
1183
+ # Before the request, never after it: the point of priming is that the import cost is
1184
+ # paid by an idle child rather than by a bound one. A failure here is a startup
1185
+ # failure and is reported on the same channel every other worker refusal uses.
1186
+ try:
1187
+ prime_parquet_support()
1188
+ except Exception as exc:
1189
+ return _worker_error(
1190
+ "SANDBOX_UNEXPECTED",
1191
+ f"pre-warm raised {type(exc).__name__}, which no refusal maps",
1192
+ )
1193
+ sys.stdout.buffer.write(WORKER_READY_MARKER)
1194
+ sys.stdout.buffer.flush()
1195
+ raw = sys.stdin.buffer.read(MAX_REQUEST_BYTES + 1)
1196
+ if len(raw) > MAX_REQUEST_BYTES:
1197
+ return _worker_error("SANDBOX_IPC_LIMIT", "request exceeds the IPC byte budget")
1198
+ try:
1199
+ request = _strict_json_object(raw, "sandbox.request")
1200
+ response = _execute_worker_request(request)
1201
+ encoded = _ipc_json_bytes(response)
1202
+ if len(encoded) > MAX_RESPONSE_BYTES:
1203
+ raise AcquisitionSecurityError(
1204
+ "SANDBOX_IPC_LIMIT",
1205
+ "response exceeds the IPC byte budget",
1206
+ )
1207
+ except (AcquisitionSecurityError, ValueError, TypeError) as exc:
1208
+ code = getattr(exc, "code", "SANDBOX_REQUEST")
1209
+ return _worker_error(code, str(exc))
1210
+ except Exception as exc:
1211
+ return _worker_error(
1212
+ "SANDBOX_UNEXPECTED",
1213
+ f"the worker raised {type(exc).__name__}, which no refusal maps",
1214
+ )
1215
+ sys.stdout.buffer.write(encoded)
1216
+ sys.stdout.buffer.flush()
1217
+ return 0
1218
+
1219
+
1220
+ def _execute_worker_request(request: dict[str, Any]) -> dict[str, Any]:
1221
+ expected = {
1222
+ "schema_version",
1223
+ "request_id",
1224
+ "operation",
1225
+ "content_base64",
1226
+ "data_format",
1227
+ "media_type",
1228
+ "filename",
1229
+ "parse_limits",
1230
+ "url",
1231
+ "allowed_hostnames",
1232
+ "approved_endpoints",
1233
+ "retrieval_limits",
1234
+ "network_probe_targets",
1235
+ # Reader authority. Every operation other than the decode sends both null and refuses
1236
+ # them non-null, exactly as ``parse`` already refuses ``url``. This literal is not
1237
+ # part of the policy document, so adding to it rotates no digest.
1238
+ "reader_pin",
1239
+ "reader_budgets",
1240
+ "range_plan",
1241
+ "cycle_survey",
1242
+ "external_network_policy_attestation",
1243
+ }
1244
+ if set(request) != expected:
1245
+ raise AcquisitionSecurityError(
1246
+ "SANDBOX_FIELDS",
1247
+ "sandbox request contains missing or extra fields",
1248
+ )
1249
+ if request["schema_version"] != SANDBOX_PROTOCOL_VERSION:
1250
+ raise AcquisitionSecurityError("SANDBOX_VERSION", "sandbox protocol version is invalid")
1251
+ request_id = _bounded_identifier(request["request_id"])
1252
+ operation = request["operation"]
1253
+ if operation not in SANDBOX_OPERATIONS:
1254
+ raise AcquisitionSecurityError("SANDBOX_OPERATION", "sandbox operation is not allowlisted")
1255
+ policy_digest = _policy_digest(operation)
1256
+ external_network_policy_attestation = _optional_digest(
1257
+ request["external_network_policy_attestation"]
1258
+ )
1259
+ visible_keys = tuple(sorted(os.environ))
1260
+ base: dict[str, Any] = {
1261
+ "schema_version": SANDBOX_PROTOCOL_VERSION,
1262
+ "request_id": request_id,
1263
+ "operation": operation,
1264
+ "policy_digest": policy_digest,
1265
+ "visible_environment_keys": list(visible_keys),
1266
+ "parsed": None,
1267
+ "content_base64": None,
1268
+ "media_type": None,
1269
+ "final_url": None,
1270
+ "transport_evidence_digest": None,
1271
+ "probe_transport": None,
1272
+ "home_directory_readable": None,
1273
+ "network_probe_errors": None,
1274
+ "decode_family_id": None,
1275
+ "decode_family_version": None,
1276
+ "decode_options_digest": None,
1277
+ "decode_flags": None,
1278
+ "decode_declared_cell_count": None,
1279
+ "fetched_content_sha256": None,
1280
+ "fetched_content_size_bytes": None,
1281
+ "fetched_total_response_body_size_bytes": None,
1282
+ "fetched_request_count": None,
1283
+ "ranges": None,
1284
+ "cycle_availability": None,
1285
+ "sidecar_content_sha256": None,
1286
+ "sidecar_size_bytes": None,
1287
+ "full_object_size_bytes": None,
1288
+ "fetched_members": None,
1289
+ "external_network_policy_attestation": external_network_policy_attestation,
1290
+ }
1291
+ if operation == "probe":
1292
+ _require_null_request_fields(request)
1293
+ probe_targets = _parse_network_probe_targets(request)
1294
+ base["home_directory_readable"] = _home_directory_readable()
1295
+ base["network_probe_errors"] = [
1296
+ {"label": label, "errno": error}
1297
+ for label, error in _probe_network_denials(probe_targets)
1298
+ ]
1299
+ return base
1300
+
1301
+ if operation == "fetch_ranges":
1302
+ _require_no_reader_authority(request)
1303
+ return _execute_range_plan(request, base)
1304
+
1305
+ if operation == "survey_cycles":
1306
+ _require_no_reader_authority(request)
1307
+ return _execute_cycle_survey(request, base)
1308
+
1309
+ if request["range_plan"] is not None or request["cycle_survey"] is not None:
1310
+ raise AcquisitionSecurityError(
1311
+ "SANDBOX_FIELDS",
1312
+ "a parsing request must not carry a range plan or a candidate survey",
1313
+ )
1314
+ limits = _parse_parse_limits(request["parse_limits"])
1315
+ data_format = _bounded_text(request["data_format"], maximum=16)
1316
+ # The ``parser_formats`` clause of this operation's policy document is enforced here,
1317
+ # before any bytes reach a parser. Without this check the clause would be attested and
1318
+ # digest-bound but never honoured: ``parse_tabular_bytes`` admits from the global format
1319
+ # tables and never sees ``operation``, so a worker bound to a csv-only attestation would
1320
+ # still decode parquet. An operation absent from the registry declares no parser formats
1321
+ # at all and therefore may parse nothing -- ``probe`` returns above and never reaches here.
1322
+ #
1323
+ # It runs before the filename is read rather than after, which is a move earlier and never
1324
+ # later: no bytes reach a parser that would not have reached one before, and the decode
1325
+ # operation declares no filename of its own because the Reader family declares it.
1326
+ admitted_formats = PARSER_FORMATS_BY_OPERATION.get(operation)
1327
+ if admitted_formats is None or data_format not in admitted_formats:
1328
+ raise AcquisitionSecurityError(
1329
+ "SANDBOX_FORMAT",
1330
+ "sandbox operation is not attested to parse this data format",
1331
+ )
1332
+ if operation == "decode_and_parse":
1333
+ return _decode_in_worker(request, base, output_format=data_format, limits=limits)
1334
+ if operation == "retrieve_decode_and_parse":
1335
+ return _retrieve_decode_in_worker(request, base, output_format=data_format, limits=limits)
1336
+
1337
+ _require_no_reader_authority(request)
1338
+ filename = _bounded_text(request["filename"], maximum=255)
1339
+ if operation == "parse":
1340
+ if request["url"] is not None:
1341
+ raise AcquisitionSecurityError(
1342
+ "SANDBOX_FIELDS",
1343
+ "parse request contains retrieval-only fields",
1344
+ )
1345
+ if (
1346
+ request["allowed_hostnames"] != []
1347
+ or request["approved_endpoints"] != []
1348
+ or request["retrieval_limits"] is not None
1349
+ or request["network_probe_targets"] != []
1350
+ ):
1351
+ raise AcquisitionSecurityError(
1352
+ "SANDBOX_FIELDS",
1353
+ "parse request contains retrieval-only authority",
1354
+ )
1355
+ media_type = _bounded_text(request["media_type"], maximum=255)
1356
+ content = _request_content(request)
1357
+ parsed = parse_tabular_bytes(
1358
+ content,
1359
+ data_format=data_format,
1360
+ media_type=media_type,
1361
+ filename=filename,
1362
+ limits=limits,
1363
+ )
1364
+ base["parsed"] = _parsed_dict(parsed)
1365
+ return base
1366
+
1367
+ if request["content_base64"] is not None or request["media_type"] is not None:
1368
+ raise AcquisitionSecurityError(
1369
+ "SANDBOX_FIELDS",
1370
+ "retrieve request cannot supply source bytes or choose response media type",
1371
+ )
1372
+ url = _bounded_text(request["url"], maximum=2_048)
1373
+ allowed_hostnames = _string_tuple(
1374
+ request["allowed_hostnames"],
1375
+ maximum_items=32,
1376
+ item_maximum=253,
1377
+ )
1378
+ _validate_egress_hosts(allowed_hostnames)
1379
+ if request["network_probe_targets"] != []:
1380
+ raise AcquisitionSecurityError(
1381
+ "SANDBOX_FIELDS",
1382
+ "retrieve request cannot carry network probes",
1383
+ )
1384
+ approved_resolver = _parse_approved_endpoints(
1385
+ request["approved_endpoints"],
1386
+ allowed_hostnames=allowed_hostnames,
1387
+ )
1388
+ retrieval_limits = _parse_retrieval_limits(request["retrieval_limits"])
1389
+ retriever = PinnedHttpsRetriever(
1390
+ resolver=approved_resolver,
1391
+ egress_policy=EgressPolicy(allowed_hostnames=allowed_hostnames),
1392
+ transport=StdlibPinnedTransport(),
1393
+ limits=retrieval_limits,
1394
+ )
1395
+ retrieved = retriever.retrieve(url)
1396
+ parsed = parse_tabular_bytes(
1397
+ retrieved.content,
1398
+ data_format=data_format,
1399
+ media_type=retrieved.media_type,
1400
+ filename=filename,
1401
+ limits=limits,
1402
+ )
1403
+ base.update(
1404
+ {
1405
+ "parsed": _parsed_dict(parsed),
1406
+ "content_base64": base64.b64encode(retrieved.content).decode("ascii"),
1407
+ "media_type": retrieved.media_type,
1408
+ "final_url": retrieved.final_url,
1409
+ "transport_evidence_digest": retrieved.transport_evidence_digest,
1410
+ "probe_transport": probe_transport_for(retrieved).to_dict(),
1411
+ }
1412
+ )
1413
+ return base
1414
+
1415
+
1416
+ def _request_content(request: dict[str, Any]) -> bytes:
1417
+ """Decode the request's carried bytes, refusing anything but canonical base64."""
1418
+
1419
+ try:
1420
+ return base64.b64decode(
1421
+ _bounded_text(request["content_base64"], maximum=MAX_REQUEST_BYTES),
1422
+ validate=True,
1423
+ )
1424
+ except (ValueError, TypeError):
1425
+ raise AcquisitionSecurityError(
1426
+ "SANDBOX_BASE64",
1427
+ "sandbox content is not canonical base64",
1428
+ ) from None
1429
+
1430
+
1431
+ def _require_no_reader_authority(request: dict[str, Any]) -> None:
1432
+ """Refuse Reader authority on an operation that is not the decode.
1433
+
1434
+ The mirror of the retrieval-only refusals: an operation that may not run a Reader must
1435
+ refuse the fields that would name one, rather than ignoring them. An ignored pin is a
1436
+ decoder a caller believes it selected.
1437
+ """
1438
+
1439
+ if request["reader_pin"] is not None or request["reader_budgets"] is not None:
1440
+ raise AcquisitionSecurityError(
1441
+ "SANDBOX_FIELDS",
1442
+ "only the decode operation may carry Reader authority",
1443
+ )
1444
+
1445
+
1446
+ def _decode_in_worker(
1447
+ request: dict[str, Any],
1448
+ base: dict[str, Any],
1449
+ *,
1450
+ output_format: str,
1451
+ limits: ParseLimits,
1452
+ ) -> dict[str, Any]:
1453
+ """Run one pinned Reader on untrusted bytes and parse what it produced.
1454
+
1455
+ Two rules live here as comments rather than in a document, because they are the security
1456
+ argument for admitting a decoder at all and not implementation detail.
1457
+
1458
+ * **The Reader never constructs a ``ParsedTable``.** It returns bytes, and the one
1459
+ allowlisted parser produces the table and its ``schema_digest`` from those bytes. A
1460
+ Reader therefore cannot attest a schema it did not derive.
1461
+ * **The normalized bytes are parsed inside this same confined process.** Untrusted bytes
1462
+ never cross to the coordinator undecoded; what crosses is the Reader's own canonical
1463
+ output and a table derived from it here, under the policy this operation is bound to.
1464
+ """
1465
+
1466
+ if (
1467
+ request["url"] is not None
1468
+ or request["allowed_hostnames"] != []
1469
+ or request["approved_endpoints"] != []
1470
+ or request["retrieval_limits"] is not None
1471
+ or request["network_probe_targets"] != []
1472
+ ):
1473
+ raise AcquisitionSecurityError(
1474
+ "SANDBOX_FIELDS",
1475
+ "decode request contains retrieval-only authority",
1476
+ )
1477
+ if request["media_type"] is not None or request["filename"] is not None:
1478
+ raise AcquisitionSecurityError(
1479
+ "SANDBOX_FIELDS",
1480
+ "decode request cannot choose the media type or filename the Reader declares",
1481
+ )
1482
+ content = _request_content(request)
1483
+ return _decode_content_in_worker(
1484
+ request,
1485
+ base,
1486
+ output_format=output_format,
1487
+ limits=limits,
1488
+ content=content,
1489
+ )
1490
+
1491
+
1492
+ def _retrieve_decode_in_worker(
1493
+ request: dict[str, Any],
1494
+ base: dict[str, Any],
1495
+ *,
1496
+ output_format: str,
1497
+ limits: ParseLimits,
1498
+ ) -> dict[str, Any]:
1499
+ """Fetch Reader input and normalize it without exporting container bytes."""
1500
+
1501
+ if request["content_base64"] is not None or request["media_type"] is not None:
1502
+ raise AcquisitionSecurityError(
1503
+ "SANDBOX_FIELDS",
1504
+ "Reader retrieval cannot supply source bytes or choose a response media type",
1505
+ )
1506
+ if request["filename"] is not None or request["network_probe_targets"] != []:
1507
+ raise AcquisitionSecurityError(
1508
+ "SANDBOX_FIELDS",
1509
+ "Reader retrieval cannot choose a filename or carry network probes",
1510
+ )
1511
+ url = _bounded_text(request["url"], maximum=2_048)
1512
+ allowed_hostnames = _string_tuple(
1513
+ request["allowed_hostnames"], maximum_items=32, item_maximum=253
1514
+ )
1515
+ _validate_egress_hosts(allowed_hostnames)
1516
+ approved_resolver = _parse_approved_endpoints(
1517
+ request["approved_endpoints"], allowed_hostnames=allowed_hostnames
1518
+ )
1519
+ retrieval_limits = _parse_retrieval_limits(request["retrieval_limits"])
1520
+ retrieved = PinnedHttpsRetriever(
1521
+ resolver=approved_resolver,
1522
+ egress_policy=EgressPolicy(allowed_hostnames=allowed_hostnames),
1523
+ transport=StdlibPinnedTransport(),
1524
+ limits=retrieval_limits,
1525
+ ).retrieve(url)
1526
+ base.update(
1527
+ {
1528
+ "final_url": retrieved.final_url,
1529
+ "transport_evidence_digest": retrieved.transport_evidence_digest,
1530
+ "probe_transport": probe_transport_for(retrieved).to_dict(),
1531
+ "fetched_content_sha256": retrieved.content_sha256,
1532
+ "fetched_content_size_bytes": len(retrieved.content),
1533
+ "fetched_total_response_body_size_bytes": (retrieved.total_response_body_size_bytes),
1534
+ # One redirect hop can contain several approved-IP transport attempts. The
1535
+ # retriever charges each of those attempts against ``max_requests``, so collection
1536
+ # evidence must count the same measured events rather than merely the number of
1537
+ # response hops.
1538
+ "fetched_request_count": sum(len(hop.peer_attempts) for hop in retrieved.hops),
1539
+ }
1540
+ )
1541
+ return _decode_content_in_worker(
1542
+ request,
1543
+ base,
1544
+ output_format=output_format,
1545
+ limits=limits,
1546
+ content=retrieved.content,
1547
+ )
1548
+
1549
+
1550
+ def _decode_content_in_worker(
1551
+ request: dict[str, Any],
1552
+ base: dict[str, Any],
1553
+ *,
1554
+ output_format: str,
1555
+ limits: ParseLimits,
1556
+ content: bytes,
1557
+ ) -> dict[str, Any]:
1558
+ """Run the exact pinned Reader and parse its normalized bytes."""
1559
+
1560
+ pin = _reader_pin_from_request(request["reader_pin"])
1561
+ caps = request["reader_budgets"]
1562
+ if caps is not None and not isinstance(caps, dict):
1563
+ raise AcquisitionSecurityError("SANDBOX_FIELDS", "decode request budgets are invalid")
1564
+
1565
+ # Resolution is against the closed Toolbox on an exact coordinate, with no fallback and
1566
+ # no discovery path, and it happens here inside the confinement. An unattested decoder
1567
+ # cannot run even if a request naming one reached the worker.
1568
+ family = TOOLBOX.resolve(pin.family_id, pin.family_version)
1569
+ # Admission before decode: the family's declared output encoding is checked against the
1570
+ # encoding this operation is attested to parse, so a family that would seal an unattested
1571
+ # encoding is refused before it sees a byte.
1572
+ if family.output_format != output_format:
1573
+ raise AcquisitionSecurityError(
1574
+ "SANDBOX_FORMAT",
1575
+ "the pinned Reader family does not emit the format this request is attested to seal",
1576
+ )
1577
+ budgets = family.default_budgets.narrowed_by(caps)
1578
+ if len(content) > budgets.max_input_bytes:
1579
+ raise AcquisitionSecurityError(
1580
+ "SANDBOX_IPC_LIMIT",
1581
+ "decode input exceeds the Reader input byte budget",
1582
+ )
1583
+ result = family.decode(content, pin, budgets)
1584
+ result.validate_for(family.output_format, budgets)
1585
+ parsed = parse_tabular_bytes(
1586
+ result.content,
1587
+ data_format=result.data_format,
1588
+ media_type=result.media_type,
1589
+ filename=result.filename,
1590
+ limits=limits,
1591
+ )
1592
+ base.update(
1593
+ {
1594
+ "parsed": _parsed_dict(parsed),
1595
+ "content_base64": base64.b64encode(result.content).decode("ascii"),
1596
+ "media_type": result.media_type,
1597
+ "decode_family_id": family.family_id,
1598
+ "decode_family_version": family.family_version,
1599
+ "decode_options_digest": pin.options_digest,
1600
+ # Already sorted and de-duplicated by ``ReaderResult``; carried as data because
1601
+ # the coordinator cannot recompute a fact about a decode it did not run.
1602
+ "decode_flags": list(result.flags),
1603
+ "decode_declared_cell_count": result.declared_cell_count,
1604
+ }
1605
+ )
1606
+ return base
1607
+
1608
+
1609
+ def _reader_pin_from_request(value: Any) -> ReaderPin:
1610
+ """Admit the request's Reader pin as the contract's own three-key object."""
1611
+
1612
+ if not isinstance(value, dict) or set(value) != {
1613
+ "family_id",
1614
+ "family_version",
1615
+ "decode_options",
1616
+ }:
1617
+ raise AcquisitionSecurityError(
1618
+ "SANDBOX_FIELDS",
1619
+ "decode request must carry exactly a three-key Reader pin",
1620
+ )
1621
+ return ReaderPin(value["family_id"], value["family_version"], value["decode_options"])
1622
+
1623
+
1624
+ def _execute_range_plan(request: dict[str, Any], base: dict[str, Any]) -> dict[str, Any]:
1625
+ """Validate a slice-fetch request's authority and its plan, then run it.
1626
+
1627
+ Ordered so that every refusal that can be made from the request alone is made before any
1628
+ address is looked at: parsing inputs first, then probe authority, then egress, then the
1629
+ pinned endpoint set, then the retrieval budget, and only then the plan those bounds apply
1630
+ to. A plan checked against a budget that had not been validated would be checked against
1631
+ nothing.
1632
+ """
1633
+
1634
+ for key in ("content_base64", "data_format", "media_type", "filename", "parse_limits", "url"):
1635
+ if request[key] is not None:
1636
+ raise AcquisitionSecurityError(
1637
+ "SANDBOX_FIELDS",
1638
+ "a slice fetch carries no parsing inputs and no single-object URL",
1639
+ )
1640
+ if request["network_probe_targets"] != [] or request["cycle_survey"] is not None:
1641
+ raise AcquisitionSecurityError(
1642
+ "SANDBOX_FIELDS",
1643
+ "a slice fetch cannot carry network probes or a candidate survey",
1644
+ )
1645
+ allowed_hostnames = _string_tuple(
1646
+ request["allowed_hostnames"],
1647
+ maximum_items=32,
1648
+ item_maximum=253,
1649
+ )
1650
+ _validate_egress_hosts(allowed_hostnames)
1651
+ approved_resolver = _parse_approved_endpoints(
1652
+ request["approved_endpoints"],
1653
+ allowed_hostnames=allowed_hostnames,
1654
+ )
1655
+ retrieval_limits = _parse_retrieval_limits(request["retrieval_limits"])
1656
+ plan = _parse_range_plan(request["range_plan"], budget_ceiling=retrieval_limits.max_ranges)
1657
+
1658
+ # One resolver, one transport and one limiter for the whole plan. The resolver is the
1659
+ # coordinator's single DNS epoch, already pinned to exact public addresses; reusing it is
1660
+ # what stops a rebind between the sidecar that says where to look and the slice taken
1661
+ # from there. The limiter is shared for the other half of the same property: N sequential
1662
+ # fetches against one source must be polite as one acquisition rather than as N bursts.
1663
+ #
1664
+ # Two retrievers rather than one, differing only in their media allowlist, because a
1665
+ # sidecar and a slice are different responses and each call should admit only the one it
1666
+ # ordered. Both narrow within whatever the coordinator already allowed and neither can
1667
+ # widen it.
1668
+ egress_policy = EgressPolicy(allowed_hostnames=allowed_hostnames)
1669
+ transport = StdlibPinnedTransport()
1670
+ limiter = AcquisitionLimiter(
1671
+ max_concurrency=retrieval_limits.max_concurrency,
1672
+ min_interval_seconds=retrieval_limits.min_interval_seconds,
1673
+ )
1674
+
1675
+ def retriever(media_types: tuple[str, ...]) -> PinnedHttpsRetriever:
1676
+ return PinnedHttpsRetriever(
1677
+ resolver=approved_resolver,
1678
+ egress_policy=egress_policy,
1679
+ transport=transport,
1680
+ limits=_narrowed_to_media_types(retrieval_limits, media_types),
1681
+ limiter=limiter,
1682
+ )
1683
+
1684
+ # Construct both narrowed retrievers before any request. A coordinator allowlist that
1685
+ # admits neither half is therefore still a request-free refusal even though the retained
1686
+ # presence probe itself admits any media type.
1687
+ sidecar_retriever = retriever(SIDECAR_MEDIA_TYPES)
1688
+ slice_retriever = retriever(SLICE_OBJECT_MEDIA_TYPES)
1689
+ member_documents: list[dict[str, Any]] = []
1690
+
1691
+ def retain_members(role: str, hops: tuple[RetrievalHop, ...]) -> None:
1692
+ for hop in hops:
1693
+ member_documents.append(
1694
+ {
1695
+ "role": role,
1696
+ "sequence": len(member_documents),
1697
+ **hop.to_dict(),
1698
+ }
1699
+ )
1700
+
1701
+ sidecar = sidecar_retriever.retrieve(plan.sidecar_url)
1702
+ retain_members("sidecar_initial", sidecar.hops)
1703
+ # ``_parse_range_plan`` bound the sidecar and the object to one origin, but it bound the
1704
+ # address that was *ordered*. An unranged retrieval follows redirects, so without this the
1705
+ # ordered host can hand the sidecar fetch to any other allowlisted host, and that host then
1706
+ # chooses which bytes of the first host's object are delivered: it renames the offsets of a
1707
+ # message the operator did not order to the descriptor the operator did order, and every
1708
+ # control downstream still passes -- exactly 206, a matching ``Content-Range``, three
1709
+ # lengths in agreement, and a structurally valid GRIB message, just the wrong field. The
1710
+ # ranged fetches are already closed to this by ``REDIRECT_ON_RANGE``; this is the same
1711
+ # property for the half of the order that says where to look. Compared against the object's
1712
+ # origin rather than the ordered sidecar's, because those two are equal by the check above
1713
+ # and the object is the thing whose bytes the index is allowed to speak for.
1714
+ if _https_origin(sidecar.final_url, code="SANDBOX_SIDECAR_ORIGIN") != _https_origin(
1715
+ plan.object_url, code="SANDBOX_SIDECAR_ORIGIN"
1716
+ ):
1717
+ raise AcquisitionSecurityError(
1718
+ "SANDBOX_SIDECAR_ORIGIN",
1719
+ "the index sidecar was answered by an origin other than the object's",
1720
+ )
1721
+ # The address that actually answered, so the receipt can state it rather than restating the
1722
+ # address that was ordered. The hop list is digested, not carried, so this is the only field
1723
+ # in which a redirect that was allowed is legible after the fact.
1724
+ base["final_url"] = sidecar.final_url
1725
+ # The sidecar is untrusted input from here on. Its numbers may become a ``ByteRange`` and
1726
+ # nothing else: never an address, never an allocation size, never a length trusted without
1727
+ # the bytes that arrive being measured against it.
1728
+ messages = parse_index_sidecar(sidecar.content, limits=SidecarLimits())
1729
+ planned = plan_ranges(messages, selectors=plan.selectors, max_ranges=plan.max_ranges)
1730
+ _require_slice_bytes_within_budget(planned, limits=retrieval_limits)
1731
+
1732
+ # The selected cycle is probed again inside the acquisition's DNS epoch. Unlike
1733
+ # ``survey_cycles`` this is retained evidence: the object has already been chosen, so the
1734
+ # exact one-byte response and every redirect body belong in the receipt and efficiency
1735
+ # numerator rather than being accepted as a caller-supplied count.
1736
+ probe_retriever = PinnedHttpsRetriever(
1737
+ resolver=approved_resolver,
1738
+ egress_policy=egress_policy,
1739
+ transport=transport,
1740
+ limits=retrieval_limits,
1741
+ limiter=limiter,
1742
+ )
1743
+ probe = probe_retriever.retrieve_presence_evidence(plan.object_url)
1744
+ retain_members("cycle_probe", probe.hops)
1745
+
1746
+ hops = [*sidecar.hops, *probe.hops]
1747
+ slices: list[dict[str, Any]] = []
1748
+ complete_sizes: set[int] = set()
1749
+ for entry in planned:
1750
+ # ``retrieve`` runs the delivery check: exactly 206, a ``Content-Range`` equal to the
1751
+ # span that was ordered, three lengths in agreement. That proves the span. It cannot
1752
+ # prove the framing, so the structural check below runs before this slice is kept, and
1753
+ # neither proves which run the bytes came from, so the run binding below runs too when
1754
+ # the order named a run. Any of the three refusing fails the whole invocation. A
1755
+ # partial slice set is not a smaller success; it is an acquisition that did not happen.
1756
+ parcel = slice_retriever.retrieve(plan.object_url, byte_range=entry.byte_range)
1757
+ hops.extend(parcel.hops)
1758
+ retain_members("selected_slice", parcel.hops)
1759
+ check_grib_message(parcel.content)
1760
+ # The run binding, when the order named a run. Ordered after the structural check
1761
+ # because that check is what makes the section offsets below meaningful at all.
1762
+ if plan.expected_reference_time is not None:
1763
+ check_grib_reference_time(parcel.content, expected=plan.expected_reference_time)
1764
+ if parcel.complete_size_bytes is None:
1765
+ raise AcquisitionSecurityError(
1766
+ "CONTENT_RANGE",
1767
+ "a slice response must name the complete object size",
1768
+ )
1769
+ complete_sizes.add(parcel.complete_size_bytes)
1770
+ slices.append(
1771
+ {
1772
+ "index": entry.index,
1773
+ # The selector this span was planned for. The plan is emitted in message-index
1774
+ # order, so without it the response says which message it delivered and never
1775
+ # which of the ordered descriptors that message answers.
1776
+ "descriptor": entry.descriptor,
1777
+ "first_byte": entry.byte_range.first_byte,
1778
+ "last_byte": entry.byte_range.last_byte,
1779
+ "sha256": parcel.content_sha256,
1780
+ "content_base64": base64.b64encode(parcel.content).decode("ascii"),
1781
+ }
1782
+ )
1783
+ # Re-read the complete sidecar only after every selected member has arrived. The source
1784
+ # revision is admitted only when the terminal bytes are identical to the bytes that chose
1785
+ # the ranges; an update during the acquisition is a refusal, never a mixed revision.
1786
+ terminal_sidecar = sidecar_retriever.retrieve(plan.sidecar_url)
1787
+ retain_members("sidecar_final", terminal_sidecar.hops)
1788
+ hops.extend(terminal_sidecar.hops)
1789
+ if (
1790
+ terminal_sidecar.content_sha256 != sidecar.content_sha256
1791
+ or terminal_sidecar.content != sidecar.content
1792
+ or terminal_sidecar.final_url != sidecar.final_url
1793
+ ):
1794
+ raise AcquisitionSecurityError(
1795
+ "SANDBOX_SOURCE_REVISION",
1796
+ "the index sidecar changed while selected messages were being fetched",
1797
+ )
1798
+ base["final_url"] = terminal_sidecar.final_url
1799
+ base["ranges"] = slices
1800
+ if probe.complete_size_bytes is None:
1801
+ raise AcquisitionSecurityError(
1802
+ "CONTENT_RANGE",
1803
+ "the retained cycle probe must name the complete object size",
1804
+ )
1805
+ complete_sizes.add(probe.complete_size_bytes)
1806
+ if len(complete_sizes) != 1:
1807
+ raise AcquisitionSecurityError(
1808
+ "CONTENT_RANGE",
1809
+ "all slice responses must name one complete object size",
1810
+ )
1811
+ base["sidecar_content_sha256"] = sidecar.content_sha256
1812
+ base["sidecar_size_bytes"] = len(sidecar.content)
1813
+ base["full_object_size_bytes"] = complete_sizes.pop()
1814
+ base["fetched_members"] = member_documents
1815
+ # One acquisition, one transport evidence digest: the sidecar hop followed by every range
1816
+ # hop in fetch order, flattened, digested exactly as a single retrieval's hop list is.
1817
+ base["transport_evidence_digest"] = canonical_sha256([hop.to_dict() for hop in hops])
1818
+ return base
1819
+
1820
+
1821
+ def _execute_cycle_survey(request: dict[str, Any], base: dict[str, Any]) -> dict[str, Any]:
1822
+ """Probe each coordinator-supplied candidate once and report presence, in their order.
1823
+
1824
+ Ordered exactly as ``_execute_range_plan`` is: every refusal that can be made from the
1825
+ request alone is made before any address is looked at, so a candidate set that was never
1826
+ going to be admitted costs no request against a source that has done nothing wrong.
1827
+ """
1828
+
1829
+ for key in ("content_base64", "data_format", "media_type", "filename", "parse_limits", "url"):
1830
+ if request[key] is not None:
1831
+ raise AcquisitionSecurityError(
1832
+ "SANDBOX_FIELDS",
1833
+ "an availability survey carries no parsing inputs and no single-object URL",
1834
+ )
1835
+ if request["network_probe_targets"] != [] or request["range_plan"] is not None:
1836
+ raise AcquisitionSecurityError(
1837
+ "SANDBOX_FIELDS",
1838
+ "an availability survey cannot carry network probes or a range plan",
1839
+ )
1840
+ allowed_hostnames = _string_tuple(
1841
+ request["allowed_hostnames"],
1842
+ maximum_items=32,
1843
+ item_maximum=253,
1844
+ )
1845
+ _validate_egress_hosts(allowed_hostnames)
1846
+ approved_resolver = _parse_approved_endpoints(
1847
+ request["approved_endpoints"],
1848
+ allowed_hostnames=allowed_hostnames,
1849
+ )
1850
+ retrieval_limits = _parse_retrieval_limits(request["retrieval_limits"])
1851
+ candidates = _parse_cycle_survey(
1852
+ request["cycle_survey"],
1853
+ allowed_hostnames=allowed_hostnames,
1854
+ )
1855
+
1856
+ # One retriever for the candidate set, so one resolver, one transport and one limiter cover
1857
+ # it: the survey is polite to a source as one look rather than as N bursts, and every probe
1858
+ # is answered inside the single DNS epoch the coordinator pinned.
1859
+ #
1860
+ # Every probe goes through ``probe_presence``, which admits any media type. The survey does
1861
+ # not know what a candidate is -- a cycle-addressed CSV run and a GRIB object are both
1862
+ # legitimate candidates, and the GRIB case answers ``application/octet-stream``, which the
1863
+ # default acquisition allowlist deliberately excludes. Judging presence by content type
1864
+ # would report a healthy published run as a hard failure on every poll. The coordinator's
1865
+ # ``retrieval_limits`` still govern this call in every other respect -- budgets, timeouts,
1866
+ # redirects, response ceiling and rate limit -- and its media allowlist governs every call
1867
+ # that actually acquires bytes.
1868
+ retriever = PinnedHttpsRetriever(
1869
+ resolver=approved_resolver,
1870
+ egress_policy=EgressPolicy(allowed_hostnames=allowed_hostnames),
1871
+ transport=StdlibPinnedTransport(),
1872
+ limits=retrieval_limits,
1873
+ )
1874
+ availability: list[dict[str, Any]] = []
1875
+ for candidate in candidates:
1876
+ # Presence and nothing else. No bytes, no digest, no status code: the survey answers one
1877
+ # question, and an address the coordinator has not locked yet is not an acquisition.
1878
+ # Absence is decided inside ``probe_presence`` from the typed ``HTTP_NOT_FOUND`` code
1879
+ # and never from message text; every other refusal propagates and fails the invocation.
1880
+ present = retriever.probe_presence(candidate.url)
1881
+ availability.append({"label": candidate.label, "present": present})
1882
+ base["cycle_availability"] = availability
1883
+ # No transport digest is emitted because not-found responses carry no hop list. A partial
1884
+ # digest would omit the absent candidates being reported.
1885
+ return base
1886
+
1887
+
1888
+ def _parse_cycle_survey(
1889
+ value: Any,
1890
+ *,
1891
+ allowed_hostnames: tuple[str, ...],
1892
+ ) -> tuple[CycleCandidate, ...]:
1893
+ """Parse the closed candidate list, or refuse it.
1894
+
1895
+ No refusal here quotes an address or a label. The refusal crosses the sandbox boundary and
1896
+ both are text that came from outside this process.
1897
+ """
1898
+
1899
+ if not isinstance(value, dict) or set(value) != _CYCLE_SURVEY_KEYS:
1900
+ raise AcquisitionSecurityError(
1901
+ "SANDBOX_CYCLE_SURVEY",
1902
+ "an availability survey must be the exact closed object of one field",
1903
+ )
1904
+ candidates = value["candidates"]
1905
+ if not isinstance(candidates, list) or not 1 <= len(candidates) <= MAX_CYCLE_CANDIDATES:
1906
+ raise AcquisitionSecurityError(
1907
+ "SANDBOX_CYCLE_SURVEY",
1908
+ f"a survey must name between one and {MAX_CYCLE_CANDIDATES} candidates",
1909
+ )
1910
+ parsed: list[CycleCandidate] = []
1911
+ labels: set[str] = set()
1912
+ for item in candidates:
1913
+ if not isinstance(item, dict) or set(item) != _CYCLE_CANDIDATE_KEYS:
1914
+ raise AcquisitionSecurityError(
1915
+ "SANDBOX_CYCLE_SURVEY",
1916
+ "a survey candidate is exactly a label and an address",
1917
+ )
1918
+ candidate = CycleCandidate(label=item["label"], url=item["url"])
1919
+ # Unique, because the label is the only key a coordinator matches a result back to.
1920
+ # Two candidates sharing one label make an answer set that cannot be read.
1921
+ if candidate.label in labels:
1922
+ raise AcquisitionSecurityError(
1923
+ "SANDBOX_CYCLE_SURVEY",
1924
+ "survey candidate labels must be unique within one request",
1925
+ )
1926
+ labels.add(candidate.label)
1927
+ # Checked here as well as by the egress policy at fetch time, so a candidate outside
1928
+ # the coordinator's allowlist costs no request at all -- including no request for the
1929
+ # legitimate candidates that would otherwise have been probed before reaching it.
1930
+ hostname = urlsplit(candidate.url).hostname
1931
+ if hostname is None or hostname.lower() not in allowed_hostnames:
1932
+ raise AcquisitionSecurityError(
1933
+ "SANDBOX_EGRESS",
1934
+ "a survey candidate names a host outside the egress allowlist",
1935
+ )
1936
+ parsed.append(candidate)
1937
+ return tuple(parsed)
1938
+
1939
+
1940
+ def _require_surveyable_candidates(
1941
+ candidates: tuple[CycleCandidate, ...],
1942
+ *,
1943
+ allowed_hostnames: tuple[str, ...],
1944
+ ) -> None:
1945
+ """Refuse a candidate set on the trusted side, before a single name is resolved."""
1946
+
1947
+ if not isinstance(candidates, tuple) or not 1 <= len(candidates) <= MAX_CYCLE_CANDIDATES:
1948
+ raise AcquisitionSecurityError(
1949
+ "SANDBOX_CYCLE_SURVEY",
1950
+ f"a survey must name between one and {MAX_CYCLE_CANDIDATES} candidates",
1951
+ )
1952
+ labels = [candidate.label for candidate in candidates]
1953
+ if len(set(labels)) != len(labels):
1954
+ raise AcquisitionSecurityError(
1955
+ "SANDBOX_CYCLE_SURVEY",
1956
+ "survey candidate labels must be unique within one request",
1957
+ )
1958
+ for candidate in candidates:
1959
+ if not isinstance(candidate, CycleCandidate):
1960
+ raise AcquisitionSecurityError(
1961
+ "SANDBOX_CYCLE_SURVEY",
1962
+ "a survey candidate must be a validated candidate value",
1963
+ )
1964
+ hostname = urlsplit(candidate.url).hostname
1965
+ if hostname is None or hostname.lower() not in allowed_hostnames:
1966
+ raise AcquisitionSecurityError(
1967
+ "SANDBOX_EGRESS",
1968
+ "a survey candidate names a host outside the egress allowlist",
1969
+ )
1970
+
1971
+
1972
+ def _narrowed_to_media_types(
1973
+ limits: RetrievalLimits,
1974
+ media_types: tuple[str, ...],
1975
+ ) -> RetrievalLimits:
1976
+ """Narrow a retrieval's media allowlist to one call's half of it, never widening it."""
1977
+
1978
+ admitted = tuple(
1979
+ media_type for media_type in media_types if media_type in limits.allowed_media_types
1980
+ )
1981
+ if not admitted:
1982
+ raise AcquisitionSecurityError(
1983
+ "SANDBOX_LIMITS",
1984
+ "the retrieval's media allowlist admits nothing this fetch would accept",
1985
+ )
1986
+ return replace(limits, allowed_media_types=admitted)
1987
+
1988
+
1989
+ def _require_slice_bytes_within_budget(
1990
+ planned: tuple[PlannedRange, ...],
1991
+ *,
1992
+ limits: RetrievalLimits,
1993
+ ) -> None:
1994
+ """Refuse a plan whose slices would not fit, before a single byte of them is fetched.
1995
+
1996
+ Two ceilings, because they bound different things. One slice may not exceed the response
1997
+ limit the retrieval already enforces per fetch -- checked here as well so an over-large
1998
+ span is refused before the request rather than after the bytes arrive.
1999
+
2000
+ The total is bounded by the IPC response cap it has to survive. Base64 expands by four
2001
+ thirds, so ``MAX_TOTAL_SLICE_BYTES`` of slices becomes 32 MiB of response text against a
2002
+ 48 MiB cap, leaving 16 MiB for a response whose other fields do not reach a kilobyte. The
2003
+ arithmetic is stated rather than left for a reader to rediscover, and the plan is refused
2004
+ rather than truncated: a shortened slice set is a different acquisition, silently.
2005
+ """
2006
+
2007
+ total = 0
2008
+ for entry in planned:
2009
+ length = entry.byte_range.length
2010
+ if length > limits.max_response_bytes:
2011
+ raise AcquisitionSecurityError(
2012
+ "SANDBOX_SLICE_BUDGET",
2013
+ "a planned slice is larger than the retrieval's own response limit",
2014
+ )
2015
+ total += length
2016
+ if total > MAX_TOTAL_SLICE_BYTES:
2017
+ raise AcquisitionSecurityError(
2018
+ "SANDBOX_SLICE_BUDGET",
2019
+ f"the planned slices total more than the {MAX_TOTAL_SLICE_BYTES} byte ceiling",
2020
+ )
2021
+
2022
+
2023
+ @dataclass(frozen=True)
2024
+ class _RangePlan:
2025
+ """A coordinator's order: which object, which sidecar describes it, and what to take."""
2026
+
2027
+ sidecar_url: str
2028
+ object_url: str
2029
+ selectors: tuple[str, ...]
2030
+ max_ranges: int
2031
+ # The model run this order is for, or ``None`` when the order names none. When it is set,
2032
+ # every delivered message must say in its own Section 1 that it came from that run; when it
2033
+ # is ``None`` the run binding is simply not in force, which is a smaller claim, not a
2034
+ # silent one -- ``mr-data acquire-slices`` prints which of the two it ran under.
2035
+ expected_reference_time: str | None
2036
+
2037
+
2038
+ def _parse_range_plan(value: Any, *, budget_ceiling: int) -> _RangePlan:
2039
+ """Parse the closed range-plan object, or refuse it.
2040
+
2041
+ No refusal here quotes an address or a selector. The refusal crosses the sandbox boundary
2042
+ and both are text that came from outside this process.
2043
+ """
2044
+
2045
+ if not isinstance(value, dict) or set(value) != _RANGE_PLAN_KEYS:
2046
+ raise AcquisitionSecurityError(
2047
+ "SANDBOX_RANGE_PLAN",
2048
+ "a range plan must be the exact closed object of five fields",
2049
+ )
2050
+ sidecar_url = _bounded_text(value["sidecar_url"], maximum=2_048)
2051
+ object_url = _bounded_text(value["object_url"], maximum=2_048)
2052
+ # Checked, never derived. Appending ``.idx`` to the object's address here would make the
2053
+ # sidecar an address this worker invented rather than one the coordinator resolved and
2054
+ # pinned, and the whole value of the pin is that every address fetched came from outside.
2055
+ if _https_origin(sidecar_url) != _https_origin(object_url):
2056
+ raise AcquisitionSecurityError(
2057
+ "SANDBOX_RANGE_PLAN",
2058
+ "a range plan's sidecar and object must share one https origin",
2059
+ )
2060
+ selectors = _string_tuple(
2061
+ value["selectors"],
2062
+ maximum_items=MAX_RANGE_PLAN,
2063
+ item_maximum=MAX_SELECTOR_BYTES,
2064
+ )
2065
+ if not selectors:
2066
+ raise AcquisitionSecurityError(
2067
+ "SANDBOX_RANGE_PLAN",
2068
+ "a range plan must order at least one message",
2069
+ )
2070
+ max_ranges = value["max_ranges"]
2071
+ # Bounded by the retrieval's own budget rather than only by the transport ceiling: a plan
2072
+ # that could name its own larger budget would make the limits it travels beside advisory.
2073
+ if type(max_ranges) is not int or not 1 <= max_ranges <= budget_ceiling:
2074
+ raise AcquisitionSecurityError(
2075
+ "SANDBOX_RANGE_PLAN",
2076
+ "a range plan's budget must lie within the retrieval's own range budget",
2077
+ )
2078
+ # How many slices are ordered is known from the request alone, so the budget is applied
2079
+ # here rather than after the sidecar fetch. ``plan_ranges`` applies the same bound again
2080
+ # to the plan it builds; this one exists so an over-long order costs no request at all.
2081
+ require_range_plan_within_budget(len(selectors), max_ranges=max_ranges)
2082
+ # Parsed here rather than trusted as it arrives, and parsed by the same function the
2083
+ # coordinator used to build it, so the two sides of the boundary cannot come to hold two
2084
+ # spellings of one run.
2085
+ expected_reference_time = value["expected_reference_time"]
2086
+ if expected_reference_time is not None:
2087
+ expected_reference_time = require_reference_time_spelling(expected_reference_time)
2088
+ return _RangePlan(
2089
+ sidecar_url=sidecar_url,
2090
+ object_url=object_url,
2091
+ selectors=selectors,
2092
+ max_ranges=max_ranges,
2093
+ expected_reference_time=expected_reference_time,
2094
+ )
2095
+
2096
+
2097
+ def _https_origin(url: str, *, code: str = "SANDBOX_RANGE_PLAN") -> tuple[str, str]:
2098
+ """Return an address's origin, refusing anything that is not an absolute https URL.
2099
+
2100
+ The refusal code is the caller's, because the two callers refuse different things and a
2101
+ survey that reported a malformed candidate as a range-plan failure would send a reader to
2102
+ the wrong half of the boundary.
2103
+ """
2104
+
2105
+ split = urlsplit(url)
2106
+ if split.scheme != "https" or not split.netloc:
2107
+ raise AcquisitionSecurityError(
2108
+ code,
2109
+ "a clean-room address must be an absolute https URL",
2110
+ )
2111
+ return split.scheme, split.netloc.lower()
2112
+
2113
+
2114
+ def _parse_response(
2115
+ response: dict[str, Any],
2116
+ *,
2117
+ expected_request_id: str,
2118
+ expected_operation: str,
2119
+ expected_external_network_policy_attestation: str | None,
2120
+ expected_cycle_labels: tuple[str, ...] | None = None,
2121
+ expected_selectors: tuple[str, ...] | None = None,
2122
+ expected_reference_time: str | None = None,
2123
+ ) -> SandboxResult:
2124
+ expected = {
2125
+ "schema_version",
2126
+ "request_id",
2127
+ "operation",
2128
+ "policy_digest",
2129
+ "visible_environment_keys",
2130
+ "parsed",
2131
+ "content_base64",
2132
+ "media_type",
2133
+ "final_url",
2134
+ "transport_evidence_digest",
2135
+ "probe_transport",
2136
+ "home_directory_readable",
2137
+ "network_probe_errors",
2138
+ "decode_family_id",
2139
+ "decode_family_version",
2140
+ "decode_options_digest",
2141
+ "decode_flags",
2142
+ "decode_declared_cell_count",
2143
+ "fetched_content_sha256",
2144
+ "fetched_content_size_bytes",
2145
+ "fetched_total_response_body_size_bytes",
2146
+ "fetched_request_count",
2147
+ "ranges",
2148
+ "cycle_availability",
2149
+ "sidecar_content_sha256",
2150
+ "sidecar_size_bytes",
2151
+ "full_object_size_bytes",
2152
+ "fetched_members",
2153
+ "external_network_policy_attestation",
2154
+ }
2155
+ if set(response) != expected or response["schema_version"] != SANDBOX_PROTOCOL_VERSION:
2156
+ raise AcquisitionSecurityError(
2157
+ "SANDBOX_RESPONSE",
2158
+ "sandbox response shape or version is invalid",
2159
+ )
2160
+ request_id = _bounded_identifier(response["request_id"])
2161
+ if request_id != expected_request_id:
2162
+ raise AcquisitionSecurityError(
2163
+ "SANDBOX_RESPONSE_BINDING",
2164
+ "sandbox response does not bind the exact request",
2165
+ )
2166
+ operation = _bounded_text(response["operation"], maximum=32)
2167
+ # The operation is bound to the one that was ordered, exactly as ``request_id`` is above.
2168
+ # Without this the response names its own operation and ``_policy_digest`` below then
2169
+ # validates it against itself: a clean room invoked for ``fetch_ranges`` -- which resolves
2170
+ # endpoints, opens the attested network clause and requires the external
2171
+ # network attestation -- could answer with ``parse``'s digest, and the sealed evidence
2172
+ # would record a stricter policy than the one that actually ran. A self-declared
2173
+ # attestation is not an attestation.
2174
+ if operation != expected_operation:
2175
+ raise AcquisitionSecurityError(
2176
+ "SANDBOX_RESPONSE_BINDING",
2177
+ "the clean room's response does not bind the operation that was requested",
2178
+ )
2179
+ expected_policy = _policy_digest(operation)
2180
+ if response["policy_digest"] != expected_policy:
2181
+ raise AcquisitionSecurityError(
2182
+ "SANDBOX_POLICY_BINDING",
2183
+ "sandbox response policy attestation is invalid",
2184
+ )
2185
+ external_network_policy_attestation = _optional_digest(
2186
+ response["external_network_policy_attestation"]
2187
+ )
2188
+ if external_network_policy_attestation != expected_external_network_policy_attestation:
2189
+ raise AcquisitionSecurityError(
2190
+ "SANDBOX_POLICY_BINDING",
2191
+ "sandbox response does not bind the external network policy attestation",
2192
+ )
2193
+ visible = _string_tuple(
2194
+ response["visible_environment_keys"],
2195
+ maximum_items=32,
2196
+ item_maximum=128,
2197
+ )
2198
+ if set(visible) - set(_ALLOWED_ENVIRONMENT):
2199
+ raise AcquisitionSecurityError(
2200
+ "SANDBOX_ENVIRONMENT",
2201
+ "sandbox inherited non-allowlisted environment keys",
2202
+ )
2203
+ home_directory_readable = response["home_directory_readable"]
2204
+ if home_directory_readable is not None and type(home_directory_readable) is not bool:
2205
+ raise AcquisitionSecurityError(
2206
+ "SANDBOX_RESPONSE",
2207
+ "sandbox home-directory probe is invalid",
2208
+ )
2209
+ # The home-directory answer is still reported, and is still bound into the response the
2210
+ # policy digest covers, but nothing enforces it here any more. It was enforced on darwin
2211
+ # alone, because the Seatbelt profile ADR 0021 deleted was the thing that made the
2212
+ # home directory unreadable; on every other host the boundary that owns the filesystem is
2213
+ # the external container the attestation names, and this process is in no position to
2214
+ # second-guess it. Keeping the field is what lets a container's own conformance check read
2215
+ # it.
2216
+ network_probe_errors = _parse_network_probe_response(
2217
+ response["network_probe_errors"],
2218
+ required=operation == "probe",
2219
+ )
2220
+ ranges = _parse_fetched_slices(
2221
+ response["ranges"],
2222
+ required=operation == "fetch_ranges",
2223
+ expected_selectors=expected_selectors,
2224
+ expected_reference_time=expected_reference_time,
2225
+ )
2226
+ cycle_availability = _parse_cycle_availability(
2227
+ response["cycle_availability"],
2228
+ required=operation == "survey_cycles",
2229
+ expected_labels=expected_cycle_labels,
2230
+ )
2231
+ fetched_members = _parse_fetched_members(
2232
+ response["fetched_members"],
2233
+ required=operation == "fetch_ranges",
2234
+ )
2235
+ sidecar_content_sha256 = _optional_digest(response["sidecar_content_sha256"])
2236
+ sidecar_size_bytes = response["sidecar_size_bytes"]
2237
+ full_object_size_bytes = response["full_object_size_bytes"]
2238
+ if operation == "fetch_ranges":
2239
+ if sidecar_content_sha256 is None:
2240
+ raise AcquisitionSecurityError(
2241
+ "SANDBOX_RESPONSE", "range response omits sidecar identity"
2242
+ )
2243
+ for name, value in (
2244
+ ("sidecar_size_bytes", sidecar_size_bytes),
2245
+ ("full_object_size_bytes", full_object_size_bytes),
2246
+ ):
2247
+ if type(value) is not int or not 1 <= value <= MAX_BYTE_RANGE_OFFSET + 1:
2248
+ raise AcquisitionSecurityError(
2249
+ "SANDBOX_RESPONSE", f"range response {name} is invalid"
2250
+ )
2251
+ elif (
2252
+ sidecar_content_sha256 is not None
2253
+ or sidecar_size_bytes is not None
2254
+ or full_object_size_bytes is not None
2255
+ ):
2256
+ raise AcquisitionSecurityError(
2257
+ "SANDBOX_RESPONSE", "non-range response carries range identity"
2258
+ )
2259
+ transport_evidence_digest = _optional_digest(response["transport_evidence_digest"])
2260
+ probe_transport = _parse_probe_transport(
2261
+ response["probe_transport"],
2262
+ required=operation in WHOLE_DOCUMENT_RETRIEVAL_OPERATIONS,
2263
+ )
2264
+ if operation == "fetch_ranges":
2265
+ assert fetched_members is not None
2266
+ expected_transport_digest = canonical_sha256(
2267
+ [
2268
+ {
2269
+ key: value
2270
+ for key, value in member.to_dict().items()
2271
+ if key not in {"role", "sequence"}
2272
+ }
2273
+ for member in fetched_members
2274
+ ]
2275
+ )
2276
+ if transport_evidence_digest != expected_transport_digest:
2277
+ raise AcquisitionSecurityError(
2278
+ "SANDBOX_RESPONSE",
2279
+ "range response transport digest does not bind every measured response",
2280
+ )
2281
+ parsed_value = response["parsed"]
2282
+ parsed = _parsed_from_dict(parsed_value) if parsed_value is not None else None
2283
+ content_value = response["content_base64"]
2284
+ content: bytes | None
2285
+ if content_value is None:
2286
+ content = None
2287
+ else:
2288
+ try:
2289
+ content = base64.b64decode(
2290
+ _bounded_text(content_value, maximum=MAX_RESPONSE_BYTES),
2291
+ validate=True,
2292
+ )
2293
+ except (ValueError, TypeError):
2294
+ raise AcquisitionSecurityError(
2295
+ "SANDBOX_RESPONSE",
2296
+ "sandbox response content is not canonical base64",
2297
+ ) from None
2298
+ decode_facts = _parse_decode_facts(response, operation=operation)
2299
+ if decode_facts is not None:
2300
+ # The returned table must describe the returned bytes. The worker parses what it
2301
+ # decoded, so this can only fail for a worker that did something else -- which is
2302
+ # exactly the case a boundary check exists for.
2303
+ if parsed is None or content is None or parsed.input_sha256 != sha256_bytes(content):
2304
+ raise AcquisitionSecurityError(
2305
+ "SANDBOX_DECODE_BINDING",
2306
+ "decode response table does not describe the bytes the decode returned",
2307
+ )
2308
+ family_id, family_version, options_digest, flags, declared_cell_count = decode_facts or (
2309
+ None,
2310
+ None,
2311
+ None,
2312
+ None,
2313
+ None,
2314
+ )
2315
+ fetched_content_sha256 = _optional_digest(response["fetched_content_sha256"])
2316
+ fetched_content_size_bytes = response["fetched_content_size_bytes"]
2317
+ fetched_total_response_body_size_bytes = response["fetched_total_response_body_size_bytes"]
2318
+ fetched_request_count = response["fetched_request_count"]
2319
+ if operation == "retrieve_decode_and_parse":
2320
+ if fetched_content_sha256 is None:
2321
+ raise AcquisitionSecurityError(
2322
+ "SANDBOX_RESPONSE", "Reader retrieval response omits fetched content identity"
2323
+ )
2324
+ if (
2325
+ type(fetched_content_size_bytes) is not int
2326
+ or not 1 <= fetched_content_size_bytes <= 1 << 34
2327
+ ):
2328
+ raise AcquisitionSecurityError(
2329
+ "SANDBOX_RESPONSE", "Reader retrieval response omits fetched content size"
2330
+ )
2331
+ if (
2332
+ type(fetched_total_response_body_size_bytes) is not int
2333
+ or not fetched_content_size_bytes <= fetched_total_response_body_size_bytes <= 1 << 34
2334
+ ):
2335
+ raise AcquisitionSecurityError(
2336
+ "SANDBOX_RESPONSE",
2337
+ "Reader retrieval response omits cumulative response-body size",
2338
+ )
2339
+ if type(fetched_request_count) is not int or not 1 <= fetched_request_count <= 256:
2340
+ raise AcquisitionSecurityError(
2341
+ "SANDBOX_RESPONSE", "Reader retrieval response omits fetched request count"
2342
+ )
2343
+ elif (
2344
+ fetched_content_sha256 is not None
2345
+ or fetched_content_size_bytes is not None
2346
+ or fetched_total_response_body_size_bytes is not None
2347
+ or fetched_request_count is not None
2348
+ ):
2349
+ raise AcquisitionSecurityError(
2350
+ "SANDBOX_RESPONSE",
2351
+ "a non-Reader-retrieval response carries fetched content identity",
2352
+ )
2353
+ return SandboxResult(
2354
+ request_id=request_id,
2355
+ operation=operation,
2356
+ policy_digest=expected_policy,
2357
+ parsed=parsed,
2358
+ content=content,
2359
+ media_type=_optional_text(response["media_type"], 255),
2360
+ final_url=_optional_text(response["final_url"], 2_048),
2361
+ transport_evidence_digest=transport_evidence_digest,
2362
+ visible_environment_keys=visible,
2363
+ probe_transport=probe_transport,
2364
+ home_directory_readable=home_directory_readable,
2365
+ network_probe_errors=network_probe_errors,
2366
+ external_network_policy_attestation=external_network_policy_attestation,
2367
+ decode_family_id=family_id,
2368
+ decode_family_version=family_version,
2369
+ decode_options_digest=options_digest,
2370
+ decode_flags=flags,
2371
+ decode_declared_cell_count=declared_cell_count,
2372
+ fetched_content_sha256=fetched_content_sha256,
2373
+ fetched_content_size_bytes=fetched_content_size_bytes,
2374
+ fetched_total_response_body_size_bytes=fetched_total_response_body_size_bytes,
2375
+ fetched_request_count=fetched_request_count,
2376
+ ranges=ranges,
2377
+ cycle_availability=cycle_availability,
2378
+ sidecar_content_sha256=sidecar_content_sha256,
2379
+ sidecar_size_bytes=sidecar_size_bytes,
2380
+ full_object_size_bytes=full_object_size_bytes,
2381
+ fetched_members=fetched_members,
2382
+ )
2383
+
2384
+
2385
+ def _parse_decode_facts(
2386
+ response: dict[str, Any],
2387
+ *,
2388
+ operation: str,
2389
+ ) -> tuple[str, str, str, tuple[str, ...], int] | None:
2390
+ """Admit the five decode facts, or refuse a response that carries them out of place.
2391
+
2392
+ The flags are checked against the closed ``DECODE_FLAGS`` vocabulary here, at the
2393
+ boundary, rather than trusted onward. A family that could name its own flag could write
2394
+ an unreviewed field into a receipt, so the vocabulary is enforced where the untrusted
2395
+ process's answer arrives and not only where it was produced.
2396
+ """
2397
+
2398
+ stated = (
2399
+ response["decode_family_id"],
2400
+ response["decode_family_version"],
2401
+ response["decode_options_digest"],
2402
+ response["decode_flags"],
2403
+ response["decode_declared_cell_count"],
2404
+ )
2405
+ if operation not in READER_PERFORMING_OPERATIONS:
2406
+ if any(item is not None for item in stated):
2407
+ raise AcquisitionSecurityError(
2408
+ "SANDBOX_RESPONSE",
2409
+ "a response for an operation that decodes nothing carries decode facts",
2410
+ )
2411
+ return None
2412
+ if any(item is None for item in stated):
2413
+ raise AcquisitionSecurityError(
2414
+ "SANDBOX_RESPONSE",
2415
+ "decode response does not carry the decode facts the receipt needs",
2416
+ )
2417
+ family_id = _bounded_text(stated[0], maximum=128)
2418
+ family_version = _bounded_text(stated[1], maximum=32)
2419
+ options_digest = _digest(stated[2])
2420
+ flags = _string_tuple(stated[3], maximum_items=32, item_maximum=64)
2421
+ unknown = sorted(flag for flag in flags if flag not in DECODE_FLAGS)
2422
+ if unknown:
2423
+ raise AcquisitionSecurityError(
2424
+ "SANDBOX_DECODE_FLAGS",
2425
+ f"decode response names no such decode flag: {', '.join(unknown)}",
2426
+ )
2427
+ if tuple(sorted(flags)) != flags:
2428
+ raise AcquisitionSecurityError(
2429
+ "SANDBOX_DECODE_FLAGS",
2430
+ "decode flags must arrive sorted, as the Reader contract produces them",
2431
+ )
2432
+ declared_cell_count = stated[4]
2433
+ if type(declared_cell_count) is not int or not 0 <= declared_cell_count <= (1 << 63) - 1:
2434
+ raise AcquisitionSecurityError(
2435
+ "SANDBOX_RESPONSE",
2436
+ "decode response declared-cell count is invalid",
2437
+ )
2438
+ return family_id, family_version, options_digest, flags, declared_cell_count
2439
+
2440
+
2441
+ def _parse_cycle_availability(
2442
+ value: Any,
2443
+ *,
2444
+ required: bool,
2445
+ expected_labels: tuple[str, ...] | None,
2446
+ ) -> tuple[CycleAvailability, ...] | None:
2447
+ """Check a survey's answers against the exact question the coordinator asked, or refuse.
2448
+
2449
+ Compare labels positionally so availability cannot be attached to a different run.
2450
+ """
2451
+
2452
+ if not required:
2453
+ if value is not None:
2454
+ raise AcquisitionSecurityError(
2455
+ "SANDBOX_RESPONSE",
2456
+ "non-survey response contains availability evidence",
2457
+ )
2458
+ return None
2459
+ if expected_labels is None:
2460
+ raise AcquisitionSecurityError(
2461
+ "SANDBOX_RESPONSE",
2462
+ "a survey response can only be read against the labels that were ordered",
2463
+ )
2464
+ if not isinstance(value, list) or len(value) != len(expected_labels):
2465
+ raise AcquisitionSecurityError(
2466
+ "SANDBOX_RESPONSE",
2467
+ "survey response does not answer the exact candidate set that was ordered",
2468
+ )
2469
+ results: list[CycleAvailability] = []
2470
+ for item, label in zip(value, expected_labels, strict=True):
2471
+ if not isinstance(item, dict) or set(item) != _CYCLE_AVAILABILITY_KEYS:
2472
+ raise AcquisitionSecurityError("SANDBOX_RESPONSE", "survey result shape is invalid")
2473
+ if item["label"] != label:
2474
+ raise AcquisitionSecurityError(
2475
+ "SANDBOX_RESPONSE",
2476
+ "survey results are not the candidates that were ordered, in that order",
2477
+ )
2478
+ present = item["present"]
2479
+ if type(present) is not bool:
2480
+ raise AcquisitionSecurityError(
2481
+ "SANDBOX_RESPONSE",
2482
+ "survey presence must be exactly true or false",
2483
+ )
2484
+ results.append(CycleAvailability(label=label, present=present))
2485
+ return tuple(results)
2486
+
2487
+
2488
+ _PROBE_TRANSPORT_KEYS = frozenset({"http_status_code", "etag_digest", "last_modified_digest"})
2489
+
2490
+
2491
+ def _parse_probe_transport(value: Any, *, required: bool) -> ProbeTransport | None:
2492
+ """Re-admit the terminal-response facts a whole-document retrieval sealed.
2493
+
2494
+ The worker computed both validator digests inside the confinement, so the coordinator never
2495
+ sees a publisher's header text and cannot be handed one here: a value that is not a bare
2496
+ lowercase SHA-256 is refused rather than carried. An operation that settled on no single
2497
+ response must report none, because a status attached to the wrong exchange would be recorded
2498
+ as an observation of the source.
2499
+ """
2500
+
2501
+ if not required:
2502
+ if value is not None:
2503
+ raise AcquisitionSecurityError(
2504
+ "SANDBOX_RESPONSE",
2505
+ "an operation that retrieves no whole document carries probe transport evidence",
2506
+ )
2507
+ return None
2508
+ if not isinstance(value, dict) or set(value) != _PROBE_TRANSPORT_KEYS:
2509
+ raise AcquisitionSecurityError(
2510
+ "SANDBOX_RESPONSE",
2511
+ "whole-document retrieval response omits its exact probe transport evidence",
2512
+ )
2513
+ try:
2514
+ transport = ProbeTransport(
2515
+ http_status_code=value["http_status_code"],
2516
+ etag_digest=_optional_digest(value["etag_digest"]),
2517
+ last_modified_digest=_optional_digest(value["last_modified_digest"]),
2518
+ )
2519
+ except AcquisitionSecurityError as error:
2520
+ raise AcquisitionSecurityError(
2521
+ "SANDBOX_RESPONSE",
2522
+ "probe transport evidence is not the exact sealed shape",
2523
+ ) from error
2524
+ # A retrieval that returned a document returned it with a successful status; the retriever
2525
+ # refuses every other class before a byte is admitted. Pinning the range here rather than
2526
+ # trusting the worker keeps a response that states 404 beside acquired bytes from becoming
2527
+ # an observation whose status contradicts its own content. The hosted Courier pins the
2528
+ # same range on its own acquired path.
2529
+ if not 200 <= transport.http_status_code <= 299:
2530
+ raise AcquisitionSecurityError(
2531
+ "SANDBOX_RESPONSE",
2532
+ "a whole-document retrieval reported a response it could not have admitted",
2533
+ )
2534
+ return transport
2535
+
2536
+
2537
+ def _parse_fetched_members(value: Any, *, required: bool) -> tuple[FetchedMember, ...] | None:
2538
+ """Admit the exact ordered HTTP responses retained by a range acquisition."""
2539
+
2540
+ if not required:
2541
+ if value is not None:
2542
+ raise AcquisitionSecurityError(
2543
+ "SANDBOX_RESPONSE",
2544
+ "non-range response contains fetched-member evidence",
2545
+ )
2546
+ return None
2547
+ if not isinstance(value, list) or not 4 <= len(value) <= 256:
2548
+ raise AcquisitionSecurityError(
2549
+ "SANDBOX_RESPONSE",
2550
+ "range response must retain a bounded complete fetched-member list",
2551
+ )
2552
+ members: list[FetchedMember] = []
2553
+ roles: list[str] = []
2554
+ for sequence, item in enumerate(value):
2555
+ if not isinstance(item, dict) or set(item) != _FETCHED_MEMBER_KEYS:
2556
+ raise AcquisitionSecurityError("SANDBOX_RESPONSE", "fetched-member shape is invalid")
2557
+ if item["sequence"] != sequence:
2558
+ raise AcquisitionSecurityError(
2559
+ "SANDBOX_RESPONSE", "fetched members are not in exact response order"
2560
+ )
2561
+ role = item["role"]
2562
+ if role not in _FETCHED_MEMBER_ROLES:
2563
+ raise AcquisitionSecurityError("SANDBOX_RESPONSE", "fetched-member role is invalid")
2564
+ status = item["status"]
2565
+ body_size = item["response_body_size_bytes"]
2566
+ if type(status) is not int or not 100 <= status <= 599:
2567
+ raise AcquisitionSecurityError("SANDBOX_RESPONSE", "fetched-member status is invalid")
2568
+ if type(body_size) is not int or not 0 <= body_size <= MAX_RESPONSE_BYTES:
2569
+ raise AcquisitionSecurityError(
2570
+ "SANDBOX_RESPONSE", "fetched-member body size is invalid"
2571
+ )
2572
+ member = FetchedMember(
2573
+ role=role,
2574
+ sequence=sequence,
2575
+ url=_bounded_text(item["url"], maximum=2_048),
2576
+ resolution_digest=_digest(item["resolution_digest"]),
2577
+ connected_peer=_bounded_text(item["connected_peer"], maximum=64),
2578
+ status=status,
2579
+ response_headers_digest=_digest(item["response_headers_digest"]),
2580
+ response_body_size_bytes=body_size,
2581
+ response_body_sha256=_digest(item["response_body_sha256"]),
2582
+ requested_range=_optional_text(item["requested_range"], 128),
2583
+ observed_content_range=_optional_text(item["observed_content_range"], 128),
2584
+ etag=_optional_text(item["etag"], 1_024),
2585
+ last_modified=_optional_text(item["last_modified"], 1_024),
2586
+ peer_attempts=_parse_peer_attempts(item["peer_attempts"]),
2587
+ )
2588
+ if member.peer_attempts[-1].response_body_size_bytes != body_size:
2589
+ raise AcquisitionSecurityError(
2590
+ "SANDBOX_RESPONSE",
2591
+ "terminal peer-attempt bytes differ from the retained response body",
2592
+ )
2593
+ members.append(member)
2594
+ roles.append(role)
2595
+ # Redirects may repeat a role; the four acquisition stages remain ordered and complete.
2596
+ collapsed = tuple(
2597
+ role for index, role in enumerate(roles) if index == 0 or role != roles[index - 1]
2598
+ )
2599
+ if collapsed != ("sidecar_initial", "cycle_probe", "selected_slice", "sidecar_final"):
2600
+ raise AcquisitionSecurityError(
2601
+ "SANDBOX_RESPONSE",
2602
+ "fetched members do not contain the complete ordered range acquisition",
2603
+ )
2604
+ return tuple(members)
2605
+
2606
+
2607
+ def _parse_peer_attempts(value: Any) -> tuple[PeerAttempt, ...]:
2608
+ """Re-admit the closed, ordered peer evidence carried by one retrieval hop."""
2609
+
2610
+ if not isinstance(value, list) or not 1 <= len(value) <= 32:
2611
+ raise AcquisitionSecurityError(
2612
+ "SANDBOX_RESPONSE",
2613
+ "fetched-member peer attempts must be a non-empty bounded list",
2614
+ )
2615
+ attempts: list[PeerAttempt] = []
2616
+ saw_response = False
2617
+ for index, item in enumerate(value):
2618
+ if not isinstance(item, dict) or set(item) != _PEER_ATTEMPT_KEYS:
2619
+ raise AcquisitionSecurityError(
2620
+ "SANDBOX_RESPONSE", "fetched-member peer-attempt shape is invalid"
2621
+ )
2622
+ approved_ip = _bounded_text(item["approved_ip"], maximum=64)
2623
+ try:
2624
+ address = ipaddress.ip_address(approved_ip)
2625
+ except ValueError as error:
2626
+ raise AcquisitionSecurityError(
2627
+ "SANDBOX_RESPONSE", "fetched-member peer-attempt address is invalid"
2628
+ ) from error
2629
+ if str(address) != approved_ip or not address.is_global:
2630
+ raise AcquisitionSecurityError(
2631
+ "SANDBOX_RESPONSE", "fetched-member peer-attempt address is not canonical public IP"
2632
+ )
2633
+ outcome = item["outcome"]
2634
+ failure_code = item["failure_code"]
2635
+ response_body_size_bytes = item["response_body_size_bytes"]
2636
+ if (
2637
+ type(response_body_size_bytes) is not int
2638
+ or not 0 <= response_body_size_bytes <= (1 << 34) + 1
2639
+ ):
2640
+ raise AcquisitionSecurityError(
2641
+ "SANDBOX_RESPONSE",
2642
+ "fetched-member peer-attempt byte evidence is invalid",
2643
+ )
2644
+ if outcome not in _PEER_ATTEMPT_OUTCOMES:
2645
+ raise AcquisitionSecurityError(
2646
+ "SANDBOX_RESPONSE", "fetched-member peer-attempt outcome is invalid"
2647
+ )
2648
+ if outcome in {"transport_failure", "response_failure"}:
2649
+ failure_code = _bounded_text(failure_code, maximum=128)
2650
+ if saw_response:
2651
+ raise AcquisitionSecurityError(
2652
+ "SANDBOX_RESPONSE", "peer attempts continue after a successful response"
2653
+ )
2654
+ if outcome == "response_failure" and index != len(value) - 1:
2655
+ raise AcquisitionSecurityError(
2656
+ "SANDBOX_RESPONSE", "a response failure must terminate peer attempts"
2657
+ )
2658
+ elif failure_code is not None or index != len(value) - 1:
2659
+ raise AcquisitionSecurityError(
2660
+ "SANDBOX_RESPONSE", "successful peer attempt must terminate the attempt list"
2661
+ )
2662
+ else:
2663
+ saw_response = True
2664
+ attempts.append(
2665
+ PeerAttempt(
2666
+ approved_ip=approved_ip,
2667
+ outcome=outcome,
2668
+ failure_code=failure_code,
2669
+ response_body_size_bytes=response_body_size_bytes,
2670
+ )
2671
+ )
2672
+ if not saw_response:
2673
+ raise AcquisitionSecurityError(
2674
+ "SANDBOX_RESPONSE", "successful fetched member has no successful peer attempt"
2675
+ )
2676
+ return tuple(attempts)
2677
+
2678
+
2679
+ def _parse_fetched_slices(
2680
+ value: Any,
2681
+ *,
2682
+ required: bool,
2683
+ expected_selectors: tuple[str, ...] | None = None,
2684
+ expected_reference_time: str | None = None,
2685
+ ) -> tuple[FetchedSlice, ...] | None:
2686
+ """Re-check every delivered slice on the trusted side of the boundary, or refuse it.
2687
+
2688
+ The worker already proved each slice's span and structure. This is not that check again:
2689
+ it is the coordinator refusing to build a result out of anything the response says that
2690
+ the response's own bytes do not support. Each slice's digest is recomputed, each span is
2691
+ compared against the bytes that claim it, the sequence must ascend, and every slice must
2692
+ still be a well-formed message -- so the ordering and structural properties this operation
2693
+ promises are facts the coordinator verified rather than ones the worker asserted.
2694
+
2695
+ ``expected_selectors`` is the exact set of descriptors the invocation ordered, which the
2696
+ coordinator knows from its own request. One selector yields exactly one slice, so the
2697
+ response must answer every selector exactly once: a response carrying fewer is a partial
2698
+ acquisition, one carrying more is a set nobody ordered, and one answering the same
2699
+ descriptor twice is not the order that was placed. This is the count check it replaces
2700
+ and more -- the count was the weakest fact the coordinator held, and holding the answer
2701
+ to the question instead makes each delivered slice say which descriptor it answers.
2702
+ ``plan_ranges`` matches by exact equality, so a descriptor that is not one of these is a
2703
+ descriptor no selector named. The exact spans still cannot be checked here -- they come
2704
+ from the sidecar, which only the worker fetched -- and neither can the descriptor-to-span
2705
+ mapping, which is the sidecar's and is the documented known gap.
2706
+
2707
+ ``expected_reference_time`` is the model run the invocation ordered, when it ordered one.
2708
+ It is re-checked here for the same reason the structural check is: the worker is the process
2709
+ that handled the untrusted bytes, so a run binding that only ever ran there is a binding the
2710
+ coordinator is taking on trust. When the order named no run, no run binding is claimed.
2711
+ """
2712
+
2713
+ if not required:
2714
+ if value is not None:
2715
+ raise AcquisitionSecurityError(
2716
+ "SANDBOX_RESPONSE",
2717
+ "non-slice response contains slice evidence",
2718
+ )
2719
+ return None
2720
+ if expected_selectors is None:
2721
+ raise AcquisitionSecurityError(
2722
+ "SANDBOX_RESPONSE",
2723
+ "a slice response can only be read against the plan that was ordered",
2724
+ )
2725
+ if not isinstance(value, list) or not 1 <= len(value) <= MAX_RANGE_PLAN:
2726
+ raise AcquisitionSecurityError(
2727
+ "SANDBOX_RESPONSE",
2728
+ "slice response is not a bounded non-empty list",
2729
+ )
2730
+ if len(value) != len(expected_selectors):
2731
+ raise AcquisitionSecurityError(
2732
+ "SANDBOX_RESPONSE",
2733
+ "slice response does not answer the exact plan that was ordered",
2734
+ )
2735
+ unanswered = list(expected_selectors)
2736
+ slices: list[FetchedSlice] = []
2737
+ for item in value:
2738
+ if not isinstance(item, dict) or set(item) != _FETCHED_SLICE_KEYS:
2739
+ raise AcquisitionSecurityError("SANDBOX_RESPONSE", "slice shape is invalid")
2740
+ index = item["index"]
2741
+ first_byte = item["first_byte"]
2742
+ last_byte = item["last_byte"]
2743
+ if type(index) is not int or index < 1:
2744
+ raise AcquisitionSecurityError("SANDBOX_RESPONSE", "slice index is invalid")
2745
+ descriptor = _bounded_text(item["descriptor"], maximum=MAX_SELECTOR_BYTES)
2746
+ if descriptor not in unanswered:
2747
+ # Either a descriptor no selector named, or one named once and answered twice.
2748
+ # Neither is the order that was placed, and both would put a slice on the receipt
2749
+ # under a label the coordinator never asked for.
2750
+ raise AcquisitionSecurityError(
2751
+ "SANDBOX_RESPONSE",
2752
+ "a delivered slice does not answer an unanswered selector that was ordered",
2753
+ )
2754
+ unanswered.remove(descriptor)
2755
+ if (
2756
+ type(first_byte) is not int
2757
+ or type(last_byte) is not int
2758
+ or not 0 <= first_byte <= last_byte
2759
+ ):
2760
+ raise AcquisitionSecurityError("SANDBOX_RESPONSE", "slice span is invalid")
2761
+ digest = _digest(item["sha256"])
2762
+ try:
2763
+ content = base64.b64decode(
2764
+ _bounded_text(item["content_base64"], maximum=MAX_RESPONSE_BYTES),
2765
+ validate=True,
2766
+ )
2767
+ except (ValueError, TypeError):
2768
+ raise AcquisitionSecurityError(
2769
+ "SANDBOX_RESPONSE",
2770
+ "slice content is not canonical base64",
2771
+ ) from None
2772
+ if len(content) != last_byte - first_byte + 1:
2773
+ raise AcquisitionSecurityError(
2774
+ "SANDBOX_RESPONSE",
2775
+ "slice bytes do not fill the span they name",
2776
+ )
2777
+ if sha256_bytes(content) != digest:
2778
+ raise AcquisitionSecurityError(
2779
+ "SANDBOX_RESPONSE",
2780
+ "slice digest does not match its own bytes",
2781
+ )
2782
+ # The structural check again, on this side of the boundary. The worker runs it before
2783
+ # it keeps a slice, but the worker is the process that handled the untrusted bytes; a
2784
+ # check that only ever ran there is a check the coordinator is taking on trust. It is
2785
+ # pure, it is cheap, and re-running it is the difference between "the worker says these
2786
+ # are messages" and "these are messages".
2787
+ try:
2788
+ check_grib_message(content)
2789
+ except AcquisitionSecurityError as exc:
2790
+ raise AcquisitionSecurityError(
2791
+ "SANDBOX_RESPONSE",
2792
+ f"slice bytes are not a well-formed message ({exc.code})",
2793
+ ) from None
2794
+ if expected_reference_time is not None:
2795
+ try:
2796
+ check_grib_reference_time(content, expected=expected_reference_time)
2797
+ except AcquisitionSecurityError as exc:
2798
+ raise AcquisitionSecurityError(
2799
+ "SANDBOX_RESPONSE",
2800
+ f"slice bytes are not from the model run that was ordered ({exc.code})",
2801
+ ) from None
2802
+ if slices and index <= slices[-1].index:
2803
+ raise AcquisitionSecurityError(
2804
+ "SANDBOX_RESPONSE",
2805
+ "slices are not emitted in ascending message-index order",
2806
+ )
2807
+ slices.append(
2808
+ FetchedSlice(
2809
+ index=index,
2810
+ descriptor=descriptor,
2811
+ first_byte=first_byte,
2812
+ last_byte=last_byte,
2813
+ sha256=digest,
2814
+ content=content,
2815
+ )
2816
+ )
2817
+ return tuple(slices)
2818
+
2819
+
2820
+ def _parsed_dict(parsed: ParsedTable) -> dict[str, Any]:
2821
+ return {
2822
+ "data_format": parsed.data_format,
2823
+ "input_sha256": parsed.input_sha256,
2824
+ "columns": list(parsed.columns),
2825
+ "rows": [list(row) for row in parsed.rows],
2826
+ "schema_digest": parsed.schema_digest,
2827
+ }
2828
+
2829
+
2830
+ def _parsed_from_dict(value: Any) -> ParsedTable:
2831
+ if not isinstance(value, dict) or set(value) != {
2832
+ "data_format",
2833
+ "input_sha256",
2834
+ "columns",
2835
+ "rows",
2836
+ "schema_digest",
2837
+ }:
2838
+ raise AcquisitionSecurityError("SANDBOX_RESPONSE", "parsed response shape is invalid")
2839
+ columns = _string_tuple(value["columns"], maximum_items=10_000, item_maximum=256)
2840
+ if not isinstance(value["rows"], list) or len(value["rows"]) > 10_000_000:
2841
+ raise AcquisitionSecurityError("SANDBOX_RESPONSE", "parsed response rows are invalid")
2842
+ rows: list[tuple[Any, ...]] = []
2843
+ for row in value["rows"]:
2844
+ if not isinstance(row, list):
2845
+ raise AcquisitionSecurityError("SANDBOX_RESPONSE", "parsed row is not an array")
2846
+ rows.append(tuple(row))
2847
+ return ParsedTable(
2848
+ data_format=_bounded_text(value["data_format"], maximum=16),
2849
+ input_sha256=_digest(value["input_sha256"]),
2850
+ columns=columns,
2851
+ rows=tuple(rows),
2852
+ schema_digest=_digest(value["schema_digest"]),
2853
+ )
2854
+
2855
+
2856
+ def _strict_json_object(raw: bytes, path: str) -> dict[str, Any]:
2857
+ try:
2858
+ value = json.loads(raw, object_pairs_hook=_unique_object, parse_constant=_reject_constant)
2859
+ except (UnicodeDecodeError, json.JSONDecodeError):
2860
+ raise AcquisitionSecurityError("SANDBOX_JSON", f"{path} is not strict JSON") from None
2861
+ if not isinstance(value, dict):
2862
+ raise AcquisitionSecurityError("SANDBOX_JSON", f"{path} must be an object")
2863
+ return value
2864
+
2865
+
2866
+ def _unique_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
2867
+ result: dict[str, Any] = {}
2868
+ for key, value in pairs:
2869
+ if key in result:
2870
+ raise AcquisitionSecurityError("SANDBOX_JSON", "JSON keys must be unique")
2871
+ result[key] = value
2872
+ return result
2873
+
2874
+
2875
+ def _reject_constant(value: str) -> Any:
2876
+ raise AcquisitionSecurityError("SANDBOX_JSON", f"non-finite number {value!r} is forbidden")
2877
+
2878
+
2879
+ def _parse_limits_dict(limits: ParseLimits) -> dict[str, int]:
2880
+ return {
2881
+ "max_input_bytes": limits.max_input_bytes,
2882
+ "max_uncompressed_bytes": limits.max_uncompressed_bytes,
2883
+ "max_expansion_ratio": limits.max_expansion_ratio,
2884
+ "max_rows": limits.max_rows,
2885
+ "max_columns": limits.max_columns,
2886
+ "max_field_bytes": limits.max_field_bytes,
2887
+ "max_json_depth": limits.max_json_depth,
2888
+ "max_total_cells": limits.max_total_cells,
2889
+ }
2890
+
2891
+
2892
+ def _parse_parse_limits(value: Any) -> ParseLimits:
2893
+ if not isinstance(value, dict) or set(value) != set(_parse_limits_dict(ParseLimits())):
2894
+ raise AcquisitionSecurityError("SANDBOX_LIMITS", "parse limits shape is invalid")
2895
+ return ParseLimits(**value)
2896
+
2897
+
2898
+ def _retrieval_limits_dict(limits: RetrievalLimits) -> dict[str, Any]:
2899
+ return {
2900
+ "max_response_bytes": limits.max_response_bytes,
2901
+ "max_aggregate_response_bytes": limits.max_aggregate_response_bytes,
2902
+ "max_redirects": limits.max_redirects,
2903
+ "max_requests": limits.max_requests,
2904
+ # Carried across the boundary like every other bound. Left out, a coordinator that
2905
+ # narrowed the range budget would have the worker silently rebuild the default, and
2906
+ # the plan's own budget would be checked against a ceiling nobody chose.
2907
+ "max_ranges": limits.max_ranges,
2908
+ "connect_timeout_seconds": limits.connect_timeout_seconds,
2909
+ "read_timeout_seconds": limits.read_timeout_seconds,
2910
+ "total_timeout_seconds": limits.total_timeout_seconds,
2911
+ "max_concurrency": limits.max_concurrency,
2912
+ "min_interval_seconds": limits.min_interval_seconds,
2913
+ "allowed_media_types": list(limits.allowed_media_types),
2914
+ "user_agent": limits.user_agent,
2915
+ }
2916
+
2917
+
2918
+ def _parse_retrieval_limits(value: Any) -> RetrievalLimits:
2919
+ if not isinstance(value, dict) or set(value) != set(_retrieval_limits_dict(RetrievalLimits())):
2920
+ raise AcquisitionSecurityError("SANDBOX_LIMITS", "retrieval limits shape is invalid")
2921
+ copied = dict(value)
2922
+ copied["allowed_media_types"] = _string_tuple(
2923
+ copied["allowed_media_types"],
2924
+ maximum_items=16,
2925
+ item_maximum=255,
2926
+ )
2927
+ return RetrievalLimits(**copied)
2928
+
2929
+
2930
+ @dataclass(frozen=True)
2931
+ class _ApprovedEndpointResolver:
2932
+ endpoints: dict[tuple[str, int], tuple[str, ...]]
2933
+
2934
+ def resolve(self, hostname: str, port: int) -> tuple[str, ...]:
2935
+ try:
2936
+ return self.endpoints[(hostname, port)]
2937
+ except KeyError as exc:
2938
+ raise AcquisitionSecurityError(
2939
+ "SANDBOX_EGRESS",
2940
+ "worker attempted an endpoint outside the coordinator DNS pin",
2941
+ ) from exc
2942
+
2943
+
2944
+ def _resolve_approved_endpoints(
2945
+ url: str,
2946
+ allowed_hostnames: tuple[str, ...],
2947
+ ) -> tuple[tuple[str, int, tuple[str, ...]], ...]:
2948
+ resolver = SystemResolver()
2949
+ policy = EgressPolicy(allowed_hostnames=allowed_hostnames)
2950
+ initial = validate_public_https_url(url, resolver=resolver, egress_policy=policy)
2951
+ resolved: dict[str, tuple[str, ...]] = {initial.hostname: initial.approved_ips}
2952
+ for hostname in allowed_hostnames:
2953
+ if hostname not in resolved:
2954
+ target = validate_public_https_url(
2955
+ _https_root_url(hostname),
2956
+ resolver=resolver,
2957
+ egress_policy=policy,
2958
+ )
2959
+ resolved[hostname] = target.approved_ips
2960
+ return tuple((hostname, 443, resolved[hostname]) for hostname in sorted(resolved))
2961
+
2962
+
2963
+ def _parse_approved_endpoints(
2964
+ value: Any,
2965
+ *,
2966
+ allowed_hostnames: tuple[str, ...],
2967
+ ) -> _ApprovedEndpointResolver:
2968
+ if not isinstance(value, list) or len(value) != len(allowed_hostnames):
2969
+ raise AcquisitionSecurityError(
2970
+ "SANDBOX_EGRESS",
2971
+ "approved endpoint set does not cover the exact hostname allowlist",
2972
+ )
2973
+ endpoints: dict[tuple[str, int], tuple[str, ...]] = {}
2974
+ for item in value:
2975
+ if not isinstance(item, dict) or set(item) != {
2976
+ "hostname",
2977
+ "port",
2978
+ "approved_ips",
2979
+ }:
2980
+ raise AcquisitionSecurityError(
2981
+ "SANDBOX_EGRESS",
2982
+ "approved endpoint shape is invalid",
2983
+ )
2984
+ hostname = _bounded_text(item["hostname"], maximum=253)
2985
+ port = item["port"]
2986
+ if hostname not in allowed_hostnames or type(port) is not int or port != 443:
2987
+ raise AcquisitionSecurityError(
2988
+ "SANDBOX_EGRESS",
2989
+ "approved endpoint is outside the hostname/port authority",
2990
+ )
2991
+ addresses = _string_tuple(
2992
+ item["approved_ips"],
2993
+ maximum_items=32,
2994
+ item_maximum=64,
2995
+ )
2996
+ if not addresses:
2997
+ raise AcquisitionSecurityError("SANDBOX_EGRESS", "approved IP set is empty")
2998
+ normalized: list[str] = []
2999
+ for raw in addresses:
3000
+ try:
3001
+ address = ipaddress.ip_address(raw)
3002
+ except ValueError:
3003
+ raise AcquisitionSecurityError(
3004
+ "SANDBOX_EGRESS",
3005
+ "approved endpoint contains a non-IP address",
3006
+ ) from None
3007
+ if not address.is_global:
3008
+ raise AcquisitionSecurityError(
3009
+ "SANDBOX_EGRESS",
3010
+ "approved endpoint contains a non-public address",
3011
+ )
3012
+ normalized.append(address.compressed)
3013
+ key = (hostname, port)
3014
+ if key in endpoints or tuple(sorted(normalized)) != addresses:
3015
+ raise AcquisitionSecurityError(
3016
+ "SANDBOX_EGRESS",
3017
+ "approved endpoint set is duplicate or non-canonical",
3018
+ )
3019
+ endpoints[key] = addresses
3020
+ if {hostname for hostname, _port in endpoints} != set(allowed_hostnames):
3021
+ raise AcquisitionSecurityError(
3022
+ "SANDBOX_EGRESS",
3023
+ "approved endpoint set differs from the hostname allowlist",
3024
+ )
3025
+ return _ApprovedEndpointResolver(endpoints)
3026
+
3027
+
3028
+ def _https_root_url(hostname: str) -> str:
3029
+ try:
3030
+ address = ipaddress.ip_address(hostname)
3031
+ except ValueError:
3032
+ address = None
3033
+ host = f"[{hostname}]" if isinstance(address, ipaddress.IPv6Address) else hostname
3034
+ return f"https://{host}/"
3035
+
3036
+
3037
+ def _parse_network_probe_targets(
3038
+ request: dict[str, Any],
3039
+ ) -> tuple[tuple[str, str, int], ...]:
3040
+ value = request["network_probe_targets"]
3041
+ if not isinstance(value, list) or len(value) != 3:
3042
+ raise AcquisitionSecurityError(
3043
+ "SANDBOX_PROBE",
3044
+ "probe must contain the exact three network-denial targets",
3045
+ )
3046
+ parsed: dict[str, tuple[str, int]] = {}
3047
+ for item in value:
3048
+ if not isinstance(item, dict) or set(item) != {"label", "ip", "port"}:
3049
+ raise AcquisitionSecurityError("SANDBOX_PROBE", "network probe shape is invalid")
3050
+ label = _bounded_text(item["label"], maximum=16)
3051
+ ip = _bounded_text(item["ip"], maximum=64)
3052
+ port = item["port"]
3053
+ if type(port) is not int or not 1 <= port <= 65_535 or label in parsed:
3054
+ raise AcquisitionSecurityError("SANDBOX_PROBE", "network probe target is invalid")
3055
+ parsed[label] = (ip, port)
3056
+ if set(parsed) != {"loopback", "private", "metadata"}:
3057
+ raise AcquisitionSecurityError(
3058
+ "SANDBOX_PROBE",
3059
+ "network probe labels are invalid",
3060
+ )
3061
+ if (
3062
+ parsed["loopback"][0] != "127.0.0.1"
3063
+ or parsed["private"] != ("10.0.0.1", 443)
3064
+ or parsed["metadata"] != ("169.254.169.254", 80)
3065
+ ):
3066
+ raise AcquisitionSecurityError(
3067
+ "SANDBOX_PROBE",
3068
+ "network probe destinations are invalid",
3069
+ )
3070
+ return tuple((label, *parsed[label]) for label in ("loopback", "private", "metadata"))
3071
+
3072
+
3073
+ def _probe_network_denials(
3074
+ targets: tuple[tuple[str, str, int], ...],
3075
+ ) -> tuple[tuple[str, int], ...]:
3076
+ results: list[tuple[str, int]] = []
3077
+ for label, ip, port in targets:
3078
+ family = socket.AF_INET6 if ":" in ip else socket.AF_INET
3079
+ probe = socket.socket(family, socket.SOCK_STREAM)
3080
+ try:
3081
+ probe.settimeout(0.25)
3082
+ error = probe.connect_ex((ip, port))
3083
+ except OSError as exc:
3084
+ error = exc.errno or 1
3085
+ finally:
3086
+ probe.close()
3087
+ results.append((label, error))
3088
+ return tuple(results)
3089
+
3090
+
3091
+ def _parse_network_probe_response(
3092
+ value: Any,
3093
+ *,
3094
+ required: bool,
3095
+ ) -> tuple[tuple[str, int], ...] | None:
3096
+ if not required:
3097
+ if value is not None:
3098
+ raise AcquisitionSecurityError(
3099
+ "SANDBOX_RESPONSE",
3100
+ "non-probe response contains network probe evidence",
3101
+ )
3102
+ return None
3103
+ if not isinstance(value, list) or len(value) != 3:
3104
+ raise AcquisitionSecurityError(
3105
+ "SANDBOX_OS_BOUNDARY",
3106
+ "sandbox did not return complete network-denial evidence",
3107
+ )
3108
+ result: list[tuple[str, int]] = []
3109
+ for item in value:
3110
+ if not isinstance(item, dict) or set(item) != {"label", "errno"}:
3111
+ raise AcquisitionSecurityError(
3112
+ "SANDBOX_OS_BOUNDARY",
3113
+ "sandbox network-denial evidence is invalid",
3114
+ )
3115
+ label = item["label"]
3116
+ error = item["errno"]
3117
+ if label not in {"loopback", "private", "metadata"} or type(error) is not int or error <= 0:
3118
+ raise AcquisitionSecurityError(
3119
+ "SANDBOX_OS_BOUNDARY",
3120
+ "sandbox completed a forbidden direct socket attempt",
3121
+ )
3122
+ result.append((label, error))
3123
+ if {label for label, _error in result} != {"loopback", "private", "metadata"}:
3124
+ raise AcquisitionSecurityError(
3125
+ "SANDBOX_OS_BOUNDARY",
3126
+ "sandbox network-denial evidence is incomplete",
3127
+ )
3128
+ return tuple(result)
3129
+
3130
+
3131
+ def _require_null_request_fields(request: dict[str, Any]) -> None:
3132
+ for key in (
3133
+ "content_base64",
3134
+ "data_format",
3135
+ "media_type",
3136
+ "filename",
3137
+ "parse_limits",
3138
+ "url",
3139
+ "approved_endpoints",
3140
+ "retrieval_limits",
3141
+ "reader_pin",
3142
+ "reader_budgets",
3143
+ "range_plan",
3144
+ "cycle_survey",
3145
+ ):
3146
+ expected = [] if key == "approved_endpoints" else None
3147
+ if request[key] != expected:
3148
+ raise AcquisitionSecurityError(
3149
+ "SANDBOX_FIELDS",
3150
+ "probe request must not carry acquisition inputs",
3151
+ )
3152
+ if request["allowed_hostnames"] != []:
3153
+ raise AcquisitionSecurityError(
3154
+ "SANDBOX_FIELDS",
3155
+ "probe request must not carry egress authority",
3156
+ )
3157
+
3158
+
3159
+ def _validate_egress_hosts(hostnames: tuple[str, ...]) -> None:
3160
+ if not hostnames or len(hostnames) > 32 or len(set(hostnames)) != len(hostnames):
3161
+ raise AcquisitionSecurityError(
3162
+ "SANDBOX_EGRESS",
3163
+ "sandbox egress hostnames must be non-empty, unique, and bounded",
3164
+ )
3165
+ for hostname in hostnames:
3166
+ normalized = hostname.rstrip(".").lower()
3167
+ if (
3168
+ normalized != hostname
3169
+ or normalized in _FORBIDDEN_EGRESS_HOSTS
3170
+ or normalized.endswith(".internal")
3171
+ ):
3172
+ raise AcquisitionSecurityError(
3173
+ "SANDBOX_EGRESS",
3174
+ "sandbox egress includes a forbidden production or metadata host",
3175
+ )
3176
+
3177
+
3178
+ def _policy_document(operation: str) -> dict[str, Any]:
3179
+ """Build the exact policy mapping whose digest binds a worker to its sandbox policy.
3180
+
3181
+ Split out from ``_policy_digest`` so a test can assert the document's shape directly --
3182
+ in particular that a non-parsing operation carries no ``parser_formats`` key -- rather
3183
+ than inferring shape from an opaque digest.
3184
+ """
3185
+
3186
+ policy: dict[str, Any] = {
3187
+ "schema_version": SANDBOX_PROTOCOL_VERSION,
3188
+ "operation": operation,
3189
+ "environment_keys": sorted(_ALLOWED_ENVIRONMENT),
3190
+ "ambient_credentials": False,
3191
+ "ambient_proxy": False,
3192
+ "studio_clients": False,
3193
+ # One value since ADR 0021. The other arm named the Seatbelt profile, which no
3194
+ # longer exists; this is the clause every non-darwin host already attested, so no
3195
+ # digest a Linux worker ever produced moves.
3196
+ "filesystem": "external-container-boundary-required",
3197
+ # Driven by the named set rather than by a comparison against one operation name, so
3198
+ # an operation cannot be given live socket authority in ``_invoke`` while its own
3199
+ # attestation keeps claiming the network is denied to it.
3200
+ # The word "seatbelt" in this constant outlived the Seatbelt backend on purpose. This
3201
+ # string is not a description that can be freshened -- it is digest input, and every
3202
+ # sealed attestation, golden, and coordinator pin in the repository and in already
3203
+ # released evidence binds the digest it produces. Renaming it would rotate four
3204
+ # operation digests to improve a label. It reads today as the historical name of the
3205
+ # approved-public-IP-plus-TCP/443 posture, which is still exactly the posture.
3206
+ "network": (
3207
+ "external-deny-default-approved-public-ip-plus-seatbelt-tcp-443"
3208
+ if operation in NETWORK_PERFORMING_OPERATIONS
3209
+ else "os-denied"
3210
+ ),
3211
+ }
3212
+ # Scoped per operation, following the pattern ``filesystem`` and ``network`` already use
3213
+ # above. An operation absent from the registry gets no key at all rather than an empty
3214
+ # list: absent parsing support produces no key. ``canonical_sha256`` sorts keys, so insertion
3215
+ # conditionally cannot perturb the digest by ordering.
3216
+ parser_formats = PARSER_FORMATS_BY_OPERATION.get(operation)
3217
+ if parser_formats is not None:
3218
+ policy["parser_formats"] = list(parser_formats)
3219
+ # The Reader contract label, present only on the operation that may run a Reader, in the
3220
+ # same conditional register the two clauses above use.
3221
+ #
3222
+ # It is the contract label and deliberately not the list of admitted families. Embedding
3223
+ # the family ids would make every family registration rotate this digest and require a
3224
+ # coordinator-configuration change -- for no security gain, because the family that ran is
3225
+ # already bound per source by three independent checks: the pin inside ``request.digest``,
3226
+ # the adapter-side cross-check in ``sources/_adapter_steps.py``, and
3227
+ # ``recipe._validate_reader_pin_binding``. What the label actually versions is the shape
3228
+ # of that authority, which is the thing an attestation should be bound to.
3229
+ if operation in READER_PERFORMING_OPERATIONS:
3230
+ policy["reader_contract"] = READER_CONTRACT_VERSION
3231
+ return policy
3232
+
3233
+
3234
+ def _policy_digest(operation: str) -> str:
3235
+ return canonical_sha256(_policy_document(operation))
3236
+
3237
+
3238
+ def _home_directory_readable() -> bool:
3239
+ try:
3240
+ next(Path.home().iterdir(), None)
3241
+ except OSError:
3242
+ return False
3243
+ return True
3244
+
3245
+
3246
+ class _WorkerProcess(Protocol):
3247
+ """The small process surface needed by the bounded IPC pump."""
3248
+
3249
+ pid: int
3250
+ stdin_fd: int
3251
+ stdout_fd: int
3252
+ stderr_fd: int
3253
+
3254
+ def close_stdin(self) -> None: ...
3255
+
3256
+ def close_output(self, descriptor: int) -> None: ...
3257
+
3258
+ def wait(self, timeout: float) -> int: ...
3259
+
3260
+ def kill_group(self) -> None: ...
3261
+
3262
+
3263
+ class _PopenWorker:
3264
+ def __init__(self, process: subprocess.Popen[bytes]) -> None:
3265
+ if process.stdin is None or process.stdout is None or process.stderr is None:
3266
+ raise RuntimeError("Clean room process pipes were not created")
3267
+ self._process = process
3268
+ self.pid = process.pid
3269
+ self.stdin_fd = process.stdin.fileno()
3270
+ self.stdout_fd = process.stdout.fileno()
3271
+ self.stderr_fd = process.stderr.fileno()
3272
+
3273
+ def close_stdin(self) -> None:
3274
+ if self._process.stdin is not None and not self._process.stdin.closed:
3275
+ self._process.stdin.close()
3276
+
3277
+ def wait(self, timeout: float) -> int:
3278
+ return self._process.wait(timeout=max(timeout, 0.0))
3279
+
3280
+ def close_output(self, descriptor: int) -> None:
3281
+ for stream in (self._process.stdout, self._process.stderr):
3282
+ if stream is not None and not stream.closed and stream.fileno() == descriptor:
3283
+ stream.close()
3284
+
3285
+ def kill_group(self) -> None:
3286
+ try:
3287
+ os.killpg(self.pid, signal.SIGKILL)
3288
+ except ProcessLookupError:
3289
+ pass
3290
+
3291
+
3292
+ class _ForkedWorker:
3293
+ def __init__(
3294
+ self,
3295
+ *,
3296
+ pid: int,
3297
+ stdin_fd: int,
3298
+ stdout_fd: int,
3299
+ stderr_fd: int,
3300
+ ) -> None:
3301
+ self.pid = pid
3302
+ self.stdin_fd = stdin_fd
3303
+ self.stdout_fd = stdout_fd
3304
+ self.stderr_fd = stderr_fd
3305
+ self._returncode: int | None = None
3306
+
3307
+ def close_stdin(self) -> None:
3308
+ if self.stdin_fd >= 0:
3309
+ try:
3310
+ os.close(self.stdin_fd)
3311
+ except OSError:
3312
+ pass
3313
+ self.stdin_fd = -1
3314
+
3315
+ def wait(self, timeout: float) -> int:
3316
+ if self._returncode is not None:
3317
+ return self._returncode
3318
+ deadline = time.monotonic() + max(timeout, 0.0)
3319
+ while True:
3320
+ pid, status = os.waitpid(self.pid, os.WNOHANG)
3321
+ if pid == self.pid:
3322
+ self._returncode = os.waitstatus_to_exitcode(status)
3323
+ return self._returncode
3324
+ if time.monotonic() >= deadline:
3325
+ raise subprocess.TimeoutExpired(("sandbox-worker",), timeout)
3326
+ time.sleep(0.005)
3327
+
3328
+ def close_output(self, descriptor: int) -> None:
3329
+ if descriptor == self.stdout_fd:
3330
+ self.stdout_fd = -1
3331
+ elif descriptor == self.stderr_fd:
3332
+ self.stderr_fd = -1
3333
+ else:
3334
+ return
3335
+ try:
3336
+ os.close(descriptor)
3337
+ except OSError:
3338
+ pass
3339
+
3340
+ def kill_group(self) -> None:
3341
+ try:
3342
+ os.killpg(self.pid, signal.SIGKILL)
3343
+ except ProcessLookupError:
3344
+ pass
3345
+
3346
+
3347
+ def _start_popen_worker(
3348
+ command: list[str],
3349
+ *,
3350
+ cwd: Path,
3351
+ env: dict[str, str],
3352
+ ) -> _WorkerProcess:
3353
+ """Start a session-isolated worker for platforms without the Linux barrier."""
3354
+
3355
+ nproc_limit = _nproc_limit()
3356
+
3357
+ def apply_limits() -> None:
3358
+ _apply_resource_limits(nproc_limit)
3359
+
3360
+ return _PopenWorker(
3361
+ subprocess.Popen(
3362
+ command,
3363
+ cwd=cwd,
3364
+ env=env,
3365
+ stdin=subprocess.PIPE,
3366
+ stdout=subprocess.PIPE,
3367
+ stderr=subprocess.PIPE,
3368
+ start_new_session=True,
3369
+ preexec_fn=apply_limits,
3370
+ )
3371
+ )
3372
+
3373
+
3374
+ def _stop_worker(process: _WorkerProcess) -> None:
3375
+ failure: BaseException | None = None
3376
+
3377
+ def attempt(operation: Any) -> None:
3378
+ nonlocal failure
3379
+ try:
3380
+ operation()
3381
+ except BaseException as error:
3382
+ if failure is None:
3383
+ failure = error
3384
+
3385
+ stdout_fd = process.stdout_fd
3386
+ stderr_fd = process.stderr_fd
3387
+ attempt(process.kill_group)
3388
+ attempt(process.close_stdin)
3389
+ try:
3390
+ process.wait(1.0)
3391
+ except subprocess.TimeoutExpired:
3392
+ # A timeout is only a failed first reap attempt. Preserve failures from the second
3393
+ # kill/reap, but do not report this timeout when the retry succeeds.
3394
+ attempt(process.kill_group)
3395
+ attempt(lambda: process.wait(1.0))
3396
+ except BaseException as error:
3397
+ if failure is None:
3398
+ failure = error
3399
+ attempt(process.close_stdin)
3400
+ attempt(lambda: process.close_output(stdout_fd))
3401
+ attempt(lambda: process.close_output(stderr_fd))
3402
+ if failure is not None:
3403
+ raise failure
3404
+
3405
+
3406
+ def _reap_owned_process_group_children(process_group: int, deadline: float) -> None:
3407
+ """Reap adopted hosted descendants without touching another warm worker's group."""
3408
+
3409
+ while True:
3410
+ try:
3411
+ pid, _status = os.waitpid(-process_group, os.WNOHANG)
3412
+ except ChildProcessError:
3413
+ return
3414
+ if pid > 0:
3415
+ continue
3416
+ if time.monotonic() >= deadline:
3417
+ raise OSError("hosted-session process-group descendants did not become reapable")
3418
+ time.sleep(0.005)
3419
+
3420
+
3421
+ def _drain_worker_ipc(
3422
+ process: _WorkerProcess,
3423
+ *,
3424
+ request_bytes: bytes,
3425
+ timeout_seconds: float,
3426
+ terminate_process_group: bool = False,
3427
+ ) -> tuple[bytes, bytes, int]:
3428
+ """Pump all three pipes concurrently and enforce byte caps during streaming."""
3429
+
3430
+ selector: selectors.BaseSelector | None = None
3431
+ stdout = bytearray()
3432
+ stderr = bytearray()
3433
+ request_offset = 0
3434
+ streams = {
3435
+ process.stdout_fd: (stdout, MAX_RESPONSE_BYTES, "stdout"),
3436
+ process.stderr_fd: (stderr, MAX_WORKER_STDERR_BYTES, "stderr"),
3437
+ }
3438
+ result: tuple[bytes, bytes, int] | None = None
3439
+ failure: BaseException | None = None
3440
+
3441
+ def remember(error: BaseException) -> None:
3442
+ nonlocal failure
3443
+ if failure is None:
3444
+ failure = error
3445
+
3446
+ try:
3447
+ selector = selectors.DefaultSelector()
3448
+ deadline = time.monotonic() + timeout_seconds
3449
+ for descriptor in (process.stdin_fd, process.stdout_fd, process.stderr_fd):
3450
+ os.set_blocking(descriptor, False)
3451
+ if request_bytes:
3452
+ selector.register(process.stdin_fd, selectors.EVENT_WRITE, "stdin")
3453
+ else:
3454
+ process.close_stdin()
3455
+ selector.register(process.stdout_fd, selectors.EVENT_READ, "stdout")
3456
+ selector.register(process.stderr_fd, selectors.EVENT_READ, "stderr")
3457
+
3458
+ while streams:
3459
+ remaining = deadline - time.monotonic()
3460
+ if remaining <= 0:
3461
+ raise AcquisitionSecurityError(
3462
+ "SANDBOX_TIMEOUT",
3463
+ "The Clean room process did not finish in time",
3464
+ )
3465
+ events = selector.select(remaining)
3466
+ if not events:
3467
+ continue
3468
+ for key, mask in events:
3469
+ descriptor = key.fd
3470
+ if key.data == "stdin" and mask & selectors.EVENT_WRITE:
3471
+ try:
3472
+ written = os.write(descriptor, request_bytes[request_offset:])
3473
+ except BrokenPipeError:
3474
+ written = 0
3475
+ request_offset += written
3476
+ if written == 0 or request_offset == len(request_bytes):
3477
+ selector.unregister(descriptor)
3478
+ process.close_stdin()
3479
+ continue
3480
+ if not mask & selectors.EVENT_READ:
3481
+ continue
3482
+ buffer, cap, _label = streams[descriptor]
3483
+ try:
3484
+ chunk = os.read(descriptor, min(65_536, cap - len(buffer) + 1))
3485
+ except BlockingIOError:
3486
+ continue
3487
+ if not chunk:
3488
+ selector.unregister(descriptor)
3489
+ process.close_output(descriptor)
3490
+ del streams[descriptor]
3491
+ continue
3492
+ buffer.extend(chunk)
3493
+ if len(buffer) > cap:
3494
+ raise AcquisitionSecurityError(
3495
+ "SANDBOX_IPC_LIMIT",
3496
+ "The Clean room process returned more data than this job allows",
3497
+ )
3498
+
3499
+ # Hosted-session children may fork bounded parser helpers, but no helper may outlive the
3500
+ # one request. Signal the still-owned process group before reaping its leader: this covers
3501
+ # the success path, catches descendants that deliberately closed all inherited pipes, and
3502
+ # avoids signalling a numeric group id after the leader pid could have been reused.
3503
+ if terminate_process_group:
3504
+ process.kill_group()
3505
+ remaining = deadline - time.monotonic()
3506
+ if remaining <= 0:
3507
+ raise AcquisitionSecurityError(
3508
+ "SANDBOX_TIMEOUT",
3509
+ "The Clean room process did not finish in time",
3510
+ )
3511
+ try:
3512
+ returncode = process.wait(remaining)
3513
+ except subprocess.TimeoutExpired as error:
3514
+ raise AcquisitionSecurityError(
3515
+ "SANDBOX_TIMEOUT",
3516
+ "The Clean room process did not finish in time",
3517
+ ) from error
3518
+ if terminate_process_group:
3519
+ _reap_owned_process_group_children(process.pid, deadline)
3520
+ result = bytes(stdout), bytes(stderr), returncode
3521
+ except BaseException as error:
3522
+ remember(error)
3523
+ try:
3524
+ _stop_worker(process)
3525
+ except BaseException as cleanup_error:
3526
+ # Cleanup is subordinate to the exception that initiated it.
3527
+ remember(cleanup_error)
3528
+ finally:
3529
+ cleanup_steps = [process.close_stdin]
3530
+ if selector is not None:
3531
+ cleanup_steps.insert(0, selector.close)
3532
+ for descriptor in tuple(streams):
3533
+ cleanup_steps.append(lambda descriptor=descriptor: process.close_output(descriptor))
3534
+ for cleanup_step in cleanup_steps:
3535
+ try:
3536
+ cleanup_step()
3537
+ except BaseException as cleanup_error:
3538
+ remember(cleanup_error)
3539
+ if failure is not None:
3540
+ raise failure
3541
+ assert result is not None
3542
+ return result
3543
+
3544
+
3545
+ def _read_control(directory_fd: int, name: str) -> str:
3546
+ descriptor = os.open(name, os.O_RDONLY | os.O_CLOEXEC | os.O_NOFOLLOW, dir_fd=directory_fd)
3547
+ try:
3548
+ chunks: list[bytes] = []
3549
+ while True:
3550
+ chunk = os.read(descriptor, 4096)
3551
+ if not chunk:
3552
+ return b"".join(chunks).decode("ascii", "strict")
3553
+ chunks.append(chunk)
3554
+ finally:
3555
+ os.close(descriptor)
3556
+
3557
+
3558
+ def _write_control(directory_fd: int, name: str, value: str) -> None:
3559
+ descriptor = os.open(name, os.O_WRONLY | os.O_CLOEXEC | os.O_NOFOLLOW, dir_fd=directory_fd)
3560
+ try:
3561
+ payload = value.encode("ascii", "strict")
3562
+ if os.write(descriptor, payload) != len(payload):
3563
+ raise OSError("short write to cgroup control")
3564
+ finally:
3565
+ os.close(descriptor)
3566
+
3567
+
3568
+ def _control_can_be_opened_for_write(directory_fd: int, name: str) -> bool:
3569
+ try:
3570
+ descriptor = os.open(
3571
+ name,
3572
+ os.O_WRONLY | os.O_CLOEXEC | os.O_NOFOLLOW,
3573
+ dir_fd=directory_fd,
3574
+ )
3575
+ except OSError:
3576
+ return False
3577
+ os.close(descriptor)
3578
+ return True
3579
+
3580
+
3581
+ def _require_hosted_session_container_boundary(expected_memory_bytes: int) -> None:
3582
+ """Validate the immutable whole-container boundary used only by research sessions.
3583
+
3584
+ Cloud Run does not delegate a writable cgroup subtree to the service. Instead, one session is
3585
+ the sole tenant of one instance and the platform's cgroup is the hard aggregate ceiling for
3586
+ the coordinator, parser, and every descendant. The same credentials that reach the parser
3587
+ must be unable to change that ceiling, create a child cgroup, or migrate a process elsewhere.
3588
+ A networkless seccomp filter is installed separately before the parser is released to exec.
3589
+ """
3590
+
3591
+ if (
3592
+ type(expected_memory_bytes) is not int
3593
+ or expected_memory_bytes != HOSTED_SESSION_CONTAINER_MEMORY_BYTES
3594
+ ):
3595
+ raise AcquisitionSecurityError(
3596
+ "SANDBOX_MEMORY_BOUNDARY",
3597
+ "hosted-session memory authority must name the exact 4 GiB service ceiling",
3598
+ )
3599
+ if sys.platform != "linux" or platform.machine().lower() not in {"x86_64", "amd64"}:
3600
+ raise AcquisitionSecurityError(
3601
+ "SANDBOX_MEMORY_BOUNDARY",
3602
+ "hosted-session container confinement requires Linux/amd64",
3603
+ )
3604
+ selected: (
3605
+ tuple[
3606
+ int,
3607
+ os.stat_result,
3608
+ str,
3609
+ str,
3610
+ tuple[str, ...],
3611
+ tuple[str, ...],
3612
+ ]
3613
+ | None
3614
+ ) = None
3615
+ last_error: BaseException | None = None
3616
+ layouts = (
3617
+ (_HOSTED_SESSION_CGROUP_ROOT, "memory.max", ("cgroup.procs",), ("cgroup.subtree_control",)),
3618
+ (
3619
+ _HOSTED_SESSION_CGROUP_ROOT / "memory",
3620
+ "memory.limit_in_bytes",
3621
+ ("cgroup.procs", "tasks"),
3622
+ (),
3623
+ ),
3624
+ )
3625
+ for root, limit_name, member_names, extra_immutable in layouts:
3626
+ root_fd = -1
3627
+ try:
3628
+ root_fd = os.open(
3629
+ root,
3630
+ os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC | os.O_NOFOLLOW,
3631
+ )
3632
+ metadata = os.fstat(root_fd)
3633
+ raw_limit = _read_control(root_fd, limit_name).strip()
3634
+ except (OSError, UnicodeError) as error:
3635
+ last_error = error
3636
+ if root_fd >= 0:
3637
+ os.close(root_fd)
3638
+ continue
3639
+ selected = (
3640
+ root_fd,
3641
+ metadata,
3642
+ raw_limit,
3643
+ limit_name,
3644
+ member_names,
3645
+ extra_immutable,
3646
+ )
3647
+ break
3648
+ if selected is None:
3649
+ raise AcquisitionSecurityError(
3650
+ "SANDBOX_MEMORY_BOUNDARY",
3651
+ "hosted-session container cgroup is unavailable or unreadable",
3652
+ ) from last_error
3653
+ root_fd, metadata, raw_limit, limit_name, member_names, extra_immutable = selected
3654
+ try:
3655
+ member_name: str | None = None
3656
+ members: set[int] | None = None
3657
+ for candidate in member_names:
3658
+ try:
3659
+ raw_members = _read_control(root_fd, candidate)
3660
+ except (OSError, UnicodeError):
3661
+ continue
3662
+ try:
3663
+ members = {int(value) for value in raw_members.split()}
3664
+ except ValueError as error:
3665
+ raise AcquisitionSecurityError(
3666
+ "SANDBOX_MEMORY_BOUNDARY",
3667
+ "hosted-session container process membership is malformed",
3668
+ ) from error
3669
+ member_name = candidate
3670
+ break
3671
+ if member_name is None or members is None:
3672
+ raise AcquisitionSecurityError(
3673
+ "SANDBOX_MEMORY_BOUNDARY",
3674
+ "hosted-session container process membership is unavailable",
3675
+ )
3676
+ if not stat.S_ISDIR(metadata.st_mode):
3677
+ raise AcquisitionSecurityError(
3678
+ "SANDBOX_MEMORY_BOUNDARY",
3679
+ "hosted-session memory authority is not a cgroup directory",
3680
+ )
3681
+ if raw_limit == "max":
3682
+ raise AcquisitionSecurityError(
3683
+ "SANDBOX_MEMORY_BOUNDARY",
3684
+ "hosted-session container cgroup has no finite memory ceiling",
3685
+ )
3686
+ if not raw_limit.isascii() or not raw_limit.isdecimal():
3687
+ raise AcquisitionSecurityError(
3688
+ "SANDBOX_MEMORY_BOUNDARY",
3689
+ "hosted-session container memory ceiling is not an unsigned byte count",
3690
+ )
3691
+ if int(raw_limit) != expected_memory_bytes:
3692
+ raise AcquisitionSecurityError(
3693
+ "SANDBOX_MEMORY_BOUNDARY",
3694
+ "hosted-session container memory ceiling does not match the exact service class",
3695
+ )
3696
+ if os.getpid() not in members:
3697
+ raise AcquisitionSecurityError(
3698
+ "SANDBOX_MEMORY_BOUNDARY",
3699
+ "hosted-session coordinator is not in the bounded container cgroup",
3700
+ )
3701
+ descriptor_path = f"/proc/self/fd/{root_fd}"
3702
+ if os.access(descriptor_path, os.W_OK, effective_ids=True) or any(
3703
+ _control_can_be_opened_for_write(root_fd, control)
3704
+ for control in (
3705
+ limit_name,
3706
+ *member_names,
3707
+ *extra_immutable,
3708
+ )
3709
+ ):
3710
+ raise AcquisitionSecurityError(
3711
+ "SANDBOX_MEMORY_BOUNDARY",
3712
+ "hosted-session container cgroup must be immutable to worker credentials",
3713
+ )
3714
+ finally:
3715
+ os.close(root_fd)
3716
+
3717
+
3718
+ def _require_bounded_cgroup_root(root_fd: int) -> None:
3719
+ """Validate one empty, operation-scoped, hierarchically bounded cgroup-v2 root.
3720
+
3721
+ The coordinator and worker share Unix credentials, so the worker can exercise the root's
3722
+ process-migration and child-creation delegation too. Containment therefore lives on the root
3723
+ itself: immutable memory/pid ceilings apply to every descendant, depth and descendant counts
3724
+ admit one protected coordinator plus exactly one job child, the no-internal-process rule keeps
3725
+ the worker out of the root, and the non-writable parent prevents escape above it.
3726
+ """
3727
+
3728
+ if type(root_fd) is not int or root_fd < 0:
3729
+ raise AcquisitionSecurityError(
3730
+ "SANDBOX_MEMORY_BOUNDARY",
3731
+ "Linux memory authority must be an open bounded cgroup root descriptor",
3732
+ )
3733
+ try:
3734
+ metadata = os.fstat(root_fd)
3735
+ controllers = set(_read_control(root_fd, "cgroup.controllers").split())
3736
+ enabled = set(_read_control(root_fd, "cgroup.subtree_control").split())
3737
+ limits = {
3738
+ "memory.max": _read_control(root_fd, "memory.max").strip(),
3739
+ "memory.swap.max": _read_control(root_fd, "memory.swap.max").strip(),
3740
+ "pids.max": _read_control(root_fd, "pids.max").strip(),
3741
+ "cgroup.max.depth": _read_control(root_fd, "cgroup.max.depth").strip(),
3742
+ "cgroup.max.descendants": _read_control(root_fd, "cgroup.max.descendants").strip(),
3743
+ }
3744
+ cgroup_stat = dict(
3745
+ line.split() for line in _read_control(root_fd, "cgroup.stat").splitlines()
3746
+ )
3747
+ except (OSError, UnicodeError) as error:
3748
+ raise AcquisitionSecurityError(
3749
+ "SANDBOX_MEMORY_BOUNDARY",
3750
+ "Linux memory authority is not a readable bounded cgroup-v2 root",
3751
+ ) from error
3752
+ expected = {
3753
+ "memory.max": str(MAX_WORKER_MEMORY_BYTES),
3754
+ "memory.swap.max": "0",
3755
+ "pids.max": str(MAX_WORKER_PIDS),
3756
+ "cgroup.max.depth": "1",
3757
+ "cgroup.max.descendants": "2",
3758
+ }
3759
+ if not stat.S_ISDIR(metadata.st_mode) or not {"memory", "pids"} <= controllers | enabled:
3760
+ raise AcquisitionSecurityError(
3761
+ "SANDBOX_MEMORY_BOUNDARY",
3762
+ "bounded cgroup root must expose the memory and pids controllers",
3763
+ )
3764
+ if limits != expected or not {"memory", "pids"} <= enabled:
3765
+ raise AcquisitionSecurityError(
3766
+ "SANDBOX_MEMORY_BOUNDARY",
3767
+ "bounded cgroup root must carry immutable hierarchy, memory, swap, and process limits",
3768
+ )
3769
+ descriptor_path = f"/proc/self/fd/{root_fd}"
3770
+ if not os.access(descriptor_path, os.W_OK, effective_ids=True):
3771
+ raise AcquisitionSecurityError(
3772
+ "SANDBOX_MEMORY_BOUNDARY",
3773
+ "bounded cgroup root must grant child-creation authority",
3774
+ )
3775
+ immutable_controls = (*expected, "cgroup.subtree_control")
3776
+ if any(_control_can_be_opened_for_write(root_fd, name) for name in immutable_controls):
3777
+ raise AcquisitionSecurityError(
3778
+ "SANDBOX_MEMORY_BOUNDARY",
3779
+ "bounded cgroup root limits must not be writable by worker credentials",
3780
+ )
3781
+ if not _control_can_be_opened_for_write(root_fd, "cgroup.procs"):
3782
+ raise AcquisitionSecurityError(
3783
+ "SANDBOX_MEMORY_BOUNDARY",
3784
+ "bounded cgroup root must grant process-migration authority",
3785
+ )
3786
+ if _control_can_be_opened_for_write(root_fd, "../cgroup.procs"):
3787
+ raise AcquisitionSecurityError(
3788
+ "SANDBOX_MEMORY_BOUNDARY",
3789
+ "worker credentials must not be able to leave the bounded cgroup root",
3790
+ )
3791
+ if _read_control(root_fd, "cgroup.procs").split() or cgroup_stat.get("nr_descendants") != "1":
3792
+ raise AcquisitionSecurityError(
3793
+ "SANDBOX_MEMORY_BOUNDARY",
3794
+ "bounded cgroup root must contain only its operator-owned coordinator child",
3795
+ )
3796
+ coordinator_fd = -1
3797
+ try:
3798
+ coordinator_fd = os.open(
3799
+ "coordinator",
3800
+ os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC | os.O_NOFOLLOW,
3801
+ dir_fd=root_fd,
3802
+ )
3803
+ coordinator_members = {
3804
+ int(value) for value in _read_control(coordinator_fd, "cgroup.procs").split()
3805
+ }
3806
+ if os.getpid() not in coordinator_members or _control_can_be_opened_for_write(
3807
+ coordinator_fd, "cgroup.procs"
3808
+ ):
3809
+ raise AcquisitionSecurityError(
3810
+ "SANDBOX_MEMORY_BOUNDARY",
3811
+ "the trusted coordinator must run in a non-writable sibling cgroup",
3812
+ )
3813
+ except OSError as error:
3814
+ raise AcquisitionSecurityError(
3815
+ "SANDBOX_MEMORY_BOUNDARY",
3816
+ "bounded cgroup root has no protected coordinator child",
3817
+ ) from error
3818
+ finally:
3819
+ if coordinator_fd >= 0:
3820
+ os.close(coordinator_fd)
3821
+
3822
+
3823
+ class _CgroupJob:
3824
+ def __init__(self, *, root_fd: int, name: str, job_fd: int) -> None:
3825
+ self.root_fd = root_fd
3826
+ self.name = name
3827
+ self.job_fd = job_fd
3828
+ self._root_oom_before = self._oom_events(root_fd)
3829
+
3830
+ @classmethod
3831
+ def create(cls, root_fd: int, pid: int) -> _CgroupJob:
3832
+ name = f"mostlyright-sandbox-{pid}-{secrets.token_hex(8)}"
3833
+ duplicated_root = os.dup(root_fd)
3834
+ job_fd = -1
3835
+ leaf_created = False
3836
+ job: _CgroupJob | None = None
3837
+ try:
3838
+ os.mkdir(name, mode=0o700, dir_fd=duplicated_root)
3839
+ leaf_created = True
3840
+ job_fd = os.open(
3841
+ name,
3842
+ os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC | os.O_NOFOLLOW,
3843
+ dir_fd=duplicated_root,
3844
+ )
3845
+ # Construction reads root telemetry. It belongs inside the ownership scope too: an
3846
+ # OSError -- or a process-level BaseException -- after the leaf and its descriptor
3847
+ # exist must not strand either merely because no `_CgroupJob` was returned to the
3848
+ # caller yet.
3849
+ job = cls(root_fd=duplicated_root, name=name, job_fd=job_fd)
3850
+ job._configure(pid)
3851
+ except BaseException:
3852
+ if job is not None:
3853
+ job._best_effort_remove()
3854
+ else:
3855
+ # These are independent owned resources. A failing close or removal must not
3856
+ # prevent the later cleanup attempts, and none may replace the construction
3857
+ # failure being re-raised below.
3858
+ if job_fd >= 0:
3859
+ try:
3860
+ os.close(job_fd)
3861
+ except BaseException:
3862
+ pass
3863
+ if leaf_created:
3864
+ try:
3865
+ os.rmdir(name, dir_fd=duplicated_root)
3866
+ except BaseException:
3867
+ pass
3868
+ try:
3869
+ os.close(duplicated_root)
3870
+ except BaseException:
3871
+ pass
3872
+ raise
3873
+ assert job is not None
3874
+ return job
3875
+
3876
+ def _configure(self, pid: int) -> None:
3877
+ expected = {
3878
+ "memory.max": str(MAX_WORKER_MEMORY_BYTES),
3879
+ "memory.swap.max": "0",
3880
+ "pids.max": str(MAX_WORKER_PIDS),
3881
+ }
3882
+ for control, value in expected.items():
3883
+ _write_control(self.job_fd, control, value)
3884
+ if _read_control(self.job_fd, control).strip() != value:
3885
+ raise OSError(f"cgroup refused {control}")
3886
+ _write_control(self.job_fd, "cgroup.procs", str(pid))
3887
+ members = {int(value) for value in _read_control(self.job_fd, "cgroup.procs").split()}
3888
+ if pid not in members:
3889
+ raise OSError("worker was not attached to its cgroup before release")
3890
+
3891
+ @staticmethod
3892
+ def _oom_events(directory_fd: int) -> int:
3893
+ events = {
3894
+ key: int(value)
3895
+ for key, value in (
3896
+ line.split() for line in _read_control(directory_fd, "memory.events").splitlines()
3897
+ )
3898
+ }
3899
+ return max(events.get("oom", 0), events.get("oom_kill", 0))
3900
+
3901
+ def memory_limit_hit(self) -> bool:
3902
+ return (
3903
+ self._oom_events(self.job_fd) > 0
3904
+ or self._oom_events(self.root_fd) > self._root_oom_before
3905
+ )
3906
+
3907
+ def cleanup(self) -> None:
3908
+ failure: BaseException | None = None
3909
+
3910
+ def remember(error: BaseException) -> None:
3911
+ nonlocal failure
3912
+ if failure is None:
3913
+ failure = error
3914
+
3915
+ def kill_members() -> None:
3916
+ try:
3917
+ members = _read_control(self.job_fd, "cgroup.procs").split()
3918
+ except BaseException as error:
3919
+ remember(error)
3920
+ return
3921
+ for pid_text in members:
3922
+ try:
3923
+ os.kill(int(pid_text), signal.SIGKILL)
3924
+ except ProcessLookupError:
3925
+ pass
3926
+ except BaseException as error:
3927
+ remember(error)
3928
+
3929
+ populated: bool | None = None
3930
+ try:
3931
+ populated = bool(
3932
+ "populated 0" not in _read_control(self.job_fd, "cgroup.events").splitlines()
3933
+ )
3934
+ except BaseException as error:
3935
+ remember(error)
3936
+
3937
+ # Unknown telemetry is not evidence that the leaf is empty. Both kill mechanisms are
3938
+ # attempted independently: cgroup.kill may be absent on an older kernel, while reading
3939
+ # cgroup.procs or killing one member may itself be the operation that was interrupted.
3940
+ if populated is not False:
3941
+ deadline: float | None = None
3942
+ try:
3943
+ deadline = time.monotonic() + _CGROUP_CLEANUP_TIMEOUT_SECONDS
3944
+ except BaseException as error:
3945
+ remember(error)
3946
+ # The attempt cap is an independent bound when clock access itself is interrupted.
3947
+ # When the clock works, the earlier of the wall-clock deadline and this cap wins.
3948
+ attempts_left = max(1, int(_CGROUP_CLEANUP_TIMEOUT_SECONDS / 0.01) + 1)
3949
+ while attempts_left > 0:
3950
+ attempts_left -= 1
3951
+ # Repeat both kill mechanisms: a descendant can appear after one procs
3952
+ # snapshot, and cgroup.kill is optional on older cgroup-v2 kernels.
3953
+ try:
3954
+ _write_control(self.job_fd, "cgroup.kill", "1")
3955
+ except OSError:
3956
+ pass
3957
+ except BaseException as error:
3958
+ remember(error)
3959
+ kill_members()
3960
+ try:
3961
+ populated = (
3962
+ "populated 0"
3963
+ not in _read_control(self.job_fd, "cgroup.events").splitlines()
3964
+ )
3965
+ except BaseException as error:
3966
+ remember(error)
3967
+ populated = None
3968
+ if populated is False:
3969
+ break
3970
+ timed_out = attempts_left == 0
3971
+ if deadline is not None:
3972
+ try:
3973
+ timed_out = timed_out or time.monotonic() >= deadline
3974
+ except BaseException as error:
3975
+ remember(error)
3976
+ if timed_out:
3977
+ remember(OSError("cgroup descendants survived cleanup"))
3978
+ break
3979
+ try:
3980
+ time.sleep(0.01)
3981
+ except BaseException as error:
3982
+ remember(error)
3983
+ continue
3984
+ try:
3985
+ os.close(self.job_fd)
3986
+ except BaseException as error:
3987
+ remember(error)
3988
+ try:
3989
+ os.rmdir(self.name, dir_fd=self.root_fd)
3990
+ except BaseException as error:
3991
+ remember(error)
3992
+ try:
3993
+ os.close(self.root_fd)
3994
+ except BaseException as error:
3995
+ remember(error)
3996
+ if failure is not None:
3997
+ raise failure
3998
+
3999
+ def _best_effort_remove(self) -> None:
4000
+ try:
4001
+ self.cleanup()
4002
+ except BaseException:
4003
+ pass
4004
+
4005
+
4006
+ def _close_child_descriptors_except(keep: set[int]) -> None:
4007
+ try:
4008
+ descriptors = [int(name) for name in os.listdir("/proc/self/fd") if name.isdigit()]
4009
+ except OSError:
4010
+ soft_limit, _hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE)
4011
+ descriptors = list(range(3, min(int(soft_limit), 65_536)))
4012
+ for descriptor in descriptors:
4013
+ if descriptor not in keep:
4014
+ try:
4015
+ os.close(descriptor)
4016
+ except OSError:
4017
+ pass
4018
+
4019
+
4020
+ def _start_linux_barrier_worker(
4021
+ command: list[str],
4022
+ *,
4023
+ cwd: Path,
4024
+ env: dict[str, str],
4025
+ hosted_session_networkless: bool = False,
4026
+ ) -> tuple[_ForkedWorker, int, int]:
4027
+ """Fork a stopped-before-exec child; return it plus ready/release pipe ends."""
4028
+
4029
+ stdin_read, stdin_write = os.pipe2(os.O_CLOEXEC)
4030
+ stdout_read, stdout_write = os.pipe2(os.O_CLOEXEC)
4031
+ stderr_read, stderr_write = os.pipe2(os.O_CLOEXEC)
4032
+ ready_read, ready_write = os.pipe2(os.O_CLOEXEC)
4033
+ release_read, release_write = os.pipe2(os.O_CLOEXEC)
4034
+ nproc_limit = _nproc_limit()
4035
+ try:
4036
+ pid = os.fork()
4037
+ except BaseException:
4038
+ for descriptor in (
4039
+ stdin_read,
4040
+ stdin_write,
4041
+ stdout_read,
4042
+ stdout_write,
4043
+ stderr_read,
4044
+ stderr_write,
4045
+ ready_read,
4046
+ ready_write,
4047
+ release_read,
4048
+ release_write,
4049
+ ):
4050
+ os.close(descriptor)
4051
+ raise
4052
+ if pid == 0:
4053
+ try:
4054
+ os.close(stdin_write)
4055
+ os.close(stdout_read)
4056
+ os.close(stderr_read)
4057
+ os.close(ready_read)
4058
+ os.close(release_write)
4059
+ os.setsid()
4060
+ os.dup2(stdin_read, 0)
4061
+ os.dup2(stdout_write, 1)
4062
+ os.dup2(stderr_write, 2)
4063
+ _close_child_descriptors_except({0, 1, 2, ready_write, release_read})
4064
+ _apply_resource_limits(nproc_limit)
4065
+ if hosted_session_networkless:
4066
+ _install_networkless_seccomp(allow_descendants=True)
4067
+ os.write(ready_write, b"R")
4068
+ if os.read(release_read, 1) != b"G":
4069
+ os._exit(126)
4070
+ os.close(ready_write)
4071
+ os.close(release_read)
4072
+ os.chdir(cwd)
4073
+ os.execve(command[0], command, env)
4074
+ except BaseException:
4075
+ try:
4076
+ os.write(2, b"sandbox worker failed before exec\n")
4077
+ except OSError:
4078
+ pass
4079
+ os._exit(127)
4080
+ os.close(stdin_read)
4081
+ os.close(stdout_write)
4082
+ os.close(stderr_write)
4083
+ os.close(ready_write)
4084
+ os.close(release_read)
4085
+ return (
4086
+ _ForkedWorker(
4087
+ pid=pid,
4088
+ stdin_fd=stdin_write,
4089
+ stdout_fd=stdout_read,
4090
+ stderr_fd=stderr_read,
4091
+ ),
4092
+ ready_read,
4093
+ release_write,
4094
+ )
4095
+
4096
+
4097
+ def _wait_for_barrier(descriptor: int, timeout_seconds: float) -> None:
4098
+ selector = selectors.DefaultSelector()
4099
+ try:
4100
+ selector.register(descriptor, selectors.EVENT_READ)
4101
+ if not selector.select(timeout_seconds) or os.read(descriptor, 1) != b"R":
4102
+ raise OSError("The Clean room process did not stop before starting its job")
4103
+ finally:
4104
+ selector.close()
4105
+ os.close(descriptor)
4106
+
4107
+
4108
+ @dataclass
4109
+ class _PreparedWorker:
4110
+ """One started, resource-limited child that has not been given a request yet.
4111
+
4112
+ The split between preparing a child and completing a request against it is what makes a
4113
+ warm pool expressible without a second launch path. Both paths build the identical
4114
+ ``_PreparedWorker``; the only difference is when it is built. A cold invocation builds one
4115
+ and binds it in the same call, a warm one built it earlier and idle. Because there is one
4116
+ constructor, a warm child cannot drift away from a cold child's environment, descriptors,
4117
+ session, or resource limits -- there is no second place for it to drift in.
4118
+
4119
+ ``deadline`` is the wall-clock budget a cold invocation already started spending on the
4120
+ barrier handshake, carried forward so the split does not silently widen the timeout. A
4121
+ warm child carries ``None``: its request budget starts when the request binds, not when
4122
+ the child was spawned, because the seconds it spent idle are not seconds the request had.
4123
+ """
4124
+
4125
+ process: _WorkerProcess
4126
+ job: _CgroupJob | None
4127
+ deadline: float | None
4128
+ launch: _WorkerLaunch
4129
+
4130
+
4131
+ @dataclass(frozen=True)
4132
+ class _WorkerLaunch:
4133
+ """The exact launch specification a child was started from.
4134
+
4135
+ A warm child may only be bound to a request whose launch specification is byte-identical
4136
+ to the one the child was started from. The boundary is the container and the cgroup, which
4137
+ are the same for every child of one coordinator, so the specification is what is left to
4138
+ compare. Comparing the whole of it rather than an operation name keeps the argument
4139
+ mechanical: nothing about a pooled child's confinement can differ from the confinement the
4140
+ binding request would have been given had it spawned its own. (Until ADR 0021 there
4141
+ was a second reason -- on darwin the Seatbelt profile was an argv element, so equal commands
4142
+ were equal kernel authority. That backend is gone; the comparison it motivated is not, because
4143
+ it is what makes a warm child indistinguishable from a fresh spawn.)
4144
+ """
4145
+
4146
+ command: tuple[str, ...]
4147
+ cwd: Path
4148
+ env: tuple[tuple[str, str], ...]
4149
+ prewarm: bool
4150
+
4151
+ @property
4152
+ def confinement(self) -> tuple[tuple[str, ...], Path, tuple[tuple[str, str], ...]]:
4153
+ """Everything about a child that a request may rely on: argv, directory, environment."""
4154
+
4155
+ return (self.command, self.cwd, self.env)
4156
+
4157
+ def argv(self) -> list[str]:
4158
+ """The exact argv this child is exec'd with.
4159
+
4160
+ ``--prewarm`` is appended rather than folded into ``command`` so the confinement a
4161
+ warm child was given and the confinement a binding request asks for stay comparable.
4162
+ It is the one enumerated difference between a warm child and a cold one, it is
4163
+ readable in ``ps``, and it grants nothing: the flag chooses when the parser imports
4164
+ happen and whether a readiness line is written, and the child's authority is decided
4165
+ entirely by the container boundary, the cgroup, the closed environment, and the request.
4166
+ """
4167
+
4168
+ return [*self.command, "--prewarm"] if self.prewarm else list(self.command)
4169
+
4170
+
4171
+ def _prepare_worker(
4172
+ launch: _WorkerLaunch,
4173
+ *,
4174
+ timeout_seconds: float,
4175
+ memory_cgroup_root_fd: int | None,
4176
+ hosted_session_container_memory_bytes: int | None = None,
4177
+ track_deadline: bool,
4178
+ ready_timeout_seconds: float | None = None,
4179
+ ) -> _PreparedWorker:
4180
+ """Start one confined child and bring it to the point of reading its request.
4181
+
4182
+ ``ready_timeout_seconds`` bounds only the pre-warm readiness wait, and a filler passes what
4183
+ is left of one budget for the whole fill rather than a fresh budget per child.
4184
+ """
4185
+
4186
+ command = launch.argv()
4187
+ cwd = launch.cwd
4188
+ env = dict(launch.env)
4189
+ deadline = time.monotonic() + timeout_seconds
4190
+ if ready_timeout_seconds is None:
4191
+ ready_timeout_seconds = min(timeout_seconds, 30.0)
4192
+ if sys.platform != "linux":
4193
+ # No deadline is carried here, and that is the pre-existing behaviour rather than an
4194
+ # oversight: this platform has no barrier handshake, so nothing was spent before the
4195
+ # request, and the drain gets the whole timeout exactly as it did before the split.
4196
+ return _become_ready_if_asked(
4197
+ _PreparedWorker(
4198
+ process=_start_popen_worker(command, cwd=cwd, env=env),
4199
+ job=None,
4200
+ deadline=None,
4201
+ launch=launch,
4202
+ ),
4203
+ timeout_seconds=ready_timeout_seconds,
4204
+ )
4205
+ if memory_cgroup_root_fd is not None and hosted_session_container_memory_bytes is not None:
4206
+ raise AcquisitionSecurityError(
4207
+ "SANDBOX_MEMORY_BOUNDARY",
4208
+ "delegated and hosted-session memory boundaries are mutually exclusive",
4209
+ )
4210
+ if memory_cgroup_root_fd is None and hosted_session_container_memory_bytes is None:
4211
+ raise AcquisitionSecurityError(
4212
+ "SANDBOX_MEMORY_BOUNDARY",
4213
+ "Linux Clean room execution requires a trusted bounded cgroup root",
4214
+ )
4215
+ if memory_cgroup_root_fd is not None:
4216
+ _require_bounded_cgroup_root(memory_cgroup_root_fd)
4217
+ else:
4218
+ assert hosted_session_container_memory_bytes is not None
4219
+ _require_hosted_session_container_boundary(hosted_session_container_memory_bytes)
4220
+ process, ready_fd, release_fd = _start_linux_barrier_worker(
4221
+ command,
4222
+ cwd=cwd,
4223
+ env=env,
4224
+ hosted_session_networkless=hosted_session_container_memory_bytes is not None,
4225
+ )
4226
+ job: _CgroupJob | None = None
4227
+ try:
4228
+ _wait_for_barrier(ready_fd, min(max(deadline - time.monotonic(), 0.0), 5.0))
4229
+ if memory_cgroup_root_fd is not None:
4230
+ job = _CgroupJob.create(memory_cgroup_root_fd, process.pid)
4231
+ if os.write(release_fd, b"G") != 1:
4232
+ raise OSError("The Clean room process could not be released to start its job")
4233
+ os.close(release_fd)
4234
+ release_fd = -1
4235
+ except BaseException as error:
4236
+ if release_fd >= 0:
4237
+ try:
4238
+ os.close(release_fd)
4239
+ except BaseException:
4240
+ pass
4241
+ try:
4242
+ _stop_worker(process)
4243
+ except BaseException:
4244
+ pass
4245
+ if job is not None:
4246
+ job._best_effort_remove()
4247
+ if not isinstance(error, Exception):
4248
+ raise error
4249
+ if isinstance(error, AcquisitionSecurityError):
4250
+ raise
4251
+ if hosted_session_container_memory_bytes is not None:
4252
+ raise AcquisitionSecurityError(
4253
+ "SANDBOX_OS_BOUNDARY",
4254
+ "hosted-session networkless process boundary could not be established before exec",
4255
+ ) from error
4256
+ raise AcquisitionSecurityError(
4257
+ "SANDBOX_MEMORY_BOUNDARY",
4258
+ "Linux cgroup memory boundary could not be established before exec",
4259
+ ) from error
4260
+
4261
+ return _become_ready_if_asked(
4262
+ _PreparedWorker(
4263
+ process=process,
4264
+ job=job,
4265
+ deadline=deadline if track_deadline else None,
4266
+ launch=launch,
4267
+ ),
4268
+ timeout_seconds=ready_timeout_seconds,
4269
+ )
4270
+
4271
+
4272
+ def _become_ready_if_asked(
4273
+ prepared: _PreparedWorker,
4274
+ *,
4275
+ timeout_seconds: float,
4276
+ ) -> _PreparedWorker:
4277
+ """Wait for a pre-warm child's readiness line, discarding it whole if it never comes.
4278
+
4279
+ After exec and inside whatever boundary this platform establishes, so the child that
4280
+ announces readiness is the confined child and not something on its way to becoming one. A
4281
+ child that fails here is torn down completely -- process group and cgroup both -- because
4282
+ it was never handed out and nothing is waiting on its response.
4283
+ """
4284
+
4285
+ if not prepared.launch.prewarm:
4286
+ return prepared
4287
+ try:
4288
+ _await_worker_ready(prepared.process, timeout_seconds=timeout_seconds)
4289
+ except BaseException:
4290
+ _discard_prepared_worker(prepared)
4291
+ raise
4292
+ return prepared
4293
+
4294
+
4295
+ def _worker_is_running(process: _WorkerProcess) -> bool:
4296
+ """Whether a child is still alive, reaping it if it is not.
4297
+
4298
+ ``wait`` with a zero budget is the one liveness question both worker shapes already answer:
4299
+ a returncode means it exited and has now been reaped, and a timeout means it is running.
4300
+
4301
+ Anything else answers neither, and the two possible guesses are not symmetric. Guessing
4302
+ "exited" sends the child down the teardown that does not signal it, which would leak a live
4303
+ process holding a cgroup. Guessing "running" sends it down the teardown that kills and
4304
+ reaps, which is correct for a live child and harmless for a dead one. So an unrecognised
4305
+ failure reports running. Interruptions are not answers at all and are re-raised.
4306
+ """
4307
+
4308
+ try:
4309
+ process.wait(0.0)
4310
+ except subprocess.TimeoutExpired:
4311
+ return True
4312
+ except (KeyboardInterrupt, SystemExit):
4313
+ raise
4314
+ except BaseException:
4315
+ return True
4316
+ return False
4317
+
4318
+
4319
+ def _discard_prepared_worker(
4320
+ prepared: _PreparedWorker,
4321
+ *,
4322
+ already_exited: bool = False,
4323
+ ) -> None:
4324
+ """Tear down a prepared child that will never be given a request.
4325
+
4326
+ ``already_exited`` changes the teardown rather than shortening it. A reaped child's pid is
4327
+ free for the kernel to hand to somebody else, so signalling its process group could reach
4328
+ whatever now holds that number; a child known to have exited therefore gets its
4329
+ descriptors closed and its cgroup removed, and nothing is signalled. Cgroup teardown is
4330
+ unaffected either way: it only ever reaches members of the leaf it owns.
4331
+ """
4332
+
4333
+ process = prepared.process
4334
+ stdout_fd = process.stdout_fd
4335
+ stderr_fd = process.stderr_fd
4336
+ steps: tuple[Any, ...] = (
4337
+ (
4338
+ process.close_stdin,
4339
+ lambda: process.close_output(stdout_fd),
4340
+ lambda: process.close_output(stderr_fd),
4341
+ )
4342
+ if already_exited
4343
+ else (lambda: _stop_worker(process),)
4344
+ )
4345
+ for step in steps:
4346
+ try:
4347
+ step()
4348
+ except BaseException:
4349
+ pass
4350
+ if prepared.job is not None:
4351
+ try:
4352
+ prepared.job.cleanup()
4353
+ except BaseException:
4354
+ prepared.job._best_effort_remove()
4355
+
4356
+
4357
+ def _await_worker_ready(process: _WorkerProcess, *, timeout_seconds: float) -> None:
4358
+ """Read exactly the readiness marker a pre-warmed child writes before its response.
4359
+
4360
+ Never more than the marker: the read length is capped at the bytes still outstanding, so
4361
+ this cannot consume a byte of the response the request will later produce, and the pump
4362
+ that drains that response sees the same stream it would have seen from a cold child.
4363
+ """
4364
+
4365
+ selector = selectors.DefaultSelector()
4366
+ seen = bytearray()
4367
+ try:
4368
+ os.set_blocking(process.stdout_fd, False)
4369
+ selector.register(process.stdout_fd, selectors.EVENT_READ)
4370
+ deadline = time.monotonic() + timeout_seconds
4371
+ while len(seen) < len(WORKER_READY_MARKER):
4372
+ remaining = deadline - time.monotonic()
4373
+ if remaining <= 0 or not selector.select(remaining):
4374
+ raise AcquisitionSecurityError(
4375
+ "SANDBOX_WARM_POOL",
4376
+ "a warm Clean room process did not become ready in time",
4377
+ )
4378
+ try:
4379
+ chunk = os.read(process.stdout_fd, len(WORKER_READY_MARKER) - len(seen))
4380
+ except BlockingIOError:
4381
+ continue
4382
+ if not chunk:
4383
+ raise AcquisitionSecurityError(
4384
+ "SANDBOX_WARM_POOL",
4385
+ "a warm Clean room process ended before it became ready",
4386
+ )
4387
+ seen.extend(chunk)
4388
+ if bytes(seen) != WORKER_READY_MARKER:
4389
+ raise AcquisitionSecurityError(
4390
+ "SANDBOX_WARM_POOL",
4391
+ "a warm Clean room process did not announce readiness",
4392
+ )
4393
+ finally:
4394
+ selector.close()
4395
+ # Left exactly as a cold child's descriptor is found. The pump sets all three
4396
+ # non-blocking itself, so this changes no behaviour -- but "a warm child is
4397
+ # indistinguishable from a fresh spawn" should not rest on the next reader repeating
4398
+ # the state this one happened to leave behind.
4399
+ try:
4400
+ os.set_blocking(process.stdout_fd, True)
4401
+ except OSError:
4402
+ pass
4403
+
4404
+
4405
+ def _complete_worker(
4406
+ prepared: _PreparedWorker,
4407
+ *,
4408
+ request_bytes: bytes,
4409
+ timeout_seconds: float,
4410
+ terminate_process_group: bool = False,
4411
+ ) -> tuple[bytes, bytes, int]:
4412
+ """Give one prepared child its one request and collect its one response."""
4413
+
4414
+ process = prepared.process
4415
+ job = prepared.job
4416
+ deadline = prepared.deadline
4417
+ if deadline is None:
4418
+ deadline = time.monotonic() + timeout_seconds
4419
+ if job is None:
4420
+ return _drain_worker_ipc(
4421
+ process,
4422
+ request_bytes=request_bytes,
4423
+ timeout_seconds=max(deadline - time.monotonic(), 0.0),
4424
+ terminate_process_group=terminate_process_group,
4425
+ )
4426
+ result: tuple[bytes, bytes, int] | None = None
4427
+ primary_error: BaseException | None = None
4428
+ primary_stage: str | None = None
4429
+ memory_limit_hit = False
4430
+ try:
4431
+ try:
4432
+ result = _drain_worker_ipc(
4433
+ process,
4434
+ request_bytes=request_bytes,
4435
+ timeout_seconds=max(deadline - time.monotonic(), 0.0),
4436
+ terminate_process_group=terminate_process_group,
4437
+ )
4438
+ except BaseException as error:
4439
+ primary_error = error
4440
+ primary_stage = "drain"
4441
+ try:
4442
+ memory_limit_hit = job.memory_limit_hit()
4443
+ except BaseException as error:
4444
+ if primary_error is None:
4445
+ primary_error = error
4446
+ primary_stage = "telemetry"
4447
+ finally:
4448
+ try:
4449
+ job.cleanup()
4450
+ except BaseException as error:
4451
+ if primary_error is None:
4452
+ primary_error = error
4453
+ primary_stage = "cleanup"
4454
+
4455
+ if primary_error is not None:
4456
+ # Interruptions are control flow, not worker failures. Teardown above still owns every
4457
+ # cgroup resource, but the exact process-level exception must reach the caller unchanged.
4458
+ if not isinstance(primary_error, Exception):
4459
+ raise primary_error
4460
+ if primary_stage == "drain" and isinstance(primary_error, AcquisitionSecurityError):
4461
+ raise primary_error
4462
+ if primary_stage == "telemetry":
4463
+ raise AcquisitionSecurityError(
4464
+ "SANDBOX_MEMORY_BOUNDARY",
4465
+ "Linux cgroup memory telemetry could not be corroborated",
4466
+ ) from primary_error
4467
+ if primary_stage == "cleanup":
4468
+ raise AcquisitionSecurityError(
4469
+ "SANDBOX_MEMORY_BOUNDARY",
4470
+ "Linux cgroup descendants could not be killed and removed",
4471
+ ) from primary_error
4472
+ raise AcquisitionSecurityError(
4473
+ "SANDBOX_FAILURE",
4474
+ "The Clean room process could not be watched safely",
4475
+ ) from primary_error
4476
+ if memory_limit_hit:
4477
+ raise AcquisitionSecurityError(
4478
+ "SANDBOX_MEMORY_LIMIT",
4479
+ "The Clean room process used more memory than this job allows",
4480
+ )
4481
+ if result is None:
4482
+ raise AcquisitionSecurityError(
4483
+ "SANDBOX_FAILURE", "The Clean room process returned no result"
4484
+ )
4485
+ return result
4486
+
4487
+
4488
+ def _run_worker_command(
4489
+ command: list[str],
4490
+ *,
4491
+ cwd: Path,
4492
+ env: dict[str, str],
4493
+ request_bytes: bytes,
4494
+ timeout_seconds: float,
4495
+ memory_cgroup_root_fd: int | None,
4496
+ hosted_session_container_memory_bytes: int | None = None,
4497
+ ) -> tuple[bytes, bytes, int]:
4498
+ """Spawn one child and run one request against it: the cold path, start to finish."""
4499
+
4500
+ launch = _WorkerLaunch(
4501
+ command=tuple(command),
4502
+ cwd=cwd,
4503
+ env=tuple(sorted(env.items())),
4504
+ prewarm=False,
4505
+ )
4506
+ prepared = _prepare_worker(
4507
+ launch,
4508
+ timeout_seconds=timeout_seconds,
4509
+ memory_cgroup_root_fd=memory_cgroup_root_fd,
4510
+ hosted_session_container_memory_bytes=hosted_session_container_memory_bytes,
4511
+ track_deadline=True,
4512
+ )
4513
+ return _complete_worker(
4514
+ prepared,
4515
+ request_bytes=request_bytes,
4516
+ timeout_seconds=timeout_seconds,
4517
+ terminate_process_group=hosted_session_container_memory_bytes is not None,
4518
+ )
4519
+
4520
+
4521
+ class _WarmSandboxPool:
4522
+ """Pre-spawned, pre-imported children that have not been bound to a request.
4523
+
4524
+ Warm means spawned early and finished importing -- it never means reused. Every child
4525
+ here is handed out at most once, serves exactly the one request it is handed, and exits;
4526
+ :func:`worker_main` reads stdin once and returns, so there is no code path by which a
4527
+ second request could reach a child that has already served one. What pooling removes is
4528
+ the interpreter start and the module imports, which are the same work for every child and
4529
+ depend on nothing a request supplies.
4530
+
4531
+ A child sitting here holds no request, no credential, and no coordinator state: it was
4532
+ exec'd with the same closed environment and closed descriptor set as a cold child and is
4533
+ blocked reading an empty stdin. The one thing a warm child has that a cold child does not
4534
+ is time already spent, and time is not authority.
4535
+
4536
+ Every method that reads or writes ``_ready`` holds ``_lock``. Handing one child to two
4537
+ callers is the one way this class could break the invariant it exists to keep, and it is
4538
+ reachable without it: the liveness check releases the GIL, so two threads could select the
4539
+ same child between the scan and the reassignment. No caller is threaded today. The lock is
4540
+ here because the invariant is stated unconditionally, and an invariant that depends on the
4541
+ caller staying single-threaded is a comment rather than a guarantee.
4542
+ """
4543
+
4544
+ def __init__(
4545
+ self,
4546
+ *,
4547
+ timeout_seconds: float,
4548
+ memory_cgroup_root_fd: int | None,
4549
+ hosted_session_container_memory_bytes: int | None = None,
4550
+ ) -> None:
4551
+ self._timeout_seconds = timeout_seconds
4552
+ self._memory_cgroup_root_fd = memory_cgroup_root_fd
4553
+ self._hosted_session_container_memory_bytes = hosted_session_container_memory_bytes
4554
+ self._ready: list[_PreparedWorker] = []
4555
+ self._lock = threading.Lock()
4556
+
4557
+ def fill(self, launch: _WorkerLaunch, *, size: int) -> int:
4558
+ """Bring the pool up to ``size`` live children ready for exactly ``launch``.
4559
+
4560
+ One readiness budget covers the whole call. A child that fails to start leaves the
4561
+ ones already started in the pool rather than unwinding them, so a caller that wraps
4562
+ this in a lifecycle still has something to close.
4563
+ """
4564
+
4565
+ with self._lock:
4566
+ self._prune(launch)
4567
+ deadline = time.monotonic() + WARM_POOL_FILL_TIMEOUT_SECONDS
4568
+ while len(self._ready) < size:
4569
+ self._ready.append(
4570
+ _prepare_worker(
4571
+ launch,
4572
+ timeout_seconds=self._timeout_seconds,
4573
+ memory_cgroup_root_fd=self._memory_cgroup_root_fd,
4574
+ hosted_session_container_memory_bytes=(
4575
+ self._hosted_session_container_memory_bytes
4576
+ ),
4577
+ track_deadline=False,
4578
+ ready_timeout_seconds=max(deadline - time.monotonic(), 0.0),
4579
+ )
4580
+ )
4581
+ return len(self._ready)
4582
+
4583
+ def take(self, launch: _WorkerLaunch) -> _PreparedWorker | None:
4584
+ """Hand out one live child started from exactly ``launch``, or nothing.
4585
+
4586
+ Nothing is a complete answer: the caller spawns its own child, which is what it would
4587
+ have done anyway. A near-match is never a match, because the launch specification is
4588
+ the whole of a child's confinement.
4589
+
4590
+ A child that died while it was idle -- an operator kill, a host-level OOM -- is reaped
4591
+ here and skipped rather than handed out, so the pool cannot turn a dead process into a
4592
+ failed probe. That is the property the pool is meant to have: every way it can be
4593
+ unhelpful degrades to the cold path, and none of them degrade to an error.
4594
+ """
4595
+
4596
+ with self._lock:
4597
+ remaining: list[_PreparedWorker] = []
4598
+ taken: _PreparedWorker | None = None
4599
+ for prepared in self._ready:
4600
+ if taken is not None or prepared.launch.confinement != launch.confinement:
4601
+ remaining.append(prepared)
4602
+ continue
4603
+ if _worker_is_running(prepared.process):
4604
+ taken = prepared
4605
+ continue
4606
+ _discard_prepared_worker(prepared, already_exited=True)
4607
+ self._ready = remaining
4608
+ return taken
4609
+
4610
+ def _prune(self, launch: _WorkerLaunch | None) -> None:
4611
+ """Drop every child that is not live, or not launched from exactly ``launch``.
4612
+
4613
+ A refill has to count live children rather than remembered ones. Without this the
4614
+ pool would report a child an operator killed hours ago as ready, and topping up to
4615
+ ``size`` would keep the corpse and add nothing. ``launch`` of ``None`` prunes on
4616
+ liveness alone, which is what a caller asking only for a count wants.
4617
+
4618
+ Callers hold ``_lock``.
4619
+ """
4620
+
4621
+ keep: list[_PreparedWorker] = []
4622
+ for prepared in self._ready:
4623
+ if launch is not None and prepared.launch.confinement != launch.confinement:
4624
+ _discard_prepared_worker(prepared)
4625
+ elif _worker_is_running(prepared.process):
4626
+ keep.append(prepared)
4627
+ else:
4628
+ _discard_prepared_worker(prepared, already_exited=True)
4629
+ self._ready = keep
4630
+
4631
+ def ready_count(self) -> int:
4632
+ """How many children are idle *and* still alive.
4633
+
4634
+ The liveness scan is the whole point: a count that includes children the host killed
4635
+ while they were idle is the misreport this method exists to avoid, and a caller sizing
4636
+ a refill from it would under-provision by exactly the number of corpses.
4637
+ """
4638
+
4639
+ with self._lock:
4640
+ self._prune(None)
4641
+ return len(self._ready)
4642
+
4643
+ def close(self) -> None:
4644
+ with self._lock:
4645
+ ready, self._ready = self._ready, []
4646
+ for prepared in ready:
4647
+ _discard_prepared_worker(prepared)
4648
+
4649
+
4650
+ def _apply_resource_limits(nproc_limit: int) -> None:
4651
+ resource.setrlimit(resource.RLIMIT_CPU, (30, 30))
4652
+ resource.setrlimit(resource.RLIMIT_NOFILE, (64, 64))
4653
+ resource.setrlimit(resource.RLIMIT_FSIZE, (64 * 1024 * 1024, 64 * 1024 * 1024))
4654
+ if hasattr(resource, "RLIMIT_NPROC"):
4655
+ resource.setrlimit(resource.RLIMIT_NPROC, (nproc_limit, nproc_limit))
4656
+
4657
+
4658
+ def _nproc_limit() -> int:
4659
+ """Pick the RLIMIT_NPROC cap in the coordinator before forking the worker.
4660
+
4661
+ macOS charges RLIMIT_NPROC only on fork, so an absolute cap of 32 blocks the
4662
+ worker from spawning processes without limiting its threads. Linux charges
4663
+ every task (process or thread) of the real UID system-wide against the cap,
4664
+ so the same absolute 32 makes the worker's first pthread_create fail on any
4665
+ host whose UID already runs that many tasks (CI runners do) and its BLAS
4666
+ runtime aborts. Linux therefore gets the UID's current tasks plus bounded
4667
+ headroom for the worker and its BLAS/Arrow thread pools: runaway task
4668
+ creation still fails closed, at the same order of magnitude as before.
4669
+ """
4670
+
4671
+ if sys.platform != "linux":
4672
+ return 32
4673
+ limit = _linux_uid_task_count() + 32 + 4 * (os.cpu_count() or 1)
4674
+ _soft, hard = resource.getrlimit(resource.RLIMIT_NPROC)
4675
+ if hard != resource.RLIM_INFINITY:
4676
+ limit = min(limit, hard)
4677
+ return limit
4678
+
4679
+
4680
+ def _linux_uid_task_count() -> int:
4681
+ """Count the real UID's current tasks via /proc.
4682
+
4683
+ Entries hidden by hidepid, owned by another UID, or gone mid-scan are
4684
+ skipped; undercounting only lowers the limit, which fails closed.
4685
+ """
4686
+
4687
+ uid = os.getuid()
4688
+ tasks = 0
4689
+ try:
4690
+ with os.scandir("/proc") as entries:
4691
+ for entry in entries:
4692
+ if not entry.name.isdigit():
4693
+ continue
4694
+ try:
4695
+ if entry.stat(follow_symlinks=False).st_uid != uid:
4696
+ continue
4697
+ tasks += len(os.listdir(os.path.join(entry.path, "task")))
4698
+ except OSError:
4699
+ continue
4700
+ except OSError:
4701
+ return 0
4702
+ return tasks
4703
+
4704
+
4705
+ def _worker_error(code: str, detail: str) -> int:
4706
+ value = {"status": "error", "code": code, "detail": detail[:MAX_CARRIED_DETAIL]}
4707
+ sys.stderr.buffer.write(_ipc_json_bytes(value))
4708
+ sys.stderr.buffer.flush()
4709
+ return 1
4710
+
4711
+
4712
+ def _bounded_worker_error(raw: bytes, *, returncode: int) -> _WorkerRefusal:
4713
+ """Recover the worker's refusal from a stderr stream it does not own alone.
4714
+
4715
+ The worker writes exactly one single-line JSON object, but the interpreter may have
4716
+ written to the same stream first: running ``-m`` on a module the package's ``__init__``
4717
+ has already imported emits a runpy ``RuntimeWarning``, and that one line was enough to
4718
+ make every fail-closed refusal arrive at the coordinator as "redacted", losing the code a
4719
+ person needs. The whole buffer is tried first and the last non-empty line second; a
4720
+ stream that yields neither is still redacted rather than guessed at.
4721
+
4722
+ **The sentence crosses too, and only for a Reader refusal.** A code alone was not enough:
4723
+ the weather family composes the pair of templates it refused, the constant to edit and the
4724
+ three-step fix entirely inside ``ReaderError.detail``, and none of those facts is
4725
+ recoverable from a static code-to-fix map, because the code is only ``READER_GRID_UNKNOWN``.
4726
+ Two documents promised the operator that sentence, so it is carried -- under the narrowest
4727
+ rule that delivers it:
4728
+
4729
+ * only when the code is in ``READER_ERROR_CODES``, the closed table a Reader may raise from.
4730
+ Every other code -- ``SANDBOX_*`` and anything a caller invented -- carries nothing, so
4731
+ no failure inside the confinement can describe the confinement;
4732
+ * only after ``_readable_detail`` accepts it, which bounds the length and admits printable
4733
+ single-line text and nothing else.
4734
+
4735
+ A Reader is handed bytes and never a path -- the container module asserts structurally that
4736
+ no filesystem-taking name appears in it -- so a Reader refusal has no staging path, no
4737
+ temporary directory and no environment to name. That is why this is the code set where
4738
+ carrying the sentence is safe, and it is asserted rather than assumed in
4739
+ ``tests/h3/test_sandbox.py``.
4740
+
4741
+ When no candidate yields a code at all the refusal is still redacted, but it carries the
4742
+ fixed-shape ``_worker_exit_diagnostic``: trusted kernel exit status plus a one-way digest
4743
+ of the first stderr line. A worker killed by a signal, or one whose stderr never became
4744
+ JSON, is then distinguishable in logs without anything the worker wrote reaching a person
4745
+ verbatim.
4746
+ """
4747
+
4748
+ lines = [line for line in raw.splitlines() if line.strip()]
4749
+ for candidate in (raw, lines[-1] if lines else b""):
4750
+ try:
4751
+ value = _strict_json_object(candidate, "sandbox.error")
4752
+ code = _bounded_text(value.get("code"), maximum=128)
4753
+ except AcquisitionSecurityError:
4754
+ continue
4755
+ detail = _readable_detail(value.get("detail")) if code in READER_ERROR_CODES else None
4756
+ return _WorkerRefusal(code, code if detail is None else f"{code}: {detail}")
4757
+ return _WorkerRefusal(
4758
+ None,
4759
+ f"worker returned a redacted failure ({_worker_exit_diagnostic(raw, returncode)})",
4760
+ )
4761
+
4762
+
4763
+ def _worker_exit_diagnostic(raw: bytes, returncode: int) -> str:
4764
+ """Fixed-shape diagnostic from the kernel exit status and a stderr digest.
4765
+
4766
+ Only trusted kernel metadata and a one-way SHA-256 prefix of the first
4767
+ stderr line appear; nothing the untrusted worker wrote reaches the error
4768
+ verbatim, so signal kills and known failure signatures are distinguishable
4769
+ in logs without weakening redaction.
4770
+ """
4771
+
4772
+ if returncode < 0:
4773
+ try:
4774
+ status = f"signal={signal.Signals(-returncode).name}"
4775
+ except ValueError:
4776
+ status = f"signal={-returncode}"
4777
+ else:
4778
+ status = f"exit={returncode}"
4779
+ if not raw:
4780
+ return f"{status}, stderr=empty"
4781
+ first_line = raw.split(b"\n", 1)[0].rstrip(b"\r")
4782
+ return f"{status}, stderr={len(raw)}B, first-line-sha256={sha256_bytes(first_line)[:16]}"
4783
+
4784
+
4785
+ # How much of a Reader's own sentence crosses the boundary. The worker truncates at the same
4786
+ # number when it writes one, so the two are the same bound stated once on each side rather than
4787
+ # two bounds that can drift. The longest refusal any shipped family composes -- the weather
4788
+ # family's grid-template halt, which carries the pair, the constant and the three-step edit --
4789
+ # is comfortably inside it.
4790
+ MAX_CARRIED_DETAIL = 1_000
4791
+
4792
+
4793
+ def _readable_detail(value: Any) -> str | None:
4794
+ """The worker's sentence if it is plain readable text, and ``None`` otherwise.
4795
+
4796
+ Deliberately strict and deliberately silent: a detail this refuses is dropped rather than
4797
+ escalated, because the code has already been recovered and losing the sentence is strictly
4798
+ better than failing the refusal itself. Printable single-line text only -- no control
4799
+ characters, so nothing can rewrite a terminal or forge a second log line, and no surrogate,
4800
+ so the string survives every encoder it will pass through on the way to a person.
4801
+ """
4802
+
4803
+ if not isinstance(value, str) or not value.strip() or len(value) > MAX_CARRIED_DETAIL:
4804
+ return None
4805
+ if any(unicodedata.category(char) in {"Cc", "Cf", "Cs", "Co", "Cn"} for char in value):
4806
+ return None
4807
+ return value.strip()
4808
+
4809
+
4810
+ def _ipc_json_bytes(value: Any) -> bytes:
4811
+ try:
4812
+ return json.dumps(
4813
+ value,
4814
+ ensure_ascii=False,
4815
+ allow_nan=False,
4816
+ sort_keys=True,
4817
+ separators=(",", ":"),
4818
+ ).encode("utf-8", errors="strict")
4819
+ except (TypeError, ValueError, UnicodeEncodeError):
4820
+ raise AcquisitionSecurityError(
4821
+ "SANDBOX_JSON",
4822
+ "sandbox message is not strict JSON",
4823
+ ) from None
4824
+
4825
+
4826
+ def _bounded_identifier(value: Any) -> str:
4827
+ text = _bounded_text(value, maximum=128)
4828
+ if not text[0].islower() or not all(
4829
+ char.islower() or char.isdigit() or char in "._-" for char in text
4830
+ ):
4831
+ raise AcquisitionSecurityError(
4832
+ "SANDBOX_IDENTIFIER",
4833
+ "request ID must be a lowercase bounded identifier",
4834
+ )
4835
+ return text
4836
+
4837
+
4838
+ def _bounded_text(value: Any, *, maximum: int) -> str:
4839
+ if not isinstance(value, str) or not value or len(value) > maximum:
4840
+ raise AcquisitionSecurityError("SANDBOX_TEXT", "sandbox text field is invalid")
4841
+ if any(0xD800 <= ord(char) <= 0xDFFF for char in value):
4842
+ raise AcquisitionSecurityError("SANDBOX_TEXT", "sandbox text contains invalid Unicode")
4843
+ return value
4844
+
4845
+
4846
+ def _optional_text(value: Any, maximum: int) -> str | None:
4847
+ return None if value is None else _bounded_text(value, maximum=maximum)
4848
+
4849
+
4850
+ def _string_tuple(value: Any, *, maximum_items: int, item_maximum: int) -> tuple[str, ...]:
4851
+ if not isinstance(value, list) or len(value) > maximum_items:
4852
+ raise AcquisitionSecurityError("SANDBOX_ARRAY", "sandbox string array is invalid")
4853
+ result = tuple(_bounded_text(item, maximum=item_maximum) for item in value)
4854
+ if len(set(result)) != len(result):
4855
+ raise AcquisitionSecurityError("SANDBOX_ARRAY", "sandbox string array has duplicates")
4856
+ return result
4857
+
4858
+
4859
+ def _digest(value: Any) -> str:
4860
+ text = _bounded_text(value, maximum=64)
4861
+ if len(text) != 64 or any(char not in "0123456789abcdef" for char in text):
4862
+ raise AcquisitionSecurityError("SANDBOX_DIGEST", "sandbox digest is invalid")
4863
+ return text
4864
+
4865
+
4866
+ def _require_digest(value: Any, label: str) -> str:
4867
+ try:
4868
+ return _digest(value)
4869
+ except AcquisitionSecurityError as exc:
4870
+ raise AcquisitionSecurityError(
4871
+ "SANDBOX_OS_BOUNDARY",
4872
+ f"{label} attestation must be a lowercase SHA-256 digest",
4873
+ ) from exc
4874
+
4875
+
4876
+ def _optional_digest(value: Any) -> str | None:
4877
+ return None if value is None else _digest(value)
4878
+
4879
+
4880
+ if __name__ == "__main__":
4881
+ # Two argv shapes and no parser: the set is closed, and anything outside it exits before a
4882
+ # byte of stdin is read. ``--prewarm`` selects when the imports happen, never what the
4883
+ # child is allowed to do.
4884
+ if tuple(sys.argv[1:]) == WORKER_ARGV:
4885
+ raise SystemExit(worker_main())
4886
+ if tuple(sys.argv[1:]) == WORKER_PREWARM_ARGV:
4887
+ raise SystemExit(worker_main(prewarm=True))
4888
+ raise SystemExit(2)