mostlyright-data 0.9.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (314) hide show
  1. mostlyright/data_harness/__init__.py +158 -0
  2. mostlyright/data_harness/acquisition/__init__.py +55 -0
  3. mostlyright/data_harness/acquisition/http.py +2773 -0
  4. mostlyright/data_harness/acquisition/parsing.py +809 -0
  5. mostlyright/data_harness/acquisition/ranges.py +495 -0
  6. mostlyright/data_harness/acquisition/result_download.py +360 -0
  7. mostlyright/data_harness/acquisition/retention_admission.py +248 -0
  8. mostlyright/data_harness/acquisition/sandbox.py +4888 -0
  9. mostlyright/data_harness/acquisition/url_policy.py +530 -0
  10. mostlyright/data_harness/agent_runtime.py +2743 -0
  11. mostlyright/data_harness/assets/logo-ink.svg +31 -0
  12. mostlyright/data_harness/backends/__init__.py +28 -0
  13. mostlyright/data_harness/backends/pandas_backend.py +350 -0
  14. mostlyright/data_harness/backends/polars_backend.py +366 -0
  15. mostlyright/data_harness/backends/protocol.py +124 -0
  16. mostlyright/data_harness/backends/reference.py +83 -0
  17. mostlyright/data_harness/backends/registry.py +55 -0
  18. mostlyright/data_harness/backends/restrictions.py +126 -0
  19. mostlyright/data_harness/canonical.py +333 -0
  20. mostlyright/data_harness/catalog_job.py +625 -0
  21. mostlyright/data_harness/cli.py +5398 -0
  22. mostlyright/data_harness/contracts.py +53 -0
  23. mostlyright/data_harness/coordinator.py +1307 -0
  24. mostlyright/data_harness/deploy.py +924 -0
  25. mostlyright/data_harness/deploy_target.py +312 -0
  26. mostlyright/data_harness/deployment_evidence.py +1067 -0
  27. mostlyright/data_harness/event_presentation.py +576 -0
  28. mostlyright/data_harness/events.py +2152 -0
  29. mostlyright/data_harness/fast_delimited.py +239 -0
  30. mostlyright/data_harness/fleet.py +237 -0
  31. mostlyright/data_harness/formats.py +236 -0
  32. mostlyright/data_harness/governors.py +1163 -0
  33. mostlyright/data_harness/hosted_bootstrap.py +972 -0
  34. mostlyright/data_harness/hosted_crawler.py +1115 -0
  35. mostlyright/data_harness/hosted_crawler_container_smoke.py +351 -0
  36. mostlyright/data_harness/hosted_crawler_fetch.py +423 -0
  37. mostlyright/data_harness/hosted_crawler_job.py +1277 -0
  38. mostlyright/data_harness/hosted_crawler_protocol.py +676 -0
  39. mostlyright/data_harness/hosted_dataset.py +1500 -0
  40. mostlyright/data_harness/hosted_deploy.py +3037 -0
  41. mostlyright/data_harness/hosted_handoff.py +62 -0
  42. mostlyright/data_harness/hosted_ingestion_contract.py +504 -0
  43. mostlyright/data_harness/hosted_ingestion_job.py +356 -0
  44. mostlyright/data_harness/hosted_ingestion_job_smoke.py +40 -0
  45. mostlyright/data_harness/hosted_session_container_smoke.py +194 -0
  46. mostlyright/data_harness/hosted_session_worker.py +3554 -0
  47. mostlyright/data_harness/hosted_session_worker_job_smoke.py +46 -0
  48. mostlyright/data_harness/hosted_worker.py +6784 -0
  49. mostlyright/data_harness/ingestion/__init__.py +56 -0
  50. mostlyright/data_harness/ingestion/contracts.py +461 -0
  51. mostlyright/data_harness/ingestion/faults.py +42 -0
  52. mostlyright/data_harness/ingestion/gcs_store.py +1162 -0
  53. mostlyright/data_harness/ingestion/spool.py +130 -0
  54. mostlyright/data_harness/ingestion/store.py +885 -0
  55. mostlyright/data_harness/key_seam.py +434 -0
  56. mostlyright/data_harness/linux_process_boundary.py +262 -0
  57. mostlyright/data_harness/local_contracts.py +2880 -0
  58. mostlyright/data_harness/local_search/__init__.py +5 -0
  59. mostlyright/data_harness/local_search/build_index.py +1087 -0
  60. mostlyright/data_harness/local_search/contracts.py +920 -0
  61. mostlyright/data_harness/local_search/query_trace.py +266 -0
  62. mostlyright/data_harness/local_search/retrieval.py +700 -0
  63. mostlyright/data_harness/local_search/sealed.py +474 -0
  64. mostlyright/data_harness/local_search/service.py +784 -0
  65. mostlyright/data_harness/nbrender/CONTRACT.md +212 -0
  66. mostlyright/data_harness/nbrender/__init__.py +12 -0
  67. mostlyright/data_harness/nbrender/chrome.py +359 -0
  68. mostlyright/data_harness/nbrender/code_body.py +266 -0
  69. mostlyright/data_harness/nbrender/document.py +407 -0
  70. mostlyright/data_harness/nbrender/frame.py +275 -0
  71. mostlyright/data_harness/nbrender/interactive.py +337 -0
  72. mostlyright/data_harness/nbrender/markdown_body.py +477 -0
  73. mostlyright/data_harness/nbrender/mr_components.py +134 -0
  74. mostlyright/data_harness/nbrender/outputs_data.py +595 -0
  75. mostlyright/data_harness/nbrender/outputs_rich.py +906 -0
  76. mostlyright/data_harness/nbrender/outputs_source.py +260 -0
  77. mostlyright/data_harness/nbrender/outputs_stage.py +176 -0
  78. mostlyright/data_harness/nbrender/outputs_text.py +400 -0
  79. mostlyright/data_harness/nbrender/parse.py +394 -0
  80. mostlyright/data_harness/nbrender/status.py +40 -0
  81. mostlyright/data_harness/nbrender/tokens.py +1295 -0
  82. mostlyright/data_harness/notebook.py +1710 -0
  83. mostlyright/data_harness/offline.py +2049 -0
  84. mostlyright/data_harness/operation_registry.py +1007 -0
  85. mostlyright/data_harness/operator_setup.py +239 -0
  86. mostlyright/data_harness/pipeline.py +6428 -0
  87. mostlyright/data_harness/plan_graph.py +2026 -0
  88. mostlyright/data_harness/preparation/__init__.py +104 -0
  89. mostlyright/data_harness/preparation/contracts.py +1017 -0
  90. mostlyright/data_harness/preparation/engine.py +221 -0
  91. mostlyright/data_harness/preparation/errors.py +14 -0
  92. mostlyright/data_harness/preparation/gates.py +751 -0
  93. mostlyright/data_harness/preparation/joins.py +574 -0
  94. mostlyright/data_harness/preparation/profile.py +384 -0
  95. mostlyright/data_harness/preparation/table.py +217 -0
  96. mostlyright/data_harness/preparation/transforms.py +568 -0
  97. mostlyright/data_harness/progress_events.py +534 -0
  98. mostlyright/data_harness/readers/__init__.py +46 -0
  99. mostlyright/data_harness/readers/containers.py +963 -0
  100. mostlyright/data_harness/readers/contracts.py +542 -0
  101. mostlyright/data_harness/readers/delimited.py +257 -0
  102. mostlyright/data_harness/readers/grib2/__init__.py +33 -0
  103. mostlyright/data_harness/readers/grib2/admission.py +722 -0
  104. mostlyright/data_harness/readers/grib2/decode.py +1009 -0
  105. mostlyright/data_harness/readers/grib2/geometry.py +1133 -0
  106. mostlyright/data_harness/readers/grib2/portable_math.py +501 -0
  107. mostlyright/data_harness/readers/json_tabular.py +485 -0
  108. mostlyright/data_harness/readers/registry.py +514 -0
  109. mostlyright/data_harness/readers/samples/README.md +110 -0
  110. mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/cities.csv.gz +0 -0
  111. mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/expected.json +24 -0
  112. mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/cities.csv.gz +0 -0
  113. mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/expected.json +24 -0
  114. mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/cities.tar +0 -0
  115. mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/expected.json +24 -0
  116. mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/cities.tar +0 -0
  117. mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/expected.json +24 -0
  118. mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/cities.zip +0 -0
  119. mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/expected.json +25 -0
  120. mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
  121. mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/expected.json +25 -0
  122. mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
  123. mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/expected.json +25 -0
  124. mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/cities.csv +3 -0
  125. mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/expected.json +23 -0
  126. mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/cities.tsv +5 -0
  127. mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/expected.json +25 -0
  128. mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/expected.json +30 -0
  129. mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/observations.csv +5 -0
  130. mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/expected.json +44 -0
  131. mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/stations.json +1 -0
  132. mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/expected.json +48 -0
  133. mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/observations.ndjson +4 -0
  134. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/cities.xlsx +0 -0
  135. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/expected.json +24 -0
  136. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/expected.json +27 -0
  137. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
  138. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/expected.json +27 -0
  139. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
  140. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/README.md +20 -0
  141. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/expected.json +55 -0
  142. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/gfs-2m-temperature.grib2 +0 -0
  143. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/expected.json +54 -0
  144. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
  145. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/expected.json +54 -0
  146. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/hrrr-categorical-rain.grib2 +0 -0
  147. mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/expected.json +54 -0
  148. mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
  149. mostlyright/data_harness/readers/samples.py +582 -0
  150. mostlyright/data_harness/readers/spreadsheet.py +803 -0
  151. mostlyright/data_harness/readers/tabular.py +510 -0
  152. mostlyright/data_harness/recipe.py +5321 -0
  153. mostlyright/data_harness/repair/__init__.py +78 -0
  154. mostlyright/data_harness/repair/adapters.py +274 -0
  155. mostlyright/data_harness/repair/contracts.py +872 -0
  156. mostlyright/data_harness/repair/coordinator.py +1099 -0
  157. mostlyright/data_harness/repair/errors.py +16 -0
  158. mostlyright/data_harness/review.py +2533 -0
  159. mostlyright/data_harness/rowset.py +283 -0
  160. mostlyright/data_harness/serving.py +1975 -0
  161. mostlyright/data_harness/serving_edge.py +590 -0
  162. mostlyright/data_harness/serving_http.py +1031 -0
  163. mostlyright/data_harness/session_probes.py +759 -0
  164. mostlyright/data_harness/signing.py +101 -0
  165. mostlyright/data_harness/source_discovery.py +898 -0
  166. mostlyright/data_harness/sources/__init__.py +209 -0
  167. mostlyright/data_harness/sources/_adapter_steps.py +213 -0
  168. mostlyright/data_harness/sources/adapters.py +1214 -0
  169. mostlyright/data_harness/sources/cadence.py +1428 -0
  170. mostlyright/data_harness/sources/cadence_emission.py +453 -0
  171. mostlyright/data_harness/sources/cadence_history.py +546 -0
  172. mostlyright/data_harness/sources/catalog/__init__.py +17 -0
  173. mostlyright/data_harness/sources/catalog/admission.py +477 -0
  174. mostlyright/data_harness/sources/catalog/authoring.py +1701 -0
  175. mostlyright/data_harness/sources/catalog/authoring_policy.py +701 -0
  176. mostlyright/data_harness/sources/catalog/authoring_shards.py +1217 -0
  177. mostlyright/data_harness/sources/catalog/bounded_io.py +231 -0
  178. mostlyright/data_harness/sources/catalog/channel.py +523 -0
  179. mostlyright/data_harness/sources/catalog/channel_client.py +296 -0
  180. mostlyright/data_harness/sources/catalog/contracts.py +825 -0
  181. mostlyright/data_harness/sources/catalog/coverage.py +137 -0
  182. mostlyright/data_harness/sources/catalog/delta.py +1340 -0
  183. mostlyright/data_harness/sources/catalog/embedding.py +532 -0
  184. mostlyright/data_harness/sources/catalog/entry_v2.py +1182 -0
  185. mostlyright/data_harness/sources/catalog/fill.py +3889 -0
  186. mostlyright/data_harness/sources/catalog/fill_partitions.py +459 -0
  187. mostlyright/data_harness/sources/catalog/fill_staging.py +1105 -0
  188. mostlyright/data_harness/sources/catalog/gating.py +374 -0
  189. mostlyright/data_harness/sources/catalog/generation_receipt.py +1607 -0
  190. mostlyright/data_harness/sources/catalog/harvest/__init__.py +7 -0
  191. mostlyright/data_harness/sources/catalog/harvest/ckan.py +384 -0
  192. mostlyright/data_harness/sources/catalog/harvest/datagov_v4.py +798 -0
  193. mostlyright/data_harness/sources/catalog/harvest/protocol.py +964 -0
  194. mostlyright/data_harness/sources/catalog/harvest/sdmx.py +445 -0
  195. mostlyright/data_harness/sources/catalog/harvest/stac.py +384 -0
  196. mostlyright/data_harness/sources/catalog/health.py +447 -0
  197. mostlyright/data_harness/sources/catalog/hosted_catalog.py +105 -0
  198. mostlyright/data_harness/sources/catalog/identity_history.py +1549 -0
  199. mostlyright/data_harness/sources/catalog/neural.py +1618 -0
  200. mostlyright/data_harness/sources/catalog/packed_catalog.py +2345 -0
  201. mostlyright/data_harness/sources/catalog/packed_retrieval.py +1517 -0
  202. mostlyright/data_harness/sources/catalog/packed_writer.py +2802 -0
  203. mostlyright/data_harness/sources/catalog/query_trace.py +1037 -0
  204. mostlyright/data_harness/sources/catalog/recommend.py +171 -0
  205. mostlyright/data_harness/sources/catalog/retrieval.py +230 -0
  206. mostlyright/data_harness/sources/catalog/retrieval_manifest.py +995 -0
  207. mostlyright/data_harness/sources/catalog/rights_decisions.py +254 -0
  208. mostlyright/data_harness/sources/catalog/sealed.py +560 -0
  209. mostlyright/data_harness/sources/catalog/search.py +230 -0
  210. mostlyright/data_harness/sources/catalog/streaming_delta.py +1097 -0
  211. mostlyright/data_harness/sources/catalog/update.py +891 -0
  212. mostlyright/data_harness/sources/collections.py +815 -0
  213. mostlyright/data_harness/sources/contracts.py +2223 -0
  214. mostlyright/data_harness/sources/deletion.py +761 -0
  215. mostlyright/data_harness/sources/fitness.py +162 -0
  216. mostlyright/data_harness/sources/governance.py +163 -0
  217. mostlyright/data_harness/sources/hosted.py +173 -0
  218. mostlyright/data_harness/sources/integration.py +218 -0
  219. mostlyright/data_harness/sources/range_reader.py +418 -0
  220. mostlyright/data_harness/sources/registry.py +514 -0
  221. mostlyright/data_harness/sources/rights_rule.py +59 -0
  222. mostlyright/data_harness/sources/source_cadence_vectors.v1.json +1 -0
  223. mostlyright/data_harness/sources/sports.py +521 -0
  224. mostlyright/data_harness/sources/stream.py +524 -0
  225. mostlyright/data_harness/sources/stream_connector.py +418 -0
  226. mostlyright/data_harness/sources/stream_recorder.py +1404 -0
  227. mostlyright/data_harness/studio_boundary.py +2019 -0
  228. mostlyright/data_harness/thin/__init__.py +37 -0
  229. mostlyright/data_harness/thin/acquire.py +1137 -0
  230. mostlyright/data_harness/thin/acquire_cancel.py +579 -0
  231. mostlyright/data_harness/thin/approvals.py +617 -0
  232. mostlyright/data_harness/thin/commands.py +406 -0
  233. mostlyright/data_harness/thin/download.py +194 -0
  234. mostlyright/data_harness/thin/narrative.py +589 -0
  235. mostlyright/data_harness/thin/parity.py +1070 -0
  236. mostlyright/data_harness/thin/propose.py +2759 -0
  237. mostlyright/data_harness/thin/research.py +1663 -0
  238. mostlyright/data_harness/thin/router.py +924 -0
  239. mostlyright/data_harness/thin/runs.py +519 -0
  240. mostlyright/data_harness/thin/session.py +281 -0
  241. mostlyright/data_harness/thin/stream.py +501 -0
  242. mostlyright/data_harness/thin/transport.py +187 -0
  243. mostlyright/data_harness/thin/vocabulary.py +368 -0
  244. mostlyright/data_harness/thin/workers.py +164 -0
  245. mostlyright/data_harness/ucum/TABLE-PIN.json +40 -0
  246. mostlyright/data_harness/ucum/ucum-subset.v1.json +632 -0
  247. mostlyright/data_harness/unit_flow.py +927 -0
  248. mostlyright/data_harness/units.py +572 -0
  249. mostlyright/data_harness/ux/__init__.py +9 -0
  250. mostlyright/data_harness/ux/approve.py +485 -0
  251. mostlyright/data_harness/ux/author_yaml.py +597 -0
  252. mostlyright/data_harness/ux/cloud_auth.py +447 -0
  253. mostlyright/data_harness/ux/commands/__init__.py +260 -0
  254. mostlyright/data_harness/ux/commands/approve.py +136 -0
  255. mostlyright/data_harness/ux/commands/auth.py +744 -0
  256. mostlyright/data_harness/ux/commands/author.py +79 -0
  257. mostlyright/data_harness/ux/commands/catalog_author.py +403 -0
  258. mostlyright/data_harness/ux/commands/catalog_fill.py +523 -0
  259. mostlyright/data_harness/ux/commands/catalog_harvest.py +545 -0
  260. mostlyright/data_harness/ux/commands/catalog_publish.py +1838 -0
  261. mostlyright/data_harness/ux/commands/catalog_search.py +71 -0
  262. mostlyright/data_harness/ux/commands/catalog_update.py +437 -0
  263. mostlyright/data_harness/ux/commands/deploy.py +134 -0
  264. mostlyright/data_harness/ux/commands/deploy_dataset.py +98 -0
  265. mostlyright/data_harness/ux/commands/deploy_plan.py +105 -0
  266. mostlyright/data_harness/ux/commands/deploy_status.py +104 -0
  267. mostlyright/data_harness/ux/commands/diff.py +74 -0
  268. mostlyright/data_harness/ux/commands/index.py +84 -0
  269. mostlyright/data_harness/ux/commands/inventory.py +47 -0
  270. mostlyright/data_harness/ux/commands/list_builds.py +143 -0
  271. mostlyright/data_harness/ux/commands/login.py +63 -0
  272. mostlyright/data_harness/ux/commands/peek.py +236 -0
  273. mostlyright/data_harness/ux/commands/plan_check.py +90 -0
  274. mostlyright/data_harness/ux/commands/preflight.py +97 -0
  275. mostlyright/data_harness/ux/commands/record.py +107 -0
  276. mostlyright/data_harness/ux/commands/review_setup.py +47 -0
  277. mostlyright/data_harness/ux/commands/search.py +440 -0
  278. mostlyright/data_harness/ux/commands/show.py +61 -0
  279. mostlyright/data_harness/ux/commands/whoami.py +37 -0
  280. mostlyright/data_harness/ux/credential_native.py +551 -0
  281. mostlyright/data_harness/ux/credential_store.py +1055 -0
  282. mostlyright/data_harness/ux/credentials.py +631 -0
  283. mostlyright/data_harness/ux/diffing.py +444 -0
  284. mostlyright/data_harness/ux/headline.py +671 -0
  285. mostlyright/data_harness/ux/hosted_acquisition.py +974 -0
  286. mostlyright/data_harness/ux/hosted_run_status.py +619 -0
  287. mostlyright/data_harness/ux/inventory.py +427 -0
  288. mostlyright/data_harness/ux/local_review.py +375 -0
  289. mostlyright/data_harness/ux/login.py +691 -0
  290. mostlyright/data_harness/ux/path_kind.py +147 -0
  291. mostlyright/data_harness/ux/peek.py +1000 -0
  292. mostlyright/data_harness/ux/plain_file.py +178 -0
  293. mostlyright/data_harness/ux/plan_check.py +311 -0
  294. mostlyright/data_harness/ux/preflight.py +918 -0
  295. mostlyright/data_harness/ux/readers.py +1124 -0
  296. mostlyright/data_harness/ux/remediation.py +2195 -0
  297. mostlyright/data_harness/ux/render.py +657 -0
  298. mostlyright/data_harness/ux/workload.py +1077 -0
  299. mostlyright/data_harness/viewer.py +3713 -0
  300. mostlyright/data_harness/visual_run/__init__.py +83 -0
  301. mostlyright/data_harness/visual_run/authoring.py +235 -0
  302. mostlyright/data_harness/visual_run/contracts.py +673 -0
  303. mostlyright/data_harness/visual_run/materialize.py +486 -0
  304. mostlyright/data_harness/visual_run/observations.py +874 -0
  305. mostlyright/data_harness/visual_run/query.py +259 -0
  306. mostlyright/data_harness/visual_run/reducer.py +280 -0
  307. mostlyright/data_harness/visual_run/sdk.py +892 -0
  308. mostlyright/data_harness/visual_run/store.py +584 -0
  309. mostlyright/data_harness/visual_run/transport.py +239 -0
  310. mostlyright/data_harness/watch.py +2999 -0
  311. mostlyright_data-0.9.0.dist-info/METADATA +607 -0
  312. mostlyright_data-0.9.0.dist-info/RECORD +314 -0
  313. mostlyright_data-0.9.0.dist-info/WHEEL +4 -0
  314. mostlyright_data-0.9.0.dist-info/entry_points.txt +12 -0
@@ -0,0 +1,434 @@
1
+ """The one seam between the two planes: cloud owns identity, the harness verifies against it.
2
+
3
+ Cloud and the harness are separate repositories with separate owners, and exactly one contract
4
+ joins them: an API key presented by a caller is verified against cloud's verify surface. The
5
+ authority for that contract is ``mostlyright-cloud/manifest/api-key.ts``. This module is the
6
+ harness side of it and must stay byte-compatible with that file.
7
+
8
+ Three rules are copied here verbatim from api-key.ts's header comment, because every one of them
9
+ is security-load-bearing and every one of them looks like dead weight to a later reader:
10
+
11
+ * Hash both sides to fixed-length digests before comparing. The comparison then runs over two
12
+ 32-byte values whatever the caller presented.
13
+ * **Never pre-check length.** A length pre-check answers "how long is the expected key" in the
14
+ time it takes to return, which is the question an attacker is asking.
15
+ * **Never wrap the compare in try/except.** Throwing is measurably faster than a full compare, so
16
+ a raising path reintroduces exactly the timing channel that hashing exists to close. Digesting
17
+ first is what makes the compare total, so there is nothing left to catch.
18
+
19
+ Fail-closed is the documented default for everything in this module. A key is authorized only by
20
+ an explicit, well-formed allow from the verify surface reached over https. Any other outcome — a
21
+ non-200 status, a redirect, a transport fault, a timeout, a body that is not strict JSON, a body
22
+ over the cap, a body that names a different key — denies.
23
+
24
+ The raw key never leaves this module. It goes out in one place, the ``x-api-key`` request header,
25
+ and it appears in no return value, no :class:`KeyVerdict`, no serialized dict, no reason string,
26
+ and no exception message. Only :func:`hash_key_id` output is safe to log.
27
+
28
+ **Where this seam is enforced, and why not in the worker.** The enforcement point is named in
29
+ :data:`CONSUMER_CALL_SITE`, and it is deliberately not this repository's hosted worker. Three
30
+ measured facts establish that, and they are recorded here so a later reader does not "fix" an
31
+ omission that is not one:
32
+
33
+ 1. ``hosted_bootstrap.launch`` hands the worker an environment of exactly three variables — PATH,
34
+ PYTHONDONTWRITEBYTECODE, PYTHONUNBUFFERED (the environment mapping at hosted_bootstrap.py:475).
35
+ No API key can reach the worker process through it. Passing one would mean deliberately
36
+ widening that barrier to carry a customer credential into the least-trusted process in the
37
+ system, which inverts the design the barrier exists to hold.
38
+ 2. ``mostlyright-studio``'s ``infra/tofu`` declares no ``MOSTLYRIGHT_API_KEY`` on the worker job,
39
+ so the credential is not present in that container today.
40
+ 3. The worker authenticates with an attempt-scoped capability token minted by Studio during the
41
+ bootstrap exchange, and its tenant coordinate is ``workspace_id``. The customer API key is
42
+ verified upstream, where the request enters the platform, before any job exists to run.
43
+
44
+ So this module is the harness-side reference implementation and parity anchor for a check that
45
+ runs at the API edge, not a library the worker forgot to call. It is intentionally not imported by
46
+ ``producer_main`` or ``verifier_main``; wiring it in there would punch a credential through the
47
+ sanitization barrier to no benefit. ``tests/test_key_seam_consumer_contract.py`` pins the two
48
+ implementations against each other.
49
+ """
50
+
51
+ from __future__ import annotations
52
+
53
+ import hashlib
54
+ import hmac
55
+ import ssl
56
+ from collections.abc import Mapping
57
+ from dataclasses import dataclass
58
+ from typing import Any, Protocol
59
+ from urllib.error import HTTPError
60
+ from urllib.parse import urlsplit
61
+ from urllib.request import (
62
+ HTTPRedirectHandler,
63
+ HTTPSHandler,
64
+ ProxyHandler,
65
+ Request,
66
+ build_opener,
67
+ )
68
+
69
+ from mostlyright.data_harness.canonical import CanonicalJSONError, parse_json
70
+
71
+ #: The environment variable holding the configured legacy v1 shared key. Matches
72
+ #: ``LEGACY_API_KEY_ENV`` at api-key.ts:27 — one shared secret, one env var name, on both the
73
+ #: sending and the receiving side.
74
+ LEGACY_API_KEY_ENV = "MOSTLYRIGHT_API_KEY"
75
+
76
+ #: The only header the credential is ever read from or sent in. Matches ``API_KEY_HEADER`` at
77
+ #: manifest-handler.ts:25. Never a query string, never a URL path segment: those are logged by
78
+ #: every proxy, load balancer, and browser history on the route.
79
+ API_KEY_HEADER = "x-api-key"
80
+
81
+ V2_LIVE_PREFIX = "mr_live_"
82
+ V2_TEST_PREFIX = "mr_test_"
83
+
84
+ #: Where a presented API key is actually refused or allowed, recorded in code because it is not in
85
+ #: this repository and an unrecorded consumer reads as a missing one. Named consumers: the Studio
86
+ #: API edge that authenticates a customer request before any run exists, and the worker-bootstrap
87
+ #: exchange that mints the attempt-scoped capability the worker carries instead of a key
88
+ #: (``mostlyright-cloud/manifest/manifest-handler.ts`` is the same check on the manifest surface).
89
+ #: This module is the harness-side reference implementation and parity anchor for that check; see
90
+ #: the module docstring for the three facts establishing why the worker must never hold a key.
91
+ CONSUMER_CALL_SITE = (
92
+ "mostlyright-studio API edge and worker-bootstrap exchange; "
93
+ "mostlyright-cloud/manifest/manifest-handler.ts on the manifest surface"
94
+ )
95
+
96
+ #: A verify response is a verdict, not a payload. Anything larger than this is not a verdict, so
97
+ #: the cap is small on purpose — it bounds memory and it bounds the time an attacker can hold the
98
+ #: connection open. Compare ``MAX_BOOTSTRAP_RESPONSE_BYTES`` (hosted_bootstrap.py:50), which is
99
+ #: large because a bootstrap envelope legitimately is.
100
+ MAX_VERIFY_RESPONSE_BYTES = 8 * 1024
101
+
102
+ #: A bounded wait. A verify surface that has not answered by now has failed, and a hang is a
103
+ #: denial of service against every caller waiting behind it.
104
+ VERIFY_TIMEOUT_SECONDS = 10.0
105
+
106
+ MAX_URL_LENGTH = 2048
107
+
108
+ #: The allow marker. A 200 alone is not an allow: an intercepting proxy, a captive portal, and a
109
+ #: misrouted request all return 200 with a body that means nothing. The body must be a JSON object
110
+ #: that says so explicitly.
111
+ ALLOW_MARKER = "ok"
112
+
113
+
114
+ class KeySeamError(RuntimeError):
115
+ """A typed key-seam failure.
116
+
117
+ Raised only for a structurally invalid call — a base URL that is not https, a key that is not
118
+ text. A rejected key is never an exception; it is a denying :class:`KeyVerdict`.
119
+
120
+ Like everything else here, the message never contains the raw key.
121
+ """
122
+
123
+
124
+ def hash_key_id(key: str) -> str:
125
+ """Return a stable, irreversible identifier for ``key``.
126
+
127
+ Exactly cloud's ``hashKeyId`` (api-key.ts:49): the first 8 bytes of the SHA-256 digest of the
128
+ key's UTF-8 encoding, hex-encoded — 16 lowercase hex characters.
129
+
130
+ This is the only value derived from a key that is safe to log, record, or return. It is not
131
+ reversible and it is not a credential.
132
+
133
+ The coupling is load-bearing and invisible from either side alone: cloud's
134
+ ``dashboard/lib/usage.ts`` stores this same value in ``usage_events.key_id``. If either
135
+ implementation changes its truncation length or its string encoding, nothing raises anywhere —
136
+ usage counting for every key simply goes to zero, and attribution for a key becomes
137
+ unresolvable between the two planes. ``tests/test_key_seam.py`` pins the derivation with
138
+ literal vectors produced by running both implementations.
139
+ """
140
+
141
+ if not isinstance(key, str):
142
+ raise KeySeamError("an API key must be text")
143
+ return hashlib.sha256(key.encode("utf-8")).hexdigest()[:16]
144
+
145
+
146
+ def constant_time_equal(a: str, b: str) -> bool:
147
+ """Return whether ``a`` and ``b`` are equal, in time independent of either value.
148
+
149
+ Mirrors ``constantTimeEqual`` (api-key.ts:38). Both operands are digested to fixed 32-byte
150
+ SHA-256 values first, and only the digests reach :func:`hmac.compare_digest`.
151
+
152
+ The digest step is not decoration. It is what makes this function *total* — every pairing of
153
+ lengths, including empty and non-ASCII inputs, returns a bool and none raises — and it is what
154
+ makes the running time independent of the presented value's length. There is deliberately no
155
+ length pre-check (it would leak the expected key's length) and deliberately no try/except
156
+ around the compare (an early raise is faster than a full compare, which is the timing channel
157
+ this design closes).
158
+ """
159
+
160
+ if not isinstance(a, str) or not isinstance(b, str):
161
+ raise KeySeamError("a credential comparison takes two text values")
162
+ digest_a = hashlib.sha256(a.encode("utf-8")).digest()
163
+ digest_b = hashlib.sha256(b.encode("utf-8")).digest()
164
+ return hmac.compare_digest(digest_a, digest_b)
165
+
166
+
167
+ def detect_key_format(key: str) -> str:
168
+ """Classify a presented credential's shape as ``v2-live``, ``v2-test``, or ``legacy``.
169
+
170
+ Mirrors ``detectKeyFormat`` (api-key.ts:87). **Purely descriptive: it authorizes nothing.**
171
+
172
+ A ``v2-live`` classification says only that a string starts with eight specified characters.
173
+ Every format still resolves through the
174
+ same verification, exactly as cloud's v0 does. This module must never mint a v2 key and must
175
+ never independently validate one; ``key_format`` on a :class:`KeyVerdict` is a label for
176
+ humans, never an input to the decision.
177
+ """
178
+
179
+ if not isinstance(key, str):
180
+ raise KeySeamError("an API key must be text")
181
+ if key.startswith(V2_LIVE_PREFIX):
182
+ return "v2-live"
183
+ if key.startswith(V2_TEST_PREFIX):
184
+ return "v2-test"
185
+ return "legacy"
186
+
187
+
188
+ @dataclass(frozen=True)
189
+ class KeyVerdict:
190
+ """The outcome of verifying one presented key.
191
+
192
+ Invariant: a verdict never carries the raw key. It carries ``key_id``, the irreversible
193
+ identifier from :func:`hash_key_id`, which is the value cloud records too — so the same key is
194
+ attributed identically on both sides of the seam.
195
+
196
+ ``reason`` is a plain-language phrase safe to show a user and safe to write to a log. It names
197
+ the class of fault only. It never contains, quotes, truncates, or measures the presented key,
198
+ and it never reports the expected key's length.
199
+
200
+ ``key_format`` is descriptive (see :func:`detect_key_format`) and never explains ``ok``.
201
+ """
202
+
203
+ ok: bool
204
+ key_id: str
205
+ key_format: str
206
+ reason: str
207
+
208
+ def __post_init__(self) -> None:
209
+ if type(self.ok) is not bool:
210
+ raise KeySeamError("a verdict must be allowed or denied, not something else")
211
+ for name in ("key_id", "key_format", "reason"):
212
+ value = getattr(self, name)
213
+ if type(value) is not str or not value:
214
+ raise KeySeamError(f"{name} must be a non-empty text value")
215
+ if len(self.key_id) != 16:
216
+ raise KeySeamError("key_id must be the 16-character identifier from hash_key_id")
217
+
218
+ def to_dict(self) -> dict[str, Any]:
219
+ """Return a plain JSON-safe mapping, carrying the key id and never the key."""
220
+
221
+ return {
222
+ "ok": self.ok,
223
+ "key_id": self.key_id,
224
+ "key_format": self.key_format,
225
+ "reason": self.reason,
226
+ }
227
+
228
+
229
+ class VerifyTransport(Protocol):
230
+ """Minimal redirect-free, proxy-free verify transport.
231
+
232
+ Deliberately the same shape as ``BootstrapTransport`` (hosted_bootstrap.py:57) so this
233
+ codebase has one transport idiom rather than two: a bootstrap transport satisfies this
234
+ protocol structurally, and a test stub written for one reads the same as a stub for the other.
235
+ """
236
+
237
+ def request(
238
+ self,
239
+ method: str,
240
+ url: str,
241
+ headers: Mapping[str, str],
242
+ body: bytes | None,
243
+ maximum: int,
244
+ ) -> tuple[int, bytes]: ...
245
+
246
+
247
+ class _NoRedirect(HTTPRedirectHandler):
248
+ """Refuse every redirect.
249
+
250
+ Following a redirect would re-send the ``x-api-key`` header to a host named by the response
251
+ rather than by the caller, which is a credential handed to whoever answered. Returning ``None``
252
+ here makes urllib surface the 3xx as an ``HTTPError``, so the status reaches the caller as a
253
+ plain non-200 and denies. Mirrors hosted_bootstrap.py:70.
254
+ """
255
+
256
+ def redirect_request(
257
+ self,
258
+ request: Request,
259
+ file_pointer: Any,
260
+ code: int,
261
+ message: str,
262
+ headers: Any,
263
+ new_url: str,
264
+ ) -> None:
265
+ return None
266
+
267
+
268
+ class UrlLibVerifyTransport:
269
+ """Small stdlib transport with ambient proxies and redirects disabled.
270
+
271
+ Ambient proxies are disabled because a proxy configured by the environment is another party
272
+ that would see the credential.
273
+ """
274
+
275
+ def __init__(self) -> None:
276
+ self._opener = build_opener(
277
+ ProxyHandler({}),
278
+ HTTPSHandler(context=ssl.create_default_context()),
279
+ _NoRedirect(),
280
+ )
281
+
282
+ def request(
283
+ self,
284
+ method: str,
285
+ url: str,
286
+ headers: Mapping[str, str],
287
+ body: bytes | None,
288
+ maximum: int,
289
+ ) -> tuple[int, bytes]:
290
+ request = Request(url, data=body, headers=dict(headers), method=method)
291
+ try:
292
+ response = self._opener.open(request, timeout=VERIFY_TIMEOUT_SECONDS)
293
+ except HTTPError as error:
294
+ response = error
295
+ with response:
296
+ status = int(response.status)
297
+ content_length = response.headers.get("Content-Length")
298
+ if content_length is not None:
299
+ try:
300
+ declared = int(content_length)
301
+ except ValueError as error:
302
+ raise KeySeamError(
303
+ "the verify response has an invalid content length"
304
+ ) from error
305
+ if declared < 0 or declared > maximum:
306
+ raise KeySeamError("the verify response exceeds its byte limit")
307
+ raw = response.read(maximum + 1)
308
+ if len(raw) > maximum:
309
+ raise KeySeamError("the verify response exceeds its byte limit")
310
+ return status, raw
311
+
312
+
313
+ def _https_url(value: str, label: str) -> str:
314
+ """Return ``value`` when it is a plain absolute https URL, otherwise refuse it.
315
+
316
+ Mirrors ``hosted_bootstrap._https_url`` (hosted_bootstrap.py:147). It is copied rather than
317
+ imported for the reason recorded in ``deploy_target._https_url``: importing that module pulls
318
+ the whole hosted-worker stack in behind it and raises a bootstrap-specific error code. The
319
+ policy itself must stay identical — https only, absolute, no user information, no fragment, no
320
+ trailing slash.
321
+
322
+ Plain http is refused *before any request is attempted*, because the failure mode is not a bad
323
+ response, it is the credential already having been sent in the clear.
324
+ """
325
+
326
+ if not isinstance(value, str) or not 1 <= len(value) <= MAX_URL_LENGTH:
327
+ raise KeySeamError(f"{label} is not a usable URL")
328
+ if value.endswith("/"):
329
+ raise KeySeamError(f"{label} must not end with a slash")
330
+ parsed = urlsplit(value)
331
+ if (
332
+ parsed.scheme != "https"
333
+ or not parsed.hostname
334
+ or parsed.username is not None
335
+ or parsed.password is not None
336
+ or parsed.fragment
337
+ ):
338
+ raise KeySeamError(
339
+ f"{label} must be an absolute https URL without user information or fragment"
340
+ )
341
+ return value
342
+
343
+
344
+ def _denied(key_id: str, key_format: str, reason: str) -> KeyVerdict:
345
+ return KeyVerdict(ok=False, key_id=key_id, key_format=key_format, reason=reason)
346
+
347
+
348
+ def _allow_body_is_valid(raw: bytes, key_id: str) -> tuple[bool, str]:
349
+ """Return whether ``raw`` is a well-formed allow body for ``key_id``, and why not if it is not.
350
+
351
+ Strict parse only: duplicate keys, byte-order marks, non-UTF-8 bytes, and non-integer numbers
352
+ are all rejected by :func:`canonical.parse_json`, so a body that two parsers would read
353
+ differently is never acted on.
354
+ """
355
+
356
+ try:
357
+ parsed = parse_json(raw)
358
+ except CanonicalJSONError:
359
+ return False, "the verify surface returned a body that is not strict JSON"
360
+ if not isinstance(parsed, dict):
361
+ return False, "the verify surface returned a body that is not a verdict object"
362
+ if parsed.get(ALLOW_MARKER) is not True:
363
+ return False, "the verify surface did not allow this key"
364
+ named = parsed.get("key_id")
365
+ if named is not None and named != key_id:
366
+ # The response is about some other key. Accepting it would let a captured allow for one
367
+ # key authorize another, and would attribute usage to the wrong key id on this side.
368
+ return False, "the verify surface answered about a different key"
369
+ return True, "the verify surface allowed this key"
370
+
371
+
372
+ def verify_api_key(
373
+ key: str,
374
+ *,
375
+ base_url: str,
376
+ transport: VerifyTransport | None = None,
377
+ ) -> KeyVerdict:
378
+ """Verify one presented key against cloud's verify surface, failing closed.
379
+
380
+ The key is sent in the ``x-api-key`` header and nowhere else: never in the URL, never in a
381
+ query string, never in a log line, never in the returned verdict.
382
+
383
+ ``key_id`` and ``key_format`` are computed locally *before* any request, so a denial still
384
+ hands back a usable non-secret identifier — the caller can report and count a rejected key
385
+ without ever holding the key again.
386
+
387
+ Fail closed is the default and is exhaustive. Every one of the following denies:
388
+
389
+ * any status other than 200, including 401, 403, 429, 5xx, and any 3xx redirect;
390
+ * any transport exception at all — connection refused, TLS failure, timeout, DNS failure;
391
+ * a body that is not strict canonical JSON, or is not an object;
392
+ * a body larger than :data:`MAX_VERIFY_RESPONSE_BYTES`;
393
+ * a body without the explicit allow marker, or naming a different key id.
394
+
395
+ A non-https ``base_url`` raises :class:`KeySeamError` before any request is attempted, because
396
+ that is a caller configuration bug and there is no safe way to proceed with it.
397
+
398
+ Raises:
399
+ KeySeamError: when ``key`` is not text or ``base_url`` is not an absolute https URL.
400
+ """
401
+
402
+ if not isinstance(key, str):
403
+ raise KeySeamError("an API key must be text")
404
+ url = _https_url(base_url, "the verify URL")
405
+ key_id = hash_key_id(key)
406
+ key_format = detect_key_format(key)
407
+ client = UrlLibVerifyTransport() if transport is None else transport
408
+
409
+ # One broad boundary catch, on purpose. Anything that goes wrong past this point becomes a
410
+ # denying verdict with a reason naming the class of fault, never a propagating exception whose
411
+ # message, traceback frame, or repr could carry the key that was in scope when it was raised.
412
+ try:
413
+ status, raw = client.request(
414
+ "POST",
415
+ url,
416
+ {API_KEY_HEADER: key, "Accept": "application/json"},
417
+ b"",
418
+ MAX_VERIFY_RESPONSE_BYTES,
419
+ )
420
+ except Exception:
421
+ return _denied(key_id, key_format, "the verify surface could not be reached")
422
+
423
+ if not isinstance(status, int) or status != 200:
424
+ if isinstance(status, int) and 300 <= status < 400:
425
+ return _denied(key_id, key_format, "the verify surface redirected, which is refused")
426
+ return _denied(key_id, key_format, "the verify surface did not confirm this key")
427
+
428
+ if not isinstance(raw, (bytes, bytearray)):
429
+ return _denied(key_id, key_format, "the verify surface returned a body that is not bytes")
430
+ if len(raw) > MAX_VERIFY_RESPONSE_BYTES:
431
+ return _denied(key_id, key_format, "the verify surface returned an oversized body")
432
+
433
+ allowed, reason = _allow_body_is_valid(bytes(raw), key_id)
434
+ return KeyVerdict(ok=allowed, key_id=key_id, key_format=key_format, reason=reason)
@@ -0,0 +1,262 @@
1
+ """Shared Linux/amd64 process hardening for hosted untrusted-data children."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import ctypes
6
+ import errno
7
+ import platform
8
+ import resource
9
+ import sys
10
+
11
+ _AUDIT_ARCH_X86_64 = 0xC000003E
12
+ _PR_SET_DUMPABLE = 4
13
+ _PR_GET_CHILD_SUBREAPER = 37
14
+ _PR_SET_CHILD_SUBREAPER = 36
15
+ _PR_SET_SECCOMP = 22
16
+ _PR_SET_NO_NEW_PRIVS = 38
17
+ _SECCOMP_MODE_FILTER = 2
18
+ _SECCOMP_RET_KILL_PROCESS = 0x80000000
19
+ _SECCOMP_RET_ERRNO = 0x00050000
20
+ _SECCOMP_RET_ALLOW = 0x7FFF0000
21
+ _BPF_LD_W_ABS = 0x20
22
+ _BPF_JMP_JEQ_K = 0x15
23
+ _BPF_JMP_JSET_K = 0x45
24
+ _BPF_RET_K = 0x06
25
+ _X32_SYSCALL_BIT = 0x40000000
26
+ _NETWORK_SYSCALLS_X86_64 = (
27
+ 41, # socket
28
+ 42, # connect
29
+ 43, # accept
30
+ 44, # sendto
31
+ 45, # recvfrom
32
+ 46, # sendmsg
33
+ 47, # recvmsg
34
+ 48, # shutdown
35
+ 49, # bind
36
+ 50, # listen
37
+ 51, # getsockname
38
+ 52, # getpeername
39
+ 53, # socketpair
40
+ 54, # setsockopt
41
+ 55, # getsockopt
42
+ 288, # accept4
43
+ 299, # recvmmsg
44
+ 307, # sendmmsg
45
+ 425, # io_uring_setup
46
+ 426, # io_uring_enter
47
+ 427, # io_uring_register
48
+ )
49
+ _PROCESS_CREATION_SYSCALLS_X86_64 = (
50
+ 56, # clone
51
+ 57, # fork
52
+ 58, # vfork
53
+ 435, # clone3
54
+ )
55
+ _PROCESS_HANDLE_SYSCALLS_X86_64 = (
56
+ 101, # ptrace
57
+ 310, # process_vm_readv
58
+ 311, # process_vm_writev
59
+ 312, # kcmp
60
+ 434, # pidfd_open
61
+ 438, # pidfd_getfd
62
+ )
63
+ _PROCESS_CONTAINMENT_SYSCALLS_X86_64 = (
64
+ 109, # setpgid
65
+ 112, # setsid
66
+ 272, # unshare
67
+ 308, # setns
68
+ )
69
+ _CLONE_NAMESPACE_FLAGS = (
70
+ 0x00000080 # CLONE_NEWTIME
71
+ | 0x00020000 # CLONE_NEWNS
72
+ | 0x02000000 # CLONE_NEWCGROUP
73
+ | 0x04000000 # CLONE_NEWUTS
74
+ | 0x08000000 # CLONE_NEWIPC
75
+ | 0x10000000 # CLONE_NEWUSER
76
+ | 0x20000000 # CLONE_NEWPID
77
+ | 0x40000000 # CLONE_NEWNET
78
+ )
79
+ _PROCESS_ESCAPE_SYSCALLS_X86_64 = tuple(
80
+ sorted(
81
+ (
82
+ *_PROCESS_CREATION_SYSCALLS_X86_64,
83
+ *_PROCESS_HANDLE_SYSCALLS_X86_64,
84
+ *_PROCESS_CONTAINMENT_SYSCALLS_X86_64,
85
+ )
86
+ )
87
+ )
88
+ _DENIED_CHILD_SYSCALLS_X86_64 = tuple(
89
+ sorted((*_NETWORK_SYSCALLS_X86_64, *_PROCESS_ESCAPE_SYSCALLS_X86_64))
90
+ )
91
+
92
+
93
+ class _SockFilter(ctypes.Structure):
94
+ _fields_ = (
95
+ ("code", ctypes.c_ushort),
96
+ ("jt", ctypes.c_ubyte),
97
+ ("jf", ctypes.c_ubyte),
98
+ ("k", ctypes.c_uint32),
99
+ )
100
+
101
+
102
+ class _SockFprog(ctypes.Structure):
103
+ _fields_ = (
104
+ ("length", ctypes.c_ushort),
105
+ ("filter", ctypes.POINTER(_SockFilter)),
106
+ )
107
+
108
+
109
+ def _child_subreaper_enabled() -> bool:
110
+ """Return this Linux process's child-subreaper state.
111
+
112
+ A short-lived, capability-free session monitor becomes a subreaper before it creates the
113
+ capability child. If that child is force-killed, its sandbox process groups are reparented to
114
+ the monitor instead of the long-lived Service or container init, where the monitor can kill and
115
+ reap only those newly adopted descendants. Other platforms have no equivalent primitive and
116
+ do not run hosted session isolation.
117
+ """
118
+
119
+ if sys.platform != "linux":
120
+ return False
121
+ value = ctypes.c_int()
122
+ libc = ctypes.CDLL(None, use_errno=True)
123
+ libc.prctl.restype = ctypes.c_int
124
+ libc.prctl.argtypes = (
125
+ ctypes.c_int,
126
+ ctypes.c_ulong,
127
+ ctypes.c_ulong,
128
+ ctypes.c_ulong,
129
+ ctypes.c_ulong,
130
+ )
131
+ if libc.prctl(_PR_GET_CHILD_SUBREAPER, ctypes.addressof(value), 0, 0, 0) != 0:
132
+ failure = ctypes.get_errno()
133
+ raise OSError(failure, "unable to inspect hosted session child-subreaper state")
134
+ return bool(value.value)
135
+
136
+
137
+ def _set_child_subreaper(enabled: bool) -> None:
138
+ """Set the Linux child-subreaper state used by the session process monitor."""
139
+
140
+ if sys.platform != "linux":
141
+ return
142
+ libc = ctypes.CDLL(None, use_errno=True)
143
+ libc.prctl.restype = ctypes.c_int
144
+ libc.prctl.argtypes = (
145
+ ctypes.c_int,
146
+ ctypes.c_ulong,
147
+ ctypes.c_ulong,
148
+ ctypes.c_ulong,
149
+ ctypes.c_ulong,
150
+ )
151
+ if libc.prctl(_PR_SET_CHILD_SUBREAPER, int(enabled), 0, 0, 0) != 0:
152
+ failure = ctypes.get_errno()
153
+ raise OSError(failure, "unable to configure hosted session child-subreaper")
154
+
155
+
156
+ def _disable_process_dumpability() -> None:
157
+ """Prevent same-UID subprocesses from reading a coordinator's live authority."""
158
+
159
+ if sys.platform != "linux":
160
+ return
161
+ libc = ctypes.CDLL(None, use_errno=True)
162
+ libc.prctl.restype = ctypes.c_int
163
+ libc.prctl.argtypes = (
164
+ ctypes.c_int,
165
+ ctypes.c_ulong,
166
+ ctypes.c_ulong,
167
+ ctypes.c_ulong,
168
+ ctypes.c_ulong,
169
+ )
170
+ if libc.prctl(_PR_SET_DUMPABLE, 0, 0, 0, 0) != 0:
171
+ failure = ctypes.get_errno()
172
+ raise OSError(failure, "unable to disable hosted coordinator dumpability")
173
+
174
+
175
+ def _install_networkless_seccomp(*, allow_descendants: bool = False) -> None:
176
+ """Install an exec-inherited Linux/amd64 filter denying network creation and I/O.
177
+
178
+ The one-shot Courier child also denies process creation because it has no reason to fork.
179
+ Long-lived research-session parser children may retain their already-bounded thread/process
180
+ allowance; their descendants inherit this filter and the outer container cgroup, so the
181
+ network denial cannot be escaped by forking. Allowing descendants never restores ptrace,
182
+ cross-process memory access, or pidfd handles against the same-UID trusted coordinator.
183
+ """
184
+
185
+ if sys.platform != "linux" or platform.machine().lower() not in {"x86_64", "amd64"}:
186
+ raise OSError(errno.ENOTSUP, "hosted seccomp requires Linux/amd64")
187
+ instructions: list[tuple[int, int, int, int]] = [
188
+ (_BPF_LD_W_ABS, 0, 0, 4),
189
+ (_BPF_JMP_JEQ_K, 1, 0, _AUDIT_ARCH_X86_64),
190
+ (_BPF_RET_K, 0, 0, _SECCOMP_RET_KILL_PROCESS),
191
+ (_BPF_LD_W_ABS, 0, 0, 0),
192
+ (_BPF_JMP_JSET_K, 0, 1, _X32_SYSCALL_BIT),
193
+ (_BPF_RET_K, 0, 0, _SECCOMP_RET_ERRNO | errno.EPERM),
194
+ ]
195
+ denied_syscalls = _DENIED_CHILD_SYSCALLS_X86_64
196
+ if allow_descendants:
197
+ denied_syscalls = tuple(
198
+ sorted(
199
+ (
200
+ *_NETWORK_SYSCALLS_X86_64,
201
+ *_PROCESS_HANDLE_SYSCALLS_X86_64,
202
+ *_PROCESS_CONTAINMENT_SYSCALLS_X86_64,
203
+ )
204
+ )
205
+ )
206
+ # Native runtimes need ordinary clone/fork for threads and bounded parser helpers. Keep
207
+ # them in the process group and namespace the trusted launcher created: classic clone is
208
+ # admitted only without namespace flags, while clone3 is reported unavailable so libc
209
+ # falls back to inspectable classic clone rather than passing an opaque struct pointer the
210
+ # classic seccomp filter cannot safely dereference.
211
+ instructions.extend(
212
+ (
213
+ (_BPF_JMP_JEQ_K, 0, 3, 56), # clone
214
+ (_BPF_LD_W_ABS, 0, 0, 16), # seccomp_data.args[0], low 32 bits
215
+ (_BPF_JMP_JSET_K, 0, 1, _CLONE_NAMESPACE_FLAGS),
216
+ (_BPF_RET_K, 0, 0, _SECCOMP_RET_ERRNO | errno.EPERM),
217
+ (_BPF_LD_W_ABS, 0, 0, 0),
218
+ (_BPF_JMP_JEQ_K, 0, 1, 435), # clone3
219
+ (_BPF_RET_K, 0, 0, _SECCOMP_RET_ERRNO | errno.ENOSYS),
220
+ )
221
+ )
222
+ for syscall_number in denied_syscalls:
223
+ instructions.extend(
224
+ (
225
+ (_BPF_JMP_JEQ_K, 0, 1, syscall_number),
226
+ (_BPF_RET_K, 0, 0, _SECCOMP_RET_ERRNO | errno.EPERM),
227
+ )
228
+ )
229
+ instructions.append((_BPF_RET_K, 0, 0, _SECCOMP_RET_ALLOW))
230
+ filters = (_SockFilter * len(instructions))(
231
+ *(_SockFilter(*instruction) for instruction in instructions)
232
+ )
233
+ program = _SockFprog(
234
+ length=len(filters),
235
+ filter=ctypes.cast(filters, ctypes.POINTER(_SockFilter)),
236
+ )
237
+ libc = ctypes.CDLL(None, use_errno=True)
238
+ libc.prctl.restype = ctypes.c_int
239
+ libc.prctl.argtypes = (
240
+ ctypes.c_int,
241
+ ctypes.c_ulong,
242
+ ctypes.c_ulong,
243
+ ctypes.c_ulong,
244
+ ctypes.c_ulong,
245
+ )
246
+ resource.setrlimit(resource.RLIMIT_CORE, (0, 0))
247
+ _disable_process_dumpability()
248
+ if libc.prctl(_PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) != 0:
249
+ failure = ctypes.get_errno()
250
+ raise OSError(failure, "unable to set hosted child no_new_privs")
251
+ if (
252
+ libc.prctl(
253
+ _PR_SET_SECCOMP,
254
+ _SECCOMP_MODE_FILTER,
255
+ ctypes.addressof(program),
256
+ 0,
257
+ 0,
258
+ )
259
+ != 0
260
+ ):
261
+ failure = ctypes.get_errno()
262
+ raise OSError(failure, "unable to install hosted child seccomp")