mostlyright-data 0.9.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (314) hide show
  1. mostlyright/data_harness/__init__.py +158 -0
  2. mostlyright/data_harness/acquisition/__init__.py +55 -0
  3. mostlyright/data_harness/acquisition/http.py +2773 -0
  4. mostlyright/data_harness/acquisition/parsing.py +809 -0
  5. mostlyright/data_harness/acquisition/ranges.py +495 -0
  6. mostlyright/data_harness/acquisition/result_download.py +360 -0
  7. mostlyright/data_harness/acquisition/retention_admission.py +248 -0
  8. mostlyright/data_harness/acquisition/sandbox.py +4888 -0
  9. mostlyright/data_harness/acquisition/url_policy.py +530 -0
  10. mostlyright/data_harness/agent_runtime.py +2743 -0
  11. mostlyright/data_harness/assets/logo-ink.svg +31 -0
  12. mostlyright/data_harness/backends/__init__.py +28 -0
  13. mostlyright/data_harness/backends/pandas_backend.py +350 -0
  14. mostlyright/data_harness/backends/polars_backend.py +366 -0
  15. mostlyright/data_harness/backends/protocol.py +124 -0
  16. mostlyright/data_harness/backends/reference.py +83 -0
  17. mostlyright/data_harness/backends/registry.py +55 -0
  18. mostlyright/data_harness/backends/restrictions.py +126 -0
  19. mostlyright/data_harness/canonical.py +333 -0
  20. mostlyright/data_harness/catalog_job.py +625 -0
  21. mostlyright/data_harness/cli.py +5398 -0
  22. mostlyright/data_harness/contracts.py +53 -0
  23. mostlyright/data_harness/coordinator.py +1307 -0
  24. mostlyright/data_harness/deploy.py +924 -0
  25. mostlyright/data_harness/deploy_target.py +312 -0
  26. mostlyright/data_harness/deployment_evidence.py +1067 -0
  27. mostlyright/data_harness/event_presentation.py +576 -0
  28. mostlyright/data_harness/events.py +2152 -0
  29. mostlyright/data_harness/fast_delimited.py +239 -0
  30. mostlyright/data_harness/fleet.py +237 -0
  31. mostlyright/data_harness/formats.py +236 -0
  32. mostlyright/data_harness/governors.py +1163 -0
  33. mostlyright/data_harness/hosted_bootstrap.py +972 -0
  34. mostlyright/data_harness/hosted_crawler.py +1115 -0
  35. mostlyright/data_harness/hosted_crawler_container_smoke.py +351 -0
  36. mostlyright/data_harness/hosted_crawler_fetch.py +423 -0
  37. mostlyright/data_harness/hosted_crawler_job.py +1277 -0
  38. mostlyright/data_harness/hosted_crawler_protocol.py +676 -0
  39. mostlyright/data_harness/hosted_dataset.py +1500 -0
  40. mostlyright/data_harness/hosted_deploy.py +3037 -0
  41. mostlyright/data_harness/hosted_handoff.py +62 -0
  42. mostlyright/data_harness/hosted_ingestion_contract.py +504 -0
  43. mostlyright/data_harness/hosted_ingestion_job.py +356 -0
  44. mostlyright/data_harness/hosted_ingestion_job_smoke.py +40 -0
  45. mostlyright/data_harness/hosted_session_container_smoke.py +194 -0
  46. mostlyright/data_harness/hosted_session_worker.py +3554 -0
  47. mostlyright/data_harness/hosted_session_worker_job_smoke.py +46 -0
  48. mostlyright/data_harness/hosted_worker.py +6784 -0
  49. mostlyright/data_harness/ingestion/__init__.py +56 -0
  50. mostlyright/data_harness/ingestion/contracts.py +461 -0
  51. mostlyright/data_harness/ingestion/faults.py +42 -0
  52. mostlyright/data_harness/ingestion/gcs_store.py +1162 -0
  53. mostlyright/data_harness/ingestion/spool.py +130 -0
  54. mostlyright/data_harness/ingestion/store.py +885 -0
  55. mostlyright/data_harness/key_seam.py +434 -0
  56. mostlyright/data_harness/linux_process_boundary.py +262 -0
  57. mostlyright/data_harness/local_contracts.py +2880 -0
  58. mostlyright/data_harness/local_search/__init__.py +5 -0
  59. mostlyright/data_harness/local_search/build_index.py +1087 -0
  60. mostlyright/data_harness/local_search/contracts.py +920 -0
  61. mostlyright/data_harness/local_search/query_trace.py +266 -0
  62. mostlyright/data_harness/local_search/retrieval.py +700 -0
  63. mostlyright/data_harness/local_search/sealed.py +474 -0
  64. mostlyright/data_harness/local_search/service.py +784 -0
  65. mostlyright/data_harness/nbrender/CONTRACT.md +212 -0
  66. mostlyright/data_harness/nbrender/__init__.py +12 -0
  67. mostlyright/data_harness/nbrender/chrome.py +359 -0
  68. mostlyright/data_harness/nbrender/code_body.py +266 -0
  69. mostlyright/data_harness/nbrender/document.py +407 -0
  70. mostlyright/data_harness/nbrender/frame.py +275 -0
  71. mostlyright/data_harness/nbrender/interactive.py +337 -0
  72. mostlyright/data_harness/nbrender/markdown_body.py +477 -0
  73. mostlyright/data_harness/nbrender/mr_components.py +134 -0
  74. mostlyright/data_harness/nbrender/outputs_data.py +595 -0
  75. mostlyright/data_harness/nbrender/outputs_rich.py +906 -0
  76. mostlyright/data_harness/nbrender/outputs_source.py +260 -0
  77. mostlyright/data_harness/nbrender/outputs_stage.py +176 -0
  78. mostlyright/data_harness/nbrender/outputs_text.py +400 -0
  79. mostlyright/data_harness/nbrender/parse.py +394 -0
  80. mostlyright/data_harness/nbrender/status.py +40 -0
  81. mostlyright/data_harness/nbrender/tokens.py +1295 -0
  82. mostlyright/data_harness/notebook.py +1710 -0
  83. mostlyright/data_harness/offline.py +2049 -0
  84. mostlyright/data_harness/operation_registry.py +1007 -0
  85. mostlyright/data_harness/operator_setup.py +239 -0
  86. mostlyright/data_harness/pipeline.py +6428 -0
  87. mostlyright/data_harness/plan_graph.py +2026 -0
  88. mostlyright/data_harness/preparation/__init__.py +104 -0
  89. mostlyright/data_harness/preparation/contracts.py +1017 -0
  90. mostlyright/data_harness/preparation/engine.py +221 -0
  91. mostlyright/data_harness/preparation/errors.py +14 -0
  92. mostlyright/data_harness/preparation/gates.py +751 -0
  93. mostlyright/data_harness/preparation/joins.py +574 -0
  94. mostlyright/data_harness/preparation/profile.py +384 -0
  95. mostlyright/data_harness/preparation/table.py +217 -0
  96. mostlyright/data_harness/preparation/transforms.py +568 -0
  97. mostlyright/data_harness/progress_events.py +534 -0
  98. mostlyright/data_harness/readers/__init__.py +46 -0
  99. mostlyright/data_harness/readers/containers.py +963 -0
  100. mostlyright/data_harness/readers/contracts.py +542 -0
  101. mostlyright/data_harness/readers/delimited.py +257 -0
  102. mostlyright/data_harness/readers/grib2/__init__.py +33 -0
  103. mostlyright/data_harness/readers/grib2/admission.py +722 -0
  104. mostlyright/data_harness/readers/grib2/decode.py +1009 -0
  105. mostlyright/data_harness/readers/grib2/geometry.py +1133 -0
  106. mostlyright/data_harness/readers/grib2/portable_math.py +501 -0
  107. mostlyright/data_harness/readers/json_tabular.py +485 -0
  108. mostlyright/data_harness/readers/registry.py +514 -0
  109. mostlyright/data_harness/readers/samples/README.md +110 -0
  110. mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/cities.csv.gz +0 -0
  111. mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/expected.json +24 -0
  112. mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/cities.csv.gz +0 -0
  113. mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/expected.json +24 -0
  114. mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/cities.tar +0 -0
  115. mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/expected.json +24 -0
  116. mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/cities.tar +0 -0
  117. mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/expected.json +24 -0
  118. mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/cities.zip +0 -0
  119. mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/expected.json +25 -0
  120. mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
  121. mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/expected.json +25 -0
  122. mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
  123. mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/expected.json +25 -0
  124. mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/cities.csv +3 -0
  125. mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/expected.json +23 -0
  126. mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/cities.tsv +5 -0
  127. mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/expected.json +25 -0
  128. mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/expected.json +30 -0
  129. mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/observations.csv +5 -0
  130. mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/expected.json +44 -0
  131. mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/stations.json +1 -0
  132. mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/expected.json +48 -0
  133. mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/observations.ndjson +4 -0
  134. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/cities.xlsx +0 -0
  135. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/expected.json +24 -0
  136. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/expected.json +27 -0
  137. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
  138. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/expected.json +27 -0
  139. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
  140. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/README.md +20 -0
  141. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/expected.json +55 -0
  142. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/gfs-2m-temperature.grib2 +0 -0
  143. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/expected.json +54 -0
  144. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
  145. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/expected.json +54 -0
  146. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/hrrr-categorical-rain.grib2 +0 -0
  147. mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/expected.json +54 -0
  148. mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
  149. mostlyright/data_harness/readers/samples.py +582 -0
  150. mostlyright/data_harness/readers/spreadsheet.py +803 -0
  151. mostlyright/data_harness/readers/tabular.py +510 -0
  152. mostlyright/data_harness/recipe.py +5321 -0
  153. mostlyright/data_harness/repair/__init__.py +78 -0
  154. mostlyright/data_harness/repair/adapters.py +274 -0
  155. mostlyright/data_harness/repair/contracts.py +872 -0
  156. mostlyright/data_harness/repair/coordinator.py +1099 -0
  157. mostlyright/data_harness/repair/errors.py +16 -0
  158. mostlyright/data_harness/review.py +2533 -0
  159. mostlyright/data_harness/rowset.py +283 -0
  160. mostlyright/data_harness/serving.py +1975 -0
  161. mostlyright/data_harness/serving_edge.py +590 -0
  162. mostlyright/data_harness/serving_http.py +1031 -0
  163. mostlyright/data_harness/session_probes.py +759 -0
  164. mostlyright/data_harness/signing.py +101 -0
  165. mostlyright/data_harness/source_discovery.py +898 -0
  166. mostlyright/data_harness/sources/__init__.py +209 -0
  167. mostlyright/data_harness/sources/_adapter_steps.py +213 -0
  168. mostlyright/data_harness/sources/adapters.py +1214 -0
  169. mostlyright/data_harness/sources/cadence.py +1428 -0
  170. mostlyright/data_harness/sources/cadence_emission.py +453 -0
  171. mostlyright/data_harness/sources/cadence_history.py +546 -0
  172. mostlyright/data_harness/sources/catalog/__init__.py +17 -0
  173. mostlyright/data_harness/sources/catalog/admission.py +477 -0
  174. mostlyright/data_harness/sources/catalog/authoring.py +1701 -0
  175. mostlyright/data_harness/sources/catalog/authoring_policy.py +701 -0
  176. mostlyright/data_harness/sources/catalog/authoring_shards.py +1217 -0
  177. mostlyright/data_harness/sources/catalog/bounded_io.py +231 -0
  178. mostlyright/data_harness/sources/catalog/channel.py +523 -0
  179. mostlyright/data_harness/sources/catalog/channel_client.py +296 -0
  180. mostlyright/data_harness/sources/catalog/contracts.py +825 -0
  181. mostlyright/data_harness/sources/catalog/coverage.py +137 -0
  182. mostlyright/data_harness/sources/catalog/delta.py +1340 -0
  183. mostlyright/data_harness/sources/catalog/embedding.py +532 -0
  184. mostlyright/data_harness/sources/catalog/entry_v2.py +1182 -0
  185. mostlyright/data_harness/sources/catalog/fill.py +3889 -0
  186. mostlyright/data_harness/sources/catalog/fill_partitions.py +459 -0
  187. mostlyright/data_harness/sources/catalog/fill_staging.py +1105 -0
  188. mostlyright/data_harness/sources/catalog/gating.py +374 -0
  189. mostlyright/data_harness/sources/catalog/generation_receipt.py +1607 -0
  190. mostlyright/data_harness/sources/catalog/harvest/__init__.py +7 -0
  191. mostlyright/data_harness/sources/catalog/harvest/ckan.py +384 -0
  192. mostlyright/data_harness/sources/catalog/harvest/datagov_v4.py +798 -0
  193. mostlyright/data_harness/sources/catalog/harvest/protocol.py +964 -0
  194. mostlyright/data_harness/sources/catalog/harvest/sdmx.py +445 -0
  195. mostlyright/data_harness/sources/catalog/harvest/stac.py +384 -0
  196. mostlyright/data_harness/sources/catalog/health.py +447 -0
  197. mostlyright/data_harness/sources/catalog/hosted_catalog.py +105 -0
  198. mostlyright/data_harness/sources/catalog/identity_history.py +1549 -0
  199. mostlyright/data_harness/sources/catalog/neural.py +1618 -0
  200. mostlyright/data_harness/sources/catalog/packed_catalog.py +2345 -0
  201. mostlyright/data_harness/sources/catalog/packed_retrieval.py +1517 -0
  202. mostlyright/data_harness/sources/catalog/packed_writer.py +2802 -0
  203. mostlyright/data_harness/sources/catalog/query_trace.py +1037 -0
  204. mostlyright/data_harness/sources/catalog/recommend.py +171 -0
  205. mostlyright/data_harness/sources/catalog/retrieval.py +230 -0
  206. mostlyright/data_harness/sources/catalog/retrieval_manifest.py +995 -0
  207. mostlyright/data_harness/sources/catalog/rights_decisions.py +254 -0
  208. mostlyright/data_harness/sources/catalog/sealed.py +560 -0
  209. mostlyright/data_harness/sources/catalog/search.py +230 -0
  210. mostlyright/data_harness/sources/catalog/streaming_delta.py +1097 -0
  211. mostlyright/data_harness/sources/catalog/update.py +891 -0
  212. mostlyright/data_harness/sources/collections.py +815 -0
  213. mostlyright/data_harness/sources/contracts.py +2223 -0
  214. mostlyright/data_harness/sources/deletion.py +761 -0
  215. mostlyright/data_harness/sources/fitness.py +162 -0
  216. mostlyright/data_harness/sources/governance.py +163 -0
  217. mostlyright/data_harness/sources/hosted.py +173 -0
  218. mostlyright/data_harness/sources/integration.py +218 -0
  219. mostlyright/data_harness/sources/range_reader.py +418 -0
  220. mostlyright/data_harness/sources/registry.py +514 -0
  221. mostlyright/data_harness/sources/rights_rule.py +59 -0
  222. mostlyright/data_harness/sources/source_cadence_vectors.v1.json +1 -0
  223. mostlyright/data_harness/sources/sports.py +521 -0
  224. mostlyright/data_harness/sources/stream.py +524 -0
  225. mostlyright/data_harness/sources/stream_connector.py +418 -0
  226. mostlyright/data_harness/sources/stream_recorder.py +1404 -0
  227. mostlyright/data_harness/studio_boundary.py +2019 -0
  228. mostlyright/data_harness/thin/__init__.py +37 -0
  229. mostlyright/data_harness/thin/acquire.py +1137 -0
  230. mostlyright/data_harness/thin/acquire_cancel.py +579 -0
  231. mostlyright/data_harness/thin/approvals.py +617 -0
  232. mostlyright/data_harness/thin/commands.py +406 -0
  233. mostlyright/data_harness/thin/download.py +194 -0
  234. mostlyright/data_harness/thin/narrative.py +589 -0
  235. mostlyright/data_harness/thin/parity.py +1070 -0
  236. mostlyright/data_harness/thin/propose.py +2759 -0
  237. mostlyright/data_harness/thin/research.py +1663 -0
  238. mostlyright/data_harness/thin/router.py +924 -0
  239. mostlyright/data_harness/thin/runs.py +519 -0
  240. mostlyright/data_harness/thin/session.py +281 -0
  241. mostlyright/data_harness/thin/stream.py +501 -0
  242. mostlyright/data_harness/thin/transport.py +187 -0
  243. mostlyright/data_harness/thin/vocabulary.py +368 -0
  244. mostlyright/data_harness/thin/workers.py +164 -0
  245. mostlyright/data_harness/ucum/TABLE-PIN.json +40 -0
  246. mostlyright/data_harness/ucum/ucum-subset.v1.json +632 -0
  247. mostlyright/data_harness/unit_flow.py +927 -0
  248. mostlyright/data_harness/units.py +572 -0
  249. mostlyright/data_harness/ux/__init__.py +9 -0
  250. mostlyright/data_harness/ux/approve.py +485 -0
  251. mostlyright/data_harness/ux/author_yaml.py +597 -0
  252. mostlyright/data_harness/ux/cloud_auth.py +447 -0
  253. mostlyright/data_harness/ux/commands/__init__.py +260 -0
  254. mostlyright/data_harness/ux/commands/approve.py +136 -0
  255. mostlyright/data_harness/ux/commands/auth.py +744 -0
  256. mostlyright/data_harness/ux/commands/author.py +79 -0
  257. mostlyright/data_harness/ux/commands/catalog_author.py +403 -0
  258. mostlyright/data_harness/ux/commands/catalog_fill.py +523 -0
  259. mostlyright/data_harness/ux/commands/catalog_harvest.py +545 -0
  260. mostlyright/data_harness/ux/commands/catalog_publish.py +1838 -0
  261. mostlyright/data_harness/ux/commands/catalog_search.py +71 -0
  262. mostlyright/data_harness/ux/commands/catalog_update.py +437 -0
  263. mostlyright/data_harness/ux/commands/deploy.py +134 -0
  264. mostlyright/data_harness/ux/commands/deploy_dataset.py +98 -0
  265. mostlyright/data_harness/ux/commands/deploy_plan.py +105 -0
  266. mostlyright/data_harness/ux/commands/deploy_status.py +104 -0
  267. mostlyright/data_harness/ux/commands/diff.py +74 -0
  268. mostlyright/data_harness/ux/commands/index.py +84 -0
  269. mostlyright/data_harness/ux/commands/inventory.py +47 -0
  270. mostlyright/data_harness/ux/commands/list_builds.py +143 -0
  271. mostlyright/data_harness/ux/commands/login.py +63 -0
  272. mostlyright/data_harness/ux/commands/peek.py +236 -0
  273. mostlyright/data_harness/ux/commands/plan_check.py +90 -0
  274. mostlyright/data_harness/ux/commands/preflight.py +97 -0
  275. mostlyright/data_harness/ux/commands/record.py +107 -0
  276. mostlyright/data_harness/ux/commands/review_setup.py +47 -0
  277. mostlyright/data_harness/ux/commands/search.py +440 -0
  278. mostlyright/data_harness/ux/commands/show.py +61 -0
  279. mostlyright/data_harness/ux/commands/whoami.py +37 -0
  280. mostlyright/data_harness/ux/credential_native.py +551 -0
  281. mostlyright/data_harness/ux/credential_store.py +1055 -0
  282. mostlyright/data_harness/ux/credentials.py +631 -0
  283. mostlyright/data_harness/ux/diffing.py +444 -0
  284. mostlyright/data_harness/ux/headline.py +671 -0
  285. mostlyright/data_harness/ux/hosted_acquisition.py +974 -0
  286. mostlyright/data_harness/ux/hosted_run_status.py +619 -0
  287. mostlyright/data_harness/ux/inventory.py +427 -0
  288. mostlyright/data_harness/ux/local_review.py +375 -0
  289. mostlyright/data_harness/ux/login.py +691 -0
  290. mostlyright/data_harness/ux/path_kind.py +147 -0
  291. mostlyright/data_harness/ux/peek.py +1000 -0
  292. mostlyright/data_harness/ux/plain_file.py +178 -0
  293. mostlyright/data_harness/ux/plan_check.py +311 -0
  294. mostlyright/data_harness/ux/preflight.py +918 -0
  295. mostlyright/data_harness/ux/readers.py +1124 -0
  296. mostlyright/data_harness/ux/remediation.py +2195 -0
  297. mostlyright/data_harness/ux/render.py +657 -0
  298. mostlyright/data_harness/ux/workload.py +1077 -0
  299. mostlyright/data_harness/viewer.py +3713 -0
  300. mostlyright/data_harness/visual_run/__init__.py +83 -0
  301. mostlyright/data_harness/visual_run/authoring.py +235 -0
  302. mostlyright/data_harness/visual_run/contracts.py +673 -0
  303. mostlyright/data_harness/visual_run/materialize.py +486 -0
  304. mostlyright/data_harness/visual_run/observations.py +874 -0
  305. mostlyright/data_harness/visual_run/query.py +259 -0
  306. mostlyright/data_harness/visual_run/reducer.py +280 -0
  307. mostlyright/data_harness/visual_run/sdk.py +892 -0
  308. mostlyright/data_harness/visual_run/store.py +584 -0
  309. mostlyright/data_harness/visual_run/transport.py +239 -0
  310. mostlyright/data_harness/watch.py +2999 -0
  311. mostlyright_data-0.9.0.dist-info/METADATA +607 -0
  312. mostlyright_data-0.9.0.dist-info/RECORD +314 -0
  313. mostlyright_data-0.9.0.dist-info/WHEEL +4 -0
  314. mostlyright_data-0.9.0.dist-info/entry_points.txt +12 -0
@@ -0,0 +1,1404 @@
1
+ """Deterministic stream capture: seal recorded messages, then read them back as batches.
2
+
3
+ Capture and build stay separate. A live socket can never satisfy the determinism contract --
4
+ a Checker cannot replay a subscription -- so a connection is a *capture* role, not an
5
+ acquisition transport. This module is the half of that split which is deterministic, and it is
6
+ the whole of what this build ships:
7
+
8
+ * :func:`seal_recorded_capture` takes an already-recorded list of provider messages and turns
9
+ it into bounded, sealed, ordered batch files plus a capture manifest. Epochs, rotation,
10
+ message classification, event identity, and gap boundaries are all decided here, from the
11
+ connector configuration, with no clock and no connection.
12
+ * :class:`RecordedStreamGateway` implements the one-method ``StreamGateway`` the existing
13
+ ``StreamBatchAdapter`` already consumes, reading those sealed batches back off disk.
14
+
15
+ What is deliberately absent is the socket. Nothing in this module opens, resolves, or reads a
16
+ network address; network access belongs to the Courier, and a Recorder that holds a long-lived
17
+ connection is a separate trusted role that does not exist yet. The recording this module reads
18
+ is therefore an input the operator supplies, not something this build goes and gets. Everything
19
+ downstream of the recording -- rotation, sealing, identity, ordering, the bookmark -- is the
20
+ real behaviour, exercised on real bytes.
21
+
22
+ The recording format is one JSON object per line, each with exactly three keys::
23
+
24
+ {"epoch": 1, "received_at": "2026-07-17T12:00:00Z", "message": {...}}
25
+
26
+ ``epoch`` is the connection the message arrived on. A disconnect closes an epoch and the next
27
+ message carries the next epoch number; that boundary is a gap and is recorded as one. Gaps stay
28
+ loud: nothing here silently splices two connections together.
29
+ """
30
+
31
+ from __future__ import annotations
32
+
33
+ import os
34
+ import re
35
+ import stat
36
+ from dataclasses import dataclass
37
+ from datetime import datetime
38
+ from pathlib import Path
39
+ from typing import Any
40
+
41
+ from mostlyright.data_harness.acquisition.http import seal_content_addressed_snapshot
42
+ from mostlyright.data_harness.canonical import (
43
+ canonical_json_bytes,
44
+ canonical_sha256,
45
+ parse_json,
46
+ sha256_bytes,
47
+ )
48
+ from mostlyright.data_harness.sources.contracts import (
49
+ AcquisitionRequest,
50
+ SourceContractError,
51
+ )
52
+ from mostlyright.data_harness.sources.stream import (
53
+ EVENT_ID_GRAMMAR,
54
+ MAX_BATCH_EVENTS,
55
+ StreamBatch,
56
+ )
57
+ from mostlyright.data_harness.sources.stream_connector import (
58
+ EVENT_TIME_KINDS,
59
+ StreamConnectorConfig,
60
+ stream_connector,
61
+ )
62
+
63
+ STREAM_CAPTURE_VERSION = "harness-stream-capture.v1"
64
+
65
+ CAPTURE_MANIFEST_NAME = "capture.json"
66
+ BATCH_DIRECTORY_NAME = "batches"
67
+ # The durable per-source bookmark ledger lives beside the batches it is about, so one capture
68
+ # directory is the whole replayable unit. ``stream_connector.RESERVED_STREAM_IDS`` keeps a
69
+ # stream from claiming this name.
70
+ LEDGER_DIRECTORY_NAME = "bookmarks"
71
+
72
+ MAX_RECORDING_BYTES = 64 * 1024 * 1024
73
+ MAX_MANIFEST_BYTES = 64 * 1024 * 1024
74
+ MAX_RECORDED_FRAMES = 200_000
75
+ MAX_EPOCHS = 4_096
76
+ MAX_BATCHES = 4_096
77
+
78
+ _TIMESTAMP = re.compile(r"[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]{1,9})?Z")
79
+ _STORAGE_NAME = re.compile(r"[0-9a-f]{64}\.(?:json|ndjson)")
80
+ # The bookmark grammar: the Recorder's own ``{epoch}:{counter}`` position, zero-padded so that
81
+ # text order and number order are the same thing. It is deliberately not the event name -- a
82
+ # venue that assigns its own identifiers assigns ``8``, ``9``, ``10``, which do not sort as
83
+ # text, and a bookmark that cannot be compared is not a bookmark.
84
+ _BOOKMARK = re.compile(r"e([0-9]{8}):([0-9]{12})")
85
+ _RECORDING_KEYS = frozenset({"epoch", "received_at", "message"})
86
+
87
+ _MEDIA_TYPES = {"ndjson": "application/x-ndjson", "json": "application/json"}
88
+
89
+ # A gap boundary is one of exactly these. ``opening`` is the first connection of a capture and
90
+ # is not a gap at all; the other two say whether the venue's own repair rule closed the hole.
91
+ GAP_BOUNDARIES = ("opening", "repaired_by_snapshot", "recorded_gap")
92
+
93
+
94
+ @dataclass(frozen=True)
95
+ class CapturedEpoch:
96
+ """One connection: what it covered and whether the boundary before it left a hole."""
97
+
98
+ epoch: int
99
+ boundary: str
100
+ event_count: int
101
+ # Null when the connection admitted no event at all. A connection that carried only
102
+ # heartbeats has no first or last event, and inventing one would be a claim about data that
103
+ # was never captured.
104
+ first_event_id: str | None
105
+ last_event_id: str | None
106
+ opened_at: str
107
+ closed_at: str
108
+
109
+ def to_dict(self) -> dict[str, Any]:
110
+ return {
111
+ "epoch": self.epoch,
112
+ "boundary": self.boundary,
113
+ "event_count": self.event_count,
114
+ "first_event_id": self.first_event_id,
115
+ "last_event_id": self.last_event_id,
116
+ "opened_at": self.opened_at,
117
+ "closed_at": self.closed_at,
118
+ }
119
+
120
+
121
+ @dataclass(frozen=True)
122
+ class CapturedBatch:
123
+ """One sealed batch file and the bookmark it advances to."""
124
+
125
+ batch_index: int
126
+ epoch: int
127
+ storage_name: str
128
+ content_sha256: str
129
+ size_bytes: int
130
+ previous_bookmark: str | None
131
+ next_bookmark: str
132
+ event_ids: tuple[str, ...]
133
+ first_event_time: str
134
+ last_event_time: str
135
+ first_received_at: str
136
+ last_received_at: str
137
+
138
+ def to_dict(self) -> dict[str, Any]:
139
+ return {
140
+ "batch_index": self.batch_index,
141
+ "epoch": self.epoch,
142
+ "storage_name": self.storage_name,
143
+ "content_sha256": self.content_sha256,
144
+ "size_bytes": self.size_bytes,
145
+ "previous_bookmark": self.previous_bookmark,
146
+ "next_bookmark": self.next_bookmark,
147
+ "event_ids": list(self.event_ids),
148
+ "first_event_time": self.first_event_time,
149
+ "last_event_time": self.last_event_time,
150
+ "first_received_at": self.first_received_at,
151
+ "last_received_at": self.last_received_at,
152
+ }
153
+
154
+
155
+ @dataclass(frozen=True)
156
+ class CaptureManifest:
157
+ """The capture's own evidence: what was configured, what arrived, and what was sealed.
158
+
159
+ This is not the chained capture-evidence contract that stream work still ahead describes. It
160
+ is the flat, digest-bound record needed to read a capture back deterministically, and it is
161
+ versioned separately so a chained contract can supersede it without rewriting captures.
162
+
163
+ ``recorded_through`` is the arrival time of the last message the capture admitted, read out
164
+ of the recording. It is deliberately not "when the recorder ran": a capture sealed from one
165
+ recording has one record whatever day it is sealed on, which is what lets a build be replayed
166
+ from the recording alone.
167
+ """
168
+
169
+ connector_id: str
170
+ connector_config_digest: str
171
+ stream_id: str
172
+ data_format: str
173
+ batch_filename: str
174
+ recorded_through: str
175
+ admitted_event_count: int
176
+ ignored_message_count: int
177
+ epochs: tuple[CapturedEpoch, ...]
178
+ batches: tuple[CapturedBatch, ...]
179
+ schema_version: str = STREAM_CAPTURE_VERSION
180
+
181
+ def to_dict(self) -> dict[str, Any]:
182
+ return {
183
+ "schema_version": self.schema_version,
184
+ "connector_id": self.connector_id,
185
+ "connector_config_digest": self.connector_config_digest,
186
+ "stream_id": self.stream_id,
187
+ "data_format": self.data_format,
188
+ "batch_filename": self.batch_filename,
189
+ "recorded_through": self.recorded_through,
190
+ "admitted_event_count": self.admitted_event_count,
191
+ "ignored_message_count": self.ignored_message_count,
192
+ "epochs": [epoch.to_dict() for epoch in self.epochs],
193
+ "batches": [batch.to_dict() for batch in self.batches],
194
+ }
195
+
196
+ @property
197
+ def digest(self) -> str:
198
+ return canonical_sha256(self.to_dict())
199
+
200
+ @property
201
+ def gap_count(self) -> int:
202
+ return sum(1 for epoch in self.epochs if epoch.boundary == "recorded_gap")
203
+
204
+
205
+ @dataclass(frozen=True)
206
+ class _Connection:
207
+ """One connection the recording saw, whether or not it contributed an event.
208
+
209
+ A connection that carried only heartbeats is still a connection, and dropping it from the
210
+ record would splice its neighbours together silently -- which is the one thing this module
211
+ promises not to do.
212
+ """
213
+
214
+ epoch: int
215
+ opened_at: str
216
+ closed_at: str
217
+ opened_with_snapshot: bool
218
+
219
+
220
+ @dataclass(frozen=True)
221
+ class _Frame:
222
+ """One recorded message after classification, with the identity the Recorder assigned."""
223
+
224
+ epoch: int
225
+ sequence: int
226
+ event_id: str
227
+ position: str
228
+ message_class: str
229
+ message_type: str
230
+ event_time: str
231
+ received_at: str
232
+ line: bytes
233
+
234
+
235
+ def capture_root(captures_root: Path, stream_id: str) -> Path:
236
+ """Return the one directory a stream's sealed capture lives in.
237
+
238
+ The whole layout is derived from the capture root, never assembled by a caller, so a build
239
+ and a recording cannot disagree about where a capture is::
240
+
241
+ <captures_root>/bookmarks/ the durable per-source ledger, shared
242
+ <captures_root>/<stream_id>/capture.json one stream's sealed record
243
+ <captures_root>/<stream_id>/batches/ its digest-addressed sealed batches
244
+
245
+ The ledger is shared across streams because one build holds one bookmark store; the reserved
246
+ names in ``stream_connector`` are what keep a stream from claiming that directory.
247
+ """
248
+
249
+ return captures_root / _capture_component(stream_id)
250
+
251
+
252
+ def ledger_root(captures_root: Path) -> Path:
253
+ """Return the durable bookmark ledger directory for a whole capture root."""
254
+
255
+ return captures_root / LEDGER_DIRECTORY_NAME
256
+
257
+
258
+ def seal_recorded_capture(
259
+ *,
260
+ config: StreamConnectorConfig,
261
+ recording: bytes,
262
+ captures_root: Path,
263
+ ) -> CaptureManifest:
264
+ """Turn one recorded message log into sealed batches beneath ``captures_root``.
265
+
266
+ Every directory the layout needs is created if missing, so one call leaves a complete
267
+ capture a build can read.
268
+
269
+ Re-running the same recording is a no-op that produces the identical manifest: batch files
270
+ are content-addressed, sealing bytes that are already there returns the existing file, and
271
+ every time in the record is read out of the recording rather than off a clock. There is no
272
+ clock input at all, which is what makes a capture reproducible from its recording alone.
273
+ """
274
+
275
+ if not captures_root.is_absolute():
276
+ raise SourceContractError(
277
+ "STREAM_CAPTURE_ROOT",
278
+ "captures_root",
279
+ "must be an absolute coordinator-owned path",
280
+ )
281
+ frames, ignored, connections = _classify_recording(config, recording)
282
+ epochs, batches_of_frames = _partition(config, frames, connections)
283
+ stream_root = capture_root(captures_root, config.stream_id)
284
+ batch_root = stream_root / BATCH_DIRECTORY_NAME
285
+ # Every directory is made and then checked to be a real directory rather than a link to
286
+ # one. ``mkdir(exist_ok=True)`` accepts a pre-existing symlink to a directory, so the
287
+ # check is what stops a planted link from deciding where a capture is written. It happens
288
+ # before any byte is written, so a refusal leaves nothing behind.
289
+ for directory in (captures_root, stream_root, batch_root, ledger_root(captures_root)):
290
+ _require_plain_directory(directory)
291
+ batches: list[CapturedBatch] = []
292
+ previous: str | None = None
293
+ for index, group in enumerate(batches_of_frames, start=1):
294
+ content = b"".join(frame.line for frame in group)
295
+ # The one sealing implementation, reused rather than written a second time: it is
296
+ # content-addressed, refuses to overwrite, follows no link, and returns the existing
297
+ # file when the bytes are already there, which is what makes re-sealing a no-op. It
298
+ # raises ``AcquisitionSecurityError`` rather than this module's own error type; both
299
+ # are typed refusals the command boundary already renders.
300
+ #
301
+ # The path is handed over unresolved on purpose. ``seal_content_addressed_snapshot``
302
+ # opens its root with ``O_NOFOLLOW``, and resolving the path here would hand it the
303
+ # link's target instead, so the guard would open a real directory and never fire.
304
+ path = seal_content_addressed_snapshot(
305
+ batch_root,
306
+ content=content,
307
+ suffix=config.data_format,
308
+ )
309
+ bookmark = group[-1].position
310
+ batches.append(
311
+ CapturedBatch(
312
+ batch_index=index,
313
+ epoch=group[0].epoch,
314
+ storage_name=path.name,
315
+ content_sha256=sha256_bytes(content),
316
+ size_bytes=len(content),
317
+ previous_bookmark=previous,
318
+ next_bookmark=bookmark,
319
+ event_ids=tuple(frame.event_id for frame in group),
320
+ first_event_time=group[0].event_time,
321
+ last_event_time=group[-1].event_time,
322
+ first_received_at=group[0].received_at,
323
+ last_received_at=group[-1].received_at,
324
+ )
325
+ )
326
+ previous = bookmark
327
+ manifest = CaptureManifest(
328
+ connector_id=config.connector_id,
329
+ connector_config_digest=config.digest,
330
+ stream_id=config.stream_id,
331
+ data_format=config.data_format,
332
+ batch_filename=f"{config.stream_id}.{config.data_format}",
333
+ recorded_through=frames[-1].received_at,
334
+ admitted_event_count=len(frames),
335
+ ignored_message_count=ignored,
336
+ epochs=epochs,
337
+ batches=tuple(batches),
338
+ )
339
+ _write_manifest(stream_root, manifest)
340
+ return manifest
341
+
342
+
343
+ @dataclass(frozen=True)
344
+ class RecordedStreamGateway:
345
+ """A ``StreamGateway`` over sealed recorded batches; it reaches no network at all.
346
+
347
+ ``observed_at`` and ``available_at`` are coordinator inputs, not capture facts. The adapter
348
+ checks the batch it gets back against the observation evidence the coordinator recorded, and
349
+ a gateway that answered with times it read out of a file would make that check a tautology.
350
+ When the capture happened is separate evidence and lives in the manifest.
351
+ """
352
+
353
+ captures_root: Path
354
+ observed_at: str
355
+ available_at: str
356
+
357
+ def __post_init__(self) -> None:
358
+ if (
359
+ not self.captures_root.is_absolute()
360
+ or self.captures_root.is_symlink()
361
+ or not self.captures_root.is_dir()
362
+ ):
363
+ raise SourceContractError(
364
+ "STREAM_CAPTURE_ROOT",
365
+ "gateway.captures_root",
366
+ "must be an existing absolute non-symlink directory",
367
+ )
368
+ for name, value in (("observed_at", self.observed_at), ("available_at", self.available_at)):
369
+ _moment(value, f"gateway.{name}")
370
+
371
+ def read_batch(self, request: AcquisitionRequest) -> StreamBatch:
372
+ """Return the one sealed batch that follows the bookmark the request carries."""
373
+
374
+ query = request.query
375
+ stream_id = query["stream_id"]
376
+ manifest = self.manifest(stream_id)
377
+ if query["data_format"] != manifest.data_format:
378
+ raise SourceContractError(
379
+ "STREAM_CAPTURE_BINDING",
380
+ "request.query.data_format",
381
+ "differs from the format this capture was sealed in",
382
+ )
383
+ if query["filename"] != manifest.batch_filename:
384
+ raise SourceContractError(
385
+ "STREAM_CAPTURE_BINDING",
386
+ "request.query.filename",
387
+ "differs from the name this capture seals its batches under",
388
+ )
389
+ expected = query["expected_watermark"]
390
+ following = [batch for batch in manifest.batches if batch.previous_bookmark == expected]
391
+ if not following:
392
+ raise SourceContractError(
393
+ "STREAM_CAPTURE_EXHAUSTED",
394
+ "request.query.expected_watermark",
395
+ "this capture holds no sealed batch after the bookmark the request carries",
396
+ )
397
+ if len(following) > 1:
398
+ # The loader already refuses a record whose batches do not form one unbroken chain,
399
+ # so this cannot fire against a record that got this far. It is kept as the second
400
+ # gate on the same invariant, at the point the choice is actually made: a bug that
401
+ # served two batches for one bookmark would have to defeat both.
402
+ raise SourceContractError(
403
+ "STREAM_CAPTURE_AMBIGUOUS",
404
+ "capture.batches",
405
+ "more than one sealed batch claims the same predecessor bookmark",
406
+ )
407
+ selected = following[0]
408
+ content = _read_sealed_batch(
409
+ capture_root(self.captures_root, stream_id) / BATCH_DIRECTORY_NAME,
410
+ storage_name=selected.storage_name,
411
+ expected_size=selected.size_bytes,
412
+ )
413
+ if sha256_bytes(content) != selected.content_sha256:
414
+ raise SourceContractError(
415
+ "STREAM_CAPTURE_DIGEST",
416
+ "capture.batches",
417
+ "sealed batch bytes do not match the digest the capture recorded",
418
+ )
419
+ return StreamBatch(
420
+ query_digest=request.query_digest,
421
+ previous_watermark=selected.previous_bookmark,
422
+ next_watermark=selected.next_bookmark,
423
+ provider_event_ids=selected.event_ids,
424
+ content=content,
425
+ content_sha256=selected.content_sha256,
426
+ media_type=_MEDIA_TYPES[manifest.data_format],
427
+ data_format=manifest.data_format,
428
+ filename=manifest.batch_filename,
429
+ observed_at=self.observed_at,
430
+ available_at=self.available_at,
431
+ )
432
+
433
+ def manifest(self, stream_id: str) -> CaptureManifest:
434
+ """Return the sealed capture manifest for one stream beneath the capture root."""
435
+
436
+ root = capture_root(self.captures_root, stream_id)
437
+ raw = _read_capture_file(root, CAPTURE_MANIFEST_NAME, MAX_MANIFEST_BYTES)
438
+ manifest = read_capture_manifest(raw)
439
+ if manifest.stream_id != stream_id:
440
+ raise SourceContractError(
441
+ "STREAM_CAPTURE_BINDING",
442
+ "capture.stream_id",
443
+ "the capture found under this name records a different stream",
444
+ )
445
+ return manifest
446
+
447
+
448
+ def read_capture_manifest(raw: bytes) -> CaptureManifest:
449
+ """Parse and re-validate one sealed capture manifest.
450
+
451
+ Every structural rule the writer applied is checked again here, because a manifest is read
452
+ back from disk long after it was written and a build that trusted it unchecked would be
453
+ trusting a file, not evidence.
454
+ """
455
+
456
+ value = parse_json(raw)
457
+ _exact_keys(
458
+ value,
459
+ {
460
+ "schema_version",
461
+ "connector_id",
462
+ "connector_config_digest",
463
+ "stream_id",
464
+ "data_format",
465
+ "batch_filename",
466
+ "recorded_through",
467
+ "admitted_event_count",
468
+ "ignored_message_count",
469
+ "epochs",
470
+ "batches",
471
+ },
472
+ "capture",
473
+ )
474
+ if value["schema_version"] != STREAM_CAPTURE_VERSION:
475
+ raise SourceContractError(
476
+ "STREAM_CAPTURE_VERSION",
477
+ "capture.schema_version",
478
+ "is unsupported",
479
+ )
480
+ config = stream_connector(value["connector_id"])
481
+ if value["connector_config_digest"] != config.digest:
482
+ raise SourceContractError(
483
+ "STREAM_CAPTURE_CONNECTOR",
484
+ "capture.connector_config_digest",
485
+ "this capture was sealed under a different connector configuration",
486
+ )
487
+ if value["stream_id"] != config.stream_id or value["data_format"] != config.data_format:
488
+ raise SourceContractError(
489
+ "STREAM_CAPTURE_CONNECTOR",
490
+ "capture.stream_id",
491
+ "capture identity differs from its own connector configuration",
492
+ )
493
+ if value["batch_filename"] != f"{config.stream_id}.{config.data_format}":
494
+ raise SourceContractError(
495
+ "STREAM_CAPTURE_CONNECTOR",
496
+ "capture.batch_filename",
497
+ "capture batch name differs from its own connector configuration",
498
+ )
499
+ _moment(value["recorded_through"], "capture.recorded_through")
500
+ for name in ("admitted_event_count", "ignored_message_count"):
501
+ if type(value[name]) is not int or not 0 <= value[name] <= MAX_RECORDED_FRAMES:
502
+ raise SourceContractError("STREAM_CAPTURE", f"capture.{name}", "is out of bounds")
503
+ epochs = _read_epochs(value["epochs"])
504
+ batches = _read_batches(value["batches"], config=config)
505
+ admitted = sum(len(batch.event_ids) for batch in batches)
506
+ if admitted != value["admitted_event_count"]:
507
+ raise SourceContractError(
508
+ "STREAM_CAPTURE",
509
+ "capture.admitted_event_count",
510
+ "does not match the events the sealed batches hold",
511
+ )
512
+ _require_consistent_capture(epochs, batches, recorded_through=value["recorded_through"])
513
+ return CaptureManifest(
514
+ connector_id=value["connector_id"],
515
+ connector_config_digest=value["connector_config_digest"],
516
+ stream_id=value["stream_id"],
517
+ data_format=value["data_format"],
518
+ batch_filename=value["batch_filename"],
519
+ recorded_through=value["recorded_through"],
520
+ admitted_event_count=value["admitted_event_count"],
521
+ ignored_message_count=value["ignored_message_count"],
522
+ epochs=epochs,
523
+ batches=batches,
524
+ )
525
+
526
+
527
+ def _classify_recording(
528
+ config: StreamConnectorConfig,
529
+ recording: bytes,
530
+ ) -> tuple[tuple[_Frame, ...], int, tuple[_Connection, ...]]:
531
+ """Read the recording, classify every message, and assign identity and event time.
532
+
533
+ Returns the admitted frames, how many messages the connector classified as ignored, and
534
+ every connection the recording saw. The ignored count is carried rather than recomputed
535
+ because it is capture evidence: a capture that dropped a thousand heartbeats and one that
536
+ dropped none are different captures. The connections are carried for the same reason and a
537
+ stronger one -- a connection that admitted nothing at all still has to appear in the record,
538
+ or its two neighbours are spliced together with nothing saying so.
539
+ """
540
+
541
+ if not isinstance(recording, bytes | bytearray):
542
+ raise SourceContractError("STREAM_RECORDING", "recording", "must be exact bytes")
543
+ if not recording or len(recording) > MAX_RECORDING_BYTES:
544
+ raise SourceContractError(
545
+ "STREAM_RECORDING",
546
+ "recording",
547
+ f"must be non-empty and no larger than {MAX_RECORDING_BYTES} bytes",
548
+ )
549
+ lines = bytes(recording).split(b"\n")
550
+ if lines[-1] != b"":
551
+ raise SourceContractError(
552
+ "STREAM_RECORDING",
553
+ "recording",
554
+ "every recorded line must end with a newline",
555
+ )
556
+ lines = lines[:-1]
557
+ if not lines or len(lines) > MAX_RECORDED_FRAMES:
558
+ raise SourceContractError(
559
+ "STREAM_RECORDING",
560
+ "recording",
561
+ f"must hold between one and {MAX_RECORDED_FRAMES} recorded messages",
562
+ )
563
+ frames: list[_Frame] = []
564
+ connections: list[dict[str, Any]] = []
565
+ ignored = 0
566
+ epoch = 0
567
+ sequence = 0
568
+ epoch_opened = False
569
+ previous_received: datetime | None = None
570
+ for number, line in enumerate(lines, start=1):
571
+ where = f"recording.line[{number}]"
572
+ value = parse_json(line)
573
+ _exact_keys(value, _RECORDING_KEYS, where)
574
+ frame_epoch = value["epoch"]
575
+ if type(frame_epoch) is not int or not 1 <= frame_epoch <= MAX_EPOCHS:
576
+ raise SourceContractError(
577
+ "STREAM_RECORDING",
578
+ f"{where}.epoch",
579
+ f"must be an integer between 1 and {MAX_EPOCHS}",
580
+ )
581
+ if frame_epoch != epoch:
582
+ if frame_epoch != epoch + 1:
583
+ raise SourceContractError(
584
+ "STREAM_RECORDING",
585
+ f"{where}.epoch",
586
+ "connections must be numbered from one with no skipped connection",
587
+ )
588
+ epoch = frame_epoch
589
+ sequence = 0
590
+ epoch_opened = False
591
+ connections.append(
592
+ {
593
+ "epoch": epoch,
594
+ "opened_at": value["received_at"],
595
+ "closed_at": value["received_at"],
596
+ "opened_with_snapshot": False,
597
+ }
598
+ )
599
+ received = _moment(value["received_at"], f"{where}.received_at")
600
+ if previous_received is not None and received < previous_received:
601
+ raise SourceContractError(
602
+ "STREAM_RECORDING",
603
+ f"{where}.received_at",
604
+ "recorded messages must not move backwards in time",
605
+ )
606
+ previous_received = received
607
+ message = value["message"]
608
+ if not isinstance(message, dict) or not message:
609
+ raise SourceContractError(
610
+ "STREAM_RECORDING",
611
+ f"{where}.message",
612
+ "must be a non-empty JSON object",
613
+ )
614
+ message_type = message.get(config.message_type_field)
615
+ if not isinstance(message_type, str):
616
+ raise SourceContractError(
617
+ "STREAM_MESSAGE_TYPE",
618
+ f"{where}.message.{config.message_type_field}",
619
+ "must be the text message type the connector classifies on",
620
+ )
621
+ message_class = config.message_class(message_type)
622
+ connections[-1]["closed_at"] = value["received_at"]
623
+ if message_class == "ignored":
624
+ ignored += 1
625
+ continue
626
+ if not epoch_opened and config.gap_repair == "snapshot_on_subscribe":
627
+ if message_class != "snapshot":
628
+ raise SourceContractError(
629
+ "STREAM_CAPTURE_GAP_UNREPAIRED",
630
+ f"{where}.message",
631
+ "this connector repairs a connection with a full snapshot and this "
632
+ "connection did not open with one",
633
+ )
634
+ if not epoch_opened:
635
+ connections[-1]["opened_with_snapshot"] = message_class == "snapshot"
636
+ epoch_opened = True
637
+ sequence += 1
638
+ event_id = _event_id(config, message, epoch=epoch, sequence=sequence, where=where)
639
+ event_time = _event_time(config, message, where=where)
640
+ frames.append(
641
+ _Frame(
642
+ epoch=epoch,
643
+ sequence=sequence,
644
+ event_id=event_id,
645
+ position=_position(epoch, sequence),
646
+ message_class=message_class,
647
+ message_type=message_type,
648
+ event_time=event_time,
649
+ received_at=value["received_at"],
650
+ line=canonical_json_bytes(
651
+ {
652
+ "epoch": epoch,
653
+ "event_id": event_id,
654
+ "event_time": event_time,
655
+ "message": message,
656
+ "message_class": message_class,
657
+ "message_type": message_type,
658
+ "received_at": value["received_at"],
659
+ "sequence": sequence,
660
+ }
661
+ )
662
+ + b"\n",
663
+ )
664
+ )
665
+ if not frames:
666
+ raise SourceContractError(
667
+ "STREAM_RECORDING",
668
+ "recording",
669
+ "every recorded message was ignored, so there is nothing to seal",
670
+ )
671
+ identities = [frame.event_id for frame in frames]
672
+ if len(set(identities)) != len(identities):
673
+ raise SourceContractError(
674
+ "STREAM_EVENT_ID",
675
+ "recording",
676
+ "the connector's identity rule did not give every message a distinct name",
677
+ )
678
+ return (
679
+ tuple(frames),
680
+ ignored,
681
+ tuple(
682
+ _Connection(
683
+ epoch=item["epoch"],
684
+ opened_at=item["opened_at"],
685
+ closed_at=item["closed_at"],
686
+ opened_with_snapshot=item["opened_with_snapshot"],
687
+ )
688
+ for item in connections
689
+ ),
690
+ )
691
+
692
+
693
+ def _partition(
694
+ config: StreamConnectorConfig,
695
+ frames: tuple[_Frame, ...],
696
+ connections: tuple[_Connection, ...],
697
+ ) -> tuple[tuple[CapturedEpoch, ...], tuple[tuple[_Frame, ...], ...]]:
698
+ """Split the classified frames into epochs and into rotation-bounded batches.
699
+
700
+ Every connection the recording saw becomes an epoch, including one that admitted nothing, so
701
+ the numbering stays unbroken and a dropped connection is visible as itself rather than as an
702
+ absence. A batch never spans a connection: an order book reconstructs only within a
703
+ contiguous epoch, so a batch that straddled a gap would carry a hole no downstream check
704
+ could see.
705
+ """
706
+
707
+ by_epoch: dict[int, list[_Frame]] = {}
708
+ for frame in frames:
709
+ by_epoch.setdefault(frame.epoch, []).append(frame)
710
+ epochs: list[CapturedEpoch] = []
711
+ batches: list[tuple[_Frame, ...]] = []
712
+ for position, connection in enumerate(connections):
713
+ members = tuple(by_epoch.get(connection.epoch, ()))
714
+ epochs.append(
715
+ CapturedEpoch(
716
+ epoch=connection.epoch,
717
+ boundary=_boundary(config, connection, first=position == 0),
718
+ event_count=len(members),
719
+ first_event_id=members[0].event_id if members else None,
720
+ last_event_id=members[-1].event_id if members else None,
721
+ opened_at=connection.opened_at,
722
+ closed_at=connection.closed_at,
723
+ )
724
+ )
725
+ current: list[_Frame] = []
726
+ size = 0
727
+ opened = None if not members else _moment(members[0].received_at, "recording.received_at")
728
+ for frame in members:
729
+ arrived = _moment(frame.received_at, "recording.received_at")
730
+ rotate = bool(current) and (
731
+ len(current) >= config.max_batch_events
732
+ or size + len(frame.line) > config.max_batch_bytes
733
+ or (arrived - opened).total_seconds() > config.max_batch_seconds
734
+ )
735
+ if rotate:
736
+ batches.append(tuple(current))
737
+ current = []
738
+ size = 0
739
+ opened = arrived
740
+ current.append(frame)
741
+ size += len(frame.line)
742
+ if current:
743
+ batches.append(tuple(current))
744
+ if len(batches) > MAX_BATCHES:
745
+ raise SourceContractError(
746
+ "STREAM_CAPTURE",
747
+ "capture.batches",
748
+ f"one capture holds at most {MAX_BATCHES} sealed batches",
749
+ )
750
+ for group in batches:
751
+ if len(group) > MAX_BATCH_EVENTS:
752
+ raise SourceContractError(
753
+ "STREAM_EVENT_LIMIT",
754
+ "capture.batches",
755
+ "a sealed batch exceeds the adapter's own event bound",
756
+ )
757
+ if sum(len(frame.line) for frame in group) > config.max_batch_bytes:
758
+ raise SourceContractError(
759
+ "STREAM_CAPTURE_LIMIT",
760
+ "capture.batches",
761
+ "one recorded message is larger than this connector's whole batch bound",
762
+ )
763
+ return tuple(epochs), tuple(batches)
764
+
765
+
766
+ def _boundary(config: StreamConnectorConfig, connection: _Connection, *, first: bool) -> str:
767
+ """Say what the boundary before one connection left behind.
768
+
769
+ The first connection of a capture opens it and has no boundary before it. After that, a
770
+ hole is closed only when the venue's own repair rule actually ran: for a venue that re-sends
771
+ full state on subscribe, that means this connection really did open with a snapshot. A
772
+ connection that admitted nothing never re-sent one, so its boundary is a recorded gap --
773
+ which is the honest answer and the reason the gap count is not decorative.
774
+ """
775
+
776
+ if first:
777
+ return "opening"
778
+ if config.gap_repair == "snapshot_on_subscribe" and connection.opened_with_snapshot:
779
+ return "repaired_by_snapshot"
780
+ return "recorded_gap"
781
+
782
+
783
+ def _event_id(
784
+ config: StreamConnectorConfig,
785
+ message: dict[str, Any],
786
+ *,
787
+ epoch: int,
788
+ sequence: int,
789
+ where: str,
790
+ ) -> str:
791
+ """Assign one event its identity under the connector's declared rule."""
792
+
793
+ if config.event_id_rule == "derived":
794
+ return _position(epoch, sequence)
795
+ field = config.provider_event_id_field
796
+ assert field is not None # the configuration contract pairs the rule with the field
797
+ value = message.get(field)
798
+ if isinstance(value, int) and not isinstance(value, bool):
799
+ value = str(value)
800
+ if not isinstance(value, str) or EVENT_ID_GRAMMAR.fullmatch(value) is None:
801
+ raise SourceContractError(
802
+ "STREAM_EVENT_ID",
803
+ f"{where}.message.{field}",
804
+ "the venue's own event identifier is missing or is not a usable name",
805
+ )
806
+ return value
807
+
808
+
809
+ def _position(epoch: int, sequence: int) -> str:
810
+ """The Recorder-assigned ``{epoch}:{counter}`` position, ordered as text.
811
+
812
+ Zero padding is what makes text order and number order the same thing, which is what lets a
813
+ bookmark be compared without knowing how to take it apart.
814
+ """
815
+
816
+ return f"e{epoch:08d}:{sequence:012d}"
817
+
818
+
819
+ def _event_time(config: StreamConnectorConfig, message: dict[str, Any], *, where: str) -> str:
820
+ """Read the event-time field and check it against its declared interpretation."""
821
+
822
+ value = message.get(config.event_time_field)
823
+ if isinstance(value, int) and not isinstance(value, bool):
824
+ value = str(value)
825
+ if not isinstance(value, str) or not 1 <= len(value) <= 64:
826
+ raise SourceContractError(
827
+ "STREAM_EVENT_TIME",
828
+ f"{where}.message.{config.event_time_field}",
829
+ "must carry the event time the connector declares",
830
+ )
831
+ if EVENT_TIME_KINDS[config.event_time_kind].fullmatch(value) is None:
832
+ raise SourceContractError(
833
+ "STREAM_EVENT_TIME",
834
+ f"{where}.message.{config.event_time_field}",
835
+ f"does not read as {config.event_time_kind}",
836
+ )
837
+ return value
838
+
839
+
840
+ def _read_epochs(value: Any) -> tuple[CapturedEpoch, ...]:
841
+ if not isinstance(value, list) or not 1 <= len(value) <= MAX_EPOCHS:
842
+ raise SourceContractError("STREAM_CAPTURE", "capture.epochs", "must be a bounded list")
843
+ epochs: list[CapturedEpoch] = []
844
+ previous_closed: datetime | None = None
845
+ for index, item in enumerate(value, start=1):
846
+ _exact_keys(
847
+ item,
848
+ {
849
+ "epoch",
850
+ "boundary",
851
+ "event_count",
852
+ "first_event_id",
853
+ "last_event_id",
854
+ "opened_at",
855
+ "closed_at",
856
+ },
857
+ "capture.epochs",
858
+ )
859
+ if item["epoch"] != index or item["boundary"] not in GAP_BOUNDARIES:
860
+ raise SourceContractError(
861
+ "STREAM_CAPTURE",
862
+ "capture.epochs",
863
+ "connections must be numbered from one and carry a known boundary",
864
+ )
865
+ if (index == 1) != (item["boundary"] == "opening"):
866
+ raise SourceContractError(
867
+ "STREAM_CAPTURE",
868
+ "capture.epochs",
869
+ "only the first connection of a capture opens without a boundary before it",
870
+ )
871
+ if (
872
+ type(item["event_count"]) is not int
873
+ or not 0 <= item["event_count"] <= MAX_RECORDED_FRAMES
874
+ ):
875
+ raise SourceContractError(
876
+ "STREAM_CAPTURE",
877
+ "capture.epochs.event_count",
878
+ "each connection must record how many events it carried",
879
+ )
880
+ # A connection that admitted nothing has no first or last event, and one that admitted
881
+ # something has both. Anything else is a record describing a connection that cannot
882
+ # exist.
883
+ named = [item["first_event_id"], item["last_event_id"]]
884
+ if item["event_count"] == 0:
885
+ if named != [None, None]:
886
+ raise SourceContractError(
887
+ "STREAM_CAPTURE",
888
+ "capture.epochs.first_event_id",
889
+ "a connection that carried no event cannot name one",
890
+ )
891
+ else:
892
+ for name in ("first_event_id", "last_event_id"):
893
+ _event_name(item[name], f"capture.epochs.{name}")
894
+ if item["first_event_id"] > item["last_event_id"]:
895
+ raise SourceContractError(
896
+ "STREAM_CAPTURE",
897
+ "capture.epochs.last_event_id",
898
+ "a connection cannot end at an event earlier than the one it began at",
899
+ )
900
+ opened = _moment(item["opened_at"], "capture.epochs.opened_at")
901
+ closed = _moment(item["closed_at"], "capture.epochs.closed_at")
902
+ if closed < opened:
903
+ raise SourceContractError(
904
+ "STREAM_CAPTURE",
905
+ "capture.epochs.closed_at",
906
+ "a connection cannot close before it opened",
907
+ )
908
+ if previous_closed is not None and opened < previous_closed:
909
+ raise SourceContractError(
910
+ "STREAM_CAPTURE",
911
+ "capture.epochs.opened_at",
912
+ "connections must not overlap in time",
913
+ )
914
+ previous_closed = closed
915
+ epochs.append(
916
+ CapturedEpoch(
917
+ epoch=item["epoch"],
918
+ boundary=item["boundary"],
919
+ event_count=item["event_count"],
920
+ first_event_id=item["first_event_id"],
921
+ last_event_id=item["last_event_id"],
922
+ opened_at=item["opened_at"],
923
+ closed_at=item["closed_at"],
924
+ )
925
+ )
926
+ return tuple(epochs)
927
+
928
+
929
+ def _read_batches(value: Any, *, config: StreamConnectorConfig) -> tuple[CapturedBatch, ...]:
930
+ if not isinstance(value, list) or not 1 <= len(value) <= MAX_BATCHES:
931
+ raise SourceContractError("STREAM_CAPTURE", "capture.batches", "must be a bounded list")
932
+ batches: list[CapturedBatch] = []
933
+ previous: str | None = None
934
+ previous_epoch = 0
935
+ seen: set[str] = set()
936
+ for index, item in enumerate(value, start=1):
937
+ _exact_keys(
938
+ item,
939
+ {
940
+ "batch_index",
941
+ "epoch",
942
+ "storage_name",
943
+ "content_sha256",
944
+ "size_bytes",
945
+ "previous_bookmark",
946
+ "next_bookmark",
947
+ "event_ids",
948
+ "first_event_time",
949
+ "last_event_time",
950
+ "first_received_at",
951
+ "last_received_at",
952
+ },
953
+ "capture.batches",
954
+ )
955
+ if item["batch_index"] != index:
956
+ raise SourceContractError(
957
+ "STREAM_CAPTURE",
958
+ "capture.batches",
959
+ "sealed batches must be numbered from one in order",
960
+ )
961
+ if (
962
+ type(item["epoch"]) is not int
963
+ or not 1 <= item["epoch"] <= MAX_EPOCHS
964
+ or item["epoch"] < previous_epoch
965
+ ):
966
+ raise SourceContractError(
967
+ "STREAM_CAPTURE",
968
+ "capture.batches.epoch",
969
+ "each sealed batch belongs to one connection, in order",
970
+ )
971
+ previous_epoch = item["epoch"]
972
+ if (
973
+ not isinstance(item["storage_name"], str)
974
+ or _STORAGE_NAME.fullmatch(item["storage_name"]) is None
975
+ ):
976
+ raise SourceContractError(
977
+ "STREAM_CAPTURE",
978
+ "capture.batches.storage_name",
979
+ "must be the digest-addressed sealed batch name",
980
+ )
981
+ if item["storage_name"] != f"{item['content_sha256']}.{config.data_format}":
982
+ raise SourceContractError(
983
+ "STREAM_CAPTURE",
984
+ "capture.batches.storage_name",
985
+ "must be the exact digest-addressed name of its own bytes",
986
+ )
987
+ if (
988
+ type(item["size_bytes"]) is not int
989
+ or not 1 <= item["size_bytes"] <= config.max_batch_bytes
990
+ ):
991
+ raise SourceContractError(
992
+ "STREAM_CAPTURE",
993
+ "capture.batches.size_bytes",
994
+ "is outside the batch bound this connector declares",
995
+ )
996
+ if item["previous_bookmark"] != previous:
997
+ raise SourceContractError(
998
+ "STREAM_CAPTURE",
999
+ "capture.batches.previous_bookmark",
1000
+ "sealed batches must form one unbroken chain of bookmarks",
1001
+ )
1002
+ if index == 1 and item["previous_bookmark"] is not None:
1003
+ raise SourceContractError(
1004
+ "STREAM_CAPTURE",
1005
+ "capture.batches.previous_bookmark",
1006
+ "the first sealed batch of a capture follows no bookmark",
1007
+ )
1008
+ identities = _read_event_ids(item["event_ids"], config=config)
1009
+ if seen & set(identities):
1010
+ raise SourceContractError(
1011
+ "STREAM_CAPTURE",
1012
+ "capture.batches.event_ids",
1013
+ "one event name appears in more than one sealed batch",
1014
+ )
1015
+ seen |= set(identities)
1016
+ marker = _BOOKMARK.fullmatch(item["next_bookmark"] or "")
1017
+ if marker is None or int(marker.group(1)) != item["epoch"]:
1018
+ raise SourceContractError(
1019
+ "STREAM_CAPTURE",
1020
+ "capture.batches.next_bookmark",
1021
+ "a batch's bookmark is its own position in its own connection",
1022
+ )
1023
+ # The bookmark is what a build commits against and then asks to continue from, so it has
1024
+ # to move one way. A record whose batches walked backwards would rewind a source that
1025
+ # has already been built, without anything downstream being able to tell.
1026
+ if previous is not None and item["next_bookmark"] <= previous:
1027
+ raise SourceContractError(
1028
+ "STREAM_CAPTURE",
1029
+ "capture.batches.next_bookmark",
1030
+ "each sealed batch must advance the bookmark, never move it back",
1031
+ )
1032
+ for name in ("first_event_time", "last_event_time"):
1033
+ _declared_event_time(item[name], config=config, path=f"capture.batches.{name}")
1034
+ first = _moment(item["first_received_at"], "capture.batches.first_received_at")
1035
+ if _moment(item["last_received_at"], "capture.batches.last_received_at") < first:
1036
+ raise SourceContractError(
1037
+ "STREAM_CAPTURE",
1038
+ "capture.batches.last_received_at",
1039
+ "a sealed batch cannot end before it began",
1040
+ )
1041
+ batches.append(
1042
+ CapturedBatch(
1043
+ batch_index=item["batch_index"],
1044
+ epoch=item["epoch"],
1045
+ storage_name=item["storage_name"],
1046
+ content_sha256=item["content_sha256"],
1047
+ size_bytes=item["size_bytes"],
1048
+ previous_bookmark=item["previous_bookmark"],
1049
+ next_bookmark=item["next_bookmark"],
1050
+ event_ids=identities,
1051
+ first_event_time=item["first_event_time"],
1052
+ last_event_time=item["last_event_time"],
1053
+ first_received_at=item["first_received_at"],
1054
+ last_received_at=item["last_received_at"],
1055
+ )
1056
+ )
1057
+ previous = item["next_bookmark"]
1058
+ return tuple(batches)
1059
+
1060
+
1061
+ def _read_event_ids(value: Any, *, config: StreamConnectorConfig) -> tuple[str, ...]:
1062
+ """Check one batch's event names are bounded, distinct, well formed, and in order."""
1063
+
1064
+ if not isinstance(value, list) or not 1 <= len(value) <= config.max_batch_events:
1065
+ raise SourceContractError(
1066
+ "STREAM_CAPTURE",
1067
+ "capture.batches.event_ids",
1068
+ "must be a bounded non-empty list of event names",
1069
+ )
1070
+ for item in value:
1071
+ _event_name(item, "capture.batches.event_ids")
1072
+ if len(set(value)) != len(value):
1073
+ raise SourceContractError(
1074
+ "STREAM_CAPTURE",
1075
+ "capture.batches.event_ids",
1076
+ "must name each event once",
1077
+ )
1078
+ return tuple(value)
1079
+
1080
+
1081
+ def _event_name(value: Any, path: str) -> str:
1082
+ if not isinstance(value, str) or EVENT_ID_GRAMMAR.fullmatch(value) is None:
1083
+ raise SourceContractError(
1084
+ "STREAM_EVENT_ID",
1085
+ path,
1086
+ "must be one event name in the grammar the ledger admits",
1087
+ )
1088
+ return value
1089
+
1090
+
1091
+ def _declared_event_time(value: Any, *, config: StreamConnectorConfig, path: str) -> str:
1092
+ if (
1093
+ not isinstance(value, str)
1094
+ or not 1 <= len(value) <= 64
1095
+ or EVENT_TIME_KINDS[config.event_time_kind].fullmatch(value) is None
1096
+ ):
1097
+ raise SourceContractError(
1098
+ "STREAM_EVENT_TIME",
1099
+ path,
1100
+ f"does not read as {config.event_time_kind}",
1101
+ )
1102
+ return value
1103
+
1104
+
1105
+ def _require_consistent_capture(
1106
+ epochs: tuple[CapturedEpoch, ...],
1107
+ batches: tuple[CapturedBatch, ...],
1108
+ *,
1109
+ recorded_through: str,
1110
+ ) -> None:
1111
+ """Check the two halves of a capture record describe the same capture.
1112
+
1113
+ Each half is well formed on its own by the time this runs. What is left is whether they
1114
+ agree: whether every batch belongs to a connection the record lists, whether each
1115
+ connection's stated event count is the number of events its batches actually hold, whether
1116
+ the connection's own first and last event names are the ones its batches begin and end with,
1117
+ and whether the capture's stated end is where its last batch actually stopped.
1118
+ """
1119
+
1120
+ numbers = {epoch.epoch for epoch in epochs}
1121
+ unknown = sorted({batch.epoch for batch in batches} - numbers)
1122
+ if unknown:
1123
+ raise SourceContractError(
1124
+ "STREAM_CAPTURE",
1125
+ "capture.batches.epoch",
1126
+ f"sealed batches name connections this capture does not record: {unknown[:10]}",
1127
+ )
1128
+ for epoch in epochs:
1129
+ members = [batch for batch in batches if batch.epoch == epoch.epoch]
1130
+ held = [identity for batch in members for identity in batch.event_ids]
1131
+ if len(held) != epoch.event_count:
1132
+ raise SourceContractError(
1133
+ "STREAM_CAPTURE",
1134
+ "capture.epochs.event_count",
1135
+ "a connection's event count is not the number its sealed batches hold",
1136
+ )
1137
+ if held and (held[0] != epoch.first_event_id or held[-1] != epoch.last_event_id):
1138
+ raise SourceContractError(
1139
+ "STREAM_CAPTURE",
1140
+ "capture.epochs.first_event_id",
1141
+ "a connection does not begin and end at the events its batches do",
1142
+ )
1143
+ if batches[-1].last_received_at != recorded_through:
1144
+ raise SourceContractError(
1145
+ "STREAM_CAPTURE",
1146
+ "capture.recorded_through",
1147
+ "a capture ends where its last sealed batch ends",
1148
+ )
1149
+
1150
+
1151
+ def _write_manifest(stream_root: Path, manifest: CaptureManifest) -> None:
1152
+ """Replace the capture manifest atomically beneath a descriptor-bound root."""
1153
+
1154
+ payload = canonical_json_bytes(manifest.to_dict())
1155
+ if len(payload) > MAX_MANIFEST_BYTES:
1156
+ raise SourceContractError(
1157
+ "STREAM_CAPTURE_LIMIT",
1158
+ "capture",
1159
+ f"the capture record exceeds its {MAX_MANIFEST_BYTES}-byte budget",
1160
+ )
1161
+ try:
1162
+ root_fd = os.open(
1163
+ stream_root,
1164
+ os.O_RDONLY
1165
+ | getattr(os, "O_DIRECTORY", 0)
1166
+ | getattr(os, "O_NOFOLLOW", 0)
1167
+ | getattr(os, "O_CLOEXEC", 0),
1168
+ )
1169
+ except OSError:
1170
+ raise SourceContractError(
1171
+ "STREAM_CAPTURE_ROOT",
1172
+ "capture",
1173
+ f"{stream_root.name} is not a plain directory this capture may be written into",
1174
+ ) from None
1175
+ temporary = f".{CAPTURE_MANIFEST_NAME}.{os.getpid()}.tmp"
1176
+ try:
1177
+ try:
1178
+ os.unlink(temporary, dir_fd=root_fd)
1179
+ except FileNotFoundError:
1180
+ pass
1181
+ descriptor = os.open(
1182
+ temporary,
1183
+ os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0),
1184
+ 0o600,
1185
+ dir_fd=root_fd,
1186
+ )
1187
+ try:
1188
+ view = memoryview(payload)
1189
+ written = 0
1190
+ while written < len(view):
1191
+ count = os.write(descriptor, view[written:])
1192
+ if count <= 0:
1193
+ raise SourceContractError(
1194
+ "STREAM_CAPTURE_WRITE",
1195
+ "capture",
1196
+ "write made no progress",
1197
+ )
1198
+ written += count
1199
+ os.fsync(descriptor)
1200
+ finally:
1201
+ os.close(descriptor)
1202
+ try:
1203
+ os.replace(temporary, CAPTURE_MANIFEST_NAME, src_dir_fd=root_fd, dst_dir_fd=root_fd)
1204
+ os.fsync(root_fd)
1205
+ finally:
1206
+ try:
1207
+ os.unlink(temporary, dir_fd=root_fd)
1208
+ except FileNotFoundError:
1209
+ pass
1210
+ except OSError:
1211
+ raise SourceContractError(
1212
+ "STREAM_CAPTURE_WRITE",
1213
+ "capture",
1214
+ "the capture record could not be written into its own directory",
1215
+ ) from None
1216
+ finally:
1217
+ os.close(root_fd)
1218
+
1219
+
1220
+ def _capture_component(stream_id: Any) -> str:
1221
+ """Check a stream name is one strict path component before it addresses a directory."""
1222
+
1223
+ if not isinstance(stream_id, str) or re.fullmatch(r"[a-z][a-z0-9-]{0,63}", stream_id) is None:
1224
+ raise SourceContractError(
1225
+ "STREAM_CAPTURE_ROOT",
1226
+ "request.query.stream_id",
1227
+ "must be one plain lowercase name, so it can address exactly one directory",
1228
+ )
1229
+ return stream_id
1230
+
1231
+
1232
+ def _require_plain_directory(path: Path) -> None:
1233
+ """Make one directory if it is missing, then check it is a directory and not a link to one.
1234
+
1235
+ ``Path.mkdir(exist_ok=True)`` is satisfied by a pre-existing symlink to a directory, because
1236
+ the existence test follows the link. So making the directory proves nothing on its own, and
1237
+ the ``lstat`` below is the check: a planted link must not get to decide where a capture is
1238
+ written. Everything this module opens afterwards uses ``O_NOFOLLOW``, so this runs once, up
1239
+ front, and a refusal leaves nothing written.
1240
+ """
1241
+
1242
+ try:
1243
+ path.mkdir(mode=0o700, parents=True, exist_ok=True)
1244
+ except OSError:
1245
+ raise SourceContractError(
1246
+ "STREAM_CAPTURE_ROOT",
1247
+ "capture",
1248
+ f"{path.name} could not be created as a plain directory",
1249
+ ) from None
1250
+ try:
1251
+ info = os.lstat(path)
1252
+ except OSError:
1253
+ raise SourceContractError(
1254
+ "STREAM_CAPTURE_ROOT",
1255
+ "capture",
1256
+ f"{path.name} could not be inspected",
1257
+ ) from None
1258
+ if not stat.S_ISDIR(info.st_mode):
1259
+ raise SourceContractError(
1260
+ "STREAM_CAPTURE_ROOT",
1261
+ "capture",
1262
+ f"{path.name} must be a plain directory, not a link to one",
1263
+ )
1264
+
1265
+
1266
+ def _read_capture_file(root: Path, name: str, maximum: int) -> bytes:
1267
+ """Read one bounded regular file beneath a root without following any link.
1268
+
1269
+ The controls are the ones ``adapters._read_confined_file`` already applies to a source file,
1270
+ for the same reasons and in the same order: open non-blocking so a planted FIFO cannot hold
1271
+ the command open forever, require one regular file with one link so a device or a hardlink
1272
+ farm is refused, and compare the full descriptor and path identity before and after the read
1273
+ so a file swapped underneath the read is caught rather than returned.
1274
+ """
1275
+
1276
+ directory_flags = (
1277
+ os.O_RDONLY
1278
+ | getattr(os, "O_DIRECTORY", 0)
1279
+ | getattr(os, "O_NOFOLLOW", 0)
1280
+ | getattr(os, "O_CLOEXEC", 0)
1281
+ )
1282
+ try:
1283
+ root_fd = os.open(root, directory_flags)
1284
+ except OSError:
1285
+ raise SourceContractError(
1286
+ "STREAM_CAPTURE_ROOT",
1287
+ "capture",
1288
+ f"no readable capture directory at {root.name}",
1289
+ ) from None
1290
+ try:
1291
+ try:
1292
+ descriptor = os.open(
1293
+ name,
1294
+ os.O_RDONLY
1295
+ | getattr(os, "O_NOFOLLOW", 0)
1296
+ | getattr(os, "O_NONBLOCK", 0)
1297
+ | getattr(os, "O_CLOEXEC", 0),
1298
+ dir_fd=root_fd,
1299
+ )
1300
+ except OSError:
1301
+ raise SourceContractError(
1302
+ "STREAM_CAPTURE_ROOT",
1303
+ "capture",
1304
+ f"{name} is missing or is not a plain readable file",
1305
+ ) from None
1306
+ try:
1307
+ before = os.fstat(descriptor)
1308
+ if not stat.S_ISREG(before.st_mode) or before.st_nlink != 1:
1309
+ raise SourceContractError(
1310
+ "STREAM_CAPTURE_ROOT",
1311
+ "capture",
1312
+ f"{name} must be one plain file with one name, not a link, a pipe, or a device",
1313
+ )
1314
+ if not 1 <= before.st_size <= maximum:
1315
+ raise SourceContractError(
1316
+ "STREAM_CAPTURE_LIMIT",
1317
+ "capture",
1318
+ f"{name} is empty or over its {maximum}-byte budget",
1319
+ )
1320
+ chunks: list[bytes] = []
1321
+ total = 0
1322
+ while total <= maximum:
1323
+ chunk = os.read(descriptor, min(64 * 1024, maximum + 1 - total))
1324
+ if not chunk:
1325
+ break
1326
+ chunks.append(chunk)
1327
+ total += len(chunk)
1328
+ after = os.fstat(descriptor)
1329
+ named = os.stat(name, dir_fd=root_fd, follow_symlinks=False)
1330
+ if (
1331
+ total != before.st_size
1332
+ or _capture_stat_signature(before) != _capture_stat_signature(after)
1333
+ or _capture_stat_signature(before) != _capture_stat_signature(named)
1334
+ ):
1335
+ raise SourceContractError(
1336
+ "STREAM_CAPTURE_MUTATED",
1337
+ "capture",
1338
+ f"{name} changed while it was being read",
1339
+ )
1340
+ return b"".join(chunks)
1341
+ finally:
1342
+ os.close(descriptor)
1343
+ except OSError:
1344
+ raise SourceContractError(
1345
+ "STREAM_CAPTURE_ROOT",
1346
+ "capture",
1347
+ f"{name} could not be read as one plain bounded file",
1348
+ ) from None
1349
+ finally:
1350
+ os.close(root_fd)
1351
+
1352
+
1353
+ def _capture_stat_signature(info: os.stat_result) -> tuple[int, ...]:
1354
+ """Bind the complete regular-file state one capture read is allowed to see."""
1355
+
1356
+ return (
1357
+ info.st_dev,
1358
+ info.st_ino,
1359
+ info.st_uid,
1360
+ info.st_gid,
1361
+ info.st_mode,
1362
+ info.st_nlink,
1363
+ info.st_size,
1364
+ info.st_mtime_ns,
1365
+ info.st_ctime_ns,
1366
+ )
1367
+
1368
+
1369
+ def _read_sealed_batch(root: Path, *, storage_name: str, expected_size: int) -> bytes:
1370
+ if _STORAGE_NAME.fullmatch(storage_name) is None:
1371
+ raise SourceContractError(
1372
+ "STREAM_CAPTURE",
1373
+ "capture.batches.storage_name",
1374
+ "must be the digest-addressed sealed batch name",
1375
+ )
1376
+ content = _read_capture_file(root, storage_name, MAX_MANIFEST_BYTES)
1377
+ if len(content) != expected_size:
1378
+ raise SourceContractError(
1379
+ "STREAM_CAPTURE_DIGEST",
1380
+ "capture.batches.size_bytes",
1381
+ "sealed batch is not the size the capture recorded",
1382
+ )
1383
+ return content
1384
+
1385
+
1386
+ def _exact_keys(value: Any, keys: frozenset[str] | set[str], path: str) -> None:
1387
+ if not isinstance(value, dict) or set(value) != set(keys):
1388
+ raise SourceContractError(
1389
+ "STREAM_CAPTURE",
1390
+ path,
1391
+ f"must be an object holding exactly {sorted(keys)}",
1392
+ )
1393
+
1394
+
1395
+ def _moment(value: Any, path: str) -> datetime:
1396
+ """Check one canonical UTC time and return it, so spans can be measured from it."""
1397
+
1398
+ if not isinstance(value, str) or _TIMESTAMP.fullmatch(value) is None:
1399
+ raise SourceContractError(
1400
+ "STREAM_CAPTURE_TIME",
1401
+ path,
1402
+ "must be a canonical RFC 3339 UTC time",
1403
+ )
1404
+ return datetime.fromisoformat(value[:-1] + "+00:00")