mostlyright-data 0.9.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (314) hide show
  1. mostlyright/data_harness/__init__.py +158 -0
  2. mostlyright/data_harness/acquisition/__init__.py +55 -0
  3. mostlyright/data_harness/acquisition/http.py +2773 -0
  4. mostlyright/data_harness/acquisition/parsing.py +809 -0
  5. mostlyright/data_harness/acquisition/ranges.py +495 -0
  6. mostlyright/data_harness/acquisition/result_download.py +360 -0
  7. mostlyright/data_harness/acquisition/retention_admission.py +248 -0
  8. mostlyright/data_harness/acquisition/sandbox.py +4888 -0
  9. mostlyright/data_harness/acquisition/url_policy.py +530 -0
  10. mostlyright/data_harness/agent_runtime.py +2743 -0
  11. mostlyright/data_harness/assets/logo-ink.svg +31 -0
  12. mostlyright/data_harness/backends/__init__.py +28 -0
  13. mostlyright/data_harness/backends/pandas_backend.py +350 -0
  14. mostlyright/data_harness/backends/polars_backend.py +366 -0
  15. mostlyright/data_harness/backends/protocol.py +124 -0
  16. mostlyright/data_harness/backends/reference.py +83 -0
  17. mostlyright/data_harness/backends/registry.py +55 -0
  18. mostlyright/data_harness/backends/restrictions.py +126 -0
  19. mostlyright/data_harness/canonical.py +333 -0
  20. mostlyright/data_harness/catalog_job.py +625 -0
  21. mostlyright/data_harness/cli.py +5398 -0
  22. mostlyright/data_harness/contracts.py +53 -0
  23. mostlyright/data_harness/coordinator.py +1307 -0
  24. mostlyright/data_harness/deploy.py +924 -0
  25. mostlyright/data_harness/deploy_target.py +312 -0
  26. mostlyright/data_harness/deployment_evidence.py +1067 -0
  27. mostlyright/data_harness/event_presentation.py +576 -0
  28. mostlyright/data_harness/events.py +2152 -0
  29. mostlyright/data_harness/fast_delimited.py +239 -0
  30. mostlyright/data_harness/fleet.py +237 -0
  31. mostlyright/data_harness/formats.py +236 -0
  32. mostlyright/data_harness/governors.py +1163 -0
  33. mostlyright/data_harness/hosted_bootstrap.py +972 -0
  34. mostlyright/data_harness/hosted_crawler.py +1115 -0
  35. mostlyright/data_harness/hosted_crawler_container_smoke.py +351 -0
  36. mostlyright/data_harness/hosted_crawler_fetch.py +423 -0
  37. mostlyright/data_harness/hosted_crawler_job.py +1277 -0
  38. mostlyright/data_harness/hosted_crawler_protocol.py +676 -0
  39. mostlyright/data_harness/hosted_dataset.py +1500 -0
  40. mostlyright/data_harness/hosted_deploy.py +3037 -0
  41. mostlyright/data_harness/hosted_handoff.py +62 -0
  42. mostlyright/data_harness/hosted_ingestion_contract.py +504 -0
  43. mostlyright/data_harness/hosted_ingestion_job.py +356 -0
  44. mostlyright/data_harness/hosted_ingestion_job_smoke.py +40 -0
  45. mostlyright/data_harness/hosted_session_container_smoke.py +194 -0
  46. mostlyright/data_harness/hosted_session_worker.py +3554 -0
  47. mostlyright/data_harness/hosted_session_worker_job_smoke.py +46 -0
  48. mostlyright/data_harness/hosted_worker.py +6784 -0
  49. mostlyright/data_harness/ingestion/__init__.py +56 -0
  50. mostlyright/data_harness/ingestion/contracts.py +461 -0
  51. mostlyright/data_harness/ingestion/faults.py +42 -0
  52. mostlyright/data_harness/ingestion/gcs_store.py +1162 -0
  53. mostlyright/data_harness/ingestion/spool.py +130 -0
  54. mostlyright/data_harness/ingestion/store.py +885 -0
  55. mostlyright/data_harness/key_seam.py +434 -0
  56. mostlyright/data_harness/linux_process_boundary.py +262 -0
  57. mostlyright/data_harness/local_contracts.py +2880 -0
  58. mostlyright/data_harness/local_search/__init__.py +5 -0
  59. mostlyright/data_harness/local_search/build_index.py +1087 -0
  60. mostlyright/data_harness/local_search/contracts.py +920 -0
  61. mostlyright/data_harness/local_search/query_trace.py +266 -0
  62. mostlyright/data_harness/local_search/retrieval.py +700 -0
  63. mostlyright/data_harness/local_search/sealed.py +474 -0
  64. mostlyright/data_harness/local_search/service.py +784 -0
  65. mostlyright/data_harness/nbrender/CONTRACT.md +212 -0
  66. mostlyright/data_harness/nbrender/__init__.py +12 -0
  67. mostlyright/data_harness/nbrender/chrome.py +359 -0
  68. mostlyright/data_harness/nbrender/code_body.py +266 -0
  69. mostlyright/data_harness/nbrender/document.py +407 -0
  70. mostlyright/data_harness/nbrender/frame.py +275 -0
  71. mostlyright/data_harness/nbrender/interactive.py +337 -0
  72. mostlyright/data_harness/nbrender/markdown_body.py +477 -0
  73. mostlyright/data_harness/nbrender/mr_components.py +134 -0
  74. mostlyright/data_harness/nbrender/outputs_data.py +595 -0
  75. mostlyright/data_harness/nbrender/outputs_rich.py +906 -0
  76. mostlyright/data_harness/nbrender/outputs_source.py +260 -0
  77. mostlyright/data_harness/nbrender/outputs_stage.py +176 -0
  78. mostlyright/data_harness/nbrender/outputs_text.py +400 -0
  79. mostlyright/data_harness/nbrender/parse.py +394 -0
  80. mostlyright/data_harness/nbrender/status.py +40 -0
  81. mostlyright/data_harness/nbrender/tokens.py +1295 -0
  82. mostlyright/data_harness/notebook.py +1710 -0
  83. mostlyright/data_harness/offline.py +2049 -0
  84. mostlyright/data_harness/operation_registry.py +1007 -0
  85. mostlyright/data_harness/operator_setup.py +239 -0
  86. mostlyright/data_harness/pipeline.py +6428 -0
  87. mostlyright/data_harness/plan_graph.py +2026 -0
  88. mostlyright/data_harness/preparation/__init__.py +104 -0
  89. mostlyright/data_harness/preparation/contracts.py +1017 -0
  90. mostlyright/data_harness/preparation/engine.py +221 -0
  91. mostlyright/data_harness/preparation/errors.py +14 -0
  92. mostlyright/data_harness/preparation/gates.py +751 -0
  93. mostlyright/data_harness/preparation/joins.py +574 -0
  94. mostlyright/data_harness/preparation/profile.py +384 -0
  95. mostlyright/data_harness/preparation/table.py +217 -0
  96. mostlyright/data_harness/preparation/transforms.py +568 -0
  97. mostlyright/data_harness/progress_events.py +534 -0
  98. mostlyright/data_harness/readers/__init__.py +46 -0
  99. mostlyright/data_harness/readers/containers.py +963 -0
  100. mostlyright/data_harness/readers/contracts.py +542 -0
  101. mostlyright/data_harness/readers/delimited.py +257 -0
  102. mostlyright/data_harness/readers/grib2/__init__.py +33 -0
  103. mostlyright/data_harness/readers/grib2/admission.py +722 -0
  104. mostlyright/data_harness/readers/grib2/decode.py +1009 -0
  105. mostlyright/data_harness/readers/grib2/geometry.py +1133 -0
  106. mostlyright/data_harness/readers/grib2/portable_math.py +501 -0
  107. mostlyright/data_harness/readers/json_tabular.py +485 -0
  108. mostlyright/data_harness/readers/registry.py +514 -0
  109. mostlyright/data_harness/readers/samples/README.md +110 -0
  110. mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/cities.csv.gz +0 -0
  111. mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/expected.json +24 -0
  112. mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/cities.csv.gz +0 -0
  113. mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/expected.json +24 -0
  114. mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/cities.tar +0 -0
  115. mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/expected.json +24 -0
  116. mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/cities.tar +0 -0
  117. mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/expected.json +24 -0
  118. mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/cities.zip +0 -0
  119. mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/expected.json +25 -0
  120. mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
  121. mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/expected.json +25 -0
  122. mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
  123. mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/expected.json +25 -0
  124. mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/cities.csv +3 -0
  125. mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/expected.json +23 -0
  126. mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/cities.tsv +5 -0
  127. mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/expected.json +25 -0
  128. mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/expected.json +30 -0
  129. mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/observations.csv +5 -0
  130. mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/expected.json +44 -0
  131. mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/stations.json +1 -0
  132. mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/expected.json +48 -0
  133. mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/observations.ndjson +4 -0
  134. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/cities.xlsx +0 -0
  135. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/expected.json +24 -0
  136. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/expected.json +27 -0
  137. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
  138. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/expected.json +27 -0
  139. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
  140. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/README.md +20 -0
  141. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/expected.json +55 -0
  142. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/gfs-2m-temperature.grib2 +0 -0
  143. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/expected.json +54 -0
  144. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
  145. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/expected.json +54 -0
  146. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/hrrr-categorical-rain.grib2 +0 -0
  147. mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/expected.json +54 -0
  148. mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
  149. mostlyright/data_harness/readers/samples.py +582 -0
  150. mostlyright/data_harness/readers/spreadsheet.py +803 -0
  151. mostlyright/data_harness/readers/tabular.py +510 -0
  152. mostlyright/data_harness/recipe.py +5321 -0
  153. mostlyright/data_harness/repair/__init__.py +78 -0
  154. mostlyright/data_harness/repair/adapters.py +274 -0
  155. mostlyright/data_harness/repair/contracts.py +872 -0
  156. mostlyright/data_harness/repair/coordinator.py +1099 -0
  157. mostlyright/data_harness/repair/errors.py +16 -0
  158. mostlyright/data_harness/review.py +2533 -0
  159. mostlyright/data_harness/rowset.py +283 -0
  160. mostlyright/data_harness/serving.py +1975 -0
  161. mostlyright/data_harness/serving_edge.py +590 -0
  162. mostlyright/data_harness/serving_http.py +1031 -0
  163. mostlyright/data_harness/session_probes.py +759 -0
  164. mostlyright/data_harness/signing.py +101 -0
  165. mostlyright/data_harness/source_discovery.py +898 -0
  166. mostlyright/data_harness/sources/__init__.py +209 -0
  167. mostlyright/data_harness/sources/_adapter_steps.py +213 -0
  168. mostlyright/data_harness/sources/adapters.py +1214 -0
  169. mostlyright/data_harness/sources/cadence.py +1428 -0
  170. mostlyright/data_harness/sources/cadence_emission.py +453 -0
  171. mostlyright/data_harness/sources/cadence_history.py +546 -0
  172. mostlyright/data_harness/sources/catalog/__init__.py +17 -0
  173. mostlyright/data_harness/sources/catalog/admission.py +477 -0
  174. mostlyright/data_harness/sources/catalog/authoring.py +1701 -0
  175. mostlyright/data_harness/sources/catalog/authoring_policy.py +701 -0
  176. mostlyright/data_harness/sources/catalog/authoring_shards.py +1217 -0
  177. mostlyright/data_harness/sources/catalog/bounded_io.py +231 -0
  178. mostlyright/data_harness/sources/catalog/channel.py +523 -0
  179. mostlyright/data_harness/sources/catalog/channel_client.py +296 -0
  180. mostlyright/data_harness/sources/catalog/contracts.py +825 -0
  181. mostlyright/data_harness/sources/catalog/coverage.py +137 -0
  182. mostlyright/data_harness/sources/catalog/delta.py +1340 -0
  183. mostlyright/data_harness/sources/catalog/embedding.py +532 -0
  184. mostlyright/data_harness/sources/catalog/entry_v2.py +1182 -0
  185. mostlyright/data_harness/sources/catalog/fill.py +3889 -0
  186. mostlyright/data_harness/sources/catalog/fill_partitions.py +459 -0
  187. mostlyright/data_harness/sources/catalog/fill_staging.py +1105 -0
  188. mostlyright/data_harness/sources/catalog/gating.py +374 -0
  189. mostlyright/data_harness/sources/catalog/generation_receipt.py +1607 -0
  190. mostlyright/data_harness/sources/catalog/harvest/__init__.py +7 -0
  191. mostlyright/data_harness/sources/catalog/harvest/ckan.py +384 -0
  192. mostlyright/data_harness/sources/catalog/harvest/datagov_v4.py +798 -0
  193. mostlyright/data_harness/sources/catalog/harvest/protocol.py +964 -0
  194. mostlyright/data_harness/sources/catalog/harvest/sdmx.py +445 -0
  195. mostlyright/data_harness/sources/catalog/harvest/stac.py +384 -0
  196. mostlyright/data_harness/sources/catalog/health.py +447 -0
  197. mostlyright/data_harness/sources/catalog/hosted_catalog.py +105 -0
  198. mostlyright/data_harness/sources/catalog/identity_history.py +1549 -0
  199. mostlyright/data_harness/sources/catalog/neural.py +1618 -0
  200. mostlyright/data_harness/sources/catalog/packed_catalog.py +2345 -0
  201. mostlyright/data_harness/sources/catalog/packed_retrieval.py +1517 -0
  202. mostlyright/data_harness/sources/catalog/packed_writer.py +2802 -0
  203. mostlyright/data_harness/sources/catalog/query_trace.py +1037 -0
  204. mostlyright/data_harness/sources/catalog/recommend.py +171 -0
  205. mostlyright/data_harness/sources/catalog/retrieval.py +230 -0
  206. mostlyright/data_harness/sources/catalog/retrieval_manifest.py +995 -0
  207. mostlyright/data_harness/sources/catalog/rights_decisions.py +254 -0
  208. mostlyright/data_harness/sources/catalog/sealed.py +560 -0
  209. mostlyright/data_harness/sources/catalog/search.py +230 -0
  210. mostlyright/data_harness/sources/catalog/streaming_delta.py +1097 -0
  211. mostlyright/data_harness/sources/catalog/update.py +891 -0
  212. mostlyright/data_harness/sources/collections.py +815 -0
  213. mostlyright/data_harness/sources/contracts.py +2223 -0
  214. mostlyright/data_harness/sources/deletion.py +761 -0
  215. mostlyright/data_harness/sources/fitness.py +162 -0
  216. mostlyright/data_harness/sources/governance.py +163 -0
  217. mostlyright/data_harness/sources/hosted.py +173 -0
  218. mostlyright/data_harness/sources/integration.py +218 -0
  219. mostlyright/data_harness/sources/range_reader.py +418 -0
  220. mostlyright/data_harness/sources/registry.py +514 -0
  221. mostlyright/data_harness/sources/rights_rule.py +59 -0
  222. mostlyright/data_harness/sources/source_cadence_vectors.v1.json +1 -0
  223. mostlyright/data_harness/sources/sports.py +521 -0
  224. mostlyright/data_harness/sources/stream.py +524 -0
  225. mostlyright/data_harness/sources/stream_connector.py +418 -0
  226. mostlyright/data_harness/sources/stream_recorder.py +1404 -0
  227. mostlyright/data_harness/studio_boundary.py +2019 -0
  228. mostlyright/data_harness/thin/__init__.py +37 -0
  229. mostlyright/data_harness/thin/acquire.py +1137 -0
  230. mostlyright/data_harness/thin/acquire_cancel.py +579 -0
  231. mostlyright/data_harness/thin/approvals.py +617 -0
  232. mostlyright/data_harness/thin/commands.py +406 -0
  233. mostlyright/data_harness/thin/download.py +194 -0
  234. mostlyright/data_harness/thin/narrative.py +589 -0
  235. mostlyright/data_harness/thin/parity.py +1070 -0
  236. mostlyright/data_harness/thin/propose.py +2759 -0
  237. mostlyright/data_harness/thin/research.py +1663 -0
  238. mostlyright/data_harness/thin/router.py +924 -0
  239. mostlyright/data_harness/thin/runs.py +519 -0
  240. mostlyright/data_harness/thin/session.py +281 -0
  241. mostlyright/data_harness/thin/stream.py +501 -0
  242. mostlyright/data_harness/thin/transport.py +187 -0
  243. mostlyright/data_harness/thin/vocabulary.py +368 -0
  244. mostlyright/data_harness/thin/workers.py +164 -0
  245. mostlyright/data_harness/ucum/TABLE-PIN.json +40 -0
  246. mostlyright/data_harness/ucum/ucum-subset.v1.json +632 -0
  247. mostlyright/data_harness/unit_flow.py +927 -0
  248. mostlyright/data_harness/units.py +572 -0
  249. mostlyright/data_harness/ux/__init__.py +9 -0
  250. mostlyright/data_harness/ux/approve.py +485 -0
  251. mostlyright/data_harness/ux/author_yaml.py +597 -0
  252. mostlyright/data_harness/ux/cloud_auth.py +447 -0
  253. mostlyright/data_harness/ux/commands/__init__.py +260 -0
  254. mostlyright/data_harness/ux/commands/approve.py +136 -0
  255. mostlyright/data_harness/ux/commands/auth.py +744 -0
  256. mostlyright/data_harness/ux/commands/author.py +79 -0
  257. mostlyright/data_harness/ux/commands/catalog_author.py +403 -0
  258. mostlyright/data_harness/ux/commands/catalog_fill.py +523 -0
  259. mostlyright/data_harness/ux/commands/catalog_harvest.py +545 -0
  260. mostlyright/data_harness/ux/commands/catalog_publish.py +1838 -0
  261. mostlyright/data_harness/ux/commands/catalog_search.py +71 -0
  262. mostlyright/data_harness/ux/commands/catalog_update.py +437 -0
  263. mostlyright/data_harness/ux/commands/deploy.py +134 -0
  264. mostlyright/data_harness/ux/commands/deploy_dataset.py +98 -0
  265. mostlyright/data_harness/ux/commands/deploy_plan.py +105 -0
  266. mostlyright/data_harness/ux/commands/deploy_status.py +104 -0
  267. mostlyright/data_harness/ux/commands/diff.py +74 -0
  268. mostlyright/data_harness/ux/commands/index.py +84 -0
  269. mostlyright/data_harness/ux/commands/inventory.py +47 -0
  270. mostlyright/data_harness/ux/commands/list_builds.py +143 -0
  271. mostlyright/data_harness/ux/commands/login.py +63 -0
  272. mostlyright/data_harness/ux/commands/peek.py +236 -0
  273. mostlyright/data_harness/ux/commands/plan_check.py +90 -0
  274. mostlyright/data_harness/ux/commands/preflight.py +97 -0
  275. mostlyright/data_harness/ux/commands/record.py +107 -0
  276. mostlyright/data_harness/ux/commands/review_setup.py +47 -0
  277. mostlyright/data_harness/ux/commands/search.py +440 -0
  278. mostlyright/data_harness/ux/commands/show.py +61 -0
  279. mostlyright/data_harness/ux/commands/whoami.py +37 -0
  280. mostlyright/data_harness/ux/credential_native.py +551 -0
  281. mostlyright/data_harness/ux/credential_store.py +1055 -0
  282. mostlyright/data_harness/ux/credentials.py +631 -0
  283. mostlyright/data_harness/ux/diffing.py +444 -0
  284. mostlyright/data_harness/ux/headline.py +671 -0
  285. mostlyright/data_harness/ux/hosted_acquisition.py +974 -0
  286. mostlyright/data_harness/ux/hosted_run_status.py +619 -0
  287. mostlyright/data_harness/ux/inventory.py +427 -0
  288. mostlyright/data_harness/ux/local_review.py +375 -0
  289. mostlyright/data_harness/ux/login.py +691 -0
  290. mostlyright/data_harness/ux/path_kind.py +147 -0
  291. mostlyright/data_harness/ux/peek.py +1000 -0
  292. mostlyright/data_harness/ux/plain_file.py +178 -0
  293. mostlyright/data_harness/ux/plan_check.py +311 -0
  294. mostlyright/data_harness/ux/preflight.py +918 -0
  295. mostlyright/data_harness/ux/readers.py +1124 -0
  296. mostlyright/data_harness/ux/remediation.py +2195 -0
  297. mostlyright/data_harness/ux/render.py +657 -0
  298. mostlyright/data_harness/ux/workload.py +1077 -0
  299. mostlyright/data_harness/viewer.py +3713 -0
  300. mostlyright/data_harness/visual_run/__init__.py +83 -0
  301. mostlyright/data_harness/visual_run/authoring.py +235 -0
  302. mostlyright/data_harness/visual_run/contracts.py +673 -0
  303. mostlyright/data_harness/visual_run/materialize.py +486 -0
  304. mostlyright/data_harness/visual_run/observations.py +874 -0
  305. mostlyright/data_harness/visual_run/query.py +259 -0
  306. mostlyright/data_harness/visual_run/reducer.py +280 -0
  307. mostlyright/data_harness/visual_run/sdk.py +892 -0
  308. mostlyright/data_harness/visual_run/store.py +584 -0
  309. mostlyright/data_harness/visual_run/transport.py +239 -0
  310. mostlyright/data_harness/watch.py +2999 -0
  311. mostlyright_data-0.9.0.dist-info/METADATA +607 -0
  312. mostlyright_data-0.9.0.dist-info/RECORD +314 -0
  313. mostlyright_data-0.9.0.dist-info/WHEEL +4 -0
  314. mostlyright_data-0.9.0.dist-info/entry_points.txt +12 -0
@@ -0,0 +1,2026 @@
1
+ """Deterministic executor for ``local-graph-table-plan.v1`` Table graphs.
2
+
3
+ The graph owns topology, budgets and observable node audits. Engines may supply typed containers,
4
+ but operation meaning lives here so row order, null behavior and reduction arithmetic cannot drift.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import hashlib
10
+ import json
11
+ import math
12
+ from collections import defaultdict
13
+ from collections.abc import Callable, Iterable, Mapping
14
+ from dataclasses import dataclass
15
+ from datetime import UTC, date, datetime, timedelta
16
+ from decimal import ROUND_HALF_EVEN, Decimal, InvalidOperation, localcontext
17
+ from types import MappingProxyType
18
+ from typing import TYPE_CHECKING, Any
19
+
20
+ from mostlyright.data_harness.local_contracts import GraphNode, GraphTablePlan
21
+ from mostlyright.data_harness.operation_registry import MAX_DATE_ADD_DAYS, thaw_parameter
22
+
23
+ if TYPE_CHECKING:
24
+ from mostlyright.data_harness.backends.protocol import Backend
25
+
26
+ # The hard ceilings are sized to the hosted acquisition contract, so a source the hosted
27
+ # clean room can lawfully hand back is a source the local engine can execute. Each value
28
+ # below names the hosted constant it mirrors; a policy may narrow any of them, and a value
29
+ # above a ceiling still refuses -- the ceilings stay hard.
30
+ #
31
+ # One source may hold every row of one hosted normalized artifact: the epa-scale Reader
32
+ # budget admits 10_000_000 rows per decode (``readers/contracts.py``), and the coordinator
33
+ # parser ceiling agrees (``acquisition/parsing.py``).
34
+ HARD_MAX_SOURCE_ROWS = 10_000_000
35
+ # One intermediate table may hold the union of several year-scale sources: four sources at
36
+ # the normalized-maximum row count, which covers a three-year union with one source of
37
+ # margin before anything is filtered or aggregated.
38
+ HARD_MAX_INTERMEDIATE_ROWS = 40_000_000
39
+ # A released dataset never carries more rows than one hosted normalized artifact may.
40
+ HARD_MAX_OUTPUT_ROWS = 10_000_000
41
+ HARD_MAX_FIELD_BYTES = 1 * 1024 * 1024
42
+ # Canonical logical bytes, not an RSS claim: room for several normalized-maximum sources
43
+ # (4 GiB each, ``readers/contracts.py`` max_output_bytes) held beside the canonical tables
44
+ # derived from them. 16 GiB = raw year-scale sources + one decode transient + the kernel's
45
+ # own tables, with the same one-source margin the intermediate row ceiling carries.
46
+ HARD_MAX_RETAINED_BYTES = 17_179_869_184
47
+ LOGICAL_TYPES = frozenset(
48
+ {"null", "boolean", "int64", "float64", "string", "date", "timestamp_utc", "decimal"}
49
+ )
50
+ DECIMAL_INTEGRAL_DIGITS = 38
51
+ DECIMAL_FRACTIONAL_DIGITS = 38
52
+ DECIMAL_WORKING_PRECISION = 160
53
+ DECIMAL_QUANTUM = Decimal(1).scaleb(-DECIMAL_FRACTIONAL_DIGITS)
54
+ # The whole trim alphabet of the local lexical contract, which is deliberately smaller than
55
+ # Python's Unicode default: ``str.strip()`` with no argument also removes NBSP, the ideographic
56
+ # space and every other Unicode space, and which code points those are depends on the Unicode
57
+ # version the interpreter was built against. These six ASCII characters do not move. The v1
58
+ # cleaning step in ``pipeline.py`` trims by this same constant, so both plan versions strip
59
+ # exactly the same bytes.
60
+ ASCII_TRIM_CHARS = "\t\n\v\f\r "
61
+ # ASCII case folding, written out rather than delegated to ``str.casefold()``. Full Unicode
62
+ # folding is locale-independent but not version-independent: its mappings grow with each Unicode
63
+ # release, and some of them change a string's length. A sealed Build must replay byte for byte on
64
+ # a later interpreter, so the fold admitted here is the twenty-six letters and nothing else; every
65
+ # other code point, including a dotted capital I and a sharp s, is returned unchanged.
66
+ _ASCII_CASE_FOLD = str.maketrans("ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz")
67
+
68
+
69
+ @dataclass(frozen=True)
70
+ class GraphResourcePolicy:
71
+ max_source_rows: int = HARD_MAX_SOURCE_ROWS
72
+ max_intermediate_rows: int = HARD_MAX_INTERMEDIATE_ROWS
73
+ max_output_rows: int = HARD_MAX_OUTPUT_ROWS
74
+ max_source_bytes: int = HARD_MAX_RETAINED_BYTES
75
+ max_intermediate_bytes: int = HARD_MAX_RETAINED_BYTES
76
+ max_output_bytes: int = HARD_MAX_RETAINED_BYTES
77
+ max_retained_bytes: int = HARD_MAX_RETAINED_BYTES
78
+
79
+ def __post_init__(self) -> None:
80
+ limits = (
81
+ ("max_source_rows", self.max_source_rows, HARD_MAX_SOURCE_ROWS),
82
+ ("max_intermediate_rows", self.max_intermediate_rows, HARD_MAX_INTERMEDIATE_ROWS),
83
+ ("max_output_rows", self.max_output_rows, HARD_MAX_OUTPUT_ROWS),
84
+ )
85
+ for name, value, ceiling in limits:
86
+ if isinstance(value, bool) or not isinstance(value, int) or not 1 <= value <= ceiling:
87
+ raise GraphExecutionError("RESOURCE_POLICY_INVALID", name, "outside hard ceiling")
88
+ byte_limits = (
89
+ ("max_source_bytes", self.max_source_bytes),
90
+ ("max_intermediate_bytes", self.max_intermediate_bytes),
91
+ ("max_output_bytes", self.max_output_bytes),
92
+ ("max_retained_bytes", self.max_retained_bytes),
93
+ )
94
+ for name, value in byte_limits:
95
+ if (
96
+ isinstance(value, bool)
97
+ or not isinstance(value, int)
98
+ or not 1 <= value <= HARD_MAX_RETAINED_BYTES
99
+ ):
100
+ raise GraphExecutionError("RESOURCE_POLICY_INVALID", name, "outside hard ceiling")
101
+
102
+
103
+ class GraphExecutionError(RuntimeError):
104
+ def __init__(self, code: str, node_id: str, detail: str) -> None:
105
+ self.code = code
106
+ self.node_id = node_id
107
+ self.detail = detail
108
+ super().__init__(f"{node_id}: {detail} [{code}]")
109
+
110
+
111
+ @dataclass(frozen=True)
112
+ class ColumnLineage:
113
+ """Sources and deterministic graph operations that produced one column."""
114
+
115
+ sources: tuple[tuple[str, str], ...]
116
+ operations: tuple[str, ...] = ()
117
+
118
+ def __iter__(self):
119
+ """Keep the v2 pipeline's source-pair iteration compatible."""
120
+
121
+ return iter(self.sources)
122
+
123
+ def __len__(self) -> int:
124
+ return len(self.sources)
125
+
126
+ def __getitem__(self, item: int | slice):
127
+ return self.sources[item]
128
+
129
+
130
+ @dataclass(frozen=True)
131
+ class GraphTable:
132
+ columns: tuple[str, ...]
133
+ rows: tuple[tuple[Any, ...], ...]
134
+ lineage: Mapping[str, ColumnLineage | tuple[tuple[str, str], ...]] = MappingProxyType({})
135
+ schema: Mapping[str, str] | None = None
136
+
137
+ def __post_init__(self) -> None:
138
+ schema = self.schema or MappingProxyType(
139
+ {
140
+ column: _infer_column_type((row[index] for row in self.rows), "table", column)
141
+ for index, column in enumerate(self.columns)
142
+ }
143
+ )
144
+ if tuple(schema) != self.columns or any(
145
+ logical_type not in LOGICAL_TYPES for logical_type in schema.values()
146
+ ):
147
+ raise GraphExecutionError(
148
+ "LOGICAL_SCHEMA_INVALID", "table", "schema must exactly type every ordered column"
149
+ )
150
+ if any(len(row) != len(self.columns) for row in self.rows):
151
+ raise GraphExecutionError(
152
+ "SOURCE_ROW_INVALID", "table", "row width differs from schema"
153
+ )
154
+ lineage = MappingProxyType(
155
+ {
156
+ column: item if isinstance(item, ColumnLineage) else ColumnLineage(tuple(item))
157
+ for column, item in self.lineage.items()
158
+ }
159
+ )
160
+ object.__setattr__(self, "schema", MappingProxyType(dict(schema)))
161
+ object.__setattr__(self, "lineage", lineage)
162
+
163
+ def mappings(self) -> tuple[Mapping[str, Any], ...]:
164
+ return tuple(
165
+ MappingProxyType(dict(zip(self.columns, row, strict=True))) for row in self.rows
166
+ )
167
+
168
+
169
+ @dataclass(frozen=True)
170
+ class NodeAudit:
171
+ node_id: str
172
+ operation: str
173
+ operation_version: str
174
+ inputs: tuple[str, ...]
175
+ parameter_digest: str
176
+ input_digests: tuple[str, ...]
177
+ output_digest: str
178
+ input_rows: int
179
+ output_rows: int
180
+ rejected_rows: int
181
+ duplicate_rows: int
182
+ order_contract: str
183
+ engine: str
184
+ input_schemas: tuple[tuple[tuple[str, str], ...], ...] = ()
185
+ output_schema: tuple[tuple[str, str], ...] = ()
186
+ input_bytes: int = 0
187
+ output_bytes: int = 0
188
+ retained_bytes: int = 0
189
+
190
+ def to_dict(self) -> dict[str, Any]:
191
+ return {
192
+ "node_id": self.node_id,
193
+ "operation": self.operation,
194
+ "operation_version": self.operation_version,
195
+ "inputs": list(self.inputs),
196
+ "parameter_digest": self.parameter_digest,
197
+ "input_digests": list(self.input_digests),
198
+ "output_digest": self.output_digest,
199
+ "input_rows": self.input_rows,
200
+ "output_rows": self.output_rows,
201
+ "rejected_rows": self.rejected_rows,
202
+ "duplicate_rows": self.duplicate_rows,
203
+ "order_contract": self.order_contract,
204
+ "engine": self.engine,
205
+ "input_schemas": [dict(schema) for schema in self.input_schemas],
206
+ "output_schema": dict(self.output_schema),
207
+ "input_bytes": self.input_bytes,
208
+ "output_bytes": self.output_bytes,
209
+ "retained_bytes": self.retained_bytes,
210
+ }
211
+
212
+
213
+ @dataclass(frozen=True)
214
+ class GraphExecution:
215
+ terminal: GraphTable
216
+ audits: tuple[NodeAudit, ...]
217
+ tables: Mapping[str, GraphTable]
218
+ peak_retained_bytes: int = 0
219
+
220
+
221
+ def _normalize(value: Any) -> Any:
222
+ if value is None or type(value) in {bool, int, str}:
223
+ return value
224
+ if isinstance(value, (date, datetime)):
225
+ return value.isoformat().replace("+00:00", "Z")
226
+ if isinstance(value, Decimal):
227
+ return format(value, "f")
228
+ if isinstance(value, float):
229
+ if not math.isfinite(value):
230
+ raise GraphExecutionError("NON_FINITE_VALUE", "digest", "non-finite floats refuse")
231
+ return {"$float": value.hex()}
232
+ if isinstance(value, tuple):
233
+ return [_normalize(item) for item in value]
234
+ if isinstance(value, list):
235
+ return [_normalize(item) for item in value]
236
+ if isinstance(value, Mapping):
237
+ return {str(key): _normalize(item) for key, item in sorted(value.items())}
238
+ raise GraphExecutionError("VALUE_UNSUPPORTED", "digest", f"unsupported value {type(value)!r}")
239
+
240
+
241
+ def _digest(value: Any) -> str:
242
+ raw = json.dumps(
243
+ _normalize(value),
244
+ ensure_ascii=False,
245
+ allow_nan=False,
246
+ sort_keys=True,
247
+ separators=(",", ":"),
248
+ ).encode()
249
+ return hashlib.sha256(raw).hexdigest()
250
+
251
+
252
+ def _value_bytes(value: Any, node_id: str) -> int:
253
+ normalized = _normalize(value)
254
+ size = len(
255
+ json.dumps(normalized, ensure_ascii=False, allow_nan=False, separators=(",", ":")).encode()
256
+ )
257
+ if size > HARD_MAX_FIELD_BYTES:
258
+ raise GraphExecutionError("FIELD_BUDGET_EXCEEDED", node_id, "field exceeds byte ceiling")
259
+ return size
260
+
261
+
262
+ def _table_bytes(table: GraphTable, node_id: str) -> int:
263
+ return sum(_value_bytes(value, node_id) for row in table.rows for value in row)
264
+
265
+
266
+ @dataclass
267
+ class _OutputBudget:
268
+ """Reserve canonical output bytes before retaining each kernel result row."""
269
+
270
+ node_id: str
271
+ max_rows: int
272
+ max_bytes: int
273
+ rows: int = 0
274
+ bytes: int = 0
275
+
276
+ def retain(self, output: list[tuple[Any, ...]], row: tuple[Any, ...]) -> None:
277
+ if self.rows >= self.max_rows:
278
+ raise GraphExecutionError(
279
+ "ROW_BUDGET_EXCEEDED", self.node_id, f"output exceeds {self.max_rows}"
280
+ )
281
+ row_bytes = sum(_value_bytes(value, self.node_id) for value in row)
282
+ if row_bytes > self.max_bytes - self.bytes:
283
+ raise GraphExecutionError(
284
+ "BYTE_BUDGET_EXCEEDED", self.node_id, f"output exceeds {self.max_bytes}"
285
+ )
286
+ output.append(row)
287
+ self.rows += 1
288
+ self.bytes += row_bytes
289
+
290
+
291
+ def table_digest(table: GraphTable) -> str:
292
+ return _digest({"columns": table.columns, "schema": dict(table.schema), "rows": table.rows})
293
+
294
+
295
+ def _logical_type(value: Any, node_id: str) -> str:
296
+ if value is None:
297
+ return "null"
298
+ if type(value) is bool:
299
+ return "boolean"
300
+ if type(value) is int:
301
+ if not -(2**63) <= value < 2**63:
302
+ raise GraphExecutionError("SOURCE_VALUE_INVALID", node_id, "integer exceeds int64")
303
+ return "int64"
304
+ if type(value) is float:
305
+ if not math.isfinite(value):
306
+ raise GraphExecutionError("NON_FINITE_VALUE", node_id, "source float is non-finite")
307
+ return "float64"
308
+ if isinstance(value, Decimal):
309
+ if not value.is_finite():
310
+ raise GraphExecutionError("NON_FINITE_VALUE", node_id, "source decimal is non-finite")
311
+ return "decimal"
312
+ if isinstance(value, datetime):
313
+ if value.tzinfo is None or value.utcoffset() is None:
314
+ raise GraphExecutionError(
315
+ "SOURCE_VALUE_INVALID", node_id, "timestamp must carry an offset"
316
+ )
317
+ return "timestamp_utc"
318
+ if isinstance(value, date):
319
+ return "date"
320
+ if isinstance(value, str):
321
+ return "string"
322
+ raise GraphExecutionError(
323
+ "SOURCE_VALUE_INVALID", node_id, f"unsupported source scalar {type(value)!r}"
324
+ )
325
+
326
+
327
+ def _infer_column_type(values: Iterable[Any], node_id: str, column: str) -> str:
328
+ observed = {_logical_type(value, node_id) for value in values} - {"null"}
329
+ if not observed:
330
+ return "null"
331
+ if len(observed) == 1:
332
+ return next(iter(observed))
333
+ raise GraphExecutionError(
334
+ "SOURCE_SCHEMA_MIXED",
335
+ node_id,
336
+ f"column {column!r} contains incompatible logical types {sorted(observed)}",
337
+ )
338
+
339
+
340
+ def _schema_items(table: GraphTable) -> tuple[tuple[str, str], ...]:
341
+ return tuple(table.schema.items())
342
+
343
+
344
+ def _operation_coordinate(node: GraphNode) -> str:
345
+ return f"{node.node_id}:{node.operation}@{node.operation_version}"
346
+
347
+
348
+ def _advance_lineage(
349
+ node: GraphNode,
350
+ lineage: Mapping[str, ColumnLineage],
351
+ ) -> Mapping[str, ColumnLineage]:
352
+ coordinate = _operation_coordinate(node)
353
+ return MappingProxyType(
354
+ {
355
+ column: ColumnLineage(item.sources, (*item.operations, coordinate))
356
+ for column, item in lineage.items()
357
+ }
358
+ )
359
+
360
+
361
+ def _parameters(node: GraphNode) -> dict[str, Any]:
362
+ value = thaw_parameter(node.parameters)
363
+ assert isinstance(value, dict)
364
+ return value
365
+
366
+
367
+ def _table_from_records(
368
+ node_id: str,
369
+ source_id: str,
370
+ records: Iterable[Mapping[str, Any]],
371
+ limit: int,
372
+ byte_limit: int,
373
+ declared_columns: tuple[str, ...] | None = None,
374
+ ) -> GraphTable:
375
+ columns: tuple[str, ...] | None = declared_columns
376
+ rows: list[tuple[Any, ...]] = []
377
+ retained_bytes = 0
378
+ for index, record in enumerate(records):
379
+ if index >= limit:
380
+ raise GraphExecutionError(
381
+ "ROW_BUDGET_EXCEEDED", node_id, f"source exceeds max_source_rows={limit}"
382
+ )
383
+ if not isinstance(record, Mapping) or any(not isinstance(key, str) for key in record):
384
+ raise GraphExecutionError("SOURCE_ROW_INVALID", node_id, "source row must be an object")
385
+ keys = tuple(record)
386
+ if columns is None:
387
+ columns = keys
388
+ elif keys != columns:
389
+ raise GraphExecutionError(
390
+ "SOURCE_SCHEMA_DRIFT", node_id, "source rows do not share exact column order"
391
+ )
392
+ rows.append(tuple(record[column] for column in columns))
393
+ for value in rows[-1]:
394
+ retained_bytes += _value_bytes(value, node_id)
395
+ if retained_bytes > byte_limit:
396
+ raise GraphExecutionError(
397
+ "BYTE_BUDGET_EXCEEDED", node_id, "source table exceeds byte ceiling"
398
+ )
399
+ final_columns = columns or ()
400
+ frozen_rows = tuple(rows)
401
+ schema = MappingProxyType(
402
+ {
403
+ column: (
404
+ _infer_column_type((row[column_index] for row in frozen_rows), node_id, column)
405
+ if frozen_rows
406
+ else "string"
407
+ )
408
+ for column_index, column in enumerate(final_columns)
409
+ }
410
+ )
411
+ return GraphTable(
412
+ final_columns,
413
+ frozen_rows,
414
+ MappingProxyType(
415
+ {column: ColumnLineage(((source_id, column),)) for column in final_columns}
416
+ ),
417
+ schema,
418
+ )
419
+
420
+
421
+ def _lineage_union(*values: Iterable[tuple[str, str]]) -> tuple[tuple[str, str], ...]:
422
+ return tuple(dict.fromkeys(item for value in values for item in value))
423
+
424
+
425
+ def _merge_lineage(node: GraphNode, *items: ColumnLineage) -> ColumnLineage:
426
+ return ColumnLineage(
427
+ _lineage_union(*(item.sources for item in items)),
428
+ tuple(
429
+ dict.fromkeys(
430
+ operation
431
+ for item in items
432
+ for operation in (*item.operations, _operation_coordinate(node))
433
+ )
434
+ ),
435
+ )
436
+
437
+
438
+ def _expression_columns(value: Any) -> tuple[str, ...]:
439
+ if isinstance(value, Mapping):
440
+ found = (str(value["column"]),) if set(value) == {"column"} else ()
441
+ return _lineage_names(found, value.values())
442
+ if isinstance(value, list):
443
+ return _lineage_names((), value)
444
+ return ()
445
+
446
+
447
+ def _lineage_names(initial: tuple[str, ...], values: Iterable[Any]) -> tuple[str, ...]:
448
+ return tuple(
449
+ dict.fromkeys(
450
+ (*initial, *(name for value in values for name in _expression_columns(value)))
451
+ )
452
+ )
453
+
454
+
455
+ def _index(table: GraphTable, node_id: str) -> dict[str, int]:
456
+ if len(table.columns) != len(set(table.columns)):
457
+ raise GraphExecutionError("SCHEMA_DUPLICATE", node_id, "column names must be unique")
458
+ return {column: index for index, column in enumerate(table.columns)}
459
+
460
+
461
+ def _columns(value: Any, node_id: str) -> tuple[str, ...]:
462
+ if not isinstance(value, list) or not value or any(not isinstance(item, str) for item in value):
463
+ raise GraphExecutionError("OPERATION_PARAMETERS", node_id, "columns must be text list")
464
+ if len(value) != len(set(value)):
465
+ raise GraphExecutionError("OPERATION_PARAMETERS", node_id, "columns must be unique")
466
+ return tuple(value)
467
+
468
+
469
+ def _require_columns(table: GraphTable, columns: Iterable[str], node_id: str) -> None:
470
+ missing = sorted(set(columns) - set(table.columns))
471
+ if missing:
472
+ raise GraphExecutionError("COLUMN_UNKNOWN", node_id, f"columns absent: {missing}")
473
+
474
+
475
+ def _canonical_decimal(value: Decimal, node_id: str) -> Decimal:
476
+ if not value.is_finite():
477
+ raise GraphExecutionError("NON_FINITE_VALUE", node_id, "decimal is non-finite")
478
+ exponent = value.as_tuple().exponent
479
+ fractional_digits = max(0, -exponent)
480
+ integral_digits = max(0, value.adjusted() + 1) if value else 0
481
+ if integral_digits > DECIMAL_INTEGRAL_DIGITS or fractional_digits > DECIMAL_FRACTIONAL_DIGITS:
482
+ raise GraphExecutionError(
483
+ "DECIMAL_RANGE",
484
+ node_id,
485
+ "decimal exceeds canonical decimal256(76,38) integral/scale bounds",
486
+ )
487
+ return value
488
+
489
+
490
+ def _cast(value: Any, kind: str, node_id: str) -> Any:
491
+ if value is None:
492
+ return None
493
+ try:
494
+ if kind == "string":
495
+ return str(value)
496
+ if kind == "int64":
497
+ if isinstance(value, bool):
498
+ raise ValueError
499
+ if isinstance(value, (float, Decimal)) and value != int(value):
500
+ raise ValueError
501
+ result = int(value)
502
+ if not -(2**63) <= result < 2**63:
503
+ raise OverflowError
504
+ return result
505
+ if kind == "float64":
506
+ result = float(value)
507
+ if not math.isfinite(result):
508
+ raise ValueError
509
+ return result
510
+ if kind == "boolean":
511
+ if type(value) is bool:
512
+ return value
513
+ if isinstance(value, str) and value.lower() in {"true", "false"}:
514
+ return value.lower() == "true"
515
+ raise ValueError
516
+ if kind == "date":
517
+ if isinstance(value, datetime):
518
+ raise ValueError
519
+ return value if isinstance(value, date) else date.fromisoformat(str(value))
520
+ if kind == "timestamp_utc":
521
+ parsed = (
522
+ value
523
+ if isinstance(value, datetime)
524
+ else datetime.fromisoformat(str(value).replace("Z", "+00:00"))
525
+ )
526
+ if parsed.tzinfo is None or parsed.utcoffset() is None:
527
+ raise ValueError
528
+ return parsed.astimezone(UTC)
529
+ if kind == "decimal":
530
+ return _canonical_decimal(Decimal(str(value)), node_id)
531
+ except (InvalidOperation, OverflowError, TypeError, ValueError) as exc:
532
+ raise GraphExecutionError("CAST_INVALID", node_id, f"cannot cast value to {kind}") from exc
533
+ raise GraphExecutionError("CAST_UNSUPPORTED", node_id, f"unknown cast {kind!r}")
534
+
535
+
536
+ def _operand(expr: Any, row: Mapping[str, Any], node_id: str) -> Any:
537
+ if isinstance(expr, Mapping) and set(expr) == {"column"} and isinstance(expr["column"], str):
538
+ if expr["column"] not in row:
539
+ raise GraphExecutionError("COLUMN_UNKNOWN", node_id, str(expr["column"]))
540
+ return row[expr["column"]]
541
+ if isinstance(expr, Mapping) and set(expr) == {"literal"}:
542
+ return expr["literal"]
543
+ return expr
544
+
545
+
546
+ def _predicate(expr: Any, row: Mapping[str, Any], node_id: str) -> bool | None:
547
+ if not isinstance(expr, Mapping) or not isinstance(expr.get("op"), str):
548
+ raise GraphExecutionError("PREDICATE_INVALID", node_id, "predicate must name an op")
549
+ op = expr["op"]
550
+ if op in {"and", "or"}:
551
+ args = expr.get("args")
552
+ if not isinstance(args, list) or len(args) < 2:
553
+ raise GraphExecutionError("PREDICATE_INVALID", node_id, f"{op} needs arguments")
554
+ values = [_predicate(item, row, node_id) for item in args]
555
+ if op == "and":
556
+ return False if False in values else None if None in values else True
557
+ return True if True in values else None if None in values else False
558
+ if op == "not":
559
+ result = _predicate(expr.get("arg"), row, node_id)
560
+ return None if result is None else not result
561
+ left = _operand(expr.get("left", {"column": expr.get("column")}), row, node_id)
562
+ if op in {"is_null", "not_null"}:
563
+ return (left is None) if op == "is_null" else (left is not None)
564
+ right = _operand(expr.get("right", expr.get("value")), row, node_id)
565
+ if left is None or right is None:
566
+ return None
567
+ if op == "in":
568
+ try:
569
+ return any(type(left) is type(candidate) and left == candidate for candidate in right)
570
+ except TypeError as exc:
571
+ raise GraphExecutionError(
572
+ "PREDICATE_TYPE", node_id, "membership operand is incompatible"
573
+ ) from exc
574
+ if type(left) is not type(right):
575
+ raise GraphExecutionError(
576
+ "PREDICATE_TYPE", node_id, "predicate operands have different logical types"
577
+ )
578
+ operations = {
579
+ "eq": lambda: left == right,
580
+ "ne": lambda: left != right,
581
+ "lt": lambda: left < right,
582
+ "le": lambda: left <= right,
583
+ "gt": lambda: left > right,
584
+ "ge": lambda: left >= right,
585
+ }
586
+ try:
587
+ return bool(operations[op]())
588
+ except KeyError as exc:
589
+ raise GraphExecutionError("PREDICATE_UNSUPPORTED", node_id, op) from exc
590
+ except (TypeError, ValueError) as exc:
591
+ raise GraphExecutionError(
592
+ "PREDICATE_TYPE", node_id, "predicate operands are incompatible"
593
+ ) from exc
594
+
595
+
596
+ def _expression_type(expr: Any, schema: Mapping[str, str], node_id: str) -> str:
597
+ if not isinstance(expr, Mapping):
598
+ return _logical_type(expr, node_id)
599
+ if set(expr) == {"column"}:
600
+ column = expr["column"]
601
+ if not isinstance(column, str) or column not in schema:
602
+ raise GraphExecutionError("COLUMN_UNKNOWN", node_id, str(column))
603
+ return schema[column]
604
+ if set(expr) == {"literal"}:
605
+ return _logical_type(expr["literal"], node_id)
606
+ op = expr.get("op")
607
+ if op == "if":
608
+ branch_types = {
609
+ _expression_type(expr[branch], schema, node_id) for branch in ("then", "else")
610
+ } - {"null"}
611
+ if len(branch_types) > 1:
612
+ raise GraphExecutionError(
613
+ "DERIVE_TYPE", node_id, "conditional branches have different logical types"
614
+ )
615
+ return next(iter(branch_types), "null")
616
+ argument_types = [_expression_type(item, schema, node_id) for item in expr.get("args", [])]
617
+ present_types = [item for item in argument_types if item != "null"]
618
+ if op in {"add", "subtract", "multiply"}:
619
+ if len(set(present_types)) > 1 or any(
620
+ item not in {"int64", "float64", "decimal"} for item in present_types
621
+ ):
622
+ raise GraphExecutionError(
623
+ "DERIVE_TYPE", node_id, "arithmetic requires one exact numeric logical type"
624
+ )
625
+ return present_types[0] if present_types else "null"
626
+ if op == "divide":
627
+ # Division takes the same one-exact-numeric-type operands as the other arithmetic and
628
+ # then leaves that type behind, because a quotient is not the domain its operands came
629
+ # from. ``int64 / int64`` is the case this makes explicit: it does not truncate towards
630
+ # zero, does not floor, and does not become a float; it produces the decimal quotient,
631
+ # exactly as ``aggregate``'s ``mean`` does over an integer column. A ratio therefore
632
+ # always declares ``output_type: decimal``.
633
+ if len(set(present_types)) > 1 or any(
634
+ item not in {"int64", "float64", "decimal"} for item in present_types
635
+ ):
636
+ raise GraphExecutionError(
637
+ "DERIVE_TYPE", node_id, "division requires one exact numeric logical type"
638
+ )
639
+ return "decimal" if present_types else "null"
640
+ if op == "concat":
641
+ if any(item != "string" for item in present_types):
642
+ raise GraphExecutionError("DERIVE_TYPE", node_id, "concat requires string operands")
643
+ return "string" if present_types else "null"
644
+ if op in {"trim", "case_fold"}:
645
+ if any(item != "string" for item in present_types):
646
+ raise GraphExecutionError("DERIVE_TYPE", node_id, f"{op} requires a string operand")
647
+ return "string" if present_types else "null"
648
+ if op in {"year", "month", "day", "hour"}:
649
+ permitted = {"timestamp_utc"} if op == "hour" else {"date", "timestamp_utc"}
650
+ if any(item not in permitted for item in present_types):
651
+ raise GraphExecutionError("DERIVE_TYPE", node_id, f"{op} requires a temporal operand")
652
+ return "int64" if present_types else "null"
653
+ raise GraphExecutionError("DERIVE_UNSUPPORTED", node_id, str(op))
654
+
655
+
656
+ def _divide(args: list[Any], node_id: str) -> Any:
657
+ """One quotient, under the two denominator policies a dividing node declares."""
658
+
659
+ present_types = {_logical_type(item, node_id) for item in args} - {"null"}
660
+ if len(present_types) > 1 or any(
661
+ item not in {"int64", "float64", "decimal"} for item in present_types
662
+ ):
663
+ raise GraphExecutionError(
664
+ "DERIVE_TYPE", node_id, "division requires one exact numeric logical type"
665
+ )
666
+ # The declared zero-denominator policy is ``reject``, and it answers before the null policy so
667
+ # that it answers for every row with a zero denominator. There is no quotient to record, and
668
+ # an absent value here would be one this operation manufactured rather than one it found.
669
+ if args[1] is not None and _decimal(args[1], node_id, code="DERIVE_TYPE") == 0:
670
+ raise GraphExecutionError(
671
+ "DERIVE_DIVIDE_BY_ZERO", node_id, "denominator is zero under the declared policy"
672
+ )
673
+ # A denominator that is absent is a different fact from one that is zero, and the declared
674
+ # null policy carries it through as absent. An absent numerator does the same.
675
+ if any(item is None for item in args):
676
+ return None
677
+ numerator = _decimal(args[0], node_id, code="DERIVE_TYPE")
678
+ denominator = _decimal(args[1], node_id, code="DERIVE_TYPE")
679
+ # Both operands came through ``_canonical_decimal``, so each is under 1e38 and no smaller than
680
+ # 1e-38; the quotient is therefore under 1e76 and the quantize below stays inside the working
681
+ # precision rather than raising an untyped decimal error.
682
+ with localcontext() as context:
683
+ context.prec = DECIMAL_WORKING_PRECISION
684
+ quotient = numerator / denominator
685
+ # A repeating quotient cannot be held exactly in decimal256(76,38), so the contract rounds
686
+ # once, at the physical scale, with banker's rounding -- the same single rounding step
687
+ # ``aggregate``'s ``mean`` takes, through the same shared kernel.
688
+ quotient = quotient.quantize(DECIMAL_QUANTUM, rounding=ROUND_HALF_EVEN)
689
+ # A negative denominator under a zero numerator, or a quotient that underflows the scale, lands
690
+ # on a signed zero. Decimal keeps the sign, ``_normalize`` renders it into the digest, and the
691
+ # decimal256 column the value is written to does not have it -- so the audit and the stored
692
+ # bytes would disagree about a value that is equal to positive zero either way. Settle it here.
693
+ result = _canonical_decimal(quotient if quotient else abs(quotient), node_id)
694
+ _value_bytes(result, node_id)
695
+ return result
696
+
697
+
698
+ def _derive(expr: Any, row: Mapping[str, Any], node_id: str) -> Any:
699
+ if not isinstance(expr, Mapping):
700
+ return expr
701
+ if "column" in expr and set(expr) == {"column"}:
702
+ return _operand(expr, row, node_id)
703
+ if "literal" in expr and set(expr) == {"literal"}:
704
+ return expr["literal"]
705
+ op = expr.get("op")
706
+ if op == "if":
707
+ decision = _predicate(expr.get("condition"), row, node_id)
708
+ if decision is None:
709
+ return None
710
+ return _derive(expr.get("then") if decision is True else expr.get("else"), row, node_id)
711
+ args = [_derive(item, row, node_id) for item in expr.get("args", [])]
712
+ # Division is settled before the blanket null propagation below, because its two policies are
713
+ # about the denominator alone. If the numerator decided whether a zero denominator refused,
714
+ # ``zero_denominator_policy`` would hold on some rows and not others for reasons having nothing
715
+ # to do with the denominator, which is the incidental behaviour a declared policy exists to
716
+ # rule out. Inside ``_divide`` the null policy still propagates; it just does not get to
717
+ # answer first.
718
+ if op == "divide" and len(args) == 2:
719
+ return _divide(args, node_id)
720
+ if any(item is None for item in args):
721
+ return None
722
+ if op in {"add", "subtract", "multiply"} and len(args) == 2:
723
+ operand_types = tuple(_logical_type(item, node_id) for item in args)
724
+ if len(set(operand_types)) != 1 or operand_types[0] not in {
725
+ "int64",
726
+ "float64",
727
+ "decimal",
728
+ }:
729
+ raise GraphExecutionError(
730
+ "DERIVE_TYPE", node_id, "arithmetic requires one exact numeric logical type"
731
+ )
732
+ try:
733
+ if operand_types[0] == "decimal":
734
+ with localcontext() as context:
735
+ context.prec = DECIMAL_WORKING_PRECISION
736
+ result = {
737
+ "add": args[0] + args[1],
738
+ "subtract": args[0] - args[1],
739
+ "multiply": args[0] * args[1],
740
+ }[op]
741
+ result = _canonical_decimal(result, node_id)
742
+ else:
743
+ result = {
744
+ "add": args[0] + args[1],
745
+ "subtract": args[0] - args[1],
746
+ "multiply": args[0] * args[1],
747
+ }[op]
748
+ except (TypeError, ValueError) as exc:
749
+ raise GraphExecutionError(
750
+ "DERIVE_TYPE", node_id, "derive operands are incompatible"
751
+ ) from exc
752
+ if isinstance(result, int) and not -(2**63) <= result < 2**63:
753
+ raise GraphExecutionError("DERIVE_OVERFLOW", node_id, "int64 arithmetic overflow")
754
+ _value_bytes(result, node_id)
755
+ return result
756
+ if op == "concat":
757
+ if any(not isinstance(item, str) for item in args):
758
+ raise GraphExecutionError("DERIVE_TYPE", node_id, "concat requires strings")
759
+ result = "".join(args)
760
+ _value_bytes(result, node_id)
761
+ return result
762
+ if op in {"trim", "case_fold"} and len(args) == 1:
763
+ if not isinstance(args[0], str):
764
+ raise GraphExecutionError("DERIVE_TYPE", node_id, f"{op} requires a string")
765
+ result = (
766
+ args[0].strip(ASCII_TRIM_CHARS) if op == "trim" else args[0].translate(_ASCII_CASE_FOLD)
767
+ )
768
+ _value_bytes(result, node_id)
769
+ return result
770
+ if op in {"year", "month", "day", "hour"} and len(args) == 1:
771
+ try:
772
+ if isinstance(args[0], datetime):
773
+ parsed = args[0]
774
+ elif isinstance(args[0], date):
775
+ if op == "hour":
776
+ raise ValueError
777
+ return getattr(args[0], op)
778
+ else:
779
+ parsed = datetime.fromisoformat(str(args[0]).replace("Z", "+00:00"))
780
+ if parsed.tzinfo is None or parsed.utcoffset() is None:
781
+ raise ValueError
782
+ except (TypeError, ValueError) as exc:
783
+ raise GraphExecutionError(
784
+ "DERIVE_TYPE", node_id, "timestamp operand is invalid"
785
+ ) from exc
786
+ return getattr(parsed.astimezone(UTC), op)
787
+ raise GraphExecutionError("DERIVE_UNSUPPORTED", node_id, str(op))
788
+
789
+
790
+ def _order_scalar(value: Any, node_id: str) -> tuple[int, Any]:
791
+ if type(value) is bool:
792
+ return (0, value)
793
+ if type(value) is int:
794
+ return (1, value)
795
+ if type(value) is float:
796
+ if not math.isfinite(value):
797
+ raise GraphExecutionError("NON_FINITE_VALUE", node_id, "non-finite floats refuse")
798
+ return (2, value)
799
+ if isinstance(value, Decimal):
800
+ return (3, value)
801
+ if isinstance(value, (date, datetime)):
802
+ return (4, value.isoformat())
803
+ if isinstance(value, str):
804
+ return (5, value)
805
+ raise GraphExecutionError("SORT_TYPE", node_id, f"unsupported sort value {type(value)!r}")
806
+
807
+
808
+ def _sort_token(value: Any, *, nulls: str, direction: str, node_id: str) -> tuple[int, Any]:
809
+ null_rank = 0 if nulls == "first" else 2
810
+ if value is None:
811
+ return (null_rank, 0)
812
+ normalized = _order_scalar(value, node_id)
813
+ if direction == "desc":
814
+ return (1, _Descending(normalized))
815
+ return (1, normalized)
816
+
817
+
818
+ @dataclass(frozen=True)
819
+ class _Descending:
820
+ value: Any
821
+
822
+ def __lt__(self, other: _Descending) -> bool:
823
+ return self.value > other.value
824
+
825
+
826
+ def _key_specs(value: Any, node_id: str) -> list[dict[str, str]]:
827
+ if not isinstance(value, list) or not value:
828
+ raise GraphExecutionError("OPERATION_PARAMETERS", node_id, "sort keys must be non-empty")
829
+ result: list[dict[str, str]] = []
830
+ for item in value:
831
+ if isinstance(item, str):
832
+ result.append({"column": item, "direction": "asc", "nulls": "last"})
833
+ elif isinstance(item, Mapping) and isinstance(item.get("column"), str):
834
+ direction = item.get("direction", "asc")
835
+ nulls = item.get("nulls", "last")
836
+ if direction not in {"asc", "desc"} or nulls not in {"first", "last"}:
837
+ raise GraphExecutionError("OPERATION_PARAMETERS", node_id, "invalid sort policy")
838
+ result.append({"column": item["column"], "direction": direction, "nulls": nulls})
839
+ else:
840
+ raise GraphExecutionError("OPERATION_PARAMETERS", node_id, "invalid sort key")
841
+ return result
842
+
843
+
844
+ def _execute_node(
845
+ node: GraphNode,
846
+ inputs: tuple[GraphTable, ...],
847
+ source: GraphTable | None,
848
+ max_rows: int,
849
+ max_bytes: int,
850
+ ) -> tuple[GraphTable, int, int, str]:
851
+ params = _parameters(node)
852
+ if node.operation == "source":
853
+ assert source is not None
854
+ # A declared unit names a column the file has to hold. The plan cannot check that -- it
855
+ # does not know the header -- so it is checked here, where the header is known. One
856
+ # transposed letter would otherwise leave the declaration attached to nothing, and the two
857
+ # unit gates silently with nothing to compare in exactly the case they were written for.
858
+ declared = thaw_parameter(params.get("column_units"))
859
+ # An empty source has no columns to check against, and reporting every declaration as
860
+ # absent would name the wrong failure: the run already stops for the missing columns.
861
+ if isinstance(declared, Mapping) and source.columns:
862
+ absent = sorted(set(declared) - set(source.columns))
863
+ if absent:
864
+ raise GraphExecutionError(
865
+ "UNIT_COLUMN_UNKNOWN",
866
+ node.node_id,
867
+ f"column_units declares columns this source does not hold: {absent}",
868
+ )
869
+ # A unit is a claim about a quantity, so the column has to hold one. The plan cannot
870
+ # tell -- it does not know the types either -- so this is the other half of the same
871
+ # check, and without it the merge gate ends up arbitrating units on strings.
872
+ unquantified = sorted(
873
+ column
874
+ for column in declared
875
+ if source.schema.get(column) not in {"int64", "float64", "decimal"}
876
+ )
877
+ if unquantified:
878
+ raise GraphExecutionError(
879
+ "UNIT_COLUMN_NOT_NUMERIC",
880
+ node.node_id,
881
+ f"column_units declares columns that hold no quantity: {unquantified}",
882
+ )
883
+ return (
884
+ GraphTable(
885
+ source.columns,
886
+ source.rows,
887
+ _advance_lineage(node, source.lineage),
888
+ source.schema,
889
+ ),
890
+ 0,
891
+ 0,
892
+ "source order",
893
+ )
894
+ table = inputs[0]
895
+ index = _index(table, node.node_id)
896
+ if node.operation == "project":
897
+ columns = _columns(params["columns"], node.node_id)
898
+ _require_columns(table, columns, node.node_id)
899
+ return (
900
+ GraphTable(
901
+ columns,
902
+ tuple(tuple(row[index[col]] for col in columns) for row in table.rows),
903
+ _advance_lineage(
904
+ node,
905
+ MappingProxyType({column: table.lineage[column] for column in columns}),
906
+ ),
907
+ MappingProxyType({column: table.schema[column] for column in columns}),
908
+ ),
909
+ 0,
910
+ 0,
911
+ "input order",
912
+ )
913
+ if node.operation == "rename":
914
+ mappings = params["mappings"]
915
+ if not isinstance(mappings, Mapping):
916
+ raise GraphExecutionError(
917
+ "OPERATION_PARAMETERS", node.node_id, "mappings must be object"
918
+ )
919
+ _require_columns(table, mappings, node.node_id)
920
+ columns = tuple(str(mappings.get(column, column)) for column in table.columns)
921
+ if len(columns) != len(set(columns)):
922
+ raise GraphExecutionError("SCHEMA_DUPLICATE", node.node_id, "rename collides")
923
+ schema = MappingProxyType(
924
+ {str(mappings.get(column, column)): table.schema[column] for column in table.columns}
925
+ )
926
+ lineage = MappingProxyType(
927
+ {
928
+ str(mappings.get(column, column)): (
929
+ _merge_lineage(node, table.lineage[column])
930
+ if column in mappings
931
+ else table.lineage[column]
932
+ )
933
+ for column in table.columns
934
+ }
935
+ )
936
+ return (
937
+ GraphTable(columns, table.rows, lineage, schema),
938
+ 0,
939
+ 0,
940
+ "input order",
941
+ )
942
+ if node.operation == "cast":
943
+ casts = params["columns"]
944
+ if not isinstance(casts, Mapping):
945
+ raise GraphExecutionError("OPERATION_PARAMETERS", node.node_id, "casts must be object")
946
+ _require_columns(table, casts, node.node_id)
947
+ rows = tuple(
948
+ tuple(
949
+ _cast(value, str(casts.get(column)), node.node_id) if column in casts else value
950
+ for column, value in zip(table.columns, row, strict=True)
951
+ )
952
+ for row in table.rows
953
+ )
954
+ schema = MappingProxyType(
955
+ {
956
+ column: str(casts[column]) if column in casts else table.schema[column]
957
+ for column in table.columns
958
+ }
959
+ )
960
+ return (
961
+ GraphTable(
962
+ table.columns,
963
+ rows,
964
+ MappingProxyType(
965
+ {
966
+ column: (
967
+ _merge_lineage(node, table.lineage[column])
968
+ if column in casts
969
+ else table.lineage[column]
970
+ )
971
+ for column in table.columns
972
+ }
973
+ ),
974
+ schema,
975
+ ),
976
+ 0,
977
+ 0,
978
+ "input order",
979
+ )
980
+ if node.operation == "filter":
981
+ selected: list[tuple[Any, ...]] = []
982
+ budget = _OutputBudget(node.node_id, max_rows, max_bytes)
983
+ for row in table.rows:
984
+ record = dict(zip(table.columns, row, strict=True))
985
+ if _predicate(params["predicate"], record, node.node_id) is True:
986
+ budget.retain(selected, row)
987
+ rows = tuple(selected)
988
+ return (
989
+ GraphTable(
990
+ table.columns,
991
+ rows,
992
+ _advance_lineage(node, table.lineage),
993
+ table.schema,
994
+ ),
995
+ len(table.rows) - len(rows),
996
+ 0,
997
+ "stable filtered input order",
998
+ )
999
+ if node.operation == "derive":
1000
+ expressions = params["expressions"]
1001
+ if not isinstance(expressions, Mapping) or not expressions:
1002
+ raise GraphExecutionError(
1003
+ "OPERATION_PARAMETERS", node.node_id, "expressions must be object"
1004
+ )
1005
+ collisions = set(expressions) & set(table.columns)
1006
+ if collisions:
1007
+ raise GraphExecutionError(
1008
+ "SCHEMA_DUPLICATE", node.node_id, f"derive collides: {sorted(collisions)}"
1009
+ )
1010
+ columns = (*table.columns, *expressions)
1011
+ for specification in expressions.values():
1012
+ expression_type = _expression_type(
1013
+ specification["expression"], table.schema, node.node_id
1014
+ )
1015
+ output_type = str(specification["output_type"])
1016
+ if expression_type not in {"null", output_type} and not (
1017
+ expression_type == "int64" and output_type == "decimal"
1018
+ ):
1019
+ raise GraphExecutionError(
1020
+ "DERIVE_TYPE",
1021
+ node.node_id,
1022
+ f"expression type {expression_type} cannot produce {output_type}",
1023
+ )
1024
+ output_rows: list[tuple[Any, ...]] = []
1025
+ budget = _OutputBudget(node.node_id, max_rows, max_bytes)
1026
+ for row in table.rows:
1027
+ record = dict(zip(table.columns, row, strict=True))
1028
+ derived = tuple(
1029
+ _cast(
1030
+ _derive(specification["expression"], record, node.node_id),
1031
+ specification["output_type"],
1032
+ node.node_id,
1033
+ )
1034
+ for specification in expressions.values()
1035
+ )
1036
+ budget.retain(output_rows, (*row, *derived))
1037
+ rows = tuple(output_rows)
1038
+ schema = dict(table.schema)
1039
+ lineage = dict(table.lineage)
1040
+ for name, specification in expressions.items():
1041
+ schema[name] = str(specification["output_type"])
1042
+ dependencies = tuple(
1043
+ table.lineage[column] for column in _expression_columns(specification["expression"])
1044
+ )
1045
+ lineage[name] = (
1046
+ _merge_lineage(node, *dependencies)
1047
+ if dependencies
1048
+ else ColumnLineage((), (_operation_coordinate(node),))
1049
+ )
1050
+ return (
1051
+ GraphTable(
1052
+ columns,
1053
+ rows,
1054
+ MappingProxyType(lineage),
1055
+ MappingProxyType(schema),
1056
+ ),
1057
+ 0,
1058
+ 0,
1059
+ "input order",
1060
+ )
1061
+ if node.operation == "date_add_days":
1062
+ # The parameter parser already settled these exact policies. Repeating the check at the
1063
+ # executable boundary keeps a forged or legacy in-memory node from acquiring a second,
1064
+ # implicit null or overflow behavior.
1065
+ if params.get("null_policy") != "propagate" or params.get("overflow_policy") != "reject":
1066
+ raise GraphExecutionError(
1067
+ "OPERATION_PARAMETERS",
1068
+ node.node_id,
1069
+ "date_add_days policies differ from the closed contract",
1070
+ )
1071
+ days = params.get("days")
1072
+ if (
1073
+ isinstance(days, bool)
1074
+ or not isinstance(days, int)
1075
+ or not -MAX_DATE_ADD_DAYS <= days <= MAX_DATE_ADD_DAYS
1076
+ ):
1077
+ raise GraphExecutionError(
1078
+ "OPERATION_PARAMETERS",
1079
+ node.node_id,
1080
+ f"days must be an integer from {-MAX_DATE_ADD_DAYS} to {MAX_DATE_ADD_DAYS}",
1081
+ )
1082
+ date_column = str(params["date_column"])
1083
+ output_column = str(params["output_column"])
1084
+ _require_columns(table, (date_column,), node.node_id)
1085
+ if output_column in table.columns:
1086
+ raise GraphExecutionError(
1087
+ "DATE_ADD_DAYS_COLUMN", node.node_id, "output column already exists"
1088
+ )
1089
+ if table.schema[date_column] != "date":
1090
+ raise GraphExecutionError(
1091
+ "DATE_ADD_DAYS_TYPE",
1092
+ node.node_id,
1093
+ "date_column must be explicitly cast to date",
1094
+ )
1095
+ output_rows: list[tuple[Any, ...]] = []
1096
+ budget = _OutputBudget(node.node_id, max_rows, max_bytes)
1097
+ for row in table.rows:
1098
+ value = row[index[date_column]]
1099
+ if value is None:
1100
+ result = None
1101
+ elif type(value) is not date:
1102
+ raise GraphExecutionError(
1103
+ "DATE_ADD_DAYS_TYPE", node.node_id, "date column contains a non-date value"
1104
+ )
1105
+ else:
1106
+ try:
1107
+ # A ``date`` has no zone or time-of-day. This is calendar arithmetic, so the
1108
+ # result never depends on a DST transition or on elapsed hours.
1109
+ result = value + timedelta(days=days)
1110
+ except OverflowError:
1111
+ raise GraphExecutionError(
1112
+ "DATE_ADD_DAYS_OVERFLOW",
1113
+ node.node_id,
1114
+ "calendar-day offset is outside the supported date range",
1115
+ ) from None
1116
+ budget.retain(output_rows, (*row, result))
1117
+ schema = dict(table.schema)
1118
+ schema[output_column] = "date"
1119
+ lineage = dict(table.lineage)
1120
+ lineage[output_column] = _merge_lineage(node, table.lineage[date_column])
1121
+ return (
1122
+ GraphTable(
1123
+ (*table.columns, output_column),
1124
+ tuple(output_rows),
1125
+ MappingProxyType(lineage),
1126
+ MappingProxyType(schema),
1127
+ ),
1128
+ 0,
1129
+ 0,
1130
+ "input order; fixed calendar-day offset",
1131
+ )
1132
+ if node.operation == "unpivot":
1133
+ value_columns = _columns(params["value_columns"], node.node_id)
1134
+ _require_columns(table, value_columns, node.node_id)
1135
+ if params.get("null_policy") != "retain":
1136
+ raise GraphExecutionError(
1137
+ "UNPIVOT_NULL_POLICY", node.node_id, "null policy must be retain"
1138
+ )
1139
+ name_column = str(params["name_column"])
1140
+ value_column = str(params["value_column"])
1141
+ melted = set(value_columns)
1142
+ identity_columns = tuple(column for column in table.columns if column not in melted)
1143
+ columns = (*identity_columns, name_column, value_column)
1144
+ if len(columns) != len(set(columns)):
1145
+ raise GraphExecutionError("SCHEMA_DUPLICATE", node.node_id, "unpivot columns collide")
1146
+ # The long value column holds one column's worth of values per row, so every declared wide
1147
+ # column must already be the type the plan declared for it. Checking against the
1148
+ # declaration rather than inferring from the columns is what keeps the output type
1149
+ # readable from the recipe: an all-empty column would otherwise type the output
1150
+ # differently from the same recipe's last run. A plan whose wide columns disagree casts
1151
+ # them first.
1152
+ value_type = str(params["value_type"])
1153
+ observed = {table.schema[column] for column in value_columns} - {"null"}
1154
+ if observed - {value_type}:
1155
+ raise GraphExecutionError(
1156
+ "UNPIVOT_VALUE_TYPE",
1157
+ node.node_id,
1158
+ f"declared value columns are not all {value_type}: {sorted(observed)}",
1159
+ )
1160
+ # The expansion factor is declared, so the refusal can come before the copy exists.
1161
+ if len(table.rows) * len(value_columns) > max_rows:
1162
+ raise GraphExecutionError(
1163
+ "ROW_BUDGET_EXCEEDED", node.node_id, f"unpivot upper bound exceeds {max_rows}"
1164
+ )
1165
+ output_rows = []
1166
+ budget = _OutputBudget(node.node_id, max_rows, max_bytes)
1167
+ for row in table.rows:
1168
+ identity = tuple(row[index[column]] for column in identity_columns)
1169
+ for column in value_columns:
1170
+ budget.retain(output_rows, (*identity, column, row[index[column]]))
1171
+ melted_lineage = _merge_lineage(node, *(table.lineage[column] for column in value_columns))
1172
+ return (
1173
+ GraphTable(
1174
+ columns,
1175
+ tuple(output_rows),
1176
+ MappingProxyType(
1177
+ {
1178
+ **dict(
1179
+ _advance_lineage(
1180
+ node,
1181
+ MappingProxyType(
1182
+ {column: table.lineage[column] for column in identity_columns}
1183
+ ),
1184
+ )
1185
+ ),
1186
+ name_column: melted_lineage,
1187
+ value_column: melted_lineage,
1188
+ }
1189
+ ),
1190
+ MappingProxyType(
1191
+ {
1192
+ **{column: table.schema[column] for column in identity_columns},
1193
+ name_column: "string",
1194
+ value_column: value_type,
1195
+ }
1196
+ ),
1197
+ ),
1198
+ 0,
1199
+ 0,
1200
+ "input order then declared value-column order",
1201
+ )
1202
+ if node.operation == "union":
1203
+ safe_casts = params["safe_casts"]
1204
+ if not isinstance(safe_casts, Mapping):
1205
+ raise GraphExecutionError(
1206
+ "OPERATION_PARAMETERS", node.node_id, "safe_casts must be an object"
1207
+ )
1208
+ unknown_casts = sorted(set(safe_casts) - set(table.columns))
1209
+ if unknown_casts:
1210
+ raise GraphExecutionError(
1211
+ "COLUMN_UNKNOWN", node.node_id, f"safe casts name absent columns: {unknown_casts}"
1212
+ )
1213
+ target_schema = {
1214
+ column: str(safe_casts.get(column, table.schema[column])) for column in table.columns
1215
+ }
1216
+ for candidate in inputs[1:]:
1217
+ if candidate.columns != table.columns:
1218
+ raise GraphExecutionError(
1219
+ "UNION_SCHEMA_MISMATCH",
1220
+ node.node_id,
1221
+ "union inputs require exact ordered logical schema",
1222
+ )
1223
+ for candidate in inputs:
1224
+ for column in table.columns:
1225
+ source_type = candidate.schema[column]
1226
+ target_type = target_schema[column]
1227
+ if source_type == target_type or source_type == "null":
1228
+ continue
1229
+ if (source_type, target_type) != ("int64", "decimal"):
1230
+ raise GraphExecutionError(
1231
+ "UNION_SCHEMA_MISMATCH",
1232
+ node.node_id,
1233
+ f"unsafe logical cast {column}: {source_type}->{target_type}",
1234
+ )
1235
+ total = sum(len(candidate.rows) for candidate in inputs)
1236
+ if total > max_rows:
1237
+ raise GraphExecutionError(
1238
+ "ROW_BUDGET_EXCEEDED", node.node_id, f"union exceeds {max_rows}"
1239
+ )
1240
+ union_rows: list[tuple[Any, ...]] = []
1241
+ budget = _OutputBudget(node.node_id, max_rows, max_bytes)
1242
+ for candidate in inputs:
1243
+ for row in candidate.rows:
1244
+ converted = tuple(
1245
+ _cast(value, target_schema[column], node.node_id)
1246
+ if candidate.schema[column] != target_schema[column]
1247
+ and candidate.schema[column] != "null"
1248
+ else value
1249
+ for column, value in zip(candidate.columns, row, strict=True)
1250
+ )
1251
+ budget.retain(union_rows, converted)
1252
+ return (
1253
+ GraphTable(
1254
+ table.columns,
1255
+ tuple(union_rows),
1256
+ MappingProxyType(
1257
+ {
1258
+ column: _merge_lineage(
1259
+ node, *(candidate.lineage[column] for candidate in inputs)
1260
+ )
1261
+ for column in table.columns
1262
+ }
1263
+ ),
1264
+ MappingProxyType(target_schema),
1265
+ ),
1266
+ 0,
1267
+ 0,
1268
+ "declared input order then source order",
1269
+ )
1270
+ if node.operation in {"sort", "deduplicate"}:
1271
+ specs = _key_specs(
1272
+ params["keys"] if node.operation == "sort" else params["authority"], node.node_id
1273
+ )
1274
+ if node.operation == "deduplicate":
1275
+ keys = _columns(params["keys"], node.node_id)
1276
+ _require_columns(table, keys, node.node_id)
1277
+ if params.get("tie_policy") != "stable_first":
1278
+ raise GraphExecutionError(
1279
+ "DEDUP_TIE_POLICY", node.node_id, "tie policy must be stable_first"
1280
+ )
1281
+ _require_columns(table, (item["column"] for item in specs), node.node_id)
1282
+ if node.operation == "sort":
1283
+ # Sorting preserves every logical scalar, so refuse before allocating its ordering
1284
+ # index rather than discovering an oversized output after the full copy exists.
1285
+ if len(table.rows) > max_rows:
1286
+ raise GraphExecutionError(
1287
+ "ROW_BUDGET_EXCEEDED", node.node_id, f"sort exceeds {max_rows}"
1288
+ )
1289
+ if _table_bytes(table, node.node_id) > max_bytes:
1290
+ raise GraphExecutionError(
1291
+ "BYTE_BUDGET_EXCEEDED", node.node_id, f"sort exceeds {max_bytes}"
1292
+ )
1293
+ ordered = sorted(
1294
+ enumerate(table.rows),
1295
+ key=lambda pair: (
1296
+ (
1297
+ *(
1298
+ _sort_token(
1299
+ pair[1][index[item["column"]]],
1300
+ nulls=item["nulls"],
1301
+ direction=item["direction"],
1302
+ node_id=node.node_id,
1303
+ )
1304
+ for item in specs
1305
+ ),
1306
+ pair[0],
1307
+ )
1308
+ ),
1309
+ )
1310
+ if node.operation == "sort":
1311
+ return (
1312
+ GraphTable(
1313
+ table.columns,
1314
+ tuple(row for _, row in ordered),
1315
+ _advance_lineage(node, table.lineage),
1316
+ table.schema,
1317
+ ),
1318
+ 0,
1319
+ 0,
1320
+ "declared keys then stable ordinal",
1321
+ )
1322
+ seen: set[tuple[tuple[str, Any], ...]] = set()
1323
+ retained: list[tuple[int, tuple[Any, ...]]] = []
1324
+ for ordinal, row in ordered:
1325
+ values = tuple(row[index[column]] for column in keys)
1326
+ if any(value is None for value in values):
1327
+ raise GraphExecutionError(
1328
+ "DEDUP_NULL_KEY", node.node_id, "deduplication key contains null"
1329
+ )
1330
+ key = tuple(
1331
+ (table.schema[column], value) for column, value in zip(keys, values, strict=True)
1332
+ )
1333
+ if key in seen:
1334
+ continue
1335
+ seen.add(key)
1336
+ retained.append((ordinal, row))
1337
+ retained.sort(key=lambda pair: pair[0])
1338
+ duplicates = len(table.rows) - len(retained)
1339
+ return (
1340
+ GraphTable(
1341
+ table.columns,
1342
+ tuple(row for _, row in retained),
1343
+ _advance_lineage(node, table.lineage),
1344
+ table.schema,
1345
+ ),
1346
+ 0,
1347
+ duplicates,
1348
+ "authority then stable original ordinal",
1349
+ )
1350
+ if node.operation == "prediction_label":
1351
+ fixed_policies = {
1352
+ "null_key_policy": "reject",
1353
+ "duplicate_key_policy": "reject",
1354
+ "missing_target_policy": "drop",
1355
+ "null_target_policy": "drop",
1356
+ "boundary_policy": "drop",
1357
+ }
1358
+ if any(params.get(name) != expected for name, expected in fixed_policies.items()):
1359
+ raise GraphExecutionError(
1360
+ "OPERATION_PARAMETERS",
1361
+ node.node_id,
1362
+ "prediction label policies differ from the closed contract",
1363
+ )
1364
+ horizon_days = params.get("horizon_days")
1365
+ if (
1366
+ isinstance(horizon_days, bool)
1367
+ or not isinstance(horizon_days, int)
1368
+ or not 1 <= horizon_days <= 3_650
1369
+ ):
1370
+ raise GraphExecutionError(
1371
+ "OPERATION_PARAMETERS",
1372
+ node.node_id,
1373
+ "horizon_days must be an integer from 1 to 3650",
1374
+ )
1375
+ raw_partition_by = params["partition_by"]
1376
+ if (
1377
+ not isinstance(raw_partition_by, list)
1378
+ or any(not isinstance(item, str) or not item for item in raw_partition_by)
1379
+ or len(raw_partition_by) != len(set(raw_partition_by))
1380
+ ):
1381
+ raise GraphExecutionError(
1382
+ "OPERATION_PARAMETERS",
1383
+ node.node_id,
1384
+ "partition_by must be a unique text list and may be empty",
1385
+ )
1386
+ partition_by = tuple(raw_partition_by)
1387
+ time_column = str(params["time_column"])
1388
+ source_column = str(params["source_column"])
1389
+ output_column = str(params["output_column"])
1390
+ _require_columns(table, (*partition_by, time_column, source_column), node.node_id)
1391
+ if output_column in table.columns:
1392
+ raise GraphExecutionError(
1393
+ "PREDICTION_LABEL_COLUMN",
1394
+ node.node_id,
1395
+ "output column already exists",
1396
+ )
1397
+ if table.schema[time_column] not in {"date", "timestamp_utc"}:
1398
+ raise GraphExecutionError(
1399
+ "PREDICTION_LABEL_TIME_TYPE",
1400
+ node.node_id,
1401
+ "time column must be explicitly cast to date or timestamp_utc",
1402
+ )
1403
+
1404
+ def temporal_key(value: Any) -> date | datetime:
1405
+ if table.schema[time_column] == "date":
1406
+ if type(value) is not date:
1407
+ raise GraphExecutionError(
1408
+ "PREDICTION_LABEL_TIME_TYPE",
1409
+ node.node_id,
1410
+ "date time column contains a non-date value",
1411
+ )
1412
+ return value
1413
+ if not isinstance(value, datetime) or value.tzinfo is None or value.utcoffset() is None:
1414
+ raise GraphExecutionError(
1415
+ "PREDICTION_LABEL_TIME_TYPE",
1416
+ node.node_id,
1417
+ "timestamp time column contains a non-UTC value",
1418
+ )
1419
+ return value.astimezone(UTC)
1420
+
1421
+ keyed: dict[tuple[Any, ...], tuple[Any, ...]] = {}
1422
+ ordered: list[tuple[tuple[Any, ...], date | datetime, tuple[Any, ...]]] = []
1423
+ for row in table.rows:
1424
+ partition = tuple(row[index[column]] for column in partition_by)
1425
+ raw_time = row[index[time_column]]
1426
+ if any(value is None for value in partition) or raw_time is None:
1427
+ raise GraphExecutionError(
1428
+ "PREDICTION_LABEL_NULL_KEY",
1429
+ node.node_id,
1430
+ "partition and time keys must be non-null",
1431
+ )
1432
+ time = temporal_key(raw_time)
1433
+ key = (*partition, time)
1434
+ if key in keyed:
1435
+ raise GraphExecutionError(
1436
+ "PREDICTION_LABEL_DUPLICATE_KEY",
1437
+ node.node_id,
1438
+ "partition and time keys must be unique",
1439
+ )
1440
+ keyed[key] = row
1441
+ ordered.append((partition, time, row))
1442
+ ordered.sort(
1443
+ key=lambda item: (
1444
+ *(
1445
+ _sort_token(
1446
+ value,
1447
+ nulls="last",
1448
+ direction="asc",
1449
+ node_id=node.node_id,
1450
+ )
1451
+ for value in item[0]
1452
+ ),
1453
+ _sort_token(
1454
+ item[1],
1455
+ nulls="last",
1456
+ direction="asc",
1457
+ node_id=node.node_id,
1458
+ ),
1459
+ )
1460
+ )
1461
+ horizon = timedelta(days=horizon_days)
1462
+ output: list[tuple[Any, ...]] = []
1463
+ budget = _OutputBudget(node.node_id, max_rows, max_bytes)
1464
+ rejected = 0
1465
+ for partition, time, row in ordered:
1466
+ try:
1467
+ target_time = time + horizon
1468
+ except (OverflowError, ValueError) as exc:
1469
+ raise GraphExecutionError(
1470
+ "PREDICTION_LABEL_TIME_RANGE",
1471
+ node.node_id,
1472
+ "future target date is outside the supported calendar range",
1473
+ ) from exc
1474
+ target = keyed.get((*partition, target_time))
1475
+ if target is None or target[index[source_column]] is None:
1476
+ rejected += 1
1477
+ continue
1478
+ budget.retain(output, (*row, target[index[source_column]]))
1479
+ return (
1480
+ GraphTable(
1481
+ (*table.columns, output_column),
1482
+ tuple(output),
1483
+ MappingProxyType(
1484
+ {
1485
+ **dict(_advance_lineage(node, table.lineage)),
1486
+ output_column: _merge_lineage(node, table.lineage[source_column]),
1487
+ }
1488
+ ),
1489
+ MappingProxyType(
1490
+ {
1491
+ **dict(table.schema),
1492
+ output_column: table.schema[source_column],
1493
+ }
1494
+ ),
1495
+ ),
1496
+ rejected,
1497
+ 0,
1498
+ "partition keys ascending then exact calendar date ascending",
1499
+ )
1500
+ if node.operation == "join":
1501
+ right = inputs[1]
1502
+ right_index = _index(right, node.node_id)
1503
+ keys = _columns(params["on"], node.node_id)
1504
+ _require_columns(table, keys, node.node_id)
1505
+ _require_columns(right, keys, node.node_id)
1506
+ incompatible = [column for column in keys if table.schema[column] != right.schema[column]]
1507
+ if incompatible:
1508
+ raise GraphExecutionError(
1509
+ "JOIN_KEY_TYPE",
1510
+ node.node_id,
1511
+ f"join keys have incompatible logical types: {incompatible}",
1512
+ )
1513
+ if len(table.rows) > max_rows:
1514
+ raise GraphExecutionError(
1515
+ "ROW_BUDGET_EXCEEDED", node.node_id, f"join upper bound exceeds {max_rows}"
1516
+ )
1517
+ left_bytes = _table_bytes(table, node.node_id)
1518
+ kind = params["kind"]
1519
+ # An anti join answers "which left rows have no match", so no right row is ever emitted
1520
+ # and the right table contributes no columns. Carrying them across as nulls would put a
1521
+ # column in the output whose every value was manufactured by the join itself.
1522
+ right_extra = (
1523
+ ()
1524
+ if kind == "anti"
1525
+ else tuple(column for column in right.columns if column not in keys)
1526
+ )
1527
+ right_extra_max = max(
1528
+ (
1529
+ sum(_value_bytes(row[right_index[column]], node.node_id) for column in right_extra)
1530
+ for row in right.rows
1531
+ ),
1532
+ default=sum(_value_bytes(None, node.node_id) for _ in right_extra),
1533
+ )
1534
+ if left_bytes + len(table.rows) * right_extra_max > max_bytes:
1535
+ raise GraphExecutionError(
1536
+ "BYTE_BUDGET_EXCEEDED", node.node_id, f"join upper bound exceeds {max_bytes}"
1537
+ )
1538
+ lookup: dict[tuple[tuple[str, Any], ...], list[tuple[Any, ...]]] = defaultdict(list)
1539
+ for row in right.rows:
1540
+ values = tuple(row[right_index[column]] for column in keys)
1541
+ if any(value is None for value in values) and params["null_key_policy"] == "reject":
1542
+ raise GraphExecutionError("JOIN_NULL_KEY", node.node_id, "right null key")
1543
+ key = tuple(
1544
+ (right.schema[column], value) for column, value in zip(keys, values, strict=True)
1545
+ )
1546
+ lookup[key].append(row)
1547
+ # What the declared cardinality is for is bounding how far the output may grow, and an
1548
+ # anti join reaches that bound by construction: it emits the left rows that matched
1549
+ # nothing, so it is a subset of the left table however many right rows share a key.
1550
+ # Matching there is existence, not lookup, and a duplicated right key cannot multiply a
1551
+ # row that is only ever emitted when no right row was found. So the right-key uniqueness
1552
+ # rule the other kinds carry is not applied to ``anti``; requiring it would refuse the
1553
+ # plainest anti join there is -- facilities with no readings, against a readings table
1554
+ # that of course holds many rows per facility. The left rule below is unchanged for
1555
+ # every kind: ``one_to_one`` still says each left key appears once.
1556
+ cardinality = params["cardinality"]
1557
+ if (
1558
+ kind != "anti"
1559
+ and cardinality in {"one_to_one", "many_to_one"}
1560
+ and any(len(rows) > 1 for rows in lookup.values())
1561
+ ):
1562
+ raise GraphExecutionError("JOIN_CARDINALITY", node.node_id, "right key is not unique")
1563
+ output_columns = (*table.columns, *right_extra)
1564
+ if len(output_columns) != len(set(output_columns)):
1565
+ raise GraphExecutionError("SCHEMA_DUPLICATE", node.node_id, "join columns collide")
1566
+ output: list[tuple[Any, ...]] = []
1567
+ budget = _OutputBudget(node.node_id, max_rows, max_bytes)
1568
+ rejected = 0
1569
+ left_seen: set[tuple[tuple[str, Any], ...]] = set()
1570
+ for left_row in table.rows:
1571
+ values = tuple(left_row[index[column]] for column in keys)
1572
+ if any(value is None for value in values):
1573
+ raise GraphExecutionError("JOIN_NULL_KEY", node.node_id, "left null key")
1574
+ key = tuple(
1575
+ (table.schema[column], value) for column, value in zip(keys, values, strict=True)
1576
+ )
1577
+ if cardinality == "one_to_one" and key in left_seen:
1578
+ raise GraphExecutionError(
1579
+ "JOIN_CARDINALITY", node.node_id, "left key is not unique"
1580
+ )
1581
+ left_seen.add(key)
1582
+ matches = lookup.get(key, [])
1583
+ if not matches:
1584
+ if params["unmatched_policy"] == "reject":
1585
+ raise GraphExecutionError(
1586
+ "JOIN_UNMATCHED", node.node_id, "left row is unmatched"
1587
+ )
1588
+ if kind in {"left", "anti"}:
1589
+ budget.retain(output, (*left_row, *(None for _ in right_extra)))
1590
+ else:
1591
+ rejected += 1
1592
+ continue
1593
+ if kind == "anti":
1594
+ rejected += 1
1595
+ continue
1596
+ for right_row in matches:
1597
+ budget.retain(
1598
+ output,
1599
+ (*left_row, *(right_row[right_index[column]] for column in right_extra)),
1600
+ )
1601
+ return (
1602
+ GraphTable(
1603
+ output_columns,
1604
+ tuple(output),
1605
+ MappingProxyType(
1606
+ {
1607
+ **{
1608
+ column: _merge_lineage(node, item)
1609
+ for column, item in table.lineage.items()
1610
+ },
1611
+ **{
1612
+ column: _merge_lineage(node, right.lineage[column])
1613
+ for column in right_extra
1614
+ },
1615
+ }
1616
+ ),
1617
+ MappingProxyType(
1618
+ {
1619
+ **dict(table.schema),
1620
+ **{column: right.schema[column] for column in right_extra},
1621
+ }
1622
+ ),
1623
+ ),
1624
+ rejected,
1625
+ 0,
1626
+ "left order" if kind == "anti" else "left order then right source order",
1627
+ )
1628
+ if node.operation in {"aggregate", "resample"}:
1629
+ return _aggregate(
1630
+ node,
1631
+ table,
1632
+ params,
1633
+ resample=node.operation == "resample",
1634
+ max_rows=max_rows,
1635
+ max_bytes=max_bytes,
1636
+ )
1637
+ raise GraphExecutionError("OPERATION_UNSUPPORTED", node.node_id, node.operation)
1638
+
1639
+
1640
+ def _decimal(value: Any, node_id: str, *, code: str = "AGGREGATE_VALUE") -> Decimal:
1641
+ """Move one numeric scalar into the canonical decimal domain.
1642
+
1643
+ ``code`` names the caller's refusal so a derive expression does not report a measure error.
1644
+ """
1645
+
1646
+ try:
1647
+ result = Decimal(str(value))
1648
+ except (InvalidOperation, ValueError) as exc:
1649
+ raise GraphExecutionError(code, node_id, "value is not numeric") from exc
1650
+ return _canonical_decimal(result, node_id)
1651
+
1652
+
1653
+ def _aggregate(
1654
+ node: GraphNode,
1655
+ table: GraphTable,
1656
+ params: dict[str, Any],
1657
+ *,
1658
+ resample: bool,
1659
+ max_rows: int,
1660
+ max_bytes: int,
1661
+ ) -> tuple[GraphTable, int, int, str]:
1662
+ group_by = _columns(params["group_by"], node.node_id) if params["group_by"] else ()
1663
+ _require_columns(table, group_by, node.node_id)
1664
+ measures = params["aggregates"] if resample else params["measures"]
1665
+ if not isinstance(measures, list) or not measures:
1666
+ raise GraphExecutionError("OPERATION_PARAMETERS", node.node_id, "measures must be list")
1667
+ event_time = str(params["event_time"]) if resample else None
1668
+ if resample:
1669
+ if params["timezone"] != "UTC":
1670
+ raise GraphExecutionError(
1671
+ "RESAMPLE_TIMEZONE_UNSUPPORTED", node.node_id, "only UTC is deterministic"
1672
+ )
1673
+ _require_columns(table, [event_time], node.node_id) # type: ignore[list-item]
1674
+ table_index = _index(table, node.node_id)
1675
+ groups: dict[tuple[Any, ...], list[tuple[Any, ...]]] = defaultdict(list)
1676
+ duration = int(params.get("duration_seconds", 0))
1677
+ epoch = datetime(1970, 1, 1, tzinfo=UTC)
1678
+ for row in table.rows:
1679
+ key = tuple(row[table_index[column]] for column in group_by)
1680
+ if resample:
1681
+ try:
1682
+ raw_time = row[table_index[event_time]] # type: ignore[index]
1683
+ if isinstance(raw_time, datetime):
1684
+ parsed = raw_time
1685
+ elif isinstance(raw_time, str) and raw_time.endswith("Z"):
1686
+ parsed = datetime.fromisoformat(raw_time.replace("Z", "+00:00"))
1687
+ else:
1688
+ raise ValueError
1689
+ except (TypeError, ValueError) as exc:
1690
+ raise GraphExecutionError(
1691
+ "RESAMPLE_TIME_INVALID", node.node_id, "event time is invalid"
1692
+ ) from exc
1693
+ if parsed.tzinfo is None or parsed.utcoffset() is None or duration <= 0:
1694
+ raise GraphExecutionError(
1695
+ "RESAMPLE_TIME_INVALID", node.node_id, "UTC timestamp/duration required"
1696
+ )
1697
+ utc_time = parsed.astimezone(UTC)
1698
+ delta = utc_time - epoch
1699
+ microseconds = (delta.days * 86_400 + delta.seconds) * 1_000_000 + delta.microseconds
1700
+ duration_microseconds = duration * 1_000_000
1701
+ bucket_microseconds = (microseconds // duration_microseconds) * duration_microseconds
1702
+ key = (
1703
+ *key,
1704
+ epoch + timedelta(microseconds=bucket_microseconds),
1705
+ )
1706
+ groups[key].append(row)
1707
+ output_names: list[str] = [*group_by]
1708
+ if resample:
1709
+ output_names.append(f"{event_time}_bucket")
1710
+ for measure in measures:
1711
+ if not isinstance(measure, Mapping) or not all(
1712
+ key in measure for key in ("column", "op", "as")
1713
+ ):
1714
+ raise GraphExecutionError("OPERATION_PARAMETERS", node.node_id, "invalid measure")
1715
+ output_names.append(str(measure["as"]))
1716
+ if len(output_names) != len(set(output_names)):
1717
+ raise GraphExecutionError("SCHEMA_DUPLICATE", node.node_id, "aggregate names collide")
1718
+ output: list[tuple[Any, ...]] = []
1719
+ budget = _OutputBudget(node.node_id, max_rows, max_bytes)
1720
+ for key in sorted(
1721
+ groups,
1722
+ key=lambda item: tuple(
1723
+ _sort_token(value, nulls="last", direction="asc", node_id=node.node_id)
1724
+ for value in item
1725
+ ),
1726
+ ):
1727
+ rows = groups[key]
1728
+ values: list[Any] = list(key)
1729
+ for measure in measures:
1730
+ column, op = str(measure["column"]), str(measure["op"])
1731
+ _require_columns(table, [column], node.node_id)
1732
+ if op in {"sum", "mean"} and table.schema[column] not in {
1733
+ "int64",
1734
+ "float64",
1735
+ "decimal",
1736
+ }:
1737
+ raise GraphExecutionError(
1738
+ "AGGREGATE_VALUE",
1739
+ node.node_id,
1740
+ "sum and mean require an explicitly numeric input column",
1741
+ )
1742
+ column_index = table_index[column]
1743
+ present = [row[column_index] for row in rows if row[column_index] is not None]
1744
+ if op == "count":
1745
+ value: Any = len(present)
1746
+ elif not present:
1747
+ value = None
1748
+ elif op == "sum":
1749
+ with localcontext() as context:
1750
+ context.prec = DECIMAL_WORKING_PRECISION
1751
+ total = sum((_decimal(item, node.node_id) for item in present), Decimal(0))
1752
+ value = _canonical_decimal(total, node.node_id)
1753
+ elif op == "mean":
1754
+ with localcontext() as context:
1755
+ context.prec = DECIMAL_WORKING_PRECISION
1756
+ mean = sum(
1757
+ (_decimal(item, node.node_id) for item in present), Decimal(0)
1758
+ ) / len(present)
1759
+ # A repeating quotient cannot be represented exactly in decimal256(76,38).
1760
+ # The operation contract therefore rounds once, at the physical scale, using
1761
+ # banker's rounding. Every engine routes through this shared scalar kernel.
1762
+ mean = mean.quantize(DECIMAL_QUANTUM, rounding=ROUND_HALF_EVEN)
1763
+ value = _canonical_decimal(mean, node.node_id)
1764
+ elif op == "min":
1765
+ try:
1766
+ value = min(present)
1767
+ except TypeError as exc:
1768
+ raise GraphExecutionError(
1769
+ "AGGREGATE_VALUE", node.node_id, "measure values are incompatible"
1770
+ ) from exc
1771
+ elif op == "max":
1772
+ try:
1773
+ value = max(present)
1774
+ except TypeError as exc:
1775
+ raise GraphExecutionError(
1776
+ "AGGREGATE_VALUE", node.node_id, "measure values are incompatible"
1777
+ ) from exc
1778
+ elif op == "first":
1779
+ value = present[0]
1780
+ elif op == "last":
1781
+ value = present[-1]
1782
+ else:
1783
+ raise GraphExecutionError("AGGREGATE_UNSUPPORTED", node.node_id, op)
1784
+ values.append(value)
1785
+ budget.retain(output, tuple(values))
1786
+ schema = {column: table.schema[column] for column in group_by}
1787
+ if resample:
1788
+ schema[f"{event_time}_bucket"] = "timestamp_utc"
1789
+ for measure in measures:
1790
+ operation = str(measure["op"])
1791
+ schema[str(measure["as"])] = (
1792
+ "int64"
1793
+ if operation == "count"
1794
+ else "decimal"
1795
+ if operation in {"sum", "mean"}
1796
+ else table.schema[str(measure["column"])]
1797
+ )
1798
+ lineage: dict[str, ColumnLineage] = {
1799
+ column: _merge_lineage(node, table.lineage[column]) for column in group_by
1800
+ }
1801
+ if resample:
1802
+ lineage[f"{event_time}_bucket"] = _merge_lineage(node, table.lineage[str(event_time)])
1803
+ for measure in measures:
1804
+ lineage[str(measure["as"])] = _merge_lineage(node, table.lineage[str(measure["column"])])
1805
+ return (
1806
+ GraphTable(
1807
+ tuple(output_names),
1808
+ tuple(output),
1809
+ MappingProxyType(lineage),
1810
+ MappingProxyType(schema),
1811
+ ),
1812
+ 0,
1813
+ 0,
1814
+ "group keys ascending; stable first/last",
1815
+ )
1816
+
1817
+
1818
+ def execute_graph(
1819
+ plan: GraphTablePlan,
1820
+ sources: Mapping[str, Iterable[Mapping[str, Any]]],
1821
+ *,
1822
+ policy: GraphResourcePolicy | None = None,
1823
+ engine: str = "reference",
1824
+ backend: Backend | None = None,
1825
+ source_columns: Mapping[str, Iterable[str]] | None = None,
1826
+ node_started: Callable[[GraphNode], None] | None = None,
1827
+ node_observer: Callable[[GraphNode, GraphTable, NodeAudit, Mapping[str, GraphTable]], None]
1828
+ | None = None,
1829
+ ) -> GraphExecution:
1830
+ if backend is None and engine not in {"reference", "reference-graph-kernel"}:
1831
+ raise GraphExecutionError(
1832
+ "ENGINE_RESTRICTED",
1833
+ "graph",
1834
+ "local-graph-table-plan.v1 currently executes only on the reference graph kernel",
1835
+ )
1836
+ backend_name = "reference" if backend is None else backend.name
1837
+ engine = f"shared-graph-kernel+{backend_name}"
1838
+ policy = policy or GraphResourcePolicy()
1839
+ declared = {source.source_id for source in plan.sources}
1840
+ if set(sources) != declared:
1841
+ raise GraphExecutionError("SOURCE_SET_MISMATCH", "graph", "source rows must match plan")
1842
+ declared_source_columns = (
1843
+ {source_id: tuple(columns) for source_id, columns in source_columns.items()}
1844
+ if source_columns is not None
1845
+ else {}
1846
+ )
1847
+ if source_columns is not None and set(declared_source_columns) != declared:
1848
+ raise GraphExecutionError("SOURCE_SET_MISMATCH", "graph", "source schemas must match plan")
1849
+ if any(
1850
+ not columns or len(columns) != len(set(columns))
1851
+ for columns in declared_source_columns.values()
1852
+ ):
1853
+ raise GraphExecutionError(
1854
+ "SOURCE_SCHEMA_INVALID", "graph", "source schema columns must be non-empty and unique"
1855
+ )
1856
+ active_tables: dict[str, GraphTable] = {}
1857
+ snapshots: dict[str, GraphTable] = {}
1858
+ audits: list[NodeAudit] = []
1859
+ retained_bytes = 0
1860
+ peak_retained_bytes = 0
1861
+ remaining_consumers = {
1862
+ node.node_id: sum(candidate.inputs.count(node.node_id) for candidate in plan.nodes)
1863
+ for node in plan.nodes
1864
+ }
1865
+ for node in plan.nodes:
1866
+ if node_started is not None:
1867
+ node_started(node)
1868
+ inputs = tuple(active_tables[input_id] for input_id in node.inputs)
1869
+ source_id = str(_parameters(node)["source"]) if node.operation == "source" else None
1870
+ limit = (
1871
+ policy.max_output_rows
1872
+ if node.node_id == plan.terminal
1873
+ else policy.max_intermediate_rows
1874
+ )
1875
+ scope_byte_limit = (
1876
+ policy.max_source_bytes
1877
+ if node.operation == "source"
1878
+ else policy.max_output_bytes
1879
+ if node.node_id == plan.terminal
1880
+ else policy.max_intermediate_bytes
1881
+ )
1882
+ source_table = (
1883
+ _table_from_records(
1884
+ node.node_id,
1885
+ source_id,
1886
+ sources[source_id],
1887
+ policy.max_source_rows,
1888
+ min(policy.max_source_bytes, policy.max_retained_bytes - retained_bytes),
1889
+ declared_source_columns.get(source_id),
1890
+ )
1891
+ if source_id is not None
1892
+ else None
1893
+ )
1894
+ table, rejected, duplicates, order = _execute_node(
1895
+ node, inputs, source_table, limit, scope_byte_limit
1896
+ )
1897
+ table_bytes = _table_bytes(table, node.node_id)
1898
+ # A backend materialization transiently holds the contract table and its engine-shaped
1899
+ # copy alongside every still-live input. This is canonical logical byte accounting, not
1900
+ # an RSS claim, and refuses before asking an engine to allocate the copy.
1901
+ transient_bytes = retained_bytes + (2 * table_bytes if backend is not None else table_bytes)
1902
+ if transient_bytes > policy.max_retained_bytes:
1903
+ raise GraphExecutionError(
1904
+ "BYTE_BUDGET_EXCEEDED",
1905
+ node.node_id,
1906
+ f"logical transient tables exceed {policy.max_retained_bytes}",
1907
+ )
1908
+ peak_retained_bytes = max(peak_retained_bytes, transient_bytes)
1909
+ if backend is not None:
1910
+ try:
1911
+ materialized = backend.materialize_graph_rows(table.columns, table.rows)
1912
+ except Exception as exc:
1913
+ raise GraphExecutionError(
1914
+ "ENGINE_EXECUTION_FAILED",
1915
+ node.node_id,
1916
+ f"{backend.name} could not materialize the graph node",
1917
+ ) from exc
1918
+ if materialized != table.rows:
1919
+ raise GraphExecutionError(
1920
+ "ENGINE_DIVERGENCE",
1921
+ node.node_id,
1922
+ f"{backend.name} changed contract-owned graph scalars or ordering",
1923
+ )
1924
+ table = GraphTable(table.columns, materialized, table.lineage, table.schema)
1925
+ if len(table.rows) > limit:
1926
+ scope = "output" if node.node_id == plan.terminal else "intermediate"
1927
+ raise GraphExecutionError(
1928
+ "ROW_BUDGET_EXCEEDED", node.node_id, f"{scope} exceeds {limit}"
1929
+ )
1930
+ if table_bytes > scope_byte_limit:
1931
+ scope = (
1932
+ "source"
1933
+ if node.operation == "source"
1934
+ else "output"
1935
+ if node.node_id == plan.terminal
1936
+ else "intermediate"
1937
+ )
1938
+ raise GraphExecutionError(
1939
+ "BYTE_BUDGET_EXCEEDED",
1940
+ node.node_id,
1941
+ f"{scope} exceeds {scope_byte_limit}",
1942
+ )
1943
+ for input_id in node.inputs:
1944
+ remaining_consumers[input_id] -= 1
1945
+ if remaining_consumers[input_id] == 0:
1946
+ retained_bytes -= _table_bytes(active_tables.pop(input_id), node.node_id)
1947
+ retained_bytes += table_bytes
1948
+ if retained_bytes > policy.max_retained_bytes:
1949
+ raise GraphExecutionError(
1950
+ "BYTE_BUDGET_EXCEEDED",
1951
+ node.node_id,
1952
+ f"retained graph tables exceed {policy.max_retained_bytes}",
1953
+ )
1954
+ input_digests = tuple(table_digest(item) for item in inputs)
1955
+ audit = NodeAudit(
1956
+ node.node_id,
1957
+ node.operation,
1958
+ node.operation_version,
1959
+ node.inputs,
1960
+ _digest(_parameters(node)),
1961
+ input_digests,
1962
+ table_digest(table),
1963
+ sum(len(item.rows) for item in inputs),
1964
+ len(table.rows),
1965
+ rejected,
1966
+ duplicates,
1967
+ order,
1968
+ engine,
1969
+ tuple(_schema_items(item) for item in inputs),
1970
+ _schema_items(table),
1971
+ sum(_table_bytes(item, node.node_id) for item in inputs),
1972
+ table_bytes,
1973
+ retained_bytes,
1974
+ )
1975
+ audits.append(audit)
1976
+ active_tables[node.node_id] = table
1977
+ snapshots[node.node_id] = GraphTable(
1978
+ table.columns,
1979
+ table.rows[:5],
1980
+ table.lineage,
1981
+ table.schema,
1982
+ )
1983
+ if node_observer is not None:
1984
+ node_observer(
1985
+ node,
1986
+ table,
1987
+ audit,
1988
+ MappingProxyType(dict(zip(node.inputs, inputs, strict=True))),
1989
+ )
1990
+ terminal = active_tables[plan.terminal]
1991
+ if terminal.columns != plan.output_columns:
1992
+ raise GraphExecutionError(
1993
+ "OUTPUT_SCHEMA_MISMATCH", plan.terminal, "terminal columns differ from output_columns"
1994
+ )
1995
+ if len(terminal.rows) < plan.quality.min_rows:
1996
+ raise GraphExecutionError(
1997
+ "QUALITY_MIN_ROWS", plan.terminal, "terminal row count is too small"
1998
+ )
1999
+ terminal_index = _index(terminal, plan.terminal)
2000
+ for column in dict.fromkeys((*plan.grain, *plan.quality.not_null)):
2001
+ if any(row[terminal_index[column]] is None for row in terminal.rows):
2002
+ raise GraphExecutionError("QUALITY_NOT_NULL", plan.terminal, f"{column} contains null")
2003
+ return GraphExecution(
2004
+ terminal,
2005
+ tuple(audits),
2006
+ MappingProxyType(snapshots),
2007
+ peak_retained_bytes,
2008
+ )
2009
+
2010
+
2011
+ __all__ = [
2012
+ "ASCII_TRIM_CHARS",
2013
+ "HARD_MAX_FIELD_BYTES",
2014
+ "HARD_MAX_INTERMEDIATE_ROWS",
2015
+ "HARD_MAX_OUTPUT_ROWS",
2016
+ "HARD_MAX_RETAINED_BYTES",
2017
+ "HARD_MAX_SOURCE_ROWS",
2018
+ "ColumnLineage",
2019
+ "GraphExecution",
2020
+ "GraphExecutionError",
2021
+ "GraphResourcePolicy",
2022
+ "GraphTable",
2023
+ "NodeAudit",
2024
+ "execute_graph",
2025
+ "table_digest",
2026
+ ]