mostlyright-data 0.9.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (314) hide show
  1. mostlyright/data_harness/__init__.py +158 -0
  2. mostlyright/data_harness/acquisition/__init__.py +55 -0
  3. mostlyright/data_harness/acquisition/http.py +2773 -0
  4. mostlyright/data_harness/acquisition/parsing.py +809 -0
  5. mostlyright/data_harness/acquisition/ranges.py +495 -0
  6. mostlyright/data_harness/acquisition/result_download.py +360 -0
  7. mostlyright/data_harness/acquisition/retention_admission.py +248 -0
  8. mostlyright/data_harness/acquisition/sandbox.py +4888 -0
  9. mostlyright/data_harness/acquisition/url_policy.py +530 -0
  10. mostlyright/data_harness/agent_runtime.py +2743 -0
  11. mostlyright/data_harness/assets/logo-ink.svg +31 -0
  12. mostlyright/data_harness/backends/__init__.py +28 -0
  13. mostlyright/data_harness/backends/pandas_backend.py +350 -0
  14. mostlyright/data_harness/backends/polars_backend.py +366 -0
  15. mostlyright/data_harness/backends/protocol.py +124 -0
  16. mostlyright/data_harness/backends/reference.py +83 -0
  17. mostlyright/data_harness/backends/registry.py +55 -0
  18. mostlyright/data_harness/backends/restrictions.py +126 -0
  19. mostlyright/data_harness/canonical.py +333 -0
  20. mostlyright/data_harness/catalog_job.py +625 -0
  21. mostlyright/data_harness/cli.py +5398 -0
  22. mostlyright/data_harness/contracts.py +53 -0
  23. mostlyright/data_harness/coordinator.py +1307 -0
  24. mostlyright/data_harness/deploy.py +924 -0
  25. mostlyright/data_harness/deploy_target.py +312 -0
  26. mostlyright/data_harness/deployment_evidence.py +1067 -0
  27. mostlyright/data_harness/event_presentation.py +576 -0
  28. mostlyright/data_harness/events.py +2152 -0
  29. mostlyright/data_harness/fast_delimited.py +239 -0
  30. mostlyright/data_harness/fleet.py +237 -0
  31. mostlyright/data_harness/formats.py +236 -0
  32. mostlyright/data_harness/governors.py +1163 -0
  33. mostlyright/data_harness/hosted_bootstrap.py +972 -0
  34. mostlyright/data_harness/hosted_crawler.py +1115 -0
  35. mostlyright/data_harness/hosted_crawler_container_smoke.py +351 -0
  36. mostlyright/data_harness/hosted_crawler_fetch.py +423 -0
  37. mostlyright/data_harness/hosted_crawler_job.py +1277 -0
  38. mostlyright/data_harness/hosted_crawler_protocol.py +676 -0
  39. mostlyright/data_harness/hosted_dataset.py +1500 -0
  40. mostlyright/data_harness/hosted_deploy.py +3037 -0
  41. mostlyright/data_harness/hosted_handoff.py +62 -0
  42. mostlyright/data_harness/hosted_ingestion_contract.py +504 -0
  43. mostlyright/data_harness/hosted_ingestion_job.py +356 -0
  44. mostlyright/data_harness/hosted_ingestion_job_smoke.py +40 -0
  45. mostlyright/data_harness/hosted_session_container_smoke.py +194 -0
  46. mostlyright/data_harness/hosted_session_worker.py +3554 -0
  47. mostlyright/data_harness/hosted_session_worker_job_smoke.py +46 -0
  48. mostlyright/data_harness/hosted_worker.py +6784 -0
  49. mostlyright/data_harness/ingestion/__init__.py +56 -0
  50. mostlyright/data_harness/ingestion/contracts.py +461 -0
  51. mostlyright/data_harness/ingestion/faults.py +42 -0
  52. mostlyright/data_harness/ingestion/gcs_store.py +1162 -0
  53. mostlyright/data_harness/ingestion/spool.py +130 -0
  54. mostlyright/data_harness/ingestion/store.py +885 -0
  55. mostlyright/data_harness/key_seam.py +434 -0
  56. mostlyright/data_harness/linux_process_boundary.py +262 -0
  57. mostlyright/data_harness/local_contracts.py +2880 -0
  58. mostlyright/data_harness/local_search/__init__.py +5 -0
  59. mostlyright/data_harness/local_search/build_index.py +1087 -0
  60. mostlyright/data_harness/local_search/contracts.py +920 -0
  61. mostlyright/data_harness/local_search/query_trace.py +266 -0
  62. mostlyright/data_harness/local_search/retrieval.py +700 -0
  63. mostlyright/data_harness/local_search/sealed.py +474 -0
  64. mostlyright/data_harness/local_search/service.py +784 -0
  65. mostlyright/data_harness/nbrender/CONTRACT.md +212 -0
  66. mostlyright/data_harness/nbrender/__init__.py +12 -0
  67. mostlyright/data_harness/nbrender/chrome.py +359 -0
  68. mostlyright/data_harness/nbrender/code_body.py +266 -0
  69. mostlyright/data_harness/nbrender/document.py +407 -0
  70. mostlyright/data_harness/nbrender/frame.py +275 -0
  71. mostlyright/data_harness/nbrender/interactive.py +337 -0
  72. mostlyright/data_harness/nbrender/markdown_body.py +477 -0
  73. mostlyright/data_harness/nbrender/mr_components.py +134 -0
  74. mostlyright/data_harness/nbrender/outputs_data.py +595 -0
  75. mostlyright/data_harness/nbrender/outputs_rich.py +906 -0
  76. mostlyright/data_harness/nbrender/outputs_source.py +260 -0
  77. mostlyright/data_harness/nbrender/outputs_stage.py +176 -0
  78. mostlyright/data_harness/nbrender/outputs_text.py +400 -0
  79. mostlyright/data_harness/nbrender/parse.py +394 -0
  80. mostlyright/data_harness/nbrender/status.py +40 -0
  81. mostlyright/data_harness/nbrender/tokens.py +1295 -0
  82. mostlyright/data_harness/notebook.py +1710 -0
  83. mostlyright/data_harness/offline.py +2049 -0
  84. mostlyright/data_harness/operation_registry.py +1007 -0
  85. mostlyright/data_harness/operator_setup.py +239 -0
  86. mostlyright/data_harness/pipeline.py +6428 -0
  87. mostlyright/data_harness/plan_graph.py +2026 -0
  88. mostlyright/data_harness/preparation/__init__.py +104 -0
  89. mostlyright/data_harness/preparation/contracts.py +1017 -0
  90. mostlyright/data_harness/preparation/engine.py +221 -0
  91. mostlyright/data_harness/preparation/errors.py +14 -0
  92. mostlyright/data_harness/preparation/gates.py +751 -0
  93. mostlyright/data_harness/preparation/joins.py +574 -0
  94. mostlyright/data_harness/preparation/profile.py +384 -0
  95. mostlyright/data_harness/preparation/table.py +217 -0
  96. mostlyright/data_harness/preparation/transforms.py +568 -0
  97. mostlyright/data_harness/progress_events.py +534 -0
  98. mostlyright/data_harness/readers/__init__.py +46 -0
  99. mostlyright/data_harness/readers/containers.py +963 -0
  100. mostlyright/data_harness/readers/contracts.py +542 -0
  101. mostlyright/data_harness/readers/delimited.py +257 -0
  102. mostlyright/data_harness/readers/grib2/__init__.py +33 -0
  103. mostlyright/data_harness/readers/grib2/admission.py +722 -0
  104. mostlyright/data_harness/readers/grib2/decode.py +1009 -0
  105. mostlyright/data_harness/readers/grib2/geometry.py +1133 -0
  106. mostlyright/data_harness/readers/grib2/portable_math.py +501 -0
  107. mostlyright/data_harness/readers/json_tabular.py +485 -0
  108. mostlyright/data_harness/readers/registry.py +514 -0
  109. mostlyright/data_harness/readers/samples/README.md +110 -0
  110. mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/cities.csv.gz +0 -0
  111. mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/expected.json +24 -0
  112. mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/cities.csv.gz +0 -0
  113. mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/expected.json +24 -0
  114. mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/cities.tar +0 -0
  115. mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/expected.json +24 -0
  116. mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/cities.tar +0 -0
  117. mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/expected.json +24 -0
  118. mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/cities.zip +0 -0
  119. mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/expected.json +25 -0
  120. mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
  121. mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/expected.json +25 -0
  122. mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
  123. mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/expected.json +25 -0
  124. mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/cities.csv +3 -0
  125. mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/expected.json +23 -0
  126. mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/cities.tsv +5 -0
  127. mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/expected.json +25 -0
  128. mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/expected.json +30 -0
  129. mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/observations.csv +5 -0
  130. mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/expected.json +44 -0
  131. mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/stations.json +1 -0
  132. mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/expected.json +48 -0
  133. mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/observations.ndjson +4 -0
  134. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/cities.xlsx +0 -0
  135. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/expected.json +24 -0
  136. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/expected.json +27 -0
  137. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
  138. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/expected.json +27 -0
  139. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
  140. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/README.md +20 -0
  141. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/expected.json +55 -0
  142. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/gfs-2m-temperature.grib2 +0 -0
  143. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/expected.json +54 -0
  144. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
  145. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/expected.json +54 -0
  146. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/hrrr-categorical-rain.grib2 +0 -0
  147. mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/expected.json +54 -0
  148. mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
  149. mostlyright/data_harness/readers/samples.py +582 -0
  150. mostlyright/data_harness/readers/spreadsheet.py +803 -0
  151. mostlyright/data_harness/readers/tabular.py +510 -0
  152. mostlyright/data_harness/recipe.py +5321 -0
  153. mostlyright/data_harness/repair/__init__.py +78 -0
  154. mostlyright/data_harness/repair/adapters.py +274 -0
  155. mostlyright/data_harness/repair/contracts.py +872 -0
  156. mostlyright/data_harness/repair/coordinator.py +1099 -0
  157. mostlyright/data_harness/repair/errors.py +16 -0
  158. mostlyright/data_harness/review.py +2533 -0
  159. mostlyright/data_harness/rowset.py +283 -0
  160. mostlyright/data_harness/serving.py +1975 -0
  161. mostlyright/data_harness/serving_edge.py +590 -0
  162. mostlyright/data_harness/serving_http.py +1031 -0
  163. mostlyright/data_harness/session_probes.py +759 -0
  164. mostlyright/data_harness/signing.py +101 -0
  165. mostlyright/data_harness/source_discovery.py +898 -0
  166. mostlyright/data_harness/sources/__init__.py +209 -0
  167. mostlyright/data_harness/sources/_adapter_steps.py +213 -0
  168. mostlyright/data_harness/sources/adapters.py +1214 -0
  169. mostlyright/data_harness/sources/cadence.py +1428 -0
  170. mostlyright/data_harness/sources/cadence_emission.py +453 -0
  171. mostlyright/data_harness/sources/cadence_history.py +546 -0
  172. mostlyright/data_harness/sources/catalog/__init__.py +17 -0
  173. mostlyright/data_harness/sources/catalog/admission.py +477 -0
  174. mostlyright/data_harness/sources/catalog/authoring.py +1701 -0
  175. mostlyright/data_harness/sources/catalog/authoring_policy.py +701 -0
  176. mostlyright/data_harness/sources/catalog/authoring_shards.py +1217 -0
  177. mostlyright/data_harness/sources/catalog/bounded_io.py +231 -0
  178. mostlyright/data_harness/sources/catalog/channel.py +523 -0
  179. mostlyright/data_harness/sources/catalog/channel_client.py +296 -0
  180. mostlyright/data_harness/sources/catalog/contracts.py +825 -0
  181. mostlyright/data_harness/sources/catalog/coverage.py +137 -0
  182. mostlyright/data_harness/sources/catalog/delta.py +1340 -0
  183. mostlyright/data_harness/sources/catalog/embedding.py +532 -0
  184. mostlyright/data_harness/sources/catalog/entry_v2.py +1182 -0
  185. mostlyright/data_harness/sources/catalog/fill.py +3889 -0
  186. mostlyright/data_harness/sources/catalog/fill_partitions.py +459 -0
  187. mostlyright/data_harness/sources/catalog/fill_staging.py +1105 -0
  188. mostlyright/data_harness/sources/catalog/gating.py +374 -0
  189. mostlyright/data_harness/sources/catalog/generation_receipt.py +1607 -0
  190. mostlyright/data_harness/sources/catalog/harvest/__init__.py +7 -0
  191. mostlyright/data_harness/sources/catalog/harvest/ckan.py +384 -0
  192. mostlyright/data_harness/sources/catalog/harvest/datagov_v4.py +798 -0
  193. mostlyright/data_harness/sources/catalog/harvest/protocol.py +964 -0
  194. mostlyright/data_harness/sources/catalog/harvest/sdmx.py +445 -0
  195. mostlyright/data_harness/sources/catalog/harvest/stac.py +384 -0
  196. mostlyright/data_harness/sources/catalog/health.py +447 -0
  197. mostlyright/data_harness/sources/catalog/hosted_catalog.py +105 -0
  198. mostlyright/data_harness/sources/catalog/identity_history.py +1549 -0
  199. mostlyright/data_harness/sources/catalog/neural.py +1618 -0
  200. mostlyright/data_harness/sources/catalog/packed_catalog.py +2345 -0
  201. mostlyright/data_harness/sources/catalog/packed_retrieval.py +1517 -0
  202. mostlyright/data_harness/sources/catalog/packed_writer.py +2802 -0
  203. mostlyright/data_harness/sources/catalog/query_trace.py +1037 -0
  204. mostlyright/data_harness/sources/catalog/recommend.py +171 -0
  205. mostlyright/data_harness/sources/catalog/retrieval.py +230 -0
  206. mostlyright/data_harness/sources/catalog/retrieval_manifest.py +995 -0
  207. mostlyright/data_harness/sources/catalog/rights_decisions.py +254 -0
  208. mostlyright/data_harness/sources/catalog/sealed.py +560 -0
  209. mostlyright/data_harness/sources/catalog/search.py +230 -0
  210. mostlyright/data_harness/sources/catalog/streaming_delta.py +1097 -0
  211. mostlyright/data_harness/sources/catalog/update.py +891 -0
  212. mostlyright/data_harness/sources/collections.py +815 -0
  213. mostlyright/data_harness/sources/contracts.py +2223 -0
  214. mostlyright/data_harness/sources/deletion.py +761 -0
  215. mostlyright/data_harness/sources/fitness.py +162 -0
  216. mostlyright/data_harness/sources/governance.py +163 -0
  217. mostlyright/data_harness/sources/hosted.py +173 -0
  218. mostlyright/data_harness/sources/integration.py +218 -0
  219. mostlyright/data_harness/sources/range_reader.py +418 -0
  220. mostlyright/data_harness/sources/registry.py +514 -0
  221. mostlyright/data_harness/sources/rights_rule.py +59 -0
  222. mostlyright/data_harness/sources/source_cadence_vectors.v1.json +1 -0
  223. mostlyright/data_harness/sources/sports.py +521 -0
  224. mostlyright/data_harness/sources/stream.py +524 -0
  225. mostlyright/data_harness/sources/stream_connector.py +418 -0
  226. mostlyright/data_harness/sources/stream_recorder.py +1404 -0
  227. mostlyright/data_harness/studio_boundary.py +2019 -0
  228. mostlyright/data_harness/thin/__init__.py +37 -0
  229. mostlyright/data_harness/thin/acquire.py +1137 -0
  230. mostlyright/data_harness/thin/acquire_cancel.py +579 -0
  231. mostlyright/data_harness/thin/approvals.py +617 -0
  232. mostlyright/data_harness/thin/commands.py +406 -0
  233. mostlyright/data_harness/thin/download.py +194 -0
  234. mostlyright/data_harness/thin/narrative.py +589 -0
  235. mostlyright/data_harness/thin/parity.py +1070 -0
  236. mostlyright/data_harness/thin/propose.py +2759 -0
  237. mostlyright/data_harness/thin/research.py +1663 -0
  238. mostlyright/data_harness/thin/router.py +924 -0
  239. mostlyright/data_harness/thin/runs.py +519 -0
  240. mostlyright/data_harness/thin/session.py +281 -0
  241. mostlyright/data_harness/thin/stream.py +501 -0
  242. mostlyright/data_harness/thin/transport.py +187 -0
  243. mostlyright/data_harness/thin/vocabulary.py +368 -0
  244. mostlyright/data_harness/thin/workers.py +164 -0
  245. mostlyright/data_harness/ucum/TABLE-PIN.json +40 -0
  246. mostlyright/data_harness/ucum/ucum-subset.v1.json +632 -0
  247. mostlyright/data_harness/unit_flow.py +927 -0
  248. mostlyright/data_harness/units.py +572 -0
  249. mostlyright/data_harness/ux/__init__.py +9 -0
  250. mostlyright/data_harness/ux/approve.py +485 -0
  251. mostlyright/data_harness/ux/author_yaml.py +597 -0
  252. mostlyright/data_harness/ux/cloud_auth.py +447 -0
  253. mostlyright/data_harness/ux/commands/__init__.py +260 -0
  254. mostlyright/data_harness/ux/commands/approve.py +136 -0
  255. mostlyright/data_harness/ux/commands/auth.py +744 -0
  256. mostlyright/data_harness/ux/commands/author.py +79 -0
  257. mostlyright/data_harness/ux/commands/catalog_author.py +403 -0
  258. mostlyright/data_harness/ux/commands/catalog_fill.py +523 -0
  259. mostlyright/data_harness/ux/commands/catalog_harvest.py +545 -0
  260. mostlyright/data_harness/ux/commands/catalog_publish.py +1838 -0
  261. mostlyright/data_harness/ux/commands/catalog_search.py +71 -0
  262. mostlyright/data_harness/ux/commands/catalog_update.py +437 -0
  263. mostlyright/data_harness/ux/commands/deploy.py +134 -0
  264. mostlyright/data_harness/ux/commands/deploy_dataset.py +98 -0
  265. mostlyright/data_harness/ux/commands/deploy_plan.py +105 -0
  266. mostlyright/data_harness/ux/commands/deploy_status.py +104 -0
  267. mostlyright/data_harness/ux/commands/diff.py +74 -0
  268. mostlyright/data_harness/ux/commands/index.py +84 -0
  269. mostlyright/data_harness/ux/commands/inventory.py +47 -0
  270. mostlyright/data_harness/ux/commands/list_builds.py +143 -0
  271. mostlyright/data_harness/ux/commands/login.py +63 -0
  272. mostlyright/data_harness/ux/commands/peek.py +236 -0
  273. mostlyright/data_harness/ux/commands/plan_check.py +90 -0
  274. mostlyright/data_harness/ux/commands/preflight.py +97 -0
  275. mostlyright/data_harness/ux/commands/record.py +107 -0
  276. mostlyright/data_harness/ux/commands/review_setup.py +47 -0
  277. mostlyright/data_harness/ux/commands/search.py +440 -0
  278. mostlyright/data_harness/ux/commands/show.py +61 -0
  279. mostlyright/data_harness/ux/commands/whoami.py +37 -0
  280. mostlyright/data_harness/ux/credential_native.py +551 -0
  281. mostlyright/data_harness/ux/credential_store.py +1055 -0
  282. mostlyright/data_harness/ux/credentials.py +631 -0
  283. mostlyright/data_harness/ux/diffing.py +444 -0
  284. mostlyright/data_harness/ux/headline.py +671 -0
  285. mostlyright/data_harness/ux/hosted_acquisition.py +974 -0
  286. mostlyright/data_harness/ux/hosted_run_status.py +619 -0
  287. mostlyright/data_harness/ux/inventory.py +427 -0
  288. mostlyright/data_harness/ux/local_review.py +375 -0
  289. mostlyright/data_harness/ux/login.py +691 -0
  290. mostlyright/data_harness/ux/path_kind.py +147 -0
  291. mostlyright/data_harness/ux/peek.py +1000 -0
  292. mostlyright/data_harness/ux/plain_file.py +178 -0
  293. mostlyright/data_harness/ux/plan_check.py +311 -0
  294. mostlyright/data_harness/ux/preflight.py +918 -0
  295. mostlyright/data_harness/ux/readers.py +1124 -0
  296. mostlyright/data_harness/ux/remediation.py +2195 -0
  297. mostlyright/data_harness/ux/render.py +657 -0
  298. mostlyright/data_harness/ux/workload.py +1077 -0
  299. mostlyright/data_harness/viewer.py +3713 -0
  300. mostlyright/data_harness/visual_run/__init__.py +83 -0
  301. mostlyright/data_harness/visual_run/authoring.py +235 -0
  302. mostlyright/data_harness/visual_run/contracts.py +673 -0
  303. mostlyright/data_harness/visual_run/materialize.py +486 -0
  304. mostlyright/data_harness/visual_run/observations.py +874 -0
  305. mostlyright/data_harness/visual_run/query.py +259 -0
  306. mostlyright/data_harness/visual_run/reducer.py +280 -0
  307. mostlyright/data_harness/visual_run/sdk.py +892 -0
  308. mostlyright/data_harness/visual_run/store.py +584 -0
  309. mostlyright/data_harness/visual_run/transport.py +239 -0
  310. mostlyright/data_harness/watch.py +2999 -0
  311. mostlyright_data-0.9.0.dist-info/METADATA +607 -0
  312. mostlyright_data-0.9.0.dist-info/RECORD +314 -0
  313. mostlyright_data-0.9.0.dist-info/WHEEL +4 -0
  314. mostlyright_data-0.9.0.dist-info/entry_points.txt +12 -0
@@ -0,0 +1,751 @@
1
+ """Deterministic temporal, target, and analysis-readiness quality gates."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from dataclasses import dataclass, replace
6
+ from datetime import date, datetime, timedelta
7
+ from itertools import pairwise
8
+ from typing import Any
9
+
10
+ from mostlyright.data_harness.canonical import canonical_sha256
11
+ from mostlyright.data_harness.preparation.contracts import (
12
+ QUALITY_CHECK_POLICY,
13
+ QUALITY_REPORT_VERSION,
14
+ TARGET_RATIONALE_VERSION,
15
+ TEMPORAL_REPORT_VERSION,
16
+ ColumnLineage,
17
+ QualitySpec,
18
+ SourceTemporalEvidence,
19
+ TargetSpec,
20
+ TemporalPolicy,
21
+ parse_utc,
22
+ )
23
+ from mostlyright.data_harness.preparation.errors import PreparationError
24
+ from mostlyright.data_harness.preparation.joins import JoinAudit
25
+ from mostlyright.data_harness.preparation.profile import TableProfile, profile_table
26
+ from mostlyright.data_harness.preparation.table import TableState, validate_columns
27
+
28
+ _CHECK_POLICY = QUALITY_CHECK_POLICY
29
+
30
+
31
+ @dataclass(frozen=True)
32
+ class TemporalFinding:
33
+ code: str
34
+ source_id: str
35
+ column: str | None
36
+ row_index: int | None
37
+ detail: str
38
+
39
+ def __post_init__(self) -> None:
40
+ if not isinstance(self.code, str) or not self.code.startswith("TEMPORAL_"):
41
+ raise PreparationError("TEMPORAL_FINDING", "temporal.finding.code", "is invalid")
42
+ if not isinstance(self.source_id, str) or not self.source_id:
43
+ raise PreparationError("TEMPORAL_FINDING", "temporal.finding.source_id", "is invalid")
44
+ if self.row_index is not None and (type(self.row_index) is not int or self.row_index < 0):
45
+ raise PreparationError(
46
+ "TEMPORAL_FINDING",
47
+ "temporal.finding.row_index",
48
+ "must be a nonnegative integer",
49
+ )
50
+ if not isinstance(self.detail, str) or not self.detail:
51
+ raise PreparationError("TEMPORAL_FINDING", "temporal.finding.detail", "is invalid")
52
+
53
+ def to_dict(self) -> dict[str, Any]:
54
+ return {
55
+ "code": self.code,
56
+ "severity": "blocker",
57
+ "source_id": self.source_id,
58
+ "column": self.column,
59
+ "row_index": self.row_index,
60
+ "detail": self.detail,
61
+ }
62
+
63
+
64
+ @dataclass(frozen=True)
65
+ class TemporalReport:
66
+ schema_version: str
67
+ analysis_kind: str
68
+ question_cutoff: str
69
+ checked_sources: tuple[str, ...]
70
+ checked_cells: int
71
+ findings: tuple[TemporalFinding, ...]
72
+ status: str
73
+
74
+ def __post_init__(self) -> None:
75
+ if self.schema_version != TEMPORAL_REPORT_VERSION:
76
+ raise PreparationError(
77
+ "TEMPORAL_REPORT_VERSION",
78
+ "temporal.schema_version",
79
+ "unsupported",
80
+ )
81
+ if self.status not in {"passed", "failed"}:
82
+ raise PreparationError("TEMPORAL_REPORT_STATUS", "temporal.status", "unsupported")
83
+ if (self.status == "passed") != (not self.findings):
84
+ raise PreparationError(
85
+ "TEMPORAL_REPORT_STATUS",
86
+ "temporal.status",
87
+ "does not reconcile",
88
+ )
89
+
90
+ @property
91
+ def digest(self) -> str:
92
+ return canonical_sha256(self.to_dict())
93
+
94
+ def to_dict(self) -> dict[str, Any]:
95
+ return {
96
+ "schema_version": self.schema_version,
97
+ "analysis_kind": self.analysis_kind,
98
+ "question_cutoff": self.question_cutoff,
99
+ "checked_sources": list(self.checked_sources),
100
+ "checked_cells": self.checked_cells,
101
+ "findings": [item.to_dict() for item in self.findings],
102
+ "status": self.status,
103
+ }
104
+
105
+
106
+ @dataclass(frozen=True)
107
+ class TargetRationale:
108
+ schema_version: str
109
+ output_column: str
110
+ operation: str
111
+ source_column: str
112
+ source_snapshot_digest: str
113
+ resolution_rule: str
114
+ horizon_seconds: int
115
+ availability_time_column: str
116
+ cutoff_column: str
117
+ mapping_digest: str | None
118
+ rationale: str
119
+ verified_rows: int
120
+
121
+ def __post_init__(self) -> None:
122
+ if self.schema_version != TARGET_RATIONALE_VERSION:
123
+ raise PreparationError(
124
+ "TARGET_RATIONALE_VERSION", "target.schema_version", "unsupported"
125
+ )
126
+ if type(self.horizon_seconds) is not int or self.horizon_seconds < 0:
127
+ raise PreparationError("TARGET_RATIONALE", "target.horizon_seconds", "is invalid")
128
+ if type(self.verified_rows) is not int or self.verified_rows < 0:
129
+ raise PreparationError("TARGET_RATIONALE", "target.verified_rows", "is invalid")
130
+ if not _is_digest(self.source_snapshot_digest):
131
+ raise PreparationError(
132
+ "TARGET_RATIONALE",
133
+ "target.source_snapshot_digest",
134
+ "is invalid",
135
+ )
136
+ if self.mapping_digest is not None and not _is_digest(self.mapping_digest):
137
+ raise PreparationError("TARGET_RATIONALE", "target.mapping_digest", "is invalid")
138
+
139
+ @property
140
+ def digest(self) -> str:
141
+ return canonical_sha256(self.to_dict())
142
+
143
+ def to_dict(self) -> dict[str, Any]:
144
+ return {
145
+ "schema_version": self.schema_version,
146
+ "output_column": self.output_column,
147
+ "operation": self.operation,
148
+ "source_column": self.source_column,
149
+ "source_snapshot_digest": self.source_snapshot_digest,
150
+ "resolution_rule": self.resolution_rule,
151
+ "horizon_seconds": self.horizon_seconds,
152
+ "availability_time_column": self.availability_time_column,
153
+ "cutoff_column": self.cutoff_column,
154
+ "mapping_digest": self.mapping_digest,
155
+ "rationale": self.rationale,
156
+ "verified_rows": self.verified_rows,
157
+ }
158
+
159
+
160
+ @dataclass(frozen=True)
161
+ class QualityCheck:
162
+ check_id: str
163
+ status: str
164
+ severity: str
165
+ threshold_id: str
166
+ observed: tuple[tuple[str, Any], ...]
167
+
168
+ def __post_init__(self) -> None:
169
+ expected = _CHECK_POLICY.get(self.check_id)
170
+ if expected != (self.severity, self.threshold_id):
171
+ raise PreparationError(
172
+ "QUALITY_POLICY",
173
+ f"quality.{self.check_id}",
174
+ "severity or threshold identity is not policy-owned",
175
+ )
176
+ if self.status not in {"passed", "failed"}:
177
+ raise PreparationError("QUALITY_STATUS", f"quality.{self.check_id}", "is unsupported")
178
+ names = tuple(name for name, _ in self.observed)
179
+ if names != tuple(sorted(set(names))):
180
+ raise PreparationError(
181
+ "QUALITY_EVIDENCE",
182
+ f"quality.{self.check_id}",
183
+ "observed evidence must be unique and canonically ordered",
184
+ )
185
+ canonical_sha256({name: value for name, value in self.observed})
186
+
187
+ def to_dict(self) -> dict[str, Any]:
188
+ return {
189
+ "check_id": self.check_id,
190
+ "status": self.status,
191
+ "severity": self.severity,
192
+ "threshold_id": self.threshold_id,
193
+ "observed": {name: value for name, value in self.observed},
194
+ }
195
+
196
+
197
+ @dataclass(frozen=True)
198
+ class QualityReport:
199
+ schema_version: str
200
+ checks: tuple[QualityCheck, ...]
201
+ status: str
202
+
203
+ def __post_init__(self) -> None:
204
+ if self.schema_version != QUALITY_REPORT_VERSION:
205
+ raise PreparationError("QUALITY_VERSION", "quality.schema_version", "is unsupported")
206
+ if tuple(item.check_id for item in self.checks) != tuple(_CHECK_POLICY):
207
+ raise PreparationError(
208
+ "QUALITY_CHECK_SET",
209
+ "quality.checks",
210
+ "check set/order is not exact",
211
+ )
212
+ expected = "failed" if any(item.status == "failed" for item in self.checks) else "passed"
213
+ if self.status != expected:
214
+ raise PreparationError("QUALITY_STATUS", "quality.status", "does not reconcile checks")
215
+
216
+ @property
217
+ def digest(self) -> str:
218
+ return canonical_sha256(self.to_dict())
219
+
220
+ @property
221
+ def findings(self) -> tuple[QualityCheck, ...]:
222
+ return tuple(item for item in self.checks if item.status == "failed")
223
+
224
+ def to_dict(self) -> dict[str, Any]:
225
+ return {
226
+ "schema_version": self.schema_version,
227
+ "checks": [item.to_dict() for item in self.checks],
228
+ "status": self.status,
229
+ }
230
+
231
+
232
+ def validate_temporal_safety(
233
+ state: TableState,
234
+ *,
235
+ evidence: tuple[SourceTemporalEvidence, ...],
236
+ policy: TemporalPolicy,
237
+ ) -> TemporalReport:
238
+ """Audit fidelity, availability, revisions, and row/question cutoffs."""
239
+
240
+ evidence_by_source = {item.source_id: item for item in evidence}
241
+ if len(evidence_by_source) != len(evidence):
242
+ raise PreparationError("TEMPORAL_EVIDENCE", "temporal.evidence", "duplicates source IDs")
243
+ sources = {item.source_id for lineage in state.lineage for item in lineage.inputs}
244
+ if set(evidence_by_source) != sources:
245
+ raise PreparationError(
246
+ "TEMPORAL_EVIDENCE",
247
+ "temporal.evidence",
248
+ "does not exactly cover output lineage sources",
249
+ )
250
+ for item in evidence:
251
+ if item.snapshot_digest not in state.snapshot_digests:
252
+ raise PreparationError(
253
+ "TEMPORAL_EVIDENCE",
254
+ item.source_id,
255
+ "snapshot digest is not bound to the table",
256
+ )
257
+ _validate_temporal_column(
258
+ state,
259
+ item.source_id,
260
+ item.available_at_column,
261
+ "available_at",
262
+ )
263
+ if item.event_time_column is not None:
264
+ _validate_temporal_column(
265
+ state,
266
+ item.source_id,
267
+ item.event_time_column,
268
+ "event_time",
269
+ )
270
+ if item.revision_time_column is not None:
271
+ _validate_temporal_column(
272
+ state,
273
+ item.source_id,
274
+ item.revision_time_column,
275
+ "revision_time",
276
+ )
277
+ question_cutoff = parse_utc(policy.question_cutoff, "temporal.question_cutoff")
278
+ row_cutoff_index = (
279
+ state.index(policy.row_cutoff_column) if policy.row_cutoff_column is not None else None
280
+ )
281
+ findings: list[TemporalFinding] = []
282
+ checked_cells = 0
283
+ for source_id in sorted(sources):
284
+ source = evidence_by_source[source_id]
285
+ retrieval = parse_utc(source.retrieval_time, f"temporal.{source_id}.retrieval_time")
286
+ if source.fidelity == "unknown":
287
+ findings.append(
288
+ TemporalFinding(
289
+ code="TEMPORAL_FIDELITY_UNKNOWN",
290
+ source_id=source_id,
291
+ column=None,
292
+ row_index=None,
293
+ detail="unknown fidelity cannot support analysis readiness",
294
+ )
295
+ )
296
+ elif source.fidelity == "latest_only" and policy.analysis_kind == "point_in_time":
297
+ findings.append(
298
+ TemporalFinding(
299
+ code="TEMPORAL_LATEST_ONLY",
300
+ source_id=source_id,
301
+ column=None,
302
+ row_index=None,
303
+ detail="latest-only evidence cannot support point-in-time analysis",
304
+ )
305
+ )
306
+ elif source.fidelity in {"reconstructed", "latest_only"} and (
307
+ policy.analysis_kind != "current_retrospective" or retrieval > question_cutoff
308
+ ):
309
+ findings.append(
310
+ TemporalFinding(
311
+ code="TEMPORAL_RECONSTRUCTED_CUTOFF",
312
+ source_id=source_id,
313
+ column=None,
314
+ row_index=None,
315
+ detail="reconstructed/latest evidence is unavailable by the declared cutoff",
316
+ )
317
+ )
318
+ for field_index, (field, lineage) in enumerate(zip(state.fields, state.lineage, strict=True)):
319
+ if field.semantic_type == "target" or all(
320
+ item.source_id not in sources for item in lineage.inputs
321
+ ):
322
+ continue
323
+ if field.temporal_role != "none":
324
+ continue
325
+ if field.availability_column is None:
326
+ findings.append(
327
+ TemporalFinding(
328
+ code="TEMPORAL_AVAILABILITY_MISSING",
329
+ source_id=field.source_id,
330
+ column=field.name,
331
+ row_index=None,
332
+ detail="selected field has no availability column",
333
+ )
334
+ )
335
+ continue
336
+ availability_index = state.index(field.availability_column)
337
+ availability_field = state.fields[availability_index]
338
+ source_evidence = evidence_by_source[field.source_id]
339
+ if (
340
+ availability_field.source_id != field.source_id
341
+ or availability_field.temporal_role != "available_at"
342
+ or field.availability_column != source_evidence.available_at_column
343
+ ):
344
+ findings.append(
345
+ TemporalFinding(
346
+ code="TEMPORAL_AVAILABILITY_MISMATCH",
347
+ source_id=field.source_id,
348
+ column=field.name,
349
+ row_index=None,
350
+ detail="availability semantics are not bound to the same source evidence",
351
+ )
352
+ )
353
+ continue
354
+ for row_index, row in enumerate(state.rows):
355
+ if row[field_index] is None:
356
+ continue
357
+ checked_cells += 1
358
+ cutoff = (
359
+ _instant(row[row_cutoff_index], "temporal.row_cutoff")
360
+ if row_cutoff_index is not None
361
+ else question_cutoff
362
+ )
363
+ if cutoff > question_cutoff:
364
+ findings.append(
365
+ TemporalFinding(
366
+ code="TEMPORAL_ROW_CUTOFF",
367
+ source_id=field.source_id,
368
+ column=field.name,
369
+ row_index=row_index,
370
+ detail="row cutoff exceeds the question cutoff",
371
+ )
372
+ )
373
+ continue
374
+ available = _instant(row[availability_index], "temporal.available_at")
375
+ if available > cutoff:
376
+ findings.append(
377
+ TemporalFinding(
378
+ code="TEMPORAL_LEAKAGE",
379
+ source_id=field.source_id,
380
+ column=field.name,
381
+ row_index=row_index,
382
+ detail="selected value became available after its cutoff",
383
+ )
384
+ )
385
+ for source in evidence:
386
+ if source.revision_time_column is None:
387
+ continue
388
+ try:
389
+ revision_index = state.index(source.revision_time_column)
390
+ except PreparationError:
391
+ continue
392
+ for row_index, row in enumerate(state.rows):
393
+ if row[revision_index] is None:
394
+ continue
395
+ cutoff = (
396
+ _instant(row[row_cutoff_index], "temporal.row_cutoff")
397
+ if row_cutoff_index is not None
398
+ else question_cutoff
399
+ )
400
+ if _instant(row[revision_index], "temporal.revision_time") > cutoff:
401
+ findings.append(
402
+ TemporalFinding(
403
+ code="TEMPORAL_FUTURE_REVISION",
404
+ source_id=source.source_id,
405
+ column=source.revision_time_column,
406
+ row_index=row_index,
407
+ detail="selected revision is later than its cutoff",
408
+ )
409
+ )
410
+ return TemporalReport(
411
+ schema_version=TEMPORAL_REPORT_VERSION,
412
+ analysis_kind=policy.analysis_kind,
413
+ question_cutoff=policy.question_cutoff,
414
+ checked_sources=tuple(sorted(sources)),
415
+ checked_cells=checked_cells,
416
+ findings=tuple(findings),
417
+ status="failed" if findings else "passed",
418
+ )
419
+
420
+
421
+ def construct_target(
422
+ state: TableState,
423
+ spec: TargetSpec | None,
424
+ *,
425
+ target_policy: str,
426
+ ) -> tuple[TableState, TargetRationale | None]:
427
+ """Construct only a justified, source-derived, cutoff-safe target."""
428
+
429
+ if target_policy == "none":
430
+ if spec is not None:
431
+ raise PreparationError("TARGET_POLICY", "target", "target is unjustified")
432
+ return state, None
433
+ if target_policy != "justified" or spec is None:
434
+ raise PreparationError("TARGET_POLICY", "target", "justified target evidence is required")
435
+ if spec.output_column in state.columns:
436
+ raise PreparationError("TARGET_COLUMN", "target.output_column", "already exists")
437
+ source_index = state.index(spec.source_column)
438
+ availability_index = state.index(spec.availability_time_column)
439
+ cutoff_index = state.index(spec.cutoff_column)
440
+ source_field = state.fields[source_index]
441
+ source_lineage = state.lineage[source_index]
442
+ if source_field.semantic_type == "target":
443
+ raise PreparationError(
444
+ "TARGET_ISOLATION",
445
+ "target.source_column",
446
+ "cannot derive from a target",
447
+ )
448
+ if spec.source_snapshot_digest not in {item.snapshot_digest for item in source_lineage.inputs}:
449
+ raise PreparationError(
450
+ "TARGET_SOURCE",
451
+ "target.source_snapshot_digest",
452
+ "does not bind the source column lineage",
453
+ )
454
+ mapping = {item.input_value: item.output_value for item in spec.mapping}
455
+ values: list[Any] = []
456
+ for row_index, row in enumerate(state.rows):
457
+ available = _instant(row[availability_index], "target.available_at")
458
+ cutoff = _instant(row[cutoff_index], "target.cutoff")
459
+ if available > cutoff:
460
+ raise PreparationError(
461
+ "TARGET_LEAKAGE",
462
+ f"target.row[{row_index}]",
463
+ "target resolution became available after the cutoff",
464
+ )
465
+ source_value = row[source_index]
466
+ if source_value is None:
467
+ if not spec.allow_null:
468
+ raise PreparationError(
469
+ "TARGET_NULL",
470
+ f"target.row[{row_index}]",
471
+ "null target source is forbidden",
472
+ )
473
+ values.append(None)
474
+ elif spec.operation == "exact_copy":
475
+ values.append(source_value)
476
+ else:
477
+ if not isinstance(source_value, str):
478
+ raise PreparationError(
479
+ "TARGET_MAPPING",
480
+ f"target.row[{row_index}]",
481
+ "categorical mapping requires string inputs",
482
+ )
483
+ try:
484
+ values.append(mapping[source_value])
485
+ except KeyError:
486
+ raise PreparationError(
487
+ "TARGET_MAPPING",
488
+ f"target.row[{row_index}]",
489
+ "categorical mapping is not total",
490
+ ) from None
491
+ output_field = replace(
492
+ source_field,
493
+ name=spec.output_column,
494
+ semantic_type="target",
495
+ nullable=spec.allow_null,
496
+ availability_column=spec.availability_time_column,
497
+ )
498
+ rows = tuple((*row, value) for row, value in zip(state.rows, values, strict=True))
499
+ operation_id = f"target.{spec.output_column}"
500
+ lineage = (
501
+ *state.lineage,
502
+ ColumnLineage(
503
+ output_column=spec.output_column,
504
+ inputs=source_lineage.inputs,
505
+ operation_ids=(*source_lineage.operation_ids, operation_id),
506
+ ),
507
+ )
508
+ output = state.replace(
509
+ fields=(*state.fields, output_field),
510
+ rows=rows,
511
+ lineage=lineage,
512
+ )
513
+ rationale = TargetRationale(
514
+ schema_version=TARGET_RATIONALE_VERSION,
515
+ output_column=spec.output_column,
516
+ operation=spec.operation,
517
+ source_column=spec.source_column,
518
+ source_snapshot_digest=spec.source_snapshot_digest,
519
+ resolution_rule=spec.resolution_rule,
520
+ horizon_seconds=spec.horizon_seconds,
521
+ availability_time_column=spec.availability_time_column,
522
+ cutoff_column=spec.cutoff_column,
523
+ mapping_digest=(
524
+ canonical_sha256([item.to_dict() for item in spec.mapping]) if spec.mapping else None
525
+ ),
526
+ rationale=spec.rationale,
527
+ verified_rows=len(state.rows),
528
+ )
529
+ return output, rationale
530
+
531
+
532
+ def validate_quality(
533
+ state: TableState,
534
+ *,
535
+ spec: QualitySpec,
536
+ joins: tuple[JoinAudit, ...],
537
+ temporal: TemporalReport,
538
+ ) -> tuple[TableProfile, QualityReport]:
539
+ """Run the complete fixed-severity analysis-readiness gate set."""
540
+
541
+ profile = profile_table(state, key_columns=spec.grain)
542
+ expected_schema = [field.to_dict() for field in spec.expected_fields]
543
+ actual_schema = [field.to_dict() for field in state.fields]
544
+ checks: list[QualityCheck] = []
545
+ checks.append(
546
+ _check(
547
+ "schema",
548
+ expected_schema == actual_schema,
549
+ (
550
+ ("actual_schema_digest", canonical_sha256(actual_schema)),
551
+ ("expected_schema_digest", canonical_sha256(expected_schema)),
552
+ ),
553
+ )
554
+ )
555
+ row_bounds_pass = spec.minimum_rows <= len(state.rows) <= spec.maximum_rows
556
+ checks.append(
557
+ _check(
558
+ "row_bounds",
559
+ row_bounds_pass,
560
+ (
561
+ ("maximum_rows", spec.maximum_rows),
562
+ ("minimum_rows", spec.minimum_rows),
563
+ ("observed_rows", len(state.rows)),
564
+ ),
565
+ )
566
+ )
567
+ field_profile = {item.name: item for item in profile.fields}
568
+ missing_failures = tuple(
569
+ (item.column, field_profile[item.column].null_count, item.maximum_nulls)
570
+ for item in spec.missingness
571
+ if field_profile[item.column].null_count > item.maximum_nulls
572
+ )
573
+ checks.append(
574
+ _check(
575
+ "missingness",
576
+ not missing_failures,
577
+ (
578
+ ("failed_columns", [list(item) for item in missing_failures]),
579
+ ("threshold_count", len(spec.missingness)),
580
+ ),
581
+ )
582
+ )
583
+ grain_indexes = validate_columns(state, spec.grain, "quality.grain")
584
+ null_grain = sum(any(row[index] is None for index in grain_indexes) for row in state.rows)
585
+ checks.append(
586
+ _check(
587
+ "grain",
588
+ null_grain == 0 and profile.duplicate_key_count == 0,
589
+ (
590
+ ("duplicate_grain_rows", profile.duplicate_key_count),
591
+ ("null_grain_rows", null_grain),
592
+ ),
593
+ )
594
+ )
595
+ checks.append(
596
+ _check(
597
+ "duplicates",
598
+ profile.duplicate_row_count == 0,
599
+ (("duplicate_rows", profile.duplicate_row_count),),
600
+ )
601
+ )
602
+ multiplier_failures = tuple(
603
+ audit.join_id
604
+ for audit in joins
605
+ if (
606
+ audit.row_multiplier_numerator * spec.maximum_join_multiplier_denominator
607
+ > audit.row_multiplier_denominator * spec.maximum_join_multiplier_numerator
608
+ )
609
+ )
610
+ checks.append(
611
+ _check(
612
+ "join_multiplier",
613
+ not multiplier_failures,
614
+ (
615
+ ("failed_join_ids", list(multiplier_failures)),
616
+ (
617
+ "maximum_denominator",
618
+ spec.maximum_join_multiplier_denominator,
619
+ ),
620
+ ("maximum_numerator", spec.maximum_join_multiplier_numerator),
621
+ ),
622
+ )
623
+ )
624
+ checks.append(
625
+ _check(
626
+ "temporal",
627
+ temporal.status == "passed",
628
+ (
629
+ ("finding_codes", [item.code for item in temporal.findings]),
630
+ ("temporal_report_digest", temporal.digest),
631
+ ),
632
+ )
633
+ )
634
+ total_outliers = sum(item.outlier_count for item in profile.fields)
635
+ checks.append(
636
+ _check(
637
+ "outliers",
638
+ total_outliers <= spec.maximum_outliers,
639
+ (
640
+ ("maximum_outliers", spec.maximum_outliers),
641
+ ("observed_outliers", total_outliers),
642
+ ),
643
+ )
644
+ )
645
+ missing_periods = _missing_periods(state, spec)
646
+ checks.append(
647
+ _check(
648
+ "missing_periods",
649
+ missing_periods <= spec.maximum_missing_periods,
650
+ (
651
+ ("maximum_missing_periods", spec.maximum_missing_periods),
652
+ ("observed_missing_periods", missing_periods),
653
+ ),
654
+ )
655
+ )
656
+ report = QualityReport(
657
+ schema_version=QUALITY_REPORT_VERSION,
658
+ checks=tuple(checks),
659
+ status="failed" if any(item.status == "failed" for item in checks) else "passed",
660
+ )
661
+ return profile, report
662
+
663
+
664
+ def _check(
665
+ check_id: str,
666
+ passed: bool,
667
+ observed: tuple[tuple[str, Any], ...],
668
+ ) -> QualityCheck:
669
+ severity, threshold = _CHECK_POLICY[check_id]
670
+ return QualityCheck(
671
+ check_id=check_id,
672
+ status="passed" if passed else "failed",
673
+ severity=severity,
674
+ threshold_id=threshold,
675
+ observed=observed,
676
+ )
677
+
678
+
679
+ def _missing_periods(state: TableState, spec: QualitySpec) -> int:
680
+ if spec.expected_frequency == "none" or spec.temporal_column is None:
681
+ return 0
682
+ temporal_index = state.index(spec.temporal_column)
683
+ partition_columns = tuple(column for column in spec.grain if column != spec.temporal_column)
684
+ partition_indexes = tuple(state.index(column) for column in partition_columns)
685
+ groups: dict[tuple[Any, ...], list[datetime | date]] = {}
686
+ for row in state.rows:
687
+ value = row[temporal_index]
688
+ if value is None:
689
+ continue
690
+ if state.fields[temporal_index].logical_type == "date":
691
+ instant: datetime | date = date.fromisoformat(value)
692
+ else:
693
+ instant = _instant(value, "quality.temporal_column")
694
+ partition = tuple(row[index] for index in partition_indexes)
695
+ groups.setdefault(partition, []).append(instant)
696
+ step = timedelta(days=1) if spec.expected_frequency == "daily" else timedelta(hours=1)
697
+ missing = 0
698
+ for values in groups.values():
699
+ ordered = sorted(set(values))
700
+ for previous, current in pairwise(ordered):
701
+ delta = current - previous
702
+ missing += max(0, int(delta / step) - 1)
703
+ return missing
704
+
705
+
706
+ def _instant(value: Any, path: str) -> datetime:
707
+ if not isinstance(value, str):
708
+ raise PreparationError("TEMPORAL_VALUE", path, "must be a UTC timestamp string")
709
+ return parse_utc(value, path)
710
+
711
+
712
+ def _validate_temporal_column(
713
+ state: TableState,
714
+ source_id: str,
715
+ column: str,
716
+ role: str,
717
+ ) -> None:
718
+ try:
719
+ field = state.field(column)
720
+ except PreparationError:
721
+ raise PreparationError(
722
+ "TEMPORAL_EVIDENCE",
723
+ source_id,
724
+ f"declared {role} column is absent from the joined table",
725
+ ) from None
726
+ if field.source_id != source_id or field.temporal_role != role:
727
+ raise PreparationError(
728
+ "TEMPORAL_EVIDENCE",
729
+ source_id,
730
+ f"declared {role} column has mismatched source or temporal role",
731
+ )
732
+
733
+
734
+ def _is_digest(value: Any) -> bool:
735
+ return (
736
+ isinstance(value, str)
737
+ and len(value) == 64
738
+ and all(character in "0123456789abcdef" for character in value)
739
+ )
740
+
741
+
742
+ __all__ = [
743
+ "QualityCheck",
744
+ "QualityReport",
745
+ "TargetRationale",
746
+ "TemporalFinding",
747
+ "TemporalReport",
748
+ "construct_target",
749
+ "validate_quality",
750
+ "validate_temporal_safety",
751
+ ]