mostlyright-data 0.9.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (314) hide show
  1. mostlyright/data_harness/__init__.py +158 -0
  2. mostlyright/data_harness/acquisition/__init__.py +55 -0
  3. mostlyright/data_harness/acquisition/http.py +2773 -0
  4. mostlyright/data_harness/acquisition/parsing.py +809 -0
  5. mostlyright/data_harness/acquisition/ranges.py +495 -0
  6. mostlyright/data_harness/acquisition/result_download.py +360 -0
  7. mostlyright/data_harness/acquisition/retention_admission.py +248 -0
  8. mostlyright/data_harness/acquisition/sandbox.py +4888 -0
  9. mostlyright/data_harness/acquisition/url_policy.py +530 -0
  10. mostlyright/data_harness/agent_runtime.py +2743 -0
  11. mostlyright/data_harness/assets/logo-ink.svg +31 -0
  12. mostlyright/data_harness/backends/__init__.py +28 -0
  13. mostlyright/data_harness/backends/pandas_backend.py +350 -0
  14. mostlyright/data_harness/backends/polars_backend.py +366 -0
  15. mostlyright/data_harness/backends/protocol.py +124 -0
  16. mostlyright/data_harness/backends/reference.py +83 -0
  17. mostlyright/data_harness/backends/registry.py +55 -0
  18. mostlyright/data_harness/backends/restrictions.py +126 -0
  19. mostlyright/data_harness/canonical.py +333 -0
  20. mostlyright/data_harness/catalog_job.py +625 -0
  21. mostlyright/data_harness/cli.py +5398 -0
  22. mostlyright/data_harness/contracts.py +53 -0
  23. mostlyright/data_harness/coordinator.py +1307 -0
  24. mostlyright/data_harness/deploy.py +924 -0
  25. mostlyright/data_harness/deploy_target.py +312 -0
  26. mostlyright/data_harness/deployment_evidence.py +1067 -0
  27. mostlyright/data_harness/event_presentation.py +576 -0
  28. mostlyright/data_harness/events.py +2152 -0
  29. mostlyright/data_harness/fast_delimited.py +239 -0
  30. mostlyright/data_harness/fleet.py +237 -0
  31. mostlyright/data_harness/formats.py +236 -0
  32. mostlyright/data_harness/governors.py +1163 -0
  33. mostlyright/data_harness/hosted_bootstrap.py +972 -0
  34. mostlyright/data_harness/hosted_crawler.py +1115 -0
  35. mostlyright/data_harness/hosted_crawler_container_smoke.py +351 -0
  36. mostlyright/data_harness/hosted_crawler_fetch.py +423 -0
  37. mostlyright/data_harness/hosted_crawler_job.py +1277 -0
  38. mostlyright/data_harness/hosted_crawler_protocol.py +676 -0
  39. mostlyright/data_harness/hosted_dataset.py +1500 -0
  40. mostlyright/data_harness/hosted_deploy.py +3037 -0
  41. mostlyright/data_harness/hosted_handoff.py +62 -0
  42. mostlyright/data_harness/hosted_ingestion_contract.py +504 -0
  43. mostlyright/data_harness/hosted_ingestion_job.py +356 -0
  44. mostlyright/data_harness/hosted_ingestion_job_smoke.py +40 -0
  45. mostlyright/data_harness/hosted_session_container_smoke.py +194 -0
  46. mostlyright/data_harness/hosted_session_worker.py +3554 -0
  47. mostlyright/data_harness/hosted_session_worker_job_smoke.py +46 -0
  48. mostlyright/data_harness/hosted_worker.py +6784 -0
  49. mostlyright/data_harness/ingestion/__init__.py +56 -0
  50. mostlyright/data_harness/ingestion/contracts.py +461 -0
  51. mostlyright/data_harness/ingestion/faults.py +42 -0
  52. mostlyright/data_harness/ingestion/gcs_store.py +1162 -0
  53. mostlyright/data_harness/ingestion/spool.py +130 -0
  54. mostlyright/data_harness/ingestion/store.py +885 -0
  55. mostlyright/data_harness/key_seam.py +434 -0
  56. mostlyright/data_harness/linux_process_boundary.py +262 -0
  57. mostlyright/data_harness/local_contracts.py +2880 -0
  58. mostlyright/data_harness/local_search/__init__.py +5 -0
  59. mostlyright/data_harness/local_search/build_index.py +1087 -0
  60. mostlyright/data_harness/local_search/contracts.py +920 -0
  61. mostlyright/data_harness/local_search/query_trace.py +266 -0
  62. mostlyright/data_harness/local_search/retrieval.py +700 -0
  63. mostlyright/data_harness/local_search/sealed.py +474 -0
  64. mostlyright/data_harness/local_search/service.py +784 -0
  65. mostlyright/data_harness/nbrender/CONTRACT.md +212 -0
  66. mostlyright/data_harness/nbrender/__init__.py +12 -0
  67. mostlyright/data_harness/nbrender/chrome.py +359 -0
  68. mostlyright/data_harness/nbrender/code_body.py +266 -0
  69. mostlyright/data_harness/nbrender/document.py +407 -0
  70. mostlyright/data_harness/nbrender/frame.py +275 -0
  71. mostlyright/data_harness/nbrender/interactive.py +337 -0
  72. mostlyright/data_harness/nbrender/markdown_body.py +477 -0
  73. mostlyright/data_harness/nbrender/mr_components.py +134 -0
  74. mostlyright/data_harness/nbrender/outputs_data.py +595 -0
  75. mostlyright/data_harness/nbrender/outputs_rich.py +906 -0
  76. mostlyright/data_harness/nbrender/outputs_source.py +260 -0
  77. mostlyright/data_harness/nbrender/outputs_stage.py +176 -0
  78. mostlyright/data_harness/nbrender/outputs_text.py +400 -0
  79. mostlyright/data_harness/nbrender/parse.py +394 -0
  80. mostlyright/data_harness/nbrender/status.py +40 -0
  81. mostlyright/data_harness/nbrender/tokens.py +1295 -0
  82. mostlyright/data_harness/notebook.py +1710 -0
  83. mostlyright/data_harness/offline.py +2049 -0
  84. mostlyright/data_harness/operation_registry.py +1007 -0
  85. mostlyright/data_harness/operator_setup.py +239 -0
  86. mostlyright/data_harness/pipeline.py +6428 -0
  87. mostlyright/data_harness/plan_graph.py +2026 -0
  88. mostlyright/data_harness/preparation/__init__.py +104 -0
  89. mostlyright/data_harness/preparation/contracts.py +1017 -0
  90. mostlyright/data_harness/preparation/engine.py +221 -0
  91. mostlyright/data_harness/preparation/errors.py +14 -0
  92. mostlyright/data_harness/preparation/gates.py +751 -0
  93. mostlyright/data_harness/preparation/joins.py +574 -0
  94. mostlyright/data_harness/preparation/profile.py +384 -0
  95. mostlyright/data_harness/preparation/table.py +217 -0
  96. mostlyright/data_harness/preparation/transforms.py +568 -0
  97. mostlyright/data_harness/progress_events.py +534 -0
  98. mostlyright/data_harness/readers/__init__.py +46 -0
  99. mostlyright/data_harness/readers/containers.py +963 -0
  100. mostlyright/data_harness/readers/contracts.py +542 -0
  101. mostlyright/data_harness/readers/delimited.py +257 -0
  102. mostlyright/data_harness/readers/grib2/__init__.py +33 -0
  103. mostlyright/data_harness/readers/grib2/admission.py +722 -0
  104. mostlyright/data_harness/readers/grib2/decode.py +1009 -0
  105. mostlyright/data_harness/readers/grib2/geometry.py +1133 -0
  106. mostlyright/data_harness/readers/grib2/portable_math.py +501 -0
  107. mostlyright/data_harness/readers/json_tabular.py +485 -0
  108. mostlyright/data_harness/readers/registry.py +514 -0
  109. mostlyright/data_harness/readers/samples/README.md +110 -0
  110. mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/cities.csv.gz +0 -0
  111. mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/expected.json +24 -0
  112. mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/cities.csv.gz +0 -0
  113. mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/expected.json +24 -0
  114. mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/cities.tar +0 -0
  115. mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/expected.json +24 -0
  116. mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/cities.tar +0 -0
  117. mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/expected.json +24 -0
  118. mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/cities.zip +0 -0
  119. mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/expected.json +25 -0
  120. mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
  121. mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/expected.json +25 -0
  122. mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
  123. mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/expected.json +25 -0
  124. mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/cities.csv +3 -0
  125. mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/expected.json +23 -0
  126. mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/cities.tsv +5 -0
  127. mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/expected.json +25 -0
  128. mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/expected.json +30 -0
  129. mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/observations.csv +5 -0
  130. mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/expected.json +44 -0
  131. mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/stations.json +1 -0
  132. mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/expected.json +48 -0
  133. mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/observations.ndjson +4 -0
  134. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/cities.xlsx +0 -0
  135. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/expected.json +24 -0
  136. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/expected.json +27 -0
  137. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
  138. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/expected.json +27 -0
  139. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
  140. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/README.md +20 -0
  141. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/expected.json +55 -0
  142. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/gfs-2m-temperature.grib2 +0 -0
  143. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/expected.json +54 -0
  144. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
  145. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/expected.json +54 -0
  146. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/hrrr-categorical-rain.grib2 +0 -0
  147. mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/expected.json +54 -0
  148. mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
  149. mostlyright/data_harness/readers/samples.py +582 -0
  150. mostlyright/data_harness/readers/spreadsheet.py +803 -0
  151. mostlyright/data_harness/readers/tabular.py +510 -0
  152. mostlyright/data_harness/recipe.py +5321 -0
  153. mostlyright/data_harness/repair/__init__.py +78 -0
  154. mostlyright/data_harness/repair/adapters.py +274 -0
  155. mostlyright/data_harness/repair/contracts.py +872 -0
  156. mostlyright/data_harness/repair/coordinator.py +1099 -0
  157. mostlyright/data_harness/repair/errors.py +16 -0
  158. mostlyright/data_harness/review.py +2533 -0
  159. mostlyright/data_harness/rowset.py +283 -0
  160. mostlyright/data_harness/serving.py +1975 -0
  161. mostlyright/data_harness/serving_edge.py +590 -0
  162. mostlyright/data_harness/serving_http.py +1031 -0
  163. mostlyright/data_harness/session_probes.py +759 -0
  164. mostlyright/data_harness/signing.py +101 -0
  165. mostlyright/data_harness/source_discovery.py +898 -0
  166. mostlyright/data_harness/sources/__init__.py +209 -0
  167. mostlyright/data_harness/sources/_adapter_steps.py +213 -0
  168. mostlyright/data_harness/sources/adapters.py +1214 -0
  169. mostlyright/data_harness/sources/cadence.py +1428 -0
  170. mostlyright/data_harness/sources/cadence_emission.py +453 -0
  171. mostlyright/data_harness/sources/cadence_history.py +546 -0
  172. mostlyright/data_harness/sources/catalog/__init__.py +17 -0
  173. mostlyright/data_harness/sources/catalog/admission.py +477 -0
  174. mostlyright/data_harness/sources/catalog/authoring.py +1701 -0
  175. mostlyright/data_harness/sources/catalog/authoring_policy.py +701 -0
  176. mostlyright/data_harness/sources/catalog/authoring_shards.py +1217 -0
  177. mostlyright/data_harness/sources/catalog/bounded_io.py +231 -0
  178. mostlyright/data_harness/sources/catalog/channel.py +523 -0
  179. mostlyright/data_harness/sources/catalog/channel_client.py +296 -0
  180. mostlyright/data_harness/sources/catalog/contracts.py +825 -0
  181. mostlyright/data_harness/sources/catalog/coverage.py +137 -0
  182. mostlyright/data_harness/sources/catalog/delta.py +1340 -0
  183. mostlyright/data_harness/sources/catalog/embedding.py +532 -0
  184. mostlyright/data_harness/sources/catalog/entry_v2.py +1182 -0
  185. mostlyright/data_harness/sources/catalog/fill.py +3889 -0
  186. mostlyright/data_harness/sources/catalog/fill_partitions.py +459 -0
  187. mostlyright/data_harness/sources/catalog/fill_staging.py +1105 -0
  188. mostlyright/data_harness/sources/catalog/gating.py +374 -0
  189. mostlyright/data_harness/sources/catalog/generation_receipt.py +1607 -0
  190. mostlyright/data_harness/sources/catalog/harvest/__init__.py +7 -0
  191. mostlyright/data_harness/sources/catalog/harvest/ckan.py +384 -0
  192. mostlyright/data_harness/sources/catalog/harvest/datagov_v4.py +798 -0
  193. mostlyright/data_harness/sources/catalog/harvest/protocol.py +964 -0
  194. mostlyright/data_harness/sources/catalog/harvest/sdmx.py +445 -0
  195. mostlyright/data_harness/sources/catalog/harvest/stac.py +384 -0
  196. mostlyright/data_harness/sources/catalog/health.py +447 -0
  197. mostlyright/data_harness/sources/catalog/hosted_catalog.py +105 -0
  198. mostlyright/data_harness/sources/catalog/identity_history.py +1549 -0
  199. mostlyright/data_harness/sources/catalog/neural.py +1618 -0
  200. mostlyright/data_harness/sources/catalog/packed_catalog.py +2345 -0
  201. mostlyright/data_harness/sources/catalog/packed_retrieval.py +1517 -0
  202. mostlyright/data_harness/sources/catalog/packed_writer.py +2802 -0
  203. mostlyright/data_harness/sources/catalog/query_trace.py +1037 -0
  204. mostlyright/data_harness/sources/catalog/recommend.py +171 -0
  205. mostlyright/data_harness/sources/catalog/retrieval.py +230 -0
  206. mostlyright/data_harness/sources/catalog/retrieval_manifest.py +995 -0
  207. mostlyright/data_harness/sources/catalog/rights_decisions.py +254 -0
  208. mostlyright/data_harness/sources/catalog/sealed.py +560 -0
  209. mostlyright/data_harness/sources/catalog/search.py +230 -0
  210. mostlyright/data_harness/sources/catalog/streaming_delta.py +1097 -0
  211. mostlyright/data_harness/sources/catalog/update.py +891 -0
  212. mostlyright/data_harness/sources/collections.py +815 -0
  213. mostlyright/data_harness/sources/contracts.py +2223 -0
  214. mostlyright/data_harness/sources/deletion.py +761 -0
  215. mostlyright/data_harness/sources/fitness.py +162 -0
  216. mostlyright/data_harness/sources/governance.py +163 -0
  217. mostlyright/data_harness/sources/hosted.py +173 -0
  218. mostlyright/data_harness/sources/integration.py +218 -0
  219. mostlyright/data_harness/sources/range_reader.py +418 -0
  220. mostlyright/data_harness/sources/registry.py +514 -0
  221. mostlyright/data_harness/sources/rights_rule.py +59 -0
  222. mostlyright/data_harness/sources/source_cadence_vectors.v1.json +1 -0
  223. mostlyright/data_harness/sources/sports.py +521 -0
  224. mostlyright/data_harness/sources/stream.py +524 -0
  225. mostlyright/data_harness/sources/stream_connector.py +418 -0
  226. mostlyright/data_harness/sources/stream_recorder.py +1404 -0
  227. mostlyright/data_harness/studio_boundary.py +2019 -0
  228. mostlyright/data_harness/thin/__init__.py +37 -0
  229. mostlyright/data_harness/thin/acquire.py +1137 -0
  230. mostlyright/data_harness/thin/acquire_cancel.py +579 -0
  231. mostlyright/data_harness/thin/approvals.py +617 -0
  232. mostlyright/data_harness/thin/commands.py +406 -0
  233. mostlyright/data_harness/thin/download.py +194 -0
  234. mostlyright/data_harness/thin/narrative.py +589 -0
  235. mostlyright/data_harness/thin/parity.py +1070 -0
  236. mostlyright/data_harness/thin/propose.py +2759 -0
  237. mostlyright/data_harness/thin/research.py +1663 -0
  238. mostlyright/data_harness/thin/router.py +924 -0
  239. mostlyright/data_harness/thin/runs.py +519 -0
  240. mostlyright/data_harness/thin/session.py +281 -0
  241. mostlyright/data_harness/thin/stream.py +501 -0
  242. mostlyright/data_harness/thin/transport.py +187 -0
  243. mostlyright/data_harness/thin/vocabulary.py +368 -0
  244. mostlyright/data_harness/thin/workers.py +164 -0
  245. mostlyright/data_harness/ucum/TABLE-PIN.json +40 -0
  246. mostlyright/data_harness/ucum/ucum-subset.v1.json +632 -0
  247. mostlyright/data_harness/unit_flow.py +927 -0
  248. mostlyright/data_harness/units.py +572 -0
  249. mostlyright/data_harness/ux/__init__.py +9 -0
  250. mostlyright/data_harness/ux/approve.py +485 -0
  251. mostlyright/data_harness/ux/author_yaml.py +597 -0
  252. mostlyright/data_harness/ux/cloud_auth.py +447 -0
  253. mostlyright/data_harness/ux/commands/__init__.py +260 -0
  254. mostlyright/data_harness/ux/commands/approve.py +136 -0
  255. mostlyright/data_harness/ux/commands/auth.py +744 -0
  256. mostlyright/data_harness/ux/commands/author.py +79 -0
  257. mostlyright/data_harness/ux/commands/catalog_author.py +403 -0
  258. mostlyright/data_harness/ux/commands/catalog_fill.py +523 -0
  259. mostlyright/data_harness/ux/commands/catalog_harvest.py +545 -0
  260. mostlyright/data_harness/ux/commands/catalog_publish.py +1838 -0
  261. mostlyright/data_harness/ux/commands/catalog_search.py +71 -0
  262. mostlyright/data_harness/ux/commands/catalog_update.py +437 -0
  263. mostlyright/data_harness/ux/commands/deploy.py +134 -0
  264. mostlyright/data_harness/ux/commands/deploy_dataset.py +98 -0
  265. mostlyright/data_harness/ux/commands/deploy_plan.py +105 -0
  266. mostlyright/data_harness/ux/commands/deploy_status.py +104 -0
  267. mostlyright/data_harness/ux/commands/diff.py +74 -0
  268. mostlyright/data_harness/ux/commands/index.py +84 -0
  269. mostlyright/data_harness/ux/commands/inventory.py +47 -0
  270. mostlyright/data_harness/ux/commands/list_builds.py +143 -0
  271. mostlyright/data_harness/ux/commands/login.py +63 -0
  272. mostlyright/data_harness/ux/commands/peek.py +236 -0
  273. mostlyright/data_harness/ux/commands/plan_check.py +90 -0
  274. mostlyright/data_harness/ux/commands/preflight.py +97 -0
  275. mostlyright/data_harness/ux/commands/record.py +107 -0
  276. mostlyright/data_harness/ux/commands/review_setup.py +47 -0
  277. mostlyright/data_harness/ux/commands/search.py +440 -0
  278. mostlyright/data_harness/ux/commands/show.py +61 -0
  279. mostlyright/data_harness/ux/commands/whoami.py +37 -0
  280. mostlyright/data_harness/ux/credential_native.py +551 -0
  281. mostlyright/data_harness/ux/credential_store.py +1055 -0
  282. mostlyright/data_harness/ux/credentials.py +631 -0
  283. mostlyright/data_harness/ux/diffing.py +444 -0
  284. mostlyright/data_harness/ux/headline.py +671 -0
  285. mostlyright/data_harness/ux/hosted_acquisition.py +974 -0
  286. mostlyright/data_harness/ux/hosted_run_status.py +619 -0
  287. mostlyright/data_harness/ux/inventory.py +427 -0
  288. mostlyright/data_harness/ux/local_review.py +375 -0
  289. mostlyright/data_harness/ux/login.py +691 -0
  290. mostlyright/data_harness/ux/path_kind.py +147 -0
  291. mostlyright/data_harness/ux/peek.py +1000 -0
  292. mostlyright/data_harness/ux/plain_file.py +178 -0
  293. mostlyright/data_harness/ux/plan_check.py +311 -0
  294. mostlyright/data_harness/ux/preflight.py +918 -0
  295. mostlyright/data_harness/ux/readers.py +1124 -0
  296. mostlyright/data_harness/ux/remediation.py +2195 -0
  297. mostlyright/data_harness/ux/render.py +657 -0
  298. mostlyright/data_harness/ux/workload.py +1077 -0
  299. mostlyright/data_harness/viewer.py +3713 -0
  300. mostlyright/data_harness/visual_run/__init__.py +83 -0
  301. mostlyright/data_harness/visual_run/authoring.py +235 -0
  302. mostlyright/data_harness/visual_run/contracts.py +673 -0
  303. mostlyright/data_harness/visual_run/materialize.py +486 -0
  304. mostlyright/data_harness/visual_run/observations.py +874 -0
  305. mostlyright/data_harness/visual_run/query.py +259 -0
  306. mostlyright/data_harness/visual_run/reducer.py +280 -0
  307. mostlyright/data_harness/visual_run/sdk.py +892 -0
  308. mostlyright/data_harness/visual_run/store.py +584 -0
  309. mostlyright/data_harness/visual_run/transport.py +239 -0
  310. mostlyright/data_harness/watch.py +2999 -0
  311. mostlyright_data-0.9.0.dist-info/METADATA +607 -0
  312. mostlyright_data-0.9.0.dist-info/RECORD +314 -0
  313. mostlyright_data-0.9.0.dist-info/WHEEL +4 -0
  314. mostlyright_data-0.9.0.dist-info/entry_points.txt +12 -0
@@ -0,0 +1,920 @@
1
+ """Strict contracts for the private local Build index.
2
+
3
+ Immutable Build facts, mutable filesystem locators, and sanitized discovery diagnostics are
4
+ different documents by construction. Nothing in this module imports the public Source Catalog:
5
+ the two indexes share canonical JSON mechanics, not contracts or admission authority.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import re
11
+ from collections.abc import Mapping, Sequence
12
+ from dataclasses import dataclass
13
+ from datetime import date
14
+ from pathlib import PurePosixPath
15
+ from typing import Any, ClassVar
16
+
17
+ from mostlyright.data_harness.canonical import canonical_json_bytes, canonical_sha256, sha256_bytes
18
+
19
+ BUILD_INDEX_ENTRY_VERSION = "harness-private-build-index-entry.v1"
20
+ BUILD_LOCATOR_VERSION = "harness-private-build-locators.v1"
21
+ BUILD_DIAGNOSTIC_VERSION = "harness-private-build-diagnostics.v1"
22
+ BUILD_INDEX_MANIFEST_VERSION = "harness-private-build-index-manifest.v1"
23
+ DATASET_PROFILE_VERSION = "dataset-profile.v1"
24
+
25
+ MAX_TEXT = 16_384
26
+ MAX_SHORT_TEXT = 512
27
+ MAX_PATH = 1_024
28
+ MAX_GRAIN = 64
29
+ MAX_FIELDS = 1_024
30
+ MAX_SOURCES = 256
31
+ MAX_ENTRIES = 100_000
32
+ MAX_LOCATORS = 200_000
33
+ MAX_DIAGNOSTICS = 100_000
34
+ MAX_SAFE_INTEGER = (1 << 53) - 1
35
+
36
+ _SHA256 = re.compile(r"^[0-9a-f]{64}$")
37
+ _IDENTIFIER = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,255}$")
38
+ _LOGICAL_TYPES = frozenset({"string", "int64", "float64", "date"})
39
+ _RIGHTS_STATUSES = frozenset(
40
+ {
41
+ "project_owned",
42
+ "public_domain",
43
+ "permissive_license",
44
+ "authorized_internal",
45
+ "user_authorized",
46
+ "unknown",
47
+ "prohibited",
48
+ }
49
+ )
50
+ _DIAGNOSTIC_CODES = frozenset(
51
+ {
52
+ "BUILD_HARDLINK",
53
+ "BUILD_INVALID",
54
+ "BUILD_LIMIT",
55
+ "BUILD_RACE",
56
+ "BUILD_SPECIAL",
57
+ "BUILD_SYMLINK",
58
+ "BUILD_UNREADABLE",
59
+ }
60
+ )
61
+
62
+
63
+ class LocalSearchContractError(ValueError):
64
+ """Typed refusal for private index contracts and sealed snapshots."""
65
+
66
+ def __init__(self, code: str, path: str, detail: str) -> None:
67
+ self.code = code
68
+ self.path = path
69
+ self.detail = detail
70
+ super().__init__(f"{path}: {detail} [{code}]")
71
+
72
+
73
+ class _CanonicalContract:
74
+ schema_version: ClassVar[str] | str
75
+
76
+ def to_dict(self) -> dict[str, Any]:
77
+ raise NotImplementedError
78
+
79
+ @property
80
+ def canonical_bytes(self) -> bytes:
81
+ return canonical_json_bytes(self.to_dict())
82
+
83
+ @property
84
+ def digest(self) -> str:
85
+ return canonical_sha256(self.to_dict())
86
+
87
+
88
+ @dataclass(frozen=True)
89
+ class BuildField(_CanonicalContract):
90
+ """One output field in the sealed profile's exact column order."""
91
+
92
+ name: str
93
+ logical_type: str
94
+
95
+ def __post_init__(self) -> None:
96
+ _text(self.name, "field.name", maximum=MAX_SHORT_TEXT)
97
+ _choice(self.logical_type, _LOGICAL_TYPES, "field.logical_type")
98
+
99
+ def to_dict(self) -> dict[str, str]:
100
+ return {"name": self.name, "logical_type": self.logical_type}
101
+
102
+
103
+ @dataclass(frozen=True)
104
+ class BuildTimeCoverage(_CanonicalContract):
105
+ """Sealed requirement coverage; absent when the Build carries no such evidence."""
106
+
107
+ start_inclusive: str
108
+ end_exclusive: str
109
+
110
+ def __post_init__(self) -> None:
111
+ start = _iso_date(self.start_inclusive, "time_coverage.start_inclusive")
112
+ end = _iso_date(self.end_exclusive, "time_coverage.end_exclusive")
113
+ if start >= end:
114
+ raise LocalSearchContractError(
115
+ "LOCAL_SEARCH_TIME",
116
+ "time_coverage",
117
+ "start_inclusive must precede end_exclusive",
118
+ )
119
+
120
+ def to_dict(self) -> dict[str, str]:
121
+ return {
122
+ "start_inclusive": self.start_inclusive,
123
+ "end_exclusive": self.end_exclusive,
124
+ }
125
+
126
+
127
+ @dataclass(frozen=True)
128
+ class BuildSourceFacts(_CanonicalContract):
129
+ """Identity, exact content, and rights evidence for one sealed Build source."""
130
+
131
+ source_id: str
132
+ content_sha256: str
133
+ rights_status: str
134
+ rights_evidence_sha256: str
135
+
136
+ def __post_init__(self) -> None:
137
+ _identifier(self.source_id, "source.source_id")
138
+ _digest(self.content_sha256, "source.content_sha256")
139
+ _choice(self.rights_status, _RIGHTS_STATUSES, "source.rights_status")
140
+ _digest(self.rights_evidence_sha256, "source.rights_evidence_sha256")
141
+
142
+ def to_dict(self) -> dict[str, str]:
143
+ return {
144
+ "source_id": self.source_id,
145
+ "content_sha256": self.content_sha256,
146
+ "rights_status": self.rights_status,
147
+ "rights_evidence_sha256": self.rights_evidence_sha256,
148
+ }
149
+
150
+
151
+ @dataclass(frozen=True)
152
+ class BuildRecipeLineage(_CanonicalContract):
153
+ """Recipe lineage only when the sealed Build carries recipe evidence."""
154
+
155
+ recipe_id: str
156
+ recipe_version: int
157
+ recipe_sha256: str
158
+ predecessor_candidate_sha256: str | None
159
+
160
+ def __post_init__(self) -> None:
161
+ _identifier(self.recipe_id, "recipe_lineage.recipe_id")
162
+ _integer(
163
+ self.recipe_version,
164
+ "recipe_lineage.recipe_version",
165
+ minimum=1,
166
+ maximum=MAX_SAFE_INTEGER,
167
+ )
168
+ _digest(self.recipe_sha256, "recipe_lineage.recipe_sha256")
169
+ if self.predecessor_candidate_sha256 is not None:
170
+ _digest(
171
+ self.predecessor_candidate_sha256,
172
+ "recipe_lineage.predecessor_candidate_sha256",
173
+ )
174
+
175
+ def to_dict(self) -> dict[str, Any]:
176
+ return {
177
+ "recipe_id": self.recipe_id,
178
+ "recipe_version": self.recipe_version,
179
+ "recipe_sha256": self.recipe_sha256,
180
+ "predecessor_candidate_sha256": self.predecessor_candidate_sha256,
181
+ }
182
+
183
+
184
+ @dataclass(frozen=True)
185
+ class SealedProfileFactsV1(_CanonicalContract):
186
+ """The exact useful subset presently sealed by ``dataset-profile.v1``.
187
+
188
+ The profile member's digest is mandatory. A later rich profile therefore adds a new
189
+ profile-facts contract instead of reinterpreting these v1 fields or rewriting the Build.
190
+ """
191
+
192
+ evidence_sha256: str
193
+ row_count: int
194
+ columns: tuple[str, ...]
195
+ logical_types: tuple[str, ...]
196
+ null_counts: tuple[int, ...]
197
+ profile_schema_version: str = DATASET_PROFILE_VERSION
198
+
199
+ def __post_init__(self) -> None:
200
+ if self.profile_schema_version != DATASET_PROFILE_VERSION:
201
+ raise LocalSearchContractError(
202
+ "LOCAL_SEARCH_PROFILE",
203
+ "profile.profile_schema_version",
204
+ f"must equal {DATASET_PROFILE_VERSION!r}",
205
+ )
206
+ _digest(self.evidence_sha256, "profile.evidence_sha256")
207
+ _integer(self.row_count, "profile.row_count", minimum=0, maximum=MAX_SAFE_INTEGER)
208
+ _texts(
209
+ self.columns,
210
+ "profile.columns",
211
+ maximum=MAX_FIELDS,
212
+ item_maximum=MAX_SHORT_TEXT,
213
+ nonempty=False,
214
+ )
215
+ _typed_choices(self.logical_types, _LOGICAL_TYPES, "profile.logical_types", MAX_FIELDS)
216
+ _integers(self.null_counts, "profile.null_counts", maximum=MAX_FIELDS)
217
+ if not (len(self.columns) == len(self.logical_types) == len(self.null_counts)):
218
+ raise LocalSearchContractError(
219
+ "LOCAL_SEARCH_PROFILE",
220
+ "profile",
221
+ "columns, logical_types, and null_counts must have equal lengths",
222
+ )
223
+ for index, count in enumerate(self.null_counts):
224
+ if count > self.row_count:
225
+ raise LocalSearchContractError(
226
+ "LOCAL_SEARCH_PROFILE",
227
+ f"profile.null_counts[{index}]",
228
+ "cannot exceed row_count",
229
+ )
230
+
231
+ def to_dict(self) -> dict[str, Any]:
232
+ return {
233
+ "profile_schema_version": self.profile_schema_version,
234
+ "evidence_sha256": self.evidence_sha256,
235
+ "row_count": self.row_count,
236
+ "columns": list(self.columns),
237
+ "logical_types": list(self.logical_types),
238
+ "null_counts": list(self.null_counts),
239
+ }
240
+
241
+
242
+ @dataclass(frozen=True)
243
+ class BuildIndexEntry(_CanonicalContract):
244
+ """Immutable sealed facts about one locally retained Build.
245
+
246
+ There is intentionally no path, indexing timestamp, username, machine, attempt, row sample,
247
+ or Parquet payload field. ``facts_sha256`` is solely the digest of this document.
248
+ """
249
+
250
+ question: str
251
+ question_sha256: str
252
+ requirements_sha256: str
253
+ grain: tuple[str, ...]
254
+ time_coverage: BuildTimeCoverage | None
255
+ fields: tuple[BuildField, ...]
256
+ sources: tuple[BuildSourceFacts, ...]
257
+ profile: SealedProfileFactsV1 | None
258
+ recipe_lineage: BuildRecipeLineage | None
259
+ candidate_sha256: str
260
+ table_sha256: str
261
+ validation_policy_sha256: str
262
+ lineage_sha256: str
263
+ schema_version: str = BUILD_INDEX_ENTRY_VERSION
264
+
265
+ def __post_init__(self) -> None:
266
+ if self.schema_version != BUILD_INDEX_ENTRY_VERSION:
267
+ raise LocalSearchContractError(
268
+ "LOCAL_SEARCH_VERSION",
269
+ "entry.schema_version",
270
+ f"must equal {BUILD_INDEX_ENTRY_VERSION!r}",
271
+ )
272
+ _text(self.question, "entry.question", maximum=MAX_TEXT)
273
+ _digest(self.question_sha256, "entry.question_sha256")
274
+ if self.question_sha256 != sha256_bytes(self.question.encode("utf-8")):
275
+ raise LocalSearchContractError(
276
+ "LOCAL_SEARCH_QUESTION_BINDING",
277
+ "entry.question_sha256",
278
+ "does not match the exact UTF-8 question",
279
+ )
280
+ _digest(self.requirements_sha256, "entry.requirements_sha256")
281
+ _texts(
282
+ self.grain,
283
+ "entry.grain",
284
+ maximum=MAX_GRAIN,
285
+ item_maximum=MAX_SHORT_TEXT,
286
+ nonempty=True,
287
+ )
288
+ _optional_type(self.time_coverage, BuildTimeCoverage, "entry.time_coverage")
289
+ _typed_tuple(self.fields, BuildField, "entry.fields", MAX_FIELDS, nonempty=True)
290
+ if len({item.name for item in self.fields}) != len(self.fields):
291
+ raise LocalSearchContractError(
292
+ "LOCAL_SEARCH_DUPLICATE", "entry.fields", "field names must be unique"
293
+ )
294
+ _typed_tuple(self.sources, BuildSourceFacts, "entry.sources", MAX_SOURCES, nonempty=True)
295
+ if len({item.source_id for item in self.sources}) != len(self.sources):
296
+ raise LocalSearchContractError(
297
+ "LOCAL_SEARCH_DUPLICATE", "entry.sources", "source ids must be unique"
298
+ )
299
+ _optional_type(self.profile, SealedProfileFactsV1, "entry.profile")
300
+ if self.profile is not None:
301
+ if self.profile.columns != tuple(field.name for field in self.fields):
302
+ raise LocalSearchContractError(
303
+ "LOCAL_SEARCH_PROFILE",
304
+ "entry.profile.columns",
305
+ "must match fields in sealed order",
306
+ )
307
+ if self.profile.logical_types != tuple(field.logical_type for field in self.fields):
308
+ raise LocalSearchContractError(
309
+ "LOCAL_SEARCH_PROFILE",
310
+ "entry.profile.logical_types",
311
+ "must match field logical types in sealed order",
312
+ )
313
+ _optional_type(self.recipe_lineage, BuildRecipeLineage, "entry.recipe_lineage")
314
+ for name in (
315
+ "candidate_sha256",
316
+ "table_sha256",
317
+ "validation_policy_sha256",
318
+ "lineage_sha256",
319
+ ):
320
+ _digest(getattr(self, name), f"entry.{name}")
321
+
322
+ @property
323
+ def facts_sha256(self) -> str:
324
+ return self.digest
325
+
326
+ def to_dict(self) -> dict[str, Any]:
327
+ return {
328
+ "schema_version": self.schema_version,
329
+ "question": self.question,
330
+ "question_sha256": self.question_sha256,
331
+ "requirements_sha256": self.requirements_sha256,
332
+ "grain": list(self.grain),
333
+ "time_coverage": None if self.time_coverage is None else self.time_coverage.to_dict(),
334
+ "fields": [item.to_dict() for item in self.fields],
335
+ "sources": [item.to_dict() for item in self.sources],
336
+ "profile": None if self.profile is None else self.profile.to_dict(),
337
+ "recipe_lineage": (
338
+ None if self.recipe_lineage is None else self.recipe_lineage.to_dict()
339
+ ),
340
+ "candidate_sha256": self.candidate_sha256,
341
+ "table_sha256": self.table_sha256,
342
+ "validation_policy_sha256": self.validation_policy_sha256,
343
+ "lineage_sha256": self.lineage_sha256,
344
+ }
345
+
346
+
347
+ @dataclass(frozen=True)
348
+ class BuildLocator(_CanonicalContract):
349
+ """One mutable root-relative location for immutable Build facts."""
350
+
351
+ facts_sha256: str
352
+ relative_path: str
353
+
354
+ def __post_init__(self) -> None:
355
+ _digest(self.facts_sha256, "locator.facts_sha256")
356
+ _relative_path(self.relative_path, "locator.relative_path")
357
+
358
+ def to_dict(self) -> dict[str, str]:
359
+ return {"facts_sha256": self.facts_sha256, "relative_path": self.relative_path}
360
+
361
+
362
+ @dataclass(frozen=True)
363
+ class BuildIndexDiagnostic(_CanonicalContract):
364
+ """A path-safe discovery result with no free-form filesystem error text."""
365
+
366
+ code: str
367
+ relative_path: str
368
+
369
+ def __post_init__(self) -> None:
370
+ _choice(self.code, _DIAGNOSTIC_CODES, "diagnostic.code")
371
+ _relative_path(self.relative_path, "diagnostic.relative_path")
372
+
373
+ def to_dict(self) -> dict[str, str]:
374
+ return {"code": self.code, "relative_path": self.relative_path}
375
+
376
+
377
+ @dataclass(frozen=True)
378
+ class BuildLocatorDocument(_CanonicalContract):
379
+ locators: tuple[BuildLocator, ...]
380
+ schema_version: str = BUILD_LOCATOR_VERSION
381
+
382
+ def __post_init__(self) -> None:
383
+ _version(self.schema_version, BUILD_LOCATOR_VERSION, "locators.schema_version")
384
+ _typed_tuple(self.locators, BuildLocator, "locators.locators", MAX_LOCATORS)
385
+ coordinates = tuple((item.facts_sha256, item.relative_path) for item in self.locators)
386
+ _sorted_unique(coordinates, "locators.locators")
387
+
388
+ def to_dict(self) -> dict[str, Any]:
389
+ return {
390
+ "schema_version": self.schema_version,
391
+ "locators": [item.to_dict() for item in self.locators],
392
+ }
393
+
394
+
395
+ @dataclass(frozen=True)
396
+ class BuildDiagnosticDocument(_CanonicalContract):
397
+ diagnostics: tuple[BuildIndexDiagnostic, ...]
398
+ schema_version: str = BUILD_DIAGNOSTIC_VERSION
399
+
400
+ def __post_init__(self) -> None:
401
+ _version(self.schema_version, BUILD_DIAGNOSTIC_VERSION, "diagnostics.schema_version")
402
+ _typed_tuple(
403
+ self.diagnostics,
404
+ BuildIndexDiagnostic,
405
+ "diagnostics.diagnostics",
406
+ MAX_DIAGNOSTICS,
407
+ )
408
+ coordinates = tuple((item.relative_path, item.code) for item in self.diagnostics)
409
+ _sorted_unique(coordinates, "diagnostics.diagnostics")
410
+
411
+ def to_dict(self) -> dict[str, Any]:
412
+ return {
413
+ "schema_version": self.schema_version,
414
+ "diagnostics": [item.to_dict() for item in self.diagnostics],
415
+ }
416
+
417
+
418
+ @dataclass(frozen=True)
419
+ class SnapshotMember(_CanonicalContract):
420
+ """Exact size and content identity of one manifest-bound snapshot member."""
421
+
422
+ path: str
423
+ bytes: int
424
+ sha256: str
425
+
426
+ def __post_init__(self) -> None:
427
+ _snapshot_member_path(self.path)
428
+ _integer(self.bytes, f"member[{self.path}].bytes", minimum=0, maximum=MAX_SAFE_INTEGER)
429
+ _digest(self.sha256, f"member[{self.path}].sha256")
430
+
431
+ def to_dict(self) -> dict[str, Any]:
432
+ return {"path": self.path, "bytes": self.bytes, "sha256": self.sha256}
433
+
434
+
435
+ def manifest_aggregate_sha256(members: Sequence[SnapshotMember]) -> str:
436
+ """Digest the explicit ordered member table, not a live filesystem listing."""
437
+
438
+ if not isinstance(members, (tuple, list)):
439
+ raise LocalSearchContractError(
440
+ "LOCAL_SEARCH_TYPE", "members", "must be an ordered sequence"
441
+ )
442
+ return canonical_sha256({"members": [item.to_dict() for item in members]})
443
+
444
+
445
+ @dataclass(frozen=True)
446
+ class BuildIndexManifest(_CanonicalContract):
447
+ """The complete explicit layout and counts for one private index snapshot."""
448
+
449
+ entries: tuple[SnapshotMember, ...]
450
+ locators: SnapshotMember
451
+ diagnostics: SnapshotMember
452
+ entry_count: int
453
+ locator_count: int
454
+ diagnostic_count: int
455
+ total_bytes: int
456
+ aggregate_sha256: str
457
+ schema_version: str = BUILD_INDEX_MANIFEST_VERSION
458
+
459
+ def __post_init__(self) -> None:
460
+ _version(self.schema_version, BUILD_INDEX_MANIFEST_VERSION, "manifest.schema_version")
461
+ _typed_tuple(self.entries, SnapshotMember, "manifest.entries", MAX_ENTRIES)
462
+ paths = tuple(member.path for member in self.entries)
463
+ if paths != tuple(sorted(paths)):
464
+ raise LocalSearchContractError(
465
+ "LOCAL_SEARCH_MANIFEST", "manifest.entries", "must be sorted by exact path"
466
+ )
467
+ if len(set(paths)) != len(paths):
468
+ raise LocalSearchContractError(
469
+ "LOCAL_SEARCH_DUPLICATE", "manifest.entries", "member paths must be unique"
470
+ )
471
+ for member in self.entries:
472
+ match = re.fullmatch(r"entries/([0-9a-f]{64})\.json", member.path)
473
+ if match is None or match.group(1) != member.sha256:
474
+ raise LocalSearchContractError(
475
+ "LOCAL_SEARCH_MANIFEST",
476
+ f"manifest.entries[{member.path}]",
477
+ "entry filename must equal its canonical member digest",
478
+ )
479
+ if not isinstance(self.locators, SnapshotMember) or self.locators.path != "locators.json":
480
+ raise LocalSearchContractError(
481
+ "LOCAL_SEARCH_MANIFEST", "manifest.locators", "must bind locators.json"
482
+ )
483
+ if (
484
+ not isinstance(self.diagnostics, SnapshotMember)
485
+ or self.diagnostics.path != "diagnostics.json"
486
+ ):
487
+ raise LocalSearchContractError(
488
+ "LOCAL_SEARCH_MANIFEST", "manifest.diagnostics", "must bind diagnostics.json"
489
+ )
490
+ _integer(self.entry_count, "manifest.entry_count", minimum=0, maximum=MAX_ENTRIES)
491
+ _integer(self.locator_count, "manifest.locator_count", minimum=0, maximum=MAX_LOCATORS)
492
+ _integer(
493
+ self.diagnostic_count,
494
+ "manifest.diagnostic_count",
495
+ minimum=0,
496
+ maximum=MAX_DIAGNOSTICS,
497
+ )
498
+ _integer(self.total_bytes, "manifest.total_bytes", minimum=0, maximum=MAX_SAFE_INTEGER)
499
+ _digest(self.aggregate_sha256, "manifest.aggregate_sha256")
500
+ if self.entry_count != len(self.entries):
501
+ raise LocalSearchContractError(
502
+ "LOCAL_SEARCH_MANIFEST",
503
+ "manifest.entry_count",
504
+ "must equal the number of bound entry members",
505
+ )
506
+ members = (*self.entries, self.locators, self.diagnostics)
507
+ if self.total_bytes != sum(item.bytes for item in members):
508
+ raise LocalSearchContractError(
509
+ "LOCAL_SEARCH_MANIFEST",
510
+ "manifest.total_bytes",
511
+ "must equal the sum of exact member sizes",
512
+ )
513
+ if self.aggregate_sha256 != manifest_aggregate_sha256(members):
514
+ raise LocalSearchContractError(
515
+ "LOCAL_SEARCH_MANIFEST",
516
+ "manifest.aggregate_sha256",
517
+ "does not match the ordered member table",
518
+ )
519
+
520
+ @property
521
+ def members(self) -> tuple[SnapshotMember, ...]:
522
+ return (*self.entries, self.locators, self.diagnostics)
523
+
524
+ def to_dict(self) -> dict[str, Any]:
525
+ return {
526
+ "schema_version": self.schema_version,
527
+ "entries": [item.to_dict() for item in self.entries],
528
+ "locators": self.locators.to_dict(),
529
+ "diagnostics": self.diagnostics.to_dict(),
530
+ "entry_count": self.entry_count,
531
+ "locator_count": self.locator_count,
532
+ "diagnostic_count": self.diagnostic_count,
533
+ "total_bytes": self.total_bytes,
534
+ "aggregate_sha256": self.aggregate_sha256,
535
+ }
536
+
537
+
538
+ def build_index_entry_from_dict(data: Mapping[str, Any]) -> BuildIndexEntry:
539
+ payload = _mapping(data, "entry")
540
+ _exact(
541
+ payload,
542
+ {
543
+ "schema_version",
544
+ "question",
545
+ "question_sha256",
546
+ "requirements_sha256",
547
+ "grain",
548
+ "time_coverage",
549
+ "fields",
550
+ "sources",
551
+ "profile",
552
+ "recipe_lineage",
553
+ "candidate_sha256",
554
+ "table_sha256",
555
+ "validation_policy_sha256",
556
+ "lineage_sha256",
557
+ },
558
+ "entry",
559
+ )
560
+ return BuildIndexEntry(
561
+ schema_version=payload["schema_version"],
562
+ question=payload["question"],
563
+ question_sha256=payload["question_sha256"],
564
+ requirements_sha256=payload["requirements_sha256"],
565
+ grain=tuple(_sequence(payload["grain"], "entry.grain")),
566
+ time_coverage=_time_coverage_from_value(payload["time_coverage"]),
567
+ fields=tuple(
568
+ _field_from_dict(item, f"entry.fields[{index}]")
569
+ for index, item in enumerate(_sequence(payload["fields"], "entry.fields"))
570
+ ),
571
+ sources=tuple(
572
+ _source_from_dict(item, f"entry.sources[{index}]")
573
+ for index, item in enumerate(_sequence(payload["sources"], "entry.sources"))
574
+ ),
575
+ profile=_profile_from_value(payload["profile"]),
576
+ recipe_lineage=_recipe_from_value(payload["recipe_lineage"]),
577
+ candidate_sha256=payload["candidate_sha256"],
578
+ table_sha256=payload["table_sha256"],
579
+ validation_policy_sha256=payload["validation_policy_sha256"],
580
+ lineage_sha256=payload["lineage_sha256"],
581
+ )
582
+
583
+
584
+ def build_locator_document_from_dict(data: Mapping[str, Any]) -> BuildLocatorDocument:
585
+ payload = _mapping(data, "locators")
586
+ _exact(payload, {"schema_version", "locators"}, "locators")
587
+ return BuildLocatorDocument(
588
+ schema_version=payload["schema_version"],
589
+ locators=tuple(
590
+ _locator_from_dict(item, f"locators.locators[{index}]")
591
+ for index, item in enumerate(_sequence(payload["locators"], "locators.locators"))
592
+ ),
593
+ )
594
+
595
+
596
+ def build_diagnostic_document_from_dict(data: Mapping[str, Any]) -> BuildDiagnosticDocument:
597
+ payload = _mapping(data, "diagnostics")
598
+ _exact(payload, {"schema_version", "diagnostics"}, "diagnostics")
599
+ return BuildDiagnosticDocument(
600
+ schema_version=payload["schema_version"],
601
+ diagnostics=tuple(
602
+ _diagnostic_from_dict(item, f"diagnostics.diagnostics[{index}]")
603
+ for index, item in enumerate(
604
+ _sequence(payload["diagnostics"], "diagnostics.diagnostics")
605
+ )
606
+ ),
607
+ )
608
+
609
+
610
+ def build_index_manifest_from_dict(data: Mapping[str, Any]) -> BuildIndexManifest:
611
+ payload = _mapping(data, "manifest")
612
+ _exact(
613
+ payload,
614
+ {
615
+ "schema_version",
616
+ "entries",
617
+ "locators",
618
+ "diagnostics",
619
+ "entry_count",
620
+ "locator_count",
621
+ "diagnostic_count",
622
+ "total_bytes",
623
+ "aggregate_sha256",
624
+ },
625
+ "manifest",
626
+ )
627
+ return BuildIndexManifest(
628
+ schema_version=payload["schema_version"],
629
+ entries=tuple(
630
+ _member_from_dict(item, f"manifest.entries[{index}]")
631
+ for index, item in enumerate(_sequence(payload["entries"], "manifest.entries"))
632
+ ),
633
+ locators=_member_from_dict(payload["locators"], "manifest.locators"),
634
+ diagnostics=_member_from_dict(payload["diagnostics"], "manifest.diagnostics"),
635
+ entry_count=payload["entry_count"],
636
+ locator_count=payload["locator_count"],
637
+ diagnostic_count=payload["diagnostic_count"],
638
+ total_bytes=payload["total_bytes"],
639
+ aggregate_sha256=payload["aggregate_sha256"],
640
+ )
641
+
642
+
643
+ def _field_from_dict(value: Any, path: str) -> BuildField:
644
+ payload = _mapping(value, path)
645
+ _exact(payload, {"name", "logical_type"}, path)
646
+ return BuildField(payload["name"], payload["logical_type"])
647
+
648
+
649
+ def _source_from_dict(value: Any, path: str) -> BuildSourceFacts:
650
+ payload = _mapping(value, path)
651
+ _exact(
652
+ payload,
653
+ {"source_id", "content_sha256", "rights_status", "rights_evidence_sha256"},
654
+ path,
655
+ )
656
+ return BuildSourceFacts(
657
+ payload["source_id"],
658
+ payload["content_sha256"],
659
+ payload["rights_status"],
660
+ payload["rights_evidence_sha256"],
661
+ )
662
+
663
+
664
+ def _time_coverage_from_value(value: Any) -> BuildTimeCoverage | None:
665
+ if value is None:
666
+ return None
667
+ payload = _mapping(value, "entry.time_coverage")
668
+ _exact(payload, {"start_inclusive", "end_exclusive"}, "entry.time_coverage")
669
+ return BuildTimeCoverage(payload["start_inclusive"], payload["end_exclusive"])
670
+
671
+
672
+ def _profile_from_value(value: Any) -> SealedProfileFactsV1 | None:
673
+ if value is None:
674
+ return None
675
+ payload = _mapping(value, "entry.profile")
676
+ _exact(
677
+ payload,
678
+ {
679
+ "profile_schema_version",
680
+ "evidence_sha256",
681
+ "row_count",
682
+ "columns",
683
+ "logical_types",
684
+ "null_counts",
685
+ },
686
+ "entry.profile",
687
+ )
688
+ return SealedProfileFactsV1(
689
+ profile_schema_version=payload["profile_schema_version"],
690
+ evidence_sha256=payload["evidence_sha256"],
691
+ row_count=payload["row_count"],
692
+ columns=tuple(_sequence(payload["columns"], "entry.profile.columns")),
693
+ logical_types=tuple(_sequence(payload["logical_types"], "entry.profile.logical_types")),
694
+ null_counts=tuple(_sequence(payload["null_counts"], "entry.profile.null_counts")),
695
+ )
696
+
697
+
698
+ def _recipe_from_value(value: Any) -> BuildRecipeLineage | None:
699
+ if value is None:
700
+ return None
701
+ payload = _mapping(value, "entry.recipe_lineage")
702
+ _exact(
703
+ payload,
704
+ {"recipe_id", "recipe_version", "recipe_sha256", "predecessor_candidate_sha256"},
705
+ "entry.recipe_lineage",
706
+ )
707
+ return BuildRecipeLineage(
708
+ payload["recipe_id"],
709
+ payload["recipe_version"],
710
+ payload["recipe_sha256"],
711
+ payload["predecessor_candidate_sha256"],
712
+ )
713
+
714
+
715
+ def _locator_from_dict(value: Any, path: str) -> BuildLocator:
716
+ payload = _mapping(value, path)
717
+ _exact(payload, {"facts_sha256", "relative_path"}, path)
718
+ return BuildLocator(payload["facts_sha256"], payload["relative_path"])
719
+
720
+
721
+ def _diagnostic_from_dict(value: Any, path: str) -> BuildIndexDiagnostic:
722
+ payload = _mapping(value, path)
723
+ _exact(payload, {"code", "relative_path"}, path)
724
+ return BuildIndexDiagnostic(payload["code"], payload["relative_path"])
725
+
726
+
727
+ def _member_from_dict(value: Any, path: str) -> SnapshotMember:
728
+ payload = _mapping(value, path)
729
+ _exact(payload, {"path", "bytes", "sha256"}, path)
730
+ return SnapshotMember(payload["path"], payload["bytes"], payload["sha256"])
731
+
732
+
733
+ def _mapping(value: Any, path: str) -> Mapping[str, Any]:
734
+ if not isinstance(value, Mapping):
735
+ raise LocalSearchContractError("LOCAL_SEARCH_TYPE", path, "must be an object")
736
+ if any(not isinstance(key, str) for key in value):
737
+ raise LocalSearchContractError("LOCAL_SEARCH_TYPE", path, "keys must be strings")
738
+ return value
739
+
740
+
741
+ def _sequence(value: Any, path: str) -> list[Any]:
742
+ if not isinstance(value, list):
743
+ raise LocalSearchContractError("LOCAL_SEARCH_TYPE", path, "must be an array")
744
+ return value
745
+
746
+
747
+ def _exact(value: Mapping[str, Any], expected: set[str], path: str) -> None:
748
+ actual = set(value)
749
+ if actual != expected:
750
+ raise LocalSearchContractError(
751
+ "LOCAL_SEARCH_FIELDS",
752
+ path,
753
+ f"must contain exactly {sorted(expected)}; missing={sorted(expected - actual)}, "
754
+ f"extra={sorted(actual - expected)}",
755
+ )
756
+
757
+
758
+ def _version(value: Any, expected: str, path: str) -> None:
759
+ if value != expected:
760
+ raise LocalSearchContractError("LOCAL_SEARCH_VERSION", path, f"must equal {expected!r}")
761
+
762
+
763
+ def _text(value: Any, path: str, *, maximum: int) -> str:
764
+ if not isinstance(value, str) or not value or len(value) > maximum:
765
+ raise LocalSearchContractError(
766
+ "LOCAL_SEARCH_TEXT", path, f"must be non-empty text no longer than {maximum}"
767
+ )
768
+ if any(ord(char) < 0x20 or 0x7F <= ord(char) <= 0x9F for char in value):
769
+ raise LocalSearchContractError(
770
+ "LOCAL_SEARCH_TEXT", path, "must not contain control characters"
771
+ )
772
+ return value
773
+
774
+
775
+ def _identifier(value: Any, path: str) -> str:
776
+ text = _text(value, path, maximum=256)
777
+ if _IDENTIFIER.fullmatch(text) is None:
778
+ raise LocalSearchContractError(
779
+ "LOCAL_SEARCH_IDENTIFIER", path, "must be a bounded identifier"
780
+ )
781
+ return text
782
+
783
+
784
+ def _digest(value: Any, path: str) -> str:
785
+ if not isinstance(value, str) or _SHA256.fullmatch(value) is None:
786
+ raise LocalSearchContractError(
787
+ "LOCAL_SEARCH_SHA256", path, "must be a lowercase 64-character SHA-256"
788
+ )
789
+ return value
790
+
791
+
792
+ def _integer(value: Any, path: str, *, minimum: int, maximum: int) -> int:
793
+ if type(value) is not int or not minimum <= value <= maximum:
794
+ raise LocalSearchContractError(
795
+ "LOCAL_SEARCH_INTEGER", path, f"must be an integer in [{minimum}, {maximum}]"
796
+ )
797
+ return value
798
+
799
+
800
+ def _integers(values: Any, path: str, *, maximum: int) -> None:
801
+ if not isinstance(values, tuple) or len(values) > maximum:
802
+ raise LocalSearchContractError(
803
+ "LOCAL_SEARCH_TYPE", path, f"must be an immutable tuple of at most {maximum} values"
804
+ )
805
+ for index, value in enumerate(values):
806
+ _integer(value, f"{path}[{index}]", minimum=0, maximum=MAX_SAFE_INTEGER)
807
+
808
+
809
+ def _choice(value: Any, allowed: frozenset[str], path: str) -> str:
810
+ text = _text(value, path, maximum=MAX_SHORT_TEXT)
811
+ if text not in allowed:
812
+ raise LocalSearchContractError(
813
+ "LOCAL_SEARCH_ENUM", path, f"must be one of {sorted(allowed)}"
814
+ )
815
+ return text
816
+
817
+
818
+ def _texts(
819
+ values: Any,
820
+ path: str,
821
+ *,
822
+ maximum: int,
823
+ item_maximum: int,
824
+ nonempty: bool,
825
+ ) -> None:
826
+ if not isinstance(values, tuple) or len(values) > maximum or (nonempty and not values):
827
+ raise LocalSearchContractError(
828
+ "LOCAL_SEARCH_TYPE", path, f"must be an immutable tuple of at most {maximum} values"
829
+ )
830
+ for index, value in enumerate(values):
831
+ _text(value, f"{path}[{index}]", maximum=item_maximum)
832
+ if len(set(values)) != len(values):
833
+ raise LocalSearchContractError("LOCAL_SEARCH_DUPLICATE", path, "values must be unique")
834
+
835
+
836
+ def _typed_choices(values: Any, allowed: frozenset[str], path: str, maximum: int) -> None:
837
+ if not isinstance(values, tuple) or len(values) > maximum:
838
+ raise LocalSearchContractError(
839
+ "LOCAL_SEARCH_TYPE", path, f"must be an immutable tuple of at most {maximum} values"
840
+ )
841
+ for index, value in enumerate(values):
842
+ _choice(value, allowed, f"{path}[{index}]")
843
+
844
+
845
+ def _typed_tuple(
846
+ values: Any,
847
+ expected: type[Any],
848
+ path: str,
849
+ maximum: int,
850
+ *,
851
+ nonempty: bool = False,
852
+ ) -> None:
853
+ if not isinstance(values, tuple) or len(values) > maximum or (nonempty and not values):
854
+ raise LocalSearchContractError(
855
+ "LOCAL_SEARCH_TYPE", path, f"must be an immutable tuple of at most {maximum} values"
856
+ )
857
+ for index, value in enumerate(values):
858
+ if not isinstance(value, expected):
859
+ raise LocalSearchContractError(
860
+ "LOCAL_SEARCH_TYPE", f"{path}[{index}]", f"must be a {expected.__name__}"
861
+ )
862
+
863
+
864
+ def _optional_type(value: Any, expected: type[Any], path: str) -> None:
865
+ if value is not None and not isinstance(value, expected):
866
+ raise LocalSearchContractError(
867
+ "LOCAL_SEARCH_TYPE", path, f"must be null or a {expected.__name__}"
868
+ )
869
+
870
+
871
+ def _sorted_unique(values: tuple[Any, ...], path: str) -> None:
872
+ if len(set(values)) != len(values):
873
+ raise LocalSearchContractError("LOCAL_SEARCH_DUPLICATE", path, "coordinates must be unique")
874
+ if values != tuple(sorted(values)):
875
+ raise LocalSearchContractError("LOCAL_SEARCH_ORDER", path, "coordinates must be sorted")
876
+
877
+
878
+ def _iso_date(value: Any, path: str) -> date:
879
+ text = _text(value, path, maximum=10)
880
+ try:
881
+ parsed = date.fromisoformat(text)
882
+ except ValueError:
883
+ raise LocalSearchContractError(
884
+ "LOCAL_SEARCH_TIME", path, "must be an exact ISO calendar date"
885
+ ) from None
886
+ if parsed.isoformat() != text:
887
+ raise LocalSearchContractError(
888
+ "LOCAL_SEARCH_TIME", path, "must be an exact ISO calendar date"
889
+ )
890
+ return parsed
891
+
892
+
893
+ def _relative_path(value: Any, path: str) -> str:
894
+ text = _text(value, path, maximum=MAX_PATH)
895
+ if "\\" in text or text.startswith("/"):
896
+ raise LocalSearchContractError(
897
+ "LOCAL_SEARCH_PATH", path, "must be a root-relative POSIX path"
898
+ )
899
+ parsed = PurePosixPath(text)
900
+ if text in {".", ".."} or any(part in {"", ".", ".."} for part in parsed.parts):
901
+ raise LocalSearchContractError(
902
+ "LOCAL_SEARCH_PATH", path, "must be a normalized root-relative POSIX path"
903
+ )
904
+ if parsed.as_posix() != text:
905
+ raise LocalSearchContractError(
906
+ "LOCAL_SEARCH_PATH", path, "must be a normalized root-relative POSIX path"
907
+ )
908
+ return text
909
+
910
+
911
+ def _snapshot_member_path(value: Any) -> str:
912
+ text = _text(value, "member.path", maximum=MAX_PATH)
913
+ allowed = text in {"locators.json", "diagnostics.json"} or re.fullmatch(
914
+ r"entries/[0-9a-f]{64}\.json", text
915
+ )
916
+ if not allowed:
917
+ raise LocalSearchContractError(
918
+ "LOCAL_SEARCH_PATH", "member.path", "is not an allowed private-index member path"
919
+ )
920
+ return text