mostlyright-data 0.9.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (314) hide show
  1. mostlyright/data_harness/__init__.py +158 -0
  2. mostlyright/data_harness/acquisition/__init__.py +55 -0
  3. mostlyright/data_harness/acquisition/http.py +2773 -0
  4. mostlyright/data_harness/acquisition/parsing.py +809 -0
  5. mostlyright/data_harness/acquisition/ranges.py +495 -0
  6. mostlyright/data_harness/acquisition/result_download.py +360 -0
  7. mostlyright/data_harness/acquisition/retention_admission.py +248 -0
  8. mostlyright/data_harness/acquisition/sandbox.py +4888 -0
  9. mostlyright/data_harness/acquisition/url_policy.py +530 -0
  10. mostlyright/data_harness/agent_runtime.py +2743 -0
  11. mostlyright/data_harness/assets/logo-ink.svg +31 -0
  12. mostlyright/data_harness/backends/__init__.py +28 -0
  13. mostlyright/data_harness/backends/pandas_backend.py +350 -0
  14. mostlyright/data_harness/backends/polars_backend.py +366 -0
  15. mostlyright/data_harness/backends/protocol.py +124 -0
  16. mostlyright/data_harness/backends/reference.py +83 -0
  17. mostlyright/data_harness/backends/registry.py +55 -0
  18. mostlyright/data_harness/backends/restrictions.py +126 -0
  19. mostlyright/data_harness/canonical.py +333 -0
  20. mostlyright/data_harness/catalog_job.py +625 -0
  21. mostlyright/data_harness/cli.py +5398 -0
  22. mostlyright/data_harness/contracts.py +53 -0
  23. mostlyright/data_harness/coordinator.py +1307 -0
  24. mostlyright/data_harness/deploy.py +924 -0
  25. mostlyright/data_harness/deploy_target.py +312 -0
  26. mostlyright/data_harness/deployment_evidence.py +1067 -0
  27. mostlyright/data_harness/event_presentation.py +576 -0
  28. mostlyright/data_harness/events.py +2152 -0
  29. mostlyright/data_harness/fast_delimited.py +239 -0
  30. mostlyright/data_harness/fleet.py +237 -0
  31. mostlyright/data_harness/formats.py +236 -0
  32. mostlyright/data_harness/governors.py +1163 -0
  33. mostlyright/data_harness/hosted_bootstrap.py +972 -0
  34. mostlyright/data_harness/hosted_crawler.py +1115 -0
  35. mostlyright/data_harness/hosted_crawler_container_smoke.py +351 -0
  36. mostlyright/data_harness/hosted_crawler_fetch.py +423 -0
  37. mostlyright/data_harness/hosted_crawler_job.py +1277 -0
  38. mostlyright/data_harness/hosted_crawler_protocol.py +676 -0
  39. mostlyright/data_harness/hosted_dataset.py +1500 -0
  40. mostlyright/data_harness/hosted_deploy.py +3037 -0
  41. mostlyright/data_harness/hosted_handoff.py +62 -0
  42. mostlyright/data_harness/hosted_ingestion_contract.py +504 -0
  43. mostlyright/data_harness/hosted_ingestion_job.py +356 -0
  44. mostlyright/data_harness/hosted_ingestion_job_smoke.py +40 -0
  45. mostlyright/data_harness/hosted_session_container_smoke.py +194 -0
  46. mostlyright/data_harness/hosted_session_worker.py +3554 -0
  47. mostlyright/data_harness/hosted_session_worker_job_smoke.py +46 -0
  48. mostlyright/data_harness/hosted_worker.py +6784 -0
  49. mostlyright/data_harness/ingestion/__init__.py +56 -0
  50. mostlyright/data_harness/ingestion/contracts.py +461 -0
  51. mostlyright/data_harness/ingestion/faults.py +42 -0
  52. mostlyright/data_harness/ingestion/gcs_store.py +1162 -0
  53. mostlyright/data_harness/ingestion/spool.py +130 -0
  54. mostlyright/data_harness/ingestion/store.py +885 -0
  55. mostlyright/data_harness/key_seam.py +434 -0
  56. mostlyright/data_harness/linux_process_boundary.py +262 -0
  57. mostlyright/data_harness/local_contracts.py +2880 -0
  58. mostlyright/data_harness/local_search/__init__.py +5 -0
  59. mostlyright/data_harness/local_search/build_index.py +1087 -0
  60. mostlyright/data_harness/local_search/contracts.py +920 -0
  61. mostlyright/data_harness/local_search/query_trace.py +266 -0
  62. mostlyright/data_harness/local_search/retrieval.py +700 -0
  63. mostlyright/data_harness/local_search/sealed.py +474 -0
  64. mostlyright/data_harness/local_search/service.py +784 -0
  65. mostlyright/data_harness/nbrender/CONTRACT.md +212 -0
  66. mostlyright/data_harness/nbrender/__init__.py +12 -0
  67. mostlyright/data_harness/nbrender/chrome.py +359 -0
  68. mostlyright/data_harness/nbrender/code_body.py +266 -0
  69. mostlyright/data_harness/nbrender/document.py +407 -0
  70. mostlyright/data_harness/nbrender/frame.py +275 -0
  71. mostlyright/data_harness/nbrender/interactive.py +337 -0
  72. mostlyright/data_harness/nbrender/markdown_body.py +477 -0
  73. mostlyright/data_harness/nbrender/mr_components.py +134 -0
  74. mostlyright/data_harness/nbrender/outputs_data.py +595 -0
  75. mostlyright/data_harness/nbrender/outputs_rich.py +906 -0
  76. mostlyright/data_harness/nbrender/outputs_source.py +260 -0
  77. mostlyright/data_harness/nbrender/outputs_stage.py +176 -0
  78. mostlyright/data_harness/nbrender/outputs_text.py +400 -0
  79. mostlyright/data_harness/nbrender/parse.py +394 -0
  80. mostlyright/data_harness/nbrender/status.py +40 -0
  81. mostlyright/data_harness/nbrender/tokens.py +1295 -0
  82. mostlyright/data_harness/notebook.py +1710 -0
  83. mostlyright/data_harness/offline.py +2049 -0
  84. mostlyright/data_harness/operation_registry.py +1007 -0
  85. mostlyright/data_harness/operator_setup.py +239 -0
  86. mostlyright/data_harness/pipeline.py +6428 -0
  87. mostlyright/data_harness/plan_graph.py +2026 -0
  88. mostlyright/data_harness/preparation/__init__.py +104 -0
  89. mostlyright/data_harness/preparation/contracts.py +1017 -0
  90. mostlyright/data_harness/preparation/engine.py +221 -0
  91. mostlyright/data_harness/preparation/errors.py +14 -0
  92. mostlyright/data_harness/preparation/gates.py +751 -0
  93. mostlyright/data_harness/preparation/joins.py +574 -0
  94. mostlyright/data_harness/preparation/profile.py +384 -0
  95. mostlyright/data_harness/preparation/table.py +217 -0
  96. mostlyright/data_harness/preparation/transforms.py +568 -0
  97. mostlyright/data_harness/progress_events.py +534 -0
  98. mostlyright/data_harness/readers/__init__.py +46 -0
  99. mostlyright/data_harness/readers/containers.py +963 -0
  100. mostlyright/data_harness/readers/contracts.py +542 -0
  101. mostlyright/data_harness/readers/delimited.py +257 -0
  102. mostlyright/data_harness/readers/grib2/__init__.py +33 -0
  103. mostlyright/data_harness/readers/grib2/admission.py +722 -0
  104. mostlyright/data_harness/readers/grib2/decode.py +1009 -0
  105. mostlyright/data_harness/readers/grib2/geometry.py +1133 -0
  106. mostlyright/data_harness/readers/grib2/portable_math.py +501 -0
  107. mostlyright/data_harness/readers/json_tabular.py +485 -0
  108. mostlyright/data_harness/readers/registry.py +514 -0
  109. mostlyright/data_harness/readers/samples/README.md +110 -0
  110. mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/cities.csv.gz +0 -0
  111. mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/expected.json +24 -0
  112. mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/cities.csv.gz +0 -0
  113. mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/expected.json +24 -0
  114. mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/cities.tar +0 -0
  115. mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/expected.json +24 -0
  116. mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/cities.tar +0 -0
  117. mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/expected.json +24 -0
  118. mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/cities.zip +0 -0
  119. mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/expected.json +25 -0
  120. mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
  121. mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/expected.json +25 -0
  122. mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
  123. mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/expected.json +25 -0
  124. mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/cities.csv +3 -0
  125. mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/expected.json +23 -0
  126. mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/cities.tsv +5 -0
  127. mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/expected.json +25 -0
  128. mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/expected.json +30 -0
  129. mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/observations.csv +5 -0
  130. mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/expected.json +44 -0
  131. mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/stations.json +1 -0
  132. mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/expected.json +48 -0
  133. mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/observations.ndjson +4 -0
  134. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/cities.xlsx +0 -0
  135. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/expected.json +24 -0
  136. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/expected.json +27 -0
  137. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
  138. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/expected.json +27 -0
  139. mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
  140. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/README.md +20 -0
  141. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/expected.json +55 -0
  142. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/gfs-2m-temperature.grib2 +0 -0
  143. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/expected.json +54 -0
  144. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
  145. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/expected.json +54 -0
  146. mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/hrrr-categorical-rain.grib2 +0 -0
  147. mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/expected.json +54 -0
  148. mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
  149. mostlyright/data_harness/readers/samples.py +582 -0
  150. mostlyright/data_harness/readers/spreadsheet.py +803 -0
  151. mostlyright/data_harness/readers/tabular.py +510 -0
  152. mostlyright/data_harness/recipe.py +5321 -0
  153. mostlyright/data_harness/repair/__init__.py +78 -0
  154. mostlyright/data_harness/repair/adapters.py +274 -0
  155. mostlyright/data_harness/repair/contracts.py +872 -0
  156. mostlyright/data_harness/repair/coordinator.py +1099 -0
  157. mostlyright/data_harness/repair/errors.py +16 -0
  158. mostlyright/data_harness/review.py +2533 -0
  159. mostlyright/data_harness/rowset.py +283 -0
  160. mostlyright/data_harness/serving.py +1975 -0
  161. mostlyright/data_harness/serving_edge.py +590 -0
  162. mostlyright/data_harness/serving_http.py +1031 -0
  163. mostlyright/data_harness/session_probes.py +759 -0
  164. mostlyright/data_harness/signing.py +101 -0
  165. mostlyright/data_harness/source_discovery.py +898 -0
  166. mostlyright/data_harness/sources/__init__.py +209 -0
  167. mostlyright/data_harness/sources/_adapter_steps.py +213 -0
  168. mostlyright/data_harness/sources/adapters.py +1214 -0
  169. mostlyright/data_harness/sources/cadence.py +1428 -0
  170. mostlyright/data_harness/sources/cadence_emission.py +453 -0
  171. mostlyright/data_harness/sources/cadence_history.py +546 -0
  172. mostlyright/data_harness/sources/catalog/__init__.py +17 -0
  173. mostlyright/data_harness/sources/catalog/admission.py +477 -0
  174. mostlyright/data_harness/sources/catalog/authoring.py +1701 -0
  175. mostlyright/data_harness/sources/catalog/authoring_policy.py +701 -0
  176. mostlyright/data_harness/sources/catalog/authoring_shards.py +1217 -0
  177. mostlyright/data_harness/sources/catalog/bounded_io.py +231 -0
  178. mostlyright/data_harness/sources/catalog/channel.py +523 -0
  179. mostlyright/data_harness/sources/catalog/channel_client.py +296 -0
  180. mostlyright/data_harness/sources/catalog/contracts.py +825 -0
  181. mostlyright/data_harness/sources/catalog/coverage.py +137 -0
  182. mostlyright/data_harness/sources/catalog/delta.py +1340 -0
  183. mostlyright/data_harness/sources/catalog/embedding.py +532 -0
  184. mostlyright/data_harness/sources/catalog/entry_v2.py +1182 -0
  185. mostlyright/data_harness/sources/catalog/fill.py +3889 -0
  186. mostlyright/data_harness/sources/catalog/fill_partitions.py +459 -0
  187. mostlyright/data_harness/sources/catalog/fill_staging.py +1105 -0
  188. mostlyright/data_harness/sources/catalog/gating.py +374 -0
  189. mostlyright/data_harness/sources/catalog/generation_receipt.py +1607 -0
  190. mostlyright/data_harness/sources/catalog/harvest/__init__.py +7 -0
  191. mostlyright/data_harness/sources/catalog/harvest/ckan.py +384 -0
  192. mostlyright/data_harness/sources/catalog/harvest/datagov_v4.py +798 -0
  193. mostlyright/data_harness/sources/catalog/harvest/protocol.py +964 -0
  194. mostlyright/data_harness/sources/catalog/harvest/sdmx.py +445 -0
  195. mostlyright/data_harness/sources/catalog/harvest/stac.py +384 -0
  196. mostlyright/data_harness/sources/catalog/health.py +447 -0
  197. mostlyright/data_harness/sources/catalog/hosted_catalog.py +105 -0
  198. mostlyright/data_harness/sources/catalog/identity_history.py +1549 -0
  199. mostlyright/data_harness/sources/catalog/neural.py +1618 -0
  200. mostlyright/data_harness/sources/catalog/packed_catalog.py +2345 -0
  201. mostlyright/data_harness/sources/catalog/packed_retrieval.py +1517 -0
  202. mostlyright/data_harness/sources/catalog/packed_writer.py +2802 -0
  203. mostlyright/data_harness/sources/catalog/query_trace.py +1037 -0
  204. mostlyright/data_harness/sources/catalog/recommend.py +171 -0
  205. mostlyright/data_harness/sources/catalog/retrieval.py +230 -0
  206. mostlyright/data_harness/sources/catalog/retrieval_manifest.py +995 -0
  207. mostlyright/data_harness/sources/catalog/rights_decisions.py +254 -0
  208. mostlyright/data_harness/sources/catalog/sealed.py +560 -0
  209. mostlyright/data_harness/sources/catalog/search.py +230 -0
  210. mostlyright/data_harness/sources/catalog/streaming_delta.py +1097 -0
  211. mostlyright/data_harness/sources/catalog/update.py +891 -0
  212. mostlyright/data_harness/sources/collections.py +815 -0
  213. mostlyright/data_harness/sources/contracts.py +2223 -0
  214. mostlyright/data_harness/sources/deletion.py +761 -0
  215. mostlyright/data_harness/sources/fitness.py +162 -0
  216. mostlyright/data_harness/sources/governance.py +163 -0
  217. mostlyright/data_harness/sources/hosted.py +173 -0
  218. mostlyright/data_harness/sources/integration.py +218 -0
  219. mostlyright/data_harness/sources/range_reader.py +418 -0
  220. mostlyright/data_harness/sources/registry.py +514 -0
  221. mostlyright/data_harness/sources/rights_rule.py +59 -0
  222. mostlyright/data_harness/sources/source_cadence_vectors.v1.json +1 -0
  223. mostlyright/data_harness/sources/sports.py +521 -0
  224. mostlyright/data_harness/sources/stream.py +524 -0
  225. mostlyright/data_harness/sources/stream_connector.py +418 -0
  226. mostlyright/data_harness/sources/stream_recorder.py +1404 -0
  227. mostlyright/data_harness/studio_boundary.py +2019 -0
  228. mostlyright/data_harness/thin/__init__.py +37 -0
  229. mostlyright/data_harness/thin/acquire.py +1137 -0
  230. mostlyright/data_harness/thin/acquire_cancel.py +579 -0
  231. mostlyright/data_harness/thin/approvals.py +617 -0
  232. mostlyright/data_harness/thin/commands.py +406 -0
  233. mostlyright/data_harness/thin/download.py +194 -0
  234. mostlyright/data_harness/thin/narrative.py +589 -0
  235. mostlyright/data_harness/thin/parity.py +1070 -0
  236. mostlyright/data_harness/thin/propose.py +2759 -0
  237. mostlyright/data_harness/thin/research.py +1663 -0
  238. mostlyright/data_harness/thin/router.py +924 -0
  239. mostlyright/data_harness/thin/runs.py +519 -0
  240. mostlyright/data_harness/thin/session.py +281 -0
  241. mostlyright/data_harness/thin/stream.py +501 -0
  242. mostlyright/data_harness/thin/transport.py +187 -0
  243. mostlyright/data_harness/thin/vocabulary.py +368 -0
  244. mostlyright/data_harness/thin/workers.py +164 -0
  245. mostlyright/data_harness/ucum/TABLE-PIN.json +40 -0
  246. mostlyright/data_harness/ucum/ucum-subset.v1.json +632 -0
  247. mostlyright/data_harness/unit_flow.py +927 -0
  248. mostlyright/data_harness/units.py +572 -0
  249. mostlyright/data_harness/ux/__init__.py +9 -0
  250. mostlyright/data_harness/ux/approve.py +485 -0
  251. mostlyright/data_harness/ux/author_yaml.py +597 -0
  252. mostlyright/data_harness/ux/cloud_auth.py +447 -0
  253. mostlyright/data_harness/ux/commands/__init__.py +260 -0
  254. mostlyright/data_harness/ux/commands/approve.py +136 -0
  255. mostlyright/data_harness/ux/commands/auth.py +744 -0
  256. mostlyright/data_harness/ux/commands/author.py +79 -0
  257. mostlyright/data_harness/ux/commands/catalog_author.py +403 -0
  258. mostlyright/data_harness/ux/commands/catalog_fill.py +523 -0
  259. mostlyright/data_harness/ux/commands/catalog_harvest.py +545 -0
  260. mostlyright/data_harness/ux/commands/catalog_publish.py +1838 -0
  261. mostlyright/data_harness/ux/commands/catalog_search.py +71 -0
  262. mostlyright/data_harness/ux/commands/catalog_update.py +437 -0
  263. mostlyright/data_harness/ux/commands/deploy.py +134 -0
  264. mostlyright/data_harness/ux/commands/deploy_dataset.py +98 -0
  265. mostlyright/data_harness/ux/commands/deploy_plan.py +105 -0
  266. mostlyright/data_harness/ux/commands/deploy_status.py +104 -0
  267. mostlyright/data_harness/ux/commands/diff.py +74 -0
  268. mostlyright/data_harness/ux/commands/index.py +84 -0
  269. mostlyright/data_harness/ux/commands/inventory.py +47 -0
  270. mostlyright/data_harness/ux/commands/list_builds.py +143 -0
  271. mostlyright/data_harness/ux/commands/login.py +63 -0
  272. mostlyright/data_harness/ux/commands/peek.py +236 -0
  273. mostlyright/data_harness/ux/commands/plan_check.py +90 -0
  274. mostlyright/data_harness/ux/commands/preflight.py +97 -0
  275. mostlyright/data_harness/ux/commands/record.py +107 -0
  276. mostlyright/data_harness/ux/commands/review_setup.py +47 -0
  277. mostlyright/data_harness/ux/commands/search.py +440 -0
  278. mostlyright/data_harness/ux/commands/show.py +61 -0
  279. mostlyright/data_harness/ux/commands/whoami.py +37 -0
  280. mostlyright/data_harness/ux/credential_native.py +551 -0
  281. mostlyright/data_harness/ux/credential_store.py +1055 -0
  282. mostlyright/data_harness/ux/credentials.py +631 -0
  283. mostlyright/data_harness/ux/diffing.py +444 -0
  284. mostlyright/data_harness/ux/headline.py +671 -0
  285. mostlyright/data_harness/ux/hosted_acquisition.py +974 -0
  286. mostlyright/data_harness/ux/hosted_run_status.py +619 -0
  287. mostlyright/data_harness/ux/inventory.py +427 -0
  288. mostlyright/data_harness/ux/local_review.py +375 -0
  289. mostlyright/data_harness/ux/login.py +691 -0
  290. mostlyright/data_harness/ux/path_kind.py +147 -0
  291. mostlyright/data_harness/ux/peek.py +1000 -0
  292. mostlyright/data_harness/ux/plain_file.py +178 -0
  293. mostlyright/data_harness/ux/plan_check.py +311 -0
  294. mostlyright/data_harness/ux/preflight.py +918 -0
  295. mostlyright/data_harness/ux/readers.py +1124 -0
  296. mostlyright/data_harness/ux/remediation.py +2195 -0
  297. mostlyright/data_harness/ux/render.py +657 -0
  298. mostlyright/data_harness/ux/workload.py +1077 -0
  299. mostlyright/data_harness/viewer.py +3713 -0
  300. mostlyright/data_harness/visual_run/__init__.py +83 -0
  301. mostlyright/data_harness/visual_run/authoring.py +235 -0
  302. mostlyright/data_harness/visual_run/contracts.py +673 -0
  303. mostlyright/data_harness/visual_run/materialize.py +486 -0
  304. mostlyright/data_harness/visual_run/observations.py +874 -0
  305. mostlyright/data_harness/visual_run/query.py +259 -0
  306. mostlyright/data_harness/visual_run/reducer.py +280 -0
  307. mostlyright/data_harness/visual_run/sdk.py +892 -0
  308. mostlyright/data_harness/visual_run/store.py +584 -0
  309. mostlyright/data_harness/visual_run/transport.py +239 -0
  310. mostlyright/data_harness/watch.py +2999 -0
  311. mostlyright_data-0.9.0.dist-info/METADATA +607 -0
  312. mostlyright_data-0.9.0.dist-info/RECORD +314 -0
  313. mostlyright_data-0.9.0.dist-info/WHEEL +4 -0
  314. mostlyright_data-0.9.0.dist-info/entry_points.txt +12 -0
@@ -0,0 +1,803 @@
1
+ """The spreadsheet Reader: strict coordinates, cached formula values surfaced and flagged.
2
+
3
+ A spreadsheet is not a table. It is a drawing surface that usually has a table on it, and
4
+ everything difficult about reading one follows from that sentence. Two design decisions
5
+ answer it, and both are unusual enough to be written down rather than left to be inferred.
6
+
7
+ Strict coordinates
8
+ ------------------
9
+ The recipe pins three things -- the sheet by name, the header row, and the cell range -- and
10
+ every one of them is checked. A workbook that grows a sheet, renames a tab, or shifts its
11
+ table down two rows produces a named refusal rather than silently different data. There is
12
+ no sniffing, no "the first sheet", no "the first non-empty row", and no index-based sheet
13
+ selection: sheet order changes without notice and an index would move a dataset from under a
14
+ recipe whose digest never changed.
15
+
16
+ The three coordinates are checked against each other as well as against the file. The header
17
+ row must be the range's first row, so every row inside the range is accounted for -- one
18
+ header and the rest data -- and no row is silently neither. A recipe whose range was edited
19
+ without its header row is therefore a refusal that names the disagreement, rather than a
20
+ dataset quietly missing its first records.
21
+
22
+ Cached formula values, surfaced and flagged
23
+ -------------------------------------------
24
+ A formula's result is not in the file in any authoritative sense. It is whatever the
25
+ authoring application last cached, and nothing here recomputes it -- this is a Reader, not a
26
+ spreadsheet engine. Using the cached value is the only useful behaviour, and pretending it is
27
+ the same as a typed value would be dishonest, so the decode reports ``contains_formula_cells``
28
+ and the receipt carries it forward. A reviewer looking at a dataset can then see that some of
29
+ its numbers were somebody else's arithmetic.
30
+
31
+ A formula with no cached value is refused rather than emitted as an empty cell. An uncomputed
32
+ formula is missing *information*; an empty cell is a stated absence of a value. Collapsing
33
+ the two would let a workbook that was never recalculated pass as a workbook full of blanks.
34
+
35
+ That flag is read from evidence, which is what the two passes buy. The workbook is loaded
36
+ twice from the same bytes: once with ``data_only=True``, which yields the cached values, and
37
+ once without, which yields each cell's type so a formula can be told from a string that merely
38
+ begins with an equals sign. Two bounded passes over an in-memory workbook is a fair price for
39
+ a flag that is honest, and the price is stated here so it is not later optimised into a guess.
40
+
41
+ The date rule, pinned
42
+ ---------------------
43
+ Excel date handling is the single most common source of quiet corruption in spreadsheet
44
+ ingestion, so the rule is stated with its examples rather than surrounded by caution.
45
+
46
+ * The 1900 date system is the only one read. A workbook declaring the 1904 system is refused
47
+ by name, because the two differ by four years and a day and the difference is invisible in
48
+ a recipe: the same coordinates would produce different dates with nothing to see.
49
+ * A date or a datetime renders as ISO 8601 with a ``T`` separator and seconds always present:
50
+ ``2026-08-07T00:00:00`` for a date, ``2026-08-07T13:05:01`` for a datetime. A date-only
51
+ cell is a datetime at midnight, because that is what the format stores.
52
+ * A time-only cell renders as ``06:30:00``.
53
+ * A duration is refused. Excel stores one temporal type and a duration is a rendering of it
54
+ rather than a value in it, so there is no honest single text form.
55
+
56
+ The float rule and the output encoding are not this module's to state. Numbers reach the
57
+ shared canonical emitter as binary64 values and are rendered there, by the one implementation
58
+ every family emits through. A spreadsheet is exactly where an engine's own string conversion
59
+ would slip into the output unnoticed, so the engine's text never reaches it.
60
+
61
+ Nothing on disk
62
+ ---------------
63
+ The workbook is constructed from ``io.BytesIO`` with ``read_only=True``, never from a path.
64
+ No API taking a filesystem path is named in this module, and a test observes two watched
65
+ directories that are empty before and after a decode.
66
+
67
+ The xlsx-is-a-zip boundary
68
+ --------------------------
69
+ An xlsx file is a zip of OOXML parts. Those parts are the format's internal structure, not a
70
+ user-visible container nesting, so the container families' "exactly one recipe-named member"
71
+ rule does not apply to them and this family does not compose with ``archive.zip``. What is
72
+ shared with that module is the zip prefix table and the bounded read, because both modules are
73
+ looking at the same bytes and a second copy of either would let them drift. What is not
74
+ shared is the member rule, and stating the difference here is cheaper than leaving a reader to
75
+ wonder whether one was forgotten.
76
+
77
+ A recipe wanting a zipped xlsx is therefore ``archive.zip`` naming the ``.xlsx`` member -- and
78
+ the container family's nesting refusal will refuse it, because an xlsx is an archive by magic.
79
+ That is a known limitation recorded in the Certification record with its reason, rather than a
80
+ special case carved into the magic table.
81
+
82
+ The engine, and why it is part of the pin
83
+ -----------------------------------------
84
+ ``openpyxl`` at an exact version, with its XML backend checked at decode time. ``openpyxl``
85
+ chooses its parser at import from what it finds on the host, so a decoder whose parser depends
86
+ on which other packages happen to be installed is a decoder that can answer differently on two
87
+ hosts. The pinned backend is the standard library's, and a host where the other one was
88
+ selected is refused rather than decoded. The full ruling, including why this family does not
89
+ follow the design document's calamine pin, is in the Certification record.
90
+ """
91
+
92
+ from __future__ import annotations
93
+
94
+ import datetime
95
+ import io
96
+ import re
97
+ import zipfile
98
+ import zlib
99
+ from collections.abc import Mapping, Sequence
100
+ from dataclasses import dataclass, field
101
+ from typing import Any
102
+
103
+ import openpyxl.xml
104
+ from openpyxl import load_workbook
105
+
106
+ from mostlyright.data_harness.formats import (
107
+ FORMAT_MEDIA_TYPES,
108
+ FORMAT_SUFFIXES,
109
+ READER_CONTRACT_VERSION,
110
+ READER_OUTPUT_FORMATS,
111
+ READER_WORKER_MEMORY_BYTES,
112
+ )
113
+ from mostlyright.data_harness.readers.containers import (
114
+ ZIP_MAGICS,
115
+ read_bounded,
116
+ screen_zip_entry,
117
+ )
118
+ from mostlyright.data_harness.readers.contracts import (
119
+ DECODE_FLAGS,
120
+ ReaderBudgets,
121
+ ReaderError,
122
+ ReaderPin,
123
+ ReaderResult,
124
+ )
125
+ from mostlyright.data_harness.readers.registry import spreadsheet_default_budgets
126
+ from mostlyright.data_harness.readers.tabular import (
127
+ check_declared_size,
128
+ encode_canonical_csv,
129
+ sealed_filename,
130
+ )
131
+
132
+ __all__ = [
133
+ "SPREADSHEET_MAX_DECLARED_CELLS",
134
+ "SPREADSHEET_MEDIA_TYPE",
135
+ "SPREADSHEET_WORST_CASE_BYTES",
136
+ "SpreadsheetReader",
137
+ ]
138
+
139
+ # The one encoding a Reader may seal, unpacked rather than restated.
140
+ (_OUTPUT_FORMAT,) = READER_OUTPUT_FORMATS
141
+ _OUTPUT_MEDIA_TYPE = sorted(FORMAT_MEDIA_TYPES[_OUTPUT_FORMAT])[0]
142
+ _OUTPUT_SUFFIX = sorted(FORMAT_SUFFIXES[_OUTPUT_FORMAT])[0]
143
+
144
+ # The one media type this family admits as input. Public because the acquisition layer asks
145
+ # the Toolbox which media types a pinned fetch may accept, and a re-spelled literal at that
146
+ # call site would be a second declaration of the same fact.
147
+ SPREADSHEET_MEDIA_TYPE = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
148
+
149
+ # Spreadsheet decoding retains the first pass's values while a second read-only pass identifies
150
+ # formula cells. The generic Reader ceiling of ten million cells permits Python-object overhead
151
+ # to approach or exceed the Linux child boundary even when the compressed workbook is small. This
152
+ # family therefore admits a quarter-million cells. The accounting deliberately charges 512 bytes
153
+ # per retained cell plus four copies of the admitted expanded package and both byte envelopes; it
154
+ # is conservative rather than self-calibrating and remains below the sandbox's exact process cap.
155
+ SPREADSHEET_MAX_DECLARED_CELLS = spreadsheet_default_budgets().max_declared_cells
156
+ _SPREADSHEET_MAX_UNCOMPRESSED_BYTES = spreadsheet_default_budgets().max_uncompressed_bytes
157
+ _SPREADSHEET_CELL_MEMORY_ALLOWANCE = 512
158
+ SPREADSHEET_WORST_CASE_BYTES = (
159
+ SPREADSHEET_MAX_DECLARED_CELLS * _SPREADSHEET_CELL_MEMORY_ALLOWANCE
160
+ + 4 * _SPREADSHEET_MAX_UNCOMPRESSED_BYTES
161
+ + 2 * 16 * 1024 * 1024
162
+ )
163
+ if SPREADSHEET_WORST_CASE_BYTES >= READER_WORKER_MEMORY_BYTES:
164
+ raise ReaderError(
165
+ "READER_BUDGET",
166
+ "reader.spreadsheet.memory",
167
+ "certified spreadsheet admission exceeds the Reader worker memory boundary",
168
+ )
169
+ # The fact this family reports about itself, taken from the closed vocabulary rather than
170
+ # spelled as a literal at the raise site. A build in which the two have drifted apart fails at
171
+ # import, which is the loudest and earliest place it can fail.
172
+ _FORMULA_FLAG = "contains_formula_cells"
173
+ if _FORMULA_FLAG not in DECODE_FLAGS: # pragma: no cover - a contradiction caught at import
174
+ raise ReaderError(
175
+ "READER_OUTPUT",
176
+ "reader.spreadsheet.flags",
177
+ f"{_FORMULA_FLAG} is not in the closed decode-flag vocabulary",
178
+ )
179
+
180
+ # The closed option set. All three are required and there is no fourth: an option that was
181
+ # accepted and ignored would be an unreviewable difference between two recipes that read
182
+ # identically to everyone who looks at them.
183
+ _OPTION_KEYS = ("header_row", "range", "sheet")
184
+
185
+ # The parts every workbook package carries. Checked by name so a zip that is not a workbook is
186
+ # refused with a sentence rather than escaping as whichever exception the engine happened to
187
+ # raise while looking for them.
188
+ _REQUIRED_PARTS = ("[Content_Types].xml", "xl/workbook.xml")
189
+
190
+ # Which parts are scanned before the engine sees them. Text parts only: a binary part that
191
+ # happened to contain these bytes is not a declaration, and refusing on one would be a false
192
+ # refusal with no fix a person could apply.
193
+ _TEXT_PART_SUFFIXES = (".xml", ".rels")
194
+
195
+ # An XML document type may define an entity that names a resource outside the file, which is
196
+ # the external-entity class. The pinned backend refuses one on its own -- a test proves that
197
+ # rather than assuming it -- and this is the layer in front of the engine, so the file is
198
+ # refused before the engine is handed it at all.
199
+ _DOCUMENT_TYPE = re.compile(rb"<!DOCTYPE|<!ENTITY", re.IGNORECASE)
200
+
201
+ # A workbook may name a resource outside itself: an external link to another workbook, or a
202
+ # relationship whose target is external. Refused rather than followed, on the same reasoning
203
+ # the acquisition parser already applies to a forbidden reference.
204
+ _EXTERNAL_REFERENCE = re.compile(
205
+ rb"TargetMode\s*=\s*[\"']External[\"']|/relationships/externalLink",
206
+ re.IGNORECASE,
207
+ )
208
+
209
+ # The 1900 date system, as the engine spells its epoch. The 1904 system is the other one the
210
+ # format permits and is refused by name.
211
+ _EPOCH_1900 = datetime.datetime(1899, 12, 30)
212
+ _EPOCH_1904 = datetime.datetime(1904, 1, 1)
213
+
214
+ # A sheet name is bounded text. Thirty-one characters is the format's own ceiling, so a longer
215
+ # one names a sheet no workbook could hold.
216
+ _SHEET_NAME_MAX_CHARACTERS = 31
217
+
218
+ # A plain cell reference: uppercase column letters and a row number, with no dollar signs and
219
+ # no whole-row or whole-column form. An absolute reference and a relative one mean the same
220
+ # cell, and admitting both spellings would let two recipes with different digests seal the same
221
+ # bytes.
222
+ _CELL_REFERENCE = re.compile(r"^([A-Z]{1,3})([1-9][0-9]{0,6})$")
223
+
224
+ # The format's own grid limits. A coordinate outside them names a cell that cannot exist.
225
+ _MAX_ROW = 1_048_576
226
+ _MAX_COLUMN = 16_384
227
+
228
+
229
+ def _column_index(letters: str) -> int:
230
+ """Turn column letters into a one-based index, on the format's base-26 rule."""
231
+
232
+ index = 0
233
+ for character in letters:
234
+ index = index * 26 + (ord(character) - ord("A") + 1)
235
+ return index
236
+
237
+
238
+ def _column_letters(index: int) -> str:
239
+ """Turn a one-based column index back into letters, for a refusal a person can act on."""
240
+
241
+ letters = ""
242
+ while index > 0:
243
+ index, remainder = divmod(index - 1, 26)
244
+ letters = chr(ord("A") + remainder) + letters
245
+ return letters
246
+
247
+
248
+ def _coordinate(row: int, column: int) -> str:
249
+ return f"{_column_letters(column)}{row}"
250
+
251
+
252
+ def _cell_reference(value: str, subject: str) -> tuple[int, int]:
253
+ """Admit one plain cell reference as a (column, row) pair, or refuse it by name."""
254
+
255
+ matched = _CELL_REFERENCE.fullmatch(value)
256
+ if matched is None:
257
+ raise ReaderError(
258
+ "READER_OPTIONS",
259
+ subject,
260
+ f"{value!r} is not a plain cell reference such as A1; a whole row, a whole "
261
+ "column, a lower-case reference, and an absolute reference are all refused",
262
+ )
263
+ column = _column_index(matched.group(1))
264
+ row = int(matched.group(2))
265
+ if column > _MAX_COLUMN or row > _MAX_ROW:
266
+ raise ReaderError(
267
+ "READER_OPTIONS",
268
+ subject,
269
+ f"{value!r} names a cell outside the format's grid",
270
+ )
271
+ return column, row
272
+
273
+
274
+ def _validate_range(value: Any, subject: str) -> tuple[int, int, int, int]:
275
+ """Admit a range as two plain cell references, the first at or before the second."""
276
+
277
+ if not isinstance(value, str) or value.count(":") != 1:
278
+ raise ReaderError(
279
+ "READER_OPTIONS",
280
+ subject,
281
+ "must be two plain cell references separated by a colon, such as A1:C4",
282
+ )
283
+ first, second = value.split(":")
284
+ start_column, start_row = _cell_reference(first, subject)
285
+ end_column, end_row = _cell_reference(second, subject)
286
+ if start_column > end_column or start_row > end_row:
287
+ raise ReaderError(
288
+ "READER_OPTIONS",
289
+ subject,
290
+ f"{value!r} runs backwards; a range states its top-left cell first",
291
+ )
292
+ return start_column, start_row, end_column, end_row
293
+
294
+
295
+ def _validate_options(options: Any) -> dict[str, Any]:
296
+ """Admit this family's closed option set, refusing a missing or unknown key by name."""
297
+
298
+ subject = "reader.spreadsheet.xlsx.decode_options"
299
+ if not isinstance(options, Mapping):
300
+ raise ReaderError("READER_OPTIONS", subject, "must be an object")
301
+ unknown = sorted(str(key) for key in options if key not in _OPTION_KEYS)
302
+ if unknown:
303
+ raise ReaderError(
304
+ "READER_OPTIONS",
305
+ subject,
306
+ f"names no such setting: {', '.join(unknown)}; this family takes "
307
+ f"{', '.join(_OPTION_KEYS)}",
308
+ )
309
+ missing = sorted(key for key in _OPTION_KEYS if key not in options)
310
+ if missing:
311
+ raise ReaderError(
312
+ "READER_OPTIONS",
313
+ subject,
314
+ f"does not state: {', '.join(missing)}; a spreadsheet is read at pinned "
315
+ "coordinates and all three are required",
316
+ )
317
+
318
+ sheet = options["sheet"]
319
+ if (
320
+ not isinstance(sheet, str)
321
+ or isinstance(sheet, bool)
322
+ or not sheet
323
+ or len(sheet) > _SHEET_NAME_MAX_CHARACTERS
324
+ or any(character < " " or character == "\x7f" for character in sheet)
325
+ ):
326
+ raise ReaderError(
327
+ "READER_OPTIONS",
328
+ f"{subject}.sheet",
329
+ "must name one sheet, as bounded text; a sheet is never selected by index, "
330
+ "because sheet order changes without notice and would move a dataset silently",
331
+ )
332
+
333
+ header_row = options["header_row"]
334
+ if type(header_row) is not int or header_row < 1 or header_row > _MAX_ROW:
335
+ raise ReaderError(
336
+ "READER_OPTIONS",
337
+ f"{subject}.header_row",
338
+ "must be the whole number of the row that names the columns",
339
+ )
340
+
341
+ _validate_range(options["range"], f"{subject}.range")
342
+ return {"sheet": sheet, "header_row": header_row, "range": options["range"]}
343
+
344
+
345
+ def _admit_workbook_bytes(content: Any, *, budgets: ReaderBudgets, subject: str) -> bytes:
346
+ """Admit the package's own bytes, before any decoder object is constructed."""
347
+
348
+ if not isinstance(content, (bytes, bytearray)):
349
+ raise ReaderError("READER_ADMISSION", subject, "must be exact bytes")
350
+ raw = bytes(content)
351
+ if len(raw) > budgets.max_input_bytes:
352
+ raise ReaderError("READER_BUDGET", subject, "exceeds the input byte budget")
353
+ if not any(raw.startswith(magic) for magic in ZIP_MAGICS):
354
+ raise ReaderError(
355
+ "READER_ADMISSION",
356
+ subject,
357
+ "does not begin as a workbook package",
358
+ )
359
+ return raw
360
+
361
+
362
+ def _inspect_package(raw: bytes, *, budgets: ReaderBudgets, subject: str) -> None:
363
+ """Decide against the package's own directory, and read its text parts, before the engine.
364
+
365
+ Everything here happens with the engine still untouched. The sizes come from the central
366
+ directory, so a package built to exhaust memory is refused with nothing inflated; the text
367
+ parts are then read under those checked declarations and scanned for the two things a
368
+ workbook may not say -- a document type, and a reference to a resource outside itself.
369
+ """
370
+
371
+ try:
372
+ archive = zipfile.ZipFile(io.BytesIO(raw))
373
+ except (zipfile.BadZipFile, EOFError, ValueError, zlib.error) as error:
374
+ raise ReaderError(
375
+ "READER_ADMISSION",
376
+ subject,
377
+ "is not a readable workbook package",
378
+ ) from error
379
+
380
+ with archive:
381
+ # Reading the directory is the second call into the engine, and the structural test in
382
+ # ``tests/h3/test_readers_xlsx.py`` requires every one of them to be translated -- not
383
+ # because this one is known to raise, but because "known to raise" is exactly the
384
+ # judgement that let the part reads escape untyped in the first place.
385
+ try:
386
+ entries = archive.infolist()
387
+ except Exception as error:
388
+ raise ReaderError(
389
+ "READER_ADMISSION",
390
+ subject,
391
+ f"has a package directory that could not be read: {type(error).__name__}",
392
+ ) from error
393
+ if len(entries) > budgets.max_container_members:
394
+ raise ReaderError(
395
+ "READER_BUDGET",
396
+ subject,
397
+ f"holds {len(entries)} parts, over the part budget of "
398
+ f"{budgets.max_container_members}",
399
+ )
400
+ declared = sum(entry.file_size for entry in entries)
401
+ if declared > budgets.max_uncompressed_bytes:
402
+ raise ReaderError(
403
+ "READER_BUDGET",
404
+ subject,
405
+ f"declares {declared} uncompressed bytes across its parts, over the "
406
+ "uncompressed byte budget",
407
+ )
408
+ if declared > max(len(raw), 1) * budgets.max_expansion_ratio:
409
+ raise ReaderError(
410
+ "READER_BUDGET",
411
+ subject,
412
+ f"declares an expansion of {declared} from {len(raw)} bytes, over the "
413
+ f"expansion ratio budget of {budgets.max_expansion_ratio}",
414
+ )
415
+
416
+ names = {entry.filename for entry in entries}
417
+ absent = [part for part in _REQUIRED_PARTS if part not in names]
418
+ if absent:
419
+ raise ReaderError(
420
+ "READER_ADMISSION",
421
+ subject,
422
+ f"is a zip that carries no workbook: it has no {', '.join(absent)}",
423
+ )
424
+
425
+ for entry in entries:
426
+ if not entry.filename.endswith(_TEXT_PART_SUFFIXES):
427
+ continue
428
+ where = f"{subject}.part"
429
+ # The same three directory screens the zip family applies, from the same
430
+ # implementation. Without them an encrypted part escapes as the bare
431
+ # ``RuntimeError`` ``ZipFile.open`` raises, and a part compressed by a method
432
+ # this interpreter was not built with escapes as ``NotImplementedError`` -- both
433
+ # outside the Reader contract's closed table and outside the worker's typed
434
+ # handler, so a refusal a recipe could repair arrives as a codeless crash.
435
+ screen_zip_entry(entry, subject=where, described=f"holds {entry.filename!r}")
436
+ part = _read_package_part(archive, entry, subject=where)
437
+ if _DOCUMENT_TYPE.search(part) is not None:
438
+ raise ReaderError(
439
+ "READER_ADMISSION",
440
+ where,
441
+ f"{entry.filename} declares a document type; a document type may define "
442
+ "an entity naming a resource outside the file, so the workbook is "
443
+ "refused rather than parsed",
444
+ )
445
+ if _EXTERNAL_REFERENCE.search(part) is not None:
446
+ raise ReaderError(
447
+ "READER_ADMISSION",
448
+ where,
449
+ f"{entry.filename} names an external resource; a workbook that points "
450
+ "outside itself is refused rather than followed",
451
+ )
452
+
453
+
454
+ def _read_package_part(archive: zipfile.ZipFile, entry: zipfile.ZipInfo, *, subject: str) -> bytes:
455
+ """Read one part of the package, mapping every engine failure to a typed refusal.
456
+
457
+ The directory screens above remove the two failures we know the name of; this removes the
458
+ class. ``ZipFile.open`` and the read behind it reach CPython's zip and zlib layers, which
459
+ raise whatever their own layers raise, and an untyped escape from a decode is a refusal
460
+ with no code -- which the clean room reports as ``SANDBOX_UNEXPECTED`` and the remediation
461
+ map cannot bind a fix to. ``ReaderError`` is re-raised first so a budget refusal from
462
+ ``read_bounded`` keeps its own code rather than being re-labelled here.
463
+ """
464
+
465
+ try:
466
+ with archive.open(entry, "r") as stream:
467
+ return read_bounded(stream, declared=entry.file_size, subject=subject)
468
+ except ReaderError:
469
+ raise
470
+ # Deliberately broad, and for the reason ``_load``'s handler is broad. ``RuntimeError``
471
+ # and ``NotImplementedError`` are the two this module has met; naming only those two
472
+ # would leave the next one to escape untyped.
473
+ except Exception as error:
474
+ raise ReaderError(
475
+ "READER_ADMISSION",
476
+ subject,
477
+ f"{entry.filename} could not be read out of the workbook package: "
478
+ f"{type(error).__name__}",
479
+ ) from error
480
+
481
+
482
+ def _check_backend(subject: str) -> None:
483
+ """Refuse a decode on a host whose XML backend is not the pinned one.
484
+
485
+ ``openpyxl`` selects its parser at import from what it finds on the host, so this is a
486
+ property of the environment rather than of the file -- which is why the refusal is a
487
+ contract refusal and why its message names both ways to fix the environment.
488
+ """
489
+
490
+ if openpyxl.xml.LXML:
491
+ raise ReaderError(
492
+ "READER_CONTRACT",
493
+ subject,
494
+ "is running on a host where openpyxl selected the lxml XML backend; this family "
495
+ "is pinned to the standard library backend, because a decoder whose parser "
496
+ "depends on which other packages are installed can answer differently on two "
497
+ "hosts. Uninstall lxml, or set OPENPYXL_LXML=0, and run the decode again",
498
+ )
499
+
500
+
501
+ def _load(raw: bytes, *, data_only: bool, subject: str) -> Any:
502
+ """Open the workbook from bytes, read-only, mapping every engine failure to a typed one."""
503
+
504
+ try:
505
+ return load_workbook(io.BytesIO(raw), read_only=True, data_only=data_only)
506
+ except ReaderError:
507
+ raise
508
+ # Deliberately broad. The engine raises whatever its own layers raise -- a missing part is
509
+ # a KeyError, a damaged package a BadZipFile, a malformed part a parse error -- and a
510
+ # refusal that escapes untyped is a refusal the remediation map cannot bind a fix to.
511
+ except Exception as error:
512
+ raise ReaderError(
513
+ "READER_ADMISSION",
514
+ subject,
515
+ f"could not be opened as a workbook: {type(error).__name__}",
516
+ ) from error
517
+
518
+
519
+ def _declared_window(worksheet: Any, subject: str) -> tuple[int, int, int, int]:
520
+ """The used area the sheet declares, as (column, row, column, row), or a refusal.
521
+
522
+ A spreadsheet states its used range in the file, so the size of the answer is knowable
523
+ before the first cell is read. A sheet that declares nothing is refused rather than
524
+ scanned to find out: the whole point of reading the declaration is to decide before
525
+ iterating, and a scan to discover the geometry is the iteration being avoided.
526
+ """
527
+
528
+ bounds = (
529
+ worksheet.min_column,
530
+ worksheet.min_row,
531
+ worksheet.max_column,
532
+ worksheet.max_row,
533
+ )
534
+ if any(type(value) is not int or value < 1 for value in bounds):
535
+ raise ReaderError(
536
+ "READER_ADMISSION",
537
+ subject,
538
+ "declares no used area, so the work it would take to read cannot be bounded "
539
+ "before it is done; re-save the workbook so it records its own dimensions",
540
+ )
541
+ return bounds
542
+
543
+
544
+ def _admit_declared_window(
545
+ worksheet: Any,
546
+ budgets: ReaderBudgets,
547
+ subject: str,
548
+ ) -> tuple[int, int, int, int]:
549
+ """Read and admit declared geometry before any cell iterator is requested."""
550
+
551
+ window = _declared_window(worksheet, subject)
552
+ start_column, start_row, end_column, end_row = window
553
+ check_declared_size(
554
+ row_count=end_row - start_row + 1,
555
+ column_count=end_column - start_column + 1,
556
+ budgets=budgets,
557
+ subject=subject,
558
+ )
559
+ return window
560
+
561
+
562
+ def _window_values(
563
+ worksheet: Any,
564
+ window: tuple[int, int, int, int],
565
+ ) -> tuple[tuple[Any, ...], ...]:
566
+ """The values of one bounded window, padded to the window's own width."""
567
+
568
+ start_column, start_row, end_column, end_row = window
569
+ width = end_column - start_column + 1
570
+ rows: list[tuple[Any, ...]] = []
571
+ for row in worksheet.iter_rows(
572
+ min_row=start_row,
573
+ max_row=end_row,
574
+ min_col=start_column,
575
+ max_col=end_column,
576
+ values_only=True,
577
+ ):
578
+ values = tuple(row)[:width]
579
+ rows.append(values + (None,) * (width - len(values)))
580
+ return tuple(rows)
581
+
582
+
583
+ def _formula_cells(
584
+ worksheet: Any,
585
+ window: tuple[int, int, int, int],
586
+ ) -> frozenset[tuple[int, int]]:
587
+ """Which cells of the window hold a formula, read from the cell type and not from text.
588
+
589
+ A typed string beginning with an equals sign is a string. The distinction is in the cell's
590
+ declared type, so that is what is read; inferring it from the value would flag exactly the
591
+ cells a careful author quoted on purpose.
592
+ """
593
+
594
+ start_column, start_row, end_column, end_row = window
595
+ found: set[tuple[int, int]] = set()
596
+ for row_offset, row in enumerate(
597
+ worksheet.iter_rows(
598
+ min_row=start_row,
599
+ max_row=end_row,
600
+ min_col=start_column,
601
+ max_col=end_column,
602
+ )
603
+ ):
604
+ for column_offset, cell in enumerate(row):
605
+ if cell.data_type == "f":
606
+ found.add((start_row + row_offset, start_column + column_offset))
607
+ return frozenset(found)
608
+
609
+
610
+ def _column_names(header: Sequence[Any], start_column: int, row: int) -> tuple[str, ...]:
611
+ """The column names the header row states, refused rather than repaired."""
612
+
613
+ subject = "reader.spreadsheet.xlsx.header_row"
614
+ names: list[str] = []
615
+ for offset, value in enumerate(header):
616
+ if not isinstance(value, str) or not value.strip():
617
+ raise ReaderError(
618
+ "READER_RANGE",
619
+ subject,
620
+ f"cell {_coordinate(row, start_column + offset)} does not hold a column name; "
621
+ "an empty or non-text header cell is refused rather than auto-named",
622
+ )
623
+ names.append(value)
624
+ if len(set(names)) != len(names):
625
+ repeated = sorted({name for name in names if names.count(name) > 1})
626
+ raise ReaderError(
627
+ "READER_RANGE",
628
+ subject,
629
+ f"repeats a column name: {', '.join(repeated)}; a duplicate header is refused "
630
+ "rather than renamed, because a renamed column is a column nobody asked for",
631
+ )
632
+ return tuple(names)
633
+
634
+
635
+ def _cell_value(value: Any, subject: str) -> Any:
636
+ """Give a temporal value its one stated text form and leave every other value alone.
637
+
638
+ Numbers, text, and booleans go on to the shared canonical emitter untouched, which is what
639
+ keeps the float rule in one place. Temporal values are converted here because the format
640
+ stores one temporal type and the choice of text form is this family's to state.
641
+ """
642
+
643
+ if isinstance(value, datetime.datetime):
644
+ return value.isoformat(sep="T")
645
+ if isinstance(value, datetime.date):
646
+ return value.isoformat()
647
+ if isinstance(value, datetime.time):
648
+ return value.isoformat()
649
+ if isinstance(value, datetime.timedelta):
650
+ raise ReaderError(
651
+ "READER_OUTPUT",
652
+ subject,
653
+ "holds a duration; a duration is a rendering of a number rather than a value in "
654
+ "the file, so it has no honest single text form and is refused",
655
+ )
656
+ return value
657
+
658
+
659
+ @dataclass(frozen=True)
660
+ class SpreadsheetReader:
661
+ """``spreadsheet.xlsx``: one pinned window of one named sheet, opened from bytes.
662
+
663
+ The order of the work is backend, options, admission, package budgets and scan, values,
664
+ formula types, header, emit -- and the function below is written in that order, because
665
+ "refused before the engine sees it" is only true if the code says so.
666
+ """
667
+
668
+ family_id: str = "spreadsheet.xlsx"
669
+ family_version: str = "1.0.0"
670
+ contract_version: str = READER_CONTRACT_VERSION
671
+ output_format: str = _OUTPUT_FORMAT
672
+ accepted_media_types: tuple[str, ...] = (SPREADSHEET_MEDIA_TYPE,)
673
+ default_budgets: ReaderBudgets = field(default_factory=spreadsheet_default_budgets)
674
+
675
+ def validate_options(self, options: Mapping[str, Any]) -> Mapping[str, Any]:
676
+ return _validate_options(options)
677
+
678
+ def decode(self, content: bytes, pin: ReaderPin, budgets: ReaderBudgets) -> ReaderResult:
679
+ subject = "reader.spreadsheet.xlsx"
680
+ _check_backend(subject)
681
+ options = _validate_options(pin.decode_options)
682
+ sheet_name = str(options["sheet"])
683
+ header_row = int(options["header_row"])
684
+ window = _validate_range(options["range"], f"{subject}.range")
685
+ start_column, start_row, end_column, end_row = window
686
+
687
+ raw = _admit_workbook_bytes(content, budgets=budgets, subject=subject)
688
+ _inspect_package(raw, budgets=budgets, subject=subject)
689
+
690
+ book = _load(raw, data_only=True, subject=subject)
691
+ try:
692
+ if book.epoch != _EPOCH_1900:
693
+ raise ReaderError(
694
+ "READER_ADMISSION",
695
+ subject,
696
+ f"declares the {_EPOCH_1904:%Y} date system; this family reads the 1900 "
697
+ "date system only, because the two differ by four years and a day and "
698
+ "the difference is invisible in a recipe. Re-save the workbook under the "
699
+ "1900 date system",
700
+ )
701
+ if sheet_name not in book.sheetnames:
702
+ raise ReaderError(
703
+ "READER_SHEET_MISSING",
704
+ f"{subject}.sheet",
705
+ f"names {sheet_name!r}, which this workbook does not hold; it holds: "
706
+ f"{', '.join(book.sheetnames)}",
707
+ )
708
+ worksheet = book[sheet_name]
709
+ used = _admit_declared_window(worksheet, budgets, f"{subject}.sheet")
710
+ used_start_column, used_start_row, used_end_column, used_end_row = used
711
+ if (
712
+ start_row < used_start_row
713
+ or start_column < used_start_column
714
+ or end_row > used_end_row
715
+ or end_column > used_end_column
716
+ ):
717
+ raise ReaderError(
718
+ "READER_RANGE",
719
+ f"{subject}.range",
720
+ f"names {options['range']!r}, which reaches outside the used area this "
721
+ f"sheet declares, "
722
+ f"{_coordinate(used_start_row, used_start_column)}:"
723
+ f"{_coordinate(used_end_row, used_end_column)}",
724
+ )
725
+ if header_row != start_row:
726
+ raise ReaderError(
727
+ "READER_RANGE",
728
+ f"{subject}.header_row",
729
+ f"names row {header_row}, which is not the first row of the range "
730
+ f"{options['range']!r}; the header row and the range must agree, so that "
731
+ "every row inside the range is either the header or data and none is "
732
+ "silently neither",
733
+ )
734
+ check_declared_size(
735
+ row_count=end_row - start_row + 1,
736
+ column_count=end_column - start_column + 1,
737
+ budgets=budgets,
738
+ subject=f"{subject}.range",
739
+ )
740
+ values = _window_values(worksheet, window)
741
+ finally:
742
+ book.close()
743
+
744
+ typed = _load(raw, data_only=False, subject=subject)
745
+ try:
746
+ formulas = _formula_cells(typed[sheet_name], window)
747
+ finally:
748
+ typed.close()
749
+
750
+ for row_index, column_index in sorted(formulas):
751
+ if values[row_index - start_row][column_index - start_column] is None:
752
+ raise ReaderError(
753
+ "READER_ADMISSION",
754
+ f"{subject}.cell",
755
+ f"{_coordinate(row_index, column_index)} holds a formula with no cached "
756
+ "value; an uncomputed formula is missing information rather than a "
757
+ "missing value, so it is refused. Open and recalculate the workbook, then "
758
+ "save it again",
759
+ )
760
+
761
+ columns = _column_names(values[0], start_column, start_row)
762
+ rows = tuple(
763
+ tuple(
764
+ _cell_value(value, f"{subject}.cell.{_coordinate(start_row + offset + 1, index)}")
765
+ for index, value in enumerate(row, start=start_column)
766
+ )
767
+ for offset, row in enumerate(values[1:])
768
+ )
769
+ return ReaderResult(
770
+ content=encode_canonical_csv(columns, rows, budgets=budgets),
771
+ data_format=_OUTPUT_FORMAT,
772
+ media_type=_OUTPUT_MEDIA_TYPE,
773
+ filename=sealed_filename(sheet_name, suffix=_OUTPUT_SUFFIX),
774
+ row_count=len(rows),
775
+ column_names=columns,
776
+ # Spreadsheet admission charges the complete used area declared by the sheet,
777
+ # including its header row, before iterating any selected cell. Preserve that
778
+ # family-specific geometry rather than reducing it to emitted data rows.
779
+ declared_cell_count=(used_end_row - used_start_row + 1)
780
+ * (used_end_column - used_start_column + 1),
781
+ flags=(_FORMULA_FLAG,) if formulas else (),
782
+ )
783
+
784
+
785
+ @dataclass(frozen=True)
786
+ class SpreadsheetReaderV1_1(SpreadsheetReader):
787
+ """``spreadsheet.xlsx@1.1.0`` admits the byte-stream label, and reads nothing new.
788
+
789
+ The same one-label delta as the container coordinates, for the same reason: a workbook
790
+ downloaded from an object store or a plain file server very often arrives as
791
+ ``application/octet-stream``, which says only that the server declined to describe it.
792
+ The admission behind the label is unchanged and decisive -- a workbook is a zip container
793
+ whose structure ``openpyxl`` validates, so a response that is not a workbook is refused
794
+ by its own bytes whatever the label said. ``1.0.0`` stays closed to the weak label, and
795
+ a label that asserts a different format outright (``application/vnd.ms-excel`` is the
796
+ legacy XLS assertion) remains refused at both coordinates.
797
+ """
798
+
799
+ family_version: str = "1.1.0"
800
+ accepted_media_types: tuple[str, ...] = (
801
+ "application/octet-stream",
802
+ SPREADSHEET_MEDIA_TYPE,
803
+ )