mostlyright-data 0.9.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mostlyright/data_harness/__init__.py +158 -0
- mostlyright/data_harness/acquisition/__init__.py +55 -0
- mostlyright/data_harness/acquisition/http.py +2773 -0
- mostlyright/data_harness/acquisition/parsing.py +809 -0
- mostlyright/data_harness/acquisition/ranges.py +495 -0
- mostlyright/data_harness/acquisition/result_download.py +360 -0
- mostlyright/data_harness/acquisition/retention_admission.py +248 -0
- mostlyright/data_harness/acquisition/sandbox.py +4888 -0
- mostlyright/data_harness/acquisition/url_policy.py +530 -0
- mostlyright/data_harness/agent_runtime.py +2743 -0
- mostlyright/data_harness/assets/logo-ink.svg +31 -0
- mostlyright/data_harness/backends/__init__.py +28 -0
- mostlyright/data_harness/backends/pandas_backend.py +350 -0
- mostlyright/data_harness/backends/polars_backend.py +366 -0
- mostlyright/data_harness/backends/protocol.py +124 -0
- mostlyright/data_harness/backends/reference.py +83 -0
- mostlyright/data_harness/backends/registry.py +55 -0
- mostlyright/data_harness/backends/restrictions.py +126 -0
- mostlyright/data_harness/canonical.py +333 -0
- mostlyright/data_harness/catalog_job.py +625 -0
- mostlyright/data_harness/cli.py +5398 -0
- mostlyright/data_harness/contracts.py +53 -0
- mostlyright/data_harness/coordinator.py +1307 -0
- mostlyright/data_harness/deploy.py +924 -0
- mostlyright/data_harness/deploy_target.py +312 -0
- mostlyright/data_harness/deployment_evidence.py +1067 -0
- mostlyright/data_harness/event_presentation.py +576 -0
- mostlyright/data_harness/events.py +2152 -0
- mostlyright/data_harness/fast_delimited.py +239 -0
- mostlyright/data_harness/fleet.py +237 -0
- mostlyright/data_harness/formats.py +236 -0
- mostlyright/data_harness/governors.py +1163 -0
- mostlyright/data_harness/hosted_bootstrap.py +972 -0
- mostlyright/data_harness/hosted_crawler.py +1115 -0
- mostlyright/data_harness/hosted_crawler_container_smoke.py +351 -0
- mostlyright/data_harness/hosted_crawler_fetch.py +423 -0
- mostlyright/data_harness/hosted_crawler_job.py +1277 -0
- mostlyright/data_harness/hosted_crawler_protocol.py +676 -0
- mostlyright/data_harness/hosted_dataset.py +1500 -0
- mostlyright/data_harness/hosted_deploy.py +3037 -0
- mostlyright/data_harness/hosted_handoff.py +62 -0
- mostlyright/data_harness/hosted_ingestion_contract.py +504 -0
- mostlyright/data_harness/hosted_ingestion_job.py +356 -0
- mostlyright/data_harness/hosted_ingestion_job_smoke.py +40 -0
- mostlyright/data_harness/hosted_session_container_smoke.py +194 -0
- mostlyright/data_harness/hosted_session_worker.py +3554 -0
- mostlyright/data_harness/hosted_session_worker_job_smoke.py +46 -0
- mostlyright/data_harness/hosted_worker.py +6784 -0
- mostlyright/data_harness/ingestion/__init__.py +56 -0
- mostlyright/data_harness/ingestion/contracts.py +461 -0
- mostlyright/data_harness/ingestion/faults.py +42 -0
- mostlyright/data_harness/ingestion/gcs_store.py +1162 -0
- mostlyright/data_harness/ingestion/spool.py +130 -0
- mostlyright/data_harness/ingestion/store.py +885 -0
- mostlyright/data_harness/key_seam.py +434 -0
- mostlyright/data_harness/linux_process_boundary.py +262 -0
- mostlyright/data_harness/local_contracts.py +2880 -0
- mostlyright/data_harness/local_search/__init__.py +5 -0
- mostlyright/data_harness/local_search/build_index.py +1087 -0
- mostlyright/data_harness/local_search/contracts.py +920 -0
- mostlyright/data_harness/local_search/query_trace.py +266 -0
- mostlyright/data_harness/local_search/retrieval.py +700 -0
- mostlyright/data_harness/local_search/sealed.py +474 -0
- mostlyright/data_harness/local_search/service.py +784 -0
- mostlyright/data_harness/nbrender/CONTRACT.md +212 -0
- mostlyright/data_harness/nbrender/__init__.py +12 -0
- mostlyright/data_harness/nbrender/chrome.py +359 -0
- mostlyright/data_harness/nbrender/code_body.py +266 -0
- mostlyright/data_harness/nbrender/document.py +407 -0
- mostlyright/data_harness/nbrender/frame.py +275 -0
- mostlyright/data_harness/nbrender/interactive.py +337 -0
- mostlyright/data_harness/nbrender/markdown_body.py +477 -0
- mostlyright/data_harness/nbrender/mr_components.py +134 -0
- mostlyright/data_harness/nbrender/outputs_data.py +595 -0
- mostlyright/data_harness/nbrender/outputs_rich.py +906 -0
- mostlyright/data_harness/nbrender/outputs_source.py +260 -0
- mostlyright/data_harness/nbrender/outputs_stage.py +176 -0
- mostlyright/data_harness/nbrender/outputs_text.py +400 -0
- mostlyright/data_harness/nbrender/parse.py +394 -0
- mostlyright/data_harness/nbrender/status.py +40 -0
- mostlyright/data_harness/nbrender/tokens.py +1295 -0
- mostlyright/data_harness/notebook.py +1710 -0
- mostlyright/data_harness/offline.py +2049 -0
- mostlyright/data_harness/operation_registry.py +1007 -0
- mostlyright/data_harness/operator_setup.py +239 -0
- mostlyright/data_harness/pipeline.py +6428 -0
- mostlyright/data_harness/plan_graph.py +2026 -0
- mostlyright/data_harness/preparation/__init__.py +104 -0
- mostlyright/data_harness/preparation/contracts.py +1017 -0
- mostlyright/data_harness/preparation/engine.py +221 -0
- mostlyright/data_harness/preparation/errors.py +14 -0
- mostlyright/data_harness/preparation/gates.py +751 -0
- mostlyright/data_harness/preparation/joins.py +574 -0
- mostlyright/data_harness/preparation/profile.py +384 -0
- mostlyright/data_harness/preparation/table.py +217 -0
- mostlyright/data_harness/preparation/transforms.py +568 -0
- mostlyright/data_harness/progress_events.py +534 -0
- mostlyright/data_harness/readers/__init__.py +46 -0
- mostlyright/data_harness/readers/containers.py +963 -0
- mostlyright/data_harness/readers/contracts.py +542 -0
- mostlyright/data_harness/readers/delimited.py +257 -0
- mostlyright/data_harness/readers/grib2/__init__.py +33 -0
- mostlyright/data_harness/readers/grib2/admission.py +722 -0
- mostlyright/data_harness/readers/grib2/decode.py +1009 -0
- mostlyright/data_harness/readers/grib2/geometry.py +1133 -0
- mostlyright/data_harness/readers/grib2/portable_math.py +501 -0
- mostlyright/data_harness/readers/json_tabular.py +485 -0
- mostlyright/data_harness/readers/registry.py +514 -0
- mostlyright/data_harness/readers/samples/README.md +110 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/cities.csv.gz +0 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/cities.csv.gz +0 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/cities.tar +0 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/cities.tar +0 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/cities.zip +0 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/expected.json +25 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/expected.json +25 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/expected.json +25 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/cities.csv +3 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/expected.json +23 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/cities.tsv +5 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/expected.json +25 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/expected.json +30 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/observations.csv +5 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/expected.json +44 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/stations.json +1 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/expected.json +48 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/observations.ndjson +4 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/cities.xlsx +0 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/expected.json +24 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/expected.json +27 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/expected.json +27 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/README.md +20 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/expected.json +55 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/gfs-2m-temperature.grib2 +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/expected.json +54 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/expected.json +54 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/hrrr-categorical-rain.grib2 +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/expected.json +54 -0
- mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
- mostlyright/data_harness/readers/samples.py +582 -0
- mostlyright/data_harness/readers/spreadsheet.py +803 -0
- mostlyright/data_harness/readers/tabular.py +510 -0
- mostlyright/data_harness/recipe.py +5321 -0
- mostlyright/data_harness/repair/__init__.py +78 -0
- mostlyright/data_harness/repair/adapters.py +274 -0
- mostlyright/data_harness/repair/contracts.py +872 -0
- mostlyright/data_harness/repair/coordinator.py +1099 -0
- mostlyright/data_harness/repair/errors.py +16 -0
- mostlyright/data_harness/review.py +2533 -0
- mostlyright/data_harness/rowset.py +283 -0
- mostlyright/data_harness/serving.py +1975 -0
- mostlyright/data_harness/serving_edge.py +590 -0
- mostlyright/data_harness/serving_http.py +1031 -0
- mostlyright/data_harness/session_probes.py +759 -0
- mostlyright/data_harness/signing.py +101 -0
- mostlyright/data_harness/source_discovery.py +898 -0
- mostlyright/data_harness/sources/__init__.py +209 -0
- mostlyright/data_harness/sources/_adapter_steps.py +213 -0
- mostlyright/data_harness/sources/adapters.py +1214 -0
- mostlyright/data_harness/sources/cadence.py +1428 -0
- mostlyright/data_harness/sources/cadence_emission.py +453 -0
- mostlyright/data_harness/sources/cadence_history.py +546 -0
- mostlyright/data_harness/sources/catalog/__init__.py +17 -0
- mostlyright/data_harness/sources/catalog/admission.py +477 -0
- mostlyright/data_harness/sources/catalog/authoring.py +1701 -0
- mostlyright/data_harness/sources/catalog/authoring_policy.py +701 -0
- mostlyright/data_harness/sources/catalog/authoring_shards.py +1217 -0
- mostlyright/data_harness/sources/catalog/bounded_io.py +231 -0
- mostlyright/data_harness/sources/catalog/channel.py +523 -0
- mostlyright/data_harness/sources/catalog/channel_client.py +296 -0
- mostlyright/data_harness/sources/catalog/contracts.py +825 -0
- mostlyright/data_harness/sources/catalog/coverage.py +137 -0
- mostlyright/data_harness/sources/catalog/delta.py +1340 -0
- mostlyright/data_harness/sources/catalog/embedding.py +532 -0
- mostlyright/data_harness/sources/catalog/entry_v2.py +1182 -0
- mostlyright/data_harness/sources/catalog/fill.py +3889 -0
- mostlyright/data_harness/sources/catalog/fill_partitions.py +459 -0
- mostlyright/data_harness/sources/catalog/fill_staging.py +1105 -0
- mostlyright/data_harness/sources/catalog/gating.py +374 -0
- mostlyright/data_harness/sources/catalog/generation_receipt.py +1607 -0
- mostlyright/data_harness/sources/catalog/harvest/__init__.py +7 -0
- mostlyright/data_harness/sources/catalog/harvest/ckan.py +384 -0
- mostlyright/data_harness/sources/catalog/harvest/datagov_v4.py +798 -0
- mostlyright/data_harness/sources/catalog/harvest/protocol.py +964 -0
- mostlyright/data_harness/sources/catalog/harvest/sdmx.py +445 -0
- mostlyright/data_harness/sources/catalog/harvest/stac.py +384 -0
- mostlyright/data_harness/sources/catalog/health.py +447 -0
- mostlyright/data_harness/sources/catalog/hosted_catalog.py +105 -0
- mostlyright/data_harness/sources/catalog/identity_history.py +1549 -0
- mostlyright/data_harness/sources/catalog/neural.py +1618 -0
- mostlyright/data_harness/sources/catalog/packed_catalog.py +2345 -0
- mostlyright/data_harness/sources/catalog/packed_retrieval.py +1517 -0
- mostlyright/data_harness/sources/catalog/packed_writer.py +2802 -0
- mostlyright/data_harness/sources/catalog/query_trace.py +1037 -0
- mostlyright/data_harness/sources/catalog/recommend.py +171 -0
- mostlyright/data_harness/sources/catalog/retrieval.py +230 -0
- mostlyright/data_harness/sources/catalog/retrieval_manifest.py +995 -0
- mostlyright/data_harness/sources/catalog/rights_decisions.py +254 -0
- mostlyright/data_harness/sources/catalog/sealed.py +560 -0
- mostlyright/data_harness/sources/catalog/search.py +230 -0
- mostlyright/data_harness/sources/catalog/streaming_delta.py +1097 -0
- mostlyright/data_harness/sources/catalog/update.py +891 -0
- mostlyright/data_harness/sources/collections.py +815 -0
- mostlyright/data_harness/sources/contracts.py +2223 -0
- mostlyright/data_harness/sources/deletion.py +761 -0
- mostlyright/data_harness/sources/fitness.py +162 -0
- mostlyright/data_harness/sources/governance.py +163 -0
- mostlyright/data_harness/sources/hosted.py +173 -0
- mostlyright/data_harness/sources/integration.py +218 -0
- mostlyright/data_harness/sources/range_reader.py +418 -0
- mostlyright/data_harness/sources/registry.py +514 -0
- mostlyright/data_harness/sources/rights_rule.py +59 -0
- mostlyright/data_harness/sources/source_cadence_vectors.v1.json +1 -0
- mostlyright/data_harness/sources/sports.py +521 -0
- mostlyright/data_harness/sources/stream.py +524 -0
- mostlyright/data_harness/sources/stream_connector.py +418 -0
- mostlyright/data_harness/sources/stream_recorder.py +1404 -0
- mostlyright/data_harness/studio_boundary.py +2019 -0
- mostlyright/data_harness/thin/__init__.py +37 -0
- mostlyright/data_harness/thin/acquire.py +1137 -0
- mostlyright/data_harness/thin/acquire_cancel.py +579 -0
- mostlyright/data_harness/thin/approvals.py +617 -0
- mostlyright/data_harness/thin/commands.py +406 -0
- mostlyright/data_harness/thin/download.py +194 -0
- mostlyright/data_harness/thin/narrative.py +589 -0
- mostlyright/data_harness/thin/parity.py +1070 -0
- mostlyright/data_harness/thin/propose.py +2759 -0
- mostlyright/data_harness/thin/research.py +1663 -0
- mostlyright/data_harness/thin/router.py +924 -0
- mostlyright/data_harness/thin/runs.py +519 -0
- mostlyright/data_harness/thin/session.py +281 -0
- mostlyright/data_harness/thin/stream.py +501 -0
- mostlyright/data_harness/thin/transport.py +187 -0
- mostlyright/data_harness/thin/vocabulary.py +368 -0
- mostlyright/data_harness/thin/workers.py +164 -0
- mostlyright/data_harness/ucum/TABLE-PIN.json +40 -0
- mostlyright/data_harness/ucum/ucum-subset.v1.json +632 -0
- mostlyright/data_harness/unit_flow.py +927 -0
- mostlyright/data_harness/units.py +572 -0
- mostlyright/data_harness/ux/__init__.py +9 -0
- mostlyright/data_harness/ux/approve.py +485 -0
- mostlyright/data_harness/ux/author_yaml.py +597 -0
- mostlyright/data_harness/ux/cloud_auth.py +447 -0
- mostlyright/data_harness/ux/commands/__init__.py +260 -0
- mostlyright/data_harness/ux/commands/approve.py +136 -0
- mostlyright/data_harness/ux/commands/auth.py +744 -0
- mostlyright/data_harness/ux/commands/author.py +79 -0
- mostlyright/data_harness/ux/commands/catalog_author.py +403 -0
- mostlyright/data_harness/ux/commands/catalog_fill.py +523 -0
- mostlyright/data_harness/ux/commands/catalog_harvest.py +545 -0
- mostlyright/data_harness/ux/commands/catalog_publish.py +1838 -0
- mostlyright/data_harness/ux/commands/catalog_search.py +71 -0
- mostlyright/data_harness/ux/commands/catalog_update.py +437 -0
- mostlyright/data_harness/ux/commands/deploy.py +134 -0
- mostlyright/data_harness/ux/commands/deploy_dataset.py +98 -0
- mostlyright/data_harness/ux/commands/deploy_plan.py +105 -0
- mostlyright/data_harness/ux/commands/deploy_status.py +104 -0
- mostlyright/data_harness/ux/commands/diff.py +74 -0
- mostlyright/data_harness/ux/commands/index.py +84 -0
- mostlyright/data_harness/ux/commands/inventory.py +47 -0
- mostlyright/data_harness/ux/commands/list_builds.py +143 -0
- mostlyright/data_harness/ux/commands/login.py +63 -0
- mostlyright/data_harness/ux/commands/peek.py +236 -0
- mostlyright/data_harness/ux/commands/plan_check.py +90 -0
- mostlyright/data_harness/ux/commands/preflight.py +97 -0
- mostlyright/data_harness/ux/commands/record.py +107 -0
- mostlyright/data_harness/ux/commands/review_setup.py +47 -0
- mostlyright/data_harness/ux/commands/search.py +440 -0
- mostlyright/data_harness/ux/commands/show.py +61 -0
- mostlyright/data_harness/ux/commands/whoami.py +37 -0
- mostlyright/data_harness/ux/credential_native.py +551 -0
- mostlyright/data_harness/ux/credential_store.py +1055 -0
- mostlyright/data_harness/ux/credentials.py +631 -0
- mostlyright/data_harness/ux/diffing.py +444 -0
- mostlyright/data_harness/ux/headline.py +671 -0
- mostlyright/data_harness/ux/hosted_acquisition.py +974 -0
- mostlyright/data_harness/ux/hosted_run_status.py +619 -0
- mostlyright/data_harness/ux/inventory.py +427 -0
- mostlyright/data_harness/ux/local_review.py +375 -0
- mostlyright/data_harness/ux/login.py +691 -0
- mostlyright/data_harness/ux/path_kind.py +147 -0
- mostlyright/data_harness/ux/peek.py +1000 -0
- mostlyright/data_harness/ux/plain_file.py +178 -0
- mostlyright/data_harness/ux/plan_check.py +311 -0
- mostlyright/data_harness/ux/preflight.py +918 -0
- mostlyright/data_harness/ux/readers.py +1124 -0
- mostlyright/data_harness/ux/remediation.py +2195 -0
- mostlyright/data_harness/ux/render.py +657 -0
- mostlyright/data_harness/ux/workload.py +1077 -0
- mostlyright/data_harness/viewer.py +3713 -0
- mostlyright/data_harness/visual_run/__init__.py +83 -0
- mostlyright/data_harness/visual_run/authoring.py +235 -0
- mostlyright/data_harness/visual_run/contracts.py +673 -0
- mostlyright/data_harness/visual_run/materialize.py +486 -0
- mostlyright/data_harness/visual_run/observations.py +874 -0
- mostlyright/data_harness/visual_run/query.py +259 -0
- mostlyright/data_harness/visual_run/reducer.py +280 -0
- mostlyright/data_harness/visual_run/sdk.py +892 -0
- mostlyright/data_harness/visual_run/store.py +584 -0
- mostlyright/data_harness/visual_run/transport.py +239 -0
- mostlyright/data_harness/watch.py +2999 -0
- mostlyright_data-0.9.0.dist-info/METADATA +607 -0
- mostlyright_data-0.9.0.dist-info/RECORD +314 -0
- mostlyright_data-0.9.0.dist-info/WHEEL +4 -0
- mostlyright_data-0.9.0.dist-info/entry_points.txt +12 -0
|
@@ -0,0 +1,803 @@
|
|
|
1
|
+
"""The spreadsheet Reader: strict coordinates, cached formula values surfaced and flagged.
|
|
2
|
+
|
|
3
|
+
A spreadsheet is not a table. It is a drawing surface that usually has a table on it, and
|
|
4
|
+
everything difficult about reading one follows from that sentence. Two design decisions
|
|
5
|
+
answer it, and both are unusual enough to be written down rather than left to be inferred.
|
|
6
|
+
|
|
7
|
+
Strict coordinates
|
|
8
|
+
------------------
|
|
9
|
+
The recipe pins three things -- the sheet by name, the header row, and the cell range -- and
|
|
10
|
+
every one of them is checked. A workbook that grows a sheet, renames a tab, or shifts its
|
|
11
|
+
table down two rows produces a named refusal rather than silently different data. There is
|
|
12
|
+
no sniffing, no "the first sheet", no "the first non-empty row", and no index-based sheet
|
|
13
|
+
selection: sheet order changes without notice and an index would move a dataset from under a
|
|
14
|
+
recipe whose digest never changed.
|
|
15
|
+
|
|
16
|
+
The three coordinates are checked against each other as well as against the file. The header
|
|
17
|
+
row must be the range's first row, so every row inside the range is accounted for -- one
|
|
18
|
+
header and the rest data -- and no row is silently neither. A recipe whose range was edited
|
|
19
|
+
without its header row is therefore a refusal that names the disagreement, rather than a
|
|
20
|
+
dataset quietly missing its first records.
|
|
21
|
+
|
|
22
|
+
Cached formula values, surfaced and flagged
|
|
23
|
+
-------------------------------------------
|
|
24
|
+
A formula's result is not in the file in any authoritative sense. It is whatever the
|
|
25
|
+
authoring application last cached, and nothing here recomputes it -- this is a Reader, not a
|
|
26
|
+
spreadsheet engine. Using the cached value is the only useful behaviour, and pretending it is
|
|
27
|
+
the same as a typed value would be dishonest, so the decode reports ``contains_formula_cells``
|
|
28
|
+
and the receipt carries it forward. A reviewer looking at a dataset can then see that some of
|
|
29
|
+
its numbers were somebody else's arithmetic.
|
|
30
|
+
|
|
31
|
+
A formula with no cached value is refused rather than emitted as an empty cell. An uncomputed
|
|
32
|
+
formula is missing *information*; an empty cell is a stated absence of a value. Collapsing
|
|
33
|
+
the two would let a workbook that was never recalculated pass as a workbook full of blanks.
|
|
34
|
+
|
|
35
|
+
That flag is read from evidence, which is what the two passes buy. The workbook is loaded
|
|
36
|
+
twice from the same bytes: once with ``data_only=True``, which yields the cached values, and
|
|
37
|
+
once without, which yields each cell's type so a formula can be told from a string that merely
|
|
38
|
+
begins with an equals sign. Two bounded passes over an in-memory workbook is a fair price for
|
|
39
|
+
a flag that is honest, and the price is stated here so it is not later optimised into a guess.
|
|
40
|
+
|
|
41
|
+
The date rule, pinned
|
|
42
|
+
---------------------
|
|
43
|
+
Excel date handling is the single most common source of quiet corruption in spreadsheet
|
|
44
|
+
ingestion, so the rule is stated with its examples rather than surrounded by caution.
|
|
45
|
+
|
|
46
|
+
* The 1900 date system is the only one read. A workbook declaring the 1904 system is refused
|
|
47
|
+
by name, because the two differ by four years and a day and the difference is invisible in
|
|
48
|
+
a recipe: the same coordinates would produce different dates with nothing to see.
|
|
49
|
+
* A date or a datetime renders as ISO 8601 with a ``T`` separator and seconds always present:
|
|
50
|
+
``2026-08-07T00:00:00`` for a date, ``2026-08-07T13:05:01`` for a datetime. A date-only
|
|
51
|
+
cell is a datetime at midnight, because that is what the format stores.
|
|
52
|
+
* A time-only cell renders as ``06:30:00``.
|
|
53
|
+
* A duration is refused. Excel stores one temporal type and a duration is a rendering of it
|
|
54
|
+
rather than a value in it, so there is no honest single text form.
|
|
55
|
+
|
|
56
|
+
The float rule and the output encoding are not this module's to state. Numbers reach the
|
|
57
|
+
shared canonical emitter as binary64 values and are rendered there, by the one implementation
|
|
58
|
+
every family emits through. A spreadsheet is exactly where an engine's own string conversion
|
|
59
|
+
would slip into the output unnoticed, so the engine's text never reaches it.
|
|
60
|
+
|
|
61
|
+
Nothing on disk
|
|
62
|
+
---------------
|
|
63
|
+
The workbook is constructed from ``io.BytesIO`` with ``read_only=True``, never from a path.
|
|
64
|
+
No API taking a filesystem path is named in this module, and a test observes two watched
|
|
65
|
+
directories that are empty before and after a decode.
|
|
66
|
+
|
|
67
|
+
The xlsx-is-a-zip boundary
|
|
68
|
+
--------------------------
|
|
69
|
+
An xlsx file is a zip of OOXML parts. Those parts are the format's internal structure, not a
|
|
70
|
+
user-visible container nesting, so the container families' "exactly one recipe-named member"
|
|
71
|
+
rule does not apply to them and this family does not compose with ``archive.zip``. What is
|
|
72
|
+
shared with that module is the zip prefix table and the bounded read, because both modules are
|
|
73
|
+
looking at the same bytes and a second copy of either would let them drift. What is not
|
|
74
|
+
shared is the member rule, and stating the difference here is cheaper than leaving a reader to
|
|
75
|
+
wonder whether one was forgotten.
|
|
76
|
+
|
|
77
|
+
A recipe wanting a zipped xlsx is therefore ``archive.zip`` naming the ``.xlsx`` member -- and
|
|
78
|
+
the container family's nesting refusal will refuse it, because an xlsx is an archive by magic.
|
|
79
|
+
That is a known limitation recorded in the Certification record with its reason, rather than a
|
|
80
|
+
special case carved into the magic table.
|
|
81
|
+
|
|
82
|
+
The engine, and why it is part of the pin
|
|
83
|
+
-----------------------------------------
|
|
84
|
+
``openpyxl`` at an exact version, with its XML backend checked at decode time. ``openpyxl``
|
|
85
|
+
chooses its parser at import from what it finds on the host, so a decoder whose parser depends
|
|
86
|
+
on which other packages happen to be installed is a decoder that can answer differently on two
|
|
87
|
+
hosts. The pinned backend is the standard library's, and a host where the other one was
|
|
88
|
+
selected is refused rather than decoded. The full ruling, including why this family does not
|
|
89
|
+
follow the design document's calamine pin, is in the Certification record.
|
|
90
|
+
"""
|
|
91
|
+
|
|
92
|
+
from __future__ import annotations
|
|
93
|
+
|
|
94
|
+
import datetime
|
|
95
|
+
import io
|
|
96
|
+
import re
|
|
97
|
+
import zipfile
|
|
98
|
+
import zlib
|
|
99
|
+
from collections.abc import Mapping, Sequence
|
|
100
|
+
from dataclasses import dataclass, field
|
|
101
|
+
from typing import Any
|
|
102
|
+
|
|
103
|
+
import openpyxl.xml
|
|
104
|
+
from openpyxl import load_workbook
|
|
105
|
+
|
|
106
|
+
from mostlyright.data_harness.formats import (
|
|
107
|
+
FORMAT_MEDIA_TYPES,
|
|
108
|
+
FORMAT_SUFFIXES,
|
|
109
|
+
READER_CONTRACT_VERSION,
|
|
110
|
+
READER_OUTPUT_FORMATS,
|
|
111
|
+
READER_WORKER_MEMORY_BYTES,
|
|
112
|
+
)
|
|
113
|
+
from mostlyright.data_harness.readers.containers import (
|
|
114
|
+
ZIP_MAGICS,
|
|
115
|
+
read_bounded,
|
|
116
|
+
screen_zip_entry,
|
|
117
|
+
)
|
|
118
|
+
from mostlyright.data_harness.readers.contracts import (
|
|
119
|
+
DECODE_FLAGS,
|
|
120
|
+
ReaderBudgets,
|
|
121
|
+
ReaderError,
|
|
122
|
+
ReaderPin,
|
|
123
|
+
ReaderResult,
|
|
124
|
+
)
|
|
125
|
+
from mostlyright.data_harness.readers.registry import spreadsheet_default_budgets
|
|
126
|
+
from mostlyright.data_harness.readers.tabular import (
|
|
127
|
+
check_declared_size,
|
|
128
|
+
encode_canonical_csv,
|
|
129
|
+
sealed_filename,
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
__all__ = [
|
|
133
|
+
"SPREADSHEET_MAX_DECLARED_CELLS",
|
|
134
|
+
"SPREADSHEET_MEDIA_TYPE",
|
|
135
|
+
"SPREADSHEET_WORST_CASE_BYTES",
|
|
136
|
+
"SpreadsheetReader",
|
|
137
|
+
]
|
|
138
|
+
|
|
139
|
+
# The one encoding a Reader may seal, unpacked rather than restated.
|
|
140
|
+
(_OUTPUT_FORMAT,) = READER_OUTPUT_FORMATS
|
|
141
|
+
_OUTPUT_MEDIA_TYPE = sorted(FORMAT_MEDIA_TYPES[_OUTPUT_FORMAT])[0]
|
|
142
|
+
_OUTPUT_SUFFIX = sorted(FORMAT_SUFFIXES[_OUTPUT_FORMAT])[0]
|
|
143
|
+
|
|
144
|
+
# The one media type this family admits as input. Public because the acquisition layer asks
|
|
145
|
+
# the Toolbox which media types a pinned fetch may accept, and a re-spelled literal at that
|
|
146
|
+
# call site would be a second declaration of the same fact.
|
|
147
|
+
SPREADSHEET_MEDIA_TYPE = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
|
|
148
|
+
|
|
149
|
+
# Spreadsheet decoding retains the first pass's values while a second read-only pass identifies
|
|
150
|
+
# formula cells. The generic Reader ceiling of ten million cells permits Python-object overhead
|
|
151
|
+
# to approach or exceed the Linux child boundary even when the compressed workbook is small. This
|
|
152
|
+
# family therefore admits a quarter-million cells. The accounting deliberately charges 512 bytes
|
|
153
|
+
# per retained cell plus four copies of the admitted expanded package and both byte envelopes; it
|
|
154
|
+
# is conservative rather than self-calibrating and remains below the sandbox's exact process cap.
|
|
155
|
+
SPREADSHEET_MAX_DECLARED_CELLS = spreadsheet_default_budgets().max_declared_cells
|
|
156
|
+
_SPREADSHEET_MAX_UNCOMPRESSED_BYTES = spreadsheet_default_budgets().max_uncompressed_bytes
|
|
157
|
+
_SPREADSHEET_CELL_MEMORY_ALLOWANCE = 512
|
|
158
|
+
SPREADSHEET_WORST_CASE_BYTES = (
|
|
159
|
+
SPREADSHEET_MAX_DECLARED_CELLS * _SPREADSHEET_CELL_MEMORY_ALLOWANCE
|
|
160
|
+
+ 4 * _SPREADSHEET_MAX_UNCOMPRESSED_BYTES
|
|
161
|
+
+ 2 * 16 * 1024 * 1024
|
|
162
|
+
)
|
|
163
|
+
if SPREADSHEET_WORST_CASE_BYTES >= READER_WORKER_MEMORY_BYTES:
|
|
164
|
+
raise ReaderError(
|
|
165
|
+
"READER_BUDGET",
|
|
166
|
+
"reader.spreadsheet.memory",
|
|
167
|
+
"certified spreadsheet admission exceeds the Reader worker memory boundary",
|
|
168
|
+
)
|
|
169
|
+
# The fact this family reports about itself, taken from the closed vocabulary rather than
|
|
170
|
+
# spelled as a literal at the raise site. A build in which the two have drifted apart fails at
|
|
171
|
+
# import, which is the loudest and earliest place it can fail.
|
|
172
|
+
_FORMULA_FLAG = "contains_formula_cells"
|
|
173
|
+
if _FORMULA_FLAG not in DECODE_FLAGS: # pragma: no cover - a contradiction caught at import
|
|
174
|
+
raise ReaderError(
|
|
175
|
+
"READER_OUTPUT",
|
|
176
|
+
"reader.spreadsheet.flags",
|
|
177
|
+
f"{_FORMULA_FLAG} is not in the closed decode-flag vocabulary",
|
|
178
|
+
)
|
|
179
|
+
|
|
180
|
+
# The closed option set. All three are required and there is no fourth: an option that was
|
|
181
|
+
# accepted and ignored would be an unreviewable difference between two recipes that read
|
|
182
|
+
# identically to everyone who looks at them.
|
|
183
|
+
_OPTION_KEYS = ("header_row", "range", "sheet")
|
|
184
|
+
|
|
185
|
+
# The parts every workbook package carries. Checked by name so a zip that is not a workbook is
|
|
186
|
+
# refused with a sentence rather than escaping as whichever exception the engine happened to
|
|
187
|
+
# raise while looking for them.
|
|
188
|
+
_REQUIRED_PARTS = ("[Content_Types].xml", "xl/workbook.xml")
|
|
189
|
+
|
|
190
|
+
# Which parts are scanned before the engine sees them. Text parts only: a binary part that
|
|
191
|
+
# happened to contain these bytes is not a declaration, and refusing on one would be a false
|
|
192
|
+
# refusal with no fix a person could apply.
|
|
193
|
+
_TEXT_PART_SUFFIXES = (".xml", ".rels")
|
|
194
|
+
|
|
195
|
+
# An XML document type may define an entity that names a resource outside the file, which is
|
|
196
|
+
# the external-entity class. The pinned backend refuses one on its own -- a test proves that
|
|
197
|
+
# rather than assuming it -- and this is the layer in front of the engine, so the file is
|
|
198
|
+
# refused before the engine is handed it at all.
|
|
199
|
+
_DOCUMENT_TYPE = re.compile(rb"<!DOCTYPE|<!ENTITY", re.IGNORECASE)
|
|
200
|
+
|
|
201
|
+
# A workbook may name a resource outside itself: an external link to another workbook, or a
|
|
202
|
+
# relationship whose target is external. Refused rather than followed, on the same reasoning
|
|
203
|
+
# the acquisition parser already applies to a forbidden reference.
|
|
204
|
+
_EXTERNAL_REFERENCE = re.compile(
|
|
205
|
+
rb"TargetMode\s*=\s*[\"']External[\"']|/relationships/externalLink",
|
|
206
|
+
re.IGNORECASE,
|
|
207
|
+
)
|
|
208
|
+
|
|
209
|
+
# The 1900 date system, as the engine spells its epoch. The 1904 system is the other one the
|
|
210
|
+
# format permits and is refused by name.
|
|
211
|
+
_EPOCH_1900 = datetime.datetime(1899, 12, 30)
|
|
212
|
+
_EPOCH_1904 = datetime.datetime(1904, 1, 1)
|
|
213
|
+
|
|
214
|
+
# A sheet name is bounded text. Thirty-one characters is the format's own ceiling, so a longer
|
|
215
|
+
# one names a sheet no workbook could hold.
|
|
216
|
+
_SHEET_NAME_MAX_CHARACTERS = 31
|
|
217
|
+
|
|
218
|
+
# A plain cell reference: uppercase column letters and a row number, with no dollar signs and
|
|
219
|
+
# no whole-row or whole-column form. An absolute reference and a relative one mean the same
|
|
220
|
+
# cell, and admitting both spellings would let two recipes with different digests seal the same
|
|
221
|
+
# bytes.
|
|
222
|
+
_CELL_REFERENCE = re.compile(r"^([A-Z]{1,3})([1-9][0-9]{0,6})$")
|
|
223
|
+
|
|
224
|
+
# The format's own grid limits. A coordinate outside them names a cell that cannot exist.
|
|
225
|
+
_MAX_ROW = 1_048_576
|
|
226
|
+
_MAX_COLUMN = 16_384
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def _column_index(letters: str) -> int:
|
|
230
|
+
"""Turn column letters into a one-based index, on the format's base-26 rule."""
|
|
231
|
+
|
|
232
|
+
index = 0
|
|
233
|
+
for character in letters:
|
|
234
|
+
index = index * 26 + (ord(character) - ord("A") + 1)
|
|
235
|
+
return index
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def _column_letters(index: int) -> str:
|
|
239
|
+
"""Turn a one-based column index back into letters, for a refusal a person can act on."""
|
|
240
|
+
|
|
241
|
+
letters = ""
|
|
242
|
+
while index > 0:
|
|
243
|
+
index, remainder = divmod(index - 1, 26)
|
|
244
|
+
letters = chr(ord("A") + remainder) + letters
|
|
245
|
+
return letters
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def _coordinate(row: int, column: int) -> str:
|
|
249
|
+
return f"{_column_letters(column)}{row}"
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def _cell_reference(value: str, subject: str) -> tuple[int, int]:
|
|
253
|
+
"""Admit one plain cell reference as a (column, row) pair, or refuse it by name."""
|
|
254
|
+
|
|
255
|
+
matched = _CELL_REFERENCE.fullmatch(value)
|
|
256
|
+
if matched is None:
|
|
257
|
+
raise ReaderError(
|
|
258
|
+
"READER_OPTIONS",
|
|
259
|
+
subject,
|
|
260
|
+
f"{value!r} is not a plain cell reference such as A1; a whole row, a whole "
|
|
261
|
+
"column, a lower-case reference, and an absolute reference are all refused",
|
|
262
|
+
)
|
|
263
|
+
column = _column_index(matched.group(1))
|
|
264
|
+
row = int(matched.group(2))
|
|
265
|
+
if column > _MAX_COLUMN or row > _MAX_ROW:
|
|
266
|
+
raise ReaderError(
|
|
267
|
+
"READER_OPTIONS",
|
|
268
|
+
subject,
|
|
269
|
+
f"{value!r} names a cell outside the format's grid",
|
|
270
|
+
)
|
|
271
|
+
return column, row
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def _validate_range(value: Any, subject: str) -> tuple[int, int, int, int]:
|
|
275
|
+
"""Admit a range as two plain cell references, the first at or before the second."""
|
|
276
|
+
|
|
277
|
+
if not isinstance(value, str) or value.count(":") != 1:
|
|
278
|
+
raise ReaderError(
|
|
279
|
+
"READER_OPTIONS",
|
|
280
|
+
subject,
|
|
281
|
+
"must be two plain cell references separated by a colon, such as A1:C4",
|
|
282
|
+
)
|
|
283
|
+
first, second = value.split(":")
|
|
284
|
+
start_column, start_row = _cell_reference(first, subject)
|
|
285
|
+
end_column, end_row = _cell_reference(second, subject)
|
|
286
|
+
if start_column > end_column or start_row > end_row:
|
|
287
|
+
raise ReaderError(
|
|
288
|
+
"READER_OPTIONS",
|
|
289
|
+
subject,
|
|
290
|
+
f"{value!r} runs backwards; a range states its top-left cell first",
|
|
291
|
+
)
|
|
292
|
+
return start_column, start_row, end_column, end_row
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def _validate_options(options: Any) -> dict[str, Any]:
|
|
296
|
+
"""Admit this family's closed option set, refusing a missing or unknown key by name."""
|
|
297
|
+
|
|
298
|
+
subject = "reader.spreadsheet.xlsx.decode_options"
|
|
299
|
+
if not isinstance(options, Mapping):
|
|
300
|
+
raise ReaderError("READER_OPTIONS", subject, "must be an object")
|
|
301
|
+
unknown = sorted(str(key) for key in options if key not in _OPTION_KEYS)
|
|
302
|
+
if unknown:
|
|
303
|
+
raise ReaderError(
|
|
304
|
+
"READER_OPTIONS",
|
|
305
|
+
subject,
|
|
306
|
+
f"names no such setting: {', '.join(unknown)}; this family takes "
|
|
307
|
+
f"{', '.join(_OPTION_KEYS)}",
|
|
308
|
+
)
|
|
309
|
+
missing = sorted(key for key in _OPTION_KEYS if key not in options)
|
|
310
|
+
if missing:
|
|
311
|
+
raise ReaderError(
|
|
312
|
+
"READER_OPTIONS",
|
|
313
|
+
subject,
|
|
314
|
+
f"does not state: {', '.join(missing)}; a spreadsheet is read at pinned "
|
|
315
|
+
"coordinates and all three are required",
|
|
316
|
+
)
|
|
317
|
+
|
|
318
|
+
sheet = options["sheet"]
|
|
319
|
+
if (
|
|
320
|
+
not isinstance(sheet, str)
|
|
321
|
+
or isinstance(sheet, bool)
|
|
322
|
+
or not sheet
|
|
323
|
+
or len(sheet) > _SHEET_NAME_MAX_CHARACTERS
|
|
324
|
+
or any(character < " " or character == "\x7f" for character in sheet)
|
|
325
|
+
):
|
|
326
|
+
raise ReaderError(
|
|
327
|
+
"READER_OPTIONS",
|
|
328
|
+
f"{subject}.sheet",
|
|
329
|
+
"must name one sheet, as bounded text; a sheet is never selected by index, "
|
|
330
|
+
"because sheet order changes without notice and would move a dataset silently",
|
|
331
|
+
)
|
|
332
|
+
|
|
333
|
+
header_row = options["header_row"]
|
|
334
|
+
if type(header_row) is not int or header_row < 1 or header_row > _MAX_ROW:
|
|
335
|
+
raise ReaderError(
|
|
336
|
+
"READER_OPTIONS",
|
|
337
|
+
f"{subject}.header_row",
|
|
338
|
+
"must be the whole number of the row that names the columns",
|
|
339
|
+
)
|
|
340
|
+
|
|
341
|
+
_validate_range(options["range"], f"{subject}.range")
|
|
342
|
+
return {"sheet": sheet, "header_row": header_row, "range": options["range"]}
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
def _admit_workbook_bytes(content: Any, *, budgets: ReaderBudgets, subject: str) -> bytes:
|
|
346
|
+
"""Admit the package's own bytes, before any decoder object is constructed."""
|
|
347
|
+
|
|
348
|
+
if not isinstance(content, (bytes, bytearray)):
|
|
349
|
+
raise ReaderError("READER_ADMISSION", subject, "must be exact bytes")
|
|
350
|
+
raw = bytes(content)
|
|
351
|
+
if len(raw) > budgets.max_input_bytes:
|
|
352
|
+
raise ReaderError("READER_BUDGET", subject, "exceeds the input byte budget")
|
|
353
|
+
if not any(raw.startswith(magic) for magic in ZIP_MAGICS):
|
|
354
|
+
raise ReaderError(
|
|
355
|
+
"READER_ADMISSION",
|
|
356
|
+
subject,
|
|
357
|
+
"does not begin as a workbook package",
|
|
358
|
+
)
|
|
359
|
+
return raw
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
def _inspect_package(raw: bytes, *, budgets: ReaderBudgets, subject: str) -> None:
|
|
363
|
+
"""Decide against the package's own directory, and read its text parts, before the engine.
|
|
364
|
+
|
|
365
|
+
Everything here happens with the engine still untouched. The sizes come from the central
|
|
366
|
+
directory, so a package built to exhaust memory is refused with nothing inflated; the text
|
|
367
|
+
parts are then read under those checked declarations and scanned for the two things a
|
|
368
|
+
workbook may not say -- a document type, and a reference to a resource outside itself.
|
|
369
|
+
"""
|
|
370
|
+
|
|
371
|
+
try:
|
|
372
|
+
archive = zipfile.ZipFile(io.BytesIO(raw))
|
|
373
|
+
except (zipfile.BadZipFile, EOFError, ValueError, zlib.error) as error:
|
|
374
|
+
raise ReaderError(
|
|
375
|
+
"READER_ADMISSION",
|
|
376
|
+
subject,
|
|
377
|
+
"is not a readable workbook package",
|
|
378
|
+
) from error
|
|
379
|
+
|
|
380
|
+
with archive:
|
|
381
|
+
# Reading the directory is the second call into the engine, and the structural test in
|
|
382
|
+
# ``tests/h3/test_readers_xlsx.py`` requires every one of them to be translated -- not
|
|
383
|
+
# because this one is known to raise, but because "known to raise" is exactly the
|
|
384
|
+
# judgement that let the part reads escape untyped in the first place.
|
|
385
|
+
try:
|
|
386
|
+
entries = archive.infolist()
|
|
387
|
+
except Exception as error:
|
|
388
|
+
raise ReaderError(
|
|
389
|
+
"READER_ADMISSION",
|
|
390
|
+
subject,
|
|
391
|
+
f"has a package directory that could not be read: {type(error).__name__}",
|
|
392
|
+
) from error
|
|
393
|
+
if len(entries) > budgets.max_container_members:
|
|
394
|
+
raise ReaderError(
|
|
395
|
+
"READER_BUDGET",
|
|
396
|
+
subject,
|
|
397
|
+
f"holds {len(entries)} parts, over the part budget of "
|
|
398
|
+
f"{budgets.max_container_members}",
|
|
399
|
+
)
|
|
400
|
+
declared = sum(entry.file_size for entry in entries)
|
|
401
|
+
if declared > budgets.max_uncompressed_bytes:
|
|
402
|
+
raise ReaderError(
|
|
403
|
+
"READER_BUDGET",
|
|
404
|
+
subject,
|
|
405
|
+
f"declares {declared} uncompressed bytes across its parts, over the "
|
|
406
|
+
"uncompressed byte budget",
|
|
407
|
+
)
|
|
408
|
+
if declared > max(len(raw), 1) * budgets.max_expansion_ratio:
|
|
409
|
+
raise ReaderError(
|
|
410
|
+
"READER_BUDGET",
|
|
411
|
+
subject,
|
|
412
|
+
f"declares an expansion of {declared} from {len(raw)} bytes, over the "
|
|
413
|
+
f"expansion ratio budget of {budgets.max_expansion_ratio}",
|
|
414
|
+
)
|
|
415
|
+
|
|
416
|
+
names = {entry.filename for entry in entries}
|
|
417
|
+
absent = [part for part in _REQUIRED_PARTS if part not in names]
|
|
418
|
+
if absent:
|
|
419
|
+
raise ReaderError(
|
|
420
|
+
"READER_ADMISSION",
|
|
421
|
+
subject,
|
|
422
|
+
f"is a zip that carries no workbook: it has no {', '.join(absent)}",
|
|
423
|
+
)
|
|
424
|
+
|
|
425
|
+
for entry in entries:
|
|
426
|
+
if not entry.filename.endswith(_TEXT_PART_SUFFIXES):
|
|
427
|
+
continue
|
|
428
|
+
where = f"{subject}.part"
|
|
429
|
+
# The same three directory screens the zip family applies, from the same
|
|
430
|
+
# implementation. Without them an encrypted part escapes as the bare
|
|
431
|
+
# ``RuntimeError`` ``ZipFile.open`` raises, and a part compressed by a method
|
|
432
|
+
# this interpreter was not built with escapes as ``NotImplementedError`` -- both
|
|
433
|
+
# outside the Reader contract's closed table and outside the worker's typed
|
|
434
|
+
# handler, so a refusal a recipe could repair arrives as a codeless crash.
|
|
435
|
+
screen_zip_entry(entry, subject=where, described=f"holds {entry.filename!r}")
|
|
436
|
+
part = _read_package_part(archive, entry, subject=where)
|
|
437
|
+
if _DOCUMENT_TYPE.search(part) is not None:
|
|
438
|
+
raise ReaderError(
|
|
439
|
+
"READER_ADMISSION",
|
|
440
|
+
where,
|
|
441
|
+
f"{entry.filename} declares a document type; a document type may define "
|
|
442
|
+
"an entity naming a resource outside the file, so the workbook is "
|
|
443
|
+
"refused rather than parsed",
|
|
444
|
+
)
|
|
445
|
+
if _EXTERNAL_REFERENCE.search(part) is not None:
|
|
446
|
+
raise ReaderError(
|
|
447
|
+
"READER_ADMISSION",
|
|
448
|
+
where,
|
|
449
|
+
f"{entry.filename} names an external resource; a workbook that points "
|
|
450
|
+
"outside itself is refused rather than followed",
|
|
451
|
+
)
|
|
452
|
+
|
|
453
|
+
|
|
454
|
+
def _read_package_part(archive: zipfile.ZipFile, entry: zipfile.ZipInfo, *, subject: str) -> bytes:
|
|
455
|
+
"""Read one part of the package, mapping every engine failure to a typed refusal.
|
|
456
|
+
|
|
457
|
+
The directory screens above remove the two failures we know the name of; this removes the
|
|
458
|
+
class. ``ZipFile.open`` and the read behind it reach CPython's zip and zlib layers, which
|
|
459
|
+
raise whatever their own layers raise, and an untyped escape from a decode is a refusal
|
|
460
|
+
with no code -- which the clean room reports as ``SANDBOX_UNEXPECTED`` and the remediation
|
|
461
|
+
map cannot bind a fix to. ``ReaderError`` is re-raised first so a budget refusal from
|
|
462
|
+
``read_bounded`` keeps its own code rather than being re-labelled here.
|
|
463
|
+
"""
|
|
464
|
+
|
|
465
|
+
try:
|
|
466
|
+
with archive.open(entry, "r") as stream:
|
|
467
|
+
return read_bounded(stream, declared=entry.file_size, subject=subject)
|
|
468
|
+
except ReaderError:
|
|
469
|
+
raise
|
|
470
|
+
# Deliberately broad, and for the reason ``_load``'s handler is broad. ``RuntimeError``
|
|
471
|
+
# and ``NotImplementedError`` are the two this module has met; naming only those two
|
|
472
|
+
# would leave the next one to escape untyped.
|
|
473
|
+
except Exception as error:
|
|
474
|
+
raise ReaderError(
|
|
475
|
+
"READER_ADMISSION",
|
|
476
|
+
subject,
|
|
477
|
+
f"{entry.filename} could not be read out of the workbook package: "
|
|
478
|
+
f"{type(error).__name__}",
|
|
479
|
+
) from error
|
|
480
|
+
|
|
481
|
+
|
|
482
|
+
def _check_backend(subject: str) -> None:
|
|
483
|
+
"""Refuse a decode on a host whose XML backend is not the pinned one.
|
|
484
|
+
|
|
485
|
+
``openpyxl`` selects its parser at import from what it finds on the host, so this is a
|
|
486
|
+
property of the environment rather than of the file -- which is why the refusal is a
|
|
487
|
+
contract refusal and why its message names both ways to fix the environment.
|
|
488
|
+
"""
|
|
489
|
+
|
|
490
|
+
if openpyxl.xml.LXML:
|
|
491
|
+
raise ReaderError(
|
|
492
|
+
"READER_CONTRACT",
|
|
493
|
+
subject,
|
|
494
|
+
"is running on a host where openpyxl selected the lxml XML backend; this family "
|
|
495
|
+
"is pinned to the standard library backend, because a decoder whose parser "
|
|
496
|
+
"depends on which other packages are installed can answer differently on two "
|
|
497
|
+
"hosts. Uninstall lxml, or set OPENPYXL_LXML=0, and run the decode again",
|
|
498
|
+
)
|
|
499
|
+
|
|
500
|
+
|
|
501
|
+
def _load(raw: bytes, *, data_only: bool, subject: str) -> Any:
|
|
502
|
+
"""Open the workbook from bytes, read-only, mapping every engine failure to a typed one."""
|
|
503
|
+
|
|
504
|
+
try:
|
|
505
|
+
return load_workbook(io.BytesIO(raw), read_only=True, data_only=data_only)
|
|
506
|
+
except ReaderError:
|
|
507
|
+
raise
|
|
508
|
+
# Deliberately broad. The engine raises whatever its own layers raise -- a missing part is
|
|
509
|
+
# a KeyError, a damaged package a BadZipFile, a malformed part a parse error -- and a
|
|
510
|
+
# refusal that escapes untyped is a refusal the remediation map cannot bind a fix to.
|
|
511
|
+
except Exception as error:
|
|
512
|
+
raise ReaderError(
|
|
513
|
+
"READER_ADMISSION",
|
|
514
|
+
subject,
|
|
515
|
+
f"could not be opened as a workbook: {type(error).__name__}",
|
|
516
|
+
) from error
|
|
517
|
+
|
|
518
|
+
|
|
519
|
+
def _declared_window(worksheet: Any, subject: str) -> tuple[int, int, int, int]:
|
|
520
|
+
"""The used area the sheet declares, as (column, row, column, row), or a refusal.
|
|
521
|
+
|
|
522
|
+
A spreadsheet states its used range in the file, so the size of the answer is knowable
|
|
523
|
+
before the first cell is read. A sheet that declares nothing is refused rather than
|
|
524
|
+
scanned to find out: the whole point of reading the declaration is to decide before
|
|
525
|
+
iterating, and a scan to discover the geometry is the iteration being avoided.
|
|
526
|
+
"""
|
|
527
|
+
|
|
528
|
+
bounds = (
|
|
529
|
+
worksheet.min_column,
|
|
530
|
+
worksheet.min_row,
|
|
531
|
+
worksheet.max_column,
|
|
532
|
+
worksheet.max_row,
|
|
533
|
+
)
|
|
534
|
+
if any(type(value) is not int or value < 1 for value in bounds):
|
|
535
|
+
raise ReaderError(
|
|
536
|
+
"READER_ADMISSION",
|
|
537
|
+
subject,
|
|
538
|
+
"declares no used area, so the work it would take to read cannot be bounded "
|
|
539
|
+
"before it is done; re-save the workbook so it records its own dimensions",
|
|
540
|
+
)
|
|
541
|
+
return bounds
|
|
542
|
+
|
|
543
|
+
|
|
544
|
+
def _admit_declared_window(
|
|
545
|
+
worksheet: Any,
|
|
546
|
+
budgets: ReaderBudgets,
|
|
547
|
+
subject: str,
|
|
548
|
+
) -> tuple[int, int, int, int]:
|
|
549
|
+
"""Read and admit declared geometry before any cell iterator is requested."""
|
|
550
|
+
|
|
551
|
+
window = _declared_window(worksheet, subject)
|
|
552
|
+
start_column, start_row, end_column, end_row = window
|
|
553
|
+
check_declared_size(
|
|
554
|
+
row_count=end_row - start_row + 1,
|
|
555
|
+
column_count=end_column - start_column + 1,
|
|
556
|
+
budgets=budgets,
|
|
557
|
+
subject=subject,
|
|
558
|
+
)
|
|
559
|
+
return window
|
|
560
|
+
|
|
561
|
+
|
|
562
|
+
def _window_values(
|
|
563
|
+
worksheet: Any,
|
|
564
|
+
window: tuple[int, int, int, int],
|
|
565
|
+
) -> tuple[tuple[Any, ...], ...]:
|
|
566
|
+
"""The values of one bounded window, padded to the window's own width."""
|
|
567
|
+
|
|
568
|
+
start_column, start_row, end_column, end_row = window
|
|
569
|
+
width = end_column - start_column + 1
|
|
570
|
+
rows: list[tuple[Any, ...]] = []
|
|
571
|
+
for row in worksheet.iter_rows(
|
|
572
|
+
min_row=start_row,
|
|
573
|
+
max_row=end_row,
|
|
574
|
+
min_col=start_column,
|
|
575
|
+
max_col=end_column,
|
|
576
|
+
values_only=True,
|
|
577
|
+
):
|
|
578
|
+
values = tuple(row)[:width]
|
|
579
|
+
rows.append(values + (None,) * (width - len(values)))
|
|
580
|
+
return tuple(rows)
|
|
581
|
+
|
|
582
|
+
|
|
583
|
+
def _formula_cells(
|
|
584
|
+
worksheet: Any,
|
|
585
|
+
window: tuple[int, int, int, int],
|
|
586
|
+
) -> frozenset[tuple[int, int]]:
|
|
587
|
+
"""Which cells of the window hold a formula, read from the cell type and not from text.
|
|
588
|
+
|
|
589
|
+
A typed string beginning with an equals sign is a string. The distinction is in the cell's
|
|
590
|
+
declared type, so that is what is read; inferring it from the value would flag exactly the
|
|
591
|
+
cells a careful author quoted on purpose.
|
|
592
|
+
"""
|
|
593
|
+
|
|
594
|
+
start_column, start_row, end_column, end_row = window
|
|
595
|
+
found: set[tuple[int, int]] = set()
|
|
596
|
+
for row_offset, row in enumerate(
|
|
597
|
+
worksheet.iter_rows(
|
|
598
|
+
min_row=start_row,
|
|
599
|
+
max_row=end_row,
|
|
600
|
+
min_col=start_column,
|
|
601
|
+
max_col=end_column,
|
|
602
|
+
)
|
|
603
|
+
):
|
|
604
|
+
for column_offset, cell in enumerate(row):
|
|
605
|
+
if cell.data_type == "f":
|
|
606
|
+
found.add((start_row + row_offset, start_column + column_offset))
|
|
607
|
+
return frozenset(found)
|
|
608
|
+
|
|
609
|
+
|
|
610
|
+
def _column_names(header: Sequence[Any], start_column: int, row: int) -> tuple[str, ...]:
|
|
611
|
+
"""The column names the header row states, refused rather than repaired."""
|
|
612
|
+
|
|
613
|
+
subject = "reader.spreadsheet.xlsx.header_row"
|
|
614
|
+
names: list[str] = []
|
|
615
|
+
for offset, value in enumerate(header):
|
|
616
|
+
if not isinstance(value, str) or not value.strip():
|
|
617
|
+
raise ReaderError(
|
|
618
|
+
"READER_RANGE",
|
|
619
|
+
subject,
|
|
620
|
+
f"cell {_coordinate(row, start_column + offset)} does not hold a column name; "
|
|
621
|
+
"an empty or non-text header cell is refused rather than auto-named",
|
|
622
|
+
)
|
|
623
|
+
names.append(value)
|
|
624
|
+
if len(set(names)) != len(names):
|
|
625
|
+
repeated = sorted({name for name in names if names.count(name) > 1})
|
|
626
|
+
raise ReaderError(
|
|
627
|
+
"READER_RANGE",
|
|
628
|
+
subject,
|
|
629
|
+
f"repeats a column name: {', '.join(repeated)}; a duplicate header is refused "
|
|
630
|
+
"rather than renamed, because a renamed column is a column nobody asked for",
|
|
631
|
+
)
|
|
632
|
+
return tuple(names)
|
|
633
|
+
|
|
634
|
+
|
|
635
|
+
def _cell_value(value: Any, subject: str) -> Any:
|
|
636
|
+
"""Give a temporal value its one stated text form and leave every other value alone.
|
|
637
|
+
|
|
638
|
+
Numbers, text, and booleans go on to the shared canonical emitter untouched, which is what
|
|
639
|
+
keeps the float rule in one place. Temporal values are converted here because the format
|
|
640
|
+
stores one temporal type and the choice of text form is this family's to state.
|
|
641
|
+
"""
|
|
642
|
+
|
|
643
|
+
if isinstance(value, datetime.datetime):
|
|
644
|
+
return value.isoformat(sep="T")
|
|
645
|
+
if isinstance(value, datetime.date):
|
|
646
|
+
return value.isoformat()
|
|
647
|
+
if isinstance(value, datetime.time):
|
|
648
|
+
return value.isoformat()
|
|
649
|
+
if isinstance(value, datetime.timedelta):
|
|
650
|
+
raise ReaderError(
|
|
651
|
+
"READER_OUTPUT",
|
|
652
|
+
subject,
|
|
653
|
+
"holds a duration; a duration is a rendering of a number rather than a value in "
|
|
654
|
+
"the file, so it has no honest single text form and is refused",
|
|
655
|
+
)
|
|
656
|
+
return value
|
|
657
|
+
|
|
658
|
+
|
|
659
|
+
@dataclass(frozen=True)
|
|
660
|
+
class SpreadsheetReader:
|
|
661
|
+
"""``spreadsheet.xlsx``: one pinned window of one named sheet, opened from bytes.
|
|
662
|
+
|
|
663
|
+
The order of the work is backend, options, admission, package budgets and scan, values,
|
|
664
|
+
formula types, header, emit -- and the function below is written in that order, because
|
|
665
|
+
"refused before the engine sees it" is only true if the code says so.
|
|
666
|
+
"""
|
|
667
|
+
|
|
668
|
+
family_id: str = "spreadsheet.xlsx"
|
|
669
|
+
family_version: str = "1.0.0"
|
|
670
|
+
contract_version: str = READER_CONTRACT_VERSION
|
|
671
|
+
output_format: str = _OUTPUT_FORMAT
|
|
672
|
+
accepted_media_types: tuple[str, ...] = (SPREADSHEET_MEDIA_TYPE,)
|
|
673
|
+
default_budgets: ReaderBudgets = field(default_factory=spreadsheet_default_budgets)
|
|
674
|
+
|
|
675
|
+
def validate_options(self, options: Mapping[str, Any]) -> Mapping[str, Any]:
|
|
676
|
+
return _validate_options(options)
|
|
677
|
+
|
|
678
|
+
def decode(self, content: bytes, pin: ReaderPin, budgets: ReaderBudgets) -> ReaderResult:
|
|
679
|
+
subject = "reader.spreadsheet.xlsx"
|
|
680
|
+
_check_backend(subject)
|
|
681
|
+
options = _validate_options(pin.decode_options)
|
|
682
|
+
sheet_name = str(options["sheet"])
|
|
683
|
+
header_row = int(options["header_row"])
|
|
684
|
+
window = _validate_range(options["range"], f"{subject}.range")
|
|
685
|
+
start_column, start_row, end_column, end_row = window
|
|
686
|
+
|
|
687
|
+
raw = _admit_workbook_bytes(content, budgets=budgets, subject=subject)
|
|
688
|
+
_inspect_package(raw, budgets=budgets, subject=subject)
|
|
689
|
+
|
|
690
|
+
book = _load(raw, data_only=True, subject=subject)
|
|
691
|
+
try:
|
|
692
|
+
if book.epoch != _EPOCH_1900:
|
|
693
|
+
raise ReaderError(
|
|
694
|
+
"READER_ADMISSION",
|
|
695
|
+
subject,
|
|
696
|
+
f"declares the {_EPOCH_1904:%Y} date system; this family reads the 1900 "
|
|
697
|
+
"date system only, because the two differ by four years and a day and "
|
|
698
|
+
"the difference is invisible in a recipe. Re-save the workbook under the "
|
|
699
|
+
"1900 date system",
|
|
700
|
+
)
|
|
701
|
+
if sheet_name not in book.sheetnames:
|
|
702
|
+
raise ReaderError(
|
|
703
|
+
"READER_SHEET_MISSING",
|
|
704
|
+
f"{subject}.sheet",
|
|
705
|
+
f"names {sheet_name!r}, which this workbook does not hold; it holds: "
|
|
706
|
+
f"{', '.join(book.sheetnames)}",
|
|
707
|
+
)
|
|
708
|
+
worksheet = book[sheet_name]
|
|
709
|
+
used = _admit_declared_window(worksheet, budgets, f"{subject}.sheet")
|
|
710
|
+
used_start_column, used_start_row, used_end_column, used_end_row = used
|
|
711
|
+
if (
|
|
712
|
+
start_row < used_start_row
|
|
713
|
+
or start_column < used_start_column
|
|
714
|
+
or end_row > used_end_row
|
|
715
|
+
or end_column > used_end_column
|
|
716
|
+
):
|
|
717
|
+
raise ReaderError(
|
|
718
|
+
"READER_RANGE",
|
|
719
|
+
f"{subject}.range",
|
|
720
|
+
f"names {options['range']!r}, which reaches outside the used area this "
|
|
721
|
+
f"sheet declares, "
|
|
722
|
+
f"{_coordinate(used_start_row, used_start_column)}:"
|
|
723
|
+
f"{_coordinate(used_end_row, used_end_column)}",
|
|
724
|
+
)
|
|
725
|
+
if header_row != start_row:
|
|
726
|
+
raise ReaderError(
|
|
727
|
+
"READER_RANGE",
|
|
728
|
+
f"{subject}.header_row",
|
|
729
|
+
f"names row {header_row}, which is not the first row of the range "
|
|
730
|
+
f"{options['range']!r}; the header row and the range must agree, so that "
|
|
731
|
+
"every row inside the range is either the header or data and none is "
|
|
732
|
+
"silently neither",
|
|
733
|
+
)
|
|
734
|
+
check_declared_size(
|
|
735
|
+
row_count=end_row - start_row + 1,
|
|
736
|
+
column_count=end_column - start_column + 1,
|
|
737
|
+
budgets=budgets,
|
|
738
|
+
subject=f"{subject}.range",
|
|
739
|
+
)
|
|
740
|
+
values = _window_values(worksheet, window)
|
|
741
|
+
finally:
|
|
742
|
+
book.close()
|
|
743
|
+
|
|
744
|
+
typed = _load(raw, data_only=False, subject=subject)
|
|
745
|
+
try:
|
|
746
|
+
formulas = _formula_cells(typed[sheet_name], window)
|
|
747
|
+
finally:
|
|
748
|
+
typed.close()
|
|
749
|
+
|
|
750
|
+
for row_index, column_index in sorted(formulas):
|
|
751
|
+
if values[row_index - start_row][column_index - start_column] is None:
|
|
752
|
+
raise ReaderError(
|
|
753
|
+
"READER_ADMISSION",
|
|
754
|
+
f"{subject}.cell",
|
|
755
|
+
f"{_coordinate(row_index, column_index)} holds a formula with no cached "
|
|
756
|
+
"value; an uncomputed formula is missing information rather than a "
|
|
757
|
+
"missing value, so it is refused. Open and recalculate the workbook, then "
|
|
758
|
+
"save it again",
|
|
759
|
+
)
|
|
760
|
+
|
|
761
|
+
columns = _column_names(values[0], start_column, start_row)
|
|
762
|
+
rows = tuple(
|
|
763
|
+
tuple(
|
|
764
|
+
_cell_value(value, f"{subject}.cell.{_coordinate(start_row + offset + 1, index)}")
|
|
765
|
+
for index, value in enumerate(row, start=start_column)
|
|
766
|
+
)
|
|
767
|
+
for offset, row in enumerate(values[1:])
|
|
768
|
+
)
|
|
769
|
+
return ReaderResult(
|
|
770
|
+
content=encode_canonical_csv(columns, rows, budgets=budgets),
|
|
771
|
+
data_format=_OUTPUT_FORMAT,
|
|
772
|
+
media_type=_OUTPUT_MEDIA_TYPE,
|
|
773
|
+
filename=sealed_filename(sheet_name, suffix=_OUTPUT_SUFFIX),
|
|
774
|
+
row_count=len(rows),
|
|
775
|
+
column_names=columns,
|
|
776
|
+
# Spreadsheet admission charges the complete used area declared by the sheet,
|
|
777
|
+
# including its header row, before iterating any selected cell. Preserve that
|
|
778
|
+
# family-specific geometry rather than reducing it to emitted data rows.
|
|
779
|
+
declared_cell_count=(used_end_row - used_start_row + 1)
|
|
780
|
+
* (used_end_column - used_start_column + 1),
|
|
781
|
+
flags=(_FORMULA_FLAG,) if formulas else (),
|
|
782
|
+
)
|
|
783
|
+
|
|
784
|
+
|
|
785
|
+
@dataclass(frozen=True)
|
|
786
|
+
class SpreadsheetReaderV1_1(SpreadsheetReader):
|
|
787
|
+
"""``spreadsheet.xlsx@1.1.0`` admits the byte-stream label, and reads nothing new.
|
|
788
|
+
|
|
789
|
+
The same one-label delta as the container coordinates, for the same reason: a workbook
|
|
790
|
+
downloaded from an object store or a plain file server very often arrives as
|
|
791
|
+
``application/octet-stream``, which says only that the server declined to describe it.
|
|
792
|
+
The admission behind the label is unchanged and decisive -- a workbook is a zip container
|
|
793
|
+
whose structure ``openpyxl`` validates, so a response that is not a workbook is refused
|
|
794
|
+
by its own bytes whatever the label said. ``1.0.0`` stays closed to the weak label, and
|
|
795
|
+
a label that asserts a different format outright (``application/vnd.ms-excel`` is the
|
|
796
|
+
legacy XLS assertion) remains refused at both coordinates.
|
|
797
|
+
"""
|
|
798
|
+
|
|
799
|
+
family_version: str = "1.1.0"
|
|
800
|
+
accepted_media_types: tuple[str, ...] = (
|
|
801
|
+
"application/octet-stream",
|
|
802
|
+
SPREADSHEET_MEDIA_TYPE,
|
|
803
|
+
)
|