mostlyright-data 0.9.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mostlyright/data_harness/__init__.py +158 -0
- mostlyright/data_harness/acquisition/__init__.py +55 -0
- mostlyright/data_harness/acquisition/http.py +2773 -0
- mostlyright/data_harness/acquisition/parsing.py +809 -0
- mostlyright/data_harness/acquisition/ranges.py +495 -0
- mostlyright/data_harness/acquisition/result_download.py +360 -0
- mostlyright/data_harness/acquisition/retention_admission.py +248 -0
- mostlyright/data_harness/acquisition/sandbox.py +4888 -0
- mostlyright/data_harness/acquisition/url_policy.py +530 -0
- mostlyright/data_harness/agent_runtime.py +2743 -0
- mostlyright/data_harness/assets/logo-ink.svg +31 -0
- mostlyright/data_harness/backends/__init__.py +28 -0
- mostlyright/data_harness/backends/pandas_backend.py +350 -0
- mostlyright/data_harness/backends/polars_backend.py +366 -0
- mostlyright/data_harness/backends/protocol.py +124 -0
- mostlyright/data_harness/backends/reference.py +83 -0
- mostlyright/data_harness/backends/registry.py +55 -0
- mostlyright/data_harness/backends/restrictions.py +126 -0
- mostlyright/data_harness/canonical.py +333 -0
- mostlyright/data_harness/catalog_job.py +625 -0
- mostlyright/data_harness/cli.py +5398 -0
- mostlyright/data_harness/contracts.py +53 -0
- mostlyright/data_harness/coordinator.py +1307 -0
- mostlyright/data_harness/deploy.py +924 -0
- mostlyright/data_harness/deploy_target.py +312 -0
- mostlyright/data_harness/deployment_evidence.py +1067 -0
- mostlyright/data_harness/event_presentation.py +576 -0
- mostlyright/data_harness/events.py +2152 -0
- mostlyright/data_harness/fast_delimited.py +239 -0
- mostlyright/data_harness/fleet.py +237 -0
- mostlyright/data_harness/formats.py +236 -0
- mostlyright/data_harness/governors.py +1163 -0
- mostlyright/data_harness/hosted_bootstrap.py +972 -0
- mostlyright/data_harness/hosted_crawler.py +1115 -0
- mostlyright/data_harness/hosted_crawler_container_smoke.py +351 -0
- mostlyright/data_harness/hosted_crawler_fetch.py +423 -0
- mostlyright/data_harness/hosted_crawler_job.py +1277 -0
- mostlyright/data_harness/hosted_crawler_protocol.py +676 -0
- mostlyright/data_harness/hosted_dataset.py +1500 -0
- mostlyright/data_harness/hosted_deploy.py +3037 -0
- mostlyright/data_harness/hosted_handoff.py +62 -0
- mostlyright/data_harness/hosted_ingestion_contract.py +504 -0
- mostlyright/data_harness/hosted_ingestion_job.py +356 -0
- mostlyright/data_harness/hosted_ingestion_job_smoke.py +40 -0
- mostlyright/data_harness/hosted_session_container_smoke.py +194 -0
- mostlyright/data_harness/hosted_session_worker.py +3554 -0
- mostlyright/data_harness/hosted_session_worker_job_smoke.py +46 -0
- mostlyright/data_harness/hosted_worker.py +6784 -0
- mostlyright/data_harness/ingestion/__init__.py +56 -0
- mostlyright/data_harness/ingestion/contracts.py +461 -0
- mostlyright/data_harness/ingestion/faults.py +42 -0
- mostlyright/data_harness/ingestion/gcs_store.py +1162 -0
- mostlyright/data_harness/ingestion/spool.py +130 -0
- mostlyright/data_harness/ingestion/store.py +885 -0
- mostlyright/data_harness/key_seam.py +434 -0
- mostlyright/data_harness/linux_process_boundary.py +262 -0
- mostlyright/data_harness/local_contracts.py +2880 -0
- mostlyright/data_harness/local_search/__init__.py +5 -0
- mostlyright/data_harness/local_search/build_index.py +1087 -0
- mostlyright/data_harness/local_search/contracts.py +920 -0
- mostlyright/data_harness/local_search/query_trace.py +266 -0
- mostlyright/data_harness/local_search/retrieval.py +700 -0
- mostlyright/data_harness/local_search/sealed.py +474 -0
- mostlyright/data_harness/local_search/service.py +784 -0
- mostlyright/data_harness/nbrender/CONTRACT.md +212 -0
- mostlyright/data_harness/nbrender/__init__.py +12 -0
- mostlyright/data_harness/nbrender/chrome.py +359 -0
- mostlyright/data_harness/nbrender/code_body.py +266 -0
- mostlyright/data_harness/nbrender/document.py +407 -0
- mostlyright/data_harness/nbrender/frame.py +275 -0
- mostlyright/data_harness/nbrender/interactive.py +337 -0
- mostlyright/data_harness/nbrender/markdown_body.py +477 -0
- mostlyright/data_harness/nbrender/mr_components.py +134 -0
- mostlyright/data_harness/nbrender/outputs_data.py +595 -0
- mostlyright/data_harness/nbrender/outputs_rich.py +906 -0
- mostlyright/data_harness/nbrender/outputs_source.py +260 -0
- mostlyright/data_harness/nbrender/outputs_stage.py +176 -0
- mostlyright/data_harness/nbrender/outputs_text.py +400 -0
- mostlyright/data_harness/nbrender/parse.py +394 -0
- mostlyright/data_harness/nbrender/status.py +40 -0
- mostlyright/data_harness/nbrender/tokens.py +1295 -0
- mostlyright/data_harness/notebook.py +1710 -0
- mostlyright/data_harness/offline.py +2049 -0
- mostlyright/data_harness/operation_registry.py +1007 -0
- mostlyright/data_harness/operator_setup.py +239 -0
- mostlyright/data_harness/pipeline.py +6428 -0
- mostlyright/data_harness/plan_graph.py +2026 -0
- mostlyright/data_harness/preparation/__init__.py +104 -0
- mostlyright/data_harness/preparation/contracts.py +1017 -0
- mostlyright/data_harness/preparation/engine.py +221 -0
- mostlyright/data_harness/preparation/errors.py +14 -0
- mostlyright/data_harness/preparation/gates.py +751 -0
- mostlyright/data_harness/preparation/joins.py +574 -0
- mostlyright/data_harness/preparation/profile.py +384 -0
- mostlyright/data_harness/preparation/table.py +217 -0
- mostlyright/data_harness/preparation/transforms.py +568 -0
- mostlyright/data_harness/progress_events.py +534 -0
- mostlyright/data_harness/readers/__init__.py +46 -0
- mostlyright/data_harness/readers/containers.py +963 -0
- mostlyright/data_harness/readers/contracts.py +542 -0
- mostlyright/data_harness/readers/delimited.py +257 -0
- mostlyright/data_harness/readers/grib2/__init__.py +33 -0
- mostlyright/data_harness/readers/grib2/admission.py +722 -0
- mostlyright/data_harness/readers/grib2/decode.py +1009 -0
- mostlyright/data_harness/readers/grib2/geometry.py +1133 -0
- mostlyright/data_harness/readers/grib2/portable_math.py +501 -0
- mostlyright/data_harness/readers/json_tabular.py +485 -0
- mostlyright/data_harness/readers/registry.py +514 -0
- mostlyright/data_harness/readers/samples/README.md +110 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/cities.csv.gz +0 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/cities.csv.gz +0 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/cities.tar +0 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/cities.tar +0 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/cities.zip +0 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/expected.json +25 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/expected.json +25 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/expected.json +25 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/cities.csv +3 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/expected.json +23 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/cities.tsv +5 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/expected.json +25 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/expected.json +30 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/observations.csv +5 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/expected.json +44 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/stations.json +1 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/expected.json +48 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/observations.ndjson +4 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/cities.xlsx +0 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/expected.json +24 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/expected.json +27 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/expected.json +27 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/README.md +20 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/expected.json +55 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/gfs-2m-temperature.grib2 +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/expected.json +54 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/expected.json +54 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/hrrr-categorical-rain.grib2 +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/expected.json +54 -0
- mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
- mostlyright/data_harness/readers/samples.py +582 -0
- mostlyright/data_harness/readers/spreadsheet.py +803 -0
- mostlyright/data_harness/readers/tabular.py +510 -0
- mostlyright/data_harness/recipe.py +5321 -0
- mostlyright/data_harness/repair/__init__.py +78 -0
- mostlyright/data_harness/repair/adapters.py +274 -0
- mostlyright/data_harness/repair/contracts.py +872 -0
- mostlyright/data_harness/repair/coordinator.py +1099 -0
- mostlyright/data_harness/repair/errors.py +16 -0
- mostlyright/data_harness/review.py +2533 -0
- mostlyright/data_harness/rowset.py +283 -0
- mostlyright/data_harness/serving.py +1975 -0
- mostlyright/data_harness/serving_edge.py +590 -0
- mostlyright/data_harness/serving_http.py +1031 -0
- mostlyright/data_harness/session_probes.py +759 -0
- mostlyright/data_harness/signing.py +101 -0
- mostlyright/data_harness/source_discovery.py +898 -0
- mostlyright/data_harness/sources/__init__.py +209 -0
- mostlyright/data_harness/sources/_adapter_steps.py +213 -0
- mostlyright/data_harness/sources/adapters.py +1214 -0
- mostlyright/data_harness/sources/cadence.py +1428 -0
- mostlyright/data_harness/sources/cadence_emission.py +453 -0
- mostlyright/data_harness/sources/cadence_history.py +546 -0
- mostlyright/data_harness/sources/catalog/__init__.py +17 -0
- mostlyright/data_harness/sources/catalog/admission.py +477 -0
- mostlyright/data_harness/sources/catalog/authoring.py +1701 -0
- mostlyright/data_harness/sources/catalog/authoring_policy.py +701 -0
- mostlyright/data_harness/sources/catalog/authoring_shards.py +1217 -0
- mostlyright/data_harness/sources/catalog/bounded_io.py +231 -0
- mostlyright/data_harness/sources/catalog/channel.py +523 -0
- mostlyright/data_harness/sources/catalog/channel_client.py +296 -0
- mostlyright/data_harness/sources/catalog/contracts.py +825 -0
- mostlyright/data_harness/sources/catalog/coverage.py +137 -0
- mostlyright/data_harness/sources/catalog/delta.py +1340 -0
- mostlyright/data_harness/sources/catalog/embedding.py +532 -0
- mostlyright/data_harness/sources/catalog/entry_v2.py +1182 -0
- mostlyright/data_harness/sources/catalog/fill.py +3889 -0
- mostlyright/data_harness/sources/catalog/fill_partitions.py +459 -0
- mostlyright/data_harness/sources/catalog/fill_staging.py +1105 -0
- mostlyright/data_harness/sources/catalog/gating.py +374 -0
- mostlyright/data_harness/sources/catalog/generation_receipt.py +1607 -0
- mostlyright/data_harness/sources/catalog/harvest/__init__.py +7 -0
- mostlyright/data_harness/sources/catalog/harvest/ckan.py +384 -0
- mostlyright/data_harness/sources/catalog/harvest/datagov_v4.py +798 -0
- mostlyright/data_harness/sources/catalog/harvest/protocol.py +964 -0
- mostlyright/data_harness/sources/catalog/harvest/sdmx.py +445 -0
- mostlyright/data_harness/sources/catalog/harvest/stac.py +384 -0
- mostlyright/data_harness/sources/catalog/health.py +447 -0
- mostlyright/data_harness/sources/catalog/hosted_catalog.py +105 -0
- mostlyright/data_harness/sources/catalog/identity_history.py +1549 -0
- mostlyright/data_harness/sources/catalog/neural.py +1618 -0
- mostlyright/data_harness/sources/catalog/packed_catalog.py +2345 -0
- mostlyright/data_harness/sources/catalog/packed_retrieval.py +1517 -0
- mostlyright/data_harness/sources/catalog/packed_writer.py +2802 -0
- mostlyright/data_harness/sources/catalog/query_trace.py +1037 -0
- mostlyright/data_harness/sources/catalog/recommend.py +171 -0
- mostlyright/data_harness/sources/catalog/retrieval.py +230 -0
- mostlyright/data_harness/sources/catalog/retrieval_manifest.py +995 -0
- mostlyright/data_harness/sources/catalog/rights_decisions.py +254 -0
- mostlyright/data_harness/sources/catalog/sealed.py +560 -0
- mostlyright/data_harness/sources/catalog/search.py +230 -0
- mostlyright/data_harness/sources/catalog/streaming_delta.py +1097 -0
- mostlyright/data_harness/sources/catalog/update.py +891 -0
- mostlyright/data_harness/sources/collections.py +815 -0
- mostlyright/data_harness/sources/contracts.py +2223 -0
- mostlyright/data_harness/sources/deletion.py +761 -0
- mostlyright/data_harness/sources/fitness.py +162 -0
- mostlyright/data_harness/sources/governance.py +163 -0
- mostlyright/data_harness/sources/hosted.py +173 -0
- mostlyright/data_harness/sources/integration.py +218 -0
- mostlyright/data_harness/sources/range_reader.py +418 -0
- mostlyright/data_harness/sources/registry.py +514 -0
- mostlyright/data_harness/sources/rights_rule.py +59 -0
- mostlyright/data_harness/sources/source_cadence_vectors.v1.json +1 -0
- mostlyright/data_harness/sources/sports.py +521 -0
- mostlyright/data_harness/sources/stream.py +524 -0
- mostlyright/data_harness/sources/stream_connector.py +418 -0
- mostlyright/data_harness/sources/stream_recorder.py +1404 -0
- mostlyright/data_harness/studio_boundary.py +2019 -0
- mostlyright/data_harness/thin/__init__.py +37 -0
- mostlyright/data_harness/thin/acquire.py +1137 -0
- mostlyright/data_harness/thin/acquire_cancel.py +579 -0
- mostlyright/data_harness/thin/approvals.py +617 -0
- mostlyright/data_harness/thin/commands.py +406 -0
- mostlyright/data_harness/thin/download.py +194 -0
- mostlyright/data_harness/thin/narrative.py +589 -0
- mostlyright/data_harness/thin/parity.py +1070 -0
- mostlyright/data_harness/thin/propose.py +2759 -0
- mostlyright/data_harness/thin/research.py +1663 -0
- mostlyright/data_harness/thin/router.py +924 -0
- mostlyright/data_harness/thin/runs.py +519 -0
- mostlyright/data_harness/thin/session.py +281 -0
- mostlyright/data_harness/thin/stream.py +501 -0
- mostlyright/data_harness/thin/transport.py +187 -0
- mostlyright/data_harness/thin/vocabulary.py +368 -0
- mostlyright/data_harness/thin/workers.py +164 -0
- mostlyright/data_harness/ucum/TABLE-PIN.json +40 -0
- mostlyright/data_harness/ucum/ucum-subset.v1.json +632 -0
- mostlyright/data_harness/unit_flow.py +927 -0
- mostlyright/data_harness/units.py +572 -0
- mostlyright/data_harness/ux/__init__.py +9 -0
- mostlyright/data_harness/ux/approve.py +485 -0
- mostlyright/data_harness/ux/author_yaml.py +597 -0
- mostlyright/data_harness/ux/cloud_auth.py +447 -0
- mostlyright/data_harness/ux/commands/__init__.py +260 -0
- mostlyright/data_harness/ux/commands/approve.py +136 -0
- mostlyright/data_harness/ux/commands/auth.py +744 -0
- mostlyright/data_harness/ux/commands/author.py +79 -0
- mostlyright/data_harness/ux/commands/catalog_author.py +403 -0
- mostlyright/data_harness/ux/commands/catalog_fill.py +523 -0
- mostlyright/data_harness/ux/commands/catalog_harvest.py +545 -0
- mostlyright/data_harness/ux/commands/catalog_publish.py +1838 -0
- mostlyright/data_harness/ux/commands/catalog_search.py +71 -0
- mostlyright/data_harness/ux/commands/catalog_update.py +437 -0
- mostlyright/data_harness/ux/commands/deploy.py +134 -0
- mostlyright/data_harness/ux/commands/deploy_dataset.py +98 -0
- mostlyright/data_harness/ux/commands/deploy_plan.py +105 -0
- mostlyright/data_harness/ux/commands/deploy_status.py +104 -0
- mostlyright/data_harness/ux/commands/diff.py +74 -0
- mostlyright/data_harness/ux/commands/index.py +84 -0
- mostlyright/data_harness/ux/commands/inventory.py +47 -0
- mostlyright/data_harness/ux/commands/list_builds.py +143 -0
- mostlyright/data_harness/ux/commands/login.py +63 -0
- mostlyright/data_harness/ux/commands/peek.py +236 -0
- mostlyright/data_harness/ux/commands/plan_check.py +90 -0
- mostlyright/data_harness/ux/commands/preflight.py +97 -0
- mostlyright/data_harness/ux/commands/record.py +107 -0
- mostlyright/data_harness/ux/commands/review_setup.py +47 -0
- mostlyright/data_harness/ux/commands/search.py +440 -0
- mostlyright/data_harness/ux/commands/show.py +61 -0
- mostlyright/data_harness/ux/commands/whoami.py +37 -0
- mostlyright/data_harness/ux/credential_native.py +551 -0
- mostlyright/data_harness/ux/credential_store.py +1055 -0
- mostlyright/data_harness/ux/credentials.py +631 -0
- mostlyright/data_harness/ux/diffing.py +444 -0
- mostlyright/data_harness/ux/headline.py +671 -0
- mostlyright/data_harness/ux/hosted_acquisition.py +974 -0
- mostlyright/data_harness/ux/hosted_run_status.py +619 -0
- mostlyright/data_harness/ux/inventory.py +427 -0
- mostlyright/data_harness/ux/local_review.py +375 -0
- mostlyright/data_harness/ux/login.py +691 -0
- mostlyright/data_harness/ux/path_kind.py +147 -0
- mostlyright/data_harness/ux/peek.py +1000 -0
- mostlyright/data_harness/ux/plain_file.py +178 -0
- mostlyright/data_harness/ux/plan_check.py +311 -0
- mostlyright/data_harness/ux/preflight.py +918 -0
- mostlyright/data_harness/ux/readers.py +1124 -0
- mostlyright/data_harness/ux/remediation.py +2195 -0
- mostlyright/data_harness/ux/render.py +657 -0
- mostlyright/data_harness/ux/workload.py +1077 -0
- mostlyright/data_harness/viewer.py +3713 -0
- mostlyright/data_harness/visual_run/__init__.py +83 -0
- mostlyright/data_harness/visual_run/authoring.py +235 -0
- mostlyright/data_harness/visual_run/contracts.py +673 -0
- mostlyright/data_harness/visual_run/materialize.py +486 -0
- mostlyright/data_harness/visual_run/observations.py +874 -0
- mostlyright/data_harness/visual_run/query.py +259 -0
- mostlyright/data_harness/visual_run/reducer.py +280 -0
- mostlyright/data_harness/visual_run/sdk.py +892 -0
- mostlyright/data_harness/visual_run/store.py +584 -0
- mostlyright/data_harness/visual_run/transport.py +239 -0
- mostlyright/data_harness/watch.py +2999 -0
- mostlyright_data-0.9.0.dist-info/METADATA +607 -0
- mostlyright_data-0.9.0.dist-info/RECORD +314 -0
- mostlyright_data-0.9.0.dist-info/WHEEL +4 -0
- mostlyright_data-0.9.0.dist-info/entry_points.txt +12 -0
|
@@ -0,0 +1,1975 @@
|
|
|
1
|
+
"""Serving: the one door into a sealed version, and the index that points at one.
|
|
2
|
+
|
|
3
|
+
A request reaches a dataset's bytes through exactly one function here, and that function verifies
|
|
4
|
+
the build before it returns anything. Four boundaries this module holds on purpose:
|
|
5
|
+
|
|
6
|
+
* **It serves only what review admits.** Every read runs :func:`recipe.verify_recipe_candidate` and
|
|
7
|
+
then requires ``drift_report.status == "passed"`` — the same pair
|
|
8
|
+
:func:`deploy.build_deployment_request` already established as what "reviewed and sealed" means in
|
|
9
|
+
this repository. Standing on that exact pair is deliberate: anything looser serves an unreviewed
|
|
10
|
+
draft, and anything different creates a second definition of reviewed, after which the two drift.
|
|
11
|
+
There is no parameter anywhere on this path that skips the gate, which is a property of the
|
|
12
|
+
signatures rather than of a filter.
|
|
13
|
+
* **It never fetches an origin.** The only bytes any path through this module reads are a sealed run
|
|
14
|
+
directory's. The network call that authorizes a caller happens at the edge, in a different module,
|
|
15
|
+
on purpose: keeping it out of here is what makes "no origin fetch" checkable by reading this file.
|
|
16
|
+
* **It never writes.** No path through this module creates, moves, or truncates anything, including
|
|
17
|
+
the version index.
|
|
18
|
+
* **The recipe defines the build, never the serving.** A dataset's shape is read from the sealed
|
|
19
|
+
tree — ``evidence/profile.json``, ``plan.json`` and ``table_card.md`` — and no field of
|
|
20
|
+
:class:`recipe.FrozenRecipe` other than identity is consulted. A serving clause in a recipe would
|
|
21
|
+
make two datasets need two serving code paths.
|
|
22
|
+
|
|
23
|
+
**The version index is evidence, never authority.** The Vault and Current do not exist in this
|
|
24
|
+
repository yet, so a read resolves one through an operator-supplied canonical JSON document that
|
|
25
|
+
this module READS and never writes. That document is deliberately weak: every field it states —
|
|
26
|
+
dataset id, version id, version number, version digest — is re-derived from the sealed run directory
|
|
27
|
+
before a byte is served, and any disagreement is a refusal naming both values. A corrupted or
|
|
28
|
+
hostile index therefore cannot cause an unsealed or wrong serve; the worst it can do is refuse.
|
|
29
|
+
Directory scanning is prohibited because target selection must be explicit.
|
|
30
|
+
|
|
31
|
+
**What this ordering costs, stated rather than hidden.** Every request runs one full verification,
|
|
32
|
+
deterministic replay included, so a read is slow in exactly the way a proof is slow — a whole
|
|
33
|
+
sealed build is re-derived and re-digested before the first byte is handed back. The resulting
|
|
34
|
+
verified snapshot retains its authenticated descriptor lease through the response and supplies the
|
|
35
|
+
Parquet reader directly; serving then holds one Arrow batch plus the bounded response window while
|
|
36
|
+
it iterates row groups. Verification is intentionally per request. Any future cross-request cache
|
|
37
|
+
would need a written invalidation contract tied to the version digest before it could preserve the
|
|
38
|
+
same proof boundary.
|
|
39
|
+
|
|
40
|
+
Refusals are quiet and typed. Every declining path raises :class:`ServingRefused` with named
|
|
41
|
+
reasons, and every reason is a plain sentence carrying at most a bracketed machine code. The
|
|
42
|
+
engineering vocabulary of the exception a reason was translated from does not travel with it; see
|
|
43
|
+
:func:`_refusal_for`.
|
|
44
|
+
|
|
45
|
+
**One envelope, and one crossing between the two vocabularies.** Every shape this surface answers
|
|
46
|
+
returns a :class:`ServingResult`, and every response is rendered from
|
|
47
|
+
:meth:`ServingResult.to_dict` — the JSON body and the human lines alike. That is what makes "every
|
|
48
|
+
response carries the version digest it was served from" a property of one class rather than a habit
|
|
49
|
+
four handlers share. It is also the single place where the engineering names on
|
|
50
|
+
:class:`SealedVersion` become the plain wire names ``version_digest`` and ``table_digest``; the
|
|
51
|
+
decision, its scope and its cost are written out on :meth:`ServingResult.to_dict`.
|
|
52
|
+
"""
|
|
53
|
+
|
|
54
|
+
from __future__ import annotations
|
|
55
|
+
|
|
56
|
+
import datetime
|
|
57
|
+
import json
|
|
58
|
+
import re
|
|
59
|
+
from collections.abc import Callable, Iterator, Mapping, Sequence
|
|
60
|
+
from contextlib import contextmanager
|
|
61
|
+
from dataclasses import dataclass
|
|
62
|
+
from pathlib import Path, PurePosixPath
|
|
63
|
+
from typing import Any
|
|
64
|
+
|
|
65
|
+
import pyarrow as pa
|
|
66
|
+
import pyarrow.compute as pc
|
|
67
|
+
|
|
68
|
+
from mostlyright.data_harness import canonical, deploy, events, pipeline, recipe, rowset
|
|
69
|
+
from mostlyright.data_harness.local_contracts import is_single_plain_line
|
|
70
|
+
|
|
71
|
+
#: Every wire name this surface reads or emits is registered under this prefix, in the convention
|
|
72
|
+
#: ``deploy.DEPLOYMENT_REQUEST_SCHEMA`` already uses: one product-scoped name and one integer
|
|
73
|
+
#: version, bumped whenever the fields change meaning.
|
|
74
|
+
SERVING_SCHEMA_PREFIX = "mostlyright-"
|
|
75
|
+
|
|
76
|
+
# Defense in depth for a synthetic or legacy result that bypassed the sealed-version opener. The
|
|
77
|
+
# authoritative recipe boundary rejects these bytes; this map keeps the human renderer one-line
|
|
78
|
+
# even if a caller constructs a result directly. U+009B and bidi controls are intentionally not
|
|
79
|
+
# escaped here: their terminal policy remains the separately recorded M16 decision.
|
|
80
|
+
_DISPLAY_LINE_BOUNDARY_ESCAPES = str.maketrans(
|
|
81
|
+
{
|
|
82
|
+
"\n": r"\n",
|
|
83
|
+
"\r": r"\r",
|
|
84
|
+
"\v": r"\v",
|
|
85
|
+
"\f": r"\f",
|
|
86
|
+
"\x1c": r"\u001c",
|
|
87
|
+
"\x1d": r"\u001d",
|
|
88
|
+
"\x1e": r"\u001e",
|
|
89
|
+
"\x85": r"\u0085",
|
|
90
|
+
"\u2028": r"\u2028",
|
|
91
|
+
"\u2029": r"\u2029",
|
|
92
|
+
}
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
#: The operator-written document that says which sealed directories may be served, and which
|
|
96
|
+
#: version each dataset's Current names.
|
|
97
|
+
VERSION_INDEX_SCHEMA = f"{SERVING_SCHEMA_PREFIX}table-version-index.v1"
|
|
98
|
+
|
|
99
|
+
#: The envelope EVERY serving response is rendered from, whatever shape answered it. One name for
|
|
100
|
+
#: one body: a caller parses one document shape and finds the version digest in the same place
|
|
101
|
+
#: whether it asked what a dataset is, for one row, or for a slice.
|
|
102
|
+
SERVING_RESULT_SCHEMA = f"{SERVING_SCHEMA_PREFIX}table-serving-result.v1"
|
|
103
|
+
|
|
104
|
+
#: The status word an answered discovery carries.
|
|
105
|
+
STATUS_DATASET_DESCRIBED = "dataset_described"
|
|
106
|
+
|
|
107
|
+
#: The status word every refusal carries, whatever declined it. A caller branches on ``ok``; the
|
|
108
|
+
#: status says which question was being answered when it was declined, and there is only one
|
|
109
|
+
#: refusing answer to give.
|
|
110
|
+
STATUS_REFUSED = "serving_refused"
|
|
111
|
+
|
|
112
|
+
#: A status is a plain lowercase machine word. It is emitted, so it is scanned by 32.6-08's sweep.
|
|
113
|
+
_STATUS = re.compile(r"\A[a-z][a-z0-9_]{0,63}\Z")
|
|
114
|
+
|
|
115
|
+
# These two bounds are not a product limit and must not be read as one. The index is a file an
|
|
116
|
+
# operator writes by hand or a future refresh path generates, and it is parsed before anything about
|
|
117
|
+
# it is known to be true; the bounds exist so a malformed or hostile file is cheap to refuse rather
|
|
118
|
+
# than expensive to walk. They are set far above any plausible real catalog for that reason: if a
|
|
119
|
+
# deployment ever approaches either number, the answer is a real catalog store, not a larger
|
|
120
|
+
# constant here.
|
|
121
|
+
MAX_INDEXED_DATASETS = 256
|
|
122
|
+
MAX_INDEXED_VERSIONS_PER_DATASET = 1_024
|
|
123
|
+
|
|
124
|
+
# The index is untrusted mutable input read by a long-lived process. This ceiling is deliberately
|
|
125
|
+
# generous for the closed dataset/version counts above while still refusing before an attacker can
|
|
126
|
+
# make a request allocate an arbitrary local file.
|
|
127
|
+
MAX_VERSION_INDEX_BYTES = 8 * 1024 * 1024
|
|
128
|
+
|
|
129
|
+
#: The largest ``run_dir`` this reader accepts, and the largest single component of one. The
|
|
130
|
+
#: component bound is POSIX ``NAME_MAX``, so nothing longer can name a directory on a filesystem
|
|
131
|
+
#: this product runs on; the whole-path bound is far above any layout an operator writes by hand and
|
|
132
|
+
#: is here for the reason the two index bounds above are.
|
|
133
|
+
MAX_RUN_DIR_CHARS = 1_024
|
|
134
|
+
MAX_RUN_DIR_COMPONENT_CHARS = 255
|
|
135
|
+
|
|
136
|
+
#: The largest dataset id this surface reads or echoes. :data:`_IDENT` bounds a dataset id at 128
|
|
137
|
+
#: characters, so nothing longer can name a listed dataset and nothing longer needs to be repeated
|
|
138
|
+
#: back to the caller who asked for it.
|
|
139
|
+
MAX_DATASET_ID_CHARS = 128
|
|
140
|
+
|
|
141
|
+
#: The largest pin this surface reads or echoes. The longest spelling the grammar admits is
|
|
142
|
+
#: ``version:`` followed by a 128-character version id.
|
|
143
|
+
#:
|
|
144
|
+
#: Both bounds live HERE, in the module that owns the grammar, and the transport references them
|
|
145
|
+
#: rather than declaring its own. A transport bound that can drift from the library's is a second
|
|
146
|
+
#: policy, and the one that binds is whichever runs first.
|
|
147
|
+
MAX_PIN_CHARS = 256
|
|
148
|
+
|
|
149
|
+
#: The pin vocabulary is CLOSED and is exactly these three spellings.
|
|
150
|
+
PIN_CURRENT = "current"
|
|
151
|
+
_PIN_VERSION_PREFIX = "version:"
|
|
152
|
+
_PIN_DIGEST_PREFIX = "sha256:"
|
|
153
|
+
|
|
154
|
+
_DIGEST = re.compile(r"\A[0-9a-f]{64}\Z")
|
|
155
|
+
|
|
156
|
+
# The identifier shape ``recipe._IDENT`` already admits for a dataset id and a version id. Reusing
|
|
157
|
+
# it rather than inventing a second one keeps an id that a sealed execution record accepts from
|
|
158
|
+
# being an id the index cannot spell.
|
|
159
|
+
_IDENT = re.compile(r"\A[A-Za-z0-9][A-Za-z0-9_.-]{0,127}\Z")
|
|
160
|
+
|
|
161
|
+
_INDEX_KEYS = frozenset({"schema_version", "datasets"})
|
|
162
|
+
_DATASET_KEYS = frozenset({"dataset_id", "table_id", "current_version_id", "versions"})
|
|
163
|
+
_VERSION_KEYS = frozenset(
|
|
164
|
+
{
|
|
165
|
+
"version_id",
|
|
166
|
+
"version_number",
|
|
167
|
+
"version_digest",
|
|
168
|
+
"run_dir",
|
|
169
|
+
"predecessor_version_id",
|
|
170
|
+
}
|
|
171
|
+
)
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
class ServingRefused(RuntimeError):
|
|
175
|
+
"""A read was declined, with every reason named. Never a partial answer.
|
|
176
|
+
|
|
177
|
+
Shaped exactly like :class:`deploy.DeployRefused`: one type for a caller to catch, at least one
|
|
178
|
+
named reason, and no path that returns half an answer. A refusal is an expected outcome — an
|
|
179
|
+
unreviewed build, a corrupted index, a pin naming nothing — and every one of them is something
|
|
180
|
+
the operator or the caller can act on.
|
|
181
|
+
"""
|
|
182
|
+
|
|
183
|
+
def __init__(self, *reasons: str) -> None:
|
|
184
|
+
cleaned = tuple(reason.strip() for reason in reasons if reason and reason.strip())
|
|
185
|
+
if not cleaned:
|
|
186
|
+
raise ValueError("a refusal must name at least one reason")
|
|
187
|
+
self.reasons: tuple[str, ...] = cleaned
|
|
188
|
+
super().__init__("; ".join(cleaned))
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
@dataclass(frozen=True)
|
|
192
|
+
class IndexedVersion:
|
|
193
|
+
"""One version's row in the index: what the operator SAYS about a sealed directory.
|
|
194
|
+
|
|
195
|
+
Every field here is a claim, not a fact. :func:`open_sealed_version` re-derives all of them from
|
|
196
|
+
the sealed tree and refuses any disagreement, so nothing downstream may treat these values as
|
|
197
|
+
authoritative.
|
|
198
|
+
|
|
199
|
+
``version_digest`` carries the wire name because this dataclass mirrors an operator-written
|
|
200
|
+
document one field for one key. :class:`SealedVersion` mirrors
|
|
201
|
+
:class:`pipeline.VerifiedCandidate` instead and therefore keeps that type's engineering names
|
|
202
|
+
(``candidate_digest``, ``table_sha256``). The asymmetry is deliberate: each dataclass carries
|
|
203
|
+
the names of the thing it mirrors, and the one place the two vocabularies meet is
|
|
204
|
+
``ServingResult.to_dict``.
|
|
205
|
+
|
|
206
|
+
``run_dir`` is the resolved absolute path. Resolution happens in the parser so no later code has
|
|
207
|
+
to remember to do it, and so a path refusal is raised before any filesystem access.
|
|
208
|
+
"""
|
|
209
|
+
|
|
210
|
+
dataset_id: str
|
|
211
|
+
version_id: str
|
|
212
|
+
version_number: int
|
|
213
|
+
version_digest: str
|
|
214
|
+
run_dir: Path
|
|
215
|
+
predecessor_version_id: str | None
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
@dataclass(frozen=True)
|
|
219
|
+
class IndexedDataset:
|
|
220
|
+
"""One dataset's block in the index: the versions it lists, and which one Current names."""
|
|
221
|
+
|
|
222
|
+
dataset_id: str
|
|
223
|
+
table_id: str
|
|
224
|
+
current_version_id: str
|
|
225
|
+
versions: tuple[IndexedVersion, ...]
|
|
226
|
+
|
|
227
|
+
def version(self, version_id: str) -> IndexedVersion | None:
|
|
228
|
+
"""Return the listed version with this id, or nothing."""
|
|
229
|
+
|
|
230
|
+
for entry in self.versions:
|
|
231
|
+
if entry.version_id == version_id:
|
|
232
|
+
return entry
|
|
233
|
+
return None
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
@dataclass(frozen=True)
|
|
237
|
+
class VersionIndex:
|
|
238
|
+
"""A whole parsed index: which datasets exist, where their versions are, and what Current is."""
|
|
239
|
+
|
|
240
|
+
schema: str
|
|
241
|
+
root: Path
|
|
242
|
+
datasets: tuple[IndexedDataset, ...]
|
|
243
|
+
|
|
244
|
+
def dataset(self, dataset_id: str) -> IndexedDataset | None:
|
|
245
|
+
"""Internal Table lookup retained while callers migrate to :meth:`table`."""
|
|
246
|
+
|
|
247
|
+
for entry in self.datasets:
|
|
248
|
+
if entry.table_id == dataset_id:
|
|
249
|
+
return entry
|
|
250
|
+
return None
|
|
251
|
+
|
|
252
|
+
def table(self, dataset_id: str, table_id: str) -> IndexedDataset | None:
|
|
253
|
+
"""Return only an exact parent Dataset/Table coordinate; never infer a parent."""
|
|
254
|
+
|
|
255
|
+
for entry in self.datasets:
|
|
256
|
+
if entry.dataset_id == dataset_id and entry.table_id == table_id:
|
|
257
|
+
return entry
|
|
258
|
+
return None
|
|
259
|
+
|
|
260
|
+
@property
|
|
261
|
+
def dataset_ids(self) -> tuple[str, ...]:
|
|
262
|
+
"""Every dataset id this index lists, in the order it lists them."""
|
|
263
|
+
|
|
264
|
+
return tuple(entry.table_id for entry in self.datasets)
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
def _one_plain_line(value: Any, limit: int) -> bool:
|
|
268
|
+
"""Whether ``value`` is text, one plain line, and within ``limit`` characters.
|
|
269
|
+
|
|
270
|
+
``deploy._PLAIN_LINE`` (``deploy.py:63``) is the rule and it is REFERENCED rather than restated:
|
|
271
|
+
text with no ASCII control, DEL, or Unicode line boundary anywhere in it, which is the same
|
|
272
|
+
regular expression ``deploy`` already applies to every name it prints. A second spelling of
|
|
273
|
+
this rule is how one surface comes to admit a character another refuses.
|
|
274
|
+
|
|
275
|
+
Two kinds of string reach this function and both need the same answer. One is a piece of an
|
|
276
|
+
operator's version index that becomes a filesystem path, where a control character is a fault
|
|
277
|
+
the path checks below cannot see: ``PurePosixPath('version-1\\x00extra').as_posix()`` is
|
|
278
|
+
byte-identical to what went in, so every shape rule that path spelling can express passes, and
|
|
279
|
+
the refusal only arrives as a ``ValueError`` out of the first ``stat``. The other is a piece of
|
|
280
|
+
a request this surface echoes back — a dataset id, a pin — which :func:`serving_lines` renders
|
|
281
|
+
beside real facts like ``version checksum:``. A value carrying a newline could forge one of
|
|
282
|
+
those lines; a value carrying a hundred kilobytes is not a name.
|
|
283
|
+
"""
|
|
284
|
+
|
|
285
|
+
return (
|
|
286
|
+
isinstance(value, str)
|
|
287
|
+
and len(value) <= limit
|
|
288
|
+
and deploy._PLAIN_LINE.fullmatch(value) is not None
|
|
289
|
+
)
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
def _text(value: Any, label: str) -> str:
|
|
293
|
+
if not isinstance(value, str) or _IDENT.fullmatch(value) is None:
|
|
294
|
+
raise ServingRefused(
|
|
295
|
+
f"the version index does not name {label} in the agreed shape: a name is up to 128 "
|
|
296
|
+
"letters, digits, dots, dashes or underscores [SERVING_INDEX_FIELD]"
|
|
297
|
+
)
|
|
298
|
+
return value
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
def _closed_keys(value: Any, expected: frozenset[str], label: str) -> Mapping[str, Any]:
|
|
302
|
+
if not isinstance(value, dict):
|
|
303
|
+
raise ServingRefused(
|
|
304
|
+
f"the version index must describe {label} as a JSON object [SERVING_INDEX_SHAPE]"
|
|
305
|
+
)
|
|
306
|
+
if "candidate_digest" in value and "version_digest" not in value:
|
|
307
|
+
# The one plausible wrong spelling, named rather than reported as an unknown key: the
|
|
308
|
+
# engineering name for this value really is candidate_digest, so an operator who read the
|
|
309
|
+
# build's own JSON output would write it in good faith.
|
|
310
|
+
raise ServingRefused(
|
|
311
|
+
"the version index names each version's digest version_digest, and this one carries "
|
|
312
|
+
"candidate_digest instead [SERVING_INDEX_FIELD]"
|
|
313
|
+
)
|
|
314
|
+
unknown = sorted(set(value) - expected)
|
|
315
|
+
if unknown:
|
|
316
|
+
raise ServingRefused(
|
|
317
|
+
f"the version index names fields on {label} that this reader does not know: "
|
|
318
|
+
+ ", ".join(unknown)
|
|
319
|
+
+ " [SERVING_INDEX_FIELD]"
|
|
320
|
+
)
|
|
321
|
+
missing = sorted(expected - set(value))
|
|
322
|
+
if missing:
|
|
323
|
+
raise ServingRefused(
|
|
324
|
+
f"the version index leaves fields off {label}: "
|
|
325
|
+
+ ", ".join(missing)
|
|
326
|
+
+ " [SERVING_INDEX_FIELD]"
|
|
327
|
+
)
|
|
328
|
+
return value
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
def _indexed_run_dir(value: Any, root: Path, label: str) -> Path:
|
|
332
|
+
"""Resolve one ``run_dir`` against the index root, refusing anything that could leave it.
|
|
333
|
+
|
|
334
|
+
The rules and their wording follow ``pipeline._validated_source_relative_path``, and the
|
|
335
|
+
absolute form is taken with ``pipeline._absolute_lexical_path`` — which makes a path absolute
|
|
336
|
+
WITHOUT resolving or following any filesystem component. Both are cited rather than
|
|
337
|
+
reimplemented so this module does not become a third path policy that can disagree with the
|
|
338
|
+
other two. All of it happens before any filesystem access, so a hostile index cannot make the
|
|
339
|
+
reader stat a path outside its own directory even once.
|
|
340
|
+
|
|
341
|
+
**Every component is one plain line, and that check comes first.** The shape rules below are
|
|
342
|
+
about where a path points, and none of them can see a control character: a component holding a
|
|
343
|
+
NUL round-trips through :class:`PurePosixPath` byte-identically and satisfies all of them, so
|
|
344
|
+
without this gate the refusal arrives as a ``ValueError`` out of the first ``stat`` — which is a
|
|
345
|
+
fault rather than a refusal, and this module's contract is that a hostile index can only ever
|
|
346
|
+
make it refuse. The rule is :func:`_one_plain_line`, which is ``deploy._PLAIN_LINE``.
|
|
347
|
+
|
|
348
|
+
The refusing sentence names the field and never repeats the value back, for the same reason the
|
|
349
|
+
rule exists: a value that failed the plain-line check is exactly the value that must not reach a
|
|
350
|
+
log line.
|
|
351
|
+
"""
|
|
352
|
+
|
|
353
|
+
if not isinstance(value, str) or not value or "\\" in value:
|
|
354
|
+
raise ServingRefused(
|
|
355
|
+
f"the version index must name {label} as a relative path using '/' separators "
|
|
356
|
+
"[SERVING_INDEX_PATH]"
|
|
357
|
+
)
|
|
358
|
+
pure = PurePosixPath(value)
|
|
359
|
+
if not _one_plain_line(value, MAX_RUN_DIR_CHARS) or any(
|
|
360
|
+
not _one_plain_line(part, MAX_RUN_DIR_COMPONENT_CHARS) for part in pure.parts
|
|
361
|
+
):
|
|
362
|
+
raise ServingRefused(
|
|
363
|
+
f"the version index must name {label} as plain path components of at most "
|
|
364
|
+
f"{MAX_RUN_DIR_COMPONENT_CHARS} characters each, with no control character in any of "
|
|
365
|
+
"them [SERVING_INDEX_PATH]"
|
|
366
|
+
)
|
|
367
|
+
if (
|
|
368
|
+
pure.is_absolute()
|
|
369
|
+
or not pure.parts
|
|
370
|
+
or any(part in {"", ".", ".."} for part in pure.parts)
|
|
371
|
+
or pure.as_posix() != value
|
|
372
|
+
):
|
|
373
|
+
raise ServingRefused(
|
|
374
|
+
f"the version index names a directory outside its own folder for {label}: {value} "
|
|
375
|
+
"[SERVING_INDEX_PATH]"
|
|
376
|
+
)
|
|
377
|
+
resolved = pipeline._absolute_lexical_path(root / pure)
|
|
378
|
+
if not pipeline._is_relative_to(resolved, root):
|
|
379
|
+
raise ServingRefused(
|
|
380
|
+
f"the version index names a directory outside its own folder for {label}: {value} "
|
|
381
|
+
"[SERVING_INDEX_PATH]"
|
|
382
|
+
)
|
|
383
|
+
return resolved
|
|
384
|
+
|
|
385
|
+
|
|
386
|
+
def _indexed_version(value: Any, *, dataset_id: str, root: Path) -> IndexedVersion:
|
|
387
|
+
version = _closed_keys(value, _VERSION_KEYS, f"a version of dataset {dataset_id}")
|
|
388
|
+
version_id = _text(version["version_id"], f"a version of dataset {dataset_id}")
|
|
389
|
+
number = version["version_number"]
|
|
390
|
+
if type(number) is not int or number < 1:
|
|
391
|
+
raise ServingRefused(
|
|
392
|
+
f"the version number of {version_id} must be a whole number of at least 1 "
|
|
393
|
+
"[SERVING_INDEX_FIELD]"
|
|
394
|
+
)
|
|
395
|
+
digest = version["version_digest"]
|
|
396
|
+
if not isinstance(digest, str) or _DIGEST.fullmatch(digest) is None:
|
|
397
|
+
raise ServingRefused(
|
|
398
|
+
f"the version digest of {version_id} must be a 64-character lowercase hex digest "
|
|
399
|
+
"[SERVING_INDEX_FIELD]"
|
|
400
|
+
)
|
|
401
|
+
predecessor = version["predecessor_version_id"]
|
|
402
|
+
if predecessor is not None:
|
|
403
|
+
predecessor = _text(predecessor, f"the version {version_id} follows")
|
|
404
|
+
return IndexedVersion(
|
|
405
|
+
dataset_id=dataset_id,
|
|
406
|
+
version_id=version_id,
|
|
407
|
+
version_number=number,
|
|
408
|
+
version_digest=digest,
|
|
409
|
+
run_dir=_indexed_run_dir(version["run_dir"], root, f"version {version_id}"),
|
|
410
|
+
predecessor_version_id=predecessor,
|
|
411
|
+
)
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
def _indexed_dataset(value: Any, *, root: Path) -> IndexedDataset:
|
|
415
|
+
dataset = _closed_keys(value, _DATASET_KEYS, "a dataset")
|
|
416
|
+
dataset_id = _text(dataset["dataset_id"], "a dataset")
|
|
417
|
+
table_id = _text(dataset["table_id"], f"a table in dataset {dataset_id}")
|
|
418
|
+
versions = dataset["versions"]
|
|
419
|
+
if not isinstance(versions, list) or not versions:
|
|
420
|
+
raise ServingRefused(
|
|
421
|
+
f"the version index lists no versions for dataset {dataset_id} [SERVING_INDEX_SHAPE]"
|
|
422
|
+
)
|
|
423
|
+
if len(versions) > MAX_INDEXED_VERSIONS_PER_DATASET:
|
|
424
|
+
raise ServingRefused(
|
|
425
|
+
f"the version index lists more than {MAX_INDEXED_VERSIONS_PER_DATASET} versions for "
|
|
426
|
+
f"dataset {dataset_id} [SERVING_INDEX_BOUNDS]"
|
|
427
|
+
)
|
|
428
|
+
parsed = tuple(_indexed_version(item, dataset_id=table_id, root=root) for item in versions)
|
|
429
|
+
seen: set[str] = set()
|
|
430
|
+
for entry in parsed:
|
|
431
|
+
if entry.version_id in seen:
|
|
432
|
+
raise ServingRefused(
|
|
433
|
+
f"the version index lists version {entry.version_id} of dataset {dataset_id} more "
|
|
434
|
+
"than once, so it does not say which directory that version is "
|
|
435
|
+
"[SERVING_INDEX_AMBIGUOUS]"
|
|
436
|
+
)
|
|
437
|
+
seen.add(entry.version_id)
|
|
438
|
+
digests: set[str] = set()
|
|
439
|
+
for entry in parsed:
|
|
440
|
+
if entry.version_digest in digests:
|
|
441
|
+
# A pin of the form sha256:<digest> names ONE version. Two entries carrying one digest
|
|
442
|
+
# would make that spelling ambiguous, and an ambiguous pin is not a pin.
|
|
443
|
+
raise ServingRefused(
|
|
444
|
+
f"the version index gives two versions of dataset {dataset_id} the same digest, so "
|
|
445
|
+
f"a read pinned to {entry.version_digest} would be ambiguous "
|
|
446
|
+
"[SERVING_INDEX_AMBIGUOUS]"
|
|
447
|
+
)
|
|
448
|
+
digests.add(entry.version_digest)
|
|
449
|
+
current = _text(dataset["current_version_id"], f"the current version of dataset {dataset_id}")
|
|
450
|
+
if current not in seen:
|
|
451
|
+
raise ServingRefused(
|
|
452
|
+
f"the version index says the current version of dataset {dataset_id} is {current}, and "
|
|
453
|
+
"does not list a version by that name [SERVING_INDEX_CURRENT]"
|
|
454
|
+
)
|
|
455
|
+
return IndexedDataset(
|
|
456
|
+
dataset_id=dataset_id,
|
|
457
|
+
table_id=table_id,
|
|
458
|
+
current_version_id=current,
|
|
459
|
+
versions=parsed,
|
|
460
|
+
)
|
|
461
|
+
|
|
462
|
+
|
|
463
|
+
def parse_version_index(raw: bytes | str, *, root: Path) -> VersionIndex:
|
|
464
|
+
"""Read the operator-supplied version index strictly, and trust it for nothing.
|
|
465
|
+
|
|
466
|
+
``root`` is the directory the index file itself lives in. Every ``run_dir`` is resolved against
|
|
467
|
+
it lexically, and anything absolute, anything carrying a parent component, and anything whose
|
|
468
|
+
resolved form is not under ``root`` is refused before a single filesystem call is made.
|
|
469
|
+
|
|
470
|
+
The bytes are read with :func:`canonical.parse_canonical_json`, so a hand-edited or
|
|
471
|
+
re-serialized file is refused rather than acted on — the same discipline
|
|
472
|
+
``governors.FileCapStore`` applies to its own store. Nothing about the returned index is treated
|
|
473
|
+
as true: it is a set of claims that
|
|
474
|
+
:func:`open_sealed_version` checks against the sealed trees they describe.
|
|
475
|
+
|
|
476
|
+
Raises:
|
|
477
|
+
ServingRefused: when the bytes are not canonical JSON, when the document is not
|
|
478
|
+
``mostlyright-version-index.v1``, when any field is not the agreed shape, when a dataset
|
|
479
|
+
or version is listed twice, when Current names nothing, or when a ``run_dir`` could
|
|
480
|
+
leave the index's own directory.
|
|
481
|
+
"""
|
|
482
|
+
|
|
483
|
+
index_root = pipeline._absolute_lexical_path(Path(root))
|
|
484
|
+
try:
|
|
485
|
+
document = canonical.parse_canonical_json(raw)
|
|
486
|
+
except canonical.CanonicalJSONError as error:
|
|
487
|
+
# Translated, not passed through: the parser's detail is written for an engineer reading a
|
|
488
|
+
# stack trace, and this sentence is printed to whoever wrote the file.
|
|
489
|
+
raise ServingRefused(
|
|
490
|
+
"the version index is not the canonical JSON this reader accepts, so it was not read "
|
|
491
|
+
f"[{error.code}]"
|
|
492
|
+
) from error
|
|
493
|
+
|
|
494
|
+
top = _closed_keys(document, _INDEX_KEYS, "the version index")
|
|
495
|
+
schema = top["schema_version"]
|
|
496
|
+
if schema != VERSION_INDEX_SCHEMA:
|
|
497
|
+
found = schema if isinstance(schema, str) else type(schema).__name__
|
|
498
|
+
raise ServingRefused(
|
|
499
|
+
f"the version index must be {VERSION_INDEX_SCHEMA}, and this one says {found} "
|
|
500
|
+
"[SERVING_INDEX_SCHEMA]"
|
|
501
|
+
)
|
|
502
|
+
datasets = top["datasets"]
|
|
503
|
+
if not isinstance(datasets, list) or not datasets:
|
|
504
|
+
raise ServingRefused("the version index lists no datasets [SERVING_INDEX_SHAPE]")
|
|
505
|
+
if len(datasets) > MAX_INDEXED_DATASETS:
|
|
506
|
+
raise ServingRefused(
|
|
507
|
+
f"the version index lists more than {MAX_INDEXED_DATASETS} datasets "
|
|
508
|
+
"[SERVING_INDEX_BOUNDS]"
|
|
509
|
+
)
|
|
510
|
+
parsed = tuple(_indexed_dataset(item, root=index_root) for item in datasets)
|
|
511
|
+
seen: set[str] = set()
|
|
512
|
+
for entry in parsed:
|
|
513
|
+
if entry.table_id in seen:
|
|
514
|
+
raise ServingRefused(
|
|
515
|
+
f"the version index lists table {entry.table_id} more than once, so it does "
|
|
516
|
+
"not say which block describes it [SERVING_INDEX_AMBIGUOUS]"
|
|
517
|
+
)
|
|
518
|
+
seen.add(entry.table_id)
|
|
519
|
+
return VersionIndex(schema=VERSION_INDEX_SCHEMA, root=index_root, datasets=parsed)
|
|
520
|
+
|
|
521
|
+
|
|
522
|
+
def read_version_index(path: Path) -> VersionIndex:
|
|
523
|
+
"""Read the index document at ``path`` and parse it, resolving every ``run_dir`` beside it.
|
|
524
|
+
|
|
525
|
+
The read path's own reader: it turns a file into an index with a refusal a CUSTOMER may see, so
|
|
526
|
+
it names no path in its sentence. ``mr-data serve``'s start-up check reads the same document
|
|
527
|
+
through the command line's own file reader, which does name the path, because an operator who
|
|
528
|
+
mistyped one is the only reader of that sentence. Both end in :func:`parse_version_index`, so
|
|
529
|
+
there is one parser and two audiences rather than two readers. It reads and never writes, like
|
|
530
|
+
everything else here.
|
|
531
|
+
|
|
532
|
+
The index is read for each request so ``current`` resolves against the latest complete document.
|
|
533
|
+
|
|
534
|
+
Raises:
|
|
535
|
+
ServingRefused: when the file cannot be read, and for every reason
|
|
536
|
+
:func:`parse_version_index` refuses.
|
|
537
|
+
"""
|
|
538
|
+
|
|
539
|
+
location = Path(path)
|
|
540
|
+
try:
|
|
541
|
+
raw = events.read_regular_bytes(location, max_bytes=MAX_VERSION_INDEX_BYTES)
|
|
542
|
+
except OSError as error:
|
|
543
|
+
# The operator's path is deliberately absent from the sentence: this refusal is rendered
|
|
544
|
+
# into a customer-facing envelope, and a response is not a place to publish where a
|
|
545
|
+
# deployment keeps its files. The errno stays on the exception this one is chained to.
|
|
546
|
+
raise ServingRefused(
|
|
547
|
+
"the version index could not be read, so no read was served [SERVING_INDEX_UNREADABLE]"
|
|
548
|
+
) from error
|
|
549
|
+
return parse_version_index(raw, root=location.parent)
|
|
550
|
+
|
|
551
|
+
|
|
552
|
+
def _elsewhere(
|
|
553
|
+
index: VersionIndex,
|
|
554
|
+
match: Callable[[IndexedVersion], bool],
|
|
555
|
+
) -> tuple[str, str] | None:
|
|
556
|
+
"""Return (dataset_id, version_id) for the first version in ANY dataset that matches."""
|
|
557
|
+
|
|
558
|
+
for dataset in index.datasets:
|
|
559
|
+
for entry in dataset.versions:
|
|
560
|
+
if match(entry):
|
|
561
|
+
return dataset.dataset_id, entry.version_id
|
|
562
|
+
return None
|
|
563
|
+
|
|
564
|
+
|
|
565
|
+
def resolve_pin(index: VersionIndex, dataset_id: str, pin: str) -> IndexedVersion:
|
|
566
|
+
"""Turn one pin into one listed version, or refuse.
|
|
567
|
+
|
|
568
|
+
The pin vocabulary is CLOSED and is exactly three spellings:
|
|
569
|
+
|
|
570
|
+
* ``"current"`` — whatever the index says this dataset's Current is right now.
|
|
571
|
+
* ``"version:<version id>"`` — one named version, whatever Current becomes.
|
|
572
|
+
* ``"sha256:<64 lowercase hex>"`` — one version by its digest.
|
|
573
|
+
|
|
574
|
+
The digest spelling leads because it is the only one that is self-authenticating against the
|
|
575
|
+
sealed tree WITHOUT the index: a caller holding a digest can check for itself that the bytes it
|
|
576
|
+
received are the bytes it asked for. The other two mean something only relative to a document.
|
|
577
|
+
|
|
578
|
+
A version NUMBER is deliberately not a pin spelling. Numbers resolve only through the index, and
|
|
579
|
+
the index is evidence; a pin has to be able to mean something without it, and "version 2" does
|
|
580
|
+
not.
|
|
581
|
+
|
|
582
|
+
**A pin and a dataset id are each one plain line, bounded, before either is repeated back.**
|
|
583
|
+
This is the library's own gate and not the transport's: the production topology is a library
|
|
584
|
+
caller — the Studio API edge, calling with an already-verified key id — so a bound that only
|
|
585
|
+
exists in ``serving_http`` is a bound production does not have. Both values reach a refusal
|
|
586
|
+
sentence here and the envelope's ``dataset_id`` and ``pin`` fields, which
|
|
587
|
+
:func:`serving_lines` renders beside ``version checksum:`` and ``table checksum:``. An
|
|
588
|
+
unbounded, unchecked echo there is how a caller writes a line into this product's own human
|
|
589
|
+
rendering that no sealed version produced. The rule is :func:`_one_plain_line`; a value that
|
|
590
|
+
fails it is refused without being quoted.
|
|
591
|
+
|
|
592
|
+
Raises:
|
|
593
|
+
ServingRefused: when the dataset is not listed, when the pin is spelled outside the closed
|
|
594
|
+
set, when it names no listed version, or when it names a version of another dataset.
|
|
595
|
+
"""
|
|
596
|
+
|
|
597
|
+
if not _one_plain_line(pin, MAX_PIN_CHARS):
|
|
598
|
+
raise ServingRefused(
|
|
599
|
+
"a version is asked for as one plain line of at most "
|
|
600
|
+
f"{MAX_PIN_CHARS} characters: current, version:<version id>, or sha256:<digest> "
|
|
601
|
+
"[SERVING_PIN]"
|
|
602
|
+
)
|
|
603
|
+
if not _one_plain_line(dataset_id, MAX_DATASET_ID_CHARS):
|
|
604
|
+
raise ServingRefused(
|
|
605
|
+
"a dataset is named by one plain line of at most "
|
|
606
|
+
f"{MAX_DATASET_ID_CHARS} characters [SERVING_DATASET_UNKNOWN]"
|
|
607
|
+
)
|
|
608
|
+
dataset = index.dataset(dataset_id)
|
|
609
|
+
if dataset is None:
|
|
610
|
+
raise ServingRefused(
|
|
611
|
+
f"there is no dataset called {dataset_id} here [SERVING_DATASET_UNKNOWN]"
|
|
612
|
+
)
|
|
613
|
+
|
|
614
|
+
if pin == PIN_CURRENT:
|
|
615
|
+
current = dataset.version(dataset.current_version_id)
|
|
616
|
+
# The parser already refused an index whose Current names nothing, so this is unreachable
|
|
617
|
+
# from a parsed index; it is restated because the alternative is an assert that a caller
|
|
618
|
+
# holding a hand-built index could trip into returning None.
|
|
619
|
+
if current is None: # pragma: no cover - refused at parse time
|
|
620
|
+
raise ServingRefused(
|
|
621
|
+
f"the current version of dataset {dataset_id} is not listed [SERVING_INDEX_CURRENT]"
|
|
622
|
+
)
|
|
623
|
+
return current
|
|
624
|
+
|
|
625
|
+
if pin.startswith(_PIN_VERSION_PREFIX):
|
|
626
|
+
wanted = pin[len(_PIN_VERSION_PREFIX) :]
|
|
627
|
+
found = dataset.version(wanted)
|
|
628
|
+
if found is not None:
|
|
629
|
+
return found
|
|
630
|
+
other = _elsewhere(index, lambda entry: entry.version_id == wanted)
|
|
631
|
+
if other is not None:
|
|
632
|
+
raise ServingRefused(
|
|
633
|
+
f"version {wanted} belongs to dataset {other[0]}, not to dataset {dataset_id} "
|
|
634
|
+
"[SERVING_PIN_DATASET]"
|
|
635
|
+
)
|
|
636
|
+
raise ServingRefused(
|
|
637
|
+
f"there is no version {wanted} of dataset {dataset_id} here [SERVING_VERSION_UNKNOWN]"
|
|
638
|
+
)
|
|
639
|
+
|
|
640
|
+
if pin.startswith(_PIN_DIGEST_PREFIX):
|
|
641
|
+
wanted = pin[len(_PIN_DIGEST_PREFIX) :]
|
|
642
|
+
if _DIGEST.fullmatch(wanted) is None:
|
|
643
|
+
raise ServingRefused(
|
|
644
|
+
"a version pinned by digest is asked for as sha256: followed by 64 lowercase hex "
|
|
645
|
+
"characters [SERVING_PIN]"
|
|
646
|
+
)
|
|
647
|
+
for entry in dataset.versions:
|
|
648
|
+
if entry.version_digest == wanted:
|
|
649
|
+
return entry
|
|
650
|
+
other = _elsewhere(index, lambda entry: entry.version_digest == wanted)
|
|
651
|
+
if other is not None:
|
|
652
|
+
raise ServingRefused(
|
|
653
|
+
f"the version with digest {wanted} belongs to dataset {other[0]}, not to dataset "
|
|
654
|
+
f"{dataset_id} [SERVING_PIN_DATASET]"
|
|
655
|
+
)
|
|
656
|
+
raise ServingRefused(
|
|
657
|
+
f"there is no version with digest {wanted} of dataset {dataset_id} here "
|
|
658
|
+
"[SERVING_VERSION_UNKNOWN]"
|
|
659
|
+
)
|
|
660
|
+
|
|
661
|
+
raise ServingRefused(
|
|
662
|
+
"a version is asked for as current, as version:<version id>, or as sha256:<digest>, and "
|
|
663
|
+
f"{pin} is none of those [SERVING_PIN]"
|
|
664
|
+
)
|
|
665
|
+
|
|
666
|
+
|
|
667
|
+
@dataclass(frozen=True)
|
|
668
|
+
class SealedVersion:
|
|
669
|
+
"""One verified, reviewed version, opened: identity, digests, shape, and small evidence.
|
|
670
|
+
|
|
671
|
+
A value of this type exists only on the far side of :func:`open_sealed_version`, which is the
|
|
672
|
+
only function that constructs one. That is the structural half of "serves only sealed versions":
|
|
673
|
+
there is no other way to obtain these facts, and no argument that produces one without the
|
|
674
|
+
gate. The potentially large Parquet member deliberately does not live on this value; row
|
|
675
|
+
serving reads it through the retained :class:`pipeline.VerifiedSnapshot` descriptor.
|
|
676
|
+
|
|
677
|
+
Its digest attributes are ``candidate_digest`` and ``table_sha256`` — the names
|
|
678
|
+
:class:`pipeline.VerifiedCandidate` uses — because this type mirrors that one. The rename to the
|
|
679
|
+
wire names ``version_digest`` and ``table_digest`` happens exactly once in
|
|
680
|
+
``ServingResult.to_dict``. This type does not carry both names.
|
|
681
|
+
|
|
682
|
+
``pin_kind`` is ``"current"`` or ``"pinned"``: which question the caller asked, recorded so a
|
|
683
|
+
response can say whether the digest it carries is a moving target or a promise.
|
|
684
|
+
|
|
685
|
+
``sources``, ``join``, ``quality`` and ``lineage`` are the sealed evidence documents decoded,
|
|
686
|
+
and they live here rather than being re-read by a shape that wants them. That is the whole
|
|
687
|
+
point of the single opening: the tree is observed once, under one lease, and every fact a
|
|
688
|
+
response can carry comes out of that observation. A second read could observe different bytes.
|
|
689
|
+
"""
|
|
690
|
+
|
|
691
|
+
dataset_id: str
|
|
692
|
+
version_id: str
|
|
693
|
+
version_number: int
|
|
694
|
+
candidate_digest: str
|
|
695
|
+
table_sha256: str
|
|
696
|
+
columns: tuple[str, ...]
|
|
697
|
+
column_types: tuple[tuple[str, str], ...]
|
|
698
|
+
grain: tuple[str, ...]
|
|
699
|
+
row_count: int
|
|
700
|
+
question: str
|
|
701
|
+
table_card: bytes
|
|
702
|
+
pin_kind: str
|
|
703
|
+
sources: tuple[Any, ...]
|
|
704
|
+
join: Mapping[str, Any]
|
|
705
|
+
quality: Mapping[str, Any]
|
|
706
|
+
lineage: Mapping[str, Any]
|
|
707
|
+
|
|
708
|
+
|
|
709
|
+
def _named(entry: IndexedVersion) -> str:
|
|
710
|
+
"""The phrase every refusal about one version opens with, written once."""
|
|
711
|
+
|
|
712
|
+
return f"version {entry.version_id} of dataset {entry.dataset_id}"
|
|
713
|
+
|
|
714
|
+
|
|
715
|
+
def _refusal_for(error: Exception, entry: IndexedVersion) -> str:
|
|
716
|
+
"""Turn a typed build or recipe failure into one plain sentence a customer can read.
|
|
717
|
+
|
|
718
|
+
TRANSLATE; do not pass through. The verifier's own free-text detail is written in the
|
|
719
|
+
engineering vocabulary — "is not a recipe candidate" — and ``docs/VOCABULARY.md`` supersedes the
|
|
720
|
+
word at its centre. These sentences are read by whoever called the serving surface. The failing
|
|
721
|
+
part and the bracketed code travel, because the code is the precise greppable handle and
|
|
722
|
+
``mr-data recipe-verify`` on the same directory prints the full engineering text; the
|
|
723
|
+
engineering sentence itself does not travel. ``deploy._refusal_for`` established this shape and
|
|
724
|
+
the reason for it, including the special case for a missing predecessor — a different next
|
|
725
|
+
action for the operator, which must not be paraphrased as a verification failure.
|
|
726
|
+
"""
|
|
727
|
+
|
|
728
|
+
if isinstance(error, recipe.RecipeError) and error.code == "RECIPE_PREDECESSOR":
|
|
729
|
+
return (
|
|
730
|
+
f"{_named(entry)} follows an earlier version, and the version index does not say which "
|
|
731
|
+
"one, so it was not served [SERVING_PREDECESSOR]"
|
|
732
|
+
)
|
|
733
|
+
if isinstance(error, recipe.RecipeError):
|
|
734
|
+
return (
|
|
735
|
+
f"{_named(entry)} is not a reviewed, sealed build that can be served: {error.path} "
|
|
736
|
+
f"[{error.code}]"
|
|
737
|
+
)
|
|
738
|
+
if isinstance(error, pipeline.BuildError):
|
|
739
|
+
# A directory with no sealed build in it at all is refused one layer below the recipe
|
|
740
|
+
# rules, so the greppable handle here is the build gate's finding id rather than a recipe
|
|
741
|
+
# code. The next action is the same either way: point the index at a reviewed build.
|
|
742
|
+
return (
|
|
743
|
+
f"{_named(entry)} is not a reviewed, sealed build that can be served "
|
|
744
|
+
f"[{error.finding_id}]"
|
|
745
|
+
)
|
|
746
|
+
return (
|
|
747
|
+
f"{_named(entry)} could not be read as a reviewed, sealed build "
|
|
748
|
+
f"[{type(error).__name__.upper()}]"
|
|
749
|
+
)
|
|
750
|
+
|
|
751
|
+
|
|
752
|
+
def _predecessor_run_dir(index: VersionIndex, entry: IndexedVersion) -> Path | None:
|
|
753
|
+
"""Resolve the directory holding the version this one follows, or refuse."""
|
|
754
|
+
|
|
755
|
+
if entry.predecessor_version_id is None:
|
|
756
|
+
return None
|
|
757
|
+
dataset = index.dataset(entry.dataset_id)
|
|
758
|
+
previous = None if dataset is None else dataset.version(entry.predecessor_version_id)
|
|
759
|
+
if previous is None:
|
|
760
|
+
raise ServingRefused(
|
|
761
|
+
f"{_named(entry)} follows {entry.predecessor_version_id}, and the version index does "
|
|
762
|
+
"not list a version by that name, so it was not served [SERVING_PREDECESSOR]"
|
|
763
|
+
)
|
|
764
|
+
return previous.run_dir
|
|
765
|
+
|
|
766
|
+
|
|
767
|
+
def _sealed_document(members: Mapping[str, bytes], path: str, entry: IndexedVersion) -> Any:
|
|
768
|
+
"""Read one sealed JSON member, keeping every non-integer number as its exact sealed text.
|
|
769
|
+
|
|
770
|
+
Its bytes are digest-bound by the manifest already verified, so the only question here is how
|
|
771
|
+
they are decoded, and ``parse_float=str`` is the whole answer. The sealed members are written
|
|
772
|
+
with ``pipeline.canonical_json_line_bytes``, which admits non-integer numbers —
|
|
773
|
+
``evidence/join.json`` really does carry ``"row_multiplier": 1.0`` — while
|
|
774
|
+
:func:`canonical.canonical_json_bytes`, the encoder every response goes out through, raises on
|
|
775
|
+
any float. Decoding such a value into a binary float and re-rendering it would be a second
|
|
776
|
+
wording of a sealed fact and would make the response unencodable besides. Taking the sealed
|
|
777
|
+
TEXT instead means the number a caller reads is the number the seal covers, character for
|
|
778
|
+
character, and no float ever exists on this path to be rounded.
|
|
779
|
+
"""
|
|
780
|
+
|
|
781
|
+
try:
|
|
782
|
+
return json.loads(members[path], parse_float=str)
|
|
783
|
+
except (KeyError, UnicodeError, json.JSONDecodeError) as error:
|
|
784
|
+
raise ServingRefused(
|
|
785
|
+
f"{_named(entry)} does not describe its own shape, so it was not served "
|
|
786
|
+
"[SERVING_VERSION_SHAPE]"
|
|
787
|
+
) from error
|
|
788
|
+
|
|
789
|
+
|
|
790
|
+
def _sealed_json(members: Mapping[str, bytes], path: str, entry: IndexedVersion) -> Any:
|
|
791
|
+
"""Read one sealed JSON member that must be an object."""
|
|
792
|
+
|
|
793
|
+
value = _sealed_document(members, path, entry)
|
|
794
|
+
if not isinstance(value, dict):
|
|
795
|
+
raise ServingRefused(
|
|
796
|
+
f"{_named(entry)} does not describe its own shape, so it was not served "
|
|
797
|
+
"[SERVING_VERSION_SHAPE]"
|
|
798
|
+
)
|
|
799
|
+
return value
|
|
800
|
+
|
|
801
|
+
|
|
802
|
+
def _disagreement(entry: IndexedVersion, field: str, stated: object, sealed: object) -> str:
|
|
803
|
+
"""One sentence for one corroboration failure, naming BOTH values in that order.
|
|
804
|
+
|
|
805
|
+
The index's value comes first because an operator reading this is holding the index file open.
|
|
806
|
+
"""
|
|
807
|
+
|
|
808
|
+
return (
|
|
809
|
+
f"the version index says the {field} of {_named(entry)} is {stated}, and the sealed files "
|
|
810
|
+
f"say it is {sealed}, so nothing was served [SERVING_INDEX_DISAGREES]"
|
|
811
|
+
)
|
|
812
|
+
|
|
813
|
+
|
|
814
|
+
def open_sealed_version(index: VersionIndex, dataset_id: str, pin: str) -> SealedVersion:
|
|
815
|
+
"""Open one reviewed sealed version from one retained replay-verified snapshot."""
|
|
816
|
+
|
|
817
|
+
entry = resolve_pin(index, dataset_id, pin)
|
|
818
|
+
with _open_sealed_version_snapshot(index, entry, pin) as (version, _snapshot):
|
|
819
|
+
return version
|
|
820
|
+
|
|
821
|
+
|
|
822
|
+
@contextmanager
|
|
823
|
+
def _open_sealed_version_snapshot(
|
|
824
|
+
index: VersionIndex,
|
|
825
|
+
entry: IndexedVersion,
|
|
826
|
+
pin: str,
|
|
827
|
+
) -> Iterator[tuple[SealedVersion, pipeline.VerifiedSnapshot]]:
|
|
828
|
+
"""Keep the verified snapshot leased for consumers that need its lazy Parquet handle."""
|
|
829
|
+
|
|
830
|
+
previous_run_dir = _predecessor_run_dir(index, entry)
|
|
831
|
+
try:
|
|
832
|
+
with pipeline.open_verified_snapshot(entry.run_dir) as verified_snapshot:
|
|
833
|
+
version = _open_sealed_version_from_snapshot(
|
|
834
|
+
entry,
|
|
835
|
+
pin,
|
|
836
|
+
previous_run_dir=previous_run_dir,
|
|
837
|
+
verified_snapshot=verified_snapshot,
|
|
838
|
+
)
|
|
839
|
+
yield version, verified_snapshot
|
|
840
|
+
except (
|
|
841
|
+
recipe.RecipeError,
|
|
842
|
+
pipeline.BuildError,
|
|
843
|
+
canonical.CanonicalJSONError,
|
|
844
|
+
OSError,
|
|
845
|
+
) as error:
|
|
846
|
+
raise ServingRefused(_refusal_for(error, entry)) from error
|
|
847
|
+
|
|
848
|
+
|
|
849
|
+
def _open_sealed_version_from_snapshot(
|
|
850
|
+
entry: IndexedVersion,
|
|
851
|
+
pin: str,
|
|
852
|
+
*,
|
|
853
|
+
previous_run_dir: Path | None,
|
|
854
|
+
verified_snapshot: pipeline.VerifiedSnapshot,
|
|
855
|
+
) -> SealedVersion:
|
|
856
|
+
"""Open one reviewed, sealed version. The only way into a version's bytes.
|
|
857
|
+
|
|
858
|
+
The order below is the contract, not an implementation detail:
|
|
859
|
+
|
|
860
|
+
1. **Resolve the pin once.** The entry is bound, and everything after is derived from the bound
|
|
861
|
+
entry — the pointer is never re-read inside one request. The failure this prevents is a
|
|
862
|
+
refresh landing between the digest stamp and the rows.
|
|
863
|
+
2. **Verify.** :func:`recipe.verify_recipe_candidate` over the bound directory, with the
|
|
864
|
+
predecessor taken from the index entry and ``None`` for an initial build. Every typed failure
|
|
865
|
+
is translated into a plain sentence; none is passed through.
|
|
866
|
+
3. **Require the build to have passed its own checks.** ``drift_report.status == "passed"``, the
|
|
867
|
+
same condition ``deploy.build_deployment_request`` applies. A sealed build that drifted is an
|
|
868
|
+
unreviewed draft and is refused with its finding codes named.
|
|
869
|
+
4. **Corroborate within the retained observation.** Compare the candidate digest derived by
|
|
870
|
+
recipe inspection with the replay-verified identity carried by the same
|
|
871
|
+
:class:`pipeline.VerifiedSnapshot`, then revalidate that snapshot's retained descriptors.
|
|
872
|
+
There is no pathname reopen and no second tree observation on this path; disagreement means
|
|
873
|
+
two derivations from the one leased snapshot failed to describe the same candidate, while a
|
|
874
|
+
lease validation failure means its namespace changed during the operation.
|
|
875
|
+
5. **Corroborate the index against the sealed execution record.** Dataset id, version id,
|
|
876
|
+
version number and digest must all agree, and any disagreement refuses naming both values.
|
|
877
|
+
The check exists because the index is a convenience whose corruption must not be able to
|
|
878
|
+
cause a wrong or unsealed serve — so it is checked against the thing it describes rather than
|
|
879
|
+
believed. Without this step, an index entry pointing one version id at another version's
|
|
880
|
+
directory would serve the wrong bytes under the right name, and every digest in the response
|
|
881
|
+
would agree with itself.
|
|
882
|
+
6. **Read the shape from the sealed tree only** — never from the recipe.
|
|
883
|
+
|
|
884
|
+
No parameter bypasses these checks. A signature test enforces that interface.
|
|
885
|
+
|
|
886
|
+
Raises:
|
|
887
|
+
ServingRefused: when the pin names nothing, when the directory holds no reviewed sealed
|
|
888
|
+
build, when the build did not pass its own checks, when its predecessor is not named,
|
|
889
|
+
when the retained snapshot lease detects a namespace change, when the snapshot's
|
|
890
|
+
derived identities disagree, or when the index disagrees with the sealed record.
|
|
891
|
+
"""
|
|
892
|
+
|
|
893
|
+
try:
|
|
894
|
+
inspection, embedded = recipe.verify_recipe_candidate(
|
|
895
|
+
entry.run_dir,
|
|
896
|
+
# Always passed explicitly: this module's boundary is where the default lives, so the
|
|
897
|
+
# verifier's own required keyword is never silently omitted.
|
|
898
|
+
previous_run_dir=previous_run_dir,
|
|
899
|
+
_snapshot=verified_snapshot,
|
|
900
|
+
)
|
|
901
|
+
except (
|
|
902
|
+
recipe.RecipeError,
|
|
903
|
+
pipeline.BuildError,
|
|
904
|
+
canonical.CanonicalJSONError,
|
|
905
|
+
OSError,
|
|
906
|
+
) as error:
|
|
907
|
+
raise ServingRefused(_refusal_for(error, entry)) from error
|
|
908
|
+
|
|
909
|
+
drift_report = embedded["drift_report"]
|
|
910
|
+
if drift_report.status != "passed":
|
|
911
|
+
# Release eligibility is what ``mr-data recipe-verify`` reports, and a build that failed its
|
|
912
|
+
# own checks is not a version to serve. Each code is bracketed separately so the whole
|
|
913
|
+
# sentence still reads as prose plus machine handles.
|
|
914
|
+
codes = " ".join(
|
|
915
|
+
f"[{code}]" for code in sorted({item.code for item in drift_report.findings})
|
|
916
|
+
)
|
|
917
|
+
raise ServingRefused(
|
|
918
|
+
f"{_named(entry)} did not pass its own checks, so it is not a version to serve: "
|
|
919
|
+
f"{codes or '[REPAIR_REQUIRED]'}"
|
|
920
|
+
)
|
|
921
|
+
|
|
922
|
+
verified = verified_snapshot.verified
|
|
923
|
+
members = verified_snapshot.members
|
|
924
|
+
verified_snapshot.validate()
|
|
925
|
+
if verified.candidate_digest != inspection.candidate_digest:
|
|
926
|
+
raise ServingRefused(
|
|
927
|
+
f"the sealed files behind {_named(entry)} changed while they were being read, so "
|
|
928
|
+
"nothing was served [SERVING_TREE_MOVED]"
|
|
929
|
+
)
|
|
930
|
+
|
|
931
|
+
execution = embedded["execution"]
|
|
932
|
+
if execution.table_id != entry.dataset_id:
|
|
933
|
+
raise ServingRefused(_disagreement(entry, "table", entry.dataset_id, execution.table_id))
|
|
934
|
+
if execution.table_version_id != entry.version_id:
|
|
935
|
+
raise ServingRefused(
|
|
936
|
+
_disagreement(entry, "table version id", entry.version_id, execution.table_version_id)
|
|
937
|
+
)
|
|
938
|
+
if execution.table_version_number != entry.version_number:
|
|
939
|
+
raise ServingRefused(
|
|
940
|
+
_disagreement(
|
|
941
|
+
entry, "table version number", entry.version_number, execution.table_version_number
|
|
942
|
+
)
|
|
943
|
+
)
|
|
944
|
+
if entry.version_digest != verified.candidate_digest:
|
|
945
|
+
# Named by the WIRE name, because whoever reads this sentence is holding the index file
|
|
946
|
+
# open and the key they are looking at is spelled version_digest.
|
|
947
|
+
raise ServingRefused(
|
|
948
|
+
_disagreement(entry, "version digest", entry.version_digest, verified.candidate_digest)
|
|
949
|
+
)
|
|
950
|
+
|
|
951
|
+
profile = _sealed_json(members, "evidence/profile.json", entry)
|
|
952
|
+
plan = _sealed_json(members, "plan.json", entry)
|
|
953
|
+
sources = _sealed_document(members, "evidence/sources.json", entry)
|
|
954
|
+
join = _sealed_json(members, "evidence/join.json", entry)
|
|
955
|
+
quality = _sealed_json(members, "evidence/quality.json", entry)
|
|
956
|
+
lineage = _sealed_json(members, "evidence/lineage.json", entry)
|
|
957
|
+
if not isinstance(sources, list):
|
|
958
|
+
raise ServingRefused(
|
|
959
|
+
f"{_named(entry)} does not describe its own shape, so it was not served "
|
|
960
|
+
"[SERVING_VERSION_SHAPE]"
|
|
961
|
+
)
|
|
962
|
+
try:
|
|
963
|
+
# Checked here rather than where the card is served, so every "can this version be
|
|
964
|
+
# answered from" question is settled in one place and a shape that renders the card cannot
|
|
965
|
+
# raise where it is supposed to refuse.
|
|
966
|
+
members["table_card.md"].decode("utf-8")
|
|
967
|
+
except UnicodeDecodeError as error:
|
|
968
|
+
raise ServingRefused(
|
|
969
|
+
f"{_named(entry)} does not describe its own shape, so it was not served "
|
|
970
|
+
"[SERVING_VERSION_SHAPE]"
|
|
971
|
+
) from error
|
|
972
|
+
columns = profile.get("columns")
|
|
973
|
+
types = profile.get("types")
|
|
974
|
+
row_count = profile.get("row_count")
|
|
975
|
+
grain = plan.get("grain")
|
|
976
|
+
question = plan.get("question")
|
|
977
|
+
if (
|
|
978
|
+
not isinstance(columns, list)
|
|
979
|
+
or not all(isinstance(item, str) for item in columns)
|
|
980
|
+
or not isinstance(types, dict)
|
|
981
|
+
or set(types) != set(columns)
|
|
982
|
+
or type(row_count) is not int
|
|
983
|
+
or not isinstance(grain, list)
|
|
984
|
+
or not all(isinstance(item, str) for item in grain)
|
|
985
|
+
or not is_single_plain_line(question)
|
|
986
|
+
):
|
|
987
|
+
raise ServingRefused(
|
|
988
|
+
f"{_named(entry)} does not describe its own shape, so it was not served "
|
|
989
|
+
"[SERVING_VERSION_SHAPE]"
|
|
990
|
+
)
|
|
991
|
+
|
|
992
|
+
return SealedVersion(
|
|
993
|
+
dataset_id=execution.table_id,
|
|
994
|
+
version_id=execution.table_version_id,
|
|
995
|
+
version_number=execution.table_version_number,
|
|
996
|
+
candidate_digest=verified.candidate_digest,
|
|
997
|
+
table_sha256=verified.table_sha256,
|
|
998
|
+
columns=tuple(columns),
|
|
999
|
+
column_types=tuple(sorted((str(key), str(value)) for key, value in types.items())),
|
|
1000
|
+
grain=tuple(grain),
|
|
1001
|
+
row_count=row_count,
|
|
1002
|
+
question=question,
|
|
1003
|
+
table_card=members["table_card.md"],
|
|
1004
|
+
pin_kind=PIN_CURRENT if pin == PIN_CURRENT else "pinned",
|
|
1005
|
+
sources=tuple(sources),
|
|
1006
|
+
join=join,
|
|
1007
|
+
quality=quality,
|
|
1008
|
+
lineage=lineage,
|
|
1009
|
+
)
|
|
1010
|
+
|
|
1011
|
+
|
|
1012
|
+
@dataclass(frozen=True)
|
|
1013
|
+
class ServingResult:
|
|
1014
|
+
"""The verdict on one serving read: what was served, or the reasons nothing was.
|
|
1015
|
+
|
|
1016
|
+
Invariant: ``ok`` is true if and only if ``refusals`` is empty and ``payload`` is not None.
|
|
1017
|
+
Only two shapes exist, exactly as in :class:`deploy.GoLiveResult`. A refusing result never
|
|
1018
|
+
hands back a payload, because a payload in hand is the thing a caller reads.
|
|
1019
|
+
|
|
1020
|
+
The bound version's identity and digests are optional and are present on **every answering
|
|
1021
|
+
shape**. They are also present on a refusal that got far enough to bind a version, and null on
|
|
1022
|
+
one that did not — a refusal before resolution has no version to name, and inventing one would
|
|
1023
|
+
be worse than saying nothing. ``table_digest`` stays null on every refusal in this plan: the
|
|
1024
|
+
table's digest is a sealed fact that only opening the version produces, and a refusal never
|
|
1025
|
+
opened it. On a refusal ``version_digest`` is the digest the read was FOR, as the version index
|
|
1026
|
+
states it; it is not a claim that anything was verified, and ``ok`` is false beside it.
|
|
1027
|
+
|
|
1028
|
+
``payload`` holds whatever the answering shape answered with. The envelope does not know or
|
|
1029
|
+
care which shape that was. The answering/refusing invariant belongs to this class rather than
|
|
1030
|
+
to individual handlers, so it cannot hold for three shapes and fail for the fourth.
|
|
1031
|
+
"""
|
|
1032
|
+
|
|
1033
|
+
ok: bool
|
|
1034
|
+
refusals: tuple[str, ...]
|
|
1035
|
+
status: str
|
|
1036
|
+
dataset_id: str
|
|
1037
|
+
pin: str
|
|
1038
|
+
pin_kind: str | None
|
|
1039
|
+
version_id: str | None
|
|
1040
|
+
version_number: int | None
|
|
1041
|
+
version_digest: str | None
|
|
1042
|
+
table_digest: str | None
|
|
1043
|
+
payload: Mapping[str, Any] | None
|
|
1044
|
+
# V3 Dataset/Table responses carry the child identity independently. Historical local reads
|
|
1045
|
+
# retain their original table-only envelope and leave this absent.
|
|
1046
|
+
table_id: str | None = None
|
|
1047
|
+
|
|
1048
|
+
def __post_init__(self) -> None:
|
|
1049
|
+
if type(self.ok) is not bool:
|
|
1050
|
+
raise ServingRefused("ok must be a boolean verdict")
|
|
1051
|
+
if type(self.refusals) is not tuple or any(
|
|
1052
|
+
type(item) is not str or not item.strip() for item in self.refusals
|
|
1053
|
+
):
|
|
1054
|
+
raise ServingRefused("refusals must be a tuple of named reasons")
|
|
1055
|
+
if not isinstance(self.status, str) or _STATUS.fullmatch(self.status) is None:
|
|
1056
|
+
raise ServingRefused("a serving result always carries one plain status word")
|
|
1057
|
+
for name in ("dataset_id", "pin"):
|
|
1058
|
+
if not isinstance(getattr(self, name), str):
|
|
1059
|
+
raise ServingRefused(
|
|
1060
|
+
"a serving result records the dataset and pin it was asked for"
|
|
1061
|
+
)
|
|
1062
|
+
if self.table_id is not None and not isinstance(self.table_id, str):
|
|
1063
|
+
raise ServingRefused("a V3 serving result records a Table identifier as text")
|
|
1064
|
+
if self.pin_kind is not None and self.pin_kind not in {PIN_CURRENT, "pinned"}:
|
|
1065
|
+
raise ServingRefused("a bound version was asked for as current or as a pin")
|
|
1066
|
+
if self.version_number is not None and (
|
|
1067
|
+
type(self.version_number) is not int or self.version_number < 1
|
|
1068
|
+
):
|
|
1069
|
+
raise ServingRefused("a bound version number is a whole number of at least 1")
|
|
1070
|
+
for name in ("version_digest", "table_digest"):
|
|
1071
|
+
value = getattr(self, name)
|
|
1072
|
+
if value is not None and (
|
|
1073
|
+
not isinstance(value, str) or _DIGEST.fullmatch(value) is None
|
|
1074
|
+
):
|
|
1075
|
+
raise ServingRefused("a digest is 64 lowercase hex characters or nothing at all")
|
|
1076
|
+
if self.payload is not None and not isinstance(self.payload, Mapping):
|
|
1077
|
+
raise ServingRefused("a payload is a mapping of named facts or nothing")
|
|
1078
|
+
if self.refusals and self.payload is not None:
|
|
1079
|
+
raise ServingRefused("a refused read never hands back a payload")
|
|
1080
|
+
clean = not self.refusals and self.payload is not None
|
|
1081
|
+
if self.ok is not clean:
|
|
1082
|
+
raise ServingRefused(
|
|
1083
|
+
"a read is ok only when there are no refusals and something was answered"
|
|
1084
|
+
)
|
|
1085
|
+
|
|
1086
|
+
def to_dict(self) -> dict[str, Any]:
|
|
1087
|
+
"""Return the one fact set both renderings are built from.
|
|
1088
|
+
|
|
1089
|
+
Every human line and every JSON field this product prints for a serving read comes from
|
|
1090
|
+
here, so the two cannot report different things.
|
|
1091
|
+
|
|
1092
|
+
This function maps engineering field names to the customer response fields:
|
|
1093
|
+
|
|
1094
|
+
================== ==========================================
|
|
1095
|
+
Wire name emitted Engineering name it is read from
|
|
1096
|
+
================== ==========================================
|
|
1097
|
+
``version_digest`` :attr:`SealedVersion.candidate_digest`
|
|
1098
|
+
``table_digest`` :attr:`SealedVersion.table_sha256`
|
|
1099
|
+
================== ==========================================
|
|
1100
|
+
|
|
1101
|
+
A JSON key a production caller parses is the most user-visible string this surface emits,
|
|
1102
|
+
and ``docs/VOCABULARY.md`` records "sealed candidate" as superseded by "Draft / Build" — so
|
|
1103
|
+
a field called ``candidate_digest`` would satisfy the digest-on-every-response gate by
|
|
1104
|
+
breaking the plain-vocabulary one. The rename happens here, in one function.
|
|
1105
|
+
``pipeline.VerifiedCandidate`` and :class:`SealedVersion` keep their own names, because the
|
|
1106
|
+
engineering vocabulary is correct where the engineering lives.
|
|
1107
|
+
|
|
1108
|
+
``table_digest`` is renamed from ``table_sha256`` for a WEAKER reason, and the weaker
|
|
1109
|
+
reason is stated as weaker: "sha256" is not superseded by anything, and the rename exists so
|
|
1110
|
+
the pair does not read as one digest and one algorithm. The cost is real — ``mr-data build``
|
|
1111
|
+
emits ``table_sha256`` in its own JSON — and it is paid off by publishing the table above
|
|
1112
|
+
in ``docs/SERVING.md``, so a caller cross-referencing the two can see the correspondence
|
|
1113
|
+
rather than guess it. Without that published mapping a later reader "fixes" the
|
|
1114
|
+
inconsistency in one direction or the other, and one of the two gates rots.
|
|
1115
|
+
|
|
1116
|
+
**What is deliberately absent.** A response carries no served-at time, no request
|
|
1117
|
+
identifier, no key id, no host, no port and no filesystem path. Each of the first three
|
|
1118
|
+
would, alone, make a pinned read impossible to keep byte-identical — the gate says the same
|
|
1119
|
+
pinned request encodes to the same bytes forever, and one clock in the payload ends that in
|
|
1120
|
+
one line. The last three would leak the deployment's internals to a caller who asked about
|
|
1121
|
+
data. This is said here, in the module, and not only in a test, because a later reader will
|
|
1122
|
+
otherwise add a served-at timestamp as an obvious improvement.
|
|
1123
|
+
|
|
1124
|
+
Every number this envelope carries is a whole number or text.
|
|
1125
|
+
:func:`canonical.canonical_json_bytes` raises on any non-integer number, so a float
|
|
1126
|
+
anywhere in a payload would not be a rendering wart — it would be a response that cannot be
|
|
1127
|
+
encoded at all.
|
|
1128
|
+
"""
|
|
1129
|
+
|
|
1130
|
+
result = {
|
|
1131
|
+
"schema_version": SERVING_RESULT_SCHEMA,
|
|
1132
|
+
"status": self.status,
|
|
1133
|
+
"ok": self.ok,
|
|
1134
|
+
"refusals": list(self.refusals),
|
|
1135
|
+
"dataset_id": self.dataset_id,
|
|
1136
|
+
"pin": self.pin,
|
|
1137
|
+
"pin_kind": self.pin_kind,
|
|
1138
|
+
"version_id": self.version_id,
|
|
1139
|
+
"version_number": self.version_number,
|
|
1140
|
+
"version_digest": self.version_digest,
|
|
1141
|
+
"table_digest": self.table_digest,
|
|
1142
|
+
"payload": None if self.payload is None else dict(self.payload),
|
|
1143
|
+
}
|
|
1144
|
+
if self.table_id is not None:
|
|
1145
|
+
result["table_id"] = self.table_id
|
|
1146
|
+
return result
|
|
1147
|
+
|
|
1148
|
+
def canonical_bytes(self) -> bytes:
|
|
1149
|
+
"""Return the response body: canonical bytes over :meth:`to_dict`.
|
|
1150
|
+
|
|
1151
|
+
Identical facts are identical bytes, because nothing time-varying, request-varying or
|
|
1152
|
+
caller-varying is in the fact set. This is the function every transport should encode
|
|
1153
|
+
through, so no shape can invent a second serialization that only mostly agrees.
|
|
1154
|
+
"""
|
|
1155
|
+
|
|
1156
|
+
return canonical.canonical_json_bytes(self.to_dict())
|
|
1157
|
+
|
|
1158
|
+
def digest(self) -> str:
|
|
1159
|
+
"""Return the content digest of this response, in ``DeploymentRequest.digest``'s shape."""
|
|
1160
|
+
|
|
1161
|
+
return canonical.canonical_sha256(self.to_dict())
|
|
1162
|
+
|
|
1163
|
+
|
|
1164
|
+
def serving_lines(result: ServingResult) -> list[str]:
|
|
1165
|
+
"""Render one serving result as plain lines, from the same facts the JSON body carries.
|
|
1166
|
+
|
|
1167
|
+
Both renderings read :meth:`ServingResult.to_dict`, so human and JSON output use the same data.
|
|
1168
|
+
|
|
1169
|
+
Only facts the envelope actually carries are rendered, so a shape whose payload does not hold a
|
|
1170
|
+
given fact simply prints one line fewer. No line is composed from anything outside ``to_dict``.
|
|
1171
|
+
"""
|
|
1172
|
+
|
|
1173
|
+
body = result.to_dict()
|
|
1174
|
+
lines = [
|
|
1175
|
+
f"dataset: {body['dataset_id']}",
|
|
1176
|
+
f"version asked for: {body['pin']}",
|
|
1177
|
+
]
|
|
1178
|
+
if body["version_id"] is not None:
|
|
1179
|
+
lines.append(f"version: {body['version_id']}")
|
|
1180
|
+
if body["version_number"] is not None:
|
|
1181
|
+
lines.append(f"version number: {body['version_number']}")
|
|
1182
|
+
if body["pin_kind"] is not None:
|
|
1183
|
+
lines.append(f"pin kind: {body['pin_kind']}")
|
|
1184
|
+
if body["version_digest"] is not None:
|
|
1185
|
+
lines.append(f"version checksum: {body['version_digest']}")
|
|
1186
|
+
if body["table_digest"] is not None:
|
|
1187
|
+
lines.append(f"table checksum: {body['table_digest']}")
|
|
1188
|
+
if not body["ok"]:
|
|
1189
|
+
lines.append("this read was refused")
|
|
1190
|
+
lines.extend(body["refusals"])
|
|
1191
|
+
return lines
|
|
1192
|
+
|
|
1193
|
+
lines.append(f"status: {body['status']}")
|
|
1194
|
+
payload = body["payload"] or {}
|
|
1195
|
+
if "question" in payload:
|
|
1196
|
+
question = payload["question"]
|
|
1197
|
+
if isinstance(question, str) and not is_single_plain_line(question):
|
|
1198
|
+
question = question.translate(_DISPLAY_LINE_BOUNDARY_ESCAPES)
|
|
1199
|
+
lines.append(f"question: {question}")
|
|
1200
|
+
if "columns" in payload:
|
|
1201
|
+
lines.append("columns: " + ", ".join(payload["columns"]))
|
|
1202
|
+
if "column_types" in payload:
|
|
1203
|
+
lines.append(
|
|
1204
|
+
"column types: "
|
|
1205
|
+
+ ", ".join(f"{name}={kind}" for name, kind in sorted(payload["column_types"].items()))
|
|
1206
|
+
)
|
|
1207
|
+
if "grain" in payload:
|
|
1208
|
+
lines.append("grain: " + ", ".join(payload["grain"]))
|
|
1209
|
+
if "row_count" in payload:
|
|
1210
|
+
lines.append(f"rows: {payload['row_count']}")
|
|
1211
|
+
# The rows shape's own counts. Each line is guarded on its key, in the same block, so a shape
|
|
1212
|
+
# whose payload does not hold a fact prints one line fewer rather than needing its own renderer.
|
|
1213
|
+
# The rows THEMSELVES are not rendered here: a table is a display decision, and the command that
|
|
1214
|
+
# owns the human surface owns it.
|
|
1215
|
+
for key, label in (
|
|
1216
|
+
("total_row_count", "rows in this version"),
|
|
1217
|
+
("matched_row_count", "rows matching this filter"),
|
|
1218
|
+
("offset", "offset"),
|
|
1219
|
+
("limit", "limit"),
|
|
1220
|
+
("truncated", "truncated"),
|
|
1221
|
+
):
|
|
1222
|
+
if key in payload:
|
|
1223
|
+
lines.append(f"{label}: {payload[key]}")
|
|
1224
|
+
return lines
|
|
1225
|
+
|
|
1226
|
+
|
|
1227
|
+
def _asked_for(value: Any, limit: int) -> str:
|
|
1228
|
+
"""Echo what the caller asked for, as text, and only when it is one plain line.
|
|
1229
|
+
|
|
1230
|
+
A dataset id or a pin that is not text is itself a refusal, and the refusal has already been
|
|
1231
|
+
raised by the time this runs; the coercion exists so the envelope reporting that refusal can
|
|
1232
|
+
still be built rather than raising a second, less useful error on top of the first.
|
|
1233
|
+
|
|
1234
|
+
The same is true one step further out, and is the reason this is not a bare ``isinstance``: a
|
|
1235
|
+
value that failed :func:`_one_plain_line` is precisely the value that must not be repeated into
|
|
1236
|
+
a response, because the envelope's ``dataset_id`` and ``pin`` fields are rendered by
|
|
1237
|
+
:func:`serving_lines` beside the digests. An unrenderable ask is echoed as nothing; the
|
|
1238
|
+
refusals beside it already say what was wrong with it.
|
|
1239
|
+
"""
|
|
1240
|
+
|
|
1241
|
+
return value if _one_plain_line(value, limit) else ""
|
|
1242
|
+
|
|
1243
|
+
|
|
1244
|
+
def _refused_result(
|
|
1245
|
+
dataset_id: Any,
|
|
1246
|
+
pin: Any,
|
|
1247
|
+
refusal: ServingRefused,
|
|
1248
|
+
entry: IndexedVersion | None,
|
|
1249
|
+
) -> ServingResult:
|
|
1250
|
+
"""Turn one raised refusal into the one shape every caller of this module renders.
|
|
1251
|
+
|
|
1252
|
+
``entry`` is the version the pin bound, or nothing when the refusal happened before a pin could
|
|
1253
|
+
resolve. When a version was bound, the envelope carries its identity and the digest the index
|
|
1254
|
+
states for it — the digest the read was FOR — so a caller can tell which version was declined.
|
|
1255
|
+
``table_digest`` stays null: the table's digest is a sealed fact that only opening the version
|
|
1256
|
+
produces, and this read did not open one. Nothing here is a claim that anything verified; ``ok``
|
|
1257
|
+
is false and ``refusals`` says why.
|
|
1258
|
+
"""
|
|
1259
|
+
|
|
1260
|
+
return ServingResult(
|
|
1261
|
+
ok=False,
|
|
1262
|
+
refusals=refusal.reasons,
|
|
1263
|
+
status=STATUS_REFUSED,
|
|
1264
|
+
dataset_id=_asked_for(dataset_id, MAX_DATASET_ID_CHARS),
|
|
1265
|
+
pin=_asked_for(pin, MAX_PIN_CHARS),
|
|
1266
|
+
pin_kind=None if entry is None else (PIN_CURRENT if pin == PIN_CURRENT else "pinned"),
|
|
1267
|
+
version_id=None if entry is None else entry.version_id,
|
|
1268
|
+
version_number=None if entry is None else entry.version_number,
|
|
1269
|
+
version_digest=None if entry is None else entry.version_digest,
|
|
1270
|
+
table_digest=None,
|
|
1271
|
+
payload=None,
|
|
1272
|
+
)
|
|
1273
|
+
|
|
1274
|
+
|
|
1275
|
+
def describe_dataset(index: VersionIndex, dataset_id: str, pin: str) -> ServingResult:
|
|
1276
|
+
"""Answer "what is this dataset" for one version: its sealed card, its schema, its evidence.
|
|
1277
|
+
|
|
1278
|
+
Everything answered here comes from one :func:`open_sealed_version` call and nothing else. If a
|
|
1279
|
+
fact belongs in a response and is not on :class:`SealedVersion`, it goes on
|
|
1280
|
+
:class:`SealedVersion` — not into a second read of the run directory from here. A second read
|
|
1281
|
+
is a second observation, and the single opening is what makes the digest this response carries
|
|
1282
|
+
the digest these bytes came from.
|
|
1283
|
+
|
|
1284
|
+
**The table card is served verbatim, and that is a decision rather than an oversight.** The
|
|
1285
|
+
card's bytes are digest-bound evidence: ``pipeline._table_card`` writes them, the manifest
|
|
1286
|
+
covers them, and the version digest is over the whole tree that includes them. Paraphrasing
|
|
1287
|
+
them here would create a second wording of a sealed fact, and rewriting them at the source
|
|
1288
|
+
would change every version digest and break every golden. So the bytes go out decoded as UTF-8
|
|
1289
|
+
and otherwise untouched.
|
|
1290
|
+
|
|
1291
|
+
The card is returned verbatim because it is part of the version digest. The response field
|
|
1292
|
+
remains ``table_card``.
|
|
1293
|
+
|
|
1294
|
+
A refusal is an expected outcome of a data read — an unreviewed build, a corrupted index, a pin
|
|
1295
|
+
naming nothing — so :class:`ServingRefused` is caught here and returned as a refusing
|
|
1296
|
+
:class:`ServingResult`. It is never allowed to propagate, for the reason ``deploy.plan_go_live``
|
|
1297
|
+
established: every caller of this surface should have exactly one shape to render.
|
|
1298
|
+
"""
|
|
1299
|
+
|
|
1300
|
+
entry: IndexedVersion | None = None
|
|
1301
|
+
try:
|
|
1302
|
+
# Resolved here as well as inside the door, so a refusal can say which version was
|
|
1303
|
+
# declined. This is not a second read of a pointer: ``index`` is an already-parsed
|
|
1304
|
+
# immutable value, so both resolutions read the same document and cannot disagree.
|
|
1305
|
+
entry = resolve_pin(index, dataset_id, pin)
|
|
1306
|
+
version = open_sealed_version(index, dataset_id, pin)
|
|
1307
|
+
except ServingRefused as refusal:
|
|
1308
|
+
return _refused_result(dataset_id, pin, refusal, entry)
|
|
1309
|
+
|
|
1310
|
+
# Plain names, all of them. P-01's ban on a superseded word covers the payload as well as the
|
|
1311
|
+
# envelope, because a caller parses both out of one body.
|
|
1312
|
+
payload: dict[str, Any] = {
|
|
1313
|
+
"table_card": version.table_card.decode("utf-8"),
|
|
1314
|
+
"columns": list(version.columns),
|
|
1315
|
+
"column_types": dict(version.column_types),
|
|
1316
|
+
"grain": list(version.grain),
|
|
1317
|
+
"row_count": version.row_count,
|
|
1318
|
+
"question": version.question,
|
|
1319
|
+
"sources": list(version.sources),
|
|
1320
|
+
"join": dict(version.join),
|
|
1321
|
+
"quality": dict(version.quality),
|
|
1322
|
+
"lineage": dict(version.lineage),
|
|
1323
|
+
}
|
|
1324
|
+
return ServingResult(
|
|
1325
|
+
ok=True,
|
|
1326
|
+
refusals=(),
|
|
1327
|
+
status=STATUS_DATASET_DESCRIBED,
|
|
1328
|
+
dataset_id=version.dataset_id,
|
|
1329
|
+
pin=pin,
|
|
1330
|
+
pin_kind=version.pin_kind,
|
|
1331
|
+
version_id=version.version_id,
|
|
1332
|
+
version_number=version.version_number,
|
|
1333
|
+
version_digest=version.candidate_digest,
|
|
1334
|
+
table_digest=version.table_sha256,
|
|
1335
|
+
payload=payload,
|
|
1336
|
+
)
|
|
1337
|
+
|
|
1338
|
+
|
|
1339
|
+
# ---------------------------------------------------------------------------
|
|
1340
|
+
# The filter: a closed structured predicate, never an expression
|
|
1341
|
+
# ---------------------------------------------------------------------------
|
|
1342
|
+
|
|
1343
|
+
# Filters are structured data, never executable expressions. The source-level safety test scans
|
|
1344
|
+
# this module for prohibited execution and dynamic-import tokens.
|
|
1345
|
+
#
|
|
1346
|
+
# So a filter is a list of ``{column, op, value}``. The column must be a member of the sealed
|
|
1347
|
+
# schema, the operator comes from the closed frozen set below, and the value is validated against
|
|
1348
|
+
# that column's logical type as the sealed profile records it. Combination is AND-only, and that is
|
|
1349
|
+
# a stated scope boundary rather than an oversight: OR and grouping are a COMBINATION GRAMMAR, and
|
|
1350
|
+
# a grammar is the thing that grows into an expression language. A caller that needs OR issues two
|
|
1351
|
+
# reads.
|
|
1352
|
+
|
|
1353
|
+
#: A filter names at most this many conditions. The number is small on purpose: every condition is
|
|
1354
|
+
#: a full pass over a sealed column, and a legitimate caller narrowing to a grain needs as many
|
|
1355
|
+
#: conditions as the grain has parts, not dozens.
|
|
1356
|
+
MAX_PREDICATES = 16
|
|
1357
|
+
|
|
1358
|
+
#: The longest value list an ``in`` may carry. Sixteen conditions of sixty-four values each is the
|
|
1359
|
+
#: worst case a single request can ask this surface to evaluate, and it is bounded on purpose.
|
|
1360
|
+
MAX_IN_VALUES = 64
|
|
1361
|
+
|
|
1362
|
+
#: The longest text a filter value may be. A sealed cell may legitimately be far longer, so this
|
|
1363
|
+
#: bounds the REQUEST rather than the data: nothing a caller sends needs to be a kilobyte to match
|
|
1364
|
+
#: a key, and an unbounded value is free work asked of a shared surface.
|
|
1365
|
+
MAX_FILTER_VALUE_CHARS = 1_024
|
|
1366
|
+
|
|
1367
|
+
# **The row ceiling, and why it is this number.**
|
|
1368
|
+
#
|
|
1369
|
+
# ``canonical.MAX_CANONICAL_MEMBERS = 100_000`` is the hard ceiling, and it counts VALUE NODES: one
|
|
1370
|
+
# row contributes its own object plus one member per column, so a row costs ``columns + 1``. At
|
|
1371
|
+
# ``pipeline.MAX_COLUMNS = 256`` that is 257 members per row, and the whole payload must also carry
|
|
1372
|
+
# the envelope. 250 rows therefore costs 250 * 257 = 64,250 members at the widest dataset this
|
|
1373
|
+
# product will build — about a third of the ceiling left as headroom, so a wide dataset is bounded
|
|
1374
|
+
# by this constant rather than by an encoder error.
|
|
1375
|
+
#
|
|
1376
|
+
# It is deliberately NOT ``hosted_worker.MAX_TABLE_PREVIEW_ROWS = 200``. That is a PREVIEW bound —
|
|
1377
|
+
# how much of a build a reviewer is shown — and reusing it here would tie a customer's page size to
|
|
1378
|
+
# an internal review affordance, so that moving one would silently move the other. 250 is a page a
|
|
1379
|
+
# production caller can work with: four reads to a thousand rows, with a stable offset.
|
|
1380
|
+
#
|
|
1381
|
+
# ``pipeline.MAX_ROWS = 100_000`` is what a whole dataset may hold, and it exceeds
|
|
1382
|
+
# ``MAX_CANONICAL_MEMBERS`` on its own at any width. That arithmetic is the reason a bounded window
|
|
1383
|
+
# with an explicit truncation flag is the only shape that can work here at all.
|
|
1384
|
+
MAX_SERVED_ROWS = 250
|
|
1385
|
+
|
|
1386
|
+
#: The byte budget one rows response is fitted to, in ``hosted_worker.MAX_TABLE_PREVIEW_BYTES``'s
|
|
1387
|
+
#: shape. A row count alone does not bound bytes — 250 rows of kilobyte strings is megabytes — so
|
|
1388
|
+
#: both bounds are applied and the byte one shrinks the window by whole rows.
|
|
1389
|
+
MAX_SERVED_BYTES = 4 * 1024 * 1024
|
|
1390
|
+
|
|
1391
|
+
#: The status word an answered rows read carries.
|
|
1392
|
+
STATUS_ROWS_SERVED = "rows_served"
|
|
1393
|
+
|
|
1394
|
+
|
|
1395
|
+
def _is_in(column: Any, value: Any) -> Any:
|
|
1396
|
+
return pc.is_in(column, value_set=value)
|
|
1397
|
+
|
|
1398
|
+
|
|
1399
|
+
def _is_null(column: Any, value: Any) -> Any:
|
|
1400
|
+
return pc.is_null(column)
|
|
1401
|
+
|
|
1402
|
+
|
|
1403
|
+
def _is_not_null(column: Any, value: Any) -> Any:
|
|
1404
|
+
return pc.is_valid(column)
|
|
1405
|
+
|
|
1406
|
+
|
|
1407
|
+
#: The CLOSED operator-to-function map: one operator name, one ``pyarrow.compute`` callable, held as
|
|
1408
|
+
#: module-level data. Every operator takes ``(column, value)`` so the apply loop has no branch of
|
|
1409
|
+
#: its own, and there is no path anywhere in this module that composes a query string — a reader can
|
|
1410
|
+
#: see that at a glance from this table rather than having to trust a claim about it.
|
|
1411
|
+
_PREDICATE_FUNCTIONS: dict[str, Callable[[Any, Any], Any]] = {
|
|
1412
|
+
"eq": pc.equal,
|
|
1413
|
+
"ne": pc.not_equal,
|
|
1414
|
+
"lt": pc.less,
|
|
1415
|
+
"le": pc.less_equal,
|
|
1416
|
+
"gt": pc.greater,
|
|
1417
|
+
"ge": pc.greater_equal,
|
|
1418
|
+
"in": _is_in,
|
|
1419
|
+
"is_null": _is_null,
|
|
1420
|
+
"is_not_null": _is_not_null,
|
|
1421
|
+
}
|
|
1422
|
+
|
|
1423
|
+
#: The operator vocabulary, derived from the function map so the two cannot disagree.
|
|
1424
|
+
PREDICATE_OPS = frozenset(_PREDICATE_FUNCTIONS)
|
|
1425
|
+
|
|
1426
|
+
#: The two operators that ask about validity rather than about a value.
|
|
1427
|
+
_NULL_OPS = frozenset({"is_null", "is_not_null"})
|
|
1428
|
+
|
|
1429
|
+
_PREDICATE_KEYS = frozenset({"column", "op"})
|
|
1430
|
+
|
|
1431
|
+
#: The sealed logical types this query surface can compare. The build pipeline can also emit
|
|
1432
|
+
#: boolean and UTC timestamp columns; keeping those outside this narrower capability set preserves
|
|
1433
|
+
#: the existing typed refusal until their serving value syntax is specified independently.
|
|
1434
|
+
_SERVED_LOGICAL_TYPES = frozenset({"string", "int64", "float64", "date"})
|
|
1435
|
+
|
|
1436
|
+
#: The logical types the recipe/build vocabulary admits but this serving surface cannot filter.
|
|
1437
|
+
#: Named rather than folded into "unknown" so the typed refusal remains explicit.
|
|
1438
|
+
_UNSERVED_LOGICAL_TYPES = frozenset(recipe.LOGICAL_TYPES) - _SERVED_LOGICAL_TYPES
|
|
1439
|
+
|
|
1440
|
+
_WHOLE_NUMBER = re.compile(r"\A-?(0|[1-9][0-9]*)\Z")
|
|
1441
|
+
_DECIMAL_NUMBER = re.compile(r"\A-?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][-+]?[0-9]+)?\Z")
|
|
1442
|
+
_CALENDAR_DATE = re.compile(r"\A\d{4}-\d{2}-\d{2}\Z")
|
|
1443
|
+
_MACHINE_CODE = re.compile(r"\A[A-Z][A-Z0-9_]{0,63}\Z")
|
|
1444
|
+
|
|
1445
|
+
|
|
1446
|
+
def _parse_string(raw: Any) -> str:
|
|
1447
|
+
if not isinstance(raw, str) or len(raw) > MAX_FILTER_VALUE_CHARS:
|
|
1448
|
+
raise ValueError
|
|
1449
|
+
return raw
|
|
1450
|
+
|
|
1451
|
+
|
|
1452
|
+
def _parse_int64(raw: Any) -> int:
|
|
1453
|
+
if type(raw) is bool:
|
|
1454
|
+
raise ValueError
|
|
1455
|
+
if type(raw) is int:
|
|
1456
|
+
value = raw
|
|
1457
|
+
elif isinstance(raw, str) and _WHOLE_NUMBER.fullmatch(raw) is not None:
|
|
1458
|
+
value = int(raw)
|
|
1459
|
+
else:
|
|
1460
|
+
raise ValueError
|
|
1461
|
+
if not -canonical.MAX_CANONICAL_INTEGER - 1 <= value <= canonical.MAX_CANONICAL_INTEGER:
|
|
1462
|
+
raise ValueError
|
|
1463
|
+
return value
|
|
1464
|
+
|
|
1465
|
+
|
|
1466
|
+
def _parse_float64(raw: Any) -> float:
|
|
1467
|
+
# Text, and only text. A float64 cell leaves this surface as text — ``rowset.render_cell``
|
|
1468
|
+
# renders it with ``.17g`` because the canonical encoder refuses every non-integer number — so a
|
|
1469
|
+
# filter value spelled as a JSON number would be written in one domain and compared against
|
|
1470
|
+
# another. One spelling in both directions is the whole point of the shared value domain.
|
|
1471
|
+
if not isinstance(raw, str) or _DECIMAL_NUMBER.fullmatch(raw) is None:
|
|
1472
|
+
raise ValueError
|
|
1473
|
+
value = float(raw)
|
|
1474
|
+
if value != value or value in (float("inf"), float("-inf")): # pragma: no cover - regex-barred
|
|
1475
|
+
raise ValueError
|
|
1476
|
+
return value
|
|
1477
|
+
|
|
1478
|
+
|
|
1479
|
+
def _parse_date(raw: Any) -> datetime.date:
|
|
1480
|
+
if not isinstance(raw, str) or _CALENDAR_DATE.fullmatch(raw) is None:
|
|
1481
|
+
raise ValueError
|
|
1482
|
+
return datetime.date.fromisoformat(raw)
|
|
1483
|
+
|
|
1484
|
+
|
|
1485
|
+
#: One parser per served logical type. Keyed by exactly ``_SERVED_LOGICAL_TYPES``, asserted by test.
|
|
1486
|
+
_VALUE_PARSERS: dict[str, Callable[[Any], Any]] = {
|
|
1487
|
+
"string": _parse_string,
|
|
1488
|
+
"int64": _parse_int64,
|
|
1489
|
+
"float64": _parse_float64,
|
|
1490
|
+
"date": _parse_date,
|
|
1491
|
+
}
|
|
1492
|
+
|
|
1493
|
+
#: What a value has to look like, per logical type, said in the words a caller would use. Held as
|
|
1494
|
+
#: data beside the parsers so a refusal cannot describe one shape while the parser accepts another.
|
|
1495
|
+
_VALUE_SHAPES: dict[str, str] = {
|
|
1496
|
+
"string": "text",
|
|
1497
|
+
"int64": "a whole number, written as text or as a number",
|
|
1498
|
+
"float64": "a number written as text",
|
|
1499
|
+
"date": "a calendar date written as text, as 2026-07-15",
|
|
1500
|
+
}
|
|
1501
|
+
|
|
1502
|
+
|
|
1503
|
+
def _arrived(value: Any) -> str:
|
|
1504
|
+
"""Say what KIND of thing arrived, never what it was.
|
|
1505
|
+
|
|
1506
|
+
A refusal sentence is a terminal line. Interpolating a caller-supplied value into one would put
|
|
1507
|
+
caller-controlled text — control characters included — in front of whoever is reading the log,
|
|
1508
|
+
so every refusal below names the column and the expected shape and describes the arrival by its
|
|
1509
|
+
kind alone.
|
|
1510
|
+
"""
|
|
1511
|
+
|
|
1512
|
+
if value is None:
|
|
1513
|
+
return "nothing"
|
|
1514
|
+
if type(value) is bool:
|
|
1515
|
+
return "a true or false"
|
|
1516
|
+
if type(value) is int:
|
|
1517
|
+
return "a whole number"
|
|
1518
|
+
if type(value) is float:
|
|
1519
|
+
return "a number"
|
|
1520
|
+
if isinstance(value, str):
|
|
1521
|
+
return "text"
|
|
1522
|
+
if isinstance(value, (list, tuple)):
|
|
1523
|
+
return "a list"
|
|
1524
|
+
if isinstance(value, Mapping):
|
|
1525
|
+
return "an object"
|
|
1526
|
+
return "something this surface does not accept"
|
|
1527
|
+
|
|
1528
|
+
|
|
1529
|
+
def _safe_column(value: Any) -> str | None:
|
|
1530
|
+
"""Return the column name only when it is safe to print, and nothing when it is not.
|
|
1531
|
+
|
|
1532
|
+
A caller-supplied column name is echoed back so a refusal is actionable, and it is echoed ONLY
|
|
1533
|
+
after it matches ``rowset.COLUMN_NAME`` — the frozen shape a sealed dataset's columns may take.
|
|
1534
|
+
A name outside that shape cannot be a sealed column anyway, so nothing is lost by refusing
|
|
1535
|
+
without repeating it, and a name carrying control characters never reaches a terminal.
|
|
1536
|
+
"""
|
|
1537
|
+
|
|
1538
|
+
if isinstance(value, str) and rowset.COLUMN_NAME.fullmatch(value) is not None:
|
|
1539
|
+
return value
|
|
1540
|
+
return None
|
|
1541
|
+
|
|
1542
|
+
|
|
1543
|
+
@dataclass(frozen=True)
|
|
1544
|
+
class Predicate:
|
|
1545
|
+
"""One parsed, typed condition: a sealed column, a closed operator, and a domain value.
|
|
1546
|
+
|
|
1547
|
+
A value of this type exists only on the far side of :func:`parse_predicates`, so anything
|
|
1548
|
+
downstream is holding a column that is in the sealed schema, an operator that is in
|
|
1549
|
+
:data:`PREDICATE_OPS`, and a value already parsed into the column's own logical type. There is
|
|
1550
|
+
no string here to interpret and nothing left to validate.
|
|
1551
|
+
|
|
1552
|
+
``value`` is ``None`` for ``is_null`` and ``is_not_null``, a tuple for ``in``, and one parsed
|
|
1553
|
+
value otherwise.
|
|
1554
|
+
"""
|
|
1555
|
+
|
|
1556
|
+
column: str
|
|
1557
|
+
op: str
|
|
1558
|
+
value: Any
|
|
1559
|
+
|
|
1560
|
+
|
|
1561
|
+
def _typed(raw: Any, *, column: str, logical: str) -> Any:
|
|
1562
|
+
try:
|
|
1563
|
+
return _VALUE_PARSERS[logical](raw)
|
|
1564
|
+
except (ValueError, TypeError) as error:
|
|
1565
|
+
raise ServingRefused(
|
|
1566
|
+
f"the filter on column {column} needs {_VALUE_SHAPES[logical]}, because that column "
|
|
1567
|
+
f"holds {logical}, and {_arrived(raw)} arrived instead [SERVING_FILTER_VALUE]"
|
|
1568
|
+
) from error
|
|
1569
|
+
|
|
1570
|
+
|
|
1571
|
+
def _predicate(
|
|
1572
|
+
raw: Any,
|
|
1573
|
+
*,
|
|
1574
|
+
column_types: Mapping[str, str],
|
|
1575
|
+
seen: set[tuple[str, str]],
|
|
1576
|
+
) -> Predicate:
|
|
1577
|
+
if not isinstance(raw, Mapping):
|
|
1578
|
+
raise ServingRefused(
|
|
1579
|
+
"a filter is a list of conditions, each written as an object naming a column, an "
|
|
1580
|
+
"operator and a value [SERVING_FILTER_SHAPE]"
|
|
1581
|
+
)
|
|
1582
|
+
op = raw.get("op")
|
|
1583
|
+
if not isinstance(op, str) or op not in PREDICATE_OPS:
|
|
1584
|
+
raise ServingRefused(
|
|
1585
|
+
"a filter condition uses one of these operators: "
|
|
1586
|
+
+ ", ".join(sorted(PREDICATE_OPS))
|
|
1587
|
+
+ " [SERVING_FILTER_OP]"
|
|
1588
|
+
)
|
|
1589
|
+
column = _safe_column(raw.get("column"))
|
|
1590
|
+
if column is None:
|
|
1591
|
+
raise ServingRefused(
|
|
1592
|
+
"a filter condition names its column as a name of up to 128 letters, digits, dots, "
|
|
1593
|
+
"dashes, colons or underscores [SERVING_FILTER_COLUMN]"
|
|
1594
|
+
)
|
|
1595
|
+
if column not in column_types:
|
|
1596
|
+
raise ServingRefused(
|
|
1597
|
+
f"this dataset has no column called {column}, so it cannot be filtered on "
|
|
1598
|
+
"[SERVING_FILTER_COLUMN]"
|
|
1599
|
+
)
|
|
1600
|
+
|
|
1601
|
+
unknown = sorted(set(raw) - _PREDICATE_KEYS - {"value"})
|
|
1602
|
+
if unknown:
|
|
1603
|
+
raise ServingRefused(
|
|
1604
|
+
f"the filter condition on column {column} names fields this surface does not know: "
|
|
1605
|
+
+ ", ".join(
|
|
1606
|
+
sorted(_safe_column(item) or "one that cannot be printed" for item in unknown)
|
|
1607
|
+
)
|
|
1608
|
+
+ " [SERVING_FILTER_SHAPE]"
|
|
1609
|
+
)
|
|
1610
|
+
if op in _NULL_OPS:
|
|
1611
|
+
if "value" in raw:
|
|
1612
|
+
raise ServingRefused(
|
|
1613
|
+
f"the {op} filter on column {column} asks whether a cell is empty, so it takes no "
|
|
1614
|
+
"value to compare against [SERVING_FILTER_VALUE]"
|
|
1615
|
+
)
|
|
1616
|
+
elif "value" not in raw:
|
|
1617
|
+
raise ServingRefused(
|
|
1618
|
+
f"the {op} filter on column {column} needs a value to compare against "
|
|
1619
|
+
"[SERVING_FILTER_VALUE]"
|
|
1620
|
+
)
|
|
1621
|
+
|
|
1622
|
+
logical = column_types[column]
|
|
1623
|
+
if logical not in _VALUE_PARSERS:
|
|
1624
|
+
if logical in _UNSERVED_LOGICAL_TYPES:
|
|
1625
|
+
raise ServingRefused(
|
|
1626
|
+
f"column {column} holds {logical}, which this surface cannot filter on yet "
|
|
1627
|
+
"[SERVING_FILTER_TYPE]"
|
|
1628
|
+
)
|
|
1629
|
+
raise ServingRefused(
|
|
1630
|
+
f"column {column} holds a kind of value this surface cannot filter on "
|
|
1631
|
+
"[SERVING_FILTER_TYPE]"
|
|
1632
|
+
)
|
|
1633
|
+
|
|
1634
|
+
pair = (column, op)
|
|
1635
|
+
if pair in seen:
|
|
1636
|
+
raise ServingRefused(
|
|
1637
|
+
f"the filter names the {op} condition on column {column} more than once, so it does "
|
|
1638
|
+
"not say which one to apply [SERVING_FILTER_DUPLICATE]"
|
|
1639
|
+
)
|
|
1640
|
+
seen.add(pair)
|
|
1641
|
+
|
|
1642
|
+
if op in _NULL_OPS:
|
|
1643
|
+
return Predicate(column=column, op=op, value=None)
|
|
1644
|
+
supplied = raw["value"]
|
|
1645
|
+
if op == "in":
|
|
1646
|
+
if not isinstance(supplied, (list, tuple)) or not supplied:
|
|
1647
|
+
raise ServingRefused(
|
|
1648
|
+
f"the in filter on column {column} needs a non-empty list of values to match "
|
|
1649
|
+
"[SERVING_FILTER_VALUE]"
|
|
1650
|
+
)
|
|
1651
|
+
if len(supplied) > MAX_IN_VALUES:
|
|
1652
|
+
raise ServingRefused(
|
|
1653
|
+
f"the in filter on column {column} lists more than {MAX_IN_VALUES} values "
|
|
1654
|
+
"[SERVING_FILTER_BOUNDS]"
|
|
1655
|
+
)
|
|
1656
|
+
return Predicate(
|
|
1657
|
+
column=column,
|
|
1658
|
+
op=op,
|
|
1659
|
+
value=tuple(_typed(item, column=column, logical=logical) for item in supplied),
|
|
1660
|
+
)
|
|
1661
|
+
return Predicate(column=column, op=op, value=_typed(supplied, column=column, logical=logical))
|
|
1662
|
+
|
|
1663
|
+
|
|
1664
|
+
def parse_predicates(raw: Any, *, column_types: Mapping[str, str]) -> tuple[Predicate, ...]:
|
|
1665
|
+
"""Read a caller's filter as DATA, and refuse everything that is not exactly that.
|
|
1666
|
+
|
|
1667
|
+
``column_types`` is the sealed schema's own logical types, as ``evidence/profile.json`` records
|
|
1668
|
+
them and :attr:`SealedVersion.column_types` carries them. Every column a filter names has to be
|
|
1669
|
+
a member of it, and every value is parsed into that column's logical type before anything
|
|
1670
|
+
touches a byte of the table. Types outside this serving surface's narrower parser set are
|
|
1671
|
+
refused by name rather than let through untyped, even when the sealed build can carry them.
|
|
1672
|
+
|
|
1673
|
+
Nothing here builds, accepts or evaluates an expression string. Combination is AND-only.
|
|
1674
|
+
|
|
1675
|
+
Raises:
|
|
1676
|
+
ServingRefused: when the filter is not a list of objects, when it names too many
|
|
1677
|
+
conditions, when an operator is outside :data:`PREDICATE_OPS`, when a column is not in
|
|
1678
|
+
the sealed schema, when a value is not the column's logical type, when a value is
|
|
1679
|
+
supplied for a validity test or left off a comparison, when an ``in`` list is empty or
|
|
1680
|
+
too long, or when one column-operator pair is named twice.
|
|
1681
|
+
"""
|
|
1682
|
+
|
|
1683
|
+
if raw is None:
|
|
1684
|
+
return ()
|
|
1685
|
+
if isinstance(raw, (str, bytes)) or not isinstance(raw, Sequence):
|
|
1686
|
+
raise ServingRefused(
|
|
1687
|
+
"a filter is a list of conditions, each written as an object naming a column, an "
|
|
1688
|
+
"operator and a value [SERVING_FILTER_SHAPE]"
|
|
1689
|
+
)
|
|
1690
|
+
if len(raw) > MAX_PREDICATES:
|
|
1691
|
+
raise ServingRefused(
|
|
1692
|
+
f"a filter names at most {MAX_PREDICATES} conditions [SERVING_FILTER_BOUNDS]"
|
|
1693
|
+
)
|
|
1694
|
+
seen: set[tuple[str, str]] = set()
|
|
1695
|
+
return tuple(_predicate(item, column_types=column_types, seen=seen) for item in raw)
|
|
1696
|
+
|
|
1697
|
+
|
|
1698
|
+
# ---------------------------------------------------------------------------
|
|
1699
|
+
# The window: bounded, deterministic rows
|
|
1700
|
+
# ---------------------------------------------------------------------------
|
|
1701
|
+
|
|
1702
|
+
#: One plain sentence per ``rowset.RowsetError`` code, held as module DATA so a new code cannot
|
|
1703
|
+
#: arrive without a sentence to answer it with.
|
|
1704
|
+
#:
|
|
1705
|
+
#: Translate internal rowset errors to serving terminology while preserving their codes.
|
|
1706
|
+
_ROWSET_REFUSALS: dict[str, str] = {
|
|
1707
|
+
rowset.ROWS_INVALID: (
|
|
1708
|
+
"the sealed table of {named} could not be read as rows within this surface's limits"
|
|
1709
|
+
),
|
|
1710
|
+
}
|
|
1711
|
+
|
|
1712
|
+
|
|
1713
|
+
def _rowset_refusal(error: rowset.RowsetError, entry: IndexedVersion) -> str:
|
|
1714
|
+
"""Turn one shared-value-domain failure into one plain sentence, in ``_refusal_for``'s shape."""
|
|
1715
|
+
|
|
1716
|
+
sentence = _ROWSET_REFUSALS.get(error.code)
|
|
1717
|
+
code = error.code if _MACHINE_CODE.fullmatch(str(error.code)) else "SERVING_ROWS"
|
|
1718
|
+
if sentence is None:
|
|
1719
|
+
sentence = "the sealed table of {named} could not be read as rows"
|
|
1720
|
+
return f"{sentence.format(named=_named(entry))} [{code}]"
|
|
1721
|
+
|
|
1722
|
+
|
|
1723
|
+
def _window(offset: Any, limit: Any) -> tuple[int, int]:
|
|
1724
|
+
"""Validate the window, and refuse anything that could ask for an unbounded payload."""
|
|
1725
|
+
|
|
1726
|
+
for name, value in (("offset", offset), ("limit", limit)):
|
|
1727
|
+
if type(value) is not int or value < 0:
|
|
1728
|
+
raise ServingRefused(
|
|
1729
|
+
f"a read's {name} is a whole number of at least 0 [SERVING_WINDOW]"
|
|
1730
|
+
)
|
|
1731
|
+
if limit < 1:
|
|
1732
|
+
raise ServingRefused("a read asks for at least one row [SERVING_WINDOW]")
|
|
1733
|
+
if limit > MAX_SERVED_ROWS:
|
|
1734
|
+
raise ServingRefused(
|
|
1735
|
+
f"a read returns at most {MAX_SERVED_ROWS} rows at a time, so ask for that many and "
|
|
1736
|
+
"page with the offset [SERVING_WINDOW]"
|
|
1737
|
+
)
|
|
1738
|
+
return offset, limit
|
|
1739
|
+
|
|
1740
|
+
|
|
1741
|
+
def _arrow_value(column: Any, predicate: Predicate) -> Any:
|
|
1742
|
+
"""Turn one typed predicate value into an Arrow value of the column's own type.
|
|
1743
|
+
|
|
1744
|
+
Built against the SEALED column's Arrow type rather than inferred, so a value that cannot be
|
|
1745
|
+
represented in that column's domain is a refusal rather than a silent widening.
|
|
1746
|
+
"""
|
|
1747
|
+
|
|
1748
|
+
if predicate.op in _NULL_OPS:
|
|
1749
|
+
return None
|
|
1750
|
+
try:
|
|
1751
|
+
if predicate.op == "in":
|
|
1752
|
+
return pa.array(list(predicate.value), type=column.type)
|
|
1753
|
+
return pa.scalar(predicate.value, type=column.type)
|
|
1754
|
+
except (pa.ArrowInvalid, pa.ArrowTypeError, OverflowError, ValueError) as error:
|
|
1755
|
+
raise ServingRefused(
|
|
1756
|
+
f"the filter on column {predicate.column} asks for a value that column cannot hold "
|
|
1757
|
+
"[SERVING_FILTER_VALUE]"
|
|
1758
|
+
) from error
|
|
1759
|
+
|
|
1760
|
+
|
|
1761
|
+
def _select_rows(table: pa.Table, predicates: Sequence[Predicate]) -> pa.Table:
|
|
1762
|
+
"""Apply every predicate to the sealed table and return the rows that match all of them.
|
|
1763
|
+
|
|
1764
|
+
Selection happens HERE, on Arrow, before anything is rendered: a filtered read must not
|
|
1765
|
+
materialize the whole dataset in the response value domain to then throw most of it away.
|
|
1766
|
+
|
|
1767
|
+
The operator reaches its ``pyarrow.compute`` function through :data:`_PREDICATE_FUNCTIONS`, a
|
|
1768
|
+
closed table of callables. There is no branch here that composes a string, and there is nothing
|
|
1769
|
+
for one to be composed from — the predicate is already typed data.
|
|
1770
|
+
|
|
1771
|
+
A cell that is null compares as null, so a null never matches ``eq`` and is dropped by the
|
|
1772
|
+
filter. ``is_null`` and ``is_not_null`` ask about VALIDITY instead, which is why they exist:
|
|
1773
|
+
there is no sentinel value that means "empty" in this domain.
|
|
1774
|
+
"""
|
|
1775
|
+
|
|
1776
|
+
if not predicates:
|
|
1777
|
+
return table
|
|
1778
|
+
mask = None
|
|
1779
|
+
for predicate in predicates:
|
|
1780
|
+
column = table.column(predicate.column)
|
|
1781
|
+
function = _PREDICATE_FUNCTIONS[predicate.op]
|
|
1782
|
+
one = function(column, _arrow_value(column, predicate))
|
|
1783
|
+
mask = one if mask is None else pc.and_(mask, one)
|
|
1784
|
+
return table.filter(mask)
|
|
1785
|
+
|
|
1786
|
+
|
|
1787
|
+
def serve_rows(
|
|
1788
|
+
index: VersionIndex,
|
|
1789
|
+
dataset_id: str,
|
|
1790
|
+
pin: str,
|
|
1791
|
+
*,
|
|
1792
|
+
predicates: Any = None,
|
|
1793
|
+
offset: int = 0,
|
|
1794
|
+
limit: int = MAX_SERVED_ROWS,
|
|
1795
|
+
max_bytes: int | None = None,
|
|
1796
|
+
) -> ServingResult:
|
|
1797
|
+
"""Answer "give me these rows" for one version: a bounded, deterministic, filtered window.
|
|
1798
|
+
|
|
1799
|
+
A point lookup is not a second shape. It is this function with an equality condition on every
|
|
1800
|
+
column of the sealed grain, and the answer is at most one row. An absent key returns an empty
|
|
1801
|
+
row list with ``ok`` true — an empty answer is an answer, not a refusal.
|
|
1802
|
+
|
|
1803
|
+
The order is the contract:
|
|
1804
|
+
|
|
1805
|
+
1. Open the sealed version through :func:`open_sealed_version` and retain
|
|
1806
|
+
its verified Parquet descriptor for bounded batch reads. The run directory is never opened
|
|
1807
|
+
again from here, because a second observation could read different bytes.
|
|
1808
|
+
2. **Parse the filter as data** against that version's own sealed schema.
|
|
1809
|
+
3. **Select on Arrow**, through the closed operator table, before anything is rendered.
|
|
1810
|
+
4. **Take the window** with ``offset`` and ``limit`` on the FILTERED order, so paging is stable.
|
|
1811
|
+
5. **Render and fit** through the one shared value domain and the one byte-budget loop.
|
|
1812
|
+
|
|
1813
|
+
**Sealed file order is the ordering, and there is no ORDER BY.** The reason there is no ORDER BY
|
|
1814
|
+
is that the sealed order already is one: the bytes are sealed and digest-bound, so row 7 of a
|
|
1815
|
+
version is row 7 of that version forever, and two adjacent windows concatenate to the wider one.
|
|
1816
|
+
A sort clause would be a second ordering that has to agree with the first.
|
|
1817
|
+
|
|
1818
|
+
``max_bytes`` may only TIGHTEN the byte budget: it is clamped to :data:`MAX_SERVED_BYTES`, so no
|
|
1819
|
+
caller can use it to ask for a larger payload than this surface's own bound. It exists so the
|
|
1820
|
+
fit loop can be exercised on a small dataset without faking the condition by reaching into the
|
|
1821
|
+
module.
|
|
1822
|
+
|
|
1823
|
+
Returns:
|
|
1824
|
+
A :class:`ServingResult` — answering with ``status`` :data:`STATUS_ROWS_SERVED`, or refusing
|
|
1825
|
+
with every reason named. A refusal is an expected outcome of a data read and is never raised
|
|
1826
|
+
out of here, for the reason ``deploy.plan_go_live`` established.
|
|
1827
|
+
"""
|
|
1828
|
+
|
|
1829
|
+
entry: IndexedVersion | None = None
|
|
1830
|
+
try:
|
|
1831
|
+
entry = resolve_pin(index, dataset_id, pin)
|
|
1832
|
+
window_offset, window_limit = _window(offset, limit)
|
|
1833
|
+
budget = _byte_budget(max_bytes)
|
|
1834
|
+
with _open_sealed_version_snapshot(index, entry, pin) as (version, snapshot):
|
|
1835
|
+
filters = parse_predicates(predicates, column_types=dict(version.column_types))
|
|
1836
|
+
return _rows_result(
|
|
1837
|
+
version,
|
|
1838
|
+
entry,
|
|
1839
|
+
parquet=snapshot.parquet_file(),
|
|
1840
|
+
pin=pin,
|
|
1841
|
+
filters=filters,
|
|
1842
|
+
offset=window_offset,
|
|
1843
|
+
limit=window_limit,
|
|
1844
|
+
budget=budget,
|
|
1845
|
+
)
|
|
1846
|
+
except ServingRefused as refusal:
|
|
1847
|
+
return _refused_result(dataset_id, pin, refusal, entry)
|
|
1848
|
+
|
|
1849
|
+
|
|
1850
|
+
def _byte_budget(max_bytes: int | None) -> int:
|
|
1851
|
+
if max_bytes is None:
|
|
1852
|
+
return MAX_SERVED_BYTES
|
|
1853
|
+
if type(max_bytes) is not int or max_bytes < 1:
|
|
1854
|
+
raise ServingRefused(
|
|
1855
|
+
"a read's byte budget is a whole number of at least 1 [SERVING_WINDOW]"
|
|
1856
|
+
)
|
|
1857
|
+
# Clamped, never taken as given: this parameter may tighten the surface's bound and may never
|
|
1858
|
+
# loosen it, so it is not a way to ask for more than the surface will serve.
|
|
1859
|
+
return min(max_bytes, MAX_SERVED_BYTES)
|
|
1860
|
+
|
|
1861
|
+
|
|
1862
|
+
def _rows_result(
|
|
1863
|
+
version: SealedVersion,
|
|
1864
|
+
entry: IndexedVersion,
|
|
1865
|
+
*,
|
|
1866
|
+
parquet: Any,
|
|
1867
|
+
pin: str,
|
|
1868
|
+
filters: tuple[Predicate, ...],
|
|
1869
|
+
offset: int,
|
|
1870
|
+
limit: int,
|
|
1871
|
+
budget: int,
|
|
1872
|
+
) -> ServingResult:
|
|
1873
|
+
"""Read, filter, window, render and fit — and hand back the one envelope."""
|
|
1874
|
+
|
|
1875
|
+
try:
|
|
1876
|
+
columns, total_row_count = rowset.inspect_sealed_parquet(
|
|
1877
|
+
parquet,
|
|
1878
|
+
max_columns=pipeline.MAX_COLUMNS,
|
|
1879
|
+
max_rows=pipeline.MAX_ROWS,
|
|
1880
|
+
max_uncompressed_bytes=pipeline.MAX_CANDIDATE_BYTES,
|
|
1881
|
+
)
|
|
1882
|
+
if tuple(columns) != version.columns:
|
|
1883
|
+
raise ServingRefused(
|
|
1884
|
+
f"{_named(entry)} does not describe its own shape, so it was not served "
|
|
1885
|
+
"[SERVING_VERSION_SHAPE]"
|
|
1886
|
+
)
|
|
1887
|
+
matched_row_count, rendered = rowset.render_filtered_window(
|
|
1888
|
+
parquet,
|
|
1889
|
+
columns,
|
|
1890
|
+
select=lambda batch: _select_rows(batch, filters),
|
|
1891
|
+
offset=offset,
|
|
1892
|
+
limit=limit,
|
|
1893
|
+
exact_match_count=total_row_count if not filters else None,
|
|
1894
|
+
)
|
|
1895
|
+
except rowset.RowsetError as error:
|
|
1896
|
+
raise ServingRefused(_rowset_refusal(error, entry)) from error
|
|
1897
|
+
|
|
1898
|
+
def result_for(prefix: int) -> ServingResult:
|
|
1899
|
+
return ServingResult(
|
|
1900
|
+
ok=True,
|
|
1901
|
+
refusals=(),
|
|
1902
|
+
status=STATUS_ROWS_SERVED,
|
|
1903
|
+
dataset_id=version.dataset_id,
|
|
1904
|
+
pin=pin,
|
|
1905
|
+
pin_kind=version.pin_kind,
|
|
1906
|
+
version_id=version.version_id,
|
|
1907
|
+
version_number=version.version_number,
|
|
1908
|
+
version_digest=version.candidate_digest,
|
|
1909
|
+
table_digest=version.table_sha256,
|
|
1910
|
+
payload={
|
|
1911
|
+
"columns": list(columns),
|
|
1912
|
+
"rows": rendered[:prefix],
|
|
1913
|
+
"total_row_count": total_row_count,
|
|
1914
|
+
"matched_row_count": matched_row_count,
|
|
1915
|
+
"offset": offset,
|
|
1916
|
+
"limit": limit,
|
|
1917
|
+
# Honest in both directions: true when the byte budget shrank the window, and true
|
|
1918
|
+
# when the window simply did not reach the end of the match set.
|
|
1919
|
+
"truncated": offset + prefix < matched_row_count,
|
|
1920
|
+
},
|
|
1921
|
+
)
|
|
1922
|
+
|
|
1923
|
+
# The fit loop searches over row PREFIXES and hands back the encoding it selected. The prefix
|
|
1924
|
+
# that produced those bytes is recovered rather than re-derived, so the response returned is the
|
|
1925
|
+
# response that was measured — no second encoding that might differ from the one that fit.
|
|
1926
|
+
measured: dict[bytes, int] = {}
|
|
1927
|
+
|
|
1928
|
+
def encode(prefix: int) -> bytes:
|
|
1929
|
+
encoded = result_for(prefix).canonical_bytes()
|
|
1930
|
+
measured[encoded] = prefix
|
|
1931
|
+
return encoded
|
|
1932
|
+
|
|
1933
|
+
try:
|
|
1934
|
+
fitted = rowset.fit_canonical_payload(encode, len(rendered), max_bytes=budget)
|
|
1935
|
+
except rowset.RowsetError as error:
|
|
1936
|
+
raise ServingRefused(_rowset_refusal(error, entry)) from error
|
|
1937
|
+
return result_for(measured[fitted])
|
|
1938
|
+
|
|
1939
|
+
|
|
1940
|
+
__all__ = [
|
|
1941
|
+
"MAX_DATASET_ID_CHARS",
|
|
1942
|
+
"MAX_FILTER_VALUE_CHARS",
|
|
1943
|
+
"MAX_INDEXED_DATASETS",
|
|
1944
|
+
"MAX_INDEXED_VERSIONS_PER_DATASET",
|
|
1945
|
+
"MAX_IN_VALUES",
|
|
1946
|
+
"MAX_PIN_CHARS",
|
|
1947
|
+
"MAX_PREDICATES",
|
|
1948
|
+
"MAX_RUN_DIR_CHARS",
|
|
1949
|
+
"MAX_RUN_DIR_COMPONENT_CHARS",
|
|
1950
|
+
"MAX_SERVED_BYTES",
|
|
1951
|
+
"MAX_SERVED_ROWS",
|
|
1952
|
+
"PIN_CURRENT",
|
|
1953
|
+
"PREDICATE_OPS",
|
|
1954
|
+
"SERVING_RESULT_SCHEMA",
|
|
1955
|
+
"SERVING_SCHEMA_PREFIX",
|
|
1956
|
+
"STATUS_DATASET_DESCRIBED",
|
|
1957
|
+
"STATUS_REFUSED",
|
|
1958
|
+
"STATUS_ROWS_SERVED",
|
|
1959
|
+
"VERSION_INDEX_SCHEMA",
|
|
1960
|
+
"IndexedDataset",
|
|
1961
|
+
"IndexedVersion",
|
|
1962
|
+
"Predicate",
|
|
1963
|
+
"SealedVersion",
|
|
1964
|
+
"ServingRefused",
|
|
1965
|
+
"ServingResult",
|
|
1966
|
+
"VersionIndex",
|
|
1967
|
+
"describe_dataset",
|
|
1968
|
+
"open_sealed_version",
|
|
1969
|
+
"parse_predicates",
|
|
1970
|
+
"parse_version_index",
|
|
1971
|
+
"read_version_index",
|
|
1972
|
+
"resolve_pin",
|
|
1973
|
+
"serve_rows",
|
|
1974
|
+
"serving_lines",
|
|
1975
|
+
]
|