mostlyright-data 0.9.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mostlyright/data_harness/__init__.py +158 -0
- mostlyright/data_harness/acquisition/__init__.py +55 -0
- mostlyright/data_harness/acquisition/http.py +2773 -0
- mostlyright/data_harness/acquisition/parsing.py +809 -0
- mostlyright/data_harness/acquisition/ranges.py +495 -0
- mostlyright/data_harness/acquisition/result_download.py +360 -0
- mostlyright/data_harness/acquisition/retention_admission.py +248 -0
- mostlyright/data_harness/acquisition/sandbox.py +4888 -0
- mostlyright/data_harness/acquisition/url_policy.py +530 -0
- mostlyright/data_harness/agent_runtime.py +2743 -0
- mostlyright/data_harness/assets/logo-ink.svg +31 -0
- mostlyright/data_harness/backends/__init__.py +28 -0
- mostlyright/data_harness/backends/pandas_backend.py +350 -0
- mostlyright/data_harness/backends/polars_backend.py +366 -0
- mostlyright/data_harness/backends/protocol.py +124 -0
- mostlyright/data_harness/backends/reference.py +83 -0
- mostlyright/data_harness/backends/registry.py +55 -0
- mostlyright/data_harness/backends/restrictions.py +126 -0
- mostlyright/data_harness/canonical.py +333 -0
- mostlyright/data_harness/catalog_job.py +625 -0
- mostlyright/data_harness/cli.py +5398 -0
- mostlyright/data_harness/contracts.py +53 -0
- mostlyright/data_harness/coordinator.py +1307 -0
- mostlyright/data_harness/deploy.py +924 -0
- mostlyright/data_harness/deploy_target.py +312 -0
- mostlyright/data_harness/deployment_evidence.py +1067 -0
- mostlyright/data_harness/event_presentation.py +576 -0
- mostlyright/data_harness/events.py +2152 -0
- mostlyright/data_harness/fast_delimited.py +239 -0
- mostlyright/data_harness/fleet.py +237 -0
- mostlyright/data_harness/formats.py +236 -0
- mostlyright/data_harness/governors.py +1163 -0
- mostlyright/data_harness/hosted_bootstrap.py +972 -0
- mostlyright/data_harness/hosted_crawler.py +1115 -0
- mostlyright/data_harness/hosted_crawler_container_smoke.py +351 -0
- mostlyright/data_harness/hosted_crawler_fetch.py +423 -0
- mostlyright/data_harness/hosted_crawler_job.py +1277 -0
- mostlyright/data_harness/hosted_crawler_protocol.py +676 -0
- mostlyright/data_harness/hosted_dataset.py +1500 -0
- mostlyright/data_harness/hosted_deploy.py +3037 -0
- mostlyright/data_harness/hosted_handoff.py +62 -0
- mostlyright/data_harness/hosted_ingestion_contract.py +504 -0
- mostlyright/data_harness/hosted_ingestion_job.py +356 -0
- mostlyright/data_harness/hosted_ingestion_job_smoke.py +40 -0
- mostlyright/data_harness/hosted_session_container_smoke.py +194 -0
- mostlyright/data_harness/hosted_session_worker.py +3554 -0
- mostlyright/data_harness/hosted_session_worker_job_smoke.py +46 -0
- mostlyright/data_harness/hosted_worker.py +6784 -0
- mostlyright/data_harness/ingestion/__init__.py +56 -0
- mostlyright/data_harness/ingestion/contracts.py +461 -0
- mostlyright/data_harness/ingestion/faults.py +42 -0
- mostlyright/data_harness/ingestion/gcs_store.py +1162 -0
- mostlyright/data_harness/ingestion/spool.py +130 -0
- mostlyright/data_harness/ingestion/store.py +885 -0
- mostlyright/data_harness/key_seam.py +434 -0
- mostlyright/data_harness/linux_process_boundary.py +262 -0
- mostlyright/data_harness/local_contracts.py +2880 -0
- mostlyright/data_harness/local_search/__init__.py +5 -0
- mostlyright/data_harness/local_search/build_index.py +1087 -0
- mostlyright/data_harness/local_search/contracts.py +920 -0
- mostlyright/data_harness/local_search/query_trace.py +266 -0
- mostlyright/data_harness/local_search/retrieval.py +700 -0
- mostlyright/data_harness/local_search/sealed.py +474 -0
- mostlyright/data_harness/local_search/service.py +784 -0
- mostlyright/data_harness/nbrender/CONTRACT.md +212 -0
- mostlyright/data_harness/nbrender/__init__.py +12 -0
- mostlyright/data_harness/nbrender/chrome.py +359 -0
- mostlyright/data_harness/nbrender/code_body.py +266 -0
- mostlyright/data_harness/nbrender/document.py +407 -0
- mostlyright/data_harness/nbrender/frame.py +275 -0
- mostlyright/data_harness/nbrender/interactive.py +337 -0
- mostlyright/data_harness/nbrender/markdown_body.py +477 -0
- mostlyright/data_harness/nbrender/mr_components.py +134 -0
- mostlyright/data_harness/nbrender/outputs_data.py +595 -0
- mostlyright/data_harness/nbrender/outputs_rich.py +906 -0
- mostlyright/data_harness/nbrender/outputs_source.py +260 -0
- mostlyright/data_harness/nbrender/outputs_stage.py +176 -0
- mostlyright/data_harness/nbrender/outputs_text.py +400 -0
- mostlyright/data_harness/nbrender/parse.py +394 -0
- mostlyright/data_harness/nbrender/status.py +40 -0
- mostlyright/data_harness/nbrender/tokens.py +1295 -0
- mostlyright/data_harness/notebook.py +1710 -0
- mostlyright/data_harness/offline.py +2049 -0
- mostlyright/data_harness/operation_registry.py +1007 -0
- mostlyright/data_harness/operator_setup.py +239 -0
- mostlyright/data_harness/pipeline.py +6428 -0
- mostlyright/data_harness/plan_graph.py +2026 -0
- mostlyright/data_harness/preparation/__init__.py +104 -0
- mostlyright/data_harness/preparation/contracts.py +1017 -0
- mostlyright/data_harness/preparation/engine.py +221 -0
- mostlyright/data_harness/preparation/errors.py +14 -0
- mostlyright/data_harness/preparation/gates.py +751 -0
- mostlyright/data_harness/preparation/joins.py +574 -0
- mostlyright/data_harness/preparation/profile.py +384 -0
- mostlyright/data_harness/preparation/table.py +217 -0
- mostlyright/data_harness/preparation/transforms.py +568 -0
- mostlyright/data_harness/progress_events.py +534 -0
- mostlyright/data_harness/readers/__init__.py +46 -0
- mostlyright/data_harness/readers/containers.py +963 -0
- mostlyright/data_harness/readers/contracts.py +542 -0
- mostlyright/data_harness/readers/delimited.py +257 -0
- mostlyright/data_harness/readers/grib2/__init__.py +33 -0
- mostlyright/data_harness/readers/grib2/admission.py +722 -0
- mostlyright/data_harness/readers/grib2/decode.py +1009 -0
- mostlyright/data_harness/readers/grib2/geometry.py +1133 -0
- mostlyright/data_harness/readers/grib2/portable_math.py +501 -0
- mostlyright/data_harness/readers/json_tabular.py +485 -0
- mostlyright/data_harness/readers/registry.py +514 -0
- mostlyright/data_harness/readers/samples/README.md +110 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/cities.csv.gz +0 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/cities.csv.gz +0 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/cities.tar +0 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/cities.tar +0 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/cities.zip +0 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/expected.json +25 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/expected.json +25 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/expected.json +25 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/cities.csv +3 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/expected.json +23 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/cities.tsv +5 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/expected.json +25 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/expected.json +30 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/observations.csv +5 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/expected.json +44 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/stations.json +1 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/expected.json +48 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/observations.ndjson +4 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/cities.xlsx +0 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/expected.json +24 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/expected.json +27 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/expected.json +27 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/README.md +20 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/expected.json +55 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/gfs-2m-temperature.grib2 +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/expected.json +54 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/expected.json +54 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/hrrr-categorical-rain.grib2 +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/expected.json +54 -0
- mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
- mostlyright/data_harness/readers/samples.py +582 -0
- mostlyright/data_harness/readers/spreadsheet.py +803 -0
- mostlyright/data_harness/readers/tabular.py +510 -0
- mostlyright/data_harness/recipe.py +5321 -0
- mostlyright/data_harness/repair/__init__.py +78 -0
- mostlyright/data_harness/repair/adapters.py +274 -0
- mostlyright/data_harness/repair/contracts.py +872 -0
- mostlyright/data_harness/repair/coordinator.py +1099 -0
- mostlyright/data_harness/repair/errors.py +16 -0
- mostlyright/data_harness/review.py +2533 -0
- mostlyright/data_harness/rowset.py +283 -0
- mostlyright/data_harness/serving.py +1975 -0
- mostlyright/data_harness/serving_edge.py +590 -0
- mostlyright/data_harness/serving_http.py +1031 -0
- mostlyright/data_harness/session_probes.py +759 -0
- mostlyright/data_harness/signing.py +101 -0
- mostlyright/data_harness/source_discovery.py +898 -0
- mostlyright/data_harness/sources/__init__.py +209 -0
- mostlyright/data_harness/sources/_adapter_steps.py +213 -0
- mostlyright/data_harness/sources/adapters.py +1214 -0
- mostlyright/data_harness/sources/cadence.py +1428 -0
- mostlyright/data_harness/sources/cadence_emission.py +453 -0
- mostlyright/data_harness/sources/cadence_history.py +546 -0
- mostlyright/data_harness/sources/catalog/__init__.py +17 -0
- mostlyright/data_harness/sources/catalog/admission.py +477 -0
- mostlyright/data_harness/sources/catalog/authoring.py +1701 -0
- mostlyright/data_harness/sources/catalog/authoring_policy.py +701 -0
- mostlyright/data_harness/sources/catalog/authoring_shards.py +1217 -0
- mostlyright/data_harness/sources/catalog/bounded_io.py +231 -0
- mostlyright/data_harness/sources/catalog/channel.py +523 -0
- mostlyright/data_harness/sources/catalog/channel_client.py +296 -0
- mostlyright/data_harness/sources/catalog/contracts.py +825 -0
- mostlyright/data_harness/sources/catalog/coverage.py +137 -0
- mostlyright/data_harness/sources/catalog/delta.py +1340 -0
- mostlyright/data_harness/sources/catalog/embedding.py +532 -0
- mostlyright/data_harness/sources/catalog/entry_v2.py +1182 -0
- mostlyright/data_harness/sources/catalog/fill.py +3889 -0
- mostlyright/data_harness/sources/catalog/fill_partitions.py +459 -0
- mostlyright/data_harness/sources/catalog/fill_staging.py +1105 -0
- mostlyright/data_harness/sources/catalog/gating.py +374 -0
- mostlyright/data_harness/sources/catalog/generation_receipt.py +1607 -0
- mostlyright/data_harness/sources/catalog/harvest/__init__.py +7 -0
- mostlyright/data_harness/sources/catalog/harvest/ckan.py +384 -0
- mostlyright/data_harness/sources/catalog/harvest/datagov_v4.py +798 -0
- mostlyright/data_harness/sources/catalog/harvest/protocol.py +964 -0
- mostlyright/data_harness/sources/catalog/harvest/sdmx.py +445 -0
- mostlyright/data_harness/sources/catalog/harvest/stac.py +384 -0
- mostlyright/data_harness/sources/catalog/health.py +447 -0
- mostlyright/data_harness/sources/catalog/hosted_catalog.py +105 -0
- mostlyright/data_harness/sources/catalog/identity_history.py +1549 -0
- mostlyright/data_harness/sources/catalog/neural.py +1618 -0
- mostlyright/data_harness/sources/catalog/packed_catalog.py +2345 -0
- mostlyright/data_harness/sources/catalog/packed_retrieval.py +1517 -0
- mostlyright/data_harness/sources/catalog/packed_writer.py +2802 -0
- mostlyright/data_harness/sources/catalog/query_trace.py +1037 -0
- mostlyright/data_harness/sources/catalog/recommend.py +171 -0
- mostlyright/data_harness/sources/catalog/retrieval.py +230 -0
- mostlyright/data_harness/sources/catalog/retrieval_manifest.py +995 -0
- mostlyright/data_harness/sources/catalog/rights_decisions.py +254 -0
- mostlyright/data_harness/sources/catalog/sealed.py +560 -0
- mostlyright/data_harness/sources/catalog/search.py +230 -0
- mostlyright/data_harness/sources/catalog/streaming_delta.py +1097 -0
- mostlyright/data_harness/sources/catalog/update.py +891 -0
- mostlyright/data_harness/sources/collections.py +815 -0
- mostlyright/data_harness/sources/contracts.py +2223 -0
- mostlyright/data_harness/sources/deletion.py +761 -0
- mostlyright/data_harness/sources/fitness.py +162 -0
- mostlyright/data_harness/sources/governance.py +163 -0
- mostlyright/data_harness/sources/hosted.py +173 -0
- mostlyright/data_harness/sources/integration.py +218 -0
- mostlyright/data_harness/sources/range_reader.py +418 -0
- mostlyright/data_harness/sources/registry.py +514 -0
- mostlyright/data_harness/sources/rights_rule.py +59 -0
- mostlyright/data_harness/sources/source_cadence_vectors.v1.json +1 -0
- mostlyright/data_harness/sources/sports.py +521 -0
- mostlyright/data_harness/sources/stream.py +524 -0
- mostlyright/data_harness/sources/stream_connector.py +418 -0
- mostlyright/data_harness/sources/stream_recorder.py +1404 -0
- mostlyright/data_harness/studio_boundary.py +2019 -0
- mostlyright/data_harness/thin/__init__.py +37 -0
- mostlyright/data_harness/thin/acquire.py +1137 -0
- mostlyright/data_harness/thin/acquire_cancel.py +579 -0
- mostlyright/data_harness/thin/approvals.py +617 -0
- mostlyright/data_harness/thin/commands.py +406 -0
- mostlyright/data_harness/thin/download.py +194 -0
- mostlyright/data_harness/thin/narrative.py +589 -0
- mostlyright/data_harness/thin/parity.py +1070 -0
- mostlyright/data_harness/thin/propose.py +2759 -0
- mostlyright/data_harness/thin/research.py +1663 -0
- mostlyright/data_harness/thin/router.py +924 -0
- mostlyright/data_harness/thin/runs.py +519 -0
- mostlyright/data_harness/thin/session.py +281 -0
- mostlyright/data_harness/thin/stream.py +501 -0
- mostlyright/data_harness/thin/transport.py +187 -0
- mostlyright/data_harness/thin/vocabulary.py +368 -0
- mostlyright/data_harness/thin/workers.py +164 -0
- mostlyright/data_harness/ucum/TABLE-PIN.json +40 -0
- mostlyright/data_harness/ucum/ucum-subset.v1.json +632 -0
- mostlyright/data_harness/unit_flow.py +927 -0
- mostlyright/data_harness/units.py +572 -0
- mostlyright/data_harness/ux/__init__.py +9 -0
- mostlyright/data_harness/ux/approve.py +485 -0
- mostlyright/data_harness/ux/author_yaml.py +597 -0
- mostlyright/data_harness/ux/cloud_auth.py +447 -0
- mostlyright/data_harness/ux/commands/__init__.py +260 -0
- mostlyright/data_harness/ux/commands/approve.py +136 -0
- mostlyright/data_harness/ux/commands/auth.py +744 -0
- mostlyright/data_harness/ux/commands/author.py +79 -0
- mostlyright/data_harness/ux/commands/catalog_author.py +403 -0
- mostlyright/data_harness/ux/commands/catalog_fill.py +523 -0
- mostlyright/data_harness/ux/commands/catalog_harvest.py +545 -0
- mostlyright/data_harness/ux/commands/catalog_publish.py +1838 -0
- mostlyright/data_harness/ux/commands/catalog_search.py +71 -0
- mostlyright/data_harness/ux/commands/catalog_update.py +437 -0
- mostlyright/data_harness/ux/commands/deploy.py +134 -0
- mostlyright/data_harness/ux/commands/deploy_dataset.py +98 -0
- mostlyright/data_harness/ux/commands/deploy_plan.py +105 -0
- mostlyright/data_harness/ux/commands/deploy_status.py +104 -0
- mostlyright/data_harness/ux/commands/diff.py +74 -0
- mostlyright/data_harness/ux/commands/index.py +84 -0
- mostlyright/data_harness/ux/commands/inventory.py +47 -0
- mostlyright/data_harness/ux/commands/list_builds.py +143 -0
- mostlyright/data_harness/ux/commands/login.py +63 -0
- mostlyright/data_harness/ux/commands/peek.py +236 -0
- mostlyright/data_harness/ux/commands/plan_check.py +90 -0
- mostlyright/data_harness/ux/commands/preflight.py +97 -0
- mostlyright/data_harness/ux/commands/record.py +107 -0
- mostlyright/data_harness/ux/commands/review_setup.py +47 -0
- mostlyright/data_harness/ux/commands/search.py +440 -0
- mostlyright/data_harness/ux/commands/show.py +61 -0
- mostlyright/data_harness/ux/commands/whoami.py +37 -0
- mostlyright/data_harness/ux/credential_native.py +551 -0
- mostlyright/data_harness/ux/credential_store.py +1055 -0
- mostlyright/data_harness/ux/credentials.py +631 -0
- mostlyright/data_harness/ux/diffing.py +444 -0
- mostlyright/data_harness/ux/headline.py +671 -0
- mostlyright/data_harness/ux/hosted_acquisition.py +974 -0
- mostlyright/data_harness/ux/hosted_run_status.py +619 -0
- mostlyright/data_harness/ux/inventory.py +427 -0
- mostlyright/data_harness/ux/local_review.py +375 -0
- mostlyright/data_harness/ux/login.py +691 -0
- mostlyright/data_harness/ux/path_kind.py +147 -0
- mostlyright/data_harness/ux/peek.py +1000 -0
- mostlyright/data_harness/ux/plain_file.py +178 -0
- mostlyright/data_harness/ux/plan_check.py +311 -0
- mostlyright/data_harness/ux/preflight.py +918 -0
- mostlyright/data_harness/ux/readers.py +1124 -0
- mostlyright/data_harness/ux/remediation.py +2195 -0
- mostlyright/data_harness/ux/render.py +657 -0
- mostlyright/data_harness/ux/workload.py +1077 -0
- mostlyright/data_harness/viewer.py +3713 -0
- mostlyright/data_harness/visual_run/__init__.py +83 -0
- mostlyright/data_harness/visual_run/authoring.py +235 -0
- mostlyright/data_harness/visual_run/contracts.py +673 -0
- mostlyright/data_harness/visual_run/materialize.py +486 -0
- mostlyright/data_harness/visual_run/observations.py +874 -0
- mostlyright/data_harness/visual_run/query.py +259 -0
- mostlyright/data_harness/visual_run/reducer.py +280 -0
- mostlyright/data_harness/visual_run/sdk.py +892 -0
- mostlyright/data_harness/visual_run/store.py +584 -0
- mostlyright/data_harness/visual_run/transport.py +239 -0
- mostlyright/data_harness/watch.py +2999 -0
- mostlyright_data-0.9.0.dist-info/METADATA +607 -0
- mostlyright_data-0.9.0.dist-info/RECORD +314 -0
- mostlyright_data-0.9.0.dist-info/WHEEL +4 -0
- mostlyright_data-0.9.0.dist-info/entry_points.txt +12 -0
|
@@ -0,0 +1,590 @@
|
|
|
1
|
+
"""The serving edge: the key, then the quota, then the read, and never out of order.
|
|
2
|
+
|
|
3
|
+
This module puts two checks in front of every data read, in the only order that makes either check
|
|
4
|
+
meaningful:
|
|
5
|
+
|
|
6
|
+
1. **The key is verified.** Either it arrived already verified from the Studio API edge, or it is
|
|
7
|
+
verified here against cloud's verify surface through :mod:`key_seam`.
|
|
8
|
+
2. **The quota is reserved.** A slot and its declared cost are charged to
|
|
9
|
+
:class:`governors.CapLedger` before anything is read.
|
|
10
|
+
3. **Only then is anything read.** :func:`serving.describe_dataset` and :func:`serving.serve_rows`
|
|
11
|
+
are entered after both gates have passed and never before.
|
|
12
|
+
|
|
13
|
+
``hosted_bootstrap.launch`` already established this discipline on the build side: the gate sits at
|
|
14
|
+
the first moment the tenant coordinate is known and before anything is allocated for the work, so a
|
|
15
|
+
refused run "creates no descriptor, holds no memory, and above all never reaches execve". The same
|
|
16
|
+
sentence in this surface's words: **a refused READ opens no run directory, reads no Parquet, and
|
|
17
|
+
allocates no buffer.** That is a property of the call order in :func:`serve_read`, and
|
|
18
|
+
``tests/test_serving_edge.py`` proves it by instrumenting the read core and asserting it is never
|
|
19
|
+
entered.
|
|
20
|
+
|
|
21
|
+
**Nothing here is a second implementation of anything.** The key check is :mod:`key_seam`, which is
|
|
22
|
+
a complete fail-closed verifier whose real enforcement point is already recorded in
|
|
23
|
+
:data:`key_seam.CONSUMER_CALL_SITE`; this module adds :data:`SERVING_CONSUMER_CALL_SITE` beside it
|
|
24
|
+
and invents no second auth mechanism. The quota is :class:`governors.CapLedger`, and this surface is
|
|
25
|
+
the caller of its ``"key"`` subject.
|
|
26
|
+
|
|
27
|
+
**This is the only module on the serving path permitted to reach the network, and it reaches exactly
|
|
28
|
+
one place:** cloud's verify surface, over https, through :func:`key_seam.verify_api_key`. The
|
|
29
|
+
carve-out is named here because an unexplained exception in a security ban is how the ban rots.
|
|
30
|
+
``serving.py`` reaches nothing; it reads sealed
|
|
31
|
+
bytes off a local run directory and that is all it can do.
|
|
32
|
+
|
|
33
|
+
**Nothing here interprets a dataset.** The edge decides whether a caller may read; ``serving.py``
|
|
34
|
+
decides what a read answers. No field of a filter, a pin or a schema is examined in this file, and a
|
|
35
|
+
refusal produced here names a credential or a cap and never a column.
|
|
36
|
+
|
|
37
|
+
**Two topologies, both supported, and the module says which is which rather than assuming.**
|
|
38
|
+
|
|
39
|
+
* **PRODUCTION.** The Studio API edge verifies the customer key before this function runs. Without
|
|
40
|
+
a verified key id, the read is refused.
|
|
41
|
+
* **LOCAL proof.** Verification runs in process against a configured https verify URL, through
|
|
42
|
+
:func:`key_seam.verify_api_key`.
|
|
43
|
+
|
|
44
|
+
There is no third mode in which verification is off. A cap a caller can switch off is not a cap
|
|
45
|
+
(``hosted_bootstrap.py:629-631``), and the same is true one gate over. No parameter, no argument and
|
|
46
|
+
no environment variable read by this module turns either check off, and
|
|
47
|
+
``tests/test_serving_edge.py`` asserts that over the signature and over the source rather than
|
|
48
|
+
leaving it to a reader's goodwill.
|
|
49
|
+
|
|
50
|
+
**The verify transport is passed through and never interpreted.** :func:`serve_read` forwards the
|
|
51
|
+
``transport`` parameter to :func:`key_seam.verify_api_key`. The transport supplies response bytes;
|
|
52
|
+
``key_seam`` computes the verdict. This module must not inspect or branch on the transport.
|
|
53
|
+
"""
|
|
54
|
+
|
|
55
|
+
from __future__ import annotations
|
|
56
|
+
|
|
57
|
+
import os
|
|
58
|
+
from dataclasses import replace
|
|
59
|
+
from datetime import datetime
|
|
60
|
+
from functools import wraps
|
|
61
|
+
from pathlib import Path
|
|
62
|
+
from typing import Any
|
|
63
|
+
|
|
64
|
+
from mostlyright.data_harness import key_seam, serving
|
|
65
|
+
from mostlyright.data_harness.governors import (
|
|
66
|
+
KEY_ID_PATTERN,
|
|
67
|
+
CapExceeded,
|
|
68
|
+
CapLedger,
|
|
69
|
+
CapPolicy,
|
|
70
|
+
CapStoreError,
|
|
71
|
+
FileCapStore,
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
#: Where a presented API key is actually refused or allowed for a SERVING read, recorded in code for
|
|
75
|
+
#: the reason :data:`key_seam.CONSUMER_CALL_SITE` is: the check runs somewhere this repository does
|
|
76
|
+
#: not contain, and an unrecorded consumer reads as a missing one.
|
|
77
|
+
#:
|
|
78
|
+
#: In production the customer key is verified at the **Studio API edge**, before a serving request
|
|
79
|
+
#: reaches the harness at all, and :func:`serve_read` receives the resulting key id. The
|
|
80
|
+
#: harness-side implementation in this module is the reference for that check and the local proof of
|
|
81
|
+
#: it — the same posture ``key_seam`` holds on the build side, one surface over.
|
|
82
|
+
SERVING_CONSUMER_CALL_SITE = (
|
|
83
|
+
"mostlyright-studio API edge verifies the customer key before a serving request reaches the "
|
|
84
|
+
"harness; serving_edge.serve_read is the harness-side reference implementation and the local "
|
|
85
|
+
"proof of that check"
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
#: What this deployment declares one READ costs, in cost units (one cent each). See
|
|
89
|
+
#: :func:`declared_read_cost_units`. Deliberately NOT ``hosted_bootstrap.RUN_COST_UNITS_ENV``: a
|
|
90
|
+
#: read and a build run are different work at different prices, and one variable for both would make
|
|
91
|
+
#: pricing a read impossible without repricing every build.
|
|
92
|
+
READ_COST_UNITS_ENV = "MOSTLYRIGHT_READ_COST_UNITS"
|
|
93
|
+
|
|
94
|
+
#: The path of the cap store the SERVING ledger counts in. Deliberately NOT
|
|
95
|
+
#: ``hosted_bootstrap.CAP_STORE_ENV``, and the separation is load-bearing rather than tidy: both
|
|
96
|
+
#: ledgers file 16-lowercase-hex coordinates into structurally identical files, and
|
|
97
|
+
#: :data:`governors.KEY_ID_PATTERN` cannot tell a workspace hash from an API key id. One file
|
|
98
|
+
#: holding both would let a workspace's builds and a key's reads share a counter — a tenant's build
|
|
99
|
+
#: allowance silently spent by somebody else's reads, with nothing anywhere raising.
|
|
100
|
+
SERVING_CAP_STORE_ENV = "MOSTLYRIGHT_SERVING_CAP_STORE"
|
|
101
|
+
|
|
102
|
+
#: The refusal code a credential refusal carries, in the bracketed idiom every ``serving.py``
|
|
103
|
+
#: refusal already uses. A transport maps it to 401; nothing else may.
|
|
104
|
+
CREDENTIAL_REFUSAL_CODE = "SERVING_CREDENTIAL"
|
|
105
|
+
|
|
106
|
+
#: The refusal code a reached cap carries. A transport maps it to 429.
|
|
107
|
+
QUOTA_REFUSAL_CODE = "SERVING_QUOTA"
|
|
108
|
+
|
|
109
|
+
#: The refusal code counters that could not be read carry. It is distinct from
|
|
110
|
+
#: :data:`QUOTA_REFUSAL_CODE` on purpose: "this key has used its allowance" and "this deployment
|
|
111
|
+
#: cannot tell what this key has used" are different facts for an operator, and only the first one
|
|
112
|
+
#: is the product working.
|
|
113
|
+
QUOTA_UNREADABLE_REFUSAL_CODE = "SERVING_QUOTA_UNREADABLE"
|
|
114
|
+
|
|
115
|
+
#: The refusal code a misconfigured declared read cost carries.
|
|
116
|
+
READ_COST_REFUSAL_CODE = "SERVING_READ_COST"
|
|
117
|
+
|
|
118
|
+
#: The refusal code a read asked for in a shape this surface does not answer carries.
|
|
119
|
+
READ_SHAPE_REFUSAL_CODE = "SERVING_READ_SHAPE"
|
|
120
|
+
|
|
121
|
+
#: Every security scheme ``mostlyright-studio/contracts/openapi/studio-v3.yaml`` declares today,
|
|
122
|
+
#: read off that document and recorded here as data so :data:`STUDIO_WIRE_BINDING`'s claim is
|
|
123
|
+
#: checkable rather than rhetorical. Not one of them is an ``x-api-key`` scheme, and the contract
|
|
124
|
+
#: declares no path for a keyed data read. These are FOREIGN identifiers quoted from another
|
|
125
|
+
#: repository's contract, so they are not bound by ``docs/VOCABULARY.md`` the way a sentence this
|
|
126
|
+
#: surface emits is; nothing here reaches a response.
|
|
127
|
+
STUDIO_SECURITY_SCHEMES = (
|
|
128
|
+
"userBearer",
|
|
129
|
+
"automationBearer",
|
|
130
|
+
"interactiveStepUp",
|
|
131
|
+
"producerCapability",
|
|
132
|
+
"verifierCapability",
|
|
133
|
+
"studioController",
|
|
134
|
+
)
|
|
135
|
+
|
|
136
|
+
#: The current Studio contract has no keyed data-read path. Adding one requires a Studio contract
|
|
137
|
+
#: update, generated-client refresh, and rotation of
|
|
138
|
+
#: :data:`studio_boundary.PINNED_OPENAPI_SOURCE_SHA256`.
|
|
139
|
+
STUDIO_WIRE_BINDING = (
|
|
140
|
+
"studio-v3.yaml declares six bearer security schemes and no x-api-key scheme, and no path for "
|
|
141
|
+
"a keyed data read; this surface therefore has no wire binding in the Studio contract yet. "
|
|
142
|
+
"Writing one means a Studio branch, a client regeneration and a rotation of "
|
|
143
|
+
"studio_boundary.PINNED_OPENAPI_SOURCE_SHA256, and it belongs to the staging apply"
|
|
144
|
+
)
|
|
145
|
+
|
|
146
|
+
#: The two shapes a read may take. Closed, and the same two functions ``serving.py`` exports: the
|
|
147
|
+
#: edge routes to one of them and can invent neither a third shape nor a variant of these two.
|
|
148
|
+
READ_SHAPES = ("describe", "rows")
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
class ServingEdgeError(RuntimeError):
|
|
152
|
+
"""The edge is misconfigured, so no read may be decided from it.
|
|
153
|
+
|
|
154
|
+
Raised only for an operator configuration fault — a declared read cost that is not a whole
|
|
155
|
+
number of cost units. It is never a rejected caller: a rejected caller is a refusing
|
|
156
|
+
:class:`serving.ServingResult`, exactly as a rejected key is a denying
|
|
157
|
+
:class:`key_seam.KeyVerdict` rather than an exception.
|
|
158
|
+
|
|
159
|
+
Like everything else on this path, the message never contains a credential.
|
|
160
|
+
"""
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def declared_read_cost_units() -> int:
|
|
164
|
+
"""Return what this deployment says one read costs, or refuse a value nobody wrote.
|
|
165
|
+
|
|
166
|
+
Same character as ``hosted_bootstrap.declared_run_cost_units``, and the same honest gap, one
|
|
167
|
+
work item over: this is a **declared budget, not a measurement**. Nothing on this path can
|
|
168
|
+
observe what a read really costs, so what it can do is charge the monthly spend ledger the
|
|
169
|
+
amount the operator says one read is worth at worst — reserve the worst case, admit only if it
|
|
170
|
+
fits.
|
|
171
|
+
|
|
172
|
+
- **Unset** — every read is charged zero, so the monthly spend ceiling in
|
|
173
|
+
:class:`governors.CapPolicy` cannot trip no matter how many reads are admitted. This is a real
|
|
174
|
+
gap, not a degraded-but-safe default. The monthly read allowance still applies where
|
|
175
|
+
:data:`SERVING_CAP_STORE_ENV` names shared storage (:func:`serving_ledger`). The concurrency
|
|
176
|
+
cap does not apply across one-process-per-request workers.
|
|
177
|
+
- **Set to a whole number of cost units** — each admitted read reserves that much against the
|
|
178
|
+
monthly ceiling before anything is read and commits the same amount on the way out, so the
|
|
179
|
+
ceiling binds at ``max_monthly_cost_units / declared`` reads.
|
|
180
|
+
|
|
181
|
+
A value that cannot be read as a whole number of cost units refuses the read rather than
|
|
182
|
+
defaulting to zero: "this budget is misconfigured" and "this read is free" must never be the
|
|
183
|
+
same answer.
|
|
184
|
+
|
|
185
|
+
Raises:
|
|
186
|
+
ServingEdgeError: when the variable is set to something that is not a whole number.
|
|
187
|
+
"""
|
|
188
|
+
|
|
189
|
+
configured = os.environ.get(READ_COST_UNITS_ENV)
|
|
190
|
+
if configured is None or configured == "":
|
|
191
|
+
return 0
|
|
192
|
+
# str.isdigit is the check rather than int(): it refuses the signs, the underscores, the
|
|
193
|
+
# surrounding whitespace and the unicode digits that int() would otherwise accept and turn
|
|
194
|
+
# into a number nobody wrote.
|
|
195
|
+
if not configured.isascii() or not configured.isdigit():
|
|
196
|
+
raise ServingEdgeError(
|
|
197
|
+
f"{READ_COST_UNITS_ENV} must be a whole number of cost units, zero or more"
|
|
198
|
+
)
|
|
199
|
+
return int(configured)
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def serving_ledger(*, policy: CapPolicy | None = None) -> CapLedger:
|
|
203
|
+
"""Build the ledger the serving gate charges, and say what it actually enforces.
|
|
204
|
+
|
|
205
|
+
The ledger counts by ``subject="key"`` and ``work_unit="read"``. Both nouns are declarations and
|
|
206
|
+
neither can be inferred: a 16-lowercase-hex coordinate looks identical whether a workspace or an
|
|
207
|
+
API key was hashed into it, and a counter looks identical whether it counted builds or reads. A
|
|
208
|
+
customer refused a read who is told they have used their monthly RUN allowance is being sent to
|
|
209
|
+
a different product surface to look for a number that will not be there, which is the failure
|
|
210
|
+
:data:`governors.CAP_SUBJECTS` and :data:`governors.CAP_WORK_UNITS` exist to prevent.
|
|
211
|
+
|
|
212
|
+
**The store is separate from the worker's, and that is not tidiness.** This ledger is backed by
|
|
213
|
+
:data:`SERVING_CAP_STORE_ENV` and the build-side admission ledger by
|
|
214
|
+
``hosted_bootstrap.CAP_STORE_ENV``. Both write structurally identical files keyed by 16-hex
|
|
215
|
+
coordinates, and :data:`governors.KEY_ID_PATTERN` accepts either kind. Pointed at one file they
|
|
216
|
+
would share counters: a workspace's build allowance would be spent by some key's reads, and
|
|
217
|
+
nothing on either side would raise. The two variables must therefore name two paths.
|
|
218
|
+
|
|
219
|
+
**The honest state of the ceiling.** ``governors.MAX_RUNS_PER_MONTH`` bounds work items per
|
|
220
|
+
calendar month per coordinate whatever the work item is called, so what this product enforces
|
|
221
|
+
today is **10,000 reads per key per month**, and for a production query API that is low. Raising
|
|
222
|
+
it is not a wording decision and is not taken here: the same constant bounds the build-side
|
|
223
|
+
allowance, so raising it weakens that cap too. Separate read counters require a
|
|
224
|
+
``governors.CAP_STORE_SCHEMA`` migration.
|
|
225
|
+
|
|
226
|
+
With :data:`SERVING_CAP_STORE_ENV` unset the counters live in this process only, so the monthly
|
|
227
|
+
read allowance is not enforced across processes at all — the same real gap
|
|
228
|
+
``hosted_bootstrap.admission_ledger`` records for the build side.
|
|
229
|
+
|
|
230
|
+
Raises:
|
|
231
|
+
CapStoreError: when the configured store cannot be read. It is deliberately allowed to
|
|
232
|
+
propagate out of construction: a ledger over counters that cannot be read must fail
|
|
233
|
+
where the caller can still refuse the read, rather than succeed with empty counters and
|
|
234
|
+
admit everything.
|
|
235
|
+
"""
|
|
236
|
+
|
|
237
|
+
configured = os.environ.get(SERVING_CAP_STORE_ENV)
|
|
238
|
+
if configured:
|
|
239
|
+
return CapLedger(
|
|
240
|
+
policy,
|
|
241
|
+
store=FileCapStore(Path(configured)),
|
|
242
|
+
subject="key",
|
|
243
|
+
work_unit="read",
|
|
244
|
+
)
|
|
245
|
+
return CapLedger(policy, subject="key", work_unit="read")
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def _refused(dataset_id: Any, pin: Any, reason: str, code: str) -> serving.ServingResult:
|
|
249
|
+
"""Turn one edge refusal into the one envelope every serving caller already renders.
|
|
250
|
+
|
|
251
|
+
Both digest fields are null and that is the honest answer: a refusal taken at the key or at the
|
|
252
|
+
cap never resolved a pin, so there is no version to name and inventing one would be worse than
|
|
253
|
+
saying nothing. A refusal taken INSIDE the read core comes back from ``serving.py`` instead,
|
|
254
|
+
already carrying the version the pin bound.
|
|
255
|
+
|
|
256
|
+
The bracketed code is appended, never substituted into the sentence: a governors' refusal must
|
|
257
|
+
reach a customer in the governors' own words (``deploy.py:390-393`` is the same refusal to write
|
|
258
|
+
a second wording of one fact), and a transport needs a machine handle to map onto a status code.
|
|
259
|
+
"""
|
|
260
|
+
|
|
261
|
+
return _envelope(dataset_id, pin, (f"{reason} [{code}]",))
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def _envelope(dataset_id: Any, pin: Any, refusals: tuple[str, ...]) -> serving.ServingResult:
|
|
265
|
+
"""The one refusing envelope this module builds, whatever declined the read.
|
|
266
|
+
|
|
267
|
+
What the caller asked for is echoed through ``serving._asked_for`` and never straight out of the
|
|
268
|
+
argument: that function holds the plain-line rule and the length bound the library's own
|
|
269
|
+
grammar owns, and a second, weaker coercion here would be the transport bound and the library
|
|
270
|
+
bound disagreeing inside one process.
|
|
271
|
+
"""
|
|
272
|
+
|
|
273
|
+
return serving.ServingResult(
|
|
274
|
+
ok=False,
|
|
275
|
+
refusals=refusals,
|
|
276
|
+
status=serving.STATUS_REFUSED,
|
|
277
|
+
dataset_id=serving._asked_for(dataset_id, serving.MAX_DATASET_ID_CHARS),
|
|
278
|
+
pin=serving._asked_for(pin, serving.MAX_PIN_CHARS),
|
|
279
|
+
pin_kind=None,
|
|
280
|
+
version_id=None,
|
|
281
|
+
version_number=None,
|
|
282
|
+
version_digest=None,
|
|
283
|
+
table_digest=None,
|
|
284
|
+
payload=None,
|
|
285
|
+
)
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def v3_coordinates(
|
|
289
|
+
result: serving.ServingResult, parent_dataset_id: Any, table_id: Any
|
|
290
|
+
) -> serving.ServingResult:
|
|
291
|
+
"""Render a V3 envelope with its requested Dataset parent and Table child.
|
|
292
|
+
|
|
293
|
+
The read core remains historical table-only. This single adapter belongs at the V3 edge so
|
|
294
|
+
every refusal -- including one taken before authentication, during an index refresh, or while
|
|
295
|
+
translating a core exception -- carries the same two requested coordinates.
|
|
296
|
+
"""
|
|
297
|
+
|
|
298
|
+
table = serving._asked_for(table_id, serving.MAX_DATASET_ID_CHARS)
|
|
299
|
+
return replace(
|
|
300
|
+
result,
|
|
301
|
+
dataset_id=serving._asked_for(parent_dataset_id, serving.MAX_DATASET_ID_CHARS),
|
|
302
|
+
table_id=table or None,
|
|
303
|
+
)
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
def _preserve_v3_coordinates(function: Any) -> Any:
|
|
307
|
+
"""Apply V3 Dataset/Table rendering after every return from one read path."""
|
|
308
|
+
|
|
309
|
+
@wraps(function)
|
|
310
|
+
def wrapped(
|
|
311
|
+
index: serving.VersionIndex, dataset_id: str, pin: str, *args: Any, **kwargs: Any
|
|
312
|
+
) -> serving.ServingResult:
|
|
313
|
+
result = function(index, dataset_id, pin, *args, **kwargs)
|
|
314
|
+
if kwargs.get("api_version") == "v3":
|
|
315
|
+
return v3_coordinates(result, kwargs.get("parent_dataset_id"), dataset_id)
|
|
316
|
+
return result
|
|
317
|
+
|
|
318
|
+
return wrapped
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
def _verified_upstream_key_id(value: Any) -> str | None:
|
|
322
|
+
"""Return an already-verified key id, or nothing when it is not one.
|
|
323
|
+
|
|
324
|
+
The Studio API edge verifies the customer key before this process receives the request. The key
|
|
325
|
+
id is checked
|
|
326
|
+
against :data:`governors.KEY_ID_PATTERN` — the same check that makes it impossible to file a raw
|
|
327
|
+
credential where an identifier belongs, since no credential this product mints is 16 lowercase
|
|
328
|
+
hex characters.
|
|
329
|
+
"""
|
|
330
|
+
|
|
331
|
+
if isinstance(value, str) and KEY_ID_PATTERN.match(value) is not None:
|
|
332
|
+
return value
|
|
333
|
+
return None
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
def _read(
|
|
337
|
+
index: serving.VersionIndex,
|
|
338
|
+
dataset_id: str,
|
|
339
|
+
pin: str,
|
|
340
|
+
*,
|
|
341
|
+
shape: str,
|
|
342
|
+
predicates: Any,
|
|
343
|
+
offset: int,
|
|
344
|
+
limit: int,
|
|
345
|
+
max_bytes: int | None,
|
|
346
|
+
) -> serving.ServingResult:
|
|
347
|
+
"""Enter the read core. Reached only after both gates have passed, and never before.
|
|
348
|
+
|
|
349
|
+
Nothing is decided here. The shape was already checked against :data:`READ_SHAPES`, so this is a
|
|
350
|
+
routing step onto the two functions ``serving.py`` exports and it can invent no third.
|
|
351
|
+
"""
|
|
352
|
+
|
|
353
|
+
if shape == "describe":
|
|
354
|
+
return serving.describe_dataset(index, dataset_id, pin)
|
|
355
|
+
return serving.serve_rows(
|
|
356
|
+
index,
|
|
357
|
+
dataset_id,
|
|
358
|
+
pin,
|
|
359
|
+
predicates=predicates,
|
|
360
|
+
offset=offset,
|
|
361
|
+
limit=limit,
|
|
362
|
+
max_bytes=max_bytes,
|
|
363
|
+
)
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
@_preserve_v3_coordinates
|
|
367
|
+
def serve_read(
|
|
368
|
+
index: serving.VersionIndex,
|
|
369
|
+
dataset_id: str,
|
|
370
|
+
pin: str,
|
|
371
|
+
*,
|
|
372
|
+
shape: str,
|
|
373
|
+
ledger: CapLedger,
|
|
374
|
+
now: datetime,
|
|
375
|
+
index_path: Path | None = None,
|
|
376
|
+
verified_key_id: str | None = None,
|
|
377
|
+
key: str | None = None,
|
|
378
|
+
verify_url: str | None = None,
|
|
379
|
+
transport: key_seam.VerifyTransport | None = None,
|
|
380
|
+
predicates: Any = None,
|
|
381
|
+
offset: int = 0,
|
|
382
|
+
limit: int = serving.MAX_SERVED_ROWS,
|
|
383
|
+
max_bytes: int | None = None,
|
|
384
|
+
parent_dataset_id: str | None = None,
|
|
385
|
+
api_version: str = "historical",
|
|
386
|
+
) -> serving.ServingResult:
|
|
387
|
+
"""Verify the key, reserve the quota, and only then read. Never in any other order.
|
|
388
|
+
|
|
389
|
+
Exactly one credential form must be supplied:
|
|
390
|
+
|
|
391
|
+
* ``verified_key_id`` — PRODUCTION. The Studio API edge already verified the customer key and
|
|
392
|
+
this is the key id it produced. Its shape is checked and nothing is re-verified, because the
|
|
393
|
+
key itself never reached this process.
|
|
394
|
+
* ``key`` with ``verify_url`` — the LOCAL proof. The key is verified in process against cloud's
|
|
395
|
+
verify surface through :func:`key_seam.verify_api_key`.
|
|
396
|
+
|
|
397
|
+
Both together are refused rather than one being picked, and neither is refused too: a read with
|
|
398
|
+
no credential is a read by nobody.
|
|
399
|
+
|
|
400
|
+
``index_path`` is how ``current`` stays current in a process that outlives a move of Current.
|
|
401
|
+
Given one, the document is read again through :func:`serving.read_version_index` for THIS read,
|
|
402
|
+
and the ``index`` argument is the copy the caller parsed at start-up — kept as the argument that
|
|
403
|
+
proves the document was readable before a socket existed, and superseded for the read itself. It
|
|
404
|
+
is read INSIDE the reservation, after both gates, so an unverified caller still causes no
|
|
405
|
+
filesystem access at all, and an index that has become unreadable or invalid refuses under the
|
|
406
|
+
same counted read as any other refusal. Without it, the passed ``index`` is what the read
|
|
407
|
+
resolves against, and ``current`` then means current as of whenever that value was parsed.
|
|
408
|
+
|
|
409
|
+
``ledger`` and ``now`` are explicit so a test can drive them. Neither is an off-switch: there is
|
|
410
|
+
no ledger a caller can pass that admits more than its policy allows, and there is no value of
|
|
411
|
+
``now`` at which the key check does not run.
|
|
412
|
+
|
|
413
|
+
The gate order is the whole contract, and every line of it is asserted rather than described. A
|
|
414
|
+
refused read **opens no run directory, reads no Parquet, and allocates no buffer** — the same
|
|
415
|
+
sentence ``hosted_bootstrap.launch`` earns on the build side, where a refused run "creates no
|
|
416
|
+
descriptor, holds no memory, and above all never reaches execve".
|
|
417
|
+
|
|
418
|
+
Returns:
|
|
419
|
+
A :class:`serving.ServingResult` — the one envelope, answering or refusing. A refusal from
|
|
420
|
+
either gate carries null digests, because no version was bound; a refusal from inside the
|
|
421
|
+
read core carries whatever ``serving.py`` bound before it declined.
|
|
422
|
+
"""
|
|
423
|
+
|
|
424
|
+
if api_version not in {"historical", "v3"}:
|
|
425
|
+
return _refused(
|
|
426
|
+
dataset_id,
|
|
427
|
+
pin,
|
|
428
|
+
"this read does not name a supported serving contract",
|
|
429
|
+
READ_SHAPE_REFUSAL_CODE,
|
|
430
|
+
)
|
|
431
|
+
if api_version == "v3" and parent_dataset_id is None:
|
|
432
|
+
return _refused(
|
|
433
|
+
dataset_id,
|
|
434
|
+
pin,
|
|
435
|
+
"a V3 read must name both its Dataset parent and Table child "
|
|
436
|
+
"[SERVING_TABLE_SCOPE_MISMATCH]",
|
|
437
|
+
"SERVING_TABLE_SCOPE_MISMATCH",
|
|
438
|
+
)
|
|
439
|
+
if shape not in READ_SHAPES:
|
|
440
|
+
return _refused(
|
|
441
|
+
dataset_id,
|
|
442
|
+
pin,
|
|
443
|
+
"a read asks a dataset to describe itself or to hand back rows, and nothing else",
|
|
444
|
+
READ_SHAPE_REFUSAL_CODE,
|
|
445
|
+
)
|
|
446
|
+
|
|
447
|
+
if parent_dataset_id is not None and index.table(parent_dataset_id, dataset_id) is None:
|
|
448
|
+
return _refused(
|
|
449
|
+
dataset_id,
|
|
450
|
+
pin,
|
|
451
|
+
"the requested Dataset/Table pair is not listed [SERVING_TABLE_SCOPE_MISMATCH]",
|
|
452
|
+
"SERVING_TABLE_SCOPE_MISMATCH",
|
|
453
|
+
)
|
|
454
|
+
|
|
455
|
+
# Verify the key before any read work.
|
|
456
|
+
# Nothing below this block runs for a caller who is not allowed to be here.
|
|
457
|
+
if verified_key_id is not None and key is not None:
|
|
458
|
+
return _refused(
|
|
459
|
+
dataset_id,
|
|
460
|
+
pin,
|
|
461
|
+
"a read presents an already-verified key or a key to verify, never both",
|
|
462
|
+
CREDENTIAL_REFUSAL_CODE,
|
|
463
|
+
)
|
|
464
|
+
if verified_key_id is not None:
|
|
465
|
+
upstream = _verified_upstream_key_id(verified_key_id)
|
|
466
|
+
if upstream is None:
|
|
467
|
+
return _refused(
|
|
468
|
+
dataset_id,
|
|
469
|
+
pin,
|
|
470
|
+
"this read did not present a usable credential",
|
|
471
|
+
CREDENTIAL_REFUSAL_CODE,
|
|
472
|
+
)
|
|
473
|
+
key_id = upstream
|
|
474
|
+
elif key is not None:
|
|
475
|
+
if not isinstance(verify_url, str) or not verify_url:
|
|
476
|
+
return _refused(
|
|
477
|
+
dataset_id,
|
|
478
|
+
pin,
|
|
479
|
+
"this deployment cannot verify a key, so no read was served",
|
|
480
|
+
CREDENTIAL_REFUSAL_CODE,
|
|
481
|
+
)
|
|
482
|
+
try:
|
|
483
|
+
# The verdict belongs to key_seam, which denies on any non-200, any redirect, any
|
|
484
|
+
# transport fault, any non-strict-JSON or oversized body, and any body whose key id does
|
|
485
|
+
# not match the presented key — so a transport can be refused but cannot be laundered
|
|
486
|
+
# into an allow. A branch here on what the transport is would be the beginning of a
|
|
487
|
+
# second verifier, which is the thing this module exists not to build.
|
|
488
|
+
verdict = key_seam.verify_api_key(key, base_url=verify_url, transport=transport)
|
|
489
|
+
except key_seam.KeySeamError:
|
|
490
|
+
return _refused(
|
|
491
|
+
dataset_id,
|
|
492
|
+
pin,
|
|
493
|
+
"this deployment cannot verify a key, so no read was served",
|
|
494
|
+
CREDENTIAL_REFUSAL_CODE,
|
|
495
|
+
)
|
|
496
|
+
if not verdict.ok:
|
|
497
|
+
return _refused(dataset_id, pin, verdict.reason, CREDENTIAL_REFUSAL_CODE)
|
|
498
|
+
key_id = verdict.key_id
|
|
499
|
+
else:
|
|
500
|
+
return _refused(
|
|
501
|
+
dataset_id,
|
|
502
|
+
pin,
|
|
503
|
+
"this read presented no credential",
|
|
504
|
+
CREDENTIAL_REFUSAL_CODE,
|
|
505
|
+
)
|
|
506
|
+
|
|
507
|
+
# The raw key stops HERE. It went into key_seam and into nothing else, and unbinding the name
|
|
508
|
+
# makes that a property of the function rather than a promise about it: any later line that
|
|
509
|
+
# reached for it would raise instead of carrying a credential further into the read path.
|
|
510
|
+
del key
|
|
511
|
+
|
|
512
|
+
# Reserve quota after authentication and before opening data.
|
|
513
|
+
try:
|
|
514
|
+
cost_units = declared_read_cost_units()
|
|
515
|
+
except ServingEdgeError:
|
|
516
|
+
# The operator's sentence names the variable and stays in the exception, where an operator
|
|
517
|
+
# reads it. What a customer gets back says the deployment could not price the read and
|
|
518
|
+
# names no environment variable: a response is not a place to publish deployment internals.
|
|
519
|
+
return _refused(
|
|
520
|
+
dataset_id,
|
|
521
|
+
pin,
|
|
522
|
+
"this deployment cannot price a read, so no read was served",
|
|
523
|
+
READ_COST_REFUSAL_CODE,
|
|
524
|
+
)
|
|
525
|
+
try:
|
|
526
|
+
reservation = ledger.reserve(key_id, now=now, estimated_cost_units=cost_units)
|
|
527
|
+
except CapExceeded as error:
|
|
528
|
+
# Preserve the governor's refusal text so all read-cap callers report the same reason.
|
|
529
|
+
return _refused(dataset_id, pin, str(error), QUOTA_REFUSAL_CODE)
|
|
530
|
+
except CapStoreError as error:
|
|
531
|
+
# Fail closed, exactly as ``hosted_bootstrap._reserve_run_slot`` does. Counters that cannot
|
|
532
|
+
# be read are not counters that say zero.
|
|
533
|
+
return _refused(dataset_id, pin, str(error), QUOTA_UNREADABLE_REFUSAL_CODE)
|
|
534
|
+
|
|
535
|
+
# Perform the read only while its quota reservation is held.
|
|
536
|
+
# Inside the reservation's block, so a fault gives the slot back instead of leaving it held by a
|
|
537
|
+
# read that never happened.
|
|
538
|
+
with reservation:
|
|
539
|
+
try:
|
|
540
|
+
bound = index if index_path is None else serving.read_version_index(index_path)
|
|
541
|
+
except serving.ServingRefused as refusal:
|
|
542
|
+
# An index that stopped being readable or valid while this process ran is a refusal
|
|
543
|
+
# with the reader's own reasons, not a fault: the module's contract is that a corrupt
|
|
544
|
+
# index can only ever make a read decline. It is still committed below, because the
|
|
545
|
+
# gates ran.
|
|
546
|
+
result = _envelope(dataset_id, pin, refusal.reasons)
|
|
547
|
+
else:
|
|
548
|
+
if parent_dataset_id is not None and bound.table(parent_dataset_id, dataset_id) is None:
|
|
549
|
+
result = _refused(
|
|
550
|
+
dataset_id,
|
|
551
|
+
pin,
|
|
552
|
+
"the requested Dataset/Table pair is not listed [SERVING_TABLE_SCOPE_MISMATCH]",
|
|
553
|
+
"SERVING_TABLE_SCOPE_MISMATCH",
|
|
554
|
+
)
|
|
555
|
+
reservation.commit(actual_cost_units=cost_units)
|
|
556
|
+
return result
|
|
557
|
+
result = _read(
|
|
558
|
+
bound,
|
|
559
|
+
dataset_id,
|
|
560
|
+
pin,
|
|
561
|
+
shape=shape,
|
|
562
|
+
predicates=predicates,
|
|
563
|
+
offset=offset,
|
|
564
|
+
limit=limit,
|
|
565
|
+
max_bytes=max_bytes,
|
|
566
|
+
)
|
|
567
|
+
# Committed whether the core answered or declined, because either way the read path ran and
|
|
568
|
+
# the work was done. Refunding a refusal would let a caller drive the whole verification and
|
|
569
|
+
# replay path without bound for free, which is the cap this gate exists to hold.
|
|
570
|
+
reservation.commit(actual_cost_units=cost_units)
|
|
571
|
+
return result
|
|
572
|
+
|
|
573
|
+
|
|
574
|
+
__all__ = [
|
|
575
|
+
"CREDENTIAL_REFUSAL_CODE",
|
|
576
|
+
"QUOTA_REFUSAL_CODE",
|
|
577
|
+
"QUOTA_UNREADABLE_REFUSAL_CODE",
|
|
578
|
+
"READ_COST_REFUSAL_CODE",
|
|
579
|
+
"READ_COST_UNITS_ENV",
|
|
580
|
+
"READ_SHAPES",
|
|
581
|
+
"READ_SHAPE_REFUSAL_CODE",
|
|
582
|
+
"SERVING_CAP_STORE_ENV",
|
|
583
|
+
"SERVING_CONSUMER_CALL_SITE",
|
|
584
|
+
"STUDIO_SECURITY_SCHEMES",
|
|
585
|
+
"STUDIO_WIRE_BINDING",
|
|
586
|
+
"ServingEdgeError",
|
|
587
|
+
"declared_read_cost_units",
|
|
588
|
+
"serve_read",
|
|
589
|
+
"serving_ledger",
|
|
590
|
+
]
|