mostlyright-data 0.9.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mostlyright/data_harness/__init__.py +158 -0
- mostlyright/data_harness/acquisition/__init__.py +55 -0
- mostlyright/data_harness/acquisition/http.py +2773 -0
- mostlyright/data_harness/acquisition/parsing.py +809 -0
- mostlyright/data_harness/acquisition/ranges.py +495 -0
- mostlyright/data_harness/acquisition/result_download.py +360 -0
- mostlyright/data_harness/acquisition/retention_admission.py +248 -0
- mostlyright/data_harness/acquisition/sandbox.py +4888 -0
- mostlyright/data_harness/acquisition/url_policy.py +530 -0
- mostlyright/data_harness/agent_runtime.py +2743 -0
- mostlyright/data_harness/assets/logo-ink.svg +31 -0
- mostlyright/data_harness/backends/__init__.py +28 -0
- mostlyright/data_harness/backends/pandas_backend.py +350 -0
- mostlyright/data_harness/backends/polars_backend.py +366 -0
- mostlyright/data_harness/backends/protocol.py +124 -0
- mostlyright/data_harness/backends/reference.py +83 -0
- mostlyright/data_harness/backends/registry.py +55 -0
- mostlyright/data_harness/backends/restrictions.py +126 -0
- mostlyright/data_harness/canonical.py +333 -0
- mostlyright/data_harness/catalog_job.py +625 -0
- mostlyright/data_harness/cli.py +5398 -0
- mostlyright/data_harness/contracts.py +53 -0
- mostlyright/data_harness/coordinator.py +1307 -0
- mostlyright/data_harness/deploy.py +924 -0
- mostlyright/data_harness/deploy_target.py +312 -0
- mostlyright/data_harness/deployment_evidence.py +1067 -0
- mostlyright/data_harness/event_presentation.py +576 -0
- mostlyright/data_harness/events.py +2152 -0
- mostlyright/data_harness/fast_delimited.py +239 -0
- mostlyright/data_harness/fleet.py +237 -0
- mostlyright/data_harness/formats.py +236 -0
- mostlyright/data_harness/governors.py +1163 -0
- mostlyright/data_harness/hosted_bootstrap.py +972 -0
- mostlyright/data_harness/hosted_crawler.py +1115 -0
- mostlyright/data_harness/hosted_crawler_container_smoke.py +351 -0
- mostlyright/data_harness/hosted_crawler_fetch.py +423 -0
- mostlyright/data_harness/hosted_crawler_job.py +1277 -0
- mostlyright/data_harness/hosted_crawler_protocol.py +676 -0
- mostlyright/data_harness/hosted_dataset.py +1500 -0
- mostlyright/data_harness/hosted_deploy.py +3037 -0
- mostlyright/data_harness/hosted_handoff.py +62 -0
- mostlyright/data_harness/hosted_ingestion_contract.py +504 -0
- mostlyright/data_harness/hosted_ingestion_job.py +356 -0
- mostlyright/data_harness/hosted_ingestion_job_smoke.py +40 -0
- mostlyright/data_harness/hosted_session_container_smoke.py +194 -0
- mostlyright/data_harness/hosted_session_worker.py +3554 -0
- mostlyright/data_harness/hosted_session_worker_job_smoke.py +46 -0
- mostlyright/data_harness/hosted_worker.py +6784 -0
- mostlyright/data_harness/ingestion/__init__.py +56 -0
- mostlyright/data_harness/ingestion/contracts.py +461 -0
- mostlyright/data_harness/ingestion/faults.py +42 -0
- mostlyright/data_harness/ingestion/gcs_store.py +1162 -0
- mostlyright/data_harness/ingestion/spool.py +130 -0
- mostlyright/data_harness/ingestion/store.py +885 -0
- mostlyright/data_harness/key_seam.py +434 -0
- mostlyright/data_harness/linux_process_boundary.py +262 -0
- mostlyright/data_harness/local_contracts.py +2880 -0
- mostlyright/data_harness/local_search/__init__.py +5 -0
- mostlyright/data_harness/local_search/build_index.py +1087 -0
- mostlyright/data_harness/local_search/contracts.py +920 -0
- mostlyright/data_harness/local_search/query_trace.py +266 -0
- mostlyright/data_harness/local_search/retrieval.py +700 -0
- mostlyright/data_harness/local_search/sealed.py +474 -0
- mostlyright/data_harness/local_search/service.py +784 -0
- mostlyright/data_harness/nbrender/CONTRACT.md +212 -0
- mostlyright/data_harness/nbrender/__init__.py +12 -0
- mostlyright/data_harness/nbrender/chrome.py +359 -0
- mostlyright/data_harness/nbrender/code_body.py +266 -0
- mostlyright/data_harness/nbrender/document.py +407 -0
- mostlyright/data_harness/nbrender/frame.py +275 -0
- mostlyright/data_harness/nbrender/interactive.py +337 -0
- mostlyright/data_harness/nbrender/markdown_body.py +477 -0
- mostlyright/data_harness/nbrender/mr_components.py +134 -0
- mostlyright/data_harness/nbrender/outputs_data.py +595 -0
- mostlyright/data_harness/nbrender/outputs_rich.py +906 -0
- mostlyright/data_harness/nbrender/outputs_source.py +260 -0
- mostlyright/data_harness/nbrender/outputs_stage.py +176 -0
- mostlyright/data_harness/nbrender/outputs_text.py +400 -0
- mostlyright/data_harness/nbrender/parse.py +394 -0
- mostlyright/data_harness/nbrender/status.py +40 -0
- mostlyright/data_harness/nbrender/tokens.py +1295 -0
- mostlyright/data_harness/notebook.py +1710 -0
- mostlyright/data_harness/offline.py +2049 -0
- mostlyright/data_harness/operation_registry.py +1007 -0
- mostlyright/data_harness/operator_setup.py +239 -0
- mostlyright/data_harness/pipeline.py +6428 -0
- mostlyright/data_harness/plan_graph.py +2026 -0
- mostlyright/data_harness/preparation/__init__.py +104 -0
- mostlyright/data_harness/preparation/contracts.py +1017 -0
- mostlyright/data_harness/preparation/engine.py +221 -0
- mostlyright/data_harness/preparation/errors.py +14 -0
- mostlyright/data_harness/preparation/gates.py +751 -0
- mostlyright/data_harness/preparation/joins.py +574 -0
- mostlyright/data_harness/preparation/profile.py +384 -0
- mostlyright/data_harness/preparation/table.py +217 -0
- mostlyright/data_harness/preparation/transforms.py +568 -0
- mostlyright/data_harness/progress_events.py +534 -0
- mostlyright/data_harness/readers/__init__.py +46 -0
- mostlyright/data_harness/readers/containers.py +963 -0
- mostlyright/data_harness/readers/contracts.py +542 -0
- mostlyright/data_harness/readers/delimited.py +257 -0
- mostlyright/data_harness/readers/grib2/__init__.py +33 -0
- mostlyright/data_harness/readers/grib2/admission.py +722 -0
- mostlyright/data_harness/readers/grib2/decode.py +1009 -0
- mostlyright/data_harness/readers/grib2/geometry.py +1133 -0
- mostlyright/data_harness/readers/grib2/portable_math.py +501 -0
- mostlyright/data_harness/readers/json_tabular.py +485 -0
- mostlyright/data_harness/readers/registry.py +514 -0
- mostlyright/data_harness/readers/samples/README.md +110 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/cities.csv.gz +0 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/cities.csv.gz +0 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/cities.tar +0 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/cities.tar +0 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/cities.zip +0 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/expected.json +25 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/expected.json +25 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/expected.json +25 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/cities.csv +3 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/expected.json +23 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/cities.tsv +5 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/expected.json +25 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/expected.json +30 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/observations.csv +5 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/expected.json +44 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/stations.json +1 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/expected.json +48 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/observations.ndjson +4 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/cities.xlsx +0 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/expected.json +24 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/expected.json +27 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/expected.json +27 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/README.md +20 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/expected.json +55 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/gfs-2m-temperature.grib2 +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/expected.json +54 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/expected.json +54 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/hrrr-categorical-rain.grib2 +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/expected.json +54 -0
- mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
- mostlyright/data_harness/readers/samples.py +582 -0
- mostlyright/data_harness/readers/spreadsheet.py +803 -0
- mostlyright/data_harness/readers/tabular.py +510 -0
- mostlyright/data_harness/recipe.py +5321 -0
- mostlyright/data_harness/repair/__init__.py +78 -0
- mostlyright/data_harness/repair/adapters.py +274 -0
- mostlyright/data_harness/repair/contracts.py +872 -0
- mostlyright/data_harness/repair/coordinator.py +1099 -0
- mostlyright/data_harness/repair/errors.py +16 -0
- mostlyright/data_harness/review.py +2533 -0
- mostlyright/data_harness/rowset.py +283 -0
- mostlyright/data_harness/serving.py +1975 -0
- mostlyright/data_harness/serving_edge.py +590 -0
- mostlyright/data_harness/serving_http.py +1031 -0
- mostlyright/data_harness/session_probes.py +759 -0
- mostlyright/data_harness/signing.py +101 -0
- mostlyright/data_harness/source_discovery.py +898 -0
- mostlyright/data_harness/sources/__init__.py +209 -0
- mostlyright/data_harness/sources/_adapter_steps.py +213 -0
- mostlyright/data_harness/sources/adapters.py +1214 -0
- mostlyright/data_harness/sources/cadence.py +1428 -0
- mostlyright/data_harness/sources/cadence_emission.py +453 -0
- mostlyright/data_harness/sources/cadence_history.py +546 -0
- mostlyright/data_harness/sources/catalog/__init__.py +17 -0
- mostlyright/data_harness/sources/catalog/admission.py +477 -0
- mostlyright/data_harness/sources/catalog/authoring.py +1701 -0
- mostlyright/data_harness/sources/catalog/authoring_policy.py +701 -0
- mostlyright/data_harness/sources/catalog/authoring_shards.py +1217 -0
- mostlyright/data_harness/sources/catalog/bounded_io.py +231 -0
- mostlyright/data_harness/sources/catalog/channel.py +523 -0
- mostlyright/data_harness/sources/catalog/channel_client.py +296 -0
- mostlyright/data_harness/sources/catalog/contracts.py +825 -0
- mostlyright/data_harness/sources/catalog/coverage.py +137 -0
- mostlyright/data_harness/sources/catalog/delta.py +1340 -0
- mostlyright/data_harness/sources/catalog/embedding.py +532 -0
- mostlyright/data_harness/sources/catalog/entry_v2.py +1182 -0
- mostlyright/data_harness/sources/catalog/fill.py +3889 -0
- mostlyright/data_harness/sources/catalog/fill_partitions.py +459 -0
- mostlyright/data_harness/sources/catalog/fill_staging.py +1105 -0
- mostlyright/data_harness/sources/catalog/gating.py +374 -0
- mostlyright/data_harness/sources/catalog/generation_receipt.py +1607 -0
- mostlyright/data_harness/sources/catalog/harvest/__init__.py +7 -0
- mostlyright/data_harness/sources/catalog/harvest/ckan.py +384 -0
- mostlyright/data_harness/sources/catalog/harvest/datagov_v4.py +798 -0
- mostlyright/data_harness/sources/catalog/harvest/protocol.py +964 -0
- mostlyright/data_harness/sources/catalog/harvest/sdmx.py +445 -0
- mostlyright/data_harness/sources/catalog/harvest/stac.py +384 -0
- mostlyright/data_harness/sources/catalog/health.py +447 -0
- mostlyright/data_harness/sources/catalog/hosted_catalog.py +105 -0
- mostlyright/data_harness/sources/catalog/identity_history.py +1549 -0
- mostlyright/data_harness/sources/catalog/neural.py +1618 -0
- mostlyright/data_harness/sources/catalog/packed_catalog.py +2345 -0
- mostlyright/data_harness/sources/catalog/packed_retrieval.py +1517 -0
- mostlyright/data_harness/sources/catalog/packed_writer.py +2802 -0
- mostlyright/data_harness/sources/catalog/query_trace.py +1037 -0
- mostlyright/data_harness/sources/catalog/recommend.py +171 -0
- mostlyright/data_harness/sources/catalog/retrieval.py +230 -0
- mostlyright/data_harness/sources/catalog/retrieval_manifest.py +995 -0
- mostlyright/data_harness/sources/catalog/rights_decisions.py +254 -0
- mostlyright/data_harness/sources/catalog/sealed.py +560 -0
- mostlyright/data_harness/sources/catalog/search.py +230 -0
- mostlyright/data_harness/sources/catalog/streaming_delta.py +1097 -0
- mostlyright/data_harness/sources/catalog/update.py +891 -0
- mostlyright/data_harness/sources/collections.py +815 -0
- mostlyright/data_harness/sources/contracts.py +2223 -0
- mostlyright/data_harness/sources/deletion.py +761 -0
- mostlyright/data_harness/sources/fitness.py +162 -0
- mostlyright/data_harness/sources/governance.py +163 -0
- mostlyright/data_harness/sources/hosted.py +173 -0
- mostlyright/data_harness/sources/integration.py +218 -0
- mostlyright/data_harness/sources/range_reader.py +418 -0
- mostlyright/data_harness/sources/registry.py +514 -0
- mostlyright/data_harness/sources/rights_rule.py +59 -0
- mostlyright/data_harness/sources/source_cadence_vectors.v1.json +1 -0
- mostlyright/data_harness/sources/sports.py +521 -0
- mostlyright/data_harness/sources/stream.py +524 -0
- mostlyright/data_harness/sources/stream_connector.py +418 -0
- mostlyright/data_harness/sources/stream_recorder.py +1404 -0
- mostlyright/data_harness/studio_boundary.py +2019 -0
- mostlyright/data_harness/thin/__init__.py +37 -0
- mostlyright/data_harness/thin/acquire.py +1137 -0
- mostlyright/data_harness/thin/acquire_cancel.py +579 -0
- mostlyright/data_harness/thin/approvals.py +617 -0
- mostlyright/data_harness/thin/commands.py +406 -0
- mostlyright/data_harness/thin/download.py +194 -0
- mostlyright/data_harness/thin/narrative.py +589 -0
- mostlyright/data_harness/thin/parity.py +1070 -0
- mostlyright/data_harness/thin/propose.py +2759 -0
- mostlyright/data_harness/thin/research.py +1663 -0
- mostlyright/data_harness/thin/router.py +924 -0
- mostlyright/data_harness/thin/runs.py +519 -0
- mostlyright/data_harness/thin/session.py +281 -0
- mostlyright/data_harness/thin/stream.py +501 -0
- mostlyright/data_harness/thin/transport.py +187 -0
- mostlyright/data_harness/thin/vocabulary.py +368 -0
- mostlyright/data_harness/thin/workers.py +164 -0
- mostlyright/data_harness/ucum/TABLE-PIN.json +40 -0
- mostlyright/data_harness/ucum/ucum-subset.v1.json +632 -0
- mostlyright/data_harness/unit_flow.py +927 -0
- mostlyright/data_harness/units.py +572 -0
- mostlyright/data_harness/ux/__init__.py +9 -0
- mostlyright/data_harness/ux/approve.py +485 -0
- mostlyright/data_harness/ux/author_yaml.py +597 -0
- mostlyright/data_harness/ux/cloud_auth.py +447 -0
- mostlyright/data_harness/ux/commands/__init__.py +260 -0
- mostlyright/data_harness/ux/commands/approve.py +136 -0
- mostlyright/data_harness/ux/commands/auth.py +744 -0
- mostlyright/data_harness/ux/commands/author.py +79 -0
- mostlyright/data_harness/ux/commands/catalog_author.py +403 -0
- mostlyright/data_harness/ux/commands/catalog_fill.py +523 -0
- mostlyright/data_harness/ux/commands/catalog_harvest.py +545 -0
- mostlyright/data_harness/ux/commands/catalog_publish.py +1838 -0
- mostlyright/data_harness/ux/commands/catalog_search.py +71 -0
- mostlyright/data_harness/ux/commands/catalog_update.py +437 -0
- mostlyright/data_harness/ux/commands/deploy.py +134 -0
- mostlyright/data_harness/ux/commands/deploy_dataset.py +98 -0
- mostlyright/data_harness/ux/commands/deploy_plan.py +105 -0
- mostlyright/data_harness/ux/commands/deploy_status.py +104 -0
- mostlyright/data_harness/ux/commands/diff.py +74 -0
- mostlyright/data_harness/ux/commands/index.py +84 -0
- mostlyright/data_harness/ux/commands/inventory.py +47 -0
- mostlyright/data_harness/ux/commands/list_builds.py +143 -0
- mostlyright/data_harness/ux/commands/login.py +63 -0
- mostlyright/data_harness/ux/commands/peek.py +236 -0
- mostlyright/data_harness/ux/commands/plan_check.py +90 -0
- mostlyright/data_harness/ux/commands/preflight.py +97 -0
- mostlyright/data_harness/ux/commands/record.py +107 -0
- mostlyright/data_harness/ux/commands/review_setup.py +47 -0
- mostlyright/data_harness/ux/commands/search.py +440 -0
- mostlyright/data_harness/ux/commands/show.py +61 -0
- mostlyright/data_harness/ux/commands/whoami.py +37 -0
- mostlyright/data_harness/ux/credential_native.py +551 -0
- mostlyright/data_harness/ux/credential_store.py +1055 -0
- mostlyright/data_harness/ux/credentials.py +631 -0
- mostlyright/data_harness/ux/diffing.py +444 -0
- mostlyright/data_harness/ux/headline.py +671 -0
- mostlyright/data_harness/ux/hosted_acquisition.py +974 -0
- mostlyright/data_harness/ux/hosted_run_status.py +619 -0
- mostlyright/data_harness/ux/inventory.py +427 -0
- mostlyright/data_harness/ux/local_review.py +375 -0
- mostlyright/data_harness/ux/login.py +691 -0
- mostlyright/data_harness/ux/path_kind.py +147 -0
- mostlyright/data_harness/ux/peek.py +1000 -0
- mostlyright/data_harness/ux/plain_file.py +178 -0
- mostlyright/data_harness/ux/plan_check.py +311 -0
- mostlyright/data_harness/ux/preflight.py +918 -0
- mostlyright/data_harness/ux/readers.py +1124 -0
- mostlyright/data_harness/ux/remediation.py +2195 -0
- mostlyright/data_harness/ux/render.py +657 -0
- mostlyright/data_harness/ux/workload.py +1077 -0
- mostlyright/data_harness/viewer.py +3713 -0
- mostlyright/data_harness/visual_run/__init__.py +83 -0
- mostlyright/data_harness/visual_run/authoring.py +235 -0
- mostlyright/data_harness/visual_run/contracts.py +673 -0
- mostlyright/data_harness/visual_run/materialize.py +486 -0
- mostlyright/data_harness/visual_run/observations.py +874 -0
- mostlyright/data_harness/visual_run/query.py +259 -0
- mostlyright/data_harness/visual_run/reducer.py +280 -0
- mostlyright/data_harness/visual_run/sdk.py +892 -0
- mostlyright/data_harness/visual_run/store.py +584 -0
- mostlyright/data_harness/visual_run/transport.py +239 -0
- mostlyright/data_harness/watch.py +2999 -0
- mostlyright_data-0.9.0.dist-info/METADATA +607 -0
- mostlyright_data-0.9.0.dist-info/RECORD +314 -0
- mostlyright_data-0.9.0.dist-info/WHEEL +4 -0
- mostlyright_data-0.9.0.dist-info/entry_points.txt +12 -0
|
@@ -0,0 +1,1031 @@
|
|
|
1
|
+
"""The loopback transport for the serving surface. It carries requests; it decides nothing.
|
|
2
|
+
|
|
3
|
+
This module is a TRANSPORT. It parses one HTTP request into the closed vocabulary ``serving.py``
|
|
4
|
+
already defines, hands it to :func:`serving_edge.serve_read`, and renders the envelope that comes
|
|
5
|
+
back. Every refusal it can produce is either a refusal the library already produced or a refusal
|
|
6
|
+
about the REQUEST'S OWN SHAPE — a path this surface does not answer, a method it does not answer on
|
|
7
|
+
that path, a body over the byte bound, a body that is not strict JSON, a field name the request
|
|
8
|
+
vocabulary does not contain. No refusal here is about data, because nothing here looks at data.
|
|
9
|
+
|
|
10
|
+
Like ``viewer.py``, it binds ``127.0.0.1`` and never a public interface, it adds no dependency
|
|
11
|
+
(``http.server`` is the standard library, and ``pyproject.toml`` is untouched by this surface), and
|
|
12
|
+
**it never executes anything it serves**: the bytes it hands back were sealed by the build path and
|
|
13
|
+
replay-verified by ``serving.open_sealed_version`` before this module saw them.
|
|
14
|
+
|
|
15
|
+
The server handles one request at a time. This keeps HTTP concurrency separate from the
|
|
16
|
+
per-process read cap in ``governors.CapPolicy.max_concurrent_runs``. Process topology is outside
|
|
17
|
+
this module.
|
|
18
|
+
|
|
19
|
+
**There is no authentication off-switch.** :func:`build_server` refuses to start without an https
|
|
20
|
+
verify URL, so an unauthenticated serving surface cannot be constructed from this module at all, and
|
|
21
|
+
``mr-data serve`` refuses the same way one layer up.
|
|
22
|
+
|
|
23
|
+
**The verify transport is a library-only injection point.** The parameter supplies response bytes;
|
|
24
|
+
:func:`key_seam.verify_api_key` computes the verdict and rejects invalid status, redirects,
|
|
25
|
+
transport faults, invalid or oversized JSON, and key-id mismatches.
|
|
26
|
+
``mr-data serve`` never passes this parameter and no command-line option maps to it; the CLI's only
|
|
27
|
+
verify configuration is a URL that must be https. The parameter exists because this environment has
|
|
28
|
+
no reachable cloud verify surface, and a serving surface nobody has driven end to end is not a
|
|
29
|
+
shipped serving surface.
|
|
30
|
+
|
|
31
|
+
**The request vocabulary is closed, and it is the library's.** Two paths, two methods, one optional
|
|
32
|
+
query key, four body fields:
|
|
33
|
+
|
|
34
|
+
========================== =========================================================
|
|
35
|
+
Wire name What it becomes
|
|
36
|
+
========================== =========================================================
|
|
37
|
+
``GET /v1/datasets/{id}`` ``serving_edge.serve_read(..., shape="describe")``
|
|
38
|
+
``POST .../{id}/rows`` ``serving_edge.serve_read(..., shape="rows")``
|
|
39
|
+
``?pin=`` / body ``pin`` the ``pin`` argument, whose grammar ``serving.resolve_pin`` owns
|
|
40
|
+
body ``filter`` the ``predicates`` argument of ``serving.serve_rows``
|
|
41
|
+
body ``offset``/``limit`` the window arguments of ``serving.serve_rows``
|
|
42
|
+
========================== =========================================================
|
|
43
|
+
|
|
44
|
+
``filter`` is the one wire name that is not spelled the way the library spells it, for the reason
|
|
45
|
+
P-01 renames ``candidate_digest`` to ``version_digest`` on the way out: a JSON key a production
|
|
46
|
+
caller types is a user-visible string, and "filter" is what ``docs/VOCABULARY.md``'s plain register
|
|
47
|
+
calls it — ``serving.py``'s own prose calls it the filter throughout. The rename happens once, in
|
|
48
|
+
:func:`_request_from_body`, and the field it maps to is the library's single filter parameter. There
|
|
49
|
+
is no second field, and a request naming anything outside the four is refused rather than ignored.
|
|
50
|
+
|
|
51
|
+
A point lookup is not a third path. It is the same POST with an equality condition on every column
|
|
52
|
+
of the sealed grain, exactly as ``serving.serve_rows`` documents.
|
|
53
|
+
"""
|
|
54
|
+
|
|
55
|
+
from __future__ import annotations
|
|
56
|
+
|
|
57
|
+
import ipaddress
|
|
58
|
+
import time
|
|
59
|
+
from collections.abc import Callable
|
|
60
|
+
from dataclasses import dataclass
|
|
61
|
+
from datetime import UTC, datetime
|
|
62
|
+
from http.server import BaseHTTPRequestHandler, HTTPServer
|
|
63
|
+
from pathlib import Path
|
|
64
|
+
from typing import Any
|
|
65
|
+
from urllib.parse import parse_qsl
|
|
66
|
+
|
|
67
|
+
from mostlyright.data_harness import canonical, key_seam, serving, serving_edge
|
|
68
|
+
from mostlyright.data_harness.governors import CapLedger
|
|
69
|
+
|
|
70
|
+
#: A request body is a FILTER, not a payload, so the bound is in the spirit of
|
|
71
|
+
#: :data:`key_seam.MAX_VERIFY_RESPONSE_BYTES` rather than of the bootstrap envelope's. It is checked
|
|
72
|
+
#: twice: against the declared content length before a byte is read, and again while reading.
|
|
73
|
+
#:
|
|
74
|
+
#: This is below the library's theoretical maximum filter size and bounds attacker-controlled
|
|
75
|
+
#: memory held per request.
|
|
76
|
+
MAX_REQUEST_BYTES = 64 * 1024
|
|
77
|
+
|
|
78
|
+
#: The largest dataset id and the largest pin this transport will carry. Both are the LIBRARY's
|
|
79
|
+
#: numbers, referenced rather than restated: ``serving`` owns the grammar of both, checks both on
|
|
80
|
+
#: every read whoever the caller is, and a transport that declared its own could only differ from
|
|
81
|
+
#: the thing it is a transport for. They are re-exported under these names because a bound a caller
|
|
82
|
+
#: reads about in this module's own documentation should be readable from this module.
|
|
83
|
+
MAX_DATASET_ID_CHARS = serving.MAX_DATASET_ID_CHARS
|
|
84
|
+
MAX_PIN_CHARS = serving.MAX_PIN_CHARS
|
|
85
|
+
|
|
86
|
+
#: How long one request may hold this surface before it is dropped, in seconds.
|
|
87
|
+
#:
|
|
88
|
+
#: **The server is single-threaded by design, so the handler in front is the only handler.** Without
|
|
89
|
+
#: a deadline, one connection that opens and then says nothing — no credential, no request line, not
|
|
90
|
+
#: one byte — blocks ``rfile.readline`` forever and every other caller is refused service for as
|
|
91
|
+
#: long as the socket is held. The bound-before-read rule on the body does not help on its own
|
|
92
|
+
#: either: a request may declare a length under :data:`MAX_REQUEST_BYTES`, pass that check, and then
|
|
93
|
+
#: never finish sending it.
|
|
94
|
+
#:
|
|
95
|
+
#: **It is ONE deadline over the WHOLE request, and not a timeout per read.** A per-read timeout is
|
|
96
|
+
#: what ``socketserver`` gives a handler that sets :attr:`_ServingHandler.timeout`, and a caller
|
|
97
|
+
#: that sends one byte every ``timeout - 1`` seconds renews it forever — in the request line, in the
|
|
98
|
+
#: headers and in the body alike. So this number is taken once when the connection is accepted and
|
|
99
|
+
#: shrinks across every segment of the request, in :class:`_DeadlineReader`, which sits in front of
|
|
100
|
+
#: every read the handler and the standard library make.
|
|
101
|
+
#:
|
|
102
|
+
#: The number is generous for a legitimate caller on loopback, where a request is written in one
|
|
103
|
+
#: pass, and short next to the roughly 30 ms a verified read costs. It bounds the request side only:
|
|
104
|
+
#: the deadline is not running while this surface verifies a build, and it is not the writer's
|
|
105
|
+
#: budget either — the answer to a request that ran out of time is still written.
|
|
106
|
+
REQUEST_TIMEOUT_SECONDS = 10.0
|
|
107
|
+
|
|
108
|
+
#: The one optional query key, on the discovery path only. A pinned discovery has to be expressible
|
|
109
|
+
#: — ``serving.describe_dataset`` takes a pin like every other read — and a single closed key is the
|
|
110
|
+
#: smallest thing that expresses it. Any other query key refuses; a query string on the rows path
|
|
111
|
+
#: refuses, because a rows read carries its pin in the body with the rest of the request.
|
|
112
|
+
DISCOVERY_QUERY_KEYS = frozenset({"pin"})
|
|
113
|
+
|
|
114
|
+
#: The closed body vocabulary of a rows read. A name outside this set refuses rather than being
|
|
115
|
+
#: ignored: a caller who misspells a filter and is silently served an unfiltered window has been
|
|
116
|
+
#: given the wrong answer with a 200 on it.
|
|
117
|
+
ROWS_BODY_KEYS = frozenset({"pin", "filter", "offset", "limit"})
|
|
118
|
+
|
|
119
|
+
#: The refusal codes this transport authors. Every one of them is about the shape of the REQUEST;
|
|
120
|
+
#: none is about data. They are declared as constants for the reason ``serving_edge``'s are: a
|
|
121
|
+
#: status mapping should be a closed lookup rather than string matching on prose.
|
|
122
|
+
REQUEST_PATH_REFUSAL_CODE = "SERVING_REQUEST_PATH"
|
|
123
|
+
REQUEST_METHOD_REFUSAL_CODE = "SERVING_REQUEST_METHOD"
|
|
124
|
+
REQUEST_DATASET_REFUSAL_CODE = "SERVING_REQUEST_DATASET"
|
|
125
|
+
REQUEST_QUERY_REFUSAL_CODE = "SERVING_REQUEST_QUERY"
|
|
126
|
+
REQUEST_SIZE_REFUSAL_CODE = "SERVING_REQUEST_SIZE"
|
|
127
|
+
REQUEST_BODY_REFUSAL_CODE = "SERVING_REQUEST_BODY"
|
|
128
|
+
REQUEST_FIELD_REFUSAL_CODE = "SERVING_REQUEST_FIELD"
|
|
129
|
+
REQUEST_SHAPE_REFUSAL_CODE = "SERVING_REQUEST_SHAPE"
|
|
130
|
+
REQUEST_FAULT_REFUSAL_CODE = "SERVING_REQUEST_FAULT"
|
|
131
|
+
REQUEST_TIMEOUT_REFUSAL_CODE = "SERVING_REQUEST_TIMEOUT"
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
@dataclass(frozen=True)
|
|
135
|
+
class ServingPath:
|
|
136
|
+
"""One path this surface answers: its template, its method, and the read shape it becomes."""
|
|
137
|
+
|
|
138
|
+
method: str
|
|
139
|
+
template: str
|
|
140
|
+
shape: str
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
#: The two paths, as data. Routing matches a template segment by segment rather than doing string
|
|
144
|
+
#: surgery on ``self.path``, so a path this table does not name cannot be reached by any spelling of
|
|
145
|
+
#: it, and adding a third shape means adding a row here and a shape ``serving_edge.READ_SHAPES``
|
|
146
|
+
#: already admits.
|
|
147
|
+
SERVING_PATHS = (
|
|
148
|
+
ServingPath(
|
|
149
|
+
method="GET", template="/v3/datasets/{dataset_id}/tables/{table_id}", shape="describe"
|
|
150
|
+
),
|
|
151
|
+
ServingPath(
|
|
152
|
+
method="POST",
|
|
153
|
+
template="/v3/datasets/{dataset_id}/tables/{table_id}/rows",
|
|
154
|
+
shape="rows",
|
|
155
|
+
),
|
|
156
|
+
)
|
|
157
|
+
|
|
158
|
+
#: How one refusal code becomes one status code. Closed, and every code not named here is a 400:
|
|
159
|
+
#: the caller asked for something this surface will not do, and the envelope says which.
|
|
160
|
+
#:
|
|
161
|
+
#: ``SERVING_QUOTA_UNREADABLE`` and ``SERVING_READ_COST`` are not listed, so both use the default
|
|
162
|
+
#: 400 status.
|
|
163
|
+
STATUS_BY_REFUSAL_CODE = {
|
|
164
|
+
serving_edge.CREDENTIAL_REFUSAL_CODE: 401,
|
|
165
|
+
serving_edge.QUOTA_REFUSAL_CODE: 429,
|
|
166
|
+
"SERVING_DATASET_UNKNOWN": 404,
|
|
167
|
+
"SERVING_VERSION_UNKNOWN": 404,
|
|
168
|
+
"SERVING_PIN_DATASET": 404,
|
|
169
|
+
"SERVING_TABLE_SCOPE_MISMATCH": 404,
|
|
170
|
+
REQUEST_PATH_REFUSAL_CODE: 404,
|
|
171
|
+
REQUEST_METHOD_REFUSAL_CODE: 405,
|
|
172
|
+
REQUEST_TIMEOUT_REFUSAL_CODE: 408,
|
|
173
|
+
REQUEST_FAULT_REFUSAL_CODE: 500,
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
#: What a refusal this table does not name answers with.
|
|
177
|
+
DEFAULT_REFUSAL_STATUS = 400
|
|
178
|
+
|
|
179
|
+
#: Read in whole chunks so a body is bounded WHILE it is read and not only after. The chunk is small
|
|
180
|
+
#: enough that a caller cannot make this surface allocate more than the bound plus one chunk.
|
|
181
|
+
_READ_CHUNK_BYTES = 8 * 1024
|
|
182
|
+
|
|
183
|
+
_LOOPBACK_RULE = (
|
|
184
|
+
"the serving surface binds loopback only, so a bind address must be a loopback address "
|
|
185
|
+
"(use 127.0.0.1)"
|
|
186
|
+
)
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
class ServingHttpError(RuntimeError):
|
|
190
|
+
"""The transport is misconfigured, so no socket may be bound from it.
|
|
191
|
+
|
|
192
|
+
Raised only for an operator configuration fault — a bind address that is not loopback, or a
|
|
193
|
+
verify URL that is not a plain absolute https URL. It is never a rejected caller: a rejected
|
|
194
|
+
caller is a refusing :class:`serving.ServingResult` rendered with a status code, exactly as a
|
|
195
|
+
rejected key is a denying :class:`key_seam.KeyVerdict` rather than an exception.
|
|
196
|
+
"""
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def _plain_single_line(value: Any, limit: int) -> bool:
|
|
200
|
+
"""Whether ``value`` is one plain line within ``limit`` characters.
|
|
201
|
+
|
|
202
|
+
One line of delegation, on purpose. The rule is ``serving._one_plain_line``, which is
|
|
203
|
+
``deploy._PLAIN_LINE`` (``deploy.py:63``) with a length bound: a control character in a dataset
|
|
204
|
+
id or a pin would otherwise be free to reach a refusal sentence, and from there any stream a
|
|
205
|
+
caller of this library writes to. This module used to spell the same rule out by hand, and a
|
|
206
|
+
hand-spelled copy of a security rule is a copy that can be relaxed on one side only.
|
|
207
|
+
|
|
208
|
+
It is applied to every piece of a request this transport reads out of a URL **or a body**. The
|
|
209
|
+
body is not the safer of the two: it is the one bounded by nothing but
|
|
210
|
+
:data:`MAX_REQUEST_BYTES`.
|
|
211
|
+
"""
|
|
212
|
+
|
|
213
|
+
return serving._one_plain_line(value, limit)
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def _refused(dataset_id: str, pin: str, reason: str, code: str) -> serving.ServingResult:
|
|
217
|
+
"""Turn one request-shape refusal into the one envelope every serving caller already renders.
|
|
218
|
+
|
|
219
|
+
Both digests are null, and that is the honest answer: a refusal taken out here never resolved a
|
|
220
|
+
pin, so there is no version to name. The bracketed code is appended in the idiom every
|
|
221
|
+
``serving.py`` and ``serving_edge.py`` refusal already uses, so :data:`STATUS_BY_REFUSAL_CODE`
|
|
222
|
+
is a lookup rather than a guess about prose.
|
|
223
|
+
"""
|
|
224
|
+
|
|
225
|
+
return serving.ServingResult(
|
|
226
|
+
ok=False,
|
|
227
|
+
refusals=(f"{reason} [{code}]",),
|
|
228
|
+
status=serving.STATUS_REFUSED,
|
|
229
|
+
dataset_id=dataset_id,
|
|
230
|
+
pin=pin,
|
|
231
|
+
pin_kind=None,
|
|
232
|
+
version_id=None,
|
|
233
|
+
version_number=None,
|
|
234
|
+
version_digest=None,
|
|
235
|
+
table_digest=None,
|
|
236
|
+
payload=None,
|
|
237
|
+
)
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def _timed_out() -> serving.ServingResult:
|
|
241
|
+
"""Return the common response for a request that exceeds its deadline."""
|
|
242
|
+
|
|
243
|
+
return _refused(
|
|
244
|
+
"",
|
|
245
|
+
"",
|
|
246
|
+
"a read sends its whole request before this surface stops waiting for it",
|
|
247
|
+
REQUEST_TIMEOUT_REFUSAL_CODE,
|
|
248
|
+
)
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def _status_for(result: serving.ServingResult) -> int:
|
|
252
|
+
"""Map one envelope onto one status code, through the closed table and nothing else.
|
|
253
|
+
|
|
254
|
+
The first refusal carrying a code this table names decides. Order matters and is the library's:
|
|
255
|
+
``serving_edge`` refuses at one gate with one reason, so a credential refusal and a quota
|
|
256
|
+
refusal are each a single-reason envelope, and a multi-reason envelope can only come from inside
|
|
257
|
+
the read core, where every code is a 400 anyway.
|
|
258
|
+
"""
|
|
259
|
+
|
|
260
|
+
if result.ok:
|
|
261
|
+
return 200
|
|
262
|
+
for reason in result.refusals:
|
|
263
|
+
start = reason.rfind("[")
|
|
264
|
+
if start == -1 or not reason.endswith("]"):
|
|
265
|
+
continue
|
|
266
|
+
status = STATUS_BY_REFUSAL_CODE.get(reason[start + 1 : -1])
|
|
267
|
+
if status is not None:
|
|
268
|
+
return status
|
|
269
|
+
return DEFAULT_REFUSAL_STATUS
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def _match_template(path: str) -> tuple[str, str, str] | None:
|
|
273
|
+
"""Return the template ``path`` matches and the dataset id it carries, or nothing.
|
|
274
|
+
|
|
275
|
+
Matching is segment by segment against :data:`SERVING_PATHS`, and the dataset id is taken from
|
|
276
|
+
the path EXACTLY as it arrived: nothing is percent-decoded, so a path cannot smuggle a byte past
|
|
277
|
+
the plain-line check by spelling it as an escape. Every dataset id ``serving._IDENT`` admits is
|
|
278
|
+
already URL-safe, so no legitimate caller needs an escape.
|
|
279
|
+
"""
|
|
280
|
+
|
|
281
|
+
arrived = path.split("/")
|
|
282
|
+
for entry in SERVING_PATHS:
|
|
283
|
+
wanted = entry.template.split("/")
|
|
284
|
+
if len(arrived) != len(wanted):
|
|
285
|
+
continue
|
|
286
|
+
dataset_id: str | None = None
|
|
287
|
+
table_id: str | None = None
|
|
288
|
+
for got, expected in zip(arrived, wanted, strict=True):
|
|
289
|
+
if expected == "{dataset_id}":
|
|
290
|
+
dataset_id = got
|
|
291
|
+
elif expected == "{table_id}":
|
|
292
|
+
table_id = got
|
|
293
|
+
elif got != expected:
|
|
294
|
+
dataset_id = None
|
|
295
|
+
break
|
|
296
|
+
if dataset_id is not None and table_id is not None:
|
|
297
|
+
return entry.template, dataset_id, table_id
|
|
298
|
+
return None
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
def _v3_coordinates_from_path(path: str) -> tuple[str, str | None] | None:
|
|
302
|
+
"""Return every V3 coordinate a malformed or unsupported path still names.
|
|
303
|
+
|
|
304
|
+
A request need not match a serving template to name its Dataset parent. Every nonempty parent
|
|
305
|
+
following ``/v3/datasets/`` is retained. A child exists only after that parent's literal
|
|
306
|
+
``tables`` segment; an unknown segment is not silently reclassified as a table. A suffix after
|
|
307
|
+
a child is still an unsupported path that must not discard either requested coordinate.
|
|
308
|
+
"""
|
|
309
|
+
|
|
310
|
+
segments = path.split("/")
|
|
311
|
+
if len(segments) < 4 or segments[:3] != ["", "v3", "datasets"] or not segments[3]:
|
|
312
|
+
return None
|
|
313
|
+
table_id = (
|
|
314
|
+
segments[5] if len(segments) > 5 and segments[4] == "tables" and segments[5] else None
|
|
315
|
+
)
|
|
316
|
+
return segments[3], table_id
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
def _request_from_body(parsed: Any) -> dict[str, Any]:
|
|
320
|
+
"""Map one parsed body onto the library's arguments, or refuse the body's shape.
|
|
321
|
+
|
|
322
|
+
This is the only place a wire name becomes a library name, and the only rename it performs is
|
|
323
|
+
``filter`` to ``predicates``. Nothing here validates a column, an operator, a value or a window:
|
|
324
|
+
those all belong to ``serving.py``, which owns their grammar and their refusal wording.
|
|
325
|
+
|
|
326
|
+
**The pin is the one exception, and it is the transport's own bound rather than a second
|
|
327
|
+
grammar.** ``serving.resolve_pin`` still decides what a pin MEANS; this checks only that the
|
|
328
|
+
thing arriving is one plain line inside :data:`MAX_PIN_CHARS`, exactly as the discovery path
|
|
329
|
+
already checks the pin it reads out of the address. Without it the only bound on this field is
|
|
330
|
+
:data:`MAX_REQUEST_BYTES`, so the body could carry a pin of some sixty-four thousand arbitrary
|
|
331
|
+
characters — 256 times this transport's own declared bound — and control characters among them.
|
|
332
|
+
|
|
333
|
+
Raises:
|
|
334
|
+
ValueError: carrying the sentence and code the caller should see.
|
|
335
|
+
"""
|
|
336
|
+
|
|
337
|
+
if parsed is None:
|
|
338
|
+
parsed = {}
|
|
339
|
+
if not isinstance(parsed, dict):
|
|
340
|
+
raise ValueError("a read's body is a JSON object of named request fields")
|
|
341
|
+
unknown = sorted(set(parsed) - ROWS_BODY_KEYS)
|
|
342
|
+
if unknown:
|
|
343
|
+
named = ", ".join(unknown)
|
|
344
|
+
raise ValueError(
|
|
345
|
+
f"a read names pin, filter, offset and limit, and this one also names {named}"
|
|
346
|
+
)
|
|
347
|
+
pin = parsed.get("pin", serving.PIN_CURRENT)
|
|
348
|
+
if not _plain_single_line(pin, MAX_PIN_CHARS):
|
|
349
|
+
# The offending value is not quoted back. It is the value that failed the plain-line rule,
|
|
350
|
+
# which is the value that must not reach a rendered line.
|
|
351
|
+
raise ValueError(
|
|
352
|
+
f"a version is asked for as one plain line of at most {MAX_PIN_CHARS} characters"
|
|
353
|
+
)
|
|
354
|
+
return {
|
|
355
|
+
"pin": pin,
|
|
356
|
+
"predicates": parsed.get("filter"),
|
|
357
|
+
"offset": parsed.get("offset", 0),
|
|
358
|
+
"limit": parsed.get("limit", serving.MAX_SERVED_ROWS),
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
class _ServingServer(HTTPServer):
|
|
363
|
+
"""A single-threaded stdlib server holding the one index, ledger and verify URL it serves.
|
|
364
|
+
|
|
365
|
+
Deliberately :class:`http.server.HTTPServer` and not ``ThreadingHTTPServer``: see the module
|
|
366
|
+
docstring's paragraph on the in-flight cap. That choice is exactly why
|
|
367
|
+
:data:`REQUEST_TIMEOUT_SECONDS` is not optional — with one handler, a request with no deadline
|
|
368
|
+
is a surface with no deadline.
|
|
369
|
+
"""
|
|
370
|
+
|
|
371
|
+
def __init__(
|
|
372
|
+
self,
|
|
373
|
+
address: tuple[str, int],
|
|
374
|
+
handler: type,
|
|
375
|
+
*,
|
|
376
|
+
index: serving.VersionIndex,
|
|
377
|
+
index_path: Path | None,
|
|
378
|
+
ledger: CapLedger,
|
|
379
|
+
verify_url: str,
|
|
380
|
+
transport: key_seam.VerifyTransport | None,
|
|
381
|
+
timeout: float,
|
|
382
|
+
) -> None:
|
|
383
|
+
super().__init__(address, handler)
|
|
384
|
+
# The index as it was when the surface was checked and bound. It is the fallback for a
|
|
385
|
+
# library caller that supplied no path; when there is a path, every read re-reads it, so
|
|
386
|
+
# ``current`` means current rather than current-as-of-start-up.
|
|
387
|
+
self.serving_index = index
|
|
388
|
+
self.serving_index_path = index_path
|
|
389
|
+
self.serving_timeout = timeout
|
|
390
|
+
self.serving_ledger = ledger
|
|
391
|
+
self.serving_verify_url = verify_url
|
|
392
|
+
# Held and never inspected. The one thing this object does is travel to
|
|
393
|
+
# ``serving_edge.serve_read``, which hands it to ``key_seam`` (P-02).
|
|
394
|
+
self.serving_transport = transport
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
class _DeadlineReader:
|
|
398
|
+
"""Every byte of one request read under ONE deadline, taken once and shrinking on every pass.
|
|
399
|
+
|
|
400
|
+
**A per-recv socket timeout is not a deadline, and the difference is the whole point of this
|
|
401
|
+
class.** ``socketserver.StreamRequestHandler.setup`` calls ``settimeout`` once with
|
|
402
|
+
:attr:`_ServingHandler.timeout`, and the number it passes bounds each individual ``recv`` rather
|
|
403
|
+
than the request. A caller that sends one byte every ``timeout - 1`` seconds renews that bound
|
|
404
|
+
forever and holds this single-threaded surface for as long as it cares to keep dripping —
|
|
405
|
+
through the request line, through the headers, and through the body alike.
|
|
406
|
+
|
|
407
|
+
The deadline lives in front of ``rfile``, where every segment of the request reads: the
|
|
408
|
+
request line ``BaseHTTPRequestHandler.handle_one_request`` reads, the headers
|
|
409
|
+
``http.client.parse_headers`` reads, and the body :meth:`_ServingHandler._body` reads. One
|
|
410
|
+
absolute instant is taken when the connection is accepted; before every underlying read the
|
|
411
|
+
time still left is computed and put on the socket, and when nothing is left the read raises
|
|
412
|
+
``TimeoutError`` instead of waiting again.
|
|
413
|
+
|
|
414
|
+
Two mechanics make that bound real rather than nominal, and both are about never blocking inside
|
|
415
|
+
one call for longer than the deadline allows:
|
|
416
|
+
|
|
417
|
+
* :meth:`read` goes through ``BufferedReader.read1``, which returns what one underlying read
|
|
418
|
+
returned rather than looping until it has the whole amount asked for.
|
|
419
|
+
* :meth:`readline` goes through ``peek``, which does at most one underlying read, and then takes
|
|
420
|
+
the line out of the buffer. ``BufferedReader.readline`` would instead sit inside a single call
|
|
421
|
+
until a newline arrives, renewing the socket timeout on every packet.
|
|
422
|
+
|
|
423
|
+
Both raise ``TimeoutError`` and set :attr:`expired`, which is how the handler knows to answer
|
|
424
|
+
the 408 envelope when the deadline expires before handler code runs.
|
|
425
|
+
|
|
426
|
+
The shape is ``coordinator._receive_exact``'s (``coordinator.py:962-990``), which is this
|
|
427
|
+
repository's existing answer to the same problem on its own transport: one absolute instant, the
|
|
428
|
+
time still left recomputed before every underlying read, and ``TimeoutError`` when there is
|
|
429
|
+
none. It is restated over a file object rather than reused because the reads being bounded are
|
|
430
|
+
the standard library's, and ``BaseHTTPRequestHandler`` makes them through ``rfile``.
|
|
431
|
+
"""
|
|
432
|
+
|
|
433
|
+
def __init__(self, source: Any, connection: Any, seconds: float) -> None:
|
|
434
|
+
self._source = source
|
|
435
|
+
self._connection = connection
|
|
436
|
+
self._seconds = seconds
|
|
437
|
+
self._deadline = time.monotonic() + seconds
|
|
438
|
+
#: Whether this request ran out of time. Read by the handler, never by a caller.
|
|
439
|
+
self.expired = False
|
|
440
|
+
|
|
441
|
+
@property
|
|
442
|
+
def seconds(self) -> float:
|
|
443
|
+
"""The whole deadline, for the writer that must not inherit what a read had left."""
|
|
444
|
+
|
|
445
|
+
return self._seconds
|
|
446
|
+
|
|
447
|
+
def _arm(self) -> None:
|
|
448
|
+
"""Put the time still left on the socket, or refuse to wait at all."""
|
|
449
|
+
|
|
450
|
+
left = self._deadline - time.monotonic()
|
|
451
|
+
if left <= 0:
|
|
452
|
+
self.expired = True
|
|
453
|
+
raise TimeoutError("this read did not arrive in time")
|
|
454
|
+
self._connection.settimeout(left)
|
|
455
|
+
|
|
456
|
+
def read(self, size: int = -1) -> bytes:
|
|
457
|
+
"""Read up to ``size`` bytes, or until end of stream, inside the deadline."""
|
|
458
|
+
|
|
459
|
+
out = bytearray()
|
|
460
|
+
while size < 0 or len(out) < size:
|
|
461
|
+
want = _READ_CHUNK_BYTES if size < 0 else min(size - len(out), _READ_CHUNK_BYTES)
|
|
462
|
+
self._arm()
|
|
463
|
+
try:
|
|
464
|
+
chunk = self._source.read1(want)
|
|
465
|
+
except TimeoutError:
|
|
466
|
+
self.expired = True
|
|
467
|
+
raise
|
|
468
|
+
if not chunk:
|
|
469
|
+
break
|
|
470
|
+
out += chunk
|
|
471
|
+
return bytes(out)
|
|
472
|
+
|
|
473
|
+
def readline(self, limit: int = -1) -> bytes:
|
|
474
|
+
"""Read one line, at most ``limit`` bytes of it, inside the deadline."""
|
|
475
|
+
|
|
476
|
+
out = bytearray()
|
|
477
|
+
while limit < 0 or len(out) < limit:
|
|
478
|
+
self._arm()
|
|
479
|
+
try:
|
|
480
|
+
available = self._source.peek(1)
|
|
481
|
+
except TimeoutError:
|
|
482
|
+
self.expired = True
|
|
483
|
+
raise
|
|
484
|
+
if not available:
|
|
485
|
+
break
|
|
486
|
+
newline = available.find(b"\n")
|
|
487
|
+
take = len(available) if newline < 0 else newline + 1
|
|
488
|
+
if limit >= 0:
|
|
489
|
+
take = min(take, limit - len(out))
|
|
490
|
+
# Out of the buffer ``peek`` just filled, so this read never touches the socket.
|
|
491
|
+
out += self._source.read(take)
|
|
492
|
+
if newline >= 0 and take == newline + 1:
|
|
493
|
+
break
|
|
494
|
+
return bytes(out)
|
|
495
|
+
|
|
496
|
+
def close(self) -> None:
|
|
497
|
+
"""Close the stream this wraps: ``socketserver.StreamRequestHandler.finish`` calls it."""
|
|
498
|
+
|
|
499
|
+
self._source.close()
|
|
500
|
+
|
|
501
|
+
@property
|
|
502
|
+
def closed(self) -> bool:
|
|
503
|
+
return bool(self._source.closed)
|
|
504
|
+
|
|
505
|
+
|
|
506
|
+
class _ServingHandler(BaseHTTPRequestHandler):
|
|
507
|
+
"""One request, parsed into the closed vocabulary and handed to the edge.
|
|
508
|
+
|
|
509
|
+
Nothing in this class reads a run directory, opens a sealed version, or looks at a column. It
|
|
510
|
+
turns bytes into arguments and an envelope into a status code.
|
|
511
|
+
"""
|
|
512
|
+
|
|
513
|
+
#: The class default, used when this handler is constructed without a server that carries one.
|
|
514
|
+
#: ``socketserver.StreamRequestHandler`` reads this attribute in ``setup`` and calls
|
|
515
|
+
#: ``settimeout`` with it; the stdlib's own default is ``None``, which is no deadline at all.
|
|
516
|
+
timeout: float = REQUEST_TIMEOUT_SECONDS
|
|
517
|
+
|
|
518
|
+
#: Whether this request has been answered. :meth:`_respond` is the only writer, and
|
|
519
|
+
#: :meth:`handle_one_request` reads it to decide whether a missed deadline still owes an
|
|
520
|
+
#: envelope. One answer per request either way: the connection is closed after one.
|
|
521
|
+
_answered: bool = False
|
|
522
|
+
|
|
523
|
+
#: The one deadline every read of this request draws down. :meth:`setup` installs it.
|
|
524
|
+
serving_deadline: _DeadlineReader
|
|
525
|
+
|
|
526
|
+
def setup(self) -> None:
|
|
527
|
+
"""Put ONE deadline in front of the request before a byte of it is read.
|
|
528
|
+
|
|
529
|
+
The number is the server's, so one value configures the surface and a test can drive a short
|
|
530
|
+
one. It is applied twice over, and the second is the one that binds: ``self.timeout`` gives
|
|
531
|
+
the stdlib its per-recv socket timeout, and :class:`_DeadlineReader` then wraps ``rfile`` so
|
|
532
|
+
the request line, the headers and the body all draw down a single shrinking deadline. A drip
|
|
533
|
+
renews the first; nothing renews the second.
|
|
534
|
+
"""
|
|
535
|
+
|
|
536
|
+
self.timeout = getattr(self.server, "serving_timeout", REQUEST_TIMEOUT_SECONDS)
|
|
537
|
+
super().setup()
|
|
538
|
+
# ``send_response_only`` reads this, and a deadline missed before the request line arrives
|
|
539
|
+
# means ``parse_request`` never set it. Its own default would suppress the 408's status
|
|
540
|
+
# line; a request that does parse overwrites this on the next statement it runs.
|
|
541
|
+
self.request_version = self.protocol_version
|
|
542
|
+
self.serving_deadline = _DeadlineReader(self.rfile, self.connection, float(self.timeout))
|
|
543
|
+
self.rfile = self.serving_deadline # type: ignore[assignment]
|
|
544
|
+
|
|
545
|
+
def handle_one_request(self) -> None:
|
|
546
|
+
"""Run the stdlib's request loop, and answer a deadline it swallowed.
|
|
547
|
+
|
|
548
|
+
``BaseHTTPRequestHandler.handle_one_request`` catches ``TimeoutError`` from the request line
|
|
549
|
+
and header reads, logs it and closes the connection without answering. Closing is the half
|
|
550
|
+
that protects the surface and it already happened; this adds the half a caller sees, so a
|
|
551
|
+
request that ran out of time in any segment gets the same envelope and the same 408 as one
|
|
552
|
+
that ran out of time in its body.
|
|
553
|
+
|
|
554
|
+
A caller that abandoned its request may well have abandoned the socket too, so the write is
|
|
555
|
+
allowed to fail: an unreachable caller is not an event this surface reports. Letting the
|
|
556
|
+
``OSError`` out would reach ``socketserver.BaseServer.handle_error``, which prints a
|
|
557
|
+
traceback to stderr — the one thing :meth:`log_message` exists to prevent.
|
|
558
|
+
"""
|
|
559
|
+
|
|
560
|
+
super().handle_one_request()
|
|
561
|
+
if self.serving_deadline.expired and not self._answered:
|
|
562
|
+
result = _timed_out()
|
|
563
|
+
try:
|
|
564
|
+
self._respond(_status_for(result), result.canonical_bytes())
|
|
565
|
+
except OSError:
|
|
566
|
+
self.close_connection = True
|
|
567
|
+
|
|
568
|
+
def log_message(self, *_args: object) -> None: # silence default stderr access logging
|
|
569
|
+
"""Say nothing.
|
|
570
|
+
|
|
571
|
+
``viewer.py:138`` silences the same log for tidiness; here it is security-load-bearing. A
|
|
572
|
+
default access log line carries the request line, and every request to this surface is
|
|
573
|
+
credential-bearing — an operator who later adds a log line must decide, deliberately, what
|
|
574
|
+
goes in it. Nothing this handler writes anywhere contains the presented key.
|
|
575
|
+
"""
|
|
576
|
+
|
|
577
|
+
return
|
|
578
|
+
|
|
579
|
+
# ---- the pieces of the request ---------------------------------------------------------
|
|
580
|
+
|
|
581
|
+
@property
|
|
582
|
+
def _server(self) -> _ServingServer:
|
|
583
|
+
return self.server # type: ignore[return-value]
|
|
584
|
+
|
|
585
|
+
def _key(self) -> tuple[str | None, str | None]:
|
|
586
|
+
"""Return the presented credential, or the reason there is not exactly one.
|
|
587
|
+
|
|
588
|
+
Read from :data:`key_seam.API_KEY_HEADER` and from nowhere else — never a query string,
|
|
589
|
+
never a path segment — for the reason recorded at ``key_seam.py:76-78``: those are logged by
|
|
590
|
+
every proxy, load balancer and browser history on the route. Two headers is not one
|
|
591
|
+
credential, so it refuses rather than picking one.
|
|
592
|
+
"""
|
|
593
|
+
|
|
594
|
+
presented = self.headers.get_all(key_seam.API_KEY_HEADER) or []
|
|
595
|
+
if len(presented) > 1:
|
|
596
|
+
return None, "a read presents one credential, and this one presented several"
|
|
597
|
+
if not presented or not presented[0]:
|
|
598
|
+
return None, None
|
|
599
|
+
return presented[0], None
|
|
600
|
+
|
|
601
|
+
def _body(self) -> bytes:
|
|
602
|
+
"""Read the request body, bounded before the read, during it, and in time.
|
|
603
|
+
|
|
604
|
+
The byte bound and the deadline are two different bounds and neither implies the other. A
|
|
605
|
+
body that declares 200 bytes and sends 199 of them is inside every byte bound this module
|
|
606
|
+
has and would hold the surface forever. The byte bound is this method's, checked twice —
|
|
607
|
+
against the declared length before a byte is read and against what arrives while it is being
|
|
608
|
+
read. The deadline is NOT this method's: it belongs to :class:`_DeadlineReader`, which
|
|
609
|
+
``setup`` put in front of every read this request makes, so the body draws down the same
|
|
610
|
+
shrinking deadline the request line and the headers already drew down. This method reads,
|
|
611
|
+
counts, and lets the timeout out.
|
|
612
|
+
|
|
613
|
+
Raises:
|
|
614
|
+
ValueError: carrying the sentence the caller should see.
|
|
615
|
+
TimeoutError: when the body does not arrive inside the deadline. The handler turns it
|
|
616
|
+
into the one envelope with a 408, because an unfinished request is the caller's
|
|
617
|
+
fault and is not an internal fault.
|
|
618
|
+
"""
|
|
619
|
+
|
|
620
|
+
if self.headers.get("Transfer-Encoding"):
|
|
621
|
+
# A chunked body has no declared length, so there is no bound to check before reading
|
|
622
|
+
# one. This surface takes a filter, and a filter has a length.
|
|
623
|
+
raise ValueError("a read declares the length of its body")
|
|
624
|
+
declared = self.headers.get("Content-Length")
|
|
625
|
+
if declared is None:
|
|
626
|
+
return b""
|
|
627
|
+
if not declared.isascii() or not declared.isdigit():
|
|
628
|
+
raise ValueError("a read declares the length of its body as a whole number of bytes")
|
|
629
|
+
length = int(declared)
|
|
630
|
+
if length > MAX_REQUEST_BYTES:
|
|
631
|
+
# Refused BEFORE a byte is read: the point of the bound is that an oversized body never
|
|
632
|
+
# reaches this process's memory, so checking it afterwards would be checking nothing.
|
|
633
|
+
raise ValueError(
|
|
634
|
+
f"a read's body is at most {MAX_REQUEST_BYTES} bytes, and this one says it is more"
|
|
635
|
+
)
|
|
636
|
+
chunks: list[bytes] = []
|
|
637
|
+
remaining = length
|
|
638
|
+
read = 0
|
|
639
|
+
while remaining > 0:
|
|
640
|
+
# The reader in front of this call carries the deadline and raises ``TimeoutError``
|
|
641
|
+
# when it is gone. The loop is here for the byte bound, which needs to be checked on
|
|
642
|
+
# what arrives and not only on what was declared.
|
|
643
|
+
chunk = self.rfile.read(min(remaining, _READ_CHUNK_BYTES))
|
|
644
|
+
if not chunk:
|
|
645
|
+
break
|
|
646
|
+
read += len(chunk)
|
|
647
|
+
if read > MAX_REQUEST_BYTES:
|
|
648
|
+
# Bounded while reading too, so a declared length that lies is bounded by the
|
|
649
|
+
# same number the declaration was checked against.
|
|
650
|
+
raise ValueError(f"a read's body is at most {MAX_REQUEST_BYTES} bytes")
|
|
651
|
+
chunks.append(chunk)
|
|
652
|
+
remaining -= len(chunk)
|
|
653
|
+
return b"".join(chunks)
|
|
654
|
+
|
|
655
|
+
# ---- the answer --------------------------------------------------------------------------
|
|
656
|
+
|
|
657
|
+
def _answer_unmapped(self) -> serving.ServingResult:
|
|
658
|
+
"""Turn this request into one envelope. Every path out of here is an envelope."""
|
|
659
|
+
|
|
660
|
+
path, _, query = self.path.partition("?")
|
|
661
|
+
matched = _match_template(path)
|
|
662
|
+
if matched is None:
|
|
663
|
+
return _refused(
|
|
664
|
+
"",
|
|
665
|
+
"",
|
|
666
|
+
"this surface answers a dataset's description and a dataset's rows, and nothing "
|
|
667
|
+
"else",
|
|
668
|
+
REQUEST_PATH_REFUSAL_CODE,
|
|
669
|
+
)
|
|
670
|
+
template, dataset_id, table_id = matched
|
|
671
|
+
if not _plain_single_line(dataset_id, MAX_DATASET_ID_CHARS) or not _plain_single_line(
|
|
672
|
+
table_id, MAX_DATASET_ID_CHARS
|
|
673
|
+
):
|
|
674
|
+
return _refused(
|
|
675
|
+
"",
|
|
676
|
+
"",
|
|
677
|
+
"a dataset is named by one plain line of at most "
|
|
678
|
+
f"{MAX_DATASET_ID_CHARS} characters",
|
|
679
|
+
REQUEST_DATASET_REFUSAL_CODE,
|
|
680
|
+
)
|
|
681
|
+
entry = next(
|
|
682
|
+
(
|
|
683
|
+
item
|
|
684
|
+
for item in SERVING_PATHS
|
|
685
|
+
if item.template == template and item.method == self.command
|
|
686
|
+
),
|
|
687
|
+
None,
|
|
688
|
+
)
|
|
689
|
+
if entry is None:
|
|
690
|
+
return _refused(
|
|
691
|
+
dataset_id,
|
|
692
|
+
"",
|
|
693
|
+
"this path answers a different method than the one this read used",
|
|
694
|
+
REQUEST_METHOD_REFUSAL_CODE,
|
|
695
|
+
)
|
|
696
|
+
|
|
697
|
+
if entry.shape == "describe":
|
|
698
|
+
request = self._discovery_request(query)
|
|
699
|
+
else:
|
|
700
|
+
if query:
|
|
701
|
+
return _refused(
|
|
702
|
+
dataset_id,
|
|
703
|
+
"",
|
|
704
|
+
"a rows read carries its whole request in its body, never in the address",
|
|
705
|
+
REQUEST_QUERY_REFUSAL_CODE,
|
|
706
|
+
)
|
|
707
|
+
request = self._rows_request()
|
|
708
|
+
if isinstance(request, serving.ServingResult):
|
|
709
|
+
return request
|
|
710
|
+
|
|
711
|
+
key, refusal = self._key()
|
|
712
|
+
if refusal is not None:
|
|
713
|
+
return _refused(dataset_id, str(request["pin"]), refusal, REQUEST_FIELD_REFUSAL_CODE)
|
|
714
|
+
|
|
715
|
+
# The whole request, handed over. The key check and the quota check are BOTH on the other
|
|
716
|
+
# side of this call, in the order ``serving_edge`` proves; nothing above this line reads
|
|
717
|
+
# anything, and a refused read opens no run directory.
|
|
718
|
+
return serving_edge.serve_read(
|
|
719
|
+
self._server.serving_index,
|
|
720
|
+
table_id,
|
|
721
|
+
request.pop("pin"),
|
|
722
|
+
shape=entry.shape,
|
|
723
|
+
index_path=self._server.serving_index_path,
|
|
724
|
+
ledger=self._server.serving_ledger,
|
|
725
|
+
now=datetime.now(UTC),
|
|
726
|
+
key=key,
|
|
727
|
+
verify_url=self._server.serving_verify_url,
|
|
728
|
+
transport=self._server.serving_transport,
|
|
729
|
+
parent_dataset_id=dataset_id,
|
|
730
|
+
api_version="v3",
|
|
731
|
+
**request,
|
|
732
|
+
)
|
|
733
|
+
|
|
734
|
+
def _answer(self) -> serving.ServingResult:
|
|
735
|
+
"""Answer once, then attach V3 Dataset/Table coordinates to every mapped refusal."""
|
|
736
|
+
|
|
737
|
+
result = self._answer_unmapped()
|
|
738
|
+
path, _, _query = self.path.partition("?")
|
|
739
|
+
coordinates = _v3_coordinates_from_path(path)
|
|
740
|
+
if coordinates is None:
|
|
741
|
+
return result
|
|
742
|
+
dataset_id, table_id = coordinates
|
|
743
|
+
return serving_edge.v3_coordinates(result, dataset_id, table_id)
|
|
744
|
+
|
|
745
|
+
def _discovery_request(self, query: str) -> dict[str, Any] | serving.ServingResult:
|
|
746
|
+
"""Read the discovery request out of the address: a pin, or Current."""
|
|
747
|
+
|
|
748
|
+
pairs = parse_qsl(query, keep_blank_values=True)
|
|
749
|
+
names = [name for name, _ in pairs]
|
|
750
|
+
if set(names) - DISCOVERY_QUERY_KEYS or len(names) != len(set(names)):
|
|
751
|
+
return _refused(
|
|
752
|
+
"",
|
|
753
|
+
"",
|
|
754
|
+
"a description is asked for with one optional pin and nothing else",
|
|
755
|
+
REQUEST_QUERY_REFUSAL_CODE,
|
|
756
|
+
)
|
|
757
|
+
pin = dict(pairs).get("pin", serving.PIN_CURRENT)
|
|
758
|
+
if not _plain_single_line(pin, MAX_PIN_CHARS):
|
|
759
|
+
return _refused(
|
|
760
|
+
"",
|
|
761
|
+
"",
|
|
762
|
+
f"a version is asked for as one plain line of at most {MAX_PIN_CHARS} characters",
|
|
763
|
+
REQUEST_QUERY_REFUSAL_CODE,
|
|
764
|
+
)
|
|
765
|
+
return {"pin": pin}
|
|
766
|
+
|
|
767
|
+
def _rows_request(self) -> dict[str, Any] | serving.ServingResult:
|
|
768
|
+
"""Read the rows request out of the body: strict JSON, closed fields, nothing inferred."""
|
|
769
|
+
|
|
770
|
+
try:
|
|
771
|
+
raw = self._body()
|
|
772
|
+
except TimeoutError:
|
|
773
|
+
return _timed_out()
|
|
774
|
+
except ValueError as error:
|
|
775
|
+
return _refused("", "", str(error), REQUEST_SIZE_REFUSAL_CODE)
|
|
776
|
+
if not raw.strip():
|
|
777
|
+
parsed: Any = {}
|
|
778
|
+
else:
|
|
779
|
+
try:
|
|
780
|
+
# Strict on purpose, and the same parser ``key_seam`` verifies a verdict with:
|
|
781
|
+
# duplicate keys, byte-order marks, non-UTF-8 bytes and non-integer numbers all
|
|
782
|
+
# refuse, so a body two parsers would read differently is never acted on.
|
|
783
|
+
parsed = canonical.parse_json(raw)
|
|
784
|
+
except canonical.CanonicalJSONError:
|
|
785
|
+
return _refused(
|
|
786
|
+
"",
|
|
787
|
+
"",
|
|
788
|
+
"a read's body is strict JSON, and this one could not be read as any",
|
|
789
|
+
REQUEST_BODY_REFUSAL_CODE,
|
|
790
|
+
)
|
|
791
|
+
try:
|
|
792
|
+
return _request_from_body(parsed)
|
|
793
|
+
except ValueError as error:
|
|
794
|
+
return _refused("", "", str(error), REQUEST_FIELD_REFUSAL_CODE)
|
|
795
|
+
|
|
796
|
+
# ---- the wire ----------------------------------------------------------------------------
|
|
797
|
+
|
|
798
|
+
def _respond(self, status: int, body: bytes) -> None:
|
|
799
|
+
"""Write one response: a status line, two headers, and the envelope.
|
|
800
|
+
|
|
801
|
+
The header set is closed and deterministic. ``send_response_only`` is used rather than
|
|
802
|
+
``send_response`` so no ``Server`` and no ``Date`` header is emitted: the first would
|
|
803
|
+
publish this deployment's Python version to anyone who asks, and the second would make the
|
|
804
|
+
response bytes time-varying. No header echoes anything the caller sent, and there is no
|
|
805
|
+
cookie on this surface because there is no session on it — the credential is per request.
|
|
806
|
+
"""
|
|
807
|
+
|
|
808
|
+
self._answered = True
|
|
809
|
+
# The deadline bounds the REQUEST, and this is the answer to it. Whatever few milliseconds
|
|
810
|
+
# the last read had left are not the writer's budget, so the socket goes back to the whole
|
|
811
|
+
# number before a byte of the envelope is written.
|
|
812
|
+
self.connection.settimeout(self.serving_deadline.seconds)
|
|
813
|
+
self.close_connection = True
|
|
814
|
+
self.send_response_only(status)
|
|
815
|
+
self.send_header("Content-Type", "application/json")
|
|
816
|
+
self.send_header("Content-Length", str(len(body)))
|
|
817
|
+
self.end_headers()
|
|
818
|
+
if getattr(self, "command", None) != "HEAD":
|
|
819
|
+
self.wfile.write(body)
|
|
820
|
+
|
|
821
|
+
def send_error(self, code: Any, message: Any = None, explain: Any = None) -> None:
|
|
822
|
+
"""Answer the stdlib's own failure paths with the same envelope, never an HTML page.
|
|
823
|
+
|
|
824
|
+
``BaseHTTPRequestHandler`` calls this for a malformed request line, an over-long header, an
|
|
825
|
+
unsupported HTTP version and an unimplemented method, and its default body is an HTML page
|
|
826
|
+
that quotes the caller's own bytes back at them. Both halves of that are refused here: the
|
|
827
|
+
body is the one envelope, and ``message`` and ``explain`` are dropped unread, because a
|
|
828
|
+
response is not a place to echo what arrived.
|
|
829
|
+
"""
|
|
830
|
+
|
|
831
|
+
status = int(code)
|
|
832
|
+
result = _refused(
|
|
833
|
+
"",
|
|
834
|
+
"",
|
|
835
|
+
"this request was not in a shape this surface answers",
|
|
836
|
+
REQUEST_SHAPE_REFUSAL_CODE,
|
|
837
|
+
)
|
|
838
|
+
path = getattr(self, "path", "")
|
|
839
|
+
coordinates = (
|
|
840
|
+
_v3_coordinates_from_path(path.partition("?")[0]) if isinstance(path, str) else None
|
|
841
|
+
)
|
|
842
|
+
if coordinates is not None:
|
|
843
|
+
dataset_id, table_id = coordinates
|
|
844
|
+
result = serving_edge.v3_coordinates(result, dataset_id, table_id)
|
|
845
|
+
self._respond(status, result.canonical_bytes())
|
|
846
|
+
|
|
847
|
+
def _handle(self) -> None:
|
|
848
|
+
try:
|
|
849
|
+
result = self._answer()
|
|
850
|
+
except Exception:
|
|
851
|
+
# One broad boundary catch, in ``key_seam.verify_api_key``'s idiom. Nothing about the
|
|
852
|
+
# exception reaches the caller: its message, its repr and its traceback could all carry
|
|
853
|
+
# a path, a column or the credential that was in scope when it was raised.
|
|
854
|
+
result = _refused(
|
|
855
|
+
"",
|
|
856
|
+
"",
|
|
857
|
+
"this read could not be completed",
|
|
858
|
+
REQUEST_FAULT_REFUSAL_CODE,
|
|
859
|
+
)
|
|
860
|
+
path = getattr(self, "path", "")
|
|
861
|
+
coordinates = (
|
|
862
|
+
_v3_coordinates_from_path(path.partition("?")[0]) if isinstance(path, str) else None
|
|
863
|
+
)
|
|
864
|
+
if coordinates is not None:
|
|
865
|
+
dataset_id, table_id = coordinates
|
|
866
|
+
result = serving_edge.v3_coordinates(result, dataset_id, table_id)
|
|
867
|
+
self._respond(_status_for(result), result.canonical_bytes())
|
|
868
|
+
|
|
869
|
+
do_GET = _handle
|
|
870
|
+
do_POST = _handle
|
|
871
|
+
do_HEAD = _handle
|
|
872
|
+
do_PUT = _handle
|
|
873
|
+
do_DELETE = _handle
|
|
874
|
+
do_PATCH = _handle
|
|
875
|
+
do_OPTIONS = _handle
|
|
876
|
+
do_TRACE = _handle
|
|
877
|
+
|
|
878
|
+
|
|
879
|
+
def _require_loopback(host: str) -> None:
|
|
880
|
+
"""Refuse a bind address that is not loopback, before any socket exists.
|
|
881
|
+
|
|
882
|
+
The rule is ``cli._require_loopback_host``'s and the wording carries the same word, but the
|
|
883
|
+
check lives here as well because this is a library entry point: a guard that trusts its caller
|
|
884
|
+
to have already checked is not a guard, and ``build_server`` is reachable without a command
|
|
885
|
+
line.
|
|
886
|
+
"""
|
|
887
|
+
|
|
888
|
+
try:
|
|
889
|
+
address = ipaddress.ip_address(host)
|
|
890
|
+
except ValueError as error:
|
|
891
|
+
raise ServingHttpError(f"{host!r} is not an IP address; {_LOOPBACK_RULE}") from error
|
|
892
|
+
if not address.is_loopback:
|
|
893
|
+
raise ServingHttpError(f"{host} is not a loopback address; {_LOOPBACK_RULE}")
|
|
894
|
+
|
|
895
|
+
|
|
896
|
+
def build_server(
|
|
897
|
+
*,
|
|
898
|
+
index: serving.VersionIndex,
|
|
899
|
+
index_path: Path | None = None,
|
|
900
|
+
ledger: CapLedger,
|
|
901
|
+
verify_url: str,
|
|
902
|
+
host: str = "127.0.0.1",
|
|
903
|
+
port: int = 0,
|
|
904
|
+
transport: key_seam.VerifyTransport | None = None,
|
|
905
|
+
timeout: float = REQUEST_TIMEOUT_SECONDS,
|
|
906
|
+
) -> HTTPServer:
|
|
907
|
+
"""Bind the serving surface and hand back the bound server, without serving anything yet.
|
|
908
|
+
|
|
909
|
+
The entry point is split in two — this, and :func:`serve_forever` — for one reason: a caller
|
|
910
|
+
that asked for an ephemeral port has to be able to learn which port it got BEFORE the loop
|
|
911
|
+
starts, and a test that cannot learn the port cannot drive a real socket. ``viewer.py`` gets
|
|
912
|
+
away without the split because it prints its URL and opens a browser; this surface is called by
|
|
913
|
+
tests and by a CLI, and both need the address back. ``server.server_address[1]`` reports the
|
|
914
|
+
actually bound port, so ``port=0`` is usable.
|
|
915
|
+
|
|
916
|
+
``verify_url`` is REQUIRED and is checked through ``key_seam``'s own https policy rather than a
|
|
917
|
+
second one, so a surface that cannot verify a key cannot be started: there is no unauthenticated
|
|
918
|
+
mode of this server and no argument that produces one.
|
|
919
|
+
|
|
920
|
+
``index_path`` is the document ``index`` was parsed from, and giving it is what makes
|
|
921
|
+
``current`` mean current: with a path, every read re-reads and re-parses the file, so a move of
|
|
922
|
+
Current is seen by a process that was already running. Without one, this server answers the
|
|
923
|
+
snapshot it was handed for its whole life — correct for a caller that built an index in memory,
|
|
924
|
+
and wrong for a long-lived surface over a document an operator maintains, which is why
|
|
925
|
+
``mr-data serve`` always passes it.
|
|
926
|
+
|
|
927
|
+
``transport`` is the P-02 seam. It is handed to :func:`serving_edge.serve_read` and is never
|
|
928
|
+
inspected, wrapped or branched on here; ``key_seam`` still computes every verdict. See the
|
|
929
|
+
module docstring.
|
|
930
|
+
|
|
931
|
+
``timeout`` is the per-request deadline in seconds, defaulting to
|
|
932
|
+
:data:`REQUEST_TIMEOUT_SECONDS`. It is a parameter so a test can drive a short one; it is not an
|
|
933
|
+
off-switch, and ``None`` is not a value it takes.
|
|
934
|
+
|
|
935
|
+
Raises:
|
|
936
|
+
ServingHttpError: when ``host`` is not a loopback address, or when ``timeout`` is not a
|
|
937
|
+
positive number of seconds.
|
|
938
|
+
key_seam.KeySeamError: when ``verify_url`` is not a plain absolute https URL.
|
|
939
|
+
"""
|
|
940
|
+
|
|
941
|
+
_require_loopback(host)
|
|
942
|
+
if not isinstance(timeout, int | float) or isinstance(timeout, bool) or timeout <= 0:
|
|
943
|
+
# A surface with no deadline is the defect this parameter exists to prevent, so the one
|
|
944
|
+
# value that reintroduces it is refused rather than passed to ``settimeout``.
|
|
945
|
+
raise ServingHttpError(
|
|
946
|
+
"the serving surface answers one request at a time, so a request deadline is a "
|
|
947
|
+
"positive number of seconds"
|
|
948
|
+
)
|
|
949
|
+
checked = key_seam._https_url(verify_url, "the verify URL")
|
|
950
|
+
return _ServingServer(
|
|
951
|
+
(host, port),
|
|
952
|
+
_ServingHandler,
|
|
953
|
+
index=index,
|
|
954
|
+
index_path=None if index_path is None else Path(index_path),
|
|
955
|
+
ledger=ledger,
|
|
956
|
+
verify_url=checked,
|
|
957
|
+
transport=transport,
|
|
958
|
+
timeout=float(timeout),
|
|
959
|
+
)
|
|
960
|
+
|
|
961
|
+
|
|
962
|
+
def serve_forever(
|
|
963
|
+
*,
|
|
964
|
+
index: serving.VersionIndex,
|
|
965
|
+
index_path: Path | None = None,
|
|
966
|
+
ledger: CapLedger,
|
|
967
|
+
verify_url: str,
|
|
968
|
+
host: str = "127.0.0.1",
|
|
969
|
+
port: int = 0,
|
|
970
|
+
transport: key_seam.VerifyTransport | None = None,
|
|
971
|
+
timeout: float = REQUEST_TIMEOUT_SECONDS,
|
|
972
|
+
announce: Callable[[str], None] | None = None,
|
|
973
|
+
) -> None:
|
|
974
|
+
"""Bind the serving surface and serve it until interrupted.
|
|
975
|
+
|
|
976
|
+
The same keyword arguments :func:`build_server` takes, so there is one set of arguments to
|
|
977
|
+
reason about rather than two that have to be kept in step.
|
|
978
|
+
|
|
979
|
+
The served-address line is flushed immediately, in ``viewer.serve_notebook``'s shape and for its
|
|
980
|
+
reason: the port may be ephemeral, so a caller that starts this in the background has to read
|
|
981
|
+
that line to learn it, and a block-buffered pipe would withhold it for the life of the server.
|
|
982
|
+
"""
|
|
983
|
+
|
|
984
|
+
server = build_server(
|
|
985
|
+
index=index,
|
|
986
|
+
index_path=index_path,
|
|
987
|
+
ledger=ledger,
|
|
988
|
+
verify_url=verify_url,
|
|
989
|
+
host=host,
|
|
990
|
+
port=port,
|
|
991
|
+
transport=transport,
|
|
992
|
+
timeout=timeout,
|
|
993
|
+
)
|
|
994
|
+
url = f"http://{host}:{server.server_address[1]}/"
|
|
995
|
+
if announce is None:
|
|
996
|
+
print(f"mr-data serve: serving {url} (Ctrl-C to stop)", flush=True)
|
|
997
|
+
else:
|
|
998
|
+
announce(url)
|
|
999
|
+
try:
|
|
1000
|
+
server.serve_forever()
|
|
1001
|
+
except KeyboardInterrupt:
|
|
1002
|
+
pass
|
|
1003
|
+
finally:
|
|
1004
|
+
server.server_close()
|
|
1005
|
+
|
|
1006
|
+
|
|
1007
|
+
__all__ = [
|
|
1008
|
+
"DEFAULT_REFUSAL_STATUS",
|
|
1009
|
+
"DISCOVERY_QUERY_KEYS",
|
|
1010
|
+
"MAX_DATASET_ID_CHARS",
|
|
1011
|
+
"MAX_PIN_CHARS",
|
|
1012
|
+
"MAX_REQUEST_BYTES",
|
|
1013
|
+
"REQUEST_BODY_REFUSAL_CODE",
|
|
1014
|
+
"REQUEST_DATASET_REFUSAL_CODE",
|
|
1015
|
+
"REQUEST_FAULT_REFUSAL_CODE",
|
|
1016
|
+
"REQUEST_FIELD_REFUSAL_CODE",
|
|
1017
|
+
"REQUEST_METHOD_REFUSAL_CODE",
|
|
1018
|
+
"REQUEST_PATH_REFUSAL_CODE",
|
|
1019
|
+
"REQUEST_QUERY_REFUSAL_CODE",
|
|
1020
|
+
"REQUEST_SHAPE_REFUSAL_CODE",
|
|
1021
|
+
"REQUEST_SIZE_REFUSAL_CODE",
|
|
1022
|
+
"REQUEST_TIMEOUT_REFUSAL_CODE",
|
|
1023
|
+
"REQUEST_TIMEOUT_SECONDS",
|
|
1024
|
+
"ROWS_BODY_KEYS",
|
|
1025
|
+
"SERVING_PATHS",
|
|
1026
|
+
"STATUS_BY_REFUSAL_CODE",
|
|
1027
|
+
"ServingHttpError",
|
|
1028
|
+
"ServingPath",
|
|
1029
|
+
"build_server",
|
|
1030
|
+
"serve_forever",
|
|
1031
|
+
]
|