mostlyright-data 0.9.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mostlyright/data_harness/__init__.py +158 -0
- mostlyright/data_harness/acquisition/__init__.py +55 -0
- mostlyright/data_harness/acquisition/http.py +2773 -0
- mostlyright/data_harness/acquisition/parsing.py +809 -0
- mostlyright/data_harness/acquisition/ranges.py +495 -0
- mostlyright/data_harness/acquisition/result_download.py +360 -0
- mostlyright/data_harness/acquisition/retention_admission.py +248 -0
- mostlyright/data_harness/acquisition/sandbox.py +4888 -0
- mostlyright/data_harness/acquisition/url_policy.py +530 -0
- mostlyright/data_harness/agent_runtime.py +2743 -0
- mostlyright/data_harness/assets/logo-ink.svg +31 -0
- mostlyright/data_harness/backends/__init__.py +28 -0
- mostlyright/data_harness/backends/pandas_backend.py +350 -0
- mostlyright/data_harness/backends/polars_backend.py +366 -0
- mostlyright/data_harness/backends/protocol.py +124 -0
- mostlyright/data_harness/backends/reference.py +83 -0
- mostlyright/data_harness/backends/registry.py +55 -0
- mostlyright/data_harness/backends/restrictions.py +126 -0
- mostlyright/data_harness/canonical.py +333 -0
- mostlyright/data_harness/catalog_job.py +625 -0
- mostlyright/data_harness/cli.py +5398 -0
- mostlyright/data_harness/contracts.py +53 -0
- mostlyright/data_harness/coordinator.py +1307 -0
- mostlyright/data_harness/deploy.py +924 -0
- mostlyright/data_harness/deploy_target.py +312 -0
- mostlyright/data_harness/deployment_evidence.py +1067 -0
- mostlyright/data_harness/event_presentation.py +576 -0
- mostlyright/data_harness/events.py +2152 -0
- mostlyright/data_harness/fast_delimited.py +239 -0
- mostlyright/data_harness/fleet.py +237 -0
- mostlyright/data_harness/formats.py +236 -0
- mostlyright/data_harness/governors.py +1163 -0
- mostlyright/data_harness/hosted_bootstrap.py +972 -0
- mostlyright/data_harness/hosted_crawler.py +1115 -0
- mostlyright/data_harness/hosted_crawler_container_smoke.py +351 -0
- mostlyright/data_harness/hosted_crawler_fetch.py +423 -0
- mostlyright/data_harness/hosted_crawler_job.py +1277 -0
- mostlyright/data_harness/hosted_crawler_protocol.py +676 -0
- mostlyright/data_harness/hosted_dataset.py +1500 -0
- mostlyright/data_harness/hosted_deploy.py +3037 -0
- mostlyright/data_harness/hosted_handoff.py +62 -0
- mostlyright/data_harness/hosted_ingestion_contract.py +504 -0
- mostlyright/data_harness/hosted_ingestion_job.py +356 -0
- mostlyright/data_harness/hosted_ingestion_job_smoke.py +40 -0
- mostlyright/data_harness/hosted_session_container_smoke.py +194 -0
- mostlyright/data_harness/hosted_session_worker.py +3554 -0
- mostlyright/data_harness/hosted_session_worker_job_smoke.py +46 -0
- mostlyright/data_harness/hosted_worker.py +6784 -0
- mostlyright/data_harness/ingestion/__init__.py +56 -0
- mostlyright/data_harness/ingestion/contracts.py +461 -0
- mostlyright/data_harness/ingestion/faults.py +42 -0
- mostlyright/data_harness/ingestion/gcs_store.py +1162 -0
- mostlyright/data_harness/ingestion/spool.py +130 -0
- mostlyright/data_harness/ingestion/store.py +885 -0
- mostlyright/data_harness/key_seam.py +434 -0
- mostlyright/data_harness/linux_process_boundary.py +262 -0
- mostlyright/data_harness/local_contracts.py +2880 -0
- mostlyright/data_harness/local_search/__init__.py +5 -0
- mostlyright/data_harness/local_search/build_index.py +1087 -0
- mostlyright/data_harness/local_search/contracts.py +920 -0
- mostlyright/data_harness/local_search/query_trace.py +266 -0
- mostlyright/data_harness/local_search/retrieval.py +700 -0
- mostlyright/data_harness/local_search/sealed.py +474 -0
- mostlyright/data_harness/local_search/service.py +784 -0
- mostlyright/data_harness/nbrender/CONTRACT.md +212 -0
- mostlyright/data_harness/nbrender/__init__.py +12 -0
- mostlyright/data_harness/nbrender/chrome.py +359 -0
- mostlyright/data_harness/nbrender/code_body.py +266 -0
- mostlyright/data_harness/nbrender/document.py +407 -0
- mostlyright/data_harness/nbrender/frame.py +275 -0
- mostlyright/data_harness/nbrender/interactive.py +337 -0
- mostlyright/data_harness/nbrender/markdown_body.py +477 -0
- mostlyright/data_harness/nbrender/mr_components.py +134 -0
- mostlyright/data_harness/nbrender/outputs_data.py +595 -0
- mostlyright/data_harness/nbrender/outputs_rich.py +906 -0
- mostlyright/data_harness/nbrender/outputs_source.py +260 -0
- mostlyright/data_harness/nbrender/outputs_stage.py +176 -0
- mostlyright/data_harness/nbrender/outputs_text.py +400 -0
- mostlyright/data_harness/nbrender/parse.py +394 -0
- mostlyright/data_harness/nbrender/status.py +40 -0
- mostlyright/data_harness/nbrender/tokens.py +1295 -0
- mostlyright/data_harness/notebook.py +1710 -0
- mostlyright/data_harness/offline.py +2049 -0
- mostlyright/data_harness/operation_registry.py +1007 -0
- mostlyright/data_harness/operator_setup.py +239 -0
- mostlyright/data_harness/pipeline.py +6428 -0
- mostlyright/data_harness/plan_graph.py +2026 -0
- mostlyright/data_harness/preparation/__init__.py +104 -0
- mostlyright/data_harness/preparation/contracts.py +1017 -0
- mostlyright/data_harness/preparation/engine.py +221 -0
- mostlyright/data_harness/preparation/errors.py +14 -0
- mostlyright/data_harness/preparation/gates.py +751 -0
- mostlyright/data_harness/preparation/joins.py +574 -0
- mostlyright/data_harness/preparation/profile.py +384 -0
- mostlyright/data_harness/preparation/table.py +217 -0
- mostlyright/data_harness/preparation/transforms.py +568 -0
- mostlyright/data_harness/progress_events.py +534 -0
- mostlyright/data_harness/readers/__init__.py +46 -0
- mostlyright/data_harness/readers/containers.py +963 -0
- mostlyright/data_harness/readers/contracts.py +542 -0
- mostlyright/data_harness/readers/delimited.py +257 -0
- mostlyright/data_harness/readers/grib2/__init__.py +33 -0
- mostlyright/data_harness/readers/grib2/admission.py +722 -0
- mostlyright/data_harness/readers/grib2/decode.py +1009 -0
- mostlyright/data_harness/readers/grib2/geometry.py +1133 -0
- mostlyright/data_harness/readers/grib2/portable_math.py +501 -0
- mostlyright/data_harness/readers/json_tabular.py +485 -0
- mostlyright/data_harness/readers/registry.py +514 -0
- mostlyright/data_harness/readers/samples/README.md +110 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/cities.csv.gz +0 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/cities.csv.gz +0 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/cities.tar +0 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/cities.tar +0 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/cities.zip +0 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/expected.json +25 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/expected.json +25 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/expected.json +25 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/cities.csv +3 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/expected.json +23 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/cities.tsv +5 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/expected.json +25 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/expected.json +30 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/observations.csv +5 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/expected.json +44 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/stations.json +1 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/expected.json +48 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/observations.ndjson +4 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/cities.xlsx +0 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/expected.json +24 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/expected.json +27 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/expected.json +27 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/README.md +20 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/expected.json +55 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/gfs-2m-temperature.grib2 +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/expected.json +54 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/expected.json +54 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/hrrr-categorical-rain.grib2 +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/expected.json +54 -0
- mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
- mostlyright/data_harness/readers/samples.py +582 -0
- mostlyright/data_harness/readers/spreadsheet.py +803 -0
- mostlyright/data_harness/readers/tabular.py +510 -0
- mostlyright/data_harness/recipe.py +5321 -0
- mostlyright/data_harness/repair/__init__.py +78 -0
- mostlyright/data_harness/repair/adapters.py +274 -0
- mostlyright/data_harness/repair/contracts.py +872 -0
- mostlyright/data_harness/repair/coordinator.py +1099 -0
- mostlyright/data_harness/repair/errors.py +16 -0
- mostlyright/data_harness/review.py +2533 -0
- mostlyright/data_harness/rowset.py +283 -0
- mostlyright/data_harness/serving.py +1975 -0
- mostlyright/data_harness/serving_edge.py +590 -0
- mostlyright/data_harness/serving_http.py +1031 -0
- mostlyright/data_harness/session_probes.py +759 -0
- mostlyright/data_harness/signing.py +101 -0
- mostlyright/data_harness/source_discovery.py +898 -0
- mostlyright/data_harness/sources/__init__.py +209 -0
- mostlyright/data_harness/sources/_adapter_steps.py +213 -0
- mostlyright/data_harness/sources/adapters.py +1214 -0
- mostlyright/data_harness/sources/cadence.py +1428 -0
- mostlyright/data_harness/sources/cadence_emission.py +453 -0
- mostlyright/data_harness/sources/cadence_history.py +546 -0
- mostlyright/data_harness/sources/catalog/__init__.py +17 -0
- mostlyright/data_harness/sources/catalog/admission.py +477 -0
- mostlyright/data_harness/sources/catalog/authoring.py +1701 -0
- mostlyright/data_harness/sources/catalog/authoring_policy.py +701 -0
- mostlyright/data_harness/sources/catalog/authoring_shards.py +1217 -0
- mostlyright/data_harness/sources/catalog/bounded_io.py +231 -0
- mostlyright/data_harness/sources/catalog/channel.py +523 -0
- mostlyright/data_harness/sources/catalog/channel_client.py +296 -0
- mostlyright/data_harness/sources/catalog/contracts.py +825 -0
- mostlyright/data_harness/sources/catalog/coverage.py +137 -0
- mostlyright/data_harness/sources/catalog/delta.py +1340 -0
- mostlyright/data_harness/sources/catalog/embedding.py +532 -0
- mostlyright/data_harness/sources/catalog/entry_v2.py +1182 -0
- mostlyright/data_harness/sources/catalog/fill.py +3889 -0
- mostlyright/data_harness/sources/catalog/fill_partitions.py +459 -0
- mostlyright/data_harness/sources/catalog/fill_staging.py +1105 -0
- mostlyright/data_harness/sources/catalog/gating.py +374 -0
- mostlyright/data_harness/sources/catalog/generation_receipt.py +1607 -0
- mostlyright/data_harness/sources/catalog/harvest/__init__.py +7 -0
- mostlyright/data_harness/sources/catalog/harvest/ckan.py +384 -0
- mostlyright/data_harness/sources/catalog/harvest/datagov_v4.py +798 -0
- mostlyright/data_harness/sources/catalog/harvest/protocol.py +964 -0
- mostlyright/data_harness/sources/catalog/harvest/sdmx.py +445 -0
- mostlyright/data_harness/sources/catalog/harvest/stac.py +384 -0
- mostlyright/data_harness/sources/catalog/health.py +447 -0
- mostlyright/data_harness/sources/catalog/hosted_catalog.py +105 -0
- mostlyright/data_harness/sources/catalog/identity_history.py +1549 -0
- mostlyright/data_harness/sources/catalog/neural.py +1618 -0
- mostlyright/data_harness/sources/catalog/packed_catalog.py +2345 -0
- mostlyright/data_harness/sources/catalog/packed_retrieval.py +1517 -0
- mostlyright/data_harness/sources/catalog/packed_writer.py +2802 -0
- mostlyright/data_harness/sources/catalog/query_trace.py +1037 -0
- mostlyright/data_harness/sources/catalog/recommend.py +171 -0
- mostlyright/data_harness/sources/catalog/retrieval.py +230 -0
- mostlyright/data_harness/sources/catalog/retrieval_manifest.py +995 -0
- mostlyright/data_harness/sources/catalog/rights_decisions.py +254 -0
- mostlyright/data_harness/sources/catalog/sealed.py +560 -0
- mostlyright/data_harness/sources/catalog/search.py +230 -0
- mostlyright/data_harness/sources/catalog/streaming_delta.py +1097 -0
- mostlyright/data_harness/sources/catalog/update.py +891 -0
- mostlyright/data_harness/sources/collections.py +815 -0
- mostlyright/data_harness/sources/contracts.py +2223 -0
- mostlyright/data_harness/sources/deletion.py +761 -0
- mostlyright/data_harness/sources/fitness.py +162 -0
- mostlyright/data_harness/sources/governance.py +163 -0
- mostlyright/data_harness/sources/hosted.py +173 -0
- mostlyright/data_harness/sources/integration.py +218 -0
- mostlyright/data_harness/sources/range_reader.py +418 -0
- mostlyright/data_harness/sources/registry.py +514 -0
- mostlyright/data_harness/sources/rights_rule.py +59 -0
- mostlyright/data_harness/sources/source_cadence_vectors.v1.json +1 -0
- mostlyright/data_harness/sources/sports.py +521 -0
- mostlyright/data_harness/sources/stream.py +524 -0
- mostlyright/data_harness/sources/stream_connector.py +418 -0
- mostlyright/data_harness/sources/stream_recorder.py +1404 -0
- mostlyright/data_harness/studio_boundary.py +2019 -0
- mostlyright/data_harness/thin/__init__.py +37 -0
- mostlyright/data_harness/thin/acquire.py +1137 -0
- mostlyright/data_harness/thin/acquire_cancel.py +579 -0
- mostlyright/data_harness/thin/approvals.py +617 -0
- mostlyright/data_harness/thin/commands.py +406 -0
- mostlyright/data_harness/thin/download.py +194 -0
- mostlyright/data_harness/thin/narrative.py +589 -0
- mostlyright/data_harness/thin/parity.py +1070 -0
- mostlyright/data_harness/thin/propose.py +2759 -0
- mostlyright/data_harness/thin/research.py +1663 -0
- mostlyright/data_harness/thin/router.py +924 -0
- mostlyright/data_harness/thin/runs.py +519 -0
- mostlyright/data_harness/thin/session.py +281 -0
- mostlyright/data_harness/thin/stream.py +501 -0
- mostlyright/data_harness/thin/transport.py +187 -0
- mostlyright/data_harness/thin/vocabulary.py +368 -0
- mostlyright/data_harness/thin/workers.py +164 -0
- mostlyright/data_harness/ucum/TABLE-PIN.json +40 -0
- mostlyright/data_harness/ucum/ucum-subset.v1.json +632 -0
- mostlyright/data_harness/unit_flow.py +927 -0
- mostlyright/data_harness/units.py +572 -0
- mostlyright/data_harness/ux/__init__.py +9 -0
- mostlyright/data_harness/ux/approve.py +485 -0
- mostlyright/data_harness/ux/author_yaml.py +597 -0
- mostlyright/data_harness/ux/cloud_auth.py +447 -0
- mostlyright/data_harness/ux/commands/__init__.py +260 -0
- mostlyright/data_harness/ux/commands/approve.py +136 -0
- mostlyright/data_harness/ux/commands/auth.py +744 -0
- mostlyright/data_harness/ux/commands/author.py +79 -0
- mostlyright/data_harness/ux/commands/catalog_author.py +403 -0
- mostlyright/data_harness/ux/commands/catalog_fill.py +523 -0
- mostlyright/data_harness/ux/commands/catalog_harvest.py +545 -0
- mostlyright/data_harness/ux/commands/catalog_publish.py +1838 -0
- mostlyright/data_harness/ux/commands/catalog_search.py +71 -0
- mostlyright/data_harness/ux/commands/catalog_update.py +437 -0
- mostlyright/data_harness/ux/commands/deploy.py +134 -0
- mostlyright/data_harness/ux/commands/deploy_dataset.py +98 -0
- mostlyright/data_harness/ux/commands/deploy_plan.py +105 -0
- mostlyright/data_harness/ux/commands/deploy_status.py +104 -0
- mostlyright/data_harness/ux/commands/diff.py +74 -0
- mostlyright/data_harness/ux/commands/index.py +84 -0
- mostlyright/data_harness/ux/commands/inventory.py +47 -0
- mostlyright/data_harness/ux/commands/list_builds.py +143 -0
- mostlyright/data_harness/ux/commands/login.py +63 -0
- mostlyright/data_harness/ux/commands/peek.py +236 -0
- mostlyright/data_harness/ux/commands/plan_check.py +90 -0
- mostlyright/data_harness/ux/commands/preflight.py +97 -0
- mostlyright/data_harness/ux/commands/record.py +107 -0
- mostlyright/data_harness/ux/commands/review_setup.py +47 -0
- mostlyright/data_harness/ux/commands/search.py +440 -0
- mostlyright/data_harness/ux/commands/show.py +61 -0
- mostlyright/data_harness/ux/commands/whoami.py +37 -0
- mostlyright/data_harness/ux/credential_native.py +551 -0
- mostlyright/data_harness/ux/credential_store.py +1055 -0
- mostlyright/data_harness/ux/credentials.py +631 -0
- mostlyright/data_harness/ux/diffing.py +444 -0
- mostlyright/data_harness/ux/headline.py +671 -0
- mostlyright/data_harness/ux/hosted_acquisition.py +974 -0
- mostlyright/data_harness/ux/hosted_run_status.py +619 -0
- mostlyright/data_harness/ux/inventory.py +427 -0
- mostlyright/data_harness/ux/local_review.py +375 -0
- mostlyright/data_harness/ux/login.py +691 -0
- mostlyright/data_harness/ux/path_kind.py +147 -0
- mostlyright/data_harness/ux/peek.py +1000 -0
- mostlyright/data_harness/ux/plain_file.py +178 -0
- mostlyright/data_harness/ux/plan_check.py +311 -0
- mostlyright/data_harness/ux/preflight.py +918 -0
- mostlyright/data_harness/ux/readers.py +1124 -0
- mostlyright/data_harness/ux/remediation.py +2195 -0
- mostlyright/data_harness/ux/render.py +657 -0
- mostlyright/data_harness/ux/workload.py +1077 -0
- mostlyright/data_harness/viewer.py +3713 -0
- mostlyright/data_harness/visual_run/__init__.py +83 -0
- mostlyright/data_harness/visual_run/authoring.py +235 -0
- mostlyright/data_harness/visual_run/contracts.py +673 -0
- mostlyright/data_harness/visual_run/materialize.py +486 -0
- mostlyright/data_harness/visual_run/observations.py +874 -0
- mostlyright/data_harness/visual_run/query.py +259 -0
- mostlyright/data_harness/visual_run/reducer.py +280 -0
- mostlyright/data_harness/visual_run/sdk.py +892 -0
- mostlyright/data_harness/visual_run/store.py +584 -0
- mostlyright/data_harness/visual_run/transport.py +239 -0
- mostlyright/data_harness/watch.py +2999 -0
- mostlyright_data-0.9.0.dist-info/METADATA +607 -0
- mostlyright_data-0.9.0.dist-info/RECORD +314 -0
- mostlyright_data-0.9.0.dist-info/WHEEL +4 -0
- mostlyright_data-0.9.0.dist-info/entry_points.txt +12 -0
|
@@ -0,0 +1,2223 @@
|
|
|
1
|
+
"""Strict Harness-private source and acquisition contracts.
|
|
2
|
+
|
|
3
|
+
These contracts describe deterministic worker evidence. They are not Studio wire models and
|
|
4
|
+
never grant acquisition, legal, human-review, or release authority.
|
|
5
|
+
|
|
6
|
+
Vocabulary map
|
|
7
|
+
--------------
|
|
8
|
+
|
|
9
|
+
Three modules carry overlapping enum sets and this one is canonical. The layers are
|
|
10
|
+
agent_runtime.py (untrusted model input and output), local_contracts.py (local proposals and
|
|
11
|
+
plans), and this module (deterministic adapter, observation, and fitness evidence). Where two
|
|
12
|
+
layers name the same concept they use the same spelling. Where the sets differ they differ
|
|
13
|
+
because the stage differs, and they are deliberately not merged.
|
|
14
|
+
|
|
15
|
+
source class
|
|
16
|
+
agent_runtime external, user
|
|
17
|
+
local_contracts external_adapter, user_file, user_url, user_api,
|
|
18
|
+
database_extract, webhook, stream (_SOURCE_CLASSES)
|
|
19
|
+
sources external_adapter, user_file, user_url, user_api, stream
|
|
20
|
+
(SOURCE_CLASSES)
|
|
21
|
+
A model names a family and the local contract narrows it: external to external_adapter, user
|
|
22
|
+
to whichever of user_file, user_url, or user_api the locator turns out to be.
|
|
23
|
+
database_extract and webhook have no adapter contract, so this module rejects them. Catalog
|
|
24
|
+
providers are a separate Harness-owned discovery boundary and never become acquisition
|
|
25
|
+
adapters merely because their metadata was indexed.
|
|
26
|
+
|
|
27
|
+
rights status
|
|
28
|
+
agent_runtime approved, conditional, unclear, prohibited
|
|
29
|
+
local_contracts approved, conditional, unclear, prohibited (_PROPOSAL_RIGHTS)
|
|
30
|
+
sources approved, conditional, unclear, prohibited (RIGHTS_STATUSES)
|
|
31
|
+
One concept, one spelling, all three layers. local_contracts._RIGHTS_STATUSES is a
|
|
32
|
+
different concept despite its name: it is the basis a build reads a file under
|
|
33
|
+
(project_owned, public_domain, permissive_license, authorized_internal, user_authorized,
|
|
34
|
+
unknown, prohibited), so its unknown is not this unclear.
|
|
35
|
+
|
|
36
|
+
fitness decision
|
|
37
|
+
agent_runtime fit, conditional, reject (assessment.fitness)
|
|
38
|
+
local_contracts selected, eligible, rejected (_FITNESS_DECISIONS)
|
|
39
|
+
sources selected, rejected, human_escalation_required (FITNESS_DECISIONS)
|
|
40
|
+
Three stages, not three spellings of one stage. The model judges a candidate, the local
|
|
41
|
+
proposal records whether it was chosen (selected) or only admissible (eligible), and the
|
|
42
|
+
fitness report adjudicates deterministically and escalates instead of guessing. fit reads
|
|
43
|
+
as eligible then selected, conditional as eligible then human_escalation_required, reject
|
|
44
|
+
as rejected.
|
|
45
|
+
|
|
46
|
+
liveness
|
|
47
|
+
agent_runtime live, delayed, dead, unknown (liveness.status)
|
|
48
|
+
local_contracts live, degraded, delayed, dead, not_applicable (_LIVE_ENDPOINTS)
|
|
49
|
+
sources none; CAPABILITIES.live is an adapter capability, not an endpoint state
|
|
50
|
+
A model may report unknown because a probe was inconclusive. A proposal may not: it states
|
|
51
|
+
a definite state or not_applicable. degraded has no model counterpart.
|
|
52
|
+
|
|
53
|
+
data format
|
|
54
|
+
agent_runtime none
|
|
55
|
+
local_contracts PLAN_DATA_FORMATS, re-exported as _DATA_FORMATS
|
|
56
|
+
sources DATA_FORMATS, re-exported from the same table
|
|
57
|
+
Not the same axis, and neither set is written here. Both live in
|
|
58
|
+
mostlyright.data_harness.formats, which is the single authority for format membership;
|
|
59
|
+
this module re-exports the wire-format set under the name its callers already use.
|
|
60
|
+
This module's set lists payload encodings an adapter emits. local_contracts mixes
|
|
61
|
+
encodings with access shapes (api, stream) that are locator-ish rather than encodings,
|
|
62
|
+
so the two sets are not shared. local_contracts.SourceSpec.format is narrower still:
|
|
63
|
+
the build reads csv only.
|
|
64
|
+
|
|
65
|
+
data classification
|
|
66
|
+
agent_runtime public, internal, confidential, restricted
|
|
67
|
+
local_contracts none
|
|
68
|
+
sources public, internal, confidential, restricted, rejected (CLASSIFICATIONS)
|
|
69
|
+
rejected is a terminal adapter state and has no model counterpart.
|
|
70
|
+
|
|
71
|
+
retention kinds
|
|
72
|
+
agent_runtime raw_days, derived_days, tombstone_required (assessment.retention)
|
|
73
|
+
local_contracts none
|
|
74
|
+
sources raw, derived, logs, review_evidence (RETENTION_KINDS)
|
|
75
|
+
RETENTION_KINDS names every evidence kind a retention policy covers; deletion.py keys its
|
|
76
|
+
policy-field lookup by the same strings. A model assessment proposes only the raw and
|
|
77
|
+
derived horizons.
|
|
78
|
+
"""
|
|
79
|
+
|
|
80
|
+
from __future__ import annotations
|
|
81
|
+
|
|
82
|
+
import ipaddress
|
|
83
|
+
import re
|
|
84
|
+
from collections.abc import Mapping
|
|
85
|
+
from dataclasses import dataclass, fields
|
|
86
|
+
from datetime import UTC, datetime
|
|
87
|
+
from typing import Any, ClassVar
|
|
88
|
+
from urllib.parse import urlsplit
|
|
89
|
+
|
|
90
|
+
from mostlyright.data_harness.canonical import canonical_json_bytes, canonical_sha256
|
|
91
|
+
from mostlyright.data_harness.formats import DATA_FORMATS as _DATA_FORMATS
|
|
92
|
+
|
|
93
|
+
# ``readers`` is a leaf: it imports ``formats``, ``canonical``, and the standard library and
|
|
94
|
+
# nothing else, and a test walks the package to keep it that way. So this direction --
|
|
95
|
+
# sources depending on readers -- is the acyclic one. Only the closed flag vocabulary is
|
|
96
|
+
# imported, deliberately not ``ReaderPin``: the pin arrives here as a sealed mapping off a
|
|
97
|
+
# recipe document, and modelling it as the Toolbox's object would couple the evidence layer
|
|
98
|
+
# to the decoder implementation for no gain.
|
|
99
|
+
from mostlyright.data_harness.readers.contracts import DECODE_FLAGS
|
|
100
|
+
|
|
101
|
+
SOURCE_CONTRACT_VERSION = "harness-source-contract.v1"
|
|
102
|
+
ADAPTER_CONTRACT_VERSION = "harness-source-adapter.v2"
|
|
103
|
+
OBSERVATION_CONTRACT_VERSION = "harness-source-observation.v1"
|
|
104
|
+
RECEIPT_CONTRACT_VERSION = "harness-acquisition-receipt.v1"
|
|
105
|
+
RECEIPT_CONTRACT_VERSION_V2 = "harness-acquisition-receipt.v2"
|
|
106
|
+
RECEIPT_CONTRACT_VERSION_V3 = "harness-acquisition-receipt.v3"
|
|
107
|
+
FITNESS_CONTRACT_VERSION = "harness-source-fitness.v1"
|
|
108
|
+
TOMBSTONE_CONTRACT_VERSION = "harness-source-deletion-tombstone.v1"
|
|
109
|
+
|
|
110
|
+
MAX_SAFE_INTEGER = (1 << 53) - 1
|
|
111
|
+
MAX_QUERY_BYTES = 16_384
|
|
112
|
+
MAX_TEXT = 4_096
|
|
113
|
+
MAX_EVIDENCE = 32
|
|
114
|
+
MAX_OBLIGATIONS = 32
|
|
115
|
+
MAX_REFERENCES = 10_000
|
|
116
|
+
|
|
117
|
+
_IDENTIFIER = re.compile(r"^[a-z][a-z0-9]*(?:[._-][a-z0-9]+)*$")
|
|
118
|
+
_SEMVER = re.compile(r"^[1-9][0-9]*\.[0-9]+\.[0-9]+$")
|
|
119
|
+
_SHA256 = re.compile(r"^[0-9a-f]{64}$")
|
|
120
|
+
_SECRET_REF = re.compile(r"^shr_[A-Za-z0-9_-]{24,120}$")
|
|
121
|
+
_MEDIA_TYPE = re.compile(r"^[a-z0-9][a-z0-9.+-]{0,126}/[a-z0-9][a-z0-9.+-]{0,126}$")
|
|
122
|
+
|
|
123
|
+
SOURCE_CLASSES = frozenset(
|
|
124
|
+
{
|
|
125
|
+
"external_adapter",
|
|
126
|
+
"user_file",
|
|
127
|
+
"user_url",
|
|
128
|
+
"user_api",
|
|
129
|
+
"stream",
|
|
130
|
+
}
|
|
131
|
+
)
|
|
132
|
+
DATA_FORMATS = _DATA_FORMATS
|
|
133
|
+
CAPABILITIES = frozenset({"snapshot", "historical", "live", "refresh", "backfill", "stream"})
|
|
134
|
+
RIGHTS_STATUSES = frozenset({"approved", "conditional", "unclear", "prohibited"})
|
|
135
|
+
CLASSIFICATIONS = frozenset({"public", "internal", "confidential", "restricted", "rejected"})
|
|
136
|
+
FITNESS_DECISIONS = frozenset({"selected", "rejected", "human_escalation_required"})
|
|
137
|
+
PEER_ATTEMPT_OUTCOMES = frozenset({"transport_failure", "response", "response_failure"})
|
|
138
|
+
FITNESS_DIMENSIONS = frozenset(
|
|
139
|
+
{
|
|
140
|
+
"question_relevance",
|
|
141
|
+
"output_grain",
|
|
142
|
+
"historical_coverage",
|
|
143
|
+
"gaps",
|
|
144
|
+
"liveness",
|
|
145
|
+
"publication_delay",
|
|
146
|
+
"update_frequency",
|
|
147
|
+
"granularity",
|
|
148
|
+
"geographic_entity_coverage",
|
|
149
|
+
"revision_behavior",
|
|
150
|
+
"schema_stability",
|
|
151
|
+
"authentication_rate_limits",
|
|
152
|
+
"rights",
|
|
153
|
+
"cost_maintenance",
|
|
154
|
+
"refresh_backfill_live_fit",
|
|
155
|
+
}
|
|
156
|
+
)
|
|
157
|
+
OBLIGATION_LIFECYCLES = frozenset({"candidate", "derivative", "redistribution", "publication"})
|
|
158
|
+
# The exact three fields a Reader pin carries, spelled the way the recipe seals them. An
|
|
159
|
+
# exact key set rather than a minimum, so a fourth field cannot ride into the request digest
|
|
160
|
+
# unnoticed and mean nothing to the adapter that reads it.
|
|
161
|
+
READER_PIN_KEYS = frozenset({"family_id", "family_version", "decode_options"})
|
|
162
|
+
# Canonical retention-evidence kinds; deletion.py keys its policy-field lookup by these strings.
|
|
163
|
+
RETENTION_KINDS = ("raw", "derived", "logs", "review_evidence")
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
class SourceContractError(ValueError):
|
|
167
|
+
"""Stable fail-closed source-contract error."""
|
|
168
|
+
|
|
169
|
+
def __init__(self, code: str, path: str, detail: str) -> None:
|
|
170
|
+
self.code = code
|
|
171
|
+
self.path = path
|
|
172
|
+
self.detail = detail
|
|
173
|
+
super().__init__(f"{path}: {detail} [{code}]")
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def decode_options_digest(options: Mapping[str, Any]) -> str:
|
|
177
|
+
"""Digest over an exact set of Reader decode settings.
|
|
178
|
+
|
|
179
|
+
One formula with one home. The recipe seals a source's ``decode_options`` and the
|
|
180
|
+
adapter records the digest of the settings a decode actually ran with; if those two
|
|
181
|
+
were computed by two textually separate rules, a drift in either would read as a
|
|
182
|
+
decoder impersonation rather than as the bug it is. ``RecipeSource`` and
|
|
183
|
+
``_adapter_steps`` both call this.
|
|
184
|
+
"""
|
|
185
|
+
|
|
186
|
+
return canonical_sha256(dict(options))
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def require_canonical_timestamp(value: Any, path: str) -> str:
|
|
190
|
+
"""The public spelling of the strict timestamp rule every temporal field obeys.
|
|
191
|
+
|
|
192
|
+
Exported so a document outside this module -- a rights decision's ``decided_at`` -- is
|
|
193
|
+
held to exactly the rule ``observed_at`` fields are held to, without a second validator
|
|
194
|
+
drifting from this one.
|
|
195
|
+
"""
|
|
196
|
+
|
|
197
|
+
return _timestamp(value, path)
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
class _CanonicalContract:
|
|
201
|
+
schema_version: ClassVar[str]
|
|
202
|
+
|
|
203
|
+
def to_dict(self) -> dict[str, Any]:
|
|
204
|
+
raise NotImplementedError
|
|
205
|
+
|
|
206
|
+
@property
|
|
207
|
+
def digest(self) -> str:
|
|
208
|
+
return canonical_sha256(self.to_dict())
|
|
209
|
+
|
|
210
|
+
@property
|
|
211
|
+
def canonical_bytes(self) -> bytes:
|
|
212
|
+
return canonical_json_bytes(self.to_dict())
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
@dataclass(frozen=True)
|
|
216
|
+
class EvidenceReference(_CanonicalContract):
|
|
217
|
+
"""Immutable citation or recorded probe/document evidence."""
|
|
218
|
+
|
|
219
|
+
uri: str
|
|
220
|
+
observed_at: str
|
|
221
|
+
content_sha256: str
|
|
222
|
+
media_type: str
|
|
223
|
+
|
|
224
|
+
def __post_init__(self) -> None:
|
|
225
|
+
_https_uri(self.uri, "evidence.uri")
|
|
226
|
+
_timestamp(self.observed_at, "evidence.observed_at")
|
|
227
|
+
_digest(self.content_sha256, "evidence.content_sha256")
|
|
228
|
+
_media_type(self.media_type, "evidence.media_type")
|
|
229
|
+
|
|
230
|
+
def to_dict(self) -> dict[str, str]:
|
|
231
|
+
return {
|
|
232
|
+
"uri": self.uri,
|
|
233
|
+
"observed_at": self.observed_at,
|
|
234
|
+
"content_sha256": self.content_sha256,
|
|
235
|
+
"media_type": self.media_type,
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
@dataclass(frozen=True)
|
|
240
|
+
class LicenseObligation(_CanonicalContract):
|
|
241
|
+
"""Machine-enforced source-license obligation."""
|
|
242
|
+
|
|
243
|
+
obligation_id: str
|
|
244
|
+
applies_at: tuple[str, ...]
|
|
245
|
+
permitted_uses: tuple[str, ...]
|
|
246
|
+
prohibited_uses: tuple[str, ...]
|
|
247
|
+
required_artifact_digests: tuple[str, ...]
|
|
248
|
+
escalation_when_ambiguous: bool
|
|
249
|
+
|
|
250
|
+
def __post_init__(self) -> None:
|
|
251
|
+
_identifier(self.obligation_id, "obligation.obligation_id")
|
|
252
|
+
_choices(
|
|
253
|
+
self.applies_at,
|
|
254
|
+
OBLIGATION_LIFECYCLES,
|
|
255
|
+
"obligation.applies_at",
|
|
256
|
+
nonempty=True,
|
|
257
|
+
# Derived for the same reason as ``adapter.data_formats`` below: the literal 4
|
|
258
|
+
# was this table's cardinality restated, so admitting a fifth lifecycle would
|
|
259
|
+
# have refused an obligation that named them all.
|
|
260
|
+
maximum=len(OBLIGATION_LIFECYCLES),
|
|
261
|
+
)
|
|
262
|
+
_texts(
|
|
263
|
+
self.permitted_uses,
|
|
264
|
+
"obligation.permitted_uses",
|
|
265
|
+
nonempty=False,
|
|
266
|
+
maximum=32,
|
|
267
|
+
item_maximum=200,
|
|
268
|
+
)
|
|
269
|
+
_texts(
|
|
270
|
+
self.prohibited_uses,
|
|
271
|
+
"obligation.prohibited_uses",
|
|
272
|
+
nonempty=False,
|
|
273
|
+
maximum=32,
|
|
274
|
+
item_maximum=200,
|
|
275
|
+
)
|
|
276
|
+
_digests(
|
|
277
|
+
self.required_artifact_digests,
|
|
278
|
+
"obligation.required_artifact_digests",
|
|
279
|
+
maximum=32,
|
|
280
|
+
)
|
|
281
|
+
if not self.permitted_uses and not self.prohibited_uses:
|
|
282
|
+
raise SourceContractError(
|
|
283
|
+
"OBLIGATION_EMPTY",
|
|
284
|
+
"obligation",
|
|
285
|
+
"an obligation must constrain at least one permitted or prohibited use",
|
|
286
|
+
)
|
|
287
|
+
|
|
288
|
+
def applies(self, lifecycle: str, intended_use: str) -> bool:
|
|
289
|
+
_choice(lifecycle, OBLIGATION_LIFECYCLES, "lifecycle")
|
|
290
|
+
_text(intended_use, "intended_use", maximum=200)
|
|
291
|
+
if lifecycle not in self.applies_at:
|
|
292
|
+
return False
|
|
293
|
+
return not self.permitted_uses or intended_use in self.permitted_uses
|
|
294
|
+
|
|
295
|
+
def to_dict(self) -> dict[str, Any]:
|
|
296
|
+
return {
|
|
297
|
+
"obligation_id": self.obligation_id,
|
|
298
|
+
"applies_at": list(self.applies_at),
|
|
299
|
+
"permitted_uses": list(self.permitted_uses),
|
|
300
|
+
"prohibited_uses": list(self.prohibited_uses),
|
|
301
|
+
"required_artifact_digests": list(self.required_artifact_digests),
|
|
302
|
+
"escalation_when_ambiguous": self.escalation_when_ambiguous,
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
@dataclass(frozen=True)
|
|
307
|
+
class RightsEvidence(_CanonicalContract):
|
|
308
|
+
"""Rights claim plus immutable evidence; never a human decision by itself."""
|
|
309
|
+
|
|
310
|
+
status: str
|
|
311
|
+
lawful_basis: str
|
|
312
|
+
allowed_uses: tuple[str, ...]
|
|
313
|
+
evidence: tuple[EvidenceReference, ...]
|
|
314
|
+
obligations: tuple[LicenseObligation, ...]
|
|
315
|
+
human_decision_reference: str | None = None
|
|
316
|
+
|
|
317
|
+
def __post_init__(self) -> None:
|
|
318
|
+
_choice(self.status, RIGHTS_STATUSES, "rights.status")
|
|
319
|
+
_text(self.lawful_basis, "rights.lawful_basis", maximum=1_000)
|
|
320
|
+
_texts(
|
|
321
|
+
self.allowed_uses,
|
|
322
|
+
"rights.allowed_uses",
|
|
323
|
+
nonempty=self.status in {"approved", "conditional"},
|
|
324
|
+
maximum=32,
|
|
325
|
+
item_maximum=200,
|
|
326
|
+
)
|
|
327
|
+
_typed_tuple(
|
|
328
|
+
self.evidence,
|
|
329
|
+
EvidenceReference,
|
|
330
|
+
"rights.evidence",
|
|
331
|
+
nonempty=True,
|
|
332
|
+
maximum=MAX_EVIDENCE,
|
|
333
|
+
)
|
|
334
|
+
_typed_tuple(
|
|
335
|
+
self.obligations,
|
|
336
|
+
LicenseObligation,
|
|
337
|
+
"rights.obligations",
|
|
338
|
+
nonempty=False,
|
|
339
|
+
maximum=MAX_OBLIGATIONS,
|
|
340
|
+
)
|
|
341
|
+
if self.human_decision_reference is not None:
|
|
342
|
+
_text(
|
|
343
|
+
self.human_decision_reference,
|
|
344
|
+
"rights.human_decision_reference",
|
|
345
|
+
maximum=200,
|
|
346
|
+
)
|
|
347
|
+
if self.status == "conditional" and not self.obligations:
|
|
348
|
+
raise SourceContractError(
|
|
349
|
+
"CONDITIONAL_WITHOUT_OBLIGATION",
|
|
350
|
+
"rights.obligations",
|
|
351
|
+
"conditional rights require at least one machine-readable obligation",
|
|
352
|
+
)
|
|
353
|
+
if self.status in {"unclear", "prohibited"} and self.human_decision_reference is not None:
|
|
354
|
+
raise SourceContractError(
|
|
355
|
+
"RIGHTS_AUTHORITY_CONFLICT",
|
|
356
|
+
"rights.human_decision_reference",
|
|
357
|
+
"unclear or prohibited source evidence cannot embed approval authority",
|
|
358
|
+
)
|
|
359
|
+
|
|
360
|
+
def to_dict(self) -> dict[str, Any]:
|
|
361
|
+
return {
|
|
362
|
+
"status": self.status,
|
|
363
|
+
"lawful_basis": self.lawful_basis,
|
|
364
|
+
"allowed_uses": list(self.allowed_uses),
|
|
365
|
+
"evidence": [item.to_dict() for item in self.evidence],
|
|
366
|
+
"obligations": [item.to_dict() for item in self.obligations],
|
|
367
|
+
"human_decision_reference": self.human_decision_reference,
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
@dataclass(frozen=True)
|
|
372
|
+
class RetentionPolicy(_CanonicalContract):
|
|
373
|
+
"""Coordinator-owned retention policy in whole days."""
|
|
374
|
+
|
|
375
|
+
raw_days: int
|
|
376
|
+
derived_days: int
|
|
377
|
+
log_days: int
|
|
378
|
+
review_evidence_days: int
|
|
379
|
+
tombstone_required: bool = True
|
|
380
|
+
|
|
381
|
+
def __post_init__(self) -> None:
|
|
382
|
+
for name, value in (
|
|
383
|
+
("raw_days", self.raw_days),
|
|
384
|
+
("derived_days", self.derived_days),
|
|
385
|
+
("log_days", self.log_days),
|
|
386
|
+
("review_evidence_days", self.review_evidence_days),
|
|
387
|
+
):
|
|
388
|
+
_integer(value, f"retention.{name}", minimum=0, maximum=3_650)
|
|
389
|
+
if self.tombstone_required is not True:
|
|
390
|
+
raise SourceContractError(
|
|
391
|
+
"TOMBSTONE_REQUIRED",
|
|
392
|
+
"retention.tombstone_required",
|
|
393
|
+
"V1 deletion must retain an immutable tombstone",
|
|
394
|
+
)
|
|
395
|
+
|
|
396
|
+
def to_dict(self) -> dict[str, Any]:
|
|
397
|
+
return {
|
|
398
|
+
"raw_days": self.raw_days,
|
|
399
|
+
"derived_days": self.derived_days,
|
|
400
|
+
"log_days": self.log_days,
|
|
401
|
+
"review_evidence_days": self.review_evidence_days,
|
|
402
|
+
"tombstone_required": self.tombstone_required,
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
|
|
406
|
+
@dataclass(frozen=True)
|
|
407
|
+
class AdapterDescriptor(_CanonicalContract):
|
|
408
|
+
"""One allowlisted source adapter identity and capability set."""
|
|
409
|
+
|
|
410
|
+
adapter_id: str
|
|
411
|
+
adapter_version: str
|
|
412
|
+
source_class: str
|
|
413
|
+
data_formats: tuple[str, ...]
|
|
414
|
+
capabilities: tuple[str, ...]
|
|
415
|
+
schema_version: str = ADAPTER_CONTRACT_VERSION
|
|
416
|
+
|
|
417
|
+
def __post_init__(self) -> None:
|
|
418
|
+
_constant(self.schema_version, ADAPTER_CONTRACT_VERSION, "adapter.schema_version")
|
|
419
|
+
_identifier(self.adapter_id, "adapter.adapter_id")
|
|
420
|
+
_semver(self.adapter_version, "adapter.adapter_version")
|
|
421
|
+
_choice(self.source_class, SOURCE_CLASSES, "adapter.source_class")
|
|
422
|
+
_choices(
|
|
423
|
+
self.data_formats,
|
|
424
|
+
DATA_FORMATS,
|
|
425
|
+
"adapter.data_formats",
|
|
426
|
+
nonempty=True,
|
|
427
|
+
# Derive the bound from the allowlist so table changes cannot leave a stale literal.
|
|
428
|
+
maximum=len(DATA_FORMATS),
|
|
429
|
+
)
|
|
430
|
+
_choices(
|
|
431
|
+
self.capabilities,
|
|
432
|
+
CAPABILITIES,
|
|
433
|
+
"adapter.capabilities",
|
|
434
|
+
nonempty=True,
|
|
435
|
+
maximum=8,
|
|
436
|
+
)
|
|
437
|
+
|
|
438
|
+
def to_dict(self) -> dict[str, Any]:
|
|
439
|
+
return {
|
|
440
|
+
"schema_version": self.schema_version,
|
|
441
|
+
"adapter_id": self.adapter_id,
|
|
442
|
+
"adapter_version": self.adapter_version,
|
|
443
|
+
"source_class": self.source_class,
|
|
444
|
+
"data_formats": list(self.data_formats),
|
|
445
|
+
"capabilities": list(self.capabilities),
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
|
|
449
|
+
@dataclass(frozen=True)
|
|
450
|
+
class AcquisitionRequest(_CanonicalContract):
|
|
451
|
+
"""Coordinator-authorized source request.
|
|
452
|
+
|
|
453
|
+
``reader_pin`` is authority, not a locator: it names which decoder the coordinator
|
|
454
|
+
entitles this acquisition to run, exactly as the recipe sealed it. It rides the
|
|
455
|
+
request rather than the adapter query because the query is the address of the bytes
|
|
456
|
+
and each adapter pins its own exact query key set; a decoder entitlement is neither.
|
|
457
|
+
|
|
458
|
+
The pin is inside ``to_dict`` and therefore inside ``digest``, which the receipt binds
|
|
459
|
+
as ``request_digest``. A pin swapped after authorization changes the digest and fails
|
|
460
|
+
that binding, so the entitlement cannot be edited in transit without leaving a trace.
|
|
461
|
+
"""
|
|
462
|
+
|
|
463
|
+
request_id: str
|
|
464
|
+
source_id: str
|
|
465
|
+
adapter_id: str
|
|
466
|
+
adapter_version: str
|
|
467
|
+
intended_use: str
|
|
468
|
+
output_grain: tuple[str, ...]
|
|
469
|
+
query: Mapping[str, Any]
|
|
470
|
+
credential_reference_id: str | None
|
|
471
|
+
requested_at: str
|
|
472
|
+
schema_version: str = SOURCE_CONTRACT_VERSION
|
|
473
|
+
reader_pin: Mapping[str, Any] | None = None
|
|
474
|
+
# The recipe's own decode budgets, carried to the clean room the same way the pin is.
|
|
475
|
+
# They are authority in the narrowing direction only -- ``ReaderBudgets.narrowed_by`` takes
|
|
476
|
+
# a per-field minimum -- so a request can tighten what the family allows and can never buy
|
|
477
|
+
# more room than it allows. ``None`` for a source that stated none, which is every source
|
|
478
|
+
# a direct fetch serves.
|
|
479
|
+
resource_caps: Mapping[str, Any] | None = None
|
|
480
|
+
|
|
481
|
+
def __post_init__(self) -> None:
|
|
482
|
+
_constant(self.schema_version, SOURCE_CONTRACT_VERSION, "request.schema_version")
|
|
483
|
+
_identifier(self.request_id, "request.request_id")
|
|
484
|
+
_identifier(self.source_id, "request.source_id")
|
|
485
|
+
_identifier(self.adapter_id, "request.adapter_id")
|
|
486
|
+
_semver(self.adapter_version, "request.adapter_version")
|
|
487
|
+
_text(self.intended_use, "request.intended_use", maximum=200)
|
|
488
|
+
_texts(
|
|
489
|
+
self.output_grain,
|
|
490
|
+
"request.output_grain",
|
|
491
|
+
nonempty=True,
|
|
492
|
+
maximum=16,
|
|
493
|
+
item_maximum=128,
|
|
494
|
+
)
|
|
495
|
+
_query(self.query, "request.query")
|
|
496
|
+
if self.credential_reference_id is not None and not _SECRET_REF.fullmatch(
|
|
497
|
+
self.credential_reference_id
|
|
498
|
+
):
|
|
499
|
+
raise SourceContractError(
|
|
500
|
+
"SECRET_REFERENCE",
|
|
501
|
+
"request.credential_reference_id",
|
|
502
|
+
"must be an opaque server-issued shr_ reference",
|
|
503
|
+
)
|
|
504
|
+
_timestamp(self.requested_at, "request.requested_at")
|
|
505
|
+
if self.reader_pin is not None:
|
|
506
|
+
_reader_pin(self.reader_pin, "request.reader_pin")
|
|
507
|
+
if self.resource_caps is not None:
|
|
508
|
+
_resource_caps(self.resource_caps, "request.resource_caps")
|
|
509
|
+
if self.resource_caps is not None and self.reader_pin is None:
|
|
510
|
+
raise SourceContractError(
|
|
511
|
+
"DECODE_CAPS",
|
|
512
|
+
"request.resource_caps",
|
|
513
|
+
"decode budgets without a Reader pin name a decode that will not happen",
|
|
514
|
+
)
|
|
515
|
+
|
|
516
|
+
@property
|
|
517
|
+
def query_digest(self) -> str:
|
|
518
|
+
return canonical_sha256(dict(self.query))
|
|
519
|
+
|
|
520
|
+
def to_dict(self) -> dict[str, Any]:
|
|
521
|
+
return {
|
|
522
|
+
"schema_version": self.schema_version,
|
|
523
|
+
"request_id": self.request_id,
|
|
524
|
+
"source_id": self.source_id,
|
|
525
|
+
"adapter_id": self.adapter_id,
|
|
526
|
+
"adapter_version": self.adapter_version,
|
|
527
|
+
"intended_use": self.intended_use,
|
|
528
|
+
"output_grain": list(self.output_grain),
|
|
529
|
+
"query": dict(self.query),
|
|
530
|
+
"credential_reference_id": self.credential_reference_id,
|
|
531
|
+
"requested_at": self.requested_at,
|
|
532
|
+
"reader_pin": None if self.reader_pin is None else dict(self.reader_pin),
|
|
533
|
+
# Present only when stated, unlike ``reader_pin``, which spells its absence as a
|
|
534
|
+
# null. The difference is deliberate and it is about evidence already written: a
|
|
535
|
+
# null here would change the canonical bytes -- and therefore ``request_digest``,
|
|
536
|
+
# which every receipt binds -- for every request in existence, including the ones
|
|
537
|
+
# that decode nothing. Omission keeps an unchanged request byte-identical to what
|
|
538
|
+
# it was, and a stated cap is inside the digest exactly as the pin is, so it cannot
|
|
539
|
+
# be edited in transit without failing the receipt's binding.
|
|
540
|
+
**({} if self.resource_caps is None else {"resource_caps": dict(self.resource_caps)}),
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
|
|
544
|
+
@dataclass(frozen=True)
|
|
545
|
+
class SourceObservation(_CanonicalContract):
|
|
546
|
+
"""Time-bound source liveness and temporal fitness evidence."""
|
|
547
|
+
|
|
548
|
+
source_id: str
|
|
549
|
+
observed_at: str
|
|
550
|
+
event_time_field: str | None
|
|
551
|
+
available_at_field: str | None
|
|
552
|
+
historical_start: str
|
|
553
|
+
historical_end: str
|
|
554
|
+
live_status: str
|
|
555
|
+
publication_delay_seconds: int
|
|
556
|
+
update_frequency_seconds: int
|
|
557
|
+
response_status: int | None
|
|
558
|
+
evidence: tuple[EvidenceReference, ...]
|
|
559
|
+
schema_version: str = OBSERVATION_CONTRACT_VERSION
|
|
560
|
+
|
|
561
|
+
def __post_init__(self) -> None:
|
|
562
|
+
_constant(self.schema_version, OBSERVATION_CONTRACT_VERSION, "observation.schema_version")
|
|
563
|
+
_identifier(self.source_id, "observation.source_id")
|
|
564
|
+
_timestamp(self.observed_at, "observation.observed_at")
|
|
565
|
+
for name, value in (
|
|
566
|
+
("event_time_field", self.event_time_field),
|
|
567
|
+
("available_at_field", self.available_at_field),
|
|
568
|
+
):
|
|
569
|
+
if value is not None:
|
|
570
|
+
_text(value, f"observation.{name}", maximum=128)
|
|
571
|
+
start = _timestamp(self.historical_start, "observation.historical_start")
|
|
572
|
+
end = _timestamp(self.historical_end, "observation.historical_end")
|
|
573
|
+
if start > end:
|
|
574
|
+
raise SourceContractError(
|
|
575
|
+
"TEMPORAL_ORDER",
|
|
576
|
+
"observation.historical_end",
|
|
577
|
+
"historical_end must not precede historical_start",
|
|
578
|
+
)
|
|
579
|
+
_choice(
|
|
580
|
+
self.live_status,
|
|
581
|
+
{"live", "delayed", "dead", "not_applicable"},
|
|
582
|
+
"observation.live_status",
|
|
583
|
+
)
|
|
584
|
+
_integer(
|
|
585
|
+
self.publication_delay_seconds,
|
|
586
|
+
"observation.publication_delay_seconds",
|
|
587
|
+
minimum=0,
|
|
588
|
+
maximum=10 * 365 * 24 * 60 * 60,
|
|
589
|
+
)
|
|
590
|
+
_integer(
|
|
591
|
+
self.update_frequency_seconds,
|
|
592
|
+
"observation.update_frequency_seconds",
|
|
593
|
+
minimum=0,
|
|
594
|
+
maximum=10 * 365 * 24 * 60 * 60,
|
|
595
|
+
)
|
|
596
|
+
if self.response_status is not None:
|
|
597
|
+
_integer(self.response_status, "observation.response_status", minimum=100, maximum=599)
|
|
598
|
+
_typed_tuple(
|
|
599
|
+
self.evidence,
|
|
600
|
+
EvidenceReference,
|
|
601
|
+
"observation.evidence",
|
|
602
|
+
nonempty=True,
|
|
603
|
+
maximum=MAX_EVIDENCE,
|
|
604
|
+
)
|
|
605
|
+
|
|
606
|
+
def to_dict(self) -> dict[str, Any]:
|
|
607
|
+
return {
|
|
608
|
+
"schema_version": self.schema_version,
|
|
609
|
+
"source_id": self.source_id,
|
|
610
|
+
"observed_at": self.observed_at,
|
|
611
|
+
"event_time_field": self.event_time_field,
|
|
612
|
+
"available_at_field": self.available_at_field,
|
|
613
|
+
"historical_start": self.historical_start,
|
|
614
|
+
"historical_end": self.historical_end,
|
|
615
|
+
"live_status": self.live_status,
|
|
616
|
+
"publication_delay_seconds": self.publication_delay_seconds,
|
|
617
|
+
"update_frequency_seconds": self.update_frequency_seconds,
|
|
618
|
+
"response_status": self.response_status,
|
|
619
|
+
"evidence": [item.to_dict() for item in self.evidence],
|
|
620
|
+
}
|
|
621
|
+
|
|
622
|
+
|
|
623
|
+
@dataclass(frozen=True)
|
|
624
|
+
class SnapshotReference(_CanonicalContract):
|
|
625
|
+
"""Immutable exact-byte snapshot reference."""
|
|
626
|
+
|
|
627
|
+
content_sha256: str
|
|
628
|
+
size_bytes: int
|
|
629
|
+
media_type: str
|
|
630
|
+
data_format: str
|
|
631
|
+
storage_name: str
|
|
632
|
+
|
|
633
|
+
def __post_init__(self) -> None:
|
|
634
|
+
_digest(self.content_sha256, "snapshot.content_sha256")
|
|
635
|
+
_integer(self.size_bytes, "snapshot.size_bytes", minimum=0, maximum=1 << 34)
|
|
636
|
+
_media_type(self.media_type, "snapshot.media_type")
|
|
637
|
+
_choice(self.data_format, DATA_FORMATS, "snapshot.data_format")
|
|
638
|
+
_safe_storage_name(self.storage_name, "snapshot.storage_name")
|
|
639
|
+
|
|
640
|
+
def to_dict(self) -> dict[str, Any]:
|
|
641
|
+
return {
|
|
642
|
+
"content_sha256": self.content_sha256,
|
|
643
|
+
"size_bytes": self.size_bytes,
|
|
644
|
+
"media_type": self.media_type,
|
|
645
|
+
"data_format": self.data_format,
|
|
646
|
+
"storage_name": self.storage_name,
|
|
647
|
+
}
|
|
648
|
+
|
|
649
|
+
|
|
650
|
+
@dataclass(frozen=True)
|
|
651
|
+
class AcquisitionReceipt(_CanonicalContract):
|
|
652
|
+
"""Digest-bound acquisition outcome, including how the bytes were decoded.
|
|
653
|
+
|
|
654
|
+
The identity chain has three links and each keeps its own name. ``fetched`` is the
|
|
655
|
+
bytes as they arrived from the source, recorded here as ``fetched_content_sha256``.
|
|
656
|
+
``acquired`` is the sealed snapshot, which under normalize-once is derived from the
|
|
657
|
+
fetched bytes rather than identical to them; its hash is ``snapshot.content_sha256``.
|
|
658
|
+
``materialized`` is the post-merge artifact the recipe layer records separately as
|
|
659
|
+
``acquired_content_sha256``. Collapsing any two of these names would let one link of
|
|
660
|
+
the chain impersonate another, so the distinction is deliberate.
|
|
661
|
+
|
|
662
|
+
Recording both hashes is what makes a derived snapshot auditable: without the fetched
|
|
663
|
+
hash, a normalized artifact has no link back to the bytes that produced it.
|
|
664
|
+
|
|
665
|
+
Decode identity -- ``family_id``, ``family_version``, ``decode_options_digest`` -- is
|
|
666
|
+
all-present or all-absent. A receipt can never claim a Reader family without pinning
|
|
667
|
+
its settings, nor settings without a family. All three are absent for an acquisition
|
|
668
|
+
that performed no decode, which is a direct fetch.
|
|
669
|
+
|
|
670
|
+
``decode_flags`` is a different kind of thing and is kept separate for that reason.
|
|
671
|
+
Decode identity records the settings a decode was *given*; a flag records a fact the
|
|
672
|
+
decode *found* -- "this spreadsheet held formulas and the values read were the ones the
|
|
673
|
+
authoring application had cached" is provenance a reader of the dataset is owed, and it
|
|
674
|
+
is not derivable from the settings. The vocabulary is closed and shared with
|
|
675
|
+
``readers.contracts.DECODE_FLAGS``, and flags are empty when nothing decoded, because a
|
|
676
|
+
flag without a decode describes nothing.
|
|
677
|
+
|
|
678
|
+
Known property, stated rather than buried: on a **Reader-decoded** acquisition,
|
|
679
|
+
classification and the rights gate scan the acquired (normalized) artifact rather than the
|
|
680
|
+
raw fetched bytes. The decode runs first and the scan reads what it produced, so a secret
|
|
681
|
+
living in a container member the Reader discards never reaches classification, and one the
|
|
682
|
+
Reader emits does. That is accepted because the normalized artifact is the data a build
|
|
683
|
+
reads, but it means a rights decision on a decoded source describes the derived form. On a
|
|
684
|
+
**direct fetch** nothing is derived and the fetched bytes are the scanned bytes, which is why
|
|
685
|
+
the two hashes above coincide there and diverge here.
|
|
686
|
+
"""
|
|
687
|
+
|
|
688
|
+
receipt_id: str
|
|
689
|
+
request_digest: str
|
|
690
|
+
source_id: str
|
|
691
|
+
adapter: AdapterDescriptor
|
|
692
|
+
query_digest: str
|
|
693
|
+
snapshot: SnapshotReference
|
|
694
|
+
fetched_content_sha256: str
|
|
695
|
+
family_id: str | None
|
|
696
|
+
family_version: str | None
|
|
697
|
+
decode_options_digest: str | None
|
|
698
|
+
source_uri: str
|
|
699
|
+
final_uri: str
|
|
700
|
+
event_time_field: str | None
|
|
701
|
+
observed_at: str
|
|
702
|
+
available_at: str
|
|
703
|
+
ingested_at: str
|
|
704
|
+
classification: str
|
|
705
|
+
rights_digest: str
|
|
706
|
+
retention_digest: str
|
|
707
|
+
observation_digest: str
|
|
708
|
+
transport_evidence_digest: str
|
|
709
|
+
schema_version: str = RECEIPT_CONTRACT_VERSION
|
|
710
|
+
decode_flags: tuple[str, ...] = ()
|
|
711
|
+
|
|
712
|
+
def __post_init__(self) -> None:
|
|
713
|
+
_constant(self.schema_version, RECEIPT_CONTRACT_VERSION, "receipt.schema_version")
|
|
714
|
+
_identifier(self.receipt_id, "receipt.receipt_id")
|
|
715
|
+
_digest(self.request_digest, "receipt.request_digest")
|
|
716
|
+
_identifier(self.source_id, "receipt.source_id")
|
|
717
|
+
if not isinstance(self.adapter, AdapterDescriptor):
|
|
718
|
+
raise SourceContractError("TYPE", "receipt.adapter", "must be an AdapterDescriptor")
|
|
719
|
+
_digest(self.query_digest, "receipt.query_digest")
|
|
720
|
+
if not isinstance(self.snapshot, SnapshotReference):
|
|
721
|
+
raise SourceContractError("TYPE", "receipt.snapshot", "must be a SnapshotReference")
|
|
722
|
+
_digest(self.fetched_content_sha256, "receipt.fetched_content_sha256")
|
|
723
|
+
decode_identity = (self.family_id, self.family_version, self.decode_options_digest)
|
|
724
|
+
present = tuple(item is not None for item in decode_identity)
|
|
725
|
+
if any(present) and not all(present):
|
|
726
|
+
raise SourceContractError(
|
|
727
|
+
"DECODE_IDENTITY",
|
|
728
|
+
"receipt.family_id",
|
|
729
|
+
"family_id, family_version, and decode_options_digest "
|
|
730
|
+
"are all present or all absent",
|
|
731
|
+
)
|
|
732
|
+
if all(present):
|
|
733
|
+
_identifier(self.family_id, "receipt.family_id")
|
|
734
|
+
_semver(self.family_version, "receipt.family_version")
|
|
735
|
+
_digest(self.decode_options_digest, "receipt.decode_options_digest")
|
|
736
|
+
_decode_flags(self.decode_flags, "receipt.decode_flags", family_id=self.family_id)
|
|
737
|
+
_acquisition_uri(
|
|
738
|
+
self.source_uri,
|
|
739
|
+
"receipt.source_uri",
|
|
740
|
+
source_class=self.adapter.source_class,
|
|
741
|
+
)
|
|
742
|
+
_acquisition_uri(
|
|
743
|
+
self.final_uri,
|
|
744
|
+
"receipt.final_uri",
|
|
745
|
+
source_class=self.adapter.source_class,
|
|
746
|
+
)
|
|
747
|
+
if self.event_time_field is not None:
|
|
748
|
+
_text(self.event_time_field, "receipt.event_time_field", maximum=128)
|
|
749
|
+
observed = _timestamp(self.observed_at, "receipt.observed_at")
|
|
750
|
+
available = _timestamp(self.available_at, "receipt.available_at")
|
|
751
|
+
ingested = _timestamp(self.ingested_at, "receipt.ingested_at")
|
|
752
|
+
if observed > ingested or available > ingested:
|
|
753
|
+
raise SourceContractError(
|
|
754
|
+
"TEMPORAL_ORDER",
|
|
755
|
+
"receipt.ingested_at",
|
|
756
|
+
"ingested_at must not precede observed_at or available_at",
|
|
757
|
+
)
|
|
758
|
+
_choice(self.classification, CLASSIFICATIONS, "receipt.classification")
|
|
759
|
+
if self.classification == "rejected":
|
|
760
|
+
raise SourceContractError(
|
|
761
|
+
"REJECTED_NOT_RECEIPT",
|
|
762
|
+
"receipt.classification",
|
|
763
|
+
"rejected content cannot receive an admitted acquisition receipt",
|
|
764
|
+
)
|
|
765
|
+
_digest(self.rights_digest, "receipt.rights_digest")
|
|
766
|
+
_digest(self.retention_digest, "receipt.retention_digest")
|
|
767
|
+
_digest(self.observation_digest, "receipt.observation_digest")
|
|
768
|
+
_digest(self.transport_evidence_digest, "receipt.transport_evidence_digest")
|
|
769
|
+
|
|
770
|
+
def to_dict(self) -> dict[str, Any]:
|
|
771
|
+
return {
|
|
772
|
+
"schema_version": self.schema_version,
|
|
773
|
+
"receipt_id": self.receipt_id,
|
|
774
|
+
"request_digest": self.request_digest,
|
|
775
|
+
"source_id": self.source_id,
|
|
776
|
+
"adapter": self.adapter.to_dict(),
|
|
777
|
+
"query_digest": self.query_digest,
|
|
778
|
+
"snapshot": self.snapshot.to_dict(),
|
|
779
|
+
"fetched_content_sha256": self.fetched_content_sha256,
|
|
780
|
+
"family_id": self.family_id,
|
|
781
|
+
"family_version": self.family_version,
|
|
782
|
+
"decode_options_digest": self.decode_options_digest,
|
|
783
|
+
"decode_flags": list(self.decode_flags),
|
|
784
|
+
"source_uri": self.source_uri,
|
|
785
|
+
"final_uri": self.final_uri,
|
|
786
|
+
"event_time_field": self.event_time_field,
|
|
787
|
+
"observed_at": self.observed_at,
|
|
788
|
+
"available_at": self.available_at,
|
|
789
|
+
"ingested_at": self.ingested_at,
|
|
790
|
+
"classification": self.classification,
|
|
791
|
+
"rights_digest": self.rights_digest,
|
|
792
|
+
"retention_digest": self.retention_digest,
|
|
793
|
+
"observation_digest": self.observation_digest,
|
|
794
|
+
"transport_evidence_digest": self.transport_evidence_digest,
|
|
795
|
+
}
|
|
796
|
+
|
|
797
|
+
|
|
798
|
+
# Exact keys written by ``AcquisitionReceipt.to_dict``. Read paths validate this shape because the
|
|
799
|
+
# schema label alone does not distinguish older key sets. Derivation prevents drift from the
|
|
800
|
+
# dataclass.
|
|
801
|
+
RECEIPT_DOCUMENT_KEYS: frozenset[str] = frozenset(
|
|
802
|
+
field.name for field in fields(AcquisitionReceipt)
|
|
803
|
+
)
|
|
804
|
+
|
|
805
|
+
|
|
806
|
+
@dataclass(frozen=True)
|
|
807
|
+
class PeerAttempt(_CanonicalContract):
|
|
808
|
+
"""One canonical attempt against a resolver-approved public address."""
|
|
809
|
+
|
|
810
|
+
approved_ip: str
|
|
811
|
+
outcome: str
|
|
812
|
+
failure_code: str | None
|
|
813
|
+
|
|
814
|
+
def __post_init__(self) -> None:
|
|
815
|
+
_text(self.approved_ip, "peer_attempt.approved_ip", maximum=64)
|
|
816
|
+
try:
|
|
817
|
+
address = ipaddress.ip_address(self.approved_ip)
|
|
818
|
+
except ValueError as error:
|
|
819
|
+
raise SourceContractError(
|
|
820
|
+
"VALUE", "peer_attempt.approved_ip", "must be an IP address"
|
|
821
|
+
) from error
|
|
822
|
+
if str(address) != self.approved_ip or not address.is_global:
|
|
823
|
+
raise SourceContractError(
|
|
824
|
+
"VALUE",
|
|
825
|
+
"peer_attempt.approved_ip",
|
|
826
|
+
"must be a canonical public IP address",
|
|
827
|
+
)
|
|
828
|
+
_choice(self.outcome, PEER_ATTEMPT_OUTCOMES, "peer_attempt.outcome")
|
|
829
|
+
if self.outcome == "response":
|
|
830
|
+
if self.failure_code is not None:
|
|
831
|
+
raise SourceContractError(
|
|
832
|
+
"VALUE", "peer_attempt.failure_code", "must be null for a response"
|
|
833
|
+
)
|
|
834
|
+
else:
|
|
835
|
+
_text(self.failure_code, "peer_attempt.failure_code", maximum=128)
|
|
836
|
+
|
|
837
|
+
def to_dict(self) -> dict[str, Any]:
|
|
838
|
+
return {
|
|
839
|
+
"approved_ip": self.approved_ip,
|
|
840
|
+
"outcome": self.outcome,
|
|
841
|
+
"failure_code": self.failure_code,
|
|
842
|
+
}
|
|
843
|
+
|
|
844
|
+
|
|
845
|
+
@dataclass(frozen=True)
|
|
846
|
+
class FetchedMember(_CanonicalContract):
|
|
847
|
+
"""One measured response body in a v2 multi-member acquisition."""
|
|
848
|
+
|
|
849
|
+
role: str
|
|
850
|
+
sequence: int
|
|
851
|
+
url: str
|
|
852
|
+
resolution_digest: str
|
|
853
|
+
connected_peer: str
|
|
854
|
+
status: int
|
|
855
|
+
response_headers_digest: str
|
|
856
|
+
response_body_size_bytes: int
|
|
857
|
+
response_body_sha256: str
|
|
858
|
+
requested_range: str | None
|
|
859
|
+
observed_content_range: str | None
|
|
860
|
+
etag: str | None
|
|
861
|
+
last_modified: str | None
|
|
862
|
+
peer_attempts: tuple[PeerAttempt, ...]
|
|
863
|
+
|
|
864
|
+
def __post_init__(self) -> None:
|
|
865
|
+
if isinstance(self.peer_attempts, list):
|
|
866
|
+
parsed_attempts: list[PeerAttempt] = []
|
|
867
|
+
for index, raw_attempt in enumerate(self.peer_attempts):
|
|
868
|
+
attempt = _object(raw_attempt, f"fetched_member.peer_attempts[{index}]")
|
|
869
|
+
_exact(
|
|
870
|
+
attempt,
|
|
871
|
+
{"approved_ip", "outcome", "failure_code"},
|
|
872
|
+
f"fetched_member.peer_attempts[{index}]",
|
|
873
|
+
)
|
|
874
|
+
parsed_attempts.append(PeerAttempt(**attempt))
|
|
875
|
+
object.__setattr__(self, "peer_attempts", tuple(parsed_attempts))
|
|
876
|
+
_choice(
|
|
877
|
+
self.role,
|
|
878
|
+
{"cycle_probe", "sidecar_initial", "selected_slice", "sidecar_final"},
|
|
879
|
+
"fetched_member.role",
|
|
880
|
+
)
|
|
881
|
+
_integer(self.sequence, "fetched_member.sequence", minimum=0, maximum=255)
|
|
882
|
+
_text(self.url, "fetched_member.url", maximum=2_048)
|
|
883
|
+
_digest(self.resolution_digest, "fetched_member.resolution_digest")
|
|
884
|
+
_text(self.connected_peer, "fetched_member.connected_peer", maximum=64)
|
|
885
|
+
_integer(self.status, "fetched_member.status", minimum=100, maximum=599)
|
|
886
|
+
_digest(self.response_headers_digest, "fetched_member.response_headers_digest")
|
|
887
|
+
_integer(
|
|
888
|
+
self.response_body_size_bytes,
|
|
889
|
+
"fetched_member.response_body_size_bytes",
|
|
890
|
+
minimum=0,
|
|
891
|
+
maximum=1 << 34,
|
|
892
|
+
)
|
|
893
|
+
_digest(self.response_body_sha256, "fetched_member.response_body_sha256")
|
|
894
|
+
for name, value, maximum in (
|
|
895
|
+
("requested_range", self.requested_range, 128),
|
|
896
|
+
("observed_content_range", self.observed_content_range, 128),
|
|
897
|
+
("etag", self.etag, 1_024),
|
|
898
|
+
("last_modified", self.last_modified, 1_024),
|
|
899
|
+
):
|
|
900
|
+
if value is not None:
|
|
901
|
+
_text(value, f"fetched_member.{name}", maximum=maximum)
|
|
902
|
+
if (
|
|
903
|
+
not isinstance(self.peer_attempts, tuple)
|
|
904
|
+
or not 1 <= len(self.peer_attempts) <= 32
|
|
905
|
+
or any(not isinstance(attempt, PeerAttempt) for attempt in self.peer_attempts)
|
|
906
|
+
):
|
|
907
|
+
raise SourceContractError(
|
|
908
|
+
"TYPE",
|
|
909
|
+
"fetched_member.peer_attempts",
|
|
910
|
+
"must be a non-empty bounded tuple of typed attempts",
|
|
911
|
+
)
|
|
912
|
+
if any(attempt.outcome != "transport_failure" for attempt in self.peer_attempts[:-1]):
|
|
913
|
+
raise SourceContractError(
|
|
914
|
+
"VALUE",
|
|
915
|
+
"fetched_member.peer_attempts",
|
|
916
|
+
"only transport failures may precede the terminal attempt",
|
|
917
|
+
)
|
|
918
|
+
if self.peer_attempts[-1].outcome != "response":
|
|
919
|
+
raise SourceContractError(
|
|
920
|
+
"VALUE",
|
|
921
|
+
"fetched_member.peer_attempts",
|
|
922
|
+
"a retained response must terminate with a successful response attempt",
|
|
923
|
+
)
|
|
924
|
+
|
|
925
|
+
def to_dict(self) -> dict[str, Any]:
|
|
926
|
+
document = {field.name: getattr(self, field.name) for field in fields(self)}
|
|
927
|
+
document["peer_attempts"] = [attempt.to_dict() for attempt in self.peer_attempts]
|
|
928
|
+
return document
|
|
929
|
+
|
|
930
|
+
def hop_document(self) -> dict[str, Any]:
|
|
931
|
+
return {
|
|
932
|
+
key: value for key, value in self.to_dict().items() if key not in {"role", "sequence"}
|
|
933
|
+
}
|
|
934
|
+
|
|
935
|
+
|
|
936
|
+
def fetched_source_revision_digest(
|
|
937
|
+
cycle: str,
|
|
938
|
+
full_object_size_bytes: int,
|
|
939
|
+
members: tuple[FetchedMember, ...],
|
|
940
|
+
) -> str:
|
|
941
|
+
"""Derive the sealed source revision solely from measured member evidence."""
|
|
942
|
+
|
|
943
|
+
return canonical_sha256(
|
|
944
|
+
{
|
|
945
|
+
"cycle": cycle,
|
|
946
|
+
"full_object_size_bytes": full_object_size_bytes,
|
|
947
|
+
"members": [
|
|
948
|
+
{
|
|
949
|
+
"role": member.role,
|
|
950
|
+
"url": member.url,
|
|
951
|
+
"response_body_size_bytes": member.response_body_size_bytes,
|
|
952
|
+
"response_body_sha256": member.response_body_sha256,
|
|
953
|
+
"requested_range": member.requested_range,
|
|
954
|
+
"observed_content_range": member.observed_content_range,
|
|
955
|
+
"etag": member.etag,
|
|
956
|
+
"last_modified": member.last_modified,
|
|
957
|
+
}
|
|
958
|
+
for member in members
|
|
959
|
+
],
|
|
960
|
+
}
|
|
961
|
+
)
|
|
962
|
+
|
|
963
|
+
|
|
964
|
+
@dataclass(frozen=True)
|
|
965
|
+
class AcquisitionReceiptV2(_CanonicalContract):
|
|
966
|
+
"""Strict multi-response receipt used by range-fetch plus Reader composition.
|
|
967
|
+
|
|
968
|
+
Version 1 remains unchanged. Version 2 retains every response body, validator and range
|
|
969
|
+
in order, so its transport digest and efficiency result are recomputable from the sealed
|
|
970
|
+
document rather than supplied as summary testimony.
|
|
971
|
+
"""
|
|
972
|
+
|
|
973
|
+
receipt_id: str
|
|
974
|
+
request_digest: str
|
|
975
|
+
source_id: str
|
|
976
|
+
adapter: AdapterDescriptor
|
|
977
|
+
query_digest: str
|
|
978
|
+
snapshot: SnapshotReference
|
|
979
|
+
fetched_content_sha256: str
|
|
980
|
+
family_id: str
|
|
981
|
+
family_version: str
|
|
982
|
+
decode_options_digest: str
|
|
983
|
+
source_uri: str
|
|
984
|
+
final_uri: str
|
|
985
|
+
event_time_field: str | None
|
|
986
|
+
observed_at: str
|
|
987
|
+
available_at: str
|
|
988
|
+
ingested_at: str
|
|
989
|
+
classification: str
|
|
990
|
+
rights_digest: str
|
|
991
|
+
retention_digest: str
|
|
992
|
+
observation_digest: str
|
|
993
|
+
transport_evidence_digest: str
|
|
994
|
+
fetched_members: tuple[FetchedMember, ...]
|
|
995
|
+
cycle: str
|
|
996
|
+
revision_lock_digest: str
|
|
997
|
+
source_revision_digest: str
|
|
998
|
+
full_object_size_bytes: int
|
|
999
|
+
efficiency_numerator: int
|
|
1000
|
+
efficiency_denominator: int
|
|
1001
|
+
fetch_policy_digest: str
|
|
1002
|
+
decode_policy_digest: str
|
|
1003
|
+
parsed_schema_digest: str
|
|
1004
|
+
schema_version: str = RECEIPT_CONTRACT_VERSION_V2
|
|
1005
|
+
decode_flags: tuple[str, ...] = ()
|
|
1006
|
+
|
|
1007
|
+
def __post_init__(self) -> None:
|
|
1008
|
+
_constant(self.schema_version, RECEIPT_CONTRACT_VERSION_V2, "receipt.schema_version")
|
|
1009
|
+
_identifier(self.receipt_id, "receipt.receipt_id")
|
|
1010
|
+
_digest(self.request_digest, "receipt.request_digest")
|
|
1011
|
+
_identifier(self.source_id, "receipt.source_id")
|
|
1012
|
+
if not isinstance(self.adapter, AdapterDescriptor):
|
|
1013
|
+
raise SourceContractError("TYPE", "receipt.adapter", "must be an AdapterDescriptor")
|
|
1014
|
+
_digest(self.query_digest, "receipt.query_digest")
|
|
1015
|
+
if not isinstance(self.snapshot, SnapshotReference):
|
|
1016
|
+
raise SourceContractError("TYPE", "receipt.snapshot", "must be a SnapshotReference")
|
|
1017
|
+
for path, value in (
|
|
1018
|
+
("receipt.fetched_content_sha256", self.fetched_content_sha256),
|
|
1019
|
+
("receipt.decode_options_digest", self.decode_options_digest),
|
|
1020
|
+
("receipt.rights_digest", self.rights_digest),
|
|
1021
|
+
("receipt.retention_digest", self.retention_digest),
|
|
1022
|
+
("receipt.observation_digest", self.observation_digest),
|
|
1023
|
+
("receipt.transport_evidence_digest", self.transport_evidence_digest),
|
|
1024
|
+
("receipt.revision_lock_digest", self.revision_lock_digest),
|
|
1025
|
+
("receipt.source_revision_digest", self.source_revision_digest),
|
|
1026
|
+
("receipt.fetch_policy_digest", self.fetch_policy_digest),
|
|
1027
|
+
("receipt.decode_policy_digest", self.decode_policy_digest),
|
|
1028
|
+
("receipt.parsed_schema_digest", self.parsed_schema_digest),
|
|
1029
|
+
):
|
|
1030
|
+
_digest(value, path)
|
|
1031
|
+
_identifier(self.family_id, "receipt.family_id")
|
|
1032
|
+
_semver(self.family_version, "receipt.family_version")
|
|
1033
|
+
_decode_flags(self.decode_flags, "receipt.decode_flags", family_id=self.family_id)
|
|
1034
|
+
_acquisition_uri(
|
|
1035
|
+
self.source_uri, "receipt.source_uri", source_class=self.adapter.source_class
|
|
1036
|
+
)
|
|
1037
|
+
_acquisition_uri(
|
|
1038
|
+
self.final_uri, "receipt.final_uri", source_class=self.adapter.source_class
|
|
1039
|
+
)
|
|
1040
|
+
if self.event_time_field is not None:
|
|
1041
|
+
_text(self.event_time_field, "receipt.event_time_field", maximum=128)
|
|
1042
|
+
observed = _timestamp(self.observed_at, "receipt.observed_at")
|
|
1043
|
+
available = _timestamp(self.available_at, "receipt.available_at")
|
|
1044
|
+
ingested = _timestamp(self.ingested_at, "receipt.ingested_at")
|
|
1045
|
+
if observed > ingested or available > ingested:
|
|
1046
|
+
raise SourceContractError(
|
|
1047
|
+
"TEMPORAL_ORDER",
|
|
1048
|
+
"receipt.ingested_at",
|
|
1049
|
+
"ingested_at must not precede observed_at or available_at",
|
|
1050
|
+
)
|
|
1051
|
+
_choice(self.classification, CLASSIFICATIONS - {"rejected"}, "receipt.classification")
|
|
1052
|
+
_text(self.cycle, "receipt.cycle", maximum=128)
|
|
1053
|
+
_integer(
|
|
1054
|
+
self.full_object_size_bytes,
|
|
1055
|
+
"receipt.full_object_size_bytes",
|
|
1056
|
+
minimum=1,
|
|
1057
|
+
maximum=1 << 44,
|
|
1058
|
+
)
|
|
1059
|
+
_integer(
|
|
1060
|
+
self.efficiency_numerator,
|
|
1061
|
+
"receipt.efficiency_numerator",
|
|
1062
|
+
minimum=1,
|
|
1063
|
+
maximum=1 << 44,
|
|
1064
|
+
)
|
|
1065
|
+
_integer(
|
|
1066
|
+
self.efficiency_denominator,
|
|
1067
|
+
"receipt.efficiency_denominator",
|
|
1068
|
+
minimum=1,
|
|
1069
|
+
maximum=1 << 44,
|
|
1070
|
+
)
|
|
1071
|
+
if self.efficiency_denominator != self.full_object_size_bytes:
|
|
1072
|
+
raise SourceContractError(
|
|
1073
|
+
"RECEIPT_ACCOUNTING",
|
|
1074
|
+
"receipt.efficiency_denominator",
|
|
1075
|
+
"must equal the complete source-object size",
|
|
1076
|
+
)
|
|
1077
|
+
if not isinstance(self.fetched_members, tuple) or not 4 <= len(self.fetched_members) <= 256:
|
|
1078
|
+
raise SourceContractError(
|
|
1079
|
+
"RECEIPT_MEMBERS", "receipt.fetched_members", "must be a bounded typed tuple"
|
|
1080
|
+
)
|
|
1081
|
+
if any(not isinstance(member, FetchedMember) for member in self.fetched_members):
|
|
1082
|
+
raise SourceContractError(
|
|
1083
|
+
"RECEIPT_MEMBERS", "receipt.fetched_members", "contains an untyped member"
|
|
1084
|
+
)
|
|
1085
|
+
if tuple(member.sequence for member in self.fetched_members) != tuple(
|
|
1086
|
+
range(len(self.fetched_members))
|
|
1087
|
+
):
|
|
1088
|
+
raise SourceContractError(
|
|
1089
|
+
"RECEIPT_MEMBERS", "receipt.fetched_members", "sequence is not contiguous"
|
|
1090
|
+
)
|
|
1091
|
+
roles = tuple(member.role for member in self.fetched_members)
|
|
1092
|
+
collapsed = tuple(
|
|
1093
|
+
role for index, role in enumerate(roles) if index == 0 or role != roles[index - 1]
|
|
1094
|
+
)
|
|
1095
|
+
if collapsed != ("sidecar_initial", "cycle_probe", "selected_slice", "sidecar_final"):
|
|
1096
|
+
raise SourceContractError(
|
|
1097
|
+
"RECEIPT_MEMBERS", "receipt.fetched_members", "acquisition stages are incomplete"
|
|
1098
|
+
)
|
|
1099
|
+
terminal_by_role = {
|
|
1100
|
+
role: self.fetched_members[
|
|
1101
|
+
max(index for index, value in enumerate(roles) if value == role)
|
|
1102
|
+
]
|
|
1103
|
+
for role in collapsed
|
|
1104
|
+
}
|
|
1105
|
+
if any(
|
|
1106
|
+
member.etag is None and member.last_modified is None
|
|
1107
|
+
for member in terminal_by_role.values()
|
|
1108
|
+
):
|
|
1109
|
+
raise SourceContractError(
|
|
1110
|
+
"RECEIPT_VALIDATOR",
|
|
1111
|
+
"receipt.fetched_members",
|
|
1112
|
+
"each fetched resource must carry ETag or Last-Modified evidence",
|
|
1113
|
+
)
|
|
1114
|
+
initial = terminal_by_role["sidecar_initial"]
|
|
1115
|
+
final = terminal_by_role["sidecar_final"]
|
|
1116
|
+
if (
|
|
1117
|
+
initial.url != final.url
|
|
1118
|
+
or initial.response_body_size_bytes != final.response_body_size_bytes
|
|
1119
|
+
or initial.response_body_sha256 != final.response_body_sha256
|
|
1120
|
+
or (initial.etag, initial.last_modified) != (final.etag, final.last_modified)
|
|
1121
|
+
):
|
|
1122
|
+
raise SourceContractError(
|
|
1123
|
+
"RECEIPT_REVISION",
|
|
1124
|
+
"receipt.fetched_members",
|
|
1125
|
+
"terminal sidecar evidence differs from the initial sidecar",
|
|
1126
|
+
)
|
|
1127
|
+
probe = terminal_by_role["cycle_probe"]
|
|
1128
|
+
selected = terminal_by_role["selected_slice"]
|
|
1129
|
+
if (probe.etag, probe.last_modified) != (selected.etag, selected.last_modified):
|
|
1130
|
+
raise SourceContractError(
|
|
1131
|
+
"RECEIPT_REVISION",
|
|
1132
|
+
"receipt.fetched_members",
|
|
1133
|
+
"cycle probe and selected object member name different revisions",
|
|
1134
|
+
)
|
|
1135
|
+
measured = sum(member.response_body_size_bytes for member in self.fetched_members)
|
|
1136
|
+
if measured != self.efficiency_numerator or measured * 100 > self.efficiency_denominator:
|
|
1137
|
+
raise SourceContractError(
|
|
1138
|
+
"RECEIPT_ACCOUNTING",
|
|
1139
|
+
"receipt.efficiency_numerator",
|
|
1140
|
+
"does not equal all response bodies or exceeds one percent",
|
|
1141
|
+
)
|
|
1142
|
+
if (
|
|
1143
|
+
fetched_source_revision_digest(
|
|
1144
|
+
self.cycle, self.full_object_size_bytes, self.fetched_members
|
|
1145
|
+
)
|
|
1146
|
+
!= self.source_revision_digest
|
|
1147
|
+
):
|
|
1148
|
+
raise SourceContractError(
|
|
1149
|
+
"RECEIPT_REVISION",
|
|
1150
|
+
"receipt.source_revision_digest",
|
|
1151
|
+
"does not rederive from the measured member evidence",
|
|
1152
|
+
)
|
|
1153
|
+
if (
|
|
1154
|
+
canonical_sha256([member.hop_document() for member in self.fetched_members])
|
|
1155
|
+
!= self.transport_evidence_digest
|
|
1156
|
+
):
|
|
1157
|
+
raise SourceContractError(
|
|
1158
|
+
"RECEIPT_TRANSPORT",
|
|
1159
|
+
"receipt.transport_evidence_digest",
|
|
1160
|
+
"does not bind the complete fetched-member sequence",
|
|
1161
|
+
)
|
|
1162
|
+
|
|
1163
|
+
def to_dict(self) -> dict[str, Any]:
|
|
1164
|
+
document = {field.name: getattr(self, field.name) for field in fields(self)}
|
|
1165
|
+
document["adapter"] = self.adapter.to_dict()
|
|
1166
|
+
document["snapshot"] = self.snapshot.to_dict()
|
|
1167
|
+
document["fetched_members"] = [member.to_dict() for member in self.fetched_members]
|
|
1168
|
+
document["decode_flags"] = list(self.decode_flags)
|
|
1169
|
+
return document
|
|
1170
|
+
|
|
1171
|
+
|
|
1172
|
+
RECEIPT_DOCUMENT_KEYS_V2: frozenset[str] = frozenset(
|
|
1173
|
+
field.name for field in fields(AcquisitionReceiptV2)
|
|
1174
|
+
)
|
|
1175
|
+
|
|
1176
|
+
|
|
1177
|
+
def parse_acquisition_receipt_v2(value: Any) -> AcquisitionReceiptV2:
|
|
1178
|
+
"""Parse and rederive every v2 receipt invariant from its sealed JSON document."""
|
|
1179
|
+
|
|
1180
|
+
data = _object(value, "receipt")
|
|
1181
|
+
_exact(data, set(RECEIPT_DOCUMENT_KEYS_V2), "receipt")
|
|
1182
|
+
snapshot_data = _object(data["snapshot"], "receipt.snapshot")
|
|
1183
|
+
_exact(
|
|
1184
|
+
snapshot_data,
|
|
1185
|
+
{"content_sha256", "size_bytes", "media_type", "data_format", "storage_name"},
|
|
1186
|
+
"receipt.snapshot",
|
|
1187
|
+
)
|
|
1188
|
+
raw_members = data["fetched_members"]
|
|
1189
|
+
if not isinstance(raw_members, list):
|
|
1190
|
+
raise SourceContractError("TYPE", "receipt.fetched_members", "must be an array")
|
|
1191
|
+
member_keys = {field.name for field in fields(FetchedMember)}
|
|
1192
|
+
members: list[FetchedMember] = []
|
|
1193
|
+
for index, raw_member in enumerate(raw_members):
|
|
1194
|
+
member_data = _object(raw_member, f"receipt.fetched_members[{index}]")
|
|
1195
|
+
_exact(member_data, member_keys, f"receipt.fetched_members[{index}]")
|
|
1196
|
+
members.append(FetchedMember(**member_data))
|
|
1197
|
+
kwargs = dict(data)
|
|
1198
|
+
kwargs["adapter"] = parse_adapter_descriptor(data["adapter"])
|
|
1199
|
+
kwargs["snapshot"] = SnapshotReference(**snapshot_data)
|
|
1200
|
+
kwargs["fetched_members"] = tuple(members)
|
|
1201
|
+
kwargs["decode_flags"] = tuple(data["decode_flags"])
|
|
1202
|
+
return AcquisitionReceiptV2(**kwargs)
|
|
1203
|
+
|
|
1204
|
+
|
|
1205
|
+
@dataclass(frozen=True)
|
|
1206
|
+
class CollectionMemberEvidence(_CanonicalContract):
|
|
1207
|
+
"""One exact, ordered input to a v3 collection acquisition.
|
|
1208
|
+
|
|
1209
|
+
A collection receipt names both the bytes that arrived and the canonical bytes the
|
|
1210
|
+
collection assembler consumed. Keeping those hashes separate preserves the same
|
|
1211
|
+
fetched-versus-normalized identity chain as a v1 receipt while making the chain explicit
|
|
1212
|
+
for every member rather than collapsing it into aggregate testimony.
|
|
1213
|
+
"""
|
|
1214
|
+
|
|
1215
|
+
member_id: str
|
|
1216
|
+
sequence: int
|
|
1217
|
+
source_uri: str
|
|
1218
|
+
final_uri: str
|
|
1219
|
+
fetched_content_sha256: str
|
|
1220
|
+
fetched_size_bytes: int
|
|
1221
|
+
normalized_content_sha256: str
|
|
1222
|
+
normalized_size_bytes: int
|
|
1223
|
+
request_count: int
|
|
1224
|
+
row_count: int
|
|
1225
|
+
column_count: int
|
|
1226
|
+
declared_cell_count: int
|
|
1227
|
+
parsed_schema_digest: str
|
|
1228
|
+
transport_evidence_digest: str
|
|
1229
|
+
decode_flags: tuple[str, ...] = ()
|
|
1230
|
+
|
|
1231
|
+
def __post_init__(self) -> None:
|
|
1232
|
+
_identifier(self.member_id, "collection_member.member_id")
|
|
1233
|
+
_integer(self.sequence, "collection_member.sequence", minimum=0, maximum=255)
|
|
1234
|
+
_https_uri(self.source_uri, "collection_member.source_uri")
|
|
1235
|
+
_https_uri(self.final_uri, "collection_member.final_uri")
|
|
1236
|
+
_digest(
|
|
1237
|
+
self.fetched_content_sha256,
|
|
1238
|
+
"collection_member.fetched_content_sha256",
|
|
1239
|
+
)
|
|
1240
|
+
_integer(
|
|
1241
|
+
self.fetched_size_bytes,
|
|
1242
|
+
"collection_member.fetched_size_bytes",
|
|
1243
|
+
minimum=1,
|
|
1244
|
+
maximum=1 << 34,
|
|
1245
|
+
)
|
|
1246
|
+
_digest(
|
|
1247
|
+
self.normalized_content_sha256,
|
|
1248
|
+
"collection_member.normalized_content_sha256",
|
|
1249
|
+
)
|
|
1250
|
+
_integer(
|
|
1251
|
+
self.normalized_size_bytes,
|
|
1252
|
+
"collection_member.normalized_size_bytes",
|
|
1253
|
+
minimum=1,
|
|
1254
|
+
maximum=1 << 34,
|
|
1255
|
+
)
|
|
1256
|
+
_integer(
|
|
1257
|
+
self.request_count,
|
|
1258
|
+
"collection_member.request_count",
|
|
1259
|
+
minimum=1,
|
|
1260
|
+
maximum=16,
|
|
1261
|
+
)
|
|
1262
|
+
_integer(
|
|
1263
|
+
self.row_count,
|
|
1264
|
+
"collection_member.row_count",
|
|
1265
|
+
minimum=0,
|
|
1266
|
+
maximum=10_000_000,
|
|
1267
|
+
)
|
|
1268
|
+
_integer(
|
|
1269
|
+
self.column_count,
|
|
1270
|
+
"collection_member.column_count",
|
|
1271
|
+
minimum=1,
|
|
1272
|
+
maximum=10_000,
|
|
1273
|
+
)
|
|
1274
|
+
_integer(
|
|
1275
|
+
self.declared_cell_count,
|
|
1276
|
+
"collection_member.declared_cell_count",
|
|
1277
|
+
minimum=0,
|
|
1278
|
+
maximum=(1 << 63) - 1,
|
|
1279
|
+
)
|
|
1280
|
+
if self.declared_cell_count != self.row_count * self.column_count:
|
|
1281
|
+
raise SourceContractError(
|
|
1282
|
+
"COLLECTION_CELL_LIMIT",
|
|
1283
|
+
"collection_member.declared_cell_count",
|
|
1284
|
+
"must equal the authenticated normalized table geometry",
|
|
1285
|
+
)
|
|
1286
|
+
_digest(self.parsed_schema_digest, "collection_member.parsed_schema_digest")
|
|
1287
|
+
_digest(
|
|
1288
|
+
self.transport_evidence_digest,
|
|
1289
|
+
"collection_member.transport_evidence_digest",
|
|
1290
|
+
)
|
|
1291
|
+
# A member cannot decide which family ran, but any stated flag still has to be a real,
|
|
1292
|
+
# canonical Reader flag. The enclosing receipt separately proves that a family was
|
|
1293
|
+
# pinned whenever any member reported one.
|
|
1294
|
+
_decode_flags(
|
|
1295
|
+
self.decode_flags,
|
|
1296
|
+
"collection_member.decode_flags",
|
|
1297
|
+
family_id="collection.member",
|
|
1298
|
+
)
|
|
1299
|
+
|
|
1300
|
+
def to_dict(self) -> dict[str, Any]:
|
|
1301
|
+
return {
|
|
1302
|
+
"member_id": self.member_id,
|
|
1303
|
+
"sequence": self.sequence,
|
|
1304
|
+
"source_uri": self.source_uri,
|
|
1305
|
+
"final_uri": self.final_uri,
|
|
1306
|
+
"fetched_content_sha256": self.fetched_content_sha256,
|
|
1307
|
+
"fetched_size_bytes": self.fetched_size_bytes,
|
|
1308
|
+
"normalized_content_sha256": self.normalized_content_sha256,
|
|
1309
|
+
"normalized_size_bytes": self.normalized_size_bytes,
|
|
1310
|
+
"request_count": self.request_count,
|
|
1311
|
+
"row_count": self.row_count,
|
|
1312
|
+
"column_count": self.column_count,
|
|
1313
|
+
"declared_cell_count": self.declared_cell_count,
|
|
1314
|
+
"parsed_schema_digest": self.parsed_schema_digest,
|
|
1315
|
+
"transport_evidence_digest": self.transport_evidence_digest,
|
|
1316
|
+
"decode_flags": list(self.decode_flags),
|
|
1317
|
+
}
|
|
1318
|
+
|
|
1319
|
+
|
|
1320
|
+
def collection_member_manifest_digest(members: tuple[CollectionMemberEvidence, ...]) -> str:
|
|
1321
|
+
"""Digest the complete ordered member evidence with the contract's one formula."""
|
|
1322
|
+
|
|
1323
|
+
return canonical_sha256([member.to_dict() for member in members])
|
|
1324
|
+
|
|
1325
|
+
|
|
1326
|
+
def collection_fetched_content_sha256(
|
|
1327
|
+
members: tuple[CollectionMemberEvidence, ...],
|
|
1328
|
+
) -> str:
|
|
1329
|
+
"""Derive the collection's fetched-byte identity without claiming bytes were concatenated."""
|
|
1330
|
+
|
|
1331
|
+
return canonical_sha256(
|
|
1332
|
+
[
|
|
1333
|
+
{
|
|
1334
|
+
"member_id": member.member_id,
|
|
1335
|
+
"fetched_content_sha256": member.fetched_content_sha256,
|
|
1336
|
+
"fetched_size_bytes": member.fetched_size_bytes,
|
|
1337
|
+
}
|
|
1338
|
+
for member in members
|
|
1339
|
+
]
|
|
1340
|
+
)
|
|
1341
|
+
|
|
1342
|
+
|
|
1343
|
+
def collection_transport_evidence_digest(
|
|
1344
|
+
members: tuple[CollectionMemberEvidence, ...],
|
|
1345
|
+
) -> str:
|
|
1346
|
+
"""Bind transport evidence to member identity and order."""
|
|
1347
|
+
|
|
1348
|
+
return canonical_sha256(
|
|
1349
|
+
[
|
|
1350
|
+
{
|
|
1351
|
+
"member_id": member.member_id,
|
|
1352
|
+
"transport_evidence_digest": member.transport_evidence_digest,
|
|
1353
|
+
}
|
|
1354
|
+
for member in members
|
|
1355
|
+
]
|
|
1356
|
+
)
|
|
1357
|
+
|
|
1358
|
+
|
|
1359
|
+
@dataclass(frozen=True)
|
|
1360
|
+
class AcquisitionReceiptV3(_CanonicalContract):
|
|
1361
|
+
"""Strict ordered receipt for one bounded homogeneous public-source collection."""
|
|
1362
|
+
|
|
1363
|
+
receipt_id: str
|
|
1364
|
+
request_digest: str
|
|
1365
|
+
source_id: str
|
|
1366
|
+
adapter: AdapterDescriptor
|
|
1367
|
+
query_digest: str
|
|
1368
|
+
snapshot: SnapshotReference
|
|
1369
|
+
fetched_content_sha256: str
|
|
1370
|
+
family_id: str
|
|
1371
|
+
family_version: str
|
|
1372
|
+
decode_options_digest: str
|
|
1373
|
+
event_time_field: str | None
|
|
1374
|
+
observed_at: str
|
|
1375
|
+
available_at: str
|
|
1376
|
+
ingested_at: str
|
|
1377
|
+
classification: str
|
|
1378
|
+
rights_digest: str
|
|
1379
|
+
retention_digest: str
|
|
1380
|
+
observation_digest: str
|
|
1381
|
+
transport_evidence_digest: str
|
|
1382
|
+
members: tuple[CollectionMemberEvidence, ...]
|
|
1383
|
+
member_manifest_digest: str
|
|
1384
|
+
parsed_schema_digest: str
|
|
1385
|
+
schema_version: str = RECEIPT_CONTRACT_VERSION_V3
|
|
1386
|
+
decode_flags: tuple[str, ...] = ()
|
|
1387
|
+
|
|
1388
|
+
def __post_init__(self) -> None:
|
|
1389
|
+
_constant(self.schema_version, RECEIPT_CONTRACT_VERSION_V3, "receipt.schema_version")
|
|
1390
|
+
_identifier(self.receipt_id, "receipt.receipt_id")
|
|
1391
|
+
_digest(self.request_digest, "receipt.request_digest")
|
|
1392
|
+
_identifier(self.source_id, "receipt.source_id")
|
|
1393
|
+
if not isinstance(self.adapter, AdapterDescriptor):
|
|
1394
|
+
raise SourceContractError("TYPE", "receipt.adapter", "must be an AdapterDescriptor")
|
|
1395
|
+
if (
|
|
1396
|
+
self.adapter.adapter_id,
|
|
1397
|
+
self.adapter.adapter_version,
|
|
1398
|
+
self.adapter.source_class,
|
|
1399
|
+
self.adapter.data_formats,
|
|
1400
|
+
self.adapter.capabilities,
|
|
1401
|
+
) != (
|
|
1402
|
+
"public.https.collection",
|
|
1403
|
+
"1.0.0",
|
|
1404
|
+
"external_adapter",
|
|
1405
|
+
("csv",),
|
|
1406
|
+
("snapshot", "refresh", "backfill"),
|
|
1407
|
+
):
|
|
1408
|
+
raise SourceContractError(
|
|
1409
|
+
"COLLECTION_ADAPTER",
|
|
1410
|
+
"receipt.adapter",
|
|
1411
|
+
"a v3 receipt requires the exact public HTTPS collection adapter",
|
|
1412
|
+
)
|
|
1413
|
+
_digest(self.query_digest, "receipt.query_digest")
|
|
1414
|
+
if not isinstance(self.snapshot, SnapshotReference):
|
|
1415
|
+
raise SourceContractError("TYPE", "receipt.snapshot", "must be a SnapshotReference")
|
|
1416
|
+
if self.snapshot.data_format != "csv" or self.snapshot.media_type != "text/csv":
|
|
1417
|
+
raise SourceContractError(
|
|
1418
|
+
"COLLECTION_SNAPSHOT",
|
|
1419
|
+
"receipt.snapshot",
|
|
1420
|
+
"a collection snapshot must be canonical CSV",
|
|
1421
|
+
)
|
|
1422
|
+
for path, value in (
|
|
1423
|
+
("receipt.fetched_content_sha256", self.fetched_content_sha256),
|
|
1424
|
+
("receipt.rights_digest", self.rights_digest),
|
|
1425
|
+
("receipt.retention_digest", self.retention_digest),
|
|
1426
|
+
("receipt.observation_digest", self.observation_digest),
|
|
1427
|
+
("receipt.transport_evidence_digest", self.transport_evidence_digest),
|
|
1428
|
+
("receipt.member_manifest_digest", self.member_manifest_digest),
|
|
1429
|
+
("receipt.parsed_schema_digest", self.parsed_schema_digest),
|
|
1430
|
+
):
|
|
1431
|
+
_digest(value, path)
|
|
1432
|
+
_identifier(self.family_id, "receipt.family_id")
|
|
1433
|
+
_semver(self.family_version, "receipt.family_version")
|
|
1434
|
+
_digest(self.decode_options_digest, "receipt.decode_options_digest")
|
|
1435
|
+
_decode_flags(self.decode_flags, "receipt.decode_flags", family_id=self.family_id)
|
|
1436
|
+
if self.event_time_field is not None:
|
|
1437
|
+
_text(self.event_time_field, "receipt.event_time_field", maximum=128)
|
|
1438
|
+
observed = _timestamp(self.observed_at, "receipt.observed_at")
|
|
1439
|
+
available = _timestamp(self.available_at, "receipt.available_at")
|
|
1440
|
+
ingested = _timestamp(self.ingested_at, "receipt.ingested_at")
|
|
1441
|
+
if observed > ingested or available > ingested:
|
|
1442
|
+
raise SourceContractError(
|
|
1443
|
+
"TEMPORAL_ORDER",
|
|
1444
|
+
"receipt.ingested_at",
|
|
1445
|
+
"ingested_at must not precede observed_at or available_at",
|
|
1446
|
+
)
|
|
1447
|
+
_choice(self.classification, CLASSIFICATIONS - {"rejected"}, "receipt.classification")
|
|
1448
|
+
_typed_tuple(
|
|
1449
|
+
self.members,
|
|
1450
|
+
CollectionMemberEvidence,
|
|
1451
|
+
"receipt.members",
|
|
1452
|
+
nonempty=True,
|
|
1453
|
+
maximum=16,
|
|
1454
|
+
)
|
|
1455
|
+
if tuple(member.sequence for member in self.members) != tuple(range(len(self.members))):
|
|
1456
|
+
raise SourceContractError(
|
|
1457
|
+
"COLLECTION_SEQUENCE",
|
|
1458
|
+
"receipt.members",
|
|
1459
|
+
"member sequence must be contiguous and start at zero",
|
|
1460
|
+
)
|
|
1461
|
+
member_ids = tuple(member.member_id for member in self.members)
|
|
1462
|
+
if len(set(member_ids)) != len(member_ids):
|
|
1463
|
+
raise SourceContractError(
|
|
1464
|
+
"COLLECTION_MEMBER_ID",
|
|
1465
|
+
"receipt.members",
|
|
1466
|
+
"member IDs must be unique",
|
|
1467
|
+
)
|
|
1468
|
+
if sum(member.request_count for member in self.members) > 16:
|
|
1469
|
+
raise SourceContractError(
|
|
1470
|
+
"COLLECTION_REQUEST_LIMIT",
|
|
1471
|
+
"receipt.members",
|
|
1472
|
+
"aggregate member requests exceed the collection ceiling",
|
|
1473
|
+
)
|
|
1474
|
+
if any(member.parsed_schema_digest != self.parsed_schema_digest for member in self.members):
|
|
1475
|
+
raise SourceContractError(
|
|
1476
|
+
"COLLECTION_SCHEMA",
|
|
1477
|
+
"receipt.members",
|
|
1478
|
+
"every member must have the aggregate parsed schema",
|
|
1479
|
+
)
|
|
1480
|
+
member_flags = tuple(
|
|
1481
|
+
sorted({flag for member in self.members for flag in member.decode_flags})
|
|
1482
|
+
)
|
|
1483
|
+
if member_flags != self.decode_flags:
|
|
1484
|
+
raise SourceContractError(
|
|
1485
|
+
"COLLECTION_DECODE_FLAGS",
|
|
1486
|
+
"receipt.decode_flags",
|
|
1487
|
+
"must equal the sorted union of member decode flags",
|
|
1488
|
+
)
|
|
1489
|
+
if collection_member_manifest_digest(self.members) != self.member_manifest_digest:
|
|
1490
|
+
raise SourceContractError(
|
|
1491
|
+
"COLLECTION_MANIFEST",
|
|
1492
|
+
"receipt.member_manifest_digest",
|
|
1493
|
+
"does not rederive from the ordered member evidence",
|
|
1494
|
+
)
|
|
1495
|
+
if collection_fetched_content_sha256(self.members) != self.fetched_content_sha256:
|
|
1496
|
+
raise SourceContractError(
|
|
1497
|
+
"COLLECTION_FETCHED_CONTENT",
|
|
1498
|
+
"receipt.fetched_content_sha256",
|
|
1499
|
+
"does not rederive from the ordered fetched-member evidence",
|
|
1500
|
+
)
|
|
1501
|
+
if collection_transport_evidence_digest(self.members) != self.transport_evidence_digest:
|
|
1502
|
+
raise SourceContractError(
|
|
1503
|
+
"COLLECTION_TRANSPORT",
|
|
1504
|
+
"receipt.transport_evidence_digest",
|
|
1505
|
+
"does not rederive from the ordered member transport evidence",
|
|
1506
|
+
)
|
|
1507
|
+
|
|
1508
|
+
def to_dict(self) -> dict[str, Any]:
|
|
1509
|
+
document = {field.name: getattr(self, field.name) for field in fields(self)}
|
|
1510
|
+
document["adapter"] = self.adapter.to_dict()
|
|
1511
|
+
document["snapshot"] = self.snapshot.to_dict()
|
|
1512
|
+
document["members"] = [member.to_dict() for member in self.members]
|
|
1513
|
+
document["decode_flags"] = list(self.decode_flags)
|
|
1514
|
+
return document
|
|
1515
|
+
|
|
1516
|
+
|
|
1517
|
+
RECEIPT_DOCUMENT_KEYS_V3: frozenset[str] = frozenset(
|
|
1518
|
+
field.name for field in fields(AcquisitionReceiptV3)
|
|
1519
|
+
)
|
|
1520
|
+
|
|
1521
|
+
|
|
1522
|
+
def parse_acquisition_receipt_v3(value: Any) -> AcquisitionReceiptV3:
|
|
1523
|
+
"""Parse and rederive every v3 collection receipt invariant."""
|
|
1524
|
+
|
|
1525
|
+
data = _object(value, "receipt")
|
|
1526
|
+
_exact(data, set(RECEIPT_DOCUMENT_KEYS_V3), "receipt")
|
|
1527
|
+
snapshot_data = _object(data["snapshot"], "receipt.snapshot")
|
|
1528
|
+
_exact(
|
|
1529
|
+
snapshot_data,
|
|
1530
|
+
{"content_sha256", "size_bytes", "media_type", "data_format", "storage_name"},
|
|
1531
|
+
"receipt.snapshot",
|
|
1532
|
+
)
|
|
1533
|
+
raw_members = _array(data["members"], "receipt.members")
|
|
1534
|
+
member_keys = {field.name for field in fields(CollectionMemberEvidence)}
|
|
1535
|
+
members: list[CollectionMemberEvidence] = []
|
|
1536
|
+
for index, raw_member in enumerate(raw_members):
|
|
1537
|
+
member_data = _object(raw_member, f"receipt.members[{index}]")
|
|
1538
|
+
_exact(member_data, member_keys, f"receipt.members[{index}]")
|
|
1539
|
+
member_kwargs = dict(member_data)
|
|
1540
|
+
member_kwargs["decode_flags"] = tuple(
|
|
1541
|
+
_array(member_data["decode_flags"], f"receipt.members[{index}].decode_flags")
|
|
1542
|
+
)
|
|
1543
|
+
members.append(CollectionMemberEvidence(**member_kwargs))
|
|
1544
|
+
kwargs = dict(data)
|
|
1545
|
+
kwargs["adapter"] = parse_adapter_descriptor(data["adapter"])
|
|
1546
|
+
kwargs["snapshot"] = SnapshotReference(**snapshot_data)
|
|
1547
|
+
kwargs["members"] = tuple(members)
|
|
1548
|
+
kwargs["decode_flags"] = tuple(_array(data["decode_flags"], "receipt.decode_flags"))
|
|
1549
|
+
return AcquisitionReceiptV3(**kwargs)
|
|
1550
|
+
|
|
1551
|
+
|
|
1552
|
+
@dataclass(frozen=True)
|
|
1553
|
+
class FitnessEvidence(_CanonicalContract):
|
|
1554
|
+
"""One fitness dimension's outcome and the evidence digests cited for it."""
|
|
1555
|
+
|
|
1556
|
+
dimension: str
|
|
1557
|
+
outcome: str
|
|
1558
|
+
summary: str
|
|
1559
|
+
evidence_digests: tuple[str, ...]
|
|
1560
|
+
|
|
1561
|
+
def __post_init__(self) -> None:
|
|
1562
|
+
_choice(self.dimension, FITNESS_DIMENSIONS, "fitness_evidence.dimension")
|
|
1563
|
+
_choice(self.outcome, {"pass", "fail", "unknown"}, "fitness_evidence.outcome")
|
|
1564
|
+
_text(self.summary, "fitness_evidence.summary", maximum=1_000)
|
|
1565
|
+
_digests(
|
|
1566
|
+
self.evidence_digests,
|
|
1567
|
+
"fitness_evidence.evidence_digests",
|
|
1568
|
+
nonempty=True,
|
|
1569
|
+
maximum=MAX_EVIDENCE,
|
|
1570
|
+
)
|
|
1571
|
+
|
|
1572
|
+
def to_dict(self) -> dict[str, Any]:
|
|
1573
|
+
return {
|
|
1574
|
+
"dimension": self.dimension,
|
|
1575
|
+
"outcome": self.outcome,
|
|
1576
|
+
"summary": self.summary,
|
|
1577
|
+
"evidence_digests": list(self.evidence_digests),
|
|
1578
|
+
}
|
|
1579
|
+
|
|
1580
|
+
|
|
1581
|
+
@dataclass(frozen=True)
|
|
1582
|
+
class FitnessReport(_CanonicalContract):
|
|
1583
|
+
"""One source's cited decision over every fitness dimension; built by build_fitness_report."""
|
|
1584
|
+
|
|
1585
|
+
report_id: str
|
|
1586
|
+
question_id: str
|
|
1587
|
+
source_id: str
|
|
1588
|
+
observed_at: str
|
|
1589
|
+
intended_use: str
|
|
1590
|
+
dimensions: tuple[FitnessEvidence, ...]
|
|
1591
|
+
decision: str
|
|
1592
|
+
decision_reason: str
|
|
1593
|
+
rights_digest: str
|
|
1594
|
+
observation_digest: str
|
|
1595
|
+
schema_version: str = FITNESS_CONTRACT_VERSION
|
|
1596
|
+
|
|
1597
|
+
def __post_init__(self) -> None:
|
|
1598
|
+
_constant(self.schema_version, FITNESS_CONTRACT_VERSION, "fitness.schema_version")
|
|
1599
|
+
_identifier(self.report_id, "fitness.report_id")
|
|
1600
|
+
_identifier(self.question_id, "fitness.question_id")
|
|
1601
|
+
_identifier(self.source_id, "fitness.source_id")
|
|
1602
|
+
_timestamp(self.observed_at, "fitness.observed_at")
|
|
1603
|
+
_text(self.intended_use, "fitness.intended_use", maximum=200)
|
|
1604
|
+
_typed_tuple(
|
|
1605
|
+
self.dimensions,
|
|
1606
|
+
FitnessEvidence,
|
|
1607
|
+
"fitness.dimensions",
|
|
1608
|
+
nonempty=True,
|
|
1609
|
+
maximum=len(FITNESS_DIMENSIONS),
|
|
1610
|
+
)
|
|
1611
|
+
seen = {item.dimension for item in self.dimensions}
|
|
1612
|
+
if seen != FITNESS_DIMENSIONS:
|
|
1613
|
+
missing = sorted(FITNESS_DIMENSIONS - seen)
|
|
1614
|
+
extra = sorted(seen - FITNESS_DIMENSIONS)
|
|
1615
|
+
raise SourceContractError(
|
|
1616
|
+
"FITNESS_DIMENSIONS",
|
|
1617
|
+
"fitness.dimensions",
|
|
1618
|
+
f"must contain every dimension exactly once; missing={missing}, extra={extra}",
|
|
1619
|
+
)
|
|
1620
|
+
if len(seen) != len(self.dimensions):
|
|
1621
|
+
raise SourceContractError(
|
|
1622
|
+
"DUPLICATE",
|
|
1623
|
+
"fitness.dimensions",
|
|
1624
|
+
"fitness dimensions must be unique",
|
|
1625
|
+
)
|
|
1626
|
+
_choice(self.decision, FITNESS_DECISIONS, "fitness.decision")
|
|
1627
|
+
_text(self.decision_reason, "fitness.decision_reason", maximum=1_000)
|
|
1628
|
+
_digest(self.rights_digest, "fitness.rights_digest")
|
|
1629
|
+
_digest(self.observation_digest, "fitness.observation_digest")
|
|
1630
|
+
outcomes = {item.outcome for item in self.dimensions}
|
|
1631
|
+
if self.decision == "selected" and outcomes != {"pass"}:
|
|
1632
|
+
raise SourceContractError(
|
|
1633
|
+
"UNFIT_SELECTED",
|
|
1634
|
+
"fitness.decision",
|
|
1635
|
+
"selected sources require every fitness dimension to pass",
|
|
1636
|
+
)
|
|
1637
|
+
|
|
1638
|
+
def to_dict(self) -> dict[str, Any]:
|
|
1639
|
+
return {
|
|
1640
|
+
"schema_version": self.schema_version,
|
|
1641
|
+
"report_id": self.report_id,
|
|
1642
|
+
"question_id": self.question_id,
|
|
1643
|
+
"source_id": self.source_id,
|
|
1644
|
+
"observed_at": self.observed_at,
|
|
1645
|
+
"intended_use": self.intended_use,
|
|
1646
|
+
"dimensions": [item.to_dict() for item in self.dimensions],
|
|
1647
|
+
"decision": self.decision,
|
|
1648
|
+
"decision_reason": self.decision_reason,
|
|
1649
|
+
"rights_digest": self.rights_digest,
|
|
1650
|
+
"observation_digest": self.observation_digest,
|
|
1651
|
+
}
|
|
1652
|
+
|
|
1653
|
+
|
|
1654
|
+
@dataclass(frozen=True)
|
|
1655
|
+
class DeletedReference(_CanonicalContract):
|
|
1656
|
+
"""One artifact covered by a deletion, named by kind and digest, listed in the tombstone."""
|
|
1657
|
+
|
|
1658
|
+
reference_kind: str
|
|
1659
|
+
reference_digest: str
|
|
1660
|
+
|
|
1661
|
+
def __post_init__(self) -> None:
|
|
1662
|
+
_choice(
|
|
1663
|
+
self.reference_kind,
|
|
1664
|
+
{"candidate", "manifest", "receipt", "derived_artifact", "review"},
|
|
1665
|
+
"deleted_reference.reference_kind",
|
|
1666
|
+
)
|
|
1667
|
+
_digest(self.reference_digest, "deleted_reference.reference_digest")
|
|
1668
|
+
|
|
1669
|
+
def to_dict(self) -> dict[str, str]:
|
|
1670
|
+
return {
|
|
1671
|
+
"reference_kind": self.reference_kind,
|
|
1672
|
+
"reference_digest": self.reference_digest,
|
|
1673
|
+
}
|
|
1674
|
+
|
|
1675
|
+
|
|
1676
|
+
@dataclass(frozen=True)
|
|
1677
|
+
class DeletionTombstone(_CanonicalContract):
|
|
1678
|
+
"""Separate immutable status envelope; referenced sealed bytes are never rewritten."""
|
|
1679
|
+
|
|
1680
|
+
tombstone_id: str
|
|
1681
|
+
removed_content_sha256: str
|
|
1682
|
+
removed_at: str
|
|
1683
|
+
reason_code: str
|
|
1684
|
+
policy_digest: str
|
|
1685
|
+
complete_reference_set_digest: str
|
|
1686
|
+
references: tuple[DeletedReference, ...]
|
|
1687
|
+
byte_removal_confirmed: bool
|
|
1688
|
+
schema_version: str = TOMBSTONE_CONTRACT_VERSION
|
|
1689
|
+
|
|
1690
|
+
def __post_init__(self) -> None:
|
|
1691
|
+
_constant(self.schema_version, TOMBSTONE_CONTRACT_VERSION, "tombstone.schema_version")
|
|
1692
|
+
_identifier(self.tombstone_id, "tombstone.tombstone_id")
|
|
1693
|
+
_digest(self.removed_content_sha256, "tombstone.removed_content_sha256")
|
|
1694
|
+
_timestamp(self.removed_at, "tombstone.removed_at")
|
|
1695
|
+
_identifier(self.reason_code, "tombstone.reason_code")
|
|
1696
|
+
_digest(self.policy_digest, "tombstone.policy_digest")
|
|
1697
|
+
_digest(
|
|
1698
|
+
self.complete_reference_set_digest,
|
|
1699
|
+
"tombstone.complete_reference_set_digest",
|
|
1700
|
+
)
|
|
1701
|
+
_typed_tuple(
|
|
1702
|
+
self.references,
|
|
1703
|
+
DeletedReference,
|
|
1704
|
+
"tombstone.references",
|
|
1705
|
+
nonempty=True,
|
|
1706
|
+
maximum=MAX_REFERENCES,
|
|
1707
|
+
)
|
|
1708
|
+
actual = canonical_sha256([item.to_dict() for item in self.references])
|
|
1709
|
+
if actual != self.complete_reference_set_digest:
|
|
1710
|
+
raise SourceContractError(
|
|
1711
|
+
"REFERENCE_SET_DIGEST",
|
|
1712
|
+
"tombstone.complete_reference_set_digest",
|
|
1713
|
+
"does not match the canonical ordered reference set",
|
|
1714
|
+
)
|
|
1715
|
+
if self.byte_removal_confirmed is not True:
|
|
1716
|
+
raise SourceContractError(
|
|
1717
|
+
"REMOVAL_UNCONFIRMED",
|
|
1718
|
+
"tombstone.byte_removal_confirmed",
|
|
1719
|
+
"a sealed tombstone requires confirmed byte removal",
|
|
1720
|
+
)
|
|
1721
|
+
|
|
1722
|
+
def to_dict(self) -> dict[str, Any]:
|
|
1723
|
+
return {
|
|
1724
|
+
"schema_version": self.schema_version,
|
|
1725
|
+
"tombstone_id": self.tombstone_id,
|
|
1726
|
+
"removed_content_sha256": self.removed_content_sha256,
|
|
1727
|
+
"removed_at": self.removed_at,
|
|
1728
|
+
"reason_code": self.reason_code,
|
|
1729
|
+
"policy_digest": self.policy_digest,
|
|
1730
|
+
"complete_reference_set_digest": self.complete_reference_set_digest,
|
|
1731
|
+
"references": [item.to_dict() for item in self.references],
|
|
1732
|
+
"byte_removal_confirmed": self.byte_removal_confirmed,
|
|
1733
|
+
}
|
|
1734
|
+
|
|
1735
|
+
|
|
1736
|
+
def parse_adapter_descriptor(value: Any) -> AdapterDescriptor:
|
|
1737
|
+
"""Parse the strict public adapter descriptor shape."""
|
|
1738
|
+
|
|
1739
|
+
data = _object(value, "adapter")
|
|
1740
|
+
_exact(
|
|
1741
|
+
data,
|
|
1742
|
+
{
|
|
1743
|
+
"schema_version",
|
|
1744
|
+
"adapter_id",
|
|
1745
|
+
"adapter_version",
|
|
1746
|
+
"source_class",
|
|
1747
|
+
"data_formats",
|
|
1748
|
+
"capabilities",
|
|
1749
|
+
},
|
|
1750
|
+
"adapter",
|
|
1751
|
+
)
|
|
1752
|
+
return AdapterDescriptor(
|
|
1753
|
+
schema_version=data["schema_version"],
|
|
1754
|
+
adapter_id=data["adapter_id"],
|
|
1755
|
+
adapter_version=data["adapter_version"],
|
|
1756
|
+
source_class=data["source_class"],
|
|
1757
|
+
data_formats=tuple(_array(data["data_formats"], "adapter.data_formats")),
|
|
1758
|
+
capabilities=tuple(_array(data["capabilities"], "adapter.capabilities")),
|
|
1759
|
+
)
|
|
1760
|
+
|
|
1761
|
+
|
|
1762
|
+
def evaluate_obligations(
|
|
1763
|
+
*,
|
|
1764
|
+
rights: RightsEvidence,
|
|
1765
|
+
lifecycle: str,
|
|
1766
|
+
intended_use: str,
|
|
1767
|
+
attached_artifact_digests: tuple[str, ...],
|
|
1768
|
+
) -> tuple[LicenseObligation, ...]:
|
|
1769
|
+
"""Return applicable obligations or fail closed when their evidence is incomplete."""
|
|
1770
|
+
|
|
1771
|
+
_choice(lifecycle, OBLIGATION_LIFECYCLES, "lifecycle")
|
|
1772
|
+
_text(intended_use, "intended_use", maximum=200)
|
|
1773
|
+
_digests(attached_artifact_digests, "attached_artifact_digests", maximum=128)
|
|
1774
|
+
if rights.status not in {"approved", "conditional"}:
|
|
1775
|
+
raise SourceContractError(
|
|
1776
|
+
"RIGHTS_NOT_SELECTABLE",
|
|
1777
|
+
"rights.status",
|
|
1778
|
+
"unclear or prohibited rights require escalation and cannot advance",
|
|
1779
|
+
)
|
|
1780
|
+
if intended_use not in rights.allowed_uses:
|
|
1781
|
+
raise SourceContractError(
|
|
1782
|
+
"USE_NOT_ALLOWED",
|
|
1783
|
+
"intended_use",
|
|
1784
|
+
"the intended use is not covered by the rights evidence",
|
|
1785
|
+
)
|
|
1786
|
+
attached = set(attached_artifact_digests)
|
|
1787
|
+
applicable: list[LicenseObligation] = []
|
|
1788
|
+
for obligation in rights.obligations:
|
|
1789
|
+
if lifecycle not in obligation.applies_at:
|
|
1790
|
+
continue
|
|
1791
|
+
if intended_use in obligation.prohibited_uses:
|
|
1792
|
+
raise SourceContractError(
|
|
1793
|
+
"USE_PROHIBITED",
|
|
1794
|
+
f"obligation.{obligation.obligation_id}",
|
|
1795
|
+
"the intended use is explicitly prohibited",
|
|
1796
|
+
)
|
|
1797
|
+
if obligation.permitted_uses and intended_use not in obligation.permitted_uses:
|
|
1798
|
+
if obligation.escalation_when_ambiguous:
|
|
1799
|
+
raise SourceContractError(
|
|
1800
|
+
"RIGHTS_ESCALATION_REQUIRED",
|
|
1801
|
+
f"obligation.{obligation.obligation_id}",
|
|
1802
|
+
"obligation applicability is ambiguous for this intended use",
|
|
1803
|
+
)
|
|
1804
|
+
continue
|
|
1805
|
+
missing = sorted(set(obligation.required_artifact_digests) - attached)
|
|
1806
|
+
if missing:
|
|
1807
|
+
raise SourceContractError(
|
|
1808
|
+
"LICENSE_ARTIFACT_MISSING",
|
|
1809
|
+
f"obligation.{obligation.obligation_id}.required_artifact_digests",
|
|
1810
|
+
f"missing required attached artifacts: {missing}",
|
|
1811
|
+
)
|
|
1812
|
+
applicable.append(obligation)
|
|
1813
|
+
return tuple(applicable)
|
|
1814
|
+
|
|
1815
|
+
|
|
1816
|
+
def _object(value: Any, path: str) -> dict[str, Any]:
|
|
1817
|
+
if not isinstance(value, dict):
|
|
1818
|
+
raise SourceContractError("TYPE", path, "must be an object")
|
|
1819
|
+
return value
|
|
1820
|
+
|
|
1821
|
+
|
|
1822
|
+
def _array(value: Any, path: str) -> list[Any]:
|
|
1823
|
+
if not isinstance(value, list):
|
|
1824
|
+
raise SourceContractError("TYPE", path, "must be an array")
|
|
1825
|
+
return value
|
|
1826
|
+
|
|
1827
|
+
|
|
1828
|
+
def _exact(data: dict[str, Any], expected: set[str], path: str) -> None:
|
|
1829
|
+
actual = set(data)
|
|
1830
|
+
if actual != expected:
|
|
1831
|
+
raise SourceContractError(
|
|
1832
|
+
"FIELDS",
|
|
1833
|
+
path,
|
|
1834
|
+
f"must contain exactly {sorted(expected)}; missing={sorted(expected - actual)}, "
|
|
1835
|
+
f"extra={sorted(actual - expected)}",
|
|
1836
|
+
)
|
|
1837
|
+
|
|
1838
|
+
|
|
1839
|
+
def _constant(value: Any, expected: str, path: str) -> None:
|
|
1840
|
+
if value != expected:
|
|
1841
|
+
raise SourceContractError("VERSION", path, f"must equal {expected!r}")
|
|
1842
|
+
|
|
1843
|
+
|
|
1844
|
+
def _text(value: Any, path: str, *, maximum: int = MAX_TEXT) -> str:
|
|
1845
|
+
if not isinstance(value, str) or not value or len(value) > maximum:
|
|
1846
|
+
raise SourceContractError(
|
|
1847
|
+
"TEXT",
|
|
1848
|
+
path,
|
|
1849
|
+
f"must be a non-empty string no longer than {maximum} characters",
|
|
1850
|
+
)
|
|
1851
|
+
for index, char in enumerate(value):
|
|
1852
|
+
if 0xD800 <= ord(char) <= 0xDFFF:
|
|
1853
|
+
raise SourceContractError(
|
|
1854
|
+
"UNICODE",
|
|
1855
|
+
path,
|
|
1856
|
+
f"contains an unpaired surrogate at character {index}",
|
|
1857
|
+
)
|
|
1858
|
+
return value
|
|
1859
|
+
|
|
1860
|
+
|
|
1861
|
+
def _texts(
|
|
1862
|
+
values: Any,
|
|
1863
|
+
path: str,
|
|
1864
|
+
*,
|
|
1865
|
+
nonempty: bool,
|
|
1866
|
+
maximum: int,
|
|
1867
|
+
item_maximum: int,
|
|
1868
|
+
) -> None:
|
|
1869
|
+
if not isinstance(values, tuple):
|
|
1870
|
+
raise SourceContractError("TYPE", path, "must be an immutable tuple")
|
|
1871
|
+
if nonempty and not values:
|
|
1872
|
+
raise SourceContractError("EMPTY", path, "must not be empty")
|
|
1873
|
+
if len(values) > maximum:
|
|
1874
|
+
raise SourceContractError("LIMIT", path, f"must contain at most {maximum} values")
|
|
1875
|
+
for index, value in enumerate(values):
|
|
1876
|
+
_text(value, f"{path}[{index}]", maximum=item_maximum)
|
|
1877
|
+
if len(set(values)) != len(values):
|
|
1878
|
+
raise SourceContractError("DUPLICATE", path, "values must be unique")
|
|
1879
|
+
|
|
1880
|
+
|
|
1881
|
+
def _identifier(value: Any, path: str) -> str:
|
|
1882
|
+
text = _text(value, path, maximum=128)
|
|
1883
|
+
if not _IDENTIFIER.fullmatch(text):
|
|
1884
|
+
raise SourceContractError(
|
|
1885
|
+
"IDENTIFIER",
|
|
1886
|
+
path,
|
|
1887
|
+
"must be a lowercase dotted/dashed/underscored identifier",
|
|
1888
|
+
)
|
|
1889
|
+
return text
|
|
1890
|
+
|
|
1891
|
+
|
|
1892
|
+
def _semver(value: Any, path: str) -> str:
|
|
1893
|
+
text = _text(value, path, maximum=64)
|
|
1894
|
+
if not _SEMVER.fullmatch(text):
|
|
1895
|
+
raise SourceContractError("SEMVER", path, "must be a positive-major semantic version")
|
|
1896
|
+
return text
|
|
1897
|
+
|
|
1898
|
+
|
|
1899
|
+
def _commit(value: Any, path: str) -> str:
|
|
1900
|
+
text = _text(value, path, maximum=64)
|
|
1901
|
+
if not re.fullmatch(r"[0-9a-f]{40,64}", text):
|
|
1902
|
+
raise SourceContractError("COMMIT", path, "must be a lowercase full commit hash")
|
|
1903
|
+
return text
|
|
1904
|
+
|
|
1905
|
+
|
|
1906
|
+
def _dotted_name(value: Any, path: str) -> str:
|
|
1907
|
+
text = _text(value, path, maximum=256)
|
|
1908
|
+
if not re.fullmatch(r"[a-zA-Z_][a-zA-Z0-9_]*(?:\.[a-zA-Z_][a-zA-Z0-9_]*)+", text):
|
|
1909
|
+
raise SourceContractError("DOTTED_NAME", path, "must be a dotted public Python name")
|
|
1910
|
+
return text
|
|
1911
|
+
|
|
1912
|
+
|
|
1913
|
+
def _digest(value: Any, path: str) -> str:
|
|
1914
|
+
text = _text(value, path, maximum=64)
|
|
1915
|
+
if not _SHA256.fullmatch(text):
|
|
1916
|
+
raise SourceContractError("SHA256", path, "must be a lowercase 64-character SHA-256")
|
|
1917
|
+
return text
|
|
1918
|
+
|
|
1919
|
+
|
|
1920
|
+
def _digests(
|
|
1921
|
+
values: Any,
|
|
1922
|
+
path: str,
|
|
1923
|
+
*,
|
|
1924
|
+
nonempty: bool = False,
|
|
1925
|
+
maximum: int,
|
|
1926
|
+
) -> None:
|
|
1927
|
+
if not isinstance(values, tuple):
|
|
1928
|
+
raise SourceContractError("TYPE", path, "must be an immutable tuple")
|
|
1929
|
+
if nonempty and not values:
|
|
1930
|
+
raise SourceContractError("EMPTY", path, "must not be empty")
|
|
1931
|
+
if len(values) > maximum:
|
|
1932
|
+
raise SourceContractError("LIMIT", path, f"must contain at most {maximum} values")
|
|
1933
|
+
for index, value in enumerate(values):
|
|
1934
|
+
_digest(value, f"{path}[{index}]")
|
|
1935
|
+
if len(set(values)) != len(values):
|
|
1936
|
+
raise SourceContractError("DUPLICATE", path, "digests must be unique")
|
|
1937
|
+
|
|
1938
|
+
|
|
1939
|
+
def _choice(value: Any, allowed: set[str] | frozenset[str], path: str) -> str:
|
|
1940
|
+
text = _text(value, path, maximum=128)
|
|
1941
|
+
if text not in allowed:
|
|
1942
|
+
raise SourceContractError("ENUM", path, f"must be one of {sorted(allowed)}")
|
|
1943
|
+
return text
|
|
1944
|
+
|
|
1945
|
+
|
|
1946
|
+
def _choices(
|
|
1947
|
+
values: Any,
|
|
1948
|
+
allowed: set[str] | frozenset[str],
|
|
1949
|
+
path: str,
|
|
1950
|
+
*,
|
|
1951
|
+
nonempty: bool,
|
|
1952
|
+
maximum: int,
|
|
1953
|
+
) -> None:
|
|
1954
|
+
if not isinstance(values, tuple):
|
|
1955
|
+
raise SourceContractError("TYPE", path, "must be an immutable tuple")
|
|
1956
|
+
if nonempty and not values:
|
|
1957
|
+
raise SourceContractError("EMPTY", path, "must not be empty")
|
|
1958
|
+
if len(values) > maximum:
|
|
1959
|
+
raise SourceContractError("LIMIT", path, f"must contain at most {maximum} values")
|
|
1960
|
+
for index, value in enumerate(values):
|
|
1961
|
+
_choice(value, allowed, f"{path}[{index}]")
|
|
1962
|
+
if len(set(values)) != len(values):
|
|
1963
|
+
raise SourceContractError("DUPLICATE", path, "values must be unique")
|
|
1964
|
+
|
|
1965
|
+
|
|
1966
|
+
def _typed_tuple(
|
|
1967
|
+
values: Any,
|
|
1968
|
+
expected_type: type[Any],
|
|
1969
|
+
path: str,
|
|
1970
|
+
*,
|
|
1971
|
+
nonempty: bool,
|
|
1972
|
+
maximum: int,
|
|
1973
|
+
) -> None:
|
|
1974
|
+
if not isinstance(values, tuple):
|
|
1975
|
+
raise SourceContractError("TYPE", path, "must be an immutable tuple")
|
|
1976
|
+
if nonempty and not values:
|
|
1977
|
+
raise SourceContractError("EMPTY", path, "must not be empty")
|
|
1978
|
+
if len(values) > maximum:
|
|
1979
|
+
raise SourceContractError("LIMIT", path, f"must contain at most {maximum} values")
|
|
1980
|
+
for index, value in enumerate(values):
|
|
1981
|
+
if not isinstance(value, expected_type):
|
|
1982
|
+
raise SourceContractError(
|
|
1983
|
+
"TYPE",
|
|
1984
|
+
f"{path}[{index}]",
|
|
1985
|
+
f"must be a {expected_type.__name__}",
|
|
1986
|
+
)
|
|
1987
|
+
|
|
1988
|
+
|
|
1989
|
+
def _integer(value: Any, path: str, *, minimum: int, maximum: int) -> int:
|
|
1990
|
+
if type(value) is not int or not minimum <= value <= maximum:
|
|
1991
|
+
raise SourceContractError(
|
|
1992
|
+
"INTEGER",
|
|
1993
|
+
path,
|
|
1994
|
+
f"must be an integer in [{minimum}, {maximum}]",
|
|
1995
|
+
)
|
|
1996
|
+
if abs(value) > MAX_SAFE_INTEGER:
|
|
1997
|
+
raise SourceContractError(
|
|
1998
|
+
"SAFE_INTEGER",
|
|
1999
|
+
path,
|
|
2000
|
+
"integer exceeds the exact Python/JavaScript JSON domain",
|
|
2001
|
+
)
|
|
2002
|
+
return value
|
|
2003
|
+
|
|
2004
|
+
|
|
2005
|
+
def _timestamp(value: Any, path: str) -> datetime:
|
|
2006
|
+
text = _text(value, path, maximum=32)
|
|
2007
|
+
if not re.fullmatch(
|
|
2008
|
+
r"[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]{1,9})?Z",
|
|
2009
|
+
text,
|
|
2010
|
+
):
|
|
2011
|
+
raise SourceContractError("TIMESTAMP", path, "must be a canonical RFC 3339 UTC timestamp")
|
|
2012
|
+
try:
|
|
2013
|
+
parsed = datetime.fromisoformat(text[:-1] + "+00:00")
|
|
2014
|
+
except ValueError:
|
|
2015
|
+
raise SourceContractError("TIMESTAMP", path, "must be a valid UTC timestamp") from None
|
|
2016
|
+
if parsed.tzinfo != UTC:
|
|
2017
|
+
raise SourceContractError("TIMESTAMP", path, "must use UTC")
|
|
2018
|
+
return parsed
|
|
2019
|
+
|
|
2020
|
+
|
|
2021
|
+
def _media_type(value: Any, path: str) -> str:
|
|
2022
|
+
text = _text(value, path, maximum=255)
|
|
2023
|
+
if not _MEDIA_TYPE.fullmatch(text):
|
|
2024
|
+
raise SourceContractError("MEDIA_TYPE", path, "must be a lowercase type/subtype")
|
|
2025
|
+
return text
|
|
2026
|
+
|
|
2027
|
+
|
|
2028
|
+
def _https_uri(value: Any, path: str) -> str:
|
|
2029
|
+
text = _text(value, path, maximum=2_048)
|
|
2030
|
+
parsed = urlsplit(text)
|
|
2031
|
+
if parsed.scheme != "https" or not parsed.hostname or parsed.username or parsed.password:
|
|
2032
|
+
raise SourceContractError(
|
|
2033
|
+
"HTTPS_URI",
|
|
2034
|
+
path,
|
|
2035
|
+
"must be an HTTPS URI without user information",
|
|
2036
|
+
)
|
|
2037
|
+
if parsed.fragment:
|
|
2038
|
+
raise SourceContractError("HTTPS_URI", path, "URI fragments are forbidden")
|
|
2039
|
+
return text
|
|
2040
|
+
|
|
2041
|
+
|
|
2042
|
+
def _acquisition_uri(value: Any, path: str, *, source_class: str) -> str:
|
|
2043
|
+
if source_class in {"user_file", "stream"}:
|
|
2044
|
+
text = _text(value, path, maximum=2_048)
|
|
2045
|
+
if not re.fullmatch(r"urn:sha256:[0-9a-f]{64}", text):
|
|
2046
|
+
raise SourceContractError(
|
|
2047
|
+
"CONTENT_URN",
|
|
2048
|
+
path,
|
|
2049
|
+
"local-file and stream receipts must expose only an exact content SHA-256 URN",
|
|
2050
|
+
)
|
|
2051
|
+
return text
|
|
2052
|
+
return _https_uri(value, path)
|
|
2053
|
+
|
|
2054
|
+
|
|
2055
|
+
def _safe_storage_name(value: Any, path: str) -> str:
|
|
2056
|
+
text = _text(value, path, maximum=255)
|
|
2057
|
+
if "/" in text or "\\" in text or text in {".", ".."} or text.startswith("."):
|
|
2058
|
+
raise SourceContractError(
|
|
2059
|
+
"STORAGE_NAME",
|
|
2060
|
+
path,
|
|
2061
|
+
"must be one non-hidden relative filename",
|
|
2062
|
+
)
|
|
2063
|
+
return text
|
|
2064
|
+
|
|
2065
|
+
|
|
2066
|
+
def _reader_pin(value: Any, path: str) -> None:
|
|
2067
|
+
"""Admit a Reader pin as an exact three-key object, never half-stated.
|
|
2068
|
+
|
|
2069
|
+
The three fields are all present or all absent as a unit, and the unit's absence is
|
|
2070
|
+
spelled by the whole pin being ``None``. A pin that names a decoder without pinning
|
|
2071
|
+
its settings, or settings without a decoder, could not be replayed from its own
|
|
2072
|
+
record -- the same rule ``RecipeSource._validate_reader_pin`` applies on the recipe
|
|
2073
|
+
side and ``AcquisitionReceipt`` applies on the evidence side.
|
|
2074
|
+
|
|
2075
|
+
Identity is screened with this module's own rules rather than the readers package's,
|
|
2076
|
+
because the value that has to survive the round trip is the one the receipt will
|
|
2077
|
+
record, and the receipt screens ``family_id`` with ``_identifier``. A pin that could
|
|
2078
|
+
be requested but never recorded would fail at the far end of the chain instead of
|
|
2079
|
+
here.
|
|
2080
|
+
|
|
2081
|
+
``decode_options`` takes exactly the screen ``request.query`` takes, so an options
|
|
2082
|
+
blob cannot smuggle a non-serialisable or cross-runtime-unsafe value into the request
|
|
2083
|
+
digest. That screen's 16 KiB ceiling is tighter than the 256 KiB the recipe applies
|
|
2084
|
+
when sealing the same options; no shipped family comes near either, and the request
|
|
2085
|
+
side is the tighter of the two on purpose because a request crosses a coordinator
|
|
2086
|
+
boundary.
|
|
2087
|
+
"""
|
|
2088
|
+
|
|
2089
|
+
if not isinstance(value, Mapping):
|
|
2090
|
+
raise SourceContractError("READER_PIN", path, "must be an object")
|
|
2091
|
+
if set(value) != READER_PIN_KEYS:
|
|
2092
|
+
raise SourceContractError(
|
|
2093
|
+
"READER_PIN",
|
|
2094
|
+
path,
|
|
2095
|
+
"must carry exactly " + ", ".join(sorted(READER_PIN_KEYS)),
|
|
2096
|
+
)
|
|
2097
|
+
if any(value[key] is None for key in READER_PIN_KEYS):
|
|
2098
|
+
raise SourceContractError(
|
|
2099
|
+
"READER_PIN",
|
|
2100
|
+
path,
|
|
2101
|
+
"family_id, family_version, and decode_options are all present or all absent",
|
|
2102
|
+
)
|
|
2103
|
+
_identifier(value["family_id"], f"{path}.family_id")
|
|
2104
|
+
_semver(value["family_version"], f"{path}.family_version")
|
|
2105
|
+
_query(value["decode_options"], f"{path}.decode_options")
|
|
2106
|
+
|
|
2107
|
+
|
|
2108
|
+
# The decode budgets a request may carry, named exactly as ``ReaderBudgets`` names them so the
|
|
2109
|
+
# request cannot invent a budget the Reader contract does not have. Kept as a literal here
|
|
2110
|
+
# rather than imported from ``readers`` for the reason the pin keys are: this module is below
|
|
2111
|
+
# that one and importing upward would close a cycle. ``tests/h3/test_adapters.py`` holds the
|
|
2112
|
+
# two spellings to agreement.
|
|
2113
|
+
RESOURCE_CAP_KEYS: frozenset[str] = frozenset(
|
|
2114
|
+
{"max_declared_cells", "max_container_members", "max_nesting_depth"}
|
|
2115
|
+
)
|
|
2116
|
+
|
|
2117
|
+
|
|
2118
|
+
def _resource_caps(value: Any, path: str) -> None:
|
|
2119
|
+
"""Admit the recipe's decode budgets as an exact, closed, positive-integer object.
|
|
2120
|
+
|
|
2121
|
+
Every key is required. A partially stated cap object would let a recipe believe it had
|
|
2122
|
+
narrowed a budget it never named, and "a silently ignored cap is a budget a recipe believes
|
|
2123
|
+
it set" is the sentence ``ReaderBudgets.narrowed_by`` exists for. The recipe layer already
|
|
2124
|
+
refuses a cap that widens a ceiling; this layer refuses one that is not a cap at all.
|
|
2125
|
+
"""
|
|
2126
|
+
|
|
2127
|
+
if not isinstance(value, Mapping):
|
|
2128
|
+
raise SourceContractError("DECODE_CAPS", path, "must be an object")
|
|
2129
|
+
if set(value) != RESOURCE_CAP_KEYS:
|
|
2130
|
+
raise SourceContractError(
|
|
2131
|
+
"DECODE_CAPS",
|
|
2132
|
+
path,
|
|
2133
|
+
"must carry exactly " + ", ".join(sorted(RESOURCE_CAP_KEYS)),
|
|
2134
|
+
)
|
|
2135
|
+
for key in sorted(RESOURCE_CAP_KEYS):
|
|
2136
|
+
item = value[key]
|
|
2137
|
+
if type(item) is not int or item < 1:
|
|
2138
|
+
raise SourceContractError(
|
|
2139
|
+
"DECODE_CAPS",
|
|
2140
|
+
f"{path}.{key}",
|
|
2141
|
+
"must be a positive whole number",
|
|
2142
|
+
)
|
|
2143
|
+
if value["max_nesting_depth"] != 1:
|
|
2144
|
+
raise SourceContractError(
|
|
2145
|
+
"DECODE_CAPS",
|
|
2146
|
+
f"{path}.max_nesting_depth",
|
|
2147
|
+
"must be exactly 1: nesting depth is a security invariant, not a budget",
|
|
2148
|
+
)
|
|
2149
|
+
|
|
2150
|
+
|
|
2151
|
+
def _decode_flags(value: Any, path: str, *, family_id: str | None) -> None:
|
|
2152
|
+
"""Admit the closed set of facts a decode reported about itself.
|
|
2153
|
+
|
|
2154
|
+
Refused rather than normalised. ``ReaderResult`` already sorts and de-duplicates its
|
|
2155
|
+
flags at the producing end, so a receipt meeting an unsorted tuple met something other
|
|
2156
|
+
than a Reader result, and silently rewriting evidence on its way into a digest is how
|
|
2157
|
+
two different decodes come to share one receipt.
|
|
2158
|
+
"""
|
|
2159
|
+
|
|
2160
|
+
if not isinstance(value, tuple):
|
|
2161
|
+
raise SourceContractError("DECODE_FLAGS", path, "must be an immutable tuple")
|
|
2162
|
+
unknown = sorted(str(flag) for flag in value if flag not in DECODE_FLAGS)
|
|
2163
|
+
if unknown:
|
|
2164
|
+
raise SourceContractError(
|
|
2165
|
+
"DECODE_FLAGS",
|
|
2166
|
+
path,
|
|
2167
|
+
f"names no such decode flag: {', '.join(unknown)}",
|
|
2168
|
+
)
|
|
2169
|
+
if tuple(sorted(set(value))) != value:
|
|
2170
|
+
raise SourceContractError("DECODE_FLAGS", path, "must be sorted and free of repeats")
|
|
2171
|
+
if value and family_id is None:
|
|
2172
|
+
raise SourceContractError(
|
|
2173
|
+
"DECODE_FLAGS",
|
|
2174
|
+
path,
|
|
2175
|
+
"a flag describes a decode, so a flag without a Reader family is incoherent",
|
|
2176
|
+
)
|
|
2177
|
+
|
|
2178
|
+
|
|
2179
|
+
def _query(value: Any, path: str) -> None:
|
|
2180
|
+
if not isinstance(value, Mapping):
|
|
2181
|
+
raise SourceContractError("TYPE", path, "must be an object")
|
|
2182
|
+
_safe_json_numbers(value, path)
|
|
2183
|
+
try:
|
|
2184
|
+
encoded = canonical_json_bytes(dict(value))
|
|
2185
|
+
except (TypeError, ValueError) as exc:
|
|
2186
|
+
raise SourceContractError("QUERY_JSON", path, str(exc)) from None
|
|
2187
|
+
if len(encoded) > MAX_QUERY_BYTES:
|
|
2188
|
+
raise SourceContractError(
|
|
2189
|
+
"QUERY_LIMIT",
|
|
2190
|
+
path,
|
|
2191
|
+
f"canonical query exceeds {MAX_QUERY_BYTES} bytes",
|
|
2192
|
+
)
|
|
2193
|
+
|
|
2194
|
+
|
|
2195
|
+
def _safe_json_numbers(value: Any, path: str) -> None:
|
|
2196
|
+
if type(value) is int:
|
|
2197
|
+
if abs(value) > MAX_SAFE_INTEGER:
|
|
2198
|
+
raise SourceContractError(
|
|
2199
|
+
"SAFE_INTEGER",
|
|
2200
|
+
path,
|
|
2201
|
+
"integer exceeds the exact Python/JavaScript JSON domain",
|
|
2202
|
+
)
|
|
2203
|
+
return
|
|
2204
|
+
if isinstance(value, dict):
|
|
2205
|
+
for key, item in value.items():
|
|
2206
|
+
if not isinstance(key, str):
|
|
2207
|
+
raise SourceContractError("QUERY_KEY", path, "query keys must be strings")
|
|
2208
|
+
_text(key, f"{path}.<key>", maximum=128)
|
|
2209
|
+
_safe_json_numbers(item, f"{path}.{key}")
|
|
2210
|
+
return
|
|
2211
|
+
if isinstance(value, (list, tuple)):
|
|
2212
|
+
for index, item in enumerate(value):
|
|
2213
|
+
_safe_json_numbers(item, f"{path}[{index}]")
|
|
2214
|
+
return
|
|
2215
|
+
if value is None or type(value) is bool or isinstance(value, str):
|
|
2216
|
+
if isinstance(value, str):
|
|
2217
|
+
_text(value, path, maximum=MAX_TEXT)
|
|
2218
|
+
return
|
|
2219
|
+
raise SourceContractError(
|
|
2220
|
+
"QUERY_VALUE",
|
|
2221
|
+
path,
|
|
2222
|
+
f"unsupported query value type {type(value).__name__}",
|
|
2223
|
+
)
|