mostlyright-data 0.9.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mostlyright/data_harness/__init__.py +158 -0
- mostlyright/data_harness/acquisition/__init__.py +55 -0
- mostlyright/data_harness/acquisition/http.py +2773 -0
- mostlyright/data_harness/acquisition/parsing.py +809 -0
- mostlyright/data_harness/acquisition/ranges.py +495 -0
- mostlyright/data_harness/acquisition/result_download.py +360 -0
- mostlyright/data_harness/acquisition/retention_admission.py +248 -0
- mostlyright/data_harness/acquisition/sandbox.py +4888 -0
- mostlyright/data_harness/acquisition/url_policy.py +530 -0
- mostlyright/data_harness/agent_runtime.py +2743 -0
- mostlyright/data_harness/assets/logo-ink.svg +31 -0
- mostlyright/data_harness/backends/__init__.py +28 -0
- mostlyright/data_harness/backends/pandas_backend.py +350 -0
- mostlyright/data_harness/backends/polars_backend.py +366 -0
- mostlyright/data_harness/backends/protocol.py +124 -0
- mostlyright/data_harness/backends/reference.py +83 -0
- mostlyright/data_harness/backends/registry.py +55 -0
- mostlyright/data_harness/backends/restrictions.py +126 -0
- mostlyright/data_harness/canonical.py +333 -0
- mostlyright/data_harness/catalog_job.py +625 -0
- mostlyright/data_harness/cli.py +5398 -0
- mostlyright/data_harness/contracts.py +53 -0
- mostlyright/data_harness/coordinator.py +1307 -0
- mostlyright/data_harness/deploy.py +924 -0
- mostlyright/data_harness/deploy_target.py +312 -0
- mostlyright/data_harness/deployment_evidence.py +1067 -0
- mostlyright/data_harness/event_presentation.py +576 -0
- mostlyright/data_harness/events.py +2152 -0
- mostlyright/data_harness/fast_delimited.py +239 -0
- mostlyright/data_harness/fleet.py +237 -0
- mostlyright/data_harness/formats.py +236 -0
- mostlyright/data_harness/governors.py +1163 -0
- mostlyright/data_harness/hosted_bootstrap.py +972 -0
- mostlyright/data_harness/hosted_crawler.py +1115 -0
- mostlyright/data_harness/hosted_crawler_container_smoke.py +351 -0
- mostlyright/data_harness/hosted_crawler_fetch.py +423 -0
- mostlyright/data_harness/hosted_crawler_job.py +1277 -0
- mostlyright/data_harness/hosted_crawler_protocol.py +676 -0
- mostlyright/data_harness/hosted_dataset.py +1500 -0
- mostlyright/data_harness/hosted_deploy.py +3037 -0
- mostlyright/data_harness/hosted_handoff.py +62 -0
- mostlyright/data_harness/hosted_ingestion_contract.py +504 -0
- mostlyright/data_harness/hosted_ingestion_job.py +356 -0
- mostlyright/data_harness/hosted_ingestion_job_smoke.py +40 -0
- mostlyright/data_harness/hosted_session_container_smoke.py +194 -0
- mostlyright/data_harness/hosted_session_worker.py +3554 -0
- mostlyright/data_harness/hosted_session_worker_job_smoke.py +46 -0
- mostlyright/data_harness/hosted_worker.py +6784 -0
- mostlyright/data_harness/ingestion/__init__.py +56 -0
- mostlyright/data_harness/ingestion/contracts.py +461 -0
- mostlyright/data_harness/ingestion/faults.py +42 -0
- mostlyright/data_harness/ingestion/gcs_store.py +1162 -0
- mostlyright/data_harness/ingestion/spool.py +130 -0
- mostlyright/data_harness/ingestion/store.py +885 -0
- mostlyright/data_harness/key_seam.py +434 -0
- mostlyright/data_harness/linux_process_boundary.py +262 -0
- mostlyright/data_harness/local_contracts.py +2880 -0
- mostlyright/data_harness/local_search/__init__.py +5 -0
- mostlyright/data_harness/local_search/build_index.py +1087 -0
- mostlyright/data_harness/local_search/contracts.py +920 -0
- mostlyright/data_harness/local_search/query_trace.py +266 -0
- mostlyright/data_harness/local_search/retrieval.py +700 -0
- mostlyright/data_harness/local_search/sealed.py +474 -0
- mostlyright/data_harness/local_search/service.py +784 -0
- mostlyright/data_harness/nbrender/CONTRACT.md +212 -0
- mostlyright/data_harness/nbrender/__init__.py +12 -0
- mostlyright/data_harness/nbrender/chrome.py +359 -0
- mostlyright/data_harness/nbrender/code_body.py +266 -0
- mostlyright/data_harness/nbrender/document.py +407 -0
- mostlyright/data_harness/nbrender/frame.py +275 -0
- mostlyright/data_harness/nbrender/interactive.py +337 -0
- mostlyright/data_harness/nbrender/markdown_body.py +477 -0
- mostlyright/data_harness/nbrender/mr_components.py +134 -0
- mostlyright/data_harness/nbrender/outputs_data.py +595 -0
- mostlyright/data_harness/nbrender/outputs_rich.py +906 -0
- mostlyright/data_harness/nbrender/outputs_source.py +260 -0
- mostlyright/data_harness/nbrender/outputs_stage.py +176 -0
- mostlyright/data_harness/nbrender/outputs_text.py +400 -0
- mostlyright/data_harness/nbrender/parse.py +394 -0
- mostlyright/data_harness/nbrender/status.py +40 -0
- mostlyright/data_harness/nbrender/tokens.py +1295 -0
- mostlyright/data_harness/notebook.py +1710 -0
- mostlyright/data_harness/offline.py +2049 -0
- mostlyright/data_harness/operation_registry.py +1007 -0
- mostlyright/data_harness/operator_setup.py +239 -0
- mostlyright/data_harness/pipeline.py +6428 -0
- mostlyright/data_harness/plan_graph.py +2026 -0
- mostlyright/data_harness/preparation/__init__.py +104 -0
- mostlyright/data_harness/preparation/contracts.py +1017 -0
- mostlyright/data_harness/preparation/engine.py +221 -0
- mostlyright/data_harness/preparation/errors.py +14 -0
- mostlyright/data_harness/preparation/gates.py +751 -0
- mostlyright/data_harness/preparation/joins.py +574 -0
- mostlyright/data_harness/preparation/profile.py +384 -0
- mostlyright/data_harness/preparation/table.py +217 -0
- mostlyright/data_harness/preparation/transforms.py +568 -0
- mostlyright/data_harness/progress_events.py +534 -0
- mostlyright/data_harness/readers/__init__.py +46 -0
- mostlyright/data_harness/readers/containers.py +963 -0
- mostlyright/data_harness/readers/contracts.py +542 -0
- mostlyright/data_harness/readers/delimited.py +257 -0
- mostlyright/data_harness/readers/grib2/__init__.py +33 -0
- mostlyright/data_harness/readers/grib2/admission.py +722 -0
- mostlyright/data_harness/readers/grib2/decode.py +1009 -0
- mostlyright/data_harness/readers/grib2/geometry.py +1133 -0
- mostlyright/data_harness/readers/grib2/portable_math.py +501 -0
- mostlyright/data_harness/readers/json_tabular.py +485 -0
- mostlyright/data_harness/readers/registry.py +514 -0
- mostlyright/data_harness/readers/samples/README.md +110 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/cities.csv.gz +0 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/cities.csv.gz +0 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/cities.tar +0 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/cities.tar +0 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/cities.zip +0 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/expected.json +25 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/expected.json +25 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/expected.json +25 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/cities.csv +3 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/expected.json +23 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/cities.tsv +5 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/expected.json +25 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/expected.json +30 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/observations.csv +5 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/expected.json +44 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/stations.json +1 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/expected.json +48 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/observations.ndjson +4 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/cities.xlsx +0 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/expected.json +24 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/expected.json +27 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/expected.json +27 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/README.md +20 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/expected.json +55 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/gfs-2m-temperature.grib2 +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/expected.json +54 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/expected.json +54 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/hrrr-categorical-rain.grib2 +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/expected.json +54 -0
- mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
- mostlyright/data_harness/readers/samples.py +582 -0
- mostlyright/data_harness/readers/spreadsheet.py +803 -0
- mostlyright/data_harness/readers/tabular.py +510 -0
- mostlyright/data_harness/recipe.py +5321 -0
- mostlyright/data_harness/repair/__init__.py +78 -0
- mostlyright/data_harness/repair/adapters.py +274 -0
- mostlyright/data_harness/repair/contracts.py +872 -0
- mostlyright/data_harness/repair/coordinator.py +1099 -0
- mostlyright/data_harness/repair/errors.py +16 -0
- mostlyright/data_harness/review.py +2533 -0
- mostlyright/data_harness/rowset.py +283 -0
- mostlyright/data_harness/serving.py +1975 -0
- mostlyright/data_harness/serving_edge.py +590 -0
- mostlyright/data_harness/serving_http.py +1031 -0
- mostlyright/data_harness/session_probes.py +759 -0
- mostlyright/data_harness/signing.py +101 -0
- mostlyright/data_harness/source_discovery.py +898 -0
- mostlyright/data_harness/sources/__init__.py +209 -0
- mostlyright/data_harness/sources/_adapter_steps.py +213 -0
- mostlyright/data_harness/sources/adapters.py +1214 -0
- mostlyright/data_harness/sources/cadence.py +1428 -0
- mostlyright/data_harness/sources/cadence_emission.py +453 -0
- mostlyright/data_harness/sources/cadence_history.py +546 -0
- mostlyright/data_harness/sources/catalog/__init__.py +17 -0
- mostlyright/data_harness/sources/catalog/admission.py +477 -0
- mostlyright/data_harness/sources/catalog/authoring.py +1701 -0
- mostlyright/data_harness/sources/catalog/authoring_policy.py +701 -0
- mostlyright/data_harness/sources/catalog/authoring_shards.py +1217 -0
- mostlyright/data_harness/sources/catalog/bounded_io.py +231 -0
- mostlyright/data_harness/sources/catalog/channel.py +523 -0
- mostlyright/data_harness/sources/catalog/channel_client.py +296 -0
- mostlyright/data_harness/sources/catalog/contracts.py +825 -0
- mostlyright/data_harness/sources/catalog/coverage.py +137 -0
- mostlyright/data_harness/sources/catalog/delta.py +1340 -0
- mostlyright/data_harness/sources/catalog/embedding.py +532 -0
- mostlyright/data_harness/sources/catalog/entry_v2.py +1182 -0
- mostlyright/data_harness/sources/catalog/fill.py +3889 -0
- mostlyright/data_harness/sources/catalog/fill_partitions.py +459 -0
- mostlyright/data_harness/sources/catalog/fill_staging.py +1105 -0
- mostlyright/data_harness/sources/catalog/gating.py +374 -0
- mostlyright/data_harness/sources/catalog/generation_receipt.py +1607 -0
- mostlyright/data_harness/sources/catalog/harvest/__init__.py +7 -0
- mostlyright/data_harness/sources/catalog/harvest/ckan.py +384 -0
- mostlyright/data_harness/sources/catalog/harvest/datagov_v4.py +798 -0
- mostlyright/data_harness/sources/catalog/harvest/protocol.py +964 -0
- mostlyright/data_harness/sources/catalog/harvest/sdmx.py +445 -0
- mostlyright/data_harness/sources/catalog/harvest/stac.py +384 -0
- mostlyright/data_harness/sources/catalog/health.py +447 -0
- mostlyright/data_harness/sources/catalog/hosted_catalog.py +105 -0
- mostlyright/data_harness/sources/catalog/identity_history.py +1549 -0
- mostlyright/data_harness/sources/catalog/neural.py +1618 -0
- mostlyright/data_harness/sources/catalog/packed_catalog.py +2345 -0
- mostlyright/data_harness/sources/catalog/packed_retrieval.py +1517 -0
- mostlyright/data_harness/sources/catalog/packed_writer.py +2802 -0
- mostlyright/data_harness/sources/catalog/query_trace.py +1037 -0
- mostlyright/data_harness/sources/catalog/recommend.py +171 -0
- mostlyright/data_harness/sources/catalog/retrieval.py +230 -0
- mostlyright/data_harness/sources/catalog/retrieval_manifest.py +995 -0
- mostlyright/data_harness/sources/catalog/rights_decisions.py +254 -0
- mostlyright/data_harness/sources/catalog/sealed.py +560 -0
- mostlyright/data_harness/sources/catalog/search.py +230 -0
- mostlyright/data_harness/sources/catalog/streaming_delta.py +1097 -0
- mostlyright/data_harness/sources/catalog/update.py +891 -0
- mostlyright/data_harness/sources/collections.py +815 -0
- mostlyright/data_harness/sources/contracts.py +2223 -0
- mostlyright/data_harness/sources/deletion.py +761 -0
- mostlyright/data_harness/sources/fitness.py +162 -0
- mostlyright/data_harness/sources/governance.py +163 -0
- mostlyright/data_harness/sources/hosted.py +173 -0
- mostlyright/data_harness/sources/integration.py +218 -0
- mostlyright/data_harness/sources/range_reader.py +418 -0
- mostlyright/data_harness/sources/registry.py +514 -0
- mostlyright/data_harness/sources/rights_rule.py +59 -0
- mostlyright/data_harness/sources/source_cadence_vectors.v1.json +1 -0
- mostlyright/data_harness/sources/sports.py +521 -0
- mostlyright/data_harness/sources/stream.py +524 -0
- mostlyright/data_harness/sources/stream_connector.py +418 -0
- mostlyright/data_harness/sources/stream_recorder.py +1404 -0
- mostlyright/data_harness/studio_boundary.py +2019 -0
- mostlyright/data_harness/thin/__init__.py +37 -0
- mostlyright/data_harness/thin/acquire.py +1137 -0
- mostlyright/data_harness/thin/acquire_cancel.py +579 -0
- mostlyright/data_harness/thin/approvals.py +617 -0
- mostlyright/data_harness/thin/commands.py +406 -0
- mostlyright/data_harness/thin/download.py +194 -0
- mostlyright/data_harness/thin/narrative.py +589 -0
- mostlyright/data_harness/thin/parity.py +1070 -0
- mostlyright/data_harness/thin/propose.py +2759 -0
- mostlyright/data_harness/thin/research.py +1663 -0
- mostlyright/data_harness/thin/router.py +924 -0
- mostlyright/data_harness/thin/runs.py +519 -0
- mostlyright/data_harness/thin/session.py +281 -0
- mostlyright/data_harness/thin/stream.py +501 -0
- mostlyright/data_harness/thin/transport.py +187 -0
- mostlyright/data_harness/thin/vocabulary.py +368 -0
- mostlyright/data_harness/thin/workers.py +164 -0
- mostlyright/data_harness/ucum/TABLE-PIN.json +40 -0
- mostlyright/data_harness/ucum/ucum-subset.v1.json +632 -0
- mostlyright/data_harness/unit_flow.py +927 -0
- mostlyright/data_harness/units.py +572 -0
- mostlyright/data_harness/ux/__init__.py +9 -0
- mostlyright/data_harness/ux/approve.py +485 -0
- mostlyright/data_harness/ux/author_yaml.py +597 -0
- mostlyright/data_harness/ux/cloud_auth.py +447 -0
- mostlyright/data_harness/ux/commands/__init__.py +260 -0
- mostlyright/data_harness/ux/commands/approve.py +136 -0
- mostlyright/data_harness/ux/commands/auth.py +744 -0
- mostlyright/data_harness/ux/commands/author.py +79 -0
- mostlyright/data_harness/ux/commands/catalog_author.py +403 -0
- mostlyright/data_harness/ux/commands/catalog_fill.py +523 -0
- mostlyright/data_harness/ux/commands/catalog_harvest.py +545 -0
- mostlyright/data_harness/ux/commands/catalog_publish.py +1838 -0
- mostlyright/data_harness/ux/commands/catalog_search.py +71 -0
- mostlyright/data_harness/ux/commands/catalog_update.py +437 -0
- mostlyright/data_harness/ux/commands/deploy.py +134 -0
- mostlyright/data_harness/ux/commands/deploy_dataset.py +98 -0
- mostlyright/data_harness/ux/commands/deploy_plan.py +105 -0
- mostlyright/data_harness/ux/commands/deploy_status.py +104 -0
- mostlyright/data_harness/ux/commands/diff.py +74 -0
- mostlyright/data_harness/ux/commands/index.py +84 -0
- mostlyright/data_harness/ux/commands/inventory.py +47 -0
- mostlyright/data_harness/ux/commands/list_builds.py +143 -0
- mostlyright/data_harness/ux/commands/login.py +63 -0
- mostlyright/data_harness/ux/commands/peek.py +236 -0
- mostlyright/data_harness/ux/commands/plan_check.py +90 -0
- mostlyright/data_harness/ux/commands/preflight.py +97 -0
- mostlyright/data_harness/ux/commands/record.py +107 -0
- mostlyright/data_harness/ux/commands/review_setup.py +47 -0
- mostlyright/data_harness/ux/commands/search.py +440 -0
- mostlyright/data_harness/ux/commands/show.py +61 -0
- mostlyright/data_harness/ux/commands/whoami.py +37 -0
- mostlyright/data_harness/ux/credential_native.py +551 -0
- mostlyright/data_harness/ux/credential_store.py +1055 -0
- mostlyright/data_harness/ux/credentials.py +631 -0
- mostlyright/data_harness/ux/diffing.py +444 -0
- mostlyright/data_harness/ux/headline.py +671 -0
- mostlyright/data_harness/ux/hosted_acquisition.py +974 -0
- mostlyright/data_harness/ux/hosted_run_status.py +619 -0
- mostlyright/data_harness/ux/inventory.py +427 -0
- mostlyright/data_harness/ux/local_review.py +375 -0
- mostlyright/data_harness/ux/login.py +691 -0
- mostlyright/data_harness/ux/path_kind.py +147 -0
- mostlyright/data_harness/ux/peek.py +1000 -0
- mostlyright/data_harness/ux/plain_file.py +178 -0
- mostlyright/data_harness/ux/plan_check.py +311 -0
- mostlyright/data_harness/ux/preflight.py +918 -0
- mostlyright/data_harness/ux/readers.py +1124 -0
- mostlyright/data_harness/ux/remediation.py +2195 -0
- mostlyright/data_harness/ux/render.py +657 -0
- mostlyright/data_harness/ux/workload.py +1077 -0
- mostlyright/data_harness/viewer.py +3713 -0
- mostlyright/data_harness/visual_run/__init__.py +83 -0
- mostlyright/data_harness/visual_run/authoring.py +235 -0
- mostlyright/data_harness/visual_run/contracts.py +673 -0
- mostlyright/data_harness/visual_run/materialize.py +486 -0
- mostlyright/data_harness/visual_run/observations.py +874 -0
- mostlyright/data_harness/visual_run/query.py +259 -0
- mostlyright/data_harness/visual_run/reducer.py +280 -0
- mostlyright/data_harness/visual_run/sdk.py +892 -0
- mostlyright/data_harness/visual_run/store.py +584 -0
- mostlyright/data_harness/visual_run/transport.py +239 -0
- mostlyright/data_harness/watch.py +2999 -0
- mostlyright_data-0.9.0.dist-info/METADATA +607 -0
- mostlyright_data-0.9.0.dist-info/RECORD +314 -0
- mostlyright_data-0.9.0.dist-info/WHEEL +4 -0
- mostlyright_data-0.9.0.dist-info/entry_points.txt +12 -0
|
@@ -0,0 +1,751 @@
|
|
|
1
|
+
"""Deterministic temporal, target, and analysis-readiness quality gates."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass, replace
|
|
6
|
+
from datetime import date, datetime, timedelta
|
|
7
|
+
from itertools import pairwise
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
from mostlyright.data_harness.canonical import canonical_sha256
|
|
11
|
+
from mostlyright.data_harness.preparation.contracts import (
|
|
12
|
+
QUALITY_CHECK_POLICY,
|
|
13
|
+
QUALITY_REPORT_VERSION,
|
|
14
|
+
TARGET_RATIONALE_VERSION,
|
|
15
|
+
TEMPORAL_REPORT_VERSION,
|
|
16
|
+
ColumnLineage,
|
|
17
|
+
QualitySpec,
|
|
18
|
+
SourceTemporalEvidence,
|
|
19
|
+
TargetSpec,
|
|
20
|
+
TemporalPolicy,
|
|
21
|
+
parse_utc,
|
|
22
|
+
)
|
|
23
|
+
from mostlyright.data_harness.preparation.errors import PreparationError
|
|
24
|
+
from mostlyright.data_harness.preparation.joins import JoinAudit
|
|
25
|
+
from mostlyright.data_harness.preparation.profile import TableProfile, profile_table
|
|
26
|
+
from mostlyright.data_harness.preparation.table import TableState, validate_columns
|
|
27
|
+
|
|
28
|
+
_CHECK_POLICY = QUALITY_CHECK_POLICY
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@dataclass(frozen=True)
|
|
32
|
+
class TemporalFinding:
|
|
33
|
+
code: str
|
|
34
|
+
source_id: str
|
|
35
|
+
column: str | None
|
|
36
|
+
row_index: int | None
|
|
37
|
+
detail: str
|
|
38
|
+
|
|
39
|
+
def __post_init__(self) -> None:
|
|
40
|
+
if not isinstance(self.code, str) or not self.code.startswith("TEMPORAL_"):
|
|
41
|
+
raise PreparationError("TEMPORAL_FINDING", "temporal.finding.code", "is invalid")
|
|
42
|
+
if not isinstance(self.source_id, str) or not self.source_id:
|
|
43
|
+
raise PreparationError("TEMPORAL_FINDING", "temporal.finding.source_id", "is invalid")
|
|
44
|
+
if self.row_index is not None and (type(self.row_index) is not int or self.row_index < 0):
|
|
45
|
+
raise PreparationError(
|
|
46
|
+
"TEMPORAL_FINDING",
|
|
47
|
+
"temporal.finding.row_index",
|
|
48
|
+
"must be a nonnegative integer",
|
|
49
|
+
)
|
|
50
|
+
if not isinstance(self.detail, str) or not self.detail:
|
|
51
|
+
raise PreparationError("TEMPORAL_FINDING", "temporal.finding.detail", "is invalid")
|
|
52
|
+
|
|
53
|
+
def to_dict(self) -> dict[str, Any]:
|
|
54
|
+
return {
|
|
55
|
+
"code": self.code,
|
|
56
|
+
"severity": "blocker",
|
|
57
|
+
"source_id": self.source_id,
|
|
58
|
+
"column": self.column,
|
|
59
|
+
"row_index": self.row_index,
|
|
60
|
+
"detail": self.detail,
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
@dataclass(frozen=True)
|
|
65
|
+
class TemporalReport:
|
|
66
|
+
schema_version: str
|
|
67
|
+
analysis_kind: str
|
|
68
|
+
question_cutoff: str
|
|
69
|
+
checked_sources: tuple[str, ...]
|
|
70
|
+
checked_cells: int
|
|
71
|
+
findings: tuple[TemporalFinding, ...]
|
|
72
|
+
status: str
|
|
73
|
+
|
|
74
|
+
def __post_init__(self) -> None:
|
|
75
|
+
if self.schema_version != TEMPORAL_REPORT_VERSION:
|
|
76
|
+
raise PreparationError(
|
|
77
|
+
"TEMPORAL_REPORT_VERSION",
|
|
78
|
+
"temporal.schema_version",
|
|
79
|
+
"unsupported",
|
|
80
|
+
)
|
|
81
|
+
if self.status not in {"passed", "failed"}:
|
|
82
|
+
raise PreparationError("TEMPORAL_REPORT_STATUS", "temporal.status", "unsupported")
|
|
83
|
+
if (self.status == "passed") != (not self.findings):
|
|
84
|
+
raise PreparationError(
|
|
85
|
+
"TEMPORAL_REPORT_STATUS",
|
|
86
|
+
"temporal.status",
|
|
87
|
+
"does not reconcile",
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
@property
|
|
91
|
+
def digest(self) -> str:
|
|
92
|
+
return canonical_sha256(self.to_dict())
|
|
93
|
+
|
|
94
|
+
def to_dict(self) -> dict[str, Any]:
|
|
95
|
+
return {
|
|
96
|
+
"schema_version": self.schema_version,
|
|
97
|
+
"analysis_kind": self.analysis_kind,
|
|
98
|
+
"question_cutoff": self.question_cutoff,
|
|
99
|
+
"checked_sources": list(self.checked_sources),
|
|
100
|
+
"checked_cells": self.checked_cells,
|
|
101
|
+
"findings": [item.to_dict() for item in self.findings],
|
|
102
|
+
"status": self.status,
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
@dataclass(frozen=True)
|
|
107
|
+
class TargetRationale:
|
|
108
|
+
schema_version: str
|
|
109
|
+
output_column: str
|
|
110
|
+
operation: str
|
|
111
|
+
source_column: str
|
|
112
|
+
source_snapshot_digest: str
|
|
113
|
+
resolution_rule: str
|
|
114
|
+
horizon_seconds: int
|
|
115
|
+
availability_time_column: str
|
|
116
|
+
cutoff_column: str
|
|
117
|
+
mapping_digest: str | None
|
|
118
|
+
rationale: str
|
|
119
|
+
verified_rows: int
|
|
120
|
+
|
|
121
|
+
def __post_init__(self) -> None:
|
|
122
|
+
if self.schema_version != TARGET_RATIONALE_VERSION:
|
|
123
|
+
raise PreparationError(
|
|
124
|
+
"TARGET_RATIONALE_VERSION", "target.schema_version", "unsupported"
|
|
125
|
+
)
|
|
126
|
+
if type(self.horizon_seconds) is not int or self.horizon_seconds < 0:
|
|
127
|
+
raise PreparationError("TARGET_RATIONALE", "target.horizon_seconds", "is invalid")
|
|
128
|
+
if type(self.verified_rows) is not int or self.verified_rows < 0:
|
|
129
|
+
raise PreparationError("TARGET_RATIONALE", "target.verified_rows", "is invalid")
|
|
130
|
+
if not _is_digest(self.source_snapshot_digest):
|
|
131
|
+
raise PreparationError(
|
|
132
|
+
"TARGET_RATIONALE",
|
|
133
|
+
"target.source_snapshot_digest",
|
|
134
|
+
"is invalid",
|
|
135
|
+
)
|
|
136
|
+
if self.mapping_digest is not None and not _is_digest(self.mapping_digest):
|
|
137
|
+
raise PreparationError("TARGET_RATIONALE", "target.mapping_digest", "is invalid")
|
|
138
|
+
|
|
139
|
+
@property
|
|
140
|
+
def digest(self) -> str:
|
|
141
|
+
return canonical_sha256(self.to_dict())
|
|
142
|
+
|
|
143
|
+
def to_dict(self) -> dict[str, Any]:
|
|
144
|
+
return {
|
|
145
|
+
"schema_version": self.schema_version,
|
|
146
|
+
"output_column": self.output_column,
|
|
147
|
+
"operation": self.operation,
|
|
148
|
+
"source_column": self.source_column,
|
|
149
|
+
"source_snapshot_digest": self.source_snapshot_digest,
|
|
150
|
+
"resolution_rule": self.resolution_rule,
|
|
151
|
+
"horizon_seconds": self.horizon_seconds,
|
|
152
|
+
"availability_time_column": self.availability_time_column,
|
|
153
|
+
"cutoff_column": self.cutoff_column,
|
|
154
|
+
"mapping_digest": self.mapping_digest,
|
|
155
|
+
"rationale": self.rationale,
|
|
156
|
+
"verified_rows": self.verified_rows,
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
@dataclass(frozen=True)
|
|
161
|
+
class QualityCheck:
|
|
162
|
+
check_id: str
|
|
163
|
+
status: str
|
|
164
|
+
severity: str
|
|
165
|
+
threshold_id: str
|
|
166
|
+
observed: tuple[tuple[str, Any], ...]
|
|
167
|
+
|
|
168
|
+
def __post_init__(self) -> None:
|
|
169
|
+
expected = _CHECK_POLICY.get(self.check_id)
|
|
170
|
+
if expected != (self.severity, self.threshold_id):
|
|
171
|
+
raise PreparationError(
|
|
172
|
+
"QUALITY_POLICY",
|
|
173
|
+
f"quality.{self.check_id}",
|
|
174
|
+
"severity or threshold identity is not policy-owned",
|
|
175
|
+
)
|
|
176
|
+
if self.status not in {"passed", "failed"}:
|
|
177
|
+
raise PreparationError("QUALITY_STATUS", f"quality.{self.check_id}", "is unsupported")
|
|
178
|
+
names = tuple(name for name, _ in self.observed)
|
|
179
|
+
if names != tuple(sorted(set(names))):
|
|
180
|
+
raise PreparationError(
|
|
181
|
+
"QUALITY_EVIDENCE",
|
|
182
|
+
f"quality.{self.check_id}",
|
|
183
|
+
"observed evidence must be unique and canonically ordered",
|
|
184
|
+
)
|
|
185
|
+
canonical_sha256({name: value for name, value in self.observed})
|
|
186
|
+
|
|
187
|
+
def to_dict(self) -> dict[str, Any]:
|
|
188
|
+
return {
|
|
189
|
+
"check_id": self.check_id,
|
|
190
|
+
"status": self.status,
|
|
191
|
+
"severity": self.severity,
|
|
192
|
+
"threshold_id": self.threshold_id,
|
|
193
|
+
"observed": {name: value for name, value in self.observed},
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
@dataclass(frozen=True)
|
|
198
|
+
class QualityReport:
|
|
199
|
+
schema_version: str
|
|
200
|
+
checks: tuple[QualityCheck, ...]
|
|
201
|
+
status: str
|
|
202
|
+
|
|
203
|
+
def __post_init__(self) -> None:
|
|
204
|
+
if self.schema_version != QUALITY_REPORT_VERSION:
|
|
205
|
+
raise PreparationError("QUALITY_VERSION", "quality.schema_version", "is unsupported")
|
|
206
|
+
if tuple(item.check_id for item in self.checks) != tuple(_CHECK_POLICY):
|
|
207
|
+
raise PreparationError(
|
|
208
|
+
"QUALITY_CHECK_SET",
|
|
209
|
+
"quality.checks",
|
|
210
|
+
"check set/order is not exact",
|
|
211
|
+
)
|
|
212
|
+
expected = "failed" if any(item.status == "failed" for item in self.checks) else "passed"
|
|
213
|
+
if self.status != expected:
|
|
214
|
+
raise PreparationError("QUALITY_STATUS", "quality.status", "does not reconcile checks")
|
|
215
|
+
|
|
216
|
+
@property
|
|
217
|
+
def digest(self) -> str:
|
|
218
|
+
return canonical_sha256(self.to_dict())
|
|
219
|
+
|
|
220
|
+
@property
|
|
221
|
+
def findings(self) -> tuple[QualityCheck, ...]:
|
|
222
|
+
return tuple(item for item in self.checks if item.status == "failed")
|
|
223
|
+
|
|
224
|
+
def to_dict(self) -> dict[str, Any]:
|
|
225
|
+
return {
|
|
226
|
+
"schema_version": self.schema_version,
|
|
227
|
+
"checks": [item.to_dict() for item in self.checks],
|
|
228
|
+
"status": self.status,
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def validate_temporal_safety(
|
|
233
|
+
state: TableState,
|
|
234
|
+
*,
|
|
235
|
+
evidence: tuple[SourceTemporalEvidence, ...],
|
|
236
|
+
policy: TemporalPolicy,
|
|
237
|
+
) -> TemporalReport:
|
|
238
|
+
"""Audit fidelity, availability, revisions, and row/question cutoffs."""
|
|
239
|
+
|
|
240
|
+
evidence_by_source = {item.source_id: item for item in evidence}
|
|
241
|
+
if len(evidence_by_source) != len(evidence):
|
|
242
|
+
raise PreparationError("TEMPORAL_EVIDENCE", "temporal.evidence", "duplicates source IDs")
|
|
243
|
+
sources = {item.source_id for lineage in state.lineage for item in lineage.inputs}
|
|
244
|
+
if set(evidence_by_source) != sources:
|
|
245
|
+
raise PreparationError(
|
|
246
|
+
"TEMPORAL_EVIDENCE",
|
|
247
|
+
"temporal.evidence",
|
|
248
|
+
"does not exactly cover output lineage sources",
|
|
249
|
+
)
|
|
250
|
+
for item in evidence:
|
|
251
|
+
if item.snapshot_digest not in state.snapshot_digests:
|
|
252
|
+
raise PreparationError(
|
|
253
|
+
"TEMPORAL_EVIDENCE",
|
|
254
|
+
item.source_id,
|
|
255
|
+
"snapshot digest is not bound to the table",
|
|
256
|
+
)
|
|
257
|
+
_validate_temporal_column(
|
|
258
|
+
state,
|
|
259
|
+
item.source_id,
|
|
260
|
+
item.available_at_column,
|
|
261
|
+
"available_at",
|
|
262
|
+
)
|
|
263
|
+
if item.event_time_column is not None:
|
|
264
|
+
_validate_temporal_column(
|
|
265
|
+
state,
|
|
266
|
+
item.source_id,
|
|
267
|
+
item.event_time_column,
|
|
268
|
+
"event_time",
|
|
269
|
+
)
|
|
270
|
+
if item.revision_time_column is not None:
|
|
271
|
+
_validate_temporal_column(
|
|
272
|
+
state,
|
|
273
|
+
item.source_id,
|
|
274
|
+
item.revision_time_column,
|
|
275
|
+
"revision_time",
|
|
276
|
+
)
|
|
277
|
+
question_cutoff = parse_utc(policy.question_cutoff, "temporal.question_cutoff")
|
|
278
|
+
row_cutoff_index = (
|
|
279
|
+
state.index(policy.row_cutoff_column) if policy.row_cutoff_column is not None else None
|
|
280
|
+
)
|
|
281
|
+
findings: list[TemporalFinding] = []
|
|
282
|
+
checked_cells = 0
|
|
283
|
+
for source_id in sorted(sources):
|
|
284
|
+
source = evidence_by_source[source_id]
|
|
285
|
+
retrieval = parse_utc(source.retrieval_time, f"temporal.{source_id}.retrieval_time")
|
|
286
|
+
if source.fidelity == "unknown":
|
|
287
|
+
findings.append(
|
|
288
|
+
TemporalFinding(
|
|
289
|
+
code="TEMPORAL_FIDELITY_UNKNOWN",
|
|
290
|
+
source_id=source_id,
|
|
291
|
+
column=None,
|
|
292
|
+
row_index=None,
|
|
293
|
+
detail="unknown fidelity cannot support analysis readiness",
|
|
294
|
+
)
|
|
295
|
+
)
|
|
296
|
+
elif source.fidelity == "latest_only" and policy.analysis_kind == "point_in_time":
|
|
297
|
+
findings.append(
|
|
298
|
+
TemporalFinding(
|
|
299
|
+
code="TEMPORAL_LATEST_ONLY",
|
|
300
|
+
source_id=source_id,
|
|
301
|
+
column=None,
|
|
302
|
+
row_index=None,
|
|
303
|
+
detail="latest-only evidence cannot support point-in-time analysis",
|
|
304
|
+
)
|
|
305
|
+
)
|
|
306
|
+
elif source.fidelity in {"reconstructed", "latest_only"} and (
|
|
307
|
+
policy.analysis_kind != "current_retrospective" or retrieval > question_cutoff
|
|
308
|
+
):
|
|
309
|
+
findings.append(
|
|
310
|
+
TemporalFinding(
|
|
311
|
+
code="TEMPORAL_RECONSTRUCTED_CUTOFF",
|
|
312
|
+
source_id=source_id,
|
|
313
|
+
column=None,
|
|
314
|
+
row_index=None,
|
|
315
|
+
detail="reconstructed/latest evidence is unavailable by the declared cutoff",
|
|
316
|
+
)
|
|
317
|
+
)
|
|
318
|
+
for field_index, (field, lineage) in enumerate(zip(state.fields, state.lineage, strict=True)):
|
|
319
|
+
if field.semantic_type == "target" or all(
|
|
320
|
+
item.source_id not in sources for item in lineage.inputs
|
|
321
|
+
):
|
|
322
|
+
continue
|
|
323
|
+
if field.temporal_role != "none":
|
|
324
|
+
continue
|
|
325
|
+
if field.availability_column is None:
|
|
326
|
+
findings.append(
|
|
327
|
+
TemporalFinding(
|
|
328
|
+
code="TEMPORAL_AVAILABILITY_MISSING",
|
|
329
|
+
source_id=field.source_id,
|
|
330
|
+
column=field.name,
|
|
331
|
+
row_index=None,
|
|
332
|
+
detail="selected field has no availability column",
|
|
333
|
+
)
|
|
334
|
+
)
|
|
335
|
+
continue
|
|
336
|
+
availability_index = state.index(field.availability_column)
|
|
337
|
+
availability_field = state.fields[availability_index]
|
|
338
|
+
source_evidence = evidence_by_source[field.source_id]
|
|
339
|
+
if (
|
|
340
|
+
availability_field.source_id != field.source_id
|
|
341
|
+
or availability_field.temporal_role != "available_at"
|
|
342
|
+
or field.availability_column != source_evidence.available_at_column
|
|
343
|
+
):
|
|
344
|
+
findings.append(
|
|
345
|
+
TemporalFinding(
|
|
346
|
+
code="TEMPORAL_AVAILABILITY_MISMATCH",
|
|
347
|
+
source_id=field.source_id,
|
|
348
|
+
column=field.name,
|
|
349
|
+
row_index=None,
|
|
350
|
+
detail="availability semantics are not bound to the same source evidence",
|
|
351
|
+
)
|
|
352
|
+
)
|
|
353
|
+
continue
|
|
354
|
+
for row_index, row in enumerate(state.rows):
|
|
355
|
+
if row[field_index] is None:
|
|
356
|
+
continue
|
|
357
|
+
checked_cells += 1
|
|
358
|
+
cutoff = (
|
|
359
|
+
_instant(row[row_cutoff_index], "temporal.row_cutoff")
|
|
360
|
+
if row_cutoff_index is not None
|
|
361
|
+
else question_cutoff
|
|
362
|
+
)
|
|
363
|
+
if cutoff > question_cutoff:
|
|
364
|
+
findings.append(
|
|
365
|
+
TemporalFinding(
|
|
366
|
+
code="TEMPORAL_ROW_CUTOFF",
|
|
367
|
+
source_id=field.source_id,
|
|
368
|
+
column=field.name,
|
|
369
|
+
row_index=row_index,
|
|
370
|
+
detail="row cutoff exceeds the question cutoff",
|
|
371
|
+
)
|
|
372
|
+
)
|
|
373
|
+
continue
|
|
374
|
+
available = _instant(row[availability_index], "temporal.available_at")
|
|
375
|
+
if available > cutoff:
|
|
376
|
+
findings.append(
|
|
377
|
+
TemporalFinding(
|
|
378
|
+
code="TEMPORAL_LEAKAGE",
|
|
379
|
+
source_id=field.source_id,
|
|
380
|
+
column=field.name,
|
|
381
|
+
row_index=row_index,
|
|
382
|
+
detail="selected value became available after its cutoff",
|
|
383
|
+
)
|
|
384
|
+
)
|
|
385
|
+
for source in evidence:
|
|
386
|
+
if source.revision_time_column is None:
|
|
387
|
+
continue
|
|
388
|
+
try:
|
|
389
|
+
revision_index = state.index(source.revision_time_column)
|
|
390
|
+
except PreparationError:
|
|
391
|
+
continue
|
|
392
|
+
for row_index, row in enumerate(state.rows):
|
|
393
|
+
if row[revision_index] is None:
|
|
394
|
+
continue
|
|
395
|
+
cutoff = (
|
|
396
|
+
_instant(row[row_cutoff_index], "temporal.row_cutoff")
|
|
397
|
+
if row_cutoff_index is not None
|
|
398
|
+
else question_cutoff
|
|
399
|
+
)
|
|
400
|
+
if _instant(row[revision_index], "temporal.revision_time") > cutoff:
|
|
401
|
+
findings.append(
|
|
402
|
+
TemporalFinding(
|
|
403
|
+
code="TEMPORAL_FUTURE_REVISION",
|
|
404
|
+
source_id=source.source_id,
|
|
405
|
+
column=source.revision_time_column,
|
|
406
|
+
row_index=row_index,
|
|
407
|
+
detail="selected revision is later than its cutoff",
|
|
408
|
+
)
|
|
409
|
+
)
|
|
410
|
+
return TemporalReport(
|
|
411
|
+
schema_version=TEMPORAL_REPORT_VERSION,
|
|
412
|
+
analysis_kind=policy.analysis_kind,
|
|
413
|
+
question_cutoff=policy.question_cutoff,
|
|
414
|
+
checked_sources=tuple(sorted(sources)),
|
|
415
|
+
checked_cells=checked_cells,
|
|
416
|
+
findings=tuple(findings),
|
|
417
|
+
status="failed" if findings else "passed",
|
|
418
|
+
)
|
|
419
|
+
|
|
420
|
+
|
|
421
|
+
def construct_target(
|
|
422
|
+
state: TableState,
|
|
423
|
+
spec: TargetSpec | None,
|
|
424
|
+
*,
|
|
425
|
+
target_policy: str,
|
|
426
|
+
) -> tuple[TableState, TargetRationale | None]:
|
|
427
|
+
"""Construct only a justified, source-derived, cutoff-safe target."""
|
|
428
|
+
|
|
429
|
+
if target_policy == "none":
|
|
430
|
+
if spec is not None:
|
|
431
|
+
raise PreparationError("TARGET_POLICY", "target", "target is unjustified")
|
|
432
|
+
return state, None
|
|
433
|
+
if target_policy != "justified" or spec is None:
|
|
434
|
+
raise PreparationError("TARGET_POLICY", "target", "justified target evidence is required")
|
|
435
|
+
if spec.output_column in state.columns:
|
|
436
|
+
raise PreparationError("TARGET_COLUMN", "target.output_column", "already exists")
|
|
437
|
+
source_index = state.index(spec.source_column)
|
|
438
|
+
availability_index = state.index(spec.availability_time_column)
|
|
439
|
+
cutoff_index = state.index(spec.cutoff_column)
|
|
440
|
+
source_field = state.fields[source_index]
|
|
441
|
+
source_lineage = state.lineage[source_index]
|
|
442
|
+
if source_field.semantic_type == "target":
|
|
443
|
+
raise PreparationError(
|
|
444
|
+
"TARGET_ISOLATION",
|
|
445
|
+
"target.source_column",
|
|
446
|
+
"cannot derive from a target",
|
|
447
|
+
)
|
|
448
|
+
if spec.source_snapshot_digest not in {item.snapshot_digest for item in source_lineage.inputs}:
|
|
449
|
+
raise PreparationError(
|
|
450
|
+
"TARGET_SOURCE",
|
|
451
|
+
"target.source_snapshot_digest",
|
|
452
|
+
"does not bind the source column lineage",
|
|
453
|
+
)
|
|
454
|
+
mapping = {item.input_value: item.output_value for item in spec.mapping}
|
|
455
|
+
values: list[Any] = []
|
|
456
|
+
for row_index, row in enumerate(state.rows):
|
|
457
|
+
available = _instant(row[availability_index], "target.available_at")
|
|
458
|
+
cutoff = _instant(row[cutoff_index], "target.cutoff")
|
|
459
|
+
if available > cutoff:
|
|
460
|
+
raise PreparationError(
|
|
461
|
+
"TARGET_LEAKAGE",
|
|
462
|
+
f"target.row[{row_index}]",
|
|
463
|
+
"target resolution became available after the cutoff",
|
|
464
|
+
)
|
|
465
|
+
source_value = row[source_index]
|
|
466
|
+
if source_value is None:
|
|
467
|
+
if not spec.allow_null:
|
|
468
|
+
raise PreparationError(
|
|
469
|
+
"TARGET_NULL",
|
|
470
|
+
f"target.row[{row_index}]",
|
|
471
|
+
"null target source is forbidden",
|
|
472
|
+
)
|
|
473
|
+
values.append(None)
|
|
474
|
+
elif spec.operation == "exact_copy":
|
|
475
|
+
values.append(source_value)
|
|
476
|
+
else:
|
|
477
|
+
if not isinstance(source_value, str):
|
|
478
|
+
raise PreparationError(
|
|
479
|
+
"TARGET_MAPPING",
|
|
480
|
+
f"target.row[{row_index}]",
|
|
481
|
+
"categorical mapping requires string inputs",
|
|
482
|
+
)
|
|
483
|
+
try:
|
|
484
|
+
values.append(mapping[source_value])
|
|
485
|
+
except KeyError:
|
|
486
|
+
raise PreparationError(
|
|
487
|
+
"TARGET_MAPPING",
|
|
488
|
+
f"target.row[{row_index}]",
|
|
489
|
+
"categorical mapping is not total",
|
|
490
|
+
) from None
|
|
491
|
+
output_field = replace(
|
|
492
|
+
source_field,
|
|
493
|
+
name=spec.output_column,
|
|
494
|
+
semantic_type="target",
|
|
495
|
+
nullable=spec.allow_null,
|
|
496
|
+
availability_column=spec.availability_time_column,
|
|
497
|
+
)
|
|
498
|
+
rows = tuple((*row, value) for row, value in zip(state.rows, values, strict=True))
|
|
499
|
+
operation_id = f"target.{spec.output_column}"
|
|
500
|
+
lineage = (
|
|
501
|
+
*state.lineage,
|
|
502
|
+
ColumnLineage(
|
|
503
|
+
output_column=spec.output_column,
|
|
504
|
+
inputs=source_lineage.inputs,
|
|
505
|
+
operation_ids=(*source_lineage.operation_ids, operation_id),
|
|
506
|
+
),
|
|
507
|
+
)
|
|
508
|
+
output = state.replace(
|
|
509
|
+
fields=(*state.fields, output_field),
|
|
510
|
+
rows=rows,
|
|
511
|
+
lineage=lineage,
|
|
512
|
+
)
|
|
513
|
+
rationale = TargetRationale(
|
|
514
|
+
schema_version=TARGET_RATIONALE_VERSION,
|
|
515
|
+
output_column=spec.output_column,
|
|
516
|
+
operation=spec.operation,
|
|
517
|
+
source_column=spec.source_column,
|
|
518
|
+
source_snapshot_digest=spec.source_snapshot_digest,
|
|
519
|
+
resolution_rule=spec.resolution_rule,
|
|
520
|
+
horizon_seconds=spec.horizon_seconds,
|
|
521
|
+
availability_time_column=spec.availability_time_column,
|
|
522
|
+
cutoff_column=spec.cutoff_column,
|
|
523
|
+
mapping_digest=(
|
|
524
|
+
canonical_sha256([item.to_dict() for item in spec.mapping]) if spec.mapping else None
|
|
525
|
+
),
|
|
526
|
+
rationale=spec.rationale,
|
|
527
|
+
verified_rows=len(state.rows),
|
|
528
|
+
)
|
|
529
|
+
return output, rationale
|
|
530
|
+
|
|
531
|
+
|
|
532
|
+
def validate_quality(
|
|
533
|
+
state: TableState,
|
|
534
|
+
*,
|
|
535
|
+
spec: QualitySpec,
|
|
536
|
+
joins: tuple[JoinAudit, ...],
|
|
537
|
+
temporal: TemporalReport,
|
|
538
|
+
) -> tuple[TableProfile, QualityReport]:
|
|
539
|
+
"""Run the complete fixed-severity analysis-readiness gate set."""
|
|
540
|
+
|
|
541
|
+
profile = profile_table(state, key_columns=spec.grain)
|
|
542
|
+
expected_schema = [field.to_dict() for field in spec.expected_fields]
|
|
543
|
+
actual_schema = [field.to_dict() for field in state.fields]
|
|
544
|
+
checks: list[QualityCheck] = []
|
|
545
|
+
checks.append(
|
|
546
|
+
_check(
|
|
547
|
+
"schema",
|
|
548
|
+
expected_schema == actual_schema,
|
|
549
|
+
(
|
|
550
|
+
("actual_schema_digest", canonical_sha256(actual_schema)),
|
|
551
|
+
("expected_schema_digest", canonical_sha256(expected_schema)),
|
|
552
|
+
),
|
|
553
|
+
)
|
|
554
|
+
)
|
|
555
|
+
row_bounds_pass = spec.minimum_rows <= len(state.rows) <= spec.maximum_rows
|
|
556
|
+
checks.append(
|
|
557
|
+
_check(
|
|
558
|
+
"row_bounds",
|
|
559
|
+
row_bounds_pass,
|
|
560
|
+
(
|
|
561
|
+
("maximum_rows", spec.maximum_rows),
|
|
562
|
+
("minimum_rows", spec.minimum_rows),
|
|
563
|
+
("observed_rows", len(state.rows)),
|
|
564
|
+
),
|
|
565
|
+
)
|
|
566
|
+
)
|
|
567
|
+
field_profile = {item.name: item for item in profile.fields}
|
|
568
|
+
missing_failures = tuple(
|
|
569
|
+
(item.column, field_profile[item.column].null_count, item.maximum_nulls)
|
|
570
|
+
for item in spec.missingness
|
|
571
|
+
if field_profile[item.column].null_count > item.maximum_nulls
|
|
572
|
+
)
|
|
573
|
+
checks.append(
|
|
574
|
+
_check(
|
|
575
|
+
"missingness",
|
|
576
|
+
not missing_failures,
|
|
577
|
+
(
|
|
578
|
+
("failed_columns", [list(item) for item in missing_failures]),
|
|
579
|
+
("threshold_count", len(spec.missingness)),
|
|
580
|
+
),
|
|
581
|
+
)
|
|
582
|
+
)
|
|
583
|
+
grain_indexes = validate_columns(state, spec.grain, "quality.grain")
|
|
584
|
+
null_grain = sum(any(row[index] is None for index in grain_indexes) for row in state.rows)
|
|
585
|
+
checks.append(
|
|
586
|
+
_check(
|
|
587
|
+
"grain",
|
|
588
|
+
null_grain == 0 and profile.duplicate_key_count == 0,
|
|
589
|
+
(
|
|
590
|
+
("duplicate_grain_rows", profile.duplicate_key_count),
|
|
591
|
+
("null_grain_rows", null_grain),
|
|
592
|
+
),
|
|
593
|
+
)
|
|
594
|
+
)
|
|
595
|
+
checks.append(
|
|
596
|
+
_check(
|
|
597
|
+
"duplicates",
|
|
598
|
+
profile.duplicate_row_count == 0,
|
|
599
|
+
(("duplicate_rows", profile.duplicate_row_count),),
|
|
600
|
+
)
|
|
601
|
+
)
|
|
602
|
+
multiplier_failures = tuple(
|
|
603
|
+
audit.join_id
|
|
604
|
+
for audit in joins
|
|
605
|
+
if (
|
|
606
|
+
audit.row_multiplier_numerator * spec.maximum_join_multiplier_denominator
|
|
607
|
+
> audit.row_multiplier_denominator * spec.maximum_join_multiplier_numerator
|
|
608
|
+
)
|
|
609
|
+
)
|
|
610
|
+
checks.append(
|
|
611
|
+
_check(
|
|
612
|
+
"join_multiplier",
|
|
613
|
+
not multiplier_failures,
|
|
614
|
+
(
|
|
615
|
+
("failed_join_ids", list(multiplier_failures)),
|
|
616
|
+
(
|
|
617
|
+
"maximum_denominator",
|
|
618
|
+
spec.maximum_join_multiplier_denominator,
|
|
619
|
+
),
|
|
620
|
+
("maximum_numerator", spec.maximum_join_multiplier_numerator),
|
|
621
|
+
),
|
|
622
|
+
)
|
|
623
|
+
)
|
|
624
|
+
checks.append(
|
|
625
|
+
_check(
|
|
626
|
+
"temporal",
|
|
627
|
+
temporal.status == "passed",
|
|
628
|
+
(
|
|
629
|
+
("finding_codes", [item.code for item in temporal.findings]),
|
|
630
|
+
("temporal_report_digest", temporal.digest),
|
|
631
|
+
),
|
|
632
|
+
)
|
|
633
|
+
)
|
|
634
|
+
total_outliers = sum(item.outlier_count for item in profile.fields)
|
|
635
|
+
checks.append(
|
|
636
|
+
_check(
|
|
637
|
+
"outliers",
|
|
638
|
+
total_outliers <= spec.maximum_outliers,
|
|
639
|
+
(
|
|
640
|
+
("maximum_outliers", spec.maximum_outliers),
|
|
641
|
+
("observed_outliers", total_outliers),
|
|
642
|
+
),
|
|
643
|
+
)
|
|
644
|
+
)
|
|
645
|
+
missing_periods = _missing_periods(state, spec)
|
|
646
|
+
checks.append(
|
|
647
|
+
_check(
|
|
648
|
+
"missing_periods",
|
|
649
|
+
missing_periods <= spec.maximum_missing_periods,
|
|
650
|
+
(
|
|
651
|
+
("maximum_missing_periods", spec.maximum_missing_periods),
|
|
652
|
+
("observed_missing_periods", missing_periods),
|
|
653
|
+
),
|
|
654
|
+
)
|
|
655
|
+
)
|
|
656
|
+
report = QualityReport(
|
|
657
|
+
schema_version=QUALITY_REPORT_VERSION,
|
|
658
|
+
checks=tuple(checks),
|
|
659
|
+
status="failed" if any(item.status == "failed" for item in checks) else "passed",
|
|
660
|
+
)
|
|
661
|
+
return profile, report
|
|
662
|
+
|
|
663
|
+
|
|
664
|
+
def _check(
|
|
665
|
+
check_id: str,
|
|
666
|
+
passed: bool,
|
|
667
|
+
observed: tuple[tuple[str, Any], ...],
|
|
668
|
+
) -> QualityCheck:
|
|
669
|
+
severity, threshold = _CHECK_POLICY[check_id]
|
|
670
|
+
return QualityCheck(
|
|
671
|
+
check_id=check_id,
|
|
672
|
+
status="passed" if passed else "failed",
|
|
673
|
+
severity=severity,
|
|
674
|
+
threshold_id=threshold,
|
|
675
|
+
observed=observed,
|
|
676
|
+
)
|
|
677
|
+
|
|
678
|
+
|
|
679
|
+
def _missing_periods(state: TableState, spec: QualitySpec) -> int:
|
|
680
|
+
if spec.expected_frequency == "none" or spec.temporal_column is None:
|
|
681
|
+
return 0
|
|
682
|
+
temporal_index = state.index(spec.temporal_column)
|
|
683
|
+
partition_columns = tuple(column for column in spec.grain if column != spec.temporal_column)
|
|
684
|
+
partition_indexes = tuple(state.index(column) for column in partition_columns)
|
|
685
|
+
groups: dict[tuple[Any, ...], list[datetime | date]] = {}
|
|
686
|
+
for row in state.rows:
|
|
687
|
+
value = row[temporal_index]
|
|
688
|
+
if value is None:
|
|
689
|
+
continue
|
|
690
|
+
if state.fields[temporal_index].logical_type == "date":
|
|
691
|
+
instant: datetime | date = date.fromisoformat(value)
|
|
692
|
+
else:
|
|
693
|
+
instant = _instant(value, "quality.temporal_column")
|
|
694
|
+
partition = tuple(row[index] for index in partition_indexes)
|
|
695
|
+
groups.setdefault(partition, []).append(instant)
|
|
696
|
+
step = timedelta(days=1) if spec.expected_frequency == "daily" else timedelta(hours=1)
|
|
697
|
+
missing = 0
|
|
698
|
+
for values in groups.values():
|
|
699
|
+
ordered = sorted(set(values))
|
|
700
|
+
for previous, current in pairwise(ordered):
|
|
701
|
+
delta = current - previous
|
|
702
|
+
missing += max(0, int(delta / step) - 1)
|
|
703
|
+
return missing
|
|
704
|
+
|
|
705
|
+
|
|
706
|
+
def _instant(value: Any, path: str) -> datetime:
|
|
707
|
+
if not isinstance(value, str):
|
|
708
|
+
raise PreparationError("TEMPORAL_VALUE", path, "must be a UTC timestamp string")
|
|
709
|
+
return parse_utc(value, path)
|
|
710
|
+
|
|
711
|
+
|
|
712
|
+
def _validate_temporal_column(
|
|
713
|
+
state: TableState,
|
|
714
|
+
source_id: str,
|
|
715
|
+
column: str,
|
|
716
|
+
role: str,
|
|
717
|
+
) -> None:
|
|
718
|
+
try:
|
|
719
|
+
field = state.field(column)
|
|
720
|
+
except PreparationError:
|
|
721
|
+
raise PreparationError(
|
|
722
|
+
"TEMPORAL_EVIDENCE",
|
|
723
|
+
source_id,
|
|
724
|
+
f"declared {role} column is absent from the joined table",
|
|
725
|
+
) from None
|
|
726
|
+
if field.source_id != source_id or field.temporal_role != role:
|
|
727
|
+
raise PreparationError(
|
|
728
|
+
"TEMPORAL_EVIDENCE",
|
|
729
|
+
source_id,
|
|
730
|
+
f"declared {role} column has mismatched source or temporal role",
|
|
731
|
+
)
|
|
732
|
+
|
|
733
|
+
|
|
734
|
+
def _is_digest(value: Any) -> bool:
|
|
735
|
+
return (
|
|
736
|
+
isinstance(value, str)
|
|
737
|
+
and len(value) == 64
|
|
738
|
+
and all(character in "0123456789abcdef" for character in value)
|
|
739
|
+
)
|
|
740
|
+
|
|
741
|
+
|
|
742
|
+
__all__ = [
|
|
743
|
+
"QualityCheck",
|
|
744
|
+
"QualityReport",
|
|
745
|
+
"TargetRationale",
|
|
746
|
+
"TemporalFinding",
|
|
747
|
+
"TemporalReport",
|
|
748
|
+
"construct_target",
|
|
749
|
+
"validate_quality",
|
|
750
|
+
"validate_temporal_safety",
|
|
751
|
+
]
|