kiri-cli 1.0.0b3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of kiri-cli might be problematic. Click here for more details.
- kiri_adapters/__init__.py +193 -0
- kiri_adapters/backends/__init__.py +0 -0
- kiri_adapters/backends/databricks.manifest.yml +55 -0
- kiri_adapters/backends/databricks.py +2980 -0
- kiri_adapters/backends/fabric_dw.manifest.yml +17 -0
- kiri_adapters/backends/fabric_dw.py +307 -0
- kiri_adapters/backends/fabric_lakehouse.manifest.yml +19 -0
- kiri_adapters/backends/fabric_lakehouse.py +365 -0
- kiri_adapters/backends/iceberg.manifest.yml +17 -0
- kiri_adapters/backends/iceberg.py +451 -0
- kiri_adapters/backends/localduckdb.manifest.yml +43 -0
- kiri_adapters/backends/localduckdb.py +1295 -0
- kiri_adapters/backends/mssql.manifest.yml +17 -0
- kiri_adapters/backends/mssql.py +366 -0
- kiri_adapters/backends/postgres.manifest.yml +17 -0
- kiri_adapters/backends/postgres.py +434 -0
- kiri_adapters/backends/trino.manifest.yml +17 -0
- kiri_adapters/backends/trino.py +542 -0
- kiri_adapters/base.py +892 -0
- kiri_adapters/doctor_checks.py +81 -0
- kiri_adapters/dv_planner.py +1205 -0
- kiri_adapters/dv_planner_bridge.py +265 -0
- kiri_adapters/dv_planner_pit.py +265 -0
- kiri_adapters/dv_planner_ref.py +403 -0
- kiri_adapters/dv_planner_sat_variants.py +668 -0
- kiri_adapters/framework.py +112 -0
- kiri_adapters/incident/__init__.py +30 -0
- kiri_adapters/incident/_http.py +121 -0
- kiri_adapters/incident/base.py +89 -0
- kiri_adapters/incident/jira.py +365 -0
- kiri_adapters/incident/servicenow.py +314 -0
- kiri_adapters/incident/zendesk.py +285 -0
- kiri_adapters/manifest.py +466 -0
- kiri_adapters/orchestration/__init__.py +30 -0
- kiri_adapters/orchestration/airflow.py +117 -0
- kiri_adapters/orchestration/artifact_emitter.py +99 -0
- kiri_adapters/orchestration/autoloader_emitter.py +141 -0
- kiri_adapters/orchestration/azure_data_factory.py +137 -0
- kiri_adapters/orchestration/base.py +626 -0
- kiri_adapters/orchestration/databricks_workflows.py +1768 -0
- kiri_adapters/orchestration/k8s_cron.py +555 -0
- kiri_adapters/orchestration/lifecycle.py +222 -0
- kiri_adapters/orchestration/pipeline_compat.py +145 -0
- kiri_adapters/orchestration/runtime_python.py +188 -0
- kiri_adapters/paths.py +38 -0
- kiri_adapters/protocols/__init__.py +64 -0
- kiri_adapters/protocols/ai_service/__init__.py +7 -0
- kiri_adapters/protocols/ai_service/conformance.py +120 -0
- kiri_adapters/protocols/base.py +220 -0
- kiri_adapters/protocols/catalog/__init__.py +7 -0
- kiri_adapters/protocols/catalog/conformance.py +138 -0
- kiri_adapters/protocols/framework/__init__.py +7 -0
- kiri_adapters/protocols/framework/conformance.py +98 -0
- kiri_adapters/protocols/incident/__init__.py +7 -0
- kiri_adapters/protocols/incident/conformance.py +64 -0
- kiri_adapters/protocols/orchestration/__init__.py +7 -0
- kiri_adapters/protocols/orchestration/conformance.py +84 -0
- kiri_adapters/protocols/platform/__init__.py +7 -0
- kiri_adapters/protocols/platform/conformance.py +120 -0
- kiri_adapters/protocols/sql_dialect/__init__.py +7 -0
- kiri_adapters/protocols/sql_dialect/conformance.py +91 -0
- kiri_adapters/protocols/vault/__init__.py +7 -0
- kiri_adapters/protocols/vault/conformance.py +65 -0
- kiri_adapters/py.typed +0 -0
- kiri_adapters/registry.py +104 -0
- kiri_adapters/semantic/__init__.py +60 -0
- kiri_adapters/semantic/apply.py +274 -0
- kiri_adapters/semantic/base.py +530 -0
- kiri_adapters/semantic/cube.py +280 -0
- kiri_adapters/semantic/dbt_semantic.py +92 -0
- kiri_adapters/semantic/metric_views.py +249 -0
- kiri_adapters/semantic/metricflow.py +345 -0
- kiri_adapters/semantic/snowflake_views.py +75 -0
- kiri_adapters/sql_dialect.py +1592 -0
- kiri_adapters/tsql_to_spark.py +653 -0
- kiri_ai_gateway/__init__.py +147 -0
- kiri_ai_gateway/audit.py +334 -0
- kiri_ai_gateway/batch.py +293 -0
- kiri_ai_gateway/config.py +165 -0
- kiri_ai_gateway/gateway.py +642 -0
- kiri_ai_gateway/placeholder_guard.py +145 -0
- kiri_ai_gateway/prices/__init__.py +77 -0
- kiri_ai_gateway/prices/prices.yaml +64 -0
- kiri_ai_gateway/prompts/__init__.py +54 -0
- kiri_ai_gateway/prompts/airbyte_source_setup.yaml +212 -0
- kiri_ai_gateway/prompts/bdv_translation.yaml +90 -0
- kiri_ai_gateway/prompts/contract_draft_mvp.yaml +117 -0
- kiri_ai_gateway/prompts/describe_table.yaml +19 -0
- kiri_ai_gateway/prompts/design_gold_from_question.yaml +130 -0
- kiri_ai_gateway/prompts/detect_pii.yaml +59 -0
- kiri_ai_gateway/prompts/discover_classify_column.yaml +69 -0
- kiri_ai_gateway/prompts/discover_describe_table.yaml +61 -0
- kiri_ai_gateway/prompts/dv_quality_summary.yaml +36 -0
- kiri_ai_gateway/prompts/incident_assist.yaml +58 -0
- kiri_ai_gateway/prompts/infer_source_schema.yaml +31 -0
- kiri_ai_gateway/prompts/ingest_backend_chooser.yaml +116 -0
- kiri_ai_gateway/prompts/interactive_source_edit.yaml +94 -0
- kiri_ai_gateway/prompts/map_needed_entity_to_source.yaml +72 -0
- kiri_ai_gateway/prompts/meet_kiri.yaml +127 -0
- kiri_ai_gateway/prompts/native_source_setup.yaml +211 -0
- kiri_ai_gateway/prompts/source_grouping_reason.yaml +29 -0
- kiri_ai_gateway/prompts/suggest_attribute_classification.yaml +42 -0
- kiri_ai_gateway/prompts/suggest_data_tests.yaml +71 -0
- kiri_ai_gateway/prompts/suggest_gold_dimension.yaml +108 -0
- kiri_ai_gateway/prompts/suggest_gold_fact.yaml +107 -0
- kiri_ai_gateway/prompts/suggest_gold_model.yaml +70 -0
- kiri_ai_gateway/prompts/suggest_semantic_intent.yaml +91 -0
- kiri_ai_gateway/prompts/suggest_silver_model.yaml +140 -0
- kiri_ai_gateway/providers/__init__.py +7 -0
- kiri_ai_gateway/providers/anthropic_provider.py +26 -0
- kiri_ai_gateway/providers/gateway_provider.py +188 -0
- kiri_ai_gateway/providers/local_provider.py +177 -0
- kiri_ai_gateway/py.typed +0 -0
- kiri_ai_gateway/service.py +283 -0
- kiri_ai_gateway/transport.py +103 -0
- kiri_ai_gateway/transports/__init__.py +12 -0
- kiri_ai_gateway/transports/anthropic_transport.py +73 -0
- kiri_ai_gateway/transports/ollama_transport.py +82 -0
- kiri_ai_gateway/transports/openai_compatible_transport.py +125 -0
- kiri_ai_gateway/workspace_policy.py +52 -0
- kiri_catalog/__init__.py +199 -0
- kiri_catalog/annotation_plan.py +183 -0
- kiri_catalog/appearances.py +155 -0
- kiri_catalog/backends/__init__.py +79 -0
- kiri_catalog/backends/base.py +167 -0
- kiri_catalog/backends/memory.py +241 -0
- kiri_catalog/backends/postgres.py +497 -0
- kiri_catalog/backends/redacting.py +195 -0
- kiri_catalog/backends/sqlite.py +480 -0
- kiri_catalog/context.py +94 -0
- kiri_catalog/federation/__init__.py +114 -0
- kiri_catalog/federation/api.py +200 -0
- kiri_catalog/federation/cache.py +149 -0
- kiri_catalog/federation/emit.py +209 -0
- kiri_catalog/federation/fs_resolver.py +295 -0
- kiri_catalog/federation/http_resolver.py +486 -0
- kiri_catalog/federation/metrics.py +123 -0
- kiri_catalog/federation/release.py +342 -0
- kiri_catalog/federation/resolver.py +332 -0
- kiri_catalog/federation/schema.py +460 -0
- kiri_catalog/importer.py +668 -0
- kiri_catalog/lineage_drift.py +280 -0
- kiri_catalog/lineage_graph.py +176 -0
- kiri_catalog/lineage_graph_explain.py +96 -0
- kiri_catalog/lineage_graph_mermaid.py +230 -0
- kiri_catalog/observed_uc_lineage.py +216 -0
- kiri_catalog/pii_detect.py +176 -0
- kiri_catalog/push.py +122 -0
- kiri_catalog/py.typed +0 -0
- kiri_catalog/redact.py +101 -0
- kiri_catalog/review_actions.py +119 -0
- kiri_catalog/schema/__init__.py +70 -0
- kiri_catalog/schema/asset.py +142 -0
- kiri_catalog/schema/column.py +247 -0
- kiri_catalog/schema/enums.py +195 -0
- kiri_catalog/schema/job.py +51 -0
- kiri_catalog/schema/lineage.py +33 -0
- kiri_catalog/schema/policy.py +69 -0
- kiri_catalog/schema/primitives.py +90 -0
- kiri_catalog/schema/review_state.py +111 -0
- kiri_catalog/schema/urn.py +45 -0
- kiri_catalog/schema_discovery.py +160 -0
- kiri_catalog/sinks/__init__.py +64 -0
- kiri_catalog/sinks/base.py +178 -0
- kiri_catalog/sinks/horizon.py +195 -0
- kiri_catalog/sinks/purview.py +216 -0
- kiri_catalog/sinks/unity.py +352 -0
- kiri_catalog/sinks/unity_executor.py +543 -0
- kiri_catalog/sources/__init__.py +42 -0
- kiri_catalog/sources/base.py +104 -0
- kiri_catalog/sources/unity.py +465 -0
- kiri_catalog/steward_writeback.py +158 -0
- kiri_catalog/tag_sql.py +99 -0
- kiri_cli/LICENSE +105 -0
- kiri_cli/__init__.py +1 -0
- kiri_cli/_apply_cli.py +63 -0
- kiri_cli/_apply_reporter.py +38 -0
- kiri_cli/_closed_menu.py +36 -0
- kiri_cli/_fixture_option.py +18 -0
- kiri_cli/_layer_detect.py +158 -0
- kiri_cli/_project_option.py +79 -0
- kiri_cli/adapter_cmd.py +192 -0
- kiri_cli/ai_batch_cmd.py +258 -0
- kiri_cli/ai_cmd.py +352 -0
- kiri_cli/approve_cmd.py +659 -0
- kiri_cli/attr_cmd.py +575 -0
- kiri_cli/audit_cmd.py +348 -0
- kiri_cli/audit_correlate_cmd.py +648 -0
- kiri_cli/audit_join.py +437 -0
- kiri_cli/audit_join_cmd.py +160 -0
- kiri_cli/auto_cmd.py +97 -0
- kiri_cli/azure_cmd.py +243 -0
- kiri_cli/catalog.py +1293 -0
- kiri_cli/cli_reference_live.py +147 -0
- kiri_cli/compile_cmd.py +493 -0
- kiri_cli/compliance_cmd.py +221 -0
- kiri_cli/contract_approval_cmd.py +94 -0
- kiri_cli/contract_cmd.py +64 -0
- kiri_cli/contract_codeowners_cmd.py +162 -0
- kiri_cli/contract_diff.py +111 -0
- kiri_cli/contract_emit.py +90 -0
- kiri_cli/contract_export.py +192 -0
- kiri_cli/contract_lint.py +1505 -0
- kiri_cli/contract_merge_cmd.py +264 -0
- kiri_cli/contract_new.py +326 -0
- kiri_cli/contract_promote.py +216 -0
- kiri_cli/contract_scaffold_cmd.py +134 -0
- kiri_cli/contract_schema_export.py +69 -0
- kiri_cli/contract_show.py +150 -0
- kiri_cli/contract_stubs.py +60 -0
- kiri_cli/contract_sync_from_live.py +327 -0
- kiri_cli/contract_validate.py +154 -0
- kiri_cli/contract_verify_live.py +242 -0
- kiri_cli/cost_cmd.py +243 -0
- kiri_cli/databricks_health.py +1254 -0
- kiri_cli/databricks_setup_cmd.py +712 -0
- kiri_cli/discover_cmd.py +600 -0
- kiri_cli/docs_cmd.py +35 -0
- kiri_cli/doctor_cmd.py +145 -0
- kiri_cli/dv_cmd.py +814 -0
- kiri_cli/dv_conformance_cmd.py +154 -0
- kiri_cli/dv_install.py +493 -0
- kiri_cli/dv_migrate_cmd.py +183 -0
- kiri_cli/dv_test_cmd.py +175 -0
- kiri_cli/federation_cmd.py +315 -0
- kiri_cli/fk_cmd.py +160 -0
- kiri_cli/flows_cmd.py +910 -0
- kiri_cli/governance_cmd.py +358 -0
- kiri_cli/incident_api_cmd.py +180 -0
- kiri_cli/incident_cmd.py +410 -0
- kiri_cli/incident_setup_cmd.py +239 -0
- kiri_cli/ingest.py +427 -0
- kiri_cli/ingest_discover.py +480 -0
- kiri_cli/ingest_new_source.py +386 -0
- kiri_cli/intent_cmd.py +122 -0
- kiri_cli/invariants_cmd.py +118 -0
- kiri_cli/kiri_banner.py +161 -0
- kiri_cli/kpi_cmd.py +222 -0
- kiri_cli/layer_cmd.py +464 -0
- kiri_cli/lineage.py +601 -0
- kiri_cli/main.py +2472 -0
- kiri_cli/maturity_cmd.py +222 -0
- kiri_cli/migrate_cmd.py +1347 -0
- kiri_cli/onboard_cmd.py +223 -0
- kiri_cli/pack_cmd.py +496 -0
- kiri_cli/pipeline_cmd.py +127 -0
- kiri_cli/pre_classify_cmd.py +274 -0
- kiri_cli/project_cmd.py +592 -0
- kiri_cli/py.typed +0 -0
- kiri_cli/quality_cmd.py +295 -0
- kiri_cli/rbac.py +139 -0
- kiri_cli/rbac_cmd.py +291 -0
- kiri_cli/release.py +850 -0
- kiri_cli/release_semver.py +265 -0
- kiri_cli/repo_cmd.py +155 -0
- kiri_cli/retention_cmd.py +333 -0
- kiri_cli/sample_data/__init__.py +12 -0
- kiri_cli/sample_data/customers.contract.yml +47 -0
- kiri_cli/sample_data/customers.csv +25 -0
- kiri_cli/scaffold_cmd.py +721 -0
- kiri_cli/schema_cmd.py +185 -0
- kiri_cli/semantic_cmd.py +605 -0
- kiri_cli/silver_cmd.py +348 -0
- kiri_cli/skills_cmd.py +243 -0
- kiri_cli/skills_data/README.md +51 -0
- kiri_cli/skills_data/add-source/SKILL.md +116 -0
- kiri_cli/skills_data/azure-deploy/SKILL.md +261 -0
- kiri_cli/skills_data/azure-status/SKILL.md +111 -0
- kiri_cli/skills_data/compliance-check/SKILL.md +99 -0
- kiri_cli/skills_data/databricks-build-pipeline/SKILL.md +112 -0
- kiri_cli/skills_data/databricks-project-setup/SKILL.md +157 -0
- kiri_cli/skills_data/debug-apply/SKILL.md +186 -0
- kiri_cli/skills_data/draft-gold-model/SKILL.md +203 -0
- kiri_cli/skills_data/explain-kpi/SKILL.md +158 -0
- kiri_cli/skills_data/incident-coach/SKILL.md +135 -0
- kiri_cli/skills_data/incident-setup/SKILL.md +112 -0
- kiri_cli/skills_data/maturity-coach/SKILL.md +120 -0
- kiri_cli/skills_data/promote-contract/SKILL.md +91 -0
- kiri_cli/skills_data/review-attributes/SKILL.md +94 -0
- kiri_cli/sla_cmd.py +769 -0
- kiri_cli/source_cmd.py +909 -0
- kiri_cli/templates/__init__.py +6 -0
- kiri_cli/templates/dv4dbt_packages.yml.j2 +11 -0
- kiri_cli/upgrade_cmd.py +896 -0
- kiri_cli/use_cmd.py +120 -0
- kiri_cli/vault_cmd.py +241 -0
- kiri_cli/verify_generated.py +241 -0
- kiri_cli-1.0.0b3.dist-info/METADATA +212 -0
- kiri_cli-1.0.0b3.dist-info/RECORD +743 -0
- kiri_cli-1.0.0b3.dist-info/WHEEL +4 -0
- kiri_cli-1.0.0b3.dist-info/entry_points.txt +4 -0
- kiri_compliance/__init__.py +33 -0
- kiri_compliance/audit_coverage.py +354 -0
- kiri_compliance/controls.py +763 -0
- kiri_compliance/evidence.py +155 -0
- kiri_compliance/py.typed +0 -0
- kiri_compliance/quality_engine.py +246 -0
- kiri_compliance/renderer.py +195 -0
- kiri_compliance/report.py +44 -0
- kiri_compliance/schema.py +129 -0
- kiri_core/__init__.py +185 -0
- kiri_core/adapter_surfaces.py +149 -0
- kiri_core/ai_query_gate/__init__.py +92 -0
- kiri_core/ai_query_gate/audit.py +153 -0
- kiri_core/ai_query_gate/evaluator.py +484 -0
- kiri_core/ai_query_gate/models.py +162 -0
- kiri_core/ai_query_gate/parser.py +436 -0
- kiri_core/ai_query_gate/resolver.py +274 -0
- kiri_core/ai_query_gate/store.py +234 -0
- kiri_core/ai_query_gate/store_postgres.py +409 -0
- kiri_core/api_keys.py +243 -0
- kiri_core/apply/__init__.py +27 -0
- kiri_core/apply/conformance.py +99 -0
- kiri_core/apply/incident_router.py +176 -0
- kiri_core/apply/models.py +140 -0
- kiri_core/apply/reporter.py +46 -0
- kiri_core/apply/runner.py +3083 -0
- kiri_core/apply_log_store.py +155 -0
- kiri_core/approval_flow/__init__.py +133 -0
- kiri_core/approval_flow/integration.py +156 -0
- kiri_core/approval_flow/models.py +443 -0
- kiri_core/approval_flow/notifier.py +263 -0
- kiri_core/approval_flow/orchestrator.py +660 -0
- kiri_core/approval_flow/reconcile_signoff.py +53 -0
- kiri_core/approval_flow/store.py +220 -0
- kiri_core/approval_flow/store_jsonl.py +217 -0
- kiri_core/approval_flow/store_postgres.py +580 -0
- kiri_core/approval_policy.py +116 -0
- kiri_core/asgi_trace.py +160 -0
- kiri_core/compliance.py +802 -0
- kiri_core/cutover_approval.py +187 -0
- kiri_core/discovery/__init__.py +109 -0
- kiri_core/discovery/ai/__init__.py +37 -0
- kiri_core/discovery/ai/_payload.py +115 -0
- kiri_core/discovery/ai/budget.py +83 -0
- kiri_core/discovery/ai/classify.py +223 -0
- kiri_core/discovery/ai/describe.py +200 -0
- kiri_core/discovery/ai/errors.py +49 -0
- kiri_core/discovery/discover.py +648 -0
- kiri_core/discovery/heuristics/__init__.py +42 -0
- kiri_core/discovery/heuristics/classification.py +174 -0
- kiri_core/discovery/heuristics/medallion.py +157 -0
- kiri_core/discovery/heuristics/pii.py +92 -0
- kiri_core/discovery/introspector.py +636 -0
- kiri_core/discovery/introspector_databricks.py +396 -0
- kiri_core/discovery/introspector_postgres.py +394 -0
- kiri_core/discovery/introspector_snowflake.py +383 -0
- kiri_core/discovery/reporting/__init__.py +17 -0
- kiri_core/discovery/reporting/discovery_report.py +190 -0
- kiri_core/discovery/scaffold/__init__.py +41 -0
- kiri_core/discovery/scaffold/contract_writer.py +632 -0
- kiri_core/discovery/scaffold/provenance_overlay.py +143 -0
- kiri_core/discovery/scaffold/source_writer.py +211 -0
- kiri_core/discovery/scaffold/todo_marker.py +98 -0
- kiri_core/domain_policy.py +122 -0
- kiri_core/goal_coverage.py +82 -0
- kiri_core/gold_registry.py +211 -0
- kiri_core/health.py +192 -0
- kiri_core/incident.py +204 -0
- kiri_core/incident_builders.py +256 -0
- kiri_core/incident_log.py +418 -0
- kiri_core/intent/__init__.py +46 -0
- kiri_core/intent/agent_runner.py +82 -0
- kiri_core/intent/project_dv.py +261 -0
- kiri_core/intent/propose.py +132 -0
- kiri_core/intent/semantic_intent.py +446 -0
- kiri_core/invariants.py +301 -0
- kiri_core/kpi.py +373 -0
- kiri_core/latency.py +150 -0
- kiri_core/lifecycle/__init__.py +555 -0
- kiri_core/lifecycle/ai_index.py +453 -0
- kiri_core/lifecycle/auto_mode.py +172 -0
- kiri_core/lifecycle/bootstrap.py +316 -0
- kiri_core/lifecycle/cli_reference.py +1064 -0
- kiri_core/lifecycle/commit.py +258 -0
- kiri_core/lifecycle/compile_dispatch.py +296 -0
- kiri_core/lifecycle/compile_range.py +665 -0
- kiri_core/lifecycle/contract_inventory.py +354 -0
- kiri_core/lifecycle/contracts.py +250 -0
- kiri_core/lifecycle/doctor.py +412 -0
- kiri_core/lifecycle/git_remote.py +948 -0
- kiri_core/lifecycle/identity.py +147 -0
- kiri_core/lifecycle/ingestion_modes.py +215 -0
- kiri_core/lifecycle/layer_contract.py +452 -0
- kiri_core/lifecycle/layer_contract_migrate.py +969 -0
- kiri_core/lifecycle/layer_contract_parser.py +436 -0
- kiri_core/lifecycle/layer_contract_validator.py +2291 -0
- kiri_core/lifecycle/layout.py +298 -0
- kiri_core/lifecycle/onboarding.py +306 -0
- kiri_core/lifecycle/openlineage_emit.py +153 -0
- kiri_core/lifecycle/patch.py +231 -0
- kiri_core/lifecycle/plan.py +241 -0
- kiri_core/lifecycle/probe.py +204 -0
- kiri_core/lifecycle/project_context.py +334 -0
- kiri_core/lifecycle/project_status.py +532 -0
- kiri_core/lifecycle/scaffold.py +455 -0
- kiri_core/lifecycle/source_grouping.py +725 -0
- kiri_core/lifecycle/source_introspection.py +488 -0
- kiri_core/lifecycle/source_inventory.py +561 -0
- kiri_core/lifecycle/source_patch.py +353 -0
- kiri_core/lifecycle/sources.py +470 -0
- kiri_core/lifecycle/sync_from_live.py +727 -0
- kiri_core/lifecycle/validate.py +116 -0
- kiri_core/lifecycle/verify_annotations.py +391 -0
- kiri_core/lifecycle/verify_live.py +389 -0
- kiri_core/lifecycle/verify_metric_views.py +321 -0
- kiri_core/lifecycle/watermark.py +371 -0
- kiri_core/lifecycle/watermark_apply.py +656 -0
- kiri_core/lineage.py +418 -0
- kiri_core/loader.py +377 -0
- kiri_core/logging.py +374 -0
- kiri_core/maturity.py +592 -0
- kiri_core/migration_layer_policy.py +285 -0
- kiri_core/names.py +163 -0
- kiri_core/observed_lineage.py +62 -0
- kiri_core/odcs/__init__.py +15 -0
- kiri_core/otel_bootstrap.py +326 -0
- kiri_core/pipeline_ir.py +332 -0
- kiri_core/pipeline_loader.py +82 -0
- kiri_core/planning.py +110 -0
- kiri_core/project.py +1576 -0
- kiri_core/project_docs.py +199 -0
- kiri_core/py.typed +0 -0
- kiri_core/rate_limit.py +456 -0
- kiri_core/release_state/__init__.py +37 -0
- kiri_core/release_state/models.py +108 -0
- kiri_core/release_state/store.py +634 -0
- kiri_core/resilience/__init__.py +288 -0
- kiri_core/resilience/retry.py +173 -0
- kiri_core/runtime.py +40 -0
- kiri_core/schema/__init__.py +62 -0
- kiri_core/schema/column.py +140 -0
- kiri_core/schema/ingestion.py +665 -0
- kiri_core/schema/model.py +44 -0
- kiri_core/schema/reporting_goal.py +160 -0
- kiri_core/schema/source.py +188 -0
- kiri_core/silver/__init__.py +148 -0
- kiri_core/silver/apply_runner.py +830 -0
- kiri_core/silver/backfill.py +280 -0
- kiri_core/silver/dv_backend.py +426 -0
- kiri_core/silver/dv_graph.py +932 -0
- kiri_core/silver/dv_graph_mermaid.py +443 -0
- kiri_core/silver/dv_graph_payload.py +79 -0
- kiri_core/silver/dv_graph_views.py +416 -0
- kiri_core/silver/graph_projector.py +566 -0
- kiri_core/silver/hash_canonicaliser.py +172 -0
- kiri_core/silver/hash_policy.py +195 -0
- kiri_core/silver/load_ts_policy.py +202 -0
- kiri_core/silver/schema_evolution.py +410 -0
- kiri_core/silver/zone_rules.py +81 -0
- kiri_core/sizing.py +134 -0
- kiri_core/storage_paths.py +130 -0
- kiri_core/technique_validator.py +700 -0
- kiri_core/trace.py +409 -0
- kiri_core/types.py +41 -0
- kiri_core/urn.py +161 -0
- kiri_dbt_bridge/__init__.py +22 -0
- kiri_dbt_bridge/_schema_resolve.py +38 -0
- kiri_dbt_bridge/bridge.py +94 -0
- kiri_dbt_bridge/dbt.manifest.yml +26 -0
- kiri_dbt_bridge/dv4dbt/__init__.py +143 -0
- kiri_dbt_bridge/dv4dbt/backend.py +204 -0
- kiri_dbt_bridge/dv4dbt/bridge_generator.py +289 -0
- kiri_dbt_bridge/dv4dbt/compat.py +232 -0
- kiri_dbt_bridge/dv4dbt/hub_generator.py +282 -0
- kiri_dbt_bridge/dv4dbt/link_generator.py +306 -0
- kiri_dbt_bridge/dv4dbt/manifest.py +203 -0
- kiri_dbt_bridge/dv4dbt/pit_generator.py +318 -0
- kiri_dbt_bridge/dv4dbt/policy_translation.py +249 -0
- kiri_dbt_bridge/dv4dbt/project_writer.py +588 -0
- kiri_dbt_bridge/dv4dbt/ref_generator.py +404 -0
- kiri_dbt_bridge/dv4dbt/render.py +180 -0
- kiri_dbt_bridge/dv4dbt/sat_generator.py +480 -0
- kiri_dbt_bridge/dv4dbt/sat_variant_generator.py +529 -0
- kiri_dbt_bridge/dv4dbt/templates/__init__.py +16 -0
- kiri_dbt_bridge/dv4dbt/templates/bridge.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/dbt_project.yml.j2 +37 -0
- kiri_dbt_bridge/dv4dbt/templates/hub.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/link.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/packages.yml.j2 +10 -0
- kiri_dbt_bridge/dv4dbt/templates/pit.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/ref_historised.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/ref_non_historised.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/sat.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/sat_effectivity.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/sat_multi_active.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/sat_non_historised.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/sat_record_tracking.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/sat_virtual_end_dated.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/sources.yml.j2 +39 -0
- kiri_dbt_bridge/framework_adapter.py +310 -0
- kiri_dbt_bridge/generate.py +372 -0
- kiri_dbt_bridge/gold/__init__.py +21 -0
- kiri_dbt_bridge/gold/_sql_emit.py +128 -0
- kiri_dbt_bridge/gold/assemble.py +90 -0
- kiri_dbt_bridge/gold/canonical_columns.py +168 -0
- kiri_dbt_bridge/gold/computed_expressions.py +179 -0
- kiri_dbt_bridge/gold/dialects/__init__.py +69 -0
- kiri_dbt_bridge/gold/dialects/databricks.py +150 -0
- kiri_dbt_bridge/gold/dialects/duckdb.py +120 -0
- kiri_dbt_bridge/gold/dialects/trino.py +136 -0
- kiri_dbt_bridge/gold/dim_generator.py +688 -0
- kiri_dbt_bridge/gold/dimension_refs.py +255 -0
- kiri_dbt_bridge/gold/dispatch.py +69 -0
- kiri_dbt_bridge/gold/fact_generator.py +534 -0
- kiri_dbt_bridge/gold/flat_generator.py +144 -0
- kiri_dbt_bridge/gold/lineage.py +484 -0
- kiri_dbt_bridge/gold/localized_twins.py +218 -0
- kiri_dbt_bridge/gold/measures.py +49 -0
- kiri_dbt_bridge/gold/role_playing.py +75 -0
- kiri_dbt_bridge/gold/surrogate_key.py +114 -0
- kiri_dbt_bridge/macros/kiri_scd1_dim.sql +57 -0
- kiri_dbt_bridge/macros/kiri_scd2_dim.sql +188 -0
- kiri_dbt_bridge/macros/kiri_sha256.sql +32 -0
- kiri_dbt_bridge/macros/kiri_string_cast.sql +37 -0
- kiri_dbt_bridge/macros/kiri_surrogate_key.sql +43 -0
- kiri_dbt_bridge/profiles.py +162 -0
- kiri_dbt_bridge/py.typed +0 -0
- kiri_dbt_bridge/safe_ident.py +44 -0
- kiri_dbt_bridge/silver_conformance.py +188 -0
- kiri_dbt_bridge/silver_kimball/__init__.py +27 -0
- kiri_dbt_bridge/silver_kimball/assemble.py +174 -0
- kiri_dbt_bridge/silver_kimball/dim_generator.py +602 -0
- kiri_dbt_bridge/silver_kimball/dispatch.py +86 -0
- kiri_dbt_bridge/silver_kimball/fact_generator.py +320 -0
- kiri_dbt_bridge/silver_kimball/flat_generator.py +156 -0
- kiri_dbt_bridge/silver_kimball/lineage.py +464 -0
- kiri_dbt_bridge/silver_kimball/watermark_window.py +199 -0
- kiri_dbt_bridge/silver_kimball/zone.py +68 -0
- kiri_dbt_bridge/silver_scd2_curated/__init__.py +46 -0
- kiri_dbt_bridge/silver_scd2_curated/assemble.py +213 -0
- kiri_dbt_bridge/silver_scd2_curated/curated_generator.py +351 -0
- kiri_dbt_bridge/silver_scd2_curated/standardized_generator.py +261 -0
- kiri_dbt_bridge/silver_scd2_curated/standardized_plan.py +254 -0
- kiri_iceberg/__init__.py +11 -0
- kiri_iceberg/catalog.py +128 -0
- kiri_iceberg/profile.py +131 -0
- kiri_iceberg/py.typed +0 -0
- kiri_ingestion/__init__.py +63 -0
- kiri_ingestion/airbyte_client.py +428 -0
- kiri_ingestion/database_fetcher.py +224 -0
- kiri_ingestion/layout.py +71 -0
- kiri_ingestion/mssql_introspect.py +565 -0
- kiri_ingestion/py.typed +0 -0
- kiri_ingestion/rest_fetcher.py +472 -0
- kiri_ingestion/runner.py +341 -0
- kiri_ingestion/schema_drift.py +222 -0
- kiri_ingestion/storage.py +621 -0
- kiri_mcp/__init__.py +5 -0
- kiri_mcp/audit.py +99 -0
- kiri_mcp/audit_decorators.py +252 -0
- kiri_mcp/main.py +22 -0
- kiri_mcp/py.typed +0 -0
- kiri_mcp/rbac.py +110 -0
- kiri_mcp/server.py +766 -0
- kiri_mcp/skills.py +297 -0
- kiri_mcp/tools/__init__.py +70 -0
- kiri_mcp/tools/_shared.py +58 -0
- kiri_mcp/tools/adapter.py +150 -0
- kiri_mcp/tools/cli_reference.py +59 -0
- kiri_mcp/tools/compliance.py +119 -0
- kiri_mcp/tools/contract.py +374 -0
- kiri_mcp/tools/dv.py +62 -0
- kiri_mcp/tools/federation.py +152 -0
- kiri_mcp/tools/intent.py +71 -0
- kiri_mcp/tools/lineage.py +204 -0
- kiri_mcp/tools/lineage_drift.py +75 -0
- kiri_mcp/tools/plan.py +66 -0
- kiri_mcp/tools/project.py +311 -0
- kiri_migration/__init__.py +56 -0
- kiri_migration/name_preservation.py +271 -0
- kiri_migration/predeploy.py +311 -0
- kiri_migration/py.typed +0 -0
- kiri_migration/reconcile.py +1891 -0
- kiri_migration/scaffold/__init__.py +90 -0
- kiri_migration/scaffold/_audit_columns.py +74 -0
- kiri_migration/scaffold/_classify.py +100 -0
- kiri_migration/scaffold/_typemap.py +71 -0
- kiri_migration/scaffold/_yaml.py +33 -0
- kiri_migration/scaffold/bronze.py +279 -0
- kiri_migration/scaffold/compile_targets/__init__.py +271 -0
- kiri_migration/scaffold/compile_targets/_common.py +193 -0
- kiri_migration/scaffold/compile_targets/copy_into.py +33 -0
- kiri_migration/scaffold/compile_targets/dlt.py +125 -0
- kiri_migration/scaffold/compile_targets/dlt_with_append_flows.py +163 -0
- kiri_migration/scaffold/compile_targets/dlt_with_apply_changes.py +192 -0
- kiri_migration/scaffold/compile_targets/dlt_with_expectations.py +276 -0
- kiri_migration/scaffold/compile_targets/layer_dispatch.py +258 -0
- kiri_migration/scaffold/compile_targets/sql_full_snapshot.py +385 -0
- kiri_migration/scaffold/compile_targets/streaming_table.py +37 -0
- kiri_migration/scaffold/mart.py +234 -0
- kiri_migration/scaffold/plans/__init__.py +49 -0
- kiri_migration/scaffold/plans/_registry.py +308 -0
- kiri_migration/scaffold/plans/minimal.py +550 -0
- kiri_migration/scaffold/plans/plan_b_layer.py +640 -0
- kiri_migration/scaffold/silver_from_marts.py +256 -0
- kiri_migration/scaffold/silver_pattern.py +451 -0
- kiri_migration/sp_translator.py +398 -0
- kiri_odcs/LICENSE +204 -0
- kiri_odcs/__init__.py +234 -0
- kiri_odcs/ai_trust_tier.py +475 -0
- kiri_odcs/canonical.py +263 -0
- kiri_odcs/diff.py +63 -0
- kiri_odcs/errors.py +51 -0
- kiri_odcs/invariant_sql.py +136 -0
- kiri_odcs/kiri_bridge.py +71 -0
- kiri_odcs/kiri_ext.py +3166 -0
- kiri_odcs/kiri_fk.py +447 -0
- kiri_odcs/kiri_pit.py +121 -0
- kiri_odcs/kiri_ref.py +57 -0
- kiri_odcs/openlineage_adapter.py +105 -0
- kiri_odcs/parser.py +1213 -0
- kiri_odcs/pii_guardrails.py +159 -0
- kiri_odcs/py.typed +0 -0
- kiri_odcs/schema_export.py +223 -0
- kiri_openlineage/__init__.py +58 -0
- kiri_openlineage/builder.py +146 -0
- kiri_openlineage/cost.py +190 -0
- kiri_openlineage/emitter.py +165 -0
- kiri_openlineage/py.typed +0 -0
- kiri_openlineage/schema.py +121 -0
- kiri_packs/__init__.py +67 -0
- kiri_packs/installer.py +500 -0
- kiri_packs/manifest.py +134 -0
- kiri_packs/publisher.py +605 -0
- kiri_packs/py.typed +0 -0
- kiri_packs/validator.py +295 -0
- kiri_quality/__init__.py +108 -0
- kiri_quality/ai_audit_correlator.py +293 -0
- kiri_quality/ai_audit_correlator_apply.py +526 -0
- kiri_quality/conformance_evidence.py +272 -0
- kiri_quality/dbt_test_collector.py +214 -0
- kiri_quality/dq_codegen.py +300 -0
- kiri_quality/dv_checks.py +393 -0
- kiri_quality/dv_column_lineage.py +279 -0
- kiri_quality/dv_column_lineage_apply.py +856 -0
- kiri_quality/dv_conformance.py +841 -0
- kiri_quality/dv_fixture_runner.py +478 -0
- kiri_quality/dv_invariants.py +986 -0
- kiri_quality/dv_kiri.py +97 -0
- kiri_quality/dv_markdown.py +169 -0
- kiri_quality/dv_observed_uc_reader.py +187 -0
- kiri_quality/dv_runner.py +389 -0
- kiri_quality/evidence_status.py +337 -0
- kiri_quality/fk_checks.py +800 -0
- kiri_quality/fk_ddl.py +402 -0
- kiri_quality/fk_pit_contract.py +168 -0
- kiri_quality/fk_runner.py +950 -0
- kiri_quality/kernel.py +376 -0
- kiri_quality/pii.py +60 -0
- kiri_quality/pre_classify/__init__.py +55 -0
- kiri_quality/pre_classify/engine.py +346 -0
- kiri_quality/pre_classify/locale_packs/en.yaml +117 -0
- kiri_quality/pre_classify/locale_packs/sv.yaml +73 -0
- kiri_quality/pre_classify/merge.py +297 -0
- kiri_quality/pre_classify/model.py +163 -0
- kiri_quality/py.typed +0 -0
- kiri_quality/reject_lane.py +261 -0
- kiri_quality/rule_runner.py +565 -0
- kiri_quality/schema_evolution_check.py +88 -0
- kiri_quality/silver_quality.py +494 -0
- kiri_quality/silver_quality_markdown.py +187 -0
- kiri_quality/uc_security_compile.py +951 -0
- kiri_quality/uc_security_compile_apply.py +1186 -0
- kiri_rbac/__init__.py +87 -0
- kiri_rbac/access.py +371 -0
- kiri_rbac/loader.py +104 -0
- kiri_rbac/pr_gate.py +213 -0
- kiri_rbac/providers/__init__.py +29 -0
- kiri_rbac/providers/_jwt.py +121 -0
- kiri_rbac/providers/azure_ad.py +93 -0
- kiri_rbac/providers/base.py +107 -0
- kiri_rbac/providers/factory.py +48 -0
- kiri_rbac/providers/github.py +134 -0
- kiri_rbac/providers/oidc_generic.py +99 -0
- kiri_rbac/py.typed +0 -0
- kiri_rbac/resolver.py +118 -0
- kiri_rbac/schema.py +219 -0
- kiri_schema_registry/__init__.py +22 -0
- kiri_schema_registry/avro.py +127 -0
- kiri_schema_registry/base.py +71 -0
- kiri_schema_registry/confluent.py +184 -0
- kiri_schema_registry/glue.py +178 -0
- kiri_schema_registry/py.typed +0 -0
- kiri_slack_bot/__init__.py +24 -0
- kiri_slack_bot/handler.py +248 -0
- kiri_slack_bot/py.typed +0 -0
- kiri_slack_bot/queries.py +90 -0
- kiri_slack_bot/signature.py +94 -0
- kiri_vault/__init__.py +59 -0
- kiri_vault/audit.py +64 -0
- kiri_vault/aws_vault.py +123 -0
- kiri_vault/azure_vault.py +236 -0
- kiri_vault/boundaries.py +219 -0
- kiri_vault/client.py +54 -0
- kiri_vault/databricks_vault.py +292 -0
- kiri_vault/env_vault.py +77 -0
- kiri_vault/py.typed +0 -0
- kiri_vault/ref.py +84 -0
- kirimana_automation/__init__.py +0 -0
- kirimana_automation/business_calendar.py +179 -0
- kirimana_automation/dag_snapshot.py +252 -0
- kirimana_automation/db/__init__.py +5 -0
- kirimana_automation/db/migrate.py +571 -0
- kirimana_automation/db/migrations/0001_release_state.down.duckdb.sql +29 -0
- kirimana_automation/db/migrations/0001_release_state.down.postgres.sql +37 -0
- kirimana_automation/db/migrations/0001_release_state.duckdb.sql +170 -0
- kirimana_automation/db/migrations/0001_release_state.postgres.sql +176 -0
- kirimana_automation/db/migrations/0002_rate_limit_buckets.down.duckdb.sql +9 -0
- kirimana_automation/db/migrations/0002_rate_limit_buckets.down.postgres.sql +9 -0
- kirimana_automation/db/migrations/0002_rate_limit_buckets.duckdb.sql +14 -0
- kirimana_automation/db/migrations/0002_rate_limit_buckets.postgres.sql +18 -0
- kirimana_automation/db/migrations/0003_flow_runs.down.duckdb.sql +18 -0
- kirimana_automation/db/migrations/0003_flow_runs.down.postgres.sql +19 -0
- kirimana_automation/db/migrations/0003_flow_runs.duckdb.sql +85 -0
- kirimana_automation/db/migrations/0003_flow_runs.postgres.sql +102 -0
- kirimana_automation/db/migrations/0004_flow_dag_snapshots.down.duckdb.sql +11 -0
- kirimana_automation/db/migrations/0004_flow_dag_snapshots.down.postgres.sql +11 -0
- kirimana_automation/db/migrations/0004_flow_dag_snapshots.duckdb.sql +15 -0
- kirimana_automation/db/migrations/0004_flow_dag_snapshots.postgres.sql +31 -0
- kirimana_automation/db/migrations/0005_calendar_skip.down.duckdb.sql +109 -0
- kirimana_automation/db/migrations/0005_calendar_skip.down.postgres.sql +42 -0
- kirimana_automation/db/migrations/0005_calendar_skip.duckdb.sql +113 -0
- kirimana_automation/db/migrations/0005_calendar_skip.postgres.sql +42 -0
- kirimana_automation/db/migrations/0006_flow_apply_log.down.duckdb.sql +9 -0
- kirimana_automation/db/migrations/0006_flow_apply_log.down.postgres.sql +11 -0
- kirimana_automation/db/migrations/0006_flow_apply_log.duckdb.sql +45 -0
- kirimana_automation/db/migrations/0006_flow_apply_log.postgres.sql +49 -0
- kirimana_automation/db/migrations/0007_flow_task_error_redacted.down.duckdb.sql +14 -0
- kirimana_automation/db/migrations/0007_flow_task_error_redacted.down.postgres.sql +2 -0
- kirimana_automation/db/migrations/0007_flow_task_error_redacted.duckdb.sql +7 -0
- kirimana_automation/db/migrations/0007_flow_task_error_redacted.postgres.sql +11 -0
- kirimana_automation/db/migrations/README.md +84 -0
- kirimana_automation/flow_loader.py +220 -0
- kirimana_automation/freeze.py +133 -0
- kirimana_automation/kiri_recommender.py +313 -0
- kirimana_automation/main.py +195 -0
- kirimana_automation/py.typed +0 -0
- kirimana_automation/run_history.py +593 -0
- kirimana_automation/run_history_analysis.py +607 -0
- kirimana_automation/schedule.py +343 -0
- kirimana_automation/schedule_validator.py +572 -0
- kirimana_automation/workflow_compiler.py +1288 -0
kiri_odcs/kiri_fk.py
ADDED
|
@@ -0,0 +1,447 @@
|
|
|
1
|
+
"""Typed foreign-key / relationship metadata — ADR 0125 §1.
|
|
2
|
+
|
|
3
|
+
A foreign key is a relationship *between two contracts*, so the
|
|
4
|
+
``kiri.relationships`` namespace lives at contract scope (like
|
|
5
|
+
``kiri.dv``), not on a single property. The model is **technique-
|
|
6
|
+
neutral**: it expresses the declared referential model a curated silver
|
|
7
|
+
layer must satisfy regardless of whether the silver shape is ``flat``,
|
|
8
|
+
``kimball_dimensional`` or ``data_vault``. The FK gate (slice 5.1b+)
|
|
9
|
+
probes those declarations on the reconcile/verify path; this module is
|
|
10
|
+
the contract-model half only.
|
|
11
|
+
|
|
12
|
+
This namespace is **distinct from ADR 0045 lineage**:
|
|
13
|
+
``kiri.lineage.from_columns`` records where a column's *value* came from
|
|
14
|
+
(derivation); ``kiri.relationships`` records what a column must
|
|
15
|
+
*reference at rest* (constraint). Conflating them would corrupt the
|
|
16
|
+
lineage graph, whose edges are data-flow.
|
|
17
|
+
|
|
18
|
+
The ``mode="after"`` validators enforce the structural invariants so a
|
|
19
|
+
malformed relationship fails at parse time, not at probe time. Each
|
|
20
|
+
raises a clear :class:`ValueError` naming the offending FK by ``name``.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import re
|
|
26
|
+
from enum import StrEnum
|
|
27
|
+
from typing import Annotated, Any
|
|
28
|
+
|
|
29
|
+
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
|
30
|
+
|
|
31
|
+
__all__ = [
|
|
32
|
+
"FkCardinality",
|
|
33
|
+
"FkKeying",
|
|
34
|
+
"FkPromotion",
|
|
35
|
+
"FkRef",
|
|
36
|
+
"FkTier",
|
|
37
|
+
"ForeignKey",
|
|
38
|
+
"KiriRelationshipsMeta",
|
|
39
|
+
]
|
|
40
|
+
|
|
41
|
+
_FROZEN = ConfigDict(frozen=True, extra="forbid")
|
|
42
|
+
|
|
43
|
+
#: A foreign-key ``name`` becomes both an emitted constraint name and a
|
|
44
|
+
#: stable finding id, so it must be identifier-shaped (no whitespace, no
|
|
45
|
+
#: quoting surface). Mirrors :meth:`Invariant._name_is_safe` in
|
|
46
|
+
#: ``kiri_ext`` — same defence-in-depth posture against injection into
|
|
47
|
+
#: SQL comments / report cells / audit rows.
|
|
48
|
+
_FK_NAME_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
|
|
49
|
+
|
|
50
|
+
#: Key columns are kept verbatim (the dialect quoter preserves source
|
|
51
|
+
#: casing / diacritics), so an empty identifier must never slip through to
|
|
52
|
+
#: the DDL / probe surface. Every key-column element is constrained
|
|
53
|
+
#: non-empty rather than only the list being non-empty.
|
|
54
|
+
_NonEmptyStr = Annotated[str, Field(min_length=1)]
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _contract_identity(contract: str) -> str:
|
|
58
|
+
"""The bare contract token of an ``FkRef.contract``.
|
|
59
|
+
|
|
60
|
+
A lineage URN is ``urn:kirimana:<layer>:<contract>::<column>``; a
|
|
61
|
+
relationship is between *contracts*, so neither the layer/URN scaffold
|
|
62
|
+
nor the trailing ``::<column>`` is part of a side's contract identity.
|
|
63
|
+
This reduces both forms to the same comparable token, so a side written
|
|
64
|
+
as ``urn:kirimana:silver:dim_customer::customer_sk`` and one written as
|
|
65
|
+
the bare ``dim_customer`` are recognised as the same contract.
|
|
66
|
+
|
|
67
|
+
The token is extracted **only** from a value that is a well-formed
|
|
68
|
+
lineage URN (matches :data:`_LINEAGE_URN_RE`). ``FkRef.contract`` is
|
|
69
|
+
already grammar-validated at field time, but this function is also
|
|
70
|
+
called on raw ``natural_key_columns`` dict keys, which are typed only
|
|
71
|
+
``str``. Gating on the full grammar means a malformed ``urn:``-ish key
|
|
72
|
+
(wrong layer, missing ``::column``, too few/many segments) is **not**
|
|
73
|
+
token-extracted: it degrades to its verbatim value, so it cannot
|
|
74
|
+
falsely match a bare-name side and the surrogate validator rejects the
|
|
75
|
+
unmatched side with a clean ``ValidationError``. A bare name has no
|
|
76
|
+
``urn:`` prefix and is returned with only a trailing ``::column``
|
|
77
|
+
stripped.
|
|
78
|
+
"""
|
|
79
|
+
# Lazy import: ``_LINEAGE_URN_RE`` lives in ``kiri_ext``, which imports
|
|
80
|
+
# this module — a module-level import would be circular. This runs only
|
|
81
|
+
# during validation, after both modules are loaded.
|
|
82
|
+
from kiri_odcs.kiri_ext import _LINEAGE_URN_RE
|
|
83
|
+
|
|
84
|
+
if contract.startswith("urn:") and _LINEAGE_URN_RE.match(contract):
|
|
85
|
+
# urn:kirimana:<layer>:<contract>::<column> → <contract>
|
|
86
|
+
return contract.split("::", 1)[0].split(":", 3)[3]
|
|
87
|
+
return contract.split("::", 1)[0]
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
class FkTier(StrEnum):
|
|
91
|
+
"""Graded enforcement posture for one declared foreign key (ADR 0125 §1).
|
|
92
|
+
|
|
93
|
+
- ``enforced`` — platform-enforced where the engine supports it.
|
|
94
|
+
- ``rely`` — ``NOT ENFORCED RELY`` (T1); orphans **FAIL** the gate.
|
|
95
|
+
- ``norely`` — ``NOT ENFORCED NORELY``; orphans are INFO (e.g. a
|
|
96
|
+
nullable self-reference).
|
|
97
|
+
- ``planned`` — logically valid but has orphans today because the
|
|
98
|
+
parent is only partially loaded; the gate WARNs and records a
|
|
99
|
+
promotion trigger. Requires ``promotion``.
|
|
100
|
+
- ``lookup`` — no physical parent: documented as a column comment
|
|
101
|
+
only, never constraint-emitting and never probed.
|
|
102
|
+
- ``pit`` — a temporal multi-column join *contract* (business
|
|
103
|
+
key + effective timestamp + current flag); never a Delta FK.
|
|
104
|
+
"""
|
|
105
|
+
|
|
106
|
+
ENFORCED = "enforced"
|
|
107
|
+
RELY = "rely"
|
|
108
|
+
NORELY = "norely"
|
|
109
|
+
PLANNED = "planned"
|
|
110
|
+
LOOKUP = "lookup"
|
|
111
|
+
PIT = "pit"
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
class FkKeying(StrEnum):
|
|
115
|
+
"""Which key the FK is validated on (ADR 0125 §1 / §7b).
|
|
116
|
+
|
|
117
|
+
- ``natural`` — validate on the verbatim business-key columns.
|
|
118
|
+
- ``surrogate`` — the FK is declared on an ``_SK`` column, but a
|
|
119
|
+
recomputed surrogate only holds if both sides recompute it
|
|
120
|
+
identically; the gate therefore validates on
|
|
121
|
+
``natural_key_columns`` instead. Hence ``surrogate`` requires
|
|
122
|
+
``natural_key_columns`` for both sides.
|
|
123
|
+
"""
|
|
124
|
+
|
|
125
|
+
NATURAL = "natural"
|
|
126
|
+
SURROGATE = "surrogate"
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
class FkCardinality(StrEnum):
|
|
130
|
+
"""Declared cardinality of the relationship (ADR 0125 §1)."""
|
|
131
|
+
|
|
132
|
+
ONE_ONE = "1:1"
|
|
133
|
+
ONE_N = "1:N"
|
|
134
|
+
ZERO_ONE_N = "0..1:N"
|
|
135
|
+
N_M = "N:M"
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
class FkRef(BaseModel):
|
|
139
|
+
"""One side (parent or child) of a relationship (ADR 0125 §1).
|
|
140
|
+
|
|
141
|
+
Fields:
|
|
142
|
+
|
|
143
|
+
- ``contract`` — the bare contract name (e.g. ``dim_customer``)
|
|
144
|
+
or the silver URN form. When a URN is given, it is validated
|
|
145
|
+
against the lineage URN grammar (:data:`_LINEAGE_URN_RE`).
|
|
146
|
+
- ``columns`` — the composite key columns on this side.
|
|
147
|
+
Order-significant (paired position-wise with the other side) and
|
|
148
|
+
kept **verbatim** so the dialect quoter preserves source casing /
|
|
149
|
+
diacritics.
|
|
150
|
+
- ``current_filter`` — optional SCD2 ``_current`` projection that the
|
|
151
|
+
runner turns into a reconcile ``CurrentFilter`` before probing.
|
|
152
|
+
"""
|
|
153
|
+
|
|
154
|
+
model_config = _FROZEN
|
|
155
|
+
|
|
156
|
+
contract: str = Field(min_length=1)
|
|
157
|
+
columns: list[_NonEmptyStr] = Field(min_length=1)
|
|
158
|
+
current_filter: dict[str, Any] | None = None
|
|
159
|
+
|
|
160
|
+
@field_validator("contract")
|
|
161
|
+
@classmethod
|
|
162
|
+
def _contract_urn_or_name(cls, value: str) -> str:
|
|
163
|
+
"""Accept a bare contract name or a lineage-grammar URN.
|
|
164
|
+
|
|
165
|
+
Per ADR 0125 §1 ``FkRef.contract`` reuses the lineage URN
|
|
166
|
+
grammar *when given in URN form*. A value that begins with
|
|
167
|
+
``urn:`` is therefore validated against :data:`_LINEAGE_URN_RE`;
|
|
168
|
+
anything else is treated as a bare contract name and accepted
|
|
169
|
+
as-is (resolution against the project happens in the runner,
|
|
170
|
+
not at parse time).
|
|
171
|
+
"""
|
|
172
|
+
# Imported lazily: ``_LINEAGE_URN_RE`` lives in ``kiri_ext``, which
|
|
173
|
+
# in turn imports ``KiriRelationshipsMeta`` from this module — a
|
|
174
|
+
# module-level import here would be circular. The validator only
|
|
175
|
+
# runs after both modules are fully loaded.
|
|
176
|
+
from kiri_odcs.kiri_ext import _LINEAGE_URN_RE
|
|
177
|
+
|
|
178
|
+
if value.startswith("urn:") and not _LINEAGE_URN_RE.match(value):
|
|
179
|
+
raise ValueError(
|
|
180
|
+
f"FkRef.contract {value!r} looks like a URN but does not "
|
|
181
|
+
"match the lineage URN grammar "
|
|
182
|
+
"``urn:kirimana:<source|bronze|silver|gold>:"
|
|
183
|
+
"<contract>::<column>``. Give a bare contract name or a "
|
|
184
|
+
"well-formed URN (ADR 0125 §1)."
|
|
185
|
+
)
|
|
186
|
+
return value
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
class FkPromotion(BaseModel):
|
|
190
|
+
"""Promotion record for a ``tier=planned`` foreign key (ADR 0125 §1).
|
|
191
|
+
|
|
192
|
+
A ``planned`` FK is logically valid but has orphans today because the
|
|
193
|
+
parent is only partially loaded. The gate WARNs (rather than
|
|
194
|
+
silently passing or hard-failing) and surfaces these two fields so
|
|
195
|
+
Kiri can reason about *why* the FK is not yet strong and *when* it
|
|
196
|
+
should be re-probed.
|
|
197
|
+
"""
|
|
198
|
+
|
|
199
|
+
model_config = _FROZEN
|
|
200
|
+
|
|
201
|
+
root_cause: str = Field(min_length=1)
|
|
202
|
+
promotion_trigger: str = Field(min_length=1)
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
class ForeignKey(BaseModel):
|
|
206
|
+
"""One declared foreign key between two contracts (ADR 0125 §1).
|
|
207
|
+
|
|
208
|
+
The structural invariants below fail at parse time so a malformed
|
|
209
|
+
relationship never reaches probe time. See the per-validator
|
|
210
|
+
docstrings for the exact rule each enforces.
|
|
211
|
+
"""
|
|
212
|
+
|
|
213
|
+
model_config = _FROZEN
|
|
214
|
+
|
|
215
|
+
name: str
|
|
216
|
+
parent: FkRef
|
|
217
|
+
child: FkRef
|
|
218
|
+
cardinality: FkCardinality
|
|
219
|
+
tier: FkTier
|
|
220
|
+
keying: FkKeying = FkKeying.NATURAL
|
|
221
|
+
bridge_table: bool = False
|
|
222
|
+
confidence: float | None = None
|
|
223
|
+
case_insensitive_match: bool = False
|
|
224
|
+
natural_key_columns: dict[str, list[_NonEmptyStr]] | None = None
|
|
225
|
+
alternate_keys: list[list[_NonEmptyStr]] | None = None
|
|
226
|
+
no_cycle: bool = False
|
|
227
|
+
promotion: FkPromotion | None = None
|
|
228
|
+
|
|
229
|
+
@field_validator("name")
|
|
230
|
+
@classmethod
|
|
231
|
+
def _name_is_identifier_shaped(cls, value: str) -> str:
|
|
232
|
+
"""``name`` must be identifier-shaped (stable constraint + finding id).
|
|
233
|
+
|
|
234
|
+
The name is emitted verbatim as a SQL constraint name and used
|
|
235
|
+
as a finding id; restricting it to ``[A-Za-z_][A-Za-z0-9_]*``
|
|
236
|
+
keeps it quoting-safe and stable across the ``--gate`` surface.
|
|
237
|
+
"""
|
|
238
|
+
if not _FK_NAME_RE.match(value):
|
|
239
|
+
raise ValueError(
|
|
240
|
+
f"ForeignKey.name must match {_FK_NAME_RE.pattern} "
|
|
241
|
+
f"(identifier-shaped, stable constraint + finding id); "
|
|
242
|
+
f"got {value!r}."
|
|
243
|
+
)
|
|
244
|
+
return value
|
|
245
|
+
|
|
246
|
+
@model_validator(mode="after")
|
|
247
|
+
def _planned_requires_promotion(self) -> ForeignKey:
|
|
248
|
+
"""``tier=planned`` requires a ``promotion`` record (ADR 0125 §1, §3).
|
|
249
|
+
|
|
250
|
+
A ``planned`` FK downgrades a FAIL to a WARN; that downgrade is
|
|
251
|
+
only honest when accompanied by a declared root cause and
|
|
252
|
+
promotion trigger, so the gate is never a silent green.
|
|
253
|
+
"""
|
|
254
|
+
if self.tier is FkTier.PLANNED and self.promotion is None:
|
|
255
|
+
raise ValueError(
|
|
256
|
+
f"ForeignKey {self.name!r}: tier=planned requires a "
|
|
257
|
+
"``promotion`` ({root_cause, promotion_trigger}) — the "
|
|
258
|
+
"explicit, recorded reason orphans are tolerated as WARN. "
|
|
259
|
+
"See ADR 0125 §1."
|
|
260
|
+
)
|
|
261
|
+
return self
|
|
262
|
+
|
|
263
|
+
@model_validator(mode="after")
|
|
264
|
+
def _lookup_is_comment_only(self) -> ForeignKey:
|
|
265
|
+
"""``tier=lookup`` is column-comment-only — never constraint-emitting.
|
|
266
|
+
|
|
267
|
+
A ``lookup`` reference has no resolvable physical parent (ADR
|
|
268
|
+
0125 §1 / §3: "no physical parent — column comment emitted"). It
|
|
269
|
+
is never probed and never emits a constraint. We still require
|
|
270
|
+
well-formed ``parent`` / ``child`` refs (enforced by the field
|
|
271
|
+
types), but a lookup FK must not carry the markers that only
|
|
272
|
+
make sense for an enforceable, probed relationship — declaring a
|
|
273
|
+
``promotion`` (a probe-tier concept) or surrogate keying on a
|
|
274
|
+
lookup is operator confusion, so reject it fail-closed.
|
|
275
|
+
"""
|
|
276
|
+
if self.tier is FkTier.LOOKUP:
|
|
277
|
+
if self.promotion is not None:
|
|
278
|
+
raise ValueError(
|
|
279
|
+
f"ForeignKey {self.name!r}: tier=lookup carries no "
|
|
280
|
+
"enforceable parent and is never probed, so a "
|
|
281
|
+
"``promotion`` record is meaningless — remove it or "
|
|
282
|
+
"choose tier=planned. See ADR 0125 §1."
|
|
283
|
+
)
|
|
284
|
+
if self.keying is FkKeying.SURROGATE:
|
|
285
|
+
raise ValueError(
|
|
286
|
+
f"ForeignKey {self.name!r}: tier=lookup is "
|
|
287
|
+
"column-comment-only and never validated, so "
|
|
288
|
+
"keying=surrogate (a probe-time key-parity concern) "
|
|
289
|
+
"does not apply. See ADR 0125 §1."
|
|
290
|
+
)
|
|
291
|
+
return self
|
|
292
|
+
|
|
293
|
+
@model_validator(mode="after")
|
|
294
|
+
def _pit_requires_temporal(self) -> ForeignKey:
|
|
295
|
+
"""``tier=pit`` requires temporal columns OR a ``current_filter``.
|
|
296
|
+
|
|
297
|
+
A PIT relationship is a temporal join contract (ADR 0125 §1 /
|
|
298
|
+
§4): it must carry a temporal predicate so the projected
|
|
299
|
+
relationship can be evaluated as-of. We accept either an
|
|
300
|
+
explicit ``current_filter`` on at least one side (the
|
|
301
|
+
``_current`` projection path) or composite (multi-column) keys
|
|
302
|
+
that include the effective-timestamp / current-flag columns —
|
|
303
|
+
i.e. more than the single business-key column a non-temporal FK
|
|
304
|
+
would carry.
|
|
305
|
+
"""
|
|
306
|
+
if self.tier is FkTier.PIT:
|
|
307
|
+
has_current_filter = (
|
|
308
|
+
self.parent.current_filter is not None or self.child.current_filter is not None
|
|
309
|
+
)
|
|
310
|
+
has_temporal_columns = len(self.parent.columns) > 1 and len(self.child.columns) > 1
|
|
311
|
+
if not (has_current_filter or has_temporal_columns):
|
|
312
|
+
raise ValueError(
|
|
313
|
+
f"ForeignKey {self.name!r}: tier=pit is a temporal "
|
|
314
|
+
"join contract and requires either a ``current_filter`` "
|
|
315
|
+
"on at least one side or composite temporal columns "
|
|
316
|
+
"(business key + effective timestamp / current flag). "
|
|
317
|
+
"See ADR 0125 §1, §4."
|
|
318
|
+
)
|
|
319
|
+
return self
|
|
320
|
+
|
|
321
|
+
@model_validator(mode="after")
|
|
322
|
+
def _surrogate_requires_natural_keys(self) -> ForeignKey:
|
|
323
|
+
"""``keying=surrogate`` requires ``natural_key_columns`` for both sides.
|
|
324
|
+
|
|
325
|
+
A recomputed ``_SK`` only holds if both sides recompute it
|
|
326
|
+
identically (ADR 0125 §1 / §7b), so the gate validates on the
|
|
327
|
+
natural business key instead. That requires the natural-key
|
|
328
|
+
columns for *both* the parent and the child side, keyed by their
|
|
329
|
+
contract identity. A side may be written as a bare name or a
|
|
330
|
+
column-qualified lineage URN, and the ``natural_key_columns`` keys
|
|
331
|
+
likewise — they match when their contract identities agree
|
|
332
|
+
(:func:`_contract_identity`), so an URN-form side resolves against
|
|
333
|
+
a name-keyed map and vice versa.
|
|
334
|
+
"""
|
|
335
|
+
if self.keying is FkKeying.SURROGATE:
|
|
336
|
+
nkc = self.natural_key_columns
|
|
337
|
+
if not nkc:
|
|
338
|
+
raise ValueError(
|
|
339
|
+
f"ForeignKey {self.name!r}: keying=surrogate requires "
|
|
340
|
+
"``natural_key_columns`` for both sides (keyed by "
|
|
341
|
+
"contract name) so the gate validates on the business "
|
|
342
|
+
"key rather than a recomputed _SK. See ADR 0125 §1, §7b."
|
|
343
|
+
)
|
|
344
|
+
by_identity: dict[str, list[str]] = {}
|
|
345
|
+
for key, key_cols in nkc.items():
|
|
346
|
+
by_identity.setdefault(_contract_identity(key), key_cols)
|
|
347
|
+
resolved: dict[str, list[str]] = {}
|
|
348
|
+
for side_name, side in (("parent", self.parent), ("child", self.child)):
|
|
349
|
+
side_cols = nkc.get(side.contract) or by_identity.get(
|
|
350
|
+
_contract_identity(side.contract)
|
|
351
|
+
)
|
|
352
|
+
if not side_cols:
|
|
353
|
+
raise ValueError(
|
|
354
|
+
f"ForeignKey {self.name!r}: keying=surrogate "
|
|
355
|
+
f"requires natural_key_columns for side "
|
|
356
|
+
f"{side.contract!r}; got keys {sorted(nkc)!r}. "
|
|
357
|
+
"See ADR 0125 §1, §7b."
|
|
358
|
+
)
|
|
359
|
+
resolved[side_name] = side_cols
|
|
360
|
+
# The surrogate gate joins the two sides' natural keys position-
|
|
361
|
+
# wise, so they must have equal arity — otherwise the renderer's
|
|
362
|
+
# paired zip would raise at probe time (a gate crash) instead of
|
|
363
|
+
# failing closed at parse time here.
|
|
364
|
+
if len(resolved["parent"]) != len(resolved["child"]):
|
|
365
|
+
raise ValueError(
|
|
366
|
+
f"ForeignKey {self.name!r}: keying=surrogate natural-key "
|
|
367
|
+
f"arity mismatch — parent has {len(resolved['parent'])} "
|
|
368
|
+
f"column(s) {resolved['parent']!r} but child has "
|
|
369
|
+
f"{len(resolved['child'])} column(s) {resolved['child']!r}. "
|
|
370
|
+
"The two natural keys join position-wise. See ADR 0125 §1, §7b."
|
|
371
|
+
)
|
|
372
|
+
return self
|
|
373
|
+
|
|
374
|
+
@model_validator(mode="after")
|
|
375
|
+
def _composite_arity_matches(self) -> ForeignKey:
|
|
376
|
+
"""Parent and child key arity must match, and so must each alternate key.
|
|
377
|
+
|
|
378
|
+
A composite FK joins parent columns to child columns position-
|
|
379
|
+
wise (ADR 0125 §1), so the two column lists must be the same
|
|
380
|
+
length. Each declared alternate-key path is an alternative
|
|
381
|
+
composite join on the same pair of tables, so it must have the
|
|
382
|
+
same arity as the primary key columns.
|
|
383
|
+
"""
|
|
384
|
+
if len(self.parent.columns) != len(self.child.columns):
|
|
385
|
+
raise ValueError(
|
|
386
|
+
f"ForeignKey {self.name!r}: composite arity mismatch — "
|
|
387
|
+
f"parent has {len(self.parent.columns)} column(s) "
|
|
388
|
+
f"{self.parent.columns!r} but child has "
|
|
389
|
+
f"{len(self.child.columns)} column(s) {self.child.columns!r}. "
|
|
390
|
+
"Parent and child key columns are paired position-wise. "
|
|
391
|
+
"See ADR 0125 §1."
|
|
392
|
+
)
|
|
393
|
+
if self.alternate_keys is not None:
|
|
394
|
+
expected = len(self.parent.columns)
|
|
395
|
+
for idx, alt in enumerate(self.alternate_keys):
|
|
396
|
+
if len(alt) != expected:
|
|
397
|
+
raise ValueError(
|
|
398
|
+
f"ForeignKey {self.name!r}: alternate_keys[{idx}] "
|
|
399
|
+
f"has arity {len(alt)} ({alt!r}) but the primary "
|
|
400
|
+
f"key columns have arity {expected}. Each "
|
|
401
|
+
"alternate-key path must match the FK's arity. "
|
|
402
|
+
"See ADR 0125 §1."
|
|
403
|
+
)
|
|
404
|
+
return self
|
|
405
|
+
|
|
406
|
+
@model_validator(mode="after")
|
|
407
|
+
def _no_cycle_only_on_self_reference(self) -> ForeignKey:
|
|
408
|
+
"""``no_cycle=true`` is meaningful only on a self-reference (ADR 0125 §1).
|
|
409
|
+
|
|
410
|
+
The cycle policy guards a self-referential hierarchy (e.g. a
|
|
411
|
+
manager chain). Setting it when the parent and child are
|
|
412
|
+
different contracts is operator confusion — there is no cycle to
|
|
413
|
+
guard — so reject it fail-closed rather than carry a no-op flag.
|
|
414
|
+
|
|
415
|
+
The two sides are compared by *contract identity*
|
|
416
|
+
(:func:`_contract_identity`), so a self-reference expressed with
|
|
417
|
+
column-qualified URNs — ``urn:…:dim_employee::employee_id`` (parent)
|
|
418
|
+
vs ``urn:…:dim_employee::manager_id`` (child) — is correctly
|
|
419
|
+
recognised as the same contract rather than wrongly rejected on
|
|
420
|
+
the differing ``::column`` segment.
|
|
421
|
+
"""
|
|
422
|
+
if self.no_cycle and _contract_identity(self.parent.contract) != _contract_identity(
|
|
423
|
+
self.child.contract
|
|
424
|
+
):
|
|
425
|
+
raise ValueError(
|
|
426
|
+
f"ForeignKey {self.name!r}: no_cycle=true is meaningful "
|
|
427
|
+
"only on a self-reference (parent.contract == "
|
|
428
|
+
f"child.contract); got parent={self.parent.contract!r} "
|
|
429
|
+
f"child={self.child.contract!r}. Remove no_cycle or point "
|
|
430
|
+
"both sides at the same contract. See ADR 0125 §1."
|
|
431
|
+
)
|
|
432
|
+
return self
|
|
433
|
+
|
|
434
|
+
|
|
435
|
+
class KiriRelationshipsMeta(BaseModel):
|
|
436
|
+
"""Aggregate of declared foreign keys for one contract — ADR 0125 §1.
|
|
437
|
+
|
|
438
|
+
Promoted from ``kiri.relationships.*`` customProperties (the
|
|
439
|
+
``foreign_keys`` list under ``kiri.relationships.foreign_keys``).
|
|
440
|
+
The default-empty instance is a no-op marker meaning "this contract
|
|
441
|
+
declares no relationships" — it must stay cheap to construct so it
|
|
442
|
+
can serve as the default factory on :class:`KiriMeta`.
|
|
443
|
+
"""
|
|
444
|
+
|
|
445
|
+
model_config = _FROZEN
|
|
446
|
+
|
|
447
|
+
foreign_keys: list[ForeignKey] = Field(default_factory=list)
|
kiri_odcs/kiri_pit.py
ADDED
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
"""Typed PIT (point-in-time) entity metadata — ADR 0056 + ADR 0062 §2.
|
|
2
|
+
|
|
3
|
+
PIT entities live in the silver ``pit`` zone (see :class:`SilverZone`) and
|
|
4
|
+
materialise query-acceleration snapshots over the raw / business vault.
|
|
5
|
+
``KiriPitMeta`` carries the per-contract shape; the silver graph planner
|
|
6
|
+
reads it to emit ANCHOR_HUB edges and to schedule PIT nodes after
|
|
7
|
+
satellites in the topological order.
|
|
8
|
+
|
|
9
|
+
The namespace is ``kiri.pit.*`` on the ODCS contract; missing keys mean
|
|
10
|
+
"not declared" — the validator only fires when the contract opts into
|
|
11
|
+
PIT semantics by setting ``anchor_hub`` and/or ``snapshot_keys``.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
from enum import StrEnum
|
|
17
|
+
|
|
18
|
+
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
|
19
|
+
|
|
20
|
+
_FROZEN = ConfigDict(frozen=True, extra="forbid")
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class PitRefreshKind(StrEnum):
|
|
24
|
+
"""How a PIT snapshot is refreshed each batch (ADR 0062 §2).
|
|
25
|
+
|
|
26
|
+
- ``snapshot`` — the default: rebuild the as-of-window from the
|
|
27
|
+
anchor hub + driving satellites on every run. Cheap on small
|
|
28
|
+
windows; honest about the window's freshness.
|
|
29
|
+
- ``incremental`` — append new as-of rows since the last batch
|
|
30
|
+
without re-reading historical windows. Requires the upstream
|
|
31
|
+
satellites to be append-only and idempotent on (parent_hk,
|
|
32
|
+
load_ts).
|
|
33
|
+
- ``full_refresh`` — drop and rebuild every row. Expensive; used
|
|
34
|
+
when the policy or the upstream satellites change shape and a
|
|
35
|
+
clean restate is cheaper than reasoning about deltas.
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
SNAPSHOT = "snapshot"
|
|
39
|
+
INCREMENTAL = "incremental"
|
|
40
|
+
FULL_REFRESH = "full_refresh"
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class SnapshotGrain(StrEnum):
|
|
44
|
+
"""Calendar grain for PIT snapshot rows (ADR 0062 §2).
|
|
45
|
+
|
|
46
|
+
``custom`` lets the operator declare a non-calendar cadence (e.g.
|
|
47
|
+
'every Monday at 06:00'); the actual cadence is then encoded in the
|
|
48
|
+
contract's ``kiri.schedule.cadence`` cron expression and the
|
|
49
|
+
PIT planner just trusts that the upstream scheduler honours it.
|
|
50
|
+
"""
|
|
51
|
+
|
|
52
|
+
DAILY = "daily"
|
|
53
|
+
HOURLY = "hourly"
|
|
54
|
+
CUSTOM = "custom"
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class KiriPitMeta(BaseModel):
|
|
58
|
+
"""PIT (point-in-time) entity metadata — ADR 0056 + ADR 0062 §2.
|
|
59
|
+
|
|
60
|
+
Promoted from ``kiri.pit.*`` customProperties on a silver contract
|
|
61
|
+
whose medallion ``silver_zone`` is ``pit``. The validator enforces
|
|
62
|
+
the minimum shape: a PIT entity must either anchor on a single hub
|
|
63
|
+
(``anchor_hub`` set) or grain on a list of snapshot keys
|
|
64
|
+
(``snapshot_keys`` non-empty); a contract that does neither is
|
|
65
|
+
structurally a no-op and is rejected at parse time.
|
|
66
|
+
|
|
67
|
+
Fields:
|
|
68
|
+
|
|
69
|
+
- ``snapshot_keys`` — columns whose distinct values define one PIT
|
|
70
|
+
row per as-of timestamp. Empty when the entity anchors on a single
|
|
71
|
+
hub instead. Order is preserved for deterministic hashing.
|
|
72
|
+
- ``anchor_hub`` — silver URN of the hub this PIT pivots around.
|
|
73
|
+
Used by the graph planner to emit an ``ANCHOR_HUB`` edge so PIT
|
|
74
|
+
nodes land after their hub + driving satellites.
|
|
75
|
+
- ``as_of_column`` — timestamp column on the resulting PIT table
|
|
76
|
+
identifying each snapshot row. Default ``as_of_date`` matches the
|
|
77
|
+
DataVault4dbt convention; rename per project taste.
|
|
78
|
+
- ``refresh`` — see :class:`PitRefreshKind`.
|
|
79
|
+
- ``snapshot_grain`` — see :class:`SnapshotGrain`. ``None`` means
|
|
80
|
+
"no calendar grain declared" — the planner falls back to the
|
|
81
|
+
contract's orchestration schedule.
|
|
82
|
+
"""
|
|
83
|
+
|
|
84
|
+
model_config = _FROZEN
|
|
85
|
+
|
|
86
|
+
snapshot_keys: list[str] = Field(default_factory=list)
|
|
87
|
+
anchor_hub: str | None = None
|
|
88
|
+
as_of_column: str = "as_of_date"
|
|
89
|
+
refresh: PitRefreshKind = PitRefreshKind.SNAPSHOT
|
|
90
|
+
snapshot_grain: SnapshotGrain | None = None
|
|
91
|
+
|
|
92
|
+
@model_validator(mode="after")
|
|
93
|
+
def _require_anchor_or_snapshot_keys(self) -> KiriPitMeta:
|
|
94
|
+
"""A PIT entity must declare an anchor hub OR at least one snapshot key.
|
|
95
|
+
|
|
96
|
+
Without either, the planner has no way to materialise rows. The
|
|
97
|
+
check is deferred until model construction so that a freshly
|
|
98
|
+
instantiated ``KiriPitMeta()`` (used as the default factory on
|
|
99
|
+
``KiriMeta``) stays a no-op marker — the validator only complains
|
|
100
|
+
when an author actually opted into PIT shape but forgot the
|
|
101
|
+
minimum fields.
|
|
102
|
+
|
|
103
|
+
Defaulting both to empty / None keeps ``KiriMeta.pit`` cheap to
|
|
104
|
+
construct for contracts that don't declare a PIT at all.
|
|
105
|
+
"""
|
|
106
|
+
# Only enforce when at least one PIT-shaped field has been
|
|
107
|
+
# explicitly set; the default-empty instance is a sentinel
|
|
108
|
+
# meaning "no PIT declared on this contract" and must remain
|
|
109
|
+
# constructable so backwards-compat parsing keeps working.
|
|
110
|
+
explicit_fields = self.model_fields_set
|
|
111
|
+
opted_in = bool(explicit_fields & {"snapshot_keys", "anchor_hub"})
|
|
112
|
+
if not opted_in:
|
|
113
|
+
return self
|
|
114
|
+
if self.anchor_hub is None and not self.snapshot_keys:
|
|
115
|
+
raise ValueError(
|
|
116
|
+
"kiri.pit requires either anchor_hub (URN of the hub the "
|
|
117
|
+
"PIT pivots around) or snapshot_keys (non-empty list of "
|
|
118
|
+
"columns defining one snapshot row). Declare at least "
|
|
119
|
+
"one — see ADR 0062 §2."
|
|
120
|
+
)
|
|
121
|
+
return self
|
kiri_odcs/kiri_ref.py
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""Typed reference entity metadata — ADR 0062 §3.
|
|
2
|
+
|
|
3
|
+
Reference entities carry small, slowly-changing lookup data (country
|
|
4
|
+
codes, product categories, currency tables) consumed by hubs / sats
|
|
5
|
+
in the silver Data Vault. They sit at the top of the graph plan
|
|
6
|
+
(``DvEdgeKind`` zero) so downstream entities can resolve their FKs.
|
|
7
|
+
|
|
8
|
+
The namespace is ``kiri.ref.*`` on the ODCS contract; missing keys
|
|
9
|
+
mean "not declared". The validator enforces that ``business_keys`` is
|
|
10
|
+
present whenever ``historised=True``, since SCD2 history requires a
|
|
11
|
+
stable identity column to attach the validity window to.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
|
17
|
+
|
|
18
|
+
_FROZEN = ConfigDict(frozen=True, extra="forbid")
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class KiriRefMeta(BaseModel):
|
|
22
|
+
"""Reference entity metadata — ADR 0062 §3.
|
|
23
|
+
|
|
24
|
+
Fields:
|
|
25
|
+
|
|
26
|
+
- ``historised`` — when ``False`` (default) the reference is
|
|
27
|
+
SCD1 (overwrite on update); when ``True`` the renderer emits an
|
|
28
|
+
SCD2 table with valid_from / valid_to columns.
|
|
29
|
+
- ``business_keys`` — columns identifying one reference entity.
|
|
30
|
+
Required when ``historised=True`` because SCD2 needs a stable
|
|
31
|
+
identity to attach the validity window to. Optional but
|
|
32
|
+
recommended for SCD1 references too — missing it forces the
|
|
33
|
+
renderer to dedup on every column.
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
model_config = _FROZEN
|
|
37
|
+
|
|
38
|
+
historised: bool = False
|
|
39
|
+
business_keys: list[str] = Field(default_factory=list)
|
|
40
|
+
|
|
41
|
+
@model_validator(mode="after")
|
|
42
|
+
def _historised_requires_business_keys(self) -> KiriRefMeta:
|
|
43
|
+
"""SCD2 references must declare business_keys.
|
|
44
|
+
|
|
45
|
+
Without them, the renderer cannot identify which physical row
|
|
46
|
+
a new version supersedes. SCD1 references can omit business
|
|
47
|
+
keys (the renderer will dedup on every column) but the lint
|
|
48
|
+
layer will warn separately — that's not a structural error.
|
|
49
|
+
"""
|
|
50
|
+
if self.historised and not self.business_keys:
|
|
51
|
+
raise ValueError(
|
|
52
|
+
"kiri.ref with historised=true requires business_keys "
|
|
53
|
+
"(non-empty list of columns identifying one reference "
|
|
54
|
+
"entity). SCD2 needs a stable identity to attach the "
|
|
55
|
+
"valid_from / valid_to window to. See ADR 0062 §3."
|
|
56
|
+
)
|
|
57
|
+
return self
|