kiri-cli 1.0.0b3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of kiri-cli might be problematic. Click here for more details.
- kiri_adapters/__init__.py +193 -0
- kiri_adapters/backends/__init__.py +0 -0
- kiri_adapters/backends/databricks.manifest.yml +55 -0
- kiri_adapters/backends/databricks.py +2980 -0
- kiri_adapters/backends/fabric_dw.manifest.yml +17 -0
- kiri_adapters/backends/fabric_dw.py +307 -0
- kiri_adapters/backends/fabric_lakehouse.manifest.yml +19 -0
- kiri_adapters/backends/fabric_lakehouse.py +365 -0
- kiri_adapters/backends/iceberg.manifest.yml +17 -0
- kiri_adapters/backends/iceberg.py +451 -0
- kiri_adapters/backends/localduckdb.manifest.yml +43 -0
- kiri_adapters/backends/localduckdb.py +1295 -0
- kiri_adapters/backends/mssql.manifest.yml +17 -0
- kiri_adapters/backends/mssql.py +366 -0
- kiri_adapters/backends/postgres.manifest.yml +17 -0
- kiri_adapters/backends/postgres.py +434 -0
- kiri_adapters/backends/trino.manifest.yml +17 -0
- kiri_adapters/backends/trino.py +542 -0
- kiri_adapters/base.py +892 -0
- kiri_adapters/doctor_checks.py +81 -0
- kiri_adapters/dv_planner.py +1205 -0
- kiri_adapters/dv_planner_bridge.py +265 -0
- kiri_adapters/dv_planner_pit.py +265 -0
- kiri_adapters/dv_planner_ref.py +403 -0
- kiri_adapters/dv_planner_sat_variants.py +668 -0
- kiri_adapters/framework.py +112 -0
- kiri_adapters/incident/__init__.py +30 -0
- kiri_adapters/incident/_http.py +121 -0
- kiri_adapters/incident/base.py +89 -0
- kiri_adapters/incident/jira.py +365 -0
- kiri_adapters/incident/servicenow.py +314 -0
- kiri_adapters/incident/zendesk.py +285 -0
- kiri_adapters/manifest.py +466 -0
- kiri_adapters/orchestration/__init__.py +30 -0
- kiri_adapters/orchestration/airflow.py +117 -0
- kiri_adapters/orchestration/artifact_emitter.py +99 -0
- kiri_adapters/orchestration/autoloader_emitter.py +141 -0
- kiri_adapters/orchestration/azure_data_factory.py +137 -0
- kiri_adapters/orchestration/base.py +626 -0
- kiri_adapters/orchestration/databricks_workflows.py +1768 -0
- kiri_adapters/orchestration/k8s_cron.py +555 -0
- kiri_adapters/orchestration/lifecycle.py +222 -0
- kiri_adapters/orchestration/pipeline_compat.py +145 -0
- kiri_adapters/orchestration/runtime_python.py +188 -0
- kiri_adapters/paths.py +38 -0
- kiri_adapters/protocols/__init__.py +64 -0
- kiri_adapters/protocols/ai_service/__init__.py +7 -0
- kiri_adapters/protocols/ai_service/conformance.py +120 -0
- kiri_adapters/protocols/base.py +220 -0
- kiri_adapters/protocols/catalog/__init__.py +7 -0
- kiri_adapters/protocols/catalog/conformance.py +138 -0
- kiri_adapters/protocols/framework/__init__.py +7 -0
- kiri_adapters/protocols/framework/conformance.py +98 -0
- kiri_adapters/protocols/incident/__init__.py +7 -0
- kiri_adapters/protocols/incident/conformance.py +64 -0
- kiri_adapters/protocols/orchestration/__init__.py +7 -0
- kiri_adapters/protocols/orchestration/conformance.py +84 -0
- kiri_adapters/protocols/platform/__init__.py +7 -0
- kiri_adapters/protocols/platform/conformance.py +120 -0
- kiri_adapters/protocols/sql_dialect/__init__.py +7 -0
- kiri_adapters/protocols/sql_dialect/conformance.py +91 -0
- kiri_adapters/protocols/vault/__init__.py +7 -0
- kiri_adapters/protocols/vault/conformance.py +65 -0
- kiri_adapters/py.typed +0 -0
- kiri_adapters/registry.py +104 -0
- kiri_adapters/semantic/__init__.py +60 -0
- kiri_adapters/semantic/apply.py +274 -0
- kiri_adapters/semantic/base.py +530 -0
- kiri_adapters/semantic/cube.py +280 -0
- kiri_adapters/semantic/dbt_semantic.py +92 -0
- kiri_adapters/semantic/metric_views.py +249 -0
- kiri_adapters/semantic/metricflow.py +345 -0
- kiri_adapters/semantic/snowflake_views.py +75 -0
- kiri_adapters/sql_dialect.py +1592 -0
- kiri_adapters/tsql_to_spark.py +653 -0
- kiri_ai_gateway/__init__.py +147 -0
- kiri_ai_gateway/audit.py +334 -0
- kiri_ai_gateway/batch.py +293 -0
- kiri_ai_gateway/config.py +165 -0
- kiri_ai_gateway/gateway.py +642 -0
- kiri_ai_gateway/placeholder_guard.py +145 -0
- kiri_ai_gateway/prices/__init__.py +77 -0
- kiri_ai_gateway/prices/prices.yaml +64 -0
- kiri_ai_gateway/prompts/__init__.py +54 -0
- kiri_ai_gateway/prompts/airbyte_source_setup.yaml +212 -0
- kiri_ai_gateway/prompts/bdv_translation.yaml +90 -0
- kiri_ai_gateway/prompts/contract_draft_mvp.yaml +117 -0
- kiri_ai_gateway/prompts/describe_table.yaml +19 -0
- kiri_ai_gateway/prompts/design_gold_from_question.yaml +130 -0
- kiri_ai_gateway/prompts/detect_pii.yaml +59 -0
- kiri_ai_gateway/prompts/discover_classify_column.yaml +69 -0
- kiri_ai_gateway/prompts/discover_describe_table.yaml +61 -0
- kiri_ai_gateway/prompts/dv_quality_summary.yaml +36 -0
- kiri_ai_gateway/prompts/incident_assist.yaml +58 -0
- kiri_ai_gateway/prompts/infer_source_schema.yaml +31 -0
- kiri_ai_gateway/prompts/ingest_backend_chooser.yaml +116 -0
- kiri_ai_gateway/prompts/interactive_source_edit.yaml +94 -0
- kiri_ai_gateway/prompts/map_needed_entity_to_source.yaml +72 -0
- kiri_ai_gateway/prompts/meet_kiri.yaml +127 -0
- kiri_ai_gateway/prompts/native_source_setup.yaml +211 -0
- kiri_ai_gateway/prompts/source_grouping_reason.yaml +29 -0
- kiri_ai_gateway/prompts/suggest_attribute_classification.yaml +42 -0
- kiri_ai_gateway/prompts/suggest_data_tests.yaml +71 -0
- kiri_ai_gateway/prompts/suggest_gold_dimension.yaml +108 -0
- kiri_ai_gateway/prompts/suggest_gold_fact.yaml +107 -0
- kiri_ai_gateway/prompts/suggest_gold_model.yaml +70 -0
- kiri_ai_gateway/prompts/suggest_semantic_intent.yaml +91 -0
- kiri_ai_gateway/prompts/suggest_silver_model.yaml +140 -0
- kiri_ai_gateway/providers/__init__.py +7 -0
- kiri_ai_gateway/providers/anthropic_provider.py +26 -0
- kiri_ai_gateway/providers/gateway_provider.py +188 -0
- kiri_ai_gateway/providers/local_provider.py +177 -0
- kiri_ai_gateway/py.typed +0 -0
- kiri_ai_gateway/service.py +283 -0
- kiri_ai_gateway/transport.py +103 -0
- kiri_ai_gateway/transports/__init__.py +12 -0
- kiri_ai_gateway/transports/anthropic_transport.py +73 -0
- kiri_ai_gateway/transports/ollama_transport.py +82 -0
- kiri_ai_gateway/transports/openai_compatible_transport.py +125 -0
- kiri_ai_gateway/workspace_policy.py +52 -0
- kiri_catalog/__init__.py +199 -0
- kiri_catalog/annotation_plan.py +183 -0
- kiri_catalog/appearances.py +155 -0
- kiri_catalog/backends/__init__.py +79 -0
- kiri_catalog/backends/base.py +167 -0
- kiri_catalog/backends/memory.py +241 -0
- kiri_catalog/backends/postgres.py +497 -0
- kiri_catalog/backends/redacting.py +195 -0
- kiri_catalog/backends/sqlite.py +480 -0
- kiri_catalog/context.py +94 -0
- kiri_catalog/federation/__init__.py +114 -0
- kiri_catalog/federation/api.py +200 -0
- kiri_catalog/federation/cache.py +149 -0
- kiri_catalog/federation/emit.py +209 -0
- kiri_catalog/federation/fs_resolver.py +295 -0
- kiri_catalog/federation/http_resolver.py +486 -0
- kiri_catalog/federation/metrics.py +123 -0
- kiri_catalog/federation/release.py +342 -0
- kiri_catalog/federation/resolver.py +332 -0
- kiri_catalog/federation/schema.py +460 -0
- kiri_catalog/importer.py +668 -0
- kiri_catalog/lineage_drift.py +280 -0
- kiri_catalog/lineage_graph.py +176 -0
- kiri_catalog/lineage_graph_explain.py +96 -0
- kiri_catalog/lineage_graph_mermaid.py +230 -0
- kiri_catalog/observed_uc_lineage.py +216 -0
- kiri_catalog/pii_detect.py +176 -0
- kiri_catalog/push.py +122 -0
- kiri_catalog/py.typed +0 -0
- kiri_catalog/redact.py +101 -0
- kiri_catalog/review_actions.py +119 -0
- kiri_catalog/schema/__init__.py +70 -0
- kiri_catalog/schema/asset.py +142 -0
- kiri_catalog/schema/column.py +247 -0
- kiri_catalog/schema/enums.py +195 -0
- kiri_catalog/schema/job.py +51 -0
- kiri_catalog/schema/lineage.py +33 -0
- kiri_catalog/schema/policy.py +69 -0
- kiri_catalog/schema/primitives.py +90 -0
- kiri_catalog/schema/review_state.py +111 -0
- kiri_catalog/schema/urn.py +45 -0
- kiri_catalog/schema_discovery.py +160 -0
- kiri_catalog/sinks/__init__.py +64 -0
- kiri_catalog/sinks/base.py +178 -0
- kiri_catalog/sinks/horizon.py +195 -0
- kiri_catalog/sinks/purview.py +216 -0
- kiri_catalog/sinks/unity.py +352 -0
- kiri_catalog/sinks/unity_executor.py +543 -0
- kiri_catalog/sources/__init__.py +42 -0
- kiri_catalog/sources/base.py +104 -0
- kiri_catalog/sources/unity.py +465 -0
- kiri_catalog/steward_writeback.py +158 -0
- kiri_catalog/tag_sql.py +99 -0
- kiri_cli/LICENSE +105 -0
- kiri_cli/__init__.py +1 -0
- kiri_cli/_apply_cli.py +63 -0
- kiri_cli/_apply_reporter.py +38 -0
- kiri_cli/_closed_menu.py +36 -0
- kiri_cli/_fixture_option.py +18 -0
- kiri_cli/_layer_detect.py +158 -0
- kiri_cli/_project_option.py +79 -0
- kiri_cli/adapter_cmd.py +192 -0
- kiri_cli/ai_batch_cmd.py +258 -0
- kiri_cli/ai_cmd.py +352 -0
- kiri_cli/approve_cmd.py +659 -0
- kiri_cli/attr_cmd.py +575 -0
- kiri_cli/audit_cmd.py +348 -0
- kiri_cli/audit_correlate_cmd.py +648 -0
- kiri_cli/audit_join.py +437 -0
- kiri_cli/audit_join_cmd.py +160 -0
- kiri_cli/auto_cmd.py +97 -0
- kiri_cli/azure_cmd.py +243 -0
- kiri_cli/catalog.py +1293 -0
- kiri_cli/cli_reference_live.py +147 -0
- kiri_cli/compile_cmd.py +493 -0
- kiri_cli/compliance_cmd.py +221 -0
- kiri_cli/contract_approval_cmd.py +94 -0
- kiri_cli/contract_cmd.py +64 -0
- kiri_cli/contract_codeowners_cmd.py +162 -0
- kiri_cli/contract_diff.py +111 -0
- kiri_cli/contract_emit.py +90 -0
- kiri_cli/contract_export.py +192 -0
- kiri_cli/contract_lint.py +1505 -0
- kiri_cli/contract_merge_cmd.py +264 -0
- kiri_cli/contract_new.py +326 -0
- kiri_cli/contract_promote.py +216 -0
- kiri_cli/contract_scaffold_cmd.py +134 -0
- kiri_cli/contract_schema_export.py +69 -0
- kiri_cli/contract_show.py +150 -0
- kiri_cli/contract_stubs.py +60 -0
- kiri_cli/contract_sync_from_live.py +327 -0
- kiri_cli/contract_validate.py +154 -0
- kiri_cli/contract_verify_live.py +242 -0
- kiri_cli/cost_cmd.py +243 -0
- kiri_cli/databricks_health.py +1254 -0
- kiri_cli/databricks_setup_cmd.py +712 -0
- kiri_cli/discover_cmd.py +600 -0
- kiri_cli/docs_cmd.py +35 -0
- kiri_cli/doctor_cmd.py +145 -0
- kiri_cli/dv_cmd.py +814 -0
- kiri_cli/dv_conformance_cmd.py +154 -0
- kiri_cli/dv_install.py +493 -0
- kiri_cli/dv_migrate_cmd.py +183 -0
- kiri_cli/dv_test_cmd.py +175 -0
- kiri_cli/federation_cmd.py +315 -0
- kiri_cli/fk_cmd.py +160 -0
- kiri_cli/flows_cmd.py +910 -0
- kiri_cli/governance_cmd.py +358 -0
- kiri_cli/incident_api_cmd.py +180 -0
- kiri_cli/incident_cmd.py +410 -0
- kiri_cli/incident_setup_cmd.py +239 -0
- kiri_cli/ingest.py +427 -0
- kiri_cli/ingest_discover.py +480 -0
- kiri_cli/ingest_new_source.py +386 -0
- kiri_cli/intent_cmd.py +122 -0
- kiri_cli/invariants_cmd.py +118 -0
- kiri_cli/kiri_banner.py +161 -0
- kiri_cli/kpi_cmd.py +222 -0
- kiri_cli/layer_cmd.py +464 -0
- kiri_cli/lineage.py +601 -0
- kiri_cli/main.py +2472 -0
- kiri_cli/maturity_cmd.py +222 -0
- kiri_cli/migrate_cmd.py +1347 -0
- kiri_cli/onboard_cmd.py +223 -0
- kiri_cli/pack_cmd.py +496 -0
- kiri_cli/pipeline_cmd.py +127 -0
- kiri_cli/pre_classify_cmd.py +274 -0
- kiri_cli/project_cmd.py +592 -0
- kiri_cli/py.typed +0 -0
- kiri_cli/quality_cmd.py +295 -0
- kiri_cli/rbac.py +139 -0
- kiri_cli/rbac_cmd.py +291 -0
- kiri_cli/release.py +850 -0
- kiri_cli/release_semver.py +265 -0
- kiri_cli/repo_cmd.py +155 -0
- kiri_cli/retention_cmd.py +333 -0
- kiri_cli/sample_data/__init__.py +12 -0
- kiri_cli/sample_data/customers.contract.yml +47 -0
- kiri_cli/sample_data/customers.csv +25 -0
- kiri_cli/scaffold_cmd.py +721 -0
- kiri_cli/schema_cmd.py +185 -0
- kiri_cli/semantic_cmd.py +605 -0
- kiri_cli/silver_cmd.py +348 -0
- kiri_cli/skills_cmd.py +243 -0
- kiri_cli/skills_data/README.md +51 -0
- kiri_cli/skills_data/add-source/SKILL.md +116 -0
- kiri_cli/skills_data/azure-deploy/SKILL.md +261 -0
- kiri_cli/skills_data/azure-status/SKILL.md +111 -0
- kiri_cli/skills_data/compliance-check/SKILL.md +99 -0
- kiri_cli/skills_data/databricks-build-pipeline/SKILL.md +112 -0
- kiri_cli/skills_data/databricks-project-setup/SKILL.md +157 -0
- kiri_cli/skills_data/debug-apply/SKILL.md +186 -0
- kiri_cli/skills_data/draft-gold-model/SKILL.md +203 -0
- kiri_cli/skills_data/explain-kpi/SKILL.md +158 -0
- kiri_cli/skills_data/incident-coach/SKILL.md +135 -0
- kiri_cli/skills_data/incident-setup/SKILL.md +112 -0
- kiri_cli/skills_data/maturity-coach/SKILL.md +120 -0
- kiri_cli/skills_data/promote-contract/SKILL.md +91 -0
- kiri_cli/skills_data/review-attributes/SKILL.md +94 -0
- kiri_cli/sla_cmd.py +769 -0
- kiri_cli/source_cmd.py +909 -0
- kiri_cli/templates/__init__.py +6 -0
- kiri_cli/templates/dv4dbt_packages.yml.j2 +11 -0
- kiri_cli/upgrade_cmd.py +896 -0
- kiri_cli/use_cmd.py +120 -0
- kiri_cli/vault_cmd.py +241 -0
- kiri_cli/verify_generated.py +241 -0
- kiri_cli-1.0.0b3.dist-info/METADATA +212 -0
- kiri_cli-1.0.0b3.dist-info/RECORD +743 -0
- kiri_cli-1.0.0b3.dist-info/WHEEL +4 -0
- kiri_cli-1.0.0b3.dist-info/entry_points.txt +4 -0
- kiri_compliance/__init__.py +33 -0
- kiri_compliance/audit_coverage.py +354 -0
- kiri_compliance/controls.py +763 -0
- kiri_compliance/evidence.py +155 -0
- kiri_compliance/py.typed +0 -0
- kiri_compliance/quality_engine.py +246 -0
- kiri_compliance/renderer.py +195 -0
- kiri_compliance/report.py +44 -0
- kiri_compliance/schema.py +129 -0
- kiri_core/__init__.py +185 -0
- kiri_core/adapter_surfaces.py +149 -0
- kiri_core/ai_query_gate/__init__.py +92 -0
- kiri_core/ai_query_gate/audit.py +153 -0
- kiri_core/ai_query_gate/evaluator.py +484 -0
- kiri_core/ai_query_gate/models.py +162 -0
- kiri_core/ai_query_gate/parser.py +436 -0
- kiri_core/ai_query_gate/resolver.py +274 -0
- kiri_core/ai_query_gate/store.py +234 -0
- kiri_core/ai_query_gate/store_postgres.py +409 -0
- kiri_core/api_keys.py +243 -0
- kiri_core/apply/__init__.py +27 -0
- kiri_core/apply/conformance.py +99 -0
- kiri_core/apply/incident_router.py +176 -0
- kiri_core/apply/models.py +140 -0
- kiri_core/apply/reporter.py +46 -0
- kiri_core/apply/runner.py +3083 -0
- kiri_core/apply_log_store.py +155 -0
- kiri_core/approval_flow/__init__.py +133 -0
- kiri_core/approval_flow/integration.py +156 -0
- kiri_core/approval_flow/models.py +443 -0
- kiri_core/approval_flow/notifier.py +263 -0
- kiri_core/approval_flow/orchestrator.py +660 -0
- kiri_core/approval_flow/reconcile_signoff.py +53 -0
- kiri_core/approval_flow/store.py +220 -0
- kiri_core/approval_flow/store_jsonl.py +217 -0
- kiri_core/approval_flow/store_postgres.py +580 -0
- kiri_core/approval_policy.py +116 -0
- kiri_core/asgi_trace.py +160 -0
- kiri_core/compliance.py +802 -0
- kiri_core/cutover_approval.py +187 -0
- kiri_core/discovery/__init__.py +109 -0
- kiri_core/discovery/ai/__init__.py +37 -0
- kiri_core/discovery/ai/_payload.py +115 -0
- kiri_core/discovery/ai/budget.py +83 -0
- kiri_core/discovery/ai/classify.py +223 -0
- kiri_core/discovery/ai/describe.py +200 -0
- kiri_core/discovery/ai/errors.py +49 -0
- kiri_core/discovery/discover.py +648 -0
- kiri_core/discovery/heuristics/__init__.py +42 -0
- kiri_core/discovery/heuristics/classification.py +174 -0
- kiri_core/discovery/heuristics/medallion.py +157 -0
- kiri_core/discovery/heuristics/pii.py +92 -0
- kiri_core/discovery/introspector.py +636 -0
- kiri_core/discovery/introspector_databricks.py +396 -0
- kiri_core/discovery/introspector_postgres.py +394 -0
- kiri_core/discovery/introspector_snowflake.py +383 -0
- kiri_core/discovery/reporting/__init__.py +17 -0
- kiri_core/discovery/reporting/discovery_report.py +190 -0
- kiri_core/discovery/scaffold/__init__.py +41 -0
- kiri_core/discovery/scaffold/contract_writer.py +632 -0
- kiri_core/discovery/scaffold/provenance_overlay.py +143 -0
- kiri_core/discovery/scaffold/source_writer.py +211 -0
- kiri_core/discovery/scaffold/todo_marker.py +98 -0
- kiri_core/domain_policy.py +122 -0
- kiri_core/goal_coverage.py +82 -0
- kiri_core/gold_registry.py +211 -0
- kiri_core/health.py +192 -0
- kiri_core/incident.py +204 -0
- kiri_core/incident_builders.py +256 -0
- kiri_core/incident_log.py +418 -0
- kiri_core/intent/__init__.py +46 -0
- kiri_core/intent/agent_runner.py +82 -0
- kiri_core/intent/project_dv.py +261 -0
- kiri_core/intent/propose.py +132 -0
- kiri_core/intent/semantic_intent.py +446 -0
- kiri_core/invariants.py +301 -0
- kiri_core/kpi.py +373 -0
- kiri_core/latency.py +150 -0
- kiri_core/lifecycle/__init__.py +555 -0
- kiri_core/lifecycle/ai_index.py +453 -0
- kiri_core/lifecycle/auto_mode.py +172 -0
- kiri_core/lifecycle/bootstrap.py +316 -0
- kiri_core/lifecycle/cli_reference.py +1064 -0
- kiri_core/lifecycle/commit.py +258 -0
- kiri_core/lifecycle/compile_dispatch.py +296 -0
- kiri_core/lifecycle/compile_range.py +665 -0
- kiri_core/lifecycle/contract_inventory.py +354 -0
- kiri_core/lifecycle/contracts.py +250 -0
- kiri_core/lifecycle/doctor.py +412 -0
- kiri_core/lifecycle/git_remote.py +948 -0
- kiri_core/lifecycle/identity.py +147 -0
- kiri_core/lifecycle/ingestion_modes.py +215 -0
- kiri_core/lifecycle/layer_contract.py +452 -0
- kiri_core/lifecycle/layer_contract_migrate.py +969 -0
- kiri_core/lifecycle/layer_contract_parser.py +436 -0
- kiri_core/lifecycle/layer_contract_validator.py +2291 -0
- kiri_core/lifecycle/layout.py +298 -0
- kiri_core/lifecycle/onboarding.py +306 -0
- kiri_core/lifecycle/openlineage_emit.py +153 -0
- kiri_core/lifecycle/patch.py +231 -0
- kiri_core/lifecycle/plan.py +241 -0
- kiri_core/lifecycle/probe.py +204 -0
- kiri_core/lifecycle/project_context.py +334 -0
- kiri_core/lifecycle/project_status.py +532 -0
- kiri_core/lifecycle/scaffold.py +455 -0
- kiri_core/lifecycle/source_grouping.py +725 -0
- kiri_core/lifecycle/source_introspection.py +488 -0
- kiri_core/lifecycle/source_inventory.py +561 -0
- kiri_core/lifecycle/source_patch.py +353 -0
- kiri_core/lifecycle/sources.py +470 -0
- kiri_core/lifecycle/sync_from_live.py +727 -0
- kiri_core/lifecycle/validate.py +116 -0
- kiri_core/lifecycle/verify_annotations.py +391 -0
- kiri_core/lifecycle/verify_live.py +389 -0
- kiri_core/lifecycle/verify_metric_views.py +321 -0
- kiri_core/lifecycle/watermark.py +371 -0
- kiri_core/lifecycle/watermark_apply.py +656 -0
- kiri_core/lineage.py +418 -0
- kiri_core/loader.py +377 -0
- kiri_core/logging.py +374 -0
- kiri_core/maturity.py +592 -0
- kiri_core/migration_layer_policy.py +285 -0
- kiri_core/names.py +163 -0
- kiri_core/observed_lineage.py +62 -0
- kiri_core/odcs/__init__.py +15 -0
- kiri_core/otel_bootstrap.py +326 -0
- kiri_core/pipeline_ir.py +332 -0
- kiri_core/pipeline_loader.py +82 -0
- kiri_core/planning.py +110 -0
- kiri_core/project.py +1576 -0
- kiri_core/project_docs.py +199 -0
- kiri_core/py.typed +0 -0
- kiri_core/rate_limit.py +456 -0
- kiri_core/release_state/__init__.py +37 -0
- kiri_core/release_state/models.py +108 -0
- kiri_core/release_state/store.py +634 -0
- kiri_core/resilience/__init__.py +288 -0
- kiri_core/resilience/retry.py +173 -0
- kiri_core/runtime.py +40 -0
- kiri_core/schema/__init__.py +62 -0
- kiri_core/schema/column.py +140 -0
- kiri_core/schema/ingestion.py +665 -0
- kiri_core/schema/model.py +44 -0
- kiri_core/schema/reporting_goal.py +160 -0
- kiri_core/schema/source.py +188 -0
- kiri_core/silver/__init__.py +148 -0
- kiri_core/silver/apply_runner.py +830 -0
- kiri_core/silver/backfill.py +280 -0
- kiri_core/silver/dv_backend.py +426 -0
- kiri_core/silver/dv_graph.py +932 -0
- kiri_core/silver/dv_graph_mermaid.py +443 -0
- kiri_core/silver/dv_graph_payload.py +79 -0
- kiri_core/silver/dv_graph_views.py +416 -0
- kiri_core/silver/graph_projector.py +566 -0
- kiri_core/silver/hash_canonicaliser.py +172 -0
- kiri_core/silver/hash_policy.py +195 -0
- kiri_core/silver/load_ts_policy.py +202 -0
- kiri_core/silver/schema_evolution.py +410 -0
- kiri_core/silver/zone_rules.py +81 -0
- kiri_core/sizing.py +134 -0
- kiri_core/storage_paths.py +130 -0
- kiri_core/technique_validator.py +700 -0
- kiri_core/trace.py +409 -0
- kiri_core/types.py +41 -0
- kiri_core/urn.py +161 -0
- kiri_dbt_bridge/__init__.py +22 -0
- kiri_dbt_bridge/_schema_resolve.py +38 -0
- kiri_dbt_bridge/bridge.py +94 -0
- kiri_dbt_bridge/dbt.manifest.yml +26 -0
- kiri_dbt_bridge/dv4dbt/__init__.py +143 -0
- kiri_dbt_bridge/dv4dbt/backend.py +204 -0
- kiri_dbt_bridge/dv4dbt/bridge_generator.py +289 -0
- kiri_dbt_bridge/dv4dbt/compat.py +232 -0
- kiri_dbt_bridge/dv4dbt/hub_generator.py +282 -0
- kiri_dbt_bridge/dv4dbt/link_generator.py +306 -0
- kiri_dbt_bridge/dv4dbt/manifest.py +203 -0
- kiri_dbt_bridge/dv4dbt/pit_generator.py +318 -0
- kiri_dbt_bridge/dv4dbt/policy_translation.py +249 -0
- kiri_dbt_bridge/dv4dbt/project_writer.py +588 -0
- kiri_dbt_bridge/dv4dbt/ref_generator.py +404 -0
- kiri_dbt_bridge/dv4dbt/render.py +180 -0
- kiri_dbt_bridge/dv4dbt/sat_generator.py +480 -0
- kiri_dbt_bridge/dv4dbt/sat_variant_generator.py +529 -0
- kiri_dbt_bridge/dv4dbt/templates/__init__.py +16 -0
- kiri_dbt_bridge/dv4dbt/templates/bridge.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/dbt_project.yml.j2 +37 -0
- kiri_dbt_bridge/dv4dbt/templates/hub.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/link.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/packages.yml.j2 +10 -0
- kiri_dbt_bridge/dv4dbt/templates/pit.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/ref_historised.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/ref_non_historised.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/sat.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/sat_effectivity.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/sat_multi_active.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/sat_non_historised.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/sat_record_tracking.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/sat_virtual_end_dated.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/sources.yml.j2 +39 -0
- kiri_dbt_bridge/framework_adapter.py +310 -0
- kiri_dbt_bridge/generate.py +372 -0
- kiri_dbt_bridge/gold/__init__.py +21 -0
- kiri_dbt_bridge/gold/_sql_emit.py +128 -0
- kiri_dbt_bridge/gold/assemble.py +90 -0
- kiri_dbt_bridge/gold/canonical_columns.py +168 -0
- kiri_dbt_bridge/gold/computed_expressions.py +179 -0
- kiri_dbt_bridge/gold/dialects/__init__.py +69 -0
- kiri_dbt_bridge/gold/dialects/databricks.py +150 -0
- kiri_dbt_bridge/gold/dialects/duckdb.py +120 -0
- kiri_dbt_bridge/gold/dialects/trino.py +136 -0
- kiri_dbt_bridge/gold/dim_generator.py +688 -0
- kiri_dbt_bridge/gold/dimension_refs.py +255 -0
- kiri_dbt_bridge/gold/dispatch.py +69 -0
- kiri_dbt_bridge/gold/fact_generator.py +534 -0
- kiri_dbt_bridge/gold/flat_generator.py +144 -0
- kiri_dbt_bridge/gold/lineage.py +484 -0
- kiri_dbt_bridge/gold/localized_twins.py +218 -0
- kiri_dbt_bridge/gold/measures.py +49 -0
- kiri_dbt_bridge/gold/role_playing.py +75 -0
- kiri_dbt_bridge/gold/surrogate_key.py +114 -0
- kiri_dbt_bridge/macros/kiri_scd1_dim.sql +57 -0
- kiri_dbt_bridge/macros/kiri_scd2_dim.sql +188 -0
- kiri_dbt_bridge/macros/kiri_sha256.sql +32 -0
- kiri_dbt_bridge/macros/kiri_string_cast.sql +37 -0
- kiri_dbt_bridge/macros/kiri_surrogate_key.sql +43 -0
- kiri_dbt_bridge/profiles.py +162 -0
- kiri_dbt_bridge/py.typed +0 -0
- kiri_dbt_bridge/safe_ident.py +44 -0
- kiri_dbt_bridge/silver_conformance.py +188 -0
- kiri_dbt_bridge/silver_kimball/__init__.py +27 -0
- kiri_dbt_bridge/silver_kimball/assemble.py +174 -0
- kiri_dbt_bridge/silver_kimball/dim_generator.py +602 -0
- kiri_dbt_bridge/silver_kimball/dispatch.py +86 -0
- kiri_dbt_bridge/silver_kimball/fact_generator.py +320 -0
- kiri_dbt_bridge/silver_kimball/flat_generator.py +156 -0
- kiri_dbt_bridge/silver_kimball/lineage.py +464 -0
- kiri_dbt_bridge/silver_kimball/watermark_window.py +199 -0
- kiri_dbt_bridge/silver_kimball/zone.py +68 -0
- kiri_dbt_bridge/silver_scd2_curated/__init__.py +46 -0
- kiri_dbt_bridge/silver_scd2_curated/assemble.py +213 -0
- kiri_dbt_bridge/silver_scd2_curated/curated_generator.py +351 -0
- kiri_dbt_bridge/silver_scd2_curated/standardized_generator.py +261 -0
- kiri_dbt_bridge/silver_scd2_curated/standardized_plan.py +254 -0
- kiri_iceberg/__init__.py +11 -0
- kiri_iceberg/catalog.py +128 -0
- kiri_iceberg/profile.py +131 -0
- kiri_iceberg/py.typed +0 -0
- kiri_ingestion/__init__.py +63 -0
- kiri_ingestion/airbyte_client.py +428 -0
- kiri_ingestion/database_fetcher.py +224 -0
- kiri_ingestion/layout.py +71 -0
- kiri_ingestion/mssql_introspect.py +565 -0
- kiri_ingestion/py.typed +0 -0
- kiri_ingestion/rest_fetcher.py +472 -0
- kiri_ingestion/runner.py +341 -0
- kiri_ingestion/schema_drift.py +222 -0
- kiri_ingestion/storage.py +621 -0
- kiri_mcp/__init__.py +5 -0
- kiri_mcp/audit.py +99 -0
- kiri_mcp/audit_decorators.py +252 -0
- kiri_mcp/main.py +22 -0
- kiri_mcp/py.typed +0 -0
- kiri_mcp/rbac.py +110 -0
- kiri_mcp/server.py +766 -0
- kiri_mcp/skills.py +297 -0
- kiri_mcp/tools/__init__.py +70 -0
- kiri_mcp/tools/_shared.py +58 -0
- kiri_mcp/tools/adapter.py +150 -0
- kiri_mcp/tools/cli_reference.py +59 -0
- kiri_mcp/tools/compliance.py +119 -0
- kiri_mcp/tools/contract.py +374 -0
- kiri_mcp/tools/dv.py +62 -0
- kiri_mcp/tools/federation.py +152 -0
- kiri_mcp/tools/intent.py +71 -0
- kiri_mcp/tools/lineage.py +204 -0
- kiri_mcp/tools/lineage_drift.py +75 -0
- kiri_mcp/tools/plan.py +66 -0
- kiri_mcp/tools/project.py +311 -0
- kiri_migration/__init__.py +56 -0
- kiri_migration/name_preservation.py +271 -0
- kiri_migration/predeploy.py +311 -0
- kiri_migration/py.typed +0 -0
- kiri_migration/reconcile.py +1891 -0
- kiri_migration/scaffold/__init__.py +90 -0
- kiri_migration/scaffold/_audit_columns.py +74 -0
- kiri_migration/scaffold/_classify.py +100 -0
- kiri_migration/scaffold/_typemap.py +71 -0
- kiri_migration/scaffold/_yaml.py +33 -0
- kiri_migration/scaffold/bronze.py +279 -0
- kiri_migration/scaffold/compile_targets/__init__.py +271 -0
- kiri_migration/scaffold/compile_targets/_common.py +193 -0
- kiri_migration/scaffold/compile_targets/copy_into.py +33 -0
- kiri_migration/scaffold/compile_targets/dlt.py +125 -0
- kiri_migration/scaffold/compile_targets/dlt_with_append_flows.py +163 -0
- kiri_migration/scaffold/compile_targets/dlt_with_apply_changes.py +192 -0
- kiri_migration/scaffold/compile_targets/dlt_with_expectations.py +276 -0
- kiri_migration/scaffold/compile_targets/layer_dispatch.py +258 -0
- kiri_migration/scaffold/compile_targets/sql_full_snapshot.py +385 -0
- kiri_migration/scaffold/compile_targets/streaming_table.py +37 -0
- kiri_migration/scaffold/mart.py +234 -0
- kiri_migration/scaffold/plans/__init__.py +49 -0
- kiri_migration/scaffold/plans/_registry.py +308 -0
- kiri_migration/scaffold/plans/minimal.py +550 -0
- kiri_migration/scaffold/plans/plan_b_layer.py +640 -0
- kiri_migration/scaffold/silver_from_marts.py +256 -0
- kiri_migration/scaffold/silver_pattern.py +451 -0
- kiri_migration/sp_translator.py +398 -0
- kiri_odcs/LICENSE +204 -0
- kiri_odcs/__init__.py +234 -0
- kiri_odcs/ai_trust_tier.py +475 -0
- kiri_odcs/canonical.py +263 -0
- kiri_odcs/diff.py +63 -0
- kiri_odcs/errors.py +51 -0
- kiri_odcs/invariant_sql.py +136 -0
- kiri_odcs/kiri_bridge.py +71 -0
- kiri_odcs/kiri_ext.py +3166 -0
- kiri_odcs/kiri_fk.py +447 -0
- kiri_odcs/kiri_pit.py +121 -0
- kiri_odcs/kiri_ref.py +57 -0
- kiri_odcs/openlineage_adapter.py +105 -0
- kiri_odcs/parser.py +1213 -0
- kiri_odcs/pii_guardrails.py +159 -0
- kiri_odcs/py.typed +0 -0
- kiri_odcs/schema_export.py +223 -0
- kiri_openlineage/__init__.py +58 -0
- kiri_openlineage/builder.py +146 -0
- kiri_openlineage/cost.py +190 -0
- kiri_openlineage/emitter.py +165 -0
- kiri_openlineage/py.typed +0 -0
- kiri_openlineage/schema.py +121 -0
- kiri_packs/__init__.py +67 -0
- kiri_packs/installer.py +500 -0
- kiri_packs/manifest.py +134 -0
- kiri_packs/publisher.py +605 -0
- kiri_packs/py.typed +0 -0
- kiri_packs/validator.py +295 -0
- kiri_quality/__init__.py +108 -0
- kiri_quality/ai_audit_correlator.py +293 -0
- kiri_quality/ai_audit_correlator_apply.py +526 -0
- kiri_quality/conformance_evidence.py +272 -0
- kiri_quality/dbt_test_collector.py +214 -0
- kiri_quality/dq_codegen.py +300 -0
- kiri_quality/dv_checks.py +393 -0
- kiri_quality/dv_column_lineage.py +279 -0
- kiri_quality/dv_column_lineage_apply.py +856 -0
- kiri_quality/dv_conformance.py +841 -0
- kiri_quality/dv_fixture_runner.py +478 -0
- kiri_quality/dv_invariants.py +986 -0
- kiri_quality/dv_kiri.py +97 -0
- kiri_quality/dv_markdown.py +169 -0
- kiri_quality/dv_observed_uc_reader.py +187 -0
- kiri_quality/dv_runner.py +389 -0
- kiri_quality/evidence_status.py +337 -0
- kiri_quality/fk_checks.py +800 -0
- kiri_quality/fk_ddl.py +402 -0
- kiri_quality/fk_pit_contract.py +168 -0
- kiri_quality/fk_runner.py +950 -0
- kiri_quality/kernel.py +376 -0
- kiri_quality/pii.py +60 -0
- kiri_quality/pre_classify/__init__.py +55 -0
- kiri_quality/pre_classify/engine.py +346 -0
- kiri_quality/pre_classify/locale_packs/en.yaml +117 -0
- kiri_quality/pre_classify/locale_packs/sv.yaml +73 -0
- kiri_quality/pre_classify/merge.py +297 -0
- kiri_quality/pre_classify/model.py +163 -0
- kiri_quality/py.typed +0 -0
- kiri_quality/reject_lane.py +261 -0
- kiri_quality/rule_runner.py +565 -0
- kiri_quality/schema_evolution_check.py +88 -0
- kiri_quality/silver_quality.py +494 -0
- kiri_quality/silver_quality_markdown.py +187 -0
- kiri_quality/uc_security_compile.py +951 -0
- kiri_quality/uc_security_compile_apply.py +1186 -0
- kiri_rbac/__init__.py +87 -0
- kiri_rbac/access.py +371 -0
- kiri_rbac/loader.py +104 -0
- kiri_rbac/pr_gate.py +213 -0
- kiri_rbac/providers/__init__.py +29 -0
- kiri_rbac/providers/_jwt.py +121 -0
- kiri_rbac/providers/azure_ad.py +93 -0
- kiri_rbac/providers/base.py +107 -0
- kiri_rbac/providers/factory.py +48 -0
- kiri_rbac/providers/github.py +134 -0
- kiri_rbac/providers/oidc_generic.py +99 -0
- kiri_rbac/py.typed +0 -0
- kiri_rbac/resolver.py +118 -0
- kiri_rbac/schema.py +219 -0
- kiri_schema_registry/__init__.py +22 -0
- kiri_schema_registry/avro.py +127 -0
- kiri_schema_registry/base.py +71 -0
- kiri_schema_registry/confluent.py +184 -0
- kiri_schema_registry/glue.py +178 -0
- kiri_schema_registry/py.typed +0 -0
- kiri_slack_bot/__init__.py +24 -0
- kiri_slack_bot/handler.py +248 -0
- kiri_slack_bot/py.typed +0 -0
- kiri_slack_bot/queries.py +90 -0
- kiri_slack_bot/signature.py +94 -0
- kiri_vault/__init__.py +59 -0
- kiri_vault/audit.py +64 -0
- kiri_vault/aws_vault.py +123 -0
- kiri_vault/azure_vault.py +236 -0
- kiri_vault/boundaries.py +219 -0
- kiri_vault/client.py +54 -0
- kiri_vault/databricks_vault.py +292 -0
- kiri_vault/env_vault.py +77 -0
- kiri_vault/py.typed +0 -0
- kiri_vault/ref.py +84 -0
- kirimana_automation/__init__.py +0 -0
- kirimana_automation/business_calendar.py +179 -0
- kirimana_automation/dag_snapshot.py +252 -0
- kirimana_automation/db/__init__.py +5 -0
- kirimana_automation/db/migrate.py +571 -0
- kirimana_automation/db/migrations/0001_release_state.down.duckdb.sql +29 -0
- kirimana_automation/db/migrations/0001_release_state.down.postgres.sql +37 -0
- kirimana_automation/db/migrations/0001_release_state.duckdb.sql +170 -0
- kirimana_automation/db/migrations/0001_release_state.postgres.sql +176 -0
- kirimana_automation/db/migrations/0002_rate_limit_buckets.down.duckdb.sql +9 -0
- kirimana_automation/db/migrations/0002_rate_limit_buckets.down.postgres.sql +9 -0
- kirimana_automation/db/migrations/0002_rate_limit_buckets.duckdb.sql +14 -0
- kirimana_automation/db/migrations/0002_rate_limit_buckets.postgres.sql +18 -0
- kirimana_automation/db/migrations/0003_flow_runs.down.duckdb.sql +18 -0
- kirimana_automation/db/migrations/0003_flow_runs.down.postgres.sql +19 -0
- kirimana_automation/db/migrations/0003_flow_runs.duckdb.sql +85 -0
- kirimana_automation/db/migrations/0003_flow_runs.postgres.sql +102 -0
- kirimana_automation/db/migrations/0004_flow_dag_snapshots.down.duckdb.sql +11 -0
- kirimana_automation/db/migrations/0004_flow_dag_snapshots.down.postgres.sql +11 -0
- kirimana_automation/db/migrations/0004_flow_dag_snapshots.duckdb.sql +15 -0
- kirimana_automation/db/migrations/0004_flow_dag_snapshots.postgres.sql +31 -0
- kirimana_automation/db/migrations/0005_calendar_skip.down.duckdb.sql +109 -0
- kirimana_automation/db/migrations/0005_calendar_skip.down.postgres.sql +42 -0
- kirimana_automation/db/migrations/0005_calendar_skip.duckdb.sql +113 -0
- kirimana_automation/db/migrations/0005_calendar_skip.postgres.sql +42 -0
- kirimana_automation/db/migrations/0006_flow_apply_log.down.duckdb.sql +9 -0
- kirimana_automation/db/migrations/0006_flow_apply_log.down.postgres.sql +11 -0
- kirimana_automation/db/migrations/0006_flow_apply_log.duckdb.sql +45 -0
- kirimana_automation/db/migrations/0006_flow_apply_log.postgres.sql +49 -0
- kirimana_automation/db/migrations/0007_flow_task_error_redacted.down.duckdb.sql +14 -0
- kirimana_automation/db/migrations/0007_flow_task_error_redacted.down.postgres.sql +2 -0
- kirimana_automation/db/migrations/0007_flow_task_error_redacted.duckdb.sql +7 -0
- kirimana_automation/db/migrations/0007_flow_task_error_redacted.postgres.sql +11 -0
- kirimana_automation/db/migrations/README.md +84 -0
- kirimana_automation/flow_loader.py +220 -0
- kirimana_automation/freeze.py +133 -0
- kirimana_automation/kiri_recommender.py +313 -0
- kirimana_automation/main.py +195 -0
- kirimana_automation/py.typed +0 -0
- kirimana_automation/run_history.py +593 -0
- kirimana_automation/run_history_analysis.py +607 -0
- kirimana_automation/schedule.py +343 -0
- kirimana_automation/schedule_validator.py +572 -0
- kirimana_automation/workflow_compiler.py +1288 -0
|
@@ -0,0 +1,653 @@
|
|
|
1
|
+
"""T-SQL → Spark SQL deterministic translator (ADR 0055).
|
|
2
|
+
|
|
3
|
+
Companion to :class:`MssqlDialect` in :mod:`kiri_adapters.sql_dialect`. The
|
|
4
|
+
dialect renders Kirimana's abstract plan objects into native T-SQL; this
|
|
5
|
+
module handles the inverse and *additive* problem: take an existing piece
|
|
6
|
+
of T-SQL (typically a stored-procedure body or a hand-written query
|
|
7
|
+
embedded in a legacy migration) and translate the deterministic parts to
|
|
8
|
+
Spark SQL. Anything ambiguous is left intact and surfaced as a
|
|
9
|
+
:class:`TranslationWarning` so a human resolves it.
|
|
10
|
+
|
|
11
|
+
Scope is strictly the deterministic rewrites listed in the BimlFlex
|
|
12
|
+
Migration Pack's closed-menu dialect rules (the "T-SQL → Spark SQL
|
|
13
|
+
syntax" section derived from a representative customer migration) plus
|
|
14
|
+
the type-mapping table:
|
|
15
|
+
|
|
16
|
+
========================== ============================================
|
|
17
|
+
T-SQL Spark SQL
|
|
18
|
+
========================== ============================================
|
|
19
|
+
``TOP N`` ``LIMIT N``
|
|
20
|
+
``GETDATE()`` ``current_timestamp()``
|
|
21
|
+
``ISNULL(x, y)`` ``COALESCE(x, y)``
|
|
22
|
+
``[schema].[table]`` ``schema.table`` (caller adds catalog)
|
|
23
|
+
``WITH (NOLOCK)`` *(removed — not meaningful in Spark)*
|
|
24
|
+
``DATEDIFF(day, a, b)`` ``DATEDIFF(b, a)`` *(arg order reversed!)*
|
|
25
|
+
``LEN(x)`` ``LENGTH(x)``
|
|
26
|
+
``CHARINDEX(x, y)`` ``INSTR(y, x)`` *(arg order reversed!)*
|
|
27
|
+
``CONVERT(t, x)`` ``CAST(x AS t)``
|
|
28
|
+
========================== ============================================
|
|
29
|
+
|
|
30
|
+
Anything beyond these — dynamic SQL, table-valued functions, MERGE
|
|
31
|
+
statements with complex WHEN clauses, CTEs that depend on T-SQL-specific
|
|
32
|
+
window functions — is **not** translated automatically. The function
|
|
33
|
+
returns a :class:`TranslationResult` whose ``warnings`` field tells the
|
|
34
|
+
caller exactly what was left untouched.
|
|
35
|
+
|
|
36
|
+
Production callers go through :func:`translate_tsql`. The AI-driven SP
|
|
37
|
+
translator (``packages/migration_pack/kiri_migration_pack/phase_c``)
|
|
38
|
+
calls this first and only invokes the AI gateway for the residual that
|
|
39
|
+
has warnings.
|
|
40
|
+
|
|
41
|
+
ADR 0055 § 6 invariants (Sprint 6 hardening, 2026-05-23):
|
|
42
|
+
|
|
43
|
+
1. **Closed-menu drift guard.** The set of built-in rewrites and the set
|
|
44
|
+
of built-in warning labels are public, frozen, and pinned by a
|
|
45
|
+
per-rewrite golden test in
|
|
46
|
+
``packages/adapters/tests/test_tsql_to_spark.py``. Adding a rewrite
|
|
47
|
+
or warning to the built-in set without updating the registry +
|
|
48
|
+
golden test breaks CI by construction.
|
|
49
|
+
2. **Per-customer override mechanism.** A customer-private YAML file
|
|
50
|
+
loaded via :func:`load_overrides` extends the closed menu with extra
|
|
51
|
+
rewrites and warnings *additively*. Overrides never replace
|
|
52
|
+
built-ins; an override that collides with a built-in ID raises
|
|
53
|
+
:class:`OverrideConflictError`. The override file path is always
|
|
54
|
+
explicit — there is no auto-globbing, no `**/*.yml` discovery, no
|
|
55
|
+
environment-variable shortcut. Customer overrides are NEVER stored
|
|
56
|
+
in the Kirimana repo.
|
|
57
|
+
3. **Every rewrite and every warning has a test.** ``test_tsql_to_spark``
|
|
58
|
+
exercises every entry in :data:`BUILTIN_REWRITE_IDS` and
|
|
59
|
+
:data:`BUILTIN_WARNING_LABELS`. A new built-in added without a test
|
|
60
|
+
fails the cross-module coverage assertion.
|
|
61
|
+
"""
|
|
62
|
+
|
|
63
|
+
from __future__ import annotations
|
|
64
|
+
|
|
65
|
+
import re
|
|
66
|
+
from dataclasses import dataclass, field
|
|
67
|
+
from pathlib import Path
|
|
68
|
+
from typing import Final
|
|
69
|
+
|
|
70
|
+
import yaml
|
|
71
|
+
|
|
72
|
+
# ---------------------------------------------------------------------------
|
|
73
|
+
# Type mapping — re-export the same map as
|
|
74
|
+
# :mod:`kiri_ingestion.mssql_introspect` so callers don't need both.
|
|
75
|
+
# ---------------------------------------------------------------------------
|
|
76
|
+
|
|
77
|
+
TSQL_TO_SPARK_TYPE: Final[dict[str, str]] = {
|
|
78
|
+
"tinyint": "INT",
|
|
79
|
+
"smallint": "INT",
|
|
80
|
+
"int": "INT",
|
|
81
|
+
"bigint": "LONG",
|
|
82
|
+
"decimal": "DECIMAL",
|
|
83
|
+
"numeric": "DECIMAL",
|
|
84
|
+
"money": "DECIMAL(19,4)",
|
|
85
|
+
"smallmoney": "DECIMAL(19,4)",
|
|
86
|
+
"float": "DOUBLE",
|
|
87
|
+
"real": "DOUBLE",
|
|
88
|
+
"varchar": "STRING",
|
|
89
|
+
"nvarchar": "STRING",
|
|
90
|
+
"char": "STRING",
|
|
91
|
+
"nchar": "STRING",
|
|
92
|
+
"text": "STRING",
|
|
93
|
+
"ntext": "STRING",
|
|
94
|
+
"datetime": "TIMESTAMP",
|
|
95
|
+
"datetime2": "TIMESTAMP",
|
|
96
|
+
"smalldatetime": "TIMESTAMP",
|
|
97
|
+
"datetimeoffset": "TIMESTAMP",
|
|
98
|
+
"date": "DATE",
|
|
99
|
+
"time": "STRING", # no native Spark TIME
|
|
100
|
+
"bit": "BOOLEAN",
|
|
101
|
+
"uniqueidentifier": "STRING",
|
|
102
|
+
"xml": "STRING",
|
|
103
|
+
"rowversion": "BINARY",
|
|
104
|
+
"timestamp": "BINARY", # T-SQL TIMESTAMP is a row-version, not a clock!
|
|
105
|
+
"binary": "BINARY",
|
|
106
|
+
"varbinary": "BINARY",
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
@dataclass(frozen=True)
|
|
111
|
+
class TranslationWarning:
|
|
112
|
+
"""One residual T-SQL construct the deterministic translator left alone."""
|
|
113
|
+
|
|
114
|
+
line: int
|
|
115
|
+
construct: str
|
|
116
|
+
detail: str
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
@dataclass
|
|
120
|
+
class TranslationResult:
|
|
121
|
+
"""Outcome of :func:`translate_tsql`."""
|
|
122
|
+
|
|
123
|
+
spark_sql: str
|
|
124
|
+
warnings: list[TranslationWarning] = field(default_factory=list)
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
# ---------------------------------------------------------------------------
|
|
128
|
+
# Closed-menu drift guard (ADR 0055 § 6 invariant 1).
|
|
129
|
+
#
|
|
130
|
+
# Every built-in rewrite has an entry in :data:`BUILTIN_REWRITE_IDS` and
|
|
131
|
+
# every built-in warning label has an entry in
|
|
132
|
+
# :data:`BUILTIN_WARNING_LABELS`. Tests in ``test_tsql_to_spark`` assert
|
|
133
|
+
# that every member has a golden in/out (rewrites) or trigger (warnings)
|
|
134
|
+
# case, so adding a built-in without a test fails CI.
|
|
135
|
+
# ---------------------------------------------------------------------------
|
|
136
|
+
|
|
137
|
+
BUILTIN_REWRITE_IDS: Final[frozenset[str]] = frozenset(
|
|
138
|
+
{
|
|
139
|
+
"get_date",
|
|
140
|
+
"isnull",
|
|
141
|
+
"with_nolock",
|
|
142
|
+
"top",
|
|
143
|
+
"brackets",
|
|
144
|
+
"len",
|
|
145
|
+
"charindex",
|
|
146
|
+
"convert",
|
|
147
|
+
"datediff_day",
|
|
148
|
+
}
|
|
149
|
+
)
|
|
150
|
+
"""Stable identifiers for the closed-menu built-in rewrites.
|
|
151
|
+
|
|
152
|
+
Adding a new rewrite means: (1) add the rewriter function, (2) wire it
|
|
153
|
+
into :func:`translate_tsql`, (3) add its id here, (4) add a golden test
|
|
154
|
+
in ``test_tsql_to_spark.py``. Steps (3) and (4) are paired — a missing
|
|
155
|
+
golden test is a CI failure via :func:`test_every_builtin_rewrite_has_golden`.
|
|
156
|
+
"""
|
|
157
|
+
|
|
158
|
+
BUILTIN_WARNING_LABELS: Final[frozenset[str]] = frozenset(
|
|
159
|
+
{
|
|
160
|
+
"DATEDIFF with non-day unit",
|
|
161
|
+
"cursor declaration",
|
|
162
|
+
"WHILE loop",
|
|
163
|
+
"dynamic SQL via sp_executesql",
|
|
164
|
+
"dynamic SQL via EXEC()",
|
|
165
|
+
"MERGE INTO statement",
|
|
166
|
+
"temp table reference",
|
|
167
|
+
"T-SQL global variable",
|
|
168
|
+
}
|
|
169
|
+
)
|
|
170
|
+
"""Stable labels for the closed-menu warning surface.
|
|
171
|
+
|
|
172
|
+
Same posture as :data:`BUILTIN_REWRITE_IDS` — adding a new warning
|
|
173
|
+
requires a paired test that triggers it.
|
|
174
|
+
"""
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def translate_tsql(
|
|
178
|
+
sql: str,
|
|
179
|
+
*,
|
|
180
|
+
overrides: TranslationOverrides | None = None,
|
|
181
|
+
) -> TranslationResult:
|
|
182
|
+
"""Apply every deterministic T-SQL → Spark SQL rewrite.
|
|
183
|
+
|
|
184
|
+
Returns the rewritten SQL plus a list of warnings for constructs that
|
|
185
|
+
look T-SQL-specific but were left alone (cursors, dynamic SQL,
|
|
186
|
+
multi-statement bodies, ``DATEDIFF`` with non-day units, etc.). The
|
|
187
|
+
rewrites are intentionally local — the function never reorders
|
|
188
|
+
statements or rewrites query plans.
|
|
189
|
+
|
|
190
|
+
:param overrides: optional :class:`TranslationOverrides` loaded via
|
|
191
|
+
:func:`load_overrides`. Override rewrites run *after* the
|
|
192
|
+
built-in pass; override warnings are appended to the result. An
|
|
193
|
+
override whose ``id`` or ``label`` collides with a built-in is
|
|
194
|
+
rejected by :func:`load_overrides` before reaching this call.
|
|
195
|
+
"""
|
|
196
|
+
|
|
197
|
+
out = sql
|
|
198
|
+
warnings: list[TranslationWarning] = []
|
|
199
|
+
|
|
200
|
+
out = _rewrite_get_date(out)
|
|
201
|
+
out = _rewrite_isnull(out)
|
|
202
|
+
out = _rewrite_with_nolock(out)
|
|
203
|
+
out = _rewrite_top(out)
|
|
204
|
+
out = _rewrite_brackets(out)
|
|
205
|
+
out = _rewrite_len(out)
|
|
206
|
+
out = _rewrite_charindex(out)
|
|
207
|
+
out = _rewrite_convert(out)
|
|
208
|
+
out, datediff_warns = _rewrite_datediff(out)
|
|
209
|
+
warnings.extend(datediff_warns)
|
|
210
|
+
|
|
211
|
+
warnings.extend(_detect_unsupported(out))
|
|
212
|
+
|
|
213
|
+
if overrides is not None:
|
|
214
|
+
out, override_warns = overrides.apply(out)
|
|
215
|
+
warnings.extend(override_warns)
|
|
216
|
+
|
|
217
|
+
return TranslationResult(spark_sql=out, warnings=warnings)
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
# ---------------------------------------------------------------------------
|
|
221
|
+
# Individual rewrite rules.
|
|
222
|
+
# ---------------------------------------------------------------------------
|
|
223
|
+
|
|
224
|
+
_GETDATE = re.compile(r"\bGETDATE\s*\(\s*\)", re.IGNORECASE)
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def _rewrite_get_date(sql: str) -> str:
|
|
228
|
+
return _GETDATE.sub("current_timestamp()", sql)
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
_ISNULL = re.compile(r"\bISNULL\s*\(", re.IGNORECASE)
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def _rewrite_isnull(sql: str) -> str:
|
|
235
|
+
return _ISNULL.sub("COALESCE(", sql)
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
_WITH_NOLOCK = re.compile(r"\bWITH\s*\(\s*NOLOCK\s*\)", re.IGNORECASE)
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def _rewrite_with_nolock(sql: str) -> str:
|
|
242
|
+
return _WITH_NOLOCK.sub("", sql)
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
_TOP = re.compile(r"\bSELECT\s+TOP\s+(\d+)\s+", re.IGNORECASE)
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def _rewrite_top(sql: str) -> str:
|
|
249
|
+
"""Rewrite ``SELECT TOP N`` to ``SELECT`` + appended ``LIMIT N``.
|
|
250
|
+
|
|
251
|
+
Limited to the simple top-of-statement form. ``TOP (N) PERCENT``,
|
|
252
|
+
``TOP WITH TIES``, and ``TOP`` inside subqueries are flagged as
|
|
253
|
+
warnings rather than rewritten — they don't have a safe local
|
|
254
|
+
equivalent in Spark SQL.
|
|
255
|
+
"""
|
|
256
|
+
matches = list(_TOP.finditer(sql))
|
|
257
|
+
if not matches:
|
|
258
|
+
return sql
|
|
259
|
+
first = matches[0]
|
|
260
|
+
n = first.group(1)
|
|
261
|
+
sql_no_top = _TOP.sub("SELECT ", sql, count=1)
|
|
262
|
+
# Append LIMIT once, only if no LIMIT clause already exists.
|
|
263
|
+
if not re.search(r"\bLIMIT\s+\d+", sql_no_top, re.IGNORECASE):
|
|
264
|
+
sql_no_top = sql_no_top.rstrip().rstrip(";") + f"\nLIMIT {n};"
|
|
265
|
+
return sql_no_top
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
_BRACKETS = re.compile(r"\[([^\]\[]+)\]")
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
def _rewrite_brackets(sql: str) -> str:
|
|
272
|
+
"""Strip T-SQL bracket-quoted identifiers.
|
|
273
|
+
|
|
274
|
+
Spark uses backticks; bracket-quoting is unsupported. Most
|
|
275
|
+
identifiers don't need quoting at all in Spark, so we strip the
|
|
276
|
+
brackets unconditionally. Callers that need backtick-quoting for
|
|
277
|
+
reserved words wrap explicitly after translation.
|
|
278
|
+
"""
|
|
279
|
+
return _BRACKETS.sub(r"\1", sql)
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
_LEN = re.compile(r"\bLEN\s*\(", re.IGNORECASE)
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def _rewrite_len(sql: str) -> str:
|
|
286
|
+
return _LEN.sub("LENGTH(", sql)
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
_CHARINDEX = re.compile(
|
|
290
|
+
r"\bCHARINDEX\s*\(\s*([^,()]+?)\s*,\s*([^,()]+?)\s*\)",
|
|
291
|
+
re.IGNORECASE,
|
|
292
|
+
)
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def _rewrite_charindex(sql: str) -> str:
|
|
296
|
+
"""``CHARINDEX(needle, haystack)`` → ``INSTR(haystack, needle)``.
|
|
297
|
+
|
|
298
|
+
Args reverse. The 3-argument form ``CHARINDEX(needle, haystack,
|
|
299
|
+
start)`` is left alone — Spark's ``INSTR`` doesn't have a start
|
|
300
|
+
parameter, so a translation requires expression rewriting that
|
|
301
|
+
can't be done locally.
|
|
302
|
+
"""
|
|
303
|
+
return _CHARINDEX.sub(r"INSTR(\2, \1)", sql)
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
_CONVERT = re.compile(
|
|
307
|
+
r"\bCONVERT\s*\(\s*([A-Za-z0-9_]+(?:\s*\(\s*\d+(?:\s*,\s*\d+)?\s*\))?)\s*,\s*([^,()]+?)\s*\)",
|
|
308
|
+
re.IGNORECASE,
|
|
309
|
+
)
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
def _rewrite_convert(sql: str) -> str:
|
|
313
|
+
"""``CONVERT(type, expr)`` → ``CAST(expr AS type)``.
|
|
314
|
+
|
|
315
|
+
The 3-argument ``CONVERT(type, expr, style)`` form (date format
|
|
316
|
+
codes) is left alone — Spark uses ``date_format`` / ``to_date``
|
|
317
|
+
with explicit pattern strings, which can't be derived locally.
|
|
318
|
+
"""
|
|
319
|
+
return _CONVERT.sub(r"CAST(\2 AS \1)", sql)
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
_DATEDIFF_DAY = re.compile(
|
|
323
|
+
r"\bDATEDIFF\s*\(\s*day\s*,\s*([^,()]+?)\s*,\s*([^,()]+?)\s*\)",
|
|
324
|
+
re.IGNORECASE,
|
|
325
|
+
)
|
|
326
|
+
_DATEDIFF_OTHER = re.compile(
|
|
327
|
+
r"\bDATEDIFF\s*\(\s*(year|quarter|month|week|hour|minute|second|millisecond)\b",
|
|
328
|
+
re.IGNORECASE,
|
|
329
|
+
)
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
def _rewrite_datediff(sql: str) -> tuple[str, list[TranslationWarning]]:
|
|
333
|
+
"""Translate ``DATEDIFF(day, a, b)`` and warn for other units.
|
|
334
|
+
|
|
335
|
+
T-SQL puts the unit first and orders ``(start, end)``. Spark's
|
|
336
|
+
``DATEDIFF`` only computes day differences and orders ``(end,
|
|
337
|
+
start)``. For year / month / week / hour / minute / second the
|
|
338
|
+
Spark equivalent uses different functions (``MONTHS_BETWEEN``,
|
|
339
|
+
``DATE_DIFF`` with units, etc.) — emit a warning rather than
|
|
340
|
+
silently dropping the unit.
|
|
341
|
+
"""
|
|
342
|
+
rewritten = _DATEDIFF_DAY.sub(r"DATEDIFF(\2, \1)", sql)
|
|
343
|
+
warnings: list[TranslationWarning] = []
|
|
344
|
+
for m in _DATEDIFF_OTHER.finditer(rewritten):
|
|
345
|
+
line = rewritten.count("\n", 0, m.start()) + 1
|
|
346
|
+
warnings.append(
|
|
347
|
+
TranslationWarning(
|
|
348
|
+
line=line,
|
|
349
|
+
construct=m.group(0),
|
|
350
|
+
detail=(
|
|
351
|
+
"DATEDIFF with non-day unit has no local Spark equivalent — "
|
|
352
|
+
"rewrite manually using MONTHS_BETWEEN, DATE_DIFF (3-arg "
|
|
353
|
+
"Spark 3.5+), or arithmetic on UNIX_TIMESTAMP."
|
|
354
|
+
),
|
|
355
|
+
)
|
|
356
|
+
)
|
|
357
|
+
return rewritten, warnings
|
|
358
|
+
|
|
359
|
+
|
|
360
|
+
# ---------------------------------------------------------------------------
|
|
361
|
+
# Residual T-SQL detection — flag constructs the local rewrites can't handle.
|
|
362
|
+
# ---------------------------------------------------------------------------
|
|
363
|
+
|
|
364
|
+
_UNSUPPORTED: Final[tuple[tuple[re.Pattern[str], str], ...]] = (
|
|
365
|
+
(re.compile(r"\bDECLARE\s+@\w+\s+CURSOR\b", re.IGNORECASE), "cursor declaration"),
|
|
366
|
+
(re.compile(r"\bWHILE\s+\w+", re.IGNORECASE), "WHILE loop"),
|
|
367
|
+
(re.compile(r"\bsp_executesql\b", re.IGNORECASE), "dynamic SQL via sp_executesql"),
|
|
368
|
+
(re.compile(r"\bEXEC\s*\(", re.IGNORECASE), "dynamic SQL via EXEC()"),
|
|
369
|
+
(re.compile(r"\bMERGE\s+INTO\b", re.IGNORECASE), "MERGE INTO statement"),
|
|
370
|
+
(re.compile(r"#\w+"), "temp table reference"),
|
|
371
|
+
(re.compile(r"@@\w+"), "T-SQL global variable"),
|
|
372
|
+
)
|
|
373
|
+
|
|
374
|
+
|
|
375
|
+
def _detect_unsupported(sql: str) -> list[TranslationWarning]:
|
|
376
|
+
found: list[TranslationWarning] = []
|
|
377
|
+
for pattern, label in _UNSUPPORTED:
|
|
378
|
+
for m in pattern.finditer(sql):
|
|
379
|
+
line = sql.count("\n", 0, m.start()) + 1
|
|
380
|
+
found.append(
|
|
381
|
+
TranslationWarning(
|
|
382
|
+
line=line,
|
|
383
|
+
construct=m.group(0),
|
|
384
|
+
detail=(
|
|
385
|
+
f"{label} is T-SQL-specific and was not translated. "
|
|
386
|
+
"Route the parent statement through the AI-driven SP "
|
|
387
|
+
"translator (`kiri migrate translate`) for a Spark-native "
|
|
388
|
+
"rewrite."
|
|
389
|
+
),
|
|
390
|
+
)
|
|
391
|
+
)
|
|
392
|
+
return found
|
|
393
|
+
|
|
394
|
+
|
|
395
|
+
# ---------------------------------------------------------------------------
|
|
396
|
+
# Per-customer override mechanism (ADR 0055 § 6 invariant 2).
|
|
397
|
+
#
|
|
398
|
+
# Customer-private rewrites live OUTSIDE this repo — typically in
|
|
399
|
+
# ``~/dev/notes/<customer>/tsql_overrides.yml`` — and are loaded via an
|
|
400
|
+
# explicit :func:`load_overrides(path)` call. No auto-discovery, no env
|
|
401
|
+
# var, no glob; the path is passed by the caller every time.
|
|
402
|
+
# ---------------------------------------------------------------------------
|
|
403
|
+
|
|
404
|
+
|
|
405
|
+
class OverrideConflictError(ValueError):
|
|
406
|
+
"""An override's ``id`` (rewrite) or ``label`` (warning) collides
|
|
407
|
+
with a built-in. The override is rejected at load time so the
|
|
408
|
+
surprise can never reach a translation."""
|
|
409
|
+
|
|
410
|
+
|
|
411
|
+
class OverrideSchemaError(ValueError):
|
|
412
|
+
"""The override file is structurally malformed (missing keys,
|
|
413
|
+
invalid YAML, unknown top-level section, bad regex, etc.)."""
|
|
414
|
+
|
|
415
|
+
|
|
416
|
+
@dataclass(frozen=True)
|
|
417
|
+
class OverrideRewrite:
|
|
418
|
+
"""One customer-private rewrite. Applied with :meth:`re.Pattern.sub`
|
|
419
|
+
after the built-in pass."""
|
|
420
|
+
|
|
421
|
+
id: str
|
|
422
|
+
pattern: re.Pattern[str]
|
|
423
|
+
replacement: str
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
@dataclass(frozen=True)
|
|
427
|
+
class OverrideWarning:
|
|
428
|
+
"""One customer-private warning. Matched against the post-rewrite
|
|
429
|
+
SQL; one warning is emitted per regex match."""
|
|
430
|
+
|
|
431
|
+
label: str
|
|
432
|
+
pattern: re.Pattern[str]
|
|
433
|
+
detail: str
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
@dataclass(frozen=True)
|
|
437
|
+
class TranslationOverrides:
|
|
438
|
+
"""A bundle of customer-private rewrites + warnings loaded from one
|
|
439
|
+
YAML file. Frozen by construction; the only way to combine two
|
|
440
|
+
customer files is to load them separately and merge explicitly."""
|
|
441
|
+
|
|
442
|
+
rewrites: tuple[OverrideRewrite, ...] = ()
|
|
443
|
+
warnings: tuple[OverrideWarning, ...] = ()
|
|
444
|
+
source_path: Path | None = None
|
|
445
|
+
|
|
446
|
+
def apply(self, sql: str) -> tuple[str, list[TranslationWarning]]:
|
|
447
|
+
"""Run override rewrites and emit override warnings."""
|
|
448
|
+
|
|
449
|
+
out = sql
|
|
450
|
+
for rewrite in self.rewrites:
|
|
451
|
+
out = rewrite.pattern.sub(rewrite.replacement, out)
|
|
452
|
+
emitted: list[TranslationWarning] = []
|
|
453
|
+
for warning in self.warnings:
|
|
454
|
+
for m in warning.pattern.finditer(out):
|
|
455
|
+
line = out.count("\n", 0, m.start()) + 1
|
|
456
|
+
emitted.append(
|
|
457
|
+
TranslationWarning(
|
|
458
|
+
line=line,
|
|
459
|
+
construct=m.group(0),
|
|
460
|
+
detail=warning.detail,
|
|
461
|
+
)
|
|
462
|
+
)
|
|
463
|
+
return out, emitted
|
|
464
|
+
|
|
465
|
+
|
|
466
|
+
_ALLOWED_FLAG_NAMES: Final[frozenset[str]] = frozenset({"IGNORECASE", "MULTILINE", "DOTALL"})
|
|
467
|
+
|
|
468
|
+
|
|
469
|
+
def _compile_flags(raw: object, *, source: str) -> int:
|
|
470
|
+
"""Build a regex flag bitmask from a list of flag names.
|
|
471
|
+
|
|
472
|
+
Closed menu: only ``IGNORECASE`` / ``MULTILINE`` / ``DOTALL`` are
|
|
473
|
+
accepted. ``VERBOSE`` is rejected because override authors are not
|
|
474
|
+
expected to need it and allowing it widens the attack surface for
|
|
475
|
+
accidentally-broken patterns.
|
|
476
|
+
"""
|
|
477
|
+
|
|
478
|
+
if raw is None:
|
|
479
|
+
return 0
|
|
480
|
+
if not isinstance(raw, list):
|
|
481
|
+
raise OverrideSchemaError(
|
|
482
|
+
f"{source}: 'flags' must be a list of flag names; got {type(raw).__name__}"
|
|
483
|
+
)
|
|
484
|
+
mask = 0
|
|
485
|
+
for entry in raw:
|
|
486
|
+
if not isinstance(entry, str) or entry not in _ALLOWED_FLAG_NAMES:
|
|
487
|
+
raise OverrideSchemaError(
|
|
488
|
+
f"{source}: unknown regex flag {entry!r}; allowed: {sorted(_ALLOWED_FLAG_NAMES)}"
|
|
489
|
+
)
|
|
490
|
+
mask |= getattr(re, entry)
|
|
491
|
+
return mask
|
|
492
|
+
|
|
493
|
+
|
|
494
|
+
def load_overrides(path: Path | str) -> TranslationOverrides:
|
|
495
|
+
"""Load a customer-private overrides YAML.
|
|
496
|
+
|
|
497
|
+
Schema::
|
|
498
|
+
|
|
499
|
+
rewrites:
|
|
500
|
+
- id: spark_dialect_specific_fn
|
|
501
|
+
pattern: '\\bMY_TSQL_FN\\s*\\('
|
|
502
|
+
replacement: 'MY_SPARK_FN('
|
|
503
|
+
flags: [IGNORECASE] # optional; closed menu
|
|
504
|
+
warnings:
|
|
505
|
+
- label: 'customer-specific xp_call'
|
|
506
|
+
pattern: '\\bxp_\\w+'
|
|
507
|
+
detail: 'xp_-prefixed extended procedure; needs manual rewrite.'
|
|
508
|
+
flags: [IGNORECASE] # optional
|
|
509
|
+
|
|
510
|
+
Both sections are optional. Empty sections are allowed (an override
|
|
511
|
+
file is permitted to ship only warnings, or only rewrites).
|
|
512
|
+
|
|
513
|
+
:raises OverrideSchemaError: if the YAML is malformed, missing
|
|
514
|
+
required keys, has an unknown top-level section, contains an
|
|
515
|
+
invalid regex, or specifies an unrecognised flag.
|
|
516
|
+
:raises OverrideConflictError: if a rewrite ``id`` or a warning
|
|
517
|
+
``label`` collides with the built-in closed menu.
|
|
518
|
+
"""
|
|
519
|
+
|
|
520
|
+
path_obj = Path(path)
|
|
521
|
+
if not path_obj.is_file():
|
|
522
|
+
raise OverrideSchemaError(f"overrides file not found: {path_obj}")
|
|
523
|
+
try:
|
|
524
|
+
raw = yaml.safe_load(path_obj.read_text(encoding="utf-8"))
|
|
525
|
+
except yaml.YAMLError as exc:
|
|
526
|
+
raise OverrideSchemaError(f"{path_obj}: invalid YAML: {exc}") from exc
|
|
527
|
+
if raw is None:
|
|
528
|
+
return TranslationOverrides(source_path=path_obj)
|
|
529
|
+
if not isinstance(raw, dict):
|
|
530
|
+
raise OverrideSchemaError(
|
|
531
|
+
f"{path_obj}: top level must be a mapping; got {type(raw).__name__}"
|
|
532
|
+
)
|
|
533
|
+
|
|
534
|
+
unknown = set(raw) - {"rewrites", "warnings"}
|
|
535
|
+
if unknown:
|
|
536
|
+
raise OverrideSchemaError(
|
|
537
|
+
f"{path_obj}: unknown top-level sections: {sorted(unknown)}; "
|
|
538
|
+
"allowed: ['rewrites', 'warnings']"
|
|
539
|
+
)
|
|
540
|
+
|
|
541
|
+
rewrites = _load_rewrites(raw.get("rewrites") or [], path_obj)
|
|
542
|
+
warnings = _load_warnings(raw.get("warnings") or [], path_obj)
|
|
543
|
+
return TranslationOverrides(
|
|
544
|
+
rewrites=tuple(rewrites),
|
|
545
|
+
warnings=tuple(warnings),
|
|
546
|
+
source_path=path_obj,
|
|
547
|
+
)
|
|
548
|
+
|
|
549
|
+
|
|
550
|
+
def _load_rewrites(
|
|
551
|
+
entries: object,
|
|
552
|
+
path_obj: Path,
|
|
553
|
+
) -> list[OverrideRewrite]:
|
|
554
|
+
if not isinstance(entries, list):
|
|
555
|
+
raise OverrideSchemaError(
|
|
556
|
+
f"{path_obj}: 'rewrites' must be a list; got {type(entries).__name__}"
|
|
557
|
+
)
|
|
558
|
+
seen_ids: set[str] = set()
|
|
559
|
+
out: list[OverrideRewrite] = []
|
|
560
|
+
for i, entry in enumerate(entries):
|
|
561
|
+
source = f"{path_obj}#rewrites[{i}]"
|
|
562
|
+
if not isinstance(entry, dict):
|
|
563
|
+
raise OverrideSchemaError(f"{source}: must be a mapping; got {type(entry).__name__}")
|
|
564
|
+
for required in ("id", "pattern", "replacement"):
|
|
565
|
+
if required not in entry:
|
|
566
|
+
raise OverrideSchemaError(f"{source}: missing required key {required!r}")
|
|
567
|
+
rid = entry["id"]
|
|
568
|
+
if not isinstance(rid, str) or not rid:
|
|
569
|
+
raise OverrideSchemaError(f"{source}: 'id' must be a non-empty string")
|
|
570
|
+
if rid in BUILTIN_REWRITE_IDS:
|
|
571
|
+
raise OverrideConflictError(
|
|
572
|
+
f"{source}: rewrite id {rid!r} collides with a built-in; "
|
|
573
|
+
"rename the override to extend rather than shadow."
|
|
574
|
+
)
|
|
575
|
+
if rid in seen_ids:
|
|
576
|
+
raise OverrideSchemaError(
|
|
577
|
+
f"{source}: duplicate rewrite id {rid!r} inside the same file"
|
|
578
|
+
)
|
|
579
|
+
seen_ids.add(rid)
|
|
580
|
+
flags = _compile_flags(entry.get("flags"), source=source)
|
|
581
|
+
pattern_str = entry["pattern"]
|
|
582
|
+
if not isinstance(pattern_str, str):
|
|
583
|
+
raise OverrideSchemaError(f"{source}: 'pattern' must be a string")
|
|
584
|
+
try:
|
|
585
|
+
pattern = re.compile(pattern_str, flags)
|
|
586
|
+
except re.error as exc:
|
|
587
|
+
raise OverrideSchemaError(f"{source}: invalid regex: {exc}") from exc
|
|
588
|
+
replacement = entry["replacement"]
|
|
589
|
+
if not isinstance(replacement, str):
|
|
590
|
+
raise OverrideSchemaError(f"{source}: 'replacement' must be a string")
|
|
591
|
+
out.append(OverrideRewrite(id=rid, pattern=pattern, replacement=replacement))
|
|
592
|
+
return out
|
|
593
|
+
|
|
594
|
+
|
|
595
|
+
def _load_warnings(
|
|
596
|
+
entries: object,
|
|
597
|
+
path_obj: Path,
|
|
598
|
+
) -> list[OverrideWarning]:
|
|
599
|
+
if not isinstance(entries, list):
|
|
600
|
+
raise OverrideSchemaError(
|
|
601
|
+
f"{path_obj}: 'warnings' must be a list; got {type(entries).__name__}"
|
|
602
|
+
)
|
|
603
|
+
seen_labels: set[str] = set()
|
|
604
|
+
out: list[OverrideWarning] = []
|
|
605
|
+
for i, entry in enumerate(entries):
|
|
606
|
+
source = f"{path_obj}#warnings[{i}]"
|
|
607
|
+
if not isinstance(entry, dict):
|
|
608
|
+
raise OverrideSchemaError(f"{source}: must be a mapping; got {type(entry).__name__}")
|
|
609
|
+
for required in ("label", "pattern", "detail"):
|
|
610
|
+
if required not in entry:
|
|
611
|
+
raise OverrideSchemaError(f"{source}: missing required key {required!r}")
|
|
612
|
+
label = entry["label"]
|
|
613
|
+
if not isinstance(label, str) or not label:
|
|
614
|
+
raise OverrideSchemaError(f"{source}: 'label' must be a non-empty string")
|
|
615
|
+
if label in BUILTIN_WARNING_LABELS:
|
|
616
|
+
raise OverrideConflictError(
|
|
617
|
+
f"{source}: warning label {label!r} collides with a built-in; "
|
|
618
|
+
"rename the override to extend rather than shadow."
|
|
619
|
+
)
|
|
620
|
+
if label in seen_labels:
|
|
621
|
+
raise OverrideSchemaError(
|
|
622
|
+
f"{source}: duplicate warning label {label!r} inside the same file"
|
|
623
|
+
)
|
|
624
|
+
seen_labels.add(label)
|
|
625
|
+
flags = _compile_flags(entry.get("flags"), source=source)
|
|
626
|
+
pattern_str = entry["pattern"]
|
|
627
|
+
if not isinstance(pattern_str, str):
|
|
628
|
+
raise OverrideSchemaError(f"{source}: 'pattern' must be a string")
|
|
629
|
+
try:
|
|
630
|
+
pattern = re.compile(pattern_str, flags)
|
|
631
|
+
except re.error as exc:
|
|
632
|
+
raise OverrideSchemaError(f"{source}: invalid regex: {exc}") from exc
|
|
633
|
+
detail = entry["detail"]
|
|
634
|
+
if not isinstance(detail, str):
|
|
635
|
+
raise OverrideSchemaError(f"{source}: 'detail' must be a string")
|
|
636
|
+
out.append(OverrideWarning(label=label, pattern=pattern, detail=detail))
|
|
637
|
+
return out
|
|
638
|
+
|
|
639
|
+
|
|
640
|
+
__all__ = [
|
|
641
|
+
"BUILTIN_REWRITE_IDS",
|
|
642
|
+
"BUILTIN_WARNING_LABELS",
|
|
643
|
+
"TSQL_TO_SPARK_TYPE",
|
|
644
|
+
"OverrideConflictError",
|
|
645
|
+
"OverrideRewrite",
|
|
646
|
+
"OverrideSchemaError",
|
|
647
|
+
"OverrideWarning",
|
|
648
|
+
"TranslationOverrides",
|
|
649
|
+
"TranslationResult",
|
|
650
|
+
"TranslationWarning",
|
|
651
|
+
"load_overrides",
|
|
652
|
+
"translate_tsql",
|
|
653
|
+
]
|