kiri-cli 1.0.0b3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of kiri-cli might be problematic. Click here for more details.
- kiri_adapters/__init__.py +193 -0
- kiri_adapters/backends/__init__.py +0 -0
- kiri_adapters/backends/databricks.manifest.yml +55 -0
- kiri_adapters/backends/databricks.py +2980 -0
- kiri_adapters/backends/fabric_dw.manifest.yml +17 -0
- kiri_adapters/backends/fabric_dw.py +307 -0
- kiri_adapters/backends/fabric_lakehouse.manifest.yml +19 -0
- kiri_adapters/backends/fabric_lakehouse.py +365 -0
- kiri_adapters/backends/iceberg.manifest.yml +17 -0
- kiri_adapters/backends/iceberg.py +451 -0
- kiri_adapters/backends/localduckdb.manifest.yml +43 -0
- kiri_adapters/backends/localduckdb.py +1295 -0
- kiri_adapters/backends/mssql.manifest.yml +17 -0
- kiri_adapters/backends/mssql.py +366 -0
- kiri_adapters/backends/postgres.manifest.yml +17 -0
- kiri_adapters/backends/postgres.py +434 -0
- kiri_adapters/backends/trino.manifest.yml +17 -0
- kiri_adapters/backends/trino.py +542 -0
- kiri_adapters/base.py +892 -0
- kiri_adapters/doctor_checks.py +81 -0
- kiri_adapters/dv_planner.py +1205 -0
- kiri_adapters/dv_planner_bridge.py +265 -0
- kiri_adapters/dv_planner_pit.py +265 -0
- kiri_adapters/dv_planner_ref.py +403 -0
- kiri_adapters/dv_planner_sat_variants.py +668 -0
- kiri_adapters/framework.py +112 -0
- kiri_adapters/incident/__init__.py +30 -0
- kiri_adapters/incident/_http.py +121 -0
- kiri_adapters/incident/base.py +89 -0
- kiri_adapters/incident/jira.py +365 -0
- kiri_adapters/incident/servicenow.py +314 -0
- kiri_adapters/incident/zendesk.py +285 -0
- kiri_adapters/manifest.py +466 -0
- kiri_adapters/orchestration/__init__.py +30 -0
- kiri_adapters/orchestration/airflow.py +117 -0
- kiri_adapters/orchestration/artifact_emitter.py +99 -0
- kiri_adapters/orchestration/autoloader_emitter.py +141 -0
- kiri_adapters/orchestration/azure_data_factory.py +137 -0
- kiri_adapters/orchestration/base.py +626 -0
- kiri_adapters/orchestration/databricks_workflows.py +1768 -0
- kiri_adapters/orchestration/k8s_cron.py +555 -0
- kiri_adapters/orchestration/lifecycle.py +222 -0
- kiri_adapters/orchestration/pipeline_compat.py +145 -0
- kiri_adapters/orchestration/runtime_python.py +188 -0
- kiri_adapters/paths.py +38 -0
- kiri_adapters/protocols/__init__.py +64 -0
- kiri_adapters/protocols/ai_service/__init__.py +7 -0
- kiri_adapters/protocols/ai_service/conformance.py +120 -0
- kiri_adapters/protocols/base.py +220 -0
- kiri_adapters/protocols/catalog/__init__.py +7 -0
- kiri_adapters/protocols/catalog/conformance.py +138 -0
- kiri_adapters/protocols/framework/__init__.py +7 -0
- kiri_adapters/protocols/framework/conformance.py +98 -0
- kiri_adapters/protocols/incident/__init__.py +7 -0
- kiri_adapters/protocols/incident/conformance.py +64 -0
- kiri_adapters/protocols/orchestration/__init__.py +7 -0
- kiri_adapters/protocols/orchestration/conformance.py +84 -0
- kiri_adapters/protocols/platform/__init__.py +7 -0
- kiri_adapters/protocols/platform/conformance.py +120 -0
- kiri_adapters/protocols/sql_dialect/__init__.py +7 -0
- kiri_adapters/protocols/sql_dialect/conformance.py +91 -0
- kiri_adapters/protocols/vault/__init__.py +7 -0
- kiri_adapters/protocols/vault/conformance.py +65 -0
- kiri_adapters/py.typed +0 -0
- kiri_adapters/registry.py +104 -0
- kiri_adapters/semantic/__init__.py +60 -0
- kiri_adapters/semantic/apply.py +274 -0
- kiri_adapters/semantic/base.py +530 -0
- kiri_adapters/semantic/cube.py +280 -0
- kiri_adapters/semantic/dbt_semantic.py +92 -0
- kiri_adapters/semantic/metric_views.py +249 -0
- kiri_adapters/semantic/metricflow.py +345 -0
- kiri_adapters/semantic/snowflake_views.py +75 -0
- kiri_adapters/sql_dialect.py +1592 -0
- kiri_adapters/tsql_to_spark.py +653 -0
- kiri_ai_gateway/__init__.py +147 -0
- kiri_ai_gateway/audit.py +334 -0
- kiri_ai_gateway/batch.py +293 -0
- kiri_ai_gateway/config.py +165 -0
- kiri_ai_gateway/gateway.py +642 -0
- kiri_ai_gateway/placeholder_guard.py +145 -0
- kiri_ai_gateway/prices/__init__.py +77 -0
- kiri_ai_gateway/prices/prices.yaml +64 -0
- kiri_ai_gateway/prompts/__init__.py +54 -0
- kiri_ai_gateway/prompts/airbyte_source_setup.yaml +212 -0
- kiri_ai_gateway/prompts/bdv_translation.yaml +90 -0
- kiri_ai_gateway/prompts/contract_draft_mvp.yaml +117 -0
- kiri_ai_gateway/prompts/describe_table.yaml +19 -0
- kiri_ai_gateway/prompts/design_gold_from_question.yaml +130 -0
- kiri_ai_gateway/prompts/detect_pii.yaml +59 -0
- kiri_ai_gateway/prompts/discover_classify_column.yaml +69 -0
- kiri_ai_gateway/prompts/discover_describe_table.yaml +61 -0
- kiri_ai_gateway/prompts/dv_quality_summary.yaml +36 -0
- kiri_ai_gateway/prompts/incident_assist.yaml +58 -0
- kiri_ai_gateway/prompts/infer_source_schema.yaml +31 -0
- kiri_ai_gateway/prompts/ingest_backend_chooser.yaml +116 -0
- kiri_ai_gateway/prompts/interactive_source_edit.yaml +94 -0
- kiri_ai_gateway/prompts/map_needed_entity_to_source.yaml +72 -0
- kiri_ai_gateway/prompts/meet_kiri.yaml +127 -0
- kiri_ai_gateway/prompts/native_source_setup.yaml +211 -0
- kiri_ai_gateway/prompts/source_grouping_reason.yaml +29 -0
- kiri_ai_gateway/prompts/suggest_attribute_classification.yaml +42 -0
- kiri_ai_gateway/prompts/suggest_data_tests.yaml +71 -0
- kiri_ai_gateway/prompts/suggest_gold_dimension.yaml +108 -0
- kiri_ai_gateway/prompts/suggest_gold_fact.yaml +107 -0
- kiri_ai_gateway/prompts/suggest_gold_model.yaml +70 -0
- kiri_ai_gateway/prompts/suggest_semantic_intent.yaml +91 -0
- kiri_ai_gateway/prompts/suggest_silver_model.yaml +140 -0
- kiri_ai_gateway/providers/__init__.py +7 -0
- kiri_ai_gateway/providers/anthropic_provider.py +26 -0
- kiri_ai_gateway/providers/gateway_provider.py +188 -0
- kiri_ai_gateway/providers/local_provider.py +177 -0
- kiri_ai_gateway/py.typed +0 -0
- kiri_ai_gateway/service.py +283 -0
- kiri_ai_gateway/transport.py +103 -0
- kiri_ai_gateway/transports/__init__.py +12 -0
- kiri_ai_gateway/transports/anthropic_transport.py +73 -0
- kiri_ai_gateway/transports/ollama_transport.py +82 -0
- kiri_ai_gateway/transports/openai_compatible_transport.py +125 -0
- kiri_ai_gateway/workspace_policy.py +52 -0
- kiri_catalog/__init__.py +199 -0
- kiri_catalog/annotation_plan.py +183 -0
- kiri_catalog/appearances.py +155 -0
- kiri_catalog/backends/__init__.py +79 -0
- kiri_catalog/backends/base.py +167 -0
- kiri_catalog/backends/memory.py +241 -0
- kiri_catalog/backends/postgres.py +497 -0
- kiri_catalog/backends/redacting.py +195 -0
- kiri_catalog/backends/sqlite.py +480 -0
- kiri_catalog/context.py +94 -0
- kiri_catalog/federation/__init__.py +114 -0
- kiri_catalog/federation/api.py +200 -0
- kiri_catalog/federation/cache.py +149 -0
- kiri_catalog/federation/emit.py +209 -0
- kiri_catalog/federation/fs_resolver.py +295 -0
- kiri_catalog/federation/http_resolver.py +486 -0
- kiri_catalog/federation/metrics.py +123 -0
- kiri_catalog/federation/release.py +342 -0
- kiri_catalog/federation/resolver.py +332 -0
- kiri_catalog/federation/schema.py +460 -0
- kiri_catalog/importer.py +668 -0
- kiri_catalog/lineage_drift.py +280 -0
- kiri_catalog/lineage_graph.py +176 -0
- kiri_catalog/lineage_graph_explain.py +96 -0
- kiri_catalog/lineage_graph_mermaid.py +230 -0
- kiri_catalog/observed_uc_lineage.py +216 -0
- kiri_catalog/pii_detect.py +176 -0
- kiri_catalog/push.py +122 -0
- kiri_catalog/py.typed +0 -0
- kiri_catalog/redact.py +101 -0
- kiri_catalog/review_actions.py +119 -0
- kiri_catalog/schema/__init__.py +70 -0
- kiri_catalog/schema/asset.py +142 -0
- kiri_catalog/schema/column.py +247 -0
- kiri_catalog/schema/enums.py +195 -0
- kiri_catalog/schema/job.py +51 -0
- kiri_catalog/schema/lineage.py +33 -0
- kiri_catalog/schema/policy.py +69 -0
- kiri_catalog/schema/primitives.py +90 -0
- kiri_catalog/schema/review_state.py +111 -0
- kiri_catalog/schema/urn.py +45 -0
- kiri_catalog/schema_discovery.py +160 -0
- kiri_catalog/sinks/__init__.py +64 -0
- kiri_catalog/sinks/base.py +178 -0
- kiri_catalog/sinks/horizon.py +195 -0
- kiri_catalog/sinks/purview.py +216 -0
- kiri_catalog/sinks/unity.py +352 -0
- kiri_catalog/sinks/unity_executor.py +543 -0
- kiri_catalog/sources/__init__.py +42 -0
- kiri_catalog/sources/base.py +104 -0
- kiri_catalog/sources/unity.py +465 -0
- kiri_catalog/steward_writeback.py +158 -0
- kiri_catalog/tag_sql.py +99 -0
- kiri_cli/LICENSE +105 -0
- kiri_cli/__init__.py +1 -0
- kiri_cli/_apply_cli.py +63 -0
- kiri_cli/_apply_reporter.py +38 -0
- kiri_cli/_closed_menu.py +36 -0
- kiri_cli/_fixture_option.py +18 -0
- kiri_cli/_layer_detect.py +158 -0
- kiri_cli/_project_option.py +79 -0
- kiri_cli/adapter_cmd.py +192 -0
- kiri_cli/ai_batch_cmd.py +258 -0
- kiri_cli/ai_cmd.py +352 -0
- kiri_cli/approve_cmd.py +659 -0
- kiri_cli/attr_cmd.py +575 -0
- kiri_cli/audit_cmd.py +348 -0
- kiri_cli/audit_correlate_cmd.py +648 -0
- kiri_cli/audit_join.py +437 -0
- kiri_cli/audit_join_cmd.py +160 -0
- kiri_cli/auto_cmd.py +97 -0
- kiri_cli/azure_cmd.py +243 -0
- kiri_cli/catalog.py +1293 -0
- kiri_cli/cli_reference_live.py +147 -0
- kiri_cli/compile_cmd.py +493 -0
- kiri_cli/compliance_cmd.py +221 -0
- kiri_cli/contract_approval_cmd.py +94 -0
- kiri_cli/contract_cmd.py +64 -0
- kiri_cli/contract_codeowners_cmd.py +162 -0
- kiri_cli/contract_diff.py +111 -0
- kiri_cli/contract_emit.py +90 -0
- kiri_cli/contract_export.py +192 -0
- kiri_cli/contract_lint.py +1505 -0
- kiri_cli/contract_merge_cmd.py +264 -0
- kiri_cli/contract_new.py +326 -0
- kiri_cli/contract_promote.py +216 -0
- kiri_cli/contract_scaffold_cmd.py +134 -0
- kiri_cli/contract_schema_export.py +69 -0
- kiri_cli/contract_show.py +150 -0
- kiri_cli/contract_stubs.py +60 -0
- kiri_cli/contract_sync_from_live.py +327 -0
- kiri_cli/contract_validate.py +154 -0
- kiri_cli/contract_verify_live.py +242 -0
- kiri_cli/cost_cmd.py +243 -0
- kiri_cli/databricks_health.py +1254 -0
- kiri_cli/databricks_setup_cmd.py +712 -0
- kiri_cli/discover_cmd.py +600 -0
- kiri_cli/docs_cmd.py +35 -0
- kiri_cli/doctor_cmd.py +145 -0
- kiri_cli/dv_cmd.py +814 -0
- kiri_cli/dv_conformance_cmd.py +154 -0
- kiri_cli/dv_install.py +493 -0
- kiri_cli/dv_migrate_cmd.py +183 -0
- kiri_cli/dv_test_cmd.py +175 -0
- kiri_cli/federation_cmd.py +315 -0
- kiri_cli/fk_cmd.py +160 -0
- kiri_cli/flows_cmd.py +910 -0
- kiri_cli/governance_cmd.py +358 -0
- kiri_cli/incident_api_cmd.py +180 -0
- kiri_cli/incident_cmd.py +410 -0
- kiri_cli/incident_setup_cmd.py +239 -0
- kiri_cli/ingest.py +427 -0
- kiri_cli/ingest_discover.py +480 -0
- kiri_cli/ingest_new_source.py +386 -0
- kiri_cli/intent_cmd.py +122 -0
- kiri_cli/invariants_cmd.py +118 -0
- kiri_cli/kiri_banner.py +161 -0
- kiri_cli/kpi_cmd.py +222 -0
- kiri_cli/layer_cmd.py +464 -0
- kiri_cli/lineage.py +601 -0
- kiri_cli/main.py +2472 -0
- kiri_cli/maturity_cmd.py +222 -0
- kiri_cli/migrate_cmd.py +1347 -0
- kiri_cli/onboard_cmd.py +223 -0
- kiri_cli/pack_cmd.py +496 -0
- kiri_cli/pipeline_cmd.py +127 -0
- kiri_cli/pre_classify_cmd.py +274 -0
- kiri_cli/project_cmd.py +592 -0
- kiri_cli/py.typed +0 -0
- kiri_cli/quality_cmd.py +295 -0
- kiri_cli/rbac.py +139 -0
- kiri_cli/rbac_cmd.py +291 -0
- kiri_cli/release.py +850 -0
- kiri_cli/release_semver.py +265 -0
- kiri_cli/repo_cmd.py +155 -0
- kiri_cli/retention_cmd.py +333 -0
- kiri_cli/sample_data/__init__.py +12 -0
- kiri_cli/sample_data/customers.contract.yml +47 -0
- kiri_cli/sample_data/customers.csv +25 -0
- kiri_cli/scaffold_cmd.py +721 -0
- kiri_cli/schema_cmd.py +185 -0
- kiri_cli/semantic_cmd.py +605 -0
- kiri_cli/silver_cmd.py +348 -0
- kiri_cli/skills_cmd.py +243 -0
- kiri_cli/skills_data/README.md +51 -0
- kiri_cli/skills_data/add-source/SKILL.md +116 -0
- kiri_cli/skills_data/azure-deploy/SKILL.md +261 -0
- kiri_cli/skills_data/azure-status/SKILL.md +111 -0
- kiri_cli/skills_data/compliance-check/SKILL.md +99 -0
- kiri_cli/skills_data/databricks-build-pipeline/SKILL.md +112 -0
- kiri_cli/skills_data/databricks-project-setup/SKILL.md +157 -0
- kiri_cli/skills_data/debug-apply/SKILL.md +186 -0
- kiri_cli/skills_data/draft-gold-model/SKILL.md +203 -0
- kiri_cli/skills_data/explain-kpi/SKILL.md +158 -0
- kiri_cli/skills_data/incident-coach/SKILL.md +135 -0
- kiri_cli/skills_data/incident-setup/SKILL.md +112 -0
- kiri_cli/skills_data/maturity-coach/SKILL.md +120 -0
- kiri_cli/skills_data/promote-contract/SKILL.md +91 -0
- kiri_cli/skills_data/review-attributes/SKILL.md +94 -0
- kiri_cli/sla_cmd.py +769 -0
- kiri_cli/source_cmd.py +909 -0
- kiri_cli/templates/__init__.py +6 -0
- kiri_cli/templates/dv4dbt_packages.yml.j2 +11 -0
- kiri_cli/upgrade_cmd.py +896 -0
- kiri_cli/use_cmd.py +120 -0
- kiri_cli/vault_cmd.py +241 -0
- kiri_cli/verify_generated.py +241 -0
- kiri_cli-1.0.0b3.dist-info/METADATA +212 -0
- kiri_cli-1.0.0b3.dist-info/RECORD +743 -0
- kiri_cli-1.0.0b3.dist-info/WHEEL +4 -0
- kiri_cli-1.0.0b3.dist-info/entry_points.txt +4 -0
- kiri_compliance/__init__.py +33 -0
- kiri_compliance/audit_coverage.py +354 -0
- kiri_compliance/controls.py +763 -0
- kiri_compliance/evidence.py +155 -0
- kiri_compliance/py.typed +0 -0
- kiri_compliance/quality_engine.py +246 -0
- kiri_compliance/renderer.py +195 -0
- kiri_compliance/report.py +44 -0
- kiri_compliance/schema.py +129 -0
- kiri_core/__init__.py +185 -0
- kiri_core/adapter_surfaces.py +149 -0
- kiri_core/ai_query_gate/__init__.py +92 -0
- kiri_core/ai_query_gate/audit.py +153 -0
- kiri_core/ai_query_gate/evaluator.py +484 -0
- kiri_core/ai_query_gate/models.py +162 -0
- kiri_core/ai_query_gate/parser.py +436 -0
- kiri_core/ai_query_gate/resolver.py +274 -0
- kiri_core/ai_query_gate/store.py +234 -0
- kiri_core/ai_query_gate/store_postgres.py +409 -0
- kiri_core/api_keys.py +243 -0
- kiri_core/apply/__init__.py +27 -0
- kiri_core/apply/conformance.py +99 -0
- kiri_core/apply/incident_router.py +176 -0
- kiri_core/apply/models.py +140 -0
- kiri_core/apply/reporter.py +46 -0
- kiri_core/apply/runner.py +3083 -0
- kiri_core/apply_log_store.py +155 -0
- kiri_core/approval_flow/__init__.py +133 -0
- kiri_core/approval_flow/integration.py +156 -0
- kiri_core/approval_flow/models.py +443 -0
- kiri_core/approval_flow/notifier.py +263 -0
- kiri_core/approval_flow/orchestrator.py +660 -0
- kiri_core/approval_flow/reconcile_signoff.py +53 -0
- kiri_core/approval_flow/store.py +220 -0
- kiri_core/approval_flow/store_jsonl.py +217 -0
- kiri_core/approval_flow/store_postgres.py +580 -0
- kiri_core/approval_policy.py +116 -0
- kiri_core/asgi_trace.py +160 -0
- kiri_core/compliance.py +802 -0
- kiri_core/cutover_approval.py +187 -0
- kiri_core/discovery/__init__.py +109 -0
- kiri_core/discovery/ai/__init__.py +37 -0
- kiri_core/discovery/ai/_payload.py +115 -0
- kiri_core/discovery/ai/budget.py +83 -0
- kiri_core/discovery/ai/classify.py +223 -0
- kiri_core/discovery/ai/describe.py +200 -0
- kiri_core/discovery/ai/errors.py +49 -0
- kiri_core/discovery/discover.py +648 -0
- kiri_core/discovery/heuristics/__init__.py +42 -0
- kiri_core/discovery/heuristics/classification.py +174 -0
- kiri_core/discovery/heuristics/medallion.py +157 -0
- kiri_core/discovery/heuristics/pii.py +92 -0
- kiri_core/discovery/introspector.py +636 -0
- kiri_core/discovery/introspector_databricks.py +396 -0
- kiri_core/discovery/introspector_postgres.py +394 -0
- kiri_core/discovery/introspector_snowflake.py +383 -0
- kiri_core/discovery/reporting/__init__.py +17 -0
- kiri_core/discovery/reporting/discovery_report.py +190 -0
- kiri_core/discovery/scaffold/__init__.py +41 -0
- kiri_core/discovery/scaffold/contract_writer.py +632 -0
- kiri_core/discovery/scaffold/provenance_overlay.py +143 -0
- kiri_core/discovery/scaffold/source_writer.py +211 -0
- kiri_core/discovery/scaffold/todo_marker.py +98 -0
- kiri_core/domain_policy.py +122 -0
- kiri_core/goal_coverage.py +82 -0
- kiri_core/gold_registry.py +211 -0
- kiri_core/health.py +192 -0
- kiri_core/incident.py +204 -0
- kiri_core/incident_builders.py +256 -0
- kiri_core/incident_log.py +418 -0
- kiri_core/intent/__init__.py +46 -0
- kiri_core/intent/agent_runner.py +82 -0
- kiri_core/intent/project_dv.py +261 -0
- kiri_core/intent/propose.py +132 -0
- kiri_core/intent/semantic_intent.py +446 -0
- kiri_core/invariants.py +301 -0
- kiri_core/kpi.py +373 -0
- kiri_core/latency.py +150 -0
- kiri_core/lifecycle/__init__.py +555 -0
- kiri_core/lifecycle/ai_index.py +453 -0
- kiri_core/lifecycle/auto_mode.py +172 -0
- kiri_core/lifecycle/bootstrap.py +316 -0
- kiri_core/lifecycle/cli_reference.py +1064 -0
- kiri_core/lifecycle/commit.py +258 -0
- kiri_core/lifecycle/compile_dispatch.py +296 -0
- kiri_core/lifecycle/compile_range.py +665 -0
- kiri_core/lifecycle/contract_inventory.py +354 -0
- kiri_core/lifecycle/contracts.py +250 -0
- kiri_core/lifecycle/doctor.py +412 -0
- kiri_core/lifecycle/git_remote.py +948 -0
- kiri_core/lifecycle/identity.py +147 -0
- kiri_core/lifecycle/ingestion_modes.py +215 -0
- kiri_core/lifecycle/layer_contract.py +452 -0
- kiri_core/lifecycle/layer_contract_migrate.py +969 -0
- kiri_core/lifecycle/layer_contract_parser.py +436 -0
- kiri_core/lifecycle/layer_contract_validator.py +2291 -0
- kiri_core/lifecycle/layout.py +298 -0
- kiri_core/lifecycle/onboarding.py +306 -0
- kiri_core/lifecycle/openlineage_emit.py +153 -0
- kiri_core/lifecycle/patch.py +231 -0
- kiri_core/lifecycle/plan.py +241 -0
- kiri_core/lifecycle/probe.py +204 -0
- kiri_core/lifecycle/project_context.py +334 -0
- kiri_core/lifecycle/project_status.py +532 -0
- kiri_core/lifecycle/scaffold.py +455 -0
- kiri_core/lifecycle/source_grouping.py +725 -0
- kiri_core/lifecycle/source_introspection.py +488 -0
- kiri_core/lifecycle/source_inventory.py +561 -0
- kiri_core/lifecycle/source_patch.py +353 -0
- kiri_core/lifecycle/sources.py +470 -0
- kiri_core/lifecycle/sync_from_live.py +727 -0
- kiri_core/lifecycle/validate.py +116 -0
- kiri_core/lifecycle/verify_annotations.py +391 -0
- kiri_core/lifecycle/verify_live.py +389 -0
- kiri_core/lifecycle/verify_metric_views.py +321 -0
- kiri_core/lifecycle/watermark.py +371 -0
- kiri_core/lifecycle/watermark_apply.py +656 -0
- kiri_core/lineage.py +418 -0
- kiri_core/loader.py +377 -0
- kiri_core/logging.py +374 -0
- kiri_core/maturity.py +592 -0
- kiri_core/migration_layer_policy.py +285 -0
- kiri_core/names.py +163 -0
- kiri_core/observed_lineage.py +62 -0
- kiri_core/odcs/__init__.py +15 -0
- kiri_core/otel_bootstrap.py +326 -0
- kiri_core/pipeline_ir.py +332 -0
- kiri_core/pipeline_loader.py +82 -0
- kiri_core/planning.py +110 -0
- kiri_core/project.py +1576 -0
- kiri_core/project_docs.py +199 -0
- kiri_core/py.typed +0 -0
- kiri_core/rate_limit.py +456 -0
- kiri_core/release_state/__init__.py +37 -0
- kiri_core/release_state/models.py +108 -0
- kiri_core/release_state/store.py +634 -0
- kiri_core/resilience/__init__.py +288 -0
- kiri_core/resilience/retry.py +173 -0
- kiri_core/runtime.py +40 -0
- kiri_core/schema/__init__.py +62 -0
- kiri_core/schema/column.py +140 -0
- kiri_core/schema/ingestion.py +665 -0
- kiri_core/schema/model.py +44 -0
- kiri_core/schema/reporting_goal.py +160 -0
- kiri_core/schema/source.py +188 -0
- kiri_core/silver/__init__.py +148 -0
- kiri_core/silver/apply_runner.py +830 -0
- kiri_core/silver/backfill.py +280 -0
- kiri_core/silver/dv_backend.py +426 -0
- kiri_core/silver/dv_graph.py +932 -0
- kiri_core/silver/dv_graph_mermaid.py +443 -0
- kiri_core/silver/dv_graph_payload.py +79 -0
- kiri_core/silver/dv_graph_views.py +416 -0
- kiri_core/silver/graph_projector.py +566 -0
- kiri_core/silver/hash_canonicaliser.py +172 -0
- kiri_core/silver/hash_policy.py +195 -0
- kiri_core/silver/load_ts_policy.py +202 -0
- kiri_core/silver/schema_evolution.py +410 -0
- kiri_core/silver/zone_rules.py +81 -0
- kiri_core/sizing.py +134 -0
- kiri_core/storage_paths.py +130 -0
- kiri_core/technique_validator.py +700 -0
- kiri_core/trace.py +409 -0
- kiri_core/types.py +41 -0
- kiri_core/urn.py +161 -0
- kiri_dbt_bridge/__init__.py +22 -0
- kiri_dbt_bridge/_schema_resolve.py +38 -0
- kiri_dbt_bridge/bridge.py +94 -0
- kiri_dbt_bridge/dbt.manifest.yml +26 -0
- kiri_dbt_bridge/dv4dbt/__init__.py +143 -0
- kiri_dbt_bridge/dv4dbt/backend.py +204 -0
- kiri_dbt_bridge/dv4dbt/bridge_generator.py +289 -0
- kiri_dbt_bridge/dv4dbt/compat.py +232 -0
- kiri_dbt_bridge/dv4dbt/hub_generator.py +282 -0
- kiri_dbt_bridge/dv4dbt/link_generator.py +306 -0
- kiri_dbt_bridge/dv4dbt/manifest.py +203 -0
- kiri_dbt_bridge/dv4dbt/pit_generator.py +318 -0
- kiri_dbt_bridge/dv4dbt/policy_translation.py +249 -0
- kiri_dbt_bridge/dv4dbt/project_writer.py +588 -0
- kiri_dbt_bridge/dv4dbt/ref_generator.py +404 -0
- kiri_dbt_bridge/dv4dbt/render.py +180 -0
- kiri_dbt_bridge/dv4dbt/sat_generator.py +480 -0
- kiri_dbt_bridge/dv4dbt/sat_variant_generator.py +529 -0
- kiri_dbt_bridge/dv4dbt/templates/__init__.py +16 -0
- kiri_dbt_bridge/dv4dbt/templates/bridge.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/dbt_project.yml.j2 +37 -0
- kiri_dbt_bridge/dv4dbt/templates/hub.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/link.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/packages.yml.j2 +10 -0
- kiri_dbt_bridge/dv4dbt/templates/pit.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/ref_historised.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/ref_non_historised.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/sat.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/sat_effectivity.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/sat_multi_active.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/sat_non_historised.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/sat_record_tracking.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/sat_virtual_end_dated.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/sources.yml.j2 +39 -0
- kiri_dbt_bridge/framework_adapter.py +310 -0
- kiri_dbt_bridge/generate.py +372 -0
- kiri_dbt_bridge/gold/__init__.py +21 -0
- kiri_dbt_bridge/gold/_sql_emit.py +128 -0
- kiri_dbt_bridge/gold/assemble.py +90 -0
- kiri_dbt_bridge/gold/canonical_columns.py +168 -0
- kiri_dbt_bridge/gold/computed_expressions.py +179 -0
- kiri_dbt_bridge/gold/dialects/__init__.py +69 -0
- kiri_dbt_bridge/gold/dialects/databricks.py +150 -0
- kiri_dbt_bridge/gold/dialects/duckdb.py +120 -0
- kiri_dbt_bridge/gold/dialects/trino.py +136 -0
- kiri_dbt_bridge/gold/dim_generator.py +688 -0
- kiri_dbt_bridge/gold/dimension_refs.py +255 -0
- kiri_dbt_bridge/gold/dispatch.py +69 -0
- kiri_dbt_bridge/gold/fact_generator.py +534 -0
- kiri_dbt_bridge/gold/flat_generator.py +144 -0
- kiri_dbt_bridge/gold/lineage.py +484 -0
- kiri_dbt_bridge/gold/localized_twins.py +218 -0
- kiri_dbt_bridge/gold/measures.py +49 -0
- kiri_dbt_bridge/gold/role_playing.py +75 -0
- kiri_dbt_bridge/gold/surrogate_key.py +114 -0
- kiri_dbt_bridge/macros/kiri_scd1_dim.sql +57 -0
- kiri_dbt_bridge/macros/kiri_scd2_dim.sql +188 -0
- kiri_dbt_bridge/macros/kiri_sha256.sql +32 -0
- kiri_dbt_bridge/macros/kiri_string_cast.sql +37 -0
- kiri_dbt_bridge/macros/kiri_surrogate_key.sql +43 -0
- kiri_dbt_bridge/profiles.py +162 -0
- kiri_dbt_bridge/py.typed +0 -0
- kiri_dbt_bridge/safe_ident.py +44 -0
- kiri_dbt_bridge/silver_conformance.py +188 -0
- kiri_dbt_bridge/silver_kimball/__init__.py +27 -0
- kiri_dbt_bridge/silver_kimball/assemble.py +174 -0
- kiri_dbt_bridge/silver_kimball/dim_generator.py +602 -0
- kiri_dbt_bridge/silver_kimball/dispatch.py +86 -0
- kiri_dbt_bridge/silver_kimball/fact_generator.py +320 -0
- kiri_dbt_bridge/silver_kimball/flat_generator.py +156 -0
- kiri_dbt_bridge/silver_kimball/lineage.py +464 -0
- kiri_dbt_bridge/silver_kimball/watermark_window.py +199 -0
- kiri_dbt_bridge/silver_kimball/zone.py +68 -0
- kiri_dbt_bridge/silver_scd2_curated/__init__.py +46 -0
- kiri_dbt_bridge/silver_scd2_curated/assemble.py +213 -0
- kiri_dbt_bridge/silver_scd2_curated/curated_generator.py +351 -0
- kiri_dbt_bridge/silver_scd2_curated/standardized_generator.py +261 -0
- kiri_dbt_bridge/silver_scd2_curated/standardized_plan.py +254 -0
- kiri_iceberg/__init__.py +11 -0
- kiri_iceberg/catalog.py +128 -0
- kiri_iceberg/profile.py +131 -0
- kiri_iceberg/py.typed +0 -0
- kiri_ingestion/__init__.py +63 -0
- kiri_ingestion/airbyte_client.py +428 -0
- kiri_ingestion/database_fetcher.py +224 -0
- kiri_ingestion/layout.py +71 -0
- kiri_ingestion/mssql_introspect.py +565 -0
- kiri_ingestion/py.typed +0 -0
- kiri_ingestion/rest_fetcher.py +472 -0
- kiri_ingestion/runner.py +341 -0
- kiri_ingestion/schema_drift.py +222 -0
- kiri_ingestion/storage.py +621 -0
- kiri_mcp/__init__.py +5 -0
- kiri_mcp/audit.py +99 -0
- kiri_mcp/audit_decorators.py +252 -0
- kiri_mcp/main.py +22 -0
- kiri_mcp/py.typed +0 -0
- kiri_mcp/rbac.py +110 -0
- kiri_mcp/server.py +766 -0
- kiri_mcp/skills.py +297 -0
- kiri_mcp/tools/__init__.py +70 -0
- kiri_mcp/tools/_shared.py +58 -0
- kiri_mcp/tools/adapter.py +150 -0
- kiri_mcp/tools/cli_reference.py +59 -0
- kiri_mcp/tools/compliance.py +119 -0
- kiri_mcp/tools/contract.py +374 -0
- kiri_mcp/tools/dv.py +62 -0
- kiri_mcp/tools/federation.py +152 -0
- kiri_mcp/tools/intent.py +71 -0
- kiri_mcp/tools/lineage.py +204 -0
- kiri_mcp/tools/lineage_drift.py +75 -0
- kiri_mcp/tools/plan.py +66 -0
- kiri_mcp/tools/project.py +311 -0
- kiri_migration/__init__.py +56 -0
- kiri_migration/name_preservation.py +271 -0
- kiri_migration/predeploy.py +311 -0
- kiri_migration/py.typed +0 -0
- kiri_migration/reconcile.py +1891 -0
- kiri_migration/scaffold/__init__.py +90 -0
- kiri_migration/scaffold/_audit_columns.py +74 -0
- kiri_migration/scaffold/_classify.py +100 -0
- kiri_migration/scaffold/_typemap.py +71 -0
- kiri_migration/scaffold/_yaml.py +33 -0
- kiri_migration/scaffold/bronze.py +279 -0
- kiri_migration/scaffold/compile_targets/__init__.py +271 -0
- kiri_migration/scaffold/compile_targets/_common.py +193 -0
- kiri_migration/scaffold/compile_targets/copy_into.py +33 -0
- kiri_migration/scaffold/compile_targets/dlt.py +125 -0
- kiri_migration/scaffold/compile_targets/dlt_with_append_flows.py +163 -0
- kiri_migration/scaffold/compile_targets/dlt_with_apply_changes.py +192 -0
- kiri_migration/scaffold/compile_targets/dlt_with_expectations.py +276 -0
- kiri_migration/scaffold/compile_targets/layer_dispatch.py +258 -0
- kiri_migration/scaffold/compile_targets/sql_full_snapshot.py +385 -0
- kiri_migration/scaffold/compile_targets/streaming_table.py +37 -0
- kiri_migration/scaffold/mart.py +234 -0
- kiri_migration/scaffold/plans/__init__.py +49 -0
- kiri_migration/scaffold/plans/_registry.py +308 -0
- kiri_migration/scaffold/plans/minimal.py +550 -0
- kiri_migration/scaffold/plans/plan_b_layer.py +640 -0
- kiri_migration/scaffold/silver_from_marts.py +256 -0
- kiri_migration/scaffold/silver_pattern.py +451 -0
- kiri_migration/sp_translator.py +398 -0
- kiri_odcs/LICENSE +204 -0
- kiri_odcs/__init__.py +234 -0
- kiri_odcs/ai_trust_tier.py +475 -0
- kiri_odcs/canonical.py +263 -0
- kiri_odcs/diff.py +63 -0
- kiri_odcs/errors.py +51 -0
- kiri_odcs/invariant_sql.py +136 -0
- kiri_odcs/kiri_bridge.py +71 -0
- kiri_odcs/kiri_ext.py +3166 -0
- kiri_odcs/kiri_fk.py +447 -0
- kiri_odcs/kiri_pit.py +121 -0
- kiri_odcs/kiri_ref.py +57 -0
- kiri_odcs/openlineage_adapter.py +105 -0
- kiri_odcs/parser.py +1213 -0
- kiri_odcs/pii_guardrails.py +159 -0
- kiri_odcs/py.typed +0 -0
- kiri_odcs/schema_export.py +223 -0
- kiri_openlineage/__init__.py +58 -0
- kiri_openlineage/builder.py +146 -0
- kiri_openlineage/cost.py +190 -0
- kiri_openlineage/emitter.py +165 -0
- kiri_openlineage/py.typed +0 -0
- kiri_openlineage/schema.py +121 -0
- kiri_packs/__init__.py +67 -0
- kiri_packs/installer.py +500 -0
- kiri_packs/manifest.py +134 -0
- kiri_packs/publisher.py +605 -0
- kiri_packs/py.typed +0 -0
- kiri_packs/validator.py +295 -0
- kiri_quality/__init__.py +108 -0
- kiri_quality/ai_audit_correlator.py +293 -0
- kiri_quality/ai_audit_correlator_apply.py +526 -0
- kiri_quality/conformance_evidence.py +272 -0
- kiri_quality/dbt_test_collector.py +214 -0
- kiri_quality/dq_codegen.py +300 -0
- kiri_quality/dv_checks.py +393 -0
- kiri_quality/dv_column_lineage.py +279 -0
- kiri_quality/dv_column_lineage_apply.py +856 -0
- kiri_quality/dv_conformance.py +841 -0
- kiri_quality/dv_fixture_runner.py +478 -0
- kiri_quality/dv_invariants.py +986 -0
- kiri_quality/dv_kiri.py +97 -0
- kiri_quality/dv_markdown.py +169 -0
- kiri_quality/dv_observed_uc_reader.py +187 -0
- kiri_quality/dv_runner.py +389 -0
- kiri_quality/evidence_status.py +337 -0
- kiri_quality/fk_checks.py +800 -0
- kiri_quality/fk_ddl.py +402 -0
- kiri_quality/fk_pit_contract.py +168 -0
- kiri_quality/fk_runner.py +950 -0
- kiri_quality/kernel.py +376 -0
- kiri_quality/pii.py +60 -0
- kiri_quality/pre_classify/__init__.py +55 -0
- kiri_quality/pre_classify/engine.py +346 -0
- kiri_quality/pre_classify/locale_packs/en.yaml +117 -0
- kiri_quality/pre_classify/locale_packs/sv.yaml +73 -0
- kiri_quality/pre_classify/merge.py +297 -0
- kiri_quality/pre_classify/model.py +163 -0
- kiri_quality/py.typed +0 -0
- kiri_quality/reject_lane.py +261 -0
- kiri_quality/rule_runner.py +565 -0
- kiri_quality/schema_evolution_check.py +88 -0
- kiri_quality/silver_quality.py +494 -0
- kiri_quality/silver_quality_markdown.py +187 -0
- kiri_quality/uc_security_compile.py +951 -0
- kiri_quality/uc_security_compile_apply.py +1186 -0
- kiri_rbac/__init__.py +87 -0
- kiri_rbac/access.py +371 -0
- kiri_rbac/loader.py +104 -0
- kiri_rbac/pr_gate.py +213 -0
- kiri_rbac/providers/__init__.py +29 -0
- kiri_rbac/providers/_jwt.py +121 -0
- kiri_rbac/providers/azure_ad.py +93 -0
- kiri_rbac/providers/base.py +107 -0
- kiri_rbac/providers/factory.py +48 -0
- kiri_rbac/providers/github.py +134 -0
- kiri_rbac/providers/oidc_generic.py +99 -0
- kiri_rbac/py.typed +0 -0
- kiri_rbac/resolver.py +118 -0
- kiri_rbac/schema.py +219 -0
- kiri_schema_registry/__init__.py +22 -0
- kiri_schema_registry/avro.py +127 -0
- kiri_schema_registry/base.py +71 -0
- kiri_schema_registry/confluent.py +184 -0
- kiri_schema_registry/glue.py +178 -0
- kiri_schema_registry/py.typed +0 -0
- kiri_slack_bot/__init__.py +24 -0
- kiri_slack_bot/handler.py +248 -0
- kiri_slack_bot/py.typed +0 -0
- kiri_slack_bot/queries.py +90 -0
- kiri_slack_bot/signature.py +94 -0
- kiri_vault/__init__.py +59 -0
- kiri_vault/audit.py +64 -0
- kiri_vault/aws_vault.py +123 -0
- kiri_vault/azure_vault.py +236 -0
- kiri_vault/boundaries.py +219 -0
- kiri_vault/client.py +54 -0
- kiri_vault/databricks_vault.py +292 -0
- kiri_vault/env_vault.py +77 -0
- kiri_vault/py.typed +0 -0
- kiri_vault/ref.py +84 -0
- kirimana_automation/__init__.py +0 -0
- kirimana_automation/business_calendar.py +179 -0
- kirimana_automation/dag_snapshot.py +252 -0
- kirimana_automation/db/__init__.py +5 -0
- kirimana_automation/db/migrate.py +571 -0
- kirimana_automation/db/migrations/0001_release_state.down.duckdb.sql +29 -0
- kirimana_automation/db/migrations/0001_release_state.down.postgres.sql +37 -0
- kirimana_automation/db/migrations/0001_release_state.duckdb.sql +170 -0
- kirimana_automation/db/migrations/0001_release_state.postgres.sql +176 -0
- kirimana_automation/db/migrations/0002_rate_limit_buckets.down.duckdb.sql +9 -0
- kirimana_automation/db/migrations/0002_rate_limit_buckets.down.postgres.sql +9 -0
- kirimana_automation/db/migrations/0002_rate_limit_buckets.duckdb.sql +14 -0
- kirimana_automation/db/migrations/0002_rate_limit_buckets.postgres.sql +18 -0
- kirimana_automation/db/migrations/0003_flow_runs.down.duckdb.sql +18 -0
- kirimana_automation/db/migrations/0003_flow_runs.down.postgres.sql +19 -0
- kirimana_automation/db/migrations/0003_flow_runs.duckdb.sql +85 -0
- kirimana_automation/db/migrations/0003_flow_runs.postgres.sql +102 -0
- kirimana_automation/db/migrations/0004_flow_dag_snapshots.down.duckdb.sql +11 -0
- kirimana_automation/db/migrations/0004_flow_dag_snapshots.down.postgres.sql +11 -0
- kirimana_automation/db/migrations/0004_flow_dag_snapshots.duckdb.sql +15 -0
- kirimana_automation/db/migrations/0004_flow_dag_snapshots.postgres.sql +31 -0
- kirimana_automation/db/migrations/0005_calendar_skip.down.duckdb.sql +109 -0
- kirimana_automation/db/migrations/0005_calendar_skip.down.postgres.sql +42 -0
- kirimana_automation/db/migrations/0005_calendar_skip.duckdb.sql +113 -0
- kirimana_automation/db/migrations/0005_calendar_skip.postgres.sql +42 -0
- kirimana_automation/db/migrations/0006_flow_apply_log.down.duckdb.sql +9 -0
- kirimana_automation/db/migrations/0006_flow_apply_log.down.postgres.sql +11 -0
- kirimana_automation/db/migrations/0006_flow_apply_log.duckdb.sql +45 -0
- kirimana_automation/db/migrations/0006_flow_apply_log.postgres.sql +49 -0
- kirimana_automation/db/migrations/0007_flow_task_error_redacted.down.duckdb.sql +14 -0
- kirimana_automation/db/migrations/0007_flow_task_error_redacted.down.postgres.sql +2 -0
- kirimana_automation/db/migrations/0007_flow_task_error_redacted.duckdb.sql +7 -0
- kirimana_automation/db/migrations/0007_flow_task_error_redacted.postgres.sql +11 -0
- kirimana_automation/db/migrations/README.md +84 -0
- kirimana_automation/flow_loader.py +220 -0
- kirimana_automation/freeze.py +133 -0
- kirimana_automation/kiri_recommender.py +313 -0
- kirimana_automation/main.py +195 -0
- kirimana_automation/py.typed +0 -0
- kirimana_automation/run_history.py +593 -0
- kirimana_automation/run_history_analysis.py +607 -0
- kirimana_automation/schedule.py +343 -0
- kirimana_automation/schedule_validator.py +572 -0
- kirimana_automation/workflow_compiler.py +1288 -0
|
@@ -0,0 +1,451 @@
|
|
|
1
|
+
"""Iceberg-direct platform adapter — ADR 0034 / #84.
|
|
2
|
+
|
|
3
|
+
Direct-write peer to TrinoPlatformAdapter. Where Trino-mediated
|
|
4
|
+
goes ``kiri apply → Trino SQL → iceberg connector → Polaris → ADLS``,
|
|
5
|
+
this adapter goes ``kiri apply → pyiceberg → Polaris → ADLS`` —
|
|
6
|
+
no SQL engine in the critical path for ingest workloads.
|
|
7
|
+
|
|
8
|
+
Key design choices (ADR 0034):
|
|
9
|
+
|
|
10
|
+
- ``dialect()`` delegates to a configurable ``query_dialect`` from
|
|
11
|
+
the profile (default TrinoDialect). Iceberg itself isn't a
|
|
12
|
+
query engine; the dialect names whoever WILL execute SQL when
|
|
13
|
+
silver materialisation runs.
|
|
14
|
+
- ``execute(sql)`` is best-effort: if ``query_endpoint`` is set in
|
|
15
|
+
the profile we pass through to a configured Trino, otherwise
|
|
16
|
+
we surface a clear "this adapter doesn't query" error so the
|
|
17
|
+
caller picks the right surface.
|
|
18
|
+
- ``ingest_bronze`` uses pyiceberg's catalog API + PyArrow to
|
|
19
|
+
register parquet files Airbyte/dlt landed.
|
|
20
|
+
- ``claim_target`` / ``log_apply`` / ``read_manifest`` write to
|
|
21
|
+
Iceberg tables in the reserved ``_kiri_meta`` namespace — same
|
|
22
|
+
pattern as DuckDB / Databricks / Trino but using catalog API
|
|
23
|
+
calls instead of SQL.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
from __future__ import annotations
|
|
27
|
+
|
|
28
|
+
import contextlib
|
|
29
|
+
import time
|
|
30
|
+
from typing import Any
|
|
31
|
+
|
|
32
|
+
from kiri_core.schema import IngestionMode, Source
|
|
33
|
+
from kiri_core.types import HealthStatus, RunStatus
|
|
34
|
+
|
|
35
|
+
from kiri_adapters.base import (
|
|
36
|
+
ApplyLogEntry,
|
|
37
|
+
ColumnDescription,
|
|
38
|
+
Credential,
|
|
39
|
+
IngestionPlan,
|
|
40
|
+
PlatformAdapter,
|
|
41
|
+
ProjectManifest,
|
|
42
|
+
QueryResult,
|
|
43
|
+
RetentionAction,
|
|
44
|
+
RetentionReport,
|
|
45
|
+
RunResult,
|
|
46
|
+
TableDescription,
|
|
47
|
+
TableRef,
|
|
48
|
+
)
|
|
49
|
+
from kiri_adapters.sql_dialect import (
|
|
50
|
+
DuckDbDialect,
|
|
51
|
+
SqlDialect,
|
|
52
|
+
TrinoDialect,
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
# Reserved namespace mirrors the convention from the other adapters.
|
|
56
|
+
_META_NAMESPACE = "_kiri_meta"
|
|
57
|
+
_MANIFEST_TABLE = "_kiri_project_manifest"
|
|
58
|
+
_APPLY_LOG_TABLE = "_kiri_apply_log"
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _load_iceberg_client_or_raise() -> Any:
|
|
62
|
+
"""Lazy-load the kiri-iceberg-catalog package.
|
|
63
|
+
|
|
64
|
+
Adapter consumers that don't use Iceberg shouldn't pay the
|
|
65
|
+
pyiceberg install cost; the import lives behind a runtime
|
|
66
|
+
check that surfaces a helpful install message.
|
|
67
|
+
"""
|
|
68
|
+
try:
|
|
69
|
+
from kiri_iceberg import (
|
|
70
|
+
KirimanaIcebergError,
|
|
71
|
+
PolarisProfile,
|
|
72
|
+
polaris_catalog,
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
return PolarisProfile, polaris_catalog, KirimanaIcebergError
|
|
76
|
+
except ImportError as e:
|
|
77
|
+
raise ImportError(
|
|
78
|
+
"kiri-adapters[iceberg] is not installed. Run "
|
|
79
|
+
"`uv sync --all-packages --extra iceberg` (or "
|
|
80
|
+
"`pip install 'kiri-iceberg-catalog[pyiceberg]'`) before "
|
|
81
|
+
"using the Iceberg-direct adapter."
|
|
82
|
+
) from e
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _dialect_for(name: str) -> SqlDialect:
|
|
86
|
+
"""Return a SqlDialect instance for the configured query engine.
|
|
87
|
+
|
|
88
|
+
Iceberg-direct doesn't render SQL itself — silver materialisation
|
|
89
|
+
happens through whichever engine the operator pairs the adapter
|
|
90
|
+
with. Per ADR 0034, default is Trino. Spark + DuckDB hooks land
|
|
91
|
+
when those engines are first-class targets; until then we raise
|
|
92
|
+
NotImplementedError so the failure mode is explicit.
|
|
93
|
+
"""
|
|
94
|
+
if name == "trino":
|
|
95
|
+
return TrinoDialect()
|
|
96
|
+
if name == "duckdb":
|
|
97
|
+
return DuckDbDialect()
|
|
98
|
+
if name == "spark":
|
|
99
|
+
raise NotImplementedError(
|
|
100
|
+
"query_dialect='spark' isn't wired yet. Use 'trino' for now; "
|
|
101
|
+
"Spark dialect lands when a customer asks."
|
|
102
|
+
)
|
|
103
|
+
raise ValueError(f"unknown query_dialect: {name!r}")
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
class IcebergPlatformAdapter(PlatformAdapter):
|
|
107
|
+
"""Direct-write Iceberg adapter (ADR 0034 / #84).
|
|
108
|
+
|
|
109
|
+
Profile shape:
|
|
110
|
+
|
|
111
|
+
```yaml
|
|
112
|
+
targets:
|
|
113
|
+
prod:
|
|
114
|
+
adapter: iceberg
|
|
115
|
+
catalog_uri: https://polaris.acme.com/api/catalog
|
|
116
|
+
warehouse: abfss://warehouse@acme.dfs.core.windows.net/
|
|
117
|
+
auth:
|
|
118
|
+
mode: oidc
|
|
119
|
+
client_id: ${vault:azure:iceberg-mi-client-id}
|
|
120
|
+
tenant_id: ${vault:azure:tenant}
|
|
121
|
+
query_dialect: trino # default; rendering target for silver SQL
|
|
122
|
+
query_endpoint: "" # optional Trino host for execute()
|
|
123
|
+
```
|
|
124
|
+
"""
|
|
125
|
+
|
|
126
|
+
name: str = "iceberg"
|
|
127
|
+
|
|
128
|
+
def __init__(self, profile: dict[str, Any]) -> None:
|
|
129
|
+
super().__init__(profile)
|
|
130
|
+
# Resolve client + profile model upfront so misconfigured
|
|
131
|
+
# deployments fail before the first actual catalog call.
|
|
132
|
+
polaris_profile_cls, polaris_catalog_fn, kirimana_err = _load_iceberg_client_or_raise()
|
|
133
|
+
self._PolarisProfile = polaris_profile_cls
|
|
134
|
+
self._polaris_catalog = polaris_catalog_fn
|
|
135
|
+
self._KirimanaIcebergError = kirimana_err
|
|
136
|
+
|
|
137
|
+
# Profile validation runs in pydantic.
|
|
138
|
+
self._polaris_profile = polaris_profile_cls(
|
|
139
|
+
catalog_uri=profile["catalog_uri"],
|
|
140
|
+
warehouse=profile["warehouse"],
|
|
141
|
+
auth=profile["auth"],
|
|
142
|
+
query_dialect=profile.get("query_dialect", "trino"),
|
|
143
|
+
)
|
|
144
|
+
self._query_endpoint: str | None = profile.get("query_endpoint")
|
|
145
|
+
self._catalog: Any | None = None # cached, opened lazily
|
|
146
|
+
|
|
147
|
+
# ---- credentials / health -------------------------------------------
|
|
148
|
+
|
|
149
|
+
def required_credentials(self) -> list[Credential]:
|
|
150
|
+
creds: list[Credential] = []
|
|
151
|
+
auth = self._polaris_profile.auth
|
|
152
|
+
if auth.mode == "static":
|
|
153
|
+
creds.append(
|
|
154
|
+
Credential(
|
|
155
|
+
env_var="POLARIS_TOKEN",
|
|
156
|
+
description=(
|
|
157
|
+
"Bearer token for the Polaris REST catalog. "
|
|
158
|
+
"Set via vault when running outside AKS."
|
|
159
|
+
),
|
|
160
|
+
)
|
|
161
|
+
)
|
|
162
|
+
elif auth.mode == "oidc":
|
|
163
|
+
creds.append(
|
|
164
|
+
Credential(
|
|
165
|
+
env_var="AZURE_FEDERATED_TOKEN_FILE",
|
|
166
|
+
description=(
|
|
167
|
+
"Workload Identity-projected token file. AKS "
|
|
168
|
+
"injects this automatically; outside AKS use "
|
|
169
|
+
"auth.mode='static' or 'oauth_client'."
|
|
170
|
+
),
|
|
171
|
+
required=False,
|
|
172
|
+
)
|
|
173
|
+
)
|
|
174
|
+
return creds
|
|
175
|
+
|
|
176
|
+
def health_check(self) -> HealthStatus:
|
|
177
|
+
"""Smoke-ping the catalog by listing namespaces."""
|
|
178
|
+
try:
|
|
179
|
+
cat = self._cat()
|
|
180
|
+
cat.list_namespaces()
|
|
181
|
+
except Exception:
|
|
182
|
+
return HealthStatus.UNHEALTHY
|
|
183
|
+
return HealthStatus.HEALTHY
|
|
184
|
+
|
|
185
|
+
def dialect(self) -> SqlDialect:
|
|
186
|
+
"""Delegate to the configured query engine's dialect (ADR 0034).
|
|
187
|
+
|
|
188
|
+
Iceberg-direct doesn't execute SQL itself. Silver-planner
|
|
189
|
+
rendered SQL targets whichever engine pairs with this
|
|
190
|
+
adapter (Trino by default). Operators querying through this
|
|
191
|
+
dialect render SQL for that engine + execute it elsewhere
|
|
192
|
+
(or via :meth:`execute` when ``query_endpoint`` is set).
|
|
193
|
+
"""
|
|
194
|
+
return _dialect_for(self._polaris_profile.query_dialect)
|
|
195
|
+
|
|
196
|
+
# ---- catalog access -------------------------------------------------
|
|
197
|
+
|
|
198
|
+
def _cat(self) -> Any:
|
|
199
|
+
"""Cached pyiceberg Catalog handle."""
|
|
200
|
+
if self._catalog is None:
|
|
201
|
+
self._catalog = self._polaris_catalog(self._polaris_profile)
|
|
202
|
+
return self._catalog
|
|
203
|
+
|
|
204
|
+
def _ensure_meta_namespace(self) -> None:
|
|
205
|
+
"""Create the ``_kiri_meta`` namespace if it doesn't exist.
|
|
206
|
+
|
|
207
|
+
Namespace-already-exists raises a pyiceberg exception whose
|
|
208
|
+
type varies by catalog implementation; suppressing every
|
|
209
|
+
Exception is acceptable because the operation is idempotent
|
|
210
|
+
+ a real connectivity failure surfaces on the next catalog
|
|
211
|
+
call (list_namespaces, create_table, etc.).
|
|
212
|
+
"""
|
|
213
|
+
cat = self._cat()
|
|
214
|
+
with contextlib.suppress(Exception):
|
|
215
|
+
cat.create_namespace(_META_NAMESPACE)
|
|
216
|
+
|
|
217
|
+
# ---- query / execute ------------------------------------------------
|
|
218
|
+
|
|
219
|
+
def query(self, sql: str, parameters: list[Any] | None = None) -> QueryResult:
|
|
220
|
+
"""Read-only query path.
|
|
221
|
+
|
|
222
|
+
Iceberg-direct doesn't run SQL. When ``query_endpoint`` is
|
|
223
|
+
configured we delegate to a Trino client; otherwise raise
|
|
224
|
+
a clear error so callers route the query elsewhere.
|
|
225
|
+
"""
|
|
226
|
+
if self._query_endpoint:
|
|
227
|
+
raise NotImplementedError(
|
|
228
|
+
"query_endpoint delegation lands when the trino-python-client "
|
|
229
|
+
"wrapper for read-only queries ships. Until then, query "
|
|
230
|
+
"Iceberg tables via the TrinoPlatformAdapter pointing at the "
|
|
231
|
+
"same Polaris catalog."
|
|
232
|
+
)
|
|
233
|
+
raise NotImplementedError(
|
|
234
|
+
"IcebergPlatformAdapter doesn't execute SQL (ADR 0034). Pair "
|
|
235
|
+
"with TrinoPlatformAdapter for the query path; this adapter "
|
|
236
|
+
"owns the write path."
|
|
237
|
+
)
|
|
238
|
+
|
|
239
|
+
def apply_retention(
|
|
240
|
+
self,
|
|
241
|
+
table_fqn: str,
|
|
242
|
+
days: int,
|
|
243
|
+
*,
|
|
244
|
+
dry_run: bool = False,
|
|
245
|
+
) -> RetentionReport:
|
|
246
|
+
"""Iceberg-direct retention dispatch (ADR 0034 + ADR 0114 §4).
|
|
247
|
+
|
|
248
|
+
Calls the platform system procedures ``expire_snapshots`` and
|
|
249
|
+
``remove_orphan_files`` against ``table_fqn`` with an
|
|
250
|
+
``older_than`` cutoff of ``now() - <days> day``. Execution
|
|
251
|
+
routes through the paired query engine — same delegation rule
|
|
252
|
+
as :meth:`execute`. When no query engine is configured the
|
|
253
|
+
adapter returns a ``NOT_IMPLEMENTED`` report listing the
|
|
254
|
+
commands so the operator can run them manually rather than
|
|
255
|
+
getting a silent skip.
|
|
256
|
+
"""
|
|
257
|
+
if days <= 0:
|
|
258
|
+
raise ValueError(f"retention days must be positive, got {days!r}")
|
|
259
|
+
cutoff = f"current_timestamp - interval '{days}' day"
|
|
260
|
+
expire_stmt = (
|
|
261
|
+
f"CALL system.expire_snapshots(table => '{table_fqn}', older_than => {cutoff})"
|
|
262
|
+
)
|
|
263
|
+
orphan_stmt = (
|
|
264
|
+
f"CALL system.remove_orphan_files(table => '{table_fqn}', older_than => {cutoff})"
|
|
265
|
+
)
|
|
266
|
+
commands = [expire_stmt, orphan_stmt]
|
|
267
|
+
if dry_run:
|
|
268
|
+
return RetentionReport(
|
|
269
|
+
action=RetentionAction.PREVIEWED,
|
|
270
|
+
mechanism="expire_snapshots",
|
|
271
|
+
table_fqn=table_fqn,
|
|
272
|
+
days=days,
|
|
273
|
+
commands=commands,
|
|
274
|
+
message=(
|
|
275
|
+
f"would call expire_snapshots + remove_orphan_files "
|
|
276
|
+
f"on {table_fqn} (older_than={days}d)"
|
|
277
|
+
),
|
|
278
|
+
)
|
|
279
|
+
# Iceberg-direct has no SQL executor of its own — the procedures
|
|
280
|
+
# live on the paired Trino/Spark engine. Surface NOT_IMPLEMENTED
|
|
281
|
+
# so the operator-visible summary makes the gap explicit; the
|
|
282
|
+
# ``commands`` list is the actionable next step.
|
|
283
|
+
return RetentionReport(
|
|
284
|
+
action=RetentionAction.NOT_IMPLEMENTED,
|
|
285
|
+
mechanism="expire_snapshots",
|
|
286
|
+
table_fqn=table_fqn,
|
|
287
|
+
days=days,
|
|
288
|
+
commands=commands,
|
|
289
|
+
message=(
|
|
290
|
+
"Iceberg-direct retention requires a paired query "
|
|
291
|
+
"engine (Trino/Spark). Run the listed commands there "
|
|
292
|
+
"or wire a query_endpoint on this adapter."
|
|
293
|
+
),
|
|
294
|
+
)
|
|
295
|
+
|
|
296
|
+
def execute(self, sql: str) -> RunResult:
|
|
297
|
+
"""Write SQL — same delegation rule as query.
|
|
298
|
+
|
|
299
|
+
Schema management (CREATE/ALTER/DROP) goes through pyiceberg's
|
|
300
|
+
catalog API, not raw SQL. This method exists for ABC
|
|
301
|
+
compliance + future Trino-delegation.
|
|
302
|
+
"""
|
|
303
|
+
return RunResult(
|
|
304
|
+
status=RunStatus.SKIPPED,
|
|
305
|
+
message=(
|
|
306
|
+
"IcebergPlatformAdapter doesn't execute SQL (ADR 0034). "
|
|
307
|
+
"Schema ops route through pyiceberg's catalog API; "
|
|
308
|
+
"DML routes through a paired query engine (Trino)."
|
|
309
|
+
),
|
|
310
|
+
)
|
|
311
|
+
|
|
312
|
+
# ---- schema + ingestion --------------------------------------------
|
|
313
|
+
|
|
314
|
+
def create_schema(self, schema_name: str) -> RunResult:
|
|
315
|
+
"""Create an Iceberg namespace (the catalog's notion of schema)."""
|
|
316
|
+
start = time.monotonic()
|
|
317
|
+
cat = self._cat()
|
|
318
|
+
try:
|
|
319
|
+
try:
|
|
320
|
+
cat.create_namespace(schema_name)
|
|
321
|
+
except Exception:
|
|
322
|
+
# Idempotent: namespace already exists. pyiceberg
|
|
323
|
+
# NamespaceAlreadyExistsError lands here, but the
|
|
324
|
+
# exception type varies — rely on listing to verify.
|
|
325
|
+
if not any(
|
|
326
|
+
ns == (schema_name,) or ns == schema_name for ns in cat.list_namespaces()
|
|
327
|
+
):
|
|
328
|
+
raise
|
|
329
|
+
except Exception as e:
|
|
330
|
+
return RunResult(
|
|
331
|
+
status=RunStatus.FAILED,
|
|
332
|
+
message=f"create_namespace({schema_name!r}) failed: {e}",
|
|
333
|
+
duration_ms=int((time.monotonic() - start) * 1000),
|
|
334
|
+
)
|
|
335
|
+
return RunResult(
|
|
336
|
+
status=RunStatus.SUCCESS,
|
|
337
|
+
message=f"namespace {schema_name} ready",
|
|
338
|
+
duration_ms=int((time.monotonic() - start) * 1000),
|
|
339
|
+
)
|
|
340
|
+
|
|
341
|
+
def ingest_bronze(self, source: Source, plan: IngestionPlan) -> RunResult:
|
|
342
|
+
"""Register Airbyte/dlt-landed parquet as an Iceberg table.
|
|
343
|
+
|
|
344
|
+
ADR 0034: this is the primary path. Parquet files at the
|
|
345
|
+
warehouse path get registered via pyiceberg's catalog. For
|
|
346
|
+
FIXTURE_FILE / DATABASE / REST_API ingest modes, return
|
|
347
|
+
SKIPPED with the same message Trino's adapter uses — those
|
|
348
|
+
modes go through Airbyte/dlt first which lands the parquet
|
|
349
|
+
we then register.
|
|
350
|
+
"""
|
|
351
|
+
ingestion = plan.effective_ingestion()
|
|
352
|
+
mode = ingestion.mode if ingestion else IngestionMode.FIXTURE_FILE
|
|
353
|
+
if mode in {
|
|
354
|
+
IngestionMode.LANDING_ZONE,
|
|
355
|
+
IngestionMode.AIRBYTE,
|
|
356
|
+
}:
|
|
357
|
+
# Real implementation: scan the warehouse path for the
|
|
358
|
+
# source's landing dir, register each parquet via
|
|
359
|
+
# ``cat.register_table(...)`` or use ``Table.append`` for
|
|
360
|
+
# incremental. Out of MVP scope — surface a clear "not
|
|
361
|
+
# implemented yet" so an operator running this adapter
|
|
362
|
+
# against a live Polaris doesn't get silent failure.
|
|
363
|
+
return RunResult(
|
|
364
|
+
status=RunStatus.SKIPPED,
|
|
365
|
+
message=(
|
|
366
|
+
f"Iceberg-direct ingest_bronze for mode={mode.value!r} "
|
|
367
|
+
"is the next iteration. Configure Airbyte to land in the "
|
|
368
|
+
"warehouse path; v1 of this adapter focuses on catalog + "
|
|
369
|
+
"manifest ops."
|
|
370
|
+
),
|
|
371
|
+
)
|
|
372
|
+
return RunResult(
|
|
373
|
+
status=RunStatus.SKIPPED,
|
|
374
|
+
message=(
|
|
375
|
+
f"ingestion mode {mode.value!r} routes through Airbyte/dlt "
|
|
376
|
+
"first. Iceberg-direct registers the resulting parquet."
|
|
377
|
+
),
|
|
378
|
+
)
|
|
379
|
+
|
|
380
|
+
def describe_table(self, ref: TableRef) -> TableDescription:
|
|
381
|
+
"""Read pyiceberg's schema for the table."""
|
|
382
|
+
cat = self._cat()
|
|
383
|
+
full_name = (ref.schema_name, ref.table_name)
|
|
384
|
+
try:
|
|
385
|
+
table = cat.load_table(full_name)
|
|
386
|
+
except Exception as e:
|
|
387
|
+
raise self._KirimanaIcebergError(f"Could not load {ref.fully_qualified()}: {e}") from e
|
|
388
|
+
columns = [
|
|
389
|
+
ColumnDescription(
|
|
390
|
+
name=field.name,
|
|
391
|
+
data_type=str(field.field_type),
|
|
392
|
+
nullable=not field.required,
|
|
393
|
+
)
|
|
394
|
+
for field in table.schema().fields
|
|
395
|
+
]
|
|
396
|
+
return TableDescription(ref=ref, row_count=None, columns=columns)
|
|
397
|
+
|
|
398
|
+
# ---- project-manifest + apply-log ----------------------------------
|
|
399
|
+
|
|
400
|
+
def claim_target(self, manifest: ProjectManifest) -> None:
|
|
401
|
+
"""Idempotently claim the catalog for ``(project, env)``.
|
|
402
|
+
|
|
403
|
+
Same semantics as the Trino + Databricks + DuckDB adapters
|
|
404
|
+
(ADR 0014 + 0017): row-per-(project, env), cross-project
|
|
405
|
+
collision raises, same-slot reclaim refreshes release fields
|
|
406
|
+
when they're set.
|
|
407
|
+
|
|
408
|
+
Implementation is metadata-only: read/write the
|
|
409
|
+
``_kiri_meta._kiri_project_manifest`` Iceberg table via
|
|
410
|
+
pyiceberg's PyArrow integration. Out of MVP scope — surface
|
|
411
|
+
a clear "schema management lands next iteration" so the
|
|
412
|
+
flow is obvious. The Trino path covers manifest ops today
|
|
413
|
+
for AKS deploys that include both adapters.
|
|
414
|
+
"""
|
|
415
|
+
self._ensure_meta_namespace()
|
|
416
|
+
# Real impl: cat.create_table_if_not_exists(...) + PyArrow
|
|
417
|
+
# append. Mark as TODO with explicit error so an operator
|
|
418
|
+
# running with no Trino in the deploy hits a clear message.
|
|
419
|
+
raise NotImplementedError(
|
|
420
|
+
"Iceberg-direct claim_target lands in the next slice. For "
|
|
421
|
+
"v1 deploys, the Trino adapter handles manifest writes; "
|
|
422
|
+
"the Iceberg adapter is for ingest registration only."
|
|
423
|
+
)
|
|
424
|
+
|
|
425
|
+
def read_manifest(self, project_name: str, environment: str) -> ProjectManifest | None:
|
|
426
|
+
"""Return the manifest row, or None when the table doesn't exist."""
|
|
427
|
+
# Same pattern as claim_target — PyArrow read of the
|
|
428
|
+
# _kiri_meta._kiri_project_manifest Iceberg table. Out of
|
|
429
|
+
# v1 scope; falls through to None so kiri release status
|
|
430
|
+
# against an Iceberg-only target reports "not applied".
|
|
431
|
+
return None
|
|
432
|
+
|
|
433
|
+
def log_apply(self, entry: ApplyLogEntry) -> None:
|
|
434
|
+
"""Append one apply-log row to the Iceberg-backed log table.
|
|
435
|
+
|
|
436
|
+
Same shape as DuckDB / Databricks / Trino. Iceberg's
|
|
437
|
+
append-only protocol is a natural fit; impl lands with
|
|
438
|
+
claim_target since they share the meta-namespace plumbing.
|
|
439
|
+
"""
|
|
440
|
+
# No-op for v1: Trino adapter handles this when paired.
|
|
441
|
+
return
|
|
442
|
+
|
|
443
|
+
def supports(self, capability: str) -> bool:
|
|
444
|
+
return capability in {
|
|
445
|
+
"iceberg",
|
|
446
|
+
"polaris-catalog",
|
|
447
|
+
"abfss-warehouse",
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
|
|
451
|
+
__all__ = ["IcebergPlatformAdapter"]
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# Adapter capability manifest — ADR 0067 §3.
|
|
2
|
+
#
|
|
3
|
+
# Declares which protocol surface this adapter implements, at which
|
|
4
|
+
# version, and which optional methods are wired. Loaders consume this
|
|
5
|
+
# at adapter-discovery time. Schema lives in
|
|
6
|
+
# packages/adapters/kiri_adapters/manifest.py (AdapterManifest).
|
|
7
|
+
#
|
|
8
|
+
# When you change capabilities here, also update the in-tree
|
|
9
|
+
# conformance run + bump conformance_suite_passed if applicable.
|
|
10
|
+
|
|
11
|
+
name: "localduckdb"
|
|
12
|
+
implementing: "PlatformAdapter"
|
|
13
|
+
surface_version: "1.6.0"
|
|
14
|
+
package_version: "0.4.2"
|
|
15
|
+
dialect: "duckdb"
|
|
16
|
+
|
|
17
|
+
# Optional method declarations against PlatformAdapter v1.4.0.
|
|
18
|
+
# This list is intentionally empty for now — the surface's optional-
|
|
19
|
+
# method catalogue is finalised when the conformance suite (ADR 0067
|
|
20
|
+
# step 5) ships. Until then, in-tree adapters declare neither
|
|
21
|
+
# implemented nor skipped optional methods, and the loader treats the
|
|
22
|
+
# absence as "no opt-in claims yet".
|
|
23
|
+
optional_methods_implemented: []
|
|
24
|
+
optional_methods_skipped: []
|
|
25
|
+
|
|
26
|
+
# MUST stay empty for stable surfaces. A non-empty value would mean
|
|
27
|
+
# the adapter claims compliance via ad-hoc shims, which the manifest
|
|
28
|
+
# validator rejects.
|
|
29
|
+
required_methods_via_workaround: []
|
|
30
|
+
|
|
31
|
+
# Surface-specific capability flags consumed by planner / runtime.
|
|
32
|
+
capability_flags:
|
|
33
|
+
# Landing-zone storage primitive (ADR 0046).
|
|
34
|
+
# Implementation: LocalLandingStorage rooted at <bronze_root>.
|
|
35
|
+
landing_storage_supported: true
|
|
36
|
+
# AI-policy-aware execution (ADR 0005 + 0059) is enforced upstream
|
|
37
|
+
# in the AI gateway today; the platform adapter does not yet
|
|
38
|
+
# participate in policy decisions.
|
|
39
|
+
ai_policy_aware: false
|
|
40
|
+
# Column-level lineage stitching (ADR 0045).
|
|
41
|
+
# Implementation pending — the silver planners emit lineage URNs but
|
|
42
|
+
# the adapter does not enrich them with platform-specific data.
|
|
43
|
+
column_lineage_capable: false
|