kiri-cli 1.0.0b3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of kiri-cli might be problematic. Click here for more details.
- kiri_adapters/__init__.py +193 -0
- kiri_adapters/backends/__init__.py +0 -0
- kiri_adapters/backends/databricks.manifest.yml +55 -0
- kiri_adapters/backends/databricks.py +2980 -0
- kiri_adapters/backends/fabric_dw.manifest.yml +17 -0
- kiri_adapters/backends/fabric_dw.py +307 -0
- kiri_adapters/backends/fabric_lakehouse.manifest.yml +19 -0
- kiri_adapters/backends/fabric_lakehouse.py +365 -0
- kiri_adapters/backends/iceberg.manifest.yml +17 -0
- kiri_adapters/backends/iceberg.py +451 -0
- kiri_adapters/backends/localduckdb.manifest.yml +43 -0
- kiri_adapters/backends/localduckdb.py +1295 -0
- kiri_adapters/backends/mssql.manifest.yml +17 -0
- kiri_adapters/backends/mssql.py +366 -0
- kiri_adapters/backends/postgres.manifest.yml +17 -0
- kiri_adapters/backends/postgres.py +434 -0
- kiri_adapters/backends/trino.manifest.yml +17 -0
- kiri_adapters/backends/trino.py +542 -0
- kiri_adapters/base.py +892 -0
- kiri_adapters/doctor_checks.py +81 -0
- kiri_adapters/dv_planner.py +1205 -0
- kiri_adapters/dv_planner_bridge.py +265 -0
- kiri_adapters/dv_planner_pit.py +265 -0
- kiri_adapters/dv_planner_ref.py +403 -0
- kiri_adapters/dv_planner_sat_variants.py +668 -0
- kiri_adapters/framework.py +112 -0
- kiri_adapters/incident/__init__.py +30 -0
- kiri_adapters/incident/_http.py +121 -0
- kiri_adapters/incident/base.py +89 -0
- kiri_adapters/incident/jira.py +365 -0
- kiri_adapters/incident/servicenow.py +314 -0
- kiri_adapters/incident/zendesk.py +285 -0
- kiri_adapters/manifest.py +466 -0
- kiri_adapters/orchestration/__init__.py +30 -0
- kiri_adapters/orchestration/airflow.py +117 -0
- kiri_adapters/orchestration/artifact_emitter.py +99 -0
- kiri_adapters/orchestration/autoloader_emitter.py +141 -0
- kiri_adapters/orchestration/azure_data_factory.py +137 -0
- kiri_adapters/orchestration/base.py +626 -0
- kiri_adapters/orchestration/databricks_workflows.py +1768 -0
- kiri_adapters/orchestration/k8s_cron.py +555 -0
- kiri_adapters/orchestration/lifecycle.py +222 -0
- kiri_adapters/orchestration/pipeline_compat.py +145 -0
- kiri_adapters/orchestration/runtime_python.py +188 -0
- kiri_adapters/paths.py +38 -0
- kiri_adapters/protocols/__init__.py +64 -0
- kiri_adapters/protocols/ai_service/__init__.py +7 -0
- kiri_adapters/protocols/ai_service/conformance.py +120 -0
- kiri_adapters/protocols/base.py +220 -0
- kiri_adapters/protocols/catalog/__init__.py +7 -0
- kiri_adapters/protocols/catalog/conformance.py +138 -0
- kiri_adapters/protocols/framework/__init__.py +7 -0
- kiri_adapters/protocols/framework/conformance.py +98 -0
- kiri_adapters/protocols/incident/__init__.py +7 -0
- kiri_adapters/protocols/incident/conformance.py +64 -0
- kiri_adapters/protocols/orchestration/__init__.py +7 -0
- kiri_adapters/protocols/orchestration/conformance.py +84 -0
- kiri_adapters/protocols/platform/__init__.py +7 -0
- kiri_adapters/protocols/platform/conformance.py +120 -0
- kiri_adapters/protocols/sql_dialect/__init__.py +7 -0
- kiri_adapters/protocols/sql_dialect/conformance.py +91 -0
- kiri_adapters/protocols/vault/__init__.py +7 -0
- kiri_adapters/protocols/vault/conformance.py +65 -0
- kiri_adapters/py.typed +0 -0
- kiri_adapters/registry.py +104 -0
- kiri_adapters/semantic/__init__.py +60 -0
- kiri_adapters/semantic/apply.py +274 -0
- kiri_adapters/semantic/base.py +530 -0
- kiri_adapters/semantic/cube.py +280 -0
- kiri_adapters/semantic/dbt_semantic.py +92 -0
- kiri_adapters/semantic/metric_views.py +249 -0
- kiri_adapters/semantic/metricflow.py +345 -0
- kiri_adapters/semantic/snowflake_views.py +75 -0
- kiri_adapters/sql_dialect.py +1592 -0
- kiri_adapters/tsql_to_spark.py +653 -0
- kiri_ai_gateway/__init__.py +147 -0
- kiri_ai_gateway/audit.py +334 -0
- kiri_ai_gateway/batch.py +293 -0
- kiri_ai_gateway/config.py +165 -0
- kiri_ai_gateway/gateway.py +642 -0
- kiri_ai_gateway/placeholder_guard.py +145 -0
- kiri_ai_gateway/prices/__init__.py +77 -0
- kiri_ai_gateway/prices/prices.yaml +64 -0
- kiri_ai_gateway/prompts/__init__.py +54 -0
- kiri_ai_gateway/prompts/airbyte_source_setup.yaml +212 -0
- kiri_ai_gateway/prompts/bdv_translation.yaml +90 -0
- kiri_ai_gateway/prompts/contract_draft_mvp.yaml +117 -0
- kiri_ai_gateway/prompts/describe_table.yaml +19 -0
- kiri_ai_gateway/prompts/design_gold_from_question.yaml +130 -0
- kiri_ai_gateway/prompts/detect_pii.yaml +59 -0
- kiri_ai_gateway/prompts/discover_classify_column.yaml +69 -0
- kiri_ai_gateway/prompts/discover_describe_table.yaml +61 -0
- kiri_ai_gateway/prompts/dv_quality_summary.yaml +36 -0
- kiri_ai_gateway/prompts/incident_assist.yaml +58 -0
- kiri_ai_gateway/prompts/infer_source_schema.yaml +31 -0
- kiri_ai_gateway/prompts/ingest_backend_chooser.yaml +116 -0
- kiri_ai_gateway/prompts/interactive_source_edit.yaml +94 -0
- kiri_ai_gateway/prompts/map_needed_entity_to_source.yaml +72 -0
- kiri_ai_gateway/prompts/meet_kiri.yaml +127 -0
- kiri_ai_gateway/prompts/native_source_setup.yaml +211 -0
- kiri_ai_gateway/prompts/source_grouping_reason.yaml +29 -0
- kiri_ai_gateway/prompts/suggest_attribute_classification.yaml +42 -0
- kiri_ai_gateway/prompts/suggest_data_tests.yaml +71 -0
- kiri_ai_gateway/prompts/suggest_gold_dimension.yaml +108 -0
- kiri_ai_gateway/prompts/suggest_gold_fact.yaml +107 -0
- kiri_ai_gateway/prompts/suggest_gold_model.yaml +70 -0
- kiri_ai_gateway/prompts/suggest_semantic_intent.yaml +91 -0
- kiri_ai_gateway/prompts/suggest_silver_model.yaml +140 -0
- kiri_ai_gateway/providers/__init__.py +7 -0
- kiri_ai_gateway/providers/anthropic_provider.py +26 -0
- kiri_ai_gateway/providers/gateway_provider.py +188 -0
- kiri_ai_gateway/providers/local_provider.py +177 -0
- kiri_ai_gateway/py.typed +0 -0
- kiri_ai_gateway/service.py +283 -0
- kiri_ai_gateway/transport.py +103 -0
- kiri_ai_gateway/transports/__init__.py +12 -0
- kiri_ai_gateway/transports/anthropic_transport.py +73 -0
- kiri_ai_gateway/transports/ollama_transport.py +82 -0
- kiri_ai_gateway/transports/openai_compatible_transport.py +125 -0
- kiri_ai_gateway/workspace_policy.py +52 -0
- kiri_catalog/__init__.py +199 -0
- kiri_catalog/annotation_plan.py +183 -0
- kiri_catalog/appearances.py +155 -0
- kiri_catalog/backends/__init__.py +79 -0
- kiri_catalog/backends/base.py +167 -0
- kiri_catalog/backends/memory.py +241 -0
- kiri_catalog/backends/postgres.py +497 -0
- kiri_catalog/backends/redacting.py +195 -0
- kiri_catalog/backends/sqlite.py +480 -0
- kiri_catalog/context.py +94 -0
- kiri_catalog/federation/__init__.py +114 -0
- kiri_catalog/federation/api.py +200 -0
- kiri_catalog/federation/cache.py +149 -0
- kiri_catalog/federation/emit.py +209 -0
- kiri_catalog/federation/fs_resolver.py +295 -0
- kiri_catalog/federation/http_resolver.py +486 -0
- kiri_catalog/federation/metrics.py +123 -0
- kiri_catalog/federation/release.py +342 -0
- kiri_catalog/federation/resolver.py +332 -0
- kiri_catalog/federation/schema.py +460 -0
- kiri_catalog/importer.py +668 -0
- kiri_catalog/lineage_drift.py +280 -0
- kiri_catalog/lineage_graph.py +176 -0
- kiri_catalog/lineage_graph_explain.py +96 -0
- kiri_catalog/lineage_graph_mermaid.py +230 -0
- kiri_catalog/observed_uc_lineage.py +216 -0
- kiri_catalog/pii_detect.py +176 -0
- kiri_catalog/push.py +122 -0
- kiri_catalog/py.typed +0 -0
- kiri_catalog/redact.py +101 -0
- kiri_catalog/review_actions.py +119 -0
- kiri_catalog/schema/__init__.py +70 -0
- kiri_catalog/schema/asset.py +142 -0
- kiri_catalog/schema/column.py +247 -0
- kiri_catalog/schema/enums.py +195 -0
- kiri_catalog/schema/job.py +51 -0
- kiri_catalog/schema/lineage.py +33 -0
- kiri_catalog/schema/policy.py +69 -0
- kiri_catalog/schema/primitives.py +90 -0
- kiri_catalog/schema/review_state.py +111 -0
- kiri_catalog/schema/urn.py +45 -0
- kiri_catalog/schema_discovery.py +160 -0
- kiri_catalog/sinks/__init__.py +64 -0
- kiri_catalog/sinks/base.py +178 -0
- kiri_catalog/sinks/horizon.py +195 -0
- kiri_catalog/sinks/purview.py +216 -0
- kiri_catalog/sinks/unity.py +352 -0
- kiri_catalog/sinks/unity_executor.py +543 -0
- kiri_catalog/sources/__init__.py +42 -0
- kiri_catalog/sources/base.py +104 -0
- kiri_catalog/sources/unity.py +465 -0
- kiri_catalog/steward_writeback.py +158 -0
- kiri_catalog/tag_sql.py +99 -0
- kiri_cli/LICENSE +105 -0
- kiri_cli/__init__.py +1 -0
- kiri_cli/_apply_cli.py +63 -0
- kiri_cli/_apply_reporter.py +38 -0
- kiri_cli/_closed_menu.py +36 -0
- kiri_cli/_fixture_option.py +18 -0
- kiri_cli/_layer_detect.py +158 -0
- kiri_cli/_project_option.py +79 -0
- kiri_cli/adapter_cmd.py +192 -0
- kiri_cli/ai_batch_cmd.py +258 -0
- kiri_cli/ai_cmd.py +352 -0
- kiri_cli/approve_cmd.py +659 -0
- kiri_cli/attr_cmd.py +575 -0
- kiri_cli/audit_cmd.py +348 -0
- kiri_cli/audit_correlate_cmd.py +648 -0
- kiri_cli/audit_join.py +437 -0
- kiri_cli/audit_join_cmd.py +160 -0
- kiri_cli/auto_cmd.py +97 -0
- kiri_cli/azure_cmd.py +243 -0
- kiri_cli/catalog.py +1293 -0
- kiri_cli/cli_reference_live.py +147 -0
- kiri_cli/compile_cmd.py +493 -0
- kiri_cli/compliance_cmd.py +221 -0
- kiri_cli/contract_approval_cmd.py +94 -0
- kiri_cli/contract_cmd.py +64 -0
- kiri_cli/contract_codeowners_cmd.py +162 -0
- kiri_cli/contract_diff.py +111 -0
- kiri_cli/contract_emit.py +90 -0
- kiri_cli/contract_export.py +192 -0
- kiri_cli/contract_lint.py +1505 -0
- kiri_cli/contract_merge_cmd.py +264 -0
- kiri_cli/contract_new.py +326 -0
- kiri_cli/contract_promote.py +216 -0
- kiri_cli/contract_scaffold_cmd.py +134 -0
- kiri_cli/contract_schema_export.py +69 -0
- kiri_cli/contract_show.py +150 -0
- kiri_cli/contract_stubs.py +60 -0
- kiri_cli/contract_sync_from_live.py +327 -0
- kiri_cli/contract_validate.py +154 -0
- kiri_cli/contract_verify_live.py +242 -0
- kiri_cli/cost_cmd.py +243 -0
- kiri_cli/databricks_health.py +1254 -0
- kiri_cli/databricks_setup_cmd.py +712 -0
- kiri_cli/discover_cmd.py +600 -0
- kiri_cli/docs_cmd.py +35 -0
- kiri_cli/doctor_cmd.py +145 -0
- kiri_cli/dv_cmd.py +814 -0
- kiri_cli/dv_conformance_cmd.py +154 -0
- kiri_cli/dv_install.py +493 -0
- kiri_cli/dv_migrate_cmd.py +183 -0
- kiri_cli/dv_test_cmd.py +175 -0
- kiri_cli/federation_cmd.py +315 -0
- kiri_cli/fk_cmd.py +160 -0
- kiri_cli/flows_cmd.py +910 -0
- kiri_cli/governance_cmd.py +358 -0
- kiri_cli/incident_api_cmd.py +180 -0
- kiri_cli/incident_cmd.py +410 -0
- kiri_cli/incident_setup_cmd.py +239 -0
- kiri_cli/ingest.py +427 -0
- kiri_cli/ingest_discover.py +480 -0
- kiri_cli/ingest_new_source.py +386 -0
- kiri_cli/intent_cmd.py +122 -0
- kiri_cli/invariants_cmd.py +118 -0
- kiri_cli/kiri_banner.py +161 -0
- kiri_cli/kpi_cmd.py +222 -0
- kiri_cli/layer_cmd.py +464 -0
- kiri_cli/lineage.py +601 -0
- kiri_cli/main.py +2472 -0
- kiri_cli/maturity_cmd.py +222 -0
- kiri_cli/migrate_cmd.py +1347 -0
- kiri_cli/onboard_cmd.py +223 -0
- kiri_cli/pack_cmd.py +496 -0
- kiri_cli/pipeline_cmd.py +127 -0
- kiri_cli/pre_classify_cmd.py +274 -0
- kiri_cli/project_cmd.py +592 -0
- kiri_cli/py.typed +0 -0
- kiri_cli/quality_cmd.py +295 -0
- kiri_cli/rbac.py +139 -0
- kiri_cli/rbac_cmd.py +291 -0
- kiri_cli/release.py +850 -0
- kiri_cli/release_semver.py +265 -0
- kiri_cli/repo_cmd.py +155 -0
- kiri_cli/retention_cmd.py +333 -0
- kiri_cli/sample_data/__init__.py +12 -0
- kiri_cli/sample_data/customers.contract.yml +47 -0
- kiri_cli/sample_data/customers.csv +25 -0
- kiri_cli/scaffold_cmd.py +721 -0
- kiri_cli/schema_cmd.py +185 -0
- kiri_cli/semantic_cmd.py +605 -0
- kiri_cli/silver_cmd.py +348 -0
- kiri_cli/skills_cmd.py +243 -0
- kiri_cli/skills_data/README.md +51 -0
- kiri_cli/skills_data/add-source/SKILL.md +116 -0
- kiri_cli/skills_data/azure-deploy/SKILL.md +261 -0
- kiri_cli/skills_data/azure-status/SKILL.md +111 -0
- kiri_cli/skills_data/compliance-check/SKILL.md +99 -0
- kiri_cli/skills_data/databricks-build-pipeline/SKILL.md +112 -0
- kiri_cli/skills_data/databricks-project-setup/SKILL.md +157 -0
- kiri_cli/skills_data/debug-apply/SKILL.md +186 -0
- kiri_cli/skills_data/draft-gold-model/SKILL.md +203 -0
- kiri_cli/skills_data/explain-kpi/SKILL.md +158 -0
- kiri_cli/skills_data/incident-coach/SKILL.md +135 -0
- kiri_cli/skills_data/incident-setup/SKILL.md +112 -0
- kiri_cli/skills_data/maturity-coach/SKILL.md +120 -0
- kiri_cli/skills_data/promote-contract/SKILL.md +91 -0
- kiri_cli/skills_data/review-attributes/SKILL.md +94 -0
- kiri_cli/sla_cmd.py +769 -0
- kiri_cli/source_cmd.py +909 -0
- kiri_cli/templates/__init__.py +6 -0
- kiri_cli/templates/dv4dbt_packages.yml.j2 +11 -0
- kiri_cli/upgrade_cmd.py +896 -0
- kiri_cli/use_cmd.py +120 -0
- kiri_cli/vault_cmd.py +241 -0
- kiri_cli/verify_generated.py +241 -0
- kiri_cli-1.0.0b3.dist-info/METADATA +212 -0
- kiri_cli-1.0.0b3.dist-info/RECORD +743 -0
- kiri_cli-1.0.0b3.dist-info/WHEEL +4 -0
- kiri_cli-1.0.0b3.dist-info/entry_points.txt +4 -0
- kiri_compliance/__init__.py +33 -0
- kiri_compliance/audit_coverage.py +354 -0
- kiri_compliance/controls.py +763 -0
- kiri_compliance/evidence.py +155 -0
- kiri_compliance/py.typed +0 -0
- kiri_compliance/quality_engine.py +246 -0
- kiri_compliance/renderer.py +195 -0
- kiri_compliance/report.py +44 -0
- kiri_compliance/schema.py +129 -0
- kiri_core/__init__.py +185 -0
- kiri_core/adapter_surfaces.py +149 -0
- kiri_core/ai_query_gate/__init__.py +92 -0
- kiri_core/ai_query_gate/audit.py +153 -0
- kiri_core/ai_query_gate/evaluator.py +484 -0
- kiri_core/ai_query_gate/models.py +162 -0
- kiri_core/ai_query_gate/parser.py +436 -0
- kiri_core/ai_query_gate/resolver.py +274 -0
- kiri_core/ai_query_gate/store.py +234 -0
- kiri_core/ai_query_gate/store_postgres.py +409 -0
- kiri_core/api_keys.py +243 -0
- kiri_core/apply/__init__.py +27 -0
- kiri_core/apply/conformance.py +99 -0
- kiri_core/apply/incident_router.py +176 -0
- kiri_core/apply/models.py +140 -0
- kiri_core/apply/reporter.py +46 -0
- kiri_core/apply/runner.py +3083 -0
- kiri_core/apply_log_store.py +155 -0
- kiri_core/approval_flow/__init__.py +133 -0
- kiri_core/approval_flow/integration.py +156 -0
- kiri_core/approval_flow/models.py +443 -0
- kiri_core/approval_flow/notifier.py +263 -0
- kiri_core/approval_flow/orchestrator.py +660 -0
- kiri_core/approval_flow/reconcile_signoff.py +53 -0
- kiri_core/approval_flow/store.py +220 -0
- kiri_core/approval_flow/store_jsonl.py +217 -0
- kiri_core/approval_flow/store_postgres.py +580 -0
- kiri_core/approval_policy.py +116 -0
- kiri_core/asgi_trace.py +160 -0
- kiri_core/compliance.py +802 -0
- kiri_core/cutover_approval.py +187 -0
- kiri_core/discovery/__init__.py +109 -0
- kiri_core/discovery/ai/__init__.py +37 -0
- kiri_core/discovery/ai/_payload.py +115 -0
- kiri_core/discovery/ai/budget.py +83 -0
- kiri_core/discovery/ai/classify.py +223 -0
- kiri_core/discovery/ai/describe.py +200 -0
- kiri_core/discovery/ai/errors.py +49 -0
- kiri_core/discovery/discover.py +648 -0
- kiri_core/discovery/heuristics/__init__.py +42 -0
- kiri_core/discovery/heuristics/classification.py +174 -0
- kiri_core/discovery/heuristics/medallion.py +157 -0
- kiri_core/discovery/heuristics/pii.py +92 -0
- kiri_core/discovery/introspector.py +636 -0
- kiri_core/discovery/introspector_databricks.py +396 -0
- kiri_core/discovery/introspector_postgres.py +394 -0
- kiri_core/discovery/introspector_snowflake.py +383 -0
- kiri_core/discovery/reporting/__init__.py +17 -0
- kiri_core/discovery/reporting/discovery_report.py +190 -0
- kiri_core/discovery/scaffold/__init__.py +41 -0
- kiri_core/discovery/scaffold/contract_writer.py +632 -0
- kiri_core/discovery/scaffold/provenance_overlay.py +143 -0
- kiri_core/discovery/scaffold/source_writer.py +211 -0
- kiri_core/discovery/scaffold/todo_marker.py +98 -0
- kiri_core/domain_policy.py +122 -0
- kiri_core/goal_coverage.py +82 -0
- kiri_core/gold_registry.py +211 -0
- kiri_core/health.py +192 -0
- kiri_core/incident.py +204 -0
- kiri_core/incident_builders.py +256 -0
- kiri_core/incident_log.py +418 -0
- kiri_core/intent/__init__.py +46 -0
- kiri_core/intent/agent_runner.py +82 -0
- kiri_core/intent/project_dv.py +261 -0
- kiri_core/intent/propose.py +132 -0
- kiri_core/intent/semantic_intent.py +446 -0
- kiri_core/invariants.py +301 -0
- kiri_core/kpi.py +373 -0
- kiri_core/latency.py +150 -0
- kiri_core/lifecycle/__init__.py +555 -0
- kiri_core/lifecycle/ai_index.py +453 -0
- kiri_core/lifecycle/auto_mode.py +172 -0
- kiri_core/lifecycle/bootstrap.py +316 -0
- kiri_core/lifecycle/cli_reference.py +1064 -0
- kiri_core/lifecycle/commit.py +258 -0
- kiri_core/lifecycle/compile_dispatch.py +296 -0
- kiri_core/lifecycle/compile_range.py +665 -0
- kiri_core/lifecycle/contract_inventory.py +354 -0
- kiri_core/lifecycle/contracts.py +250 -0
- kiri_core/lifecycle/doctor.py +412 -0
- kiri_core/lifecycle/git_remote.py +948 -0
- kiri_core/lifecycle/identity.py +147 -0
- kiri_core/lifecycle/ingestion_modes.py +215 -0
- kiri_core/lifecycle/layer_contract.py +452 -0
- kiri_core/lifecycle/layer_contract_migrate.py +969 -0
- kiri_core/lifecycle/layer_contract_parser.py +436 -0
- kiri_core/lifecycle/layer_contract_validator.py +2291 -0
- kiri_core/lifecycle/layout.py +298 -0
- kiri_core/lifecycle/onboarding.py +306 -0
- kiri_core/lifecycle/openlineage_emit.py +153 -0
- kiri_core/lifecycle/patch.py +231 -0
- kiri_core/lifecycle/plan.py +241 -0
- kiri_core/lifecycle/probe.py +204 -0
- kiri_core/lifecycle/project_context.py +334 -0
- kiri_core/lifecycle/project_status.py +532 -0
- kiri_core/lifecycle/scaffold.py +455 -0
- kiri_core/lifecycle/source_grouping.py +725 -0
- kiri_core/lifecycle/source_introspection.py +488 -0
- kiri_core/lifecycle/source_inventory.py +561 -0
- kiri_core/lifecycle/source_patch.py +353 -0
- kiri_core/lifecycle/sources.py +470 -0
- kiri_core/lifecycle/sync_from_live.py +727 -0
- kiri_core/lifecycle/validate.py +116 -0
- kiri_core/lifecycle/verify_annotations.py +391 -0
- kiri_core/lifecycle/verify_live.py +389 -0
- kiri_core/lifecycle/verify_metric_views.py +321 -0
- kiri_core/lifecycle/watermark.py +371 -0
- kiri_core/lifecycle/watermark_apply.py +656 -0
- kiri_core/lineage.py +418 -0
- kiri_core/loader.py +377 -0
- kiri_core/logging.py +374 -0
- kiri_core/maturity.py +592 -0
- kiri_core/migration_layer_policy.py +285 -0
- kiri_core/names.py +163 -0
- kiri_core/observed_lineage.py +62 -0
- kiri_core/odcs/__init__.py +15 -0
- kiri_core/otel_bootstrap.py +326 -0
- kiri_core/pipeline_ir.py +332 -0
- kiri_core/pipeline_loader.py +82 -0
- kiri_core/planning.py +110 -0
- kiri_core/project.py +1576 -0
- kiri_core/project_docs.py +199 -0
- kiri_core/py.typed +0 -0
- kiri_core/rate_limit.py +456 -0
- kiri_core/release_state/__init__.py +37 -0
- kiri_core/release_state/models.py +108 -0
- kiri_core/release_state/store.py +634 -0
- kiri_core/resilience/__init__.py +288 -0
- kiri_core/resilience/retry.py +173 -0
- kiri_core/runtime.py +40 -0
- kiri_core/schema/__init__.py +62 -0
- kiri_core/schema/column.py +140 -0
- kiri_core/schema/ingestion.py +665 -0
- kiri_core/schema/model.py +44 -0
- kiri_core/schema/reporting_goal.py +160 -0
- kiri_core/schema/source.py +188 -0
- kiri_core/silver/__init__.py +148 -0
- kiri_core/silver/apply_runner.py +830 -0
- kiri_core/silver/backfill.py +280 -0
- kiri_core/silver/dv_backend.py +426 -0
- kiri_core/silver/dv_graph.py +932 -0
- kiri_core/silver/dv_graph_mermaid.py +443 -0
- kiri_core/silver/dv_graph_payload.py +79 -0
- kiri_core/silver/dv_graph_views.py +416 -0
- kiri_core/silver/graph_projector.py +566 -0
- kiri_core/silver/hash_canonicaliser.py +172 -0
- kiri_core/silver/hash_policy.py +195 -0
- kiri_core/silver/load_ts_policy.py +202 -0
- kiri_core/silver/schema_evolution.py +410 -0
- kiri_core/silver/zone_rules.py +81 -0
- kiri_core/sizing.py +134 -0
- kiri_core/storage_paths.py +130 -0
- kiri_core/technique_validator.py +700 -0
- kiri_core/trace.py +409 -0
- kiri_core/types.py +41 -0
- kiri_core/urn.py +161 -0
- kiri_dbt_bridge/__init__.py +22 -0
- kiri_dbt_bridge/_schema_resolve.py +38 -0
- kiri_dbt_bridge/bridge.py +94 -0
- kiri_dbt_bridge/dbt.manifest.yml +26 -0
- kiri_dbt_bridge/dv4dbt/__init__.py +143 -0
- kiri_dbt_bridge/dv4dbt/backend.py +204 -0
- kiri_dbt_bridge/dv4dbt/bridge_generator.py +289 -0
- kiri_dbt_bridge/dv4dbt/compat.py +232 -0
- kiri_dbt_bridge/dv4dbt/hub_generator.py +282 -0
- kiri_dbt_bridge/dv4dbt/link_generator.py +306 -0
- kiri_dbt_bridge/dv4dbt/manifest.py +203 -0
- kiri_dbt_bridge/dv4dbt/pit_generator.py +318 -0
- kiri_dbt_bridge/dv4dbt/policy_translation.py +249 -0
- kiri_dbt_bridge/dv4dbt/project_writer.py +588 -0
- kiri_dbt_bridge/dv4dbt/ref_generator.py +404 -0
- kiri_dbt_bridge/dv4dbt/render.py +180 -0
- kiri_dbt_bridge/dv4dbt/sat_generator.py +480 -0
- kiri_dbt_bridge/dv4dbt/sat_variant_generator.py +529 -0
- kiri_dbt_bridge/dv4dbt/templates/__init__.py +16 -0
- kiri_dbt_bridge/dv4dbt/templates/bridge.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/dbt_project.yml.j2 +37 -0
- kiri_dbt_bridge/dv4dbt/templates/hub.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/link.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/packages.yml.j2 +10 -0
- kiri_dbt_bridge/dv4dbt/templates/pit.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/ref_historised.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/ref_non_historised.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/sat.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/sat_effectivity.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/sat_multi_active.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/sat_non_historised.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/sat_record_tracking.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/sat_virtual_end_dated.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/sources.yml.j2 +39 -0
- kiri_dbt_bridge/framework_adapter.py +310 -0
- kiri_dbt_bridge/generate.py +372 -0
- kiri_dbt_bridge/gold/__init__.py +21 -0
- kiri_dbt_bridge/gold/_sql_emit.py +128 -0
- kiri_dbt_bridge/gold/assemble.py +90 -0
- kiri_dbt_bridge/gold/canonical_columns.py +168 -0
- kiri_dbt_bridge/gold/computed_expressions.py +179 -0
- kiri_dbt_bridge/gold/dialects/__init__.py +69 -0
- kiri_dbt_bridge/gold/dialects/databricks.py +150 -0
- kiri_dbt_bridge/gold/dialects/duckdb.py +120 -0
- kiri_dbt_bridge/gold/dialects/trino.py +136 -0
- kiri_dbt_bridge/gold/dim_generator.py +688 -0
- kiri_dbt_bridge/gold/dimension_refs.py +255 -0
- kiri_dbt_bridge/gold/dispatch.py +69 -0
- kiri_dbt_bridge/gold/fact_generator.py +534 -0
- kiri_dbt_bridge/gold/flat_generator.py +144 -0
- kiri_dbt_bridge/gold/lineage.py +484 -0
- kiri_dbt_bridge/gold/localized_twins.py +218 -0
- kiri_dbt_bridge/gold/measures.py +49 -0
- kiri_dbt_bridge/gold/role_playing.py +75 -0
- kiri_dbt_bridge/gold/surrogate_key.py +114 -0
- kiri_dbt_bridge/macros/kiri_scd1_dim.sql +57 -0
- kiri_dbt_bridge/macros/kiri_scd2_dim.sql +188 -0
- kiri_dbt_bridge/macros/kiri_sha256.sql +32 -0
- kiri_dbt_bridge/macros/kiri_string_cast.sql +37 -0
- kiri_dbt_bridge/macros/kiri_surrogate_key.sql +43 -0
- kiri_dbt_bridge/profiles.py +162 -0
- kiri_dbt_bridge/py.typed +0 -0
- kiri_dbt_bridge/safe_ident.py +44 -0
- kiri_dbt_bridge/silver_conformance.py +188 -0
- kiri_dbt_bridge/silver_kimball/__init__.py +27 -0
- kiri_dbt_bridge/silver_kimball/assemble.py +174 -0
- kiri_dbt_bridge/silver_kimball/dim_generator.py +602 -0
- kiri_dbt_bridge/silver_kimball/dispatch.py +86 -0
- kiri_dbt_bridge/silver_kimball/fact_generator.py +320 -0
- kiri_dbt_bridge/silver_kimball/flat_generator.py +156 -0
- kiri_dbt_bridge/silver_kimball/lineage.py +464 -0
- kiri_dbt_bridge/silver_kimball/watermark_window.py +199 -0
- kiri_dbt_bridge/silver_kimball/zone.py +68 -0
- kiri_dbt_bridge/silver_scd2_curated/__init__.py +46 -0
- kiri_dbt_bridge/silver_scd2_curated/assemble.py +213 -0
- kiri_dbt_bridge/silver_scd2_curated/curated_generator.py +351 -0
- kiri_dbt_bridge/silver_scd2_curated/standardized_generator.py +261 -0
- kiri_dbt_bridge/silver_scd2_curated/standardized_plan.py +254 -0
- kiri_iceberg/__init__.py +11 -0
- kiri_iceberg/catalog.py +128 -0
- kiri_iceberg/profile.py +131 -0
- kiri_iceberg/py.typed +0 -0
- kiri_ingestion/__init__.py +63 -0
- kiri_ingestion/airbyte_client.py +428 -0
- kiri_ingestion/database_fetcher.py +224 -0
- kiri_ingestion/layout.py +71 -0
- kiri_ingestion/mssql_introspect.py +565 -0
- kiri_ingestion/py.typed +0 -0
- kiri_ingestion/rest_fetcher.py +472 -0
- kiri_ingestion/runner.py +341 -0
- kiri_ingestion/schema_drift.py +222 -0
- kiri_ingestion/storage.py +621 -0
- kiri_mcp/__init__.py +5 -0
- kiri_mcp/audit.py +99 -0
- kiri_mcp/audit_decorators.py +252 -0
- kiri_mcp/main.py +22 -0
- kiri_mcp/py.typed +0 -0
- kiri_mcp/rbac.py +110 -0
- kiri_mcp/server.py +766 -0
- kiri_mcp/skills.py +297 -0
- kiri_mcp/tools/__init__.py +70 -0
- kiri_mcp/tools/_shared.py +58 -0
- kiri_mcp/tools/adapter.py +150 -0
- kiri_mcp/tools/cli_reference.py +59 -0
- kiri_mcp/tools/compliance.py +119 -0
- kiri_mcp/tools/contract.py +374 -0
- kiri_mcp/tools/dv.py +62 -0
- kiri_mcp/tools/federation.py +152 -0
- kiri_mcp/tools/intent.py +71 -0
- kiri_mcp/tools/lineage.py +204 -0
- kiri_mcp/tools/lineage_drift.py +75 -0
- kiri_mcp/tools/plan.py +66 -0
- kiri_mcp/tools/project.py +311 -0
- kiri_migration/__init__.py +56 -0
- kiri_migration/name_preservation.py +271 -0
- kiri_migration/predeploy.py +311 -0
- kiri_migration/py.typed +0 -0
- kiri_migration/reconcile.py +1891 -0
- kiri_migration/scaffold/__init__.py +90 -0
- kiri_migration/scaffold/_audit_columns.py +74 -0
- kiri_migration/scaffold/_classify.py +100 -0
- kiri_migration/scaffold/_typemap.py +71 -0
- kiri_migration/scaffold/_yaml.py +33 -0
- kiri_migration/scaffold/bronze.py +279 -0
- kiri_migration/scaffold/compile_targets/__init__.py +271 -0
- kiri_migration/scaffold/compile_targets/_common.py +193 -0
- kiri_migration/scaffold/compile_targets/copy_into.py +33 -0
- kiri_migration/scaffold/compile_targets/dlt.py +125 -0
- kiri_migration/scaffold/compile_targets/dlt_with_append_flows.py +163 -0
- kiri_migration/scaffold/compile_targets/dlt_with_apply_changes.py +192 -0
- kiri_migration/scaffold/compile_targets/dlt_with_expectations.py +276 -0
- kiri_migration/scaffold/compile_targets/layer_dispatch.py +258 -0
- kiri_migration/scaffold/compile_targets/sql_full_snapshot.py +385 -0
- kiri_migration/scaffold/compile_targets/streaming_table.py +37 -0
- kiri_migration/scaffold/mart.py +234 -0
- kiri_migration/scaffold/plans/__init__.py +49 -0
- kiri_migration/scaffold/plans/_registry.py +308 -0
- kiri_migration/scaffold/plans/minimal.py +550 -0
- kiri_migration/scaffold/plans/plan_b_layer.py +640 -0
- kiri_migration/scaffold/silver_from_marts.py +256 -0
- kiri_migration/scaffold/silver_pattern.py +451 -0
- kiri_migration/sp_translator.py +398 -0
- kiri_odcs/LICENSE +204 -0
- kiri_odcs/__init__.py +234 -0
- kiri_odcs/ai_trust_tier.py +475 -0
- kiri_odcs/canonical.py +263 -0
- kiri_odcs/diff.py +63 -0
- kiri_odcs/errors.py +51 -0
- kiri_odcs/invariant_sql.py +136 -0
- kiri_odcs/kiri_bridge.py +71 -0
- kiri_odcs/kiri_ext.py +3166 -0
- kiri_odcs/kiri_fk.py +447 -0
- kiri_odcs/kiri_pit.py +121 -0
- kiri_odcs/kiri_ref.py +57 -0
- kiri_odcs/openlineage_adapter.py +105 -0
- kiri_odcs/parser.py +1213 -0
- kiri_odcs/pii_guardrails.py +159 -0
- kiri_odcs/py.typed +0 -0
- kiri_odcs/schema_export.py +223 -0
- kiri_openlineage/__init__.py +58 -0
- kiri_openlineage/builder.py +146 -0
- kiri_openlineage/cost.py +190 -0
- kiri_openlineage/emitter.py +165 -0
- kiri_openlineage/py.typed +0 -0
- kiri_openlineage/schema.py +121 -0
- kiri_packs/__init__.py +67 -0
- kiri_packs/installer.py +500 -0
- kiri_packs/manifest.py +134 -0
- kiri_packs/publisher.py +605 -0
- kiri_packs/py.typed +0 -0
- kiri_packs/validator.py +295 -0
- kiri_quality/__init__.py +108 -0
- kiri_quality/ai_audit_correlator.py +293 -0
- kiri_quality/ai_audit_correlator_apply.py +526 -0
- kiri_quality/conformance_evidence.py +272 -0
- kiri_quality/dbt_test_collector.py +214 -0
- kiri_quality/dq_codegen.py +300 -0
- kiri_quality/dv_checks.py +393 -0
- kiri_quality/dv_column_lineage.py +279 -0
- kiri_quality/dv_column_lineage_apply.py +856 -0
- kiri_quality/dv_conformance.py +841 -0
- kiri_quality/dv_fixture_runner.py +478 -0
- kiri_quality/dv_invariants.py +986 -0
- kiri_quality/dv_kiri.py +97 -0
- kiri_quality/dv_markdown.py +169 -0
- kiri_quality/dv_observed_uc_reader.py +187 -0
- kiri_quality/dv_runner.py +389 -0
- kiri_quality/evidence_status.py +337 -0
- kiri_quality/fk_checks.py +800 -0
- kiri_quality/fk_ddl.py +402 -0
- kiri_quality/fk_pit_contract.py +168 -0
- kiri_quality/fk_runner.py +950 -0
- kiri_quality/kernel.py +376 -0
- kiri_quality/pii.py +60 -0
- kiri_quality/pre_classify/__init__.py +55 -0
- kiri_quality/pre_classify/engine.py +346 -0
- kiri_quality/pre_classify/locale_packs/en.yaml +117 -0
- kiri_quality/pre_classify/locale_packs/sv.yaml +73 -0
- kiri_quality/pre_classify/merge.py +297 -0
- kiri_quality/pre_classify/model.py +163 -0
- kiri_quality/py.typed +0 -0
- kiri_quality/reject_lane.py +261 -0
- kiri_quality/rule_runner.py +565 -0
- kiri_quality/schema_evolution_check.py +88 -0
- kiri_quality/silver_quality.py +494 -0
- kiri_quality/silver_quality_markdown.py +187 -0
- kiri_quality/uc_security_compile.py +951 -0
- kiri_quality/uc_security_compile_apply.py +1186 -0
- kiri_rbac/__init__.py +87 -0
- kiri_rbac/access.py +371 -0
- kiri_rbac/loader.py +104 -0
- kiri_rbac/pr_gate.py +213 -0
- kiri_rbac/providers/__init__.py +29 -0
- kiri_rbac/providers/_jwt.py +121 -0
- kiri_rbac/providers/azure_ad.py +93 -0
- kiri_rbac/providers/base.py +107 -0
- kiri_rbac/providers/factory.py +48 -0
- kiri_rbac/providers/github.py +134 -0
- kiri_rbac/providers/oidc_generic.py +99 -0
- kiri_rbac/py.typed +0 -0
- kiri_rbac/resolver.py +118 -0
- kiri_rbac/schema.py +219 -0
- kiri_schema_registry/__init__.py +22 -0
- kiri_schema_registry/avro.py +127 -0
- kiri_schema_registry/base.py +71 -0
- kiri_schema_registry/confluent.py +184 -0
- kiri_schema_registry/glue.py +178 -0
- kiri_schema_registry/py.typed +0 -0
- kiri_slack_bot/__init__.py +24 -0
- kiri_slack_bot/handler.py +248 -0
- kiri_slack_bot/py.typed +0 -0
- kiri_slack_bot/queries.py +90 -0
- kiri_slack_bot/signature.py +94 -0
- kiri_vault/__init__.py +59 -0
- kiri_vault/audit.py +64 -0
- kiri_vault/aws_vault.py +123 -0
- kiri_vault/azure_vault.py +236 -0
- kiri_vault/boundaries.py +219 -0
- kiri_vault/client.py +54 -0
- kiri_vault/databricks_vault.py +292 -0
- kiri_vault/env_vault.py +77 -0
- kiri_vault/py.typed +0 -0
- kiri_vault/ref.py +84 -0
- kirimana_automation/__init__.py +0 -0
- kirimana_automation/business_calendar.py +179 -0
- kirimana_automation/dag_snapshot.py +252 -0
- kirimana_automation/db/__init__.py +5 -0
- kirimana_automation/db/migrate.py +571 -0
- kirimana_automation/db/migrations/0001_release_state.down.duckdb.sql +29 -0
- kirimana_automation/db/migrations/0001_release_state.down.postgres.sql +37 -0
- kirimana_automation/db/migrations/0001_release_state.duckdb.sql +170 -0
- kirimana_automation/db/migrations/0001_release_state.postgres.sql +176 -0
- kirimana_automation/db/migrations/0002_rate_limit_buckets.down.duckdb.sql +9 -0
- kirimana_automation/db/migrations/0002_rate_limit_buckets.down.postgres.sql +9 -0
- kirimana_automation/db/migrations/0002_rate_limit_buckets.duckdb.sql +14 -0
- kirimana_automation/db/migrations/0002_rate_limit_buckets.postgres.sql +18 -0
- kirimana_automation/db/migrations/0003_flow_runs.down.duckdb.sql +18 -0
- kirimana_automation/db/migrations/0003_flow_runs.down.postgres.sql +19 -0
- kirimana_automation/db/migrations/0003_flow_runs.duckdb.sql +85 -0
- kirimana_automation/db/migrations/0003_flow_runs.postgres.sql +102 -0
- kirimana_automation/db/migrations/0004_flow_dag_snapshots.down.duckdb.sql +11 -0
- kirimana_automation/db/migrations/0004_flow_dag_snapshots.down.postgres.sql +11 -0
- kirimana_automation/db/migrations/0004_flow_dag_snapshots.duckdb.sql +15 -0
- kirimana_automation/db/migrations/0004_flow_dag_snapshots.postgres.sql +31 -0
- kirimana_automation/db/migrations/0005_calendar_skip.down.duckdb.sql +109 -0
- kirimana_automation/db/migrations/0005_calendar_skip.down.postgres.sql +42 -0
- kirimana_automation/db/migrations/0005_calendar_skip.duckdb.sql +113 -0
- kirimana_automation/db/migrations/0005_calendar_skip.postgres.sql +42 -0
- kirimana_automation/db/migrations/0006_flow_apply_log.down.duckdb.sql +9 -0
- kirimana_automation/db/migrations/0006_flow_apply_log.down.postgres.sql +11 -0
- kirimana_automation/db/migrations/0006_flow_apply_log.duckdb.sql +45 -0
- kirimana_automation/db/migrations/0006_flow_apply_log.postgres.sql +49 -0
- kirimana_automation/db/migrations/0007_flow_task_error_redacted.down.duckdb.sql +14 -0
- kirimana_automation/db/migrations/0007_flow_task_error_redacted.down.postgres.sql +2 -0
- kirimana_automation/db/migrations/0007_flow_task_error_redacted.duckdb.sql +7 -0
- kirimana_automation/db/migrations/0007_flow_task_error_redacted.postgres.sql +11 -0
- kirimana_automation/db/migrations/README.md +84 -0
- kirimana_automation/flow_loader.py +220 -0
- kirimana_automation/freeze.py +133 -0
- kirimana_automation/kiri_recommender.py +313 -0
- kirimana_automation/main.py +195 -0
- kirimana_automation/py.typed +0 -0
- kirimana_automation/run_history.py +593 -0
- kirimana_automation/run_history_analysis.py +607 -0
- kirimana_automation/schedule.py +343 -0
- kirimana_automation/schedule_validator.py +572 -0
- kirimana_automation/workflow_compiler.py +1288 -0
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
name: suggest_attribute_classification
|
|
2
|
+
version: 1
|
|
3
|
+
model: default
|
|
4
|
+
system: |
|
|
5
|
+
You are Kiri — Kirimana's governance assistant. Given one catalog
|
|
6
|
+
column on a data asset, suggest the strictest defensible
|
|
7
|
+
classification and a short business description.
|
|
8
|
+
|
|
9
|
+
Classifications (strictest first):
|
|
10
|
+
restricted — regulated PII / PCI / PHI, Art. 9 special category,
|
|
11
|
+
payment-card PANs, auth secrets, health data.
|
|
12
|
+
confidential — ordinary PII (email, phone, address, name), salary,
|
|
13
|
+
non-public business data.
|
|
14
|
+
internal — operational data visible to the company but not to
|
|
15
|
+
the public.
|
|
16
|
+
public — catalogue data, marketing pages, anything already
|
|
17
|
+
on the company website.
|
|
18
|
+
|
|
19
|
+
Pick "confidential" or "restricted" when the column name suggests
|
|
20
|
+
PII or regulated data. Default to "internal" only when you are
|
|
21
|
+
confident the column is neither PII nor sensitive. Never pick
|
|
22
|
+
"public" unless the name strongly implies publicly-visible content.
|
|
23
|
+
|
|
24
|
+
Keep the description to 1-2 sentences in plain English — readable
|
|
25
|
+
by a business steward, not a data engineer. Do not speculate about
|
|
26
|
+
values you cannot infer.
|
|
27
|
+
user: |
|
|
28
|
+
Asset: $asset_qualified_name
|
|
29
|
+
Asset type: $asset_type
|
|
30
|
+
Domain: $asset_domain
|
|
31
|
+
Asset classification: $asset_classification
|
|
32
|
+
|
|
33
|
+
Column:
|
|
34
|
+
name: $column_name
|
|
35
|
+
data_type: $column_data_type
|
|
36
|
+
|
|
37
|
+
Return JSON exactly matching this shape (no prose, no code fences):
|
|
38
|
+
{"classification": "<one of: public | internal | confidential | restricted>",
|
|
39
|
+
"description": "<1-2 sentence business description>",
|
|
40
|
+
"reason": "<one short sentence explaining the classification choice>"}
|
|
41
|
+
stub: |
|
|
42
|
+
{"classification": "internal", "description": "Stub suggestion for $column_name on $asset_qualified_name — set ANTHROPIC_API_KEY for a real Kiri classification.", "reason": "stub mode"}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
name: suggest_data_tests
|
|
2
|
+
version: 1
|
|
3
|
+
model: heavy
|
|
4
|
+
system: |
|
|
5
|
+
You are a senior analytics engineer. Given a bronze table's schema, a small
|
|
6
|
+
data sample, and (optionally) the business context from a ReportingGoal,
|
|
7
|
+
propose dbt tests for the table as a single dbt-core `models:`-style YAML
|
|
8
|
+
document.
|
|
9
|
+
|
|
10
|
+
Output rules (strict):
|
|
11
|
+
|
|
12
|
+
1. Output EXACTLY ONE YAML document. No markdown fences. No prose.
|
|
13
|
+
The first characters of your response must be `version: 2`.
|
|
14
|
+
2. Top-level shape:
|
|
15
|
+
|
|
16
|
+
version: 2
|
|
17
|
+
models:
|
|
18
|
+
- name: <source_name>__<table_name>
|
|
19
|
+
description: <one sentence derived from business context>
|
|
20
|
+
columns:
|
|
21
|
+
- name: <col>
|
|
22
|
+
description: <one sentence if helpful>
|
|
23
|
+
data_tests:
|
|
24
|
+
- <test_name>
|
|
25
|
+
- <test_name>:
|
|
26
|
+
<arg>: <value>
|
|
27
|
+
|
|
28
|
+
3. Only use dbt-core built-in tests unless noted otherwise:
|
|
29
|
+
not_null, unique, accepted_values (with {values: [...]}),
|
|
30
|
+
relationships (with {to: "ref('<model>')", field: "<col>"}).
|
|
31
|
+
4. Recommend `not_null` for columns that look like primary keys,
|
|
32
|
+
foreign keys, or business-critical identifiers.
|
|
33
|
+
5. Recommend `unique` for columns named like primary keys
|
|
34
|
+
(`*_id`, `id`, `uuid`) OR appearing in the provided dedupe_key.
|
|
35
|
+
6. Recommend `accepted_values` when the sample suggests a small fixed
|
|
36
|
+
domain (e.g. status codes, country codes, booleans).
|
|
37
|
+
7. Recommend `relationships` when a column ends in `_id` and the goal's
|
|
38
|
+
needed_silver_entities or declared sources suggest the referenced
|
|
39
|
+
model. Use `ref('silver_<entity>')`. If you can't identify a target
|
|
40
|
+
with confidence, omit the test rather than guess.
|
|
41
|
+
8. DO NOT invent tests for columns that are not in the provided schema.
|
|
42
|
+
9. DO NOT test the kiri-internal provenance columns (`_kiri_*`) —
|
|
43
|
+
they are system-managed.
|
|
44
|
+
10. Keep the number of tests proportionate: don't pile up noise. Aim for
|
|
45
|
+
1-4 tests per column of interest; skip columns that need nothing.
|
|
46
|
+
user: |
|
|
47
|
+
Source: $source_name
|
|
48
|
+
Table (bronze model): ${source_name}__${table_name}
|
|
49
|
+
Classification: $classification
|
|
50
|
+
Domain: $domain
|
|
51
|
+
Dedupe key declared on the source: $dedupe_key
|
|
52
|
+
|
|
53
|
+
Bronze columns available (name : type):
|
|
54
|
+
$columns
|
|
55
|
+
|
|
56
|
+
Sample rows (first few, JSON array — may be empty if no data yet):
|
|
57
|
+
$sample_rows
|
|
58
|
+
|
|
59
|
+
Reporting-goal context (may be "(none)"):
|
|
60
|
+
$goal_context
|
|
61
|
+
|
|
62
|
+
Produce the dbt tests YAML now.
|
|
63
|
+
stub: |
|
|
64
|
+
version: 2
|
|
65
|
+
models:
|
|
66
|
+
- name: ${source_name}__${table_name}
|
|
67
|
+
description: Stub tests — set ANTHROPIC_API_KEY for AI-generated tests.
|
|
68
|
+
columns:
|
|
69
|
+
- name: id
|
|
70
|
+
data_tests:
|
|
71
|
+
- not_null
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
name: suggest_gold_dimension
|
|
2
|
+
version: 1
|
|
3
|
+
model: heavy
|
|
4
|
+
system: |
|
|
5
|
+
You are a senior analytics engineer. Generate a dbt gold **dimension**
|
|
6
|
+
model that implements Kimball's star-schema dim per ADR 0031. The
|
|
7
|
+
SQL shape depends on the declared ``scd_type`` — follow the right
|
|
8
|
+
pattern.
|
|
9
|
+
|
|
10
|
+
### SCD patterns
|
|
11
|
+
|
|
12
|
+
- **type_1** (overwrite) — thin ``MERGE`` on business_keys; latest
|
|
13
|
+
values win; no history. Use ``materialized='table'`` with a
|
|
14
|
+
``unique_key`` = surrogate key column.
|
|
15
|
+
- **type_2** (historise) — every change creates a new row. Use the
|
|
16
|
+
``{{ kiri_scd2_dim(...) }}`` macro that emits ``valid_from``,
|
|
17
|
+
``valid_to`` (NULL on current), and ``is_current`` columns +
|
|
18
|
+
handles late-arriving + out-of-order updates. ``unique_key`` is
|
|
19
|
+
the surrogate key column (changes per version).
|
|
20
|
+
- **type_3** (limited history) — maintain ``<attr>`` + ``prev_<attr>``
|
|
21
|
+
per tracked attribute. Overwrite on every change. One row per
|
|
22
|
+
business_key; the ``prev_`` columns carry the immediately
|
|
23
|
+
preceding value.
|
|
24
|
+
- **type_7** (dual) — emit both SCD1-style current snapshot AND
|
|
25
|
+
SCD2-style history in the same model via a UNION on two CTEs.
|
|
26
|
+
Queries filter on ``is_current = true`` for the flat view or
|
|
27
|
+
join on surrogate_key for historical.
|
|
28
|
+
|
|
29
|
+
### Surrogate keys
|
|
30
|
+
|
|
31
|
+
Always call ``{{ kiri_surrogate_key(business_keys) }}`` as the first
|
|
32
|
+
computed column in the final SELECT. The macro picks the project-
|
|
33
|
+
configured strategy:
|
|
34
|
+
|
|
35
|
+
- ``hash`` — deterministic ``sha256(concat_ws('||', business_keys))``
|
|
36
|
+
(matches ADR 0013 DV hashdiff).
|
|
37
|
+
- ``sequence`` — platform-managed identity.
|
|
38
|
+
- ``natural`` — the first business_key itself (small dims only).
|
|
39
|
+
|
|
40
|
+
Do NOT reinvent the surrogate logic. Delegate; that's what the
|
|
41
|
+
macro exists for.
|
|
42
|
+
|
|
43
|
+
### Conformed + role-playing
|
|
44
|
+
|
|
45
|
+
- When ``conformed: true``, add a SQL comment at the top:
|
|
46
|
+
``-- CONFORMED DIM: shared across goals; do not diverge fields``.
|
|
47
|
+
- ``role_playing_aliases`` do NOT change this model's SQL — facts
|
|
48
|
+
that reference the dim under alias X will derive the alias-
|
|
49
|
+
specific FK on their side via ``{{ kiri_surrogate_key(...) }}``.
|
|
50
|
+
Mention aliases in a SQL comment for traceability.
|
|
51
|
+
|
|
52
|
+
### Rules
|
|
53
|
+
|
|
54
|
+
1. First line: ``{{ config(materialized='table', schema='gold',
|
|
55
|
+
unique_key='<surrogate_key_column>') }}`` (or ``incremental``
|
|
56
|
+
strategy for SCD2 — delegate to the macro's config).
|
|
57
|
+
2. Read ONLY from silver via ``{{ ref('silver_<entity>') }}``.
|
|
58
|
+
3. Structure as CTEs: one per silver source, a dedup/latest CTE if
|
|
59
|
+
needed, final SELECT.
|
|
60
|
+
4. The surrogate-key column MUST be named per the declared template
|
|
61
|
+
(``{name}`` expands to the contract name → ``customer_key``).
|
|
62
|
+
5. Business-key columns stay on the dim — they're not hidden by the
|
|
63
|
+
surrogate.
|
|
64
|
+
6. Include descriptive attributes from the silver entity. Keep the
|
|
65
|
+
dim thin — a dim is a lookup, not a fact; don't copy transactional
|
|
66
|
+
columns.
|
|
67
|
+
7. Output pure dbt SQL — no markdown fences, no prose.
|
|
68
|
+
|
|
69
|
+
### MUST NOT
|
|
70
|
+
|
|
71
|
+
- Hand-roll SCD2 logic. Use ``{{ kiri_scd2_dim(...) }}``.
|
|
72
|
+
- Omit the surrogate key. Every dim has one; even
|
|
73
|
+
``surrogate_key_strategy=natural`` still produces a named column.
|
|
74
|
+
- Change business_keys order — surrogate-key hashes depend on it.
|
|
75
|
+
- Invent silver models outside ``needed_silver_entities``.
|
|
76
|
+
user: |
|
|
77
|
+
Goal: $goal_name
|
|
78
|
+
Question: $question
|
|
79
|
+
Persona: $persona | Tool: $tool | Refresh: $refresh
|
|
80
|
+
|
|
81
|
+
DIMENSION target:
|
|
82
|
+
name: $dim_name
|
|
83
|
+
semantic_entity: $semantic_entity
|
|
84
|
+
scd_type: $scd_type
|
|
85
|
+
business_keys: $business_keys
|
|
86
|
+
surrogate_key_strategy: $surrogate_key_strategy
|
|
87
|
+
surrogate_key_column: $surrogate_key_column
|
|
88
|
+
conformed: $conformed
|
|
89
|
+
role_playing_aliases: $role_playing_aliases
|
|
90
|
+
description: $dim_description
|
|
91
|
+
|
|
92
|
+
Silver entities this goal needs (ref as silver_<name>):
|
|
93
|
+
$needed_silver_entities
|
|
94
|
+
|
|
95
|
+
Generate the gold dimension SQL now.
|
|
96
|
+
stub: |
|
|
97
|
+
-- AI stub gold dimension for goal $goal_name (target: $dim_name)
|
|
98
|
+
-- Set ANTHROPIC_API_KEY for a real AI-generated model.
|
|
99
|
+
--
|
|
100
|
+
-- scd_type=$scd_type · keys=$business_keys
|
|
101
|
+
{{ config(materialized='table', schema='gold', unique_key='$surrogate_key_column') }}
|
|
102
|
+
|
|
103
|
+
WITH base AS (
|
|
104
|
+
-- TODO: replace with real silver source + kiri_surrogate_key().
|
|
105
|
+
-- For SCD2, wrap in {{ kiri_scd2_dim(...) }} macro instead.
|
|
106
|
+
SELECT 1 AS placeholder
|
|
107
|
+
)
|
|
108
|
+
SELECT * FROM base
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
name: suggest_gold_fact
|
|
2
|
+
version: 1
|
|
3
|
+
model: heavy
|
|
4
|
+
system: |
|
|
5
|
+
You are a senior analytics engineer. Generate a dbt gold **fact**
|
|
6
|
+
model that implements a Kimball star-schema fact per ADR 0031.
|
|
7
|
+
|
|
8
|
+
You get structured metadata: grain, measures (each with additivity),
|
|
9
|
+
dimension_refs (URNs, with optional ``::<alias>`` for role-playing),
|
|
10
|
+
degenerate_dimensions, and the fact_kind. Respect ALL of it —
|
|
11
|
+
violating the metadata is a bug, not creative license.
|
|
12
|
+
|
|
13
|
+
### Fact kinds and their SQL shapes
|
|
14
|
+
|
|
15
|
+
- **transaction** — one row per business event. Typical GROUP BY is
|
|
16
|
+
the grain; measures summed with whatever aggregation each declares.
|
|
17
|
+
- **periodic_snapshot** — one row per (grain-entity, snapshot_date).
|
|
18
|
+
Grain MUST include a date/period column. Measures are usually
|
|
19
|
+
semi-additive (balances) — preserve their additivity in the SQL
|
|
20
|
+
comments so BI tools don't silently SUM them across time.
|
|
21
|
+
- **accumulating_snapshot** — one row per pipeline instance (grain =
|
|
22
|
+
process_id), updated as milestones progress. Generate a SQL pattern
|
|
23
|
+
that can MERGE rows as dates fill in (use incremental config with
|
|
24
|
+
unique_key=grain).
|
|
25
|
+
- **factless_fact** — coverage/event tables. NO measures column
|
|
26
|
+
(declared count is always 0). Grain alone.
|
|
27
|
+
|
|
28
|
+
### Dimensional keys
|
|
29
|
+
|
|
30
|
+
For every entry in dimension_refs:
|
|
31
|
+
|
|
32
|
+
- Parse the URN: ``urn:kirimana:gold:<dim>`` or
|
|
33
|
+
``urn:kirimana:gold:<dim>::<alias>``. Use ``<alias>`` as the FK
|
|
34
|
+
column prefix when present; otherwise use ``<dim>``.
|
|
35
|
+
- Call ``{{ kiri_surrogate_key(['<biz_key>']) }}`` to compute each FK
|
|
36
|
+
— the macro picks the project's surrogate strategy (hash/
|
|
37
|
+
sequence/natural) from kiri.yml. Delegate; do not reinvent.
|
|
38
|
+
- Emit one FK column per ref as ``<prefix>_key``.
|
|
39
|
+
|
|
40
|
+
degenerate_dimensions are columns that live on the fact but don't
|
|
41
|
+
FK to a dim — select them verbatim from the matching silver.
|
|
42
|
+
|
|
43
|
+
### Rules
|
|
44
|
+
|
|
45
|
+
1. First line: ``{{ config(materialized='table', schema='gold'
|
|
46
|
+
<extra_config>) }}``. For ``accumulating_snapshot`` use
|
|
47
|
+
``materialized='incremental'`` with a ``unique_key`` matching
|
|
48
|
+
the grain.
|
|
49
|
+
2. Read ONLY from silver via ``{{ ref('silver_<entity>') }}``; never
|
|
50
|
+
bronze or ``source()``.
|
|
51
|
+
3. Structure as CTEs: one per silver source, a joining CTE, final
|
|
52
|
+
SELECT producing the grain + FKs + measures + degenerate_dims.
|
|
53
|
+
4. Emit every declared measure with its declared ``aggregation``
|
|
54
|
+
verb and a one-line comment stating its ``additivity``.
|
|
55
|
+
Semi-additive measures also comment which axes they're summable
|
|
56
|
+
over (``-- semi_additive over: customer``).
|
|
57
|
+
5. Carry ``MAX(_kiri_ingested_at) AS _kiri_ingested_at`` into the gold
|
|
58
|
+
row for freshness.
|
|
59
|
+
6. Output pure dbt SQL — no markdown fences, no prose.
|
|
60
|
+
|
|
61
|
+
### MUST NOT
|
|
62
|
+
|
|
63
|
+
- Invent silver models outside ``needed_silver_entities``.
|
|
64
|
+
- Add measures beyond the declared list.
|
|
65
|
+
- Reorder grain columns — their sequence is load-bearing for
|
|
66
|
+
surrogate-key hashes.
|
|
67
|
+
- Expand role-playing aliases into one shared FK — each alias must
|
|
68
|
+
become its own FK column.
|
|
69
|
+
user: |
|
|
70
|
+
Goal: $goal_name
|
|
71
|
+
Question: $question
|
|
72
|
+
Persona: $persona | Tool: $tool | Refresh: $refresh
|
|
73
|
+
|
|
74
|
+
FACT target:
|
|
75
|
+
name: $fact_name
|
|
76
|
+
semantic_entity: $semantic_entity
|
|
77
|
+
fact_kind: $fact_kind
|
|
78
|
+
grain: $grain
|
|
79
|
+
description: $fact_description
|
|
80
|
+
|
|
81
|
+
Measures (name · column · aggregation · additivity · semi_additive_axis):
|
|
82
|
+
$measures
|
|
83
|
+
|
|
84
|
+
Dimension references (URN, with ::alias for role-playing):
|
|
85
|
+
$dimension_refs
|
|
86
|
+
|
|
87
|
+
Degenerate dimensions (fact-local columns that aren't FKs/measures):
|
|
88
|
+
$degenerate_dimensions
|
|
89
|
+
|
|
90
|
+
Silver entities this goal needs (ref as silver_<name>):
|
|
91
|
+
$needed_silver_entities
|
|
92
|
+
|
|
93
|
+
Generate the gold fact SQL now.
|
|
94
|
+
stub: |
|
|
95
|
+
-- AI stub gold fact for goal $goal_name (target: $fact_name)
|
|
96
|
+
-- Set ANTHROPIC_API_KEY for a real AI-generated model.
|
|
97
|
+
--
|
|
98
|
+
-- Rename + connect the placeholders to your actual silver models.
|
|
99
|
+
{{ config(materialized='table', schema='gold') }}
|
|
100
|
+
|
|
101
|
+
WITH base AS (
|
|
102
|
+
-- TODO: replace with joins over real silver entities + surrogate
|
|
103
|
+
-- keys from kiri_surrogate_key(). Add measures respecting their
|
|
104
|
+
-- declared additivity.
|
|
105
|
+
SELECT 1 AS placeholder
|
|
106
|
+
)
|
|
107
|
+
SELECT * FROM base
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
name: suggest_gold_model
|
|
2
|
+
version: 1
|
|
3
|
+
model: heavy
|
|
4
|
+
system: |
|
|
5
|
+
You are a senior analytics engineer. Generate a dbt gold model for one
|
|
6
|
+
fact or dimension of a ReportingGoal. Gold is the report-facing layer: it
|
|
7
|
+
serves a BI user's specific question.
|
|
8
|
+
|
|
9
|
+
Rules — follow all of them:
|
|
10
|
+
|
|
11
|
+
1. Start the file with `{{ config(materialized='table', schema='gold') }}`.
|
|
12
|
+
2. Read from silver models by convention: `{{ ref('silver_<entity>') }}`
|
|
13
|
+
for each silver entity named in the context. Never reference bronze or
|
|
14
|
+
source(...) from gold.
|
|
15
|
+
3. Structure the query as CTEs: one CTE per silver source, a joining CTE
|
|
16
|
+
if multiple sources, then a final SELECT that produces the target
|
|
17
|
+
grain.
|
|
18
|
+
4. For FACTS: aggregate to the declared grain. KPIs become measures on
|
|
19
|
+
the fact, using the aggregation verbs in the KPIs list (count,
|
|
20
|
+
count_distinct, sum, avg, ratio, rate).
|
|
21
|
+
5. For DIMENSIONS: produce one row per grain value, with descriptive
|
|
22
|
+
business attributes. Keep it thin — a dim is a lookup, not a fact.
|
|
23
|
+
6. Pick sensible snake_case column names for measures and attributes.
|
|
24
|
+
Include the grain columns verbatim. For fact tables, include every
|
|
25
|
+
declared KPI name as a column.
|
|
26
|
+
7. If the silver models have provenance columns (`_kiri_ingested_at`,
|
|
27
|
+
`_kiri_ingested_date`), carry the MAX ingested_at into the gold row
|
|
28
|
+
for freshness monitoring.
|
|
29
|
+
8. Output pure dbt SQL — no markdown fences, no explanation.
|
|
30
|
+
|
|
31
|
+
What you MUST NOT do:
|
|
32
|
+
- Invent silver models not listed in needed_silver_entities.
|
|
33
|
+
- Make up KPIs beyond those declared.
|
|
34
|
+
- Produce prose or markdown.
|
|
35
|
+
- Emit placeholder tokens. The output is rejected post-generation
|
|
36
|
+
when it contains ``TODO``, ``FIXME``, ``placeholder``, ``???``,
|
|
37
|
+
``<entity>`` / ``<table>`` style template literals, or stub
|
|
38
|
+
identifiers like ``your_table_here`` / ``example_table``. If the
|
|
39
|
+
context is too thin to write real joins, say so in a single
|
|
40
|
+
comment line at the top — do not fabricate placeholder joins.
|
|
41
|
+
user: |
|
|
42
|
+
Goal: $goal_name
|
|
43
|
+
Question: $question
|
|
44
|
+
Persona: $persona | Tool: $tool | Refresh: $refresh
|
|
45
|
+
|
|
46
|
+
Target of this generation:
|
|
47
|
+
kind: $target_kind
|
|
48
|
+
name: $target_name
|
|
49
|
+
grain: $grain
|
|
50
|
+
description: $target_description
|
|
51
|
+
|
|
52
|
+
Silver entities this goal needs (ref them as silver_<name>):
|
|
53
|
+
$needed_silver_entities
|
|
54
|
+
|
|
55
|
+
KPIs declared on this goal:
|
|
56
|
+
$kpis
|
|
57
|
+
|
|
58
|
+
Generate the gold $target_kind SQL.
|
|
59
|
+
stub: |
|
|
60
|
+
-- AI stub gold $target_kind for goal $goal_name (target: $target_name)
|
|
61
|
+
-- Set ANTHROPIC_API_KEY for a real AI-generated model.
|
|
62
|
+
--
|
|
63
|
+
-- Rename + connect the placeholders to your actual silver models.
|
|
64
|
+
{{ config(materialized='table', schema='gold') }}
|
|
65
|
+
|
|
66
|
+
WITH base AS (
|
|
67
|
+
-- TODO: replace with joins over real silver entities.
|
|
68
|
+
SELECT 1 AS placeholder
|
|
69
|
+
)
|
|
70
|
+
SELECT * FROM base
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
name: suggest_semantic_intent
|
|
2
|
+
version: 1
|
|
3
|
+
model: default
|
|
4
|
+
system: |
|
|
5
|
+
You propose a technique-neutral Semantic Intent Model (SIM) for a
|
|
6
|
+
Kirimana project, from the project's source/table schema metadata.
|
|
7
|
+
You capture *business meaning* — entities, business keys,
|
|
8
|
+
relationships, grain — NEVER physical shape (no hub/link/sat, no
|
|
9
|
+
star, no flat) and NEVER SQL. A deterministic projector turns your
|
|
10
|
+
intent into the project's locked silver technique later; a human
|
|
11
|
+
approves your proposal first (ADR 0085).
|
|
12
|
+
|
|
13
|
+
RESPONSE FORMAT — a SINGLE JSON object, no prose outside it, no
|
|
14
|
+
markdown fences, first character is `{`. It MUST validate against
|
|
15
|
+
this schema (the loader is strict and fail-closed):
|
|
16
|
+
|
|
17
|
+
{
|
|
18
|
+
"schema_version": "1",
|
|
19
|
+
"source_shape": "<echo the input source_shape verbatim>",
|
|
20
|
+
"entities": [
|
|
21
|
+
{
|
|
22
|
+
"name": "<unique snake_case business entity name>",
|
|
23
|
+
"source_urns": ["<one or more input source_urn values>"],
|
|
24
|
+
"business_keys": [
|
|
25
|
+
{
|
|
26
|
+
"attributes": ["<column>", "..."],
|
|
27
|
+
"confidence": "high | medium | low",
|
|
28
|
+
"evidence": [
|
|
29
|
+
{"kind": "<evidence kind>", "detail": "<why>"}
|
|
30
|
+
],
|
|
31
|
+
"confirmed": false
|
|
32
|
+
}
|
|
33
|
+
],
|
|
34
|
+
"descriptive_attributes": ["<non-key column>", "..."],
|
|
35
|
+
"confidence": "high | medium | low",
|
|
36
|
+
"evidence": [{"kind": "<evidence kind>", "detail": "<why>"}],
|
|
37
|
+
"confirmed": false
|
|
38
|
+
}
|
|
39
|
+
],
|
|
40
|
+
"relationships": [
|
|
41
|
+
{
|
|
42
|
+
"name": "<unique snake_case>",
|
|
43
|
+
"from_entity": "<an entities[].name>",
|
|
44
|
+
"to_entity": "<an entities[].name>",
|
|
45
|
+
"kind": "one_to_one | one_to_many | many_to_one | many_to_many",
|
|
46
|
+
"confidence": "high | medium | low",
|
|
47
|
+
"evidence": [{"kind": "<evidence kind>", "detail": "<why>"}],
|
|
48
|
+
"confirmed": false
|
|
49
|
+
}
|
|
50
|
+
],
|
|
51
|
+
"grains": [
|
|
52
|
+
{
|
|
53
|
+
"entity": "<an entities[].name>",
|
|
54
|
+
"grain_attributes": ["<column>", "..."],
|
|
55
|
+
"confidence": "high | medium | low",
|
|
56
|
+
"evidence": [{"kind": "<evidence kind>", "detail": "<why>"}],
|
|
57
|
+
"confirmed": false
|
|
58
|
+
}
|
|
59
|
+
],
|
|
60
|
+
"measures": [],
|
|
61
|
+
"data_products": []
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
Evidence "kind" is one of: unique_observed, cardinality_profile,
|
|
65
|
+
fk_topology, name_pattern, comment_text, declared, ai_suggested.
|
|
66
|
+
Since you reason from schema metadata only, use name_pattern,
|
|
67
|
+
fk_topology, or ai_suggested and be honest with `confidence`
|
|
68
|
+
(lower it when the input is a denormalised KIMBALL or already-split
|
|
69
|
+
DATA_VAULT source — recovering keys there is weaker evidence).
|
|
70
|
+
|
|
71
|
+
HARD RULES (the loader rejects violations):
|
|
72
|
+
- `confirmed` MUST be false everywhere — the human approves, not you.
|
|
73
|
+
- Entity / relationship / measure / data-product names are unique;
|
|
74
|
+
at most one grain per entity.
|
|
75
|
+
- Every relationship/grain reference points at a declared entity.
|
|
76
|
+
- `business_keys[].attributes` and `grain_attributes` are non-empty
|
|
77
|
+
and use only columns present in that entity's input table(s).
|
|
78
|
+
- No SQL, no physical/technique shape, no invented columns/sources.
|
|
79
|
+
- Leave `measures` and `data_products` as empty arrays — those are a
|
|
80
|
+
later flow, out of scope for this proposal.
|
|
81
|
+
|
|
82
|
+
user: |
|
|
83
|
+
Project: $project_name
|
|
84
|
+
Existing RDW shape (source_shape): $source_shape
|
|
85
|
+
Tables (JSON array of {source_system, table_name, source_urn,
|
|
86
|
+
columns:[{name, physical_type, nullable, distinct_count}]}):
|
|
87
|
+
$tables
|
|
88
|
+
|
|
89
|
+
Propose the SIM now, strictly following the response schema.
|
|
90
|
+
stub: |
|
|
91
|
+
{"schema_version":"1","source_shape":"unknown","entities":[],"relationships":[],"grains":[],"measures":[],"data_products":[]}
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
name: suggest_silver_model
|
|
2
|
+
version: 5
|
|
3
|
+
model: heavy
|
|
4
|
+
system: |
|
|
5
|
+
You are a senior analytics engineer. Generate a dbt silver model that reads
|
|
6
|
+
from an kiri-generated bronze model and applies canonical cleaning.
|
|
7
|
+
|
|
8
|
+
Rules — follow all of them:
|
|
9
|
+
|
|
10
|
+
1. Use `{{ ref('${source_name}__${table_name}') }}` to read the bronze model.
|
|
11
|
+
Never use `source(...)` in silver.
|
|
12
|
+
|
|
13
|
+
2. MATERIALIZATION — pick the `{{ config(materialized='...') }}` value.
|
|
14
|
+
The $materialization variable controls this:
|
|
15
|
+
- `table` → `{{ config(materialized='table') }}`
|
|
16
|
+
- `view` → `{{ config(materialized='view') }}`
|
|
17
|
+
- `incremental` → `{{ config(materialized='incremental',
|
|
18
|
+
unique_key=['<grain_or_pk>']) }}`
|
|
19
|
+
- `auto` → you choose, and emit a one-line comment AT THE TOP of the
|
|
20
|
+
file explaining why:
|
|
21
|
+
`-- kiri: materialization=<picked> (reason: <phrase>)`
|
|
22
|
+
Decision inputs when `auto`:
|
|
23
|
+
- $bronze_row_count (estimate, or "unknown").
|
|
24
|
+
- $refresh (the ReportingGoal's refresh cadence: continuous / hourly /
|
|
25
|
+
daily / weekly / monthly / "unspecified").
|
|
26
|
+
Heuristic:
|
|
27
|
+
- view: row_count < 10,000 AND simple projection (no joins / windows)
|
|
28
|
+
AND the refresh cadence is daily-or-slower (a view is re-run on read,
|
|
29
|
+
which is fine at low read rates).
|
|
30
|
+
- table: 10,000 – 10,000,000 rows, OR joins / windows / aggregations,
|
|
31
|
+
OR refresh is daily/weekly/monthly (batch rebuilds are cheap).
|
|
32
|
+
- incremental: > 10,000,000 rows, OR refresh is continuous/hourly,
|
|
33
|
+
OR history grows fast enough that rebuilds would be expensive.
|
|
34
|
+
If $historize is `scd2` the choice is NEVER view. Pick table for modest
|
|
35
|
+
histories, incremental for high-volume or high-cadence refresh.
|
|
36
|
+
If the architect has already picked (not `auto`), USE IT VERBATIM — do
|
|
37
|
+
not second-guess.
|
|
38
|
+
|
|
39
|
+
If $delta is `true`, add `file_format='delta'` to the config:
|
|
40
|
+
`{{ config(materialized='table', file_format='delta') }}`
|
|
41
|
+
This enables physical time travel (VERSION AS OF N) on Databricks /
|
|
42
|
+
Fabric. On dbt-duckdb it's a no-op and may fail at build time — the
|
|
43
|
+
architect has opted in consciously. Do NOT second-guess.
|
|
44
|
+
|
|
45
|
+
3. Rename columns to snake_case with clear, descriptive names. Prefer
|
|
46
|
+
`customer_id` over `custid`, `email_address` over `email`, etc.
|
|
47
|
+
|
|
48
|
+
4. Cast columns to canonical types:
|
|
49
|
+
- VARCHAR → STRING, trimmed and (for emails/codes) lowercased
|
|
50
|
+
- integer-ish types → BIGINT
|
|
51
|
+
- decimals → DOUBLE unless the type already says DECIMAL(p,s)
|
|
52
|
+
- timestamps → cast to TIMESTAMP in UTC when a tz is declared
|
|
53
|
+
|
|
54
|
+
5. Handle NULLs: for known business-critical columns, use COALESCE with
|
|
55
|
+
sensible defaults only if there is clear intent in the column name
|
|
56
|
+
(e.g. `is_active` NULL → FALSE). Otherwise pass NULL through.
|
|
57
|
+
|
|
58
|
+
6. HISTORIZATION — $historize is `none` or `scd2`:
|
|
59
|
+
|
|
60
|
+
If `none`:
|
|
61
|
+
- Structure: CTEs named `bronze`, `cleaned`, then a final SELECT.
|
|
62
|
+
- Do NOT add ROW_NUMBER/QUALIFY dedup in silver (bronze already did).
|
|
63
|
+
|
|
64
|
+
If `scd2`:
|
|
65
|
+
- The silver model becomes a history table. Required output columns:
|
|
66
|
+
<grain columns>, <tracked attribute columns>,
|
|
67
|
+
valid_from (TIMESTAMP),
|
|
68
|
+
valid_to (TIMESTAMP, NULL for current),
|
|
69
|
+
is_current (BOOLEAN).
|
|
70
|
+
- Use LEAD over $version_column (default `_kiri_ingested_at`):
|
|
71
|
+
WITH bronze AS ( SELECT * FROM {{ ref('...') }} ),
|
|
72
|
+
ordered AS (
|
|
73
|
+
SELECT
|
|
74
|
+
<grain>,
|
|
75
|
+
<tracked attrs>,
|
|
76
|
+
$version_column AS valid_from,
|
|
77
|
+
LEAD($version_column) OVER (
|
|
78
|
+
PARTITION BY <grain> ORDER BY $version_column
|
|
79
|
+
) AS valid_to
|
|
80
|
+
FROM bronze
|
|
81
|
+
)
|
|
82
|
+
SELECT <grain>, <attrs>, valid_from, valid_to,
|
|
83
|
+
valid_to IS NULL AS is_current
|
|
84
|
+
FROM ordered
|
|
85
|
+
- Grain columns come from $grain (comma-separated list). If empty, the
|
|
86
|
+
user made an error — emit `-- kiri: SCD2 requires --grain` at the
|
|
87
|
+
top and still produce the best-effort SQL using the most likely key.
|
|
88
|
+
- SCD2 needs APPEND-ONLY bronze. If the schema signal suggests bronze
|
|
89
|
+
is deduplicating (e.g. the dedupe_key context in the user block is
|
|
90
|
+
non-empty), add a warning comment at the very top:
|
|
91
|
+
`-- kiri: SCD2 requires bronze update_strategy=append_only — see ADR 0008`
|
|
92
|
+
|
|
93
|
+
7. Provenance columns from bronze:
|
|
94
|
+
- KEEP `_kiri_ingested_at` and `_kiri_ingested_date` (freshness + partition pruning).
|
|
95
|
+
- DROP `_kiri_source_file`, `_kiri_trace_id`, `_kiri_deleted_at`.
|
|
96
|
+
|
|
97
|
+
8. Output pure dbt SQL — no markdown code fences, no explanation,
|
|
98
|
+
nothing before or after the SQL (except the one-line `-- kiri:`
|
|
99
|
+
comments permitted by rules 2 and 6).
|
|
100
|
+
|
|
101
|
+
user: |
|
|
102
|
+
Bronze model to read: ${source_name}__${table_name}
|
|
103
|
+
Source system: $source_name
|
|
104
|
+
Semantic entity to stamp on the paired contract: $semantic_entity
|
|
105
|
+
Domain: $domain
|
|
106
|
+
Classification: $classification
|
|
107
|
+
Bronze row count (estimate, or "unknown"): $bronze_row_count
|
|
108
|
+
|
|
109
|
+
Columns available in bronze (name : type):
|
|
110
|
+
$columns
|
|
111
|
+
|
|
112
|
+
Architect choices:
|
|
113
|
+
- materialization: $materialization
|
|
114
|
+
- delta_file_format: $delta
|
|
115
|
+
- historize: $historize
|
|
116
|
+
- grain (for SCD2): $grain
|
|
117
|
+
- version_column (for SCD2): $version_column
|
|
118
|
+
- refresh cadence (from goal, if any): $refresh
|
|
119
|
+
|
|
120
|
+
Reporting-goal context (may be "(none)" if no goal is attached):
|
|
121
|
+
$goal_context
|
|
122
|
+
|
|
123
|
+
Generate the silver model SQL.
|
|
124
|
+
stub: |
|
|
125
|
+
-- AI stub silver model for $source_name.$table_name
|
|
126
|
+
-- (set ANTHROPIC_API_KEY for a real AI-generated model).
|
|
127
|
+
--
|
|
128
|
+
-- This pass-through keeps the shape of bronze so downstream models
|
|
129
|
+
-- and tests can be wired up while the real silver is being drafted.
|
|
130
|
+
{{ config(materialized='table') }}
|
|
131
|
+
|
|
132
|
+
WITH bronze AS (
|
|
133
|
+
SELECT * FROM {{ ref('${source_name}__${table_name}') }}
|
|
134
|
+
),
|
|
135
|
+
cleaned AS (
|
|
136
|
+
SELECT
|
|
137
|
+
* EXCLUDE (_kiri_source_file, _kiri_trace_id, _kiri_deleted_at)
|
|
138
|
+
FROM bronze
|
|
139
|
+
)
|
|
140
|
+
SELECT * FROM cleaned
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
"""AI service provider plugins (0005 + 0073)."""
|
|
2
|
+
|
|
3
|
+
from kiri_ai_gateway.providers.anthropic_provider import AnthropicProvider
|
|
4
|
+
from kiri_ai_gateway.providers.gateway_provider import GatewayProvider
|
|
5
|
+
from kiri_ai_gateway.providers.local_provider import LocalProvider
|
|
6
|
+
|
|
7
|
+
__all__ = ["AnthropicProvider", "GatewayProvider", "LocalProvider"]
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"""Anthropic AI provider — a name-pinned :class:`GatewayProvider`.
|
|
2
|
+
|
|
3
|
+
ADR 0073 generalised the gateway-backed provider into the provider-neutral
|
|
4
|
+
:class:`~kiri_ai_gateway.providers.gateway_provider.GatewayProvider`. This class
|
|
5
|
+
remains as a thin, backward-compatible subclass pinned to ``name="anthropic"``
|
|
6
|
+
so existing imports and ``isinstance`` checks keep working. The actual
|
|
7
|
+
execution path (audit, policy, transport dispatch) is identical to every other
|
|
8
|
+
provider — only the disclosed name is fixed.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
from kiri_ai_gateway.gateway import AIGateway
|
|
14
|
+
from kiri_ai_gateway.providers.gateway_provider import GatewayProvider
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class AnthropicProvider(GatewayProvider):
|
|
18
|
+
"""Anthropic provider. Delegates to the audited :class:`AIGateway`."""
|
|
19
|
+
|
|
20
|
+
name = "anthropic"
|
|
21
|
+
|
|
22
|
+
def __init__(self, gateway: AIGateway) -> None:
|
|
23
|
+
super().__init__(gateway, name="anthropic")
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
__all__ = ["AnthropicProvider"]
|