kiri-cli 1.0.0b3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of kiri-cli might be problematic. Click here for more details.

Files changed (743) hide show
  1. kiri_adapters/__init__.py +193 -0
  2. kiri_adapters/backends/__init__.py +0 -0
  3. kiri_adapters/backends/databricks.manifest.yml +55 -0
  4. kiri_adapters/backends/databricks.py +2980 -0
  5. kiri_adapters/backends/fabric_dw.manifest.yml +17 -0
  6. kiri_adapters/backends/fabric_dw.py +307 -0
  7. kiri_adapters/backends/fabric_lakehouse.manifest.yml +19 -0
  8. kiri_adapters/backends/fabric_lakehouse.py +365 -0
  9. kiri_adapters/backends/iceberg.manifest.yml +17 -0
  10. kiri_adapters/backends/iceberg.py +451 -0
  11. kiri_adapters/backends/localduckdb.manifest.yml +43 -0
  12. kiri_adapters/backends/localduckdb.py +1295 -0
  13. kiri_adapters/backends/mssql.manifest.yml +17 -0
  14. kiri_adapters/backends/mssql.py +366 -0
  15. kiri_adapters/backends/postgres.manifest.yml +17 -0
  16. kiri_adapters/backends/postgres.py +434 -0
  17. kiri_adapters/backends/trino.manifest.yml +17 -0
  18. kiri_adapters/backends/trino.py +542 -0
  19. kiri_adapters/base.py +892 -0
  20. kiri_adapters/doctor_checks.py +81 -0
  21. kiri_adapters/dv_planner.py +1205 -0
  22. kiri_adapters/dv_planner_bridge.py +265 -0
  23. kiri_adapters/dv_planner_pit.py +265 -0
  24. kiri_adapters/dv_planner_ref.py +403 -0
  25. kiri_adapters/dv_planner_sat_variants.py +668 -0
  26. kiri_adapters/framework.py +112 -0
  27. kiri_adapters/incident/__init__.py +30 -0
  28. kiri_adapters/incident/_http.py +121 -0
  29. kiri_adapters/incident/base.py +89 -0
  30. kiri_adapters/incident/jira.py +365 -0
  31. kiri_adapters/incident/servicenow.py +314 -0
  32. kiri_adapters/incident/zendesk.py +285 -0
  33. kiri_adapters/manifest.py +466 -0
  34. kiri_adapters/orchestration/__init__.py +30 -0
  35. kiri_adapters/orchestration/airflow.py +117 -0
  36. kiri_adapters/orchestration/artifact_emitter.py +99 -0
  37. kiri_adapters/orchestration/autoloader_emitter.py +141 -0
  38. kiri_adapters/orchestration/azure_data_factory.py +137 -0
  39. kiri_adapters/orchestration/base.py +626 -0
  40. kiri_adapters/orchestration/databricks_workflows.py +1768 -0
  41. kiri_adapters/orchestration/k8s_cron.py +555 -0
  42. kiri_adapters/orchestration/lifecycle.py +222 -0
  43. kiri_adapters/orchestration/pipeline_compat.py +145 -0
  44. kiri_adapters/orchestration/runtime_python.py +188 -0
  45. kiri_adapters/paths.py +38 -0
  46. kiri_adapters/protocols/__init__.py +64 -0
  47. kiri_adapters/protocols/ai_service/__init__.py +7 -0
  48. kiri_adapters/protocols/ai_service/conformance.py +120 -0
  49. kiri_adapters/protocols/base.py +220 -0
  50. kiri_adapters/protocols/catalog/__init__.py +7 -0
  51. kiri_adapters/protocols/catalog/conformance.py +138 -0
  52. kiri_adapters/protocols/framework/__init__.py +7 -0
  53. kiri_adapters/protocols/framework/conformance.py +98 -0
  54. kiri_adapters/protocols/incident/__init__.py +7 -0
  55. kiri_adapters/protocols/incident/conformance.py +64 -0
  56. kiri_adapters/protocols/orchestration/__init__.py +7 -0
  57. kiri_adapters/protocols/orchestration/conformance.py +84 -0
  58. kiri_adapters/protocols/platform/__init__.py +7 -0
  59. kiri_adapters/protocols/platform/conformance.py +120 -0
  60. kiri_adapters/protocols/sql_dialect/__init__.py +7 -0
  61. kiri_adapters/protocols/sql_dialect/conformance.py +91 -0
  62. kiri_adapters/protocols/vault/__init__.py +7 -0
  63. kiri_adapters/protocols/vault/conformance.py +65 -0
  64. kiri_adapters/py.typed +0 -0
  65. kiri_adapters/registry.py +104 -0
  66. kiri_adapters/semantic/__init__.py +60 -0
  67. kiri_adapters/semantic/apply.py +274 -0
  68. kiri_adapters/semantic/base.py +530 -0
  69. kiri_adapters/semantic/cube.py +280 -0
  70. kiri_adapters/semantic/dbt_semantic.py +92 -0
  71. kiri_adapters/semantic/metric_views.py +249 -0
  72. kiri_adapters/semantic/metricflow.py +345 -0
  73. kiri_adapters/semantic/snowflake_views.py +75 -0
  74. kiri_adapters/sql_dialect.py +1592 -0
  75. kiri_adapters/tsql_to_spark.py +653 -0
  76. kiri_ai_gateway/__init__.py +147 -0
  77. kiri_ai_gateway/audit.py +334 -0
  78. kiri_ai_gateway/batch.py +293 -0
  79. kiri_ai_gateway/config.py +165 -0
  80. kiri_ai_gateway/gateway.py +642 -0
  81. kiri_ai_gateway/placeholder_guard.py +145 -0
  82. kiri_ai_gateway/prices/__init__.py +77 -0
  83. kiri_ai_gateway/prices/prices.yaml +64 -0
  84. kiri_ai_gateway/prompts/__init__.py +54 -0
  85. kiri_ai_gateway/prompts/airbyte_source_setup.yaml +212 -0
  86. kiri_ai_gateway/prompts/bdv_translation.yaml +90 -0
  87. kiri_ai_gateway/prompts/contract_draft_mvp.yaml +117 -0
  88. kiri_ai_gateway/prompts/describe_table.yaml +19 -0
  89. kiri_ai_gateway/prompts/design_gold_from_question.yaml +130 -0
  90. kiri_ai_gateway/prompts/detect_pii.yaml +59 -0
  91. kiri_ai_gateway/prompts/discover_classify_column.yaml +69 -0
  92. kiri_ai_gateway/prompts/discover_describe_table.yaml +61 -0
  93. kiri_ai_gateway/prompts/dv_quality_summary.yaml +36 -0
  94. kiri_ai_gateway/prompts/incident_assist.yaml +58 -0
  95. kiri_ai_gateway/prompts/infer_source_schema.yaml +31 -0
  96. kiri_ai_gateway/prompts/ingest_backend_chooser.yaml +116 -0
  97. kiri_ai_gateway/prompts/interactive_source_edit.yaml +94 -0
  98. kiri_ai_gateway/prompts/map_needed_entity_to_source.yaml +72 -0
  99. kiri_ai_gateway/prompts/meet_kiri.yaml +127 -0
  100. kiri_ai_gateway/prompts/native_source_setup.yaml +211 -0
  101. kiri_ai_gateway/prompts/source_grouping_reason.yaml +29 -0
  102. kiri_ai_gateway/prompts/suggest_attribute_classification.yaml +42 -0
  103. kiri_ai_gateway/prompts/suggest_data_tests.yaml +71 -0
  104. kiri_ai_gateway/prompts/suggest_gold_dimension.yaml +108 -0
  105. kiri_ai_gateway/prompts/suggest_gold_fact.yaml +107 -0
  106. kiri_ai_gateway/prompts/suggest_gold_model.yaml +70 -0
  107. kiri_ai_gateway/prompts/suggest_semantic_intent.yaml +91 -0
  108. kiri_ai_gateway/prompts/suggest_silver_model.yaml +140 -0
  109. kiri_ai_gateway/providers/__init__.py +7 -0
  110. kiri_ai_gateway/providers/anthropic_provider.py +26 -0
  111. kiri_ai_gateway/providers/gateway_provider.py +188 -0
  112. kiri_ai_gateway/providers/local_provider.py +177 -0
  113. kiri_ai_gateway/py.typed +0 -0
  114. kiri_ai_gateway/service.py +283 -0
  115. kiri_ai_gateway/transport.py +103 -0
  116. kiri_ai_gateway/transports/__init__.py +12 -0
  117. kiri_ai_gateway/transports/anthropic_transport.py +73 -0
  118. kiri_ai_gateway/transports/ollama_transport.py +82 -0
  119. kiri_ai_gateway/transports/openai_compatible_transport.py +125 -0
  120. kiri_ai_gateway/workspace_policy.py +52 -0
  121. kiri_catalog/__init__.py +199 -0
  122. kiri_catalog/annotation_plan.py +183 -0
  123. kiri_catalog/appearances.py +155 -0
  124. kiri_catalog/backends/__init__.py +79 -0
  125. kiri_catalog/backends/base.py +167 -0
  126. kiri_catalog/backends/memory.py +241 -0
  127. kiri_catalog/backends/postgres.py +497 -0
  128. kiri_catalog/backends/redacting.py +195 -0
  129. kiri_catalog/backends/sqlite.py +480 -0
  130. kiri_catalog/context.py +94 -0
  131. kiri_catalog/federation/__init__.py +114 -0
  132. kiri_catalog/federation/api.py +200 -0
  133. kiri_catalog/federation/cache.py +149 -0
  134. kiri_catalog/federation/emit.py +209 -0
  135. kiri_catalog/federation/fs_resolver.py +295 -0
  136. kiri_catalog/federation/http_resolver.py +486 -0
  137. kiri_catalog/federation/metrics.py +123 -0
  138. kiri_catalog/federation/release.py +342 -0
  139. kiri_catalog/federation/resolver.py +332 -0
  140. kiri_catalog/federation/schema.py +460 -0
  141. kiri_catalog/importer.py +668 -0
  142. kiri_catalog/lineage_drift.py +280 -0
  143. kiri_catalog/lineage_graph.py +176 -0
  144. kiri_catalog/lineage_graph_explain.py +96 -0
  145. kiri_catalog/lineage_graph_mermaid.py +230 -0
  146. kiri_catalog/observed_uc_lineage.py +216 -0
  147. kiri_catalog/pii_detect.py +176 -0
  148. kiri_catalog/push.py +122 -0
  149. kiri_catalog/py.typed +0 -0
  150. kiri_catalog/redact.py +101 -0
  151. kiri_catalog/review_actions.py +119 -0
  152. kiri_catalog/schema/__init__.py +70 -0
  153. kiri_catalog/schema/asset.py +142 -0
  154. kiri_catalog/schema/column.py +247 -0
  155. kiri_catalog/schema/enums.py +195 -0
  156. kiri_catalog/schema/job.py +51 -0
  157. kiri_catalog/schema/lineage.py +33 -0
  158. kiri_catalog/schema/policy.py +69 -0
  159. kiri_catalog/schema/primitives.py +90 -0
  160. kiri_catalog/schema/review_state.py +111 -0
  161. kiri_catalog/schema/urn.py +45 -0
  162. kiri_catalog/schema_discovery.py +160 -0
  163. kiri_catalog/sinks/__init__.py +64 -0
  164. kiri_catalog/sinks/base.py +178 -0
  165. kiri_catalog/sinks/horizon.py +195 -0
  166. kiri_catalog/sinks/purview.py +216 -0
  167. kiri_catalog/sinks/unity.py +352 -0
  168. kiri_catalog/sinks/unity_executor.py +543 -0
  169. kiri_catalog/sources/__init__.py +42 -0
  170. kiri_catalog/sources/base.py +104 -0
  171. kiri_catalog/sources/unity.py +465 -0
  172. kiri_catalog/steward_writeback.py +158 -0
  173. kiri_catalog/tag_sql.py +99 -0
  174. kiri_cli/LICENSE +105 -0
  175. kiri_cli/__init__.py +1 -0
  176. kiri_cli/_apply_cli.py +63 -0
  177. kiri_cli/_apply_reporter.py +38 -0
  178. kiri_cli/_closed_menu.py +36 -0
  179. kiri_cli/_fixture_option.py +18 -0
  180. kiri_cli/_layer_detect.py +158 -0
  181. kiri_cli/_project_option.py +79 -0
  182. kiri_cli/adapter_cmd.py +192 -0
  183. kiri_cli/ai_batch_cmd.py +258 -0
  184. kiri_cli/ai_cmd.py +352 -0
  185. kiri_cli/approve_cmd.py +659 -0
  186. kiri_cli/attr_cmd.py +575 -0
  187. kiri_cli/audit_cmd.py +348 -0
  188. kiri_cli/audit_correlate_cmd.py +648 -0
  189. kiri_cli/audit_join.py +437 -0
  190. kiri_cli/audit_join_cmd.py +160 -0
  191. kiri_cli/auto_cmd.py +97 -0
  192. kiri_cli/azure_cmd.py +243 -0
  193. kiri_cli/catalog.py +1293 -0
  194. kiri_cli/cli_reference_live.py +147 -0
  195. kiri_cli/compile_cmd.py +493 -0
  196. kiri_cli/compliance_cmd.py +221 -0
  197. kiri_cli/contract_approval_cmd.py +94 -0
  198. kiri_cli/contract_cmd.py +64 -0
  199. kiri_cli/contract_codeowners_cmd.py +162 -0
  200. kiri_cli/contract_diff.py +111 -0
  201. kiri_cli/contract_emit.py +90 -0
  202. kiri_cli/contract_export.py +192 -0
  203. kiri_cli/contract_lint.py +1505 -0
  204. kiri_cli/contract_merge_cmd.py +264 -0
  205. kiri_cli/contract_new.py +326 -0
  206. kiri_cli/contract_promote.py +216 -0
  207. kiri_cli/contract_scaffold_cmd.py +134 -0
  208. kiri_cli/contract_schema_export.py +69 -0
  209. kiri_cli/contract_show.py +150 -0
  210. kiri_cli/contract_stubs.py +60 -0
  211. kiri_cli/contract_sync_from_live.py +327 -0
  212. kiri_cli/contract_validate.py +154 -0
  213. kiri_cli/contract_verify_live.py +242 -0
  214. kiri_cli/cost_cmd.py +243 -0
  215. kiri_cli/databricks_health.py +1254 -0
  216. kiri_cli/databricks_setup_cmd.py +712 -0
  217. kiri_cli/discover_cmd.py +600 -0
  218. kiri_cli/docs_cmd.py +35 -0
  219. kiri_cli/doctor_cmd.py +145 -0
  220. kiri_cli/dv_cmd.py +814 -0
  221. kiri_cli/dv_conformance_cmd.py +154 -0
  222. kiri_cli/dv_install.py +493 -0
  223. kiri_cli/dv_migrate_cmd.py +183 -0
  224. kiri_cli/dv_test_cmd.py +175 -0
  225. kiri_cli/federation_cmd.py +315 -0
  226. kiri_cli/fk_cmd.py +160 -0
  227. kiri_cli/flows_cmd.py +910 -0
  228. kiri_cli/governance_cmd.py +358 -0
  229. kiri_cli/incident_api_cmd.py +180 -0
  230. kiri_cli/incident_cmd.py +410 -0
  231. kiri_cli/incident_setup_cmd.py +239 -0
  232. kiri_cli/ingest.py +427 -0
  233. kiri_cli/ingest_discover.py +480 -0
  234. kiri_cli/ingest_new_source.py +386 -0
  235. kiri_cli/intent_cmd.py +122 -0
  236. kiri_cli/invariants_cmd.py +118 -0
  237. kiri_cli/kiri_banner.py +161 -0
  238. kiri_cli/kpi_cmd.py +222 -0
  239. kiri_cli/layer_cmd.py +464 -0
  240. kiri_cli/lineage.py +601 -0
  241. kiri_cli/main.py +2472 -0
  242. kiri_cli/maturity_cmd.py +222 -0
  243. kiri_cli/migrate_cmd.py +1347 -0
  244. kiri_cli/onboard_cmd.py +223 -0
  245. kiri_cli/pack_cmd.py +496 -0
  246. kiri_cli/pipeline_cmd.py +127 -0
  247. kiri_cli/pre_classify_cmd.py +274 -0
  248. kiri_cli/project_cmd.py +592 -0
  249. kiri_cli/py.typed +0 -0
  250. kiri_cli/quality_cmd.py +295 -0
  251. kiri_cli/rbac.py +139 -0
  252. kiri_cli/rbac_cmd.py +291 -0
  253. kiri_cli/release.py +850 -0
  254. kiri_cli/release_semver.py +265 -0
  255. kiri_cli/repo_cmd.py +155 -0
  256. kiri_cli/retention_cmd.py +333 -0
  257. kiri_cli/sample_data/__init__.py +12 -0
  258. kiri_cli/sample_data/customers.contract.yml +47 -0
  259. kiri_cli/sample_data/customers.csv +25 -0
  260. kiri_cli/scaffold_cmd.py +721 -0
  261. kiri_cli/schema_cmd.py +185 -0
  262. kiri_cli/semantic_cmd.py +605 -0
  263. kiri_cli/silver_cmd.py +348 -0
  264. kiri_cli/skills_cmd.py +243 -0
  265. kiri_cli/skills_data/README.md +51 -0
  266. kiri_cli/skills_data/add-source/SKILL.md +116 -0
  267. kiri_cli/skills_data/azure-deploy/SKILL.md +261 -0
  268. kiri_cli/skills_data/azure-status/SKILL.md +111 -0
  269. kiri_cli/skills_data/compliance-check/SKILL.md +99 -0
  270. kiri_cli/skills_data/databricks-build-pipeline/SKILL.md +112 -0
  271. kiri_cli/skills_data/databricks-project-setup/SKILL.md +157 -0
  272. kiri_cli/skills_data/debug-apply/SKILL.md +186 -0
  273. kiri_cli/skills_data/draft-gold-model/SKILL.md +203 -0
  274. kiri_cli/skills_data/explain-kpi/SKILL.md +158 -0
  275. kiri_cli/skills_data/incident-coach/SKILL.md +135 -0
  276. kiri_cli/skills_data/incident-setup/SKILL.md +112 -0
  277. kiri_cli/skills_data/maturity-coach/SKILL.md +120 -0
  278. kiri_cli/skills_data/promote-contract/SKILL.md +91 -0
  279. kiri_cli/skills_data/review-attributes/SKILL.md +94 -0
  280. kiri_cli/sla_cmd.py +769 -0
  281. kiri_cli/source_cmd.py +909 -0
  282. kiri_cli/templates/__init__.py +6 -0
  283. kiri_cli/templates/dv4dbt_packages.yml.j2 +11 -0
  284. kiri_cli/upgrade_cmd.py +896 -0
  285. kiri_cli/use_cmd.py +120 -0
  286. kiri_cli/vault_cmd.py +241 -0
  287. kiri_cli/verify_generated.py +241 -0
  288. kiri_cli-1.0.0b3.dist-info/METADATA +212 -0
  289. kiri_cli-1.0.0b3.dist-info/RECORD +743 -0
  290. kiri_cli-1.0.0b3.dist-info/WHEEL +4 -0
  291. kiri_cli-1.0.0b3.dist-info/entry_points.txt +4 -0
  292. kiri_compliance/__init__.py +33 -0
  293. kiri_compliance/audit_coverage.py +354 -0
  294. kiri_compliance/controls.py +763 -0
  295. kiri_compliance/evidence.py +155 -0
  296. kiri_compliance/py.typed +0 -0
  297. kiri_compliance/quality_engine.py +246 -0
  298. kiri_compliance/renderer.py +195 -0
  299. kiri_compliance/report.py +44 -0
  300. kiri_compliance/schema.py +129 -0
  301. kiri_core/__init__.py +185 -0
  302. kiri_core/adapter_surfaces.py +149 -0
  303. kiri_core/ai_query_gate/__init__.py +92 -0
  304. kiri_core/ai_query_gate/audit.py +153 -0
  305. kiri_core/ai_query_gate/evaluator.py +484 -0
  306. kiri_core/ai_query_gate/models.py +162 -0
  307. kiri_core/ai_query_gate/parser.py +436 -0
  308. kiri_core/ai_query_gate/resolver.py +274 -0
  309. kiri_core/ai_query_gate/store.py +234 -0
  310. kiri_core/ai_query_gate/store_postgres.py +409 -0
  311. kiri_core/api_keys.py +243 -0
  312. kiri_core/apply/__init__.py +27 -0
  313. kiri_core/apply/conformance.py +99 -0
  314. kiri_core/apply/incident_router.py +176 -0
  315. kiri_core/apply/models.py +140 -0
  316. kiri_core/apply/reporter.py +46 -0
  317. kiri_core/apply/runner.py +3083 -0
  318. kiri_core/apply_log_store.py +155 -0
  319. kiri_core/approval_flow/__init__.py +133 -0
  320. kiri_core/approval_flow/integration.py +156 -0
  321. kiri_core/approval_flow/models.py +443 -0
  322. kiri_core/approval_flow/notifier.py +263 -0
  323. kiri_core/approval_flow/orchestrator.py +660 -0
  324. kiri_core/approval_flow/reconcile_signoff.py +53 -0
  325. kiri_core/approval_flow/store.py +220 -0
  326. kiri_core/approval_flow/store_jsonl.py +217 -0
  327. kiri_core/approval_flow/store_postgres.py +580 -0
  328. kiri_core/approval_policy.py +116 -0
  329. kiri_core/asgi_trace.py +160 -0
  330. kiri_core/compliance.py +802 -0
  331. kiri_core/cutover_approval.py +187 -0
  332. kiri_core/discovery/__init__.py +109 -0
  333. kiri_core/discovery/ai/__init__.py +37 -0
  334. kiri_core/discovery/ai/_payload.py +115 -0
  335. kiri_core/discovery/ai/budget.py +83 -0
  336. kiri_core/discovery/ai/classify.py +223 -0
  337. kiri_core/discovery/ai/describe.py +200 -0
  338. kiri_core/discovery/ai/errors.py +49 -0
  339. kiri_core/discovery/discover.py +648 -0
  340. kiri_core/discovery/heuristics/__init__.py +42 -0
  341. kiri_core/discovery/heuristics/classification.py +174 -0
  342. kiri_core/discovery/heuristics/medallion.py +157 -0
  343. kiri_core/discovery/heuristics/pii.py +92 -0
  344. kiri_core/discovery/introspector.py +636 -0
  345. kiri_core/discovery/introspector_databricks.py +396 -0
  346. kiri_core/discovery/introspector_postgres.py +394 -0
  347. kiri_core/discovery/introspector_snowflake.py +383 -0
  348. kiri_core/discovery/reporting/__init__.py +17 -0
  349. kiri_core/discovery/reporting/discovery_report.py +190 -0
  350. kiri_core/discovery/scaffold/__init__.py +41 -0
  351. kiri_core/discovery/scaffold/contract_writer.py +632 -0
  352. kiri_core/discovery/scaffold/provenance_overlay.py +143 -0
  353. kiri_core/discovery/scaffold/source_writer.py +211 -0
  354. kiri_core/discovery/scaffold/todo_marker.py +98 -0
  355. kiri_core/domain_policy.py +122 -0
  356. kiri_core/goal_coverage.py +82 -0
  357. kiri_core/gold_registry.py +211 -0
  358. kiri_core/health.py +192 -0
  359. kiri_core/incident.py +204 -0
  360. kiri_core/incident_builders.py +256 -0
  361. kiri_core/incident_log.py +418 -0
  362. kiri_core/intent/__init__.py +46 -0
  363. kiri_core/intent/agent_runner.py +82 -0
  364. kiri_core/intent/project_dv.py +261 -0
  365. kiri_core/intent/propose.py +132 -0
  366. kiri_core/intent/semantic_intent.py +446 -0
  367. kiri_core/invariants.py +301 -0
  368. kiri_core/kpi.py +373 -0
  369. kiri_core/latency.py +150 -0
  370. kiri_core/lifecycle/__init__.py +555 -0
  371. kiri_core/lifecycle/ai_index.py +453 -0
  372. kiri_core/lifecycle/auto_mode.py +172 -0
  373. kiri_core/lifecycle/bootstrap.py +316 -0
  374. kiri_core/lifecycle/cli_reference.py +1064 -0
  375. kiri_core/lifecycle/commit.py +258 -0
  376. kiri_core/lifecycle/compile_dispatch.py +296 -0
  377. kiri_core/lifecycle/compile_range.py +665 -0
  378. kiri_core/lifecycle/contract_inventory.py +354 -0
  379. kiri_core/lifecycle/contracts.py +250 -0
  380. kiri_core/lifecycle/doctor.py +412 -0
  381. kiri_core/lifecycle/git_remote.py +948 -0
  382. kiri_core/lifecycle/identity.py +147 -0
  383. kiri_core/lifecycle/ingestion_modes.py +215 -0
  384. kiri_core/lifecycle/layer_contract.py +452 -0
  385. kiri_core/lifecycle/layer_contract_migrate.py +969 -0
  386. kiri_core/lifecycle/layer_contract_parser.py +436 -0
  387. kiri_core/lifecycle/layer_contract_validator.py +2291 -0
  388. kiri_core/lifecycle/layout.py +298 -0
  389. kiri_core/lifecycle/onboarding.py +306 -0
  390. kiri_core/lifecycle/openlineage_emit.py +153 -0
  391. kiri_core/lifecycle/patch.py +231 -0
  392. kiri_core/lifecycle/plan.py +241 -0
  393. kiri_core/lifecycle/probe.py +204 -0
  394. kiri_core/lifecycle/project_context.py +334 -0
  395. kiri_core/lifecycle/project_status.py +532 -0
  396. kiri_core/lifecycle/scaffold.py +455 -0
  397. kiri_core/lifecycle/source_grouping.py +725 -0
  398. kiri_core/lifecycle/source_introspection.py +488 -0
  399. kiri_core/lifecycle/source_inventory.py +561 -0
  400. kiri_core/lifecycle/source_patch.py +353 -0
  401. kiri_core/lifecycle/sources.py +470 -0
  402. kiri_core/lifecycle/sync_from_live.py +727 -0
  403. kiri_core/lifecycle/validate.py +116 -0
  404. kiri_core/lifecycle/verify_annotations.py +391 -0
  405. kiri_core/lifecycle/verify_live.py +389 -0
  406. kiri_core/lifecycle/verify_metric_views.py +321 -0
  407. kiri_core/lifecycle/watermark.py +371 -0
  408. kiri_core/lifecycle/watermark_apply.py +656 -0
  409. kiri_core/lineage.py +418 -0
  410. kiri_core/loader.py +377 -0
  411. kiri_core/logging.py +374 -0
  412. kiri_core/maturity.py +592 -0
  413. kiri_core/migration_layer_policy.py +285 -0
  414. kiri_core/names.py +163 -0
  415. kiri_core/observed_lineage.py +62 -0
  416. kiri_core/odcs/__init__.py +15 -0
  417. kiri_core/otel_bootstrap.py +326 -0
  418. kiri_core/pipeline_ir.py +332 -0
  419. kiri_core/pipeline_loader.py +82 -0
  420. kiri_core/planning.py +110 -0
  421. kiri_core/project.py +1576 -0
  422. kiri_core/project_docs.py +199 -0
  423. kiri_core/py.typed +0 -0
  424. kiri_core/rate_limit.py +456 -0
  425. kiri_core/release_state/__init__.py +37 -0
  426. kiri_core/release_state/models.py +108 -0
  427. kiri_core/release_state/store.py +634 -0
  428. kiri_core/resilience/__init__.py +288 -0
  429. kiri_core/resilience/retry.py +173 -0
  430. kiri_core/runtime.py +40 -0
  431. kiri_core/schema/__init__.py +62 -0
  432. kiri_core/schema/column.py +140 -0
  433. kiri_core/schema/ingestion.py +665 -0
  434. kiri_core/schema/model.py +44 -0
  435. kiri_core/schema/reporting_goal.py +160 -0
  436. kiri_core/schema/source.py +188 -0
  437. kiri_core/silver/__init__.py +148 -0
  438. kiri_core/silver/apply_runner.py +830 -0
  439. kiri_core/silver/backfill.py +280 -0
  440. kiri_core/silver/dv_backend.py +426 -0
  441. kiri_core/silver/dv_graph.py +932 -0
  442. kiri_core/silver/dv_graph_mermaid.py +443 -0
  443. kiri_core/silver/dv_graph_payload.py +79 -0
  444. kiri_core/silver/dv_graph_views.py +416 -0
  445. kiri_core/silver/graph_projector.py +566 -0
  446. kiri_core/silver/hash_canonicaliser.py +172 -0
  447. kiri_core/silver/hash_policy.py +195 -0
  448. kiri_core/silver/load_ts_policy.py +202 -0
  449. kiri_core/silver/schema_evolution.py +410 -0
  450. kiri_core/silver/zone_rules.py +81 -0
  451. kiri_core/sizing.py +134 -0
  452. kiri_core/storage_paths.py +130 -0
  453. kiri_core/technique_validator.py +700 -0
  454. kiri_core/trace.py +409 -0
  455. kiri_core/types.py +41 -0
  456. kiri_core/urn.py +161 -0
  457. kiri_dbt_bridge/__init__.py +22 -0
  458. kiri_dbt_bridge/_schema_resolve.py +38 -0
  459. kiri_dbt_bridge/bridge.py +94 -0
  460. kiri_dbt_bridge/dbt.manifest.yml +26 -0
  461. kiri_dbt_bridge/dv4dbt/__init__.py +143 -0
  462. kiri_dbt_bridge/dv4dbt/backend.py +204 -0
  463. kiri_dbt_bridge/dv4dbt/bridge_generator.py +289 -0
  464. kiri_dbt_bridge/dv4dbt/compat.py +232 -0
  465. kiri_dbt_bridge/dv4dbt/hub_generator.py +282 -0
  466. kiri_dbt_bridge/dv4dbt/link_generator.py +306 -0
  467. kiri_dbt_bridge/dv4dbt/manifest.py +203 -0
  468. kiri_dbt_bridge/dv4dbt/pit_generator.py +318 -0
  469. kiri_dbt_bridge/dv4dbt/policy_translation.py +249 -0
  470. kiri_dbt_bridge/dv4dbt/project_writer.py +588 -0
  471. kiri_dbt_bridge/dv4dbt/ref_generator.py +404 -0
  472. kiri_dbt_bridge/dv4dbt/render.py +180 -0
  473. kiri_dbt_bridge/dv4dbt/sat_generator.py +480 -0
  474. kiri_dbt_bridge/dv4dbt/sat_variant_generator.py +529 -0
  475. kiri_dbt_bridge/dv4dbt/templates/__init__.py +16 -0
  476. kiri_dbt_bridge/dv4dbt/templates/bridge.sql.j2 +9 -0
  477. kiri_dbt_bridge/dv4dbt/templates/dbt_project.yml.j2 +37 -0
  478. kiri_dbt_bridge/dv4dbt/templates/hub.sql.j2 +9 -0
  479. kiri_dbt_bridge/dv4dbt/templates/link.sql.j2 +9 -0
  480. kiri_dbt_bridge/dv4dbt/templates/packages.yml.j2 +10 -0
  481. kiri_dbt_bridge/dv4dbt/templates/pit.sql.j2 +9 -0
  482. kiri_dbt_bridge/dv4dbt/templates/ref_historised.sql.j2 +9 -0
  483. kiri_dbt_bridge/dv4dbt/templates/ref_non_historised.sql.j2 +9 -0
  484. kiri_dbt_bridge/dv4dbt/templates/sat.sql.j2 +9 -0
  485. kiri_dbt_bridge/dv4dbt/templates/sat_effectivity.sql.j2 +9 -0
  486. kiri_dbt_bridge/dv4dbt/templates/sat_multi_active.sql.j2 +9 -0
  487. kiri_dbt_bridge/dv4dbt/templates/sat_non_historised.sql.j2 +9 -0
  488. kiri_dbt_bridge/dv4dbt/templates/sat_record_tracking.sql.j2 +9 -0
  489. kiri_dbt_bridge/dv4dbt/templates/sat_virtual_end_dated.sql.j2 +9 -0
  490. kiri_dbt_bridge/dv4dbt/templates/sources.yml.j2 +39 -0
  491. kiri_dbt_bridge/framework_adapter.py +310 -0
  492. kiri_dbt_bridge/generate.py +372 -0
  493. kiri_dbt_bridge/gold/__init__.py +21 -0
  494. kiri_dbt_bridge/gold/_sql_emit.py +128 -0
  495. kiri_dbt_bridge/gold/assemble.py +90 -0
  496. kiri_dbt_bridge/gold/canonical_columns.py +168 -0
  497. kiri_dbt_bridge/gold/computed_expressions.py +179 -0
  498. kiri_dbt_bridge/gold/dialects/__init__.py +69 -0
  499. kiri_dbt_bridge/gold/dialects/databricks.py +150 -0
  500. kiri_dbt_bridge/gold/dialects/duckdb.py +120 -0
  501. kiri_dbt_bridge/gold/dialects/trino.py +136 -0
  502. kiri_dbt_bridge/gold/dim_generator.py +688 -0
  503. kiri_dbt_bridge/gold/dimension_refs.py +255 -0
  504. kiri_dbt_bridge/gold/dispatch.py +69 -0
  505. kiri_dbt_bridge/gold/fact_generator.py +534 -0
  506. kiri_dbt_bridge/gold/flat_generator.py +144 -0
  507. kiri_dbt_bridge/gold/lineage.py +484 -0
  508. kiri_dbt_bridge/gold/localized_twins.py +218 -0
  509. kiri_dbt_bridge/gold/measures.py +49 -0
  510. kiri_dbt_bridge/gold/role_playing.py +75 -0
  511. kiri_dbt_bridge/gold/surrogate_key.py +114 -0
  512. kiri_dbt_bridge/macros/kiri_scd1_dim.sql +57 -0
  513. kiri_dbt_bridge/macros/kiri_scd2_dim.sql +188 -0
  514. kiri_dbt_bridge/macros/kiri_sha256.sql +32 -0
  515. kiri_dbt_bridge/macros/kiri_string_cast.sql +37 -0
  516. kiri_dbt_bridge/macros/kiri_surrogate_key.sql +43 -0
  517. kiri_dbt_bridge/profiles.py +162 -0
  518. kiri_dbt_bridge/py.typed +0 -0
  519. kiri_dbt_bridge/safe_ident.py +44 -0
  520. kiri_dbt_bridge/silver_conformance.py +188 -0
  521. kiri_dbt_bridge/silver_kimball/__init__.py +27 -0
  522. kiri_dbt_bridge/silver_kimball/assemble.py +174 -0
  523. kiri_dbt_bridge/silver_kimball/dim_generator.py +602 -0
  524. kiri_dbt_bridge/silver_kimball/dispatch.py +86 -0
  525. kiri_dbt_bridge/silver_kimball/fact_generator.py +320 -0
  526. kiri_dbt_bridge/silver_kimball/flat_generator.py +156 -0
  527. kiri_dbt_bridge/silver_kimball/lineage.py +464 -0
  528. kiri_dbt_bridge/silver_kimball/watermark_window.py +199 -0
  529. kiri_dbt_bridge/silver_kimball/zone.py +68 -0
  530. kiri_dbt_bridge/silver_scd2_curated/__init__.py +46 -0
  531. kiri_dbt_bridge/silver_scd2_curated/assemble.py +213 -0
  532. kiri_dbt_bridge/silver_scd2_curated/curated_generator.py +351 -0
  533. kiri_dbt_bridge/silver_scd2_curated/standardized_generator.py +261 -0
  534. kiri_dbt_bridge/silver_scd2_curated/standardized_plan.py +254 -0
  535. kiri_iceberg/__init__.py +11 -0
  536. kiri_iceberg/catalog.py +128 -0
  537. kiri_iceberg/profile.py +131 -0
  538. kiri_iceberg/py.typed +0 -0
  539. kiri_ingestion/__init__.py +63 -0
  540. kiri_ingestion/airbyte_client.py +428 -0
  541. kiri_ingestion/database_fetcher.py +224 -0
  542. kiri_ingestion/layout.py +71 -0
  543. kiri_ingestion/mssql_introspect.py +565 -0
  544. kiri_ingestion/py.typed +0 -0
  545. kiri_ingestion/rest_fetcher.py +472 -0
  546. kiri_ingestion/runner.py +341 -0
  547. kiri_ingestion/schema_drift.py +222 -0
  548. kiri_ingestion/storage.py +621 -0
  549. kiri_mcp/__init__.py +5 -0
  550. kiri_mcp/audit.py +99 -0
  551. kiri_mcp/audit_decorators.py +252 -0
  552. kiri_mcp/main.py +22 -0
  553. kiri_mcp/py.typed +0 -0
  554. kiri_mcp/rbac.py +110 -0
  555. kiri_mcp/server.py +766 -0
  556. kiri_mcp/skills.py +297 -0
  557. kiri_mcp/tools/__init__.py +70 -0
  558. kiri_mcp/tools/_shared.py +58 -0
  559. kiri_mcp/tools/adapter.py +150 -0
  560. kiri_mcp/tools/cli_reference.py +59 -0
  561. kiri_mcp/tools/compliance.py +119 -0
  562. kiri_mcp/tools/contract.py +374 -0
  563. kiri_mcp/tools/dv.py +62 -0
  564. kiri_mcp/tools/federation.py +152 -0
  565. kiri_mcp/tools/intent.py +71 -0
  566. kiri_mcp/tools/lineage.py +204 -0
  567. kiri_mcp/tools/lineage_drift.py +75 -0
  568. kiri_mcp/tools/plan.py +66 -0
  569. kiri_mcp/tools/project.py +311 -0
  570. kiri_migration/__init__.py +56 -0
  571. kiri_migration/name_preservation.py +271 -0
  572. kiri_migration/predeploy.py +311 -0
  573. kiri_migration/py.typed +0 -0
  574. kiri_migration/reconcile.py +1891 -0
  575. kiri_migration/scaffold/__init__.py +90 -0
  576. kiri_migration/scaffold/_audit_columns.py +74 -0
  577. kiri_migration/scaffold/_classify.py +100 -0
  578. kiri_migration/scaffold/_typemap.py +71 -0
  579. kiri_migration/scaffold/_yaml.py +33 -0
  580. kiri_migration/scaffold/bronze.py +279 -0
  581. kiri_migration/scaffold/compile_targets/__init__.py +271 -0
  582. kiri_migration/scaffold/compile_targets/_common.py +193 -0
  583. kiri_migration/scaffold/compile_targets/copy_into.py +33 -0
  584. kiri_migration/scaffold/compile_targets/dlt.py +125 -0
  585. kiri_migration/scaffold/compile_targets/dlt_with_append_flows.py +163 -0
  586. kiri_migration/scaffold/compile_targets/dlt_with_apply_changes.py +192 -0
  587. kiri_migration/scaffold/compile_targets/dlt_with_expectations.py +276 -0
  588. kiri_migration/scaffold/compile_targets/layer_dispatch.py +258 -0
  589. kiri_migration/scaffold/compile_targets/sql_full_snapshot.py +385 -0
  590. kiri_migration/scaffold/compile_targets/streaming_table.py +37 -0
  591. kiri_migration/scaffold/mart.py +234 -0
  592. kiri_migration/scaffold/plans/__init__.py +49 -0
  593. kiri_migration/scaffold/plans/_registry.py +308 -0
  594. kiri_migration/scaffold/plans/minimal.py +550 -0
  595. kiri_migration/scaffold/plans/plan_b_layer.py +640 -0
  596. kiri_migration/scaffold/silver_from_marts.py +256 -0
  597. kiri_migration/scaffold/silver_pattern.py +451 -0
  598. kiri_migration/sp_translator.py +398 -0
  599. kiri_odcs/LICENSE +204 -0
  600. kiri_odcs/__init__.py +234 -0
  601. kiri_odcs/ai_trust_tier.py +475 -0
  602. kiri_odcs/canonical.py +263 -0
  603. kiri_odcs/diff.py +63 -0
  604. kiri_odcs/errors.py +51 -0
  605. kiri_odcs/invariant_sql.py +136 -0
  606. kiri_odcs/kiri_bridge.py +71 -0
  607. kiri_odcs/kiri_ext.py +3166 -0
  608. kiri_odcs/kiri_fk.py +447 -0
  609. kiri_odcs/kiri_pit.py +121 -0
  610. kiri_odcs/kiri_ref.py +57 -0
  611. kiri_odcs/openlineage_adapter.py +105 -0
  612. kiri_odcs/parser.py +1213 -0
  613. kiri_odcs/pii_guardrails.py +159 -0
  614. kiri_odcs/py.typed +0 -0
  615. kiri_odcs/schema_export.py +223 -0
  616. kiri_openlineage/__init__.py +58 -0
  617. kiri_openlineage/builder.py +146 -0
  618. kiri_openlineage/cost.py +190 -0
  619. kiri_openlineage/emitter.py +165 -0
  620. kiri_openlineage/py.typed +0 -0
  621. kiri_openlineage/schema.py +121 -0
  622. kiri_packs/__init__.py +67 -0
  623. kiri_packs/installer.py +500 -0
  624. kiri_packs/manifest.py +134 -0
  625. kiri_packs/publisher.py +605 -0
  626. kiri_packs/py.typed +0 -0
  627. kiri_packs/validator.py +295 -0
  628. kiri_quality/__init__.py +108 -0
  629. kiri_quality/ai_audit_correlator.py +293 -0
  630. kiri_quality/ai_audit_correlator_apply.py +526 -0
  631. kiri_quality/conformance_evidence.py +272 -0
  632. kiri_quality/dbt_test_collector.py +214 -0
  633. kiri_quality/dq_codegen.py +300 -0
  634. kiri_quality/dv_checks.py +393 -0
  635. kiri_quality/dv_column_lineage.py +279 -0
  636. kiri_quality/dv_column_lineage_apply.py +856 -0
  637. kiri_quality/dv_conformance.py +841 -0
  638. kiri_quality/dv_fixture_runner.py +478 -0
  639. kiri_quality/dv_invariants.py +986 -0
  640. kiri_quality/dv_kiri.py +97 -0
  641. kiri_quality/dv_markdown.py +169 -0
  642. kiri_quality/dv_observed_uc_reader.py +187 -0
  643. kiri_quality/dv_runner.py +389 -0
  644. kiri_quality/evidence_status.py +337 -0
  645. kiri_quality/fk_checks.py +800 -0
  646. kiri_quality/fk_ddl.py +402 -0
  647. kiri_quality/fk_pit_contract.py +168 -0
  648. kiri_quality/fk_runner.py +950 -0
  649. kiri_quality/kernel.py +376 -0
  650. kiri_quality/pii.py +60 -0
  651. kiri_quality/pre_classify/__init__.py +55 -0
  652. kiri_quality/pre_classify/engine.py +346 -0
  653. kiri_quality/pre_classify/locale_packs/en.yaml +117 -0
  654. kiri_quality/pre_classify/locale_packs/sv.yaml +73 -0
  655. kiri_quality/pre_classify/merge.py +297 -0
  656. kiri_quality/pre_classify/model.py +163 -0
  657. kiri_quality/py.typed +0 -0
  658. kiri_quality/reject_lane.py +261 -0
  659. kiri_quality/rule_runner.py +565 -0
  660. kiri_quality/schema_evolution_check.py +88 -0
  661. kiri_quality/silver_quality.py +494 -0
  662. kiri_quality/silver_quality_markdown.py +187 -0
  663. kiri_quality/uc_security_compile.py +951 -0
  664. kiri_quality/uc_security_compile_apply.py +1186 -0
  665. kiri_rbac/__init__.py +87 -0
  666. kiri_rbac/access.py +371 -0
  667. kiri_rbac/loader.py +104 -0
  668. kiri_rbac/pr_gate.py +213 -0
  669. kiri_rbac/providers/__init__.py +29 -0
  670. kiri_rbac/providers/_jwt.py +121 -0
  671. kiri_rbac/providers/azure_ad.py +93 -0
  672. kiri_rbac/providers/base.py +107 -0
  673. kiri_rbac/providers/factory.py +48 -0
  674. kiri_rbac/providers/github.py +134 -0
  675. kiri_rbac/providers/oidc_generic.py +99 -0
  676. kiri_rbac/py.typed +0 -0
  677. kiri_rbac/resolver.py +118 -0
  678. kiri_rbac/schema.py +219 -0
  679. kiri_schema_registry/__init__.py +22 -0
  680. kiri_schema_registry/avro.py +127 -0
  681. kiri_schema_registry/base.py +71 -0
  682. kiri_schema_registry/confluent.py +184 -0
  683. kiri_schema_registry/glue.py +178 -0
  684. kiri_schema_registry/py.typed +0 -0
  685. kiri_slack_bot/__init__.py +24 -0
  686. kiri_slack_bot/handler.py +248 -0
  687. kiri_slack_bot/py.typed +0 -0
  688. kiri_slack_bot/queries.py +90 -0
  689. kiri_slack_bot/signature.py +94 -0
  690. kiri_vault/__init__.py +59 -0
  691. kiri_vault/audit.py +64 -0
  692. kiri_vault/aws_vault.py +123 -0
  693. kiri_vault/azure_vault.py +236 -0
  694. kiri_vault/boundaries.py +219 -0
  695. kiri_vault/client.py +54 -0
  696. kiri_vault/databricks_vault.py +292 -0
  697. kiri_vault/env_vault.py +77 -0
  698. kiri_vault/py.typed +0 -0
  699. kiri_vault/ref.py +84 -0
  700. kirimana_automation/__init__.py +0 -0
  701. kirimana_automation/business_calendar.py +179 -0
  702. kirimana_automation/dag_snapshot.py +252 -0
  703. kirimana_automation/db/__init__.py +5 -0
  704. kirimana_automation/db/migrate.py +571 -0
  705. kirimana_automation/db/migrations/0001_release_state.down.duckdb.sql +29 -0
  706. kirimana_automation/db/migrations/0001_release_state.down.postgres.sql +37 -0
  707. kirimana_automation/db/migrations/0001_release_state.duckdb.sql +170 -0
  708. kirimana_automation/db/migrations/0001_release_state.postgres.sql +176 -0
  709. kirimana_automation/db/migrations/0002_rate_limit_buckets.down.duckdb.sql +9 -0
  710. kirimana_automation/db/migrations/0002_rate_limit_buckets.down.postgres.sql +9 -0
  711. kirimana_automation/db/migrations/0002_rate_limit_buckets.duckdb.sql +14 -0
  712. kirimana_automation/db/migrations/0002_rate_limit_buckets.postgres.sql +18 -0
  713. kirimana_automation/db/migrations/0003_flow_runs.down.duckdb.sql +18 -0
  714. kirimana_automation/db/migrations/0003_flow_runs.down.postgres.sql +19 -0
  715. kirimana_automation/db/migrations/0003_flow_runs.duckdb.sql +85 -0
  716. kirimana_automation/db/migrations/0003_flow_runs.postgres.sql +102 -0
  717. kirimana_automation/db/migrations/0004_flow_dag_snapshots.down.duckdb.sql +11 -0
  718. kirimana_automation/db/migrations/0004_flow_dag_snapshots.down.postgres.sql +11 -0
  719. kirimana_automation/db/migrations/0004_flow_dag_snapshots.duckdb.sql +15 -0
  720. kirimana_automation/db/migrations/0004_flow_dag_snapshots.postgres.sql +31 -0
  721. kirimana_automation/db/migrations/0005_calendar_skip.down.duckdb.sql +109 -0
  722. kirimana_automation/db/migrations/0005_calendar_skip.down.postgres.sql +42 -0
  723. kirimana_automation/db/migrations/0005_calendar_skip.duckdb.sql +113 -0
  724. kirimana_automation/db/migrations/0005_calendar_skip.postgres.sql +42 -0
  725. kirimana_automation/db/migrations/0006_flow_apply_log.down.duckdb.sql +9 -0
  726. kirimana_automation/db/migrations/0006_flow_apply_log.down.postgres.sql +11 -0
  727. kirimana_automation/db/migrations/0006_flow_apply_log.duckdb.sql +45 -0
  728. kirimana_automation/db/migrations/0006_flow_apply_log.postgres.sql +49 -0
  729. kirimana_automation/db/migrations/0007_flow_task_error_redacted.down.duckdb.sql +14 -0
  730. kirimana_automation/db/migrations/0007_flow_task_error_redacted.down.postgres.sql +2 -0
  731. kirimana_automation/db/migrations/0007_flow_task_error_redacted.duckdb.sql +7 -0
  732. kirimana_automation/db/migrations/0007_flow_task_error_redacted.postgres.sql +11 -0
  733. kirimana_automation/db/migrations/README.md +84 -0
  734. kirimana_automation/flow_loader.py +220 -0
  735. kirimana_automation/freeze.py +133 -0
  736. kirimana_automation/kiri_recommender.py +313 -0
  737. kirimana_automation/main.py +195 -0
  738. kirimana_automation/py.typed +0 -0
  739. kirimana_automation/run_history.py +593 -0
  740. kirimana_automation/run_history_analysis.py +607 -0
  741. kirimana_automation/schedule.py +343 -0
  742. kirimana_automation/schedule_validator.py +572 -0
  743. kirimana_automation/workflow_compiler.py +1288 -0
@@ -0,0 +1,969 @@
1
+ """Atomic migration from per-table ``03-contracts/<schema>/<table>.yml`` files
2
+ into per-(domain, layer) ``01-config/<domain>/<layer>.yml`` manifests.
3
+
4
+ ADR 0116 slice 20.2. The verb is intentionally one-way and atomic:
5
+
6
+ 1. Read every YAML under ``<contracts-dir>`` through
7
+ :func:`kiri_odcs.parser.parse_contract`.
8
+ 2. Group by ``(domain, layer)`` — layer derived from a closed-menu set of
9
+ ``kiri.lifecycle.state`` / ``kiri.medallion.state`` / ``kiri.layer``
10
+ customProperties (in that precedence). Missing on all three == fatal.
11
+ 3. For each group compute the common-subset customProperties (values that
12
+ appear identically on every contract in the group) — they become the
13
+ layer-wide defaults. Per-contract deltas survive as the per-table
14
+ :class:`~kiri_core.lifecycle.layer_contract.LayerTableEntry`
15
+ customProperties.
16
+ 4. Per-table ``kiri.target.catalog`` / ``kiri.target.schema`` are stripped
17
+ and hoisted into the layer file's ``targets:`` block — multiple
18
+ distinct values for the same target rejected as fatal.
19
+ 5. Emit one byte-deterministic YAML file per group at
20
+ ``<out>/<domain>/<layer>.yml``.
21
+ 6. Delete (or archive) the source directory.
22
+ 7. Re-validate the emitted project with
23
+ :func:`kiri_core.lifecycle.layer_contract_validator.validate_project`.
24
+
25
+ The emitter is **canonical**: it is the only function that converts a
26
+ :class:`LayerContract` into YAML bytes. Re-running the verb on its own
27
+ output (or running it twice on the same exploded set) produces identical
28
+ bytes — this is pinned by a named round-trip test.
29
+ """
30
+
31
+ from __future__ import annotations
32
+
33
+ import difflib
34
+ import re
35
+ import shutil
36
+ from dataclasses import dataclass, field
37
+ from pathlib import Path
38
+ from typing import Any, Literal
39
+
40
+ import yaml
41
+ from kiri_odcs.errors import (
42
+ ContractParseError,
43
+ SemanticValidationError,
44
+ StructuralValidationError,
45
+ ValidationIssue,
46
+ )
47
+ from kiri_odcs.parser import parse_contract
48
+
49
+ from kiri_core.lifecycle.layer_contract import (
50
+ CONFIG_SUBDIR,
51
+ Layer,
52
+ LayerContract,
53
+ )
54
+ from kiri_core.lifecycle.layer_contract_validator import (
55
+ ValidationReport,
56
+ validate_project,
57
+ )
58
+
59
+ __all__ = [
60
+ "BrokenReference",
61
+ "MigrationConflict",
62
+ "MigrationError",
63
+ "MigrationPlan",
64
+ "MigrationResult",
65
+ "PlannedLayerFile",
66
+ "dump_layer_contract_yaml",
67
+ "explode_layer_to_per_table",
68
+ "migrate_from_contracts",
69
+ "plan_migration",
70
+ ]
71
+
72
+ _LAYER_VALUES: list[str] = [m.value for m in Layer]
73
+
74
+ # Customproperty keys we consult to derive the layer for a per-table contract.
75
+ # Precedence is high → low; the first present (case-sensitive) wins.
76
+ _LAYER_PROP_KEYS: tuple[str, ...] = (
77
+ "kiri.layer",
78
+ "kiri.lifecycle.state",
79
+ "kiri.medallion.state",
80
+ )
81
+
82
+ # Per-table catalog binding keys hoisted into the layer ``targets:`` block.
83
+ _TARGET_CATALOG_KEY = "kiri.target.catalog"
84
+ _TARGET_SCHEMA_KEY = "kiri.target.schema"
85
+ _HOISTED_TARGET_KEYS: set[str] = {_TARGET_CATALOG_KEY, _TARGET_SCHEMA_KEY}
86
+
87
+
88
+ class MigrationError(Exception):
89
+ """Atomic migration could not complete — no destination files written."""
90
+
91
+ def __init__(self, issues: list[ValidationIssue]) -> None:
92
+ self.issues = issues
93
+ super().__init__("; ".join(f"{i.path}: {i.message}" for i in issues))
94
+
95
+
96
+ @dataclass(frozen=True)
97
+ class MigrationConflict:
98
+ """One ``kiri.target.catalog`` disagreement among contracts in a group."""
99
+
100
+ target: str
101
+ layer: Layer
102
+ domain: str
103
+ contracts: tuple[Path, ...]
104
+ values: tuple[str, ...]
105
+
106
+
107
+ @dataclass
108
+ class PlannedLayerFile:
109
+ """One ``(domain, layer)`` output planned by :func:`plan_migration`."""
110
+
111
+ domain: str
112
+ layer: Layer
113
+ out_path: Path
114
+ source_contracts: list[Path]
115
+ target_catalogs: dict[str, str] # target name -> catalog
116
+
117
+
118
+ @dataclass
119
+ class MigrationPlan:
120
+ """Result of :func:`plan_migration` — the dry-run view."""
121
+
122
+ planned: list[PlannedLayerFile] = field(default_factory=list)
123
+ issues: list[ValidationIssue] = field(default_factory=list)
124
+
125
+ @property
126
+ def ok(self) -> bool:
127
+ return not self.issues
128
+
129
+
130
+ @dataclass
131
+ class MigrationResult:
132
+ """Result of :func:`migrate_from_contracts`."""
133
+
134
+ written: list[Path] = field(default_factory=list)
135
+ archived_to: Path | None = None
136
+ deleted: bool = False
137
+ validation: ValidationReport | None = None
138
+ #: Issue #278 — intra-project references (flow selectors, silver
139
+ #: column-lineage URNs) that point at a contract name no migrated
140
+ #: layer entry exposes. Listed for the operator to fix manually;
141
+ #: the migration itself does not rewrite YAML it didn't write.
142
+ broken_references: list[BrokenReference] = field(default_factory=list)
143
+
144
+
145
+ @dataclass(frozen=True)
146
+ class BrokenReference:
147
+ """A YAML reference that didn't resolve after migration (issue #278).
148
+
149
+ The migration moves per-table contracts into a per-(domain, layer)
150
+ layer file. Anything elsewhere in the project that references the
151
+ old per-table contract by **name** silently breaks unless the
152
+ migration preserves the contract name as the layer-entry name
153
+ (see issue #278 fix). This dataclass records each surviving broken
154
+ reference so :func:`migrate_from_contracts` can list them in
155
+ :attr:`MigrationResult.broken_references` and the CLI can warn the
156
+ operator.
157
+ """
158
+
159
+ kind: Literal["flow_selector", "lineage_urn"]
160
+ source_file: Path
161
+ reference: str
162
+ detail: str = ""
163
+
164
+
165
+ # ---------------------------------------------------------------------------
166
+ # Pure helpers (no I/O)
167
+ # ---------------------------------------------------------------------------
168
+
169
+
170
+ def _did_you_mean(value: str, choices: list[str], cutoff: float = 0.6) -> str:
171
+ close = difflib.get_close_matches(value, choices, n=1, cutoff=cutoff)
172
+ if not close:
173
+ return ""
174
+ return f" Did you mean {close[0]!r}?"
175
+
176
+
177
+ def _resolve_layer_value(props: dict[str, Any]) -> str | None:
178
+ """Return the first present layer-hint customProperty value, or ``None``."""
179
+ for key in _LAYER_PROP_KEYS:
180
+ if key in props:
181
+ value = props[key]
182
+ if isinstance(value, str) and value:
183
+ return value
184
+ return None
185
+
186
+
187
+ def _props_dict(custom_properties: list[Any]) -> dict[str, Any]:
188
+ out: dict[str, Any] = {}
189
+ for entry in custom_properties:
190
+ # Support both CustomProperty pydantic instances and raw dicts.
191
+ if hasattr(entry, "property"):
192
+ out[entry.property] = entry.value
193
+ elif isinstance(entry, dict) and "property" in entry:
194
+ out[entry["property"]] = entry.get("value")
195
+ return out
196
+
197
+
198
+ def _custom_properties_to_list(props: dict[str, Any]) -> list[dict[str, Any]]:
199
+ """Emit a stable-order list of ``{property, value}`` dicts."""
200
+ return [{"property": k, "value": props[k]} for k in sorted(props)]
201
+
202
+
203
+ def _common_subset(per_contract: list[dict[str, Any]]) -> dict[str, Any]:
204
+ """Return key→value pairs present (and equal) in every contract.
205
+
206
+ Identity is by ``==`` so nested lists/dicts that re-serialise identically
207
+ are folded into the layer defaults.
208
+ """
209
+ if not per_contract:
210
+ return {}
211
+ head = per_contract[0]
212
+ common: dict[str, Any] = {}
213
+ for key, value in head.items():
214
+ if all(key in other and other[key] == value for other in per_contract[1:]):
215
+ common[key] = value
216
+ return common
217
+
218
+
219
+ # ---------------------------------------------------------------------------
220
+ # Canonical YAML emit (the only function that writes layer-file bytes)
221
+ # ---------------------------------------------------------------------------
222
+
223
+
224
+ def dump_layer_contract_yaml(
225
+ *,
226
+ domain: str,
227
+ layer: Layer,
228
+ contract_id: str,
229
+ version: str,
230
+ status: str,
231
+ owner: str | None,
232
+ targets: dict[str, dict[str, str]],
233
+ layer_custom_properties: dict[str, Any],
234
+ tables: list[dict[str, Any]],
235
+ schemas: list[dict[str, Any]] | None = None,
236
+ api_version: str = "3.0.0",
237
+ ) -> str:
238
+ """Render a single ``01-config/<domain>/<layer>.yml`` YAML document.
239
+
240
+ The output is byte-deterministic for the same input — keys appear in a
241
+ fixed order, customProperties are sorted, and tables preserve the
242
+ caller-supplied order. This is the canonical emit format; the layer
243
+ parser round-trips it via ``yaml.safe_load → parse_layer_contract``.
244
+
245
+ Parameters:
246
+ tables: each entry is ``{name, logicalType, physicalName?,
247
+ customProperties: dict, properties: list[dict]}``.
248
+ schemas: optional ADR 0134 declared-schema registry — each entry is
249
+ ``{name, physical?, role?, locale?}``. Emitted as the top-level
250
+ ``schemas:`` block (after ``targets:``) when non-empty, preserving
251
+ caller-supplied order. Omitted entirely when ``None`` / empty, so a
252
+ free-name manifest round-trips byte-identically to pre-ADR-0134.
253
+ """
254
+ doc: dict[str, Any] = {
255
+ "apiVersion": api_version,
256
+ "kind": "DataContract",
257
+ "id": contract_id,
258
+ "name": layer.value,
259
+ "version": version,
260
+ "status": status,
261
+ "domain": domain,
262
+ }
263
+ if owner is not None:
264
+ doc["owner"] = owner
265
+
266
+ if targets:
267
+ doc["targets"] = {
268
+ name: {
269
+ "catalog": binding["catalog"],
270
+ "governance_schema": binding["governance_schema"],
271
+ }
272
+ for name, binding in sorted(targets.items())
273
+ }
274
+
275
+ # ADR 0134 — re-emit the declared-schema registry so a registry manifest
276
+ # round-trips byte-stable. Order is caller-supplied; only set keys are
277
+ # emitted (a bare ``{name}`` schema stays bare). Absent → omitted entirely,
278
+ # keeping the free-name manifest byte-identical to pre-0134.
279
+ if schemas:
280
+ schema_decls: list[dict[str, Any]] = []
281
+ for decl in schemas:
282
+ decl_entry: dict[str, Any] = {"name": decl["name"]}
283
+ for optional in ("physical", "role", "locale"):
284
+ value = decl.get(optional)
285
+ if value is not None:
286
+ decl_entry[optional] = value
287
+ schema_decls.append(decl_entry)
288
+ doc["schemas"] = schema_decls
289
+
290
+ if layer_custom_properties:
291
+ doc["customProperties"] = _custom_properties_to_list(layer_custom_properties)
292
+
293
+ schema_list: list[dict[str, Any]] = []
294
+ for table in tables:
295
+ entry: dict[str, Any] = {
296
+ "name": table["name"],
297
+ "logicalType": table.get("logicalType", "table"),
298
+ }
299
+ physical = table.get("physicalName")
300
+ if physical is not None and physical != table["name"]:
301
+ entry["physicalName"] = physical
302
+ tprops = table.get("customProperties") or {}
303
+ if tprops:
304
+ entry["customProperties"] = _custom_properties_to_list(tprops)
305
+ properties = table.get("properties") or []
306
+ if properties:
307
+ entry["properties"] = list(properties)
308
+ schema_list.append(entry)
309
+ doc["schema"] = schema_list
310
+
311
+ return str(
312
+ yaml.safe_dump(
313
+ doc,
314
+ sort_keys=False,
315
+ allow_unicode=True,
316
+ default_flow_style=False,
317
+ )
318
+ )
319
+
320
+
321
+ # ---------------------------------------------------------------------------
322
+ # Round-trip helper used by tests: blow a LayerContract back into per-table
323
+ # YAML docs (in the legacy ``03-contracts/<schema>/<table>.yml`` shape).
324
+ # ---------------------------------------------------------------------------
325
+
326
+
327
+ def explode_layer_to_per_table(
328
+ layer_contract: LayerContract,
329
+ ) -> list[tuple[str, str]]:
330
+ """Return one per-table ``(relative_path, yaml_text)`` per table.
331
+
332
+ The yaml is shape-compatible with the legacy per-table layout and
333
+ carries the ``kiri.layer`` customProperty + hoisted-down
334
+ ``kiri.target.catalog`` / ``kiri.target.schema`` so a subsequent
335
+ :func:`migrate_from_contracts` reconstructs the original layer file.
336
+
337
+ Used by tests + by anyone needing to interoperate with a pre-20.1
338
+ repo via a one-shot blow-up.
339
+ """
340
+ layer_props = _props_dict(layer_contract.canonical.contract.custom_properties)
341
+ # Pick a target whose catalog name we stamp onto each contract. We pick
342
+ # the alphabetically-first target deterministically.
343
+ first_target_name = sorted(layer_contract.targets.keys())[0]
344
+ first_target = layer_contract.targets[first_target_name]
345
+
346
+ out: list[tuple[str, str]] = []
347
+ for entry in layer_contract.tables:
348
+ # Layer defaults sit on the contract-level customProperties bucket.
349
+ # We add the hoisted catalog so the migration can put it back in
350
+ # ``targets:`` and a kiri.layer marker so the migrator groups
351
+ # the contract under the right layer.
352
+ contract_props = dict(layer_props)
353
+ contract_props["kiri.layer"] = layer_contract.layer.value
354
+ contract_props[_TARGET_CATALOG_KEY] = first_target.catalog
355
+
356
+ # Per-table customProperties stay on the schema entry — this mirrors
357
+ # ``migrate_from_contracts`` (which reads per-table deltas off the
358
+ # schema entry), keeping the layer<->explode round-trip byte-stable.
359
+ # The load path lifts these to the contract top-level where the parser
360
+ # reads them; see ``materialize_layer_contract`` (#270).
361
+ table_props = _props_dict(entry.custom_properties)
362
+
363
+ doc: dict[str, Any] = {
364
+ "apiVersion": layer_contract.canonical.contract.api_version,
365
+ "kind": "DataContract",
366
+ "id": f"{layer_contract.canonical.contract.id}.{entry.name}",
367
+ "name": entry.name,
368
+ "version": layer_contract.version,
369
+ "status": layer_contract.canonical.contract.status,
370
+ "domain": layer_contract.domain,
371
+ }
372
+ if layer_contract.owner is not None:
373
+ doc["owner"] = layer_contract.owner
374
+ doc["customProperties"] = _custom_properties_to_list(contract_props)
375
+ schema_entry: dict[str, Any] = {
376
+ "name": entry.name,
377
+ "logicalType": entry.logical_type,
378
+ }
379
+ if entry.physical_name is not None and entry.physical_name != entry.name:
380
+ schema_entry["physicalName"] = entry.physical_name
381
+ if table_props:
382
+ schema_entry["customProperties"] = _custom_properties_to_list(table_props)
383
+ if entry.properties:
384
+ schema_entry["properties"] = [
385
+ p.model_dump(by_alias=True, exclude_none=True, mode="json")
386
+ for p in entry.properties
387
+ ]
388
+ doc["schema"] = [schema_entry]
389
+
390
+ rel = f"{entry.name}.yml"
391
+ text = yaml.safe_dump(doc, sort_keys=False, allow_unicode=True, default_flow_style=False)
392
+ out.append((rel, text))
393
+ return out
394
+
395
+
396
+ # ---------------------------------------------------------------------------
397
+ # Plan
398
+ # ---------------------------------------------------------------------------
399
+
400
+
401
+ def _walk_contract_files(contracts_dir: Path) -> list[Path]:
402
+ if not contracts_dir.is_dir():
403
+ return []
404
+ return sorted(
405
+ p for p in contracts_dir.rglob("*") if p.is_file() and p.suffix in (".yml", ".yaml")
406
+ )
407
+
408
+
409
+ def _parse_all(files: list[Path]) -> tuple[list[tuple[Path, Any]], list[ValidationIssue]]:
410
+ parsed: list[tuple[Path, Any]] = []
411
+ issues: list[ValidationIssue] = []
412
+ for path in files:
413
+ try:
414
+ canonical = parse_contract(path)
415
+ except (ContractParseError, StructuralValidationError, SemanticValidationError) as exc:
416
+ if isinstance(exc, (StructuralValidationError, SemanticValidationError)):
417
+ issues.extend(exc.issues)
418
+ else:
419
+ issues.append(ValidationIssue(path=str(path), message=str(exc)))
420
+ continue
421
+ parsed.append((path, canonical))
422
+ return parsed, issues
423
+
424
+
425
+ def plan_migration(
426
+ contracts_dir: Path,
427
+ *,
428
+ out_dir: Path,
429
+ ) -> MigrationPlan:
430
+ """Compute the migration plan without writing anything.
431
+
432
+ Discovers every per-table YAML, groups by ``(domain, layer)``,
433
+ resolves the per-target catalog hoist, and reports every fatal
434
+ finding upfront. Caller raises :class:`MigrationError` to abort.
435
+ """
436
+ plan = MigrationPlan()
437
+ files = _walk_contract_files(contracts_dir)
438
+ if not files:
439
+ plan.issues.append(
440
+ ValidationIssue(
441
+ path=str(contracts_dir),
442
+ message=(
443
+ f"no contract YAML files found under {contracts_dir!s}; nothing to migrate."
444
+ ),
445
+ )
446
+ )
447
+ return plan
448
+
449
+ parsed, parse_issues = _parse_all(files)
450
+ plan.issues.extend(parse_issues)
451
+
452
+ # Group: (domain, layer) -> [(path, canonical)]
453
+ groups: dict[tuple[str, Layer], list[tuple[Path, Any]]] = {}
454
+ for path, canonical in parsed:
455
+ tprops = _props_dict(canonical.contract.custom_properties)
456
+ layer_raw = _resolve_layer_value(tprops)
457
+ if layer_raw is None:
458
+ plan.issues.append(
459
+ ValidationIssue(
460
+ path=str(path),
461
+ message=(
462
+ "contract has no layer hint: declare one of "
463
+ f"{list(_LAYER_PROP_KEYS)!r} as a customProperty "
464
+ "before migration (closed menu)."
465
+ ),
466
+ )
467
+ )
468
+ continue
469
+ if layer_raw not in _LAYER_VALUES:
470
+ plan.issues.append(
471
+ ValidationIssue(
472
+ path=str(path),
473
+ message=(
474
+ f"layer hint {layer_raw!r} is not a valid layer. "
475
+ f"Valid values: {_LAYER_VALUES!r}."
476
+ + _did_you_mean(layer_raw, _LAYER_VALUES)
477
+ ),
478
+ )
479
+ )
480
+ continue
481
+ domain = canonical.contract.domain
482
+ if not domain:
483
+ plan.issues.append(
484
+ ValidationIssue(
485
+ path=str(path),
486
+ message=(
487
+ "contract has no top-level 'domain:' field; "
488
+ "required for migration grouping."
489
+ ),
490
+ )
491
+ )
492
+ continue
493
+ groups.setdefault((domain, Layer(layer_raw)), []).append((path, canonical))
494
+
495
+ # Resolve the target-catalog hoist per group. Each group MUST agree on
496
+ # the value of ``kiri.target.catalog`` across every contract within it,
497
+ # otherwise the migration is ambiguous.
498
+ for (domain, layer), members in sorted(
499
+ groups.items(), key=lambda kv: (kv[0][0], kv[0][1].value)
500
+ ):
501
+ # Per spec — the legacy per-table layout stamps catalog/schema in one
502
+ # bucket; we hoist to a single conceptual 'prod' target binding.
503
+ # When multiple targets exist in the OLD model, the operator's
504
+ # kiri.yml is consulted for the target names (handled by the writer
505
+ # via the supplied `target_names` arg in migrate_from_contracts).
506
+ target_values: set[str] = set()
507
+ for _path, canonical in members:
508
+ tprops = _props_dict(canonical.contract.custom_properties)
509
+ cat = tprops.get(_TARGET_CATALOG_KEY)
510
+ if isinstance(cat, str) and cat:
511
+ target_values.add(cat)
512
+ if len(target_values) > 1:
513
+ paths = sorted(str(p) for p, _ in members)
514
+ plan.issues.append(
515
+ ValidationIssue(
516
+ path=f"{domain}/{layer.value}",
517
+ message=(
518
+ f"contracts in this group disagree on "
519
+ f"{_TARGET_CATALOG_KEY!r}: values "
520
+ f"{sorted(target_values)!r} across contracts "
521
+ f"{paths!r}. Resolve by aligning to a single value "
522
+ "before migrating."
523
+ ),
524
+ )
525
+ )
526
+ continue
527
+ catalog_value = next(iter(target_values)) if target_values else f"{domain}_catalog"
528
+
529
+ out_path = out_dir / domain / f"{layer.value}.yml"
530
+ planned = PlannedLayerFile(
531
+ domain=domain,
532
+ layer=layer,
533
+ out_path=out_path,
534
+ source_contracts=sorted(p for p, _ in members),
535
+ target_catalogs={"prod": catalog_value},
536
+ )
537
+ plan.planned.append(planned)
538
+
539
+ return plan
540
+
541
+
542
+ # ---------------------------------------------------------------------------
543
+ # Migration (writes)
544
+ # ---------------------------------------------------------------------------
545
+
546
+
547
+ def _archive_dir(contracts_dir: Path) -> Path:
548
+ return contracts_dir.with_name(contracts_dir.name + ".archived")
549
+
550
+
551
+ def _build_layer_doc(
552
+ *,
553
+ domain: str,
554
+ layer: Layer,
555
+ members: list[tuple[Path, Any]],
556
+ target_catalogs: dict[str, str],
557
+ governance_schema: str,
558
+ ) -> str:
559
+ """Compose one layer YAML from a parsed group."""
560
+ # Per-contract property dicts (each per-table customProperties bucket).
561
+ per_contract_layer_props: list[dict[str, Any]] = []
562
+ table_buckets: list[dict[str, Any]] = []
563
+ base_id_parts: list[str] = []
564
+
565
+ # Sort by table name for determinism.
566
+ members_sorted = sorted(members, key=lambda pair: _first_table_name(pair[1]) or pair[0].name)
567
+
568
+ for _path, canonical in members_sorted:
569
+ contract = canonical.contract
570
+ all_props = _props_dict(contract.custom_properties)
571
+ # Strip layer hints + hoisted target bindings from per-contract props
572
+ # that become layer defaults / per-table bucket.
573
+ carryable = {
574
+ k: v
575
+ for k, v in all_props.items()
576
+ if k not in _LAYER_PROP_KEYS and k not in _HOISTED_TARGET_KEYS
577
+ }
578
+ per_contract_layer_props.append(carryable)
579
+
580
+ # Each legacy contract is one table (schema entry).
581
+ if not contract.schema_:
582
+ continue
583
+ s = contract.schema_[0]
584
+ table_props_raw = _props_dict(s.custom_properties)
585
+ # If the per-table catalog was on the contract level (legacy),
586
+ # we've already stripped it. kiri.target_schema may still appear
587
+ # on either level — leave it on the table.
588
+ per_table_bucket = dict(table_props_raw)
589
+ table_buckets.append(per_table_bucket)
590
+
591
+ base_id_parts.append(contract.id)
592
+
593
+ # Common subset = layer-wide customProperties (sorted in emit).
594
+ common_props = _common_subset(per_contract_layer_props)
595
+ # Always stamp kiri.layer for clarity / round-trip-ability.
596
+ common_props["kiri.layer"] = layer.value
597
+
598
+ # Per-table overlay = layer-level prop bucket minus the common subset,
599
+ # MERGED with the actual per-table customProperties.
600
+ final_tables: list[dict[str, Any]] = []
601
+ for (_path, canonical), per_layer_props, raw_table_props in zip(
602
+ members_sorted, per_contract_layer_props, table_buckets, strict=True
603
+ ):
604
+ contract = canonical.contract
605
+ s = contract.schema_[0]
606
+ # Issue #278 — preserve the source contract's name as the layer-entry
607
+ # name when it differs from ``schema[0].name``. Flow selectors
608
+ # (``flows/*.yml`` ``select:``) and Kimball silver column-lineage URNs
609
+ # (``urn:kirimana:bronze:<contract_name>::<col>``) reference the
610
+ # *contract* name; collapsing every entry onto the table name silently
611
+ # broke them in #269 + #277 (had to be hand-fixed). When contract.name
612
+ # equals the table name there's no rename — keep the existing behaviour.
613
+ entry_name = contract.name if contract.name and contract.name != s.name else s.name
614
+ # When the entry name diverges from the table name we MUST surface the
615
+ # original table name as ``physicalName`` so the warehouse still knows
616
+ # what to look up. Otherwise the rename would just move the problem
617
+ # from "flow selectors break" to "the DW can't resolve the table".
618
+ # ``dump_layer_contract_yaml`` omits physicalName when it equals the
619
+ # entry name, so the no-rename case stays byte-identical.
620
+ physical_name = s.physical_name if s.physical_name else s.name
621
+ overlay = {k: v for k, v in per_layer_props.items() if k not in common_props}
622
+ merged_props = {**overlay, **raw_table_props}
623
+ final_tables.append(
624
+ {
625
+ "name": entry_name,
626
+ "logicalType": s.logical_type,
627
+ "physicalName": physical_name,
628
+ "customProperties": merged_props,
629
+ "properties": [
630
+ p.model_dump(by_alias=True, exclude_none=True, mode="json")
631
+ for p in s.properties
632
+ ],
633
+ }
634
+ )
635
+
636
+ # Deterministic contract id: derive from domain + layer (stable).
637
+ contract_id = f"com.example.{domain}.{layer.value}"
638
+ # If every source contract shares an id-prefix (e.g. com.example.wfm.bronze),
639
+ # prefer that prefix for round-trip determinism. The prefix already
640
+ # contains the layer suffix; we do NOT re-append it.
641
+ if base_id_parts:
642
+ prefix = base_id_parts[0].rsplit(".", 1)[0]
643
+ if all(b.startswith(prefix) for b in base_id_parts):
644
+ contract_id = prefix
645
+
646
+ # Owner: take the first non-None owner across the group.
647
+ owner: str | None = None
648
+ for _path, canonical in members_sorted:
649
+ if canonical.contract.owner:
650
+ owner = canonical.contract.owner
651
+ break
652
+
653
+ # Targets block — one binding per requested target. governance_schema is
654
+ # synthesised as ``<domain>_governance`` (matches fixtures).
655
+ targets_block: dict[str, dict[str, str]] = {
656
+ name: {"catalog": catalog, "governance_schema": governance_schema}
657
+ for name, catalog in target_catalogs.items()
658
+ }
659
+
660
+ # api_version / status / version from first member.
661
+ first_contract = members_sorted[0][1].contract
662
+ return dump_layer_contract_yaml(
663
+ domain=domain,
664
+ layer=layer,
665
+ contract_id=contract_id,
666
+ version=first_contract.version,
667
+ status=first_contract.status,
668
+ owner=owner,
669
+ targets=targets_block,
670
+ layer_custom_properties=common_props,
671
+ tables=final_tables,
672
+ api_version=first_contract.api_version,
673
+ )
674
+
675
+
676
+ # Issue #278 — bronze lineage URN shape:
677
+ # ``urn:kirimana:bronze:<contract-name>::<column-name>``.
678
+ _BRONZE_LINEAGE_URN_RE = re.compile(
679
+ r"\burn:kirimana:bronze:(?P<name>[^:\s\"']+)::(?P<col>[^\s\"']+)"
680
+ )
681
+
682
+
683
+ def _walk_flow_selectors(doc: Any) -> list[str]:
684
+ """Yield every ``select:`` string inside a flow YAML document.
685
+
686
+ Flow docs put their selectors at ``flow.select`` or ``select:`` at the
687
+ top level, possibly as a string, list of strings, or list of dicts
688
+ with a ``name`` field. We accept all three shapes — the goal is a
689
+ best-effort warning, not a strict parser (selectors are validated
690
+ elsewhere).
691
+ """
692
+ selectors: list[str] = []
693
+
694
+ def _flatten(value: Any) -> None:
695
+ if isinstance(value, str):
696
+ selectors.append(value)
697
+ elif isinstance(value, list):
698
+ for item in value:
699
+ if isinstance(item, str):
700
+ selectors.append(item)
701
+ elif isinstance(item, dict) and "name" in item:
702
+ name = item["name"]
703
+ if isinstance(name, str):
704
+ selectors.append(name)
705
+
706
+ def _visit(node: Any) -> None:
707
+ if isinstance(node, dict):
708
+ if "select" in node:
709
+ _flatten(node["select"])
710
+ for v in node.values():
711
+ _visit(v)
712
+ elif isinstance(node, list):
713
+ for item in node:
714
+ _visit(item)
715
+
716
+ _visit(doc)
717
+ return selectors
718
+
719
+
720
+ def _walk_bronze_lineage_urns(node: Any) -> list[str]:
721
+ """Yield every ``urn:kirimana:bronze:<name>::<col>`` string in a YAML doc."""
722
+
723
+ found: list[str] = []
724
+
725
+ def _visit(value: Any) -> None:
726
+ if isinstance(value, str):
727
+ for match in _BRONZE_LINEAGE_URN_RE.finditer(value):
728
+ found.append(match.group(0))
729
+ elif isinstance(value, dict):
730
+ for v in value.values():
731
+ _visit(v)
732
+ elif isinstance(value, list):
733
+ for item in value:
734
+ _visit(item)
735
+
736
+ _visit(node)
737
+ return found
738
+
739
+
740
+ def _scan_broken_references(
741
+ *,
742
+ project_root: Path,
743
+ valid_names_by_layer: dict[Layer, set[str]],
744
+ skip_files: set[Path],
745
+ ) -> list[BrokenReference]:
746
+ """Find flow selectors + bronze lineage URNs that don't resolve after migration.
747
+
748
+ Issue #278 — runs after the migration writes its layer files. ``skip_files``
749
+ lists the freshly written outputs so we don't flag the migration's own
750
+ output (those table-entry names are by definition the new truth).
751
+ """
752
+ broken: list[BrokenReference] = []
753
+ valid_all: set[str] = set()
754
+ for names in valid_names_by_layer.values():
755
+ valid_all.update(names)
756
+ valid_bronze = valid_names_by_layer.get(Layer.BRONZE, set())
757
+
758
+ # --- flow selectors -----------------------------------------------------
759
+ flows_dir = project_root / "flows"
760
+ if flows_dir.is_dir():
761
+ for flow_file in sorted(flows_dir.glob("*.yml")):
762
+ try:
763
+ doc = yaml.safe_load(flow_file.read_text(encoding="utf-8"))
764
+ except (yaml.YAMLError, OSError):
765
+ continue
766
+ for selector in _walk_flow_selectors(doc):
767
+ if selector and selector not in valid_all:
768
+ broken.append(
769
+ BrokenReference(
770
+ kind="flow_selector",
771
+ source_file=flow_file,
772
+ reference=selector,
773
+ detail=(
774
+ f"flow selector '{selector}' does not resolve "
775
+ "to any migrated layer-entry name"
776
+ ),
777
+ )
778
+ )
779
+
780
+ # --- bronze lineage URNs ------------------------------------------------
781
+ # Scan every YAML in the project root except the files the migration just
782
+ # wrote (those are the new source of truth and reference themselves by
783
+ # the new entry names).
784
+ skip_resolved = {p.resolve() for p in skip_files}
785
+ for yml in sorted(project_root.rglob("*.yml")):
786
+ if yml.resolve() in skip_resolved:
787
+ continue
788
+ try:
789
+ doc = yaml.safe_load(yml.read_text(encoding="utf-8"))
790
+ except (yaml.YAMLError, OSError):
791
+ continue
792
+ for urn in _walk_bronze_lineage_urns(doc):
793
+ match = _BRONZE_LINEAGE_URN_RE.search(urn)
794
+ if not match:
795
+ continue
796
+ bronze_name = match.group("name")
797
+ if bronze_name not in valid_bronze:
798
+ broken.append(
799
+ BrokenReference(
800
+ kind="lineage_urn",
801
+ source_file=yml,
802
+ reference=urn,
803
+ detail=(f"bronze contract '{bronze_name}' not in migrated bronze layer"),
804
+ )
805
+ )
806
+ return broken
807
+
808
+
809
+ def _first_table_name(canonical: Any) -> str | None:
810
+ if canonical.contract.schema_:
811
+ name = canonical.contract.schema_[0].name
812
+ return str(name) if name is not None else None
813
+ return None
814
+
815
+
816
+ def migrate_from_contracts(
817
+ contracts_dir: Path,
818
+ *,
819
+ out_dir: Path,
820
+ dry_run: bool = False,
821
+ keep_archive: bool = False,
822
+ target_names: tuple[str, ...] = ("prod",),
823
+ governance_schema_template: str = "{domain}_governance",
824
+ ) -> MigrationResult:
825
+ """Atomically migrate per-table contracts into layer manifests.
826
+
827
+ See module docstring for the full algorithm. Raises
828
+ :class:`MigrationError` on any fatal condition; no destination files
829
+ written on the error path.
830
+
831
+ Parameters:
832
+ contracts_dir: existing per-table contract directory to read.
833
+ out_dir: directory to write the ``01-config`` tree into (typically
834
+ the project root or its ``01-config`` subdir; we write to
835
+ ``out_dir`` directly without re-appending ``01-config``, so
836
+ the caller decides the layout).
837
+ dry_run: plan only — no writes, no source-dir mutation. Returns
838
+ an empty :class:`MigrationResult`; the plan lives on the
839
+ raised event log printed by the CLI.
840
+ keep_archive: rename the source dir to ``<contracts-dir>.archived``
841
+ instead of deleting it.
842
+ target_names: target names to declare in the emitted ``targets:``
843
+ blocks. The per-table catalog value is replicated across
844
+ every named target (the schema mapping is identical — only
845
+ ``catalog:`` differs at the kiri.yml level).
846
+ governance_schema_template: format string for the per-target
847
+ ``governance_schema``; receives ``{domain}``.
848
+ """
849
+ plan = plan_migration(contracts_dir, out_dir=out_dir)
850
+ if not plan.ok:
851
+ raise MigrationError(plan.issues)
852
+
853
+ if dry_run:
854
+ return MigrationResult()
855
+
856
+ # Re-parse to build per-group buffers (plan_migration already validated).
857
+ files = _walk_contract_files(contracts_dir)
858
+ parsed, _issues = _parse_all(files)
859
+ groups: dict[tuple[str, Layer], list[tuple[Path, Any]]] = {}
860
+ for path, canonical in parsed:
861
+ tprops = _props_dict(canonical.contract.custom_properties)
862
+ layer_raw = _resolve_layer_value(tprops)
863
+ if not layer_raw or layer_raw not in _LAYER_VALUES:
864
+ continue
865
+ domain = canonical.contract.domain
866
+ if not domain:
867
+ continue
868
+ groups.setdefault((domain, Layer(layer_raw)), []).append((path, canonical))
869
+
870
+ # Cross-reference planned files to know catalogs per group.
871
+ planned_by_key = {(p.domain, p.layer): p for p in plan.planned}
872
+
873
+ # Compose every YAML in memory BEFORE writing anything (atomic).
874
+ composed: list[tuple[Path, str]] = []
875
+ for key, members in sorted(groups.items(), key=lambda kv: (kv[0][0], kv[0][1].value)):
876
+ domain, layer = key
877
+ planned = planned_by_key[key]
878
+ catalog_value = planned.target_catalogs["prod"]
879
+ governance = governance_schema_template.format(domain=domain)
880
+ target_catalogs: dict[str, str] = {name: catalog_value for name in target_names}
881
+ yaml_text = _build_layer_doc(
882
+ domain=domain,
883
+ layer=layer,
884
+ members=members,
885
+ target_catalogs=target_catalogs,
886
+ governance_schema=governance,
887
+ )
888
+ composed.append((planned.out_path, yaml_text))
889
+
890
+ # Write phase — all-or-nothing. We don't have a true 2PC but we minimise
891
+ # the window: stage in a sibling dir then move into place.
892
+ written: list[Path] = []
893
+ for out_path, text in composed:
894
+ out_path.parent.mkdir(parents=True, exist_ok=True)
895
+ out_path.write_text(text, encoding="utf-8")
896
+ written.append(out_path)
897
+
898
+ # Re-validate the emitted layer-file project to refuse to declare
899
+ # success if the result is invalid.
900
+ project_root = out_dir.parent if out_dir.name == CONFIG_SUBDIR else out_dir
901
+ if (project_root / CONFIG_SUBDIR).is_dir():
902
+ report = validate_project(project_root)
903
+ else:
904
+ # Caller targeted an arbitrary directory; validate against the parent
905
+ # of every written file's grandparent (best effort). For correctness
906
+ # the CLI always points out_dir at the canonical 01-config dir.
907
+ report = ValidationReport()
908
+
909
+ if not report.ok:
910
+ # Roll back the writes — atomic posture.
911
+ import contextlib
912
+
913
+ for p in written:
914
+ with contextlib.suppress(FileNotFoundError):
915
+ p.unlink()
916
+ raise MigrationError(list(report.issues))
917
+
918
+ # Issue #278 — scan for intra-project references that no migrated layer
919
+ # entry exposes. Most projects come out clean (Option A above preserves
920
+ # the contract name as the layer-entry name); the scan catches drift /
921
+ # hand-edited projects / migrations that pre-date the Option A fix.
922
+ valid_names_by_layer: dict[Layer, set[str]] = {}
923
+ for key, members in groups.items():
924
+ _, layer = key
925
+ names_for_layer = valid_names_by_layer.setdefault(layer, set())
926
+ for _path, canonical in members:
927
+ contract_name = canonical.contract.name
928
+ if contract_name:
929
+ names_for_layer.add(contract_name)
930
+ schema_list = canonical.contract.schema_
931
+ if schema_list:
932
+ first_name = schema_list[0].name
933
+ if first_name:
934
+ names_for_layer.add(str(first_name))
935
+ broken_references = _scan_broken_references(
936
+ project_root=project_root,
937
+ valid_names_by_layer=valid_names_by_layer,
938
+ skip_files=set(written),
939
+ )
940
+
941
+ # Delete or archive the source dir.
942
+ archived_to: Path | None = None
943
+ deleted = False
944
+ if keep_archive:
945
+ archived_to = _archive_dir(contracts_dir)
946
+ if archived_to.exists():
947
+ raise MigrationError(
948
+ [
949
+ ValidationIssue(
950
+ path=str(archived_to),
951
+ message=(
952
+ f"archive path {archived_to!s} already exists; "
953
+ "refusing to overwrite (move it aside and retry)."
954
+ ),
955
+ )
956
+ ]
957
+ )
958
+ contracts_dir.rename(archived_to)
959
+ else:
960
+ shutil.rmtree(contracts_dir)
961
+ deleted = True
962
+
963
+ return MigrationResult(
964
+ written=written,
965
+ archived_to=archived_to,
966
+ deleted=deleted,
967
+ validation=report,
968
+ broken_references=broken_references,
969
+ )