contractforge-databricks 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (220) hide show
  1. contractforge_databricks/__init__.py +172 -0
  2. contractforge_databricks/adapter.py +69 -0
  3. contractforge_databricks/annotations/__init__.py +10 -0
  4. contractforge_databricks/annotations/application.py +52 -0
  5. contractforge_databricks/annotations/audit.py +49 -0
  6. contractforge_databricks/annotations/sql.py +142 -0
  7. contractforge_databricks/api.py +65 -0
  8. contractforge_databricks/bundles/__init__.py +9 -0
  9. contractforge_databricks/bundles/assets.py +47 -0
  10. contractforge_databricks/bundles/project.py +213 -0
  11. contractforge_databricks/bundles/project_config.py +133 -0
  12. contractforge_databricks/capabilities/__init__.py +17 -0
  13. contractforge_databricks/capabilities/builders.py +43 -0
  14. contractforge_databricks/capabilities/evaluate.py +162 -0
  15. contractforge_databricks/capabilities/mapping.py +36 -0
  16. contractforge_databricks/capabilities/models.py +44 -0
  17. contractforge_databricks/capabilities/runtime.py +111 -0
  18. contractforge_databricks/capabilities/uc.py +47 -0
  19. contractforge_databricks/cli.py +196 -0
  20. contractforge_databricks/cli_deploy.py +98 -0
  21. contractforge_databricks/cli_governance.py +142 -0
  22. contractforge_databricks/cli_io.py +91 -0
  23. contractforge_databricks/cli_maintenance.py +69 -0
  24. contractforge_databricks/coercion.py +31 -0
  25. contractforge_databricks/contract_extensions.py +70 -0
  26. contractforge_databricks/cost/__init__.py +11 -0
  27. contractforge_databricks/cost/model.py +22 -0
  28. contractforge_databricks/cost/report.py +65 -0
  29. contractforge_databricks/cost/sql.py +136 -0
  30. contractforge_databricks/dashboards/__init__.py +15 -0
  31. contractforge_databricks/dashboards/control_tables.py +150 -0
  32. contractforge_databricks/diagnostics/__init__.py +7 -0
  33. contractforge_databricks/diagnostics/explain.py +40 -0
  34. contractforge_databricks/environment.py +53 -0
  35. contractforge_databricks/evidence/__init__.py +98 -0
  36. contractforge_databricks/evidence/ddl.py +35 -0
  37. contractforge_databricks/evidence/governance_log.py +175 -0
  38. contractforge_databricks/evidence/helpers.py +29 -0
  39. contractforge_databricks/evidence/ops_log.py +210 -0
  40. contractforge_databricks/evidence/records.py +27 -0
  41. contractforge_databricks/evidence/run_log.py +74 -0
  42. contractforge_databricks/evidence/schemas.py +7 -0
  43. contractforge_databricks/evidence/sql.py +144 -0
  44. contractforge_databricks/evidence/tables.py +20 -0
  45. contractforge_databricks/evidence/writer.py +118 -0
  46. contractforge_databricks/execution/__init__.py +70 -0
  47. contractforge_databricks/execution/delta_basic.py +57 -0
  48. contractforge_databricks/execution/hash_diff.py +126 -0
  49. contractforge_databricks/execution/hash_diff_latest.py +142 -0
  50. contractforge_databricks/execution/replace_partitions.py +40 -0
  51. contractforge_databricks/execution/results.py +5 -0
  52. contractforge_databricks/execution/retry.py +36 -0
  53. contractforge_databricks/execution/scd2.py +213 -0
  54. contractforge_databricks/execution/scd2_deletes.py +65 -0
  55. contractforge_databricks/execution/scd2_late.py +30 -0
  56. contractforge_databricks/execution/snapshot.py +77 -0
  57. contractforge_databricks/execution/sql_merge.py +85 -0
  58. contractforge_databricks/execution/tables.py +98 -0
  59. contractforge_databricks/execution/windows.py +58 -0
  60. contractforge_databricks/governance/__init__.py +30 -0
  61. contractforge_databricks/governance/access.py +185 -0
  62. contractforge_databricks/governance/application.py +93 -0
  63. contractforge_databricks/governance/drift.py +49 -0
  64. contractforge_databricks/governance/runtime.py +60 -0
  65. contractforge_databricks/governance/sql.py +31 -0
  66. contractforge_databricks/governance/validation.py +135 -0
  67. contractforge_databricks/lakeflow/__init__.py +21 -0
  68. contractforge_databricks/lakeflow/compatibility.py +194 -0
  69. contractforge_databricks/lakeflow/rendering.py +175 -0
  70. contractforge_databricks/lineage/__init__.py +7 -0
  71. contractforge_databricks/lineage/openlineage.py +182 -0
  72. contractforge_databricks/maintenance/__init__.py +27 -0
  73. contractforge_databricks/maintenance/retention.py +90 -0
  74. contractforge_databricks/maintenance/sql.py +68 -0
  75. contractforge_databricks/metrics/__init__.py +19 -0
  76. contractforge_databricks/metrics/history.py +21 -0
  77. contractforge_databricks/metrics/write.py +63 -0
  78. contractforge_databricks/operations/__init__.py +4 -0
  79. contractforge_databricks/operations/application.py +38 -0
  80. contractforge_databricks/operations/sql.py +95 -0
  81. contractforge_databricks/parity/__init__.py +18 -0
  82. contractforge_databricks/parity/catalog.py +59 -0
  83. contractforge_databricks/parity/models.py +7 -0
  84. contractforge_databricks/parity/scenarios.py +111 -0
  85. contractforge_databricks/partitioning/__init__.py +3 -0
  86. contractforge_databricks/partitioning/predicates.py +28 -0
  87. contractforge_databricks/preparation/__init__.py +47 -0
  88. contractforge_databricks/preparation/deduplicate.py +87 -0
  89. contractforge_databricks/preparation/encoding.py +37 -0
  90. contractforge_databricks/preparation/hashing.py +18 -0
  91. contractforge_databricks/preparation/pyspark.py +178 -0
  92. contractforge_databricks/preparation/pyspark_staging.py +70 -0
  93. contractforge_databricks/preparation/shape.py +209 -0
  94. contractforge_databricks/preparation/shape_validation.py +94 -0
  95. contractforge_databricks/preparation/staging.py +17 -0
  96. contractforge_databricks/preparation/zip_arrays.py +51 -0
  97. contractforge_databricks/presets/__init__.py +3 -0
  98. contractforge_databricks/presets/base.py +24 -0
  99. contractforge_databricks/presets/bronze.py +57 -0
  100. contractforge_databricks/presets/catalog.py +22 -0
  101. contractforge_databricks/presets/core.py +134 -0
  102. contractforge_databricks/presets/gold.py +62 -0
  103. contractforge_databricks/presets/modifiers.py +51 -0
  104. contractforge_databricks/presets/runtime.py +22 -0
  105. contractforge_databricks/presets/silver.py +101 -0
  106. contractforge_databricks/presets/write_engine.py +57 -0
  107. contractforge_databricks/quality/__init__.py +41 -0
  108. contractforge_databricks/quality/evaluation.py +178 -0
  109. contractforge_databricks/quality/persistence.py +81 -0
  110. contractforge_databricks/quality/registry.py +134 -0
  111. contractforge_databricks/quality/results.py +17 -0
  112. contractforge_databricks/quality/sql.py +113 -0
  113. contractforge_databricks/rendering/__init__.py +11 -0
  114. contractforge_databricks/rendering/bundle.py +93 -0
  115. contractforge_databricks/rendering/markdown.py +50 -0
  116. contractforge_databricks/rendering/names.py +56 -0
  117. contractforge_databricks/results.py +15 -0
  118. contractforge_databricks/runtime/__init__.py +101 -0
  119. contractforge_databricks/runtime/available_now.py +147 -0
  120. contractforge_databricks/runtime/bundles.py +211 -0
  121. contractforge_databricks/runtime/cache.py +20 -0
  122. contractforge_databricks/runtime/control_tables.py +19 -0
  123. contractforge_databricks/runtime/deploy.py +197 -0
  124. contractforge_databricks/runtime/detection.py +114 -0
  125. contractforge_databricks/runtime/dry_run.py +46 -0
  126. contractforge_databricks/runtime/errors.py +54 -0
  127. contractforge_databricks/runtime/file_selection.py +109 -0
  128. contractforge_databricks/runtime/finalization.py +168 -0
  129. contractforge_databricks/runtime/governance.py +37 -0
  130. contractforge_databricks/runtime/hooks.py +45 -0
  131. contractforge_databricks/runtime/http_file.py +37 -0
  132. contractforge_databricks/runtime/http_retry.py +15 -0
  133. contractforge_databricks/runtime/http_safety.py +9 -0
  134. contractforge_databricks/runtime/json_materialization.py +97 -0
  135. contractforge_databricks/runtime/lineage.py +164 -0
  136. contractforge_databricks/runtime/maintenance.py +43 -0
  137. contractforge_databricks/runtime/merge_validation.py +98 -0
  138. contractforge_databricks/runtime/metadata.py +21 -0
  139. contractforge_databricks/runtime/metrics.py +34 -0
  140. contractforge_databricks/runtime/models.py +32 -0
  141. contractforge_databricks/runtime/options.py +33 -0
  142. contractforge_databricks/runtime/orchestration_context.py +185 -0
  143. contractforge_databricks/runtime/orchestrator.py +147 -0
  144. contractforge_databricks/runtime/partitioning.py +93 -0
  145. contractforge_databricks/runtime/quality_quarantine.py +92 -0
  146. contractforge_databricks/runtime/rest_api.py +46 -0
  147. contractforge_databricks/runtime/rest_auth.py +21 -0
  148. contractforge_databricks/runtime/rest_pagination.py +21 -0
  149. contractforge_databricks/runtime/run_payload.py +177 -0
  150. contractforge_databricks/runtime/schema.py +106 -0
  151. contractforge_databricks/runtime/source_metadata.py +30 -0
  152. contractforge_databricks/runtime/source_registry.py +43 -0
  153. contractforge_databricks/runtime/source_schema.py +24 -0
  154. contractforge_databricks/runtime/sources.py +208 -0
  155. contractforge_databricks/runtime/spark.py +183 -0
  156. contractforge_databricks/runtime/spark_defaults.py +35 -0
  157. contractforge_databricks/runtime/storage_auth.py +132 -0
  158. contractforge_databricks/runtime/streaming.py +131 -0
  159. contractforge_databricks/runtime/success.py +104 -0
  160. contractforge_databricks/runtime/utils.py +52 -0
  161. contractforge_databricks/runtime/watermark.py +71 -0
  162. contractforge_databricks/runtime/windows.py +184 -0
  163. contractforge_databricks/runtime/write.py +66 -0
  164. contractforge_databricks/runtime/write_flow.py +146 -0
  165. contractforge_databricks/runtime/write_strategy.py +40 -0
  166. contractforge_databricks/schema/__init__.py +21 -0
  167. contractforge_databricks/schema/diff.py +11 -0
  168. contractforge_databricks/schema/policy.py +33 -0
  169. contractforge_databricks/schema/sync.py +23 -0
  170. contractforge_databricks/security/__init__.py +21 -0
  171. contractforge_databricks/security/errors.py +5 -0
  172. contractforge_databricks/security/redaction.py +5 -0
  173. contractforge_databricks/security/secrets.py +114 -0
  174. contractforge_databricks/security/source_policy.py +17 -0
  175. contractforge_databricks/shapes/__init__.py +3 -0
  176. contractforge_databricks/shapes/sql.py +123 -0
  177. contractforge_databricks/sources/__init__.py +67 -0
  178. contractforge_databricks/sources/artifacts.py +100 -0
  179. contractforge_databricks/sources/autoloader.py +48 -0
  180. contractforge_databricks/sources/bounded_streams.py +44 -0
  181. contractforge_databricks/sources/classification.py +115 -0
  182. contractforge_databricks/sources/delta_share.py +21 -0
  183. contractforge_databricks/sources/files.py +48 -0
  184. contractforge_databricks/sources/http_file.py +46 -0
  185. contractforge_databricks/sources/interpret.py +76 -0
  186. contractforge_databricks/sources/jdbc.py +32 -0
  187. contractforge_databricks/sources/metadata.py +18 -0
  188. contractforge_databricks/sources/native_passthrough.py +33 -0
  189. contractforge_databricks/sources/rds_iam.py +15 -0
  190. contractforge_databricks/sources/rds_iam_runtime.py +191 -0
  191. contractforge_databricks/sources/rest_api.py +33 -0
  192. contractforge_databricks/sources/support.py +50 -0
  193. contractforge_databricks/sources/table_refs.py +65 -0
  194. contractforge_databricks/sql/__init__.py +4 -0
  195. contractforge_databricks/sql/identifiers.py +17 -0
  196. contractforge_databricks/sql/literals.py +36 -0
  197. contractforge_databricks/state/__init__.py +39 -0
  198. contractforge_databricks/state/ddl.py +24 -0
  199. contractforge_databricks/state/migrations.py +146 -0
  200. contractforge_databricks/state/queries.py +149 -0
  201. contractforge_databricks/state/sql.py +116 -0
  202. contractforge_databricks/state/tables.py +9 -0
  203. contractforge_databricks/state/writer.py +83 -0
  204. contractforge_databricks/templates/__init__.py +15 -0
  205. contractforge_databricks/templates/catalog.py +205 -0
  206. contractforge_databricks/templates/catalog_parity.py +85 -0
  207. contractforge_databricks/templates/core.py +83 -0
  208. contractforge_databricks/templates/enrichment.py +175 -0
  209. contractforge_databricks/transforms/__init__.py +3 -0
  210. contractforge_databricks/transforms/sql.py +118 -0
  211. contractforge_databricks/watermark/__init__.py +6 -0
  212. contractforge_databricks/watermark/sql.py +91 -0
  213. contractforge_databricks/write_modes/__init__.py +20 -0
  214. contractforge_databricks/write_modes/registry.py +44 -0
  215. contractforge_databricks/write_modes/sql.py +33 -0
  216. contractforge_databricks/write_modes/strategy.py +192 -0
  217. contractforge_databricks-0.1.0.dist-info/METADATA +34 -0
  218. contractforge_databricks-0.1.0.dist-info/RECORD +220 -0
  219. contractforge_databricks-0.1.0.dist-info/WHEEL +4 -0
  220. contractforge_databricks-0.1.0.dist-info/entry_points.txt +2 -0
@@ -0,0 +1,111 @@
1
+ """Runtime classification helpers for Databricks capability evaluation."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Any
6
+
7
+ from contractforge_databricks.capabilities.models import CapabilityEvidence, RuntimeKind
8
+
9
+ SERVERLESS_TRUE_KEYS = (
10
+ "spark.databricks.serverless.enabled",
11
+ "spark.databricks.compute.serverless.enabled",
12
+ )
13
+ DATABRICKS_ENVIRONMENT_KEYS = (
14
+ "DB_INSTANCE_TYPE",
15
+ "DATABRICKS_RUNTIME_VERSION",
16
+ "DATABRICKS_ENV_VERSION",
17
+ "SPARK_CONNECT_MODE_ENABLED",
18
+ "SPARK_EXECUTOR_ATTRIBUTE_POD_NAME",
19
+ )
20
+ CLASSIC_CLUSTER_KEYS = (
21
+ "spark.databricks.clusterUsageTags.clusterId",
22
+ "spark.databricks.clusterUsageTags.clusterName",
23
+ "spark.databricks.clusterUsageTags.clusterType",
24
+ )
25
+ JOB_METADATA_KEY_FRAGMENTS = ("job", "run")
26
+
27
+
28
+ def runtime_kind(
29
+ *,
30
+ runtime_type: str | None,
31
+ spark_conf: dict[str, str],
32
+ environment: dict[str, str] | None = None,
33
+ ) -> RuntimeKind:
34
+ normalized = (runtime_type or "").strip().lower()
35
+ if normalized in {"serverless", "serverless_job", "databricks_serverless"}:
36
+ return "databricks_serverless"
37
+ if normalized == "classic":
38
+ return "databricks_classic" if has_databricks_conf(spark_conf) else "spark"
39
+ if normalized in {"classic_cluster", "classic_existing_cluster", "databricks_classic"}:
40
+ return "databricks_classic"
41
+ if is_serverless_runtime(spark_conf, environment=environment):
42
+ return "databricks_serverless"
43
+ if has_databricks_conf(spark_conf):
44
+ return "databricks_classic"
45
+ return "unknown" if not spark_conf else "spark"
46
+
47
+
48
+ def runtime_evidence(
49
+ *,
50
+ runtime_kind: RuntimeKind,
51
+ spark_version: str | None,
52
+ spark_conf: dict[str, str],
53
+ environment: dict[str, str] | None = None,
54
+ ) -> tuple[CapabilityEvidence, ...]:
55
+ evidence = [_e("runtime_kind", "Runtime classified from provided evidence.", runtime_kind)]
56
+ if spark_version:
57
+ evidence.append(_e("spark_version", "Spark version was provided.", spark_version))
58
+ for key in sorted(spark_conf):
59
+ if key.startswith("spark.databricks."):
60
+ evidence.append(_e("spark_conf", "Databricks Spark configuration key detected.", key))
61
+ for key in sorted(environment or {}):
62
+ if key in DATABRICKS_ENVIRONMENT_KEYS:
63
+ evidence.append(_e("environment", "Databricks runtime environment key detected.", key))
64
+ return tuple(evidence)
65
+
66
+
67
+ def has_databricks_conf(spark_conf: dict[str, str]) -> bool:
68
+ return any(key.startswith("spark.databricks.") and str(value).strip() for key, value in spark_conf.items())
69
+
70
+
71
+ def is_serverless_runtime(
72
+ spark_conf: dict[str, str],
73
+ *,
74
+ environment: dict[str, str] | None = None,
75
+ ) -> bool:
76
+ return is_serverless_conf(spark_conf) or is_serverless_environment(environment or {})
77
+
78
+
79
+ def is_serverless_conf(spark_conf: dict[str, str]) -> bool:
80
+ normalized = {str(key): str(value) for key, value in spark_conf.items()}
81
+ if any(normalized.get(key, "").strip().lower() == "true" for key in SERVERLESS_TRUE_KEYS):
82
+ return True
83
+ if any(key.startswith("spark.databricks.") and "serverless" in value.lower() for key, value in normalized.items()):
84
+ return True
85
+ return _looks_like_databricks_serverless_job(normalized)
86
+
87
+
88
+ def is_serverless_environment(environment: dict[str, str]) -> bool:
89
+ normalized = {str(key): str(value) for key, value in environment.items()}
90
+ return any(key in DATABRICKS_ENVIRONMENT_KEYS and "serverless" in value.lower() for key, value in normalized.items())
91
+
92
+
93
+ def is_three_part_name(target_table: str | None) -> bool:
94
+ return bool(target_table and len([part for part in target_table.split(".") if part.strip()]) >= 3)
95
+
96
+
97
+ def _looks_like_databricks_serverless_job(spark_conf: dict[str, str]) -> bool:
98
+ if not has_databricks_conf(spark_conf):
99
+ return False
100
+ if any(spark_conf.get(key, "").strip() for key in CLASSIC_CLUSTER_KEYS):
101
+ return False
102
+ return any(
103
+ key.startswith("spark.databricks.")
104
+ and all(fragment in key.lower() for fragment in JOB_METADATA_KEY_FRAGMENTS)
105
+ and str(value).strip()
106
+ for key, value in spark_conf.items()
107
+ )
108
+
109
+
110
+ def _e(source: str, message: str, value: Any = None) -> CapabilityEvidence:
111
+ return CapabilityEvidence(source=source, message=message, value=None if value is None else str(value))
@@ -0,0 +1,47 @@
1
+ """Unity Catalog capability issue helpers."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Any, Iterable
6
+
7
+ from contractforge_databricks.capabilities.evaluate import evaluate_databricks_capabilities
8
+ from contractforge_databricks.capabilities.models import DatabricksCapabilities
9
+
10
+ UC_CAPABILITY_ALIASES = {
11
+ "table_comments": "uc_table_comments",
12
+ "column_comments": "uc_column_comments",
13
+ "table_tags": "uc_table_tags",
14
+ "column_tags": "uc_column_tags",
15
+ "grants": "uc_grants",
16
+ "row_filters": "uc_row_filters",
17
+ "column_masks": "uc_column_masks",
18
+ }
19
+
20
+
21
+ def uc_capability_issues(
22
+ target_table: str,
23
+ requirements: Iterable[tuple[str, str, str, str]],
24
+ *,
25
+ capabilities: DatabricksCapabilities | None = None,
26
+ runtime_type: str | None = "serverless",
27
+ ) -> list[dict[str, Any]]:
28
+ caps = capabilities or evaluate_databricks_capabilities(target_table=target_table, runtime_type=runtime_type)
29
+ issues = []
30
+ for capability, scope, obj, severity in requirements:
31
+ native_name = UC_CAPABILITY_ALIASES.get(capability, capability)
32
+ if caps.supports(native_name):
33
+ continue
34
+ issues.append(
35
+ {
36
+ "severity": severity,
37
+ "scope": scope,
38
+ "object": obj,
39
+ "capability": native_name,
40
+ "status": caps.status(native_name),
41
+ "message": (
42
+ f"{native_name} is not supported for {target_table}. "
43
+ "Use a three-part Unity Catalog table or remove the feature from the contract."
44
+ ),
45
+ }
46
+ )
47
+ return issues
@@ -0,0 +1,196 @@
1
+ """CLI entrypoint for Databricks adapter utilities."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+ import json
7
+ import sys
8
+ from pathlib import Path
9
+ from typing import Any
10
+
11
+ from contractforge_databricks.api import render_databricks_contract
12
+ from contractforge_databricks.cli_deploy import add_deploy_parser, deploy_command
13
+ from contractforge_databricks.cli_governance import add_governance_parser, governance_command
14
+ from contractforge_databricks.cli_io import load_contract_input, load_mapping, write_artifacts, write_mapping
15
+ from contractforge_databricks.cli_maintenance import add_maintenance_parser, maintenance_command
16
+ from contractforge_databricks.dashboards import render_control_dashboard_artifacts
17
+ from contractforge_databricks.presets import list_presets, preset_details
18
+ from contractforge_databricks.templates import (
19
+ contract_template_details,
20
+ contract_template_files,
21
+ get_contract_template,
22
+ list_contract_templates,
23
+ recommend_contract_templates,
24
+ )
25
+
26
+
27
+ def build_parser() -> argparse.ArgumentParser:
28
+ parser = argparse.ArgumentParser(prog="contractforge-databricks")
29
+ sub = parser.add_subparsers(dest="command", required=True)
30
+
31
+ presets = sub.add_parser("presets", help="List or inspect Databricks adapter presets")
32
+ presets_sub = presets.add_subparsers(dest="preset_command", required=True)
33
+ presets_list = presets_sub.add_parser("list")
34
+ presets_list.add_argument("--indent", type=int, default=2)
35
+ presets_show = presets_sub.add_parser("show")
36
+ presets_show.add_argument("names", nargs="+")
37
+ presets_show.add_argument("--indent", type=int, default=2)
38
+
39
+ templates = sub.add_parser("templates", help="List, show, recommend or write Databricks contract templates")
40
+ templates_sub = templates.add_subparsers(dest="template_command", required=True)
41
+ templates_list = templates_sub.add_parser("list")
42
+ templates_list.add_argument("--indent", type=int, default=2)
43
+ templates_show = templates_sub.add_parser("show")
44
+ templates_show.add_argument("name")
45
+ templates_show.add_argument("--metadata-only", action="store_true")
46
+ templates_show.add_argument("--indent", type=int, default=2)
47
+ templates_write = templates_sub.add_parser("write")
48
+ templates_write.add_argument("name")
49
+ templates_write.add_argument("--output", required=True, type=Path)
50
+ templates_write.add_argument("--force", action="store_true")
51
+ templates_write.add_argument("--indent", type=int, default=2)
52
+ templates_wizard = templates_sub.add_parser("wizard")
53
+ templates_wizard.add_argument("--layer")
54
+ templates_wizard.add_argument("--source")
55
+ templates_wizard.add_argument("--mode")
56
+ templates_wizard.add_argument("--pattern")
57
+ templates_wizard.add_argument("--limit", type=int, default=5)
58
+ templates_wizard.add_argument("--name")
59
+ templates_wizard.add_argument("--output", type=Path)
60
+ templates_wizard.add_argument("--force", action="store_true")
61
+ templates_wizard.add_argument("--indent", type=int, default=2)
62
+
63
+ render = sub.add_parser("render", help="Render Databricks review artifacts for a contract JSON/YAML file")
64
+ render.add_argument("contract", type=Path)
65
+ render.add_argument("--environment", type=Path)
66
+ render.add_argument("--output-dir", type=Path)
67
+ render.add_argument("--indent", type=int, default=2)
68
+
69
+ dashboard = sub.add_parser("dashboard", help="Render Databricks control-table dashboard artifacts")
70
+ dashboard.add_argument("--catalog", default="main")
71
+ dashboard.add_argument("--schema", default="ops")
72
+ dashboard.add_argument("--lookback-days", type=int, default=7)
73
+ dashboard.add_argument("--output-dir", type=Path)
74
+ dashboard.add_argument("--indent", type=int, default=2)
75
+
76
+ add_governance_parser(sub)
77
+ add_maintenance_parser(sub)
78
+ add_deploy_parser(sub)
79
+
80
+ return parser
81
+
82
+
83
+ def main(argv: list[str] | None = None) -> int:
84
+ parser = build_parser()
85
+ args = parser.parse_args(argv)
86
+ try:
87
+ return _dispatch(args)
88
+ except Exception as exc:
89
+ print(f"ERROR: {exc}", file=sys.stderr)
90
+ return 1
91
+
92
+
93
+ def _dispatch(args: argparse.Namespace) -> int:
94
+ if args.command == "presets":
95
+ return _presets(args)
96
+ if args.command == "templates":
97
+ return _templates(args)
98
+ if args.command == "render":
99
+ return _render_contract(args)
100
+ if args.command == "dashboard":
101
+ return _dashboard(args)
102
+ governance_result = governance_command(args)
103
+ if governance_result is not None:
104
+ return governance_result
105
+ if args.command == "maintenance":
106
+ return maintenance_command(args)
107
+ deploy_result = deploy_command(args)
108
+ if deploy_result is not None:
109
+ return deploy_result
110
+ raise ValueError(f"unsupported command: {args.command}")
111
+
112
+
113
+ def _presets(args: argparse.Namespace) -> int:
114
+ if args.preset_command == "list":
115
+ return _print([preset_details(name) for name in list_presets()], args.indent)
116
+ if args.preset_command == "show":
117
+ return _print([preset_details(name) for name in args.names], args.indent)
118
+ raise ValueError(f"unsupported presets command: {args.preset_command}")
119
+
120
+
121
+ def _templates(args: argparse.Namespace) -> int:
122
+ if args.template_command == "list":
123
+ return _print([contract_template_details(name) for name in list_contract_templates()], args.indent)
124
+ if args.template_command == "show":
125
+ payload = contract_template_details(args.name) if args.metadata_only else get_contract_template(args.name)
126
+ return _print(payload, args.indent)
127
+ if args.template_command == "write":
128
+ written = _write_template_files(args.name, args.output, force=args.force)
129
+ return _print({"status": "SUCCESS", "template": args.name, "written": written}, args.indent)
130
+ if args.template_command == "wizard":
131
+ recommendations = recommend_contract_templates(
132
+ layer=args.layer,
133
+ source=args.source,
134
+ mode=args.mode,
135
+ pattern=args.pattern,
136
+ limit=args.limit,
137
+ )
138
+ result: dict[str, Any] = {
139
+ "status": "SUCCESS",
140
+ "criteria": {
141
+ "layer": args.layer,
142
+ "source": args.source,
143
+ "mode": args.mode,
144
+ "pattern": args.pattern,
145
+ },
146
+ "recommendations": recommendations,
147
+ }
148
+ if args.output:
149
+ if not recommendations:
150
+ raise ValueError("no compatible template found for the provided criteria")
151
+ selected = args.name or str(recommendations[0]["name"])
152
+ result["selected_template"] = selected
153
+ result["written"] = _write_template_files(selected, args.output, force=args.force)
154
+ return _print(result, args.indent)
155
+ raise ValueError(f"unsupported templates command: {args.template_command}")
156
+
157
+
158
+ def _render_contract(args: argparse.Namespace) -> int:
159
+ contract, bundle_environment = load_contract_input(args.contract)
160
+ environment = load_mapping(args.environment) if args.environment else bundle_environment
161
+ artifacts = render_databricks_contract(contract, environment=environment).artifacts
162
+ if args.output_dir:
163
+ written = write_artifacts(args.output_dir, artifacts)
164
+ return _print({"status": "SUCCESS", "written": written}, args.indent)
165
+ return _print(artifacts, args.indent)
166
+
167
+
168
+ def _dashboard(args: argparse.Namespace) -> int:
169
+ artifacts = render_control_dashboard_artifacts(
170
+ catalog=args.catalog,
171
+ schema=args.schema,
172
+ lookback_days=args.lookback_days,
173
+ )
174
+ if args.output_dir:
175
+ written = write_artifacts(args.output_dir, artifacts)
176
+ return _print({"status": "SUCCESS", "written": written}, args.indent)
177
+ return _print(artifacts, args.indent)
178
+
179
+
180
+ def _write_template_files(name: str, output: Path, *, force: bool) -> list[str]:
181
+ files = contract_template_files(name)
182
+ written = []
183
+ for kind, payload in files.items():
184
+ path = output.with_suffix(f".{kind}.yaml")
185
+ write_mapping(path, payload, force=force)
186
+ written.append(str(path))
187
+ return written
188
+
189
+
190
+ def _print(payload: object, indent: int) -> int:
191
+ print(json.dumps(payload, indent=indent, sort_keys=True, default=str))
192
+ return 0
193
+
194
+
195
+ if __name__ == "__main__": # pragma: no cover
196
+ raise SystemExit(main())
@@ -0,0 +1,98 @@
1
+ """Databricks deployment CLI commands."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+ import json
7
+ from pathlib import Path
8
+ from typing import Callable
9
+
10
+ from contractforge_databricks.runtime.deploy import (
11
+ deploy_databricks_bundle,
12
+ deploy_databricks_project,
13
+ render_databricks_project_bundle_file,
14
+ )
15
+
16
+
17
+ def add_deploy_parser(subparsers: argparse._SubParsersAction[argparse.ArgumentParser]) -> None:
18
+ project = subparsers.add_parser("deploy-project", help="Validate/deploy/run a project Databricks Asset Bundle.")
19
+ project.add_argument("project", type=Path)
20
+ project.add_argument("--render-bundle", action="store_true", help="Render databricks.yml from project.yaml before deployment.")
21
+ project.add_argument("--force-render", action="store_true", help="Overwrite the rendered bundle when --render-bundle is used.")
22
+ _add_common_deploy_args(project)
23
+
24
+ bundle = subparsers.add_parser("deploy-bundle", help="Validate/deploy/run a Databricks Asset Bundle directory.")
25
+ bundle.add_argument("bundle", type=Path)
26
+ _add_common_deploy_args(bundle)
27
+
28
+ render = subparsers.add_parser("render-project-bundle", help="Render databricks.yml from project.yaml scheduling metadata.")
29
+ render.add_argument("project", type=Path)
30
+ render.add_argument("--output", type=Path, default=Path("databricks.yml"))
31
+ render.add_argument("--target", default="dev")
32
+ render.add_argument("--force", action="store_true")
33
+ render.add_argument("--indent", type=int, default=2)
34
+
35
+
36
+ def deploy_command(args: argparse.Namespace) -> int | None:
37
+ handler = _COMMANDS.get(args.command)
38
+ return None if handler is None else handler(args)
39
+
40
+
41
+ def _handle_project(args: argparse.Namespace) -> int:
42
+ return _print(
43
+ deploy_databricks_project(
44
+ args.project,
45
+ profile=args.profile,
46
+ target=args.target,
47
+ run=args.run,
48
+ validate=not args.skip_validate,
49
+ render_bundle=args.render_bundle,
50
+ force_render=args.force_render,
51
+ ),
52
+ args.indent,
53
+ )
54
+
55
+
56
+ def _handle_bundle(args: argparse.Namespace) -> int:
57
+ return _print(
58
+ deploy_databricks_bundle(
59
+ args.bundle,
60
+ profile=args.profile,
61
+ target=args.target,
62
+ run=args.run,
63
+ validate=not args.skip_validate,
64
+ ),
65
+ args.indent,
66
+ )
67
+
68
+
69
+ def _handle_render_project_bundle(args: argparse.Namespace) -> int:
70
+ return _print(
71
+ render_databricks_project_bundle_file(
72
+ args.project,
73
+ args.output,
74
+ target=args.target,
75
+ force=args.force,
76
+ ),
77
+ args.indent,
78
+ )
79
+
80
+
81
+ def _add_common_deploy_args(parser: argparse.ArgumentParser) -> None:
82
+ parser.add_argument("--profile")
83
+ parser.add_argument("--target")
84
+ parser.add_argument("--run", action="store_true", help="Run the deployed bundle job after deployment.")
85
+ parser.add_argument("--skip-validate", action="store_true")
86
+ parser.add_argument("--indent", type=int, default=2)
87
+
88
+
89
+ def _print(payload: object, indent: int) -> int:
90
+ print(json.dumps(payload, indent=indent, sort_keys=True, default=str))
91
+ return 0
92
+
93
+
94
+ _COMMANDS: dict[str, Callable[[argparse.Namespace], int]] = {
95
+ "deploy-project": _handle_project,
96
+ "deploy-bundle": _handle_bundle,
97
+ "render-project-bundle": _handle_render_project_bundle,
98
+ }
@@ -0,0 +1,142 @@
1
+ """Databricks governance review commands."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ from pathlib import Path
7
+ from typing import Any
8
+
9
+ from contractforge_core.cli_io import yaml_load
10
+ from contractforge_core.contracts import compose_contract_sections, semantic_contract_from_mapping
11
+ from contractforge_core.semantic import SemanticContract
12
+ from contractforge_databricks.annotations import render_annotations_sql
13
+ from contractforge_databricks.environment import DatabricksEnvironment
14
+ from contractforge_databricks.governance import render_access_sql, render_governance_sql
15
+ from contractforge_databricks.operations import render_operations_insert_sql, render_operations_json
16
+
17
+
18
+ def add_governance_parser(subparsers: Any) -> None:
19
+ preview = subparsers.add_parser("governance-preview", help="Render Databricks governance review artifacts")
20
+ preview.add_argument("paths", nargs="+", type=Path)
21
+ preview.add_argument("--output-dir", type=Path)
22
+ preview.add_argument("--indent", type=int, default=2)
23
+
24
+ apply_plan = subparsers.add_parser("governance-apply-plan", help="Render Databricks SQL that a runtime executor can apply")
25
+ apply_plan.add_argument("paths", nargs="+", type=Path)
26
+ apply_plan.add_argument("--output-dir", type=Path)
27
+ apply_plan.add_argument("--indent", type=int, default=2)
28
+
29
+
30
+ def governance_command(args: Any) -> int | None:
31
+ if args.command == "governance-preview":
32
+ return _governance_preview(args)
33
+ if args.command == "governance-apply-plan":
34
+ return _governance_apply_plan(args)
35
+ return None
36
+
37
+
38
+ def _governance_preview(args: Any) -> int:
39
+ artifacts = {}
40
+ for path in args.paths:
41
+ contract, environment = _load_contract(path)
42
+ prefix = _artifact_prefix(path)
43
+ env = DatabricksEnvironment.from_contract(environment)
44
+ artifacts[f"{prefix}.governance.sql"] = render_governance_sql(contract)
45
+ artifacts[f"{prefix}.annotations.sql"] = render_annotations_sql(contract)
46
+ artifacts[f"{prefix}.access.sql"] = render_access_sql(contract)
47
+ artifacts[f"{prefix}.operations.json"] = render_operations_json(contract)
48
+ artifacts[f"{prefix}.operations_evidence.sql"] = render_operations_insert_sql(
49
+ contract,
50
+ catalog=env.evidence_catalog,
51
+ schema=env.evidence_schema,
52
+ )
53
+ return _emit(artifacts, args.output_dir, args.indent)
54
+
55
+
56
+ def _governance_apply_plan(args: Any) -> int:
57
+ artifacts = {}
58
+ for path in args.paths:
59
+ contract, environment = _load_contract(path)
60
+ prefix = _artifact_prefix(path)
61
+ env = DatabricksEnvironment.from_contract(environment)
62
+ sections = [
63
+ "-- ContractForge Databricks governance apply plan",
64
+ "-- Execute with a Databricks SQL/Spark runner owned by the adapter runtime.",
65
+ "",
66
+ render_governance_sql(contract),
67
+ render_operations_insert_sql(contract, catalog=env.evidence_catalog, schema=env.evidence_schema) + ";",
68
+ ]
69
+ artifacts[f"{prefix}.governance_apply_plan.sql"] = "\n".join(sections)
70
+ return _emit(artifacts, args.output_dir, args.indent)
71
+
72
+
73
+ def _load_contract(path: Path) -> tuple[SemanticContract, dict[str, Any] | None]:
74
+ if _is_split_ingestion(path):
75
+ bundle = _load_bundle(path)
76
+ return bundle.semantic, bundle.environment
77
+ payload = _load_mapping(path)
78
+ environment = payload.get("environment") if isinstance(payload.get("environment"), dict) else None
79
+ return semantic_contract_from_mapping(payload), environment
80
+
81
+
82
+ def _load_bundle(path: Path):
83
+ base = _bundle_base(path)
84
+ paths = {
85
+ "ingestion": base.with_suffix(".ingestion.yaml"),
86
+ "annotations": base.with_suffix(".annotations.yaml"),
87
+ "operations": base.with_suffix(".operations.yaml"),
88
+ "access": base.with_suffix(".access.yaml"),
89
+ "environment": base.with_suffix(".environment.yaml"),
90
+ }
91
+ ingestion = _load_mapping(paths["ingestion"])
92
+ optional = {name: _load_mapping(item) for name, item in paths.items() if name != "ingestion" and item.exists()}
93
+ return compose_contract_sections(
94
+ ingestion=ingestion,
95
+ annotations=optional.get("annotations"),
96
+ operations=optional.get("operations"),
97
+ access=optional.get("access"),
98
+ environment=optional.get("environment"),
99
+ )
100
+
101
+
102
+ def _emit(artifacts: dict[str, str], output_dir: Path | None, indent: int) -> int:
103
+ if output_dir:
104
+ output_dir.mkdir(parents=True, exist_ok=True)
105
+ written = []
106
+ for name, body in artifacts.items():
107
+ path = output_dir / name
108
+ path.write_text(body, encoding="utf-8")
109
+ written.append(str(path))
110
+ return _print({"status": "SUCCESS", "written": written}, indent)
111
+ return _print(artifacts, indent)
112
+
113
+
114
+ def _load_mapping(path: Path) -> dict[str, Any]:
115
+ text = path.read_text(encoding="utf-8")
116
+ payload = json.loads(text) if path.suffix.lower() == ".json" else yaml_load(text)
117
+ if not isinstance(payload, dict):
118
+ raise ValueError(f"{path} must contain an object")
119
+ return payload
120
+
121
+
122
+ def _artifact_prefix(path: Path) -> str:
123
+ name = _bundle_base(path).name
124
+ return name.replace(".", "_")
125
+
126
+
127
+ def _bundle_base(path: Path) -> Path:
128
+ if _is_split_ingestion(path):
129
+ name = path.name
130
+ for suffix in (".ingestion.yaml", ".ingestion.yml", ".ingestion.json"):
131
+ if name.endswith(suffix):
132
+ return path.with_name(name[: -len(suffix)])
133
+ return path
134
+
135
+
136
+ def _is_split_ingestion(path: Path) -> bool:
137
+ return path.is_file() and ".ingestion." in path.name
138
+
139
+
140
+ def _print(payload: object, indent: int) -> int:
141
+ print(json.dumps(payload, indent=indent, sort_keys=True, default=str))
142
+ return 0
@@ -0,0 +1,91 @@
1
+ """Databricks CLI file IO helpers."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ from pathlib import Path
7
+ from typing import Any
8
+
9
+ from contractforge_core.contracts import load_contract_bundle
10
+
11
+
12
+ def load_mapping(path: Path | None) -> dict[str, Any]:
13
+ if path is None:
14
+ return {}
15
+ text = path.read_text(encoding="utf-8")
16
+ if path.suffix.lower() == ".json":
17
+ payload = json.loads(text)
18
+ else:
19
+ payload = yaml_load(text)
20
+ if not isinstance(payload, dict):
21
+ raise ValueError(f"{path} must contain an object")
22
+ return payload
23
+
24
+
25
+ def load_contract_input(path: Path) -> tuple[dict[str, Any], dict[str, Any] | None]:
26
+ if looks_like_split_contract(path):
27
+ bundle = load_contract_bundle(bundle_base(path))
28
+ environment = bundle.environment if isinstance(bundle.environment, dict) else None
29
+ return bundle.contract, environment
30
+ return load_mapping(path), None
31
+
32
+
33
+ def write_mapping(path: Path, payload: dict[str, Any], *, force: bool) -> None:
34
+ if path.exists() and not force:
35
+ raise FileExistsError(f"{path} already exists; use --force to overwrite it")
36
+ path.parent.mkdir(parents=True, exist_ok=True)
37
+ path.write_text(yaml_dump(payload), encoding="utf-8")
38
+
39
+
40
+ def write_artifacts(output_dir: Path, artifacts: dict[str, str]) -> list[str]:
41
+ output_dir.mkdir(parents=True, exist_ok=True)
42
+ written = []
43
+ for name, body in artifacts.items():
44
+ path = output_dir / name
45
+ path.write_text(body, encoding="utf-8")
46
+ written.append(str(path))
47
+ return written
48
+
49
+
50
+ def yaml_load(text: str) -> Any:
51
+ try:
52
+ import yaml # type: ignore
53
+ except Exception as exc: # pragma: no cover
54
+ raise RuntimeError("YAML support requires PyYAML; use JSON files or install PyYAML") from exc
55
+ return yaml.safe_load(text)
56
+
57
+
58
+ def yaml_dump(payload: dict[str, Any]) -> str:
59
+ try:
60
+ import yaml # type: ignore
61
+ except Exception: # pragma: no cover
62
+ return json.dumps(payload, indent=2, sort_keys=False) + "\n"
63
+ return yaml.safe_dump(payload, allow_unicode=True, sort_keys=False)
64
+
65
+
66
+ def looks_like_split_contract(path: Path) -> bool:
67
+ return any(marker in path.name for marker in (".ingestion.", ".annotations.", ".operations.", ".access.", ".environment."))
68
+
69
+
70
+ def bundle_base(path: Path) -> Path:
71
+ name = path.name
72
+ for suffix in (
73
+ ".ingestion.yaml",
74
+ ".ingestion.yml",
75
+ ".ingestion.json",
76
+ ".annotations.yaml",
77
+ ".annotations.yml",
78
+ ".annotations.json",
79
+ ".operations.yaml",
80
+ ".operations.yml",
81
+ ".operations.json",
82
+ ".access.yaml",
83
+ ".access.yml",
84
+ ".access.json",
85
+ ".environment.yaml",
86
+ ".environment.yml",
87
+ ".environment.json",
88
+ ):
89
+ if name.endswith(suffix):
90
+ return path.with_name(name[: -len(suffix)])
91
+ return path