contractforge-databricks 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- contractforge_databricks/__init__.py +172 -0
- contractforge_databricks/adapter.py +69 -0
- contractforge_databricks/annotations/__init__.py +10 -0
- contractforge_databricks/annotations/application.py +52 -0
- contractforge_databricks/annotations/audit.py +49 -0
- contractforge_databricks/annotations/sql.py +142 -0
- contractforge_databricks/api.py +65 -0
- contractforge_databricks/bundles/__init__.py +9 -0
- contractforge_databricks/bundles/assets.py +47 -0
- contractforge_databricks/bundles/project.py +213 -0
- contractforge_databricks/bundles/project_config.py +133 -0
- contractforge_databricks/capabilities/__init__.py +17 -0
- contractforge_databricks/capabilities/builders.py +43 -0
- contractforge_databricks/capabilities/evaluate.py +162 -0
- contractforge_databricks/capabilities/mapping.py +36 -0
- contractforge_databricks/capabilities/models.py +44 -0
- contractforge_databricks/capabilities/runtime.py +111 -0
- contractforge_databricks/capabilities/uc.py +47 -0
- contractforge_databricks/cli.py +196 -0
- contractforge_databricks/cli_deploy.py +98 -0
- contractforge_databricks/cli_governance.py +142 -0
- contractforge_databricks/cli_io.py +91 -0
- contractforge_databricks/cli_maintenance.py +69 -0
- contractforge_databricks/coercion.py +31 -0
- contractforge_databricks/contract_extensions.py +70 -0
- contractforge_databricks/cost/__init__.py +11 -0
- contractforge_databricks/cost/model.py +22 -0
- contractforge_databricks/cost/report.py +65 -0
- contractforge_databricks/cost/sql.py +136 -0
- contractforge_databricks/dashboards/__init__.py +15 -0
- contractforge_databricks/dashboards/control_tables.py +150 -0
- contractforge_databricks/diagnostics/__init__.py +7 -0
- contractforge_databricks/diagnostics/explain.py +40 -0
- contractforge_databricks/environment.py +53 -0
- contractforge_databricks/evidence/__init__.py +98 -0
- contractforge_databricks/evidence/ddl.py +35 -0
- contractforge_databricks/evidence/governance_log.py +175 -0
- contractforge_databricks/evidence/helpers.py +29 -0
- contractforge_databricks/evidence/ops_log.py +210 -0
- contractforge_databricks/evidence/records.py +27 -0
- contractforge_databricks/evidence/run_log.py +74 -0
- contractforge_databricks/evidence/schemas.py +7 -0
- contractforge_databricks/evidence/sql.py +144 -0
- contractforge_databricks/evidence/tables.py +20 -0
- contractforge_databricks/evidence/writer.py +118 -0
- contractforge_databricks/execution/__init__.py +70 -0
- contractforge_databricks/execution/delta_basic.py +57 -0
- contractforge_databricks/execution/hash_diff.py +126 -0
- contractforge_databricks/execution/hash_diff_latest.py +142 -0
- contractforge_databricks/execution/replace_partitions.py +40 -0
- contractforge_databricks/execution/results.py +5 -0
- contractforge_databricks/execution/retry.py +36 -0
- contractforge_databricks/execution/scd2.py +213 -0
- contractforge_databricks/execution/scd2_deletes.py +65 -0
- contractforge_databricks/execution/scd2_late.py +30 -0
- contractforge_databricks/execution/snapshot.py +77 -0
- contractforge_databricks/execution/sql_merge.py +85 -0
- contractforge_databricks/execution/tables.py +98 -0
- contractforge_databricks/execution/windows.py +58 -0
- contractforge_databricks/governance/__init__.py +30 -0
- contractforge_databricks/governance/access.py +185 -0
- contractforge_databricks/governance/application.py +93 -0
- contractforge_databricks/governance/drift.py +49 -0
- contractforge_databricks/governance/runtime.py +60 -0
- contractforge_databricks/governance/sql.py +31 -0
- contractforge_databricks/governance/validation.py +135 -0
- contractforge_databricks/lakeflow/__init__.py +21 -0
- contractforge_databricks/lakeflow/compatibility.py +194 -0
- contractforge_databricks/lakeflow/rendering.py +175 -0
- contractforge_databricks/lineage/__init__.py +7 -0
- contractforge_databricks/lineage/openlineage.py +182 -0
- contractforge_databricks/maintenance/__init__.py +27 -0
- contractforge_databricks/maintenance/retention.py +90 -0
- contractforge_databricks/maintenance/sql.py +68 -0
- contractforge_databricks/metrics/__init__.py +19 -0
- contractforge_databricks/metrics/history.py +21 -0
- contractforge_databricks/metrics/write.py +63 -0
- contractforge_databricks/operations/__init__.py +4 -0
- contractforge_databricks/operations/application.py +38 -0
- contractforge_databricks/operations/sql.py +95 -0
- contractforge_databricks/parity/__init__.py +18 -0
- contractforge_databricks/parity/catalog.py +59 -0
- contractforge_databricks/parity/models.py +7 -0
- contractforge_databricks/parity/scenarios.py +111 -0
- contractforge_databricks/partitioning/__init__.py +3 -0
- contractforge_databricks/partitioning/predicates.py +28 -0
- contractforge_databricks/preparation/__init__.py +47 -0
- contractforge_databricks/preparation/deduplicate.py +87 -0
- contractforge_databricks/preparation/encoding.py +37 -0
- contractforge_databricks/preparation/hashing.py +18 -0
- contractforge_databricks/preparation/pyspark.py +178 -0
- contractforge_databricks/preparation/pyspark_staging.py +70 -0
- contractforge_databricks/preparation/shape.py +209 -0
- contractforge_databricks/preparation/shape_validation.py +94 -0
- contractforge_databricks/preparation/staging.py +17 -0
- contractforge_databricks/preparation/zip_arrays.py +51 -0
- contractforge_databricks/presets/__init__.py +3 -0
- contractforge_databricks/presets/base.py +24 -0
- contractforge_databricks/presets/bronze.py +57 -0
- contractforge_databricks/presets/catalog.py +22 -0
- contractforge_databricks/presets/core.py +134 -0
- contractforge_databricks/presets/gold.py +62 -0
- contractforge_databricks/presets/modifiers.py +51 -0
- contractforge_databricks/presets/runtime.py +22 -0
- contractforge_databricks/presets/silver.py +101 -0
- contractforge_databricks/presets/write_engine.py +57 -0
- contractforge_databricks/quality/__init__.py +41 -0
- contractforge_databricks/quality/evaluation.py +178 -0
- contractforge_databricks/quality/persistence.py +81 -0
- contractforge_databricks/quality/registry.py +134 -0
- contractforge_databricks/quality/results.py +17 -0
- contractforge_databricks/quality/sql.py +113 -0
- contractforge_databricks/rendering/__init__.py +11 -0
- contractforge_databricks/rendering/bundle.py +93 -0
- contractforge_databricks/rendering/markdown.py +50 -0
- contractforge_databricks/rendering/names.py +56 -0
- contractforge_databricks/results.py +15 -0
- contractforge_databricks/runtime/__init__.py +101 -0
- contractforge_databricks/runtime/available_now.py +147 -0
- contractforge_databricks/runtime/bundles.py +211 -0
- contractforge_databricks/runtime/cache.py +20 -0
- contractforge_databricks/runtime/control_tables.py +19 -0
- contractforge_databricks/runtime/deploy.py +197 -0
- contractforge_databricks/runtime/detection.py +114 -0
- contractforge_databricks/runtime/dry_run.py +46 -0
- contractforge_databricks/runtime/errors.py +54 -0
- contractforge_databricks/runtime/file_selection.py +109 -0
- contractforge_databricks/runtime/finalization.py +168 -0
- contractforge_databricks/runtime/governance.py +37 -0
- contractforge_databricks/runtime/hooks.py +45 -0
- contractforge_databricks/runtime/http_file.py +37 -0
- contractforge_databricks/runtime/http_retry.py +15 -0
- contractforge_databricks/runtime/http_safety.py +9 -0
- contractforge_databricks/runtime/json_materialization.py +97 -0
- contractforge_databricks/runtime/lineage.py +164 -0
- contractforge_databricks/runtime/maintenance.py +43 -0
- contractforge_databricks/runtime/merge_validation.py +98 -0
- contractforge_databricks/runtime/metadata.py +21 -0
- contractforge_databricks/runtime/metrics.py +34 -0
- contractforge_databricks/runtime/models.py +32 -0
- contractforge_databricks/runtime/options.py +33 -0
- contractforge_databricks/runtime/orchestration_context.py +185 -0
- contractforge_databricks/runtime/orchestrator.py +147 -0
- contractforge_databricks/runtime/partitioning.py +93 -0
- contractforge_databricks/runtime/quality_quarantine.py +92 -0
- contractforge_databricks/runtime/rest_api.py +46 -0
- contractforge_databricks/runtime/rest_auth.py +21 -0
- contractforge_databricks/runtime/rest_pagination.py +21 -0
- contractforge_databricks/runtime/run_payload.py +177 -0
- contractforge_databricks/runtime/schema.py +106 -0
- contractforge_databricks/runtime/source_metadata.py +30 -0
- contractforge_databricks/runtime/source_registry.py +43 -0
- contractforge_databricks/runtime/source_schema.py +24 -0
- contractforge_databricks/runtime/sources.py +208 -0
- contractforge_databricks/runtime/spark.py +183 -0
- contractforge_databricks/runtime/spark_defaults.py +35 -0
- contractforge_databricks/runtime/storage_auth.py +132 -0
- contractforge_databricks/runtime/streaming.py +131 -0
- contractforge_databricks/runtime/success.py +104 -0
- contractforge_databricks/runtime/utils.py +52 -0
- contractforge_databricks/runtime/watermark.py +71 -0
- contractforge_databricks/runtime/windows.py +184 -0
- contractforge_databricks/runtime/write.py +66 -0
- contractforge_databricks/runtime/write_flow.py +146 -0
- contractforge_databricks/runtime/write_strategy.py +40 -0
- contractforge_databricks/schema/__init__.py +21 -0
- contractforge_databricks/schema/diff.py +11 -0
- contractforge_databricks/schema/policy.py +33 -0
- contractforge_databricks/schema/sync.py +23 -0
- contractforge_databricks/security/__init__.py +21 -0
- contractforge_databricks/security/errors.py +5 -0
- contractforge_databricks/security/redaction.py +5 -0
- contractforge_databricks/security/secrets.py +114 -0
- contractforge_databricks/security/source_policy.py +17 -0
- contractforge_databricks/shapes/__init__.py +3 -0
- contractforge_databricks/shapes/sql.py +123 -0
- contractforge_databricks/sources/__init__.py +67 -0
- contractforge_databricks/sources/artifacts.py +100 -0
- contractforge_databricks/sources/autoloader.py +48 -0
- contractforge_databricks/sources/bounded_streams.py +44 -0
- contractforge_databricks/sources/classification.py +115 -0
- contractforge_databricks/sources/delta_share.py +21 -0
- contractforge_databricks/sources/files.py +48 -0
- contractforge_databricks/sources/http_file.py +46 -0
- contractforge_databricks/sources/interpret.py +76 -0
- contractforge_databricks/sources/jdbc.py +32 -0
- contractforge_databricks/sources/metadata.py +18 -0
- contractforge_databricks/sources/native_passthrough.py +33 -0
- contractforge_databricks/sources/rds_iam.py +15 -0
- contractforge_databricks/sources/rds_iam_runtime.py +191 -0
- contractforge_databricks/sources/rest_api.py +33 -0
- contractforge_databricks/sources/support.py +50 -0
- contractforge_databricks/sources/table_refs.py +65 -0
- contractforge_databricks/sql/__init__.py +4 -0
- contractforge_databricks/sql/identifiers.py +17 -0
- contractforge_databricks/sql/literals.py +36 -0
- contractforge_databricks/state/__init__.py +39 -0
- contractforge_databricks/state/ddl.py +24 -0
- contractforge_databricks/state/migrations.py +146 -0
- contractforge_databricks/state/queries.py +149 -0
- contractforge_databricks/state/sql.py +116 -0
- contractforge_databricks/state/tables.py +9 -0
- contractforge_databricks/state/writer.py +83 -0
- contractforge_databricks/templates/__init__.py +15 -0
- contractforge_databricks/templates/catalog.py +205 -0
- contractforge_databricks/templates/catalog_parity.py +85 -0
- contractforge_databricks/templates/core.py +83 -0
- contractforge_databricks/templates/enrichment.py +175 -0
- contractforge_databricks/transforms/__init__.py +3 -0
- contractforge_databricks/transforms/sql.py +118 -0
- contractforge_databricks/watermark/__init__.py +6 -0
- contractforge_databricks/watermark/sql.py +91 -0
- contractforge_databricks/write_modes/__init__.py +20 -0
- contractforge_databricks/write_modes/registry.py +44 -0
- contractforge_databricks/write_modes/sql.py +33 -0
- contractforge_databricks/write_modes/strategy.py +192 -0
- contractforge_databricks-0.1.0.dist-info/METADATA +34 -0
- contractforge_databricks-0.1.0.dist-info/RECORD +220 -0
- contractforge_databricks-0.1.0.dist-info/WHEEL +4 -0
- contractforge_databricks-0.1.0.dist-info/entry_points.txt +2 -0
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
"""Databricks SQL MERGE execution helpers."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any, Protocol
|
|
6
|
+
|
|
7
|
+
from contractforge_core.semantic import SemanticContract
|
|
8
|
+
from contractforge_core.execution import ExecutionOutcome
|
|
9
|
+
from contractforge_databricks.rendering.names import target_full_name
|
|
10
|
+
from contractforge_databricks.sql import quote_identifier, quote_table_name
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class SqlRunner(Protocol):
|
|
14
|
+
def sql(self, statement: str) -> Any:
|
|
15
|
+
...
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def render_scd1_merge_sql(
|
|
19
|
+
*,
|
|
20
|
+
target_table: str,
|
|
21
|
+
source_view: str,
|
|
22
|
+
merge_keys: tuple[str, ...],
|
|
23
|
+
source_columns: tuple[str, ...],
|
|
24
|
+
target_partition_predicate: str | None = None,
|
|
25
|
+
) -> str:
|
|
26
|
+
"""Render deterministic SCD1 current-state MERGE SQL."""
|
|
27
|
+
if not merge_keys:
|
|
28
|
+
raise ValueError("scd1_upsert requires merge_keys")
|
|
29
|
+
if not source_columns:
|
|
30
|
+
raise ValueError("scd1_upsert requires source_columns")
|
|
31
|
+
|
|
32
|
+
missing_keys = [key for key in merge_keys if key not in source_columns]
|
|
33
|
+
if missing_keys:
|
|
34
|
+
raise ValueError(f"merge_keys missing from source_columns: {missing_keys}")
|
|
35
|
+
|
|
36
|
+
update_columns = tuple(column for column in source_columns if column not in merge_keys)
|
|
37
|
+
key_condition = " AND ".join(
|
|
38
|
+
f"t.{quote_identifier(key)} <=> s.{quote_identifier(key)}" for key in merge_keys
|
|
39
|
+
)
|
|
40
|
+
if target_partition_predicate:
|
|
41
|
+
key_condition = f"{key_condition} AND {target_partition_predicate}"
|
|
42
|
+
update_set = ", ".join(
|
|
43
|
+
f"t.{quote_identifier(column)} = s.{quote_identifier(column)}" for column in update_columns
|
|
44
|
+
)
|
|
45
|
+
insert_columns = ", ".join(quote_identifier(column) for column in source_columns)
|
|
46
|
+
insert_values = ", ".join(f"s.{quote_identifier(column)}" for column in source_columns)
|
|
47
|
+
|
|
48
|
+
lines = [
|
|
49
|
+
f"MERGE INTO {quote_table_name(target_table)} t",
|
|
50
|
+
f"USING {quote_table_name(source_view)} s",
|
|
51
|
+
f"ON {key_condition}",
|
|
52
|
+
]
|
|
53
|
+
if update_set:
|
|
54
|
+
lines.append(f"WHEN MATCHED THEN UPDATE SET {update_set}")
|
|
55
|
+
lines.append(f"WHEN NOT MATCHED THEN INSERT ({insert_columns}) VALUES ({insert_values})")
|
|
56
|
+
return "\n".join(lines)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def execute_scd1_merge(
|
|
60
|
+
*,
|
|
61
|
+
runner: SqlRunner,
|
|
62
|
+
contract: SemanticContract,
|
|
63
|
+
source_view: str,
|
|
64
|
+
source_columns: tuple[str, ...],
|
|
65
|
+
target_partition_predicate: str | None = None,
|
|
66
|
+
) -> ExecutionOutcome:
|
|
67
|
+
if contract.write.mode != "scd1_upsert":
|
|
68
|
+
raise ValueError(f"execute_scd1_merge only supports scd1_upsert, got {contract.write.mode}")
|
|
69
|
+
|
|
70
|
+
target = target_full_name(contract)
|
|
71
|
+
statement = render_scd1_merge_sql(
|
|
72
|
+
target_table=target,
|
|
73
|
+
source_view=source_view,
|
|
74
|
+
merge_keys=contract.write.merge_keys,
|
|
75
|
+
source_columns=source_columns,
|
|
76
|
+
target_partition_predicate=target_partition_predicate,
|
|
77
|
+
)
|
|
78
|
+
runner.sql(statement)
|
|
79
|
+
return ExecutionOutcome(
|
|
80
|
+
status="SUCCESS",
|
|
81
|
+
operation="scd1_sql_merge",
|
|
82
|
+
target=target,
|
|
83
|
+
metrics={"source_columns": len(source_columns), "merge_keys": len(contract.write.merge_keys)},
|
|
84
|
+
sql=statement,
|
|
85
|
+
)
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
"""Databricks Delta table setup SQL helpers."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from contractforge_core.semantic import SemanticContract
|
|
8
|
+
from contractforge_databricks.contract_extensions import databricks_extensions
|
|
9
|
+
from contractforge_databricks.execution.sql_merge import SqlRunner
|
|
10
|
+
from contractforge_databricks.rendering.names import target_full_name
|
|
11
|
+
from contractforge_databricks.sql import quote_identifier, quote_table_name, sql_literal
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def render_create_delta_table_sql(
|
|
15
|
+
*,
|
|
16
|
+
target_table: str,
|
|
17
|
+
columns: dict[str, str],
|
|
18
|
+
partition_column: str | None = None,
|
|
19
|
+
partition_columns: tuple[str, ...] = (),
|
|
20
|
+
) -> str:
|
|
21
|
+
if not columns:
|
|
22
|
+
raise ValueError("Delta table creation requires at least one column")
|
|
23
|
+
partitions = tuple(partition_columns or ((partition_column,) if partition_column else ()))
|
|
24
|
+
missing_partition_columns = [column for column in partitions if column not in columns]
|
|
25
|
+
if missing_partition_columns:
|
|
26
|
+
raise ValueError(f"partition_columns missing from source columns: {missing_partition_columns}")
|
|
27
|
+
cols_sql = ", ".join(f"{quote_identifier(name)} {data_type}" for name, data_type in columns.items())
|
|
28
|
+
partition_sql = f" PARTITIONED BY ({', '.join(quote_identifier(column) for column in partitions)})" if partitions else ""
|
|
29
|
+
return f"CREATE TABLE IF NOT EXISTS {quote_table_name(target_table)} ({cols_sql}) USING DELTA{partition_sql}"
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def render_create_schema_sql(*, namespace: str | None) -> str | None:
|
|
33
|
+
if not namespace:
|
|
34
|
+
return None
|
|
35
|
+
return f"CREATE SCHEMA IF NOT EXISTS {quote_table_name(namespace)}"
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def render_cluster_by_sql(*, target_table: str, cluster_columns: tuple[str, ...]) -> str | None:
|
|
39
|
+
if not cluster_columns:
|
|
40
|
+
return None
|
|
41
|
+
columns_sql = ", ".join(quote_identifier(column) for column in cluster_columns)
|
|
42
|
+
return f"ALTER TABLE {quote_table_name(target_table)} CLUSTER BY ({columns_sql})"
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def render_delta_properties_sql(*, target_table: str, properties: dict[str, Any] | None) -> str | None:
|
|
46
|
+
if not properties:
|
|
47
|
+
return None
|
|
48
|
+
props_sql = ", ".join(f"{sql_literal(key)} = {sql_literal(value)}" for key, value in sorted(properties.items()))
|
|
49
|
+
return f"ALTER TABLE {quote_table_name(target_table)} SET TBLPROPERTIES ({props_sql})"
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def render_table_setup_sql(
|
|
53
|
+
contract: SemanticContract,
|
|
54
|
+
*,
|
|
55
|
+
columns: dict[str, str],
|
|
56
|
+
) -> tuple[str, ...]:
|
|
57
|
+
extensions = databricks_extensions(contract)
|
|
58
|
+
target = target_full_name(contract)
|
|
59
|
+
cluster_columns = tuple(str(column) for column in extensions.get("cluster_columns") or ())
|
|
60
|
+
partition_columns = _partition_columns(extensions)
|
|
61
|
+
missing_cluster_columns = [column for column in cluster_columns if column not in columns]
|
|
62
|
+
if missing_cluster_columns:
|
|
63
|
+
raise ValueError(f"cluster_columns missing from source columns: {missing_cluster_columns}")
|
|
64
|
+
if cluster_columns and partition_columns:
|
|
65
|
+
partition_columns = ()
|
|
66
|
+
statements = [
|
|
67
|
+
render_create_schema_sql(namespace=contract.target.namespace),
|
|
68
|
+
render_create_delta_table_sql(
|
|
69
|
+
target_table=target,
|
|
70
|
+
columns=columns,
|
|
71
|
+
partition_columns=partition_columns,
|
|
72
|
+
)
|
|
73
|
+
]
|
|
74
|
+
cluster_sql = render_cluster_by_sql(target_table=target, cluster_columns=cluster_columns)
|
|
75
|
+
properties_sql = render_delta_properties_sql(target_table=target, properties=extensions.get("delta_properties"))
|
|
76
|
+
return tuple(statement for statement in (*statements, cluster_sql, properties_sql) if statement)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def execute_table_setup(
|
|
80
|
+
*,
|
|
81
|
+
runner: SqlRunner,
|
|
82
|
+
contract: SemanticContract,
|
|
83
|
+
columns: dict[str, str],
|
|
84
|
+
) -> tuple[str, ...]:
|
|
85
|
+
statements = render_table_setup_sql(contract, columns=columns)
|
|
86
|
+
for statement in statements:
|
|
87
|
+
runner.sql(statement)
|
|
88
|
+
return statements
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _partition_columns(extensions: dict[str, Any]) -> tuple[str, ...]:
|
|
92
|
+
value = extensions.get("partition_columns")
|
|
93
|
+
if value:
|
|
94
|
+
if isinstance(value, str):
|
|
95
|
+
return (value,)
|
|
96
|
+
return tuple(str(column) for column in value)
|
|
97
|
+
single = str(extensions.get("partition_column") or "")
|
|
98
|
+
return (single,) if single else ()
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
"""Databricks execution window SQL helpers."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
|
|
7
|
+
from contractforge_core.execution import (
|
|
8
|
+
ChildWindowPlan,
|
|
9
|
+
ExecutionWindow,
|
|
10
|
+
build_time_windows,
|
|
11
|
+
combine_filter,
|
|
12
|
+
summarize_window_results,
|
|
13
|
+
)
|
|
14
|
+
from contractforge_core.execution import build_child_window_plan as build_core_child_window_plan
|
|
15
|
+
from contractforge_databricks.sql import quote_identifier, sql_string
|
|
16
|
+
|
|
17
|
+
_SIMPLE_COLUMN_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def render_window_filter_sql(column: str, window: ExecutionWindow) -> str:
|
|
21
|
+
if not _SIMPLE_COLUMN_RE.match(column):
|
|
22
|
+
raise ValueError("execution window column must be a simple column name")
|
|
23
|
+
quoted = quote_identifier(column)
|
|
24
|
+
return (
|
|
25
|
+
f"(CAST({quoted} AS TIMESTAMP) >= CAST({sql_string(window.start)} AS TIMESTAMP) "
|
|
26
|
+
f"AND CAST({quoted} AS TIMESTAMP) < CAST({sql_string(window.end)} AS TIMESTAMP))"
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def build_child_window_plan(
|
|
31
|
+
*,
|
|
32
|
+
parent_run_id: str,
|
|
33
|
+
column: str,
|
|
34
|
+
window: ExecutionWindow,
|
|
35
|
+
index: int,
|
|
36
|
+
existing_filter: str | None = None,
|
|
37
|
+
base_idempotency_key: str | None = None,
|
|
38
|
+
) -> ChildWindowPlan:
|
|
39
|
+
return build_core_child_window_plan(
|
|
40
|
+
parent_run_id=parent_run_id,
|
|
41
|
+
column=column,
|
|
42
|
+
window=window,
|
|
43
|
+
index=index,
|
|
44
|
+
window_filter=render_window_filter_sql(column, window),
|
|
45
|
+
existing_filter=existing_filter,
|
|
46
|
+
base_idempotency_key=base_idempotency_key,
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
__all__ = [
|
|
51
|
+
"ChildWindowPlan",
|
|
52
|
+
"ExecutionWindow",
|
|
53
|
+
"build_child_window_plan",
|
|
54
|
+
"build_time_windows",
|
|
55
|
+
"combine_filter",
|
|
56
|
+
"render_window_filter_sql",
|
|
57
|
+
"summarize_window_results",
|
|
58
|
+
]
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
from contractforge_core.results import GovernanceApplyResult
|
|
2
|
+
from contractforge_databricks.governance.access import (
|
|
3
|
+
access_steps,
|
|
4
|
+
render_access_audit_insert_sql,
|
|
5
|
+
render_access_sql,
|
|
6
|
+
revoke_grant_steps,
|
|
7
|
+
)
|
|
8
|
+
from contractforge_databricks.governance.application import apply_access_contract
|
|
9
|
+
from contractforge_databricks.governance.runtime import apply_governance_contract, check_governance_contract
|
|
10
|
+
from contractforge_databricks.governance.sql import render_governance_sql
|
|
11
|
+
from contractforge_databricks.governance.validation import (
|
|
12
|
+
access_drift_report,
|
|
13
|
+
governance_referenced_columns,
|
|
14
|
+
validate_governance_contract,
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
__all__ = [
|
|
18
|
+
"GovernanceApplyResult",
|
|
19
|
+
"access_steps",
|
|
20
|
+
"apply_access_contract",
|
|
21
|
+
"apply_governance_contract",
|
|
22
|
+
"check_governance_contract",
|
|
23
|
+
"access_drift_report",
|
|
24
|
+
"governance_referenced_columns",
|
|
25
|
+
"render_access_audit_insert_sql",
|
|
26
|
+
"render_access_sql",
|
|
27
|
+
"revoke_grant_steps",
|
|
28
|
+
"render_governance_sql",
|
|
29
|
+
"validate_governance_contract",
|
|
30
|
+
]
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
"""Render Unity Catalog access steps and audit SQL."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from datetime import datetime
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
from contractforge_core.semantic import SemanticContract
|
|
10
|
+
from contractforge_databricks.coercion import string_list
|
|
11
|
+
from contractforge_databricks.evidence.tables import evidence_table_names
|
|
12
|
+
from contractforge_databricks.rendering.names import target_full_name
|
|
13
|
+
from contractforge_databricks.sql import quote_identifier, quote_table_name, sql_string
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def access_steps(contract: SemanticContract) -> list[dict[str, Any]]:
|
|
17
|
+
governance = contract.governance
|
|
18
|
+
access = governance.access if governance else None
|
|
19
|
+
if not isinstance(access, dict):
|
|
20
|
+
return []
|
|
21
|
+
target = target_full_name(contract)
|
|
22
|
+
quoted_target = quote_table_name(target)
|
|
23
|
+
steps: list[dict[str, Any]] = []
|
|
24
|
+
for grant in access.get("grants", []):
|
|
25
|
+
principal = str(grant["principal"])
|
|
26
|
+
for privilege in string_list(grant.get("privileges")):
|
|
27
|
+
steps.append(_grant_step(quoted_target, target, principal, privilege, access))
|
|
28
|
+
for row_filter in access.get("row_filters", []):
|
|
29
|
+
steps.append(_row_filter_step(quoted_target, row_filter, access))
|
|
30
|
+
for column_mask in access.get("column_masks", []):
|
|
31
|
+
steps.append(_column_mask_step(quoted_target, column_mask, access))
|
|
32
|
+
return steps
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def revoke_grant_steps(contract: SemanticContract, unmanaged_grants: list[tuple[str, str]]) -> list[dict[str, Any]]:
|
|
36
|
+
access = contract.governance.access if contract.governance else None
|
|
37
|
+
if not isinstance(access, dict):
|
|
38
|
+
return []
|
|
39
|
+
target = target_full_name(contract)
|
|
40
|
+
quoted_target = quote_table_name(target)
|
|
41
|
+
return [_revoke_step(quoted_target, target, principal, privilege, access) for principal, privilege in unmanaged_grants]
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def render_access_sql(contract: SemanticContract) -> str:
|
|
45
|
+
return "\n".join(f"{step['sql']};" for step in access_steps(contract))
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def render_access_audit_insert_sql(
|
|
49
|
+
contract: SemanticContract,
|
|
50
|
+
*,
|
|
51
|
+
run_id: str = "${run_id}",
|
|
52
|
+
status: str = "PLANNED",
|
|
53
|
+
captured_at_utc: datetime | None = None,
|
|
54
|
+
catalog: str = "main",
|
|
55
|
+
schema: str = "ops",
|
|
56
|
+
) -> str:
|
|
57
|
+
steps = access_steps(contract)
|
|
58
|
+
if not steps:
|
|
59
|
+
return "-- No access intent declared.\n"
|
|
60
|
+
table = evidence_table_names(catalog, schema)["access"]
|
|
61
|
+
captured_at_utc = captured_at_utc or datetime(1970, 1, 1, 0, 0, 0)
|
|
62
|
+
statements = [
|
|
63
|
+
_audit_insert(table, run_id, target_full_name(contract), step, status, captured_at_utc)
|
|
64
|
+
for step in steps
|
|
65
|
+
]
|
|
66
|
+
return ";\n".join(statements) + ";\n"
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _grant_step(
|
|
70
|
+
quoted_target: str,
|
|
71
|
+
target: str,
|
|
72
|
+
principal: str,
|
|
73
|
+
privilege: str,
|
|
74
|
+
access: dict[str, Any],
|
|
75
|
+
) -> dict[str, Any]:
|
|
76
|
+
return {
|
|
77
|
+
"access_type": "grant",
|
|
78
|
+
"principal": principal,
|
|
79
|
+
"privilege": privilege,
|
|
80
|
+
"column_name": None,
|
|
81
|
+
"function_name": None,
|
|
82
|
+
"object_name": target,
|
|
83
|
+
"new_value": "GRANTED",
|
|
84
|
+
"mode": _access_policy(access, "mode", "apply"),
|
|
85
|
+
"drift_policy": _access_policy(access, "on_drift", "warn"),
|
|
86
|
+
"revoke_unmanaged": _access_policy(access, "revoke_unmanaged", False),
|
|
87
|
+
"sql": f"GRANT {privilege} ON TABLE {quoted_target} TO {_principal(principal)}",
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _row_filter_step(quoted_target: str, row_filter: dict[str, Any], access: dict[str, Any]) -> dict[str, Any]:
|
|
92
|
+
columns = ", ".join(quote_identifier(column) for column in string_list(row_filter.get("columns")))
|
|
93
|
+
function = str(row_filter["function"])
|
|
94
|
+
return {
|
|
95
|
+
"access_type": "row_filter",
|
|
96
|
+
"principal": "|".join(string_list(row_filter.get("applies_to", {}).get("principals"))),
|
|
97
|
+
"privilege": "ROW_FILTER",
|
|
98
|
+
"column_name": "|".join(string_list(row_filter.get("columns"))),
|
|
99
|
+
"function_name": function,
|
|
100
|
+
"object_name": row_filter.get("name"),
|
|
101
|
+
"new_value": function,
|
|
102
|
+
"mode": _access_policy(access, "mode", "apply"),
|
|
103
|
+
"drift_policy": _access_policy(access, "on_drift", "warn"),
|
|
104
|
+
"revoke_unmanaged": _access_policy(access, "revoke_unmanaged", False),
|
|
105
|
+
"sql": f"ALTER TABLE {quoted_target} SET ROW FILTER {function} ON ({columns})",
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _column_mask_step(quoted_target: str, column_mask: dict[str, Any], access: dict[str, Any]) -> dict[str, Any]:
|
|
110
|
+
column = str(column_mask["column"])
|
|
111
|
+
using_columns = string_list(column_mask.get("using_columns"))
|
|
112
|
+
using_sql = ""
|
|
113
|
+
if using_columns:
|
|
114
|
+
using_sql = " USING COLUMNS (" + ", ".join(quote_identifier(item) for item in using_columns) + ")"
|
|
115
|
+
function = str(column_mask["function"])
|
|
116
|
+
return {
|
|
117
|
+
"access_type": "column_mask",
|
|
118
|
+
"principal": "|".join(string_list(column_mask.get("applies_to", {}).get("principals"))),
|
|
119
|
+
"privilege": "COLUMN_MASK",
|
|
120
|
+
"column_name": column,
|
|
121
|
+
"function_name": function,
|
|
122
|
+
"object_name": column,
|
|
123
|
+
"new_value": function,
|
|
124
|
+
"mode": _access_policy(access, "mode", "apply"),
|
|
125
|
+
"drift_policy": _access_policy(access, "on_drift", "warn"),
|
|
126
|
+
"revoke_unmanaged": _access_policy(access, "revoke_unmanaged", False),
|
|
127
|
+
"sql": f"ALTER TABLE {quoted_target} ALTER COLUMN {quote_identifier(column)} SET MASK {function}{using_sql}",
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _revoke_step(
|
|
132
|
+
quoted_target: str,
|
|
133
|
+
target: str,
|
|
134
|
+
principal: str,
|
|
135
|
+
privilege: str,
|
|
136
|
+
access: dict[str, Any],
|
|
137
|
+
) -> dict[str, Any]:
|
|
138
|
+
return {
|
|
139
|
+
"access_type": "revoke",
|
|
140
|
+
"principal": principal,
|
|
141
|
+
"privilege": privilege,
|
|
142
|
+
"column_name": None,
|
|
143
|
+
"function_name": None,
|
|
144
|
+
"object_name": target,
|
|
145
|
+
"previous_value": "GRANTED",
|
|
146
|
+
"new_value": "REVOKED",
|
|
147
|
+
"mode": _access_policy(access, "mode", "apply"),
|
|
148
|
+
"drift_policy": _access_policy(access, "on_drift", "warn"),
|
|
149
|
+
"revoke_unmanaged": True,
|
|
150
|
+
"sql": f"REVOKE {privilege} ON TABLE {quoted_target} FROM {_principal(principal)}",
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def _audit_insert(table: str, run_id: str, target: str, step: dict[str, Any], status: str, captured_at_utc: datetime) -> str:
|
|
155
|
+
payload = json.dumps(step, sort_keys=True, separators=(",", ":"))
|
|
156
|
+
columns = "run_id, target_table, action, status, payload_json, applied_at_utc, access_type, principal, privilege, column_name, function_name, object_name, applied_sql, new_value, mode, drift_policy, revoke_unmanaged"
|
|
157
|
+
values = [
|
|
158
|
+
sql_string(run_id),
|
|
159
|
+
sql_string(target),
|
|
160
|
+
sql_string(step["access_type"]),
|
|
161
|
+
sql_string(status),
|
|
162
|
+
sql_string(payload),
|
|
163
|
+
f"TIMESTAMP {sql_string(captured_at_utc.strftime('%Y-%m-%d %H:%M:%S'))}",
|
|
164
|
+
sql_string(step["access_type"]),
|
|
165
|
+
sql_string(step.get("principal")),
|
|
166
|
+
sql_string(step.get("privilege")),
|
|
167
|
+
sql_string(step.get("column_name")),
|
|
168
|
+
sql_string(step.get("function_name")),
|
|
169
|
+
sql_string(step.get("object_name")),
|
|
170
|
+
sql_string(step.get("sql")),
|
|
171
|
+
sql_string(step.get("new_value")),
|
|
172
|
+
sql_string(step.get("mode")),
|
|
173
|
+
sql_string(step.get("drift_policy")),
|
|
174
|
+
"true" if bool(step.get("revoke_unmanaged")) else "false",
|
|
175
|
+
]
|
|
176
|
+
return f"INSERT INTO {quote_table_name(table)} ({columns}) VALUES ({', '.join(values)})"
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def _access_policy(access: dict[str, Any], key: str, default: object) -> object:
|
|
180
|
+
policy = access.get("access_policy", {})
|
|
181
|
+
return policy.get(key, access.get(key, default)) if isinstance(policy, dict) else access.get(key, default)
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def _principal(value: str) -> str:
|
|
185
|
+
return "`" + value.replace("`", "``") + "`"
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
"""Apply Databricks access governance with an injected SQL runner."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from contractforge_core.semantic import SemanticContract
|
|
6
|
+
from contractforge_core.results import GovernanceApplyResult
|
|
7
|
+
from contractforge_databricks.execution.sql_merge import SqlRunner
|
|
8
|
+
from contractforge_databricks.governance.access import access_steps, revoke_grant_steps
|
|
9
|
+
from contractforge_databricks.governance.drift import current_contract_grants
|
|
10
|
+
from contractforge_databricks.governance.validation import access_drift_report
|
|
11
|
+
from contractforge_databricks.security import exception_message
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def apply_access_contract(
|
|
15
|
+
*,
|
|
16
|
+
runner: SqlRunner,
|
|
17
|
+
contract: SemanticContract,
|
|
18
|
+
allow_revoke_unmanaged: bool = False,
|
|
19
|
+
) -> GovernanceApplyResult:
|
|
20
|
+
steps = access_steps(contract)
|
|
21
|
+
if not steps:
|
|
22
|
+
return GovernanceApplyResult(status="NOT_CONFIGURED")
|
|
23
|
+
drift = _drift(runner, contract)
|
|
24
|
+
if drift and drift["status"] == "DRIFTED" and _access_setting(contract, "on_drift", "warn") == "fail":
|
|
25
|
+
return GovernanceApplyResult(
|
|
26
|
+
status="FAILED",
|
|
27
|
+
failed=len(drift["issues"]) or 1,
|
|
28
|
+
sql_preview=tuple(str(step["sql"]) for step in steps),
|
|
29
|
+
errors=tuple(str(issue["message"]) for issue in drift["issues"]),
|
|
30
|
+
)
|
|
31
|
+
if _revoke_unmanaged(contract) and drift and drift["unmanaged_grants"]:
|
|
32
|
+
if not allow_revoke_unmanaged:
|
|
33
|
+
return GovernanceApplyResult(
|
|
34
|
+
status="FAILED",
|
|
35
|
+
failed=len(drift["unmanaged_grants"]),
|
|
36
|
+
sql_preview=tuple(str(step["sql"]) for step in steps),
|
|
37
|
+
errors=("access.revoke_unmanaged requires explicit allow_revoke_unmanaged=True",),
|
|
38
|
+
)
|
|
39
|
+
steps = [*steps, *revoke_grant_steps(contract, drift["unmanaged_grants"])]
|
|
40
|
+
sql_preview = tuple(str(step["sql"]) for step in steps)
|
|
41
|
+
mode = _access_setting(contract, "mode", "apply")
|
|
42
|
+
if mode == "ignore":
|
|
43
|
+
return GovernanceApplyResult(status="IGNORED", ignored=len(steps), sql_preview=sql_preview)
|
|
44
|
+
if mode == "validate_only":
|
|
45
|
+
return GovernanceApplyResult(status="VALIDATED", validated=len(steps), sql_preview=sql_preview)
|
|
46
|
+
|
|
47
|
+
applied = 0
|
|
48
|
+
errors: list[str] = []
|
|
49
|
+
fail_fast = _access_setting(contract, "on_drift", "warn") == "fail"
|
|
50
|
+
for statement in sql_preview:
|
|
51
|
+
try:
|
|
52
|
+
runner.sql(statement)
|
|
53
|
+
applied += 1
|
|
54
|
+
except Exception as exc:
|
|
55
|
+
errors.append(exception_message(exc))
|
|
56
|
+
if fail_fast:
|
|
57
|
+
return GovernanceApplyResult(
|
|
58
|
+
status="FAILED",
|
|
59
|
+
applied=applied,
|
|
60
|
+
failed=len(errors),
|
|
61
|
+
sql_preview=sql_preview,
|
|
62
|
+
errors=tuple(errors),
|
|
63
|
+
)
|
|
64
|
+
if errors:
|
|
65
|
+
return GovernanceApplyResult(
|
|
66
|
+
status="WARNED",
|
|
67
|
+
applied=applied,
|
|
68
|
+
failed=len(errors),
|
|
69
|
+
sql_preview=sql_preview,
|
|
70
|
+
errors=tuple(errors),
|
|
71
|
+
)
|
|
72
|
+
return GovernanceApplyResult(status="SUCCESS", applied=applied, sql_preview=sql_preview)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _access_setting(contract: SemanticContract, key: str, default: str) -> str:
|
|
76
|
+
access = contract.governance.access if contract.governance else None
|
|
77
|
+
if not isinstance(access, dict):
|
|
78
|
+
return default
|
|
79
|
+
policy = access.get("access_policy", {})
|
|
80
|
+
if isinstance(policy, dict) and policy.get(key) is not None:
|
|
81
|
+
return str(policy[key])
|
|
82
|
+
return str(access.get(key, default))
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _revoke_unmanaged(contract: SemanticContract) -> bool:
|
|
86
|
+
return _access_setting(contract, "revoke_unmanaged", "false").lower() == "true"
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _drift(runner: SqlRunner, contract: SemanticContract) -> dict[str, object] | None:
|
|
90
|
+
current = current_contract_grants(runner, contract)
|
|
91
|
+
if current is None:
|
|
92
|
+
return None
|
|
93
|
+
return access_drift_report(contract, current_grants=current)
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"""Unity Catalog access drift inspection helpers."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from contractforge_databricks.rendering.names import target_full_name
|
|
8
|
+
from contractforge_databricks.sql import quote_table_name
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def current_table_grants(runner: Any, target_table: str) -> set[tuple[str, str]] | None:
|
|
12
|
+
"""Return current table grants when the runner exposes a query interface."""
|
|
13
|
+
|
|
14
|
+
query = getattr(runner, "query", None)
|
|
15
|
+
if not callable(query):
|
|
16
|
+
return None
|
|
17
|
+
rows = query(f"SHOW GRANTS ON TABLE {quote_table_name(target_table)}")
|
|
18
|
+
return {_grant_tuple(row) for row in rows if _grant_tuple(row) != (None, None)} # type: ignore[misc]
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def current_contract_grants(runner: Any, contract: Any) -> set[tuple[str, str]] | None:
|
|
22
|
+
return current_table_grants(runner, target_full_name(contract))
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _grant_tuple(row: Any) -> tuple[str | None, str | None]:
|
|
26
|
+
principal = _row_value(row, "Principal", "principal", "grantee")
|
|
27
|
+
privilege = _row_value(row, "ActionType", "actionType", "Privilege", "privilege")
|
|
28
|
+
if principal is None or privilege is None:
|
|
29
|
+
return (None, None)
|
|
30
|
+
return (str(principal), str(privilege).upper())
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _row_value(row: Any, *names: str) -> Any:
|
|
34
|
+
if isinstance(row, dict):
|
|
35
|
+
data = row
|
|
36
|
+
elif hasattr(row, "asDict"):
|
|
37
|
+
data = row.asDict(recursive=True)
|
|
38
|
+
else:
|
|
39
|
+
try:
|
|
40
|
+
data = dict(row)
|
|
41
|
+
except Exception:
|
|
42
|
+
data = {}
|
|
43
|
+
lower = {str(key).lower(): value for key, value in data.items()}
|
|
44
|
+
for name in names:
|
|
45
|
+
if name in data:
|
|
46
|
+
return data[name]
|
|
47
|
+
if name.lower() in lower:
|
|
48
|
+
return lower[name.lower()]
|
|
49
|
+
return None
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
"""Runtime check/apply facade for Databricks governance contracts."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from contractforge_core.results import GovernanceApplyResult
|
|
8
|
+
from contractforge_core.semantic import SemanticContract
|
|
9
|
+
from contractforge_databricks.annotations import annotation_steps, apply_annotations_contract
|
|
10
|
+
from contractforge_databricks.execution.sql_merge import SqlRunner
|
|
11
|
+
from contractforge_databricks.governance.access import access_steps
|
|
12
|
+
from contractforge_databricks.governance.application import apply_access_contract
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def check_governance_contract(contract: SemanticContract) -> dict[str, Any]:
|
|
16
|
+
"""Return reviewable governance SQL without executing it."""
|
|
17
|
+
|
|
18
|
+
annotation_sql = tuple(str(step["sql"]) for step in annotation_steps(contract))
|
|
19
|
+
access_sql = tuple(str(step["sql"]) for step in access_steps(contract))
|
|
20
|
+
total = len(annotation_sql) + len(access_sql)
|
|
21
|
+
return {
|
|
22
|
+
"status": "VALIDATED" if total else "NOT_CONFIGURED",
|
|
23
|
+
"annotations": GovernanceApplyResult(status="VALIDATED" if annotation_sql else "NOT_CONFIGURED", validated=len(annotation_sql), sql_preview=annotation_sql),
|
|
24
|
+
"access": GovernanceApplyResult(status="VALIDATED" if access_sql else "NOT_CONFIGURED", validated=len(access_sql), sql_preview=access_sql),
|
|
25
|
+
"sql_preview": annotation_sql + access_sql,
|
|
26
|
+
"validated": total,
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def apply_governance_contract(*, runner: SqlRunner, contract: SemanticContract) -> dict[str, Any]:
|
|
31
|
+
"""Apply annotations and access governance using an injected Databricks SQL runner."""
|
|
32
|
+
|
|
33
|
+
annotations = apply_annotations_contract(runner=runner, contract=contract)
|
|
34
|
+
access = apply_access_contract(runner=runner, contract=contract)
|
|
35
|
+
return {
|
|
36
|
+
"status": _combined_status(annotations, access),
|
|
37
|
+
"annotations": annotations,
|
|
38
|
+
"access": access,
|
|
39
|
+
"applied": annotations.applied + access.applied,
|
|
40
|
+
"validated": annotations.validated + access.validated,
|
|
41
|
+
"ignored": annotations.ignored + access.ignored,
|
|
42
|
+
"failed": annotations.failed + access.failed,
|
|
43
|
+
"sql_preview": annotations.sql_preview + access.sql_preview,
|
|
44
|
+
"errors": annotations.errors + access.errors,
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _combined_status(*results: GovernanceApplyResult) -> str:
|
|
49
|
+
statuses = {result.status for result in results}
|
|
50
|
+
if "FAILED" in statuses:
|
|
51
|
+
return "FAILED"
|
|
52
|
+
if "WARNED" in statuses:
|
|
53
|
+
return "WARNED"
|
|
54
|
+
if "SUCCESS" in statuses:
|
|
55
|
+
return "SUCCESS"
|
|
56
|
+
if "VALIDATED" in statuses:
|
|
57
|
+
return "VALIDATED"
|
|
58
|
+
if "IGNORED" in statuses:
|
|
59
|
+
return "IGNORED"
|
|
60
|
+
return "NOT_CONFIGURED"
|