databricks-schema 0.10.0__tar.gz → 0.11.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {databricks_schema-0.10.0 → databricks_schema-0.11.0}/CLAUDE.md +1 -1
- {databricks_schema-0.10.0 → databricks_schema-0.11.0}/PKG-INFO +2 -2
- {databricks_schema-0.10.0 → databricks_schema-0.11.0}/README.md +1 -1
- {databricks_schema-0.10.0 → databricks_schema-0.11.0}/databricks_schema/extractor.py +25 -4
- {databricks_schema-0.10.0 → databricks_schema-0.11.0}/databricks_schema/sql_gen.py +44 -25
- {databricks_schema-0.10.0 → databricks_schema-0.11.0}/pyproject.toml +1 -1
- {databricks_schema-0.10.0 → databricks_schema-0.11.0}/skills/databricks-schema-cli/SKILL.md +4 -3
- {databricks_schema-0.10.0 → databricks_schema-0.11.0}/tests/test_extractor.py +52 -10
- {databricks_schema-0.10.0 → databricks_schema-0.11.0}/tests/test_sql_gen.py +110 -2
- {databricks_schema-0.10.0 → databricks_schema-0.11.0}/uv.lock +1 -1
- {databricks_schema-0.10.0 → databricks_schema-0.11.0}/.github/workflows/ci.yml +0 -0
- {databricks_schema-0.10.0 → databricks_schema-0.11.0}/.github/workflows/release.yml +0 -0
- {databricks_schema-0.10.0 → databricks_schema-0.11.0}/.gitignore +0 -0
- {databricks_schema-0.10.0 → databricks_schema-0.11.0}/.python-version +0 -0
- {databricks_schema-0.10.0 → databricks_schema-0.11.0}/LICENSE +0 -0
- {databricks_schema-0.10.0 → databricks_schema-0.11.0}/databricks_schema/__init__.py +0 -0
- {databricks_schema-0.10.0 → databricks_schema-0.11.0}/databricks_schema/cli.py +0 -0
- {databricks_schema-0.10.0 → databricks_schema-0.11.0}/databricks_schema/dbml_io.py +0 -0
- {databricks_schema-0.10.0 → databricks_schema-0.11.0}/databricks_schema/diff.py +0 -0
- {databricks_schema-0.10.0 → databricks_schema-0.11.0}/databricks_schema/models.py +0 -0
- {databricks_schema-0.10.0 → databricks_schema-0.11.0}/databricks_schema/validate.py +0 -0
- {databricks_schema-0.10.0 → databricks_schema-0.11.0}/databricks_schema/yaml_io.py +0 -0
- {databricks_schema-0.10.0 → databricks_schema-0.11.0}/examples/01_extract_schema.py +0 -0
- {databricks_schema-0.10.0 → databricks_schema-0.11.0}/examples/02_save_load_yaml.py +0 -0
- {databricks_schema-0.10.0 → databricks_schema-0.11.0}/examples/03_diff_schemas.py +0 -0
- {databricks_schema-0.10.0 → databricks_schema-0.11.0}/examples/04_generate_sql.py +0 -0
- {databricks_schema-0.10.0 → databricks_schema-0.11.0}/tests/__init__.py +0 -0
- {databricks_schema-0.10.0 → databricks_schema-0.11.0}/tests/test_cli.py +0 -0
- {databricks_schema-0.10.0 → databricks_schema-0.11.0}/tests/test_dbml_io.py +0 -0
- {databricks_schema-0.10.0 → databricks_schema-0.11.0}/tests/test_diff.py +0 -0
- {databricks_schema-0.10.0 → databricks_schema-0.11.0}/tests/test_models.py +0 -0
- {databricks_schema-0.10.0 → databricks_schema-0.11.0}/tests/test_validate.py +0 -0
- {databricks_schema-0.10.0 → databricks_schema-0.11.0}/tests/test_yaml_io.py +0 -0
|
@@ -70,7 +70,7 @@ skills/databricks-schema-cli/SKILL.md # Agent Skill: teaches an AI agent to dri
|
|
|
70
70
|
- `diff` command exits 0 (no changes) or 1 (differences found) — useful in CI
|
|
71
71
|
- Diff result types are dataclasses (not Pydantic); comparison functions are pure (no SDK calls)
|
|
72
72
|
- `generate-sql` auto-detects format from files; exits 2 on mixed formats or empty directory; invalid DBML exits 2
|
|
73
|
-
- `generate-sql` destructive statements commented out by default; `--allow-drop` emits real DROPs
|
|
73
|
+
- `generate-sql` destructive statements commented out by default; `--allow-drop` emits real DROPs (incl. `DROP PRIMARY KEY`/`DROP FOREIGN KEY`); a replaced PK/FK's `ADD` is commented out with its DROP
|
|
74
74
|
- `sql_gen.py` is pure (no SDK, no I/O); diff direction: `FieldChange.old` = stored (target), `.new` = live
|
|
75
75
|
- FK refs in SQL use same catalog as the source table (`ref_schema` + `ref_table` from model)
|
|
76
76
|
- Unsupported field changes (`table_type`) emit `-- TODO: unsupported change: …` comments
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: databricks-schema
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.11.0
|
|
4
4
|
Summary: Databricks Unity Catalog schema extractor
|
|
5
5
|
Project-URL: Homepage, https://github.com/frizzleqq/databricks-schema
|
|
6
6
|
Project-URL: Repository, https://github.com/frizzleqq/databricks-schema
|
|
@@ -394,7 +394,7 @@ Write one `.sql` file per schema to a directory instead:
|
|
|
394
394
|
databricks-schema generate-sql <catalog> ./schemas/ --output-dir ./migrations/
|
|
395
395
|
```
|
|
396
396
|
|
|
397
|
-
Destructive statements (`DROP SCHEMA`, `DROP TABLE`, `DROP COLUMN`, `REVOKE`) are emitted as SQL comments by default. Pass `--allow-drop` to emit them as executable statements:
|
|
397
|
+
Destructive statements (`DROP SCHEMA`, `DROP TABLE`, `DROP COLUMN`, `DROP PRIMARY KEY`, `DROP FOREIGN KEY`, `REVOKE`) are emitted as SQL comments by default. When a primary or foreign key has to be replaced (dropped and re-added), its `ADD CONSTRAINT` is commented out as well, since it would fail while the old key still exists. Pass `--allow-drop` to emit them as executable statements:
|
|
398
398
|
|
|
399
399
|
```bash
|
|
400
400
|
databricks-schema generate-sql <catalog> ./schemas/ --allow-drop
|
|
@@ -367,7 +367,7 @@ Write one `.sql` file per schema to a directory instead:
|
|
|
367
367
|
databricks-schema generate-sql <catalog> ./schemas/ --output-dir ./migrations/
|
|
368
368
|
```
|
|
369
369
|
|
|
370
|
-
Destructive statements (`DROP SCHEMA`, `DROP TABLE`, `DROP COLUMN`, `REVOKE`) are emitted as SQL comments by default. Pass `--allow-drop` to emit them as executable statements:
|
|
370
|
+
Destructive statements (`DROP SCHEMA`, `DROP TABLE`, `DROP COLUMN`, `DROP PRIMARY KEY`, `DROP FOREIGN KEY`, `REVOKE`) are emitted as SQL comments by default. When a primary or foreign key has to be replaced (dropped and re-added), its `ADD CONSTRAINT` is commented out as well, since it would fail while the old key still exists. Pass `--allow-drop` to emit them as executable statements:
|
|
371
371
|
|
|
372
372
|
```bash
|
|
373
373
|
databricks-schema generate-sql <catalog> ./schemas/ --allow-drop
|
|
@@ -7,7 +7,7 @@ from urllib.parse import quote
|
|
|
7
7
|
|
|
8
8
|
from databricks.sdk import WorkspaceClient
|
|
9
9
|
from databricks.sdk.errors import NotFound, PermissionDenied
|
|
10
|
-
from databricks.sdk.service.catalog import SecurableType
|
|
10
|
+
from databricks.sdk.service.catalog import SecurableType, TableType
|
|
11
11
|
|
|
12
12
|
from databricks_schema.models import Catalog, Column, ForeignKey, PrimaryKey, Schema, Table
|
|
13
13
|
|
|
@@ -15,6 +15,9 @@ logger = logging.getLogger(__name__)
|
|
|
15
15
|
|
|
16
16
|
_SYSTEM_SCHEMAS = {"information_schema"}
|
|
17
17
|
|
|
18
|
+
# Views cannot carry PK/FK constraints, so their tables.get() lookup is skipped.
|
|
19
|
+
_NO_CONSTRAINT_TABLE_TYPES = {TableType.VIEW, TableType.METRIC_VIEW}
|
|
20
|
+
|
|
18
21
|
|
|
19
22
|
class CatalogExtractor:
|
|
20
23
|
def __init__(self, client: WorkspaceClient | None = None, max_workers: int = 4) -> None:
|
|
@@ -72,6 +75,23 @@ class CatalogExtractor:
|
|
|
72
75
|
)
|
|
73
76
|
return {p: sorted(privs) for p, privs in sorted(grants.items()) if privs}
|
|
74
77
|
|
|
78
|
+
def _fetch_constraints(self, full_name: str, table_type) -> list:
|
|
79
|
+
"""Return a table's PK/FK constraints via tables.get().
|
|
80
|
+
|
|
81
|
+
tables.list() never populates table_constraints, so a per-table get is required.
|
|
82
|
+
"""
|
|
83
|
+
if table_type in _NO_CONSTRAINT_TABLE_TYPES:
|
|
84
|
+
return []
|
|
85
|
+
try:
|
|
86
|
+
sdk_table = self.client.tables.get(quote(full_name, safe="."))
|
|
87
|
+
except NotFound:
|
|
88
|
+
logger.warning("Not found when fetching constraints for table '%s'", full_name)
|
|
89
|
+
return []
|
|
90
|
+
except PermissionDenied:
|
|
91
|
+
logger.warning("Permission denied when fetching constraints for table '%s'", full_name)
|
|
92
|
+
return []
|
|
93
|
+
return list(getattr(sdk_table, "table_constraints", None) or [])
|
|
94
|
+
|
|
75
95
|
def iter_schemas(
|
|
76
96
|
self,
|
|
77
97
|
catalog_name: str,
|
|
@@ -145,7 +165,7 @@ class CatalogExtractor:
|
|
|
145
165
|
else {}
|
|
146
166
|
)
|
|
147
167
|
|
|
148
|
-
# tables.list returns
|
|
168
|
+
# tables.list returns columns but never table_constraints; those come from tables.get
|
|
149
169
|
sdk_tables = list(
|
|
150
170
|
self.client.tables.list(catalog_name=catalog_name, schema_name=schema_name)
|
|
151
171
|
)
|
|
@@ -225,7 +245,8 @@ class CatalogExtractor:
|
|
|
225
245
|
primary_key: PrimaryKey | None = None
|
|
226
246
|
foreign_keys: list[ForeignKey] = []
|
|
227
247
|
|
|
228
|
-
|
|
248
|
+
table_type = getattr(sdk_table, "table_type", None)
|
|
249
|
+
for constraint in self._fetch_constraints(full_name, table_type):
|
|
229
250
|
pk = getattr(constraint, "primary_key_constraint", None)
|
|
230
251
|
if pk is not None:
|
|
231
252
|
primary_key = PrimaryKey(
|
|
@@ -261,7 +282,7 @@ class CatalogExtractor:
|
|
|
261
282
|
table_name = full_name.split(".")[-1]
|
|
262
283
|
return Table(
|
|
263
284
|
name=table_name,
|
|
264
|
-
table_type=
|
|
285
|
+
table_type=table_type,
|
|
265
286
|
comment=getattr(sdk_table, "comment", None),
|
|
266
287
|
owner=getattr(sdk_table, "owner", None) if include_metadata else None,
|
|
267
288
|
columns=columns,
|
|
@@ -95,21 +95,28 @@ def _grant_stmts(
|
|
|
95
95
|
|
|
96
96
|
|
|
97
97
|
def _pk_stmts(
|
|
98
|
-
tref: str,
|
|
98
|
+
tref: str,
|
|
99
|
+
table_name: str,
|
|
100
|
+
old_pk: PrimaryKey | None,
|
|
101
|
+
new_pk: PrimaryKey | None,
|
|
102
|
+
allow_drop: bool,
|
|
99
103
|
) -> list[str]:
|
|
100
104
|
"""Generate DROP / ADD PRIMARY KEY statements.
|
|
101
105
|
|
|
102
|
-
old_pk = stored/target, new_pk = live/current.
|
|
106
|
+
old_pk = stored/target, new_pk = live/current. DROP is destructive, so it is commented out
|
|
107
|
+
unless allow_drop is True. When replacing a live PK, the ADD would fail while the old PK
|
|
108
|
+
still exists, so it is commented out along with the DROP.
|
|
103
109
|
"""
|
|
104
110
|
stmts: list[str] = []
|
|
111
|
+
drop_skipped = new_pk is not None and not allow_drop
|
|
105
112
|
if new_pk is not None:
|
|
106
|
-
|
|
113
|
+
drop_stmt = f"ALTER TABLE {tref} DROP PRIMARY KEY IF EXISTS;"
|
|
114
|
+
stmts.append(drop_stmt if allow_drop else f"-- {drop_stmt}")
|
|
107
115
|
if old_pk is not None:
|
|
108
116
|
cols = ", ".join(_q(c) for c in old_pk.columns)
|
|
109
117
|
constraint_name = old_pk.name or f"pk_{table_name}"
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
)
|
|
118
|
+
add_stmt = f"ALTER TABLE {tref} ADD CONSTRAINT {_q(constraint_name)} PRIMARY KEY ({cols});"
|
|
119
|
+
stmts.append(f"-- {add_stmt}" if drop_skipped else add_stmt)
|
|
113
120
|
return stmts
|
|
114
121
|
|
|
115
122
|
|
|
@@ -119,33 +126,41 @@ def _fk_stmts(
|
|
|
119
126
|
table_name: str,
|
|
120
127
|
old_fks: list[ForeignKey],
|
|
121
128
|
new_fks: list[ForeignKey],
|
|
129
|
+
allow_drop: bool,
|
|
122
130
|
) -> list[str]:
|
|
123
131
|
"""Generate DROP / ADD FOREIGN KEY statements.
|
|
124
132
|
|
|
125
|
-
FKs are matched by frozenset of their column names
|
|
126
|
-
|
|
133
|
+
FKs are matched by frozenset of their column names; a match whose definition differs
|
|
134
|
+
(name or referenced table/columns) is replaced via DROP + ADD.
|
|
135
|
+
old_fks = stored/target, new_fks = live/current. DROP is destructive, so it is commented
|
|
136
|
+
out unless allow_drop is True; the ADD of a replaced FK is then commented out too.
|
|
127
137
|
"""
|
|
128
138
|
old_by_cols = {frozenset(fk.columns): fk for fk in old_fks}
|
|
129
139
|
new_by_cols = {frozenset(fk.columns): fk for fk in new_fks}
|
|
130
140
|
|
|
131
141
|
stmts: list[str] = []
|
|
132
|
-
# FKs to drop: in live (new) but not in stored (old)
|
|
142
|
+
# FKs to drop: in live (new) but not in stored (old), or defined differently
|
|
133
143
|
for cols_key, fk in new_by_cols.items():
|
|
134
|
-
if cols_key
|
|
144
|
+
if old_by_cols.get(cols_key) != fk:
|
|
135
145
|
cols_list = ", ".join(_q(c) for c in fk.columns)
|
|
136
|
-
|
|
146
|
+
drop_stmt = f"ALTER TABLE {tref} DROP FOREIGN KEY IF EXISTS ({cols_list});"
|
|
147
|
+
stmts.append(drop_stmt if allow_drop else f"-- {drop_stmt}")
|
|
137
148
|
|
|
138
|
-
# FKs to add: in stored (old) but not in live (new)
|
|
149
|
+
# FKs to add: in stored (old) but not in live (new), or defined differently
|
|
139
150
|
for cols_key, fk in old_by_cols.items():
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
)
|
|
151
|
+
live_fk = new_by_cols.get(cols_key)
|
|
152
|
+
if live_fk == fk:
|
|
153
|
+
continue
|
|
154
|
+
fk_cols = ", ".join(_q(c) for c in fk.columns)
|
|
155
|
+
ref_cols = ", ".join(_q(c) for c in fk.ref_columns)
|
|
156
|
+
constraint_name = fk.name or f"fk_{table_name}_{'_'.join(fk.columns)}"
|
|
157
|
+
ref_tref = _table_ref(catalog, fk.ref_schema, fk.ref_table)
|
|
158
|
+
add_stmt = (
|
|
159
|
+
f"ALTER TABLE {tref} ADD CONSTRAINT {_q(constraint_name)} "
|
|
160
|
+
f"FOREIGN KEY ({fk_cols}) REFERENCES {ref_tref} ({ref_cols});"
|
|
161
|
+
)
|
|
162
|
+
replaces_live = live_fk is not None
|
|
163
|
+
stmts.append(f"-- {add_stmt}" if replaces_live and not allow_drop else add_stmt)
|
|
149
164
|
|
|
150
165
|
return stmts
|
|
151
166
|
|
|
@@ -259,9 +274,13 @@ def _table_diff_stmts(
|
|
|
259
274
|
elif fc.field == "grants":
|
|
260
275
|
stmts.extend(_grant_stmts("TABLE", tref, fc.old or {}, fc.new or {}, allow_drop))
|
|
261
276
|
elif fc.field == "primary_key":
|
|
262
|
-
stmts.extend(_pk_stmts(tref, table_diff.name, fc.old, fc.new))
|
|
277
|
+
stmts.extend(_pk_stmts(tref, table_diff.name, fc.old, fc.new, allow_drop))
|
|
263
278
|
elif fc.field == "foreign_keys":
|
|
264
|
-
stmts.extend(
|
|
279
|
+
stmts.extend(
|
|
280
|
+
_fk_stmts(
|
|
281
|
+
tref, catalog, table_diff.name, fc.old or [], fc.new or [], allow_drop
|
|
282
|
+
)
|
|
283
|
+
)
|
|
265
284
|
elif fc.field == "table_type":
|
|
266
285
|
stmts.append(f"-- TODO: unsupported change: table_type {fc.new!r} -> {fc.old!r}")
|
|
267
286
|
|
|
@@ -286,8 +305,8 @@ def schema_diff_to_sql(
|
|
|
286
305
|
catalog_name: Catalog name for fully-qualified references.
|
|
287
306
|
schema_diff: Diff result from diff_schemas() or a manually constructed SchemaDiff.
|
|
288
307
|
stored_schema: The stored (local) Schema; required for "removed" and "modified" status.
|
|
289
|
-
allow_drop: If True, emit real DROP and REVOKE statements;
|
|
290
|
-
versions.
|
|
308
|
+
allow_drop: If True, emit real DROP (including PK/FK drops) and REVOKE statements;
|
|
309
|
+
otherwise emit commented-out versions.
|
|
291
310
|
|
|
292
311
|
Returns:
|
|
293
312
|
SQL string with statements separated by newlines.
|
|
@@ -184,9 +184,10 @@ databricks-schema generate-sql <catalog> ./schemas/ --output-dir ./migrations/
|
|
|
184
184
|
|
|
185
185
|
Databricks Spark SQL DDL to bring the *live* catalog in line with the *stored* files
|
|
186
186
|
(create/alter tables and columns, add/drop keys, `GRANT`/`REVOKE` with `--include-grants`,
|
|
187
|
-
etc.). Destructive statements (`DROP SCHEMA`, `DROP TABLE`, `DROP COLUMN`,
|
|
188
|
-
commented out by default — pass
|
|
189
|
-
executable.
|
|
187
|
+
etc.). Destructive statements (`DROP SCHEMA`, `DROP TABLE`, `DROP COLUMN`,
|
|
188
|
+
`DROP PRIMARY KEY`, `DROP FOREIGN KEY`, `REVOKE`) are commented out by default — pass
|
|
189
|
+
`--allow-drop` to make them executable. A PK/FK that must be replaced (drop + re-add) has its
|
|
190
|
+
`ADD` commented out too unless `--allow-drop` is set, since it would fail while the old key exists. Treat this as review-then-run, not something to pipe straight into execution,
|
|
190
191
|
especially with `--allow-drop`. Unsupported changes (e.g. `table_type`) become
|
|
191
192
|
`-- TODO: unsupported change: ...` comments rather than being silently dropped.
|
|
192
193
|
|
|
@@ -165,8 +165,9 @@ class TestCatalogExtractor:
|
|
|
165
165
|
pk_c.child_columns = ["order_id"]
|
|
166
166
|
constraint = _make_constraint(pk=pk_c)
|
|
167
167
|
|
|
168
|
-
|
|
169
|
-
client.tables.list.return_value = [
|
|
168
|
+
# tables.list never returns constraints; they only come from tables.get
|
|
169
|
+
client.tables.list.return_value = [_make_sdk_table("orders")]
|
|
170
|
+
client.tables.get.return_value = _make_sdk_table("orders", constraints=[constraint])
|
|
170
171
|
|
|
171
172
|
catalog = extractor.extract_catalog("mycat")
|
|
172
173
|
table = catalog.schemas[0].tables[0]
|
|
@@ -193,8 +194,9 @@ class TestCatalogExtractor:
|
|
|
193
194
|
fk_c.parent_columns = ["id"]
|
|
194
195
|
constraint = _make_constraint(fk=fk_c)
|
|
195
196
|
|
|
196
|
-
|
|
197
|
-
client.tables.list.return_value = [
|
|
197
|
+
# tables.list never returns constraints; they only come from tables.get
|
|
198
|
+
client.tables.list.return_value = [_make_sdk_table("orders")]
|
|
199
|
+
client.tables.get.return_value = _make_sdk_table("orders", constraints=[constraint])
|
|
198
200
|
|
|
199
201
|
catalog = extractor.extract_catalog("mycat")
|
|
200
202
|
table = catalog.schemas[0].tables[0]
|
|
@@ -399,25 +401,65 @@ class TestCatalogExtractor:
|
|
|
399
401
|
assert catalog.schemas[0].tables[0].tags == {}
|
|
400
402
|
assert catalog.schemas[0].tables[0].columns[0].tags == {}
|
|
401
403
|
|
|
402
|
-
def
|
|
403
|
-
"""tables.list results are used directly — tables.get should never be called."""
|
|
404
|
-
extractor, client = self._extractor()
|
|
404
|
+
def _single_schema(self, client):
|
|
405
405
|
sdk_catalog = MagicMock()
|
|
406
406
|
sdk_catalog.comment = None
|
|
407
407
|
client.catalogs.get.return_value = sdk_catalog
|
|
408
|
-
|
|
409
408
|
s = MagicMock()
|
|
410
409
|
s.name = "main"
|
|
411
410
|
s.comment = None
|
|
412
411
|
s.owner = None
|
|
413
412
|
client.schemas.list.return_value = [s]
|
|
414
413
|
|
|
415
|
-
|
|
416
|
-
|
|
414
|
+
def test_constraints_fetched_via_tables_get(self):
|
|
415
|
+
"""Constraints come from tables.get with the percent-encoded full name."""
|
|
416
|
+
extractor, client = self._extractor()
|
|
417
|
+
self._single_schema(client)
|
|
418
|
+
client.tables.list.return_value = [_make_sdk_table("my/orders")]
|
|
419
|
+
client.tables.get.return_value = _make_sdk_table("my/orders")
|
|
417
420
|
|
|
418
421
|
extractor.extract_catalog("mycat")
|
|
419
422
|
|
|
423
|
+
client.tables.get.assert_called_once_with("mycat.main.my%2Forders")
|
|
424
|
+
|
|
425
|
+
def test_constraints_not_fetched_for_views(self):
|
|
426
|
+
extractor, client = self._extractor()
|
|
427
|
+
self._single_schema(client)
|
|
428
|
+
client.tables.list.return_value = [
|
|
429
|
+
_make_sdk_table("v", table_type_val="VIEW"),
|
|
430
|
+
_make_sdk_table("mv", table_type_val="METRIC_VIEW"),
|
|
431
|
+
]
|
|
432
|
+
|
|
433
|
+
catalog = extractor.extract_catalog("mycat")
|
|
434
|
+
|
|
420
435
|
client.tables.get.assert_not_called()
|
|
436
|
+
assert all(t.primary_key is None for t in catalog.schemas[0].tables)
|
|
437
|
+
|
|
438
|
+
def test_constraints_not_found_logs_warning_and_continues(self, caplog):
|
|
439
|
+
extractor, client = self._extractor()
|
|
440
|
+
self._single_schema(client)
|
|
441
|
+
client.tables.list.return_value = [_make_sdk_table("orders")]
|
|
442
|
+
client.tables.get.side_effect = NotFound("gone")
|
|
443
|
+
|
|
444
|
+
with caplog.at_level(logging.WARNING, logger="databricks_schema.extractor"):
|
|
445
|
+
catalog = extractor.extract_catalog("mycat")
|
|
446
|
+
|
|
447
|
+
table = catalog.schemas[0].tables[0]
|
|
448
|
+
assert table.primary_key is None
|
|
449
|
+
assert table.foreign_keys == []
|
|
450
|
+
assert "mycat.main.orders" in caplog.text
|
|
451
|
+
|
|
452
|
+
def test_constraints_permission_denied_logs_warning_and_continues(self, caplog):
|
|
453
|
+
extractor, client = self._extractor()
|
|
454
|
+
self._single_schema(client)
|
|
455
|
+
client.tables.list.return_value = [_make_sdk_table("orders")]
|
|
456
|
+
client.tables.get.side_effect = PermissionDenied("nope")
|
|
457
|
+
|
|
458
|
+
with caplog.at_level(logging.WARNING, logger="databricks_schema.extractor"):
|
|
459
|
+
catalog = extractor.extract_catalog("mycat")
|
|
460
|
+
|
|
461
|
+
assert catalog.schemas[0].tables[0].primary_key is None
|
|
462
|
+
assert "Permission denied" in caplog.text
|
|
421
463
|
|
|
422
464
|
def test_column_tags_slash_in_name(self):
|
|
423
465
|
"""Column names containing slashes must be percent-encoded before tag lookup."""
|
|
@@ -252,7 +252,47 @@ class TestTableModified:
|
|
|
252
252
|
)
|
|
253
253
|
sd = diff_schemas(live, stored)
|
|
254
254
|
sql = schema_diff_to_sql("prod", sd, stored)
|
|
255
|
-
assert "DROP PRIMARY KEY IF EXISTS" in sql
|
|
255
|
+
assert "-- ALTER TABLE `prod`.`main`.`t` DROP PRIMARY KEY IF EXISTS;" in sql
|
|
256
|
+
|
|
257
|
+
def test_primary_key_dropped_allow_drop(self):
|
|
258
|
+
stored = _schema("main", tables=[Table(name="t", primary_key=None)])
|
|
259
|
+
live = _schema(
|
|
260
|
+
"main", tables=[Table(name="t", primary_key=PrimaryKey(name="pk_t", columns=["id"]))]
|
|
261
|
+
)
|
|
262
|
+
sd = diff_schemas(live, stored)
|
|
263
|
+
sql = schema_diff_to_sql("prod", sd, stored, allow_drop=True)
|
|
264
|
+
assert "ALTER TABLE `prod`.`main`.`t` DROP PRIMARY KEY IF EXISTS;" in sql.splitlines()
|
|
265
|
+
assert "-- ALTER TABLE `prod`.`main`.`t` DROP PRIMARY KEY" not in sql
|
|
266
|
+
|
|
267
|
+
def test_primary_key_replaced_commented_without_allow_drop(self):
|
|
268
|
+
"""The ADD would fail while the live PK exists, so it is commented out with the DROP."""
|
|
269
|
+
stored = _schema(
|
|
270
|
+
"main",
|
|
271
|
+
tables=[Table(name="t", primary_key=PrimaryKey(name="pk_t", columns=["id", "v"]))],
|
|
272
|
+
)
|
|
273
|
+
live = _schema(
|
|
274
|
+
"main", tables=[Table(name="t", primary_key=PrimaryKey(name="pk_t", columns=["id"]))]
|
|
275
|
+
)
|
|
276
|
+
sd = diff_schemas(live, stored)
|
|
277
|
+
sql = schema_diff_to_sql("prod", sd, stored)
|
|
278
|
+
assert "-- ALTER TABLE `prod`.`main`.`t` DROP PRIMARY KEY IF EXISTS;" in sql
|
|
279
|
+
assert "-- ALTER TABLE `prod`.`main`.`t` ADD CONSTRAINT `pk_t` PRIMARY KEY" in sql
|
|
280
|
+
|
|
281
|
+
def test_primary_key_replaced_allow_drop(self):
|
|
282
|
+
stored = _schema(
|
|
283
|
+
"main",
|
|
284
|
+
tables=[Table(name="t", primary_key=PrimaryKey(name="pk_t", columns=["id", "v"]))],
|
|
285
|
+
)
|
|
286
|
+
live = _schema(
|
|
287
|
+
"main", tables=[Table(name="t", primary_key=PrimaryKey(name="pk_t", columns=["id"]))]
|
|
288
|
+
)
|
|
289
|
+
sd = diff_schemas(live, stored)
|
|
290
|
+
sql = schema_diff_to_sql("prod", sd, stored, allow_drop=True)
|
|
291
|
+
lines = [ln for ln in sql.splitlines() if "PRIMARY KEY" in ln]
|
|
292
|
+
assert lines == [
|
|
293
|
+
"ALTER TABLE `prod`.`main`.`t` DROP PRIMARY KEY IF EXISTS;",
|
|
294
|
+
"ALTER TABLE `prod`.`main`.`t` ADD CONSTRAINT `pk_t` PRIMARY KEY (`id`, `v`);",
|
|
295
|
+
]
|
|
256
296
|
|
|
257
297
|
def test_primary_key_auto_name(self):
|
|
258
298
|
stored = _schema(
|
|
@@ -290,7 +330,75 @@ class TestTableModified:
|
|
|
290
330
|
live = _schema("main", tables=[Table(name="t", foreign_keys=[fk])])
|
|
291
331
|
sd = diff_schemas(live, stored)
|
|
292
332
|
sql = schema_diff_to_sql("prod", sd, stored)
|
|
293
|
-
assert "DROP FOREIGN KEY IF EXISTS (`org_id`);" in sql
|
|
333
|
+
assert "-- ALTER TABLE `prod`.`main`.`t` DROP FOREIGN KEY IF EXISTS (`org_id`);" in sql
|
|
334
|
+
|
|
335
|
+
def test_foreign_key_dropped_allow_drop(self):
|
|
336
|
+
fk = ForeignKey(
|
|
337
|
+
name="fk_org",
|
|
338
|
+
columns=["org_id"],
|
|
339
|
+
ref_schema="orgs",
|
|
340
|
+
ref_table="orgs",
|
|
341
|
+
ref_columns=["id"],
|
|
342
|
+
)
|
|
343
|
+
stored = _schema("main", tables=[Table(name="t", foreign_keys=[])])
|
|
344
|
+
live = _schema("main", tables=[Table(name="t", foreign_keys=[fk])])
|
|
345
|
+
sd = diff_schemas(live, stored)
|
|
346
|
+
sql = schema_diff_to_sql("prod", sd, stored, allow_drop=True)
|
|
347
|
+
assert (
|
|
348
|
+
"ALTER TABLE `prod`.`main`.`t` DROP FOREIGN KEY IF EXISTS (`org_id`);"
|
|
349
|
+
in sql.splitlines()
|
|
350
|
+
)
|
|
351
|
+
assert "-- " not in sql
|
|
352
|
+
|
|
353
|
+
def test_foreign_key_ref_changed_same_columns(self):
|
|
354
|
+
"""An FK on the same columns pointing elsewhere is replaced, not silently ignored."""
|
|
355
|
+
stored_fk = ForeignKey(
|
|
356
|
+
name="fk_org",
|
|
357
|
+
columns=["org_id"],
|
|
358
|
+
ref_schema="orgs",
|
|
359
|
+
ref_table="orgs",
|
|
360
|
+
ref_columns=["id"],
|
|
361
|
+
)
|
|
362
|
+
live_fk = ForeignKey(
|
|
363
|
+
name="fk_org",
|
|
364
|
+
columns=["org_id"],
|
|
365
|
+
ref_schema="old",
|
|
366
|
+
ref_table="orgs",
|
|
367
|
+
ref_columns=["id"],
|
|
368
|
+
)
|
|
369
|
+
stored = _schema("main", tables=[Table(name="t", foreign_keys=[stored_fk])])
|
|
370
|
+
live = _schema("main", tables=[Table(name="t", foreign_keys=[live_fk])])
|
|
371
|
+
sd = diff_schemas(live, stored)
|
|
372
|
+
|
|
373
|
+
sql = schema_diff_to_sql("prod", sd, stored)
|
|
374
|
+
assert "-- ALTER TABLE `prod`.`main`.`t` DROP FOREIGN KEY IF EXISTS (`org_id`);" in sql
|
|
375
|
+
assert "-- ALTER TABLE `prod`.`main`.`t` ADD CONSTRAINT `fk_org`" in sql
|
|
376
|
+
assert "REFERENCES `prod`.`orgs`.`orgs` (`id`);" in sql
|
|
377
|
+
|
|
378
|
+
sql = schema_diff_to_sql("prod", sd, stored, allow_drop=True)
|
|
379
|
+
assert sql.splitlines() == [
|
|
380
|
+
"ALTER TABLE `prod`.`main`.`t` DROP FOREIGN KEY IF EXISTS (`org_id`);",
|
|
381
|
+
"ALTER TABLE `prod`.`main`.`t` ADD CONSTRAINT `fk_org` FOREIGN KEY (`org_id`) "
|
|
382
|
+
"REFERENCES `prod`.`orgs`.`orgs` (`id`);",
|
|
383
|
+
]
|
|
384
|
+
|
|
385
|
+
def test_foreign_key_added_alongside_existing_not_commented(self):
|
|
386
|
+
"""A new FK on other columns needs no DROP, so it is emitted even without allow_drop."""
|
|
387
|
+
keep = ForeignKey(
|
|
388
|
+
name="fk_a", columns=["a"], ref_schema="s", ref_table="x", ref_columns=["id"]
|
|
389
|
+
)
|
|
390
|
+
new = ForeignKey(
|
|
391
|
+
name="fk_b", columns=["b"], ref_schema="s", ref_table="y", ref_columns=["id"]
|
|
392
|
+
)
|
|
393
|
+
stored = _schema("main", tables=[Table(name="t", foreign_keys=[keep, new])])
|
|
394
|
+
live = _schema("main", tables=[Table(name="t", foreign_keys=[keep])])
|
|
395
|
+
sd = diff_schemas(live, stored)
|
|
396
|
+
sql = schema_diff_to_sql("prod", sd, stored)
|
|
397
|
+
assert sql.splitlines() == [
|
|
398
|
+
"ALTER TABLE `prod`.`main`.`t` ADD CONSTRAINT `fk_b` FOREIGN KEY (`b`) "
|
|
399
|
+
"REFERENCES `prod`.`s`.`y` (`id`);"
|
|
400
|
+
]
|
|
401
|
+
assert "DROP" not in sql
|
|
294
402
|
|
|
295
403
|
def test_foreign_key_auto_constraint_name(self):
|
|
296
404
|
fk = ForeignKey(
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|