trellis-datamodel 0.3.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- trellis_datamodel/__init__.py +8 -0
- trellis_datamodel/adapters/__init__.py +41 -0
- trellis_datamodel/adapters/base.py +147 -0
- trellis_datamodel/adapters/dbt_core.py +975 -0
- trellis_datamodel/cli.py +292 -0
- trellis_datamodel/config.py +239 -0
- trellis_datamodel/models/__init__.py +13 -0
- trellis_datamodel/models/schemas.py +28 -0
- trellis_datamodel/routes/__init__.py +11 -0
- trellis_datamodel/routes/data_model.py +221 -0
- trellis_datamodel/routes/manifest.py +110 -0
- trellis_datamodel/routes/schema.py +183 -0
- trellis_datamodel/server.py +101 -0
- trellis_datamodel/static/_app/env.js +1 -0
- trellis_datamodel/static/_app/immutable/assets/0.ByDwyx3a.css +1 -0
- trellis_datamodel/static/_app/immutable/assets/2.DLAp_5AW.css +1 -0
- trellis_datamodel/static/_app/immutable/assets/trellis_squared.CTOnsdDx.svg +127 -0
- trellis_datamodel/static/_app/immutable/chunks/8ZaN1sxc.js +1 -0
- trellis_datamodel/static/_app/immutable/chunks/BfBfOTnK.js +1 -0
- trellis_datamodel/static/_app/immutable/chunks/C3yhlRfZ.js +2 -0
- trellis_datamodel/static/_app/immutable/chunks/CK3bXPEX.js +1 -0
- trellis_datamodel/static/_app/immutable/chunks/CXDUumOQ.js +1 -0
- trellis_datamodel/static/_app/immutable/chunks/DDNfEvut.js +1 -0
- trellis_datamodel/static/_app/immutable/chunks/DUdVct7e.js +1 -0
- trellis_datamodel/static/_app/immutable/chunks/QRltG_J6.js +2 -0
- trellis_datamodel/static/_app/immutable/chunks/zXDdy2c_.js +1 -0
- trellis_datamodel/static/_app/immutable/entry/app.abCkWeAJ.js +2 -0
- trellis_datamodel/static/_app/immutable/entry/start.B7CjH6Z7.js +1 -0
- trellis_datamodel/static/_app/immutable/nodes/0.bFI_DI3G.js +1 -0
- trellis_datamodel/static/_app/immutable/nodes/1.J_r941Qf.js +1 -0
- trellis_datamodel/static/_app/immutable/nodes/2.WqbMkq6o.js +27 -0
- trellis_datamodel/static/_app/version.json +1 -0
- trellis_datamodel/static/index.html +40 -0
- trellis_datamodel/static/robots.txt +3 -0
- trellis_datamodel/static/trellis_squared.svg +127 -0
- trellis_datamodel/tests/__init__.py +2 -0
- trellis_datamodel/tests/conftest.py +132 -0
- trellis_datamodel/tests/test_cli.py +526 -0
- trellis_datamodel/tests/test_data_model.py +151 -0
- trellis_datamodel/tests/test_dbt_schema.py +892 -0
- trellis_datamodel/tests/test_manifest.py +72 -0
- trellis_datamodel/tests/test_server_static.py +44 -0
- trellis_datamodel/tests/test_yaml_handler.py +228 -0
- trellis_datamodel/utils/__init__.py +2 -0
- trellis_datamodel/utils/yaml_handler.py +365 -0
- trellis_datamodel-0.3.3.dist-info/METADATA +333 -0
- trellis_datamodel-0.3.3.dist-info/RECORD +52 -0
- trellis_datamodel-0.3.3.dist-info/WHEEL +5 -0
- trellis_datamodel-0.3.3.dist-info/entry_points.txt +2 -0
- trellis_datamodel-0.3.3.dist-info/licenses/LICENSE +661 -0
- trellis_datamodel-0.3.3.dist-info/licenses/NOTICE +6 -0
- trellis_datamodel-0.3.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,892 @@
|
|
|
1
|
+
"""Tests for dbt schema API endpoints."""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import shutil
|
|
5
|
+
import yaml
|
|
6
|
+
import json
|
|
7
|
+
import pytest
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class TestSaveDbtSchema:
|
|
11
|
+
"""Tests for POST /api/dbt-schema endpoint."""
|
|
12
|
+
|
|
13
|
+
def test_creates_schema_file(self, test_client, temp_dir):
|
|
14
|
+
request_data = {
|
|
15
|
+
"entity_id": "users",
|
|
16
|
+
"model_name": "users",
|
|
17
|
+
"fields": [
|
|
18
|
+
{"name": "id", "datatype": "int"},
|
|
19
|
+
{"name": "email", "datatype": "text", "description": "User email"},
|
|
20
|
+
],
|
|
21
|
+
"description": "User entity",
|
|
22
|
+
}
|
|
23
|
+
response = test_client.post("/api/dbt-schema", json=request_data)
|
|
24
|
+
assert response.status_code == 200
|
|
25
|
+
|
|
26
|
+
result = response.json()
|
|
27
|
+
assert result["status"] == "success"
|
|
28
|
+
assert "file_path" in result
|
|
29
|
+
|
|
30
|
+
# Verify file content
|
|
31
|
+
with open(result["file_path"], "r") as f:
|
|
32
|
+
schema = yaml.safe_load(f)
|
|
33
|
+
|
|
34
|
+
assert schema["version"] == 2
|
|
35
|
+
assert len(schema["models"]) == 1
|
|
36
|
+
model = schema["models"][0]
|
|
37
|
+
assert model["name"] == "users"
|
|
38
|
+
assert model["description"] == "User entity"
|
|
39
|
+
assert len(model["columns"]) == 2
|
|
40
|
+
|
|
41
|
+
def test_preserves_versioned_models_and_versions(
|
|
42
|
+
self, test_client, temp_dir, temp_data_model_path
|
|
43
|
+
):
|
|
44
|
+
# Overwrite manifest with versioned model pointing to player_v2.sql
|
|
45
|
+
manifest_path = os.path.join(temp_dir, "manifest.json")
|
|
46
|
+
manifest_data = {
|
|
47
|
+
"nodes": {
|
|
48
|
+
"model.project.player.v1": {
|
|
49
|
+
"unique_id": "model.project.player.v1",
|
|
50
|
+
"resource_type": "model",
|
|
51
|
+
"name": "player",
|
|
52
|
+
"version": 1,
|
|
53
|
+
"schema": "public",
|
|
54
|
+
"alias": "player",
|
|
55
|
+
"original_file_path": "models/3_core/all/player_v1.sql",
|
|
56
|
+
"columns": {},
|
|
57
|
+
"description": "Player v1",
|
|
58
|
+
"config": {"materialized": "table"},
|
|
59
|
+
"tags": [],
|
|
60
|
+
},
|
|
61
|
+
"model.project.player.v2": {
|
|
62
|
+
"unique_id": "model.project.player.v2",
|
|
63
|
+
"resource_type": "model",
|
|
64
|
+
"name": "player",
|
|
65
|
+
"version": 2,
|
|
66
|
+
"schema": "public",
|
|
67
|
+
"alias": "player",
|
|
68
|
+
"original_file_path": "models/3_core/all/player_v2.sql",
|
|
69
|
+
"columns": {},
|
|
70
|
+
"description": "Player v2",
|
|
71
|
+
"config": {"materialized": "table"},
|
|
72
|
+
"tags": [],
|
|
73
|
+
},
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
with open(manifest_path, "w") as f:
|
|
77
|
+
json.dump(manifest_data, f)
|
|
78
|
+
|
|
79
|
+
# Existing schema with v1 definition (stored in player.yml)
|
|
80
|
+
models_dir = os.path.join(temp_dir, "models", "3_core", "all")
|
|
81
|
+
os.makedirs(models_dir, exist_ok=True)
|
|
82
|
+
yml_path = os.path.join(models_dir, "player.yml")
|
|
83
|
+
existing_schema = {
|
|
84
|
+
"version": 2,
|
|
85
|
+
"models": [
|
|
86
|
+
{
|
|
87
|
+
"name": "player",
|
|
88
|
+
"latest_version": 1,
|
|
89
|
+
"versions": [
|
|
90
|
+
{
|
|
91
|
+
"v": 1,
|
|
92
|
+
"description": "v1 description",
|
|
93
|
+
"columns": [{"name": "player_id", "data_type": "text"}],
|
|
94
|
+
}
|
|
95
|
+
],
|
|
96
|
+
}
|
|
97
|
+
],
|
|
98
|
+
}
|
|
99
|
+
with open(yml_path, "w") as f:
|
|
100
|
+
yaml.dump(existing_schema, f)
|
|
101
|
+
|
|
102
|
+
# Data model binds entity to v2
|
|
103
|
+
data_model = {
|
|
104
|
+
"version": 0.1,
|
|
105
|
+
"entities": [
|
|
106
|
+
{
|
|
107
|
+
"id": "player",
|
|
108
|
+
"label": "Player",
|
|
109
|
+
"description": "Players competing in the NBA",
|
|
110
|
+
"dbt_model": "model.project.player.v2",
|
|
111
|
+
}
|
|
112
|
+
],
|
|
113
|
+
"relationships": [],
|
|
114
|
+
}
|
|
115
|
+
with open(temp_data_model_path, "w") as f:
|
|
116
|
+
yaml.dump(data_model, f)
|
|
117
|
+
|
|
118
|
+
request_data = {
|
|
119
|
+
"entity_id": "player",
|
|
120
|
+
"model_name": "player",
|
|
121
|
+
"fields": [
|
|
122
|
+
{
|
|
123
|
+
"name": "player_uuid",
|
|
124
|
+
"datatype": "text",
|
|
125
|
+
"description": "New PK",
|
|
126
|
+
}
|
|
127
|
+
],
|
|
128
|
+
"description": "Players v2",
|
|
129
|
+
"tags": ["core"],
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
response = test_client.post("/api/dbt-schema", json=request_data)
|
|
133
|
+
assert response.status_code == 200
|
|
134
|
+
|
|
135
|
+
with open(yml_path, "r") as f:
|
|
136
|
+
schema = yaml.safe_load(f)
|
|
137
|
+
|
|
138
|
+
model = schema["models"][0]
|
|
139
|
+
assert model["latest_version"] == 2
|
|
140
|
+
|
|
141
|
+
versions = {v["v"]: v for v in model["versions"]}
|
|
142
|
+
assert 1 in versions # keep existing v1
|
|
143
|
+
assert 2 in versions # add/update v2
|
|
144
|
+
|
|
145
|
+
# v1 is unchanged
|
|
146
|
+
assert versions[1]["columns"][0]["name"] == "player_id"
|
|
147
|
+
|
|
148
|
+
# v2 reflects new request
|
|
149
|
+
v2_columns = versions[2]["columns"]
|
|
150
|
+
assert v2_columns[0]["name"] == "player_uuid"
|
|
151
|
+
assert versions[2].get("config", {}).get("tags") == ["core"]
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
class TestSyncDbtTests:
|
|
155
|
+
"""Tests for POST /api/sync-dbt-tests endpoint."""
|
|
156
|
+
|
|
157
|
+
def test_syncs_relationship_tests(
|
|
158
|
+
self, test_client, temp_dir, temp_data_model_path
|
|
159
|
+
):
|
|
160
|
+
# Create data model with entities and relationships
|
|
161
|
+
data_model = {
|
|
162
|
+
"version": 0.1,
|
|
163
|
+
"entities": [
|
|
164
|
+
{"id": "users", "label": "Users", "position": {"x": 0, "y": 0}},
|
|
165
|
+
{
|
|
166
|
+
"id": "orders",
|
|
167
|
+
"label": "Orders",
|
|
168
|
+
"position": {"x": 100, "y": 0},
|
|
169
|
+
"drafted_fields": [{"name": "user_id", "datatype": "int"}],
|
|
170
|
+
},
|
|
171
|
+
],
|
|
172
|
+
"relationships": [
|
|
173
|
+
{
|
|
174
|
+
"source": "users",
|
|
175
|
+
"target": "orders",
|
|
176
|
+
"type": "one_to_many",
|
|
177
|
+
"source_field": "id",
|
|
178
|
+
"target_field": "user_id",
|
|
179
|
+
}
|
|
180
|
+
],
|
|
181
|
+
}
|
|
182
|
+
with open(temp_data_model_path, "w") as f:
|
|
183
|
+
yaml.dump(data_model, f)
|
|
184
|
+
|
|
185
|
+
response = test_client.post("/api/sync-dbt-tests")
|
|
186
|
+
assert response.status_code == 200
|
|
187
|
+
|
|
188
|
+
result = response.json()
|
|
189
|
+
assert result["status"] == "success"
|
|
190
|
+
assert len(result["files"]) == 2 # One for each entity
|
|
191
|
+
|
|
192
|
+
def test_syncs_using_dbt_model_names(
|
|
193
|
+
self, test_client, temp_dir, temp_data_model_path
|
|
194
|
+
):
|
|
195
|
+
"""
|
|
196
|
+
Ensure relationship tests reference bound dbt model names, not raw entity IDs.
|
|
197
|
+
"""
|
|
198
|
+
data_model = {
|
|
199
|
+
"version": 0.1,
|
|
200
|
+
"entities": [
|
|
201
|
+
{
|
|
202
|
+
"id": "customer_entity",
|
|
203
|
+
"label": "Customers",
|
|
204
|
+
"dbt_model": "model.project.customers",
|
|
205
|
+
"position": {"x": 0, "y": 0},
|
|
206
|
+
},
|
|
207
|
+
{
|
|
208
|
+
"id": "order_entity",
|
|
209
|
+
"label": "Orders",
|
|
210
|
+
"dbt_model": "model.project.orders",
|
|
211
|
+
"drafted_fields": [{"name": "customer_id", "datatype": "int"}],
|
|
212
|
+
},
|
|
213
|
+
],
|
|
214
|
+
"relationships": [
|
|
215
|
+
{
|
|
216
|
+
"source": "customer_entity",
|
|
217
|
+
"target": "order_entity",
|
|
218
|
+
"type": "one_to_many",
|
|
219
|
+
"source_field": "id",
|
|
220
|
+
"target_field": "customer_id",
|
|
221
|
+
}
|
|
222
|
+
],
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
# Persist data model
|
|
226
|
+
with open(temp_data_model_path, "w") as f:
|
|
227
|
+
yaml.dump(data_model, f)
|
|
228
|
+
|
|
229
|
+
response = test_client.post("/api/sync-dbt-tests")
|
|
230
|
+
assert response.status_code == 200
|
|
231
|
+
|
|
232
|
+
# orders.yml should contain a relationship test pointing to customers (dbt model name)
|
|
233
|
+
orders_yml = os.path.join(temp_dir, "models", "3_core", "orders.yml")
|
|
234
|
+
assert os.path.exists(orders_yml)
|
|
235
|
+
with open(orders_yml, "r") as f:
|
|
236
|
+
schema = yaml.safe_load(f)
|
|
237
|
+
|
|
238
|
+
rel_tests = schema["models"][0]["columns"][0]["data_tests"]
|
|
239
|
+
assert rel_tests == [
|
|
240
|
+
{
|
|
241
|
+
"relationships": {
|
|
242
|
+
"arguments": {"to": "ref('customers')", "field": "id"},
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
]
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
class TestGetModelSchema:
|
|
249
|
+
"""Tests for GET /api/models/{model_name}/schema endpoint."""
|
|
250
|
+
|
|
251
|
+
def test_returns_empty_for_missing_yml(self, test_client, temp_dir, mock_manifest):
|
|
252
|
+
# Create SQL file path structure (manifest points to this)
|
|
253
|
+
sql_dir = os.path.join(temp_dir, "models", "3_core")
|
|
254
|
+
os.makedirs(sql_dir, exist_ok=True)
|
|
255
|
+
with open(os.path.join(sql_dir, "users.sql"), "w") as f:
|
|
256
|
+
f.write("SELECT 1")
|
|
257
|
+
|
|
258
|
+
response = test_client.get("/api/models/users/schema")
|
|
259
|
+
assert response.status_code == 200
|
|
260
|
+
|
|
261
|
+
data = response.json()
|
|
262
|
+
assert data["model_name"] == "users"
|
|
263
|
+
assert data["columns"] == []
|
|
264
|
+
|
|
265
|
+
def test_returns_404_for_unknown_model(self, test_client):
|
|
266
|
+
response = test_client.get("/api/models/nonexistent/schema")
|
|
267
|
+
assert response.status_code == 404
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
class TestUpdateModelSchema:
|
|
271
|
+
"""Tests for POST /api/models/{model_name}/schema endpoint."""
|
|
272
|
+
|
|
273
|
+
def test_updates_schema(self, test_client, temp_dir, mock_manifest):
|
|
274
|
+
# Create the SQL file that manifest points to
|
|
275
|
+
sql_dir = os.path.join(temp_dir, "models", "3_core")
|
|
276
|
+
os.makedirs(sql_dir, exist_ok=True)
|
|
277
|
+
with open(os.path.join(sql_dir, "users.sql"), "w") as f:
|
|
278
|
+
f.write("SELECT 1")
|
|
279
|
+
|
|
280
|
+
request_data = {
|
|
281
|
+
"columns": [
|
|
282
|
+
{"name": "id", "data_type": "int", "description": "Primary key"},
|
|
283
|
+
],
|
|
284
|
+
"description": "Updated description",
|
|
285
|
+
}
|
|
286
|
+
response = test_client.post("/api/models/users/schema", json=request_data)
|
|
287
|
+
assert response.status_code == 200
|
|
288
|
+
|
|
289
|
+
result = response.json()
|
|
290
|
+
assert result["status"] == "success"
|
|
291
|
+
|
|
292
|
+
# Verify the YML file was created
|
|
293
|
+
yml_path = os.path.join(sql_dir, "users.yml")
|
|
294
|
+
assert os.path.exists(yml_path)
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
class TestInferRelationships:
|
|
298
|
+
"""Tests for GET /api/infer-relationships endpoint."""
|
|
299
|
+
|
|
300
|
+
def test_returns_empty_for_no_yml_files(self, test_client, temp_dir):
|
|
301
|
+
response = test_client.get("/api/infer-relationships")
|
|
302
|
+
assert response.status_code == 400
|
|
303
|
+
assert "No schema yml files found" in response.json()["detail"]
|
|
304
|
+
|
|
305
|
+
def test_infers_relationships_from_tests(
|
|
306
|
+
self, test_client, temp_dir, temp_data_model_path
|
|
307
|
+
):
|
|
308
|
+
# Data model with bound entities
|
|
309
|
+
data_model = {
|
|
310
|
+
"version": 0.1,
|
|
311
|
+
"entities": [
|
|
312
|
+
{"id": "users", "dbt_model": "model.project.users"},
|
|
313
|
+
{"id": "orders", "dbt_model": "model.project.orders"},
|
|
314
|
+
],
|
|
315
|
+
}
|
|
316
|
+
with open(temp_data_model_path, "w") as f:
|
|
317
|
+
yaml.dump(data_model, f)
|
|
318
|
+
|
|
319
|
+
# Create a YML file with relationship tests
|
|
320
|
+
models_dir = os.path.join(temp_dir, "models", "3_core")
|
|
321
|
+
os.makedirs(models_dir, exist_ok=True)
|
|
322
|
+
|
|
323
|
+
schema = {
|
|
324
|
+
"version": 2,
|
|
325
|
+
"models": [
|
|
326
|
+
{
|
|
327
|
+
"name": "orders",
|
|
328
|
+
"columns": [
|
|
329
|
+
{
|
|
330
|
+
"name": "user_id",
|
|
331
|
+
"data_type": "int",
|
|
332
|
+
"tests": [
|
|
333
|
+
{
|
|
334
|
+
"relationships": {
|
|
335
|
+
"arguments": {
|
|
336
|
+
"to": "ref('users')",
|
|
337
|
+
"field": "id",
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
],
|
|
342
|
+
}
|
|
343
|
+
],
|
|
344
|
+
}
|
|
345
|
+
],
|
|
346
|
+
}
|
|
347
|
+
with open(os.path.join(models_dir, "orders.yml"), "w") as f:
|
|
348
|
+
yaml.dump(schema, f)
|
|
349
|
+
|
|
350
|
+
response = test_client.get("/api/infer-relationships")
|
|
351
|
+
assert response.status_code == 200
|
|
352
|
+
|
|
353
|
+
rels = response.json()["relationships"]
|
|
354
|
+
assert len(rels) == 1
|
|
355
|
+
assert rels[0]["source"] == "users"
|
|
356
|
+
assert rels[0]["target"] == "orders"
|
|
357
|
+
assert rels[0]["source_field"] == "id"
|
|
358
|
+
assert rels[0]["target_field"] == "user_id"
|
|
359
|
+
|
|
360
|
+
def test_infers_relationships_from_nested_directories(
|
|
361
|
+
self, test_client, temp_dir, temp_data_model_path
|
|
362
|
+
):
|
|
363
|
+
# Ensure nested model directories are also scanned
|
|
364
|
+
nested_dir = os.path.join(temp_dir, "models", "3_core", "all")
|
|
365
|
+
os.makedirs(nested_dir, exist_ok=True)
|
|
366
|
+
|
|
367
|
+
data_model = {
|
|
368
|
+
"version": 0.1,
|
|
369
|
+
"entities": [
|
|
370
|
+
{"id": "team", "dbt_model": "model.project.team"},
|
|
371
|
+
{"id": "game", "dbt_model": "model.project.game"},
|
|
372
|
+
],
|
|
373
|
+
}
|
|
374
|
+
with open(temp_data_model_path, "w") as f:
|
|
375
|
+
yaml.dump(data_model, f)
|
|
376
|
+
|
|
377
|
+
schema = {
|
|
378
|
+
"version": 2,
|
|
379
|
+
"models": [
|
|
380
|
+
{
|
|
381
|
+
"name": "game",
|
|
382
|
+
"columns": [
|
|
383
|
+
{
|
|
384
|
+
"name": "home_team_id",
|
|
385
|
+
"data_type": "text",
|
|
386
|
+
"data_tests": [
|
|
387
|
+
{
|
|
388
|
+
"relationships": {
|
|
389
|
+
"arguments": {
|
|
390
|
+
"to": "ref('team')",
|
|
391
|
+
"field": "team_id",
|
|
392
|
+
},
|
|
393
|
+
}
|
|
394
|
+
}
|
|
395
|
+
],
|
|
396
|
+
},
|
|
397
|
+
{
|
|
398
|
+
"name": "away_team_id",
|
|
399
|
+
"data_type": "text",
|
|
400
|
+
"data_tests": [
|
|
401
|
+
{
|
|
402
|
+
"relationships": {
|
|
403
|
+
"arguments": {
|
|
404
|
+
"to": "ref('team')",
|
|
405
|
+
"field": "team_id",
|
|
406
|
+
},
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
],
|
|
410
|
+
},
|
|
411
|
+
],
|
|
412
|
+
}
|
|
413
|
+
],
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
with open(os.path.join(nested_dir, "game.yml"), "w") as f:
|
|
417
|
+
yaml.dump(schema, f)
|
|
418
|
+
|
|
419
|
+
response = test_client.get("/api/infer-relationships")
|
|
420
|
+
assert response.status_code == 200
|
|
421
|
+
|
|
422
|
+
rels = response.json()["relationships"]
|
|
423
|
+
assert len(rels) == 2
|
|
424
|
+
assert {
|
|
425
|
+
"source": "team",
|
|
426
|
+
"target": "game",
|
|
427
|
+
"source_field": "team_id",
|
|
428
|
+
"target_field": "home_team_id",
|
|
429
|
+
} in [
|
|
430
|
+
{
|
|
431
|
+
"source": r["source"],
|
|
432
|
+
"target": r["target"],
|
|
433
|
+
"source_field": r["source_field"],
|
|
434
|
+
"target_field": r["target_field"],
|
|
435
|
+
}
|
|
436
|
+
for r in rels
|
|
437
|
+
]
|
|
438
|
+
assert {
|
|
439
|
+
"source": "team",
|
|
440
|
+
"target": "game",
|
|
441
|
+
"source_field": "team_id",
|
|
442
|
+
"target_field": "away_team_id",
|
|
443
|
+
} in [
|
|
444
|
+
{
|
|
445
|
+
"source": r["source"],
|
|
446
|
+
"target": r["target"],
|
|
447
|
+
"source_field": r["source_field"],
|
|
448
|
+
"target_field": r["target_field"],
|
|
449
|
+
}
|
|
450
|
+
for r in rels
|
|
451
|
+
]
|
|
452
|
+
|
|
453
|
+
def test_infers_relationships_across_multiple_model_paths(
|
|
454
|
+
self, test_client, temp_dir, temp_data_model_path
|
|
455
|
+
):
|
|
456
|
+
"""
|
|
457
|
+
When multiple dbt model paths are configured (including with a models/ prefix),
|
|
458
|
+
all should be scanned.
|
|
459
|
+
"""
|
|
460
|
+
from trellis_datamodel import config as cfg
|
|
461
|
+
|
|
462
|
+
# Add an extra model path and point to a different directory
|
|
463
|
+
extra_models_dir = os.path.join(temp_dir, "models", "3_entity")
|
|
464
|
+
os.makedirs(extra_models_dir, exist_ok=True)
|
|
465
|
+
|
|
466
|
+
original_paths = list(cfg.DBT_MODEL_PATHS)
|
|
467
|
+
try:
|
|
468
|
+
cfg.DBT_MODEL_PATHS = ["3_core", "models/3_entity"]
|
|
469
|
+
|
|
470
|
+
data_model = {
|
|
471
|
+
"version": 0.1,
|
|
472
|
+
"entities": [
|
|
473
|
+
{"id": "product", "dbt_model": "model.project.product"},
|
|
474
|
+
{"id": "opportunity", "dbt_model": "model.project.opportunity"},
|
|
475
|
+
],
|
|
476
|
+
}
|
|
477
|
+
with open(temp_data_model_path, "w") as f:
|
|
478
|
+
yaml.dump(data_model, f)
|
|
479
|
+
|
|
480
|
+
schema = {
|
|
481
|
+
"version": 2,
|
|
482
|
+
"models": [
|
|
483
|
+
{
|
|
484
|
+
"name": "opportunity",
|
|
485
|
+
"columns": [
|
|
486
|
+
{
|
|
487
|
+
"name": "product_id",
|
|
488
|
+
"data_tests": [
|
|
489
|
+
{
|
|
490
|
+
"relationships": {
|
|
491
|
+
"arguments": {
|
|
492
|
+
"to": "ref('product')",
|
|
493
|
+
"field": "product_id",
|
|
494
|
+
}
|
|
495
|
+
}
|
|
496
|
+
}
|
|
497
|
+
],
|
|
498
|
+
}
|
|
499
|
+
],
|
|
500
|
+
}
|
|
501
|
+
],
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
with open(os.path.join(extra_models_dir, "opportunity.yml"), "w") as f:
|
|
505
|
+
yaml.dump(schema, f)
|
|
506
|
+
|
|
507
|
+
response = test_client.get("/api/infer-relationships")
|
|
508
|
+
assert response.status_code == 200
|
|
509
|
+
|
|
510
|
+
rels = response.json()["relationships"]
|
|
511
|
+
assert {"source": "product", "target": "opportunity"} in [
|
|
512
|
+
{"source": r["source"], "target": r["target"]} for r in rels
|
|
513
|
+
]
|
|
514
|
+
finally:
|
|
515
|
+
cfg.DBT_MODEL_PATHS = original_paths
|
|
516
|
+
shutil.rmtree(extra_models_dir, ignore_errors=True)
|
|
517
|
+
|
|
518
|
+
def test_infers_relationships_with_arguments_block(
|
|
519
|
+
self, test_client, temp_dir, temp_data_model_path
|
|
520
|
+
):
|
|
521
|
+
"""
|
|
522
|
+
The app should recognize dbt's arguments syntax for relationship tests.
|
|
523
|
+
"""
|
|
524
|
+
models_dir = os.path.join(temp_dir, "models", "3_core")
|
|
525
|
+
# Clean out prior test artifacts to avoid cross-test contamination
|
|
526
|
+
shutil.rmtree(models_dir, ignore_errors=True)
|
|
527
|
+
os.makedirs(models_dir, exist_ok=True)
|
|
528
|
+
|
|
529
|
+
data_model = {
|
|
530
|
+
"version": 0.1,
|
|
531
|
+
"entities": [
|
|
532
|
+
{"id": "customers", "dbt_model": "model.project.customers"},
|
|
533
|
+
{"id": "orders", "dbt_model": "model.project.orders"},
|
|
534
|
+
],
|
|
535
|
+
}
|
|
536
|
+
with open(temp_data_model_path, "w") as f:
|
|
537
|
+
yaml.dump(data_model, f)
|
|
538
|
+
|
|
539
|
+
schema = {
|
|
540
|
+
"version": 2,
|
|
541
|
+
"models": [
|
|
542
|
+
{
|
|
543
|
+
"name": "orders",
|
|
544
|
+
"columns": [
|
|
545
|
+
{
|
|
546
|
+
"name": "customer_id",
|
|
547
|
+
"data_type": "int",
|
|
548
|
+
"data_tests": [
|
|
549
|
+
{
|
|
550
|
+
"relationships": {
|
|
551
|
+
"arguments": {
|
|
552
|
+
"to": "ref('customers')",
|
|
553
|
+
"field": "id",
|
|
554
|
+
},
|
|
555
|
+
"config": {"severity": "error"},
|
|
556
|
+
}
|
|
557
|
+
}
|
|
558
|
+
],
|
|
559
|
+
}
|
|
560
|
+
],
|
|
561
|
+
}
|
|
562
|
+
],
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
with open(os.path.join(models_dir, "orders.yml"), "w") as f:
|
|
566
|
+
yaml.dump(schema, f)
|
|
567
|
+
|
|
568
|
+
response = test_client.get("/api/infer-relationships")
|
|
569
|
+
assert response.status_code == 200
|
|
570
|
+
|
|
571
|
+
rels = response.json()["relationships"]
|
|
572
|
+
assert len(rels) == 1
|
|
573
|
+
assert rels[0]["source"] == "customers"
|
|
574
|
+
assert rels[0]["target"] == "orders"
|
|
575
|
+
assert rels[0]["source_field"] == "id"
|
|
576
|
+
assert rels[0]["target_field"] == "customer_id"
|
|
577
|
+
|
|
578
|
+
def test_can_include_unbound_entities_when_requested(
|
|
579
|
+
self, test_client, temp_dir, temp_data_model_path
|
|
580
|
+
):
|
|
581
|
+
"""
|
|
582
|
+
When include_unbound=true is passed, relationships are returned even if
|
|
583
|
+
the entities have not yet been persisted with dbt_model bindings.
|
|
584
|
+
"""
|
|
585
|
+
models_dir = os.path.join(temp_dir, "models", "3_core")
|
|
586
|
+
os.makedirs(models_dir, exist_ok=True)
|
|
587
|
+
|
|
588
|
+
# Data model without dbt_model bindings (e.g. right after a drag+drop)
|
|
589
|
+
data_model = {
|
|
590
|
+
"version": 0.1,
|
|
591
|
+
"entities": [
|
|
592
|
+
{"id": "customers"},
|
|
593
|
+
{"id": "orders"},
|
|
594
|
+
],
|
|
595
|
+
}
|
|
596
|
+
with open(temp_data_model_path, "w") as f:
|
|
597
|
+
yaml.dump(data_model, f)
|
|
598
|
+
|
|
599
|
+
# Relationship test between the two models
|
|
600
|
+
schema = {
|
|
601
|
+
"version": 2,
|
|
602
|
+
"models": [
|
|
603
|
+
{
|
|
604
|
+
"name": "orders",
|
|
605
|
+
"columns": [
|
|
606
|
+
{
|
|
607
|
+
"name": "customer_id",
|
|
608
|
+
"tests": [
|
|
609
|
+
{
|
|
610
|
+
"relationships": {
|
|
611
|
+
"arguments": {
|
|
612
|
+
"to": "ref('customers')",
|
|
613
|
+
"field": "id",
|
|
614
|
+
}
|
|
615
|
+
}
|
|
616
|
+
}
|
|
617
|
+
],
|
|
618
|
+
}
|
|
619
|
+
],
|
|
620
|
+
}
|
|
621
|
+
],
|
|
622
|
+
}
|
|
623
|
+
with open(os.path.join(models_dir, "orders.yml"), "w") as f:
|
|
624
|
+
yaml.dump(schema, f)
|
|
625
|
+
|
|
626
|
+
# Default behaviour should still filter unbound entities
|
|
627
|
+
default_response = test_client.get("/api/infer-relationships")
|
|
628
|
+
assert default_response.status_code == 200
|
|
629
|
+
assert default_response.json()["relationships"] == []
|
|
630
|
+
|
|
631
|
+
# With the flag enabled we should get the inferred relationship back
|
|
632
|
+
response = test_client.get("/api/infer-relationships?include_unbound=true")
|
|
633
|
+
assert response.status_code == 200
|
|
634
|
+
rels = response.json()["relationships"]
|
|
635
|
+
assert len(rels) == 1
|
|
636
|
+
assert rels[0]["source"] == "customers"
|
|
637
|
+
assert rels[0]["target"] == "orders"
|
|
638
|
+
assert rels[0]["source_field"] == "id"
|
|
639
|
+
assert rels[0]["target_field"] == "customer_id"
|
|
640
|
+
|
|
641
|
+
def test_maps_additional_models_to_entity_ids(
|
|
642
|
+
self, test_client, temp_dir, temp_data_model_path
|
|
643
|
+
):
|
|
644
|
+
"""
|
|
645
|
+
Relationship inference should translate additional_models to their entity IDs.
|
|
646
|
+
"""
|
|
647
|
+
# Data model maps additional model to entity
|
|
648
|
+
data_model = {
|
|
649
|
+
"version": 0.1,
|
|
650
|
+
"entities": [
|
|
651
|
+
{
|
|
652
|
+
"id": "customers",
|
|
653
|
+
"label": "Customers",
|
|
654
|
+
"additional_models": ["model.project.customers_alt"],
|
|
655
|
+
},
|
|
656
|
+
{"id": "orders", "label": "Orders", "dbt_model": "model.project.orders"},
|
|
657
|
+
],
|
|
658
|
+
}
|
|
659
|
+
with open(temp_data_model_path, "w") as f:
|
|
660
|
+
yaml.dump(data_model, f)
|
|
661
|
+
|
|
662
|
+
# Create YML for additional model name with relationship test
|
|
663
|
+
models_dir = os.path.join(temp_dir, "models", "3_core")
|
|
664
|
+
os.makedirs(models_dir, exist_ok=True)
|
|
665
|
+
schema = {
|
|
666
|
+
"version": 2,
|
|
667
|
+
"models": [
|
|
668
|
+
{
|
|
669
|
+
"name": "customers_alt",
|
|
670
|
+
"columns": [
|
|
671
|
+
{
|
|
672
|
+
"name": "id",
|
|
673
|
+
"data_type": "int",
|
|
674
|
+
"data_tests": [
|
|
675
|
+
{
|
|
676
|
+
"relationships": {
|
|
677
|
+
"arguments": {
|
|
678
|
+
"to": "ref('orders')",
|
|
679
|
+
"field": "order_id",
|
|
680
|
+
},
|
|
681
|
+
}
|
|
682
|
+
}
|
|
683
|
+
],
|
|
684
|
+
}
|
|
685
|
+
],
|
|
686
|
+
}
|
|
687
|
+
],
|
|
688
|
+
}
|
|
689
|
+
with open(os.path.join(models_dir, "customers_alt.yml"), "w") as f:
|
|
690
|
+
yaml.dump(schema, f)
|
|
691
|
+
|
|
692
|
+
response = test_client.get("/api/infer-relationships")
|
|
693
|
+
assert response.status_code == 200
|
|
694
|
+
|
|
695
|
+
rels = response.json()["relationships"]
|
|
696
|
+
# Find the relationship coming from the additional model file
|
|
697
|
+
rel = next(
|
|
698
|
+
r
|
|
699
|
+
for r in rels
|
|
700
|
+
if r["source_field"] == "order_id"
|
|
701
|
+
and r["target_field"] == "id"
|
|
702
|
+
and r["target"] == "customers"
|
|
703
|
+
and r["source"] == "orders"
|
|
704
|
+
)
|
|
705
|
+
assert rel
|
|
706
|
+
|
|
707
|
+
def test_resolves_versioned_refs_to_existing_entity(
|
|
708
|
+
self, test_client, temp_dir, temp_data_model_path
|
|
709
|
+
):
|
|
710
|
+
"""
|
|
711
|
+
ref('model', v=1) should resolve to an entity bound to v2 (or vice-versa)
|
|
712
|
+
instead of creating a duplicate entity.
|
|
713
|
+
"""
|
|
714
|
+
# Bind player to v2 in the data model
|
|
715
|
+
data_model = {
|
|
716
|
+
"version": 0.1,
|
|
717
|
+
"entities": [
|
|
718
|
+
{
|
|
719
|
+
"id": "player",
|
|
720
|
+
"label": "Player",
|
|
721
|
+
"dbt_model": "model.test.player.v2",
|
|
722
|
+
},
|
|
723
|
+
{
|
|
724
|
+
"id": "game_stats",
|
|
725
|
+
"label": "Game Stats",
|
|
726
|
+
"dbt_model": "model.test.game_stats",
|
|
727
|
+
},
|
|
728
|
+
],
|
|
729
|
+
}
|
|
730
|
+
with open(temp_data_model_path, "w") as f:
|
|
731
|
+
yaml.dump(data_model, f)
|
|
732
|
+
|
|
733
|
+
# YML with versioned ref to player v1
|
|
734
|
+
schema = {
|
|
735
|
+
"version": 2,
|
|
736
|
+
"models": [
|
|
737
|
+
{
|
|
738
|
+
"name": "game_stats",
|
|
739
|
+
"columns": [
|
|
740
|
+
{
|
|
741
|
+
"name": "player_id",
|
|
742
|
+
"data_tests": [
|
|
743
|
+
{
|
|
744
|
+
"relationships": {
|
|
745
|
+
"arguments": {
|
|
746
|
+
"to": "ref('player', v=1)",
|
|
747
|
+
"field": "player_id",
|
|
748
|
+
}
|
|
749
|
+
}
|
|
750
|
+
}
|
|
751
|
+
],
|
|
752
|
+
}
|
|
753
|
+
],
|
|
754
|
+
}
|
|
755
|
+
],
|
|
756
|
+
}
|
|
757
|
+
|
|
758
|
+
models_dir = os.path.join(temp_dir, "models", "3_core")
|
|
759
|
+
os.makedirs(models_dir, exist_ok=True)
|
|
760
|
+
with open(os.path.join(models_dir, "game_stats.yml"), "w") as f:
|
|
761
|
+
yaml.dump(schema, f)
|
|
762
|
+
|
|
763
|
+
response = test_client.get("/api/infer-relationships")
|
|
764
|
+
assert response.status_code == 200
|
|
765
|
+
|
|
766
|
+
rels = response.json()["relationships"]
|
|
767
|
+
assert len(rels) == 1
|
|
768
|
+
rel = rels[0]
|
|
769
|
+
assert rel["source"] == "player"
|
|
770
|
+
assert rel["target"] == "game_stats"
|
|
771
|
+
assert rel["source_field"] == "player_id"
|
|
772
|
+
assert rel["target_field"] == "player_id"
|
|
773
|
+
|
|
774
|
+
|
|
775
|
+
class TestModelSchemaVersionHandling:
|
|
776
|
+
"""Ensure schema read/write honors requested dbt model version."""
|
|
777
|
+
|
|
778
|
+
def _write_versioned_manifest(self, temp_dir: str):
|
|
779
|
+
manifest_data = {
|
|
780
|
+
"nodes": {
|
|
781
|
+
"model.project.player.v1": {
|
|
782
|
+
"unique_id": "model.project.player.v1",
|
|
783
|
+
"resource_type": "model",
|
|
784
|
+
"name": "player",
|
|
785
|
+
"version": 1,
|
|
786
|
+
"schema": "public",
|
|
787
|
+
"alias": "player",
|
|
788
|
+
"original_file_path": "models/3_core/all/player_v1.sql",
|
|
789
|
+
"columns": {},
|
|
790
|
+
"description": "Player v1",
|
|
791
|
+
"config": {"materialized": "table"},
|
|
792
|
+
"tags": [],
|
|
793
|
+
},
|
|
794
|
+
"model.project.player.v2": {
|
|
795
|
+
"unique_id": "model.project.player.v2",
|
|
796
|
+
"resource_type": "model",
|
|
797
|
+
"name": "player",
|
|
798
|
+
"version": 2,
|
|
799
|
+
"schema": "public",
|
|
800
|
+
"alias": "player",
|
|
801
|
+
"original_file_path": "models/3_core/all/player_v2.sql",
|
|
802
|
+
"columns": {},
|
|
803
|
+
"description": "Player v2",
|
|
804
|
+
"config": {"materialized": "table"},
|
|
805
|
+
"tags": [],
|
|
806
|
+
},
|
|
807
|
+
}
|
|
808
|
+
}
|
|
809
|
+
|
|
810
|
+
manifest_path = os.path.join(temp_dir, "manifest.json")
|
|
811
|
+
with open(manifest_path, "w") as f:
|
|
812
|
+
json.dump(manifest_data, f)
|
|
813
|
+
|
|
814
|
+
def _write_versioned_schema(self, temp_dir: str) -> str:
|
|
815
|
+
models_dir = os.path.join(temp_dir, "models", "3_core", "all")
|
|
816
|
+
os.makedirs(models_dir, exist_ok=True)
|
|
817
|
+
yml_path = os.path.join(models_dir, "player.yml")
|
|
818
|
+
|
|
819
|
+
existing_schema = {
|
|
820
|
+
"version": 2,
|
|
821
|
+
"models": [
|
|
822
|
+
{
|
|
823
|
+
"name": "player",
|
|
824
|
+
"latest_version": 2,
|
|
825
|
+
"versions": [
|
|
826
|
+
{
|
|
827
|
+
"v": 1,
|
|
828
|
+
"description": "v1 description",
|
|
829
|
+
"columns": [{"name": "player_id", "data_type": "text"}],
|
|
830
|
+
},
|
|
831
|
+
{
|
|
832
|
+
"v": 2,
|
|
833
|
+
"description": "v2 description",
|
|
834
|
+
"columns": [{"name": "player_uuid", "data_type": "text"}],
|
|
835
|
+
},
|
|
836
|
+
],
|
|
837
|
+
}
|
|
838
|
+
],
|
|
839
|
+
}
|
|
840
|
+
|
|
841
|
+
with open(yml_path, "w") as f:
|
|
842
|
+
yaml.dump(existing_schema, f)
|
|
843
|
+
|
|
844
|
+
return yml_path
|
|
845
|
+
|
|
846
|
+
def test_get_model_schema_uses_requested_version(self, test_client, temp_dir):
|
|
847
|
+
self._write_versioned_manifest(temp_dir)
|
|
848
|
+
self._write_versioned_schema(temp_dir)
|
|
849
|
+
|
|
850
|
+
response = test_client.get(
|
|
851
|
+
"/api/models/player/schema", params={"version": 2}
|
|
852
|
+
)
|
|
853
|
+
assert response.status_code == 200
|
|
854
|
+
|
|
855
|
+
schema = response.json()
|
|
856
|
+
col_names = [col["name"] for col in schema["columns"]]
|
|
857
|
+
assert "player_uuid" in col_names
|
|
858
|
+
assert "player_id" not in col_names
|
|
859
|
+
assert schema["description"] == "v2 description"
|
|
860
|
+
|
|
861
|
+
def test_save_model_schema_targets_requested_version(self, test_client, temp_dir):
|
|
862
|
+
self._write_versioned_manifest(temp_dir)
|
|
863
|
+
yml_path = self._write_versioned_schema(temp_dir)
|
|
864
|
+
|
|
865
|
+
response = test_client.post(
|
|
866
|
+
"/api/models/player/schema",
|
|
867
|
+
json={
|
|
868
|
+
"columns": [
|
|
869
|
+
{
|
|
870
|
+
"name": "player_uuid",
|
|
871
|
+
"data_type": "text",
|
|
872
|
+
"description": "Updated PK",
|
|
873
|
+
}
|
|
874
|
+
],
|
|
875
|
+
"description": "Players v2 updated",
|
|
876
|
+
"tags": ["core"],
|
|
877
|
+
"version": 2,
|
|
878
|
+
},
|
|
879
|
+
)
|
|
880
|
+
assert response.status_code == 200
|
|
881
|
+
|
|
882
|
+
with open(yml_path, "r") as f:
|
|
883
|
+
updated = yaml.safe_load(f)
|
|
884
|
+
|
|
885
|
+
versions = {v["v"]: v for v in updated["models"][0]["versions"]}
|
|
886
|
+
assert versions[1]["columns"][0]["name"] == "player_id"
|
|
887
|
+
|
|
888
|
+
v2_cols = versions[2]["columns"]
|
|
889
|
+
assert v2_cols[0]["name"] == "player_uuid"
|
|
890
|
+
assert v2_cols[0]["description"] == "Updated PK"
|
|
891
|
+
assert versions[2].get("config", {}).get("tags") == ["core"]
|
|
892
|
+
assert updated["models"][0]["latest_version"] == 2
|