fast2dev-dbt-runner 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fast2dev_dbt_runner/__init__.py +47 -0
- fast2dev_dbt_runner/_compat.py +41 -0
- fast2dev_dbt_runner/airbyte_task_group_builder.py +259 -0
- fast2dev_dbt_runner/bash_operator/__init__.py +3 -0
- fast2dev_dbt_runner/bash_operator/datawarehouse_secrets.py +490 -0
- fast2dev_dbt_runner/bash_operator/dbt_hook.py +615 -0
- fast2dev_dbt_runner/bash_operator/dbt_operator.py +248 -0
- fast2dev_dbt_runner/cached_manifest_loader.py +275 -0
- fast2dev_dbt_runner/dbt_manifest_parser_api_operator.py +452 -0
- fast2dev_dbt_runner/dbt_manifest_parser_bash_operator.py +361 -0
- fast2dev_dbt_runner/dbt_manifest_parser_gke_operator.py +401 -0
- fast2dev_dbt_runner/dbt_manifest_parser_k8s_operator.py +448 -0
- fast2dev_dbt_runner/utils.py +451 -0
- fast2dev_dbt_runner-0.0.1.dist-info/METADATA +398 -0
- fast2dev_dbt_runner-0.0.1.dist-info/RECORD +18 -0
- fast2dev_dbt_runner-0.0.1.dist-info/WHEEL +5 -0
- fast2dev_dbt_runner-0.0.1.dist-info/licenses/LICENSE +21 -0
- fast2dev_dbt_runner-0.0.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,451 @@
|
|
|
1
|
+
import re
|
|
2
|
+
import json
|
|
3
|
+
import datetime
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def days_ago(n, hour=0, minute=0, second=0, microsecond=0):
|
|
7
|
+
"""Timezone-aware stand-in for Airflow 2's removed ``airflow.utils.dates.days_ago``.
|
|
8
|
+
|
|
9
|
+
Returns UTC midnight (or the given clock) ``n`` days before now. Never imports
|
|
10
|
+
``airflow.utils.dates`` — that module still exists in Airflow 3.2 but no longer
|
|
11
|
+
exports ``days_ago``, which raises ``ImportError: cannot import name 'days_ago'``.
|
|
12
|
+
"""
|
|
13
|
+
now = datetime.datetime.now(datetime.timezone.utc).replace(
|
|
14
|
+
hour=hour, minute=minute, second=second, microsecond=microsecond
|
|
15
|
+
)
|
|
16
|
+
return now - datetime.timedelta(days=int(n))
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def get_valid_start_date(start_date_raw):
|
|
20
|
+
"""
|
|
21
|
+
Tries to parse the START_DATE from Airflow Variables.
|
|
22
|
+
Supports:
|
|
23
|
+
- days_ago() function
|
|
24
|
+
- ISO format (YYYY-MM-DDTHH:MM:SS)
|
|
25
|
+
"""
|
|
26
|
+
# Check if start_date_raw follows days_ago(N) pattern
|
|
27
|
+
if re.fullmatch(r"days_ago\(\d+\)", start_date_raw):
|
|
28
|
+
days_value = int(start_date_raw[9:-1]) # Extract the number from days_ago(N)
|
|
29
|
+
return days_ago(days_value)
|
|
30
|
+
|
|
31
|
+
# Check if start_date_raw follows the correct ISO format
|
|
32
|
+
try:
|
|
33
|
+
parsed = datetime.datetime.fromisoformat(start_date_raw)
|
|
34
|
+
except ValueError:
|
|
35
|
+
raise ValueError(
|
|
36
|
+
f"Invalid start_date format: {start_date_raw}. Must be ISO format (YYYY-MM-DDTHH:MM:SS) or 'days_ago(N)'.")
|
|
37
|
+
# date-only ISO ("2026-10-04") can yield datetime.date on some Python builds.
|
|
38
|
+
# DAG.start_date accepts datetime; DAG.params must never receive a date object.
|
|
39
|
+
if isinstance(parsed, datetime.date) and not isinstance(parsed, datetime.datetime):
|
|
40
|
+
parsed = datetime.datetime(parsed.year, parsed.month, parsed.day, tzinfo=datetime.timezone.utc)
|
|
41
|
+
elif parsed.tzinfo is None:
|
|
42
|
+
parsed = parsed.replace(tzinfo=datetime.timezone.utc)
|
|
43
|
+
return parsed
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def json_safe_param(value):
|
|
47
|
+
"""Coerce date/datetime values to ISO strings for Airflow 3 DAG.params.
|
|
48
|
+
|
|
49
|
+
Airflow 3 Param defaults must be JSON-serializable. ``datetime.date``
|
|
50
|
+
(including ``date.today()``) fails with ParamValidationError even though
|
|
51
|
+
its ``str()`` looks like ``'2026-10-04'``. Use this (or ``.isoformat()``)
|
|
52
|
+
for params; ``dag.start_date`` may stay a datetime.
|
|
53
|
+
"""
|
|
54
|
+
if isinstance(value, datetime.datetime):
|
|
55
|
+
return value.date().isoformat()
|
|
56
|
+
if isinstance(value, datetime.date):
|
|
57
|
+
return value.isoformat()
|
|
58
|
+
return value
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
# CHANGE INPUT PARAM EVERYWHERE
|
|
62
|
+
def is_resource_type_in_manifest(manifest_data, resource_type):
|
|
63
|
+
if any(resource_type in sub_dict.values() for sub_dict in manifest_data.values()):
|
|
64
|
+
return 1
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def check_dbt_tag(dbt_tag):
|
|
68
|
+
"""
|
|
69
|
+
Cover next cases: ->>>> ADD TO DOC AND TO TEMPLATES
|
|
70
|
+
None->[]
|
|
71
|
+
""->[]
|
|
72
|
+
"tag"->["tag"]
|
|
73
|
+
["tag1", ""]->["tag1"]
|
|
74
|
+
["", " "]->[]
|
|
75
|
+
"""
|
|
76
|
+
if isinstance(dbt_tag, str):
|
|
77
|
+
dbt_tag = [dbt_tag.strip()] if dbt_tag.strip() else []
|
|
78
|
+
elif isinstance(dbt_tag, list):
|
|
79
|
+
dbt_tag = [tag.strip() for tag in dbt_tag if tag.strip()]
|
|
80
|
+
else:
|
|
81
|
+
dbt_tag = []
|
|
82
|
+
return dbt_tag
|
|
83
|
+
|
|
84
|
+
def to_bool(value):
|
|
85
|
+
"""Convert string value to boolean."""
|
|
86
|
+
if isinstance(value, bool):
|
|
87
|
+
return value
|
|
88
|
+
if not isinstance(value, str):
|
|
89
|
+
raise ValueError(f"Expected string or bool, got {type(value)}")
|
|
90
|
+
|
|
91
|
+
true_vals = {"true", "1", "yes", "y", "on"}
|
|
92
|
+
false_vals = {"false", "0", "no", "n", "off"}
|
|
93
|
+
|
|
94
|
+
val = value.strip().lower()
|
|
95
|
+
if val in true_vals:
|
|
96
|
+
return True
|
|
97
|
+
elif val in false_vals:
|
|
98
|
+
return False
|
|
99
|
+
else:
|
|
100
|
+
raise ValueError(f"Cannot convert string to bool: {value}")
|
|
101
|
+
|
|
102
|
+
# CHANGE INPUT PARAM EVERYWHERE
|
|
103
|
+
def filter_models(manifest_data, models_list):
|
|
104
|
+
filtered_dict = {}
|
|
105
|
+
for i in manifest_data.keys():
|
|
106
|
+
if manifest_data[i]['name'] in models_list:
|
|
107
|
+
filtered_dict[i] = manifest_data[i]
|
|
108
|
+
if manifest_data[i]['depends_on'] and manifest_data[i]['resource_type'] == 'test':
|
|
109
|
+
for depends_on_model in manifest_data[i]['depends_on']:
|
|
110
|
+
if depends_on_model.split(".")[-1] in models_list:
|
|
111
|
+
filtered_dict[i] = manifest_data[i]
|
|
112
|
+
return filtered_dict
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def change_to_test_in_models_depends_on(nodes_list):
|
|
116
|
+
models_tests_dict = {}
|
|
117
|
+
for k, v in nodes_list.items():
|
|
118
|
+
if v["resource_type"] == 'test':
|
|
119
|
+
for depends_on_model in v['depends_on']:
|
|
120
|
+
models_tests_dict.setdefault(depends_on_model, []).append(k)
|
|
121
|
+
|
|
122
|
+
for k, v in nodes_list.items():
|
|
123
|
+
temp_models_list = []
|
|
124
|
+
if v["resource_type"] != 'test':
|
|
125
|
+
for i in v['depends_on']:
|
|
126
|
+
if i.split(".")[1] != "re_data":
|
|
127
|
+
if i not in temp_models_list:
|
|
128
|
+
if i in models_tests_dict:
|
|
129
|
+
temp_models_list.extend(models_tests_dict[i])
|
|
130
|
+
else:
|
|
131
|
+
temp_models_list.append(i)
|
|
132
|
+
if temp_models_list:
|
|
133
|
+
v['depends_on'] = list(set(temp_models_list))
|
|
134
|
+
return nodes_list
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def weave_snapshots_into_model_group(nodes_list):
|
|
138
|
+
"""
|
|
139
|
+
Re-group snapshots (and their tests) that a selected model depends on so they
|
|
140
|
+
run interleaved inside the 'model' task group instead of the separate snapshot
|
|
141
|
+
phase.
|
|
142
|
+
|
|
143
|
+
This mirrors how tests are already interleaved: grouping is driven by
|
|
144
|
+
``group_type`` rather than ``resource_type``. A snapshot is considered
|
|
145
|
+
"model-blocking" when some selected model has the snapshot's node id in its
|
|
146
|
+
raw ``depends_on``. For each such snapshot we add ``"model"`` to its
|
|
147
|
+
``group_type`` (and drop ``"snapshot"`` so it is not also built in the
|
|
148
|
+
snapshot phase). Tests that test those snapshots are moved too, so the
|
|
149
|
+
``snapshot -> its tests -> downstream model`` ordering is preserved (matching
|
|
150
|
+
``dbt build`` semantics).
|
|
151
|
+
|
|
152
|
+
NOTE: must run on the raw ``depends_on`` (before
|
|
153
|
+
``change_to_test_in_models_depends_on`` rewrites model deps to point at tests),
|
|
154
|
+
otherwise a snapshot that has tests would no longer be visible as a direct
|
|
155
|
+
model dependency.
|
|
156
|
+
"""
|
|
157
|
+
snapshot_ids = {k for k, v in nodes_list.items() if v["resource_type"] == "snapshot"}
|
|
158
|
+
if not snapshot_ids:
|
|
159
|
+
return nodes_list
|
|
160
|
+
|
|
161
|
+
blocking = set()
|
|
162
|
+
for v in nodes_list.values():
|
|
163
|
+
if v["resource_type"] == "model":
|
|
164
|
+
blocking.update(dep for dep in v["depends_on"] if dep in snapshot_ids)
|
|
165
|
+
if not blocking:
|
|
166
|
+
return nodes_list
|
|
167
|
+
|
|
168
|
+
test_ids = {
|
|
169
|
+
k for k, v in nodes_list.items()
|
|
170
|
+
if v["resource_type"] == "test" and any(dep in blocking for dep in v["depends_on"])
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
for nid in blocking | test_ids:
|
|
174
|
+
group_type = nodes_list[nid]["group_type"]
|
|
175
|
+
if "model" not in group_type:
|
|
176
|
+
group_type.append("model")
|
|
177
|
+
if "snapshot" in group_type:
|
|
178
|
+
group_type.remove("snapshot")
|
|
179
|
+
return nodes_list
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def manifest_has_model_depends_on_snapshot(nodes_list):
|
|
183
|
+
"""
|
|
184
|
+
Return True when some selected model directly depends on a snapshot.
|
|
185
|
+
|
|
186
|
+
Used to decide whether the batch (non-sharded) path must switch from the
|
|
187
|
+
default ``dbt run`` + ``dbt snapshot`` phases to a single ``dbt build`` so the
|
|
188
|
+
``model -> snapshot -> model`` ordering is honoured. Detection is on
|
|
189
|
+
``resource_type`` (unchanged by weaving) so it works whether or not the
|
|
190
|
+
snapshot has been woven into the model group.
|
|
191
|
+
"""
|
|
192
|
+
snapshot_ids = {k for k, v in nodes_list.items() if v.get("resource_type") == "snapshot"}
|
|
193
|
+
if not snapshot_ids:
|
|
194
|
+
return False
|
|
195
|
+
for v in nodes_list.values():
|
|
196
|
+
if v.get("resource_type") == "model":
|
|
197
|
+
if any(dep in snapshot_ids for dep in v.get("depends_on", [])):
|
|
198
|
+
return True
|
|
199
|
+
return False
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def get_file_tests(nodes_list):
|
|
203
|
+
for k, v in nodes_list.items():
|
|
204
|
+
if v["resource_type"] == 'test':
|
|
205
|
+
for i in v.get('depends_on'):
|
|
206
|
+
if v.get('file_key_name'):
|
|
207
|
+
if v.get('file_key_name') not in i.split("."):
|
|
208
|
+
v['depends_on'].remove(i)
|
|
209
|
+
v['group_type'].remove(i.split(".")[0])
|
|
210
|
+
return nodes_list
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def check_node_cycle_for_tests(nodes_list):
|
|
214
|
+
tmp_list = []
|
|
215
|
+
new_depends_on_list = []
|
|
216
|
+
for k, v in nodes_list.items():
|
|
217
|
+
if v["resource_type"] == 'test':
|
|
218
|
+
|
|
219
|
+
has_model = any(node.startswith("model") for node in v["depends_on"])
|
|
220
|
+
has_source = any(node.startswith("source") for node in v["depends_on"])
|
|
221
|
+
if has_model and has_source:
|
|
222
|
+
v['depends_on'] = [node for node in v['depends_on'] if not node.startswith("source")]
|
|
223
|
+
|
|
224
|
+
for node in v["depends_on"]:
|
|
225
|
+
for parent_node_k, parent_node_v in nodes_list.items():
|
|
226
|
+
if parent_node_k == node:
|
|
227
|
+
for item in parent_node_v["depends_on"]:
|
|
228
|
+
if item not in tmp_list:
|
|
229
|
+
tmp_list.append(item)
|
|
230
|
+
|
|
231
|
+
for node in v["depends_on"]:
|
|
232
|
+
if node not in tmp_list:
|
|
233
|
+
new_depends_on_list.append(node)
|
|
234
|
+
v["depends_on"] = new_depends_on_list
|
|
235
|
+
new_depends_on_list = []
|
|
236
|
+
tmp_list = []
|
|
237
|
+
return nodes_list
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
# NEED TO BE BEFORE change_to_test_in_models_depends_on
|
|
241
|
+
def remove_ephemeral_dependencies(manifest_data_transformed):
|
|
242
|
+
# Identify ephemeral models
|
|
243
|
+
ephemeral_models = {
|
|
244
|
+
node_id for node_id, node in manifest_data_transformed.items()
|
|
245
|
+
if node.get("resource_type") == "model" and node.get("materialized", {}) == "ephemeral"
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
# Build a lookup for ephemeral -> its dependencies (parents)
|
|
249
|
+
ephemeral_to_parents = {
|
|
250
|
+
node_id: node.get("depends_on", {})
|
|
251
|
+
for node_id, node in manifest_data_transformed.items()
|
|
252
|
+
if node_id in ephemeral_models
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
# Remove ephemeral models tests
|
|
256
|
+
tests_to_remove = {
|
|
257
|
+
node_id for node_id, node in manifest_data_transformed.items()
|
|
258
|
+
if node.get("resource_type") == "test" and any(
|
|
259
|
+
dep in ephemeral_models for dep in manifest_data_transformed[node_id].get("depends_on"))
|
|
260
|
+
|
|
261
|
+
}
|
|
262
|
+
# Replace ephemeral dependencies
|
|
263
|
+
for node_id, node in manifest_data_transformed.items():
|
|
264
|
+
new_deps = []
|
|
265
|
+
for dep in node["depends_on"]:
|
|
266
|
+
if dep in ephemeral_models:
|
|
267
|
+
# Replace with ephemeral's parents (if they exist)
|
|
268
|
+
lifted = ephemeral_to_parents.get(dep, [])
|
|
269
|
+
if lifted:
|
|
270
|
+
new_deps.extend(lifted)
|
|
271
|
+
else:
|
|
272
|
+
new_deps.append(dep)
|
|
273
|
+
# Deduplicate dependencies
|
|
274
|
+
node["depends_on"] = list(set(new_deps))
|
|
275
|
+
|
|
276
|
+
# Delete ephemeral models and their tests
|
|
277
|
+
nodes_to_remove = ephemeral_models.union(tests_to_remove)
|
|
278
|
+
cleaned_manifest = {
|
|
279
|
+
node_id: node for node_id, node in manifest_data_transformed.items()
|
|
280
|
+
if node_id not in nodes_to_remove
|
|
281
|
+
}
|
|
282
|
+
return cleaned_manifest
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def filter_tasks_by_tag(manifest_data, dbt_tag):
|
|
286
|
+
return {k: v for k, v in manifest_data.items() if any(tag in v["tags"] for tag in dbt_tag)}
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def get_tests_for_tagged_models(manifest_data, filtered_manifest):
|
|
290
|
+
dependent_node_list = {}
|
|
291
|
+
for node_id, node in manifest_data.items():
|
|
292
|
+
if node.get('resource_type') == 'test':
|
|
293
|
+
for dependent_node in node.get('depends_on'):
|
|
294
|
+
for filtered_item in filtered_manifest.keys():
|
|
295
|
+
if filtered_item == dependent_node:
|
|
296
|
+
if node_id not in dependent_node_list.keys():
|
|
297
|
+
dependent_node_list[node_id] = node
|
|
298
|
+
result = filtered_manifest | dependent_node_list
|
|
299
|
+
return result
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
def collect_relatives(manifest_data, model_id, relatives_map, visited):
|
|
303
|
+
# Recursively collect all relatives
|
|
304
|
+
for item in relatives_map.get(model_id, []):
|
|
305
|
+
if item not in visited and item in manifest_data:
|
|
306
|
+
visited.add(item)
|
|
307
|
+
collect_relatives(manifest_data, item, relatives_map, visited)
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
def get_tagged_models(manifest_data, relatives_map, dbt_tag):
|
|
311
|
+
# Find models with the given tag
|
|
312
|
+
filtered_tasks = filter_tasks_by_tag(manifest_data, dbt_tag)
|
|
313
|
+
tagged_models = list(filtered_tasks.keys())
|
|
314
|
+
result = set(tagged_models)
|
|
315
|
+
for model in tagged_models:
|
|
316
|
+
collect_relatives(manifest_data, model, relatives_map, result)
|
|
317
|
+
return {node_id: manifest_data[node_id] for node_id in result}
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
def get_models_with_tag_and_parents(manifest_data, dbt_tag):
|
|
321
|
+
# Take parents
|
|
322
|
+
parent_map = {
|
|
323
|
+
node_id: node.get("depends_on", {})
|
|
324
|
+
for node_id, node in manifest_data.items()
|
|
325
|
+
}
|
|
326
|
+
result = get_tagged_models(manifest_data, parent_map, dbt_tag)
|
|
327
|
+
result_with_test = get_tests_for_tagged_models(manifest_data, result)
|
|
328
|
+
return result_with_test
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
def get_models_with_tag_and_children(manifest_data, dbt_tag):
|
|
332
|
+
# Build a child map: for each model, who depends on it
|
|
333
|
+
child_map = {}
|
|
334
|
+
for node_id, node in manifest_data.items():
|
|
335
|
+
for dep in node.get("depends_on", {}):
|
|
336
|
+
if dep not in child_map:
|
|
337
|
+
child_map[dep] = []
|
|
338
|
+
child_map[dep].append(node_id)
|
|
339
|
+
result = get_tagged_models(manifest_data, child_map, dbt_tag)
|
|
340
|
+
result_with_test = get_tests_for_tagged_models(manifest_data, result)
|
|
341
|
+
return result_with_test
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
def get_models_with_tag(manifest_data, dbt_tag):
|
|
345
|
+
filtered_tasks = filter_tasks_by_tag(manifest_data, dbt_tag)
|
|
346
|
+
result_with_test = get_tests_for_tagged_models(manifest_data, filtered_tasks)
|
|
347
|
+
return result_with_test
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
def change_macros_dependencies_to_source_dependencies(current_node, source_dict):
|
|
351
|
+
depends_list = []
|
|
352
|
+
depends_list_full_path = []
|
|
353
|
+
if current_node['config'].get('depends_on'):
|
|
354
|
+
depends_list = depends_list + current_node['config'].get('depends_on')
|
|
355
|
+
|
|
356
|
+
if current_node["depends_on"].get("macros"):
|
|
357
|
+
for i in current_node["depends_on"].get("macros"):
|
|
358
|
+
if i and i.split('.')[-1] == "generate_columns_from_airbyte_yml":
|
|
359
|
+
source_table_name = current_node["name"].replace("stg", "raw")
|
|
360
|
+
depends_list.append(source_table_name)
|
|
361
|
+
depends_list = list(set(depends_list))
|
|
362
|
+
for key in depends_list:
|
|
363
|
+
for k, v in source_dict.items():
|
|
364
|
+
if key in v['name']:
|
|
365
|
+
depends_list_full_path.append(k)
|
|
366
|
+
if current_node["depends_on"].get("nodes"):
|
|
367
|
+
depends_list_full_path = list(set(depends_list_full_path + current_node["depends_on"].get("nodes")))
|
|
368
|
+
return depends_list_full_path
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
def load_dbt_manifest(manifest_path,
|
|
372
|
+
dbt_tag=[],
|
|
373
|
+
dbt_tag_ancestors=False,
|
|
374
|
+
dbt_tag_descendants=False,
|
|
375
|
+
dbt_model_depends_on_snapshot=False):
|
|
376
|
+
"""
|
|
377
|
+
Helper function to load the dbt manifest file.
|
|
378
|
+
Returns: A JSON object containing the dbt manifest content.
|
|
379
|
+
"""
|
|
380
|
+
with open(manifest_path, encoding="utf-8") as file:
|
|
381
|
+
file_content = json.load(file)
|
|
382
|
+
|
|
383
|
+
node_dependency = {
|
|
384
|
+
k: v
|
|
385
|
+
for k, v in file_content["nodes"].items()
|
|
386
|
+
if k.split(".")[0] in ["model", "seed", "snapshot", "test", "source"]}
|
|
387
|
+
|
|
388
|
+
get_sources = {
|
|
389
|
+
k: v
|
|
390
|
+
for k, v in file_content["sources"].items()
|
|
391
|
+
if k.split(".")[0] in ["source"] and (v.get("freshness")
|
|
392
|
+
and ((v.get("freshness").get("warn_after")
|
|
393
|
+
and v.get("freshness").get("warn_after").get("count"))
|
|
394
|
+
or (v.get("freshness").get("error_after")
|
|
395
|
+
and v.get("freshness").get("error_after").get("count"))))
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
get_source_new_dict = {
|
|
399
|
+
k: {
|
|
400
|
+
"name": v["name"],
|
|
401
|
+
"alias": v["name"],
|
|
402
|
+
"package_name": v["package_name"],
|
|
403
|
+
"resource_type": v["resource_type"],
|
|
404
|
+
"schema": v["schema"],
|
|
405
|
+
"fqn": v["fqn"],
|
|
406
|
+
"group_type": ["source"],
|
|
407
|
+
"depends_on": [],
|
|
408
|
+
"tags": v["tags"],
|
|
409
|
+
"file_key_name": ""
|
|
410
|
+
}
|
|
411
|
+
for k, v in get_sources.items()}
|
|
412
|
+
|
|
413
|
+
node_dependency_unique = {
|
|
414
|
+
k: {
|
|
415
|
+
"name": v["name"],
|
|
416
|
+
"alias": v["alias"],
|
|
417
|
+
"package_name": v["package_name"],
|
|
418
|
+
"resource_type": v["resource_type"],
|
|
419
|
+
"materialized": v.get("config").get("materialized"),
|
|
420
|
+
"schema": v["schema"],
|
|
421
|
+
"fqn": v["fqn"],
|
|
422
|
+
"group_type": [v["resource_type"]] if v["resource_type"] != "test" else list(
|
|
423
|
+
set([i.split('.')[0] for i in v["depends_on"].get("nodes", [])])),
|
|
424
|
+
"depends_on": change_macros_dependencies_to_source_dependencies(v, get_source_new_dict),
|
|
425
|
+
"tags": v["tags"],
|
|
426
|
+
"file_key_name": v.get("file_key_name", "").split(".")[-1]
|
|
427
|
+
}
|
|
428
|
+
for k, v in node_dependency.items()
|
|
429
|
+
if 'depends_on' in v}
|
|
430
|
+
|
|
431
|
+
node_dependency_unique = check_node_cycle_for_tests(node_dependency_unique)
|
|
432
|
+
node_dependency_unique = {**node_dependency_unique, **get_source_new_dict}
|
|
433
|
+
if dbt_tag:
|
|
434
|
+
if to_bool(dbt_tag_ancestors) and not to_bool(dbt_tag_descendants):
|
|
435
|
+
node_dependency_unique_filtered = get_models_with_tag_and_parents(node_dependency_unique, dbt_tag)
|
|
436
|
+
if to_bool(dbt_tag_descendants) and not to_bool(dbt_tag_ancestors):
|
|
437
|
+
node_dependency_unique_filtered = get_models_with_tag_and_children(node_dependency_unique, dbt_tag)
|
|
438
|
+
if to_bool(dbt_tag_ancestors) and to_bool(dbt_tag_descendants):
|
|
439
|
+
node_dependency_unique_filtered = {**get_models_with_tag_and_parents(node_dependency_unique, dbt_tag), **get_models_with_tag_and_children(node_dependency_unique, dbt_tag)}
|
|
440
|
+
if not to_bool(dbt_tag_ancestors) and not to_bool(dbt_tag_descendants):
|
|
441
|
+
node_dependency_unique_filtered = get_models_with_tag(node_dependency_unique, dbt_tag)
|
|
442
|
+
else:
|
|
443
|
+
node_dependency_unique_filtered = node_dependency_unique
|
|
444
|
+
|
|
445
|
+
node_without_ephemeral = remove_ephemeral_dependencies(node_dependency_unique_filtered)
|
|
446
|
+
node_with_get_file = get_file_tests(node_without_ephemeral)
|
|
447
|
+
if to_bool(dbt_model_depends_on_snapshot):
|
|
448
|
+
node_with_get_file = weave_snapshots_into_model_group(node_with_get_file)
|
|
449
|
+
result = change_to_test_in_models_depends_on(node_with_get_file)
|
|
450
|
+
|
|
451
|
+
return result
|