fast2dev-dbt-runner 0.0.2__tar.gz → 0.0.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fast2dev_dbt_runner-0.0.2/fast2dev_dbt_runner.egg-info → fast2dev_dbt_runner-0.0.3}/PKG-INFO +1 -1
- {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/fast2dev_dbt_runner/_compat.py +6 -0
- {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/fast2dev_dbt_runner/bash_operator/dbt_operator.py +1 -1
- {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/fast2dev_dbt_runner/dbt_manifest_parser_api_operator.py +15 -11
- {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/fast2dev_dbt_runner/dbt_manifest_parser_bash_operator.py +34 -10
- {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/fast2dev_dbt_runner/dbt_manifest_parser_gke_operator.py +7 -3
- {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/fast2dev_dbt_runner/dbt_manifest_parser_k8s_operator.py +7 -3
- fast2dev_dbt_runner-0.0.3/fast2dev_dbt_runner/shared_task_guard.py +349 -0
- {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/fast2dev_dbt_runner/utils.py +67 -0
- {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3/fast2dev_dbt_runner.egg-info}/PKG-INFO +1 -1
- {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/fast2dev_dbt_runner.egg-info/SOURCES.txt +3 -0
- {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/pyproject.toml +1 -1
- {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/tests/test_manifest_parser_bash.py +12 -0
- fast2dev_dbt_runner-0.0.3/tests/test_manifest_parser_pod_api.py +97 -0
- fast2dev_dbt_runner-0.0.3/tests/test_shared_task_guard.py +274 -0
- {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/CHANGELOG.md +0 -0
- {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/LICENSE +0 -0
- {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/MANIFEST.in +0 -0
- {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/README.md +0 -0
- {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/docs/README.md +0 -0
- {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/docs/examples.md +0 -0
- {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/docs/installation.md +0 -0
- {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/docs/operators.md +0 -0
- {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/docs/publishing.md +0 -0
- {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/fast2dev_dbt_runner/__init__.py +0 -0
- {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/fast2dev_dbt_runner/airbyte_task_group_builder.py +0 -0
- {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/fast2dev_dbt_runner/bash_operator/__init__.py +0 -0
- {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/fast2dev_dbt_runner/bash_operator/datawarehouse_secrets.py +0 -0
- {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/fast2dev_dbt_runner/bash_operator/dbt_hook.py +0 -0
- {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/fast2dev_dbt_runner/cached_manifest_loader.py +0 -0
- {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/fast2dev_dbt_runner.egg-info/dependency_links.txt +0 -0
- {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/fast2dev_dbt_runner.egg-info/requires.txt +0 -0
- {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/fast2dev_dbt_runner.egg-info/top_level.txt +0 -0
- {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/requirements.txt +0 -0
- {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/setup.cfg +0 -0
- {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/setup.py +0 -0
- {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/tests/test_cached_manifest_loader.py +0 -0
- {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/tests/test_utils.py +0 -0
{fast2dev_dbt_runner-0.0.2/fast2dev_dbt_runner.egg-info → fast2dev_dbt_runner-0.0.3}/PKG-INFO
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: fast2dev-dbt-runner
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.3
|
|
4
4
|
Summary: Airflow 3 operators that turn a dbt 1.12 manifest into DAGs for Fast2.Dev (k8s, bash, API, and GKE).
|
|
5
5
|
Author-email: M7XLab <support@fast2.dev>
|
|
6
6
|
Maintainer-email: "Fast2.Dev" <support@fast2.dev>
|
|
@@ -29,6 +29,11 @@ try:
|
|
|
29
29
|
except ImportError:
|
|
30
30
|
from airflow.operators.python import BranchPythonOperator, PythonOperator
|
|
31
31
|
|
|
32
|
+
try:
|
|
33
|
+
from airflow.sdk import TriggerRule
|
|
34
|
+
except ImportError:
|
|
35
|
+
from airflow.utils.trigger_rule import TriggerRule
|
|
36
|
+
|
|
32
37
|
__all__ = [
|
|
33
38
|
"AirflowException",
|
|
34
39
|
"AirflowFailException",
|
|
@@ -38,4 +43,5 @@ __all__ = [
|
|
|
38
43
|
"EmptyOperator",
|
|
39
44
|
"PythonOperator",
|
|
40
45
|
"TaskGroup",
|
|
46
|
+
"TriggerRule",
|
|
41
47
|
]
|
|
@@ -188,7 +188,7 @@ class DbtReDataOperator(DbtBaseOperator):
|
|
|
188
188
|
super(DbtReDataOperator, self).__init__(dbt_project_dir=dbt_project_dir, profiles_dir=profiles_dir, target=target, *args, **kwargs)
|
|
189
189
|
|
|
190
190
|
def execute(self, context):
|
|
191
|
-
self.create_hook().run_cli('run --select package:re_data')
|
|
191
|
+
self.create_hook().run_cli('run --select package:re_data package:f2dq')
|
|
192
192
|
|
|
193
193
|
|
|
194
194
|
class DbtDocsGenerateOperator(DbtBaseOperator):
|
|
@@ -11,6 +11,7 @@ from time import sleep
|
|
|
11
11
|
from fast2dev_dbt_runner._compat import AirflowFailException, BaseHook, PythonOperator, TaskGroup
|
|
12
12
|
import fast2dev_dbt_runner.utils as utils
|
|
13
13
|
from fast2dev_dbt_runner.cached_manifest_loader import load_dbt_manifest_cached
|
|
14
|
+
from fast2dev_dbt_runner.shared_task_guard import build_shared_task_guard, guard_task_kwargs
|
|
14
15
|
|
|
15
16
|
|
|
16
17
|
|
|
@@ -51,6 +52,7 @@ class DbtManifestParser:
|
|
|
51
52
|
dbt_tag_descendants=self.dbt_tag_descendants,
|
|
52
53
|
dbt_model_depends_on_snapshot=self.dbt_model_depends_on_snapshot)
|
|
53
54
|
self.dbt_tasks = {}
|
|
55
|
+
self.shared_task_guard = build_shared_task_guard(airflow_vars, manifest_path)
|
|
54
56
|
self.connection_id = connection_id
|
|
55
57
|
self.dbt_log_format_file = "debug" if self.airflow_vars.get("MODEL_DEBUG_LOG") \
|
|
56
58
|
and self.airflow_vars.get("MODEL_DEBUG_LOG").lower() == "true" \
|
|
@@ -120,7 +122,7 @@ class DbtManifestParser:
|
|
|
120
122
|
full_command_list=None,
|
|
121
123
|
node_name=None,
|
|
122
124
|
node_alias=None,
|
|
123
|
-
parent_group=None
|
|
125
|
+
parent_group=None, shared_node=False
|
|
124
126
|
):
|
|
125
127
|
"""
|
|
126
128
|
Takes the manifest JSON content and returns a KubernetesPodOperator task
|
|
@@ -158,14 +160,14 @@ class DbtManifestParser:
|
|
|
158
160
|
self.dbt_log_format_file,
|
|
159
161
|
"run",
|
|
160
162
|
"--select",
|
|
161
|
-
"package:re_data"]
|
|
163
|
+
"package:re_data package:f2dq"]
|
|
162
164
|
elif dbt_command == "freshness":
|
|
163
165
|
full_command_list = ["--log-level-file",
|
|
164
166
|
self.dbt_log_format_file,
|
|
165
167
|
"source",
|
|
166
168
|
dbt_command,
|
|
167
169
|
"--exclude",
|
|
168
|
-
"package:re_data"]
|
|
170
|
+
"package:re_data package:f2dq"]
|
|
169
171
|
elif dbt_command == "snapshot":
|
|
170
172
|
full_command_list = ["--log-level-file",
|
|
171
173
|
self.dbt_log_format_file,
|
|
@@ -175,7 +177,7 @@ class DbtManifestParser:
|
|
|
175
177
|
self.dbt_log_format_file,
|
|
176
178
|
dbt_command,
|
|
177
179
|
"--exclude",
|
|
178
|
-
"package:re_data"]
|
|
180
|
+
"package:re_data package:f2dq"]
|
|
179
181
|
|
|
180
182
|
if dbt_command not in ("test", "freshness", "snapshot") and task_params.get('full_refresh') is True:
|
|
181
183
|
full_command_list.append("-f")
|
|
@@ -193,6 +195,7 @@ class DbtManifestParser:
|
|
|
193
195
|
print(f"node_name: {node_name}")
|
|
194
196
|
print(f"node_alias: {node_alias}")
|
|
195
197
|
|
|
198
|
+
running_rule, guard_kwargs = guard_task_kwargs(self.shared_task_guard, running_rule, shared_node)
|
|
196
199
|
dbt_task = PythonOperator(
|
|
197
200
|
task_id=node_alias,
|
|
198
201
|
python_callable=self.post_to_api_server,
|
|
@@ -202,6 +205,7 @@ class DbtManifestParser:
|
|
|
202
205
|
"command": full_command_list
|
|
203
206
|
},
|
|
204
207
|
trigger_rule=running_rule,
|
|
208
|
+
**guard_kwargs,
|
|
205
209
|
task_group=parent_group
|
|
206
210
|
)
|
|
207
211
|
return dbt_task
|
|
@@ -221,7 +225,7 @@ class DbtManifestParser:
|
|
|
221
225
|
full_command_list=full_command_with_model_name,
|
|
222
226
|
node_name=task_name,
|
|
223
227
|
node_alias=task_alias,
|
|
224
|
-
parent_group=parent_group)
|
|
228
|
+
parent_group=parent_group, shared_node=True)
|
|
225
229
|
return
|
|
226
230
|
|
|
227
231
|
# Get the current level and create a subgroup
|
|
@@ -302,7 +306,7 @@ class DbtManifestParser:
|
|
|
302
306
|
self.dbt_log_format_file,
|
|
303
307
|
"build",
|
|
304
308
|
"--exclude",
|
|
305
|
-
"package:re_data",
|
|
309
|
+
"package:re_data package:f2dq",
|
|
306
310
|
"--select",
|
|
307
311
|
select_string
|
|
308
312
|
]
|
|
@@ -331,12 +335,12 @@ class DbtManifestParser:
|
|
|
331
335
|
self.dbt_log_format_file,
|
|
332
336
|
dbt_command,
|
|
333
337
|
"--exclude",
|
|
334
|
-
"package:re_data",
|
|
338
|
+
"package:re_data package:f2dq",
|
|
335
339
|
"--select",
|
|
336
340
|
select_string
|
|
337
341
|
]
|
|
338
342
|
|
|
339
|
-
task_alias = f"{dbt_command}_all_models"
|
|
343
|
+
task_alias = "freshness_all_sources" if dbt_command == "source freshness" else f"{dbt_command}_all_models"
|
|
340
344
|
return self.create_api_task(
|
|
341
345
|
project_dir=project_dir,
|
|
342
346
|
dbt_command=dbt_command,
|
|
@@ -421,7 +425,7 @@ class DbtManifestParser:
|
|
|
421
425
|
self.dbt_log_format_file,
|
|
422
426
|
node_command,
|
|
423
427
|
"--exclude",
|
|
424
|
-
"package:re_data",
|
|
428
|
+
"package:re_data package:f2dq",
|
|
425
429
|
"--select",
|
|
426
430
|
task_name]
|
|
427
431
|
|
|
@@ -432,7 +436,7 @@ class DbtManifestParser:
|
|
|
432
436
|
"source",
|
|
433
437
|
node_command,
|
|
434
438
|
"--exclude",
|
|
435
|
-
"package:re_data"]
|
|
439
|
+
"package:re_data package:f2dq"]
|
|
436
440
|
elif node_command == "snapshot":
|
|
437
441
|
full_command_with_model_name = ["--log-level-file",
|
|
438
442
|
self.dbt_log_format_file,
|
|
@@ -442,7 +446,7 @@ class DbtManifestParser:
|
|
|
442
446
|
self.dbt_log_format_file,
|
|
443
447
|
node_command,
|
|
444
448
|
"--exclude",
|
|
445
|
-
"package:re_data"]
|
|
449
|
+
"package:re_data package:f2dq"]
|
|
446
450
|
# Create the dynamic task groups under root_group
|
|
447
451
|
self.create_task_groups(root_group, fqn, node_id, task_name, task_alias, node_command, running_rule,
|
|
448
452
|
task_params, full_command_with_model_name, project_dir)
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
import fast2dev_dbt_runner.utils as utils
|
|
3
3
|
from fast2dev_dbt_runner.cached_manifest_loader import load_dbt_manifest_cached
|
|
4
|
-
from fast2dev_dbt_runner.
|
|
4
|
+
from fast2dev_dbt_runner.shared_task_guard import build_shared_task_guard, guard_task_kwargs
|
|
5
|
+
from fast2dev_dbt_runner._compat import TaskGroup, TriggerRule
|
|
5
6
|
from fast2dev_dbt_runner.bash_operator.dbt_operator import (
|
|
6
7
|
DbtSeedOperator,
|
|
7
8
|
DbtSnapshotOperator,
|
|
@@ -40,6 +41,7 @@ class DbtManifestParser:
|
|
|
40
41
|
dbt_tag_ancestors=self.dbt_tag_ancestors,
|
|
41
42
|
dbt_tag_descendants=self.dbt_tag_descendants,
|
|
42
43
|
dbt_model_depends_on_snapshot=self.dbt_model_depends_on_snapshot)
|
|
44
|
+
self.shared_task_guard = build_shared_task_guard(airflow_vars, manifest_path)
|
|
43
45
|
self.dbt_tasks = {}
|
|
44
46
|
self.fqn_unique_list = []
|
|
45
47
|
self.existing_task_groups = {}
|
|
@@ -53,8 +55,10 @@ class DbtManifestParser:
|
|
|
53
55
|
return utils.get_valid_start_date(start_date_raw)
|
|
54
56
|
|
|
55
57
|
def create_dbt_bash_task(self, dbt_command, running_rule, task_params=None, node_name=None, node_alias=None,
|
|
56
|
-
parent_group=None):
|
|
58
|
+
parent_group=None, shared_node=False):
|
|
57
59
|
try:
|
|
60
|
+
running_rule, guard_kwargs = guard_task_kwargs(self.shared_task_guard, running_rule, shared_node)
|
|
61
|
+
|
|
58
62
|
if self.airflow_vars.get("TARGET"):
|
|
59
63
|
target = self.airflow_vars.get("TARGET")
|
|
60
64
|
else:
|
|
@@ -80,8 +84,10 @@ class DbtManifestParser:
|
|
|
80
84
|
git_branch=git_branch,
|
|
81
85
|
warehouse_type=warehouse_type,
|
|
82
86
|
task_group=parent_group,
|
|
87
|
+
trigger_rule=running_rule,
|
|
83
88
|
debug=self.debug,
|
|
84
|
-
empty=True if self.airflow_vars.get("E2E_MODE_EMPTY") else False
|
|
89
|
+
empty=True if self.airflow_vars.get("E2E_MODE_EMPTY") else False,
|
|
90
|
+
**guard_kwargs
|
|
85
91
|
)
|
|
86
92
|
elif dbt_command == 'build':
|
|
87
93
|
task_id = node_alias if node_alias else "build_all_models"
|
|
@@ -92,6 +98,7 @@ class DbtManifestParser:
|
|
|
92
98
|
target=target,
|
|
93
99
|
git_branch=git_branch,
|
|
94
100
|
warehouse_type=warehouse_type,
|
|
101
|
+
trigger_rule=running_rule,
|
|
95
102
|
task_group=parent_group,
|
|
96
103
|
debug=self.debug,
|
|
97
104
|
empty=True if self.airflow_vars.get("E2E_MODE_EMPTY") else False
|
|
@@ -104,8 +111,10 @@ class DbtManifestParser:
|
|
|
104
111
|
target=target,
|
|
105
112
|
git_branch=git_branch,
|
|
106
113
|
warehouse_type=warehouse_type,
|
|
114
|
+
trigger_rule=running_rule,
|
|
107
115
|
task_group=parent_group,
|
|
108
|
-
debug=self.debug
|
|
116
|
+
debug=self.debug,
|
|
117
|
+
**guard_kwargs
|
|
109
118
|
)
|
|
110
119
|
elif dbt_command == 'seed':
|
|
111
120
|
task_id = node_alias if node_name and node_alias else "seed_all_models"
|
|
@@ -118,8 +127,10 @@ class DbtManifestParser:
|
|
|
118
127
|
target=target,
|
|
119
128
|
git_branch=git_branch,
|
|
120
129
|
warehouse_type=warehouse_type,
|
|
130
|
+
trigger_rule=running_rule,
|
|
121
131
|
task_group=parent_group,
|
|
122
|
-
debug=self.debug
|
|
132
|
+
debug=self.debug,
|
|
133
|
+
**guard_kwargs
|
|
123
134
|
)
|
|
124
135
|
elif dbt_command == 'snapshot':
|
|
125
136
|
task_id = node_alias if node_name and node_alias else "snapshot_all_models"
|
|
@@ -133,7 +144,9 @@ class DbtManifestParser:
|
|
|
133
144
|
git_branch=git_branch,
|
|
134
145
|
warehouse_type=warehouse_type,
|
|
135
146
|
task_group=parent_group,
|
|
136
|
-
|
|
147
|
+
trigger_rule=running_rule,
|
|
148
|
+
debug=self.debug,
|
|
149
|
+
**guard_kwargs
|
|
137
150
|
)
|
|
138
151
|
elif dbt_command == 'source freshness':
|
|
139
152
|
task_id = node_alias if node_name and node_alias else "source_all_models"
|
|
@@ -145,7 +158,9 @@ class DbtManifestParser:
|
|
|
145
158
|
dbt_project_dir=self.dbt_project_dir,
|
|
146
159
|
warehouse_type=warehouse_type,
|
|
147
160
|
task_group=parent_group,
|
|
148
|
-
|
|
161
|
+
trigger_rule=running_rule,
|
|
162
|
+
debug=self.debug,
|
|
163
|
+
**guard_kwargs
|
|
149
164
|
)
|
|
150
165
|
elif dbt_command == 're_data':
|
|
151
166
|
dbt_run = DbtReDataOperator(
|
|
@@ -153,6 +168,7 @@ class DbtManifestParser:
|
|
|
153
168
|
dbt_project_dir=self.dbt_project_dir,
|
|
154
169
|
warehouse_type=warehouse_type,
|
|
155
170
|
task_group=parent_group,
|
|
171
|
+
trigger_rule=running_rule,
|
|
156
172
|
debug=self.debug
|
|
157
173
|
)
|
|
158
174
|
elif dbt_command == 'deps':
|
|
@@ -160,6 +176,7 @@ class DbtManifestParser:
|
|
|
160
176
|
task_id=node_alias,
|
|
161
177
|
dbt_project_dir=self.dbt_project_dir,
|
|
162
178
|
warehouse_type=warehouse_type,
|
|
179
|
+
trigger_rule=running_rule,
|
|
163
180
|
debug=self.debug
|
|
164
181
|
)
|
|
165
182
|
elif dbt_command == 'debug':
|
|
@@ -170,6 +187,7 @@ class DbtManifestParser:
|
|
|
170
187
|
git_branch=git_branch,
|
|
171
188
|
warehouse_type=warehouse_type,
|
|
172
189
|
task_group=parent_group,
|
|
190
|
+
trigger_rule=running_rule,
|
|
173
191
|
debug=self.debug
|
|
174
192
|
)
|
|
175
193
|
else:
|
|
@@ -195,7 +213,8 @@ class DbtManifestParser:
|
|
|
195
213
|
task_params=task_params,
|
|
196
214
|
node_name=task_name,
|
|
197
215
|
node_alias=task_alias,
|
|
198
|
-
parent_group=parent_group
|
|
216
|
+
parent_group=parent_group,
|
|
217
|
+
shared_node=True)
|
|
199
218
|
return
|
|
200
219
|
|
|
201
220
|
# Get the current level and create a subgroup
|
|
@@ -270,7 +289,7 @@ class DbtManifestParser:
|
|
|
270
289
|
if not select_string:
|
|
271
290
|
return None
|
|
272
291
|
|
|
273
|
-
task_alias = f"{dbt_command}_all_models"
|
|
292
|
+
task_alias = "freshness_all_sources" if dbt_command == "source freshness" else f"{dbt_command}_all_models"
|
|
274
293
|
return self.create_dbt_bash_task(
|
|
275
294
|
dbt_command=dbt_command,
|
|
276
295
|
running_rule=running_rule,
|
|
@@ -342,16 +361,21 @@ class DbtManifestParser:
|
|
|
342
361
|
# Use a per-node command so it does not leak across loop
|
|
343
362
|
# iterations (a model iterated after a test must stay 'run').
|
|
344
363
|
node_command = dbt_command
|
|
364
|
+
node_running_rule = running_rule
|
|
345
365
|
if node_data['resource_type'] == "test":
|
|
346
366
|
node_command = "test"
|
|
347
367
|
elif node_data['resource_type'] == "snapshot":
|
|
348
368
|
node_command = "snapshot"
|
|
369
|
+
# Snapshots woven into the models group (DBT_MODEL_DEPENDS_ON_SNAPSHOT)
|
|
370
|
+
# should run regardless of upstream success/failure.
|
|
371
|
+
if node_data.get("woven_into_models") or resource_type == "model":
|
|
372
|
+
node_running_rule = TriggerRule.ALL_DONE
|
|
349
373
|
elif node_data['resource_type'] == "source":
|
|
350
374
|
task_name = f"source:{node_data['schema']}.{task_name}"
|
|
351
375
|
node_command = "source freshness"
|
|
352
376
|
# Create the dynamic task groups under root_group
|
|
353
377
|
self.create_task_groups(root_group, fqn, node_id, task_name, task_alias, node_command,
|
|
354
|
-
|
|
378
|
+
node_running_rule,
|
|
355
379
|
task_params)
|
|
356
380
|
self.set_dependencies(resource_type)
|
|
357
381
|
# Guard against an empty group (e.g. all snapshots were woven into the
|
|
@@ -6,6 +6,7 @@ from fast2dev_dbt_runner._compat import AirflowException, TaskGroup
|
|
|
6
6
|
from airflow.providers.google.cloud.operators.kubernetes_engine import GKEStartPodOperator
|
|
7
7
|
import fast2dev_dbt_runner.utils as utils
|
|
8
8
|
from fast2dev_dbt_runner.cached_manifest_loader import load_dbt_manifest_cached
|
|
9
|
+
from fast2dev_dbt_runner.shared_task_guard import build_shared_task_guard, guard_task_kwargs
|
|
9
10
|
|
|
10
11
|
|
|
11
12
|
class CustomGKEStartPodOperator(GKEStartPodOperator):
|
|
@@ -77,6 +78,7 @@ class DbtManifestParser:
|
|
|
77
78
|
dbt_tag_descendants=self.dbt_tag_descendants,
|
|
78
79
|
dbt_model_depends_on_snapshot=self.dbt_model_depends_on_snapshot)
|
|
79
80
|
self.dbt_tasks = {}
|
|
81
|
+
self.shared_task_guard = build_shared_task_guard(airflow_vars, manifest_path)
|
|
80
82
|
self.fqn_unique_list = []
|
|
81
83
|
self.existing_task_groups = {}
|
|
82
84
|
|
|
@@ -142,7 +144,7 @@ class DbtManifestParser:
|
|
|
142
144
|
env_vars_with_model.append(k8s.V1EnvVar(name="SNAPSHOT", value="false"))
|
|
143
145
|
return env_vars_with_model
|
|
144
146
|
|
|
145
|
-
def create_dbt_task(self, dbt_command, running_rule, task_params={}, node_name=None, node_alias=None, parent_group=None):
|
|
147
|
+
def create_dbt_task(self, dbt_command, running_rule, task_params={}, node_name=None, node_alias=None, parent_group=None, shared_node=False):
|
|
146
148
|
"""
|
|
147
149
|
Takes the manifest JSON content and returns a GKEStartPodOperator task
|
|
148
150
|
to run a dbt command.
|
|
@@ -195,6 +197,7 @@ class DbtManifestParser:
|
|
|
195
197
|
}
|
|
196
198
|
}
|
|
197
199
|
|
|
200
|
+
running_rule, guard_kwargs = guard_task_kwargs(self.shared_task_guard, running_rule, shared_node)
|
|
198
201
|
dbt_task = CustomGKEStartPodOperator(
|
|
199
202
|
task_id=node_alias,
|
|
200
203
|
name=self.pod_name + "_" + node_alias,
|
|
@@ -211,6 +214,7 @@ class DbtManifestParser:
|
|
|
211
214
|
# Optional, run with specific K8S SA Account
|
|
212
215
|
# service_account_name="airflow",
|
|
213
216
|
trigger_rule=running_rule,
|
|
217
|
+
**guard_kwargs,
|
|
214
218
|
secrets=[],
|
|
215
219
|
labels={"app": "dbt"},
|
|
216
220
|
startup_timeout_seconds=240,
|
|
@@ -242,7 +246,7 @@ class DbtManifestParser:
|
|
|
242
246
|
task_params=task_params,
|
|
243
247
|
node_name=task_name,
|
|
244
248
|
node_alias=task_alias,
|
|
245
|
-
parent_group=parent_group)
|
|
249
|
+
parent_group=parent_group, shared_node=True)
|
|
246
250
|
return
|
|
247
251
|
|
|
248
252
|
# Get the current level and create a subgroup
|
|
@@ -317,7 +321,7 @@ class DbtManifestParser:
|
|
|
317
321
|
if not select_string:
|
|
318
322
|
return None
|
|
319
323
|
|
|
320
|
-
task_alias = f"{dbt_command}_all_models"
|
|
324
|
+
task_alias = "freshness_all_sources" if dbt_command == "source freshness" else f"{dbt_command}_all_models"
|
|
321
325
|
return self.create_dbt_task(
|
|
322
326
|
dbt_command=dbt_command,
|
|
323
327
|
running_rule=running_rule,
|
|
@@ -5,6 +5,7 @@ from kubernetes.client import models as k8s
|
|
|
5
5
|
from fast2dev_dbt_runner._compat import AirflowException, TaskGroup
|
|
6
6
|
import fast2dev_dbt_runner.utils as utils
|
|
7
7
|
from fast2dev_dbt_runner.cached_manifest_loader import load_dbt_manifest_cached
|
|
8
|
+
from fast2dev_dbt_runner.shared_task_guard import build_shared_task_guard, guard_task_kwargs
|
|
8
9
|
|
|
9
10
|
try:
|
|
10
11
|
# Airflow 2.10+
|
|
@@ -77,6 +78,7 @@ class DbtManifestParser:
|
|
|
77
78
|
dbt_tag_descendants=self.dbt_tag_descendants,
|
|
78
79
|
dbt_model_depends_on_snapshot=self.dbt_model_depends_on_snapshot)
|
|
79
80
|
self.dbt_tasks = {}
|
|
81
|
+
self.shared_task_guard = build_shared_task_guard(airflow_vars, manifest_path)
|
|
80
82
|
self.fqn_unique_list = []
|
|
81
83
|
self.existing_task_groups = {}
|
|
82
84
|
|
|
@@ -143,7 +145,7 @@ class DbtManifestParser:
|
|
|
143
145
|
return env_vars_with_model
|
|
144
146
|
|
|
145
147
|
def create_dbt_kuberoperator_task(self, dbt_command, running_rule, task_params={}, node_name=None, node_alias=None,
|
|
146
|
-
parent_group=None):
|
|
148
|
+
parent_group=None, shared_node=False):
|
|
147
149
|
"""
|
|
148
150
|
Takes the manifest JSON content and returns a KubernetesPodOperator task
|
|
149
151
|
to run a dbt command.
|
|
@@ -245,6 +247,7 @@ class DbtManifestParser:
|
|
|
245
247
|
volume_mounts.append(volume_mount_val)
|
|
246
248
|
|
|
247
249
|
# Create the pod operator with conditional volumes
|
|
250
|
+
running_rule, guard_kwargs = guard_task_kwargs(self.shared_task_guard, running_rule, shared_node)
|
|
248
251
|
dbt_task = CustomKubernetesPodOperator(
|
|
249
252
|
task_id=node_alias,
|
|
250
253
|
name=self.pod_name + "_" + node_alias,
|
|
@@ -257,6 +260,7 @@ class DbtManifestParser:
|
|
|
257
260
|
# Optional, run with specific K8S SA Account
|
|
258
261
|
service_account_name=self.airflow_vars.get("DATA_ORCHESTRATOR_WORKER_SA", "data-orchestration-worker"),
|
|
259
262
|
trigger_rule=running_rule,
|
|
263
|
+
**guard_kwargs,
|
|
260
264
|
labels={"app": "dbt"},
|
|
261
265
|
env_vars=env_vars_with_model,
|
|
262
266
|
image_pull_policy="IfNotPresent",
|
|
@@ -288,7 +292,7 @@ class DbtManifestParser:
|
|
|
288
292
|
task_params=task_params,
|
|
289
293
|
node_name=task_name,
|
|
290
294
|
node_alias=task_alias,
|
|
291
|
-
parent_group=parent_group)
|
|
295
|
+
parent_group=parent_group, shared_node=True)
|
|
292
296
|
return
|
|
293
297
|
|
|
294
298
|
# Get the current level and create a subgroup
|
|
@@ -363,7 +367,7 @@ class DbtManifestParser:
|
|
|
363
367
|
if not select_string:
|
|
364
368
|
return None
|
|
365
369
|
|
|
366
|
-
task_alias = f"{dbt_command}_all_models"
|
|
370
|
+
task_alias = "freshness_all_sources" if dbt_command == "source freshness" else f"{dbt_command}_all_models"
|
|
367
371
|
return self.create_dbt_kuberoperator_task(
|
|
368
372
|
dbt_command=dbt_command,
|
|
369
373
|
running_rule=running_rule,
|