fast2dev-dbt-runner 0.0.2__tar.gz → 0.0.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. {fast2dev_dbt_runner-0.0.2/fast2dev_dbt_runner.egg-info → fast2dev_dbt_runner-0.0.3}/PKG-INFO +1 -1
  2. {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/fast2dev_dbt_runner/_compat.py +6 -0
  3. {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/fast2dev_dbt_runner/bash_operator/dbt_operator.py +1 -1
  4. {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/fast2dev_dbt_runner/dbt_manifest_parser_api_operator.py +15 -11
  5. {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/fast2dev_dbt_runner/dbt_manifest_parser_bash_operator.py +34 -10
  6. {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/fast2dev_dbt_runner/dbt_manifest_parser_gke_operator.py +7 -3
  7. {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/fast2dev_dbt_runner/dbt_manifest_parser_k8s_operator.py +7 -3
  8. fast2dev_dbt_runner-0.0.3/fast2dev_dbt_runner/shared_task_guard.py +349 -0
  9. {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/fast2dev_dbt_runner/utils.py +67 -0
  10. {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3/fast2dev_dbt_runner.egg-info}/PKG-INFO +1 -1
  11. {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/fast2dev_dbt_runner.egg-info/SOURCES.txt +3 -0
  12. {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/pyproject.toml +1 -1
  13. {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/tests/test_manifest_parser_bash.py +12 -0
  14. fast2dev_dbt_runner-0.0.3/tests/test_manifest_parser_pod_api.py +97 -0
  15. fast2dev_dbt_runner-0.0.3/tests/test_shared_task_guard.py +274 -0
  16. {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/CHANGELOG.md +0 -0
  17. {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/LICENSE +0 -0
  18. {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/MANIFEST.in +0 -0
  19. {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/README.md +0 -0
  20. {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/docs/README.md +0 -0
  21. {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/docs/examples.md +0 -0
  22. {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/docs/installation.md +0 -0
  23. {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/docs/operators.md +0 -0
  24. {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/docs/publishing.md +0 -0
  25. {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/fast2dev_dbt_runner/__init__.py +0 -0
  26. {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/fast2dev_dbt_runner/airbyte_task_group_builder.py +0 -0
  27. {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/fast2dev_dbt_runner/bash_operator/__init__.py +0 -0
  28. {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/fast2dev_dbt_runner/bash_operator/datawarehouse_secrets.py +0 -0
  29. {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/fast2dev_dbt_runner/bash_operator/dbt_hook.py +0 -0
  30. {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/fast2dev_dbt_runner/cached_manifest_loader.py +0 -0
  31. {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/fast2dev_dbt_runner.egg-info/dependency_links.txt +0 -0
  32. {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/fast2dev_dbt_runner.egg-info/requires.txt +0 -0
  33. {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/fast2dev_dbt_runner.egg-info/top_level.txt +0 -0
  34. {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/requirements.txt +0 -0
  35. {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/setup.cfg +0 -0
  36. {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/setup.py +0 -0
  37. {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/tests/test_cached_manifest_loader.py +0 -0
  38. {fast2dev_dbt_runner-0.0.2 → fast2dev_dbt_runner-0.0.3}/tests/test_utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: fast2dev-dbt-runner
3
- Version: 0.0.2
3
+ Version: 0.0.3
4
4
  Summary: Airflow 3 operators that turn a dbt 1.12 manifest into DAGs for Fast2.Dev (k8s, bash, API, and GKE).
5
5
  Author-email: M7XLab <support@fast2.dev>
6
6
  Maintainer-email: "Fast2.Dev" <support@fast2.dev>
@@ -29,6 +29,11 @@ try:
29
29
  except ImportError:
30
30
  from airflow.operators.python import BranchPythonOperator, PythonOperator
31
31
 
32
+ try:
33
+ from airflow.sdk import TriggerRule
34
+ except ImportError:
35
+ from airflow.utils.trigger_rule import TriggerRule
36
+
32
37
  __all__ = [
33
38
  "AirflowException",
34
39
  "AirflowFailException",
@@ -38,4 +43,5 @@ __all__ = [
38
43
  "EmptyOperator",
39
44
  "PythonOperator",
40
45
  "TaskGroup",
46
+ "TriggerRule",
41
47
  ]
@@ -188,7 +188,7 @@ class DbtReDataOperator(DbtBaseOperator):
188
188
  super(DbtReDataOperator, self).__init__(dbt_project_dir=dbt_project_dir, profiles_dir=profiles_dir, target=target, *args, **kwargs)
189
189
 
190
190
  def execute(self, context):
191
- self.create_hook().run_cli('run --select package:re_data')
191
+ self.create_hook().run_cli('run --select package:re_data package:f2dq')
192
192
 
193
193
 
194
194
  class DbtDocsGenerateOperator(DbtBaseOperator):
@@ -11,6 +11,7 @@ from time import sleep
11
11
  from fast2dev_dbt_runner._compat import AirflowFailException, BaseHook, PythonOperator, TaskGroup
12
12
  import fast2dev_dbt_runner.utils as utils
13
13
  from fast2dev_dbt_runner.cached_manifest_loader import load_dbt_manifest_cached
14
+ from fast2dev_dbt_runner.shared_task_guard import build_shared_task_guard, guard_task_kwargs
14
15
 
15
16
 
16
17
 
@@ -51,6 +52,7 @@ class DbtManifestParser:
51
52
  dbt_tag_descendants=self.dbt_tag_descendants,
52
53
  dbt_model_depends_on_snapshot=self.dbt_model_depends_on_snapshot)
53
54
  self.dbt_tasks = {}
55
+ self.shared_task_guard = build_shared_task_guard(airflow_vars, manifest_path)
54
56
  self.connection_id = connection_id
55
57
  self.dbt_log_format_file = "debug" if self.airflow_vars.get("MODEL_DEBUG_LOG") \
56
58
  and self.airflow_vars.get("MODEL_DEBUG_LOG").lower() == "true" \
@@ -120,7 +122,7 @@ class DbtManifestParser:
120
122
  full_command_list=None,
121
123
  node_name=None,
122
124
  node_alias=None,
123
- parent_group=None
125
+ parent_group=None, shared_node=False
124
126
  ):
125
127
  """
126
128
  Takes the manifest JSON content and returns a KubernetesPodOperator task
@@ -158,14 +160,14 @@ class DbtManifestParser:
158
160
  self.dbt_log_format_file,
159
161
  "run",
160
162
  "--select",
161
- "package:re_data"]
163
+ "package:re_data package:f2dq"]
162
164
  elif dbt_command == "freshness":
163
165
  full_command_list = ["--log-level-file",
164
166
  self.dbt_log_format_file,
165
167
  "source",
166
168
  dbt_command,
167
169
  "--exclude",
168
- "package:re_data"]
170
+ "package:re_data package:f2dq"]
169
171
  elif dbt_command == "snapshot":
170
172
  full_command_list = ["--log-level-file",
171
173
  self.dbt_log_format_file,
@@ -175,7 +177,7 @@ class DbtManifestParser:
175
177
  self.dbt_log_format_file,
176
178
  dbt_command,
177
179
  "--exclude",
178
- "package:re_data"]
180
+ "package:re_data package:f2dq"]
179
181
 
180
182
  if dbt_command not in ("test", "freshness", "snapshot") and task_params.get('full_refresh') is True:
181
183
  full_command_list.append("-f")
@@ -193,6 +195,7 @@ class DbtManifestParser:
193
195
  print(f"node_name: {node_name}")
194
196
  print(f"node_alias: {node_alias}")
195
197
 
198
+ running_rule, guard_kwargs = guard_task_kwargs(self.shared_task_guard, running_rule, shared_node)
196
199
  dbt_task = PythonOperator(
197
200
  task_id=node_alias,
198
201
  python_callable=self.post_to_api_server,
@@ -202,6 +205,7 @@ class DbtManifestParser:
202
205
  "command": full_command_list
203
206
  },
204
207
  trigger_rule=running_rule,
208
+ **guard_kwargs,
205
209
  task_group=parent_group
206
210
  )
207
211
  return dbt_task
@@ -221,7 +225,7 @@ class DbtManifestParser:
221
225
  full_command_list=full_command_with_model_name,
222
226
  node_name=task_name,
223
227
  node_alias=task_alias,
224
- parent_group=parent_group)
228
+ parent_group=parent_group, shared_node=True)
225
229
  return
226
230
 
227
231
  # Get the current level and create a subgroup
@@ -302,7 +306,7 @@ class DbtManifestParser:
302
306
  self.dbt_log_format_file,
303
307
  "build",
304
308
  "--exclude",
305
- "package:re_data",
309
+ "package:re_data package:f2dq",
306
310
  "--select",
307
311
  select_string
308
312
  ]
@@ -331,12 +335,12 @@ class DbtManifestParser:
331
335
  self.dbt_log_format_file,
332
336
  dbt_command,
333
337
  "--exclude",
334
- "package:re_data",
338
+ "package:re_data package:f2dq",
335
339
  "--select",
336
340
  select_string
337
341
  ]
338
342
 
339
- task_alias = f"{dbt_command}_all_models"
343
+ task_alias = "freshness_all_sources" if dbt_command == "source freshness" else f"{dbt_command}_all_models"
340
344
  return self.create_api_task(
341
345
  project_dir=project_dir,
342
346
  dbt_command=dbt_command,
@@ -421,7 +425,7 @@ class DbtManifestParser:
421
425
  self.dbt_log_format_file,
422
426
  node_command,
423
427
  "--exclude",
424
- "package:re_data",
428
+ "package:re_data package:f2dq",
425
429
  "--select",
426
430
  task_name]
427
431
 
@@ -432,7 +436,7 @@ class DbtManifestParser:
432
436
  "source",
433
437
  node_command,
434
438
  "--exclude",
435
- "package:re_data"]
439
+ "package:re_data package:f2dq"]
436
440
  elif node_command == "snapshot":
437
441
  full_command_with_model_name = ["--log-level-file",
438
442
  self.dbt_log_format_file,
@@ -442,7 +446,7 @@ class DbtManifestParser:
442
446
  self.dbt_log_format_file,
443
447
  node_command,
444
448
  "--exclude",
445
- "package:re_data"]
449
+ "package:re_data package:f2dq"]
446
450
  # Create the dynamic task groups under root_group
447
451
  self.create_task_groups(root_group, fqn, node_id, task_name, task_alias, node_command, running_rule,
448
452
  task_params, full_command_with_model_name, project_dir)
@@ -1,7 +1,8 @@
1
1
  import logging
2
2
  import fast2dev_dbt_runner.utils as utils
3
3
  from fast2dev_dbt_runner.cached_manifest_loader import load_dbt_manifest_cached
4
- from fast2dev_dbt_runner._compat import TaskGroup
4
+ from fast2dev_dbt_runner.shared_task_guard import build_shared_task_guard, guard_task_kwargs
5
+ from fast2dev_dbt_runner._compat import TaskGroup, TriggerRule
5
6
  from fast2dev_dbt_runner.bash_operator.dbt_operator import (
6
7
  DbtSeedOperator,
7
8
  DbtSnapshotOperator,
@@ -40,6 +41,7 @@ class DbtManifestParser:
40
41
  dbt_tag_ancestors=self.dbt_tag_ancestors,
41
42
  dbt_tag_descendants=self.dbt_tag_descendants,
42
43
  dbt_model_depends_on_snapshot=self.dbt_model_depends_on_snapshot)
44
+ self.shared_task_guard = build_shared_task_guard(airflow_vars, manifest_path)
43
45
  self.dbt_tasks = {}
44
46
  self.fqn_unique_list = []
45
47
  self.existing_task_groups = {}
@@ -53,8 +55,10 @@ class DbtManifestParser:
53
55
  return utils.get_valid_start_date(start_date_raw)
54
56
 
55
57
  def create_dbt_bash_task(self, dbt_command, running_rule, task_params=None, node_name=None, node_alias=None,
56
- parent_group=None):
58
+ parent_group=None, shared_node=False):
57
59
  try:
60
+ running_rule, guard_kwargs = guard_task_kwargs(self.shared_task_guard, running_rule, shared_node)
61
+
58
62
  if self.airflow_vars.get("TARGET"):
59
63
  target = self.airflow_vars.get("TARGET")
60
64
  else:
@@ -80,8 +84,10 @@ class DbtManifestParser:
80
84
  git_branch=git_branch,
81
85
  warehouse_type=warehouse_type,
82
86
  task_group=parent_group,
87
+ trigger_rule=running_rule,
83
88
  debug=self.debug,
84
- empty=True if self.airflow_vars.get("E2E_MODE_EMPTY") else False
89
+ empty=True if self.airflow_vars.get("E2E_MODE_EMPTY") else False,
90
+ **guard_kwargs
85
91
  )
86
92
  elif dbt_command == 'build':
87
93
  task_id = node_alias if node_alias else "build_all_models"
@@ -92,6 +98,7 @@ class DbtManifestParser:
92
98
  target=target,
93
99
  git_branch=git_branch,
94
100
  warehouse_type=warehouse_type,
101
+ trigger_rule=running_rule,
95
102
  task_group=parent_group,
96
103
  debug=self.debug,
97
104
  empty=True if self.airflow_vars.get("E2E_MODE_EMPTY") else False
@@ -104,8 +111,10 @@ class DbtManifestParser:
104
111
  target=target,
105
112
  git_branch=git_branch,
106
113
  warehouse_type=warehouse_type,
114
+ trigger_rule=running_rule,
107
115
  task_group=parent_group,
108
- debug=self.debug
116
+ debug=self.debug,
117
+ **guard_kwargs
109
118
  )
110
119
  elif dbt_command == 'seed':
111
120
  task_id = node_alias if node_name and node_alias else "seed_all_models"
@@ -118,8 +127,10 @@ class DbtManifestParser:
118
127
  target=target,
119
128
  git_branch=git_branch,
120
129
  warehouse_type=warehouse_type,
130
+ trigger_rule=running_rule,
121
131
  task_group=parent_group,
122
- debug=self.debug
132
+ debug=self.debug,
133
+ **guard_kwargs
123
134
  )
124
135
  elif dbt_command == 'snapshot':
125
136
  task_id = node_alias if node_name and node_alias else "snapshot_all_models"
@@ -133,7 +144,9 @@ class DbtManifestParser:
133
144
  git_branch=git_branch,
134
145
  warehouse_type=warehouse_type,
135
146
  task_group=parent_group,
136
- debug=self.debug
147
+ trigger_rule=running_rule,
148
+ debug=self.debug,
149
+ **guard_kwargs
137
150
  )
138
151
  elif dbt_command == 'source freshness':
139
152
  task_id = node_alias if node_name and node_alias else "source_all_models"
@@ -145,7 +158,9 @@ class DbtManifestParser:
145
158
  dbt_project_dir=self.dbt_project_dir,
146
159
  warehouse_type=warehouse_type,
147
160
  task_group=parent_group,
148
- debug=self.debug
161
+ trigger_rule=running_rule,
162
+ debug=self.debug,
163
+ **guard_kwargs
149
164
  )
150
165
  elif dbt_command == 're_data':
151
166
  dbt_run = DbtReDataOperator(
@@ -153,6 +168,7 @@ class DbtManifestParser:
153
168
  dbt_project_dir=self.dbt_project_dir,
154
169
  warehouse_type=warehouse_type,
155
170
  task_group=parent_group,
171
+ trigger_rule=running_rule,
156
172
  debug=self.debug
157
173
  )
158
174
  elif dbt_command == 'deps':
@@ -160,6 +176,7 @@ class DbtManifestParser:
160
176
  task_id=node_alias,
161
177
  dbt_project_dir=self.dbt_project_dir,
162
178
  warehouse_type=warehouse_type,
179
+ trigger_rule=running_rule,
163
180
  debug=self.debug
164
181
  )
165
182
  elif dbt_command == 'debug':
@@ -170,6 +187,7 @@ class DbtManifestParser:
170
187
  git_branch=git_branch,
171
188
  warehouse_type=warehouse_type,
172
189
  task_group=parent_group,
190
+ trigger_rule=running_rule,
173
191
  debug=self.debug
174
192
  )
175
193
  else:
@@ -195,7 +213,8 @@ class DbtManifestParser:
195
213
  task_params=task_params,
196
214
  node_name=task_name,
197
215
  node_alias=task_alias,
198
- parent_group=parent_group)
216
+ parent_group=parent_group,
217
+ shared_node=True)
199
218
  return
200
219
 
201
220
  # Get the current level and create a subgroup
@@ -270,7 +289,7 @@ class DbtManifestParser:
270
289
  if not select_string:
271
290
  return None
272
291
 
273
- task_alias = f"{dbt_command}_all_models"
292
+ task_alias = "freshness_all_sources" if dbt_command == "source freshness" else f"{dbt_command}_all_models"
274
293
  return self.create_dbt_bash_task(
275
294
  dbt_command=dbt_command,
276
295
  running_rule=running_rule,
@@ -342,16 +361,21 @@ class DbtManifestParser:
342
361
  # Use a per-node command so it does not leak across loop
343
362
  # iterations (a model iterated after a test must stay 'run').
344
363
  node_command = dbt_command
364
+ node_running_rule = running_rule
345
365
  if node_data['resource_type'] == "test":
346
366
  node_command = "test"
347
367
  elif node_data['resource_type'] == "snapshot":
348
368
  node_command = "snapshot"
369
+ # Snapshots woven into the models group (DBT_MODEL_DEPENDS_ON_SNAPSHOT)
370
+ # should run regardless of upstream success/failure.
371
+ if node_data.get("woven_into_models") or resource_type == "model":
372
+ node_running_rule = TriggerRule.ALL_DONE
349
373
  elif node_data['resource_type'] == "source":
350
374
  task_name = f"source:{node_data['schema']}.{task_name}"
351
375
  node_command = "source freshness"
352
376
  # Create the dynamic task groups under root_group
353
377
  self.create_task_groups(root_group, fqn, node_id, task_name, task_alias, node_command,
354
- running_rule,
378
+ node_running_rule,
355
379
  task_params)
356
380
  self.set_dependencies(resource_type)
357
381
  # Guard against an empty group (e.g. all snapshots were woven into the
@@ -6,6 +6,7 @@ from fast2dev_dbt_runner._compat import AirflowException, TaskGroup
6
6
  from airflow.providers.google.cloud.operators.kubernetes_engine import GKEStartPodOperator
7
7
  import fast2dev_dbt_runner.utils as utils
8
8
  from fast2dev_dbt_runner.cached_manifest_loader import load_dbt_manifest_cached
9
+ from fast2dev_dbt_runner.shared_task_guard import build_shared_task_guard, guard_task_kwargs
9
10
 
10
11
 
11
12
  class CustomGKEStartPodOperator(GKEStartPodOperator):
@@ -77,6 +78,7 @@ class DbtManifestParser:
77
78
  dbt_tag_descendants=self.dbt_tag_descendants,
78
79
  dbt_model_depends_on_snapshot=self.dbt_model_depends_on_snapshot)
79
80
  self.dbt_tasks = {}
81
+ self.shared_task_guard = build_shared_task_guard(airflow_vars, manifest_path)
80
82
  self.fqn_unique_list = []
81
83
  self.existing_task_groups = {}
82
84
 
@@ -142,7 +144,7 @@ class DbtManifestParser:
142
144
  env_vars_with_model.append(k8s.V1EnvVar(name="SNAPSHOT", value="false"))
143
145
  return env_vars_with_model
144
146
 
145
- def create_dbt_task(self, dbt_command, running_rule, task_params={}, node_name=None, node_alias=None, parent_group=None):
147
+ def create_dbt_task(self, dbt_command, running_rule, task_params={}, node_name=None, node_alias=None, parent_group=None, shared_node=False):
146
148
  """
147
149
  Takes the manifest JSON content and returns a GKEStartPodOperator task
148
150
  to run a dbt command.
@@ -195,6 +197,7 @@ class DbtManifestParser:
195
197
  }
196
198
  }
197
199
 
200
+ running_rule, guard_kwargs = guard_task_kwargs(self.shared_task_guard, running_rule, shared_node)
198
201
  dbt_task = CustomGKEStartPodOperator(
199
202
  task_id=node_alias,
200
203
  name=self.pod_name + "_" + node_alias,
@@ -211,6 +214,7 @@ class DbtManifestParser:
211
214
  # Optional, run with specific K8S SA Account
212
215
  # service_account_name="airflow",
213
216
  trigger_rule=running_rule,
217
+ **guard_kwargs,
214
218
  secrets=[],
215
219
  labels={"app": "dbt"},
216
220
  startup_timeout_seconds=240,
@@ -242,7 +246,7 @@ class DbtManifestParser:
242
246
  task_params=task_params,
243
247
  node_name=task_name,
244
248
  node_alias=task_alias,
245
- parent_group=parent_group)
249
+ parent_group=parent_group, shared_node=True)
246
250
  return
247
251
 
248
252
  # Get the current level and create a subgroup
@@ -317,7 +321,7 @@ class DbtManifestParser:
317
321
  if not select_string:
318
322
  return None
319
323
 
320
- task_alias = f"{dbt_command}_all_models"
324
+ task_alias = "freshness_all_sources" if dbt_command == "source freshness" else f"{dbt_command}_all_models"
321
325
  return self.create_dbt_task(
322
326
  dbt_command=dbt_command,
323
327
  running_rule=running_rule,
@@ -5,6 +5,7 @@ from kubernetes.client import models as k8s
5
5
  from fast2dev_dbt_runner._compat import AirflowException, TaskGroup
6
6
  import fast2dev_dbt_runner.utils as utils
7
7
  from fast2dev_dbt_runner.cached_manifest_loader import load_dbt_manifest_cached
8
+ from fast2dev_dbt_runner.shared_task_guard import build_shared_task_guard, guard_task_kwargs
8
9
 
9
10
  try:
10
11
  # Airflow 2.10+
@@ -77,6 +78,7 @@ class DbtManifestParser:
77
78
  dbt_tag_descendants=self.dbt_tag_descendants,
78
79
  dbt_model_depends_on_snapshot=self.dbt_model_depends_on_snapshot)
79
80
  self.dbt_tasks = {}
81
+ self.shared_task_guard = build_shared_task_guard(airflow_vars, manifest_path)
80
82
  self.fqn_unique_list = []
81
83
  self.existing_task_groups = {}
82
84
 
@@ -143,7 +145,7 @@ class DbtManifestParser:
143
145
  return env_vars_with_model
144
146
 
145
147
  def create_dbt_kuberoperator_task(self, dbt_command, running_rule, task_params={}, node_name=None, node_alias=None,
146
- parent_group=None):
148
+ parent_group=None, shared_node=False):
147
149
  """
148
150
  Takes the manifest JSON content and returns a KubernetesPodOperator task
149
151
  to run a dbt command.
@@ -245,6 +247,7 @@ class DbtManifestParser:
245
247
  volume_mounts.append(volume_mount_val)
246
248
 
247
249
  # Create the pod operator with conditional volumes
250
+ running_rule, guard_kwargs = guard_task_kwargs(self.shared_task_guard, running_rule, shared_node)
248
251
  dbt_task = CustomKubernetesPodOperator(
249
252
  task_id=node_alias,
250
253
  name=self.pod_name + "_" + node_alias,
@@ -257,6 +260,7 @@ class DbtManifestParser:
257
260
  # Optional, run with specific K8S SA Account
258
261
  service_account_name=self.airflow_vars.get("DATA_ORCHESTRATOR_WORKER_SA", "data-orchestration-worker"),
259
262
  trigger_rule=running_rule,
263
+ **guard_kwargs,
260
264
  labels={"app": "dbt"},
261
265
  env_vars=env_vars_with_model,
262
266
  image_pull_policy="IfNotPresent",
@@ -288,7 +292,7 @@ class DbtManifestParser:
288
292
  task_params=task_params,
289
293
  node_name=task_name,
290
294
  node_alias=task_alias,
291
- parent_group=parent_group)
295
+ parent_group=parent_group, shared_node=True)
292
296
  return
293
297
 
294
298
  # Get the current level and create a subgroup
@@ -363,7 +367,7 @@ class DbtManifestParser:
363
367
  if not select_string:
364
368
  return None
365
369
 
366
- task_alias = f"{dbt_command}_all_models"
370
+ task_alias = "freshness_all_sources" if dbt_command == "source freshness" else f"{dbt_command}_all_models"
367
371
  return self.create_dbt_kuberoperator_task(
368
372
  dbt_command=dbt_command,
369
373
  running_rule=running_rule,