fast2dev-dbt-runner 0.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,361 @@
1
+ import logging
2
+ import fast2dev_dbt_runner.utils as utils
3
+ from fast2dev_dbt_runner.cached_manifest_loader import load_dbt_manifest_cached
4
+ from fast2dev_dbt_runner._compat import TaskGroup
5
+ from fast2dev_dbt_runner.bash_operator.dbt_operator import (
6
+ DbtSeedOperator,
7
+ DbtSnapshotOperator,
8
+ DbtRunOperator,
9
+ DbtBuildOperator,
10
+ DbtTestOperator,
11
+ DbtDepsOperator,
12
+ DbtSourceFreshnessOperator,
13
+ DbtReDataOperator,
14
+ DbtDebugOperator
15
+ )
16
+
17
+ class DbtManifestParser:
18
+ """
19
+ A class analyses dbt project and parses manifest.json and creates the respective task groups
20
+ """
21
+
22
+ def __init__(
23
+ self,
24
+ dbt_project_dir,
25
+ dbt_tag,
26
+ airflow_vars,
27
+ manifest_path=None,
28
+ debug=False,
29
+ **kwargs
30
+ ) -> None:
31
+ self.dbt_project_dir = dbt_project_dir
32
+ self.dbt_tag = utils.check_dbt_tag(dbt_tag)
33
+ self.airflow_vars = airflow_vars
34
+ self.manifest_path = manifest_path
35
+ self.dbt_tag_ancestors = kwargs.get("dbt_tag_ancestors", False)
36
+ self.dbt_tag_descendants = kwargs.get("dbt_tag_descendants", False)
37
+ self.dbt_model_depends_on_snapshot = airflow_vars.get("DBT_MODEL_DEPENDS_ON_SNAPSHOT", "False")
38
+ self.manifest_data = load_dbt_manifest_cached(self.manifest_path,
39
+ dbt_tag=self.dbt_tag,
40
+ dbt_tag_ancestors=self.dbt_tag_ancestors,
41
+ dbt_tag_descendants=self.dbt_tag_descendants,
42
+ dbt_model_depends_on_snapshot=self.dbt_model_depends_on_snapshot)
43
+ self.dbt_tasks = {}
44
+ self.fqn_unique_list = []
45
+ self.existing_task_groups = {}
46
+ self.log = logging.getLogger(__name__)
47
+ self.debug = debug
48
+
49
+ def is_resource_type_in_manifest(self, resource_type):
50
+ return utils.is_resource_type_in_manifest(self.manifest_data, resource_type)
51
+
52
+ def get_valid_start_date(self, start_date_raw):
53
+ return utils.get_valid_start_date(start_date_raw)
54
+
55
+ def create_dbt_bash_task(self, dbt_command, running_rule, task_params=None, node_name=None, node_alias=None,
56
+ parent_group=None):
57
+ try:
58
+ if self.airflow_vars.get("TARGET"):
59
+ target = self.airflow_vars.get("TARGET")
60
+ else:
61
+ target = None
62
+
63
+ if self.airflow_vars.get("GIT_BRANCH"):
64
+ git_branch = self.airflow_vars.get("GIT_BRANCH")
65
+ else:
66
+ git_branch = None
67
+
68
+ # Get warehouse type from airflow variables
69
+ warehouse_type = self.airflow_vars.get("DATA_WAREHOUSE_PLATFORM")
70
+ if warehouse_type:
71
+ if self.debug:
72
+ self.log.info(f"Using DATA_WAREHOUSE_PLATFORM={warehouse_type} for warehouse type")
73
+
74
+ if dbt_command == 'run':
75
+ dbt_run = DbtRunOperator(
76
+ task_id=node_alias,
77
+ models=node_name,
78
+ dbt_project_dir=self.dbt_project_dir,
79
+ target=target,
80
+ git_branch=git_branch,
81
+ warehouse_type=warehouse_type,
82
+ task_group=parent_group,
83
+ debug=self.debug,
84
+ empty=True if self.airflow_vars.get("E2E_MODE_EMPTY") else False
85
+ )
86
+ elif dbt_command == 'build':
87
+ task_id = node_alias if node_alias else "build_all_models"
88
+ dbt_run = DbtBuildOperator(
89
+ task_id=task_id,
90
+ models=node_name,
91
+ dbt_project_dir=self.dbt_project_dir,
92
+ target=target,
93
+ git_branch=git_branch,
94
+ warehouse_type=warehouse_type,
95
+ task_group=parent_group,
96
+ debug=self.debug,
97
+ empty=True if self.airflow_vars.get("E2E_MODE_EMPTY") else False
98
+ )
99
+ elif dbt_command == 'test':
100
+ dbt_run = DbtTestOperator(
101
+ task_id=node_alias,
102
+ models=node_name,
103
+ dbt_project_dir=self.dbt_project_dir,
104
+ target=target,
105
+ git_branch=git_branch,
106
+ warehouse_type=warehouse_type,
107
+ task_group=parent_group,
108
+ debug=self.debug
109
+ )
110
+ elif dbt_command == 'seed':
111
+ task_id = node_alias if node_name and node_alias else "seed_all_models"
112
+ models = node_name if node_name and node_alias else None
113
+
114
+ dbt_run = DbtSeedOperator(
115
+ task_id=task_id,
116
+ models=models,
117
+ dbt_project_dir=self.dbt_project_dir,
118
+ target=target,
119
+ git_branch=git_branch,
120
+ warehouse_type=warehouse_type,
121
+ task_group=parent_group,
122
+ debug=self.debug
123
+ )
124
+ elif dbt_command == 'snapshot':
125
+ task_id = node_alias if node_name and node_alias else "snapshot_all_models"
126
+ models = node_name if node_name and node_alias else None
127
+
128
+ dbt_run = DbtSnapshotOperator(
129
+ task_id=task_id,
130
+ models=models,
131
+ dbt_project_dir=self.dbt_project_dir,
132
+ target=target,
133
+ git_branch=git_branch,
134
+ warehouse_type=warehouse_type,
135
+ task_group=parent_group,
136
+ debug=self.debug
137
+ )
138
+ elif dbt_command == 'source freshness':
139
+ task_id = node_alias if node_name and node_alias else "source_all_models"
140
+ models = node_name if node_name and node_alias else None
141
+
142
+ dbt_run = DbtSourceFreshnessOperator(
143
+ task_id=task_id,
144
+ models=models,
145
+ dbt_project_dir=self.dbt_project_dir,
146
+ warehouse_type=warehouse_type,
147
+ task_group=parent_group,
148
+ debug=self.debug
149
+ )
150
+ elif dbt_command == 're_data':
151
+ dbt_run = DbtReDataOperator(
152
+ task_id='re_data_quality_checks',
153
+ dbt_project_dir=self.dbt_project_dir,
154
+ warehouse_type=warehouse_type,
155
+ task_group=parent_group,
156
+ debug=self.debug
157
+ )
158
+ elif dbt_command == 'deps':
159
+ dbt_run = DbtDepsOperator(
160
+ task_id=node_alias,
161
+ dbt_project_dir=self.dbt_project_dir,
162
+ warehouse_type=warehouse_type,
163
+ debug=self.debug
164
+ )
165
+ elif dbt_command == 'debug':
166
+ dbt_run = DbtDebugOperator(
167
+ task_id=node_alias,
168
+ dbt_project_dir=self.dbt_project_dir,
169
+ target=target,
170
+ git_branch=git_branch,
171
+ warehouse_type=warehouse_type,
172
+ task_group=parent_group,
173
+ debug=self.debug
174
+ )
175
+ else:
176
+ self.log.error(f"Invalid dbt command: {dbt_command}")
177
+ raise ValueError(f"Invalid dbt command: {dbt_command}")
178
+
179
+ return dbt_run
180
+
181
+ except Exception as e:
182
+ self.log.error(f"Failed to create dbt task: {str(e)}")
183
+ raise
184
+
185
+ def create_task_groups(self, parent_group, fqn, node, task_name, task_alias, dbt_command, running_rule,
186
+ task_params):
187
+ """
188
+ Recursively creates task groups from FQN and adds the task to the final group.
189
+ """
190
+ if not fqn:
191
+ # Base case: If FQN is empty, add a task
192
+ if node not in self.dbt_tasks:
193
+ self.dbt_tasks[node] = self.create_dbt_bash_task(dbt_command=dbt_command,
194
+ running_rule=running_rule,
195
+ task_params=task_params,
196
+ node_name=task_name,
197
+ node_alias=task_alias,
198
+ parent_group=parent_group)
199
+ return
200
+
201
+ # Get the current level and create a subgroup
202
+ current_level = fqn[0]
203
+ if current_level != "re_data":
204
+ subgroup = getattr(parent_group, current_level, None)
205
+
206
+ # If the subgroup does not exist, create it
207
+ if not subgroup:
208
+ subgroup = TaskGroup(group_id=current_level, parent_group=parent_group)
209
+ setattr(parent_group, current_level, subgroup)
210
+
211
+ # Recurse into the next level
212
+ self.create_task_groups(subgroup, fqn[1:], node, task_name, task_alias, dbt_command, running_rule,
213
+ task_params)
214
+
215
+ def set_dependencies(self, resource_type):
216
+ """
217
+ Sets the dependencies between tasks based on the `depends_on` attribute in the manifest data.
218
+ """
219
+ for node in self.manifest_data.keys():
220
+ if resource_type in self.manifest_data[node]['group_type']:
221
+ # Only set dependencies for nodes that were actually created as tasks
222
+ if node not in self.dbt_tasks:
223
+ continue
224
+
225
+ for upstream_node in self.manifest_data[node].get("depends_on", []):
226
+ if self.dbt_tasks.get(upstream_node, []):
227
+ self.dbt_tasks[upstream_node] >> self.dbt_tasks[node]
228
+
229
+ def manifest_has_model_depends_on_snapshot(self):
230
+ """True when a selected model directly depends on a snapshot (batch build trigger)."""
231
+ return utils.manifest_has_model_depends_on_snapshot(self.manifest_data)
232
+
233
+ def get_model_names_for_resource_types(self, resource_types, task_params=None):
234
+ """
235
+ Collects node names from the filtered manifest for the given resource type(s).
236
+ Returns a space-separated string suitable for dbt --select, or None if empty.
237
+ """
238
+ if isinstance(resource_types, str):
239
+ resource_types = [resource_types]
240
+ resource_types = set(resource_types)
241
+
242
+ if task_params is None:
243
+ task_params = {}
244
+ task_params = {k: v for k, v in task_params.items() if v}
245
+
246
+ manifest = self.manifest_data
247
+ if "full_refresh_model_name" in task_params:
248
+ manifest = utils.filter_models(manifest, task_params["full_refresh_model_name"])
249
+
250
+ model_names = []
251
+ for node_id, node_data in manifest.items():
252
+ if node_data.get("resource_type") in resource_types:
253
+ model_names.append(node_data["name"])
254
+ return " ".join(model_names) if model_names else None
255
+
256
+ def get_model_names_for_resource_type(self, resource_type, task_params=None):
257
+ """
258
+ Collects all model names from the filtered manifest for a given resource type.
259
+ Returns a space-separated string suitable for dbt --select.
260
+ """
261
+ return self.get_model_names_for_resource_types([resource_type], task_params)
262
+
263
+ def create_dbt_batch_task(self, resource_type, dbt_command, running_rule, task_params=None):
264
+ """
265
+ Creates a single Airflow task that runs all models of the given resource_type
266
+ in one batch using dbt --select "model1 model2 model3 ...".
267
+ Bypasses per-model lineage — dbt handles internal dependency ordering.
268
+ """
269
+ select_string = self.get_model_names_for_resource_type(resource_type, task_params)
270
+ if not select_string:
271
+ return None
272
+
273
+ task_alias = f"{dbt_command}_all_models"
274
+ return self.create_dbt_bash_task(
275
+ dbt_command=dbt_command,
276
+ running_rule=running_rule,
277
+ task_params=task_params,
278
+ node_name=select_string,
279
+ node_alias=task_alias
280
+ )
281
+
282
+ def create_dbt_build_batch_task(self, running_rule, task_params=None,
283
+ resource_types=("model", "snapshot")):
284
+ """
285
+ Creates a single ``dbt build --select "<models> <snapshots>"`` batch task.
286
+
287
+ Used in non-sharded mode when DBT_MODEL_DEPENDS_ON_SNAPSHOT is enabled and a
288
+ model depends on a snapshot. ``dbt build`` runs the selected models and
289
+ snapshots (and their tests) in lineage order in one process, so a
290
+ ``model -> snapshot -> model`` chain resolves correctly - which a separate
291
+ ``dbt run`` + ``dbt snapshot`` cannot do. Selection is the explicit node
292
+ list already resolved in the manifest (tag + ancestors), so scope is
293
+ identical to the run batch it replaces, just with snapshots interleaved.
294
+ """
295
+ select_string = self.get_model_names_for_resource_types(list(resource_types), task_params)
296
+ if not select_string:
297
+ return None
298
+
299
+ return self.create_dbt_bash_task(
300
+ dbt_command="build",
301
+ running_rule=running_rule,
302
+ task_params=task_params,
303
+ node_name=select_string,
304
+ node_alias="build_all_models"
305
+ )
306
+
307
+ def create_dbt_task_groups(
308
+ self,
309
+ group_name,
310
+ resource_type,
311
+ dbt_command,
312
+ running_rule,
313
+ task_params={}):
314
+ """
315
+ Parse out a JSON file and populates the task groups with dbt tasks
316
+ Args:
317
+ group_name: name of Task Groups uses for DAGs graph view in the Airflow UI.
318
+ resource_type: type of manifest nodes group: model, seed, snapshot
319
+ package_name: project name, tag that define by which certain records
320
+ from the manifest nodes will be selected
321
+ dbt_command: dbt command run or test
322
+ running_rule: trigger rule
323
+ task_params: parameters that was added in the task
324
+ Returns: task group
325
+ """
326
+ task_params = {k: v for k, v in task_params.items() if v} # Filter out empty values
327
+ if "full_refresh_model_name" in task_params:
328
+ self.manifest_data = utils.filter_models(self.manifest_data, task_params["full_refresh_model_name"])
329
+
330
+ if utils.is_resource_type_in_manifest(self.manifest_data, resource_type):
331
+ # Initialize the root TaskGroup from `group_name` (should be a TaskGroup instance, not a string)
332
+ with TaskGroup(group_id=group_name, parent_group=None) as root_group:
333
+ for node_id, node_data in self.manifest_data.items():
334
+ # Skip tests when creating source task groups - tests should run separately, not as part of source freshness
335
+ if resource_type == "source" and node_data.get('resource_type') == 'test':
336
+ continue
337
+ if group_name[:-1] in node_data["group_type"]:
338
+ # Extract FQN and task name
339
+ fqn = node_data["fqn"][:-1] # Remove model name from the FQN
340
+ task_name = node_data["name"]
341
+ task_alias = node_data["alias"]
342
+ # Use a per-node command so it does not leak across loop
343
+ # iterations (a model iterated after a test must stay 'run').
344
+ node_command = dbt_command
345
+ if node_data['resource_type'] == "test":
346
+ node_command = "test"
347
+ elif node_data['resource_type'] == "snapshot":
348
+ node_command = "snapshot"
349
+ elif node_data['resource_type'] == "source":
350
+ task_name = f"source:{node_data['schema']}.{task_name}"
351
+ node_command = "source freshness"
352
+ # Create the dynamic task groups under root_group
353
+ self.create_task_groups(root_group, fqn, node_id, task_name, task_alias, node_command,
354
+ running_rule,
355
+ task_params)
356
+ self.set_dependencies(resource_type)
357
+ # Guard against an empty group (e.g. all snapshots were woven into the
358
+ # model group). The DAG drops None via list(filter(None, task_list)).
359
+ if not root_group.children:
360
+ return None
361
+ return root_group