fast2dev-dbt-runner 0.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,452 @@
1
+ import json
2
+ import datetime
3
+ import re
4
+ import logging
5
+ import requests
6
+ import uuid
7
+ from datetime import datetime
8
+ from kubernetes.client import models as k8s
9
+ from itertools import chain
10
+ from time import sleep
11
+ from fast2dev_dbt_runner._compat import AirflowFailException, BaseHook, PythonOperator, TaskGroup
12
+ import fast2dev_dbt_runner.utils as utils
13
+ from fast2dev_dbt_runner.cached_manifest_loader import load_dbt_manifest_cached
14
+
15
+
16
+
17
+
18
+ class DbtManifestParser:
19
+ """
20
+ A class analyses dbt project and parses manifest.json and creates the respective task groups
21
+ :param manifest_path: Path to the directory containing the manifest files
22
+ :param dbt_tag: define different parts of a project. Have to be set as
23
+ a list of one or a few values
24
+ :param image: docker image where define dbt command
25
+ :param namespace: default
26
+ """
27
+
28
+ def __init__(
29
+ self,
30
+ dbt_tag,
31
+ env_vars,
32
+ airflow_vars,
33
+ manifest_path=None,
34
+ image=None,
35
+ namespace=None,
36
+ connection_id=None,
37
+ **kwargs
38
+ ) -> None:
39
+ self.env_vars = env_vars
40
+ self.airflow_vars = airflow_vars
41
+ self.dbt_tag = utils.check_dbt_tag(dbt_tag)
42
+ self.image = image
43
+ self.namespace = namespace
44
+ self.manifest_path = manifest_path
45
+ self.dbt_tag_ancestors = kwargs.get("dbt_tag_ancestors", False)
46
+ self.dbt_tag_descendants = kwargs.get("dbt_tag_descendants", False)
47
+ self.dbt_model_depends_on_snapshot = airflow_vars.get("DBT_MODEL_DEPENDS_ON_SNAPSHOT", "False")
48
+ self.manifest_data = load_dbt_manifest_cached(self.manifest_path,
49
+ dbt_tag=self.dbt_tag,
50
+ dbt_tag_ancestors=self.dbt_tag_ancestors,
51
+ dbt_tag_descendants=self.dbt_tag_descendants,
52
+ dbt_model_depends_on_snapshot=self.dbt_model_depends_on_snapshot)
53
+ self.dbt_tasks = {}
54
+ self.connection_id = connection_id
55
+ self.dbt_log_format_file = "debug" if self.airflow_vars.get("MODEL_DEBUG_LOG") \
56
+ and self.airflow_vars.get("MODEL_DEBUG_LOG").lower() == "true" \
57
+ else "info"
58
+ self.fqn_unique_list = []
59
+ self.existing_task_groups = {}
60
+
61
+ def is_resource_type_in_manifest(self, resource_type):
62
+ return utils.is_resource_type_in_manifest(self.manifest_data, resource_type)
63
+
64
+ def get_valid_start_date(self, start_date_raw):
65
+ return utils.get_valid_start_date(start_date_raw)
66
+
67
+ def post_api(self, task_id, project_dir, command, host, login, password):
68
+ data = {"command": command, "task_id": task_id, "project_dir": project_dir}
69
+ for i in range(1, 3600):
70
+ post_task_response = requests.post(host, json=data, auth=(login, password))
71
+ if post_task_response.status_code == 200:
72
+ post_task_response.close()
73
+ return post_task_response.json()
74
+ raise Exception("POST API server call failed")
75
+
76
+ def get_api(self, task_id, host, login, password):
77
+ print("Start get api")
78
+ for attempt in range(1, 3600):
79
+ get_response = requests.get(host + "/" + task_id, auth=(login, password))
80
+ if get_response.status_code == 200:
81
+ state = get_response.json().get("state")
82
+ if state == "SUCCESS":
83
+ print("Finish get api")
84
+ return get_response.json()["log_content"]
85
+ elif state == "FAILURE":
86
+ print(get_response.json()["log_content"])
87
+ raise AirflowFailException("FAILED TASK")
88
+ get_response.close()
89
+ print("Finish get api")
90
+ return None
91
+
92
+ def post_to_api_server(
93
+ self, task_id, project_dir, command, **kwargs
94
+ ):
95
+ gcd_conn = BaseHook.get_connection(self.connection_id)
96
+ host = gcd_conn.host
97
+ login = gcd_conn.login
98
+ password = gcd_conn.password
99
+
100
+ unique_task_id = task_id + str(uuid.uuid4())
101
+ print(f"task_id: {unique_task_id}")
102
+ print(f"FULL DBT COMMAND: {command}")
103
+ if not self.post_api(
104
+ unique_task_id, project_dir, command, host, login, password
105
+ ):
106
+ raise Exception("POST API server call failed")
107
+ else:
108
+ response = self.get_api(unique_task_id, host, login, password)
109
+ if not response:
110
+ raise Exception("GET API server call failed")
111
+ else:
112
+ return response
113
+
114
+ def create_api_task(
115
+ self,
116
+ project_dir,
117
+ dbt_command,
118
+ running_rule,
119
+ task_params={},
120
+ full_command_list=None,
121
+ node_name=None,
122
+ node_alias=None,
123
+ parent_group=None
124
+ ):
125
+ """
126
+ Takes the manifest JSON content and returns a KubernetesPodOperator task
127
+ to run a dbt command.
128
+ Args:
129
+ node_name: The name of the node from manifest.json. By default, is equal to None
130
+ dbt_command: dbt command: run, test
131
+ running_rule: argument which defines the rule by which the generated task get triggered
132
+ Returns: A KubernetesPodOperator task that runs the respective dbt command
133
+ :param node_alias:
134
+ :param full_command_list:
135
+ :param project_dir:
136
+ :param dbt_command:
137
+ :param running_rule:
138
+ :param node_name:
139
+ :param task_params:
140
+ """
141
+
142
+ if node_name is None:
143
+ if "re_data" not in dbt_command:
144
+ if not isinstance(dbt_command, list):
145
+ node_name = dbt_command + "_all_models"
146
+ node_alias = dbt_command + "_all_models"
147
+ if "freshness" in dbt_command:
148
+ node_name = dbt_command + "_all_sources"
149
+ node_alias = dbt_command + "_all_sources"
150
+
151
+ else:
152
+ node_alias = "re_data_all_models"
153
+
154
+ if full_command_list is None:
155
+ if dbt_command == "re_data":
156
+ full_command_list = [
157
+ "--log-level-file",
158
+ self.dbt_log_format_file,
159
+ "run",
160
+ "--select",
161
+ "package:re_data"]
162
+ elif dbt_command == "freshness":
163
+ full_command_list = ["--log-level-file",
164
+ self.dbt_log_format_file,
165
+ "source",
166
+ dbt_command,
167
+ "--exclude",
168
+ "package:re_data"]
169
+ elif dbt_command == "snapshot":
170
+ full_command_list = ["--log-level-file",
171
+ self.dbt_log_format_file,
172
+ "snapshot"]
173
+ else:
174
+ full_command_list = ["--log-level-file",
175
+ self.dbt_log_format_file,
176
+ dbt_command,
177
+ "--exclude",
178
+ "package:re_data"]
179
+
180
+ if dbt_command not in ("test", "freshness", "snapshot") and task_params.get('full_refresh') is True:
181
+ full_command_list.append("-f")
182
+
183
+ if dbt_command in ("run", "build") and self.airflow_vars.get("E2E_MODE_EMPTY"):
184
+ full_command_list.append("--empty")
185
+
186
+ if task_params.get("DBT_VAR"):
187
+ full_command_list.extend(["--vars", task_params.get("DBT_VAR")])
188
+
189
+ if self.airflow_vars.get("TARGET"):
190
+ full_command_list.extend(["--target", self.airflow_vars.get("TARGET")])
191
+
192
+ print(f"FULL DBT COMMAND: {str(full_command_list)}")
193
+ print(f"node_name: {node_name}")
194
+ print(f"node_alias: {node_alias}")
195
+
196
+ dbt_task = PythonOperator(
197
+ task_id=node_alias,
198
+ python_callable=self.post_to_api_server,
199
+ op_kwargs={
200
+ "task_id": node_alias,
201
+ "project_dir": project_dir,
202
+ "command": full_command_list
203
+ },
204
+ trigger_rule=running_rule,
205
+ task_group=parent_group
206
+ )
207
+ return dbt_task
208
+
209
+ def create_task_groups(self, parent_group, fqn, node, task_name, task_alias, dbt_command, running_rule,
210
+ task_params, full_command_with_model_name, project_dir):
211
+ """
212
+ Recursively creates task groups from FQN and adds the task to the final group.
213
+ """
214
+ if not fqn:
215
+ # Base case: If FQN is empty, add a task
216
+ if node not in self.dbt_tasks:
217
+ self.dbt_tasks[node] = self.create_api_task(project_dir=project_dir,
218
+ dbt_command=dbt_command,
219
+ running_rule=running_rule,
220
+ task_params=task_params,
221
+ full_command_list=full_command_with_model_name,
222
+ node_name=task_name,
223
+ node_alias=task_alias,
224
+ parent_group=parent_group)
225
+ return
226
+
227
+ # Get the current level and create a subgroup
228
+ current_level = fqn[0]
229
+ if current_level != "re_data":
230
+ subgroup = getattr(parent_group, current_level, None)
231
+
232
+ # If the subgroup does not exist, create it
233
+ if not subgroup:
234
+ subgroup = TaskGroup(group_id=current_level, parent_group=parent_group)
235
+ setattr(parent_group, current_level, subgroup)
236
+
237
+ # Recurse into the next level
238
+ self.create_task_groups(subgroup, fqn[1:], node, task_name, task_alias, dbt_command, running_rule,
239
+ task_params, full_command_with_model_name, project_dir)
240
+
241
+ def set_dependencies(self, resource_type):
242
+ """
243
+ Sets the dependencies between tasks based on the `depends_on` attribute in the manifest data.
244
+ """
245
+ for node in self.manifest_data.keys():
246
+ if resource_type in self.manifest_data[node]['group_type']:
247
+ # Only set dependencies for nodes that were actually created as tasks
248
+ if node not in self.dbt_tasks:
249
+ continue
250
+
251
+ for upstream_node in self.manifest_data[node].get("depends_on", []):
252
+ if self.dbt_tasks.get(upstream_node, []):
253
+ self.dbt_tasks[upstream_node] >> self.dbt_tasks[node]
254
+
255
+ def manifest_has_model_depends_on_snapshot(self):
256
+ """True when a selected model directly depends on a snapshot (batch build trigger)."""
257
+ return utils.manifest_has_model_depends_on_snapshot(self.manifest_data)
258
+
259
+ def get_model_names_for_resource_types(self, resource_types, task_params=None):
260
+ """
261
+ Collects node names from the filtered manifest for the given resource type(s).
262
+ Returns a space-separated string suitable for dbt --select, or None if empty.
263
+ """
264
+ if isinstance(resource_types, str):
265
+ resource_types = [resource_types]
266
+ resource_types = set(resource_types)
267
+
268
+ if task_params is None:
269
+ task_params = {}
270
+ task_params = {k: v for k, v in task_params.items() if v}
271
+
272
+ manifest = self.manifest_data
273
+ if "full_refresh_model_name" in task_params:
274
+ manifest = utils.filter_models(manifest, task_params["full_refresh_model_name"])
275
+
276
+ model_names = []
277
+ for node_id, node_data in manifest.items():
278
+ if node_data.get("resource_type") in resource_types:
279
+ model_names.append(node_data["name"])
280
+ return " ".join(model_names) if model_names else None
281
+
282
+ def get_model_names_for_resource_type(self, resource_type, task_params=None):
283
+ """
284
+ Collects all model names from the filtered manifest for a given resource type.
285
+ Returns a space-separated string suitable for dbt --select.
286
+ """
287
+ return self.get_model_names_for_resource_types([resource_type], task_params)
288
+
289
+ def create_dbt_build_batch_task(self, running_rule, project_dir, task_params=None,
290
+ resource_types=("model", "snapshot")):
291
+ """
292
+ Creates a single ``dbt build`` batch API task over models + snapshots, so a
293
+ ``model -> snapshot -> model`` chain runs in lineage order in one call
294
+ (used in non-sharded mode when DBT_MODEL_DEPENDS_ON_SNAPSHOT is enabled).
295
+ """
296
+ select_string = self.get_model_names_for_resource_types(list(resource_types), task_params)
297
+ if not select_string:
298
+ return None
299
+
300
+ full_command_list = [
301
+ "--log-level-file",
302
+ self.dbt_log_format_file,
303
+ "build",
304
+ "--exclude",
305
+ "package:re_data",
306
+ "--select",
307
+ select_string
308
+ ]
309
+ return self.create_api_task(
310
+ project_dir=project_dir,
311
+ dbt_command="build",
312
+ running_rule=running_rule,
313
+ task_params=task_params if task_params else {},
314
+ full_command_list=full_command_list,
315
+ node_name=select_string,
316
+ node_alias="build_all_models"
317
+ )
318
+
319
+ def create_dbt_batch_task(self, resource_type, dbt_command, running_rule, project_dir, task_params=None):
320
+ """
321
+ Creates a single Airflow task that runs all models of the given resource_type
322
+ in one batch using dbt --select "model1 model2 model3 ...".
323
+ Bypasses per-model lineage — dbt handles internal dependency ordering.
324
+ """
325
+ select_string = self.get_model_names_for_resource_type(resource_type, task_params)
326
+ if not select_string:
327
+ return None
328
+
329
+ full_command_list = [
330
+ "--log-level-file",
331
+ self.dbt_log_format_file,
332
+ dbt_command,
333
+ "--exclude",
334
+ "package:re_data",
335
+ "--select",
336
+ select_string
337
+ ]
338
+
339
+ task_alias = f"{dbt_command}_all_models"
340
+ return self.create_api_task(
341
+ project_dir=project_dir,
342
+ dbt_command=dbt_command,
343
+ running_rule=running_rule,
344
+ task_params=task_params if task_params else {},
345
+ full_command_list=full_command_list,
346
+ node_name=select_string,
347
+ node_alias=task_alias
348
+ )
349
+
350
+ def create_dbt_task_groups(
351
+ self,
352
+ group_name,
353
+ resource_type,
354
+ project_dir,
355
+ dbt_command,
356
+ running_rule,
357
+ task_params={}):
358
+ """
359
+ Parse out a JSON file and populates the task groups with dbt tasks
360
+ Args:
361
+ group_name: name of Task Groups uses for DAGs graph view in the Airflow UI.
362
+ resource_type: type of manifest nodes group: model, seed, snapshot
363
+ package_name: project name, tag that define by which certain records
364
+ from the manifest nodes will be selected
365
+ dbt_command: dbt command run or test
366
+ running_rule: trigger rule
367
+ task_params: parameters that was added in the task
368
+ Returns: task group
369
+ :param task_params:
370
+ :param running_rule:
371
+ :param group_name:
372
+ :param resource_type:
373
+ :param dbt_command:
374
+ :param project_dir:
375
+ """
376
+
377
+ task_params = {k: v for k, v in task_params.items() if v} # get only not empty value in task_params
378
+ if "full_refresh_model_name" in task_params:
379
+ self.manifest_data = utils.filter_models(self.manifest_data, task_params["full_refresh_model_name"])
380
+
381
+ if utils.is_resource_type_in_manifest(self.manifest_data, resource_type):
382
+ # Initialize the root TaskGroup from `group_name` (should be a TaskGroup instance, not a string)
383
+ with TaskGroup(group_id=group_name, parent_group=None) as root_group:
384
+ for node_id, node_data in self.manifest_data.items():
385
+ # Skip tests when creating source task groups - tests should run separately, not as part of source freshness
386
+ if resource_type == "source" and node_data.get('resource_type') == 'test':
387
+ continue
388
+ if group_name[:-1] in node_data["group_type"]:
389
+ # Extract FQN and task name
390
+ fqn = node_data["fqn"][:-1] # Remove model name from the FQN
391
+ task_name = node_data["name"]
392
+ task_alias = node_data["alias"]
393
+ # Use a per-node command so overrides (test/snapshot/source)
394
+ # do not leak into subsequent iterations of this loop.
395
+ node_command = dbt_command
396
+ if node_data['resource_type'] == "test":
397
+ node_command = "test"
398
+ elif node_data['resource_type'] == "snapshot":
399
+ # A snapshot woven into the model group (DBT_MODEL_DEPENDS_ON_SNAPSHOT)
400
+ # must still run as `dbt snapshot`, not `dbt run`.
401
+ node_command = "snapshot"
402
+ elif node_data['resource_type'] == "source":
403
+ task_name = f"source:{node_data['schema']}.{task_name}"
404
+ node_command = "freshness"
405
+ if task_name:
406
+ if node_data['resource_type'] == "source":
407
+ full_command_with_model_name = ["--log-level-file",
408
+ self.dbt_log_format_file,
409
+ "source",
410
+ node_command,
411
+ "--select",
412
+ task_name]
413
+ elif node_command == "snapshot":
414
+ full_command_with_model_name = ["--log-level-file",
415
+ self.dbt_log_format_file,
416
+ "snapshot",
417
+ "--select",
418
+ task_name]
419
+ else:
420
+ full_command_with_model_name = ["--log-level-file",
421
+ self.dbt_log_format_file,
422
+ node_command,
423
+ "--exclude",
424
+ "package:re_data",
425
+ "--select",
426
+ task_name]
427
+
428
+ else:
429
+ if node_data['resource_type'] == "source":
430
+ full_command_with_model_name = ["--log-level-file",
431
+ self.dbt_log_format_file,
432
+ "source",
433
+ node_command,
434
+ "--exclude",
435
+ "package:re_data"]
436
+ elif node_command == "snapshot":
437
+ full_command_with_model_name = ["--log-level-file",
438
+ self.dbt_log_format_file,
439
+ "snapshot"]
440
+ else:
441
+ full_command_with_model_name = ["--log-level-file",
442
+ self.dbt_log_format_file,
443
+ node_command,
444
+ "--exclude",
445
+ "package:re_data"]
446
+ # Create the dynamic task groups under root_group
447
+ self.create_task_groups(root_group, fqn, node_id, task_name, task_alias, node_command, running_rule,
448
+ task_params, full_command_with_model_name, project_dir)
449
+ self.set_dependencies(resource_type)
450
+ if not root_group.children:
451
+ return None
452
+ return root_group