fast2dev-dbt-runner 0.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,448 @@
1
+ import datetime
2
+ import textwrap
3
+ from datetime import datetime
4
+ from kubernetes.client import models as k8s
5
+ from fast2dev_dbt_runner._compat import AirflowException, TaskGroup
6
+ import fast2dev_dbt_runner.utils as utils
7
+ from fast2dev_dbt_runner.cached_manifest_loader import load_dbt_manifest_cached
8
+
9
+ try:
10
+ # Airflow 2.10+
11
+ from airflow.providers.cncf.kubernetes.operators.pod import KubernetesPodOperator
12
+ except ImportError:
13
+ # Older versions
14
+ from airflow.providers.cncf.kubernetes.operators.kubernetes_pod import KubernetesPodOperator
15
+
16
+
17
+ class CustomKubernetesPodOperator(KubernetesPodOperator):
18
+ def execute(self, context):
19
+ """
20
+ Executes a KubernetesPodOperator task with customized exception handling.
21
+
22
+ This custom operator extends the KubernetesPodOperator from the Airflow library.
23
+ It overrides the execute method to catch AirflowException raised by the parent class.
24
+ The error message is then shortened and a new AirflowException is raised with the
25
+ shortened message. This approach allows for more concise error reporting while
26
+ preserving the detailed error information.
27
+
28
+ :param context: The context dictionary provided by Airflow.
29
+ """
30
+ try:
31
+ super().execute(context)
32
+ except AirflowException as e:
33
+ # Shorten the error message for clearer reporting
34
+ msg = str(e)
35
+ short_msg = textwrap.shorten(msg, width=160)
36
+ raise AirflowException(short_msg) from None
37
+
38
+
39
+ class DbtManifestParser:
40
+ """
41
+ A class analyses dbt project and parses manifest.json and creates the respective task groups
42
+ :param manifest_path: Path to the directory containing the manifest files
43
+ :param pod_name: name of Kubercluster pod
44
+ :param dbt_tag: define different parts of a project. Have to be set as
45
+ a list of one or a few values
46
+ :param env_var: airflow variables
47
+ :param image: docker image where define dbt command
48
+ :param namespace: default
49
+ :param CLUSTER_ZONE: Getting from Airflow environment variables {{ var.value.CLUSTER_ZONE }}
50
+ :param CLUSTER_NAME: Getting from Airflow environment variables {{ var.value.CLUSTER_NAME_DBT }}
51
+ """
52
+
53
+ def __init__(
54
+ self,
55
+ dbt_tag,
56
+ env_vars,
57
+ airflow_vars,
58
+ manifest_path=None,
59
+ pod_name=None,
60
+ image=None,
61
+ namespace=None,
62
+ **kwargs
63
+ ) -> None:
64
+ self.env_vars = env_vars
65
+ self.airflow_vars = airflow_vars
66
+ self.pod_name = pod_name
67
+ self.dbt_tag = utils.check_dbt_tag(dbt_tag)
68
+ self.image = image
69
+ self.namespace = namespace
70
+ self.manifest_path = manifest_path
71
+ self.dbt_tag_ancestors = kwargs.get("dbt_tag_ancestors", False)
72
+ self.dbt_tag_descendants = kwargs.get("dbt_tag_descendants", False)
73
+ self.dbt_model_depends_on_snapshot = airflow_vars.get("DBT_MODEL_DEPENDS_ON_SNAPSHOT", "False")
74
+ self.manifest_data = load_dbt_manifest_cached(self.manifest_path,
75
+ dbt_tag=self.dbt_tag,
76
+ dbt_tag_ancestors=self.dbt_tag_ancestors,
77
+ dbt_tag_descendants=self.dbt_tag_descendants,
78
+ dbt_model_depends_on_snapshot=self.dbt_model_depends_on_snapshot)
79
+ self.dbt_tasks = {}
80
+ self.fqn_unique_list = []
81
+ self.existing_task_groups = {}
82
+
83
+ def is_resource_type_in_manifest(self, resource_type):
84
+ return utils.is_resource_type_in_manifest(self.manifest_data, resource_type)
85
+
86
+ def get_valid_start_date(self, start_date_raw):
87
+ return utils.get_valid_start_date(start_date_raw)
88
+
89
+ def add_additional_env_variables(self, task_params, dbt_command, node_name):
90
+ # create env_vars_with_model list as a copy of env_vars
91
+ # to avoid redefinition of env_vars list
92
+ env_vars_with_model = self.env_vars.copy()
93
+ airflow_var = self.airflow_vars.copy()
94
+
95
+ if task_params.get('full_refresh', None):
96
+ if dbt_command != "test":
97
+ dbt_command = task_params['full_refresh']
98
+
99
+ env_vars_with_model.append(k8s.V1EnvVar(name="DBT_COMMAND", value=f"{dbt_command}"))
100
+ # add to env_vars_with_model new variable MODEL that equal
101
+ # to current DBT project model node_name
102
+ if node_name:
103
+ env_vars_with_model.append(k8s.V1EnvVar(name="MODEL", value=f"{node_name}"))
104
+ if dbt_command == "seed":
105
+ env_vars_with_model.append(k8s.V1EnvVar(name="SEED", value="true"))
106
+ else:
107
+ env_vars_with_model.append(k8s.V1EnvVar(name="SEED", value="false"))
108
+
109
+ if dbt_command in ("run", "build") and self.airflow_vars.get("E2E_MODE_EMPTY"):
110
+ env_vars_with_model.append(k8s.V1EnvVar(name="E2E_MODE_EMPTY", value="true"))
111
+
112
+ if task_params:
113
+ env_vars_with_model_keys = [i.name for i in env_vars_with_model]
114
+ kuber_dag_new_params = [k8s.V1EnvVar(name=k, value=str(v)) for k, v in task_params.items() if
115
+ k not in env_vars_with_model_keys]
116
+ env_vars_with_model.extend(kuber_dag_new_params)
117
+ airflow_var = {**task_params, **airflow_var}
118
+
119
+ if dbt_command == "snapshot":
120
+ env_vars_with_model.append(k8s.V1EnvVar(name="SNAPSHOT", value="true"))
121
+ if self.airflow_vars.get("GIT_BRANCH"):
122
+ env_vars_with_model.append(k8s.V1EnvVar(name="GIT_BRANCH", value=str(self.airflow_vars.get("GIT_BRANCH"))))
123
+ if not airflow_var.get('DBT_SNAPSHOT_INTERVAL') or airflow_var['DBT_SNAPSHOT_INTERVAL'] == 'daily':
124
+ env_vars_with_model.append(k8s.V1EnvVar(name="SNAPSHOT_RUN_PERIOD", value="true"))
125
+ else:
126
+ env_vars_with_model.append(k8s.V1EnvVar(name="SNAPSHOT_RUN_PERIOD", value="false"))
127
+
128
+ if airflow_var.get('DBT_DAG_RUN_DATE'):
129
+ date_input = datetime.datetime.strptime(airflow_var['DBT_DAG_RUN_DATE'], "%Y-%m-%d")
130
+ if airflow_var.get('DBT_SNAPSHOT_VALID_FROM'):
131
+ date_input_from = datetime.datetime.strptime(airflow_var['DBT_SNAPSHOT_VALID_FROM'], "%Y-%m-%d")
132
+ if airflow_var.get('DBT_SNAPSHOT_VALID_TO'):
133
+ date_input_to = datetime.datetime.strptime(airflow_var['DBT_SNAPSHOT_VALID_TO'], "%Y-%m-%d")
134
+
135
+ if airflow_var['DBT_SNAPSHOT_INTERVAL'] == 'monthly' \
136
+ and date_input_from.day <= date_input.day <= date_input_to.day:
137
+ env_vars_with_model.append(k8s.V1EnvVar(name="SNAPSHOT_RUN_PERIOD", value="true"))
138
+ elif airflow_var['DBT_SNAPSHOT_INTERVAL'] == 'weekly' \
139
+ and date_input_from.weekday() <= date_input.weekday() <= date_input_to.weekday():
140
+ env_vars_with_model.append(k8s.V1EnvVar(name="SNAPSHOT_RUN_PERIOD", value="true"))
141
+ else:
142
+ env_vars_with_model.append(k8s.V1EnvVar(name="SNAPSHOT", value="false"))
143
+ return env_vars_with_model
144
+
145
+ def create_dbt_kuberoperator_task(self, dbt_command, running_rule, task_params={}, node_name=None, node_alias=None,
146
+ parent_group=None):
147
+ """
148
+ Takes the manifest JSON content and returns a KubernetesPodOperator task
149
+ to run a dbt command.
150
+ Args:
151
+ node_name: The name of the node from manifest.json. By default is equal to None
152
+ dbt_command: dbt command: run, test
153
+ running_rule: argument which defines the rule by which the generated task get triggered
154
+ Returns: A KubernetesPodOperator task that runs the respective dbt command
155
+ :param dbt_command:
156
+ :param running_rule:
157
+ :param node_name:
158
+ :param task_params:
159
+ """
160
+ env_vars_with_model = self.add_additional_env_variables(task_params, dbt_command, node_name)
161
+ if node_name is None:
162
+ node_name = dbt_command + "_all_models"
163
+ node_alias = dbt_command + "_all_models"
164
+ if dbt_command == "source freshness":
165
+ node_name = "freshness_all_sources"
166
+ node_alias = "freshness_all_sources"
167
+
168
+ """
169
+ Create KubernetesPodOperator operators
170
+ Args:
171
+ task_id: The ID specified for the task
172
+ name: Name of task you want to run, used to generate Pod ID
173
+ namespace: The namespace to run within Kubernetes
174
+ image: Docker image specified
175
+ trigger_rule: the conditions that Airflow applies to tasks to
176
+ determine whether they are ready to execute.
177
+ ALL_DONE - all upstream tasks are done with their execution
178
+ """
179
+
180
+ affinity = {
181
+ 'podAntiAffinity': {
182
+ 'preferredDuringSchedulingIgnoredDuringExecution': [
183
+ {
184
+ 'weight': 100,
185
+ 'podAffinityTerm': {
186
+ 'labelSelector': {
187
+ 'matchExpressions': [
188
+ {
189
+ 'key': 'component',
190
+ 'operator': 'In',
191
+ 'values': ['scheduler', 'triggerer', 'worker']
192
+ }
193
+ ]
194
+ },
195
+ 'topologyKey': 'kubernetes.io/hostname'
196
+ }
197
+ }
198
+ ]
199
+ }
200
+ }
201
+
202
+ # Initialize empty lists for volumes and volume_mounts
203
+ volumes = []
204
+ volume_mounts = []
205
+
206
+ # Only add volume and volume_mount if DATA_WAREHOUSE_PLATFORM is provided
207
+ dwh = self.airflow_vars.get("DATA_WAREHOUSE_PLATFORM", None)
208
+ if dwh is not None:
209
+ # Universal mapping for both numeric and string values
210
+ dwh_mapping = {
211
+ # Numeric to string mapping
212
+ "1": "bigquery",
213
+ "2": "snowflake",
214
+ "3": "redshift",
215
+ "4": "fabric",
216
+ # String to string mapping (case-insensitive)
217
+ "bigquery": "bigquery",
218
+ "snowflake": "snowflake",
219
+ "redshift": "redshift",
220
+ "fabric": "fabric"
221
+ }
222
+
223
+ # Convert input to lowercase if it's a string
224
+ dwh_key = str(dwh).lower()
225
+
226
+ # Get the warehouse name from mapping
227
+ warehouse_name = dwh_mapping.get(dwh_key)
228
+
229
+ if warehouse_name is None:
230
+ raise ValueError(
231
+ f"Invalid DATA_WAREHOUSE_PLATFORM value: {dwh}. Must be one of: {list(dwh_mapping.keys())}")
232
+
233
+ secret_name_dwh_specific = f"data-warehouse-{warehouse_name}-secrets"
234
+ volume_val = k8s.V1Volume(
235
+ name="data-warehouse-secrets",
236
+ secret=k8s.V1SecretVolumeSource(secret_name=secret_name_dwh_specific)
237
+ )
238
+ volume_mount_val = k8s.V1VolumeMount(
239
+ name="data-warehouse-secrets",
240
+ mount_path="/fast2dev/secrets",
241
+ sub_path=None,
242
+ read_only=True
243
+ )
244
+ volumes.append(volume_val)
245
+ volume_mounts.append(volume_mount_val)
246
+
247
+ # Create the pod operator with conditional volumes
248
+ dbt_task = CustomKubernetesPodOperator(
249
+ task_id=node_alias,
250
+ name=self.pod_name + "_" + node_alias,
251
+ namespace=self.namespace,
252
+ image=self.image,
253
+ # Debug - uncomment for debug the container.
254
+ # cmds=["bash", "-cx"],
255
+ # arguments=[f" tail -f /dev/null "],
256
+ # trigger_rule=TriggerRule.ALL_DONE,
257
+ # Optional, run with specific K8S SA Account
258
+ service_account_name=self.airflow_vars.get("DATA_ORCHESTRATOR_WORKER_SA", "data-orchestration-worker"),
259
+ trigger_rule=running_rule,
260
+ labels={"app": "dbt"},
261
+ env_vars=env_vars_with_model,
262
+ image_pull_policy="IfNotPresent",
263
+ do_xcom_push=False,
264
+ startup_timeout_seconds=600,
265
+ is_delete_operator_pod=True,
266
+ get_logs=True,
267
+ container_resources=k8s.V1ResourceRequirements(
268
+ requests={"memory": "128Mi", "cpu": "100m"},
269
+ limits={"memory": "512Mi", "cpu": "400m"}
270
+ ),
271
+ affinity=affinity,
272
+ task_group=parent_group,
273
+ volumes=volumes, # This will be an empty list if dwh is None
274
+ volume_mounts=volume_mounts # This will be an empty list if dwh is None
275
+ )
276
+ return dbt_task
277
+
278
+ def create_task_groups(self, parent_group, fqn, node, task_name, task_alias, dbt_command, running_rule,
279
+ task_params):
280
+ """
281
+ Recursively creates task groups from FQN and adds the task to the final group.
282
+ """
283
+ if not fqn:
284
+ # Base case: If FQN is empty, add a task
285
+ if node not in self.dbt_tasks:
286
+ self.dbt_tasks[node] = self.create_dbt_kuberoperator_task(dbt_command=dbt_command,
287
+ running_rule=running_rule,
288
+ task_params=task_params,
289
+ node_name=task_name,
290
+ node_alias=task_alias,
291
+ parent_group=parent_group)
292
+ return
293
+
294
+ # Get the current level and create a subgroup
295
+ current_level = fqn[0]
296
+ if current_level != "re_data":
297
+ subgroup = getattr(parent_group, current_level, None)
298
+
299
+ # If the subgroup does not exist, create it
300
+ if not subgroup:
301
+ subgroup = TaskGroup(group_id=current_level, parent_group=parent_group)
302
+ setattr(parent_group, current_level, subgroup)
303
+
304
+ # Recurse into the next level
305
+ self.create_task_groups(subgroup, fqn[1:], node, task_name, task_alias, dbt_command, running_rule,
306
+ task_params)
307
+
308
+ def set_dependencies(self, resource_type):
309
+ """
310
+ Sets the dependencies between tasks based on the `depends_on` attribute in the manifest data.
311
+ """
312
+ for node in self.manifest_data.keys():
313
+ if resource_type in self.manifest_data[node]['group_type']:
314
+ # Only set dependencies for nodes that were actually created as tasks
315
+ if node not in self.dbt_tasks:
316
+ continue
317
+
318
+ for upstream_node in self.manifest_data[node].get("depends_on", []):
319
+ if self.dbt_tasks.get(upstream_node, []):
320
+ self.dbt_tasks[upstream_node] >> self.dbt_tasks[node]
321
+
322
+ def manifest_has_model_depends_on_snapshot(self):
323
+ """True when a selected model directly depends on a snapshot (batch build trigger)."""
324
+ return utils.manifest_has_model_depends_on_snapshot(self.manifest_data)
325
+
326
+ def get_model_names_for_resource_types(self, resource_types, task_params=None):
327
+ """
328
+ Collects node names from the filtered manifest for the given resource type(s).
329
+ Returns a space-separated string suitable for dbt --select, or None if empty.
330
+ """
331
+ if isinstance(resource_types, str):
332
+ resource_types = [resource_types]
333
+ resource_types = set(resource_types)
334
+
335
+ if task_params is None:
336
+ task_params = {}
337
+ task_params = {k: v for k, v in task_params.items() if v}
338
+
339
+ manifest = self.manifest_data
340
+ if "full_refresh_model_name" in task_params:
341
+ manifest = utils.filter_models(manifest, task_params["full_refresh_model_name"])
342
+
343
+ model_names = []
344
+ for node_id, node_data in manifest.items():
345
+ if node_data.get("resource_type") in resource_types:
346
+ model_names.append(node_data["name"])
347
+ return " ".join(model_names) if model_names else None
348
+
349
+ def get_model_names_for_resource_type(self, resource_type, task_params=None):
350
+ """
351
+ Collects all model names from the filtered manifest for a given resource type.
352
+ Returns a space-separated string suitable for dbt --select.
353
+ """
354
+ return self.get_model_names_for_resource_types([resource_type], task_params)
355
+
356
+ def create_dbt_batch_task(self, resource_type, dbt_command, running_rule, task_params=None):
357
+ """
358
+ Creates a single Airflow task that runs all models of the given resource_type
359
+ in one batch using dbt --select "model1 model2 model3 ...".
360
+ Bypasses per-model lineage — dbt handles internal dependency ordering.
361
+ """
362
+ select_string = self.get_model_names_for_resource_type(resource_type, task_params)
363
+ if not select_string:
364
+ return None
365
+
366
+ task_alias = f"{dbt_command}_all_models"
367
+ return self.create_dbt_kuberoperator_task(
368
+ dbt_command=dbt_command,
369
+ running_rule=running_rule,
370
+ task_params=task_params if task_params else {},
371
+ node_name=select_string,
372
+ node_alias=task_alias
373
+ )
374
+
375
+ def create_dbt_build_batch_task(self, running_rule, task_params=None,
376
+ resource_types=("model", "snapshot")):
377
+ """
378
+ Creates a single ``dbt build`` batch pod over models + snapshots, so a
379
+ ``model -> snapshot -> model`` chain runs in lineage order in one pod
380
+ (used in non-sharded mode when DBT_MODEL_DEPENDS_ON_SNAPSHOT is enabled).
381
+ """
382
+ select_string = self.get_model_names_for_resource_types(list(resource_types), task_params)
383
+ if not select_string:
384
+ return None
385
+
386
+ return self.create_dbt_kuberoperator_task(
387
+ dbt_command="build",
388
+ running_rule=running_rule,
389
+ task_params=task_params if task_params else {},
390
+ node_name=select_string,
391
+ node_alias="build_all_models"
392
+ )
393
+
394
+ def create_dbt_task_groups(
395
+ self,
396
+ group_name,
397
+ resource_type,
398
+ dbt_command,
399
+ running_rule,
400
+ task_params={}):
401
+ """
402
+ Parse out a JSON file and populates the task groups with dbt tasks
403
+ Args:
404
+ group_name: name of Task Groups uses for DAGs graph view in the Airflow UI.
405
+ resource_type: type of manifest nodes group: model, seed, snapshot
406
+ package_name: project name, tag that define by which certain records
407
+ from the manifest nodes will be selected
408
+ dbt_command: dbt command run or test
409
+ running_rule: trigger rule
410
+ task_params: parameters that was added in the task
411
+ Returns: task group
412
+ """
413
+ task_params = {k: v for k, v in task_params.items() if v} # Filter out empty values
414
+ if "full_refresh_model_name" in task_params:
415
+ self.manifest_data = utils.filter_models(self.manifest_data, task_params["full_refresh_model_name"])
416
+
417
+ if utils.is_resource_type_in_manifest(self.manifest_data, resource_type):
418
+ # Initialize the root TaskGroup from `group_name` (should be a TaskGroup instance, not a string)
419
+ with TaskGroup(group_id=group_name, parent_group=None) as root_group:
420
+ for node_id, node_data in self.manifest_data.items():
421
+ # Skip tests when creating source task groups - tests should run separately, not as part of source freshness
422
+ if resource_type == "source" and node_data.get('resource_type') == 'test':
423
+ continue
424
+ if group_name[:-1] in node_data["group_type"]:
425
+ # Extract FQN and task name
426
+ fqn = node_data["fqn"][:-1] # Remove model name from the FQN
427
+ task_name = node_data["name"]
428
+ task_alias = node_data["alias"]
429
+ # Use a per-node command so overrides (test/snapshot/source)
430
+ # do not leak into subsequent iterations of this loop.
431
+ node_command = dbt_command
432
+ if node_data['resource_type'] == "test":
433
+ node_command = "test"
434
+ elif node_data['resource_type'] == "snapshot":
435
+ # A snapshot woven into the model group (DBT_MODEL_DEPENDS_ON_SNAPSHOT)
436
+ # must still run as `dbt snapshot`, not `dbt run`.
437
+ node_command = "snapshot"
438
+ elif node_data['resource_type'] == "source":
439
+ task_name = f"source:{node_data['schema']}.{task_name}"
440
+ node_command = "source freshness"
441
+ # Create the dynamic task groups under root_group
442
+ self.create_task_groups(root_group, fqn, node_id, task_name, task_alias, node_command,
443
+ running_rule,
444
+ task_params)
445
+ self.set_dependencies(resource_type)
446
+ if not root_group.children:
447
+ return None
448
+ return root_group