fast2dev-dbt-runner 0.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,451 @@
1
+ import re
2
+ import json
3
+ import datetime
4
+
5
+
6
+ def days_ago(n, hour=0, minute=0, second=0, microsecond=0):
7
+ """Timezone-aware stand-in for Airflow 2's removed ``airflow.utils.dates.days_ago``.
8
+
9
+ Returns UTC midnight (or the given clock) ``n`` days before now. Never imports
10
+ ``airflow.utils.dates`` — that module still exists in Airflow 3.2 but no longer
11
+ exports ``days_ago``, which raises ``ImportError: cannot import name 'days_ago'``.
12
+ """
13
+ now = datetime.datetime.now(datetime.timezone.utc).replace(
14
+ hour=hour, minute=minute, second=second, microsecond=microsecond
15
+ )
16
+ return now - datetime.timedelta(days=int(n))
17
+
18
+
19
+ def get_valid_start_date(start_date_raw):
20
+ """
21
+ Tries to parse the START_DATE from Airflow Variables.
22
+ Supports:
23
+ - days_ago() function
24
+ - ISO format (YYYY-MM-DDTHH:MM:SS)
25
+ """
26
+ # Check if start_date_raw follows days_ago(N) pattern
27
+ if re.fullmatch(r"days_ago\(\d+\)", start_date_raw):
28
+ days_value = int(start_date_raw[9:-1]) # Extract the number from days_ago(N)
29
+ return days_ago(days_value)
30
+
31
+ # Check if start_date_raw follows the correct ISO format
32
+ try:
33
+ parsed = datetime.datetime.fromisoformat(start_date_raw)
34
+ except ValueError:
35
+ raise ValueError(
36
+ f"Invalid start_date format: {start_date_raw}. Must be ISO format (YYYY-MM-DDTHH:MM:SS) or 'days_ago(N)'.")
37
+ # date-only ISO ("2026-10-04") can yield datetime.date on some Python builds.
38
+ # DAG.start_date accepts datetime; DAG.params must never receive a date object.
39
+ if isinstance(parsed, datetime.date) and not isinstance(parsed, datetime.datetime):
40
+ parsed = datetime.datetime(parsed.year, parsed.month, parsed.day, tzinfo=datetime.timezone.utc)
41
+ elif parsed.tzinfo is None:
42
+ parsed = parsed.replace(tzinfo=datetime.timezone.utc)
43
+ return parsed
44
+
45
+
46
+ def json_safe_param(value):
47
+ """Coerce date/datetime values to ISO strings for Airflow 3 DAG.params.
48
+
49
+ Airflow 3 Param defaults must be JSON-serializable. ``datetime.date``
50
+ (including ``date.today()``) fails with ParamValidationError even though
51
+ its ``str()`` looks like ``'2026-10-04'``. Use this (or ``.isoformat()``)
52
+ for params; ``dag.start_date`` may stay a datetime.
53
+ """
54
+ if isinstance(value, datetime.datetime):
55
+ return value.date().isoformat()
56
+ if isinstance(value, datetime.date):
57
+ return value.isoformat()
58
+ return value
59
+
60
+
61
+ # CHANGE INPUT PARAM EVERYWHERE
62
+ def is_resource_type_in_manifest(manifest_data, resource_type):
63
+ if any(resource_type in sub_dict.values() for sub_dict in manifest_data.values()):
64
+ return 1
65
+
66
+
67
+ def check_dbt_tag(dbt_tag):
68
+ """
69
+ Cover next cases: ->>>> ADD TO DOC AND TO TEMPLATES
70
+ None->[]
71
+ ""->[]
72
+ "tag"->["tag"]
73
+ ["tag1", ""]->["tag1"]
74
+ ["", " "]->[]
75
+ """
76
+ if isinstance(dbt_tag, str):
77
+ dbt_tag = [dbt_tag.strip()] if dbt_tag.strip() else []
78
+ elif isinstance(dbt_tag, list):
79
+ dbt_tag = [tag.strip() for tag in dbt_tag if tag.strip()]
80
+ else:
81
+ dbt_tag = []
82
+ return dbt_tag
83
+
84
+ def to_bool(value):
85
+ """Convert string value to boolean."""
86
+ if isinstance(value, bool):
87
+ return value
88
+ if not isinstance(value, str):
89
+ raise ValueError(f"Expected string or bool, got {type(value)}")
90
+
91
+ true_vals = {"true", "1", "yes", "y", "on"}
92
+ false_vals = {"false", "0", "no", "n", "off"}
93
+
94
+ val = value.strip().lower()
95
+ if val in true_vals:
96
+ return True
97
+ elif val in false_vals:
98
+ return False
99
+ else:
100
+ raise ValueError(f"Cannot convert string to bool: {value}")
101
+
102
+ # CHANGE INPUT PARAM EVERYWHERE
103
+ def filter_models(manifest_data, models_list):
104
+ filtered_dict = {}
105
+ for i in manifest_data.keys():
106
+ if manifest_data[i]['name'] in models_list:
107
+ filtered_dict[i] = manifest_data[i]
108
+ if manifest_data[i]['depends_on'] and manifest_data[i]['resource_type'] == 'test':
109
+ for depends_on_model in manifest_data[i]['depends_on']:
110
+ if depends_on_model.split(".")[-1] in models_list:
111
+ filtered_dict[i] = manifest_data[i]
112
+ return filtered_dict
113
+
114
+
115
+ def change_to_test_in_models_depends_on(nodes_list):
116
+ models_tests_dict = {}
117
+ for k, v in nodes_list.items():
118
+ if v["resource_type"] == 'test':
119
+ for depends_on_model in v['depends_on']:
120
+ models_tests_dict.setdefault(depends_on_model, []).append(k)
121
+
122
+ for k, v in nodes_list.items():
123
+ temp_models_list = []
124
+ if v["resource_type"] != 'test':
125
+ for i in v['depends_on']:
126
+ if i.split(".")[1] != "re_data":
127
+ if i not in temp_models_list:
128
+ if i in models_tests_dict:
129
+ temp_models_list.extend(models_tests_dict[i])
130
+ else:
131
+ temp_models_list.append(i)
132
+ if temp_models_list:
133
+ v['depends_on'] = list(set(temp_models_list))
134
+ return nodes_list
135
+
136
+
137
+ def weave_snapshots_into_model_group(nodes_list):
138
+ """
139
+ Re-group snapshots (and their tests) that a selected model depends on so they
140
+ run interleaved inside the 'model' task group instead of the separate snapshot
141
+ phase.
142
+
143
+ This mirrors how tests are already interleaved: grouping is driven by
144
+ ``group_type`` rather than ``resource_type``. A snapshot is considered
145
+ "model-blocking" when some selected model has the snapshot's node id in its
146
+ raw ``depends_on``. For each such snapshot we add ``"model"`` to its
147
+ ``group_type`` (and drop ``"snapshot"`` so it is not also built in the
148
+ snapshot phase). Tests that test those snapshots are moved too, so the
149
+ ``snapshot -> its tests -> downstream model`` ordering is preserved (matching
150
+ ``dbt build`` semantics).
151
+
152
+ NOTE: must run on the raw ``depends_on`` (before
153
+ ``change_to_test_in_models_depends_on`` rewrites model deps to point at tests),
154
+ otherwise a snapshot that has tests would no longer be visible as a direct
155
+ model dependency.
156
+ """
157
+ snapshot_ids = {k for k, v in nodes_list.items() if v["resource_type"] == "snapshot"}
158
+ if not snapshot_ids:
159
+ return nodes_list
160
+
161
+ blocking = set()
162
+ for v in nodes_list.values():
163
+ if v["resource_type"] == "model":
164
+ blocking.update(dep for dep in v["depends_on"] if dep in snapshot_ids)
165
+ if not blocking:
166
+ return nodes_list
167
+
168
+ test_ids = {
169
+ k for k, v in nodes_list.items()
170
+ if v["resource_type"] == "test" and any(dep in blocking for dep in v["depends_on"])
171
+ }
172
+
173
+ for nid in blocking | test_ids:
174
+ group_type = nodes_list[nid]["group_type"]
175
+ if "model" not in group_type:
176
+ group_type.append("model")
177
+ if "snapshot" in group_type:
178
+ group_type.remove("snapshot")
179
+ return nodes_list
180
+
181
+
182
+ def manifest_has_model_depends_on_snapshot(nodes_list):
183
+ """
184
+ Return True when some selected model directly depends on a snapshot.
185
+
186
+ Used to decide whether the batch (non-sharded) path must switch from the
187
+ default ``dbt run`` + ``dbt snapshot`` phases to a single ``dbt build`` so the
188
+ ``model -> snapshot -> model`` ordering is honoured. Detection is on
189
+ ``resource_type`` (unchanged by weaving) so it works whether or not the
190
+ snapshot has been woven into the model group.
191
+ """
192
+ snapshot_ids = {k for k, v in nodes_list.items() if v.get("resource_type") == "snapshot"}
193
+ if not snapshot_ids:
194
+ return False
195
+ for v in nodes_list.values():
196
+ if v.get("resource_type") == "model":
197
+ if any(dep in snapshot_ids for dep in v.get("depends_on", [])):
198
+ return True
199
+ return False
200
+
201
+
202
+ def get_file_tests(nodes_list):
203
+ for k, v in nodes_list.items():
204
+ if v["resource_type"] == 'test':
205
+ for i in v.get('depends_on'):
206
+ if v.get('file_key_name'):
207
+ if v.get('file_key_name') not in i.split("."):
208
+ v['depends_on'].remove(i)
209
+ v['group_type'].remove(i.split(".")[0])
210
+ return nodes_list
211
+
212
+
213
+ def check_node_cycle_for_tests(nodes_list):
214
+ tmp_list = []
215
+ new_depends_on_list = []
216
+ for k, v in nodes_list.items():
217
+ if v["resource_type"] == 'test':
218
+
219
+ has_model = any(node.startswith("model") for node in v["depends_on"])
220
+ has_source = any(node.startswith("source") for node in v["depends_on"])
221
+ if has_model and has_source:
222
+ v['depends_on'] = [node for node in v['depends_on'] if not node.startswith("source")]
223
+
224
+ for node in v["depends_on"]:
225
+ for parent_node_k, parent_node_v in nodes_list.items():
226
+ if parent_node_k == node:
227
+ for item in parent_node_v["depends_on"]:
228
+ if item not in tmp_list:
229
+ tmp_list.append(item)
230
+
231
+ for node in v["depends_on"]:
232
+ if node not in tmp_list:
233
+ new_depends_on_list.append(node)
234
+ v["depends_on"] = new_depends_on_list
235
+ new_depends_on_list = []
236
+ tmp_list = []
237
+ return nodes_list
238
+
239
+
240
+ # NEED TO BE BEFORE change_to_test_in_models_depends_on
241
+ def remove_ephemeral_dependencies(manifest_data_transformed):
242
+ # Identify ephemeral models
243
+ ephemeral_models = {
244
+ node_id for node_id, node in manifest_data_transformed.items()
245
+ if node.get("resource_type") == "model" and node.get("materialized", {}) == "ephemeral"
246
+ }
247
+
248
+ # Build a lookup for ephemeral -> its dependencies (parents)
249
+ ephemeral_to_parents = {
250
+ node_id: node.get("depends_on", {})
251
+ for node_id, node in manifest_data_transformed.items()
252
+ if node_id in ephemeral_models
253
+ }
254
+
255
+ # Remove ephemeral models tests
256
+ tests_to_remove = {
257
+ node_id for node_id, node in manifest_data_transformed.items()
258
+ if node.get("resource_type") == "test" and any(
259
+ dep in ephemeral_models for dep in manifest_data_transformed[node_id].get("depends_on"))
260
+
261
+ }
262
+ # Replace ephemeral dependencies
263
+ for node_id, node in manifest_data_transformed.items():
264
+ new_deps = []
265
+ for dep in node["depends_on"]:
266
+ if dep in ephemeral_models:
267
+ # Replace with ephemeral's parents (if they exist)
268
+ lifted = ephemeral_to_parents.get(dep, [])
269
+ if lifted:
270
+ new_deps.extend(lifted)
271
+ else:
272
+ new_deps.append(dep)
273
+ # Deduplicate dependencies
274
+ node["depends_on"] = list(set(new_deps))
275
+
276
+ # Delete ephemeral models and their tests
277
+ nodes_to_remove = ephemeral_models.union(tests_to_remove)
278
+ cleaned_manifest = {
279
+ node_id: node for node_id, node in manifest_data_transformed.items()
280
+ if node_id not in nodes_to_remove
281
+ }
282
+ return cleaned_manifest
283
+
284
+
285
+ def filter_tasks_by_tag(manifest_data, dbt_tag):
286
+ return {k: v for k, v in manifest_data.items() if any(tag in v["tags"] for tag in dbt_tag)}
287
+
288
+
289
+ def get_tests_for_tagged_models(manifest_data, filtered_manifest):
290
+ dependent_node_list = {}
291
+ for node_id, node in manifest_data.items():
292
+ if node.get('resource_type') == 'test':
293
+ for dependent_node in node.get('depends_on'):
294
+ for filtered_item in filtered_manifest.keys():
295
+ if filtered_item == dependent_node:
296
+ if node_id not in dependent_node_list.keys():
297
+ dependent_node_list[node_id] = node
298
+ result = filtered_manifest | dependent_node_list
299
+ return result
300
+
301
+
302
+ def collect_relatives(manifest_data, model_id, relatives_map, visited):
303
+ # Recursively collect all relatives
304
+ for item in relatives_map.get(model_id, []):
305
+ if item not in visited and item in manifest_data:
306
+ visited.add(item)
307
+ collect_relatives(manifest_data, item, relatives_map, visited)
308
+
309
+
310
+ def get_tagged_models(manifest_data, relatives_map, dbt_tag):
311
+ # Find models with the given tag
312
+ filtered_tasks = filter_tasks_by_tag(manifest_data, dbt_tag)
313
+ tagged_models = list(filtered_tasks.keys())
314
+ result = set(tagged_models)
315
+ for model in tagged_models:
316
+ collect_relatives(manifest_data, model, relatives_map, result)
317
+ return {node_id: manifest_data[node_id] for node_id in result}
318
+
319
+
320
+ def get_models_with_tag_and_parents(manifest_data, dbt_tag):
321
+ # Take parents
322
+ parent_map = {
323
+ node_id: node.get("depends_on", {})
324
+ for node_id, node in manifest_data.items()
325
+ }
326
+ result = get_tagged_models(manifest_data, parent_map, dbt_tag)
327
+ result_with_test = get_tests_for_tagged_models(manifest_data, result)
328
+ return result_with_test
329
+
330
+
331
+ def get_models_with_tag_and_children(manifest_data, dbt_tag):
332
+ # Build a child map: for each model, who depends on it
333
+ child_map = {}
334
+ for node_id, node in manifest_data.items():
335
+ for dep in node.get("depends_on", {}):
336
+ if dep not in child_map:
337
+ child_map[dep] = []
338
+ child_map[dep].append(node_id)
339
+ result = get_tagged_models(manifest_data, child_map, dbt_tag)
340
+ result_with_test = get_tests_for_tagged_models(manifest_data, result)
341
+ return result_with_test
342
+
343
+
344
+ def get_models_with_tag(manifest_data, dbt_tag):
345
+ filtered_tasks = filter_tasks_by_tag(manifest_data, dbt_tag)
346
+ result_with_test = get_tests_for_tagged_models(manifest_data, filtered_tasks)
347
+ return result_with_test
348
+
349
+
350
+ def change_macros_dependencies_to_source_dependencies(current_node, source_dict):
351
+ depends_list = []
352
+ depends_list_full_path = []
353
+ if current_node['config'].get('depends_on'):
354
+ depends_list = depends_list + current_node['config'].get('depends_on')
355
+
356
+ if current_node["depends_on"].get("macros"):
357
+ for i in current_node["depends_on"].get("macros"):
358
+ if i and i.split('.')[-1] == "generate_columns_from_airbyte_yml":
359
+ source_table_name = current_node["name"].replace("stg", "raw")
360
+ depends_list.append(source_table_name)
361
+ depends_list = list(set(depends_list))
362
+ for key in depends_list:
363
+ for k, v in source_dict.items():
364
+ if key in v['name']:
365
+ depends_list_full_path.append(k)
366
+ if current_node["depends_on"].get("nodes"):
367
+ depends_list_full_path = list(set(depends_list_full_path + current_node["depends_on"].get("nodes")))
368
+ return depends_list_full_path
369
+
370
+
371
+ def load_dbt_manifest(manifest_path,
372
+ dbt_tag=[],
373
+ dbt_tag_ancestors=False,
374
+ dbt_tag_descendants=False,
375
+ dbt_model_depends_on_snapshot=False):
376
+ """
377
+ Helper function to load the dbt manifest file.
378
+ Returns: A JSON object containing the dbt manifest content.
379
+ """
380
+ with open(manifest_path, encoding="utf-8") as file:
381
+ file_content = json.load(file)
382
+
383
+ node_dependency = {
384
+ k: v
385
+ for k, v in file_content["nodes"].items()
386
+ if k.split(".")[0] in ["model", "seed", "snapshot", "test", "source"]}
387
+
388
+ get_sources = {
389
+ k: v
390
+ for k, v in file_content["sources"].items()
391
+ if k.split(".")[0] in ["source"] and (v.get("freshness")
392
+ and ((v.get("freshness").get("warn_after")
393
+ and v.get("freshness").get("warn_after").get("count"))
394
+ or (v.get("freshness").get("error_after")
395
+ and v.get("freshness").get("error_after").get("count"))))
396
+ }
397
+
398
+ get_source_new_dict = {
399
+ k: {
400
+ "name": v["name"],
401
+ "alias": v["name"],
402
+ "package_name": v["package_name"],
403
+ "resource_type": v["resource_type"],
404
+ "schema": v["schema"],
405
+ "fqn": v["fqn"],
406
+ "group_type": ["source"],
407
+ "depends_on": [],
408
+ "tags": v["tags"],
409
+ "file_key_name": ""
410
+ }
411
+ for k, v in get_sources.items()}
412
+
413
+ node_dependency_unique = {
414
+ k: {
415
+ "name": v["name"],
416
+ "alias": v["alias"],
417
+ "package_name": v["package_name"],
418
+ "resource_type": v["resource_type"],
419
+ "materialized": v.get("config").get("materialized"),
420
+ "schema": v["schema"],
421
+ "fqn": v["fqn"],
422
+ "group_type": [v["resource_type"]] if v["resource_type"] != "test" else list(
423
+ set([i.split('.')[0] for i in v["depends_on"].get("nodes", [])])),
424
+ "depends_on": change_macros_dependencies_to_source_dependencies(v, get_source_new_dict),
425
+ "tags": v["tags"],
426
+ "file_key_name": v.get("file_key_name", "").split(".")[-1]
427
+ }
428
+ for k, v in node_dependency.items()
429
+ if 'depends_on' in v}
430
+
431
+ node_dependency_unique = check_node_cycle_for_tests(node_dependency_unique)
432
+ node_dependency_unique = {**node_dependency_unique, **get_source_new_dict}
433
+ if dbt_tag:
434
+ if to_bool(dbt_tag_ancestors) and not to_bool(dbt_tag_descendants):
435
+ node_dependency_unique_filtered = get_models_with_tag_and_parents(node_dependency_unique, dbt_tag)
436
+ if to_bool(dbt_tag_descendants) and not to_bool(dbt_tag_ancestors):
437
+ node_dependency_unique_filtered = get_models_with_tag_and_children(node_dependency_unique, dbt_tag)
438
+ if to_bool(dbt_tag_ancestors) and to_bool(dbt_tag_descendants):
439
+ node_dependency_unique_filtered = {**get_models_with_tag_and_parents(node_dependency_unique, dbt_tag), **get_models_with_tag_and_children(node_dependency_unique, dbt_tag)}
440
+ if not to_bool(dbt_tag_ancestors) and not to_bool(dbt_tag_descendants):
441
+ node_dependency_unique_filtered = get_models_with_tag(node_dependency_unique, dbt_tag)
442
+ else:
443
+ node_dependency_unique_filtered = node_dependency_unique
444
+
445
+ node_without_ephemeral = remove_ephemeral_dependencies(node_dependency_unique_filtered)
446
+ node_with_get_file = get_file_tests(node_without_ephemeral)
447
+ if to_bool(dbt_model_depends_on_snapshot):
448
+ node_with_get_file = weave_snapshots_into_model_group(node_with_get_file)
449
+ result = change_to_test_in_models_depends_on(node_with_get_file)
450
+
451
+ return result