fast2dev-dbt-runner 0.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,248 @@
1
+ import os
2
+ from fast2dev_dbt_runner._compat import BaseOperator
3
+ from fast2dev_dbt_runner.bash_operator.dbt_hook import DbtCliHook
4
+
5
+ class DbtBaseOperator(BaseOperator):
6
+ """
7
+ Base dbt operator
8
+ All other dbt operators are derived from this operator.
9
+
10
+ :param env: If set, passes the env variables to the subprocess handler
11
+ :type env: dict
12
+ :param profiles_dir: If set, passed as the `--profiles-dir` argument to the `dbt` command
13
+ :type profiles_dir: str
14
+ :param target: If set, passed as the `--target` argument to the `dbt` command
15
+ :type dir: str
16
+ :param dir: The directory to run the CLI in
17
+ :type vars: str
18
+ :param vars: If set, passed as the `--vars` argument to the `dbt` command
19
+ :type vars: dict
20
+ :param full_refresh: If `True`, will fully-refresh incremental models.
21
+ :type full_refresh: bool
22
+ :param models: If set, passed as the `--models` argument to the `dbt` command
23
+ :type models: str
24
+ :param warn_error: If `True`, treat warnings as errors.
25
+ :type warn_error: bool
26
+ :param exclude: If set, passed as the `--exclude` argument to the `dbt` command
27
+ :type exclude: str
28
+ :param select: If set, passed as the `--select` argument to the `dbt` command
29
+ :type select: str
30
+ :param selector: If set, passed as the `--selector` argument to the `dbt` command
31
+ :type selector: str
32
+ :param dbt_bin: The `dbt` CLI. Defaults to `dbt`, so assumes it's on your `PATH`
33
+ :type dbt_bin: str
34
+ :param verbose: The operator will log verbosely to the Airflow logs
35
+ :type verbose: bool
36
+ :param warehouse_type: The type of data warehouse to use (bigquery, snowflake, redshift, fabric)
37
+ :type warehouse_type: str
38
+ :param debug: If `True`, passes the debug flag to the DbtCliHook
39
+ :type debug: bool
40
+ :param empty: If `True`, pass `--empty` to dbt run (E2E empty mode). Run command only.
41
+ :type empty: bool
42
+ """
43
+
44
+ def __init__(self,
45
+ env=None,
46
+ profiles_dir=None,
47
+ target=None,
48
+ git_branch=None,
49
+ dbt_project_dir='',
50
+ vars=None,
51
+ models=None,
52
+ exclude=None,
53
+ select=None,
54
+ selector=None,
55
+ dbt_bin='/home/airflow/.local/bin/dbt',
56
+ verbose=True,
57
+ warn_error=False,
58
+ full_refresh=False,
59
+ data=False,
60
+ schema=False,
61
+ warehouse_type=None,
62
+ debug=False,
63
+ empty=False,
64
+ *args,
65
+ **kwargs):
66
+ super(DbtBaseOperator, self).__init__(*args, **kwargs)
67
+
68
+ # Initialize environment variables
69
+ self.env = env or {}
70
+
71
+ # Check for DATA_WAREHOUSE_PLATFORM in environment
72
+ dwh_platform = os.environ.get("DATA_WAREHOUSE_PLATFORM")
73
+ if dwh_platform and not warehouse_type:
74
+ if debug:
75
+ self.log.info(f"Using DATA_WAREHOUSE_PLATFORM={dwh_platform} for warehouse type")
76
+
77
+ self.profiles_dir = profiles_dir
78
+ self.target = target
79
+ self.git_branch = git_branch
80
+ self.dbt_project_dir = dbt_project_dir
81
+ self.vars = vars
82
+ self.models = models
83
+ self.full_refresh = full_refresh
84
+ self.data = data
85
+ self.schema = schema
86
+ self.exclude = exclude
87
+ self.select = select
88
+ self.selector = selector
89
+ self.dbt_bin = dbt_bin
90
+ self.verbose = verbose
91
+ self.warn_error = warn_error
92
+ self.warehouse_type = warehouse_type
93
+ self.debug = debug
94
+ self.empty = empty
95
+ self.create_hook()
96
+
97
+ def create_hook(self):
98
+ self.hook = DbtCliHook(
99
+ env=self.env,
100
+ profiles_dir=self.profiles_dir,
101
+ target=self.target,
102
+ git_branch=self.git_branch,
103
+ dbt_project_dir=self.dbt_project_dir,
104
+ vars=self.vars,
105
+ full_refresh=self.full_refresh,
106
+ data=self.data,
107
+ schema=self.schema,
108
+ models=self.models,
109
+ exclude=self.exclude,
110
+ select=self.select,
111
+ selector=self.selector,
112
+ dbt_bin=self.dbt_bin,
113
+ verbose=self.verbose,
114
+ warn_error=self.warn_error,
115
+ warehouse_type=self.warehouse_type,
116
+ debug=self.debug,
117
+ dag_id=getattr(self, 'dag_id', None),
118
+ task_id=getattr(self, 'task_id', None),
119
+ empty=self.empty)
120
+
121
+ return self.hook
122
+
123
+ class DbtRunOperator(DbtBaseOperator):
124
+ def __init__(self, dbt_project_dir, profiles_dir=None, target=None, git_branch=None, *args, **kwargs):
125
+ super(DbtRunOperator, self).__init__(dbt_project_dir=dbt_project_dir,
126
+ profiles_dir=profiles_dir,
127
+ target=target,
128
+ git_branch=git_branch,
129
+ *args, **kwargs)
130
+
131
+ def execute(self, context):
132
+ full_refresh = context['ti'].xcom_pull(task_ids="show_input_data", key="full_refresh_model")
133
+ execution_date = context['ti'].xcom_pull(task_ids="show_input_data", key="execution_date")
134
+ if str(full_refresh) == 'True':
135
+ self.full_refresh = True
136
+
137
+ self.vars = {"execution_date": execution_date}
138
+ self.create_hook().run_cli('run')
139
+
140
+
141
+ class DbtBuildOperator(DbtBaseOperator):
142
+ """
143
+ Runs ``dbt build`` for the provided selection.
144
+
145
+ Unlike ``DbtRunOperator``, ``dbt build`` runs the full pipeline for the
146
+ selected nodes (models, snapshots, seeds and their tests) in a single process
147
+ and in lineage order. This is what makes a ``model -> snapshot -> model`` chain
148
+ resolve correctly in a single batch task (see DBT_MODEL_DEPENDS_ON_SNAPSHOT in
149
+ batch mode).
150
+ """
151
+ def __init__(self, dbt_project_dir, profiles_dir=None, target=None, git_branch=None, *args, **kwargs):
152
+ super(DbtBuildOperator, self).__init__(dbt_project_dir=dbt_project_dir,
153
+ profiles_dir=profiles_dir,
154
+ target=target,
155
+ git_branch=git_branch,
156
+ *args, **kwargs)
157
+
158
+ def execute(self, context):
159
+ full_refresh = context['ti'].xcom_pull(task_ids="show_input_data", key="full_refresh_model")
160
+ execution_date = context['ti'].xcom_pull(task_ids="show_input_data", key="execution_date")
161
+ if str(full_refresh) == 'True':
162
+ self.full_refresh = True
163
+
164
+ self.vars = {"execution_date": execution_date}
165
+ self.create_hook().run_cli('build')
166
+
167
+
168
+ class DbtTestOperator(DbtBaseOperator):
169
+ def __init__(self, dbt_project_dir, profiles_dir=None, target=None, git_branch=None, *args, **kwargs):
170
+ super(DbtTestOperator, self).__init__(dbt_project_dir=dbt_project_dir,
171
+ profiles_dir=profiles_dir,
172
+ target=target,
173
+ git_branch=git_branch,*args, **kwargs)
174
+
175
+ def execute(self, context):
176
+ self.create_hook().run_cli('test')
177
+
178
+
179
+ class DbtSourceFreshnessOperator(DbtBaseOperator):
180
+ def __init__(self, dbt_project_dir, profiles_dir=None, target=None, *args, **kwargs):
181
+ super(DbtSourceFreshnessOperator, self).__init__(dbt_project_dir=dbt_project_dir, profiles_dir=profiles_dir, target=target, *args, **kwargs)
182
+
183
+ def execute(self, context):
184
+ self.create_hook().run_cli('source freshness')
185
+
186
+ class DbtReDataOperator(DbtBaseOperator):
187
+ def __init__(self, dbt_project_dir, profiles_dir=None, target=None, *args, **kwargs):
188
+ super(DbtReDataOperator, self).__init__(dbt_project_dir=dbt_project_dir, profiles_dir=profiles_dir, target=target, *args, **kwargs)
189
+
190
+ def execute(self, context):
191
+ self.create_hook().run_cli('run --select package:re_data')
192
+
193
+
194
+ class DbtDocsGenerateOperator(DbtBaseOperator):
195
+ def __init__(self, dbt_project_dir, profiles_dir=None, target=None, *args, **kwargs):
196
+ super(DbtDocsGenerateOperator, self).__init__(dbt_project_dir=dbt_project_dir, profiles_dir=profiles_dir, target=target, *args,
197
+ **kwargs)
198
+
199
+ def execute(self, context):
200
+ self.create_hook().run_cli('docs', 'generate')
201
+
202
+
203
+ class DbtSnapshotOperator(DbtBaseOperator):
204
+ def __init__(self, dbt_project_dir, profiles_dir=None, target=None, *args, **kwargs):
205
+ super(DbtSnapshotOperator, self).__init__(dbt_project_dir=dbt_project_dir, profiles_dir=profiles_dir, target=target, *args, **kwargs)
206
+
207
+ def execute(self, context):
208
+ self.create_hook().run_cli('snapshot')
209
+
210
+
211
+ class DbtSeedOperator(DbtBaseOperator):
212
+ def __init__(self, dbt_project_dir, profiles_dir=None, target=None, *args, **kwargs):
213
+ super(DbtSeedOperator, self).__init__(dbt_project_dir=dbt_project_dir, profiles_dir=profiles_dir, target=target, *args, **kwargs)
214
+
215
+ def execute(self, context):
216
+ full_refresh = context['ti'].xcom_pull(task_ids="show_input_data", key="full_refresh_seed")
217
+ if str(full_refresh) == 'True':
218
+ self.full_refresh = True
219
+
220
+ self.create_hook().run_cli('seed')
221
+
222
+
223
+ class DbtDepsOperator(DbtBaseOperator):
224
+ def __init__(self, dbt_project_dir, profiles_dir=None, target=None, *args, **kwargs):
225
+ super(DbtDepsOperator, self).__init__(dbt_project_dir=dbt_project_dir, profiles_dir=profiles_dir, target=target, *args, **kwargs)
226
+
227
+ def execute(self, context):
228
+ self.create_hook().run_cli('deps')
229
+
230
+
231
+ class DbtCleanOperator(DbtBaseOperator):
232
+ def __init__(self, dbt_project_dir, profiles_dir=None, target=None, *args, **kwargs):
233
+ super(DbtCleanOperator, self).__init__(dbt_project_dir=dbt_project_dir, profiles_dir=profiles_dir, target=target, *args, **kwargs)
234
+
235
+ def execute(self, context):
236
+ self.create_hook().run_cli('clean')
237
+
238
+
239
+ class DbtDebugOperator(DbtBaseOperator):
240
+ def __init__(self, dbt_project_dir, profiles_dir=None, target=None, git_branch=None, *args, **kwargs):
241
+ super(DbtDebugOperator, self).__init__(dbt_project_dir=dbt_project_dir,
242
+ profiles_dir=profiles_dir,
243
+ target=target,
244
+ git_branch=git_branch,
245
+ *args, **kwargs)
246
+
247
+ def execute(self, context):
248
+ self.create_hook().run_cli('debug')
@@ -0,0 +1,275 @@
1
+ """
2
+ Cached manifest loader for dbt manifest files.
3
+
4
+ This module provides a cached version of load_dbt_manifest that uses file hashing
5
+ to avoid redundant parsing of manifest files. The cache is stored at module level
6
+ and persists across DAG imports within the same Python process.
7
+
8
+ Performance: ~99% cache hit rate in production, reducing DAG import time from 2-4s to <10ms.
9
+ """
10
+ import copy
11
+ import hashlib
12
+ import logging
13
+ import os
14
+ import threading
15
+ from typing import Dict, Any, List, Optional, Tuple
16
+ import fast2dev_dbt_runner.utils as utils
17
+
18
+ # Module-level cache storage (persists across DAG imports in same process)
19
+ _MANIFEST_CACHE: Dict[Tuple, Dict[str, Any]] = {}
20
+ _CACHE_LOCK = threading.Lock()
21
+ _CACHE_STATS = {"hits": 0, "misses": 0, "errors": 0}
22
+
23
+ # Configuration from environment variables
24
+ _CACHE_ENABLED = os.getenv("AIRFLOW__CORE__MANIFEST_CACHE_ENABLED", "True").lower() == "true"
25
+ _CACHE_DEBUG = os.getenv("AIRFLOW__CORE__MANIFEST_CACHE_DEBUG", "False").lower() == "true"
26
+ _CACHE_MAX_SIZE = int(os.getenv("AIRFLOW__CORE__MANIFEST_CACHE_MAX_SIZE", "50"))
27
+
28
+ log = logging.getLogger(__name__)
29
+
30
+
31
+ def _get_file_hash(manifest_path: str) -> str:
32
+ """
33
+ Calculate MD5 hash of manifest file.
34
+
35
+ Reads file in chunks to handle large files efficiently without loading
36
+ entire file into memory.
37
+
38
+ Args:
39
+ manifest_path: Path to the manifest JSON file
40
+
41
+ Returns:
42
+ MD5 hash of file contents as hex string
43
+ """
44
+ hash_md5 = hashlib.md5()
45
+
46
+ try:
47
+ with open(manifest_path, "rb") as f:
48
+ # Read in 64KB chunks for efficiency
49
+ for chunk in iter(lambda: f.read(65536), b""):
50
+ hash_md5.update(chunk)
51
+ return hash_md5.hexdigest()
52
+ except Exception as e:
53
+ log.error(f"Error calculating hash for {manifest_path}: {e}")
54
+ # Return a unique value that won't match any cache
55
+ return f"error_{id(e)}"
56
+
57
+
58
+ def _norm_bool(value: Any) -> bool:
59
+ """
60
+ Normalize a flag to a canonical bool for cache-key construction.
61
+
62
+ Airflow variables may arrive as real bools or as strings ("True"/"false"/...).
63
+ Normalizing here guarantees that semantically-equal values (e.g. True and
64
+ "true") collapse to a single cache entry instead of creating duplicates or,
65
+ worse, reusing an entry built under a different semantic value.
66
+ """
67
+ try:
68
+ return utils.to_bool(value)
69
+ except Exception:
70
+ return bool(value)
71
+
72
+
73
+ def _create_cache_key(
74
+ manifest_path: str,
75
+ file_hash: str,
76
+ dbt_tag: List[str],
77
+ dbt_tag_ancestors: bool,
78
+ dbt_tag_descendants: bool,
79
+ dbt_model_depends_on_snapshot: bool = False
80
+ ) -> Tuple:
81
+ """
82
+ Create a cache key from manifest parameters.
83
+
84
+ The cache key includes:
85
+ - File hash (changes only when file content changes)
86
+ - DBT tags (different tags = different filtered results)
87
+ - Ancestor/descendant flags (affects dependency resolution)
88
+ - Model-depends-on-snapshot flag (changes node grouping)
89
+
90
+ Args:
91
+ manifest_path: Path to manifest file (for logging)
92
+ file_hash: MD5 hash of file contents
93
+ dbt_tag: List of DBT tags to filter
94
+ dbt_tag_ancestors: Include ancestors of tagged models
95
+ dbt_tag_descendants: Include descendants of tagged models
96
+ dbt_model_depends_on_snapshot: Weave snapshots into the model group
97
+
98
+ Returns:
99
+ Tuple suitable for use as dict key
100
+ """
101
+ # Use frozenset for tags so order doesn't matter
102
+ tags_key = frozenset(dbt_tag) if dbt_tag else frozenset()
103
+
104
+ return (
105
+ file_hash,
106
+ tags_key,
107
+ _norm_bool(dbt_tag_ancestors),
108
+ _norm_bool(dbt_tag_descendants),
109
+ _norm_bool(dbt_model_depends_on_snapshot)
110
+ )
111
+
112
+
113
+ def _evict_lru_cache_entries() -> None:
114
+ """
115
+ Evict least recently used cache entries if cache is too large.
116
+
117
+ This is called when cache size exceeds MAX_SIZE. We don't track LRU properly
118
+ for simplicity, so we just clear the oldest half of entries.
119
+ """
120
+ if len(_MANIFEST_CACHE) > _CACHE_MAX_SIZE:
121
+ # Simple strategy: keep newest half based on insertion order
122
+ entries_to_keep = _CACHE_MAX_SIZE // 2
123
+ keys_to_remove = list(_MANIFEST_CACHE.keys())[:-entries_to_keep]
124
+
125
+ for key in keys_to_remove:
126
+ del _MANIFEST_CACHE[key]
127
+
128
+ log.info(f"Evicted {len(keys_to_remove)} entries from manifest cache. "
129
+ f"Cache size: {len(_MANIFEST_CACHE)}/{_CACHE_MAX_SIZE}")
130
+
131
+
132
+ def load_dbt_manifest_cached(
133
+ manifest_path: str,
134
+ dbt_tag: Optional[List[str]] = None,
135
+ dbt_tag_ancestors: bool = False,
136
+ dbt_tag_descendants: bool = False,
137
+ dbt_model_depends_on_snapshot: bool = False
138
+ ) -> Dict[str, Any]:
139
+ """
140
+ Load and parse dbt manifest with caching.
141
+
142
+ This is a drop-in replacement for utils.load_dbt_manifest() that adds
143
+ file hash-based caching. The manifest is only re-parsed when the file
144
+ content actually changes.
145
+
146
+ Args:
147
+ manifest_path: Path to the manifest JSON file
148
+ dbt_tag: List of DBT tags to filter (default: [])
149
+ dbt_tag_ancestors: Include ancestors of tagged models (default: False)
150
+ dbt_tag_descendants: Include descendants of tagged models (default: False)
151
+ dbt_model_depends_on_snapshot: Weave snapshots into the model group (default: False)
152
+
153
+ Returns:
154
+ Parsed manifest data dictionary. A deep copy is returned so callers may
155
+ freely mutate the result without corrupting the shared cached object.
156
+
157
+ Performance:
158
+ - Cache hit: <10ms
159
+ - Cache miss: 2-4 seconds (full parse)
160
+ - Expected hit rate: >99% in production
161
+ """
162
+ # Handle default values
163
+ if dbt_tag is None:
164
+ dbt_tag = []
165
+
166
+ # If caching is disabled, call original function directly
167
+ if not _CACHE_ENABLED:
168
+ return utils.load_dbt_manifest(
169
+ manifest_path,
170
+ dbt_tag=dbt_tag,
171
+ dbt_tag_ancestors=dbt_tag_ancestors,
172
+ dbt_tag_descendants=dbt_tag_descendants,
173
+ dbt_model_depends_on_snapshot=dbt_model_depends_on_snapshot
174
+ )
175
+
176
+ # Calculate file hash
177
+ file_hash = _get_file_hash(manifest_path)
178
+ cache_key = _create_cache_key(
179
+ manifest_path,
180
+ file_hash,
181
+ dbt_tag,
182
+ dbt_tag_ancestors,
183
+ dbt_tag_descendants,
184
+ dbt_model_depends_on_snapshot
185
+ )
186
+
187
+ # Thread-safe cache lookup
188
+ with _CACHE_LOCK:
189
+ if cache_key in _MANIFEST_CACHE:
190
+ # CACHE HIT - return cached data
191
+ _CACHE_STATS["hits"] += 1
192
+
193
+ if _CACHE_DEBUG:
194
+ hit_rate = (_CACHE_STATS["hits"] /
195
+ (_CACHE_STATS["hits"] + _CACHE_STATS["misses"]) * 100)
196
+ log.info(
197
+ f"Manifest cache HIT for {os.path.basename(manifest_path)} "
198
+ f"(hash: {file_hash[:8]}..., tags: {dbt_tag}, "
199
+ f"cache_size: {len(_MANIFEST_CACHE)}, "
200
+ f"hit_rate: {hit_rate:.1f}%)"
201
+ )
202
+
203
+ # Return an owned copy so caller-side mutation can't corrupt the cache.
204
+ return copy.deepcopy(_MANIFEST_CACHE[cache_key])
205
+
206
+ # CACHE MISS - parse manifest
207
+ _CACHE_STATS["misses"] += 1
208
+
209
+ if _CACHE_DEBUG:
210
+ log.info(
211
+ f"Manifest cache MISS for {os.path.basename(manifest_path)} "
212
+ f"(hash: {file_hash[:8]}..., tags: {dbt_tag}) - parsing..."
213
+ )
214
+
215
+ try:
216
+ # Parse manifest using original function
217
+ result = utils.load_dbt_manifest(
218
+ manifest_path,
219
+ dbt_tag=dbt_tag,
220
+ dbt_tag_ancestors=dbt_tag_ancestors,
221
+ dbt_tag_descendants=dbt_tag_descendants,
222
+ dbt_model_depends_on_snapshot=dbt_model_depends_on_snapshot
223
+ )
224
+
225
+ # Store in cache (thread-safe)
226
+ with _CACHE_LOCK:
227
+ _MANIFEST_CACHE[cache_key] = result
228
+
229
+ # Evict old entries if cache is too large
230
+ if len(_MANIFEST_CACHE) > _CACHE_MAX_SIZE:
231
+ _evict_lru_cache_entries()
232
+
233
+ if _CACHE_DEBUG:
234
+ log.info(
235
+ f"Manifest cached successfully for {os.path.basename(manifest_path)} "
236
+ f"(cache_size: {len(_MANIFEST_CACHE)})"
237
+ )
238
+
239
+ # Return an owned copy so caller-side mutation can't corrupt the cache.
240
+ return copy.deepcopy(result)
241
+
242
+ except Exception as e:
243
+ _CACHE_STATS["errors"] += 1
244
+ log.error(f"Error parsing manifest {manifest_path}: {e}")
245
+ raise
246
+
247
+
248
+ def get_cache_stats() -> Dict[str, Any]:
249
+ """
250
+ Get cache statistics for monitoring.
251
+
252
+ Returns:
253
+ Dictionary with cache hits, misses, errors, size, and hit rate
254
+ """
255
+ with _CACHE_LOCK:
256
+ total_requests = _CACHE_STATS["hits"] + _CACHE_STATS["misses"]
257
+ hit_rate = (_CACHE_STATS["hits"] / total_requests * 100) if total_requests > 0 else 0
258
+
259
+ return {
260
+ "hits": _CACHE_STATS["hits"],
261
+ "misses": _CACHE_STATS["misses"],
262
+ "errors": _CACHE_STATS["errors"],
263
+ "cache_size": len(_MANIFEST_CACHE),
264
+ "hit_rate_percent": round(hit_rate, 2),
265
+ "enabled": _CACHE_ENABLED,
266
+ "max_size": _CACHE_MAX_SIZE
267
+ }
268
+
269
+
270
+ def clear_cache() -> None:
271
+ """Clear all cached manifests. Useful for testing or troubleshooting."""
272
+ with _CACHE_LOCK:
273
+ _MANIFEST_CACHE.clear()
274
+ log.info("Manifest cache cleared")
275
+