fast2dev-dbt-runner 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fast2dev_dbt_runner/__init__.py +47 -0
- fast2dev_dbt_runner/_compat.py +41 -0
- fast2dev_dbt_runner/airbyte_task_group_builder.py +259 -0
- fast2dev_dbt_runner/bash_operator/__init__.py +3 -0
- fast2dev_dbt_runner/bash_operator/datawarehouse_secrets.py +490 -0
- fast2dev_dbt_runner/bash_operator/dbt_hook.py +615 -0
- fast2dev_dbt_runner/bash_operator/dbt_operator.py +248 -0
- fast2dev_dbt_runner/cached_manifest_loader.py +275 -0
- fast2dev_dbt_runner/dbt_manifest_parser_api_operator.py +452 -0
- fast2dev_dbt_runner/dbt_manifest_parser_bash_operator.py +361 -0
- fast2dev_dbt_runner/dbt_manifest_parser_gke_operator.py +401 -0
- fast2dev_dbt_runner/dbt_manifest_parser_k8s_operator.py +448 -0
- fast2dev_dbt_runner/utils.py +451 -0
- fast2dev_dbt_runner-0.0.1.dist-info/METADATA +398 -0
- fast2dev_dbt_runner-0.0.1.dist-info/RECORD +18 -0
- fast2dev_dbt_runner-0.0.1.dist-info/WHEEL +5 -0
- fast2dev_dbt_runner-0.0.1.dist-info/licenses/LICENSE +21 -0
- fast2dev_dbt_runner-0.0.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,248 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from fast2dev_dbt_runner._compat import BaseOperator
|
|
3
|
+
from fast2dev_dbt_runner.bash_operator.dbt_hook import DbtCliHook
|
|
4
|
+
|
|
5
|
+
class DbtBaseOperator(BaseOperator):
|
|
6
|
+
"""
|
|
7
|
+
Base dbt operator
|
|
8
|
+
All other dbt operators are derived from this operator.
|
|
9
|
+
|
|
10
|
+
:param env: If set, passes the env variables to the subprocess handler
|
|
11
|
+
:type env: dict
|
|
12
|
+
:param profiles_dir: If set, passed as the `--profiles-dir` argument to the `dbt` command
|
|
13
|
+
:type profiles_dir: str
|
|
14
|
+
:param target: If set, passed as the `--target` argument to the `dbt` command
|
|
15
|
+
:type dir: str
|
|
16
|
+
:param dir: The directory to run the CLI in
|
|
17
|
+
:type vars: str
|
|
18
|
+
:param vars: If set, passed as the `--vars` argument to the `dbt` command
|
|
19
|
+
:type vars: dict
|
|
20
|
+
:param full_refresh: If `True`, will fully-refresh incremental models.
|
|
21
|
+
:type full_refresh: bool
|
|
22
|
+
:param models: If set, passed as the `--models` argument to the `dbt` command
|
|
23
|
+
:type models: str
|
|
24
|
+
:param warn_error: If `True`, treat warnings as errors.
|
|
25
|
+
:type warn_error: bool
|
|
26
|
+
:param exclude: If set, passed as the `--exclude` argument to the `dbt` command
|
|
27
|
+
:type exclude: str
|
|
28
|
+
:param select: If set, passed as the `--select` argument to the `dbt` command
|
|
29
|
+
:type select: str
|
|
30
|
+
:param selector: If set, passed as the `--selector` argument to the `dbt` command
|
|
31
|
+
:type selector: str
|
|
32
|
+
:param dbt_bin: The `dbt` CLI. Defaults to `dbt`, so assumes it's on your `PATH`
|
|
33
|
+
:type dbt_bin: str
|
|
34
|
+
:param verbose: The operator will log verbosely to the Airflow logs
|
|
35
|
+
:type verbose: bool
|
|
36
|
+
:param warehouse_type: The type of data warehouse to use (bigquery, snowflake, redshift, fabric)
|
|
37
|
+
:type warehouse_type: str
|
|
38
|
+
:param debug: If `True`, passes the debug flag to the DbtCliHook
|
|
39
|
+
:type debug: bool
|
|
40
|
+
:param empty: If `True`, pass `--empty` to dbt run (E2E empty mode). Run command only.
|
|
41
|
+
:type empty: bool
|
|
42
|
+
"""
|
|
43
|
+
|
|
44
|
+
def __init__(self,
|
|
45
|
+
env=None,
|
|
46
|
+
profiles_dir=None,
|
|
47
|
+
target=None,
|
|
48
|
+
git_branch=None,
|
|
49
|
+
dbt_project_dir='',
|
|
50
|
+
vars=None,
|
|
51
|
+
models=None,
|
|
52
|
+
exclude=None,
|
|
53
|
+
select=None,
|
|
54
|
+
selector=None,
|
|
55
|
+
dbt_bin='/home/airflow/.local/bin/dbt',
|
|
56
|
+
verbose=True,
|
|
57
|
+
warn_error=False,
|
|
58
|
+
full_refresh=False,
|
|
59
|
+
data=False,
|
|
60
|
+
schema=False,
|
|
61
|
+
warehouse_type=None,
|
|
62
|
+
debug=False,
|
|
63
|
+
empty=False,
|
|
64
|
+
*args,
|
|
65
|
+
**kwargs):
|
|
66
|
+
super(DbtBaseOperator, self).__init__(*args, **kwargs)
|
|
67
|
+
|
|
68
|
+
# Initialize environment variables
|
|
69
|
+
self.env = env or {}
|
|
70
|
+
|
|
71
|
+
# Check for DATA_WAREHOUSE_PLATFORM in environment
|
|
72
|
+
dwh_platform = os.environ.get("DATA_WAREHOUSE_PLATFORM")
|
|
73
|
+
if dwh_platform and not warehouse_type:
|
|
74
|
+
if debug:
|
|
75
|
+
self.log.info(f"Using DATA_WAREHOUSE_PLATFORM={dwh_platform} for warehouse type")
|
|
76
|
+
|
|
77
|
+
self.profiles_dir = profiles_dir
|
|
78
|
+
self.target = target
|
|
79
|
+
self.git_branch = git_branch
|
|
80
|
+
self.dbt_project_dir = dbt_project_dir
|
|
81
|
+
self.vars = vars
|
|
82
|
+
self.models = models
|
|
83
|
+
self.full_refresh = full_refresh
|
|
84
|
+
self.data = data
|
|
85
|
+
self.schema = schema
|
|
86
|
+
self.exclude = exclude
|
|
87
|
+
self.select = select
|
|
88
|
+
self.selector = selector
|
|
89
|
+
self.dbt_bin = dbt_bin
|
|
90
|
+
self.verbose = verbose
|
|
91
|
+
self.warn_error = warn_error
|
|
92
|
+
self.warehouse_type = warehouse_type
|
|
93
|
+
self.debug = debug
|
|
94
|
+
self.empty = empty
|
|
95
|
+
self.create_hook()
|
|
96
|
+
|
|
97
|
+
def create_hook(self):
|
|
98
|
+
self.hook = DbtCliHook(
|
|
99
|
+
env=self.env,
|
|
100
|
+
profiles_dir=self.profiles_dir,
|
|
101
|
+
target=self.target,
|
|
102
|
+
git_branch=self.git_branch,
|
|
103
|
+
dbt_project_dir=self.dbt_project_dir,
|
|
104
|
+
vars=self.vars,
|
|
105
|
+
full_refresh=self.full_refresh,
|
|
106
|
+
data=self.data,
|
|
107
|
+
schema=self.schema,
|
|
108
|
+
models=self.models,
|
|
109
|
+
exclude=self.exclude,
|
|
110
|
+
select=self.select,
|
|
111
|
+
selector=self.selector,
|
|
112
|
+
dbt_bin=self.dbt_bin,
|
|
113
|
+
verbose=self.verbose,
|
|
114
|
+
warn_error=self.warn_error,
|
|
115
|
+
warehouse_type=self.warehouse_type,
|
|
116
|
+
debug=self.debug,
|
|
117
|
+
dag_id=getattr(self, 'dag_id', None),
|
|
118
|
+
task_id=getattr(self, 'task_id', None),
|
|
119
|
+
empty=self.empty)
|
|
120
|
+
|
|
121
|
+
return self.hook
|
|
122
|
+
|
|
123
|
+
class DbtRunOperator(DbtBaseOperator):
|
|
124
|
+
def __init__(self, dbt_project_dir, profiles_dir=None, target=None, git_branch=None, *args, **kwargs):
|
|
125
|
+
super(DbtRunOperator, self).__init__(dbt_project_dir=dbt_project_dir,
|
|
126
|
+
profiles_dir=profiles_dir,
|
|
127
|
+
target=target,
|
|
128
|
+
git_branch=git_branch,
|
|
129
|
+
*args, **kwargs)
|
|
130
|
+
|
|
131
|
+
def execute(self, context):
|
|
132
|
+
full_refresh = context['ti'].xcom_pull(task_ids="show_input_data", key="full_refresh_model")
|
|
133
|
+
execution_date = context['ti'].xcom_pull(task_ids="show_input_data", key="execution_date")
|
|
134
|
+
if str(full_refresh) == 'True':
|
|
135
|
+
self.full_refresh = True
|
|
136
|
+
|
|
137
|
+
self.vars = {"execution_date": execution_date}
|
|
138
|
+
self.create_hook().run_cli('run')
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
class DbtBuildOperator(DbtBaseOperator):
|
|
142
|
+
"""
|
|
143
|
+
Runs ``dbt build`` for the provided selection.
|
|
144
|
+
|
|
145
|
+
Unlike ``DbtRunOperator``, ``dbt build`` runs the full pipeline for the
|
|
146
|
+
selected nodes (models, snapshots, seeds and their tests) in a single process
|
|
147
|
+
and in lineage order. This is what makes a ``model -> snapshot -> model`` chain
|
|
148
|
+
resolve correctly in a single batch task (see DBT_MODEL_DEPENDS_ON_SNAPSHOT in
|
|
149
|
+
batch mode).
|
|
150
|
+
"""
|
|
151
|
+
def __init__(self, dbt_project_dir, profiles_dir=None, target=None, git_branch=None, *args, **kwargs):
|
|
152
|
+
super(DbtBuildOperator, self).__init__(dbt_project_dir=dbt_project_dir,
|
|
153
|
+
profiles_dir=profiles_dir,
|
|
154
|
+
target=target,
|
|
155
|
+
git_branch=git_branch,
|
|
156
|
+
*args, **kwargs)
|
|
157
|
+
|
|
158
|
+
def execute(self, context):
|
|
159
|
+
full_refresh = context['ti'].xcom_pull(task_ids="show_input_data", key="full_refresh_model")
|
|
160
|
+
execution_date = context['ti'].xcom_pull(task_ids="show_input_data", key="execution_date")
|
|
161
|
+
if str(full_refresh) == 'True':
|
|
162
|
+
self.full_refresh = True
|
|
163
|
+
|
|
164
|
+
self.vars = {"execution_date": execution_date}
|
|
165
|
+
self.create_hook().run_cli('build')
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
class DbtTestOperator(DbtBaseOperator):
|
|
169
|
+
def __init__(self, dbt_project_dir, profiles_dir=None, target=None, git_branch=None, *args, **kwargs):
|
|
170
|
+
super(DbtTestOperator, self).__init__(dbt_project_dir=dbt_project_dir,
|
|
171
|
+
profiles_dir=profiles_dir,
|
|
172
|
+
target=target,
|
|
173
|
+
git_branch=git_branch,*args, **kwargs)
|
|
174
|
+
|
|
175
|
+
def execute(self, context):
|
|
176
|
+
self.create_hook().run_cli('test')
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
class DbtSourceFreshnessOperator(DbtBaseOperator):
|
|
180
|
+
def __init__(self, dbt_project_dir, profiles_dir=None, target=None, *args, **kwargs):
|
|
181
|
+
super(DbtSourceFreshnessOperator, self).__init__(dbt_project_dir=dbt_project_dir, profiles_dir=profiles_dir, target=target, *args, **kwargs)
|
|
182
|
+
|
|
183
|
+
def execute(self, context):
|
|
184
|
+
self.create_hook().run_cli('source freshness')
|
|
185
|
+
|
|
186
|
+
class DbtReDataOperator(DbtBaseOperator):
|
|
187
|
+
def __init__(self, dbt_project_dir, profiles_dir=None, target=None, *args, **kwargs):
|
|
188
|
+
super(DbtReDataOperator, self).__init__(dbt_project_dir=dbt_project_dir, profiles_dir=profiles_dir, target=target, *args, **kwargs)
|
|
189
|
+
|
|
190
|
+
def execute(self, context):
|
|
191
|
+
self.create_hook().run_cli('run --select package:re_data')
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
class DbtDocsGenerateOperator(DbtBaseOperator):
|
|
195
|
+
def __init__(self, dbt_project_dir, profiles_dir=None, target=None, *args, **kwargs):
|
|
196
|
+
super(DbtDocsGenerateOperator, self).__init__(dbt_project_dir=dbt_project_dir, profiles_dir=profiles_dir, target=target, *args,
|
|
197
|
+
**kwargs)
|
|
198
|
+
|
|
199
|
+
def execute(self, context):
|
|
200
|
+
self.create_hook().run_cli('docs', 'generate')
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
class DbtSnapshotOperator(DbtBaseOperator):
|
|
204
|
+
def __init__(self, dbt_project_dir, profiles_dir=None, target=None, *args, **kwargs):
|
|
205
|
+
super(DbtSnapshotOperator, self).__init__(dbt_project_dir=dbt_project_dir, profiles_dir=profiles_dir, target=target, *args, **kwargs)
|
|
206
|
+
|
|
207
|
+
def execute(self, context):
|
|
208
|
+
self.create_hook().run_cli('snapshot')
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
class DbtSeedOperator(DbtBaseOperator):
|
|
212
|
+
def __init__(self, dbt_project_dir, profiles_dir=None, target=None, *args, **kwargs):
|
|
213
|
+
super(DbtSeedOperator, self).__init__(dbt_project_dir=dbt_project_dir, profiles_dir=profiles_dir, target=target, *args, **kwargs)
|
|
214
|
+
|
|
215
|
+
def execute(self, context):
|
|
216
|
+
full_refresh = context['ti'].xcom_pull(task_ids="show_input_data", key="full_refresh_seed")
|
|
217
|
+
if str(full_refresh) == 'True':
|
|
218
|
+
self.full_refresh = True
|
|
219
|
+
|
|
220
|
+
self.create_hook().run_cli('seed')
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
class DbtDepsOperator(DbtBaseOperator):
|
|
224
|
+
def __init__(self, dbt_project_dir, profiles_dir=None, target=None, *args, **kwargs):
|
|
225
|
+
super(DbtDepsOperator, self).__init__(dbt_project_dir=dbt_project_dir, profiles_dir=profiles_dir, target=target, *args, **kwargs)
|
|
226
|
+
|
|
227
|
+
def execute(self, context):
|
|
228
|
+
self.create_hook().run_cli('deps')
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
class DbtCleanOperator(DbtBaseOperator):
|
|
232
|
+
def __init__(self, dbt_project_dir, profiles_dir=None, target=None, *args, **kwargs):
|
|
233
|
+
super(DbtCleanOperator, self).__init__(dbt_project_dir=dbt_project_dir, profiles_dir=profiles_dir, target=target, *args, **kwargs)
|
|
234
|
+
|
|
235
|
+
def execute(self, context):
|
|
236
|
+
self.create_hook().run_cli('clean')
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
class DbtDebugOperator(DbtBaseOperator):
|
|
240
|
+
def __init__(self, dbt_project_dir, profiles_dir=None, target=None, git_branch=None, *args, **kwargs):
|
|
241
|
+
super(DbtDebugOperator, self).__init__(dbt_project_dir=dbt_project_dir,
|
|
242
|
+
profiles_dir=profiles_dir,
|
|
243
|
+
target=target,
|
|
244
|
+
git_branch=git_branch,
|
|
245
|
+
*args, **kwargs)
|
|
246
|
+
|
|
247
|
+
def execute(self, context):
|
|
248
|
+
self.create_hook().run_cli('debug')
|
|
@@ -0,0 +1,275 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Cached manifest loader for dbt manifest files.
|
|
3
|
+
|
|
4
|
+
This module provides a cached version of load_dbt_manifest that uses file hashing
|
|
5
|
+
to avoid redundant parsing of manifest files. The cache is stored at module level
|
|
6
|
+
and persists across DAG imports within the same Python process.
|
|
7
|
+
|
|
8
|
+
Performance: ~99% cache hit rate in production, reducing DAG import time from 2-4s to <10ms.
|
|
9
|
+
"""
|
|
10
|
+
import copy
|
|
11
|
+
import hashlib
|
|
12
|
+
import logging
|
|
13
|
+
import os
|
|
14
|
+
import threading
|
|
15
|
+
from typing import Dict, Any, List, Optional, Tuple
|
|
16
|
+
import fast2dev_dbt_runner.utils as utils
|
|
17
|
+
|
|
18
|
+
# Module-level cache storage (persists across DAG imports in same process)
|
|
19
|
+
_MANIFEST_CACHE: Dict[Tuple, Dict[str, Any]] = {}
|
|
20
|
+
_CACHE_LOCK = threading.Lock()
|
|
21
|
+
_CACHE_STATS = {"hits": 0, "misses": 0, "errors": 0}
|
|
22
|
+
|
|
23
|
+
# Configuration from environment variables
|
|
24
|
+
_CACHE_ENABLED = os.getenv("AIRFLOW__CORE__MANIFEST_CACHE_ENABLED", "True").lower() == "true"
|
|
25
|
+
_CACHE_DEBUG = os.getenv("AIRFLOW__CORE__MANIFEST_CACHE_DEBUG", "False").lower() == "true"
|
|
26
|
+
_CACHE_MAX_SIZE = int(os.getenv("AIRFLOW__CORE__MANIFEST_CACHE_MAX_SIZE", "50"))
|
|
27
|
+
|
|
28
|
+
log = logging.getLogger(__name__)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _get_file_hash(manifest_path: str) -> str:
|
|
32
|
+
"""
|
|
33
|
+
Calculate MD5 hash of manifest file.
|
|
34
|
+
|
|
35
|
+
Reads file in chunks to handle large files efficiently without loading
|
|
36
|
+
entire file into memory.
|
|
37
|
+
|
|
38
|
+
Args:
|
|
39
|
+
manifest_path: Path to the manifest JSON file
|
|
40
|
+
|
|
41
|
+
Returns:
|
|
42
|
+
MD5 hash of file contents as hex string
|
|
43
|
+
"""
|
|
44
|
+
hash_md5 = hashlib.md5()
|
|
45
|
+
|
|
46
|
+
try:
|
|
47
|
+
with open(manifest_path, "rb") as f:
|
|
48
|
+
# Read in 64KB chunks for efficiency
|
|
49
|
+
for chunk in iter(lambda: f.read(65536), b""):
|
|
50
|
+
hash_md5.update(chunk)
|
|
51
|
+
return hash_md5.hexdigest()
|
|
52
|
+
except Exception as e:
|
|
53
|
+
log.error(f"Error calculating hash for {manifest_path}: {e}")
|
|
54
|
+
# Return a unique value that won't match any cache
|
|
55
|
+
return f"error_{id(e)}"
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _norm_bool(value: Any) -> bool:
|
|
59
|
+
"""
|
|
60
|
+
Normalize a flag to a canonical bool for cache-key construction.
|
|
61
|
+
|
|
62
|
+
Airflow variables may arrive as real bools or as strings ("True"/"false"/...).
|
|
63
|
+
Normalizing here guarantees that semantically-equal values (e.g. True and
|
|
64
|
+
"true") collapse to a single cache entry instead of creating duplicates or,
|
|
65
|
+
worse, reusing an entry built under a different semantic value.
|
|
66
|
+
"""
|
|
67
|
+
try:
|
|
68
|
+
return utils.to_bool(value)
|
|
69
|
+
except Exception:
|
|
70
|
+
return bool(value)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _create_cache_key(
|
|
74
|
+
manifest_path: str,
|
|
75
|
+
file_hash: str,
|
|
76
|
+
dbt_tag: List[str],
|
|
77
|
+
dbt_tag_ancestors: bool,
|
|
78
|
+
dbt_tag_descendants: bool,
|
|
79
|
+
dbt_model_depends_on_snapshot: bool = False
|
|
80
|
+
) -> Tuple:
|
|
81
|
+
"""
|
|
82
|
+
Create a cache key from manifest parameters.
|
|
83
|
+
|
|
84
|
+
The cache key includes:
|
|
85
|
+
- File hash (changes only when file content changes)
|
|
86
|
+
- DBT tags (different tags = different filtered results)
|
|
87
|
+
- Ancestor/descendant flags (affects dependency resolution)
|
|
88
|
+
- Model-depends-on-snapshot flag (changes node grouping)
|
|
89
|
+
|
|
90
|
+
Args:
|
|
91
|
+
manifest_path: Path to manifest file (for logging)
|
|
92
|
+
file_hash: MD5 hash of file contents
|
|
93
|
+
dbt_tag: List of DBT tags to filter
|
|
94
|
+
dbt_tag_ancestors: Include ancestors of tagged models
|
|
95
|
+
dbt_tag_descendants: Include descendants of tagged models
|
|
96
|
+
dbt_model_depends_on_snapshot: Weave snapshots into the model group
|
|
97
|
+
|
|
98
|
+
Returns:
|
|
99
|
+
Tuple suitable for use as dict key
|
|
100
|
+
"""
|
|
101
|
+
# Use frozenset for tags so order doesn't matter
|
|
102
|
+
tags_key = frozenset(dbt_tag) if dbt_tag else frozenset()
|
|
103
|
+
|
|
104
|
+
return (
|
|
105
|
+
file_hash,
|
|
106
|
+
tags_key,
|
|
107
|
+
_norm_bool(dbt_tag_ancestors),
|
|
108
|
+
_norm_bool(dbt_tag_descendants),
|
|
109
|
+
_norm_bool(dbt_model_depends_on_snapshot)
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _evict_lru_cache_entries() -> None:
|
|
114
|
+
"""
|
|
115
|
+
Evict least recently used cache entries if cache is too large.
|
|
116
|
+
|
|
117
|
+
This is called when cache size exceeds MAX_SIZE. We don't track LRU properly
|
|
118
|
+
for simplicity, so we just clear the oldest half of entries.
|
|
119
|
+
"""
|
|
120
|
+
if len(_MANIFEST_CACHE) > _CACHE_MAX_SIZE:
|
|
121
|
+
# Simple strategy: keep newest half based on insertion order
|
|
122
|
+
entries_to_keep = _CACHE_MAX_SIZE // 2
|
|
123
|
+
keys_to_remove = list(_MANIFEST_CACHE.keys())[:-entries_to_keep]
|
|
124
|
+
|
|
125
|
+
for key in keys_to_remove:
|
|
126
|
+
del _MANIFEST_CACHE[key]
|
|
127
|
+
|
|
128
|
+
log.info(f"Evicted {len(keys_to_remove)} entries from manifest cache. "
|
|
129
|
+
f"Cache size: {len(_MANIFEST_CACHE)}/{_CACHE_MAX_SIZE}")
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def load_dbt_manifest_cached(
|
|
133
|
+
manifest_path: str,
|
|
134
|
+
dbt_tag: Optional[List[str]] = None,
|
|
135
|
+
dbt_tag_ancestors: bool = False,
|
|
136
|
+
dbt_tag_descendants: bool = False,
|
|
137
|
+
dbt_model_depends_on_snapshot: bool = False
|
|
138
|
+
) -> Dict[str, Any]:
|
|
139
|
+
"""
|
|
140
|
+
Load and parse dbt manifest with caching.
|
|
141
|
+
|
|
142
|
+
This is a drop-in replacement for utils.load_dbt_manifest() that adds
|
|
143
|
+
file hash-based caching. The manifest is only re-parsed when the file
|
|
144
|
+
content actually changes.
|
|
145
|
+
|
|
146
|
+
Args:
|
|
147
|
+
manifest_path: Path to the manifest JSON file
|
|
148
|
+
dbt_tag: List of DBT tags to filter (default: [])
|
|
149
|
+
dbt_tag_ancestors: Include ancestors of tagged models (default: False)
|
|
150
|
+
dbt_tag_descendants: Include descendants of tagged models (default: False)
|
|
151
|
+
dbt_model_depends_on_snapshot: Weave snapshots into the model group (default: False)
|
|
152
|
+
|
|
153
|
+
Returns:
|
|
154
|
+
Parsed manifest data dictionary. A deep copy is returned so callers may
|
|
155
|
+
freely mutate the result without corrupting the shared cached object.
|
|
156
|
+
|
|
157
|
+
Performance:
|
|
158
|
+
- Cache hit: <10ms
|
|
159
|
+
- Cache miss: 2-4 seconds (full parse)
|
|
160
|
+
- Expected hit rate: >99% in production
|
|
161
|
+
"""
|
|
162
|
+
# Handle default values
|
|
163
|
+
if dbt_tag is None:
|
|
164
|
+
dbt_tag = []
|
|
165
|
+
|
|
166
|
+
# If caching is disabled, call original function directly
|
|
167
|
+
if not _CACHE_ENABLED:
|
|
168
|
+
return utils.load_dbt_manifest(
|
|
169
|
+
manifest_path,
|
|
170
|
+
dbt_tag=dbt_tag,
|
|
171
|
+
dbt_tag_ancestors=dbt_tag_ancestors,
|
|
172
|
+
dbt_tag_descendants=dbt_tag_descendants,
|
|
173
|
+
dbt_model_depends_on_snapshot=dbt_model_depends_on_snapshot
|
|
174
|
+
)
|
|
175
|
+
|
|
176
|
+
# Calculate file hash
|
|
177
|
+
file_hash = _get_file_hash(manifest_path)
|
|
178
|
+
cache_key = _create_cache_key(
|
|
179
|
+
manifest_path,
|
|
180
|
+
file_hash,
|
|
181
|
+
dbt_tag,
|
|
182
|
+
dbt_tag_ancestors,
|
|
183
|
+
dbt_tag_descendants,
|
|
184
|
+
dbt_model_depends_on_snapshot
|
|
185
|
+
)
|
|
186
|
+
|
|
187
|
+
# Thread-safe cache lookup
|
|
188
|
+
with _CACHE_LOCK:
|
|
189
|
+
if cache_key in _MANIFEST_CACHE:
|
|
190
|
+
# CACHE HIT - return cached data
|
|
191
|
+
_CACHE_STATS["hits"] += 1
|
|
192
|
+
|
|
193
|
+
if _CACHE_DEBUG:
|
|
194
|
+
hit_rate = (_CACHE_STATS["hits"] /
|
|
195
|
+
(_CACHE_STATS["hits"] + _CACHE_STATS["misses"]) * 100)
|
|
196
|
+
log.info(
|
|
197
|
+
f"Manifest cache HIT for {os.path.basename(manifest_path)} "
|
|
198
|
+
f"(hash: {file_hash[:8]}..., tags: {dbt_tag}, "
|
|
199
|
+
f"cache_size: {len(_MANIFEST_CACHE)}, "
|
|
200
|
+
f"hit_rate: {hit_rate:.1f}%)"
|
|
201
|
+
)
|
|
202
|
+
|
|
203
|
+
# Return an owned copy so caller-side mutation can't corrupt the cache.
|
|
204
|
+
return copy.deepcopy(_MANIFEST_CACHE[cache_key])
|
|
205
|
+
|
|
206
|
+
# CACHE MISS - parse manifest
|
|
207
|
+
_CACHE_STATS["misses"] += 1
|
|
208
|
+
|
|
209
|
+
if _CACHE_DEBUG:
|
|
210
|
+
log.info(
|
|
211
|
+
f"Manifest cache MISS for {os.path.basename(manifest_path)} "
|
|
212
|
+
f"(hash: {file_hash[:8]}..., tags: {dbt_tag}) - parsing..."
|
|
213
|
+
)
|
|
214
|
+
|
|
215
|
+
try:
|
|
216
|
+
# Parse manifest using original function
|
|
217
|
+
result = utils.load_dbt_manifest(
|
|
218
|
+
manifest_path,
|
|
219
|
+
dbt_tag=dbt_tag,
|
|
220
|
+
dbt_tag_ancestors=dbt_tag_ancestors,
|
|
221
|
+
dbt_tag_descendants=dbt_tag_descendants,
|
|
222
|
+
dbt_model_depends_on_snapshot=dbt_model_depends_on_snapshot
|
|
223
|
+
)
|
|
224
|
+
|
|
225
|
+
# Store in cache (thread-safe)
|
|
226
|
+
with _CACHE_LOCK:
|
|
227
|
+
_MANIFEST_CACHE[cache_key] = result
|
|
228
|
+
|
|
229
|
+
# Evict old entries if cache is too large
|
|
230
|
+
if len(_MANIFEST_CACHE) > _CACHE_MAX_SIZE:
|
|
231
|
+
_evict_lru_cache_entries()
|
|
232
|
+
|
|
233
|
+
if _CACHE_DEBUG:
|
|
234
|
+
log.info(
|
|
235
|
+
f"Manifest cached successfully for {os.path.basename(manifest_path)} "
|
|
236
|
+
f"(cache_size: {len(_MANIFEST_CACHE)})"
|
|
237
|
+
)
|
|
238
|
+
|
|
239
|
+
# Return an owned copy so caller-side mutation can't corrupt the cache.
|
|
240
|
+
return copy.deepcopy(result)
|
|
241
|
+
|
|
242
|
+
except Exception as e:
|
|
243
|
+
_CACHE_STATS["errors"] += 1
|
|
244
|
+
log.error(f"Error parsing manifest {manifest_path}: {e}")
|
|
245
|
+
raise
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def get_cache_stats() -> Dict[str, Any]:
|
|
249
|
+
"""
|
|
250
|
+
Get cache statistics for monitoring.
|
|
251
|
+
|
|
252
|
+
Returns:
|
|
253
|
+
Dictionary with cache hits, misses, errors, size, and hit rate
|
|
254
|
+
"""
|
|
255
|
+
with _CACHE_LOCK:
|
|
256
|
+
total_requests = _CACHE_STATS["hits"] + _CACHE_STATS["misses"]
|
|
257
|
+
hit_rate = (_CACHE_STATS["hits"] / total_requests * 100) if total_requests > 0 else 0
|
|
258
|
+
|
|
259
|
+
return {
|
|
260
|
+
"hits": _CACHE_STATS["hits"],
|
|
261
|
+
"misses": _CACHE_STATS["misses"],
|
|
262
|
+
"errors": _CACHE_STATS["errors"],
|
|
263
|
+
"cache_size": len(_MANIFEST_CACHE),
|
|
264
|
+
"hit_rate_percent": round(hit_rate, 2),
|
|
265
|
+
"enabled": _CACHE_ENABLED,
|
|
266
|
+
"max_size": _CACHE_MAX_SIZE
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
def clear_cache() -> None:
|
|
271
|
+
"""Clear all cached manifests. Useful for testing or troubleshooting."""
|
|
272
|
+
with _CACHE_LOCK:
|
|
273
|
+
_MANIFEST_CACHE.clear()
|
|
274
|
+
log.info("Manifest cache cleared")
|
|
275
|
+
|