fast2dev-dbt-runner 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fast2dev_dbt_runner/__init__.py +47 -0
- fast2dev_dbt_runner/_compat.py +41 -0
- fast2dev_dbt_runner/airbyte_task_group_builder.py +259 -0
- fast2dev_dbt_runner/bash_operator/__init__.py +3 -0
- fast2dev_dbt_runner/bash_operator/datawarehouse_secrets.py +490 -0
- fast2dev_dbt_runner/bash_operator/dbt_hook.py +615 -0
- fast2dev_dbt_runner/bash_operator/dbt_operator.py +248 -0
- fast2dev_dbt_runner/cached_manifest_loader.py +275 -0
- fast2dev_dbt_runner/dbt_manifest_parser_api_operator.py +452 -0
- fast2dev_dbt_runner/dbt_manifest_parser_bash_operator.py +361 -0
- fast2dev_dbt_runner/dbt_manifest_parser_gke_operator.py +401 -0
- fast2dev_dbt_runner/dbt_manifest_parser_k8s_operator.py +448 -0
- fast2dev_dbt_runner/utils.py +451 -0
- fast2dev_dbt_runner-0.0.1.dist-info/METADATA +398 -0
- fast2dev_dbt_runner-0.0.1.dist-info/RECORD +18 -0
- fast2dev_dbt_runner-0.0.1.dist-info/WHEEL +5 -0
- fast2dev_dbt_runner-0.0.1.dist-info/licenses/LICENSE +21 -0
- fast2dev_dbt_runner-0.0.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,615 @@
|
|
|
1
|
+
from __future__ import print_function
|
|
2
|
+
import os
|
|
3
|
+
import re
|
|
4
|
+
import signal
|
|
5
|
+
import subprocess
|
|
6
|
+
import json
|
|
7
|
+
import socket
|
|
8
|
+
import sys
|
|
9
|
+
import hashlib
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from filelock import FileLock
|
|
12
|
+
from fast2dev_dbt_runner._compat import AirflowException, BaseHook
|
|
13
|
+
from fast2dev_dbt_runner.bash_operator.datawarehouse_secrets import DataWarehouseSecretsManager
|
|
14
|
+
|
|
15
|
+
class DbtCliHook(BaseHook):
|
|
16
|
+
"""
|
|
17
|
+
Simple wrapper around the dbt CLI.
|
|
18
|
+
|
|
19
|
+
:param env: If set, passes the env variables to the subprocess handler
|
|
20
|
+
:type env: dict
|
|
21
|
+
:param profiles_dir: If set, passed as the `--profiles-dir` argument to the `dbt` command
|
|
22
|
+
:type profiles_dir: str
|
|
23
|
+
:param target: If set, passed as the `--target` argument to the `dbt` command
|
|
24
|
+
:type dbt_project_dir: str
|
|
25
|
+
:param dbt_project_dir: The directory to run the CLI in
|
|
26
|
+
:type vars: str
|
|
27
|
+
:param vars: If set, passed as the `--vars` argument to the `dbt` command
|
|
28
|
+
:type vars: dict
|
|
29
|
+
:param full_refresh: If `True`, will fully-refresh incremental models.
|
|
30
|
+
:type full_refresh: bool
|
|
31
|
+
:param models: If set, passed as the `--select` argument to the `dbt` command (dbt recommends --select over deprecated --models)
|
|
32
|
+
:type models: str
|
|
33
|
+
:param warn_error: If `True`, treat warnings as errors.
|
|
34
|
+
:type warn_error: bool
|
|
35
|
+
:param exclude: If set, passed as the `--exclude` argument to the `dbt` command
|
|
36
|
+
:type exclude: str
|
|
37
|
+
:param select: If set, passed as the `--select` argument to the `dbt` command
|
|
38
|
+
:type select: str
|
|
39
|
+
:param selector: If set, passed as the `--selector` argument to the `dbt` command
|
|
40
|
+
:type selector: str
|
|
41
|
+
:param dbt_bin: The `dbt` CLI. Defaults to `dbt`, so assumes it's on your `PATH`
|
|
42
|
+
:type dbt_bin: str
|
|
43
|
+
:param output_encoding: Output encoding of bash command. Defaults to utf-8
|
|
44
|
+
:type output_encoding: str
|
|
45
|
+
:param verbose: The operator will log verbosely to the Airflow logs
|
|
46
|
+
:type verbose: bool
|
|
47
|
+
:param warehouse_type: The type of data warehouse to use (bigquery, snowflake, redshift, fabric)
|
|
48
|
+
:type warehouse_type: str
|
|
49
|
+
:param debug: If `True`, enables debug logging
|
|
50
|
+
:type debug: bool
|
|
51
|
+
:param dag_id: Airflow DAG ID; used (via short hash) together with task_id
|
|
52
|
+
to build a stable, per-DAG+task --target-path while keeping paths short.
|
|
53
|
+
:type dag_id: str
|
|
54
|
+
:param task_id: Airflow task ID; used to build a per-task --target-path.
|
|
55
|
+
:type task_id: str
|
|
56
|
+
:param target_path: Override directory for dbt target output. If None and dag_id/task_id are set,
|
|
57
|
+
defaults to /tmp/target_{sanitized_dag_id}_{sanitized_task_id_last_segment}/.
|
|
58
|
+
:type target_path: str
|
|
59
|
+
:param empty: If ``True``, pass ``--empty`` to the dbt command (run only). Used for E2E empty runs.
|
|
60
|
+
:type empty: bool
|
|
61
|
+
"""
|
|
62
|
+
|
|
63
|
+
# Map for backward compatibility with DATA_WAREHOUSE_PLATFORM
|
|
64
|
+
DWH_PLATFORM_MAP = {
|
|
65
|
+
"1": "bigquery",
|
|
66
|
+
"2": "snowflake",
|
|
67
|
+
"3": "redshift",
|
|
68
|
+
"4": "fabric"
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
def __init__(self,
|
|
72
|
+
env=None,
|
|
73
|
+
profiles_dir=None,
|
|
74
|
+
target=None,
|
|
75
|
+
dbt_project_dir=None,
|
|
76
|
+
vars=None,
|
|
77
|
+
full_refresh=False,
|
|
78
|
+
data=False,
|
|
79
|
+
schema=False,
|
|
80
|
+
models=None,
|
|
81
|
+
exclude=None,
|
|
82
|
+
select=None,
|
|
83
|
+
selector=None,
|
|
84
|
+
dbt_bin='dbt',
|
|
85
|
+
output_encoding='utf-8',
|
|
86
|
+
verbose=True,
|
|
87
|
+
warn_error=False,
|
|
88
|
+
git_branch=None,
|
|
89
|
+
warehouse_type=None,
|
|
90
|
+
debug=False,
|
|
91
|
+
dag_id=None,
|
|
92
|
+
task_id=None,
|
|
93
|
+
target_path=None,
|
|
94
|
+
empty=False):
|
|
95
|
+
self.env = env or {}
|
|
96
|
+
self.profiles_dir = profiles_dir
|
|
97
|
+
self.dbt_project_dir = dbt_project_dir
|
|
98
|
+
self.target = target
|
|
99
|
+
# Per-task target path to avoid concurrent dbt runs sharing the same target directory.
|
|
100
|
+
# Use last segment of task_id (e.g. models.foo.bar.model_name -> model_name) for shorter paths.
|
|
101
|
+
# If dag_id is available, prefix with a short, filesystem-safe hash of dag_id so that
|
|
102
|
+
# the same DAG/task pair reuses the same target directory while keeping the path short.
|
|
103
|
+
if target_path is not None:
|
|
104
|
+
self.target_path = target_path.rstrip('/')
|
|
105
|
+
elif task_id is not None:
|
|
106
|
+
short_id = str(task_id).rsplit('.', 1)[-1] if '.' in str(task_id) else str(task_id)
|
|
107
|
+
safe_task = re.sub(r'[^\w\-.]', '_', short_id)
|
|
108
|
+
dag_hash = None
|
|
109
|
+
if dag_id is not None:
|
|
110
|
+
# Use a short hex digest to keep paths well under OS limits.
|
|
111
|
+
dag_hash = hashlib.sha1(str(dag_id).encode("utf-8")).hexdigest()[:8]
|
|
112
|
+
if dag_hash:
|
|
113
|
+
self.target_path = f"/tmp/target_{dag_hash}_{safe_task}"
|
|
114
|
+
else:
|
|
115
|
+
self.target_path = f"/tmp/target_{safe_task}"
|
|
116
|
+
else:
|
|
117
|
+
self.target_path = None
|
|
118
|
+
self.empty = empty
|
|
119
|
+
self.vars = vars
|
|
120
|
+
self.full_refresh = full_refresh
|
|
121
|
+
self.data = data
|
|
122
|
+
self.schema = schema
|
|
123
|
+
self.models = models
|
|
124
|
+
self.exclude = exclude
|
|
125
|
+
self.select = select
|
|
126
|
+
self.selector = selector
|
|
127
|
+
self.dbt_bin = dbt_bin
|
|
128
|
+
self.verbose = verbose
|
|
129
|
+
self.warn_error = warn_error
|
|
130
|
+
self.output_encoding = output_encoding
|
|
131
|
+
self.git_branch = git_branch
|
|
132
|
+
self.dag_id = dag_id
|
|
133
|
+
self.worker_id = socket.gethostname()
|
|
134
|
+
self._debug = debug
|
|
135
|
+
|
|
136
|
+
# Add debug logging
|
|
137
|
+
self._log_debug(f"[Worker: {self.worker_id}] Initializing DbtCliHook")
|
|
138
|
+
self._log_debug(f"[Worker: {self.worker_id}] Python path: {sys.path}")
|
|
139
|
+
self._log_debug(f"[Worker: {self.worker_id}] Current directory: {os.getcwd()}")
|
|
140
|
+
|
|
141
|
+
try:
|
|
142
|
+
# Check if we're using the new secrets management system
|
|
143
|
+
self._using_new_secrets = os.path.exists("/fast2dev/secrets")
|
|
144
|
+
self._log_debug(f"[Worker: {self.worker_id}] Using new secrets management: {self._using_new_secrets}")
|
|
145
|
+
|
|
146
|
+
# Handle warehouse type with backward compatibility
|
|
147
|
+
self.warehouse_type = self._determine_warehouse_type(warehouse_type)
|
|
148
|
+
self._log_debug(f"[Worker: {self.worker_id}] Determined warehouse type: {self.warehouse_type}")
|
|
149
|
+
|
|
150
|
+
if self._using_new_secrets:
|
|
151
|
+
# Try importing the secrets manager
|
|
152
|
+
self._log_debug(f"[Worker: {self.worker_id}] Successfully imported DataWarehouseSecretsManager")
|
|
153
|
+
|
|
154
|
+
# Initialize secrets manager if warehouse type is determined
|
|
155
|
+
if self.warehouse_type:
|
|
156
|
+
try:
|
|
157
|
+
self.secrets_manager = DataWarehouseSecretsManager(debug=self._debug)
|
|
158
|
+
self.secrets_manager.setup_secrets(self.warehouse_type)
|
|
159
|
+
self._log_debug(f"[Worker: {self.worker_id}] Successfully initialized secrets manager for {self.warehouse_type}")
|
|
160
|
+
|
|
161
|
+
# Check what environment variables are available
|
|
162
|
+
env_vars = self.secrets_manager.get_env_vars()
|
|
163
|
+
self._log_debug(f"[Worker: {self.worker_id}] Available environment variables: {list(env_vars.keys())}")
|
|
164
|
+
except Exception as e:
|
|
165
|
+
self.log.error(f"[Worker: {self.worker_id}] Failed to initialize secrets manager: {str(e)}")
|
|
166
|
+
raise
|
|
167
|
+
else:
|
|
168
|
+
self.log.warning(f"[Worker: {self.worker_id}] No warehouse type determined, skipping secrets manager initialization")
|
|
169
|
+
else:
|
|
170
|
+
self._log_debug(f"[Worker: {self.worker_id}] Using legacy secrets management (workload identity)")
|
|
171
|
+
# For backward compatibility, we don't need to initialize secrets manager
|
|
172
|
+
# The workload identity will handle authentication
|
|
173
|
+
self.secrets_manager = None
|
|
174
|
+
except ImportError as e:
|
|
175
|
+
self.log.error(f"[Worker: {self.worker_id}] Failed to import DataWarehouseSecretsManager: {str(e)}")
|
|
176
|
+
raise
|
|
177
|
+
|
|
178
|
+
def _log_debug(self, message: str) -> None:
|
|
179
|
+
"""Log message only if debug is enabled"""
|
|
180
|
+
if self._debug:
|
|
181
|
+
self.log.info(message)
|
|
182
|
+
|
|
183
|
+
def _log_info(self, message: str, *args) -> None:
|
|
184
|
+
"""Log message regardless of debug setting"""
|
|
185
|
+
if args:
|
|
186
|
+
self.log.info(message, *args)
|
|
187
|
+
else:
|
|
188
|
+
self.log.info(message)
|
|
189
|
+
|
|
190
|
+
def _log_warning(self, message: str) -> None:
|
|
191
|
+
"""Log warning regardless of debug setting"""
|
|
192
|
+
self.log.warning(message)
|
|
193
|
+
|
|
194
|
+
def _log_error(self, message: str) -> None:
|
|
195
|
+
"""Log error regardless of debug setting"""
|
|
196
|
+
self.log.error(message)
|
|
197
|
+
|
|
198
|
+
def _log_compiled_sql_context(self, paths: list) -> None:
|
|
199
|
+
"""
|
|
200
|
+
On dbt failure, log contents of compiled/run SQL files referenced in dbt output
|
|
201
|
+
so that Airflow logs show the failing SQL context before the exception.
|
|
202
|
+
"""
|
|
203
|
+
for path in paths:
|
|
204
|
+
path = path.strip()
|
|
205
|
+
if not path:
|
|
206
|
+
continue
|
|
207
|
+
if not os.path.isfile(path):
|
|
208
|
+
self._log_warning(
|
|
209
|
+
"[Worker: %s] Compiled SQL path not found (may be on another worker): %s",
|
|
210
|
+
self.worker_id,
|
|
211
|
+
path,
|
|
212
|
+
)
|
|
213
|
+
continue
|
|
214
|
+
try:
|
|
215
|
+
with open(path, "r", encoding=self.output_encoding) as f:
|
|
216
|
+
content = f.read()
|
|
217
|
+
self._log_info(
|
|
218
|
+
"[Worker: %s] ----- Compiled SQL context: %s -----",
|
|
219
|
+
self.worker_id,
|
|
220
|
+
path,
|
|
221
|
+
)
|
|
222
|
+
for sql_line in content.splitlines():
|
|
223
|
+
self._log_info(" %s", sql_line)
|
|
224
|
+
self._log_info(
|
|
225
|
+
"[Worker: %s] ----- End compiled SQL: %s -----",
|
|
226
|
+
self.worker_id,
|
|
227
|
+
path,
|
|
228
|
+
)
|
|
229
|
+
except OSError as e:
|
|
230
|
+
self._log_warning(
|
|
231
|
+
"[Worker: %s] Could not read compiled SQL file %s: %s",
|
|
232
|
+
self.worker_id,
|
|
233
|
+
path,
|
|
234
|
+
e,
|
|
235
|
+
)
|
|
236
|
+
|
|
237
|
+
def _determine_warehouse_type(self, warehouse_type: str = None) -> str:
|
|
238
|
+
"""
|
|
239
|
+
Determine the warehouse type with backward compatibility.
|
|
240
|
+
First checks warehouse_type parameter, then falls back to DATA_WAREHOUSE_PLATFORM.
|
|
241
|
+
If neither is provided, defaults to "bigquery" for backward compatibility.
|
|
242
|
+
"""
|
|
243
|
+
try:
|
|
244
|
+
self._log_debug(f"[Worker: {self.worker_id}] Determining warehouse type from input: {warehouse_type}")
|
|
245
|
+
|
|
246
|
+
# First try the direct warehouse_type parameter
|
|
247
|
+
if warehouse_type:
|
|
248
|
+
normalized = warehouse_type.lower()
|
|
249
|
+
if normalized not in self.DWH_PLATFORM_MAP.values():
|
|
250
|
+
self._log_warning(f"[Worker: {self.worker_id}] Invalid warehouse_type parameter: {warehouse_type}")
|
|
251
|
+
self._log_debug(f"[Worker: {self.worker_id}] Using warehouse type from parameter: {normalized}")
|
|
252
|
+
return normalized
|
|
253
|
+
|
|
254
|
+
# Then check for DATA_WAREHOUSE_PLATFORM in environment
|
|
255
|
+
dwh_platform = os.environ.get("DATA_WAREHOUSE_PLATFORM")
|
|
256
|
+
if dwh_platform:
|
|
257
|
+
normalized = self.DWH_PLATFORM_MAP.get(str(dwh_platform).lower())
|
|
258
|
+
if not normalized:
|
|
259
|
+
self._log_warning(f"[Worker: {self.worker_id}] Invalid DATA_WAREHOUSE_PLATFORM value: {dwh_platform}")
|
|
260
|
+
self._log_debug(f"[Worker: {self.worker_id}] Using warehouse type from environment: {normalized}")
|
|
261
|
+
return normalized
|
|
262
|
+
|
|
263
|
+
# Default to bigquery for backward compatibility
|
|
264
|
+
self._log_info(f"[Worker: {self.worker_id}] No warehouse type specified, defaulting to bigquery")
|
|
265
|
+
return "bigquery"
|
|
266
|
+
except Exception as e:
|
|
267
|
+
self._log_error(f"[Worker: {self.worker_id}] Error determining warehouse type: {str(e)}")
|
|
268
|
+
return "bigquery" # Default to bigquery on error for backward compatibility
|
|
269
|
+
|
|
270
|
+
def _check_packages_dir(self):
|
|
271
|
+
"""
|
|
272
|
+
Check if dbt_packages directory exists and install if missing.
|
|
273
|
+
Uses file locking to prevent concurrent installations.
|
|
274
|
+
"""
|
|
275
|
+
exec_cwd = f'/opt/airflow/dbt/{self.dbt_project_dir}'
|
|
276
|
+
packages_dir = os.path.join(exec_cwd, 'dbt_packages')
|
|
277
|
+
lock_file = os.path.join(exec_cwd, '.dbt_deps_lock')
|
|
278
|
+
packages_yml = os.path.join(exec_cwd, 'packages.yml')
|
|
279
|
+
|
|
280
|
+
self._log_debug(f"[Worker: {self.worker_id}] Checking dbt packages in {packages_dir}")
|
|
281
|
+
|
|
282
|
+
# Check if packages.yml exists and if dbt_packages directory exists or is empty
|
|
283
|
+
if os.path.exists(packages_yml) and (not os.path.exists(packages_dir) or not os.listdir(packages_dir)):
|
|
284
|
+
self._log_debug(f"[Worker: {self.worker_id}] packages.yml found and dbt_packages missing or empty, initiating installation")
|
|
285
|
+
|
|
286
|
+
# Create lock file if doesn't exist
|
|
287
|
+
Path(lock_file).touch(exist_ok=True)
|
|
288
|
+
|
|
289
|
+
# Use file lock with timeout
|
|
290
|
+
lock = FileLock(lock_file, timeout=300)
|
|
291
|
+
|
|
292
|
+
try:
|
|
293
|
+
with lock:
|
|
294
|
+
# Double-check after acquiring lock
|
|
295
|
+
if not os.path.exists(packages_dir) or not os.listdir(packages_dir):
|
|
296
|
+
# Prepare dbt deps command
|
|
297
|
+
deps_cmd = [self.dbt_bin, 'deps']
|
|
298
|
+
if self.profiles_dir is not None:
|
|
299
|
+
deps_cmd.extend(['--profiles-dir', self.profiles_dir])
|
|
300
|
+
if self.target is not None:
|
|
301
|
+
deps_cmd.extend(['--target', self.target])
|
|
302
|
+
|
|
303
|
+
# Handle test/e2e environments
|
|
304
|
+
if self.target in ['test', 'e2e', 'e2e_test']:
|
|
305
|
+
cmd_prefix = f"export GIT_BRANCH={self.git_branch} && "
|
|
306
|
+
else:
|
|
307
|
+
cmd_prefix = ""
|
|
308
|
+
|
|
309
|
+
wrapped_cmd = ['bash', '-c', f"{cmd_prefix}{' '.join(deps_cmd)}"]
|
|
310
|
+
|
|
311
|
+
if self.verbose:
|
|
312
|
+
self._log_debug(f"[Worker: {self.worker_id}] Running: {' '.join(wrapped_cmd)}")
|
|
313
|
+
|
|
314
|
+
# Use same subprocess approach as main command
|
|
315
|
+
sp = subprocess.Popen(
|
|
316
|
+
wrapped_cmd,
|
|
317
|
+
env=self.env if self.env or self.env == {} else os.environ,
|
|
318
|
+
stdout=subprocess.PIPE,
|
|
319
|
+
stderr=subprocess.STDOUT,
|
|
320
|
+
cwd=exec_cwd,
|
|
321
|
+
close_fds=True
|
|
322
|
+
)
|
|
323
|
+
|
|
324
|
+
# Handle output same way as main command
|
|
325
|
+
for line in iter(sp.stdout.readline, b''):
|
|
326
|
+
line = line.decode(self.output_encoding).rstrip()
|
|
327
|
+
self._log_info(f"[deps] {line}")
|
|
328
|
+
|
|
329
|
+
sp.wait()
|
|
330
|
+
|
|
331
|
+
if sp.returncode:
|
|
332
|
+
raise AirflowException("dbt deps command failed")
|
|
333
|
+
|
|
334
|
+
self._log_debug(f"[Worker: {self.worker_id}] Successfully installed dbt packages")
|
|
335
|
+
|
|
336
|
+
except Exception as e:
|
|
337
|
+
self._log_error(f"[Worker: {self.worker_id}] Error during package installation: {str(e)}")
|
|
338
|
+
raise AirflowException(f"Failed to install dbt packages: {str(e)}")
|
|
339
|
+
else:
|
|
340
|
+
self._log_debug(f"[Worker: {self.worker_id}] dbt_packages directory exists and not empty, skipping installation")
|
|
341
|
+
|
|
342
|
+
def _check_datawarehouse_connection(self):
|
|
343
|
+
"""
|
|
344
|
+
Check data warehouse connection using dbt debug command.
|
|
345
|
+
Only runs when debug mode is enabled.
|
|
346
|
+
Uses file locking to prevent concurrent debug checks.
|
|
347
|
+
"""
|
|
348
|
+
if not self._debug:
|
|
349
|
+
self._log_debug(f"[Worker: {self.worker_id}] Debug mode not enabled, skipping connection check")
|
|
350
|
+
return
|
|
351
|
+
|
|
352
|
+
exec_cwd = f'/opt/airflow/dbt/{self.dbt_project_dir}'
|
|
353
|
+
lock_file = os.path.join(exec_cwd, '.dbt_debug_lock')
|
|
354
|
+
|
|
355
|
+
self._log_debug(f"[Worker: {self.worker_id}] Checking data warehouse connection")
|
|
356
|
+
|
|
357
|
+
# Create lock file if doesn't exist
|
|
358
|
+
Path(lock_file).touch(exist_ok=True)
|
|
359
|
+
|
|
360
|
+
# Use file lock with timeout
|
|
361
|
+
lock = FileLock(lock_file, timeout=300)
|
|
362
|
+
|
|
363
|
+
try:
|
|
364
|
+
with lock:
|
|
365
|
+
# Prepare dbt debug command
|
|
366
|
+
debug_cmd = [self.dbt_bin, 'debug']
|
|
367
|
+
if self.profiles_dir is not None:
|
|
368
|
+
debug_cmd.extend(['--profiles-dir', self.profiles_dir])
|
|
369
|
+
if self.target is not None:
|
|
370
|
+
debug_cmd.extend(['--target', self.target])
|
|
371
|
+
|
|
372
|
+
# Initialize cmd_prefix with exports
|
|
373
|
+
exports = []
|
|
374
|
+
|
|
375
|
+
# Get warehouse environment variables first
|
|
376
|
+
warehouse_env_vars = {}
|
|
377
|
+
if hasattr(self, 'secrets_manager'):
|
|
378
|
+
warehouse_env_vars = self.secrets_manager.get_env_vars()
|
|
379
|
+
self._log_debug(f"[Worker: {self.worker_id}] Retrieved warehouse env vars: {list(warehouse_env_vars.keys())}")
|
|
380
|
+
|
|
381
|
+
# Add warehouse environment variables
|
|
382
|
+
if warehouse_env_vars:
|
|
383
|
+
for key, value in warehouse_env_vars.items():
|
|
384
|
+
if value is not None:
|
|
385
|
+
# Escape single quotes in the value
|
|
386
|
+
escaped_value = str(value).replace("'", "'\\''")
|
|
387
|
+
exports.append(f"export {key}='{escaped_value}'")
|
|
388
|
+
self._log_debug(f"[Worker: {self.worker_id}] Added export for {key}")
|
|
389
|
+
else:
|
|
390
|
+
self._log_warning(f"[Worker: {self.worker_id}] No warehouse environment variables available")
|
|
391
|
+
|
|
392
|
+
# Handle test/e2e environments - only add GIT_BRANCH for these targets
|
|
393
|
+
if self.target in ['test', 'e2e', 'e2e_test']:
|
|
394
|
+
exports.append(f"export GIT_BRANCH={self.git_branch}")
|
|
395
|
+
self._log_debug(f"[Worker: {self.worker_id}] Added export for GIT_BRANCH")
|
|
396
|
+
|
|
397
|
+
# Combine all exports with && between them
|
|
398
|
+
cmd_prefix = ""
|
|
399
|
+
if exports:
|
|
400
|
+
cmd_prefix = " && ".join(exports) + " && "
|
|
401
|
+
self._log_debug(f"[Worker: {self.worker_id}] Final exports count: {len(exports)}")
|
|
402
|
+
|
|
403
|
+
wrapped_cmd = ['bash', '-c', f"{cmd_prefix}{' '.join(debug_cmd)}"]
|
|
404
|
+
|
|
405
|
+
if self.verbose:
|
|
406
|
+
# Log the command but mask sensitive values
|
|
407
|
+
log_cmd = wrapped_cmd[:]
|
|
408
|
+
log_cmd[2] = log_cmd[2].replace(self.git_branch, "***") if self.git_branch else log_cmd[2]
|
|
409
|
+
if hasattr(self, 'secrets_manager') and warehouse_env_vars:
|
|
410
|
+
for key in warehouse_env_vars.keys():
|
|
411
|
+
if key in log_cmd[2]:
|
|
412
|
+
log_cmd[2] = log_cmd[2].replace(f"{key}='{warehouse_env_vars[key]}'", f"{key}='***'")
|
|
413
|
+
self._log_debug(f"[Worker: {self.worker_id}] Running command (with masked values): {' '.join(log_cmd)}")
|
|
414
|
+
|
|
415
|
+
# Create environment dict for subprocess
|
|
416
|
+
env = os.environ.copy()
|
|
417
|
+
env.setdefault("PYTHONWARNINGS", "ignore")
|
|
418
|
+
if self.env:
|
|
419
|
+
env.update(self.env)
|
|
420
|
+
|
|
421
|
+
# Add warehouse environment variables to subprocess environment
|
|
422
|
+
if hasattr(self, 'secrets_manager') and warehouse_env_vars:
|
|
423
|
+
self._log_debug(f"[Worker: {self.worker_id}] Adding to subprocess env: {list(warehouse_env_vars.keys())}")
|
|
424
|
+
env.update(warehouse_env_vars)
|
|
425
|
+
|
|
426
|
+
# Use same subprocess approach as main command
|
|
427
|
+
sp = subprocess.Popen(
|
|
428
|
+
wrapped_cmd,
|
|
429
|
+
env=env,
|
|
430
|
+
stdout=subprocess.PIPE,
|
|
431
|
+
stderr=subprocess.STDOUT,
|
|
432
|
+
cwd=exec_cwd,
|
|
433
|
+
close_fds=True
|
|
434
|
+
)
|
|
435
|
+
|
|
436
|
+
# Handle output same way as main command
|
|
437
|
+
for line in iter(sp.stdout.readline, b''):
|
|
438
|
+
line = line.decode(self.output_encoding).rstrip()
|
|
439
|
+
self._log_info(f"[debug] {line}")
|
|
440
|
+
|
|
441
|
+
sp.wait()
|
|
442
|
+
|
|
443
|
+
if sp.returncode:
|
|
444
|
+
raise AirflowException("dbt debug command failed")
|
|
445
|
+
|
|
446
|
+
self._log_debug(f"[Worker: {self.worker_id}] Successfully verified data warehouse connection")
|
|
447
|
+
|
|
448
|
+
except Exception as e:
|
|
449
|
+
self._log_error(f"[Worker: {self.worker_id}] Error during connection check: {str(e)}")
|
|
450
|
+
raise AirflowException(f"Failed to verify data warehouse connection: {str(e)}")
|
|
451
|
+
|
|
452
|
+
def _dump_vars(self):
|
|
453
|
+
|
|
454
|
+
return f"'{json.dumps(self.vars)}'"
|
|
455
|
+
|
|
456
|
+
def run_cli(self, *command):
|
|
457
|
+
"""
|
|
458
|
+
Run the dbt cli
|
|
459
|
+
|
|
460
|
+
:param command: The dbt command to run
|
|
461
|
+
:type command: str
|
|
462
|
+
"""
|
|
463
|
+
self._log_debug(f"[Worker: {self.worker_id}] Starting run_cli with command: {command}")
|
|
464
|
+
|
|
465
|
+
# Check for dependencies before running any command
|
|
466
|
+
self._check_packages_dir()
|
|
467
|
+
|
|
468
|
+
# Refresh secrets and environment variables only if using new secrets management
|
|
469
|
+
warehouse_env_vars = {}
|
|
470
|
+
if self._using_new_secrets and hasattr(self, 'secrets_manager'):
|
|
471
|
+
self._log_debug(f"[Worker: {self.worker_id}] Refreshing secrets for warehouse type: {self.warehouse_type}")
|
|
472
|
+
self.secrets_manager.setup_secrets(self.warehouse_type)
|
|
473
|
+
warehouse_env_vars = self.secrets_manager.get_env_vars()
|
|
474
|
+
self._log_debug(f"[Worker: {self.worker_id}] Got warehouse env vars: {list(warehouse_env_vars.keys())}")
|
|
475
|
+
else:
|
|
476
|
+
self._log_debug(f"[Worker: {self.worker_id}] Using legacy secrets management (workload identity)")
|
|
477
|
+
|
|
478
|
+
dbt_cmd = [self.dbt_bin, *command]
|
|
479
|
+
|
|
480
|
+
if self.profiles_dir is not None:
|
|
481
|
+
dbt_cmd.extend(['--profiles-dir', self.profiles_dir])
|
|
482
|
+
|
|
483
|
+
if self.target is not None:
|
|
484
|
+
dbt_cmd.extend(['--target', self.target])
|
|
485
|
+
|
|
486
|
+
if self.target_path is not None and command and command[0] in ('run', 'test', 'seed', 'snapshot', 'build'):
|
|
487
|
+
dbt_cmd.extend(['--target-path', self.target_path])
|
|
488
|
+
|
|
489
|
+
if command and command[0] in ('run', 'build') and self.empty:
|
|
490
|
+
dbt_cmd.extend(['--empty'])
|
|
491
|
+
|
|
492
|
+
if self.vars is not None:
|
|
493
|
+
dbt_cmd.extend(['--vars', self._dump_vars()])
|
|
494
|
+
|
|
495
|
+
if self.data:
|
|
496
|
+
dbt_cmd.extend(['--data'])
|
|
497
|
+
|
|
498
|
+
if self.schema:
|
|
499
|
+
dbt_cmd.extend(['--schema'])
|
|
500
|
+
|
|
501
|
+
if self.models is not None:
|
|
502
|
+
dbt_cmd.extend(['--select', self.models])
|
|
503
|
+
|
|
504
|
+
if self.exclude is not None:
|
|
505
|
+
dbt_cmd.extend(['--exclude', self.exclude])
|
|
506
|
+
|
|
507
|
+
if self.select is not None:
|
|
508
|
+
dbt_cmd.extend(['--select', self.select])
|
|
509
|
+
|
|
510
|
+
if self.selector is not None:
|
|
511
|
+
dbt_cmd.extend(['--selector', self.selector])
|
|
512
|
+
|
|
513
|
+
if self.full_refresh:
|
|
514
|
+
dbt_cmd.extend(['--full-refresh'])
|
|
515
|
+
|
|
516
|
+
if self.warn_error:
|
|
517
|
+
dbt_cmd.insert(1, '--warn-error')
|
|
518
|
+
|
|
519
|
+
exec_cwd = f'/opt/airflow/dbt/{self.dbt_project_dir}'
|
|
520
|
+
|
|
521
|
+
# Build environment exports
|
|
522
|
+
exports = []
|
|
523
|
+
|
|
524
|
+
# Add warehouse environment variables only if using new secrets management
|
|
525
|
+
if self._using_new_secrets and warehouse_env_vars:
|
|
526
|
+
for key, value in warehouse_env_vars.items():
|
|
527
|
+
if value is not None:
|
|
528
|
+
# Escape single quotes in the value
|
|
529
|
+
escaped_value = str(value).replace("'", "'\\''")
|
|
530
|
+
exports.append(f"export {key}='{escaped_value}'")
|
|
531
|
+
self._log_debug(f"[Worker: {self.worker_id}] Adding export for {key}")
|
|
532
|
+
|
|
533
|
+
# Then add GIT_BRANCH if needed
|
|
534
|
+
if self.target in ['test', 'e2e', 'e2e_test']:
|
|
535
|
+
exports.append(f"export GIT_BRANCH={self.git_branch}")
|
|
536
|
+
self._log_debug(f"[Worker: {self.worker_id}] Adding export for GIT_BRANCH")
|
|
537
|
+
|
|
538
|
+
# Build the command prefix
|
|
539
|
+
cmd_prefix = ""
|
|
540
|
+
if exports:
|
|
541
|
+
cmd_prefix = " && ".join(exports) + " && "
|
|
542
|
+
self._log_debug(f"[Worker: {self.worker_id}] Command prefix will include {len(exports)} exports")
|
|
543
|
+
|
|
544
|
+
wrapped_cmd = ['bash', '-c', f"{cmd_prefix}{' '.join(dbt_cmd)}"]
|
|
545
|
+
|
|
546
|
+
if self.verbose:
|
|
547
|
+
# Log the command but mask sensitive values
|
|
548
|
+
log_cmd = wrapped_cmd[:]
|
|
549
|
+
log_cmd[2] = log_cmd[2].replace(self.git_branch, "***") if self.git_branch else log_cmd[2]
|
|
550
|
+
if warehouse_env_vars:
|
|
551
|
+
for key in warehouse_env_vars.keys():
|
|
552
|
+
if key in log_cmd[2]:
|
|
553
|
+
log_cmd[2] = log_cmd[2].replace(f"{key}='{warehouse_env_vars[key]}'", f"{key}='***'")
|
|
554
|
+
self._log_info(f"[Worker: {self.worker_id}] Running command (with masked values): {' '.join(log_cmd)}")
|
|
555
|
+
|
|
556
|
+
# Create environment dict for subprocess
|
|
557
|
+
env = os.environ.copy()
|
|
558
|
+
env.setdefault("PYTHONWARNINGS", "ignore")
|
|
559
|
+
if self.env:
|
|
560
|
+
env.update(self.env)
|
|
561
|
+
self._log_debug(f"[Worker: {self.worker_id}] Added {len(self.env)} variables from self.env")
|
|
562
|
+
|
|
563
|
+
# Add warehouse environment variables to subprocess environment only if using new secrets management
|
|
564
|
+
if self._using_new_secrets and warehouse_env_vars:
|
|
565
|
+
env.update(warehouse_env_vars)
|
|
566
|
+
self._log_debug(f"[Worker: {self.worker_id}] Added {len(warehouse_env_vars)} warehouse variables to subprocess env")
|
|
567
|
+
|
|
568
|
+
# Log final environment state
|
|
569
|
+
if self.verbose:
|
|
570
|
+
env_keys = set(env.keys())
|
|
571
|
+
self._log_debug(f"[Worker: {self.worker_id}] Final subprocess environment will have {len(env_keys)} variables")
|
|
572
|
+
if warehouse_env_vars:
|
|
573
|
+
for key in warehouse_env_vars.keys():
|
|
574
|
+
if key in env:
|
|
575
|
+
self._log_debug(f"[Worker: {self.worker_id}] Environment will include {key}")
|
|
576
|
+
else:
|
|
577
|
+
self._log_warning(f"[Worker: {self.worker_id}] Environment is missing {key}")
|
|
578
|
+
|
|
579
|
+
sp = subprocess.Popen(
|
|
580
|
+
wrapped_cmd,
|
|
581
|
+
env=env, # Use the updated environment
|
|
582
|
+
stdout=subprocess.PIPE,
|
|
583
|
+
stderr=subprocess.STDOUT,
|
|
584
|
+
cwd=exec_cwd,
|
|
585
|
+
close_fds=True)
|
|
586
|
+
|
|
587
|
+
self.sp = sp
|
|
588
|
+
self._log_info("Output:")
|
|
589
|
+
compiled_code_paths = []
|
|
590
|
+
line = ''
|
|
591
|
+
for line in iter(sp.stdout.readline, b''):
|
|
592
|
+
line = line.decode(self.output_encoding).rstrip()
|
|
593
|
+
self._log_info(line)
|
|
594
|
+
if "compiled code at " in line:
|
|
595
|
+
path = line.split("compiled code at ", 1)[1].strip()
|
|
596
|
+
if path and path not in compiled_code_paths:
|
|
597
|
+
compiled_code_paths.append(path)
|
|
598
|
+
sp.wait()
|
|
599
|
+
|
|
600
|
+
# Only log compiled SQL when command failed and dbt printed "compiled code at" paths
|
|
601
|
+
# (e.g. run/test do; source freshness, deps, debug do not - no-op for those)
|
|
602
|
+
if sp.returncode and compiled_code_paths:
|
|
603
|
+
self._log_compiled_sql_context(compiled_code_paths)
|
|
604
|
+
|
|
605
|
+
self._log_info(
|
|
606
|
+
"Command exited with return code %s",
|
|
607
|
+
sp.returncode
|
|
608
|
+
)
|
|
609
|
+
|
|
610
|
+
if sp.returncode:
|
|
611
|
+
raise AirflowException("dbt command failed")
|
|
612
|
+
|
|
613
|
+
def on_kill(self):
|
|
614
|
+
self._log_info('Sending SIGTERM signal to dbt command')
|
|
615
|
+
os.killpg(os.getpgid(self.sp.pid), signal.SIGTERM)
|