fast2dev-dbt-runner 0.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,615 @@
1
+ from __future__ import print_function
2
+ import os
3
+ import re
4
+ import signal
5
+ import subprocess
6
+ import json
7
+ import socket
8
+ import sys
9
+ import hashlib
10
+ from pathlib import Path
11
+ from filelock import FileLock
12
+ from fast2dev_dbt_runner._compat import AirflowException, BaseHook
13
+ from fast2dev_dbt_runner.bash_operator.datawarehouse_secrets import DataWarehouseSecretsManager
14
+
15
+ class DbtCliHook(BaseHook):
16
+ """
17
+ Simple wrapper around the dbt CLI.
18
+
19
+ :param env: If set, passes the env variables to the subprocess handler
20
+ :type env: dict
21
+ :param profiles_dir: If set, passed as the `--profiles-dir` argument to the `dbt` command
22
+ :type profiles_dir: str
23
+ :param target: If set, passed as the `--target` argument to the `dbt` command
24
+ :type dbt_project_dir: str
25
+ :param dbt_project_dir: The directory to run the CLI in
26
+ :type vars: str
27
+ :param vars: If set, passed as the `--vars` argument to the `dbt` command
28
+ :type vars: dict
29
+ :param full_refresh: If `True`, will fully-refresh incremental models.
30
+ :type full_refresh: bool
31
+ :param models: If set, passed as the `--select` argument to the `dbt` command (dbt recommends --select over deprecated --models)
32
+ :type models: str
33
+ :param warn_error: If `True`, treat warnings as errors.
34
+ :type warn_error: bool
35
+ :param exclude: If set, passed as the `--exclude` argument to the `dbt` command
36
+ :type exclude: str
37
+ :param select: If set, passed as the `--select` argument to the `dbt` command
38
+ :type select: str
39
+ :param selector: If set, passed as the `--selector` argument to the `dbt` command
40
+ :type selector: str
41
+ :param dbt_bin: The `dbt` CLI. Defaults to `dbt`, so assumes it's on your `PATH`
42
+ :type dbt_bin: str
43
+ :param output_encoding: Output encoding of bash command. Defaults to utf-8
44
+ :type output_encoding: str
45
+ :param verbose: The operator will log verbosely to the Airflow logs
46
+ :type verbose: bool
47
+ :param warehouse_type: The type of data warehouse to use (bigquery, snowflake, redshift, fabric)
48
+ :type warehouse_type: str
49
+ :param debug: If `True`, enables debug logging
50
+ :type debug: bool
51
+ :param dag_id: Airflow DAG ID; used (via short hash) together with task_id
52
+ to build a stable, per-DAG+task --target-path while keeping paths short.
53
+ :type dag_id: str
54
+ :param task_id: Airflow task ID; used to build a per-task --target-path.
55
+ :type task_id: str
56
+ :param target_path: Override directory for dbt target output. If None and dag_id/task_id are set,
57
+ defaults to /tmp/target_{sanitized_dag_id}_{sanitized_task_id_last_segment}/.
58
+ :type target_path: str
59
+ :param empty: If ``True``, pass ``--empty`` to the dbt command (run only). Used for E2E empty runs.
60
+ :type empty: bool
61
+ """
62
+
63
+ # Map for backward compatibility with DATA_WAREHOUSE_PLATFORM
64
+ DWH_PLATFORM_MAP = {
65
+ "1": "bigquery",
66
+ "2": "snowflake",
67
+ "3": "redshift",
68
+ "4": "fabric"
69
+ }
70
+
71
+ def __init__(self,
72
+ env=None,
73
+ profiles_dir=None,
74
+ target=None,
75
+ dbt_project_dir=None,
76
+ vars=None,
77
+ full_refresh=False,
78
+ data=False,
79
+ schema=False,
80
+ models=None,
81
+ exclude=None,
82
+ select=None,
83
+ selector=None,
84
+ dbt_bin='dbt',
85
+ output_encoding='utf-8',
86
+ verbose=True,
87
+ warn_error=False,
88
+ git_branch=None,
89
+ warehouse_type=None,
90
+ debug=False,
91
+ dag_id=None,
92
+ task_id=None,
93
+ target_path=None,
94
+ empty=False):
95
+ self.env = env or {}
96
+ self.profiles_dir = profiles_dir
97
+ self.dbt_project_dir = dbt_project_dir
98
+ self.target = target
99
+ # Per-task target path to avoid concurrent dbt runs sharing the same target directory.
100
+ # Use last segment of task_id (e.g. models.foo.bar.model_name -> model_name) for shorter paths.
101
+ # If dag_id is available, prefix with a short, filesystem-safe hash of dag_id so that
102
+ # the same DAG/task pair reuses the same target directory while keeping the path short.
103
+ if target_path is not None:
104
+ self.target_path = target_path.rstrip('/')
105
+ elif task_id is not None:
106
+ short_id = str(task_id).rsplit('.', 1)[-1] if '.' in str(task_id) else str(task_id)
107
+ safe_task = re.sub(r'[^\w\-.]', '_', short_id)
108
+ dag_hash = None
109
+ if dag_id is not None:
110
+ # Use a short hex digest to keep paths well under OS limits.
111
+ dag_hash = hashlib.sha1(str(dag_id).encode("utf-8")).hexdigest()[:8]
112
+ if dag_hash:
113
+ self.target_path = f"/tmp/target_{dag_hash}_{safe_task}"
114
+ else:
115
+ self.target_path = f"/tmp/target_{safe_task}"
116
+ else:
117
+ self.target_path = None
118
+ self.empty = empty
119
+ self.vars = vars
120
+ self.full_refresh = full_refresh
121
+ self.data = data
122
+ self.schema = schema
123
+ self.models = models
124
+ self.exclude = exclude
125
+ self.select = select
126
+ self.selector = selector
127
+ self.dbt_bin = dbt_bin
128
+ self.verbose = verbose
129
+ self.warn_error = warn_error
130
+ self.output_encoding = output_encoding
131
+ self.git_branch = git_branch
132
+ self.dag_id = dag_id
133
+ self.worker_id = socket.gethostname()
134
+ self._debug = debug
135
+
136
+ # Add debug logging
137
+ self._log_debug(f"[Worker: {self.worker_id}] Initializing DbtCliHook")
138
+ self._log_debug(f"[Worker: {self.worker_id}] Python path: {sys.path}")
139
+ self._log_debug(f"[Worker: {self.worker_id}] Current directory: {os.getcwd()}")
140
+
141
+ try:
142
+ # Check if we're using the new secrets management system
143
+ self._using_new_secrets = os.path.exists("/fast2dev/secrets")
144
+ self._log_debug(f"[Worker: {self.worker_id}] Using new secrets management: {self._using_new_secrets}")
145
+
146
+ # Handle warehouse type with backward compatibility
147
+ self.warehouse_type = self._determine_warehouse_type(warehouse_type)
148
+ self._log_debug(f"[Worker: {self.worker_id}] Determined warehouse type: {self.warehouse_type}")
149
+
150
+ if self._using_new_secrets:
151
+ # Try importing the secrets manager
152
+ self._log_debug(f"[Worker: {self.worker_id}] Successfully imported DataWarehouseSecretsManager")
153
+
154
+ # Initialize secrets manager if warehouse type is determined
155
+ if self.warehouse_type:
156
+ try:
157
+ self.secrets_manager = DataWarehouseSecretsManager(debug=self._debug)
158
+ self.secrets_manager.setup_secrets(self.warehouse_type)
159
+ self._log_debug(f"[Worker: {self.worker_id}] Successfully initialized secrets manager for {self.warehouse_type}")
160
+
161
+ # Check what environment variables are available
162
+ env_vars = self.secrets_manager.get_env_vars()
163
+ self._log_debug(f"[Worker: {self.worker_id}] Available environment variables: {list(env_vars.keys())}")
164
+ except Exception as e:
165
+ self.log.error(f"[Worker: {self.worker_id}] Failed to initialize secrets manager: {str(e)}")
166
+ raise
167
+ else:
168
+ self.log.warning(f"[Worker: {self.worker_id}] No warehouse type determined, skipping secrets manager initialization")
169
+ else:
170
+ self._log_debug(f"[Worker: {self.worker_id}] Using legacy secrets management (workload identity)")
171
+ # For backward compatibility, we don't need to initialize secrets manager
172
+ # The workload identity will handle authentication
173
+ self.secrets_manager = None
174
+ except ImportError as e:
175
+ self.log.error(f"[Worker: {self.worker_id}] Failed to import DataWarehouseSecretsManager: {str(e)}")
176
+ raise
177
+
178
+ def _log_debug(self, message: str) -> None:
179
+ """Log message only if debug is enabled"""
180
+ if self._debug:
181
+ self.log.info(message)
182
+
183
+ def _log_info(self, message: str, *args) -> None:
184
+ """Log message regardless of debug setting"""
185
+ if args:
186
+ self.log.info(message, *args)
187
+ else:
188
+ self.log.info(message)
189
+
190
+ def _log_warning(self, message: str) -> None:
191
+ """Log warning regardless of debug setting"""
192
+ self.log.warning(message)
193
+
194
+ def _log_error(self, message: str) -> None:
195
+ """Log error regardless of debug setting"""
196
+ self.log.error(message)
197
+
198
+ def _log_compiled_sql_context(self, paths: list) -> None:
199
+ """
200
+ On dbt failure, log contents of compiled/run SQL files referenced in dbt output
201
+ so that Airflow logs show the failing SQL context before the exception.
202
+ """
203
+ for path in paths:
204
+ path = path.strip()
205
+ if not path:
206
+ continue
207
+ if not os.path.isfile(path):
208
+ self._log_warning(
209
+ "[Worker: %s] Compiled SQL path not found (may be on another worker): %s",
210
+ self.worker_id,
211
+ path,
212
+ )
213
+ continue
214
+ try:
215
+ with open(path, "r", encoding=self.output_encoding) as f:
216
+ content = f.read()
217
+ self._log_info(
218
+ "[Worker: %s] ----- Compiled SQL context: %s -----",
219
+ self.worker_id,
220
+ path,
221
+ )
222
+ for sql_line in content.splitlines():
223
+ self._log_info(" %s", sql_line)
224
+ self._log_info(
225
+ "[Worker: %s] ----- End compiled SQL: %s -----",
226
+ self.worker_id,
227
+ path,
228
+ )
229
+ except OSError as e:
230
+ self._log_warning(
231
+ "[Worker: %s] Could not read compiled SQL file %s: %s",
232
+ self.worker_id,
233
+ path,
234
+ e,
235
+ )
236
+
237
+ def _determine_warehouse_type(self, warehouse_type: str = None) -> str:
238
+ """
239
+ Determine the warehouse type with backward compatibility.
240
+ First checks warehouse_type parameter, then falls back to DATA_WAREHOUSE_PLATFORM.
241
+ If neither is provided, defaults to "bigquery" for backward compatibility.
242
+ """
243
+ try:
244
+ self._log_debug(f"[Worker: {self.worker_id}] Determining warehouse type from input: {warehouse_type}")
245
+
246
+ # First try the direct warehouse_type parameter
247
+ if warehouse_type:
248
+ normalized = warehouse_type.lower()
249
+ if normalized not in self.DWH_PLATFORM_MAP.values():
250
+ self._log_warning(f"[Worker: {self.worker_id}] Invalid warehouse_type parameter: {warehouse_type}")
251
+ self._log_debug(f"[Worker: {self.worker_id}] Using warehouse type from parameter: {normalized}")
252
+ return normalized
253
+
254
+ # Then check for DATA_WAREHOUSE_PLATFORM in environment
255
+ dwh_platform = os.environ.get("DATA_WAREHOUSE_PLATFORM")
256
+ if dwh_platform:
257
+ normalized = self.DWH_PLATFORM_MAP.get(str(dwh_platform).lower())
258
+ if not normalized:
259
+ self._log_warning(f"[Worker: {self.worker_id}] Invalid DATA_WAREHOUSE_PLATFORM value: {dwh_platform}")
260
+ self._log_debug(f"[Worker: {self.worker_id}] Using warehouse type from environment: {normalized}")
261
+ return normalized
262
+
263
+ # Default to bigquery for backward compatibility
264
+ self._log_info(f"[Worker: {self.worker_id}] No warehouse type specified, defaulting to bigquery")
265
+ return "bigquery"
266
+ except Exception as e:
267
+ self._log_error(f"[Worker: {self.worker_id}] Error determining warehouse type: {str(e)}")
268
+ return "bigquery" # Default to bigquery on error for backward compatibility
269
+
270
+ def _check_packages_dir(self):
271
+ """
272
+ Check if dbt_packages directory exists and install if missing.
273
+ Uses file locking to prevent concurrent installations.
274
+ """
275
+ exec_cwd = f'/opt/airflow/dbt/{self.dbt_project_dir}'
276
+ packages_dir = os.path.join(exec_cwd, 'dbt_packages')
277
+ lock_file = os.path.join(exec_cwd, '.dbt_deps_lock')
278
+ packages_yml = os.path.join(exec_cwd, 'packages.yml')
279
+
280
+ self._log_debug(f"[Worker: {self.worker_id}] Checking dbt packages in {packages_dir}")
281
+
282
+ # Check if packages.yml exists and if dbt_packages directory exists or is empty
283
+ if os.path.exists(packages_yml) and (not os.path.exists(packages_dir) or not os.listdir(packages_dir)):
284
+ self._log_debug(f"[Worker: {self.worker_id}] packages.yml found and dbt_packages missing or empty, initiating installation")
285
+
286
+ # Create lock file if doesn't exist
287
+ Path(lock_file).touch(exist_ok=True)
288
+
289
+ # Use file lock with timeout
290
+ lock = FileLock(lock_file, timeout=300)
291
+
292
+ try:
293
+ with lock:
294
+ # Double-check after acquiring lock
295
+ if not os.path.exists(packages_dir) or not os.listdir(packages_dir):
296
+ # Prepare dbt deps command
297
+ deps_cmd = [self.dbt_bin, 'deps']
298
+ if self.profiles_dir is not None:
299
+ deps_cmd.extend(['--profiles-dir', self.profiles_dir])
300
+ if self.target is not None:
301
+ deps_cmd.extend(['--target', self.target])
302
+
303
+ # Handle test/e2e environments
304
+ if self.target in ['test', 'e2e', 'e2e_test']:
305
+ cmd_prefix = f"export GIT_BRANCH={self.git_branch} && "
306
+ else:
307
+ cmd_prefix = ""
308
+
309
+ wrapped_cmd = ['bash', '-c', f"{cmd_prefix}{' '.join(deps_cmd)}"]
310
+
311
+ if self.verbose:
312
+ self._log_debug(f"[Worker: {self.worker_id}] Running: {' '.join(wrapped_cmd)}")
313
+
314
+ # Use same subprocess approach as main command
315
+ sp = subprocess.Popen(
316
+ wrapped_cmd,
317
+ env=self.env if self.env or self.env == {} else os.environ,
318
+ stdout=subprocess.PIPE,
319
+ stderr=subprocess.STDOUT,
320
+ cwd=exec_cwd,
321
+ close_fds=True
322
+ )
323
+
324
+ # Handle output same way as main command
325
+ for line in iter(sp.stdout.readline, b''):
326
+ line = line.decode(self.output_encoding).rstrip()
327
+ self._log_info(f"[deps] {line}")
328
+
329
+ sp.wait()
330
+
331
+ if sp.returncode:
332
+ raise AirflowException("dbt deps command failed")
333
+
334
+ self._log_debug(f"[Worker: {self.worker_id}] Successfully installed dbt packages")
335
+
336
+ except Exception as e:
337
+ self._log_error(f"[Worker: {self.worker_id}] Error during package installation: {str(e)}")
338
+ raise AirflowException(f"Failed to install dbt packages: {str(e)}")
339
+ else:
340
+ self._log_debug(f"[Worker: {self.worker_id}] dbt_packages directory exists and not empty, skipping installation")
341
+
342
+ def _check_datawarehouse_connection(self):
343
+ """
344
+ Check data warehouse connection using dbt debug command.
345
+ Only runs when debug mode is enabled.
346
+ Uses file locking to prevent concurrent debug checks.
347
+ """
348
+ if not self._debug:
349
+ self._log_debug(f"[Worker: {self.worker_id}] Debug mode not enabled, skipping connection check")
350
+ return
351
+
352
+ exec_cwd = f'/opt/airflow/dbt/{self.dbt_project_dir}'
353
+ lock_file = os.path.join(exec_cwd, '.dbt_debug_lock')
354
+
355
+ self._log_debug(f"[Worker: {self.worker_id}] Checking data warehouse connection")
356
+
357
+ # Create lock file if doesn't exist
358
+ Path(lock_file).touch(exist_ok=True)
359
+
360
+ # Use file lock with timeout
361
+ lock = FileLock(lock_file, timeout=300)
362
+
363
+ try:
364
+ with lock:
365
+ # Prepare dbt debug command
366
+ debug_cmd = [self.dbt_bin, 'debug']
367
+ if self.profiles_dir is not None:
368
+ debug_cmd.extend(['--profiles-dir', self.profiles_dir])
369
+ if self.target is not None:
370
+ debug_cmd.extend(['--target', self.target])
371
+
372
+ # Initialize cmd_prefix with exports
373
+ exports = []
374
+
375
+ # Get warehouse environment variables first
376
+ warehouse_env_vars = {}
377
+ if hasattr(self, 'secrets_manager'):
378
+ warehouse_env_vars = self.secrets_manager.get_env_vars()
379
+ self._log_debug(f"[Worker: {self.worker_id}] Retrieved warehouse env vars: {list(warehouse_env_vars.keys())}")
380
+
381
+ # Add warehouse environment variables
382
+ if warehouse_env_vars:
383
+ for key, value in warehouse_env_vars.items():
384
+ if value is not None:
385
+ # Escape single quotes in the value
386
+ escaped_value = str(value).replace("'", "'\\''")
387
+ exports.append(f"export {key}='{escaped_value}'")
388
+ self._log_debug(f"[Worker: {self.worker_id}] Added export for {key}")
389
+ else:
390
+ self._log_warning(f"[Worker: {self.worker_id}] No warehouse environment variables available")
391
+
392
+ # Handle test/e2e environments - only add GIT_BRANCH for these targets
393
+ if self.target in ['test', 'e2e', 'e2e_test']:
394
+ exports.append(f"export GIT_BRANCH={self.git_branch}")
395
+ self._log_debug(f"[Worker: {self.worker_id}] Added export for GIT_BRANCH")
396
+
397
+ # Combine all exports with && between them
398
+ cmd_prefix = ""
399
+ if exports:
400
+ cmd_prefix = " && ".join(exports) + " && "
401
+ self._log_debug(f"[Worker: {self.worker_id}] Final exports count: {len(exports)}")
402
+
403
+ wrapped_cmd = ['bash', '-c', f"{cmd_prefix}{' '.join(debug_cmd)}"]
404
+
405
+ if self.verbose:
406
+ # Log the command but mask sensitive values
407
+ log_cmd = wrapped_cmd[:]
408
+ log_cmd[2] = log_cmd[2].replace(self.git_branch, "***") if self.git_branch else log_cmd[2]
409
+ if hasattr(self, 'secrets_manager') and warehouse_env_vars:
410
+ for key in warehouse_env_vars.keys():
411
+ if key in log_cmd[2]:
412
+ log_cmd[2] = log_cmd[2].replace(f"{key}='{warehouse_env_vars[key]}'", f"{key}='***'")
413
+ self._log_debug(f"[Worker: {self.worker_id}] Running command (with masked values): {' '.join(log_cmd)}")
414
+
415
+ # Create environment dict for subprocess
416
+ env = os.environ.copy()
417
+ env.setdefault("PYTHONWARNINGS", "ignore")
418
+ if self.env:
419
+ env.update(self.env)
420
+
421
+ # Add warehouse environment variables to subprocess environment
422
+ if hasattr(self, 'secrets_manager') and warehouse_env_vars:
423
+ self._log_debug(f"[Worker: {self.worker_id}] Adding to subprocess env: {list(warehouse_env_vars.keys())}")
424
+ env.update(warehouse_env_vars)
425
+
426
+ # Use same subprocess approach as main command
427
+ sp = subprocess.Popen(
428
+ wrapped_cmd,
429
+ env=env,
430
+ stdout=subprocess.PIPE,
431
+ stderr=subprocess.STDOUT,
432
+ cwd=exec_cwd,
433
+ close_fds=True
434
+ )
435
+
436
+ # Handle output same way as main command
437
+ for line in iter(sp.stdout.readline, b''):
438
+ line = line.decode(self.output_encoding).rstrip()
439
+ self._log_info(f"[debug] {line}")
440
+
441
+ sp.wait()
442
+
443
+ if sp.returncode:
444
+ raise AirflowException("dbt debug command failed")
445
+
446
+ self._log_debug(f"[Worker: {self.worker_id}] Successfully verified data warehouse connection")
447
+
448
+ except Exception as e:
449
+ self._log_error(f"[Worker: {self.worker_id}] Error during connection check: {str(e)}")
450
+ raise AirflowException(f"Failed to verify data warehouse connection: {str(e)}")
451
+
452
+ def _dump_vars(self):
453
+
454
+ return f"'{json.dumps(self.vars)}'"
455
+
456
+ def run_cli(self, *command):
457
+ """
458
+ Run the dbt cli
459
+
460
+ :param command: The dbt command to run
461
+ :type command: str
462
+ """
463
+ self._log_debug(f"[Worker: {self.worker_id}] Starting run_cli with command: {command}")
464
+
465
+ # Check for dependencies before running any command
466
+ self._check_packages_dir()
467
+
468
+ # Refresh secrets and environment variables only if using new secrets management
469
+ warehouse_env_vars = {}
470
+ if self._using_new_secrets and hasattr(self, 'secrets_manager'):
471
+ self._log_debug(f"[Worker: {self.worker_id}] Refreshing secrets for warehouse type: {self.warehouse_type}")
472
+ self.secrets_manager.setup_secrets(self.warehouse_type)
473
+ warehouse_env_vars = self.secrets_manager.get_env_vars()
474
+ self._log_debug(f"[Worker: {self.worker_id}] Got warehouse env vars: {list(warehouse_env_vars.keys())}")
475
+ else:
476
+ self._log_debug(f"[Worker: {self.worker_id}] Using legacy secrets management (workload identity)")
477
+
478
+ dbt_cmd = [self.dbt_bin, *command]
479
+
480
+ if self.profiles_dir is not None:
481
+ dbt_cmd.extend(['--profiles-dir', self.profiles_dir])
482
+
483
+ if self.target is not None:
484
+ dbt_cmd.extend(['--target', self.target])
485
+
486
+ if self.target_path is not None and command and command[0] in ('run', 'test', 'seed', 'snapshot', 'build'):
487
+ dbt_cmd.extend(['--target-path', self.target_path])
488
+
489
+ if command and command[0] in ('run', 'build') and self.empty:
490
+ dbt_cmd.extend(['--empty'])
491
+
492
+ if self.vars is not None:
493
+ dbt_cmd.extend(['--vars', self._dump_vars()])
494
+
495
+ if self.data:
496
+ dbt_cmd.extend(['--data'])
497
+
498
+ if self.schema:
499
+ dbt_cmd.extend(['--schema'])
500
+
501
+ if self.models is not None:
502
+ dbt_cmd.extend(['--select', self.models])
503
+
504
+ if self.exclude is not None:
505
+ dbt_cmd.extend(['--exclude', self.exclude])
506
+
507
+ if self.select is not None:
508
+ dbt_cmd.extend(['--select', self.select])
509
+
510
+ if self.selector is not None:
511
+ dbt_cmd.extend(['--selector', self.selector])
512
+
513
+ if self.full_refresh:
514
+ dbt_cmd.extend(['--full-refresh'])
515
+
516
+ if self.warn_error:
517
+ dbt_cmd.insert(1, '--warn-error')
518
+
519
+ exec_cwd = f'/opt/airflow/dbt/{self.dbt_project_dir}'
520
+
521
+ # Build environment exports
522
+ exports = []
523
+
524
+ # Add warehouse environment variables only if using new secrets management
525
+ if self._using_new_secrets and warehouse_env_vars:
526
+ for key, value in warehouse_env_vars.items():
527
+ if value is not None:
528
+ # Escape single quotes in the value
529
+ escaped_value = str(value).replace("'", "'\\''")
530
+ exports.append(f"export {key}='{escaped_value}'")
531
+ self._log_debug(f"[Worker: {self.worker_id}] Adding export for {key}")
532
+
533
+ # Then add GIT_BRANCH if needed
534
+ if self.target in ['test', 'e2e', 'e2e_test']:
535
+ exports.append(f"export GIT_BRANCH={self.git_branch}")
536
+ self._log_debug(f"[Worker: {self.worker_id}] Adding export for GIT_BRANCH")
537
+
538
+ # Build the command prefix
539
+ cmd_prefix = ""
540
+ if exports:
541
+ cmd_prefix = " && ".join(exports) + " && "
542
+ self._log_debug(f"[Worker: {self.worker_id}] Command prefix will include {len(exports)} exports")
543
+
544
+ wrapped_cmd = ['bash', '-c', f"{cmd_prefix}{' '.join(dbt_cmd)}"]
545
+
546
+ if self.verbose:
547
+ # Log the command but mask sensitive values
548
+ log_cmd = wrapped_cmd[:]
549
+ log_cmd[2] = log_cmd[2].replace(self.git_branch, "***") if self.git_branch else log_cmd[2]
550
+ if warehouse_env_vars:
551
+ for key in warehouse_env_vars.keys():
552
+ if key in log_cmd[2]:
553
+ log_cmd[2] = log_cmd[2].replace(f"{key}='{warehouse_env_vars[key]}'", f"{key}='***'")
554
+ self._log_info(f"[Worker: {self.worker_id}] Running command (with masked values): {' '.join(log_cmd)}")
555
+
556
+ # Create environment dict for subprocess
557
+ env = os.environ.copy()
558
+ env.setdefault("PYTHONWARNINGS", "ignore")
559
+ if self.env:
560
+ env.update(self.env)
561
+ self._log_debug(f"[Worker: {self.worker_id}] Added {len(self.env)} variables from self.env")
562
+
563
+ # Add warehouse environment variables to subprocess environment only if using new secrets management
564
+ if self._using_new_secrets and warehouse_env_vars:
565
+ env.update(warehouse_env_vars)
566
+ self._log_debug(f"[Worker: {self.worker_id}] Added {len(warehouse_env_vars)} warehouse variables to subprocess env")
567
+
568
+ # Log final environment state
569
+ if self.verbose:
570
+ env_keys = set(env.keys())
571
+ self._log_debug(f"[Worker: {self.worker_id}] Final subprocess environment will have {len(env_keys)} variables")
572
+ if warehouse_env_vars:
573
+ for key in warehouse_env_vars.keys():
574
+ if key in env:
575
+ self._log_debug(f"[Worker: {self.worker_id}] Environment will include {key}")
576
+ else:
577
+ self._log_warning(f"[Worker: {self.worker_id}] Environment is missing {key}")
578
+
579
+ sp = subprocess.Popen(
580
+ wrapped_cmd,
581
+ env=env, # Use the updated environment
582
+ stdout=subprocess.PIPE,
583
+ stderr=subprocess.STDOUT,
584
+ cwd=exec_cwd,
585
+ close_fds=True)
586
+
587
+ self.sp = sp
588
+ self._log_info("Output:")
589
+ compiled_code_paths = []
590
+ line = ''
591
+ for line in iter(sp.stdout.readline, b''):
592
+ line = line.decode(self.output_encoding).rstrip()
593
+ self._log_info(line)
594
+ if "compiled code at " in line:
595
+ path = line.split("compiled code at ", 1)[1].strip()
596
+ if path and path not in compiled_code_paths:
597
+ compiled_code_paths.append(path)
598
+ sp.wait()
599
+
600
+ # Only log compiled SQL when command failed and dbt printed "compiled code at" paths
601
+ # (e.g. run/test do; source freshness, deps, debug do not - no-op for those)
602
+ if sp.returncode and compiled_code_paths:
603
+ self._log_compiled_sql_context(compiled_code_paths)
604
+
605
+ self._log_info(
606
+ "Command exited with return code %s",
607
+ sp.returncode
608
+ )
609
+
610
+ if sp.returncode:
611
+ raise AirflowException("dbt command failed")
612
+
613
+ def on_kill(self):
614
+ self._log_info('Sending SIGTERM signal to dbt command')
615
+ os.killpg(os.getpgid(self.sp.pid), signal.SIGTERM)