fast2dev-dbt-runner 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fast2dev_dbt_runner/__init__.py +47 -0
- fast2dev_dbt_runner/_compat.py +41 -0
- fast2dev_dbt_runner/airbyte_task_group_builder.py +259 -0
- fast2dev_dbt_runner/bash_operator/__init__.py +3 -0
- fast2dev_dbt_runner/bash_operator/datawarehouse_secrets.py +490 -0
- fast2dev_dbt_runner/bash_operator/dbt_hook.py +615 -0
- fast2dev_dbt_runner/bash_operator/dbt_operator.py +248 -0
- fast2dev_dbt_runner/cached_manifest_loader.py +275 -0
- fast2dev_dbt_runner/dbt_manifest_parser_api_operator.py +452 -0
- fast2dev_dbt_runner/dbt_manifest_parser_bash_operator.py +361 -0
- fast2dev_dbt_runner/dbt_manifest_parser_gke_operator.py +401 -0
- fast2dev_dbt_runner/dbt_manifest_parser_k8s_operator.py +448 -0
- fast2dev_dbt_runner/utils.py +451 -0
- fast2dev_dbt_runner-0.0.1.dist-info/METADATA +398 -0
- fast2dev_dbt_runner-0.0.1.dist-info/RECORD +18 -0
- fast2dev_dbt_runner-0.0.1.dist-info/WHEEL +5 -0
- fast2dev_dbt_runner-0.0.1.dist-info/licenses/LICENSE +21 -0
- fast2dev_dbt_runner-0.0.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,490 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import threading
|
|
3
|
+
import base64
|
|
4
|
+
import logging
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Optional, Dict
|
|
7
|
+
from filelock import FileLock
|
|
8
|
+
|
|
9
|
+
class DataWarehouseSecretsManager:
|
|
10
|
+
"""
|
|
11
|
+
Thread-safe manager for handling data warehouse secrets.
|
|
12
|
+
Uses file locking to prevent concurrent access issues.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
_instance = None
|
|
16
|
+
_lock = threading.Lock()
|
|
17
|
+
_setup_logged = False # Track if we've already logged setup messages
|
|
18
|
+
|
|
19
|
+
# Map for both numeric and string warehouse types
|
|
20
|
+
WAREHOUSE_TYPE_MAP = {
|
|
21
|
+
"1": "bigquery",
|
|
22
|
+
"2": "snowflake",
|
|
23
|
+
"3": "redshift",
|
|
24
|
+
"4": "fabric",
|
|
25
|
+
"bigquery": "bigquery",
|
|
26
|
+
"snowflake": "snowflake",
|
|
27
|
+
"redshift": "redshift",
|
|
28
|
+
"fabric": "fabric"
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
def __new__(cls, *args, **kwargs):
|
|
32
|
+
if cls._instance is None:
|
|
33
|
+
with cls._lock:
|
|
34
|
+
if cls._instance is None:
|
|
35
|
+
cls._instance = super(DataWarehouseSecretsManager, cls).__new__(cls)
|
|
36
|
+
# Store the debug flag for initialization
|
|
37
|
+
cls._instance._debug_flag = kwargs.get('debug', False)
|
|
38
|
+
return cls._instance
|
|
39
|
+
|
|
40
|
+
def __init__(self, debug=False):
|
|
41
|
+
# Skip initialization if already initialized
|
|
42
|
+
if hasattr(self, '_initialized'):
|
|
43
|
+
# Update debug flag if it's different
|
|
44
|
+
if self._debug != debug:
|
|
45
|
+
self._debug = debug
|
|
46
|
+
return
|
|
47
|
+
|
|
48
|
+
self.logger = logging.getLogger(__name__)
|
|
49
|
+
self.secrets_base_path = "/fast2dev/secrets"
|
|
50
|
+
self.secrets_ready_file = Path("/tmp/secrets_ready")
|
|
51
|
+
# Use home directory for snowsql secrets
|
|
52
|
+
self.snowsql_base_path = "/snowsql"
|
|
53
|
+
# Store environment variables that need to be passed to child processes
|
|
54
|
+
self._env_vars = {}
|
|
55
|
+
# Store warehouse-specific environment variables
|
|
56
|
+
self._warehouse_env_vars = {
|
|
57
|
+
"snowflake": {
|
|
58
|
+
"SNOWSQL_PRIVATE_KEY_PASSPHRASE": None,
|
|
59
|
+
"SNOWFLAKE_PRIVATE_KEY_PATH": None
|
|
60
|
+
},
|
|
61
|
+
"bigquery": {
|
|
62
|
+
"GOOGLE_APPLICATION_CREDENTIALS": None
|
|
63
|
+
},
|
|
64
|
+
"redshift": {
|
|
65
|
+
"REDSHIFT_PASSWORD": None,
|
|
66
|
+
"REDSHIFT_USER": None,
|
|
67
|
+
"REDSHIFT_HOST": None,
|
|
68
|
+
"REDSHIFT_PORT": None
|
|
69
|
+
},
|
|
70
|
+
"fabric": {
|
|
71
|
+
"FABRIC_USER": None,
|
|
72
|
+
"FABRIC_PASSWORD": None,
|
|
73
|
+
"FABRIC_SERVER": None,
|
|
74
|
+
"FABRIC_DATABASE": None,
|
|
75
|
+
"FABRIC_PORT": None,
|
|
76
|
+
"FABRIC_AUTHENTICATION": None
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
self._current_warehouse_type = None
|
|
80
|
+
self._debug = getattr(self, '_debug_flag', debug)
|
|
81
|
+
self._initialized = True
|
|
82
|
+
|
|
83
|
+
def _log_debug(self, message: str) -> None:
|
|
84
|
+
"""Log message only if debug is enabled"""
|
|
85
|
+
if self._debug:
|
|
86
|
+
self.logger.info(message)
|
|
87
|
+
|
|
88
|
+
def _log_info(self, message: str) -> None:
|
|
89
|
+
"""Log message regardless of debug setting"""
|
|
90
|
+
self.logger.info(message)
|
|
91
|
+
|
|
92
|
+
def _log_warning(self, message: str) -> None:
|
|
93
|
+
"""Log warning regardless of debug setting"""
|
|
94
|
+
self.logger.warning(message)
|
|
95
|
+
|
|
96
|
+
def _log_error(self, message: str) -> None:
|
|
97
|
+
"""Log error regardless of debug setting"""
|
|
98
|
+
self.logger.error(message)
|
|
99
|
+
|
|
100
|
+
def _are_secrets_ready(self) -> bool:
|
|
101
|
+
"""Check if secrets are ready by looking for the ready file"""
|
|
102
|
+
return self.secrets_ready_file.exists()
|
|
103
|
+
|
|
104
|
+
def _mark_secrets_ready(self) -> None:
|
|
105
|
+
"""Mark secrets as ready by creating the ready file"""
|
|
106
|
+
self.secrets_ready_file.touch()
|
|
107
|
+
|
|
108
|
+
def _normalize_warehouse_type(self, warehouse_type: str) -> str:
|
|
109
|
+
"""Normalize warehouse type to standard string format"""
|
|
110
|
+
if not warehouse_type:
|
|
111
|
+
return None
|
|
112
|
+
try:
|
|
113
|
+
normalized = self.WAREHOUSE_TYPE_MAP.get(str(warehouse_type).lower())
|
|
114
|
+
if not normalized:
|
|
115
|
+
self._log_warning(f"Invalid warehouse type provided: {warehouse_type}")
|
|
116
|
+
return normalized
|
|
117
|
+
except Exception as e:
|
|
118
|
+
self._log_error(f"Error normalizing warehouse type: {str(e)}")
|
|
119
|
+
return None
|
|
120
|
+
|
|
121
|
+
def _read_secret(self, secret_path: str) -> str:
|
|
122
|
+
"""Safely read a secret file"""
|
|
123
|
+
try:
|
|
124
|
+
with open(secret_path, 'r') as f:
|
|
125
|
+
return f.read().strip()
|
|
126
|
+
except Exception as e:
|
|
127
|
+
self._log_error(f"Failed to read secret from {secret_path}: {str(e)}")
|
|
128
|
+
raise
|
|
129
|
+
|
|
130
|
+
def _setup_bigquery(self) -> None:
|
|
131
|
+
"""Configure BigQuery secrets"""
|
|
132
|
+
try:
|
|
133
|
+
if not self._setup_logged:
|
|
134
|
+
self._log_debug("Setting up BigQuery secrets")
|
|
135
|
+
|
|
136
|
+
# Create secret directory if it doesn't exist
|
|
137
|
+
secret_dir = Path("/bigquery/secret")
|
|
138
|
+
sa_json_path = secret_dir / "sa.json"
|
|
139
|
+
|
|
140
|
+
# Only create and write sa.json if it doesn't exist
|
|
141
|
+
if not sa_json_path.exists():
|
|
142
|
+
secret_dir.mkdir(exist_ok=True)
|
|
143
|
+
|
|
144
|
+
# Handle service account key
|
|
145
|
+
sa_key_path = Path(self.secrets_base_path) / "bigquery" / "DBT_DEPLOY_GCP_SA_SECRET"
|
|
146
|
+
if sa_key_path.exists():
|
|
147
|
+
# Store lock file in the writable bigquery directory
|
|
148
|
+
lock_path = Path("/bigquery") / "DBT_DEPLOY_GCP_SA_SECRET.lock"
|
|
149
|
+
with FileLock(str(lock_path)):
|
|
150
|
+
# Read and decode base64 service account key
|
|
151
|
+
decoded_key = base64.b64decode(self._read_secret(str(sa_key_path)))
|
|
152
|
+
# Write to sa.json
|
|
153
|
+
with open(sa_json_path, "wb") as f:
|
|
154
|
+
f.write(decoded_key)
|
|
155
|
+
else:
|
|
156
|
+
raise ValueError("BigQuery service account secret not found")
|
|
157
|
+
|
|
158
|
+
# Store environment variable
|
|
159
|
+
self._warehouse_env_vars["bigquery"]["GOOGLE_APPLICATION_CREDENTIALS"] = str(sa_json_path)
|
|
160
|
+
if not self._setup_logged:
|
|
161
|
+
self._log_debug("Successfully set up BigQuery secrets")
|
|
162
|
+
|
|
163
|
+
except Exception as e:
|
|
164
|
+
self._log_error(f"Failed to setup BigQuery secrets: {str(e)}")
|
|
165
|
+
raise
|
|
166
|
+
|
|
167
|
+
def _setup_snowflake(self) -> None:
|
|
168
|
+
"""Configure Snowflake secrets"""
|
|
169
|
+
try:
|
|
170
|
+
self._log_debug("Setting up Snowflake secrets")
|
|
171
|
+
|
|
172
|
+
# Check if secrets are already prepared
|
|
173
|
+
snowsql_dir = Path(self.snowsql_base_path) / "secrets"
|
|
174
|
+
key_path = snowsql_dir / "rsa_key.p8"
|
|
175
|
+
|
|
176
|
+
# If key file doesn't exist, prepare it
|
|
177
|
+
if not key_path.exists():
|
|
178
|
+
self._log_debug("Preparing Snowflake secrets - RSA key not found, creating it")
|
|
179
|
+
# Create directories if they don't exist
|
|
180
|
+
snowsql_dir.mkdir(parents=True, exist_ok=True)
|
|
181
|
+
|
|
182
|
+
# Get private key from source
|
|
183
|
+
secret_key_path = Path(self.secrets_base_path) / "snowflake" / "SNOWFLAKE_PRIVATE_KEY"
|
|
184
|
+
if not secret_key_path.exists():
|
|
185
|
+
self._log_warning(f"Snowflake private key secret not found at path: {secret_key_path}")
|
|
186
|
+
# Instead of raising an error, set a default empty key
|
|
187
|
+
self._log_warning("Using default empty key for Snowflake")
|
|
188
|
+
with open(key_path, "w") as f:
|
|
189
|
+
f.write("")
|
|
190
|
+
os.chmod(key_path, 0o600)
|
|
191
|
+
else:
|
|
192
|
+
# Copy private key to target location
|
|
193
|
+
with open(key_path, "w") as f:
|
|
194
|
+
f.write(self._read_secret(str(secret_key_path)))
|
|
195
|
+
os.chmod(key_path, 0o600)
|
|
196
|
+
self._log_debug(f"Successfully created RSA key at {key_path}")
|
|
197
|
+
else:
|
|
198
|
+
self._log_debug(f"RSA key already exists at {key_path}")
|
|
199
|
+
|
|
200
|
+
# Set passphrase environment variable
|
|
201
|
+
passphrase_path = Path(self.secrets_base_path) / "snowflake" / "SNOWFLAKE_PASSPHRASE"
|
|
202
|
+
if not passphrase_path.exists():
|
|
203
|
+
self._log_warning(f"Snowflake passphrase secret not found at path: {passphrase_path}")
|
|
204
|
+
# Use empty passphrase as fallback
|
|
205
|
+
passphrase = ""
|
|
206
|
+
else:
|
|
207
|
+
# Read passphrase
|
|
208
|
+
passphrase = self._read_secret(str(passphrase_path))
|
|
209
|
+
self._log_debug("Read passphrase from secret file")
|
|
210
|
+
|
|
211
|
+
# Initialize snowflake env vars if not already initialized
|
|
212
|
+
if "snowflake" not in self._warehouse_env_vars:
|
|
213
|
+
self._warehouse_env_vars["snowflake"] = {}
|
|
214
|
+
|
|
215
|
+
# Set the environment variables
|
|
216
|
+
self._warehouse_env_vars["snowflake"]["SNOWSQL_PRIVATE_KEY_PASSPHRASE"] = passphrase
|
|
217
|
+
self._warehouse_env_vars["snowflake"]["SNOWFLAKE_PRIVATE_KEY_PATH"] = str(key_path)
|
|
218
|
+
|
|
219
|
+
self._log_debug("Environment variables set:")
|
|
220
|
+
for key, value in self._warehouse_env_vars["snowflake"].items():
|
|
221
|
+
if value:
|
|
222
|
+
self._log_debug(f" - {key}: [value is set]")
|
|
223
|
+
else:
|
|
224
|
+
self._log_warning(f" - {key}: [not set]")
|
|
225
|
+
|
|
226
|
+
self._log_debug("Successfully set up Snowflake secrets")
|
|
227
|
+
|
|
228
|
+
except Exception as e:
|
|
229
|
+
self._log_error(f"Failed to setup Snowflake secrets: {str(e)}")
|
|
230
|
+
# Instead of raising the error, log it and continue
|
|
231
|
+
self._log_warning("Continuing with default Snowflake configuration")
|
|
232
|
+
|
|
233
|
+
def _setup_redshift(self) -> None:
|
|
234
|
+
"""Configure Redshift secrets"""
|
|
235
|
+
try:
|
|
236
|
+
if not self._setup_logged:
|
|
237
|
+
self._log_debug("Setting up Redshift secrets")
|
|
238
|
+
|
|
239
|
+
# Create a writable directory for Redshift secrets
|
|
240
|
+
redshift_dir = Path("/redshift") / "secrets"
|
|
241
|
+
redshift_dir.mkdir(parents=True, exist_ok=True)
|
|
242
|
+
|
|
243
|
+
required_vars = ["REDSHIFT_PASSWORD", "REDSHIFT_USER", "REDSHIFT_HOST", "REDSHIFT_PORT"]
|
|
244
|
+
missing_vars = []
|
|
245
|
+
|
|
246
|
+
for var in required_vars:
|
|
247
|
+
secret_path = Path(self.secrets_base_path) / "redshift" / var
|
|
248
|
+
if secret_path.exists():
|
|
249
|
+
# Store lock file in the writable redshift directory
|
|
250
|
+
lock_path = redshift_dir / f"{var}.lock"
|
|
251
|
+
with FileLock(str(lock_path)):
|
|
252
|
+
value = self._read_secret(str(secret_path))
|
|
253
|
+
self._warehouse_env_vars["redshift"][var] = value
|
|
254
|
+
else:
|
|
255
|
+
missing_vars.append(var)
|
|
256
|
+
|
|
257
|
+
if missing_vars:
|
|
258
|
+
raise ValueError(f"Missing required Redshift secrets: {', '.join(missing_vars)}")
|
|
259
|
+
|
|
260
|
+
if not self._setup_logged:
|
|
261
|
+
self._log_debug("Successfully set up Redshift secrets")
|
|
262
|
+
|
|
263
|
+
except Exception as e:
|
|
264
|
+
self._log_error(f"Failed to setup Redshift secrets: {str(e)}")
|
|
265
|
+
raise
|
|
266
|
+
|
|
267
|
+
def _setup_fabric(self) -> None:
|
|
268
|
+
"""Configure Fabric secrets"""
|
|
269
|
+
try:
|
|
270
|
+
if not self._setup_logged:
|
|
271
|
+
self._log_debug("Setting up Fabric secrets")
|
|
272
|
+
|
|
273
|
+
# Create a writable directory for Fabric secrets
|
|
274
|
+
fabric_dir = Path("/fabric") / "secrets"
|
|
275
|
+
fabric_dir.mkdir(parents=True, exist_ok=True)
|
|
276
|
+
|
|
277
|
+
required_vars = ["FABRIC_USER", "FABRIC_PASSWORD", "FABRIC_SERVER",
|
|
278
|
+
"FABRIC_DATABASE", "FABRIC_PORT", "FABRIC_AUTHENTICATION"]
|
|
279
|
+
missing_vars = []
|
|
280
|
+
|
|
281
|
+
for var in required_vars:
|
|
282
|
+
secret_path = Path(self.secrets_base_path) / "fabric" / var
|
|
283
|
+
if secret_path.exists():
|
|
284
|
+
# Store lock file in the writable fabric directory
|
|
285
|
+
lock_path = fabric_dir / f"{var}.lock"
|
|
286
|
+
with FileLock(str(lock_path)):
|
|
287
|
+
value = self._read_secret(str(secret_path))
|
|
288
|
+
self._warehouse_env_vars["fabric"][var] = value
|
|
289
|
+
else:
|
|
290
|
+
missing_vars.append(var)
|
|
291
|
+
|
|
292
|
+
if missing_vars:
|
|
293
|
+
raise ValueError(f"Missing required Fabric secrets: {', '.join(missing_vars)}")
|
|
294
|
+
|
|
295
|
+
if not self._setup_logged:
|
|
296
|
+
self._log_debug("Successfully set up Fabric secrets")
|
|
297
|
+
|
|
298
|
+
except Exception as e:
|
|
299
|
+
self._log_error(f"Failed to setup Fabric secrets: {str(e)}")
|
|
300
|
+
raise
|
|
301
|
+
|
|
302
|
+
def setup_secrets(self, warehouse_type: str) -> None:
|
|
303
|
+
"""
|
|
304
|
+
Set up secrets for the specified warehouse type.
|
|
305
|
+
This only handles the file-based secrets setup.
|
|
306
|
+
"""
|
|
307
|
+
self._log_debug(f"Setting up secrets for warehouse type: {warehouse_type}")
|
|
308
|
+
|
|
309
|
+
# Normalize warehouse type
|
|
310
|
+
warehouse_type = self._normalize_warehouse_type(warehouse_type)
|
|
311
|
+
if not warehouse_type:
|
|
312
|
+
raise ValueError(f"Invalid warehouse type: {warehouse_type}")
|
|
313
|
+
|
|
314
|
+
# Get the appropriate setup function
|
|
315
|
+
setup_functions = {
|
|
316
|
+
'bigquery': self._setup_bigquery,
|
|
317
|
+
'snowflake': self._setup_snowflake,
|
|
318
|
+
'redshift': self._setup_redshift,
|
|
319
|
+
'fabric': self._setup_fabric
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
setup_func = setup_functions.get(warehouse_type.lower())
|
|
323
|
+
if not setup_func:
|
|
324
|
+
raise ValueError(f"Unsupported warehouse type: {warehouse_type}")
|
|
325
|
+
|
|
326
|
+
# If secrets are already set up for this warehouse type, skip
|
|
327
|
+
if self._are_secrets_ready() and self._current_warehouse_type == warehouse_type:
|
|
328
|
+
self._log_debug("Secrets files already set up, skipping")
|
|
329
|
+
return
|
|
330
|
+
|
|
331
|
+
# Call the setup function
|
|
332
|
+
setup_func()
|
|
333
|
+
|
|
334
|
+
# Update current warehouse type
|
|
335
|
+
self._current_warehouse_type = warehouse_type
|
|
336
|
+
self._mark_secrets_ready()
|
|
337
|
+
|
|
338
|
+
def _setup_snowflake_secrets(self) -> None:
|
|
339
|
+
"""Configure Snowflake secret files"""
|
|
340
|
+
try:
|
|
341
|
+
# Check if secrets are already prepared
|
|
342
|
+
snowsql_dir = Path(self.snowsql_base_path) / "secrets"
|
|
343
|
+
key_path = snowsql_dir / "rsa_key.p8"
|
|
344
|
+
|
|
345
|
+
# If key file doesn't exist, prepare it
|
|
346
|
+
if not key_path.exists():
|
|
347
|
+
self._log_debug("Preparing Snowflake secrets - RSA key not found, creating it")
|
|
348
|
+
# Create directories if they don't exist
|
|
349
|
+
snowsql_dir.mkdir(parents=True, exist_ok=True)
|
|
350
|
+
|
|
351
|
+
# Get private key from source
|
|
352
|
+
secret_key_path = Path(self.secrets_base_path) / "snowflake" / "SNOWFLAKE_PRIVATE_KEY"
|
|
353
|
+
if not secret_key_path.exists():
|
|
354
|
+
self._log_error("Snowflake private key secret not found at path: " + str(secret_key_path))
|
|
355
|
+
raise ValueError("Snowflake private key secret not found")
|
|
356
|
+
|
|
357
|
+
# Copy private key to target location
|
|
358
|
+
with open(key_path, "w") as f:
|
|
359
|
+
f.write(self._read_secret(str(secret_key_path)))
|
|
360
|
+
os.chmod(key_path, 0o600)
|
|
361
|
+
self._log_debug(f"Successfully created RSA key at {key_path}")
|
|
362
|
+
else:
|
|
363
|
+
self._log_debug(f"RSA key already exists at {key_path}")
|
|
364
|
+
|
|
365
|
+
self._log_debug("Successfully set up Snowflake secret files")
|
|
366
|
+
|
|
367
|
+
except Exception as e:
|
|
368
|
+
self._log_error(f"Failed to setup Snowflake secrets: {str(e)}")
|
|
369
|
+
raise
|
|
370
|
+
|
|
371
|
+
def _setup_bigquery_secrets(self) -> None:
|
|
372
|
+
"""Configure BigQuery secret files"""
|
|
373
|
+
try:
|
|
374
|
+
# Create secret directory if it doesn't exist
|
|
375
|
+
secret_dir = Path("/bigquery/secret")
|
|
376
|
+
sa_json_path = secret_dir / "sa.json"
|
|
377
|
+
|
|
378
|
+
# Only create and write sa.json if it doesn't exist
|
|
379
|
+
if not sa_json_path.exists():
|
|
380
|
+
self._log_debug("Preparing BigQuery secrets - service account JSON not found, creating it")
|
|
381
|
+
secret_dir.mkdir(exist_ok=True)
|
|
382
|
+
|
|
383
|
+
# Handle service account key
|
|
384
|
+
sa_key_path = Path(self.secrets_base_path) / "bigquery" / "DBT_DEPLOY_GCP_SA_SECRET"
|
|
385
|
+
if not sa_key_path.exists():
|
|
386
|
+
self._log_error("BigQuery service account secret not found at path: " + str(sa_key_path))
|
|
387
|
+
raise ValueError("BigQuery service account secret not found")
|
|
388
|
+
|
|
389
|
+
# Read and decode base64 service account key
|
|
390
|
+
decoded_key = base64.b64decode(self._read_secret(str(sa_key_path)))
|
|
391
|
+
|
|
392
|
+
# Write to sa.json
|
|
393
|
+
with open(sa_json_path, "wb") as f:
|
|
394
|
+
f.write(decoded_key)
|
|
395
|
+
os.chmod(sa_json_path, 0o600)
|
|
396
|
+
self._log_debug(f"Successfully created service account JSON at {sa_json_path}")
|
|
397
|
+
else:
|
|
398
|
+
self._log_debug(f"Service account JSON already exists at {sa_json_path}")
|
|
399
|
+
|
|
400
|
+
self._log_debug("Successfully set up BigQuery secret files")
|
|
401
|
+
|
|
402
|
+
except Exception as e:
|
|
403
|
+
self._log_error(f"Failed to setup BigQuery secrets: {str(e)}")
|
|
404
|
+
raise
|
|
405
|
+
|
|
406
|
+
def get_env_vars(self) -> Dict[str, str]:
|
|
407
|
+
"""
|
|
408
|
+
Get environment variables for the current warehouse type.
|
|
409
|
+
This should be called every time we need environment variables.
|
|
410
|
+
"""
|
|
411
|
+
if not self._current_warehouse_type:
|
|
412
|
+
self._log_warning("No warehouse type set, returning empty dict")
|
|
413
|
+
return {}
|
|
414
|
+
|
|
415
|
+
self._log_debug(f"Getting environment variables for warehouse type: {self._current_warehouse_type}")
|
|
416
|
+
|
|
417
|
+
# Get the appropriate environment variables
|
|
418
|
+
if self._current_warehouse_type == 'snowflake':
|
|
419
|
+
# Get passphrase environment variable
|
|
420
|
+
passphrase_path = Path(self.secrets_base_path) / "snowflake" / "SNOWFLAKE_PASSPHRASE"
|
|
421
|
+
if not passphrase_path.exists():
|
|
422
|
+
self._log_warning(f"Snowflake passphrase secret not found at path: {passphrase_path}")
|
|
423
|
+
passphrase = "" # Use empty passphrase as fallback
|
|
424
|
+
else:
|
|
425
|
+
# Read passphrase
|
|
426
|
+
passphrase = self._read_secret(str(passphrase_path))
|
|
427
|
+
self._log_debug("Read passphrase from secret file")
|
|
428
|
+
|
|
429
|
+
# Get key path
|
|
430
|
+
key_path = Path(self.snowsql_base_path) / "secrets" / "rsa_key.p8"
|
|
431
|
+
|
|
432
|
+
env_vars = {
|
|
433
|
+
'SNOWSQL_PRIVATE_KEY_PASSPHRASE': passphrase,
|
|
434
|
+
'SNOWFLAKE_PRIVATE_KEY_PATH': str(key_path)
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
# Log which variables are set
|
|
438
|
+
for key, value in env_vars.items():
|
|
439
|
+
if value:
|
|
440
|
+
self._log_debug(f"Environment variable {key} is set")
|
|
441
|
+
else:
|
|
442
|
+
self._log_warning(f"Environment variable {key} is NOT set")
|
|
443
|
+
|
|
444
|
+
return env_vars
|
|
445
|
+
|
|
446
|
+
elif self._current_warehouse_type == 'bigquery':
|
|
447
|
+
# Check if service account JSON exists
|
|
448
|
+
sa_json_path = Path("/bigquery/secret/sa.json")
|
|
449
|
+
if not sa_json_path.exists():
|
|
450
|
+
self._log_warning(f"BigQuery service account JSON not found at {sa_json_path}")
|
|
451
|
+
return {}
|
|
452
|
+
|
|
453
|
+
env_vars = {
|
|
454
|
+
'GOOGLE_APPLICATION_CREDENTIALS': str(sa_json_path)
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
# Log which variables are set
|
|
458
|
+
for key, value in env_vars.items():
|
|
459
|
+
if value:
|
|
460
|
+
self._log_debug(f"Environment variable {key} is set to {value}")
|
|
461
|
+
else:
|
|
462
|
+
self._log_warning(f"Environment variable {key} is NOT set")
|
|
463
|
+
|
|
464
|
+
return env_vars
|
|
465
|
+
|
|
466
|
+
elif self._current_warehouse_type == 'redshift':
|
|
467
|
+
env_vars = {}
|
|
468
|
+
for var in ["REDSHIFT_PASSWORD", "REDSHIFT_USER", "REDSHIFT_HOST", "REDSHIFT_PORT"]:
|
|
469
|
+
secret_path = Path(self.secrets_base_path) / "redshift" / var
|
|
470
|
+
if secret_path.exists():
|
|
471
|
+
env_vars[var] = self._read_secret(str(secret_path))
|
|
472
|
+
self._log_debug(f"Read {var} from {secret_path}")
|
|
473
|
+
else:
|
|
474
|
+
self._log_warning(f"Secret not found at {secret_path}")
|
|
475
|
+
return env_vars
|
|
476
|
+
|
|
477
|
+
elif self._current_warehouse_type == 'fabric':
|
|
478
|
+
env_vars = {}
|
|
479
|
+
for var in ["FABRIC_USER", "FABRIC_PASSWORD", "FABRIC_SERVER",
|
|
480
|
+
"FABRIC_DATABASE", "FABRIC_PORT", "FABRIC_AUTHENTICATION"]:
|
|
481
|
+
secret_path = Path(self.secrets_base_path) / "fabric" / var
|
|
482
|
+
if secret_path.exists():
|
|
483
|
+
env_vars[var] = self._read_secret(str(secret_path))
|
|
484
|
+
self._log_debug(f"Read {var} from {secret_path}")
|
|
485
|
+
else:
|
|
486
|
+
self._log_warning(f"Secret not found at {secret_path}")
|
|
487
|
+
return env_vars
|
|
488
|
+
|
|
489
|
+
self._log_warning(f"No environment variables found for warehouse type: {self._current_warehouse_type}")
|
|
490
|
+
return {}
|