fast2dev-dbt-runner 0.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,490 @@
1
+ import os
2
+ import threading
3
+ import base64
4
+ import logging
5
+ from pathlib import Path
6
+ from typing import Optional, Dict
7
+ from filelock import FileLock
8
+
9
+ class DataWarehouseSecretsManager:
10
+ """
11
+ Thread-safe manager for handling data warehouse secrets.
12
+ Uses file locking to prevent concurrent access issues.
13
+ """
14
+
15
+ _instance = None
16
+ _lock = threading.Lock()
17
+ _setup_logged = False # Track if we've already logged setup messages
18
+
19
+ # Map for both numeric and string warehouse types
20
+ WAREHOUSE_TYPE_MAP = {
21
+ "1": "bigquery",
22
+ "2": "snowflake",
23
+ "3": "redshift",
24
+ "4": "fabric",
25
+ "bigquery": "bigquery",
26
+ "snowflake": "snowflake",
27
+ "redshift": "redshift",
28
+ "fabric": "fabric"
29
+ }
30
+
31
+ def __new__(cls, *args, **kwargs):
32
+ if cls._instance is None:
33
+ with cls._lock:
34
+ if cls._instance is None:
35
+ cls._instance = super(DataWarehouseSecretsManager, cls).__new__(cls)
36
+ # Store the debug flag for initialization
37
+ cls._instance._debug_flag = kwargs.get('debug', False)
38
+ return cls._instance
39
+
40
+ def __init__(self, debug=False):
41
+ # Skip initialization if already initialized
42
+ if hasattr(self, '_initialized'):
43
+ # Update debug flag if it's different
44
+ if self._debug != debug:
45
+ self._debug = debug
46
+ return
47
+
48
+ self.logger = logging.getLogger(__name__)
49
+ self.secrets_base_path = "/fast2dev/secrets"
50
+ self.secrets_ready_file = Path("/tmp/secrets_ready")
51
+ # Use home directory for snowsql secrets
52
+ self.snowsql_base_path = "/snowsql"
53
+ # Store environment variables that need to be passed to child processes
54
+ self._env_vars = {}
55
+ # Store warehouse-specific environment variables
56
+ self._warehouse_env_vars = {
57
+ "snowflake": {
58
+ "SNOWSQL_PRIVATE_KEY_PASSPHRASE": None,
59
+ "SNOWFLAKE_PRIVATE_KEY_PATH": None
60
+ },
61
+ "bigquery": {
62
+ "GOOGLE_APPLICATION_CREDENTIALS": None
63
+ },
64
+ "redshift": {
65
+ "REDSHIFT_PASSWORD": None,
66
+ "REDSHIFT_USER": None,
67
+ "REDSHIFT_HOST": None,
68
+ "REDSHIFT_PORT": None
69
+ },
70
+ "fabric": {
71
+ "FABRIC_USER": None,
72
+ "FABRIC_PASSWORD": None,
73
+ "FABRIC_SERVER": None,
74
+ "FABRIC_DATABASE": None,
75
+ "FABRIC_PORT": None,
76
+ "FABRIC_AUTHENTICATION": None
77
+ }
78
+ }
79
+ self._current_warehouse_type = None
80
+ self._debug = getattr(self, '_debug_flag', debug)
81
+ self._initialized = True
82
+
83
+ def _log_debug(self, message: str) -> None:
84
+ """Log message only if debug is enabled"""
85
+ if self._debug:
86
+ self.logger.info(message)
87
+
88
+ def _log_info(self, message: str) -> None:
89
+ """Log message regardless of debug setting"""
90
+ self.logger.info(message)
91
+
92
+ def _log_warning(self, message: str) -> None:
93
+ """Log warning regardless of debug setting"""
94
+ self.logger.warning(message)
95
+
96
+ def _log_error(self, message: str) -> None:
97
+ """Log error regardless of debug setting"""
98
+ self.logger.error(message)
99
+
100
+ def _are_secrets_ready(self) -> bool:
101
+ """Check if secrets are ready by looking for the ready file"""
102
+ return self.secrets_ready_file.exists()
103
+
104
+ def _mark_secrets_ready(self) -> None:
105
+ """Mark secrets as ready by creating the ready file"""
106
+ self.secrets_ready_file.touch()
107
+
108
+ def _normalize_warehouse_type(self, warehouse_type: str) -> str:
109
+ """Normalize warehouse type to standard string format"""
110
+ if not warehouse_type:
111
+ return None
112
+ try:
113
+ normalized = self.WAREHOUSE_TYPE_MAP.get(str(warehouse_type).lower())
114
+ if not normalized:
115
+ self._log_warning(f"Invalid warehouse type provided: {warehouse_type}")
116
+ return normalized
117
+ except Exception as e:
118
+ self._log_error(f"Error normalizing warehouse type: {str(e)}")
119
+ return None
120
+
121
+ def _read_secret(self, secret_path: str) -> str:
122
+ """Safely read a secret file"""
123
+ try:
124
+ with open(secret_path, 'r') as f:
125
+ return f.read().strip()
126
+ except Exception as e:
127
+ self._log_error(f"Failed to read secret from {secret_path}: {str(e)}")
128
+ raise
129
+
130
+ def _setup_bigquery(self) -> None:
131
+ """Configure BigQuery secrets"""
132
+ try:
133
+ if not self._setup_logged:
134
+ self._log_debug("Setting up BigQuery secrets")
135
+
136
+ # Create secret directory if it doesn't exist
137
+ secret_dir = Path("/bigquery/secret")
138
+ sa_json_path = secret_dir / "sa.json"
139
+
140
+ # Only create and write sa.json if it doesn't exist
141
+ if not sa_json_path.exists():
142
+ secret_dir.mkdir(exist_ok=True)
143
+
144
+ # Handle service account key
145
+ sa_key_path = Path(self.secrets_base_path) / "bigquery" / "DBT_DEPLOY_GCP_SA_SECRET"
146
+ if sa_key_path.exists():
147
+ # Store lock file in the writable bigquery directory
148
+ lock_path = Path("/bigquery") / "DBT_DEPLOY_GCP_SA_SECRET.lock"
149
+ with FileLock(str(lock_path)):
150
+ # Read and decode base64 service account key
151
+ decoded_key = base64.b64decode(self._read_secret(str(sa_key_path)))
152
+ # Write to sa.json
153
+ with open(sa_json_path, "wb") as f:
154
+ f.write(decoded_key)
155
+ else:
156
+ raise ValueError("BigQuery service account secret not found")
157
+
158
+ # Store environment variable
159
+ self._warehouse_env_vars["bigquery"]["GOOGLE_APPLICATION_CREDENTIALS"] = str(sa_json_path)
160
+ if not self._setup_logged:
161
+ self._log_debug("Successfully set up BigQuery secrets")
162
+
163
+ except Exception as e:
164
+ self._log_error(f"Failed to setup BigQuery secrets: {str(e)}")
165
+ raise
166
+
167
+ def _setup_snowflake(self) -> None:
168
+ """Configure Snowflake secrets"""
169
+ try:
170
+ self._log_debug("Setting up Snowflake secrets")
171
+
172
+ # Check if secrets are already prepared
173
+ snowsql_dir = Path(self.snowsql_base_path) / "secrets"
174
+ key_path = snowsql_dir / "rsa_key.p8"
175
+
176
+ # If key file doesn't exist, prepare it
177
+ if not key_path.exists():
178
+ self._log_debug("Preparing Snowflake secrets - RSA key not found, creating it")
179
+ # Create directories if they don't exist
180
+ snowsql_dir.mkdir(parents=True, exist_ok=True)
181
+
182
+ # Get private key from source
183
+ secret_key_path = Path(self.secrets_base_path) / "snowflake" / "SNOWFLAKE_PRIVATE_KEY"
184
+ if not secret_key_path.exists():
185
+ self._log_warning(f"Snowflake private key secret not found at path: {secret_key_path}")
186
+ # Instead of raising an error, set a default empty key
187
+ self._log_warning("Using default empty key for Snowflake")
188
+ with open(key_path, "w") as f:
189
+ f.write("")
190
+ os.chmod(key_path, 0o600)
191
+ else:
192
+ # Copy private key to target location
193
+ with open(key_path, "w") as f:
194
+ f.write(self._read_secret(str(secret_key_path)))
195
+ os.chmod(key_path, 0o600)
196
+ self._log_debug(f"Successfully created RSA key at {key_path}")
197
+ else:
198
+ self._log_debug(f"RSA key already exists at {key_path}")
199
+
200
+ # Set passphrase environment variable
201
+ passphrase_path = Path(self.secrets_base_path) / "snowflake" / "SNOWFLAKE_PASSPHRASE"
202
+ if not passphrase_path.exists():
203
+ self._log_warning(f"Snowflake passphrase secret not found at path: {passphrase_path}")
204
+ # Use empty passphrase as fallback
205
+ passphrase = ""
206
+ else:
207
+ # Read passphrase
208
+ passphrase = self._read_secret(str(passphrase_path))
209
+ self._log_debug("Read passphrase from secret file")
210
+
211
+ # Initialize snowflake env vars if not already initialized
212
+ if "snowflake" not in self._warehouse_env_vars:
213
+ self._warehouse_env_vars["snowflake"] = {}
214
+
215
+ # Set the environment variables
216
+ self._warehouse_env_vars["snowflake"]["SNOWSQL_PRIVATE_KEY_PASSPHRASE"] = passphrase
217
+ self._warehouse_env_vars["snowflake"]["SNOWFLAKE_PRIVATE_KEY_PATH"] = str(key_path)
218
+
219
+ self._log_debug("Environment variables set:")
220
+ for key, value in self._warehouse_env_vars["snowflake"].items():
221
+ if value:
222
+ self._log_debug(f" - {key}: [value is set]")
223
+ else:
224
+ self._log_warning(f" - {key}: [not set]")
225
+
226
+ self._log_debug("Successfully set up Snowflake secrets")
227
+
228
+ except Exception as e:
229
+ self._log_error(f"Failed to setup Snowflake secrets: {str(e)}")
230
+ # Instead of raising the error, log it and continue
231
+ self._log_warning("Continuing with default Snowflake configuration")
232
+
233
+ def _setup_redshift(self) -> None:
234
+ """Configure Redshift secrets"""
235
+ try:
236
+ if not self._setup_logged:
237
+ self._log_debug("Setting up Redshift secrets")
238
+
239
+ # Create a writable directory for Redshift secrets
240
+ redshift_dir = Path("/redshift") / "secrets"
241
+ redshift_dir.mkdir(parents=True, exist_ok=True)
242
+
243
+ required_vars = ["REDSHIFT_PASSWORD", "REDSHIFT_USER", "REDSHIFT_HOST", "REDSHIFT_PORT"]
244
+ missing_vars = []
245
+
246
+ for var in required_vars:
247
+ secret_path = Path(self.secrets_base_path) / "redshift" / var
248
+ if secret_path.exists():
249
+ # Store lock file in the writable redshift directory
250
+ lock_path = redshift_dir / f"{var}.lock"
251
+ with FileLock(str(lock_path)):
252
+ value = self._read_secret(str(secret_path))
253
+ self._warehouse_env_vars["redshift"][var] = value
254
+ else:
255
+ missing_vars.append(var)
256
+
257
+ if missing_vars:
258
+ raise ValueError(f"Missing required Redshift secrets: {', '.join(missing_vars)}")
259
+
260
+ if not self._setup_logged:
261
+ self._log_debug("Successfully set up Redshift secrets")
262
+
263
+ except Exception as e:
264
+ self._log_error(f"Failed to setup Redshift secrets: {str(e)}")
265
+ raise
266
+
267
+ def _setup_fabric(self) -> None:
268
+ """Configure Fabric secrets"""
269
+ try:
270
+ if not self._setup_logged:
271
+ self._log_debug("Setting up Fabric secrets")
272
+
273
+ # Create a writable directory for Fabric secrets
274
+ fabric_dir = Path("/fabric") / "secrets"
275
+ fabric_dir.mkdir(parents=True, exist_ok=True)
276
+
277
+ required_vars = ["FABRIC_USER", "FABRIC_PASSWORD", "FABRIC_SERVER",
278
+ "FABRIC_DATABASE", "FABRIC_PORT", "FABRIC_AUTHENTICATION"]
279
+ missing_vars = []
280
+
281
+ for var in required_vars:
282
+ secret_path = Path(self.secrets_base_path) / "fabric" / var
283
+ if secret_path.exists():
284
+ # Store lock file in the writable fabric directory
285
+ lock_path = fabric_dir / f"{var}.lock"
286
+ with FileLock(str(lock_path)):
287
+ value = self._read_secret(str(secret_path))
288
+ self._warehouse_env_vars["fabric"][var] = value
289
+ else:
290
+ missing_vars.append(var)
291
+
292
+ if missing_vars:
293
+ raise ValueError(f"Missing required Fabric secrets: {', '.join(missing_vars)}")
294
+
295
+ if not self._setup_logged:
296
+ self._log_debug("Successfully set up Fabric secrets")
297
+
298
+ except Exception as e:
299
+ self._log_error(f"Failed to setup Fabric secrets: {str(e)}")
300
+ raise
301
+
302
+ def setup_secrets(self, warehouse_type: str) -> None:
303
+ """
304
+ Set up secrets for the specified warehouse type.
305
+ This only handles the file-based secrets setup.
306
+ """
307
+ self._log_debug(f"Setting up secrets for warehouse type: {warehouse_type}")
308
+
309
+ # Normalize warehouse type
310
+ warehouse_type = self._normalize_warehouse_type(warehouse_type)
311
+ if not warehouse_type:
312
+ raise ValueError(f"Invalid warehouse type: {warehouse_type}")
313
+
314
+ # Get the appropriate setup function
315
+ setup_functions = {
316
+ 'bigquery': self._setup_bigquery,
317
+ 'snowflake': self._setup_snowflake,
318
+ 'redshift': self._setup_redshift,
319
+ 'fabric': self._setup_fabric
320
+ }
321
+
322
+ setup_func = setup_functions.get(warehouse_type.lower())
323
+ if not setup_func:
324
+ raise ValueError(f"Unsupported warehouse type: {warehouse_type}")
325
+
326
+ # If secrets are already set up for this warehouse type, skip
327
+ if self._are_secrets_ready() and self._current_warehouse_type == warehouse_type:
328
+ self._log_debug("Secrets files already set up, skipping")
329
+ return
330
+
331
+ # Call the setup function
332
+ setup_func()
333
+
334
+ # Update current warehouse type
335
+ self._current_warehouse_type = warehouse_type
336
+ self._mark_secrets_ready()
337
+
338
+ def _setup_snowflake_secrets(self) -> None:
339
+ """Configure Snowflake secret files"""
340
+ try:
341
+ # Check if secrets are already prepared
342
+ snowsql_dir = Path(self.snowsql_base_path) / "secrets"
343
+ key_path = snowsql_dir / "rsa_key.p8"
344
+
345
+ # If key file doesn't exist, prepare it
346
+ if not key_path.exists():
347
+ self._log_debug("Preparing Snowflake secrets - RSA key not found, creating it")
348
+ # Create directories if they don't exist
349
+ snowsql_dir.mkdir(parents=True, exist_ok=True)
350
+
351
+ # Get private key from source
352
+ secret_key_path = Path(self.secrets_base_path) / "snowflake" / "SNOWFLAKE_PRIVATE_KEY"
353
+ if not secret_key_path.exists():
354
+ self._log_error("Snowflake private key secret not found at path: " + str(secret_key_path))
355
+ raise ValueError("Snowflake private key secret not found")
356
+
357
+ # Copy private key to target location
358
+ with open(key_path, "w") as f:
359
+ f.write(self._read_secret(str(secret_key_path)))
360
+ os.chmod(key_path, 0o600)
361
+ self._log_debug(f"Successfully created RSA key at {key_path}")
362
+ else:
363
+ self._log_debug(f"RSA key already exists at {key_path}")
364
+
365
+ self._log_debug("Successfully set up Snowflake secret files")
366
+
367
+ except Exception as e:
368
+ self._log_error(f"Failed to setup Snowflake secrets: {str(e)}")
369
+ raise
370
+
371
+ def _setup_bigquery_secrets(self) -> None:
372
+ """Configure BigQuery secret files"""
373
+ try:
374
+ # Create secret directory if it doesn't exist
375
+ secret_dir = Path("/bigquery/secret")
376
+ sa_json_path = secret_dir / "sa.json"
377
+
378
+ # Only create and write sa.json if it doesn't exist
379
+ if not sa_json_path.exists():
380
+ self._log_debug("Preparing BigQuery secrets - service account JSON not found, creating it")
381
+ secret_dir.mkdir(exist_ok=True)
382
+
383
+ # Handle service account key
384
+ sa_key_path = Path(self.secrets_base_path) / "bigquery" / "DBT_DEPLOY_GCP_SA_SECRET"
385
+ if not sa_key_path.exists():
386
+ self._log_error("BigQuery service account secret not found at path: " + str(sa_key_path))
387
+ raise ValueError("BigQuery service account secret not found")
388
+
389
+ # Read and decode base64 service account key
390
+ decoded_key = base64.b64decode(self._read_secret(str(sa_key_path)))
391
+
392
+ # Write to sa.json
393
+ with open(sa_json_path, "wb") as f:
394
+ f.write(decoded_key)
395
+ os.chmod(sa_json_path, 0o600)
396
+ self._log_debug(f"Successfully created service account JSON at {sa_json_path}")
397
+ else:
398
+ self._log_debug(f"Service account JSON already exists at {sa_json_path}")
399
+
400
+ self._log_debug("Successfully set up BigQuery secret files")
401
+
402
+ except Exception as e:
403
+ self._log_error(f"Failed to setup BigQuery secrets: {str(e)}")
404
+ raise
405
+
406
+ def get_env_vars(self) -> Dict[str, str]:
407
+ """
408
+ Get environment variables for the current warehouse type.
409
+ This should be called every time we need environment variables.
410
+ """
411
+ if not self._current_warehouse_type:
412
+ self._log_warning("No warehouse type set, returning empty dict")
413
+ return {}
414
+
415
+ self._log_debug(f"Getting environment variables for warehouse type: {self._current_warehouse_type}")
416
+
417
+ # Get the appropriate environment variables
418
+ if self._current_warehouse_type == 'snowflake':
419
+ # Get passphrase environment variable
420
+ passphrase_path = Path(self.secrets_base_path) / "snowflake" / "SNOWFLAKE_PASSPHRASE"
421
+ if not passphrase_path.exists():
422
+ self._log_warning(f"Snowflake passphrase secret not found at path: {passphrase_path}")
423
+ passphrase = "" # Use empty passphrase as fallback
424
+ else:
425
+ # Read passphrase
426
+ passphrase = self._read_secret(str(passphrase_path))
427
+ self._log_debug("Read passphrase from secret file")
428
+
429
+ # Get key path
430
+ key_path = Path(self.snowsql_base_path) / "secrets" / "rsa_key.p8"
431
+
432
+ env_vars = {
433
+ 'SNOWSQL_PRIVATE_KEY_PASSPHRASE': passphrase,
434
+ 'SNOWFLAKE_PRIVATE_KEY_PATH': str(key_path)
435
+ }
436
+
437
+ # Log which variables are set
438
+ for key, value in env_vars.items():
439
+ if value:
440
+ self._log_debug(f"Environment variable {key} is set")
441
+ else:
442
+ self._log_warning(f"Environment variable {key} is NOT set")
443
+
444
+ return env_vars
445
+
446
+ elif self._current_warehouse_type == 'bigquery':
447
+ # Check if service account JSON exists
448
+ sa_json_path = Path("/bigquery/secret/sa.json")
449
+ if not sa_json_path.exists():
450
+ self._log_warning(f"BigQuery service account JSON not found at {sa_json_path}")
451
+ return {}
452
+
453
+ env_vars = {
454
+ 'GOOGLE_APPLICATION_CREDENTIALS': str(sa_json_path)
455
+ }
456
+
457
+ # Log which variables are set
458
+ for key, value in env_vars.items():
459
+ if value:
460
+ self._log_debug(f"Environment variable {key} is set to {value}")
461
+ else:
462
+ self._log_warning(f"Environment variable {key} is NOT set")
463
+
464
+ return env_vars
465
+
466
+ elif self._current_warehouse_type == 'redshift':
467
+ env_vars = {}
468
+ for var in ["REDSHIFT_PASSWORD", "REDSHIFT_USER", "REDSHIFT_HOST", "REDSHIFT_PORT"]:
469
+ secret_path = Path(self.secrets_base_path) / "redshift" / var
470
+ if secret_path.exists():
471
+ env_vars[var] = self._read_secret(str(secret_path))
472
+ self._log_debug(f"Read {var} from {secret_path}")
473
+ else:
474
+ self._log_warning(f"Secret not found at {secret_path}")
475
+ return env_vars
476
+
477
+ elif self._current_warehouse_type == 'fabric':
478
+ env_vars = {}
479
+ for var in ["FABRIC_USER", "FABRIC_PASSWORD", "FABRIC_SERVER",
480
+ "FABRIC_DATABASE", "FABRIC_PORT", "FABRIC_AUTHENTICATION"]:
481
+ secret_path = Path(self.secrets_base_path) / "fabric" / var
482
+ if secret_path.exists():
483
+ env_vars[var] = self._read_secret(str(secret_path))
484
+ self._log_debug(f"Read {var} from {secret_path}")
485
+ else:
486
+ self._log_warning(f"Secret not found at {secret_path}")
487
+ return env_vars
488
+
489
+ self._log_warning(f"No environment variables found for warehouse type: {self._current_warehouse_type}")
490
+ return {}