dataquery-sdk 0.0.7__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
dataquery/config.py ADDED
@@ -0,0 +1,571 @@
1
+ """
2
+ Environment-based configuration for the DATAQUERY SDK.
3
+
4
+ This module provides a comprehensive configuration system that loads all settings
5
+ from environment variables with proper defaults, validation, and type conversion.
6
+ """
7
+
8
+ import os
9
+ from pathlib import Path
10
+ from typing import Any, Dict, Optional
11
+
12
+ from dotenv import load_dotenv
13
+
14
+ from .exceptions import ConfigurationError
15
+ from .models import ClientConfig
16
+
17
+
18
+ class EnvConfig:
19
+ """Environment-based configuration loader for DATAQUERY SDK."""
20
+
21
+ # Environment variable prefix
22
+ PREFIX = "DATAQUERY_"
23
+
24
+ # Default values for configuration
25
+ DEFAULTS = {
26
+ # API Configuration
27
+ "BASE_URL": "https://api-developer.jpmorgan.com",
28
+ "CONTEXT_PATH": "/research/dataquery-authe/api/v2",
29
+ "API_VERSION": "2.0.0",
30
+ # Optional separate files host
31
+ "FILES_BASE_URL": "https://api-strm-gw01.jpmchase.com",
32
+ "FILES_CONTEXT_PATH": "/research/dataquery-authe/api/v2",
33
+ # OAuth Configuration
34
+ "OAUTH_ENABLED": "true",
35
+ "OAUTH_TOKEN_URL": "https://authe.jpmorgan.com/as/token.oauth2",
36
+ "CLIENT_ID": None,
37
+ "CLIENT_SECRET": None,
38
+ # scope removed
39
+ "OAUTH_AUD": "JPMC:URI:RS-06785-DataQueryExternalApi-PROD",
40
+ "GRANT_TYPE": "client_credentials",
41
+ # Bearer Token Configuration
42
+ "BEARER_TOKEN": None,
43
+ "TOKEN_REFRESH_THRESHOLD": "300",
44
+ # HTTP Configuration
45
+ "TIMEOUT": "6000.0",
46
+ "MAX_RETRIES": "3",
47
+ "RETRY_DELAY": "1.0",
48
+ # Connection Pooling
49
+ "POOL_CONNECTIONS": "10",
50
+ "POOL_MAXSIZE": "20",
51
+ # Rate Limiting
52
+ "REQUESTS_PER_MINUTE": "100",
53
+ "BURST_CAPACITY": "20",
54
+ # Proxy Configuration
55
+ "PROXY_ENABLED": "false",
56
+ "PROXY_URL": "",
57
+ "PROXY_USERNAME": "",
58
+ "PROXY_PASSWORD": "",
59
+ "PROXY_VERIFY_SSL": "true",
60
+ # Logging
61
+ "LOG_LEVEL": "INFO",
62
+ "ENABLE_DEBUG_LOGGING": "false",
63
+ # Download Configuration
64
+ "DOWNLOAD_DIR": "./downloads",
65
+ "CREATE_DIRECTORIES": "true",
66
+ "OVERWRITE_EXISTING": "false",
67
+ # Batch Download Configuration
68
+ "MAX_CONCURRENT_DOWNLOADS": "5",
69
+ "BATCH_SIZE": "10",
70
+ "RETRY_FAILED": "true",
71
+ "MAX_RETRY_ATTEMPTS": "2",
72
+ "CREATE_DATE_FOLDERS": "true",
73
+ "PRESERVE_PATH_STRUCTURE": "true",
74
+ "FLATTEN_STRUCTURE": "false",
75
+ "SHOW_BATCH_PROGRESS": "true",
76
+ "SHOW_INDIVIDUAL_PROGRESS": "true",
77
+ "CONTINUE_ON_ERROR": "true",
78
+ "LOG_ERRORS": "true",
79
+ "SAVE_ERROR_LOG": "true",
80
+ "USE_ASYNC_DOWNLOADS": "true",
81
+ "CHUNK_SIZE": "8192",
82
+ # Download Options
83
+ "ENABLE_RANGE_REQUESTS": "true",
84
+ "SHOW_PROGRESS": "true",
85
+ # Workflow Configuration
86
+ "WORKFLOW_DIR": "workflow",
87
+ "GROUPS_DIR": "groups",
88
+ "AVAILABILITY_DIR": "availability",
89
+ "DEFAULT_DIR": "files",
90
+ # Security
91
+ "MASK_SECRETS": "true",
92
+ "TOKEN_STORAGE_ENABLED": "false",
93
+ "TOKEN_STORAGE_DIR": ".tokens",
94
+ }
95
+
96
+ # Environment variable names
97
+ ENV_VARS = {
98
+ # API Configuration
99
+ "base_url": "DATAQUERY_BASE_URL",
100
+ "context_path": "DATAQUERY_CONTEXT_PATH",
101
+ "api_version": "DATAQUERY_API_VERSION",
102
+ # Optional separate files host
103
+ "files_base_url": "DATAQUERY_FILES_BASE_URL",
104
+ "files_context_path": "DATAQUERY_FILES_CONTEXT_PATH",
105
+ # OAuth Configuration
106
+ "oauth_enabled": "DATAQUERY_OAUTH_ENABLED",
107
+ "oauth_token_url": "DATAQUERY_OAUTH_TOKEN_URL",
108
+ "client_id": "DATAQUERY_CLIENT_ID",
109
+ "client_secret": "DATAQUERY_CLIENT_SECRET",
110
+ # scope removed
111
+ "aud": "DATAQUERY_OAUTH_AUD",
112
+ "grant_type": "DATAQUERY_OAUTH_GRANT_TYPE",
113
+ # Bearer Token Configuration
114
+ "bearer_token": "DATAQUERY_BEARER_TOKEN",
115
+ "token_refresh_threshold": "DATAQUERY_TOKEN_REFRESH_THRESHOLD",
116
+ # HTTP Configuration
117
+ "timeout": "DATAQUERY_TIMEOUT",
118
+ "max_retries": "DATAQUERY_MAX_RETRIES",
119
+ "retry_delay": "DATAQUERY_RETRY_DELAY",
120
+ # Connection Pooling
121
+ "pool_connections": "DATAQUERY_POOL_CONNECTIONS",
122
+ "pool_maxsize": "DATAQUERY_POOL_MAXSIZE",
123
+ # Rate Limiting
124
+ "requests_per_minute": "DATAQUERY_REQUESTS_PER_MINUTE",
125
+ "burst_capacity": "DATAQUERY_BURST_CAPACITY",
126
+ # Proxy Configuration
127
+ "proxy_enabled": "DATAQUERY_PROXY_ENABLED",
128
+ "proxy_url": "DATAQUERY_PROXY_URL",
129
+ "proxy_username": "DATAQUERY_PROXY_USERNAME",
130
+ "proxy_password": "DATAQUERY_PROXY_PASSWORD",
131
+ "proxy_verify_ssl": "DATAQUERY_PROXY_VERIFY_SSL",
132
+ # Logging
133
+ "log_level": "DATAQUERY_LOG_LEVEL",
134
+ "enable_debug_logging": "DATAQUERY_ENABLE_DEBUG_LOGGING",
135
+ # Download Configuration
136
+ "download_dir": "DATAQUERY_DOWNLOAD_DIR",
137
+ "create_directories": "DATAQUERY_CREATE_DIRECTORIES",
138
+ "overwrite_existing": "DATAQUERY_OVERWRITE_EXISTING",
139
+ }
140
+
141
+ @classmethod
142
+ def load_env_file(cls, env_file: Optional[Path] = None) -> None:
143
+ """Load environment variables from .env file."""
144
+ if env_file is None:
145
+ env_file = Path(".env")
146
+
147
+ if env_file.exists():
148
+ load_dotenv(env_file)
149
+
150
+ @classmethod
151
+ def get_env_var(cls, key: str, default: Optional[str] = None) -> Optional[str]:
152
+ """Get environment variable with prefix, using DEFAULTS if no default provided."""
153
+ env_key = f"{cls.PREFIX}{key}"
154
+
155
+ # Use default from DEFAULTS if no default provided
156
+ if default is None:
157
+ default = cls.DEFAULTS.get(key)
158
+
159
+ value = os.getenv(env_key, default)
160
+
161
+ # Handle empty strings as None
162
+ if value == "":
163
+ return None
164
+
165
+ return value
166
+
167
+ @classmethod
168
+ def get_bool(cls, key: str, default: Optional[str] = None) -> bool:
169
+ """Get boolean environment variable."""
170
+ if default is None:
171
+ default = cls.DEFAULTS.get(key, "false")
172
+ value = cls.get_env_var(key, default)
173
+ if value is None:
174
+ return False
175
+ return value.lower() in ("true", "1", "yes", "on")
176
+
177
+ @classmethod
178
+ def get_int(cls, key: str, default: Optional[str] = None) -> int:
179
+ """Get integer environment variable."""
180
+ if default is None:
181
+ default = cls.DEFAULTS.get(key, "0")
182
+ value = cls.get_env_var(key, default)
183
+ if value is None:
184
+ return 0
185
+ try:
186
+ return int(value)
187
+ except ValueError:
188
+ raise ConfigurationError(
189
+ f"Invalid integer value for {cls.PREFIX}{key}: {value}"
190
+ )
191
+
192
+ @classmethod
193
+ def get_float(cls, key: str, default: Optional[str] = None) -> float:
194
+ """Get float environment variable."""
195
+ if default is None:
196
+ default = cls.DEFAULTS.get(key, "0.0")
197
+ value = cls.get_env_var(key, default)
198
+ if value is None:
199
+ return 0.0
200
+ try:
201
+ return float(value)
202
+ except ValueError:
203
+ raise ConfigurationError(
204
+ f"Invalid float value for {cls.PREFIX}{key}: {value}"
205
+ )
206
+
207
+ @classmethod
208
+ def get_path(cls, key: str, default: str = ".") -> Path:
209
+ """Get path environment variable."""
210
+ value = cls.get_env_var(key, default)
211
+ if value is None:
212
+ return Path(".")
213
+ return Path(value)
214
+
215
+ @classmethod
216
+ def create_client_config_with_defaults(cls, base_url: str) -> ClientConfig:
217
+ """
218
+ Create ClientConfig with just base_url and all other defaults.
219
+
220
+ Args:
221
+ base_url: Base URL of the DataQuery API
222
+
223
+ Returns:
224
+ ClientConfig instance with defaults
225
+ """
226
+ return ClientConfig(
227
+ base_url=base_url,
228
+ # All other fields will use their default values from the model
229
+ )
230
+
231
+ @classmethod
232
+ def create_client_config(
233
+ cls,
234
+ config_data: Optional[Dict[str, Any]] = None,
235
+ env_file: Optional[Path] = None,
236
+ ) -> ClientConfig:
237
+ """
238
+ Create ClientConfig from environment variables or provided config data.
239
+
240
+ Args:
241
+ config_data: Optional dictionary with configuration data
242
+ env_file: Optional path to .env file
243
+
244
+ Returns:
245
+ ClientConfig instance
246
+
247
+ Raises:
248
+ ConfigurationError: If required configuration is missing or invalid
249
+ """
250
+ if config_data is None:
251
+ # Load .env file if provided
252
+ if env_file is not None:
253
+ cls.load_env_file(env_file)
254
+
255
+ # Get base URL (required)
256
+ base_url = cls.get_env_var("BASE_URL")
257
+ if not base_url:
258
+ raise ConfigurationError(
259
+ f"{cls.PREFIX}BASE_URL environment variable is required"
260
+ )
261
+
262
+ # Auto-generate OAuth token URL if not provided
263
+ oauth_token_url = cls.get_env_var("OAUTH_TOKEN_URL")
264
+ if not oauth_token_url and cls.get_bool("OAUTH_ENABLED"):
265
+ oauth_token_url = f"{base_url}/oauth/token"
266
+
267
+ return ClientConfig(
268
+ # API configuration
269
+ base_url=base_url,
270
+ context_path=cls.get_env_var("CONTEXT_PATH"),
271
+ api_version=cls.get_env_var("API_VERSION") or "2.0.0",
272
+ files_base_url=cls.get_env_var("FILES_BASE_URL"),
273
+ files_context_path=cls.get_env_var("FILES_CONTEXT_PATH"),
274
+ # OAuth configuration
275
+ oauth_enabled=cls.get_bool("OAUTH_ENABLED"),
276
+ oauth_token_url=oauth_token_url,
277
+ client_id=cls.get_env_var("CLIENT_ID"),
278
+ client_secret=cls.get_env_var("CLIENT_SECRET"),
279
+ # scope removed
280
+ aud=cls.get_env_var("OAUTH_AUD"),
281
+ grant_type=cls.get_env_var("GRANT_TYPE") or "client_credentials",
282
+ # Bearer token configuration
283
+ bearer_token=cls.get_env_var("BEARER_TOKEN"),
284
+ token_refresh_threshold=cls.get_int("TOKEN_REFRESH_THRESHOLD"),
285
+ # HTTP configuration
286
+ timeout=cls.get_float("TIMEOUT"),
287
+ max_retries=cls.get_int("MAX_RETRIES"),
288
+ retry_delay=cls.get_float("RETRY_DELAY"),
289
+ # Connection pooling
290
+ pool_connections=cls.get_int("POOL_CONNECTIONS"),
291
+ pool_maxsize=cls.get_int("POOL_MAXSIZE"),
292
+ # Rate limiting
293
+ requests_per_minute=cls.get_int("REQUESTS_PER_MINUTE"),
294
+ burst_capacity=cls.get_int("BURST_CAPACITY"),
295
+ # Proxy configuration
296
+ proxy_enabled=cls.get_bool("PROXY_ENABLED"),
297
+ proxy_url=cls.get_env_var("PROXY_URL"),
298
+ proxy_username=cls.get_env_var("PROXY_USERNAME"),
299
+ proxy_password=cls.get_env_var("PROXY_PASSWORD"),
300
+ proxy_verify_ssl=cls.get_bool("PROXY_VERIFY_SSL"),
301
+ # Logging
302
+ log_level=cls.get_env_var("LOG_LEVEL") or "INFO",
303
+ enable_debug_logging=cls.get_bool("ENABLE_DEBUG_LOGGING"),
304
+ # Download configuration
305
+ download_dir=cls.get_env_var("DOWNLOAD_DIR") or "./downloads",
306
+ create_directories=cls.get_bool("CREATE_DIRECTORIES"),
307
+ overwrite_existing=cls.get_bool("OVERWRITE_EXISTING"),
308
+ # Batch Download Configuration
309
+ max_concurrent_downloads=cls.get_int("MAX_CONCURRENT_DOWNLOADS"),
310
+ batch_size=cls.get_int("BATCH_SIZE"),
311
+ retry_failed=cls.get_bool("RETRY_FAILED"),
312
+ max_retry_attempts=cls.get_int("MAX_RETRY_ATTEMPTS"),
313
+ create_date_folders=cls.get_bool("CREATE_DATE_FOLDERS"),
314
+ preserve_path_structure=cls.get_bool("PRESERVE_PATH_STRUCTURE"),
315
+ flatten_structure=cls.get_bool("FLATTEN_STRUCTURE"),
316
+ show_batch_progress=cls.get_bool("SHOW_BATCH_PROGRESS"),
317
+ show_individual_progress=cls.get_bool("SHOW_INDIVIDUAL_PROGRESS"),
318
+ continue_on_error=cls.get_bool("CONTINUE_ON_ERROR"),
319
+ log_errors=cls.get_bool("LOG_ERRORS"),
320
+ save_error_log=cls.get_bool("SAVE_ERROR_LOG"),
321
+ use_async_downloads=cls.get_bool("USE_ASYNC_DOWNLOADS"),
322
+ chunk_size=cls.get_int("CHUNK_SIZE"),
323
+ # Download Options
324
+ enable_range_requests=cls.get_bool("ENABLE_RANGE_REQUESTS"),
325
+ show_progress=cls.get_bool("SHOW_PROGRESS"),
326
+ # Workflow Configuration
327
+ workflow_dir=cls.get_env_var("WORKFLOW_DIR") or "workflow",
328
+ groups_dir=cls.get_env_var("GROUPS_DIR") or "groups",
329
+ availability_dir=cls.get_env_var("AVAILABILITY_DIR") or "availability",
330
+ default_dir=cls.get_env_var("DEFAULT_DIR") or "files",
331
+ # Security
332
+ mask_secrets=cls.get_bool("MASK_SECRETS"),
333
+ token_storage_enabled=cls.get_bool("TOKEN_STORAGE_ENABLED"),
334
+ token_storage_dir=cls.get_env_var("TOKEN_STORAGE_DIR"),
335
+ )
336
+ else:
337
+ # Use provided config data
338
+ return ClientConfig(**config_data)
339
+
340
+ @classmethod
341
+ def get_download_options(cls) -> Dict[str, Any]:
342
+ """Get download options from environment variables."""
343
+ return {
344
+ "chunk_size": cls.get_int("CHUNK_SIZE", "8192"),
345
+ "max_retries": cls.get_int("MAX_RETRIES", "3"),
346
+ "retry_delay": cls.get_float("RETRY_DELAY", "1.0"),
347
+ "timeout": cls.get_float("TIMEOUT", "600.0"),
348
+ "enable_range_requests": cls.get_bool("ENABLE_RANGE_REQUESTS"),
349
+ "show_progress": cls.get_bool("SHOW_PROGRESS"),
350
+ "create_directories": cls.get_bool("CREATE_DIRECTORIES"),
351
+ "overwrite_existing": cls.get_bool("OVERWRITE_EXISTING"),
352
+ }
353
+
354
+ @classmethod
355
+ def get_batch_download_options(cls) -> Dict[str, Any]:
356
+ """Get batch download options from environment variables."""
357
+ return {
358
+ "max_concurrent_downloads": cls.get_int("MAX_CONCURRENT_DOWNLOADS", "3"),
359
+ "batch_size": cls.get_int("BATCH_SIZE", "10"),
360
+ "retry_failed": cls.get_bool("RETRY_FAILED"),
361
+ "max_retry_attempts": cls.get_int("MAX_RETRY_ATTEMPTS", "2"),
362
+ "create_date_folders": cls.get_bool("CREATE_DATE_FOLDERS"),
363
+ "preserve_path_structure": cls.get_bool("PRESERVE_PATH_STRUCTURE"),
364
+ "flatten_structure": cls.get_bool("FLATTEN_STRUCTURE"),
365
+ "show_batch_progress": cls.get_bool("SHOW_BATCH_PROGRESS"),
366
+ "show_individual_progress": cls.get_bool("SHOW_INDIVIDUAL_PROGRESS"),
367
+ "continue_on_error": cls.get_bool("CONTINUE_ON_ERROR"),
368
+ "log_errors": cls.get_bool("LOG_ERRORS"),
369
+ "save_error_log": cls.get_bool("SAVE_ERROR_LOG"),
370
+ "use_async_downloads": cls.get_bool("USE_ASYNC_DOWNLOADS"),
371
+ "chunk_size": cls.get_int("CHUNK_SIZE", "8192"),
372
+ }
373
+
374
+ @classmethod
375
+ def get_workflow_paths(cls) -> Dict[str, Path]:
376
+ """Get workflow directory paths from environment variables."""
377
+ base_download_dir = cls.get_path("DOWNLOAD_DIR", "./downloads")
378
+
379
+ return {
380
+ "base": base_download_dir,
381
+ "workflow": base_download_dir
382
+ / (cls.get_env_var("WORKFLOW_DIR") or "workflow"),
383
+ "groups": base_download_dir / (cls.get_env_var("GROUPS_DIR") or "groups"),
384
+ "availability": base_download_dir
385
+ / (cls.get_env_var("AVAILABILITY_DIR") or "availability"),
386
+ "default": base_download_dir / (cls.get_env_var("DEFAULT_DIR") or "files"),
387
+ }
388
+
389
+ @classmethod
390
+ def get_token_storage_config(cls) -> Dict[str, Any]:
391
+ """Get token storage configuration from environment variables."""
392
+ return {
393
+ "enabled": cls.get_bool("TOKEN_STORAGE_ENABLED"),
394
+ "directory": cls.get_env_var("TOKEN_STORAGE_DIR", ".tokens"),
395
+ }
396
+
397
+ @classmethod
398
+ def validate_config(cls, config: ClientConfig) -> None:
399
+ """Validate configuration and raise ConfigurationError if invalid."""
400
+ errors = []
401
+
402
+ # Check required fields
403
+ if not config.base_url:
404
+ errors.append("BASE_URL is required")
405
+
406
+ # Check OAuth configuration
407
+ if config.oauth_enabled:
408
+ if not config.client_id:
409
+ errors.append("CLIENT_ID is required when OAuth is enabled")
410
+ if not config.client_secret:
411
+ errors.append("CLIENT_SECRET is required when OAuth is enabled")
412
+ if not config.oauth_token_url:
413
+ errors.append("OAUTH_TOKEN_URL is required when OAuth is enabled")
414
+
415
+ # Check Bearer token configuration
416
+ if not config.has_oauth_credentials and not config.has_bearer_token:
417
+ errors.append("Either OAuth credentials or Bearer token must be configured")
418
+
419
+ # Check numeric values
420
+ if config.timeout <= 0:
421
+ errors.append("TIMEOUT must be positive")
422
+ if config.max_retries < 0:
423
+ errors.append("MAX_RETRIES must be non-negative")
424
+ if config.retry_delay < 0:
425
+ errors.append("RETRY_DELAY must be non-negative")
426
+ if config.pool_connections <= 0:
427
+ errors.append("POOL_CONNECTIONS must be positive")
428
+ if config.pool_maxsize <= 0:
429
+ errors.append("POOL_MAXSIZE must be positive")
430
+ if config.requests_per_minute <= 0:
431
+ errors.append("REQUESTS_PER_MINUTE must be positive")
432
+ if config.burst_capacity <= 0:
433
+ errors.append("BURST_CAPACITY must be positive")
434
+
435
+ if errors:
436
+ raise ConfigurationError(
437
+ f"Configuration validation failed: {'; '.join(errors)}"
438
+ )
439
+
440
+ @classmethod
441
+ def create_env_template(cls, output_path: Optional[Path] = None) -> Path:
442
+ """
443
+ Create a .env template file with all available configuration options.
444
+
445
+ Args:
446
+ output_path: Optional output path for the template
447
+
448
+ Returns:
449
+ Path to the created template file
450
+ """
451
+ if output_path is None:
452
+ output_path = Path(".env.template")
453
+
454
+ # Ensure output_path is a Path object
455
+ if not isinstance(output_path, Path):
456
+ output_path = Path(output_path)
457
+
458
+ template_content = f"""# DATAQUERY SDK Environment Configuration Template
459
+ # Copy this file to .env and update the values as needed
460
+
461
+ # API Configuration
462
+ {cls.PREFIX}BASE_URL=https://api-developer.jpmorgan.com
463
+ {cls.PREFIX}CONTEXT_PATH=/research/dataquery-authe/api/v2
464
+ # Optional separate Files API host
465
+ {cls.PREFIX}FILES_BASE_URL=https://api-strm-gw01.jpmchase.com
466
+ {cls.PREFIX}FILES_CONTEXT_PATH=/research/dataquery-authe/api/v2
467
+
468
+ # OAuth Configuration
469
+ {cls.PREFIX}OAUTH_ENABLED=true
470
+ {cls.PREFIX}OAUTH_TOKEN_URL=https://authe.jpmorgan.com/as/token.oauth2
471
+ {cls.PREFIX}CLIENT_ID=your_client_id_here
472
+ {cls.PREFIX}CLIENT_SECRET=your_client_secret_here
473
+ {cls.PREFIX}GRANT_TYPE=client_credentials
474
+
475
+ # Bearer Token Configuration (alternative to OAuth)
476
+ {cls.PREFIX}BEARER_TOKEN=your_bearer_token_here
477
+ {cls.PREFIX}TOKEN_REFRESH_THRESHOLD=300
478
+
479
+ # HTTP Configuration
480
+ {cls.PREFIX}TIMEOUT=600.0
481
+ {cls.PREFIX}MAX_RETRIES=3
482
+ {cls.PREFIX}RETRY_DELAY=1.0
483
+
484
+ # Connection Pooling
485
+ {cls.PREFIX}POOL_CONNECTIONS=10
486
+ {cls.PREFIX}POOL_MAXSIZE=20
487
+
488
+ # Rate Limiting
489
+ {cls.PREFIX}REQUESTS_PER_MINUTE=100
490
+ {cls.PREFIX}BURST_CAPACITY=20
491
+
492
+ # Proxy Configuration
493
+ {cls.PREFIX}PROXY_ENABLED=false
494
+ {cls.PREFIX}PROXY_URL=
495
+ {cls.PREFIX}PROXY_USERNAME=
496
+ {cls.PREFIX}PROXY_PASSWORD=
497
+ {cls.PREFIX}PROXY_VERIFY_SSL=true
498
+
499
+ # Logging
500
+ {cls.PREFIX}LOG_LEVEL=INFO
501
+ {cls.PREFIX}ENABLE_DEBUG_LOGGING=false
502
+
503
+ # Download Configuration
504
+ {cls.PREFIX}DOWNLOAD_DIR=./downloads
505
+ {cls.PREFIX}CREATE_DIRECTORIES=true
506
+ {cls.PREFIX}OVERWRITE_EXISTING=false
507
+
508
+ # Batch Download Configuration
509
+ {cls.PREFIX}MAX_CONCURRENT_DOWNLOADS=5
510
+ {cls.PREFIX}BATCH_SIZE=10
511
+ {cls.PREFIX}RETRY_FAILED=true
512
+ {cls.PREFIX}MAX_RETRY_ATTEMPTS=2
513
+ {cls.PREFIX}CREATE_DATE_FOLDERS=true
514
+ {cls.PREFIX}PRESERVE_PATH_STRUCTURE=true
515
+ {cls.PREFIX}FLATTEN_STRUCTURE=false
516
+ {cls.PREFIX}SHOW_BATCH_PROGRESS=true
517
+ {cls.PREFIX}SHOW_INDIVIDUAL_PROGRESS=true
518
+ {cls.PREFIX}CONTINUE_ON_ERROR=true
519
+ {cls.PREFIX}LOG_ERRORS=true
520
+ {cls.PREFIX}SAVE_ERROR_LOG=true
521
+ {cls.PREFIX}USE_ASYNC_DOWNLOADS=true
522
+ {cls.PREFIX}CHUNK_SIZE=8192
523
+
524
+ # Download Options
525
+ {cls.PREFIX}ENABLE_RANGE_REQUESTS=true
526
+ {cls.PREFIX}SHOW_PROGRESS=true
527
+
528
+ # Workflow Configuration
529
+ {cls.PREFIX}WORKFLOW_DIR=workflow
530
+ {cls.PREFIX}GROUPS_DIR=groups
531
+ {cls.PREFIX}AVAILABILITY_DIR=availability
532
+ {cls.PREFIX}DEFAULT_DIR=files
533
+
534
+ # Security
535
+ {cls.PREFIX}MASK_SECRETS=true
536
+ {cls.PREFIX}TOKEN_STORAGE_ENABLED=true
537
+ {cls.PREFIX}TOKEN_STORAGE_DIR=.tokens
538
+ """
539
+
540
+ output_path.write_text(template_content)
541
+ return output_path
542
+
543
+ @classmethod
544
+ def get_all_env_vars(cls) -> Dict[str, str]:
545
+ """Get all DATAQUERY environment variables."""
546
+ env_vars: Dict[str, str] = {}
547
+ # Return only variables that are explicitly set in the environment, not defaults
548
+ prefix = cls.PREFIX
549
+ for k, v in os.environ.items():
550
+ if k.startswith(prefix):
551
+ env_key = k[len(prefix) :]
552
+ env_vars[env_key] = v
553
+ return env_vars
554
+
555
+ @classmethod
556
+ def mask_secrets(cls, config_dict: Dict[str, Any]) -> Dict[str, Any]:
557
+ """Mask sensitive configuration values."""
558
+ sensitive_keys = {
559
+ "client_id",
560
+ "client_secret",
561
+ "bearer_token",
562
+ "oauth_token_url",
563
+ "aud",
564
+ }
565
+
566
+ masked_config = config_dict.copy()
567
+ for key in sensitive_keys:
568
+ if key in masked_config and masked_config[key]:
569
+ masked_config[key] = "***"
570
+
571
+ return masked_config