dataquery-sdk 0.0.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dataquery/__init__.py +215 -0
- dataquery/auth.py +380 -0
- dataquery/auto_download.py +448 -0
- dataquery/cli.py +244 -0
- dataquery/client.py +2825 -0
- dataquery/config.py +571 -0
- dataquery/connection_pool.py +538 -0
- dataquery/dataquery.py +2467 -0
- dataquery/exceptions.py +169 -0
- dataquery/logging_config.py +415 -0
- dataquery/models.py +1303 -0
- dataquery/py.typed +2 -0
- dataquery/rate_limiter.py +520 -0
- dataquery/retry.py +406 -0
- dataquery/utils.py +525 -0
- dataquery_sdk-0.0.7.dist-info/METADATA +996 -0
- dataquery_sdk-0.0.7.dist-info/RECORD +21 -0
- dataquery_sdk-0.0.7.dist-info/WHEEL +5 -0
- dataquery_sdk-0.0.7.dist-info/entry_points.txt +2 -0
- dataquery_sdk-0.0.7.dist-info/licenses/LICENSE +201 -0
- dataquery_sdk-0.0.7.dist-info/top_level.txt +1 -0
dataquery/config.py
ADDED
|
@@ -0,0 +1,571 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Environment-based configuration for the DATAQUERY SDK.
|
|
3
|
+
|
|
4
|
+
This module provides a comprehensive configuration system that loads all settings
|
|
5
|
+
from environment variables with proper defaults, validation, and type conversion.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import os
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Any, Dict, Optional
|
|
11
|
+
|
|
12
|
+
from dotenv import load_dotenv
|
|
13
|
+
|
|
14
|
+
from .exceptions import ConfigurationError
|
|
15
|
+
from .models import ClientConfig
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class EnvConfig:
|
|
19
|
+
"""Environment-based configuration loader for DATAQUERY SDK."""
|
|
20
|
+
|
|
21
|
+
# Environment variable prefix
|
|
22
|
+
PREFIX = "DATAQUERY_"
|
|
23
|
+
|
|
24
|
+
# Default values for configuration
|
|
25
|
+
DEFAULTS = {
|
|
26
|
+
# API Configuration
|
|
27
|
+
"BASE_URL": "https://api-developer.jpmorgan.com",
|
|
28
|
+
"CONTEXT_PATH": "/research/dataquery-authe/api/v2",
|
|
29
|
+
"API_VERSION": "2.0.0",
|
|
30
|
+
# Optional separate files host
|
|
31
|
+
"FILES_BASE_URL": "https://api-strm-gw01.jpmchase.com",
|
|
32
|
+
"FILES_CONTEXT_PATH": "/research/dataquery-authe/api/v2",
|
|
33
|
+
# OAuth Configuration
|
|
34
|
+
"OAUTH_ENABLED": "true",
|
|
35
|
+
"OAUTH_TOKEN_URL": "https://authe.jpmorgan.com/as/token.oauth2",
|
|
36
|
+
"CLIENT_ID": None,
|
|
37
|
+
"CLIENT_SECRET": None,
|
|
38
|
+
# scope removed
|
|
39
|
+
"OAUTH_AUD": "JPMC:URI:RS-06785-DataQueryExternalApi-PROD",
|
|
40
|
+
"GRANT_TYPE": "client_credentials",
|
|
41
|
+
# Bearer Token Configuration
|
|
42
|
+
"BEARER_TOKEN": None,
|
|
43
|
+
"TOKEN_REFRESH_THRESHOLD": "300",
|
|
44
|
+
# HTTP Configuration
|
|
45
|
+
"TIMEOUT": "6000.0",
|
|
46
|
+
"MAX_RETRIES": "3",
|
|
47
|
+
"RETRY_DELAY": "1.0",
|
|
48
|
+
# Connection Pooling
|
|
49
|
+
"POOL_CONNECTIONS": "10",
|
|
50
|
+
"POOL_MAXSIZE": "20",
|
|
51
|
+
# Rate Limiting
|
|
52
|
+
"REQUESTS_PER_MINUTE": "100",
|
|
53
|
+
"BURST_CAPACITY": "20",
|
|
54
|
+
# Proxy Configuration
|
|
55
|
+
"PROXY_ENABLED": "false",
|
|
56
|
+
"PROXY_URL": "",
|
|
57
|
+
"PROXY_USERNAME": "",
|
|
58
|
+
"PROXY_PASSWORD": "",
|
|
59
|
+
"PROXY_VERIFY_SSL": "true",
|
|
60
|
+
# Logging
|
|
61
|
+
"LOG_LEVEL": "INFO",
|
|
62
|
+
"ENABLE_DEBUG_LOGGING": "false",
|
|
63
|
+
# Download Configuration
|
|
64
|
+
"DOWNLOAD_DIR": "./downloads",
|
|
65
|
+
"CREATE_DIRECTORIES": "true",
|
|
66
|
+
"OVERWRITE_EXISTING": "false",
|
|
67
|
+
# Batch Download Configuration
|
|
68
|
+
"MAX_CONCURRENT_DOWNLOADS": "5",
|
|
69
|
+
"BATCH_SIZE": "10",
|
|
70
|
+
"RETRY_FAILED": "true",
|
|
71
|
+
"MAX_RETRY_ATTEMPTS": "2",
|
|
72
|
+
"CREATE_DATE_FOLDERS": "true",
|
|
73
|
+
"PRESERVE_PATH_STRUCTURE": "true",
|
|
74
|
+
"FLATTEN_STRUCTURE": "false",
|
|
75
|
+
"SHOW_BATCH_PROGRESS": "true",
|
|
76
|
+
"SHOW_INDIVIDUAL_PROGRESS": "true",
|
|
77
|
+
"CONTINUE_ON_ERROR": "true",
|
|
78
|
+
"LOG_ERRORS": "true",
|
|
79
|
+
"SAVE_ERROR_LOG": "true",
|
|
80
|
+
"USE_ASYNC_DOWNLOADS": "true",
|
|
81
|
+
"CHUNK_SIZE": "8192",
|
|
82
|
+
# Download Options
|
|
83
|
+
"ENABLE_RANGE_REQUESTS": "true",
|
|
84
|
+
"SHOW_PROGRESS": "true",
|
|
85
|
+
# Workflow Configuration
|
|
86
|
+
"WORKFLOW_DIR": "workflow",
|
|
87
|
+
"GROUPS_DIR": "groups",
|
|
88
|
+
"AVAILABILITY_DIR": "availability",
|
|
89
|
+
"DEFAULT_DIR": "files",
|
|
90
|
+
# Security
|
|
91
|
+
"MASK_SECRETS": "true",
|
|
92
|
+
"TOKEN_STORAGE_ENABLED": "false",
|
|
93
|
+
"TOKEN_STORAGE_DIR": ".tokens",
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
# Environment variable names
|
|
97
|
+
ENV_VARS = {
|
|
98
|
+
# API Configuration
|
|
99
|
+
"base_url": "DATAQUERY_BASE_URL",
|
|
100
|
+
"context_path": "DATAQUERY_CONTEXT_PATH",
|
|
101
|
+
"api_version": "DATAQUERY_API_VERSION",
|
|
102
|
+
# Optional separate files host
|
|
103
|
+
"files_base_url": "DATAQUERY_FILES_BASE_URL",
|
|
104
|
+
"files_context_path": "DATAQUERY_FILES_CONTEXT_PATH",
|
|
105
|
+
# OAuth Configuration
|
|
106
|
+
"oauth_enabled": "DATAQUERY_OAUTH_ENABLED",
|
|
107
|
+
"oauth_token_url": "DATAQUERY_OAUTH_TOKEN_URL",
|
|
108
|
+
"client_id": "DATAQUERY_CLIENT_ID",
|
|
109
|
+
"client_secret": "DATAQUERY_CLIENT_SECRET",
|
|
110
|
+
# scope removed
|
|
111
|
+
"aud": "DATAQUERY_OAUTH_AUD",
|
|
112
|
+
"grant_type": "DATAQUERY_OAUTH_GRANT_TYPE",
|
|
113
|
+
# Bearer Token Configuration
|
|
114
|
+
"bearer_token": "DATAQUERY_BEARER_TOKEN",
|
|
115
|
+
"token_refresh_threshold": "DATAQUERY_TOKEN_REFRESH_THRESHOLD",
|
|
116
|
+
# HTTP Configuration
|
|
117
|
+
"timeout": "DATAQUERY_TIMEOUT",
|
|
118
|
+
"max_retries": "DATAQUERY_MAX_RETRIES",
|
|
119
|
+
"retry_delay": "DATAQUERY_RETRY_DELAY",
|
|
120
|
+
# Connection Pooling
|
|
121
|
+
"pool_connections": "DATAQUERY_POOL_CONNECTIONS",
|
|
122
|
+
"pool_maxsize": "DATAQUERY_POOL_MAXSIZE",
|
|
123
|
+
# Rate Limiting
|
|
124
|
+
"requests_per_minute": "DATAQUERY_REQUESTS_PER_MINUTE",
|
|
125
|
+
"burst_capacity": "DATAQUERY_BURST_CAPACITY",
|
|
126
|
+
# Proxy Configuration
|
|
127
|
+
"proxy_enabled": "DATAQUERY_PROXY_ENABLED",
|
|
128
|
+
"proxy_url": "DATAQUERY_PROXY_URL",
|
|
129
|
+
"proxy_username": "DATAQUERY_PROXY_USERNAME",
|
|
130
|
+
"proxy_password": "DATAQUERY_PROXY_PASSWORD",
|
|
131
|
+
"proxy_verify_ssl": "DATAQUERY_PROXY_VERIFY_SSL",
|
|
132
|
+
# Logging
|
|
133
|
+
"log_level": "DATAQUERY_LOG_LEVEL",
|
|
134
|
+
"enable_debug_logging": "DATAQUERY_ENABLE_DEBUG_LOGGING",
|
|
135
|
+
# Download Configuration
|
|
136
|
+
"download_dir": "DATAQUERY_DOWNLOAD_DIR",
|
|
137
|
+
"create_directories": "DATAQUERY_CREATE_DIRECTORIES",
|
|
138
|
+
"overwrite_existing": "DATAQUERY_OVERWRITE_EXISTING",
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
@classmethod
|
|
142
|
+
def load_env_file(cls, env_file: Optional[Path] = None) -> None:
|
|
143
|
+
"""Load environment variables from .env file."""
|
|
144
|
+
if env_file is None:
|
|
145
|
+
env_file = Path(".env")
|
|
146
|
+
|
|
147
|
+
if env_file.exists():
|
|
148
|
+
load_dotenv(env_file)
|
|
149
|
+
|
|
150
|
+
@classmethod
|
|
151
|
+
def get_env_var(cls, key: str, default: Optional[str] = None) -> Optional[str]:
|
|
152
|
+
"""Get environment variable with prefix, using DEFAULTS if no default provided."""
|
|
153
|
+
env_key = f"{cls.PREFIX}{key}"
|
|
154
|
+
|
|
155
|
+
# Use default from DEFAULTS if no default provided
|
|
156
|
+
if default is None:
|
|
157
|
+
default = cls.DEFAULTS.get(key)
|
|
158
|
+
|
|
159
|
+
value = os.getenv(env_key, default)
|
|
160
|
+
|
|
161
|
+
# Handle empty strings as None
|
|
162
|
+
if value == "":
|
|
163
|
+
return None
|
|
164
|
+
|
|
165
|
+
return value
|
|
166
|
+
|
|
167
|
+
@classmethod
|
|
168
|
+
def get_bool(cls, key: str, default: Optional[str] = None) -> bool:
|
|
169
|
+
"""Get boolean environment variable."""
|
|
170
|
+
if default is None:
|
|
171
|
+
default = cls.DEFAULTS.get(key, "false")
|
|
172
|
+
value = cls.get_env_var(key, default)
|
|
173
|
+
if value is None:
|
|
174
|
+
return False
|
|
175
|
+
return value.lower() in ("true", "1", "yes", "on")
|
|
176
|
+
|
|
177
|
+
@classmethod
|
|
178
|
+
def get_int(cls, key: str, default: Optional[str] = None) -> int:
|
|
179
|
+
"""Get integer environment variable."""
|
|
180
|
+
if default is None:
|
|
181
|
+
default = cls.DEFAULTS.get(key, "0")
|
|
182
|
+
value = cls.get_env_var(key, default)
|
|
183
|
+
if value is None:
|
|
184
|
+
return 0
|
|
185
|
+
try:
|
|
186
|
+
return int(value)
|
|
187
|
+
except ValueError:
|
|
188
|
+
raise ConfigurationError(
|
|
189
|
+
f"Invalid integer value for {cls.PREFIX}{key}: {value}"
|
|
190
|
+
)
|
|
191
|
+
|
|
192
|
+
@classmethod
|
|
193
|
+
def get_float(cls, key: str, default: Optional[str] = None) -> float:
|
|
194
|
+
"""Get float environment variable."""
|
|
195
|
+
if default is None:
|
|
196
|
+
default = cls.DEFAULTS.get(key, "0.0")
|
|
197
|
+
value = cls.get_env_var(key, default)
|
|
198
|
+
if value is None:
|
|
199
|
+
return 0.0
|
|
200
|
+
try:
|
|
201
|
+
return float(value)
|
|
202
|
+
except ValueError:
|
|
203
|
+
raise ConfigurationError(
|
|
204
|
+
f"Invalid float value for {cls.PREFIX}{key}: {value}"
|
|
205
|
+
)
|
|
206
|
+
|
|
207
|
+
@classmethod
|
|
208
|
+
def get_path(cls, key: str, default: str = ".") -> Path:
|
|
209
|
+
"""Get path environment variable."""
|
|
210
|
+
value = cls.get_env_var(key, default)
|
|
211
|
+
if value is None:
|
|
212
|
+
return Path(".")
|
|
213
|
+
return Path(value)
|
|
214
|
+
|
|
215
|
+
@classmethod
|
|
216
|
+
def create_client_config_with_defaults(cls, base_url: str) -> ClientConfig:
|
|
217
|
+
"""
|
|
218
|
+
Create ClientConfig with just base_url and all other defaults.
|
|
219
|
+
|
|
220
|
+
Args:
|
|
221
|
+
base_url: Base URL of the DataQuery API
|
|
222
|
+
|
|
223
|
+
Returns:
|
|
224
|
+
ClientConfig instance with defaults
|
|
225
|
+
"""
|
|
226
|
+
return ClientConfig(
|
|
227
|
+
base_url=base_url,
|
|
228
|
+
# All other fields will use their default values from the model
|
|
229
|
+
)
|
|
230
|
+
|
|
231
|
+
@classmethod
|
|
232
|
+
def create_client_config(
|
|
233
|
+
cls,
|
|
234
|
+
config_data: Optional[Dict[str, Any]] = None,
|
|
235
|
+
env_file: Optional[Path] = None,
|
|
236
|
+
) -> ClientConfig:
|
|
237
|
+
"""
|
|
238
|
+
Create ClientConfig from environment variables or provided config data.
|
|
239
|
+
|
|
240
|
+
Args:
|
|
241
|
+
config_data: Optional dictionary with configuration data
|
|
242
|
+
env_file: Optional path to .env file
|
|
243
|
+
|
|
244
|
+
Returns:
|
|
245
|
+
ClientConfig instance
|
|
246
|
+
|
|
247
|
+
Raises:
|
|
248
|
+
ConfigurationError: If required configuration is missing or invalid
|
|
249
|
+
"""
|
|
250
|
+
if config_data is None:
|
|
251
|
+
# Load .env file if provided
|
|
252
|
+
if env_file is not None:
|
|
253
|
+
cls.load_env_file(env_file)
|
|
254
|
+
|
|
255
|
+
# Get base URL (required)
|
|
256
|
+
base_url = cls.get_env_var("BASE_URL")
|
|
257
|
+
if not base_url:
|
|
258
|
+
raise ConfigurationError(
|
|
259
|
+
f"{cls.PREFIX}BASE_URL environment variable is required"
|
|
260
|
+
)
|
|
261
|
+
|
|
262
|
+
# Auto-generate OAuth token URL if not provided
|
|
263
|
+
oauth_token_url = cls.get_env_var("OAUTH_TOKEN_URL")
|
|
264
|
+
if not oauth_token_url and cls.get_bool("OAUTH_ENABLED"):
|
|
265
|
+
oauth_token_url = f"{base_url}/oauth/token"
|
|
266
|
+
|
|
267
|
+
return ClientConfig(
|
|
268
|
+
# API configuration
|
|
269
|
+
base_url=base_url,
|
|
270
|
+
context_path=cls.get_env_var("CONTEXT_PATH"),
|
|
271
|
+
api_version=cls.get_env_var("API_VERSION") or "2.0.0",
|
|
272
|
+
files_base_url=cls.get_env_var("FILES_BASE_URL"),
|
|
273
|
+
files_context_path=cls.get_env_var("FILES_CONTEXT_PATH"),
|
|
274
|
+
# OAuth configuration
|
|
275
|
+
oauth_enabled=cls.get_bool("OAUTH_ENABLED"),
|
|
276
|
+
oauth_token_url=oauth_token_url,
|
|
277
|
+
client_id=cls.get_env_var("CLIENT_ID"),
|
|
278
|
+
client_secret=cls.get_env_var("CLIENT_SECRET"),
|
|
279
|
+
# scope removed
|
|
280
|
+
aud=cls.get_env_var("OAUTH_AUD"),
|
|
281
|
+
grant_type=cls.get_env_var("GRANT_TYPE") or "client_credentials",
|
|
282
|
+
# Bearer token configuration
|
|
283
|
+
bearer_token=cls.get_env_var("BEARER_TOKEN"),
|
|
284
|
+
token_refresh_threshold=cls.get_int("TOKEN_REFRESH_THRESHOLD"),
|
|
285
|
+
# HTTP configuration
|
|
286
|
+
timeout=cls.get_float("TIMEOUT"),
|
|
287
|
+
max_retries=cls.get_int("MAX_RETRIES"),
|
|
288
|
+
retry_delay=cls.get_float("RETRY_DELAY"),
|
|
289
|
+
# Connection pooling
|
|
290
|
+
pool_connections=cls.get_int("POOL_CONNECTIONS"),
|
|
291
|
+
pool_maxsize=cls.get_int("POOL_MAXSIZE"),
|
|
292
|
+
# Rate limiting
|
|
293
|
+
requests_per_minute=cls.get_int("REQUESTS_PER_MINUTE"),
|
|
294
|
+
burst_capacity=cls.get_int("BURST_CAPACITY"),
|
|
295
|
+
# Proxy configuration
|
|
296
|
+
proxy_enabled=cls.get_bool("PROXY_ENABLED"),
|
|
297
|
+
proxy_url=cls.get_env_var("PROXY_URL"),
|
|
298
|
+
proxy_username=cls.get_env_var("PROXY_USERNAME"),
|
|
299
|
+
proxy_password=cls.get_env_var("PROXY_PASSWORD"),
|
|
300
|
+
proxy_verify_ssl=cls.get_bool("PROXY_VERIFY_SSL"),
|
|
301
|
+
# Logging
|
|
302
|
+
log_level=cls.get_env_var("LOG_LEVEL") or "INFO",
|
|
303
|
+
enable_debug_logging=cls.get_bool("ENABLE_DEBUG_LOGGING"),
|
|
304
|
+
# Download configuration
|
|
305
|
+
download_dir=cls.get_env_var("DOWNLOAD_DIR") or "./downloads",
|
|
306
|
+
create_directories=cls.get_bool("CREATE_DIRECTORIES"),
|
|
307
|
+
overwrite_existing=cls.get_bool("OVERWRITE_EXISTING"),
|
|
308
|
+
# Batch Download Configuration
|
|
309
|
+
max_concurrent_downloads=cls.get_int("MAX_CONCURRENT_DOWNLOADS"),
|
|
310
|
+
batch_size=cls.get_int("BATCH_SIZE"),
|
|
311
|
+
retry_failed=cls.get_bool("RETRY_FAILED"),
|
|
312
|
+
max_retry_attempts=cls.get_int("MAX_RETRY_ATTEMPTS"),
|
|
313
|
+
create_date_folders=cls.get_bool("CREATE_DATE_FOLDERS"),
|
|
314
|
+
preserve_path_structure=cls.get_bool("PRESERVE_PATH_STRUCTURE"),
|
|
315
|
+
flatten_structure=cls.get_bool("FLATTEN_STRUCTURE"),
|
|
316
|
+
show_batch_progress=cls.get_bool("SHOW_BATCH_PROGRESS"),
|
|
317
|
+
show_individual_progress=cls.get_bool("SHOW_INDIVIDUAL_PROGRESS"),
|
|
318
|
+
continue_on_error=cls.get_bool("CONTINUE_ON_ERROR"),
|
|
319
|
+
log_errors=cls.get_bool("LOG_ERRORS"),
|
|
320
|
+
save_error_log=cls.get_bool("SAVE_ERROR_LOG"),
|
|
321
|
+
use_async_downloads=cls.get_bool("USE_ASYNC_DOWNLOADS"),
|
|
322
|
+
chunk_size=cls.get_int("CHUNK_SIZE"),
|
|
323
|
+
# Download Options
|
|
324
|
+
enable_range_requests=cls.get_bool("ENABLE_RANGE_REQUESTS"),
|
|
325
|
+
show_progress=cls.get_bool("SHOW_PROGRESS"),
|
|
326
|
+
# Workflow Configuration
|
|
327
|
+
workflow_dir=cls.get_env_var("WORKFLOW_DIR") or "workflow",
|
|
328
|
+
groups_dir=cls.get_env_var("GROUPS_DIR") or "groups",
|
|
329
|
+
availability_dir=cls.get_env_var("AVAILABILITY_DIR") or "availability",
|
|
330
|
+
default_dir=cls.get_env_var("DEFAULT_DIR") or "files",
|
|
331
|
+
# Security
|
|
332
|
+
mask_secrets=cls.get_bool("MASK_SECRETS"),
|
|
333
|
+
token_storage_enabled=cls.get_bool("TOKEN_STORAGE_ENABLED"),
|
|
334
|
+
token_storage_dir=cls.get_env_var("TOKEN_STORAGE_DIR"),
|
|
335
|
+
)
|
|
336
|
+
else:
|
|
337
|
+
# Use provided config data
|
|
338
|
+
return ClientConfig(**config_data)
|
|
339
|
+
|
|
340
|
+
@classmethod
|
|
341
|
+
def get_download_options(cls) -> Dict[str, Any]:
|
|
342
|
+
"""Get download options from environment variables."""
|
|
343
|
+
return {
|
|
344
|
+
"chunk_size": cls.get_int("CHUNK_SIZE", "8192"),
|
|
345
|
+
"max_retries": cls.get_int("MAX_RETRIES", "3"),
|
|
346
|
+
"retry_delay": cls.get_float("RETRY_DELAY", "1.0"),
|
|
347
|
+
"timeout": cls.get_float("TIMEOUT", "600.0"),
|
|
348
|
+
"enable_range_requests": cls.get_bool("ENABLE_RANGE_REQUESTS"),
|
|
349
|
+
"show_progress": cls.get_bool("SHOW_PROGRESS"),
|
|
350
|
+
"create_directories": cls.get_bool("CREATE_DIRECTORIES"),
|
|
351
|
+
"overwrite_existing": cls.get_bool("OVERWRITE_EXISTING"),
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
@classmethod
|
|
355
|
+
def get_batch_download_options(cls) -> Dict[str, Any]:
|
|
356
|
+
"""Get batch download options from environment variables."""
|
|
357
|
+
return {
|
|
358
|
+
"max_concurrent_downloads": cls.get_int("MAX_CONCURRENT_DOWNLOADS", "3"),
|
|
359
|
+
"batch_size": cls.get_int("BATCH_SIZE", "10"),
|
|
360
|
+
"retry_failed": cls.get_bool("RETRY_FAILED"),
|
|
361
|
+
"max_retry_attempts": cls.get_int("MAX_RETRY_ATTEMPTS", "2"),
|
|
362
|
+
"create_date_folders": cls.get_bool("CREATE_DATE_FOLDERS"),
|
|
363
|
+
"preserve_path_structure": cls.get_bool("PRESERVE_PATH_STRUCTURE"),
|
|
364
|
+
"flatten_structure": cls.get_bool("FLATTEN_STRUCTURE"),
|
|
365
|
+
"show_batch_progress": cls.get_bool("SHOW_BATCH_PROGRESS"),
|
|
366
|
+
"show_individual_progress": cls.get_bool("SHOW_INDIVIDUAL_PROGRESS"),
|
|
367
|
+
"continue_on_error": cls.get_bool("CONTINUE_ON_ERROR"),
|
|
368
|
+
"log_errors": cls.get_bool("LOG_ERRORS"),
|
|
369
|
+
"save_error_log": cls.get_bool("SAVE_ERROR_LOG"),
|
|
370
|
+
"use_async_downloads": cls.get_bool("USE_ASYNC_DOWNLOADS"),
|
|
371
|
+
"chunk_size": cls.get_int("CHUNK_SIZE", "8192"),
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
@classmethod
|
|
375
|
+
def get_workflow_paths(cls) -> Dict[str, Path]:
|
|
376
|
+
"""Get workflow directory paths from environment variables."""
|
|
377
|
+
base_download_dir = cls.get_path("DOWNLOAD_DIR", "./downloads")
|
|
378
|
+
|
|
379
|
+
return {
|
|
380
|
+
"base": base_download_dir,
|
|
381
|
+
"workflow": base_download_dir
|
|
382
|
+
/ (cls.get_env_var("WORKFLOW_DIR") or "workflow"),
|
|
383
|
+
"groups": base_download_dir / (cls.get_env_var("GROUPS_DIR") or "groups"),
|
|
384
|
+
"availability": base_download_dir
|
|
385
|
+
/ (cls.get_env_var("AVAILABILITY_DIR") or "availability"),
|
|
386
|
+
"default": base_download_dir / (cls.get_env_var("DEFAULT_DIR") or "files"),
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
@classmethod
|
|
390
|
+
def get_token_storage_config(cls) -> Dict[str, Any]:
|
|
391
|
+
"""Get token storage configuration from environment variables."""
|
|
392
|
+
return {
|
|
393
|
+
"enabled": cls.get_bool("TOKEN_STORAGE_ENABLED"),
|
|
394
|
+
"directory": cls.get_env_var("TOKEN_STORAGE_DIR", ".tokens"),
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
@classmethod
|
|
398
|
+
def validate_config(cls, config: ClientConfig) -> None:
|
|
399
|
+
"""Validate configuration and raise ConfigurationError if invalid."""
|
|
400
|
+
errors = []
|
|
401
|
+
|
|
402
|
+
# Check required fields
|
|
403
|
+
if not config.base_url:
|
|
404
|
+
errors.append("BASE_URL is required")
|
|
405
|
+
|
|
406
|
+
# Check OAuth configuration
|
|
407
|
+
if config.oauth_enabled:
|
|
408
|
+
if not config.client_id:
|
|
409
|
+
errors.append("CLIENT_ID is required when OAuth is enabled")
|
|
410
|
+
if not config.client_secret:
|
|
411
|
+
errors.append("CLIENT_SECRET is required when OAuth is enabled")
|
|
412
|
+
if not config.oauth_token_url:
|
|
413
|
+
errors.append("OAUTH_TOKEN_URL is required when OAuth is enabled")
|
|
414
|
+
|
|
415
|
+
# Check Bearer token configuration
|
|
416
|
+
if not config.has_oauth_credentials and not config.has_bearer_token:
|
|
417
|
+
errors.append("Either OAuth credentials or Bearer token must be configured")
|
|
418
|
+
|
|
419
|
+
# Check numeric values
|
|
420
|
+
if config.timeout <= 0:
|
|
421
|
+
errors.append("TIMEOUT must be positive")
|
|
422
|
+
if config.max_retries < 0:
|
|
423
|
+
errors.append("MAX_RETRIES must be non-negative")
|
|
424
|
+
if config.retry_delay < 0:
|
|
425
|
+
errors.append("RETRY_DELAY must be non-negative")
|
|
426
|
+
if config.pool_connections <= 0:
|
|
427
|
+
errors.append("POOL_CONNECTIONS must be positive")
|
|
428
|
+
if config.pool_maxsize <= 0:
|
|
429
|
+
errors.append("POOL_MAXSIZE must be positive")
|
|
430
|
+
if config.requests_per_minute <= 0:
|
|
431
|
+
errors.append("REQUESTS_PER_MINUTE must be positive")
|
|
432
|
+
if config.burst_capacity <= 0:
|
|
433
|
+
errors.append("BURST_CAPACITY must be positive")
|
|
434
|
+
|
|
435
|
+
if errors:
|
|
436
|
+
raise ConfigurationError(
|
|
437
|
+
f"Configuration validation failed: {'; '.join(errors)}"
|
|
438
|
+
)
|
|
439
|
+
|
|
440
|
+
@classmethod
|
|
441
|
+
def create_env_template(cls, output_path: Optional[Path] = None) -> Path:
|
|
442
|
+
"""
|
|
443
|
+
Create a .env template file with all available configuration options.
|
|
444
|
+
|
|
445
|
+
Args:
|
|
446
|
+
output_path: Optional output path for the template
|
|
447
|
+
|
|
448
|
+
Returns:
|
|
449
|
+
Path to the created template file
|
|
450
|
+
"""
|
|
451
|
+
if output_path is None:
|
|
452
|
+
output_path = Path(".env.template")
|
|
453
|
+
|
|
454
|
+
# Ensure output_path is a Path object
|
|
455
|
+
if not isinstance(output_path, Path):
|
|
456
|
+
output_path = Path(output_path)
|
|
457
|
+
|
|
458
|
+
template_content = f"""# DATAQUERY SDK Environment Configuration Template
|
|
459
|
+
# Copy this file to .env and update the values as needed
|
|
460
|
+
|
|
461
|
+
# API Configuration
|
|
462
|
+
{cls.PREFIX}BASE_URL=https://api-developer.jpmorgan.com
|
|
463
|
+
{cls.PREFIX}CONTEXT_PATH=/research/dataquery-authe/api/v2
|
|
464
|
+
# Optional separate Files API host
|
|
465
|
+
{cls.PREFIX}FILES_BASE_URL=https://api-strm-gw01.jpmchase.com
|
|
466
|
+
{cls.PREFIX}FILES_CONTEXT_PATH=/research/dataquery-authe/api/v2
|
|
467
|
+
|
|
468
|
+
# OAuth Configuration
|
|
469
|
+
{cls.PREFIX}OAUTH_ENABLED=true
|
|
470
|
+
{cls.PREFIX}OAUTH_TOKEN_URL=https://authe.jpmorgan.com/as/token.oauth2
|
|
471
|
+
{cls.PREFIX}CLIENT_ID=your_client_id_here
|
|
472
|
+
{cls.PREFIX}CLIENT_SECRET=your_client_secret_here
|
|
473
|
+
{cls.PREFIX}GRANT_TYPE=client_credentials
|
|
474
|
+
|
|
475
|
+
# Bearer Token Configuration (alternative to OAuth)
|
|
476
|
+
{cls.PREFIX}BEARER_TOKEN=your_bearer_token_here
|
|
477
|
+
{cls.PREFIX}TOKEN_REFRESH_THRESHOLD=300
|
|
478
|
+
|
|
479
|
+
# HTTP Configuration
|
|
480
|
+
{cls.PREFIX}TIMEOUT=600.0
|
|
481
|
+
{cls.PREFIX}MAX_RETRIES=3
|
|
482
|
+
{cls.PREFIX}RETRY_DELAY=1.0
|
|
483
|
+
|
|
484
|
+
# Connection Pooling
|
|
485
|
+
{cls.PREFIX}POOL_CONNECTIONS=10
|
|
486
|
+
{cls.PREFIX}POOL_MAXSIZE=20
|
|
487
|
+
|
|
488
|
+
# Rate Limiting
|
|
489
|
+
{cls.PREFIX}REQUESTS_PER_MINUTE=100
|
|
490
|
+
{cls.PREFIX}BURST_CAPACITY=20
|
|
491
|
+
|
|
492
|
+
# Proxy Configuration
|
|
493
|
+
{cls.PREFIX}PROXY_ENABLED=false
|
|
494
|
+
{cls.PREFIX}PROXY_URL=
|
|
495
|
+
{cls.PREFIX}PROXY_USERNAME=
|
|
496
|
+
{cls.PREFIX}PROXY_PASSWORD=
|
|
497
|
+
{cls.PREFIX}PROXY_VERIFY_SSL=true
|
|
498
|
+
|
|
499
|
+
# Logging
|
|
500
|
+
{cls.PREFIX}LOG_LEVEL=INFO
|
|
501
|
+
{cls.PREFIX}ENABLE_DEBUG_LOGGING=false
|
|
502
|
+
|
|
503
|
+
# Download Configuration
|
|
504
|
+
{cls.PREFIX}DOWNLOAD_DIR=./downloads
|
|
505
|
+
{cls.PREFIX}CREATE_DIRECTORIES=true
|
|
506
|
+
{cls.PREFIX}OVERWRITE_EXISTING=false
|
|
507
|
+
|
|
508
|
+
# Batch Download Configuration
|
|
509
|
+
{cls.PREFIX}MAX_CONCURRENT_DOWNLOADS=5
|
|
510
|
+
{cls.PREFIX}BATCH_SIZE=10
|
|
511
|
+
{cls.PREFIX}RETRY_FAILED=true
|
|
512
|
+
{cls.PREFIX}MAX_RETRY_ATTEMPTS=2
|
|
513
|
+
{cls.PREFIX}CREATE_DATE_FOLDERS=true
|
|
514
|
+
{cls.PREFIX}PRESERVE_PATH_STRUCTURE=true
|
|
515
|
+
{cls.PREFIX}FLATTEN_STRUCTURE=false
|
|
516
|
+
{cls.PREFIX}SHOW_BATCH_PROGRESS=true
|
|
517
|
+
{cls.PREFIX}SHOW_INDIVIDUAL_PROGRESS=true
|
|
518
|
+
{cls.PREFIX}CONTINUE_ON_ERROR=true
|
|
519
|
+
{cls.PREFIX}LOG_ERRORS=true
|
|
520
|
+
{cls.PREFIX}SAVE_ERROR_LOG=true
|
|
521
|
+
{cls.PREFIX}USE_ASYNC_DOWNLOADS=true
|
|
522
|
+
{cls.PREFIX}CHUNK_SIZE=8192
|
|
523
|
+
|
|
524
|
+
# Download Options
|
|
525
|
+
{cls.PREFIX}ENABLE_RANGE_REQUESTS=true
|
|
526
|
+
{cls.PREFIX}SHOW_PROGRESS=true
|
|
527
|
+
|
|
528
|
+
# Workflow Configuration
|
|
529
|
+
{cls.PREFIX}WORKFLOW_DIR=workflow
|
|
530
|
+
{cls.PREFIX}GROUPS_DIR=groups
|
|
531
|
+
{cls.PREFIX}AVAILABILITY_DIR=availability
|
|
532
|
+
{cls.PREFIX}DEFAULT_DIR=files
|
|
533
|
+
|
|
534
|
+
# Security
|
|
535
|
+
{cls.PREFIX}MASK_SECRETS=true
|
|
536
|
+
{cls.PREFIX}TOKEN_STORAGE_ENABLED=true
|
|
537
|
+
{cls.PREFIX}TOKEN_STORAGE_DIR=.tokens
|
|
538
|
+
"""
|
|
539
|
+
|
|
540
|
+
output_path.write_text(template_content)
|
|
541
|
+
return output_path
|
|
542
|
+
|
|
543
|
+
@classmethod
|
|
544
|
+
def get_all_env_vars(cls) -> Dict[str, str]:
|
|
545
|
+
"""Get all DATAQUERY environment variables."""
|
|
546
|
+
env_vars: Dict[str, str] = {}
|
|
547
|
+
# Return only variables that are explicitly set in the environment, not defaults
|
|
548
|
+
prefix = cls.PREFIX
|
|
549
|
+
for k, v in os.environ.items():
|
|
550
|
+
if k.startswith(prefix):
|
|
551
|
+
env_key = k[len(prefix) :]
|
|
552
|
+
env_vars[env_key] = v
|
|
553
|
+
return env_vars
|
|
554
|
+
|
|
555
|
+
@classmethod
|
|
556
|
+
def mask_secrets(cls, config_dict: Dict[str, Any]) -> Dict[str, Any]:
|
|
557
|
+
"""Mask sensitive configuration values."""
|
|
558
|
+
sensitive_keys = {
|
|
559
|
+
"client_id",
|
|
560
|
+
"client_secret",
|
|
561
|
+
"bearer_token",
|
|
562
|
+
"oauth_token_url",
|
|
563
|
+
"aud",
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
masked_config = config_dict.copy()
|
|
567
|
+
for key in sensitive_keys:
|
|
568
|
+
if key in masked_config and masked_config[key]:
|
|
569
|
+
masked_config[key] = "***"
|
|
570
|
+
|
|
571
|
+
return masked_config
|