dataquery-sdk 0.0.9__tar.gz → 0.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dataquery_sdk-0.0.9 → dataquery_sdk-0.1.1}/CHANGELOG.md +5 -1
- {dataquery_sdk-0.0.9 → dataquery_sdk-0.1.1}/PKG-INFO +2 -2
- {dataquery_sdk-0.0.9 → dataquery_sdk-0.1.1}/README.md +1 -1
- {dataquery_sdk-0.0.9 → dataquery_sdk-0.1.1}/dataquery/__init__.py +4 -3
- {dataquery_sdk-0.0.9 → dataquery_sdk-0.1.1}/dataquery/auth.py +0 -1
- {dataquery_sdk-0.0.9 → dataquery_sdk-0.1.1}/dataquery/auto_download.py +32 -28
- {dataquery_sdk-0.0.9 → dataquery_sdk-0.1.1}/dataquery/client.py +148 -122
- {dataquery_sdk-0.0.9 → dataquery_sdk-0.1.1}/dataquery/config.py +8 -8
- {dataquery_sdk-0.0.9 → dataquery_sdk-0.1.1}/dataquery/dataquery.py +260 -194
- {dataquery_sdk-0.0.9 → dataquery_sdk-0.1.1}/dataquery/exceptions.py +6 -2
- {dataquery_sdk-0.0.9 → dataquery_sdk-0.1.1}/dataquery/models.py +12 -125
- {dataquery_sdk-0.0.9 → dataquery_sdk-0.1.1}/dataquery/rate_limiter.py +2 -40
- {dataquery_sdk-0.0.9 → dataquery_sdk-0.1.1}/dataquery/utils.py +4 -5
- {dataquery_sdk-0.0.9 → dataquery_sdk-0.1.1}/dataquery_sdk.egg-info/PKG-INFO +2 -2
- {dataquery_sdk-0.0.9 → dataquery_sdk-0.1.1}/CONTRIBUTING.md +0 -0
- {dataquery_sdk-0.0.9 → dataquery_sdk-0.1.1}/LICENSE +0 -0
- {dataquery_sdk-0.0.9 → dataquery_sdk-0.1.1}/MANIFEST.in +0 -0
- {dataquery_sdk-0.0.9 → dataquery_sdk-0.1.1}/dataquery/cli.py +0 -0
- {dataquery_sdk-0.0.9 → dataquery_sdk-0.1.1}/dataquery/connection_pool.py +0 -0
- {dataquery_sdk-0.0.9 → dataquery_sdk-0.1.1}/dataquery/logging_config.py +0 -0
- {dataquery_sdk-0.0.9 → dataquery_sdk-0.1.1}/dataquery/py.typed +0 -0
- {dataquery_sdk-0.0.9 → dataquery_sdk-0.1.1}/dataquery/retry.py +0 -0
- {dataquery_sdk-0.0.9 → dataquery_sdk-0.1.1}/dataquery_sdk.egg-info/SOURCES.txt +0 -0
- {dataquery_sdk-0.0.9 → dataquery_sdk-0.1.1}/dataquery_sdk.egg-info/dependency_links.txt +0 -0
- {dataquery_sdk-0.0.9 → dataquery_sdk-0.1.1}/dataquery_sdk.egg-info/entry_points.txt +0 -0
- {dataquery_sdk-0.0.9 → dataquery_sdk-0.1.1}/dataquery_sdk.egg-info/requires.txt +0 -0
- {dataquery_sdk-0.0.9 → dataquery_sdk-0.1.1}/dataquery_sdk.egg-info/top_level.txt +0 -0
- {dataquery_sdk-0.0.9 → dataquery_sdk-0.1.1}/pyproject.toml +0 -0
- {dataquery_sdk-0.0.9 → dataquery_sdk-0.1.1}/setup.cfg +0 -0
|
@@ -14,4 +14,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
14
14
|
## [0.0.8] - 2025-10-21
|
|
15
15
|
- Made label and expression parameters optional in attribute api response
|
|
16
16
|
## [0.0.9] - 2025-12-09
|
|
17
|
-
-
|
|
17
|
+
- Performance optimizations
|
|
18
|
+
## [0.1.0] - 2025-12-10
|
|
19
|
+
- Time series data made optional
|
|
20
|
+
## [0.1.1] - 2026-02-18
|
|
21
|
+
- Non blocking IO
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dataquery-sdk
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.1.1
|
|
4
4
|
Summary: Python SDK for DATAQUERY Data API - Query, download, and check availability of economic data files
|
|
5
5
|
Author-email: DATAQUERY SDK Team <support@dataquery.com>
|
|
6
6
|
Project-URL: Homepage, https://github.com/dataquery/dataquery-sdk
|
|
@@ -279,7 +279,7 @@ DATAQUERY_CLIENT_SECRET=your_client_secret
|
|
|
279
279
|
|
|
280
280
|
# Optional - API Endpoints
|
|
281
281
|
DATAQUERY_BASE_URL=https://api-developer.jpmorgan.com
|
|
282
|
-
DATAQUERY_FILES_BASE_URL=https://api-
|
|
282
|
+
DATAQUERY_FILES_BASE_URL=https://api-dataquery.jpmchase.com
|
|
283
283
|
|
|
284
284
|
# Optional - Performance
|
|
285
285
|
DATAQUERY_MAX_RETRIES=3
|
|
@@ -210,7 +210,7 @@ DATAQUERY_CLIENT_SECRET=your_client_secret
|
|
|
210
210
|
|
|
211
211
|
# Optional - API Endpoints
|
|
212
212
|
DATAQUERY_BASE_URL=https://api-developer.jpmorgan.com
|
|
213
|
-
DATAQUERY_FILES_BASE_URL=https://api-
|
|
213
|
+
DATAQUERY_FILES_BASE_URL=https://api-dataquery.jpmchase.com
|
|
214
214
|
|
|
215
215
|
# Optional - Performance
|
|
216
216
|
DATAQUERY_MAX_RETRIES=3
|
|
@@ -14,7 +14,7 @@ Quick Start:
|
|
|
14
14
|
For more information, visit: https://github.com/dataquery/dataquery-sdk
|
|
15
15
|
"""
|
|
16
16
|
|
|
17
|
-
__version__ = "0.
|
|
17
|
+
__version__ = "0.1.1"
|
|
18
18
|
__author__ = "DATAQUERY SDK Team"
|
|
19
19
|
__email__ = "dataquery@jpmorgan.com"
|
|
20
20
|
__license__ = "MIT"
|
|
@@ -43,7 +43,7 @@ from .connection_pool import (
|
|
|
43
43
|
)
|
|
44
44
|
|
|
45
45
|
# Core imports
|
|
46
|
-
from .dataquery import DataQuery
|
|
46
|
+
from .dataquery import DataQuery
|
|
47
47
|
from .exceptions import (
|
|
48
48
|
AuthenticationError,
|
|
49
49
|
AvailabilityError,
|
|
@@ -52,6 +52,7 @@ from .exceptions import (
|
|
|
52
52
|
DateRangeError,
|
|
53
53
|
DownloadError,
|
|
54
54
|
FileNotFoundError,
|
|
55
|
+
FileNotFoundInGroupError,
|
|
55
56
|
FileTypeError,
|
|
56
57
|
GroupNotFoundError,
|
|
57
58
|
NetworkError,
|
|
@@ -124,7 +125,6 @@ from .utils import (
|
|
|
124
125
|
__all__ = [
|
|
125
126
|
# Core
|
|
126
127
|
"DataQuery",
|
|
127
|
-
"setup_logging",
|
|
128
128
|
# Models
|
|
129
129
|
"ClientConfig",
|
|
130
130
|
"Group",
|
|
@@ -153,6 +153,7 @@ __all__ = [
|
|
|
153
153
|
"AvailabilityError",
|
|
154
154
|
"GroupNotFoundError",
|
|
155
155
|
"FileNotFoundError",
|
|
156
|
+
"FileNotFoundInGroupError",
|
|
156
157
|
"DateRangeError",
|
|
157
158
|
"FileTypeError",
|
|
158
159
|
"WorkflowError",
|
|
@@ -166,7 +166,7 @@ class AutoDownloadManager:
|
|
|
166
166
|
self.stats["last_check_time"] = datetime.now()
|
|
167
167
|
|
|
168
168
|
except Exception as e:
|
|
169
|
-
self.logger.error(
|
|
169
|
+
self.logger.error("Error in monitoring loop: %s", e)
|
|
170
170
|
self.stats["errors"].append(
|
|
171
171
|
{
|
|
172
172
|
"timestamp": datetime.now().isoformat(),
|
|
@@ -180,7 +180,7 @@ class AutoDownloadManager:
|
|
|
180
180
|
try:
|
|
181
181
|
await self._safe_call_callback(self.error_callback, e)
|
|
182
182
|
except Exception as cb_error:
|
|
183
|
-
self.logger.error(
|
|
183
|
+
self.logger.error("Error in error callback: %s", cb_error)
|
|
184
184
|
|
|
185
185
|
# Wait for the specified interval or until stopped
|
|
186
186
|
try:
|
|
@@ -196,7 +196,7 @@ class AutoDownloadManager:
|
|
|
196
196
|
except asyncio.CancelledError:
|
|
197
197
|
self.logger.info("Monitoring loop cancelled")
|
|
198
198
|
except Exception as e:
|
|
199
|
-
self.logger.error(
|
|
199
|
+
self.logger.error("Fatal error in monitoring loop: %s", e)
|
|
200
200
|
finally:
|
|
201
201
|
self._running = False
|
|
202
202
|
|
|
@@ -230,28 +230,30 @@ class AutoDownloadManager:
|
|
|
230
230
|
)
|
|
231
231
|
|
|
232
232
|
# Build eligible downloads
|
|
233
|
-
eligible: List[tuple[str, str, str]] =
|
|
233
|
+
eligible: List[tuple[str, str, str]] = (
|
|
234
|
+
[]
|
|
235
|
+
) # (file_group_id, date_str, file_key)
|
|
234
236
|
for item in available_files:
|
|
235
237
|
if not self._running:
|
|
236
238
|
break
|
|
237
|
-
|
|
239
|
+
file_group_id = item.get("file-group-id")
|
|
238
240
|
date_str = item.get("file-datetime")
|
|
239
241
|
avail_flag = item.get("is-available")
|
|
240
242
|
is_available = bool(avail_flag)
|
|
241
|
-
if not
|
|
243
|
+
if not file_group_id or not date_str or not is_available:
|
|
242
244
|
continue
|
|
243
245
|
if self.file_filter and not self.file_filter(item):
|
|
244
246
|
continue
|
|
245
|
-
file_key = f"{
|
|
247
|
+
file_key = f"{file_group_id}_{date_str}"
|
|
246
248
|
if file_key in self._downloaded_files:
|
|
247
249
|
continue
|
|
248
250
|
if self._failed_files.get(file_key, 0) >= self.max_retries:
|
|
249
251
|
continue
|
|
250
|
-
if self._file_exists_locally(
|
|
252
|
+
if self._file_exists_locally(file_group_id, date_str):
|
|
251
253
|
self.stats["files_skipped"] += 1
|
|
252
254
|
self._downloaded_files.add(file_key)
|
|
253
255
|
continue
|
|
254
|
-
eligible.append((
|
|
256
|
+
eligible.append((file_group_id, date_str, file_key))
|
|
255
257
|
|
|
256
258
|
if not eligible:
|
|
257
259
|
return
|
|
@@ -272,13 +274,9 @@ class AutoDownloadManager:
|
|
|
272
274
|
await asyncio.gather(*tasks, return_exceptions=True)
|
|
273
275
|
|
|
274
276
|
except Exception as e:
|
|
275
|
-
self.logger.error(
|
|
277
|
+
self.logger.error("Error checking available files: %s", e)
|
|
276
278
|
raise
|
|
277
279
|
|
|
278
|
-
async def _process_file(self, file_info):
|
|
279
|
-
"""Deprecated path; available-files flow supersedes this."""
|
|
280
|
-
return
|
|
281
|
-
|
|
282
280
|
def _get_dates_to_check(self) -> List[str]:
|
|
283
281
|
"""Get list of dates to check for files."""
|
|
284
282
|
if self.check_current_date_only:
|
|
@@ -293,11 +291,7 @@ class AutoDownloadManager:
|
|
|
293
291
|
dates.append(date.strftime("%Y%m%d"))
|
|
294
292
|
return dates
|
|
295
293
|
|
|
296
|
-
|
|
297
|
-
"""Deprecated path; available-files flow supersedes this."""
|
|
298
|
-
return
|
|
299
|
-
|
|
300
|
-
def _file_exists_locally(self, file_id: str, date_str: str) -> bool:
|
|
294
|
+
def _file_exists_locally(self, file_group_id: str, date_str: str) -> bool:
|
|
301
295
|
"""Check if file already exists in the destination directory."""
|
|
302
296
|
# Look for files that might match this file_group_id and date
|
|
303
297
|
# This is a heuristic approach since we don't know the exact filename
|
|
@@ -306,14 +300,14 @@ class AutoDownloadManager:
|
|
|
306
300
|
if file_path.is_file():
|
|
307
301
|
filename = file_path.name
|
|
308
302
|
# Check if filename contains the file_group_id and date
|
|
309
|
-
if
|
|
303
|
+
if file_group_id in filename and date_str in filename:
|
|
310
304
|
return True
|
|
311
305
|
|
|
312
306
|
return False
|
|
313
307
|
|
|
314
|
-
async def _download_file(self,
|
|
308
|
+
async def _download_file(self, file_group_id: str, date_str: str, file_key: str):
|
|
315
309
|
"""Download a file."""
|
|
316
|
-
self.logger.info(
|
|
310
|
+
self.logger.info("Downloading '%s' for %s", file_group_id, date_str)
|
|
317
311
|
|
|
318
312
|
# Create progress callback that updates statistics
|
|
319
313
|
def progress_wrapper(progress: DownloadProgress):
|
|
@@ -331,11 +325,11 @@ class AutoDownloadManager:
|
|
|
331
325
|
else:
|
|
332
326
|
self.progress_callback(progress)
|
|
333
327
|
except Exception as e:
|
|
334
|
-
self.logger.error(
|
|
328
|
+
self.logger.error("Error in progress callback: %s", e)
|
|
335
329
|
|
|
336
330
|
try:
|
|
337
331
|
result = await self.client.download_file_async(
|
|
338
|
-
file_group_id=
|
|
332
|
+
file_group_id=file_group_id,
|
|
339
333
|
file_datetime=date_str,
|
|
340
334
|
options=self.download_options,
|
|
341
335
|
progress_callback=progress_wrapper,
|
|
@@ -353,7 +347,9 @@ class AutoDownloadManager:
|
|
|
353
347
|
succeeded = False
|
|
354
348
|
|
|
355
349
|
if succeeded:
|
|
356
|
-
self.logger.info(
|
|
350
|
+
self.logger.info(
|
|
351
|
+
"Successfully downloaded '%s' for %s", file_group_id, date_str
|
|
352
|
+
)
|
|
357
353
|
self.stats["files_downloaded"] += 1
|
|
358
354
|
self._downloaded_files.add(file_key)
|
|
359
355
|
# Accumulate total bytes if provided
|
|
@@ -376,13 +372,21 @@ class AutoDownloadManager:
|
|
|
376
372
|
else "No result"
|
|
377
373
|
)
|
|
378
374
|
self.logger.error(
|
|
379
|
-
|
|
375
|
+
"Download failed for '%s' for %s: %s",
|
|
376
|
+
file_group_id,
|
|
377
|
+
date_str,
|
|
378
|
+
error_msg,
|
|
380
379
|
)
|
|
381
380
|
self._failed_files[file_key] = self._failed_files.get(file_key, 0) + 1
|
|
382
381
|
self.stats["download_failures"] += 1
|
|
383
382
|
|
|
384
383
|
except Exception as e:
|
|
385
|
-
self.logger.error(
|
|
384
|
+
self.logger.error(
|
|
385
|
+
"Exception downloading '%s' for %s: %s",
|
|
386
|
+
file_group_id,
|
|
387
|
+
date_str,
|
|
388
|
+
e,
|
|
389
|
+
)
|
|
386
390
|
self._failed_files[file_key] = self._failed_files.get(file_key, 0) + 1
|
|
387
391
|
self.stats["download_failures"] += 1
|
|
388
392
|
raise
|
|
@@ -395,7 +399,7 @@ class AutoDownloadManager:
|
|
|
395
399
|
else:
|
|
396
400
|
callback(*args, **kwargs)
|
|
397
401
|
except Exception as e:
|
|
398
|
-
self.logger.error(
|
|
402
|
+
self.logger.error("Error in callback: %s", e)
|
|
399
403
|
|
|
400
404
|
def get_stats(self) -> Dict[str, Any]:
|
|
401
405
|
"""Get auto-download statistics."""
|
|
@@ -5,6 +5,7 @@ Main client for the DATAQUERY SDK.
|
|
|
5
5
|
import asyncio
|
|
6
6
|
import socket
|
|
7
7
|
import time
|
|
8
|
+
from collections import OrderedDict
|
|
8
9
|
from datetime import datetime
|
|
9
10
|
from pathlib import Path
|
|
10
11
|
from typing import Any, Callable, Dict, List, Optional, Tuple, Union
|
|
@@ -26,7 +27,7 @@ from .connection_pool import ConnectionPoolConfig, ConnectionPoolMonitor
|
|
|
26
27
|
from .exceptions import (
|
|
27
28
|
AuthenticationError,
|
|
28
29
|
ConfigurationError,
|
|
29
|
-
|
|
30
|
+
FileNotFoundInGroupError,
|
|
30
31
|
NetworkError,
|
|
31
32
|
NotFoundError,
|
|
32
33
|
RateLimitError,
|
|
@@ -158,8 +159,11 @@ class DataQueryClient:
|
|
|
158
159
|
self.pool_monitor = ConnectionPoolMonitor(pool_config)
|
|
159
160
|
|
|
160
161
|
# Initialize response cache for read-only operations
|
|
161
|
-
self._response_cache:
|
|
162
|
+
self._response_cache: OrderedDict[str, tuple] = (
|
|
163
|
+
OrderedDict()
|
|
164
|
+
) # key -> (data, timestamp)
|
|
162
165
|
self._cache_ttl = 300 # 5 minutes cache TTL
|
|
166
|
+
self._cache_max_size = 256 # Maximum cache entries (LRU eviction)
|
|
163
167
|
|
|
164
168
|
self.logger.info(
|
|
165
169
|
"Enhanced client components initialized",
|
|
@@ -278,6 +282,7 @@ class DataQueryClient:
|
|
|
278
282
|
if cache_key in self._response_cache:
|
|
279
283
|
data, timestamp = self._response_cache[cache_key]
|
|
280
284
|
if time.time() - timestamp < self._cache_ttl:
|
|
285
|
+
self._response_cache.move_to_end(cache_key)
|
|
281
286
|
return data
|
|
282
287
|
else:
|
|
283
288
|
# Remove expired entry
|
|
@@ -285,8 +290,12 @@ class DataQueryClient:
|
|
|
285
290
|
return None
|
|
286
291
|
|
|
287
292
|
def _set_cache(self, cache_key: str, data: Any) -> None:
|
|
288
|
-
"""Store data in cache."""
|
|
293
|
+
"""Store data in cache with LRU eviction."""
|
|
294
|
+
if cache_key in self._response_cache:
|
|
295
|
+
self._response_cache.move_to_end(cache_key)
|
|
289
296
|
self._response_cache[cache_key] = (data, time.time())
|
|
297
|
+
while len(self._response_cache) > self._cache_max_size:
|
|
298
|
+
self._response_cache.popitem(last=False)
|
|
290
299
|
|
|
291
300
|
def clear_cache(self) -> None:
|
|
292
301
|
"""Clear all cached responses."""
|
|
@@ -297,13 +306,7 @@ class DataQueryClient:
|
|
|
297
306
|
if self.session is None:
|
|
298
307
|
# Optimize timeout configuration
|
|
299
308
|
timeout = aiohttp.ClientTimeout(
|
|
300
|
-
total=self.config.timeout,
|
|
301
|
-
connect=min(
|
|
302
|
-
300.0, self.config.timeout * 0.5
|
|
303
|
-
), # 50% of total timeout for connect
|
|
304
|
-
sock_read=min(
|
|
305
|
-
300.0, self.config.timeout * 0.5
|
|
306
|
-
), # 50% for read operations
|
|
309
|
+
total=self.config.timeout, connect=300.0, sock_read=self.config.timeout
|
|
307
310
|
)
|
|
308
311
|
|
|
309
312
|
# Optimize connector configuration for better performance
|
|
@@ -315,7 +318,6 @@ class DataQueryClient:
|
|
|
315
318
|
use_dns_cache=True, # Enable DNS caching
|
|
316
319
|
ttl_dns_cache=300, # 5 minutes DNS cache
|
|
317
320
|
family=socket.AF_UNSPEC, # Allow both IPv4 and IPv6
|
|
318
|
-
ssl=False, # Let aiohttp handle SSL context
|
|
319
321
|
local_addr=None, # Let OS choose local address
|
|
320
322
|
force_close=False, # Keep connections alive
|
|
321
323
|
)
|
|
@@ -347,18 +349,6 @@ class DataQueryClient:
|
|
|
347
349
|
|
|
348
350
|
self.session = aiohttp.ClientSession(**session_kwargs) # type: ignore[arg-type]
|
|
349
351
|
|
|
350
|
-
# For compatibility tests expecting BasicAuth construction when proxy auth is configured
|
|
351
|
-
if self.config.proxy_enabled and self.config.has_proxy_credentials:
|
|
352
|
-
try:
|
|
353
|
-
from aiohttp import BasicAuth
|
|
354
|
-
|
|
355
|
-
_ = BasicAuth(
|
|
356
|
-
login=self.config.proxy_username or "",
|
|
357
|
-
password=self.config.proxy_password or "",
|
|
358
|
-
)
|
|
359
|
-
except Exception:
|
|
360
|
-
pass
|
|
361
|
-
|
|
362
352
|
self.logger.info(
|
|
363
353
|
"Client connected with optimized configuration",
|
|
364
354
|
base_url=self.config.base_url,
|
|
@@ -550,19 +540,23 @@ class DataQueryClient:
|
|
|
550
540
|
raise NetworkError("Failed to establish connection")
|
|
551
541
|
|
|
552
542
|
try:
|
|
553
|
-
|
|
554
|
-
except Exception:
|
|
555
|
-
# For proxy-auth related tests, construct BasicAuth so tests see it was used
|
|
556
|
-
if self.config.proxy_enabled and self.config.has_proxy_credentials:
|
|
557
|
-
try:
|
|
558
|
-
from aiohttp import BasicAuth
|
|
543
|
+
response = await self.session.request(method, url, **kwargs)
|
|
559
544
|
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
)
|
|
545
|
+
# Check for retryable server errors (5xx) - these should trigger retry
|
|
546
|
+
if response.status >= 500:
|
|
547
|
+
try:
|
|
548
|
+
await response.text()
|
|
564
549
|
except Exception:
|
|
565
550
|
pass
|
|
551
|
+
raise NetworkError(
|
|
552
|
+
f"Server error: {response.status}", status_code=response.status
|
|
553
|
+
)
|
|
554
|
+
|
|
555
|
+
return response
|
|
556
|
+
except NetworkError:
|
|
557
|
+
# Re-raise NetworkError to trigger retry
|
|
558
|
+
raise
|
|
559
|
+
except Exception:
|
|
566
560
|
raise
|
|
567
561
|
|
|
568
562
|
async def list_groups_async(self, limit: Optional[int] = None) -> List[Group]:
|
|
@@ -758,7 +752,7 @@ class DataQueryClient:
|
|
|
758
752
|
file_list = await self.list_files_async(group_id, file_group_id)
|
|
759
753
|
|
|
760
754
|
if not file_list.file_group_ids:
|
|
761
|
-
raise
|
|
755
|
+
raise FileNotFoundInGroupError(file_group_id, group_id)
|
|
762
756
|
|
|
763
757
|
return file_list.file_group_ids[0]
|
|
764
758
|
|
|
@@ -933,6 +927,15 @@ class DataQueryClient:
|
|
|
933
927
|
temp_destination: Optional[Path] = None
|
|
934
928
|
total_bytes: int = 0
|
|
935
929
|
|
|
930
|
+
# Check if range downloads are disabled - use single stream instead
|
|
931
|
+
if not self.config.enable_range_downloads:
|
|
932
|
+
return await self._download_file_single_stream(
|
|
933
|
+
file_group_id=file_group_id,
|
|
934
|
+
file_datetime=file_datetime,
|
|
935
|
+
options=options,
|
|
936
|
+
progress_callback=progress_callback,
|
|
937
|
+
)
|
|
938
|
+
|
|
936
939
|
try:
|
|
937
940
|
url = self._build_files_api_url("group/file/download")
|
|
938
941
|
|
|
@@ -959,18 +962,21 @@ class DataQueryClient:
|
|
|
959
962
|
progress_callback=progress_callback,
|
|
960
963
|
)
|
|
961
964
|
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
965
|
+
# If file is small (<10MB), prefer a single-stream download
|
|
966
|
+
ten_mb = 10 * 1024 * 1024
|
|
967
|
+
if total_bytes and total_bytes < ten_mb:
|
|
968
|
+
return await self._download_file_single_stream(
|
|
969
|
+
file_group_id=file_group_id,
|
|
970
|
+
file_datetime=file_datetime,
|
|
971
|
+
options=options,
|
|
972
|
+
progress_callback=progress_callback,
|
|
973
|
+
)
|
|
974
|
+
|
|
975
|
+
# Scale num_parts with file size for large files
|
|
976
|
+
if total_bytes > 500 * 1024 * 1024: # >500MB
|
|
977
|
+
num_parts = max(
|
|
978
|
+
num_parts, min(total_bytes // (100 * 1024 * 1024), 20)
|
|
979
|
+
) # 1 part per 100MB, max 20
|
|
974
980
|
|
|
975
981
|
# Determine filename and destination
|
|
976
982
|
filename = get_filename_from_response(
|
|
@@ -1011,74 +1017,118 @@ class DataQueryClient:
|
|
|
1011
1017
|
start_time=datetime.now(),
|
|
1012
1018
|
)
|
|
1013
1019
|
|
|
1020
|
+
# Scale chunk size with file size for parallel parts
|
|
1021
|
+
part_size = total_bytes // num_parts
|
|
1022
|
+
chunk_size = options.chunk_size or 1048576 # 1MB default
|
|
1023
|
+
if part_size > 100 * 1024 * 1024: # Part >100MB: use 4MB chunks
|
|
1024
|
+
chunk_size = max(chunk_size, 4 * 1024 * 1024)
|
|
1025
|
+
elif part_size > 50 * 1024 * 1024: # Part >50MB: use 2MB chunks
|
|
1026
|
+
chunk_size = max(chunk_size, 2 * 1024 * 1024)
|
|
1027
|
+
|
|
1014
1028
|
# Progress callback optimization: track last callback state
|
|
1015
1029
|
last_callback_bytes = 0
|
|
1016
1030
|
last_callback_time = time.time()
|
|
1017
1031
|
callback_threshold_bytes = 1024 * 1024 # 1MB
|
|
1018
1032
|
callback_threshold_time = 0.5 # 0.5 seconds
|
|
1019
1033
|
|
|
1034
|
+
# Per-part retry configuration
|
|
1035
|
+
max_part_retries = options.max_retries
|
|
1036
|
+
part_retry_delay = options.retry_delay
|
|
1037
|
+
|
|
1038
|
+
# For large file parts, disable the total timeout (which caps the entire
|
|
1039
|
+
# transfer) but keep sock_read so stalled connections are detected.
|
|
1040
|
+
range_timeout = aiohttp.ClientTimeout(
|
|
1041
|
+
total=None,
|
|
1042
|
+
sock_read=self.config.timeout,
|
|
1043
|
+
)
|
|
1044
|
+
|
|
1020
1045
|
# Get the current event loop
|
|
1021
1046
|
loop = asyncio.get_running_loop()
|
|
1022
1047
|
bytes_lock = asyncio.Lock()
|
|
1023
1048
|
|
|
1024
1049
|
async def download_range(start_byte: int, end_byte: int):
|
|
1025
1050
|
nonlocal bytes_downloaded, last_callback_bytes, last_callback_time
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
with open(temp_destination, "r+b") as part_fh:
|
|
1031
|
-
# each part request
|
|
1032
|
-
async with await self._enter_request_cm(
|
|
1033
|
-
"GET", url, params=params, headers=headers
|
|
1034
|
-
) as resp:
|
|
1035
|
-
await self._handle_response(resp)
|
|
1036
|
-
# Stream and write to correct offset
|
|
1037
|
-
current_pos = start_byte
|
|
1038
|
-
chunk_size = (
|
|
1039
|
-
options.chunk_size or 1048576
|
|
1040
|
-
) # 1MB default for better performance
|
|
1041
|
-
|
|
1042
|
-
async for chunk in resp.content.iter_chunked(chunk_size):
|
|
1043
|
-
# Write directly to file at correct position
|
|
1044
|
-
# Since we have a private handle, we don't need a file lock
|
|
1045
|
-
# But we should still run IO in executor to avoid blocking the loop
|
|
1046
|
-
await loop.run_in_executor(None, part_fh.seek, current_pos)
|
|
1047
|
-
await loop.run_in_executor(None, part_fh.write, chunk)
|
|
1048
|
-
|
|
1049
|
-
chunk_len = len(chunk)
|
|
1050
|
-
current_pos += chunk_len
|
|
1051
|
+
range_headers = {"Range": f"bytes={start_byte}-{end_byte}"}
|
|
1052
|
+
part_bytes_written = (
|
|
1053
|
+
0 # Track bytes written by this part for retry rollback
|
|
1054
|
+
)
|
|
1051
1055
|
|
|
1056
|
+
for attempt in range(max_part_retries + 1):
|
|
1057
|
+
try:
|
|
1058
|
+
# On retry, subtract previously counted bytes for this part
|
|
1059
|
+
if attempt > 0 and part_bytes_written > 0:
|
|
1052
1060
|
async with bytes_lock:
|
|
1053
|
-
bytes_downloaded
|
|
1061
|
+
bytes_downloaded -= part_bytes_written
|
|
1054
1062
|
progress.update_progress(bytes_downloaded)
|
|
1063
|
+
part_bytes_written = 0
|
|
1064
|
+
await asyncio.sleep(part_retry_delay * (2 ** (attempt - 1)))
|
|
1065
|
+
|
|
1066
|
+
with open(temp_destination, "r+b") as part_fh:
|
|
1067
|
+
async with await self._enter_request_cm(
|
|
1068
|
+
"GET",
|
|
1069
|
+
url,
|
|
1070
|
+
params=params,
|
|
1071
|
+
headers=range_headers,
|
|
1072
|
+
timeout=range_timeout,
|
|
1073
|
+
) as resp:
|
|
1074
|
+
await self._handle_response(resp)
|
|
1075
|
+
current_pos = start_byte
|
|
1076
|
+
|
|
1077
|
+
def _seek_write(fh, pos, data):
|
|
1078
|
+
fh.seek(pos)
|
|
1079
|
+
fh.write(data)
|
|
1080
|
+
|
|
1081
|
+
async for chunk in resp.content.iter_chunked(
|
|
1082
|
+
chunk_size
|
|
1083
|
+
):
|
|
1084
|
+
await loop.run_in_executor(
|
|
1085
|
+
None, _seek_write, part_fh, current_pos, chunk
|
|
1086
|
+
)
|
|
1087
|
+
|
|
1088
|
+
chunk_len = len(chunk)
|
|
1089
|
+
current_pos += chunk_len
|
|
1090
|
+
part_bytes_written += chunk_len
|
|
1091
|
+
|
|
1092
|
+
# Update counters under lock, invoke callback outside
|
|
1093
|
+
should_callback = False
|
|
1094
|
+
async with bytes_lock:
|
|
1095
|
+
bytes_downloaded += chunk_len
|
|
1096
|
+
progress.update_progress(bytes_downloaded)
|
|
1097
|
+
|
|
1098
|
+
current_time = time.time()
|
|
1099
|
+
bytes_diff = (
|
|
1100
|
+
bytes_downloaded - last_callback_bytes
|
|
1101
|
+
)
|
|
1102
|
+
time_diff = current_time - last_callback_time
|
|
1055
1103
|
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
should_callback = (
|
|
1062
|
-
bytes_diff >= callback_threshold_bytes
|
|
1063
|
-
or time_diff >= callback_threshold_time
|
|
1064
|
-
or bytes_downloaded
|
|
1065
|
-
== total_bytes # Always callback on completion
|
|
1066
|
-
)
|
|
1067
|
-
|
|
1068
|
-
if should_callback:
|
|
1069
|
-
if progress_callback:
|
|
1070
|
-
progress_callback(progress)
|
|
1071
|
-
elif options.show_progress:
|
|
1072
|
-
self.logger.info(
|
|
1073
|
-
"Download progress",
|
|
1074
|
-
file=file_group_id,
|
|
1075
|
-
percentage=f"{progress.percentage:.1f}%",
|
|
1076
|
-
downloaded=format_file_size(
|
|
1077
|
-
bytes_downloaded
|
|
1078
|
-
),
|
|
1104
|
+
should_callback = (
|
|
1105
|
+
bytes_diff >= callback_threshold_bytes
|
|
1106
|
+
or time_diff >= callback_threshold_time
|
|
1107
|
+
or bytes_downloaded
|
|
1108
|
+
== total_bytes # Always callback on completion
|
|
1079
1109
|
)
|
|
1080
|
-
|
|
1081
|
-
|
|
1110
|
+
|
|
1111
|
+
if should_callback:
|
|
1112
|
+
last_callback_bytes = bytes_downloaded
|
|
1113
|
+
last_callback_time = current_time
|
|
1114
|
+
|
|
1115
|
+
if should_callback:
|
|
1116
|
+
if progress_callback:
|
|
1117
|
+
progress_callback(progress)
|
|
1118
|
+
elif options.show_progress:
|
|
1119
|
+
self.logger.info(
|
|
1120
|
+
"Download progress",
|
|
1121
|
+
file=file_group_id,
|
|
1122
|
+
percentage=f"{progress.percentage:.1f}%",
|
|
1123
|
+
downloaded=format_file_size(
|
|
1124
|
+
bytes_downloaded
|
|
1125
|
+
),
|
|
1126
|
+
)
|
|
1127
|
+
# Part completed successfully
|
|
1128
|
+
return
|
|
1129
|
+
except Exception:
|
|
1130
|
+
if attempt == max_part_retries:
|
|
1131
|
+
raise # Exhausted retries, propagate to gather
|
|
1082
1132
|
|
|
1083
1133
|
# Run all parts concurrently
|
|
1084
1134
|
await asyncio.gather(*(download_range(s, e) for s, e in ranges))
|
|
@@ -1099,10 +1149,7 @@ class DataQueryClient:
|
|
|
1099
1149
|
if temp_destination and temp_destination.exists():
|
|
1100
1150
|
try:
|
|
1101
1151
|
# Ensure any open handle is closed
|
|
1102
|
-
if
|
|
1103
|
-
total_bytes
|
|
1104
|
-
and temp_destination.stat().st_size >= total_bytes
|
|
1105
|
-
):
|
|
1152
|
+
if total_bytes and bytes_downloaded >= total_bytes:
|
|
1106
1153
|
if destination is None:
|
|
1107
1154
|
destination = temp_destination.with_suffix("")
|
|
1108
1155
|
temp_destination.replace(destination)
|
|
@@ -1453,25 +1500,7 @@ class DataQueryClient:
|
|
|
1453
1500
|
validate_date_format(start_date, "start-date")
|
|
1454
1501
|
if end_date is not None:
|
|
1455
1502
|
validate_date_format(end_date, "end-date")
|
|
1456
|
-
"""
|
|
1457
|
-
Retrieve time-series data for explicit list of instruments and attributes using identifiers.
|
|
1458
1503
|
|
|
1459
|
-
Args:
|
|
1460
|
-
instruments: List of instrument identifiers
|
|
1461
|
-
attributes: List of attribute identifiers
|
|
1462
|
-
data: Data type (REFERENCE_DATA, NO_REFERENCE_DATA, ALL)
|
|
1463
|
-
format: Response format (JSON)
|
|
1464
|
-
start_date: Start date in YYYYMMDD or TODAY-Nx format
|
|
1465
|
-
end_date: End date in YYYYMMDD or TODAY-Nx format
|
|
1466
|
-
calendar: Calendar convention
|
|
1467
|
-
frequency: Frequency convention
|
|
1468
|
-
conversion: Conversion convention
|
|
1469
|
-
nan_treatment: Missing data treatment
|
|
1470
|
-
page: Optional page token for pagination
|
|
1471
|
-
|
|
1472
|
-
Returns:
|
|
1473
|
-
TimeSeriesResponse containing the time series data
|
|
1474
|
-
"""
|
|
1475
1504
|
params = {
|
|
1476
1505
|
"instruments": instruments,
|
|
1477
1506
|
"attributes": attributes,
|
|
@@ -2461,9 +2490,6 @@ class DataQueryClient:
|
|
|
2461
2490
|
)
|
|
2462
2491
|
)
|
|
2463
2492
|
|
|
2464
|
-
# Note: download_multiple_files method removed as it's not part of the new API spec
|
|
2465
|
-
# Use individual download_file calls for batch operations
|
|
2466
|
-
|
|
2467
2493
|
def list_available_files(
|
|
2468
2494
|
self,
|
|
2469
2495
|
group_id: str,
|