dataquery-sdk 0.0.9__tar.gz → 0.1.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (29) hide show
  1. {dataquery_sdk-0.0.9 → dataquery_sdk-0.1.1}/CHANGELOG.md +5 -1
  2. {dataquery_sdk-0.0.9 → dataquery_sdk-0.1.1}/PKG-INFO +2 -2
  3. {dataquery_sdk-0.0.9 → dataquery_sdk-0.1.1}/README.md +1 -1
  4. {dataquery_sdk-0.0.9 → dataquery_sdk-0.1.1}/dataquery/__init__.py +4 -3
  5. {dataquery_sdk-0.0.9 → dataquery_sdk-0.1.1}/dataquery/auth.py +0 -1
  6. {dataquery_sdk-0.0.9 → dataquery_sdk-0.1.1}/dataquery/auto_download.py +32 -28
  7. {dataquery_sdk-0.0.9 → dataquery_sdk-0.1.1}/dataquery/client.py +148 -122
  8. {dataquery_sdk-0.0.9 → dataquery_sdk-0.1.1}/dataquery/config.py +8 -8
  9. {dataquery_sdk-0.0.9 → dataquery_sdk-0.1.1}/dataquery/dataquery.py +260 -194
  10. {dataquery_sdk-0.0.9 → dataquery_sdk-0.1.1}/dataquery/exceptions.py +6 -2
  11. {dataquery_sdk-0.0.9 → dataquery_sdk-0.1.1}/dataquery/models.py +12 -125
  12. {dataquery_sdk-0.0.9 → dataquery_sdk-0.1.1}/dataquery/rate_limiter.py +2 -40
  13. {dataquery_sdk-0.0.9 → dataquery_sdk-0.1.1}/dataquery/utils.py +4 -5
  14. {dataquery_sdk-0.0.9 → dataquery_sdk-0.1.1}/dataquery_sdk.egg-info/PKG-INFO +2 -2
  15. {dataquery_sdk-0.0.9 → dataquery_sdk-0.1.1}/CONTRIBUTING.md +0 -0
  16. {dataquery_sdk-0.0.9 → dataquery_sdk-0.1.1}/LICENSE +0 -0
  17. {dataquery_sdk-0.0.9 → dataquery_sdk-0.1.1}/MANIFEST.in +0 -0
  18. {dataquery_sdk-0.0.9 → dataquery_sdk-0.1.1}/dataquery/cli.py +0 -0
  19. {dataquery_sdk-0.0.9 → dataquery_sdk-0.1.1}/dataquery/connection_pool.py +0 -0
  20. {dataquery_sdk-0.0.9 → dataquery_sdk-0.1.1}/dataquery/logging_config.py +0 -0
  21. {dataquery_sdk-0.0.9 → dataquery_sdk-0.1.1}/dataquery/py.typed +0 -0
  22. {dataquery_sdk-0.0.9 → dataquery_sdk-0.1.1}/dataquery/retry.py +0 -0
  23. {dataquery_sdk-0.0.9 → dataquery_sdk-0.1.1}/dataquery_sdk.egg-info/SOURCES.txt +0 -0
  24. {dataquery_sdk-0.0.9 → dataquery_sdk-0.1.1}/dataquery_sdk.egg-info/dependency_links.txt +0 -0
  25. {dataquery_sdk-0.0.9 → dataquery_sdk-0.1.1}/dataquery_sdk.egg-info/entry_points.txt +0 -0
  26. {dataquery_sdk-0.0.9 → dataquery_sdk-0.1.1}/dataquery_sdk.egg-info/requires.txt +0 -0
  27. {dataquery_sdk-0.0.9 → dataquery_sdk-0.1.1}/dataquery_sdk.egg-info/top_level.txt +0 -0
  28. {dataquery_sdk-0.0.9 → dataquery_sdk-0.1.1}/pyproject.toml +0 -0
  29. {dataquery_sdk-0.0.9 → dataquery_sdk-0.1.1}/setup.cfg +0 -0
@@ -14,4 +14,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
14
14
  ## [0.0.8] - 2025-10-21
15
15
  - Made label and expression parameters optional in attribute api response
16
16
  ## [0.0.9] - 2025-12-09
17
- - Perfomance optimizations
17
+ - Performance optimizations
18
+ ## [0.1.0] - 2025-12-10
19
+ - Time series data made optional
20
+ ## [0.1.1] - 2026-02-18
21
+ - Non blocking IO
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dataquery-sdk
3
- Version: 0.0.9
3
+ Version: 0.1.1
4
4
  Summary: Python SDK for DATAQUERY Data API - Query, download, and check availability of economic data files
5
5
  Author-email: DATAQUERY SDK Team <support@dataquery.com>
6
6
  Project-URL: Homepage, https://github.com/dataquery/dataquery-sdk
@@ -279,7 +279,7 @@ DATAQUERY_CLIENT_SECRET=your_client_secret
279
279
 
280
280
  # Optional - API Endpoints
281
281
  DATAQUERY_BASE_URL=https://api-developer.jpmorgan.com
282
- DATAQUERY_FILES_BASE_URL=https://api-strm-gw01.jpmchase.com
282
+ DATAQUERY_FILES_BASE_URL=https://api-dataquery.jpmchase.com
283
283
 
284
284
  # Optional - Performance
285
285
  DATAQUERY_MAX_RETRIES=3
@@ -210,7 +210,7 @@ DATAQUERY_CLIENT_SECRET=your_client_secret
210
210
 
211
211
  # Optional - API Endpoints
212
212
  DATAQUERY_BASE_URL=https://api-developer.jpmorgan.com
213
- DATAQUERY_FILES_BASE_URL=https://api-strm-gw01.jpmchase.com
213
+ DATAQUERY_FILES_BASE_URL=https://api-dataquery.jpmchase.com
214
214
 
215
215
  # Optional - Performance
216
216
  DATAQUERY_MAX_RETRIES=3
@@ -14,7 +14,7 @@ Quick Start:
14
14
  For more information, visit: https://github.com/dataquery/dataquery-sdk
15
15
  """
16
16
 
17
- __version__ = "0.0.9"
17
+ __version__ = "0.1.1"
18
18
  __author__ = "DATAQUERY SDK Team"
19
19
  __email__ = "dataquery@jpmorgan.com"
20
20
  __license__ = "MIT"
@@ -43,7 +43,7 @@ from .connection_pool import (
43
43
  )
44
44
 
45
45
  # Core imports
46
- from .dataquery import DataQuery, setup_logging
46
+ from .dataquery import DataQuery
47
47
  from .exceptions import (
48
48
  AuthenticationError,
49
49
  AvailabilityError,
@@ -52,6 +52,7 @@ from .exceptions import (
52
52
  DateRangeError,
53
53
  DownloadError,
54
54
  FileNotFoundError,
55
+ FileNotFoundInGroupError,
55
56
  FileTypeError,
56
57
  GroupNotFoundError,
57
58
  NetworkError,
@@ -124,7 +125,6 @@ from .utils import (
124
125
  __all__ = [
125
126
  # Core
126
127
  "DataQuery",
127
- "setup_logging",
128
128
  # Models
129
129
  "ClientConfig",
130
130
  "Group",
@@ -153,6 +153,7 @@ __all__ = [
153
153
  "AvailabilityError",
154
154
  "GroupNotFoundError",
155
155
  "FileNotFoundError",
156
+ "FileNotFoundInGroupError",
156
157
  "DateRangeError",
157
158
  "FileTypeError",
158
159
  "WorkflowError",
@@ -106,7 +106,6 @@ class TokenManager:
106
106
  grant_type=self.config.grant_type,
107
107
  client_id=self.config.client_id,
108
108
  client_secret=self.config.client_secret,
109
- # scope removed
110
109
  aud=getattr(self.config, "aud", None),
111
110
  )
112
111
 
@@ -166,7 +166,7 @@ class AutoDownloadManager:
166
166
  self.stats["last_check_time"] = datetime.now()
167
167
 
168
168
  except Exception as e:
169
- self.logger.error(f"Error in monitoring loop: {e}")
169
+ self.logger.error("Error in monitoring loop: %s", e)
170
170
  self.stats["errors"].append(
171
171
  {
172
172
  "timestamp": datetime.now().isoformat(),
@@ -180,7 +180,7 @@ class AutoDownloadManager:
180
180
  try:
181
181
  await self._safe_call_callback(self.error_callback, e)
182
182
  except Exception as cb_error:
183
- self.logger.error(f"Error in error callback: {cb_error}")
183
+ self.logger.error("Error in error callback: %s", cb_error)
184
184
 
185
185
  # Wait for the specified interval or until stopped
186
186
  try:
@@ -196,7 +196,7 @@ class AutoDownloadManager:
196
196
  except asyncio.CancelledError:
197
197
  self.logger.info("Monitoring loop cancelled")
198
198
  except Exception as e:
199
- self.logger.error(f"Fatal error in monitoring loop: {e}")
199
+ self.logger.error("Fatal error in monitoring loop: %s", e)
200
200
  finally:
201
201
  self._running = False
202
202
 
@@ -230,28 +230,30 @@ class AutoDownloadManager:
230
230
  )
231
231
 
232
232
  # Build eligible downloads
233
- eligible: List[tuple[str, str, str]] = [] # (file_id, date_str, file_key)
233
+ eligible: List[tuple[str, str, str]] = (
234
+ []
235
+ ) # (file_group_id, date_str, file_key)
234
236
  for item in available_files:
235
237
  if not self._running:
236
238
  break
237
- file_id = item.get("file-group-id")
239
+ file_group_id = item.get("file-group-id")
238
240
  date_str = item.get("file-datetime")
239
241
  avail_flag = item.get("is-available")
240
242
  is_available = bool(avail_flag)
241
- if not file_id or not date_str or not is_available:
243
+ if not file_group_id or not date_str or not is_available:
242
244
  continue
243
245
  if self.file_filter and not self.file_filter(item):
244
246
  continue
245
- file_key = f"{file_id}_{date_str}"
247
+ file_key = f"{file_group_id}_{date_str}"
246
248
  if file_key in self._downloaded_files:
247
249
  continue
248
250
  if self._failed_files.get(file_key, 0) >= self.max_retries:
249
251
  continue
250
- if self._file_exists_locally(file_id, date_str):
252
+ if self._file_exists_locally(file_group_id, date_str):
251
253
  self.stats["files_skipped"] += 1
252
254
  self._downloaded_files.add(file_key)
253
255
  continue
254
- eligible.append((file_id, date_str, file_key))
256
+ eligible.append((file_group_id, date_str, file_key))
255
257
 
256
258
  if not eligible:
257
259
  return
@@ -272,13 +274,9 @@ class AutoDownloadManager:
272
274
  await asyncio.gather(*tasks, return_exceptions=True)
273
275
 
274
276
  except Exception as e:
275
- self.logger.error(f"Error checking available files: {e}")
277
+ self.logger.error("Error checking available files: %s", e)
276
278
  raise
277
279
 
278
- async def _process_file(self, file_info):
279
- """Deprecated path; available-files flow supersedes this."""
280
- return
281
-
282
280
  def _get_dates_to_check(self) -> List[str]:
283
281
  """Get list of dates to check for files."""
284
282
  if self.check_current_date_only:
@@ -293,11 +291,7 @@ class AutoDownloadManager:
293
291
  dates.append(date.strftime("%Y%m%d"))
294
292
  return dates
295
293
 
296
- async def _check_and_download_file_for_date(self, file_id: str, date_str: str):
297
- """Deprecated path; available-files flow supersedes this."""
298
- return
299
-
300
- def _file_exists_locally(self, file_id: str, date_str: str) -> bool:
294
+ def _file_exists_locally(self, file_group_id: str, date_str: str) -> bool:
301
295
  """Check if file already exists in the destination directory."""
302
296
  # Look for files that might match this file_group_id and date
303
297
  # This is a heuristic approach since we don't know the exact filename
@@ -306,14 +300,14 @@ class AutoDownloadManager:
306
300
  if file_path.is_file():
307
301
  filename = file_path.name
308
302
  # Check if filename contains the file_group_id and date
309
- if file_id in filename and date_str in filename:
303
+ if file_group_id in filename and date_str in filename:
310
304
  return True
311
305
 
312
306
  return False
313
307
 
314
- async def _download_file(self, file_id: str, date_str: str, file_key: str):
308
+ async def _download_file(self, file_group_id: str, date_str: str, file_key: str):
315
309
  """Download a file."""
316
- self.logger.info(f"Downloading '{file_id}' for {date_str}...")
310
+ self.logger.info("Downloading '%s' for %s", file_group_id, date_str)
317
311
 
318
312
  # Create progress callback that updates statistics
319
313
  def progress_wrapper(progress: DownloadProgress):
@@ -331,11 +325,11 @@ class AutoDownloadManager:
331
325
  else:
332
326
  self.progress_callback(progress)
333
327
  except Exception as e:
334
- self.logger.error(f"Error in progress callback: {e}")
328
+ self.logger.error("Error in progress callback: %s", e)
335
329
 
336
330
  try:
337
331
  result = await self.client.download_file_async(
338
- file_group_id=file_id,
332
+ file_group_id=file_group_id,
339
333
  file_datetime=date_str,
340
334
  options=self.download_options,
341
335
  progress_callback=progress_wrapper,
@@ -353,7 +347,9 @@ class AutoDownloadManager:
353
347
  succeeded = False
354
348
 
355
349
  if succeeded:
356
- self.logger.info(f"Successfully downloaded '{file_id}' for {date_str}")
350
+ self.logger.info(
351
+ "Successfully downloaded '%s' for %s", file_group_id, date_str
352
+ )
357
353
  self.stats["files_downloaded"] += 1
358
354
  self._downloaded_files.add(file_key)
359
355
  # Accumulate total bytes if provided
@@ -376,13 +372,21 @@ class AutoDownloadManager:
376
372
  else "No result"
377
373
  )
378
374
  self.logger.error(
379
- f"Download failed for '{file_id}' for {date_str}: {error_msg}"
375
+ "Download failed for '%s' for %s: %s",
376
+ file_group_id,
377
+ date_str,
378
+ error_msg,
380
379
  )
381
380
  self._failed_files[file_key] = self._failed_files.get(file_key, 0) + 1
382
381
  self.stats["download_failures"] += 1
383
382
 
384
383
  except Exception as e:
385
- self.logger.error(f"Exception downloading '{file_id}' for {date_str}: {e}")
384
+ self.logger.error(
385
+ "Exception downloading '%s' for %s: %s",
386
+ file_group_id,
387
+ date_str,
388
+ e,
389
+ )
386
390
  self._failed_files[file_key] = self._failed_files.get(file_key, 0) + 1
387
391
  self.stats["download_failures"] += 1
388
392
  raise
@@ -395,7 +399,7 @@ class AutoDownloadManager:
395
399
  else:
396
400
  callback(*args, **kwargs)
397
401
  except Exception as e:
398
- self.logger.error(f"Error in callback: {e}")
402
+ self.logger.error("Error in callback: %s", e)
399
403
 
400
404
  def get_stats(self) -> Dict[str, Any]:
401
405
  """Get auto-download statistics."""
@@ -5,6 +5,7 @@ Main client for the DATAQUERY SDK.
5
5
  import asyncio
6
6
  import socket
7
7
  import time
8
+ from collections import OrderedDict
8
9
  from datetime import datetime
9
10
  from pathlib import Path
10
11
  from typing import Any, Callable, Dict, List, Optional, Tuple, Union
@@ -26,7 +27,7 @@ from .connection_pool import ConnectionPoolConfig, ConnectionPoolMonitor
26
27
  from .exceptions import (
27
28
  AuthenticationError,
28
29
  ConfigurationError,
29
- FileNotFoundError,
30
+ FileNotFoundInGroupError,
30
31
  NetworkError,
31
32
  NotFoundError,
32
33
  RateLimitError,
@@ -158,8 +159,11 @@ class DataQueryClient:
158
159
  self.pool_monitor = ConnectionPoolMonitor(pool_config)
159
160
 
160
161
  # Initialize response cache for read-only operations
161
- self._response_cache: Dict[str, tuple] = {} # key -> (data, timestamp)
162
+ self._response_cache: OrderedDict[str, tuple] = (
163
+ OrderedDict()
164
+ ) # key -> (data, timestamp)
162
165
  self._cache_ttl = 300 # 5 minutes cache TTL
166
+ self._cache_max_size = 256 # Maximum cache entries (LRU eviction)
163
167
 
164
168
  self.logger.info(
165
169
  "Enhanced client components initialized",
@@ -278,6 +282,7 @@ class DataQueryClient:
278
282
  if cache_key in self._response_cache:
279
283
  data, timestamp = self._response_cache[cache_key]
280
284
  if time.time() - timestamp < self._cache_ttl:
285
+ self._response_cache.move_to_end(cache_key)
281
286
  return data
282
287
  else:
283
288
  # Remove expired entry
@@ -285,8 +290,12 @@ class DataQueryClient:
285
290
  return None
286
291
 
287
292
  def _set_cache(self, cache_key: str, data: Any) -> None:
288
- """Store data in cache."""
293
+ """Store data in cache with LRU eviction."""
294
+ if cache_key in self._response_cache:
295
+ self._response_cache.move_to_end(cache_key)
289
296
  self._response_cache[cache_key] = (data, time.time())
297
+ while len(self._response_cache) > self._cache_max_size:
298
+ self._response_cache.popitem(last=False)
290
299
 
291
300
  def clear_cache(self) -> None:
292
301
  """Clear all cached responses."""
@@ -297,13 +306,7 @@ class DataQueryClient:
297
306
  if self.session is None:
298
307
  # Optimize timeout configuration
299
308
  timeout = aiohttp.ClientTimeout(
300
- total=self.config.timeout,
301
- connect=min(
302
- 300.0, self.config.timeout * 0.5
303
- ), # 50% of total timeout for connect
304
- sock_read=min(
305
- 300.0, self.config.timeout * 0.5
306
- ), # 50% for read operations
309
+ total=self.config.timeout, connect=300.0, sock_read=self.config.timeout
307
310
  )
308
311
 
309
312
  # Optimize connector configuration for better performance
@@ -315,7 +318,6 @@ class DataQueryClient:
315
318
  use_dns_cache=True, # Enable DNS caching
316
319
  ttl_dns_cache=300, # 5 minutes DNS cache
317
320
  family=socket.AF_UNSPEC, # Allow both IPv4 and IPv6
318
- ssl=False, # Let aiohttp handle SSL context
319
321
  local_addr=None, # Let OS choose local address
320
322
  force_close=False, # Keep connections alive
321
323
  )
@@ -347,18 +349,6 @@ class DataQueryClient:
347
349
 
348
350
  self.session = aiohttp.ClientSession(**session_kwargs) # type: ignore[arg-type]
349
351
 
350
- # For compatibility tests expecting BasicAuth construction when proxy auth is configured
351
- if self.config.proxy_enabled and self.config.has_proxy_credentials:
352
- try:
353
- from aiohttp import BasicAuth
354
-
355
- _ = BasicAuth(
356
- login=self.config.proxy_username or "",
357
- password=self.config.proxy_password or "",
358
- )
359
- except Exception:
360
- pass
361
-
362
352
  self.logger.info(
363
353
  "Client connected with optimized configuration",
364
354
  base_url=self.config.base_url,
@@ -550,19 +540,23 @@ class DataQueryClient:
550
540
  raise NetworkError("Failed to establish connection")
551
541
 
552
542
  try:
553
- return await self.session.request(method, url, **kwargs)
554
- except Exception:
555
- # For proxy-auth related tests, construct BasicAuth so tests see it was used
556
- if self.config.proxy_enabled and self.config.has_proxy_credentials:
557
- try:
558
- from aiohttp import BasicAuth
543
+ response = await self.session.request(method, url, **kwargs)
559
544
 
560
- _ = BasicAuth(
561
- login=self.config.proxy_username or "",
562
- password=self.config.proxy_password or "",
563
- )
545
+ # Check for retryable server errors (5xx) - these should trigger retry
546
+ if response.status >= 500:
547
+ try:
548
+ await response.text()
564
549
  except Exception:
565
550
  pass
551
+ raise NetworkError(
552
+ f"Server error: {response.status}", status_code=response.status
553
+ )
554
+
555
+ return response
556
+ except NetworkError:
557
+ # Re-raise NetworkError to trigger retry
558
+ raise
559
+ except Exception:
566
560
  raise
567
561
 
568
562
  async def list_groups_async(self, limit: Optional[int] = None) -> List[Group]:
@@ -758,7 +752,7 @@ class DataQueryClient:
758
752
  file_list = await self.list_files_async(group_id, file_group_id)
759
753
 
760
754
  if not file_list.file_group_ids:
761
- raise FileNotFoundError(file_group_id, group_id)
755
+ raise FileNotFoundInGroupError(file_group_id, group_id)
762
756
 
763
757
  return file_list.file_group_ids[0]
764
758
 
@@ -933,6 +927,15 @@ class DataQueryClient:
933
927
  temp_destination: Optional[Path] = None
934
928
  total_bytes: int = 0
935
929
 
930
+ # Check if range downloads are disabled - use single stream instead
931
+ if not self.config.enable_range_downloads:
932
+ return await self._download_file_single_stream(
933
+ file_group_id=file_group_id,
934
+ file_datetime=file_datetime,
935
+ options=options,
936
+ progress_callback=progress_callback,
937
+ )
938
+
936
939
  try:
937
940
  url = self._build_files_api_url("group/file/download")
938
941
 
@@ -959,18 +962,21 @@ class DataQueryClient:
959
962
  progress_callback=progress_callback,
960
963
  )
961
964
 
962
- # If file is small (<10MB), prefer a single-stream download
963
- try:
964
- ten_mb = 10 * 1024 * 1024
965
- if total_bytes and total_bytes < ten_mb:
966
- return await self._download_file_single_stream(
967
- file_group_id=file_group_id,
968
- file_datetime=file_datetime,
969
- options=options,
970
- progress_callback=progress_callback,
971
- )
972
- except Exception:
973
- pass
965
+ # If file is small (<10MB), prefer a single-stream download
966
+ ten_mb = 10 * 1024 * 1024
967
+ if total_bytes and total_bytes < ten_mb:
968
+ return await self._download_file_single_stream(
969
+ file_group_id=file_group_id,
970
+ file_datetime=file_datetime,
971
+ options=options,
972
+ progress_callback=progress_callback,
973
+ )
974
+
975
+ # Scale num_parts with file size for large files
976
+ if total_bytes > 500 * 1024 * 1024: # >500MB
977
+ num_parts = max(
978
+ num_parts, min(total_bytes // (100 * 1024 * 1024), 20)
979
+ ) # 1 part per 100MB, max 20
974
980
 
975
981
  # Determine filename and destination
976
982
  filename = get_filename_from_response(
@@ -1011,74 +1017,118 @@ class DataQueryClient:
1011
1017
  start_time=datetime.now(),
1012
1018
  )
1013
1019
 
1020
+ # Scale chunk size with file size for parallel parts
1021
+ part_size = total_bytes // num_parts
1022
+ chunk_size = options.chunk_size or 1048576 # 1MB default
1023
+ if part_size > 100 * 1024 * 1024: # Part >100MB: use 4MB chunks
1024
+ chunk_size = max(chunk_size, 4 * 1024 * 1024)
1025
+ elif part_size > 50 * 1024 * 1024: # Part >50MB: use 2MB chunks
1026
+ chunk_size = max(chunk_size, 2 * 1024 * 1024)
1027
+
1014
1028
  # Progress callback optimization: track last callback state
1015
1029
  last_callback_bytes = 0
1016
1030
  last_callback_time = time.time()
1017
1031
  callback_threshold_bytes = 1024 * 1024 # 1MB
1018
1032
  callback_threshold_time = 0.5 # 0.5 seconds
1019
1033
 
1034
+ # Per-part retry configuration
1035
+ max_part_retries = options.max_retries
1036
+ part_retry_delay = options.retry_delay
1037
+
1038
+ # For large file parts, disable the total timeout (which caps the entire
1039
+ # transfer) but keep sock_read so stalled connections are detected.
1040
+ range_timeout = aiohttp.ClientTimeout(
1041
+ total=None,
1042
+ sock_read=self.config.timeout,
1043
+ )
1044
+
1020
1045
  # Get the current event loop
1021
1046
  loop = asyncio.get_running_loop()
1022
1047
  bytes_lock = asyncio.Lock()
1023
1048
 
1024
1049
  async def download_range(start_byte: int, end_byte: int):
1025
1050
  nonlocal bytes_downloaded, last_callback_bytes, last_callback_time
1026
- headers = {"Range": f"bytes={start_byte}-{end_byte}"}
1027
-
1028
- # Open a separate file handle for this part to avoid lock contention
1029
- # Use 'r+b' to write to the existing file without truncating
1030
- with open(temp_destination, "r+b") as part_fh:
1031
- # each part request
1032
- async with await self._enter_request_cm(
1033
- "GET", url, params=params, headers=headers
1034
- ) as resp:
1035
- await self._handle_response(resp)
1036
- # Stream and write to correct offset
1037
- current_pos = start_byte
1038
- chunk_size = (
1039
- options.chunk_size or 1048576
1040
- ) # 1MB default for better performance
1041
-
1042
- async for chunk in resp.content.iter_chunked(chunk_size):
1043
- # Write directly to file at correct position
1044
- # Since we have a private handle, we don't need a file lock
1045
- # But we should still run IO in executor to avoid blocking the loop
1046
- await loop.run_in_executor(None, part_fh.seek, current_pos)
1047
- await loop.run_in_executor(None, part_fh.write, chunk)
1048
-
1049
- chunk_len = len(chunk)
1050
- current_pos += chunk_len
1051
+ range_headers = {"Range": f"bytes={start_byte}-{end_byte}"}
1052
+ part_bytes_written = (
1053
+ 0 # Track bytes written by this part for retry rollback
1054
+ )
1051
1055
 
1056
+ for attempt in range(max_part_retries + 1):
1057
+ try:
1058
+ # On retry, subtract previously counted bytes for this part
1059
+ if attempt > 0 and part_bytes_written > 0:
1052
1060
  async with bytes_lock:
1053
- bytes_downloaded += chunk_len
1061
+ bytes_downloaded -= part_bytes_written
1054
1062
  progress.update_progress(bytes_downloaded)
1063
+ part_bytes_written = 0
1064
+ await asyncio.sleep(part_retry_delay * (2 ** (attempt - 1)))
1065
+
1066
+ with open(temp_destination, "r+b") as part_fh:
1067
+ async with await self._enter_request_cm(
1068
+ "GET",
1069
+ url,
1070
+ params=params,
1071
+ headers=range_headers,
1072
+ timeout=range_timeout,
1073
+ ) as resp:
1074
+ await self._handle_response(resp)
1075
+ current_pos = start_byte
1076
+
1077
+ def _seek_write(fh, pos, data):
1078
+ fh.seek(pos)
1079
+ fh.write(data)
1080
+
1081
+ async for chunk in resp.content.iter_chunked(
1082
+ chunk_size
1083
+ ):
1084
+ await loop.run_in_executor(
1085
+ None, _seek_write, part_fh, current_pos, chunk
1086
+ )
1087
+
1088
+ chunk_len = len(chunk)
1089
+ current_pos += chunk_len
1090
+ part_bytes_written += chunk_len
1091
+
1092
+ # Update counters under lock, invoke callback outside
1093
+ should_callback = False
1094
+ async with bytes_lock:
1095
+ bytes_downloaded += chunk_len
1096
+ progress.update_progress(bytes_downloaded)
1097
+
1098
+ current_time = time.time()
1099
+ bytes_diff = (
1100
+ bytes_downloaded - last_callback_bytes
1101
+ )
1102
+ time_diff = current_time - last_callback_time
1055
1103
 
1056
- # Optimized progress callback: only call every 1MB or 0.5s
1057
- current_time = time.time()
1058
- bytes_diff = bytes_downloaded - last_callback_bytes
1059
- time_diff = current_time - last_callback_time
1060
-
1061
- should_callback = (
1062
- bytes_diff >= callback_threshold_bytes
1063
- or time_diff >= callback_threshold_time
1064
- or bytes_downloaded
1065
- == total_bytes # Always callback on completion
1066
- )
1067
-
1068
- if should_callback:
1069
- if progress_callback:
1070
- progress_callback(progress)
1071
- elif options.show_progress:
1072
- self.logger.info(
1073
- "Download progress",
1074
- file=file_group_id,
1075
- percentage=f"{progress.percentage:.1f}%",
1076
- downloaded=format_file_size(
1077
- bytes_downloaded
1078
- ),
1104
+ should_callback = (
1105
+ bytes_diff >= callback_threshold_bytes
1106
+ or time_diff >= callback_threshold_time
1107
+ or bytes_downloaded
1108
+ == total_bytes # Always callback on completion
1079
1109
  )
1080
- last_callback_bytes = bytes_downloaded
1081
- last_callback_time = current_time
1110
+
1111
+ if should_callback:
1112
+ last_callback_bytes = bytes_downloaded
1113
+ last_callback_time = current_time
1114
+
1115
+ if should_callback:
1116
+ if progress_callback:
1117
+ progress_callback(progress)
1118
+ elif options.show_progress:
1119
+ self.logger.info(
1120
+ "Download progress",
1121
+ file=file_group_id,
1122
+ percentage=f"{progress.percentage:.1f}%",
1123
+ downloaded=format_file_size(
1124
+ bytes_downloaded
1125
+ ),
1126
+ )
1127
+ # Part completed successfully
1128
+ return
1129
+ except Exception:
1130
+ if attempt == max_part_retries:
1131
+ raise # Exhausted retries, propagate to gather
1082
1132
 
1083
1133
  # Run all parts concurrently
1084
1134
  await asyncio.gather(*(download_range(s, e) for s, e in ranges))
@@ -1099,10 +1149,7 @@ class DataQueryClient:
1099
1149
  if temp_destination and temp_destination.exists():
1100
1150
  try:
1101
1151
  # Ensure any open handle is closed
1102
- if (
1103
- total_bytes
1104
- and temp_destination.stat().st_size >= total_bytes
1105
- ):
1152
+ if total_bytes and bytes_downloaded >= total_bytes:
1106
1153
  if destination is None:
1107
1154
  destination = temp_destination.with_suffix("")
1108
1155
  temp_destination.replace(destination)
@@ -1453,25 +1500,7 @@ class DataQueryClient:
1453
1500
  validate_date_format(start_date, "start-date")
1454
1501
  if end_date is not None:
1455
1502
  validate_date_format(end_date, "end-date")
1456
- """
1457
- Retrieve time-series data for explicit list of instruments and attributes using identifiers.
1458
1503
 
1459
- Args:
1460
- instruments: List of instrument identifiers
1461
- attributes: List of attribute identifiers
1462
- data: Data type (REFERENCE_DATA, NO_REFERENCE_DATA, ALL)
1463
- format: Response format (JSON)
1464
- start_date: Start date in YYYYMMDD or TODAY-Nx format
1465
- end_date: End date in YYYYMMDD or TODAY-Nx format
1466
- calendar: Calendar convention
1467
- frequency: Frequency convention
1468
- conversion: Conversion convention
1469
- nan_treatment: Missing data treatment
1470
- page: Optional page token for pagination
1471
-
1472
- Returns:
1473
- TimeSeriesResponse containing the time series data
1474
- """
1475
1504
  params = {
1476
1505
  "instruments": instruments,
1477
1506
  "attributes": attributes,
@@ -2461,9 +2490,6 @@ class DataQueryClient:
2461
2490
  )
2462
2491
  )
2463
2492
 
2464
- # Note: download_multiple_files method removed as it's not part of the new API spec
2465
- # Use individual download_file calls for batch operations
2466
-
2467
2493
  def list_available_files(
2468
2494
  self,
2469
2495
  group_id: str,