cryptodatapy 0.2.32__py3-none-any.whl → 0.2.34__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cryptodatapy/conf/fields.csv +1 -1
- cryptodatapy/extract/libraries/ccxt_api.py +249 -78
- cryptodatapy/transform/wrangle.py +2 -5
- {cryptodatapy-0.2.32.dist-info → cryptodatapy-0.2.34.dist-info}/METADATA +1 -1
- {cryptodatapy-0.2.32.dist-info → cryptodatapy-0.2.34.dist-info}/RECORD +7 -7
- {cryptodatapy-0.2.32.dist-info → cryptodatapy-0.2.34.dist-info}/LICENSE +0 -0
- {cryptodatapy-0.2.32.dist-info → cryptodatapy-0.2.34.dist-info}/WHEEL +0 -0
cryptodatapy/conf/fields.csv
CHANGED
|
@@ -27,7 +27,7 @@ vwap,volume-weighted avg price,average volume-weighted price of quote asset with
|
|
|
27
27
|
dividend,dividend,dividend paid out on ex-ante date,market,ohlc_bars,d,quote currency units,Float64,,,,,,divCash,,,,,,,,
|
|
28
28
|
split,split,"factor used to adjust prices when a company splits, reverse splits, or pays a distribution",market,ohlc_bars,d,factor,Float64,,,,,,splitFactor,,,,,,,,
|
|
29
29
|
ref_rate_usd,reference rate in USD,price of asset in USD using an aggregation methodology,market,ohlc_bars,d,quote currency units,Float64,,ReferenceRateUSD,,,,,,,,,,,,
|
|
30
|
-
oi,open interest,number of outstanding futures contracts that are open and have yet to be settled,market,derivatives,"1min, 5min, 10min, 15min, 30min, 1h, 2h, 4h, 8h, d, w, m, q",number of contracts,Float64,,contract_count
|
|
30
|
+
oi,open interest,number of outstanding futures contracts that are open and have yet to be settled,market,derivatives,"1min, 5min, 10min, 15min, 30min, 1h, 2h, 4h, 8h, d, w, m, q",number of contracts,Float64,,contract_count,openInterestAmount,,,,,,,,,,,
|
|
31
31
|
funding_rate,funding rate,interest rate for holding derivative contract within time interval,market,derivatives,"1h, 2h, 4h, 8h, d, w, m, q",interest rate over time interval,Float64,,rate,fundingRate,derivatives/futures_funding_rate_perpetual,,,,,,,,,,
|
|
32
32
|
mkt_cap,market capitalization,usd value of circulating supply,market,market capitalization,"d, w, m, q",usd,Float64,,CapMrktCurUSD,,market/marketcap_usd,,,,,,,,,,
|
|
33
33
|
mkt_cap_real,"market capitalization, reaized","The sum USD value based on the USD closing price on the day that a native unit last moved (i.e., last transacted) for all native units",market,market capitalization,"d, w, m, q",usd,Float64,,CapRealUSD,,,,,,,,,,,,
|
|
@@ -37,6 +37,9 @@ class CCXT(Library):
|
|
|
37
37
|
api_key: Optional[str] = None,
|
|
38
38
|
max_obs_per_call: Optional[int] = 1000,
|
|
39
39
|
rate_limit: Optional[Any] = None,
|
|
40
|
+
ip_ban_wait_time_s: float = 320.0, # 5.3 minutes for IP ban
|
|
41
|
+
recovery_base_delay_s: float = 2.0, # base delay for exponential backoff
|
|
42
|
+
max_recovery_delay_s: float = 60.0 # max delay for exponential backoff
|
|
40
43
|
):
|
|
41
44
|
"""
|
|
42
45
|
Constructor
|
|
@@ -71,6 +74,12 @@ class CCXT(Library):
|
|
|
71
74
|
api_limit stored in DataCredentials.
|
|
72
75
|
rate_limit: Any, optional, Default None
|
|
73
76
|
Number of API calls made and left, by time frequency.
|
|
77
|
+
ip_ban_wait_time_s: float, default 320.0
|
|
78
|
+
Time in seconds to wait if IP ban is detected (HTTP 403).
|
|
79
|
+
recovery_base_delay_s: float, default 2.0
|
|
80
|
+
Base delay in seconds for exponential backoff strategy.
|
|
81
|
+
max_recovery_delay_s: float, default 60.0
|
|
82
|
+
Maximum delay in seconds for exponential backoff strategy.
|
|
74
83
|
"""
|
|
75
84
|
super().__init__(
|
|
76
85
|
categories, exchanges, indexes, assets, markets, market_types,
|
|
@@ -79,6 +88,11 @@ class CCXT(Library):
|
|
|
79
88
|
self.exchange = None
|
|
80
89
|
self.exchange_async = None
|
|
81
90
|
self.data_req = None
|
|
91
|
+
|
|
92
|
+
self.ip_ban_wait_time_s = ip_ban_wait_time_s
|
|
93
|
+
self.recovery_base_delay_s = recovery_base_delay_s
|
|
94
|
+
self.max_recovery_delay_s = max_recovery_delay_s
|
|
95
|
+
|
|
82
96
|
self.data_resp = []
|
|
83
97
|
self.data = pd.DataFrame()
|
|
84
98
|
|
|
@@ -313,17 +327,170 @@ class CCXT(Library):
|
|
|
313
327
|
if self.rate_limit is None:
|
|
314
328
|
self.rate_limit = self.exchange.rateLimit
|
|
315
329
|
|
|
316
|
-
@staticmethod
|
|
317
|
-
def exponential_backoff_with_jitter(base_delay: float, max_delay: int, attempts: int) -> None:
|
|
330
|
+
# @staticmethod
|
|
331
|
+
# def exponential_backoff_with_jitter(base_delay: float, max_delay: int, attempts: int) -> None:
|
|
332
|
+
# delay = min(max_delay, base_delay * (2 ** attempts))
|
|
333
|
+
# delay_with_jitter = delay + random.uniform(0, delay * 0.5)
|
|
334
|
+
# sleep(delay_with_jitter)
|
|
335
|
+
#
|
|
336
|
+
# @staticmethod
|
|
337
|
+
# async def exponential_backoff_with_jitter_async(base_delay: float, max_delay: int, attempts: int) -> None:
|
|
338
|
+
# delay = min(max_delay, base_delay * (2 ** attempts))
|
|
339
|
+
# delay_with_jitter = delay + random.uniform(0, delay * 0.5)
|
|
340
|
+
# await asyncio.sleep(delay_with_jitter)
|
|
341
|
+
#
|
|
342
|
+
|
|
343
|
+
def exponential_backoff_with_jitter(
|
|
344
|
+
self,
|
|
345
|
+
attempts: int,
|
|
346
|
+
status_code: Optional[int] = None
|
|
347
|
+
) -> float:
|
|
348
|
+
"""
|
|
349
|
+
Calculates and applies exponential backoff with full jitter, honoring
|
|
350
|
+
specific error codes (403/429) using configurable instance properties.
|
|
351
|
+
|
|
352
|
+
Args:
|
|
353
|
+
attempts: The current retry number (starting at 1 after the first failure).
|
|
354
|
+
status_code: The HTTP status code received (e.g., 403, 429).
|
|
355
|
+
|
|
356
|
+
Returns:
|
|
357
|
+
The actual time slept in seconds.
|
|
358
|
+
"""
|
|
359
|
+
|
|
360
|
+
# 1. CRITICAL: Hard IP ban handling (403)
|
|
361
|
+
if status_code == 403:
|
|
362
|
+
sleep_time = self.ip_ban_wait_time_s
|
|
363
|
+
logging.error(
|
|
364
|
+
f"🚨 CRITICAL: HTTP 403 Forbidden (IP Ban) detected. Pausing for mandatory {sleep_time} seconds.")
|
|
365
|
+
sleep(sleep_time)
|
|
366
|
+
return sleep_time
|
|
367
|
+
|
|
368
|
+
# 2. Standard Exponential Backoff for 429 or generic exceptions
|
|
369
|
+
base_delay = self.recovery_base_delay_s
|
|
370
|
+
max_delay = self.max_recovery_delay_s
|
|
371
|
+
|
|
372
|
+
# Calculate exponential growth
|
|
318
373
|
delay = min(max_delay, base_delay * (2 ** attempts))
|
|
319
|
-
delay_with_jitter = delay + random.uniform(0, delay * 0.5)
|
|
320
|
-
sleep(delay_with_jitter)
|
|
321
374
|
|
|
322
|
-
|
|
323
|
-
|
|
375
|
+
# Add full jitter (random delay between 0 and the calculated delay)
|
|
376
|
+
sleep_time = random.uniform(0, delay)
|
|
377
|
+
|
|
378
|
+
logging.warning(
|
|
379
|
+
f"Applying recovery backoff (Attempt {attempts}, Code {status_code if status_code else 'N/A'}): "
|
|
380
|
+
f"{sleep_time:.2f} seconds."
|
|
381
|
+
)
|
|
382
|
+
|
|
383
|
+
sleep(sleep_time)
|
|
384
|
+
return sleep_time
|
|
385
|
+
|
|
386
|
+
async def exponential_backoff_with_jitter_async(
|
|
387
|
+
self,
|
|
388
|
+
attempts: int,
|
|
389
|
+
status_code: Optional[int] = None
|
|
390
|
+
) -> float:
|
|
391
|
+
"""
|
|
392
|
+
Async version of exponential_backoff_with_jitter, using configurable instance properties.
|
|
393
|
+
"""
|
|
394
|
+
|
|
395
|
+
# 1. CRITICAL: Hard IP ban handling (403)
|
|
396
|
+
if status_code == 403:
|
|
397
|
+
sleep_time = self.ip_ban_wait_time_s
|
|
398
|
+
logging.error(
|
|
399
|
+
f"🚨 CRITICAL: HTTP 403 Forbidden (IP Ban) detected. Pausing for mandatory {sleep_time} seconds.")
|
|
400
|
+
await asyncio.sleep(sleep_time)
|
|
401
|
+
return sleep_time
|
|
402
|
+
|
|
403
|
+
# 2. Standard Exponential Backoff for 429 or generic exceptions
|
|
404
|
+
base_delay = self.recovery_base_delay_s
|
|
405
|
+
max_delay = self.max_recovery_delay_s
|
|
406
|
+
|
|
407
|
+
# Calculate exponential growth
|
|
324
408
|
delay = min(max_delay, base_delay * (2 ** attempts))
|
|
325
|
-
|
|
326
|
-
|
|
409
|
+
|
|
410
|
+
# Add full jitter (random delay between 0 and the calculated delay)
|
|
411
|
+
sleep_time = random.uniform(0, delay)
|
|
412
|
+
|
|
413
|
+
logging.warning(
|
|
414
|
+
f"Applying recovery backoff (Attempt {attempts}, Code {status_code if status_code else 'N/A'}): "
|
|
415
|
+
f"{sleep_time:.2f} seconds."
|
|
416
|
+
)
|
|
417
|
+
|
|
418
|
+
await asyncio.sleep(sleep_time)
|
|
419
|
+
return sleep_time
|
|
420
|
+
|
|
421
|
+
# --- New Exception Helper Methods for Modularity ---
|
|
422
|
+
|
|
423
|
+
def _handle_exception_and_backoff(self, e: Exception, attempts: int) -> bool:
|
|
424
|
+
"""
|
|
425
|
+
Analyzes a synchronous exception, applies backoff if recoverable, and logs the result.
|
|
426
|
+
|
|
427
|
+
Returns:
|
|
428
|
+
True if the error was recoverable (retries should continue).
|
|
429
|
+
False if the error is terminal (retries should stop).
|
|
430
|
+
"""
|
|
431
|
+
# 1. CRITICAL: 403 IP Ban Check (Must parse string due to CCXT wrapping)
|
|
432
|
+
error_message = str(e)
|
|
433
|
+
if '403' in error_message or 'Forbidden' in error_message:
|
|
434
|
+
logging.error(f"CCXT Exception: Critical 403 IP Ban detected via parsing.")
|
|
435
|
+
self.exponential_backoff_with_jitter(attempts, status_code=403)
|
|
436
|
+
return True # Recoverable with mandatory long pause
|
|
437
|
+
|
|
438
|
+
# 2. Recoverable CCXT Exceptions (Standard Backoff: proxy 429)
|
|
439
|
+
if isinstance(e, (
|
|
440
|
+
ccxt.RequestTimeout,
|
|
441
|
+
ccxt.RateLimitExceeded,
|
|
442
|
+
ccxt.DDoSProtection,
|
|
443
|
+
ccxt.ExchangeNotAvailable,
|
|
444
|
+
ccxt.OperationFailed,
|
|
445
|
+
)):
|
|
446
|
+
logging.warning(f"CCXT Exception: Recoverable error ({e.__class__.__name__}) on attempt {attempts}.")
|
|
447
|
+
self.exponential_backoff_with_jitter(attempts, status_code=429)
|
|
448
|
+
return True # Recoverable, continue retries
|
|
449
|
+
|
|
450
|
+
# 3. Unrecoverable CCXT Errors (Terminal: e.g., bad symbol)
|
|
451
|
+
if isinstance(e, ccxt.ExchangeError):
|
|
452
|
+
logging.error(f"CCXT Exception: Terminal ExchangeError ({e.__class__.__name__}). Halting retries.")
|
|
453
|
+
return False # Unrecoverable, stop retries
|
|
454
|
+
|
|
455
|
+
# 4. Other/Unknown Exceptions (Terminal)
|
|
456
|
+
logging.error(f"CCXT Exception: Unhandled error ({e.__class__.__name__}). Halting retries: {e}")
|
|
457
|
+
return False
|
|
458
|
+
|
|
459
|
+
async def _handle_exception_and_backoff_async(self, e: Exception, attempts: int) -> bool:
|
|
460
|
+
"""
|
|
461
|
+
Analyzes an asynchronous exception, applies backoff if recoverable, and logs the result.
|
|
462
|
+
|
|
463
|
+
Returns:
|
|
464
|
+
True if the error was recoverable (retries should continue).
|
|
465
|
+
False if the error is terminal (retries should stop).
|
|
466
|
+
"""
|
|
467
|
+
# 1. CRITICAL: 403 IP Ban Check (Must parse string due to CCXT wrapping)
|
|
468
|
+
error_message = str(e)
|
|
469
|
+
if '403' in error_message or 'Forbidden' in error_message:
|
|
470
|
+
logging.error(f"CCXT Exception: Critical 403 IP Ban detected via parsing.")
|
|
471
|
+
await self.exponential_backoff_with_jitter_async(attempts, status_code=403)
|
|
472
|
+
return True # Recoverable with mandatory long pause
|
|
473
|
+
|
|
474
|
+
# 2. Recoverable CCXT Exceptions (Standard Backoff: proxy 429)
|
|
475
|
+
if isinstance(e, (
|
|
476
|
+
ccxt.RequestTimeout,
|
|
477
|
+
ccxt.RateLimitExceeded,
|
|
478
|
+
ccxt.DDoSProtection,
|
|
479
|
+
ccxt.ExchangeNotAvailable,
|
|
480
|
+
ccxt.OperationFailed,
|
|
481
|
+
)):
|
|
482
|
+
logging.warning(f"CCXT Exception: Recoverable error ({e.__class__.__name__}) on attempt {attempts}.")
|
|
483
|
+
await self.exponential_backoff_with_jitter_async(attempts, status_code=429)
|
|
484
|
+
return True # Recoverable, continue retries
|
|
485
|
+
|
|
486
|
+
# 3. Unrecoverable CCXT Errors (Terminal: e.g., bad symbol)
|
|
487
|
+
if isinstance(e, ccxt.ExchangeError):
|
|
488
|
+
logging.error(f"CCXT Exception: Terminal ExchangeError ({e.__class__.__name__}). Halting retries.")
|
|
489
|
+
return False # Unrecoverable, stop retries
|
|
490
|
+
|
|
491
|
+
# 4. Other/Unknown Exceptions (Terminal)
|
|
492
|
+
logging.error(f"CCXT Exception: Unhandled error ({e.__class__.__name__}). Halting retries: {e}")
|
|
493
|
+
return False
|
|
327
494
|
|
|
328
495
|
async def _fetch_ohlcv_async(self,
|
|
329
496
|
ticker: str,
|
|
@@ -332,7 +499,6 @@ class CCXT(Library):
|
|
|
332
499
|
end_date: int,
|
|
333
500
|
exch: str,
|
|
334
501
|
trials: int = 3,
|
|
335
|
-
pause: int = 1
|
|
336
502
|
) -> Union[List, None]:
|
|
337
503
|
"""
|
|
338
504
|
Fetches OHLCV data for a specific ticker.
|
|
@@ -351,8 +517,6 @@ class CCXT(Library):
|
|
|
351
517
|
Name of exchange.
|
|
352
518
|
trials: int, default 3
|
|
353
519
|
Number of attempts to fetch data.
|
|
354
|
-
pause: int, default 60
|
|
355
|
-
Pause in seconds to respect the rate limit.
|
|
356
520
|
|
|
357
521
|
Returns
|
|
358
522
|
-------
|
|
@@ -383,17 +547,15 @@ class CCXT(Library):
|
|
|
383
547
|
|
|
384
548
|
# add data to list
|
|
385
549
|
if data:
|
|
550
|
+
# noinspection PyUnusedLocal
|
|
386
551
|
start_date = data[-1][0] + 1
|
|
387
552
|
data_resp.extend(data)
|
|
388
553
|
else:
|
|
389
554
|
break
|
|
390
555
|
|
|
391
556
|
except Exception as e:
|
|
392
|
-
logging.warning(
|
|
393
|
-
f"Failed to get OHLCV data from {self.exchange_async.id} for {ticker} "
|
|
394
|
-
f"on attempt #{attempts + 1}: {e}."
|
|
395
|
-
)
|
|
396
557
|
attempts += 1
|
|
558
|
+
|
|
397
559
|
if attempts >= trials:
|
|
398
560
|
logging.warning(
|
|
399
561
|
f"Failed to get OHLCV data from {self.exchange_async.id} "
|
|
@@ -401,17 +563,17 @@ class CCXT(Library):
|
|
|
401
563
|
)
|
|
402
564
|
break
|
|
403
565
|
|
|
404
|
-
|
|
405
|
-
await self.
|
|
406
|
-
|
|
407
|
-
|
|
566
|
+
# exception handling
|
|
567
|
+
if not await self._handle_exception_and_backoff_async(e, attempts):
|
|
568
|
+
# If the helper returns False, the error is terminal (ExchangeError, etc.)
|
|
569
|
+
break
|
|
408
570
|
|
|
409
|
-
|
|
410
|
-
|
|
571
|
+
await self.exchange_async.close()
|
|
572
|
+
return data_resp
|
|
411
573
|
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
574
|
+
else:
|
|
575
|
+
logging.warning(f"OHLCV data is not available for {self.exchange_async.id}.")
|
|
576
|
+
return None
|
|
415
577
|
|
|
416
578
|
def _fetch_ohlcv(self,
|
|
417
579
|
ticker: str,
|
|
@@ -420,7 +582,6 @@ class CCXT(Library):
|
|
|
420
582
|
end_date: str,
|
|
421
583
|
exch: str,
|
|
422
584
|
trials: int = 3,
|
|
423
|
-
pause: int = 1
|
|
424
585
|
) -> Union[List, None]:
|
|
425
586
|
"""
|
|
426
587
|
Fetches OHLCV data for a specific ticker.
|
|
@@ -439,8 +600,6 @@ class CCXT(Library):
|
|
|
439
600
|
Name of exchange.
|
|
440
601
|
trials: int, default 3
|
|
441
602
|
Number of attempts to fetch data.
|
|
442
|
-
pause: int, default 60
|
|
443
|
-
Pause in seconds to respect the rate limit.
|
|
444
603
|
|
|
445
604
|
Returns
|
|
446
605
|
-------
|
|
@@ -480,11 +639,8 @@ class CCXT(Library):
|
|
|
480
639
|
break
|
|
481
640
|
|
|
482
641
|
except Exception as e:
|
|
483
|
-
logging.warning(
|
|
484
|
-
f"Failed to get OHLCV data from {self.exchange.id} for {ticker} "
|
|
485
|
-
f"on attempt #{attempts + 1}: {e}."
|
|
486
|
-
)
|
|
487
642
|
attempts += 1
|
|
643
|
+
|
|
488
644
|
if attempts >= trials:
|
|
489
645
|
logging.warning(
|
|
490
646
|
f"Failed to get OHLCV data from {self.exchange.id} "
|
|
@@ -492,8 +648,10 @@ class CCXT(Library):
|
|
|
492
648
|
)
|
|
493
649
|
break
|
|
494
650
|
|
|
495
|
-
|
|
496
|
-
self.
|
|
651
|
+
# exception handling
|
|
652
|
+
if not self._handle_exception_and_backoff(e, attempts):
|
|
653
|
+
# If the helper returns False, the error is terminal (ExchangeError, etc.)
|
|
654
|
+
break
|
|
497
655
|
|
|
498
656
|
return data_resp
|
|
499
657
|
|
|
@@ -609,7 +767,6 @@ class CCXT(Library):
|
|
|
609
767
|
end_date: int,
|
|
610
768
|
exch: str,
|
|
611
769
|
trials: int = 3,
|
|
612
|
-
pause: int = 1
|
|
613
770
|
) -> Union[List, None]:
|
|
614
771
|
"""
|
|
615
772
|
Fetches funding rates data for a specific ticker.
|
|
@@ -624,8 +781,6 @@ class CCXT(Library):
|
|
|
624
781
|
End date in integers in milliseconds since Unix epoch.
|
|
625
782
|
trials: int, default 3
|
|
626
783
|
Number of attempts to fetch data.
|
|
627
|
-
pause: int, default 1
|
|
628
|
-
Pause in seconds to respect the rate limit.
|
|
629
784
|
|
|
630
785
|
Returns
|
|
631
786
|
-------
|
|
@@ -661,11 +816,8 @@ class CCXT(Library):
|
|
|
661
816
|
break
|
|
662
817
|
|
|
663
818
|
except Exception as e:
|
|
664
|
-
logging.warning(
|
|
665
|
-
f"Failed to get funding rates from {self.exchange_async.id} for {ticker} "
|
|
666
|
-
f"on attempt #{attempts + 1}: {e}."
|
|
667
|
-
)
|
|
668
819
|
attempts += 1
|
|
820
|
+
|
|
669
821
|
if attempts >= trials:
|
|
670
822
|
logging.warning(
|
|
671
823
|
f"Failed to get funding rates from {self.exchange_async.id} "
|
|
@@ -673,10 +825,9 @@ class CCXT(Library):
|
|
|
673
825
|
)
|
|
674
826
|
break
|
|
675
827
|
|
|
676
|
-
|
|
677
|
-
await self.
|
|
678
|
-
|
|
679
|
-
attempts)
|
|
828
|
+
# exception handling
|
|
829
|
+
if not await self._handle_exception_and_backoff_async(e, attempts):
|
|
830
|
+
break
|
|
680
831
|
|
|
681
832
|
await self.exchange_async.close()
|
|
682
833
|
return data_resp
|
|
@@ -691,7 +842,6 @@ class CCXT(Library):
|
|
|
691
842
|
end_date: int,
|
|
692
843
|
exch: str,
|
|
693
844
|
trials: int = 3,
|
|
694
|
-
pause: int = 1
|
|
695
845
|
) -> Union[List, None]:
|
|
696
846
|
"""
|
|
697
847
|
Fetches funding rates data for a specific ticker.
|
|
@@ -706,8 +856,6 @@ class CCXT(Library):
|
|
|
706
856
|
End date in integers in milliseconds since Unix epoch.
|
|
707
857
|
trials: int, default 3
|
|
708
858
|
Number of attempts to fetch data.
|
|
709
|
-
pause: int, default 1
|
|
710
|
-
Pause in seconds to respect the rate limit.
|
|
711
859
|
|
|
712
860
|
Returns
|
|
713
861
|
-------
|
|
@@ -743,11 +891,8 @@ class CCXT(Library):
|
|
|
743
891
|
break
|
|
744
892
|
|
|
745
893
|
except Exception as e:
|
|
746
|
-
logging.warning(
|
|
747
|
-
f"Failed to get funding rates from {self.exchange.id} for {ticker} "
|
|
748
|
-
f"on attempt #{attempts + 1}: {e}."
|
|
749
|
-
)
|
|
750
894
|
attempts += 1
|
|
895
|
+
|
|
751
896
|
if attempts >= trials:
|
|
752
897
|
logging.warning(
|
|
753
898
|
f"Failed to get funding rates from {self.exchange.id} "
|
|
@@ -755,8 +900,9 @@ class CCXT(Library):
|
|
|
755
900
|
)
|
|
756
901
|
break
|
|
757
902
|
|
|
758
|
-
|
|
759
|
-
self.
|
|
903
|
+
# exception handling
|
|
904
|
+
if not self._handle_exception_and_backoff(e, attempts):
|
|
905
|
+
break
|
|
760
906
|
|
|
761
907
|
return data_resp
|
|
762
908
|
|
|
@@ -868,7 +1014,6 @@ class CCXT(Library):
|
|
|
868
1014
|
end_date: int,
|
|
869
1015
|
exch: str,
|
|
870
1016
|
trials: int = 3,
|
|
871
|
-
pause: int = 1
|
|
872
1017
|
) -> Union[List, None]:
|
|
873
1018
|
"""
|
|
874
1019
|
Fetches open interest data for a specific ticker.
|
|
@@ -887,8 +1032,6 @@ class CCXT(Library):
|
|
|
887
1032
|
Name of exchange.
|
|
888
1033
|
trials: int, default 3
|
|
889
1034
|
Number of attempts to fetch data.
|
|
890
|
-
pause: int, default 1
|
|
891
|
-
Pause in seconds to respect the rate limit.
|
|
892
1035
|
|
|
893
1036
|
Returns
|
|
894
1037
|
-------
|
|
@@ -899,6 +1042,26 @@ class CCXT(Library):
|
|
|
899
1042
|
attempts = 0
|
|
900
1043
|
data_resp = []
|
|
901
1044
|
|
|
1045
|
+
# Maximum historical range for Binance OI is 30 days
|
|
1046
|
+
# 30 days in milliseconds: 30 * 24 * 60 * 60 * 1000 = 2,592,000,000 ms
|
|
1047
|
+
SAFE_OI_RANGE_MS = 25 * 24 * 60 * 60 * 1000
|
|
1048
|
+
|
|
1049
|
+
# inst exch
|
|
1050
|
+
if self.exchange_async is None:
|
|
1051
|
+
self.exchange_async = getattr(ccxt_async, exch)()
|
|
1052
|
+
|
|
1053
|
+
# --- Binance 30-day Limit Enforcement ---
|
|
1054
|
+
if exch.lower() == 'binanceusdm': # Binance USDM Futures
|
|
1055
|
+
requested_range_ms = end_date - start_date
|
|
1056
|
+
if requested_range_ms > SAFE_OI_RANGE_MS:
|
|
1057
|
+
# Adjust start_date to fit within the 30-day window ending at end_date
|
|
1058
|
+
new_start_date = end_date - SAFE_OI_RANGE_MS
|
|
1059
|
+
logging.warning(
|
|
1060
|
+
f"Exchange '{exch}' Open Interest historical data is limited to 30 days. "
|
|
1061
|
+
f"Adjusting start_date for {ticker} from {start_date} to {new_start_date}."
|
|
1062
|
+
)
|
|
1063
|
+
start_date = new_start_date
|
|
1064
|
+
|
|
902
1065
|
# inst exch
|
|
903
1066
|
if self.exchange_async is None:
|
|
904
1067
|
self.exchange_async = getattr(ccxt_async, exch)()
|
|
@@ -926,28 +1089,24 @@ class CCXT(Library):
|
|
|
926
1089
|
break
|
|
927
1090
|
|
|
928
1091
|
except Exception as e:
|
|
929
|
-
logging.warning(
|
|
930
|
-
f"Failed to get open interest from {self.exchange_async.id} for {ticker} "
|
|
931
|
-
f"on attempt #{attempts + 1}: {e}."
|
|
932
|
-
)
|
|
933
1092
|
attempts += 1
|
|
1093
|
+
|
|
934
1094
|
if attempts >= trials:
|
|
935
1095
|
logging.warning(
|
|
936
|
-
f"Failed to get
|
|
1096
|
+
f"Failed to get funding rates from {self.exchange_async.id} "
|
|
937
1097
|
f"for {ticker} after {trials} attempts."
|
|
938
1098
|
)
|
|
939
1099
|
break
|
|
940
1100
|
|
|
941
|
-
|
|
942
|
-
await self.
|
|
943
|
-
|
|
944
|
-
attempts)
|
|
1101
|
+
# exception handling
|
|
1102
|
+
if not await self._handle_exception_and_backoff_async(e, attempts):
|
|
1103
|
+
break
|
|
945
1104
|
|
|
946
1105
|
await self.exchange_async.close()
|
|
947
1106
|
return data_resp
|
|
948
1107
|
|
|
949
1108
|
else:
|
|
950
|
-
logging.warning(f"
|
|
1109
|
+
logging.warning(f"Funding rates are not available for {self.exchange_async.id}.")
|
|
951
1110
|
return None
|
|
952
1111
|
|
|
953
1112
|
def _fetch_open_interest(self,
|
|
@@ -957,7 +1116,6 @@ class CCXT(Library):
|
|
|
957
1116
|
end_date: int,
|
|
958
1117
|
exch: str,
|
|
959
1118
|
trials: int = 3,
|
|
960
|
-
pause: int = 1
|
|
961
1119
|
) -> Union[List, None]:
|
|
962
1120
|
"""
|
|
963
1121
|
Fetches open interest data for a specific ticker.
|
|
@@ -976,8 +1134,6 @@ class CCXT(Library):
|
|
|
976
1134
|
Name of exchange.
|
|
977
1135
|
trials: int, default 3
|
|
978
1136
|
Number of attempts to fetch data.
|
|
979
|
-
pause: int, default 1
|
|
980
|
-
Pause in seconds to respect the rate limit.
|
|
981
1137
|
|
|
982
1138
|
Returns
|
|
983
1139
|
-------
|
|
@@ -988,8 +1144,25 @@ class CCXT(Library):
|
|
|
988
1144
|
attempts = 0
|
|
989
1145
|
data_resp = []
|
|
990
1146
|
|
|
1147
|
+
# Maximum historical range for Binance OI is 30 days
|
|
1148
|
+
# 30 days in milliseconds: 30 * 24 * 60 * 60 * 1000 = 2,592,000,000 ms
|
|
1149
|
+
SAFE_OI_RANGE_MS = 25 * 24 * 60 * 60 * 1000
|
|
1150
|
+
|
|
991
1151
|
# inst exch
|
|
992
|
-
self.
|
|
1152
|
+
if self.exchange_async is None:
|
|
1153
|
+
self.exchange_async = getattr(ccxt_async, exch)()
|
|
1154
|
+
|
|
1155
|
+
# --- Binance 30-day Limit Enforcement ---
|
|
1156
|
+
if exch.lower() == 'binanceusdm': # Binance USDM Futures
|
|
1157
|
+
requested_range_ms = end_date - start_date
|
|
1158
|
+
if requested_range_ms > SAFE_OI_RANGE_MS:
|
|
1159
|
+
# Adjust start_date to fit within the 30-day window ending at end_date
|
|
1160
|
+
new_start_date = end_date - SAFE_OI_RANGE_MS
|
|
1161
|
+
logging.warning(
|
|
1162
|
+
f"Exchange '{exch}' open interest historical data is limited to 30 days. "
|
|
1163
|
+
f"Adjusting start_date for {ticker} from {start_date} to {new_start_date}."
|
|
1164
|
+
)
|
|
1165
|
+
start_date = new_start_date
|
|
993
1166
|
|
|
994
1167
|
# fetch data
|
|
995
1168
|
if self.exchange.has['fetchOpenInterestHistory']:
|
|
@@ -1014,25 +1187,23 @@ class CCXT(Library):
|
|
|
1014
1187
|
break
|
|
1015
1188
|
|
|
1016
1189
|
except Exception as e:
|
|
1017
|
-
logging.warning(
|
|
1018
|
-
f"Failed to get open interest from {self.exchange.id} for {ticker} "
|
|
1019
|
-
f"on attempt #{attempts + 1}: {e}."
|
|
1020
|
-
)
|
|
1021
1190
|
attempts += 1
|
|
1191
|
+
|
|
1022
1192
|
if attempts >= trials:
|
|
1023
1193
|
logging.warning(
|
|
1024
|
-
f"Failed to get
|
|
1194
|
+
f"Failed to get funding rates from {self.exchange.id} "
|
|
1025
1195
|
f"for {ticker} after {trials} attempts."
|
|
1026
1196
|
)
|
|
1027
1197
|
break
|
|
1028
1198
|
|
|
1029
|
-
|
|
1030
|
-
self.
|
|
1199
|
+
# exception handling
|
|
1200
|
+
if not self._handle_exception_and_backoff(e, attempts):
|
|
1201
|
+
break
|
|
1031
1202
|
|
|
1032
1203
|
return data_resp
|
|
1033
1204
|
|
|
1034
1205
|
else:
|
|
1035
|
-
logging.warning(f"
|
|
1206
|
+
logging.warning(f"Funding rates are not available for {self.exchange.id}.")
|
|
1036
1207
|
return None
|
|
1037
1208
|
|
|
1038
1209
|
async def _fetch_all_open_interest_async(self,
|
|
@@ -857,10 +857,7 @@ class WrangleData:
|
|
|
857
857
|
Dataframe with tidy data format.
|
|
858
858
|
"""
|
|
859
859
|
# add tickers
|
|
860
|
-
|
|
861
|
-
df = pd.DataFrame(self.data_resp[i])
|
|
862
|
-
self.tidy_data = pd.concat([self.tidy_data, df])
|
|
863
|
-
self.tidy_data = self.tidy_data[['symbol', 'openInterestAmount', 'datetime']]
|
|
860
|
+
self.tidy_data = pd.DataFrame(self.data_resp)[['symbol', 'openInterestAmount', 'datetime']]
|
|
864
861
|
self.data_resp = self.tidy_data
|
|
865
862
|
|
|
866
863
|
# convert to lib fields
|
|
@@ -1245,4 +1242,4 @@ class WrangleData:
|
|
|
1245
1242
|
if self.data_req.end_date is not None:
|
|
1246
1243
|
self.data_resp = self.data_resp[(self.data_resp.index <= self.data_req.end_date)]
|
|
1247
1244
|
|
|
1248
|
-
return self
|
|
1245
|
+
return self
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
cryptodatapy/__init__.py,sha256=ee1UaINHZn1A_SZ96XM3hCguQEJgiPTvKlnYsk3mmS4,185
|
|
2
2
|
cryptodatapy/conf/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
3
|
-
cryptodatapy/conf/fields.csv,sha256=
|
|
3
|
+
cryptodatapy/conf/fields.csv,sha256=HyVTpFhiTZyAUbk9xPNcpPNU3ZG9J31iIewjzImBhLQ,25963
|
|
4
4
|
cryptodatapy/conf/tickers.csv,sha256=5iEg--AyIhSF9XkscKrbdhj-hDASkKzda8tqpWNYMrE,357956
|
|
5
5
|
cryptodatapy/datasets/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
6
6
|
cryptodatapy/datasets/br_econ_calendar.csv,sha256=mSM0IOIByI-0gIIL1CbDQPqHYI5lK6vavrY1ODj3Jlk,1185318
|
|
@@ -39,7 +39,7 @@ cryptodatapy/extract/exchanges/dydx.py,sha256=tBp60PG24tUZI949nHSiJQwjsP0zI2Oyz9
|
|
|
39
39
|
cryptodatapy/extract/exchanges/exchange.py,sha256=Cicj3KS4zLbwmXX5fu89byXNwqqU4TH31GFv0zj3D4s,13010
|
|
40
40
|
cryptodatapy/extract/getdata.py,sha256=_8Hi4vdkj2xGykb_2fBcqzJTNROzX0QnQE2hxPfe690,11543
|
|
41
41
|
cryptodatapy/extract/libraries/__init__.py,sha256=KG2Rr3c8CcDq-nbhT-ItssqZE9U65xQXH0Wv0g86SVg,254
|
|
42
|
-
cryptodatapy/extract/libraries/ccxt_api.py,sha256=
|
|
42
|
+
cryptodatapy/extract/libraries/ccxt_api.py,sha256=eeA7xPN9DBY61LvUM5jW_2MOxo_3vIIK-h5VdIDm0n0,63146
|
|
43
43
|
cryptodatapy/extract/libraries/dbnomics_api.py,sha256=M6kPIH-hKqkmeBQb-g56dY9jatqLCtSl_MnvPblHtAc,9421
|
|
44
44
|
cryptodatapy/extract/libraries/investpy_api.py,sha256=qtGm3LDluXxJorvFv0w1bm1oBrcZIfE5cZSYzNYvttY,18409
|
|
45
45
|
cryptodatapy/extract/libraries/library.py,sha256=eU8NnQZ9luLGdIF5hms6j8VPCWc50evkREc4xdh-g1I,12301
|
|
@@ -53,12 +53,12 @@ cryptodatapy/transform/convertparams.py,sha256=yrm9Gr6Fm7CaVTfxHGs0TJx6ZtP7llrlI
|
|
|
53
53
|
cryptodatapy/transform/filter.py,sha256=37MjUKUay3dwwyn47rnNOU51X_OFzmWq_N9buALzq9k,9058
|
|
54
54
|
cryptodatapy/transform/impute.py,sha256=_0-SX5nnPrYgJYT-HKwBGNkmWXRMy9-C2oeU6VqkQp0,5537
|
|
55
55
|
cryptodatapy/transform/od.py,sha256=mI1oojMbfmdO9ZewL3AvMxoXuMM05Ut2oGm_ogMf2XU,30386
|
|
56
|
-
cryptodatapy/transform/wrangle.py,sha256=
|
|
56
|
+
cryptodatapy/transform/wrangle.py,sha256=o_VcH90sHXn7Hf3u9O6O1LKd8lEMy2o84A3MYrmGcjQ,44653
|
|
57
57
|
cryptodatapy/util/__init__.py,sha256=zSQ2HU2QIXzCuptJjknmrClwtQKCvIj4aNysZljIgrU,116
|
|
58
58
|
cryptodatapy/util/datacatalog.py,sha256=qCCX6srXvaAbVAKuA0M2y5IK_2OEx5xA3yRahDZlC-g,13157
|
|
59
59
|
cryptodatapy/util/datacredentials.py,sha256=BnoQlUchbP0vfXqXRuhCOOsHyUTMuH5T4RAKBbHzMyo,3140
|
|
60
60
|
cryptodatapy/util/utils.py,sha256=OTTa4YvRj7Cb_2h5h8xoDy9Ap0LB1rg_wFgsDwy9R9o,4244
|
|
61
|
-
cryptodatapy-0.2.
|
|
62
|
-
cryptodatapy-0.2.
|
|
63
|
-
cryptodatapy-0.2.
|
|
64
|
-
cryptodatapy-0.2.
|
|
61
|
+
cryptodatapy-0.2.34.dist-info/LICENSE,sha256=sw4oVq8bDjT3uMtaFebQ-xeIVP4H-bXldTs9q-Jjeks,11344
|
|
62
|
+
cryptodatapy-0.2.34.dist-info/METADATA,sha256=UDAk1Hgyu9nrObLEfI_iQWsA1y9Wi3CrBk_Hx1JIMiw,6486
|
|
63
|
+
cryptodatapy-0.2.34.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
|
|
64
|
+
cryptodatapy-0.2.34.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|