cryptodatapy 0.2.8__py3-none-any.whl → 0.2.9__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cryptodatapy/extract/datarequest.py +0 -1
- cryptodatapy/extract/exchanges/__init__.py +2 -0
- cryptodatapy/extract/exchanges/dydx.py +137 -0
- cryptodatapy/extract/exchanges/exchange.py +439 -0
- cryptodatapy/extract/libraries/ccxt_api.py +667 -172
- cryptodatapy/extract/libraries/library.py +1 -3
- cryptodatapy/extract/web/web.py +62 -0
- {cryptodatapy-0.2.8.dist-info → cryptodatapy-0.2.9.dist-info}/METADATA +1 -1
- {cryptodatapy-0.2.8.dist-info → cryptodatapy-0.2.9.dist-info}/RECORD +11 -22
- cryptodatapy/conf/fx_tickers.csv +0 -31
- cryptodatapy/extract/data_vendors/CoinMetrics.ipynb +0 -747
- cryptodatapy/extract/libraries/Untitled.ipynb +0 -199
- cryptodatapy/extract/libraries/ccxt.ipynb +0 -747
- cryptodatapy/extract/libraries/yfinance_api.py +0 -511
- cryptodatapy/transform/cc_onchain_data.csv +0 -118423
- cryptodatapy/transform/clean_onchain_data.ipynb +0 -4750
- cryptodatapy/transform/clean_perp_futures_ohlcv.ipynb +0 -2819
- cryptodatapy/transform/cmdty_data.ipynb +0 -402
- cryptodatapy/transform/credit_data.ipynb +0 -291
- cryptodatapy/transform/eqty_data.ipynb +0 -836
- cryptodatapy/transform/global_credit_data_daily.parquet +0 -0
- cryptodatapy/transform/rates_data.ipynb +0 -465
- cryptodatapy/transform/us_rates_daily.csv +0 -227752
- {cryptodatapy-0.2.8.dist-info → cryptodatapy-0.2.9.dist-info}/LICENSE +0 -0
- {cryptodatapy-0.2.8.dist-info → cryptodatapy-0.2.9.dist-info}/WHEEL +0 -0
@@ -1,11 +1,11 @@
|
|
1
1
|
import logging
|
2
2
|
from typing import Any, Dict, List, Optional, Union
|
3
|
-
|
4
3
|
import pandas as pd
|
5
4
|
import asyncio
|
5
|
+
from time import sleep
|
6
6
|
import ccxt
|
7
7
|
import ccxt.async_support as ccxt_async
|
8
|
-
from tqdm.asyncio import tqdm
|
8
|
+
from tqdm.asyncio import tqdm
|
9
9
|
|
10
10
|
from cryptodatapy.extract.datarequest import DataRequest
|
11
11
|
from cryptodatapy.extract.libraries.library import Library
|
@@ -21,6 +21,7 @@ class CCXT(Library):
|
|
21
21
|
"""
|
22
22
|
Retrieves data from CCXT API.
|
23
23
|
"""
|
24
|
+
|
24
25
|
def __init__(
|
25
26
|
self,
|
26
27
|
categories: Union[str, List[str]] = "crypto",
|
@@ -70,24 +71,12 @@ class CCXT(Library):
|
|
70
71
|
rate_limit: Any, optional, Default None
|
71
72
|
Number of API calls made and left, by time frequency.
|
72
73
|
"""
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
exchanges,
|
77
|
-
indexes,
|
78
|
-
assets,
|
79
|
-
markets,
|
80
|
-
market_types,
|
81
|
-
fields,
|
82
|
-
frequencies,
|
83
|
-
base_url,
|
84
|
-
api_key,
|
85
|
-
max_obs_per_call,
|
86
|
-
rate_limit,
|
74
|
+
super().__init__(
|
75
|
+
categories, exchanges, indexes, assets, markets, market_types,
|
76
|
+
fields, frequencies, base_url, api_key, max_obs_per_call, rate_limit
|
87
77
|
)
|
88
|
-
|
89
78
|
self.exchange = None
|
90
|
-
self.exchange_async = None
|
79
|
+
# self.exchange_async = None
|
91
80
|
self.data_req = None
|
92
81
|
self.data = pd.DataFrame()
|
93
82
|
|
@@ -322,14 +311,14 @@ class CCXT(Library):
|
|
322
311
|
if self.rate_limit is None:
|
323
312
|
self.rate_limit = self.exchange.rateLimit
|
324
313
|
|
325
|
-
async def
|
326
|
-
|
327
|
-
|
328
|
-
|
329
|
-
|
330
|
-
|
331
|
-
|
332
|
-
|
314
|
+
async def _fetch_ohlcv_async(self,
|
315
|
+
ticker: str,
|
316
|
+
freq: str,
|
317
|
+
start_date: str,
|
318
|
+
end_date: str,
|
319
|
+
exch: str,
|
320
|
+
trials: int = 3
|
321
|
+
) -> List:
|
333
322
|
"""
|
334
323
|
Fetches OHLCV data for a specific ticker.
|
335
324
|
|
@@ -357,17 +346,16 @@ class CCXT(Library):
|
|
357
346
|
data = []
|
358
347
|
|
359
348
|
# inst exch
|
360
|
-
|
361
|
-
self.exchange_async = getattr(ccxt_async, exch)()
|
349
|
+
self.exchange = getattr(ccxt_async, exch)()
|
362
350
|
|
363
351
|
# fetch data
|
364
|
-
if self.
|
352
|
+
if self.exchange.has['fetchOHLCV']:
|
365
353
|
|
366
354
|
# while loop to fetch all data
|
367
355
|
while start_date < end_date and attempts < trials:
|
368
356
|
|
369
357
|
try:
|
370
|
-
data_resp = await
|
358
|
+
data_resp = await self.exchange.fetch_ohlcv(
|
371
359
|
ticker,
|
372
360
|
freq,
|
373
361
|
since=start_date,
|
@@ -375,48 +363,126 @@ class CCXT(Library):
|
|
375
363
|
params={'until': end_date}
|
376
364
|
)
|
377
365
|
|
366
|
+
# add data to list
|
367
|
+
if data_resp:
|
368
|
+
start_date = data_resp[-1][0] + 1
|
369
|
+
data.extend(data_resp)
|
370
|
+
else:
|
371
|
+
break
|
372
|
+
|
378
373
|
except Exception as e:
|
379
374
|
logging.warning(
|
380
|
-
f"Failed to get OHLCV data from {self.
|
375
|
+
f"Failed to get OHLCV data from {self.exchange.id} for {ticker} "
|
376
|
+
f"on attempt #{attempts + 1}: {e}."
|
381
377
|
)
|
382
|
-
logging.warning(e)
|
383
378
|
attempts += 1
|
384
|
-
if attempts
|
379
|
+
if attempts >= trials:
|
385
380
|
logging.warning(
|
386
|
-
f"Failed to get OHLCV data from {self.
|
381
|
+
f"Failed to get OHLCV data from {self.exchange.id} "
|
387
382
|
f"for {ticker} after {trials} attempts."
|
388
383
|
)
|
389
|
-
|
384
|
+
break
|
390
385
|
|
391
|
-
|
392
|
-
|
386
|
+
finally:
|
387
|
+
await asyncio.sleep(self.exchange.rateLimit / 1000)
|
393
388
|
|
394
|
-
|
395
|
-
|
396
|
-
|
397
|
-
|
389
|
+
await self.exchange.close()
|
390
|
+
return data
|
391
|
+
|
392
|
+
else:
|
393
|
+
logging.warning(f"OHLCV data is not available for {self.exchange.id}.")
|
394
|
+
return None
|
395
|
+
|
396
|
+
def _fetch_ohlcv(self,
|
397
|
+
ticker: str,
|
398
|
+
freq: str,
|
399
|
+
start_date: str,
|
400
|
+
end_date: str,
|
401
|
+
exch: str,
|
402
|
+
trials: int = 3
|
403
|
+
) -> List:
|
404
|
+
"""
|
405
|
+
Fetches OHLCV data for a specific ticker.
|
406
|
+
|
407
|
+
Parameters
|
408
|
+
----------
|
409
|
+
ticker: str
|
410
|
+
Ticker symbol.
|
411
|
+
freq: str
|
412
|
+
Frequency of data, e.g. '1m', '5m', '1h', '1d'.
|
413
|
+
start_date: str
|
414
|
+
Start date in integers in milliseconds since Unix epoch.
|
415
|
+
end_date: str
|
416
|
+
End date in integers in milliseconds since Unix epoch.
|
417
|
+
exch: str
|
418
|
+
Name of exchange.
|
419
|
+
trials: int, default 3
|
420
|
+
Number of attempts to fetch data.
|
421
|
+
|
422
|
+
Returns
|
423
|
+
-------
|
424
|
+
data: list
|
425
|
+
List of timestamps with OHLCV data.
|
426
|
+
"""
|
427
|
+
attempts = 0
|
428
|
+
data = []
|
429
|
+
|
430
|
+
# inst exch
|
431
|
+
self.exchange = getattr(ccxt, exch)()
|
432
|
+
|
433
|
+
# fetch data
|
434
|
+
if self.exchange.has['fetchOHLCV']:
|
435
|
+
|
436
|
+
# while loop to fetch all data
|
437
|
+
while start_date < end_date and attempts < trials:
|
438
|
+
|
439
|
+
try:
|
440
|
+
data_resp = self.exchange.fetch_ohlcv(
|
441
|
+
ticker,
|
442
|
+
freq,
|
443
|
+
since=start_date,
|
444
|
+
limit=self.max_obs_per_call,
|
445
|
+
params={'until': end_date}
|
446
|
+
)
|
447
|
+
|
448
|
+
# add data to list
|
449
|
+
if data_resp:
|
398
450
|
start_date = data_resp[-1][0] + 1
|
399
451
|
data.extend(data_resp)
|
400
|
-
await asyncio.sleep(self.exchange_async.rateLimit / 1000)
|
401
|
-
|
402
452
|
else:
|
403
453
|
break
|
404
454
|
|
455
|
+
except Exception as e:
|
456
|
+
logging.warning(
|
457
|
+
f"Failed to get OHLCV data from {self.exchange.id} for {ticker} "
|
458
|
+
f"on attempt #{attempts + 1}: {e}."
|
459
|
+
)
|
460
|
+
attempts += 1
|
461
|
+
if attempts >= trials:
|
462
|
+
logging.warning(
|
463
|
+
f"Failed to get OHLCV data from {self.exchange.id} "
|
464
|
+
f"for {ticker} after {trials} attempts."
|
465
|
+
)
|
466
|
+
break
|
467
|
+
|
468
|
+
finally:
|
469
|
+
sleep(self.exchange.rateLimit / 1000)
|
470
|
+
|
405
471
|
return data
|
406
472
|
|
407
473
|
else:
|
408
|
-
logging.warning(f"OHLCV data is not available for {self.
|
474
|
+
logging.warning(f"OHLCV data is not available for {self.exchange.id}.")
|
409
475
|
return None
|
410
476
|
|
411
|
-
async def
|
412
|
-
|
413
|
-
|
414
|
-
|
415
|
-
|
416
|
-
|
417
|
-
|
418
|
-
|
419
|
-
|
477
|
+
async def _fetch_all_ohlcv_async(self,
|
478
|
+
tickers,
|
479
|
+
freq: str,
|
480
|
+
start_date: str,
|
481
|
+
end_date: str,
|
482
|
+
exch: str,
|
483
|
+
trials: int = 3,
|
484
|
+
pause: int = 0.5
|
485
|
+
):
|
420
486
|
"""
|
421
487
|
Fetches OHLCV data for a list of tickers.
|
422
488
|
|
@@ -443,8 +509,7 @@ class CCXT(Library):
|
|
443
509
|
List of lists of timestamps and OHLCV data for each ticker.
|
444
510
|
"""
|
445
511
|
# inst exch
|
446
|
-
|
447
|
-
self.exchange_async = getattr(ccxt_async, exch)()
|
512
|
+
self.exchange = getattr(ccxt_async, exch)()
|
448
513
|
|
449
514
|
data = []
|
450
515
|
|
@@ -453,22 +518,68 @@ class CCXT(Library):
|
|
453
518
|
|
454
519
|
# loop through tickers
|
455
520
|
for ticker in tickers:
|
456
|
-
data_resp = await self.
|
521
|
+
data_resp = await self._fetch_ohlcv_async(ticker, freq, start_date, end_date, trials=trials, exch=exch)
|
522
|
+
await asyncio.sleep(pause)
|
457
523
|
data.append(data_resp)
|
458
524
|
pbar.update(1)
|
459
|
-
await asyncio.sleep(pause) # pause between ticker requests to respect the rate limit
|
460
525
|
|
461
|
-
await self.
|
526
|
+
await self.exchange.close()
|
527
|
+
|
528
|
+
return data
|
529
|
+
|
530
|
+
def _fetch_all_ohlcv(self,
|
531
|
+
tickers,
|
532
|
+
freq: str,
|
533
|
+
start_date: str,
|
534
|
+
end_date: str,
|
535
|
+
exch: str,
|
536
|
+
trials: int = 3,
|
537
|
+
pause: int = 0.5
|
538
|
+
):
|
539
|
+
"""
|
540
|
+
Fetches OHLCV data for a list of tickers.
|
541
|
+
|
542
|
+
Parameters
|
543
|
+
----------
|
544
|
+
tickers: list
|
545
|
+
List of ticker symbols.
|
546
|
+
freq: str
|
547
|
+
Frequency of data, e.g. '1m', '5m', '1h', '1d'.
|
548
|
+
start_date: str
|
549
|
+
Start date in integers in milliseconds since Unix epoch.
|
550
|
+
end_date: str
|
551
|
+
End date in integers in milliseconds since Unix epoch.
|
552
|
+
exch: str
|
553
|
+
Name of exchange.
|
554
|
+
trials: int, default 3
|
555
|
+
Number of attempts to fetch data.
|
556
|
+
pause: int, default 0.5
|
557
|
+
Pause in seconds to respect the rate limit.
|
558
|
+
"""
|
559
|
+
# inst exch
|
560
|
+
self.exchange = getattr(ccxt, exch)()
|
561
|
+
|
562
|
+
data = []
|
563
|
+
|
564
|
+
# create progress bar
|
565
|
+
pbar = tqdm(total=len(tickers), desc="Fetching OHLCV data", unit="ticker")
|
566
|
+
|
567
|
+
# loop through tickers
|
568
|
+
for ticker in tickers:
|
569
|
+
data_resp = self._fetch_ohlcv(ticker, freq, start_date, end_date, trials=trials, exch=exch)
|
570
|
+
sleep(pause)
|
571
|
+
data.append(data_resp)
|
572
|
+
pbar.update(1)
|
462
573
|
|
463
574
|
return data
|
464
575
|
|
465
|
-
async def
|
466
|
-
|
467
|
-
|
468
|
-
|
469
|
-
|
470
|
-
|
471
|
-
|
576
|
+
async def _fetch_funding_rates_async(self,
|
577
|
+
ticker: str,
|
578
|
+
start_date: str,
|
579
|
+
end_date: str,
|
580
|
+
exch: str,
|
581
|
+
trials: int = 3
|
582
|
+
) -> List:
|
472
583
|
"""
|
473
584
|
Fetches funding rates data for a specific ticker.
|
474
585
|
|
@@ -492,64 +603,135 @@ class CCXT(Library):
|
|
492
603
|
data = []
|
493
604
|
|
494
605
|
# inst exch
|
495
|
-
|
496
|
-
self.exchange_async = getattr(ccxt_async, exch)()
|
606
|
+
self.exchange = getattr(ccxt_async, exch)()
|
497
607
|
|
498
608
|
# fetch data
|
499
|
-
if self.
|
609
|
+
if self.exchange.has['fetchFundingRateHistory']:
|
500
610
|
|
501
611
|
# while loop to get all data
|
502
612
|
while start_date < end_date and attempts < trials:
|
503
613
|
|
504
614
|
try:
|
505
|
-
data_resp = await
|
615
|
+
data_resp = await self.exchange.fetch_funding_rate_history(
|
506
616
|
ticker,
|
507
617
|
since=start_date,
|
508
618
|
limit=self.max_obs_per_call,
|
509
619
|
params={'until': end_date}
|
510
620
|
)
|
511
621
|
|
622
|
+
# add data to list
|
623
|
+
if data_resp:
|
624
|
+
start_date = data_resp[-1]['timestamp'] + 1
|
625
|
+
data.extend(data_resp)
|
626
|
+
else:
|
627
|
+
break
|
628
|
+
|
512
629
|
except Exception as e:
|
513
630
|
logging.warning(
|
514
|
-
f"Failed to get funding rates from {self.
|
515
|
-
f"
|
631
|
+
f"Failed to get funding rates from {self.exchange.id} for {ticker} "
|
632
|
+
f"on attempt #{attempts + 1}: {e}."
|
516
633
|
)
|
517
|
-
logging.warning(e)
|
518
634
|
attempts += 1
|
519
|
-
if attempts
|
635
|
+
if attempts >= trials:
|
520
636
|
logging.warning(
|
521
|
-
f"Failed to get funding rates from {self.
|
637
|
+
f"Failed to get funding rates from {self.exchange.id} "
|
522
638
|
f"for {ticker} after {trials} attempts."
|
523
639
|
)
|
524
|
-
|
640
|
+
break
|
525
641
|
|
526
|
-
|
527
|
-
|
642
|
+
finally:
|
643
|
+
await asyncio.sleep(self.exchange.rateLimit / 1000)
|
528
644
|
|
529
|
-
|
530
|
-
|
531
|
-
|
532
|
-
|
645
|
+
await self.exchange.close()
|
646
|
+
return data
|
647
|
+
|
648
|
+
else:
|
649
|
+
logging.warning(f"Funding rates are not available for {self.exchange.id}.")
|
650
|
+
return None
|
651
|
+
|
652
|
+
def _fetch_funding_rates(self,
|
653
|
+
ticker: str,
|
654
|
+
start_date: str,
|
655
|
+
end_date: str,
|
656
|
+
exch: str,
|
657
|
+
trials: int = 3
|
658
|
+
) -> List:
|
659
|
+
"""
|
660
|
+
Fetches funding rates data for a specific ticker.
|
661
|
+
|
662
|
+
Parameters
|
663
|
+
----------
|
664
|
+
ticker: str
|
665
|
+
Ticker symbol.
|
666
|
+
start_date: str
|
667
|
+
Start date in integers in milliseconds since Unix epoch.
|
668
|
+
end_date: str
|
669
|
+
End date in integers in milliseconds since Unix epoch.
|
670
|
+
trials: int, default 3
|
671
|
+
Number of attempts to fetch data.
|
672
|
+
|
673
|
+
Returns
|
674
|
+
-------
|
675
|
+
data: list
|
676
|
+
List of dictionaries with timestamps and funding rates data.
|
677
|
+
"""
|
678
|
+
attempts = 0
|
679
|
+
data = []
|
680
|
+
|
681
|
+
# inst exch
|
682
|
+
self.exchange = getattr(ccxt, exch)()
|
683
|
+
|
684
|
+
# fetch data
|
685
|
+
if self.exchange.has['fetchFundingRateHistory']:
|
686
|
+
|
687
|
+
# while loop to get all data
|
688
|
+
while start_date < end_date and attempts < trials:
|
689
|
+
|
690
|
+
try:
|
691
|
+
data_resp = self.exchange.fetch_funding_rate_history(
|
692
|
+
ticker,
|
693
|
+
since=start_date,
|
694
|
+
limit=self.max_obs_per_call,
|
695
|
+
params={'until': end_date}
|
696
|
+
)
|
697
|
+
|
698
|
+
# add data to list
|
699
|
+
if data_resp:
|
533
700
|
start_date = data_resp[-1]['timestamp'] + 1
|
534
701
|
data.extend(data_resp)
|
535
|
-
await asyncio.sleep(self.exchange_async.rateLimit / 1000)
|
536
702
|
else:
|
537
703
|
break
|
538
704
|
|
705
|
+
except Exception as e:
|
706
|
+
logging.warning(
|
707
|
+
f"Failed to get funding rates from {self.exchange.id} for {ticker} "
|
708
|
+
f"on attempt #{attempts + 1}: {e}."
|
709
|
+
)
|
710
|
+
attempts += 1
|
711
|
+
if attempts >= trials:
|
712
|
+
logging.warning(
|
713
|
+
f"Failed to get funding rates from {self.exchange.id} "
|
714
|
+
f"for {ticker} after {trials} attempts."
|
715
|
+
)
|
716
|
+
break
|
717
|
+
|
718
|
+
finally:
|
719
|
+
sleep(self.exchange.rateLimit / 1000)
|
720
|
+
|
539
721
|
return data
|
540
722
|
|
541
723
|
else:
|
542
|
-
logging.warning(f"Funding rates are not available for {self.
|
724
|
+
logging.warning(f"Funding rates are not available for {self.exchange.id}.")
|
543
725
|
return None
|
544
726
|
|
545
|
-
async def
|
546
|
-
|
547
|
-
|
548
|
-
|
549
|
-
|
550
|
-
|
551
|
-
|
552
|
-
|
727
|
+
async def _fetch_all_funding_rates_async(self,
|
728
|
+
tickers,
|
729
|
+
start_date: str,
|
730
|
+
end_date: str,
|
731
|
+
exch: str,
|
732
|
+
trials: int = 3,
|
733
|
+
pause: int = 0.5
|
734
|
+
):
|
553
735
|
"""
|
554
736
|
Fetches funding rates data for a list of tickers.
|
555
737
|
|
@@ -574,8 +756,7 @@ class CCXT(Library):
|
|
574
756
|
List of lists of dictionaries with timestamps and funding rates data for each ticker.
|
575
757
|
"""
|
576
758
|
# inst exch
|
577
|
-
|
578
|
-
self.exchange_async = getattr(ccxt_async, exch)()
|
759
|
+
self.exchange = getattr(ccxt_async, exch)()
|
579
760
|
|
580
761
|
data = []
|
581
762
|
|
@@ -584,23 +765,72 @@ class CCXT(Library):
|
|
584
765
|
|
585
766
|
# loop through tickers
|
586
767
|
for ticker in tickers:
|
587
|
-
data_resp = await self.
|
768
|
+
data_resp = await self._fetch_funding_rates_async(ticker, start_date, end_date, trials=trials, exch=exch)
|
588
769
|
data.append(data_resp)
|
589
770
|
pbar.update(1)
|
590
|
-
await asyncio.sleep(pause)
|
771
|
+
await asyncio.sleep(pause)
|
772
|
+
|
773
|
+
await self.exchange.close()
|
774
|
+
|
775
|
+
return data
|
776
|
+
|
777
|
+
def _fetch_all_funding_rates(self,
|
778
|
+
tickers,
|
779
|
+
start_date: str,
|
780
|
+
end_date: str,
|
781
|
+
exch: str,
|
782
|
+
trials: int = 3,
|
783
|
+
pause: int = 0.5
|
784
|
+
):
|
785
|
+
"""
|
786
|
+
Fetches funding rates data for a list of tickers.
|
787
|
+
|
788
|
+
Parameters
|
789
|
+
----------
|
790
|
+
tickers: list
|
791
|
+
List of ticker symbols.
|
792
|
+
start_date: str
|
793
|
+
Start date in integers in milliseconds since Unix epoch.
|
794
|
+
end_date: str
|
795
|
+
End date in integers in milliseconds since Unix epoch.
|
796
|
+
exch: str
|
797
|
+
Name of exchange.
|
798
|
+
trials: int, default 3
|
799
|
+
Number of attempts to fetch data.
|
800
|
+
pause: int, default 0.5
|
801
|
+
Pause in seconds to respect the rate limit.
|
802
|
+
|
803
|
+
Returns
|
804
|
+
-------
|
805
|
+
data: list
|
806
|
+
List of lists of dictionaries with timestamps and funding rates data for each ticker.
|
807
|
+
"""
|
808
|
+
|
809
|
+
# inst exch
|
810
|
+
self.exchange = getattr(ccxt, exch)()
|
811
|
+
|
812
|
+
data = []
|
591
813
|
|
592
|
-
|
814
|
+
# create progress bar
|
815
|
+
pbar = tqdm(total=len(tickers), desc="Fetching funding rates", unit="ticker")
|
816
|
+
|
817
|
+
# loop through tickers
|
818
|
+
for ticker in tickers:
|
819
|
+
data_resp = self._fetch_funding_rates(ticker, start_date, end_date, trials=trials, exch=exch)
|
820
|
+
data.append(data_resp)
|
821
|
+
pbar.update(1)
|
822
|
+
sleep(pause)
|
593
823
|
|
594
824
|
return data
|
595
825
|
|
596
|
-
async def
|
597
|
-
|
598
|
-
|
599
|
-
|
600
|
-
|
601
|
-
|
602
|
-
|
603
|
-
|
826
|
+
async def _fetch_open_interest_async(self,
|
827
|
+
ticker: str,
|
828
|
+
freq: str,
|
829
|
+
start_date: str,
|
830
|
+
end_date: str,
|
831
|
+
exch: str,
|
832
|
+
trials: int = 3
|
833
|
+
) -> List:
|
604
834
|
"""
|
605
835
|
Fetches open interest data for a specific ticker.
|
606
836
|
|
@@ -629,17 +859,16 @@ class CCXT(Library):
|
|
629
859
|
data = []
|
630
860
|
|
631
861
|
# inst exch
|
632
|
-
|
633
|
-
self.exchange_async = getattr(ccxt_async, exch)()
|
862
|
+
self.exchange = getattr(ccxt_async, exch)()
|
634
863
|
|
635
864
|
# fetch data
|
636
|
-
if self.
|
865
|
+
if self.exchange.has['fetchOpenInterestHistory']:
|
637
866
|
|
638
867
|
# while loop to get all data
|
639
868
|
while start_date < end_date and attempts < trials:
|
640
869
|
|
641
870
|
try:
|
642
|
-
data_resp = await
|
871
|
+
data_resp = await self.exchange.fetch_open_interest_history(
|
643
872
|
ticker,
|
644
873
|
freq,
|
645
874
|
since=start_date,
|
@@ -647,49 +876,127 @@ class CCXT(Library):
|
|
647
876
|
params={'until': end_date}
|
648
877
|
)
|
649
878
|
|
879
|
+
# add data to list
|
880
|
+
if data_resp:
|
881
|
+
start_date = data_resp[-1]['timestamp'] + 1
|
882
|
+
data.extend(data_resp)
|
883
|
+
else:
|
884
|
+
break
|
885
|
+
|
650
886
|
except Exception as e:
|
651
887
|
logging.warning(
|
652
|
-
f"Failed to get open interest from {self.
|
653
|
-
f"
|
888
|
+
f"Failed to get open interest from {self.exchange.id} for {ticker} "
|
889
|
+
f"on attempt #{attempts + 1}: {e}."
|
654
890
|
)
|
655
|
-
logging.warning(e)
|
656
891
|
attempts += 1
|
657
|
-
if attempts
|
892
|
+
if attempts >= trials:
|
658
893
|
logging.warning(
|
659
|
-
f"Failed to get open interest from {self.
|
894
|
+
f"Failed to get open interest from {self.exchange.id} "
|
660
895
|
f"for {ticker} after {trials} attempts."
|
661
896
|
)
|
662
|
-
|
897
|
+
break
|
663
898
|
|
664
|
-
|
665
|
-
|
899
|
+
finally:
|
900
|
+
await asyncio.sleep(self.exchange.rateLimit / 1000)
|
666
901
|
|
667
|
-
|
668
|
-
|
669
|
-
|
670
|
-
|
902
|
+
await self.exchange.close()
|
903
|
+
return data
|
904
|
+
|
905
|
+
else:
|
906
|
+
logging.warning(f"Open interest is not available for {self.exchange.id}.")
|
907
|
+
return None
|
908
|
+
|
909
|
+
def _fetch_open_interest(self,
|
910
|
+
ticker: str,
|
911
|
+
freq: str,
|
912
|
+
start_date: str,
|
913
|
+
end_date: str,
|
914
|
+
exch: str,
|
915
|
+
trials: int = 3
|
916
|
+
) -> List:
|
917
|
+
"""
|
918
|
+
Fetches open interest data for a specific ticker.
|
919
|
+
|
920
|
+
Parameters
|
921
|
+
----------
|
922
|
+
ticker: str
|
923
|
+
Ticker symbol.
|
924
|
+
freq: str
|
925
|
+
Frequency of data, e.g. '1m', '5m', '1h', '1d'.
|
926
|
+
start_date: str
|
927
|
+
Start date in integers in milliseconds since Unix epoch.
|
928
|
+
end_date: str
|
929
|
+
End date in integers in milliseconds since Unix epoch.
|
930
|
+
exch: str
|
931
|
+
Name of exchange.
|
932
|
+
trials: int, default 3
|
933
|
+
Number of attempts to fetch data.
|
934
|
+
|
935
|
+
Returns
|
936
|
+
-------
|
937
|
+
data: list
|
938
|
+
List of dictionaries with timestamps and open interest data.
|
939
|
+
"""
|
940
|
+
# number of attempts
|
941
|
+
attempts = 0
|
942
|
+
data = []
|
943
|
+
|
944
|
+
# inst exch
|
945
|
+
self.exchange = getattr(ccxt, exch)()
|
946
|
+
|
947
|
+
# fetch data
|
948
|
+
if self.exchange.has['fetchOpenInterestHistory']:
|
949
|
+
|
950
|
+
# while loop to get all data
|
951
|
+
while start_date < end_date and attempts < trials:
|
952
|
+
|
953
|
+
try:
|
954
|
+
data_resp = self.exchange.fetch_open_interest_history(
|
955
|
+
ticker,
|
956
|
+
freq,
|
957
|
+
since=start_date,
|
958
|
+
limit=500,
|
959
|
+
params={'until': end_date}
|
960
|
+
)
|
961
|
+
|
962
|
+
# add data to list
|
963
|
+
if data_resp:
|
671
964
|
start_date = data_resp[-1]['timestamp'] + 1
|
672
965
|
data.extend(data_resp)
|
673
|
-
await asyncio.sleep(self.exchange_async.rateLimit / 1000)
|
674
966
|
else:
|
675
967
|
break
|
676
968
|
|
969
|
+
except Exception as e:
|
970
|
+
logging.warning(
|
971
|
+
f"Failed to get open interest from {self.exchange.id} for {ticker} "
|
972
|
+
f"on attempt #{attempts + 1}: {e}."
|
973
|
+
)
|
974
|
+
attempts += 1
|
975
|
+
if attempts >= trials:
|
976
|
+
logging.warning(
|
977
|
+
f"Failed to get open interest from {self.exchange.id} "
|
978
|
+
f"for {ticker} after {trials} attempts."
|
979
|
+
)
|
980
|
+
break
|
981
|
+
|
982
|
+
finally:
|
983
|
+
sleep(self.exchange.rateLimit / 1000)
|
984
|
+
|
677
985
|
return data
|
678
986
|
|
679
987
|
else:
|
680
|
-
logging.warning(f"Open interest is not available for {self.
|
988
|
+
logging.warning(f"Open interest is not available for {self.exchange.id}.")
|
681
989
|
return None
|
682
990
|
|
683
|
-
async def
|
684
|
-
|
685
|
-
|
686
|
-
|
687
|
-
|
688
|
-
|
689
|
-
|
690
|
-
|
691
|
-
|
692
|
-
|
991
|
+
async def _fetch_all_open_interest_async(self,
|
992
|
+
tickers,
|
993
|
+
freq: str,
|
994
|
+
start_date: str,
|
995
|
+
end_date: str,
|
996
|
+
exch: str,
|
997
|
+
trials: int = 3,
|
998
|
+
pause: int = 0.5
|
999
|
+
):
|
693
1000
|
"""
|
694
1001
|
Fetches open interest data for a list of tickers.
|
695
1002
|
|
@@ -716,8 +1023,7 @@ class CCXT(Library):
|
|
716
1023
|
List of lists of dictionaries with timestamps and open interest data for each ticker.
|
717
1024
|
"""
|
718
1025
|
# inst exch
|
719
|
-
|
720
|
-
self.exchange_async = getattr(ccxt_async, exch)()
|
1026
|
+
self.exchange = getattr(ccxt_async, exch)()
|
721
1027
|
|
722
1028
|
data = []
|
723
1029
|
|
@@ -726,12 +1032,64 @@ class CCXT(Library):
|
|
726
1032
|
|
727
1033
|
# loop through tickers
|
728
1034
|
for ticker in tickers:
|
729
|
-
data_resp = await self.
|
1035
|
+
data_resp = await self._fetch_open_interest_async(ticker, freq, start_date, end_date, trials=trials,
|
1036
|
+
exch=exch)
|
730
1037
|
data.append(data_resp)
|
731
1038
|
pbar.update(1)
|
732
|
-
await asyncio.sleep(pause)
|
1039
|
+
await asyncio.sleep(pause)
|
733
1040
|
|
734
|
-
await self.
|
1041
|
+
await self.exchange.close()
|
1042
|
+
|
1043
|
+
return data
|
1044
|
+
|
1045
|
+
def _fetch_all_open_interest(self,
|
1046
|
+
tickers,
|
1047
|
+
freq: str,
|
1048
|
+
start_date: str,
|
1049
|
+
end_date: str,
|
1050
|
+
exch: str,
|
1051
|
+
trials: int = 3,
|
1052
|
+
pause: int = 0.5
|
1053
|
+
):
|
1054
|
+
"""
|
1055
|
+
Fetches open interest data for a list of tickers.
|
1056
|
+
|
1057
|
+
Parameters
|
1058
|
+
----------
|
1059
|
+
tickers: list
|
1060
|
+
List of ticker symbols.
|
1061
|
+
freq: str
|
1062
|
+
Frequency of data, e.g. '1m', '5m', '1h', '1d'.
|
1063
|
+
start_date: str
|
1064
|
+
Start date in integers in milliseconds since Unix epoch.
|
1065
|
+
end_date: str
|
1066
|
+
End date in integers in milliseconds since Unix epoch.
|
1067
|
+
exch: str
|
1068
|
+
Name of exchange.
|
1069
|
+
trials: int, default 3
|
1070
|
+
Number of attempts to fetch data.
|
1071
|
+
pause: int, default 0.5
|
1072
|
+
Pause in seconds to respect the rate limit.
|
1073
|
+
|
1074
|
+
Returns
|
1075
|
+
-------
|
1076
|
+
data: list
|
1077
|
+
List of lists of dictionaries with timestamps and open interest data for each ticker.
|
1078
|
+
"""
|
1079
|
+
# inst exch
|
1080
|
+
self.exchange = getattr(ccxt, exch)()
|
1081
|
+
|
1082
|
+
data = []
|
1083
|
+
|
1084
|
+
# create progress bar
|
1085
|
+
pbar = tqdm(total=len(tickers), desc="Fetching open interest", unit="ticker")
|
1086
|
+
|
1087
|
+
# loop through tickers
|
1088
|
+
for ticker in tickers:
|
1089
|
+
data_resp = self._fetch_open_interest(ticker, freq, start_date, end_date, trials=trials, exch=exch)
|
1090
|
+
data.append(data_resp)
|
1091
|
+
pbar.update(1)
|
1092
|
+
sleep(pause)
|
735
1093
|
|
736
1094
|
return data
|
737
1095
|
|
@@ -754,6 +1112,10 @@ class CCXT(Library):
|
|
754
1112
|
# get metadata
|
755
1113
|
self.get_metadata(self.data_req.exch)
|
756
1114
|
|
1115
|
+
print(self.data_req.exch)
|
1116
|
+
|
1117
|
+
print(self.exchange)
|
1118
|
+
|
757
1119
|
# check markets
|
758
1120
|
if not any([market in self.markets for market in self.data_req.source_markets]):
|
759
1121
|
raise ValueError(
|
@@ -767,14 +1129,6 @@ class CCXT(Library):
|
|
767
1129
|
f"Use the '.frequencies' attribute to check available frequencies."
|
768
1130
|
)
|
769
1131
|
|
770
|
-
# check quote ccy
|
771
|
-
if self.data_req.quote_ccy is not None:
|
772
|
-
if self.data_req.quote_ccy not in self.assets:
|
773
|
-
raise ValueError(
|
774
|
-
f"{self.data_req.quote_ccy} is not supported. "
|
775
|
-
f"Use the '.assets' attribute to check supported currencies."
|
776
|
-
)
|
777
|
-
|
778
1132
|
# check mkt type
|
779
1133
|
if self.data_req.mkt_type not in self.market_types:
|
780
1134
|
raise ValueError(
|
@@ -853,7 +1207,41 @@ class CCXT(Library):
|
|
853
1207
|
|
854
1208
|
return WrangleData(self.data_req, data_resp).ccxt(data_type=data_type)
|
855
1209
|
|
856
|
-
async def
|
1210
|
+
async def fetch_tidy_ohlcv_async(self, data_req: DataRequest) -> pd.DataFrame:
|
1211
|
+
"""
|
1212
|
+
Gets entire OHLCV history and wrangles the data response into tidy data format.
|
1213
|
+
|
1214
|
+
Parameters
|
1215
|
+
----------
|
1216
|
+
data_req: DataRequest
|
1217
|
+
Parameters of data request in CryptoDataPy format.
|
1218
|
+
|
1219
|
+
Returns
|
1220
|
+
-------
|
1221
|
+
df: pd.DataFrame
|
1222
|
+
Dataframe with entire OHLCV data history retrieved and wrangled into tidy data format.
|
1223
|
+
"""
|
1224
|
+
# convert data request parameters to CCXT format
|
1225
|
+
if self.data_req is None:
|
1226
|
+
self.convert_params(data_req)
|
1227
|
+
|
1228
|
+
# get entire data history
|
1229
|
+
data_resp = await self._fetch_all_ohlcv_async(self.data_req.source_markets,
|
1230
|
+
self.data_req.source_freq,
|
1231
|
+
self.data_req.source_start_date,
|
1232
|
+
self.data_req.source_end_date,
|
1233
|
+
self.data_req.exch,
|
1234
|
+
trials=self.data_req.trials,
|
1235
|
+
pause=self.data_req.pause)
|
1236
|
+
|
1237
|
+
# wrangle df
|
1238
|
+
if any(data_resp):
|
1239
|
+
df = self.wrangle_data_resp(data_resp, data_type='ohlcv')
|
1240
|
+
return df
|
1241
|
+
else:
|
1242
|
+
logging.warning("Failed to get requested OHLCV data.")
|
1243
|
+
|
1244
|
+
def fetch_tidy_ohlcv(self, data_req: DataRequest) -> pd.DataFrame:
|
857
1245
|
"""
|
858
1246
|
Gets entire OHLCV history and wrangles the data response into tidy data format.
|
859
1247
|
|
@@ -872,13 +1260,13 @@ class CCXT(Library):
|
|
872
1260
|
self.convert_params(data_req)
|
873
1261
|
|
874
1262
|
# get entire data history
|
875
|
-
data_resp =
|
876
|
-
|
877
|
-
|
878
|
-
|
879
|
-
|
880
|
-
|
881
|
-
|
1263
|
+
data_resp = self._fetch_all_ohlcv(self.data_req.source_markets,
|
1264
|
+
self.data_req.source_freq,
|
1265
|
+
self.data_req.source_start_date,
|
1266
|
+
self.data_req.source_end_date,
|
1267
|
+
self.data_req.exch,
|
1268
|
+
trials=self.data_req.trials,
|
1269
|
+
pause=self.data_req.pause)
|
882
1270
|
|
883
1271
|
# wrangle df
|
884
1272
|
if any(data_resp):
|
@@ -887,7 +1275,7 @@ class CCXT(Library):
|
|
887
1275
|
else:
|
888
1276
|
logging.warning("Failed to get requested OHLCV data.")
|
889
1277
|
|
890
|
-
async def
|
1278
|
+
async def fetch_tidy_funding_rates_async(self, data_req: DataRequest) -> pd.DataFrame:
|
891
1279
|
"""
|
892
1280
|
Gets entire funding rates history and wrangles the data response into tidy data format.
|
893
1281
|
|
@@ -906,12 +1294,12 @@ class CCXT(Library):
|
|
906
1294
|
self.convert_params(data_req)
|
907
1295
|
|
908
1296
|
# get entire data history
|
909
|
-
data_resp = await self.
|
910
|
-
|
911
|
-
|
912
|
-
|
913
|
-
|
914
|
-
|
1297
|
+
data_resp = await self._fetch_all_funding_rates_async(self.data_req.source_markets,
|
1298
|
+
self.data_req.source_start_date,
|
1299
|
+
self.data_req.source_end_date,
|
1300
|
+
self.data_req.exch,
|
1301
|
+
trials=self.data_req.trials,
|
1302
|
+
pause=self.data_req.pause)
|
915
1303
|
|
916
1304
|
# wrangle df
|
917
1305
|
if any(data_resp):
|
@@ -920,7 +1308,40 @@ class CCXT(Library):
|
|
920
1308
|
else:
|
921
1309
|
logging.warning("Failed to get requested funding rates.")
|
922
1310
|
|
923
|
-
|
1311
|
+
def fetch_tidy_funding_rates(self, data_req: DataRequest) -> pd.DataFrame:
|
1312
|
+
"""
|
1313
|
+
Gets entire funding rates history and wrangles the data response into tidy data format.
|
1314
|
+
|
1315
|
+
Parameters
|
1316
|
+
----------
|
1317
|
+
data_req: DataRequest
|
1318
|
+
Parameters of data request in CryptoDataPy format.
|
1319
|
+
|
1320
|
+
Returns
|
1321
|
+
-------
|
1322
|
+
df: pd.DataFrame
|
1323
|
+
Dataframe with entire data history retrieved and wrangled into tidy data format.
|
1324
|
+
"""
|
1325
|
+
# convert data request parameters to CCXT format
|
1326
|
+
if self.data_req is None:
|
1327
|
+
self.convert_params(data_req)
|
1328
|
+
|
1329
|
+
# get entire data history
|
1330
|
+
data_resp = self._fetch_all_funding_rates(self.data_req.source_markets,
|
1331
|
+
self.data_req.source_start_date,
|
1332
|
+
self.data_req.source_end_date,
|
1333
|
+
self.data_req.exch,
|
1334
|
+
trials=self.data_req.trials,
|
1335
|
+
pause=self.data_req.pause)
|
1336
|
+
|
1337
|
+
# wrangle df
|
1338
|
+
if any(data_resp):
|
1339
|
+
df = self.wrangle_data_resp(data_resp, data_type='funding_rates')
|
1340
|
+
return df
|
1341
|
+
else:
|
1342
|
+
logging.warning("Failed to get requested funding rates.")
|
1343
|
+
|
1344
|
+
async def fetch_tidy_open_interest_async(self, data_req: DataRequest) -> pd.DataFrame:
|
924
1345
|
"""
|
925
1346
|
Gets entire open interest history and wrangles the data response into tidy data format.
|
926
1347
|
|
@@ -939,13 +1360,13 @@ class CCXT(Library):
|
|
939
1360
|
self.convert_params(data_req)
|
940
1361
|
|
941
1362
|
# get entire data history
|
942
|
-
data_resp = await self.
|
943
|
-
|
944
|
-
|
945
|
-
|
946
|
-
|
947
|
-
|
948
|
-
|
1363
|
+
data_resp = await self._fetch_all_open_interest_async(self.data_req.source_markets,
|
1364
|
+
self.data_req.source_freq,
|
1365
|
+
self.data_req.source_start_date,
|
1366
|
+
self.data_req.source_end_date,
|
1367
|
+
self.data_req.exch,
|
1368
|
+
trials=self.data_req.trials,
|
1369
|
+
pause=self.data_req.pause)
|
949
1370
|
|
950
1371
|
# wrangle df
|
951
1372
|
if any(data_resp):
|
@@ -954,7 +1375,81 @@ class CCXT(Library):
|
|
954
1375
|
else:
|
955
1376
|
logging.warning("Failed to get requested open interest.")
|
956
1377
|
|
957
|
-
|
1378
|
+
def fetch_tidy_open_interest(self, data_req: DataRequest) -> pd.DataFrame:
|
1379
|
+
"""
|
1380
|
+
Gets entire open interest history and wrangles the data response into tidy data format.
|
1381
|
+
|
1382
|
+
Parameters
|
1383
|
+
----------
|
1384
|
+
data_req: DataRequest
|
1385
|
+
Parameters of data request in CryptoDataPy format.
|
1386
|
+
|
1387
|
+
Returns
|
1388
|
+
-------
|
1389
|
+
df: pd.DataFrame
|
1390
|
+
Dataframe with entire data history retrieved and wrangled into tidy data format.
|
1391
|
+
"""
|
1392
|
+
# convert data request parameters to CCXT format
|
1393
|
+
if self.data_req is None:
|
1394
|
+
self.convert_params(data_req)
|
1395
|
+
|
1396
|
+
# get entire data history
|
1397
|
+
data_resp = self._fetch_all_open_interest(self.data_req.source_markets,
|
1398
|
+
self.data_req.source_freq,
|
1399
|
+
self.data_req.source_start_date,
|
1400
|
+
self.data_req.source_end_date,
|
1401
|
+
self.data_req.exch,
|
1402
|
+
trials=self.data_req.trials,
|
1403
|
+
pause=self.data_req.pause)
|
1404
|
+
|
1405
|
+
# wrangle df
|
1406
|
+
if any(data_resp):
|
1407
|
+
df = self.wrangle_data_resp(data_resp, data_type='open_interest')
|
1408
|
+
return df
|
1409
|
+
else:
|
1410
|
+
logging.warning("Failed to get requested open interest.")
|
1411
|
+
|
1412
|
+
async def get_data_async(self, data_req: DataRequest) -> pd.DataFrame:
|
1413
|
+
"""
|
1414
|
+
Get data specified by data request.
|
1415
|
+
|
1416
|
+
Parameters
|
1417
|
+
data_req: DataRequest
|
1418
|
+
Parameters of data request in CryptoDataPy format.
|
1419
|
+
|
1420
|
+
Returns
|
1421
|
+
-------
|
1422
|
+
df: pd.DataFrame - MultiIndex
|
1423
|
+
DataFrame with DatetimeIndex (level 0), ticker (level 1), and values for selected fields (cols).
|
1424
|
+
"""
|
1425
|
+
# get OHLCV
|
1426
|
+
if any([field in ["open", "high", "low", "close", "volume"] for field in data_req.fields]):
|
1427
|
+
df = await self.fetch_tidy_ohlcv_async(data_req)
|
1428
|
+
self.data = pd.concat([self.data, df])
|
1429
|
+
|
1430
|
+
# get funding rates
|
1431
|
+
if any([field == "funding_rate" for field in data_req.fields]):
|
1432
|
+
df = await self.fetch_tidy_funding_rates_async(data_req)
|
1433
|
+
self.data = pd.concat([self.data, df], axis=1)
|
1434
|
+
|
1435
|
+
# get open interest
|
1436
|
+
if any([field == "oi" for field in data_req.fields]):
|
1437
|
+
df = await self.fetch_tidy_open_interest_async(data_req)
|
1438
|
+
self.data = pd.concat([self.data, df], axis=1)
|
1439
|
+
|
1440
|
+
# check df
|
1441
|
+
if self.data.empty:
|
1442
|
+
raise Exception(
|
1443
|
+
"No data returned. Check data request parameters and try again."
|
1444
|
+
)
|
1445
|
+
|
1446
|
+
# filter df for desired fields and typecast
|
1447
|
+
fields = [field for field in data_req.fields if field in self.data.columns]
|
1448
|
+
self.data = self.data.loc[:, fields]
|
1449
|
+
|
1450
|
+
return self.data.sort_index()
|
1451
|
+
|
1452
|
+
def get_data(self, data_req: DataRequest) -> pd.DataFrame:
|
958
1453
|
"""
|
959
1454
|
Get data specified by data request.
|
960
1455
|
|
@@ -969,17 +1464,17 @@ class CCXT(Library):
|
|
969
1464
|
"""
|
970
1465
|
# get OHLCV
|
971
1466
|
if any([field in ["open", "high", "low", "close", "volume"] for field in data_req.fields]):
|
972
|
-
df =
|
1467
|
+
df = self.fetch_tidy_ohlcv(data_req)
|
973
1468
|
self.data = pd.concat([self.data, df])
|
974
1469
|
|
975
1470
|
# get funding rates
|
976
1471
|
if any([field == "funding_rate" for field in data_req.fields]):
|
977
|
-
df =
|
1472
|
+
df = self.fetch_tidy_funding_rates(data_req)
|
978
1473
|
self.data = pd.concat([self.data, df], axis=1)
|
979
1474
|
|
980
1475
|
# get open interest
|
981
1476
|
if any([field == "oi" for field in data_req.fields]):
|
982
|
-
df =
|
1477
|
+
df = self.fetch_tidy_open_interest(data_req)
|
983
1478
|
self.data = pd.concat([self.data, df], axis=1)
|
984
1479
|
|
985
1480
|
# check df
|