cryptodatapy 0.2.8__py3-none-any.whl → 0.2.9__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,11 +1,11 @@
1
1
  import logging
2
2
  from typing import Any, Dict, List, Optional, Union
3
-
4
3
  import pandas as pd
5
4
  import asyncio
5
+ from time import sleep
6
6
  import ccxt
7
7
  import ccxt.async_support as ccxt_async
8
- from tqdm.asyncio import tqdm # Progress bar for async
8
+ from tqdm.asyncio import tqdm
9
9
 
10
10
  from cryptodatapy.extract.datarequest import DataRequest
11
11
  from cryptodatapy.extract.libraries.library import Library
@@ -21,6 +21,7 @@ class CCXT(Library):
21
21
  """
22
22
  Retrieves data from CCXT API.
23
23
  """
24
+
24
25
  def __init__(
25
26
  self,
26
27
  categories: Union[str, List[str]] = "crypto",
@@ -70,24 +71,12 @@ class CCXT(Library):
70
71
  rate_limit: Any, optional, Default None
71
72
  Number of API calls made and left, by time frequency.
72
73
  """
73
- Library.__init__(
74
- self,
75
- categories,
76
- exchanges,
77
- indexes,
78
- assets,
79
- markets,
80
- market_types,
81
- fields,
82
- frequencies,
83
- base_url,
84
- api_key,
85
- max_obs_per_call,
86
- rate_limit,
74
+ super().__init__(
75
+ categories, exchanges, indexes, assets, markets, market_types,
76
+ fields, frequencies, base_url, api_key, max_obs_per_call, rate_limit
87
77
  )
88
-
89
78
  self.exchange = None
90
- self.exchange_async = None
79
+ # self.exchange_async = None
91
80
  self.data_req = None
92
81
  self.data = pd.DataFrame()
93
82
 
@@ -322,14 +311,14 @@ class CCXT(Library):
322
311
  if self.rate_limit is None:
323
312
  self.rate_limit = self.exchange.rateLimit
324
313
 
325
- async def _fetch_ohlcv(self,
326
- ticker: str,
327
- freq: str,
328
- start_date: str,
329
- end_date: str,
330
- exch: str,
331
- trials: int = 3
332
- ) -> List:
314
+ async def _fetch_ohlcv_async(self,
315
+ ticker: str,
316
+ freq: str,
317
+ start_date: str,
318
+ end_date: str,
319
+ exch: str,
320
+ trials: int = 3
321
+ ) -> List:
333
322
  """
334
323
  Fetches OHLCV data for a specific ticker.
335
324
 
@@ -357,17 +346,16 @@ class CCXT(Library):
357
346
  data = []
358
347
 
359
348
  # inst exch
360
- if self.exchange_async is None:
361
- self.exchange_async = getattr(ccxt_async, exch)()
349
+ self.exchange = getattr(ccxt_async, exch)()
362
350
 
363
351
  # fetch data
364
- if self.exchange_async.has['fetchOHLCV']:
352
+ if self.exchange.has['fetchOHLCV']:
365
353
 
366
354
  # while loop to fetch all data
367
355
  while start_date < end_date and attempts < trials:
368
356
 
369
357
  try:
370
- data_resp = await getattr(self.exchange_async, 'fetchOHLCV')(
358
+ data_resp = await self.exchange.fetch_ohlcv(
371
359
  ticker,
372
360
  freq,
373
361
  since=start_date,
@@ -375,48 +363,126 @@ class CCXT(Library):
375
363
  params={'until': end_date}
376
364
  )
377
365
 
366
+ # add data to list
367
+ if data_resp:
368
+ start_date = data_resp[-1][0] + 1
369
+ data.extend(data_resp)
370
+ else:
371
+ break
372
+
378
373
  except Exception as e:
379
374
  logging.warning(
380
- f"Failed to get OHLCV data from {self.exchange_async.id} for {ticker} on attempt #{attempts+1}."
375
+ f"Failed to get OHLCV data from {self.exchange.id} for {ticker} "
376
+ f"on attempt #{attempts + 1}: {e}."
381
377
  )
382
- logging.warning(e)
383
378
  attempts += 1
384
- if attempts == trials:
379
+ if attempts >= trials:
385
380
  logging.warning(
386
- f"Failed to get OHLCV data from {self.exchange_async.id} "
381
+ f"Failed to get OHLCV data from {self.exchange.id} "
387
382
  f"for {ticker} after {trials} attempts."
388
383
  )
389
- return data
384
+ break
390
385
 
391
- await asyncio.sleep(self.exchange_async.rateLimit / 1000)
392
- continue
386
+ finally:
387
+ await asyncio.sleep(self.exchange.rateLimit / 1000)
393
388
 
394
- else:
395
- # check if data resp is empty
396
- if len(data_resp):
397
- # next start date
389
+ await self.exchange.close()
390
+ return data
391
+
392
+ else:
393
+ logging.warning(f"OHLCV data is not available for {self.exchange.id}.")
394
+ return None
395
+
396
+ def _fetch_ohlcv(self,
397
+ ticker: str,
398
+ freq: str,
399
+ start_date: str,
400
+ end_date: str,
401
+ exch: str,
402
+ trials: int = 3
403
+ ) -> List:
404
+ """
405
+ Fetches OHLCV data for a specific ticker.
406
+
407
+ Parameters
408
+ ----------
409
+ ticker: str
410
+ Ticker symbol.
411
+ freq: str
412
+ Frequency of data, e.g. '1m', '5m', '1h', '1d'.
413
+ start_date: str
414
+ Start date in integers in milliseconds since Unix epoch.
415
+ end_date: str
416
+ End date in integers in milliseconds since Unix epoch.
417
+ exch: str
418
+ Name of exchange.
419
+ trials: int, default 3
420
+ Number of attempts to fetch data.
421
+
422
+ Returns
423
+ -------
424
+ data: list
425
+ List of timestamps with OHLCV data.
426
+ """
427
+ attempts = 0
428
+ data = []
429
+
430
+ # inst exch
431
+ self.exchange = getattr(ccxt, exch)()
432
+
433
+ # fetch data
434
+ if self.exchange.has['fetchOHLCV']:
435
+
436
+ # while loop to fetch all data
437
+ while start_date < end_date and attempts < trials:
438
+
439
+ try:
440
+ data_resp = self.exchange.fetch_ohlcv(
441
+ ticker,
442
+ freq,
443
+ since=start_date,
444
+ limit=self.max_obs_per_call,
445
+ params={'until': end_date}
446
+ )
447
+
448
+ # add data to list
449
+ if data_resp:
398
450
  start_date = data_resp[-1][0] + 1
399
451
  data.extend(data_resp)
400
- await asyncio.sleep(self.exchange_async.rateLimit / 1000)
401
-
402
452
  else:
403
453
  break
404
454
 
455
+ except Exception as e:
456
+ logging.warning(
457
+ f"Failed to get OHLCV data from {self.exchange.id} for {ticker} "
458
+ f"on attempt #{attempts + 1}: {e}."
459
+ )
460
+ attempts += 1
461
+ if attempts >= trials:
462
+ logging.warning(
463
+ f"Failed to get OHLCV data from {self.exchange.id} "
464
+ f"for {ticker} after {trials} attempts."
465
+ )
466
+ break
467
+
468
+ finally:
469
+ sleep(self.exchange.rateLimit / 1000)
470
+
405
471
  return data
406
472
 
407
473
  else:
408
- logging.warning(f"OHLCV data is not available for {self.exchange_async.id}.")
474
+ logging.warning(f"OHLCV data is not available for {self.exchange.id}.")
409
475
  return None
410
476
 
411
- async def fetch_all_ohlcv(self,
412
- tickers,
413
- freq: str,
414
- start_date: str,
415
- end_date: str,
416
- exch: str,
417
- trials: int = 3,
418
- pause: int = 0.5
419
- ):
477
+ async def _fetch_all_ohlcv_async(self,
478
+ tickers,
479
+ freq: str,
480
+ start_date: str,
481
+ end_date: str,
482
+ exch: str,
483
+ trials: int = 3,
484
+ pause: int = 0.5
485
+ ):
420
486
  """
421
487
  Fetches OHLCV data for a list of tickers.
422
488
 
@@ -443,8 +509,7 @@ class CCXT(Library):
443
509
  List of lists of timestamps and OHLCV data for each ticker.
444
510
  """
445
511
  # inst exch
446
- if self.exchange_async is None:
447
- self.exchange_async = getattr(ccxt_async, exch)()
512
+ self.exchange = getattr(ccxt_async, exch)()
448
513
 
449
514
  data = []
450
515
 
@@ -453,22 +518,68 @@ class CCXT(Library):
453
518
 
454
519
  # loop through tickers
455
520
  for ticker in tickers:
456
- data_resp = await self._fetch_ohlcv(ticker, freq, start_date, end_date, trials=trials, exch=exch)
521
+ data_resp = await self._fetch_ohlcv_async(ticker, freq, start_date, end_date, trials=trials, exch=exch)
522
+ await asyncio.sleep(pause)
457
523
  data.append(data_resp)
458
524
  pbar.update(1)
459
- await asyncio.sleep(pause) # pause between ticker requests to respect the rate limit
460
525
 
461
- await self.exchange_async.close()
526
+ await self.exchange.close()
527
+
528
+ return data
529
+
530
+ def _fetch_all_ohlcv(self,
531
+ tickers,
532
+ freq: str,
533
+ start_date: str,
534
+ end_date: str,
535
+ exch: str,
536
+ trials: int = 3,
537
+ pause: int = 0.5
538
+ ):
539
+ """
540
+ Fetches OHLCV data for a list of tickers.
541
+
542
+ Parameters
543
+ ----------
544
+ tickers: list
545
+ List of ticker symbols.
546
+ freq: str
547
+ Frequency of data, e.g. '1m', '5m', '1h', '1d'.
548
+ start_date: str
549
+ Start date in integers in milliseconds since Unix epoch.
550
+ end_date: str
551
+ End date in integers in milliseconds since Unix epoch.
552
+ exch: str
553
+ Name of exchange.
554
+ trials: int, default 3
555
+ Number of attempts to fetch data.
556
+ pause: int, default 0.5
557
+ Pause in seconds to respect the rate limit.
558
+ """
559
+ # inst exch
560
+ self.exchange = getattr(ccxt, exch)()
561
+
562
+ data = []
563
+
564
+ # create progress bar
565
+ pbar = tqdm(total=len(tickers), desc="Fetching OHLCV data", unit="ticker")
566
+
567
+ # loop through tickers
568
+ for ticker in tickers:
569
+ data_resp = self._fetch_ohlcv(ticker, freq, start_date, end_date, trials=trials, exch=exch)
570
+ sleep(pause)
571
+ data.append(data_resp)
572
+ pbar.update(1)
462
573
 
463
574
  return data
464
575
 
465
- async def _fetch_funding_rates(self,
466
- ticker: str,
467
- start_date: str,
468
- end_date: str,
469
- exch: str,
470
- trials: int = 3
471
- ) -> List:
576
+ async def _fetch_funding_rates_async(self,
577
+ ticker: str,
578
+ start_date: str,
579
+ end_date: str,
580
+ exch: str,
581
+ trials: int = 3
582
+ ) -> List:
472
583
  """
473
584
  Fetches funding rates data for a specific ticker.
474
585
 
@@ -492,64 +603,135 @@ class CCXT(Library):
492
603
  data = []
493
604
 
494
605
  # inst exch
495
- if self.exchange_async is None:
496
- self.exchange_async = getattr(ccxt_async, exch)()
606
+ self.exchange = getattr(ccxt_async, exch)()
497
607
 
498
608
  # fetch data
499
- if self.exchange_async.has['fetchFundingRateHistory']:
609
+ if self.exchange.has['fetchFundingRateHistory']:
500
610
 
501
611
  # while loop to get all data
502
612
  while start_date < end_date and attempts < trials:
503
613
 
504
614
  try:
505
- data_resp = await getattr(self.exchange_async, 'fetchFundingRateHistory')(
615
+ data_resp = await self.exchange.fetch_funding_rate_history(
506
616
  ticker,
507
617
  since=start_date,
508
618
  limit=self.max_obs_per_call,
509
619
  params={'until': end_date}
510
620
  )
511
621
 
622
+ # add data to list
623
+ if data_resp:
624
+ start_date = data_resp[-1]['timestamp'] + 1
625
+ data.extend(data_resp)
626
+ else:
627
+ break
628
+
512
629
  except Exception as e:
513
630
  logging.warning(
514
- f"Failed to get funding rates from {self.exchange_async.id} "
515
- f"for {ticker} on attempt #{attempts+1}."
631
+ f"Failed to get funding rates from {self.exchange.id} for {ticker} "
632
+ f"on attempt #{attempts + 1}: {e}."
516
633
  )
517
- logging.warning(e)
518
634
  attempts += 1
519
- if attempts == trials:
635
+ if attempts >= trials:
520
636
  logging.warning(
521
- f"Failed to get funding rates from {self.exchange_async.id} "
637
+ f"Failed to get funding rates from {self.exchange.id} "
522
638
  f"for {ticker} after {trials} attempts."
523
639
  )
524
- return data
640
+ break
525
641
 
526
- await asyncio.sleep(self.exchange_async.rateLimit / 1000)
527
- continue
642
+ finally:
643
+ await asyncio.sleep(self.exchange.rateLimit / 1000)
528
644
 
529
- else:
530
- # check if data resp is empty
531
- if len(data_resp):
532
- # next start date
645
+ await self.exchange.close()
646
+ return data
647
+
648
+ else:
649
+ logging.warning(f"Funding rates are not available for {self.exchange.id}.")
650
+ return None
651
+
652
+ def _fetch_funding_rates(self,
653
+ ticker: str,
654
+ start_date: str,
655
+ end_date: str,
656
+ exch: str,
657
+ trials: int = 3
658
+ ) -> List:
659
+ """
660
+ Fetches funding rates data for a specific ticker.
661
+
662
+ Parameters
663
+ ----------
664
+ ticker: str
665
+ Ticker symbol.
666
+ start_date: str
667
+ Start date in integers in milliseconds since Unix epoch.
668
+ end_date: str
669
+ End date in integers in milliseconds since Unix epoch.
670
+ trials: int, default 3
671
+ Number of attempts to fetch data.
672
+
673
+ Returns
674
+ -------
675
+ data: list
676
+ List of dictionaries with timestamps and funding rates data.
677
+ """
678
+ attempts = 0
679
+ data = []
680
+
681
+ # inst exch
682
+ self.exchange = getattr(ccxt, exch)()
683
+
684
+ # fetch data
685
+ if self.exchange.has['fetchFundingRateHistory']:
686
+
687
+ # while loop to get all data
688
+ while start_date < end_date and attempts < trials:
689
+
690
+ try:
691
+ data_resp = self.exchange.fetch_funding_rate_history(
692
+ ticker,
693
+ since=start_date,
694
+ limit=self.max_obs_per_call,
695
+ params={'until': end_date}
696
+ )
697
+
698
+ # add data to list
699
+ if data_resp:
533
700
  start_date = data_resp[-1]['timestamp'] + 1
534
701
  data.extend(data_resp)
535
- await asyncio.sleep(self.exchange_async.rateLimit / 1000)
536
702
  else:
537
703
  break
538
704
 
705
+ except Exception as e:
706
+ logging.warning(
707
+ f"Failed to get funding rates from {self.exchange.id} for {ticker} "
708
+ f"on attempt #{attempts + 1}: {e}."
709
+ )
710
+ attempts += 1
711
+ if attempts >= trials:
712
+ logging.warning(
713
+ f"Failed to get funding rates from {self.exchange.id} "
714
+ f"for {ticker} after {trials} attempts."
715
+ )
716
+ break
717
+
718
+ finally:
719
+ sleep(self.exchange.rateLimit / 1000)
720
+
539
721
  return data
540
722
 
541
723
  else:
542
- logging.warning(f"Funding rates are not available for {self.exchange_async.id}.")
724
+ logging.warning(f"Funding rates are not available for {self.exchange.id}.")
543
725
  return None
544
726
 
545
- async def fetch_all_funding_rates(self,
546
- tickers,
547
- start_date: str,
548
- end_date: str,
549
- exch: str,
550
- trials: int = 3,
551
- pause: int = 0.5
552
- ):
727
+ async def _fetch_all_funding_rates_async(self,
728
+ tickers,
729
+ start_date: str,
730
+ end_date: str,
731
+ exch: str,
732
+ trials: int = 3,
733
+ pause: int = 0.5
734
+ ):
553
735
  """
554
736
  Fetches funding rates data for a list of tickers.
555
737
 
@@ -574,8 +756,7 @@ class CCXT(Library):
574
756
  List of lists of dictionaries with timestamps and funding rates data for each ticker.
575
757
  """
576
758
  # inst exch
577
- if self.exchange_async is None:
578
- self.exchange_async = getattr(ccxt_async, exch)()
759
+ self.exchange = getattr(ccxt_async, exch)()
579
760
 
580
761
  data = []
581
762
 
@@ -584,23 +765,72 @@ class CCXT(Library):
584
765
 
585
766
  # loop through tickers
586
767
  for ticker in tickers:
587
- data_resp = await self._fetch_funding_rates(ticker, start_date, end_date, trials=trials, exch=exch)
768
+ data_resp = await self._fetch_funding_rates_async(ticker, start_date, end_date, trials=trials, exch=exch)
588
769
  data.append(data_resp)
589
770
  pbar.update(1)
590
- await asyncio.sleep(pause) # pause between ticker requests to respect the rate limit
771
+ await asyncio.sleep(pause)
772
+
773
+ await self.exchange.close()
774
+
775
+ return data
776
+
777
+ def _fetch_all_funding_rates(self,
778
+ tickers,
779
+ start_date: str,
780
+ end_date: str,
781
+ exch: str,
782
+ trials: int = 3,
783
+ pause: int = 0.5
784
+ ):
785
+ """
786
+ Fetches funding rates data for a list of tickers.
787
+
788
+ Parameters
789
+ ----------
790
+ tickers: list
791
+ List of ticker symbols.
792
+ start_date: str
793
+ Start date in integers in milliseconds since Unix epoch.
794
+ end_date: str
795
+ End date in integers in milliseconds since Unix epoch.
796
+ exch: str
797
+ Name of exchange.
798
+ trials: int, default 3
799
+ Number of attempts to fetch data.
800
+ pause: int, default 0.5
801
+ Pause in seconds to respect the rate limit.
802
+
803
+ Returns
804
+ -------
805
+ data: list
806
+ List of lists of dictionaries with timestamps and funding rates data for each ticker.
807
+ """
808
+
809
+ # inst exch
810
+ self.exchange = getattr(ccxt, exch)()
811
+
812
+ data = []
591
813
 
592
- await self.exchange_async.close()
814
+ # create progress bar
815
+ pbar = tqdm(total=len(tickers), desc="Fetching funding rates", unit="ticker")
816
+
817
+ # loop through tickers
818
+ for ticker in tickers:
819
+ data_resp = self._fetch_funding_rates(ticker, start_date, end_date, trials=trials, exch=exch)
820
+ data.append(data_resp)
821
+ pbar.update(1)
822
+ sleep(pause)
593
823
 
594
824
  return data
595
825
 
596
- async def _fetch_open_interest(self,
597
- ticker: str,
598
- freq: str,
599
- start_date: str,
600
- end_date: str,
601
- exch: str,
602
- trials: int = 3
603
- ) -> List:
826
+ async def _fetch_open_interest_async(self,
827
+ ticker: str,
828
+ freq: str,
829
+ start_date: str,
830
+ end_date: str,
831
+ exch: str,
832
+ trials: int = 3
833
+ ) -> List:
604
834
  """
605
835
  Fetches open interest data for a specific ticker.
606
836
 
@@ -629,17 +859,16 @@ class CCXT(Library):
629
859
  data = []
630
860
 
631
861
  # inst exch
632
- if self.exchange_async is None:
633
- self.exchange_async = getattr(ccxt_async, exch)()
862
+ self.exchange = getattr(ccxt_async, exch)()
634
863
 
635
864
  # fetch data
636
- if self.exchange_async.has['fetchOpenInterestHistory']:
865
+ if self.exchange.has['fetchOpenInterestHistory']:
637
866
 
638
867
  # while loop to get all data
639
868
  while start_date < end_date and attempts < trials:
640
869
 
641
870
  try:
642
- data_resp = await getattr(self.exchange_async, 'fetchOpenInterestHistory')(
871
+ data_resp = await self.exchange.fetch_open_interest_history(
643
872
  ticker,
644
873
  freq,
645
874
  since=start_date,
@@ -647,49 +876,127 @@ class CCXT(Library):
647
876
  params={'until': end_date}
648
877
  )
649
878
 
879
+ # add data to list
880
+ if data_resp:
881
+ start_date = data_resp[-1]['timestamp'] + 1
882
+ data.extend(data_resp)
883
+ else:
884
+ break
885
+
650
886
  except Exception as e:
651
887
  logging.warning(
652
- f"Failed to get open interest from {self.exchange_async.id} "
653
- f"for {ticker} on attempt #{attempts + 1}."
888
+ f"Failed to get open interest from {self.exchange.id} for {ticker} "
889
+ f"on attempt #{attempts + 1}: {e}."
654
890
  )
655
- logging.warning(e)
656
891
  attempts += 1
657
- if attempts == trials:
892
+ if attempts >= trials:
658
893
  logging.warning(
659
- f"Failed to get open interest from {self.exchange_async.id} "
894
+ f"Failed to get open interest from {self.exchange.id} "
660
895
  f"for {ticker} after {trials} attempts."
661
896
  )
662
- return data
897
+ break
663
898
 
664
- await asyncio.sleep(self.exchange_async.rateLimit / 1000)
665
- continue
899
+ finally:
900
+ await asyncio.sleep(self.exchange.rateLimit / 1000)
666
901
 
667
- else:
668
- # check if data resp is empty
669
- if len(data_resp):
670
- # next start date
902
+ await self.exchange.close()
903
+ return data
904
+
905
+ else:
906
+ logging.warning(f"Open interest is not available for {self.exchange.id}.")
907
+ return None
908
+
909
+ def _fetch_open_interest(self,
910
+ ticker: str,
911
+ freq: str,
912
+ start_date: str,
913
+ end_date: str,
914
+ exch: str,
915
+ trials: int = 3
916
+ ) -> List:
917
+ """
918
+ Fetches open interest data for a specific ticker.
919
+
920
+ Parameters
921
+ ----------
922
+ ticker: str
923
+ Ticker symbol.
924
+ freq: str
925
+ Frequency of data, e.g. '1m', '5m', '1h', '1d'.
926
+ start_date: str
927
+ Start date in integers in milliseconds since Unix epoch.
928
+ end_date: str
929
+ End date in integers in milliseconds since Unix epoch.
930
+ exch: str
931
+ Name of exchange.
932
+ trials: int, default 3
933
+ Number of attempts to fetch data.
934
+
935
+ Returns
936
+ -------
937
+ data: list
938
+ List of dictionaries with timestamps and open interest data.
939
+ """
940
+ # number of attempts
941
+ attempts = 0
942
+ data = []
943
+
944
+ # inst exch
945
+ self.exchange = getattr(ccxt, exch)()
946
+
947
+ # fetch data
948
+ if self.exchange.has['fetchOpenInterestHistory']:
949
+
950
+ # while loop to get all data
951
+ while start_date < end_date and attempts < trials:
952
+
953
+ try:
954
+ data_resp = self.exchange.fetch_open_interest_history(
955
+ ticker,
956
+ freq,
957
+ since=start_date,
958
+ limit=500,
959
+ params={'until': end_date}
960
+ )
961
+
962
+ # add data to list
963
+ if data_resp:
671
964
  start_date = data_resp[-1]['timestamp'] + 1
672
965
  data.extend(data_resp)
673
- await asyncio.sleep(self.exchange_async.rateLimit / 1000)
674
966
  else:
675
967
  break
676
968
 
969
+ except Exception as e:
970
+ logging.warning(
971
+ f"Failed to get open interest from {self.exchange.id} for {ticker} "
972
+ f"on attempt #{attempts + 1}: {e}."
973
+ )
974
+ attempts += 1
975
+ if attempts >= trials:
976
+ logging.warning(
977
+ f"Failed to get open interest from {self.exchange.id} "
978
+ f"for {ticker} after {trials} attempts."
979
+ )
980
+ break
981
+
982
+ finally:
983
+ sleep(self.exchange.rateLimit / 1000)
984
+
677
985
  return data
678
986
 
679
987
  else:
680
- logging.warning(f"Open interest is not available for {self.exchange_async.id}.")
988
+ logging.warning(f"Open interest is not available for {self.exchange.id}.")
681
989
  return None
682
990
 
683
- async def fetch_all_open_interest(self,
684
- tickers,
685
- freq: str,
686
- start_date: str,
687
- end_date: str,
688
- exch: str,
689
- trials: int = 3,
690
- pause: int = 0.5
691
- ):
692
-
991
+ async def _fetch_all_open_interest_async(self,
992
+ tickers,
993
+ freq: str,
994
+ start_date: str,
995
+ end_date: str,
996
+ exch: str,
997
+ trials: int = 3,
998
+ pause: int = 0.5
999
+ ):
693
1000
  """
694
1001
  Fetches open interest data for a list of tickers.
695
1002
 
@@ -716,8 +1023,7 @@ class CCXT(Library):
716
1023
  List of lists of dictionaries with timestamps and open interest data for each ticker.
717
1024
  """
718
1025
  # inst exch
719
- if self.exchange_async is None:
720
- self.exchange_async = getattr(ccxt_async, exch)()
1026
+ self.exchange = getattr(ccxt_async, exch)()
721
1027
 
722
1028
  data = []
723
1029
 
@@ -726,12 +1032,64 @@ class CCXT(Library):
726
1032
 
727
1033
  # loop through tickers
728
1034
  for ticker in tickers:
729
- data_resp = await self._fetch_open_interest(ticker, freq, start_date, end_date, trials=trials, exch=exch)
1035
+ data_resp = await self._fetch_open_interest_async(ticker, freq, start_date, end_date, trials=trials,
1036
+ exch=exch)
730
1037
  data.append(data_resp)
731
1038
  pbar.update(1)
732
- await asyncio.sleep(pause) # pause between ticker requests to respect the rate limit
1039
+ await asyncio.sleep(pause)
733
1040
 
734
- await self.exchange_async.close()
1041
+ await self.exchange.close()
1042
+
1043
+ return data
1044
+
1045
+ def _fetch_all_open_interest(self,
1046
+ tickers,
1047
+ freq: str,
1048
+ start_date: str,
1049
+ end_date: str,
1050
+ exch: str,
1051
+ trials: int = 3,
1052
+ pause: int = 0.5
1053
+ ):
1054
+ """
1055
+ Fetches open interest data for a list of tickers.
1056
+
1057
+ Parameters
1058
+ ----------
1059
+ tickers: list
1060
+ List of ticker symbols.
1061
+ freq: str
1062
+ Frequency of data, e.g. '1m', '5m', '1h', '1d'.
1063
+ start_date: str
1064
+ Start date in integers in milliseconds since Unix epoch.
1065
+ end_date: str
1066
+ End date in integers in milliseconds since Unix epoch.
1067
+ exch: str
1068
+ Name of exchange.
1069
+ trials: int, default 3
1070
+ Number of attempts to fetch data.
1071
+ pause: int, default 0.5
1072
+ Pause in seconds to respect the rate limit.
1073
+
1074
+ Returns
1075
+ -------
1076
+ data: list
1077
+ List of lists of dictionaries with timestamps and open interest data for each ticker.
1078
+ """
1079
+ # inst exch
1080
+ self.exchange = getattr(ccxt, exch)()
1081
+
1082
+ data = []
1083
+
1084
+ # create progress bar
1085
+ pbar = tqdm(total=len(tickers), desc="Fetching open interest", unit="ticker")
1086
+
1087
+ # loop through tickers
1088
+ for ticker in tickers:
1089
+ data_resp = self._fetch_open_interest(ticker, freq, start_date, end_date, trials=trials, exch=exch)
1090
+ data.append(data_resp)
1091
+ pbar.update(1)
1092
+ sleep(pause)
735
1093
 
736
1094
  return data
737
1095
 
@@ -754,6 +1112,10 @@ class CCXT(Library):
754
1112
  # get metadata
755
1113
  self.get_metadata(self.data_req.exch)
756
1114
 
1115
+ print(self.data_req.exch)
1116
+
1117
+ print(self.exchange)
1118
+
757
1119
  # check markets
758
1120
  if not any([market in self.markets for market in self.data_req.source_markets]):
759
1121
  raise ValueError(
@@ -767,14 +1129,6 @@ class CCXT(Library):
767
1129
  f"Use the '.frequencies' attribute to check available frequencies."
768
1130
  )
769
1131
 
770
- # check quote ccy
771
- if self.data_req.quote_ccy is not None:
772
- if self.data_req.quote_ccy not in self.assets:
773
- raise ValueError(
774
- f"{self.data_req.quote_ccy} is not supported. "
775
- f"Use the '.assets' attribute to check supported currencies."
776
- )
777
-
778
1132
  # check mkt type
779
1133
  if self.data_req.mkt_type not in self.market_types:
780
1134
  raise ValueError(
@@ -853,7 +1207,41 @@ class CCXT(Library):
853
1207
 
854
1208
  return WrangleData(self.data_req, data_resp).ccxt(data_type=data_type)
855
1209
 
856
- async def fetch_tidy_ohlcv(self, data_req: DataRequest) -> pd.DataFrame:
1210
+ async def fetch_tidy_ohlcv_async(self, data_req: DataRequest) -> pd.DataFrame:
1211
+ """
1212
+ Gets entire OHLCV history and wrangles the data response into tidy data format.
1213
+
1214
+ Parameters
1215
+ ----------
1216
+ data_req: DataRequest
1217
+ Parameters of data request in CryptoDataPy format.
1218
+
1219
+ Returns
1220
+ -------
1221
+ df: pd.DataFrame
1222
+ Dataframe with entire OHLCV data history retrieved and wrangled into tidy data format.
1223
+ """
1224
+ # convert data request parameters to CCXT format
1225
+ if self.data_req is None:
1226
+ self.convert_params(data_req)
1227
+
1228
+ # get entire data history
1229
+ data_resp = await self._fetch_all_ohlcv_async(self.data_req.source_markets,
1230
+ self.data_req.source_freq,
1231
+ self.data_req.source_start_date,
1232
+ self.data_req.source_end_date,
1233
+ self.data_req.exch,
1234
+ trials=self.data_req.trials,
1235
+ pause=self.data_req.pause)
1236
+
1237
+ # wrangle df
1238
+ if any(data_resp):
1239
+ df = self.wrangle_data_resp(data_resp, data_type='ohlcv')
1240
+ return df
1241
+ else:
1242
+ logging.warning("Failed to get requested OHLCV data.")
1243
+
1244
+ def fetch_tidy_ohlcv(self, data_req: DataRequest) -> pd.DataFrame:
857
1245
  """
858
1246
  Gets entire OHLCV history and wrangles the data response into tidy data format.
859
1247
 
@@ -872,13 +1260,13 @@ class CCXT(Library):
872
1260
  self.convert_params(data_req)
873
1261
 
874
1262
  # get entire data history
875
- data_resp = await self.fetch_all_ohlcv(self.data_req.source_markets,
876
- self.data_req.source_freq,
877
- self.data_req.source_start_date,
878
- self.data_req.source_end_date,
879
- self.data_req.exch,
880
- trials=self.data_req.trials,
881
- pause=self.data_req.pause)
1263
+ data_resp = self._fetch_all_ohlcv(self.data_req.source_markets,
1264
+ self.data_req.source_freq,
1265
+ self.data_req.source_start_date,
1266
+ self.data_req.source_end_date,
1267
+ self.data_req.exch,
1268
+ trials=self.data_req.trials,
1269
+ pause=self.data_req.pause)
882
1270
 
883
1271
  # wrangle df
884
1272
  if any(data_resp):
@@ -887,7 +1275,7 @@ class CCXT(Library):
887
1275
  else:
888
1276
  logging.warning("Failed to get requested OHLCV data.")
889
1277
 
890
- async def fetch_tidy_funding_rates(self, data_req: DataRequest) -> pd.DataFrame:
1278
+ async def fetch_tidy_funding_rates_async(self, data_req: DataRequest) -> pd.DataFrame:
891
1279
  """
892
1280
  Gets entire funding rates history and wrangles the data response into tidy data format.
893
1281
 
@@ -906,12 +1294,12 @@ class CCXT(Library):
906
1294
  self.convert_params(data_req)
907
1295
 
908
1296
  # get entire data history
909
- data_resp = await self.fetch_all_funding_rates(self.data_req.source_markets,
910
- self.data_req.source_start_date,
911
- self.data_req.source_end_date,
912
- self.data_req.exch,
913
- trials=self.data_req.trials,
914
- pause=self.data_req.pause)
1297
+ data_resp = await self._fetch_all_funding_rates_async(self.data_req.source_markets,
1298
+ self.data_req.source_start_date,
1299
+ self.data_req.source_end_date,
1300
+ self.data_req.exch,
1301
+ trials=self.data_req.trials,
1302
+ pause=self.data_req.pause)
915
1303
 
916
1304
  # wrangle df
917
1305
  if any(data_resp):
@@ -920,7 +1308,40 @@ class CCXT(Library):
920
1308
  else:
921
1309
  logging.warning("Failed to get requested funding rates.")
922
1310
 
923
- async def fetch_tidy_open_interest(self, data_req: DataRequest) -> pd.DataFrame:
1311
+ def fetch_tidy_funding_rates(self, data_req: DataRequest) -> pd.DataFrame:
1312
+ """
1313
+ Gets entire funding rates history and wrangles the data response into tidy data format.
1314
+
1315
+ Parameters
1316
+ ----------
1317
+ data_req: DataRequest
1318
+ Parameters of data request in CryptoDataPy format.
1319
+
1320
+ Returns
1321
+ -------
1322
+ df: pd.DataFrame
1323
+ Dataframe with entire data history retrieved and wrangled into tidy data format.
1324
+ """
1325
+ # convert data request parameters to CCXT format
1326
+ if self.data_req is None:
1327
+ self.convert_params(data_req)
1328
+
1329
+ # get entire data history
1330
+ data_resp = self._fetch_all_funding_rates(self.data_req.source_markets,
1331
+ self.data_req.source_start_date,
1332
+ self.data_req.source_end_date,
1333
+ self.data_req.exch,
1334
+ trials=self.data_req.trials,
1335
+ pause=self.data_req.pause)
1336
+
1337
+ # wrangle df
1338
+ if any(data_resp):
1339
+ df = self.wrangle_data_resp(data_resp, data_type='funding_rates')
1340
+ return df
1341
+ else:
1342
+ logging.warning("Failed to get requested funding rates.")
1343
+
1344
+ async def fetch_tidy_open_interest_async(self, data_req: DataRequest) -> pd.DataFrame:
924
1345
  """
925
1346
  Gets entire open interest history and wrangles the data response into tidy data format.
926
1347
 
@@ -939,13 +1360,13 @@ class CCXT(Library):
939
1360
  self.convert_params(data_req)
940
1361
 
941
1362
  # get entire data history
942
- data_resp = await self.fetch_all_open_interest(self.data_req.source_markets,
943
- self.data_req.source_freq,
944
- self.data_req.source_start_date,
945
- self.data_req.source_end_date,
946
- self.data_req.exch,
947
- trials=self.data_req.trials,
948
- pause=self.data_req.pause)
1363
+ data_resp = await self._fetch_all_open_interest_async(self.data_req.source_markets,
1364
+ self.data_req.source_freq,
1365
+ self.data_req.source_start_date,
1366
+ self.data_req.source_end_date,
1367
+ self.data_req.exch,
1368
+ trials=self.data_req.trials,
1369
+ pause=self.data_req.pause)
949
1370
 
950
1371
  # wrangle df
951
1372
  if any(data_resp):
@@ -954,7 +1375,81 @@ class CCXT(Library):
954
1375
  else:
955
1376
  logging.warning("Failed to get requested open interest.")
956
1377
 
957
- async def get_data(self, data_req: DataRequest) -> pd.DataFrame:
1378
+ def fetch_tidy_open_interest(self, data_req: DataRequest) -> pd.DataFrame:
1379
+ """
1380
+ Gets entire open interest history and wrangles the data response into tidy data format.
1381
+
1382
+ Parameters
1383
+ ----------
1384
+ data_req: DataRequest
1385
+ Parameters of data request in CryptoDataPy format.
1386
+
1387
+ Returns
1388
+ -------
1389
+ df: pd.DataFrame
1390
+ Dataframe with entire data history retrieved and wrangled into tidy data format.
1391
+ """
1392
+ # convert data request parameters to CCXT format
1393
+ if self.data_req is None:
1394
+ self.convert_params(data_req)
1395
+
1396
+ # get entire data history
1397
+ data_resp = self._fetch_all_open_interest(self.data_req.source_markets,
1398
+ self.data_req.source_freq,
1399
+ self.data_req.source_start_date,
1400
+ self.data_req.source_end_date,
1401
+ self.data_req.exch,
1402
+ trials=self.data_req.trials,
1403
+ pause=self.data_req.pause)
1404
+
1405
+ # wrangle df
1406
+ if any(data_resp):
1407
+ df = self.wrangle_data_resp(data_resp, data_type='open_interest')
1408
+ return df
1409
+ else:
1410
+ logging.warning("Failed to get requested open interest.")
1411
+
1412
+ async def get_data_async(self, data_req: DataRequest) -> pd.DataFrame:
1413
+ """
1414
+ Get data specified by data request.
1415
+
1416
+ Parameters
1417
+ data_req: DataRequest
1418
+ Parameters of data request in CryptoDataPy format.
1419
+
1420
+ Returns
1421
+ -------
1422
+ df: pd.DataFrame - MultiIndex
1423
+ DataFrame with DatetimeIndex (level 0), ticker (level 1), and values for selected fields (cols).
1424
+ """
1425
+ # get OHLCV
1426
+ if any([field in ["open", "high", "low", "close", "volume"] for field in data_req.fields]):
1427
+ df = await self.fetch_tidy_ohlcv_async(data_req)
1428
+ self.data = pd.concat([self.data, df])
1429
+
1430
+ # get funding rates
1431
+ if any([field == "funding_rate" for field in data_req.fields]):
1432
+ df = await self.fetch_tidy_funding_rates_async(data_req)
1433
+ self.data = pd.concat([self.data, df], axis=1)
1434
+
1435
+ # get open interest
1436
+ if any([field == "oi" for field in data_req.fields]):
1437
+ df = await self.fetch_tidy_open_interest_async(data_req)
1438
+ self.data = pd.concat([self.data, df], axis=1)
1439
+
1440
+ # check df
1441
+ if self.data.empty:
1442
+ raise Exception(
1443
+ "No data returned. Check data request parameters and try again."
1444
+ )
1445
+
1446
+ # filter df for desired fields and typecast
1447
+ fields = [field for field in data_req.fields if field in self.data.columns]
1448
+ self.data = self.data.loc[:, fields]
1449
+
1450
+ return self.data.sort_index()
1451
+
1452
+ def get_data(self, data_req: DataRequest) -> pd.DataFrame:
958
1453
  """
959
1454
  Get data specified by data request.
960
1455
 
@@ -969,17 +1464,17 @@ class CCXT(Library):
969
1464
  """
970
1465
  # get OHLCV
971
1466
  if any([field in ["open", "high", "low", "close", "volume"] for field in data_req.fields]):
972
- df = await self.fetch_tidy_ohlcv(data_req)
1467
+ df = self.fetch_tidy_ohlcv(data_req)
973
1468
  self.data = pd.concat([self.data, df])
974
1469
 
975
1470
  # get funding rates
976
1471
  if any([field == "funding_rate" for field in data_req.fields]):
977
- df = await self.fetch_tidy_funding_rates(data_req)
1472
+ df = self.fetch_tidy_funding_rates(data_req)
978
1473
  self.data = pd.concat([self.data, df], axis=1)
979
1474
 
980
1475
  # get open interest
981
1476
  if any([field == "oi" for field in data_req.fields]):
982
- df = await self.fetch_tidy_open_interest(data_req)
1477
+ df = self.fetch_tidy_open_interest(data_req)
983
1478
  self.data = pd.concat([self.data, df], axis=1)
984
1479
 
985
1480
  # check df