cryptodatapy 0.2.12__py3-none-any.whl → 0.2.14__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cryptodatapy/extract/data_vendors/coinmetrics_api.py +109 -198
- cryptodatapy/extract/data_vendors/tiingo_api.py +12 -7
- cryptodatapy/extract/datarequest.py +0 -23
- cryptodatapy/extract/libraries/ccxt_api.py +4 -0
- cryptodatapy/transform/convertparams.py +110 -123
- cryptodatapy/transform/impute.py +10 -4
- cryptodatapy/transform/od.py +30 -30
- cryptodatapy/transform/wrangle.py +15 -7
- {cryptodatapy-0.2.12.dist-info → cryptodatapy-0.2.14.dist-info}/METADATA +1 -1
- {cryptodatapy-0.2.12.dist-info → cryptodatapy-0.2.14.dist-info}/RECORD +12 -12
- {cryptodatapy-0.2.12.dist-info → cryptodatapy-0.2.14.dist-info}/LICENSE +0 -0
- {cryptodatapy-0.2.12.dist-info → cryptodatapy-0.2.14.dist-info}/WHEEL +0 -0
@@ -71,21 +71,12 @@ class CoinMetrics(DataVendor):
|
|
71
71
|
rate_limit: Any, optional, Default None
|
72
72
|
Number of API calls made and left, by time frequency.
|
73
73
|
"""
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
exchanges,
|
78
|
-
indexes,
|
79
|
-
assets,
|
80
|
-
markets,
|
81
|
-
market_types,
|
82
|
-
fields,
|
83
|
-
frequencies,
|
84
|
-
base_url,
|
85
|
-
api_key,
|
86
|
-
max_obs_per_call,
|
87
|
-
rate_limit,
|
74
|
+
super().__init__(
|
75
|
+
categories, exchanges, indexes, assets, markets, market_types,
|
76
|
+
fields, frequencies, base_url, api_key, max_obs_per_call, rate_limit
|
88
77
|
)
|
78
|
+
self.data_req = None
|
79
|
+
self.data = pd.DataFrame()
|
89
80
|
|
90
81
|
@staticmethod
|
91
82
|
def req_meta(data_type: str) -> Dict[str, Any]:
|
@@ -176,27 +167,6 @@ class CoinMetrics(DataVendor):
|
|
176
167
|
|
177
168
|
return self.assets
|
178
169
|
|
179
|
-
def get_inst_info(self, as_dict: bool = False) -> Union[Dict[str, List[str]], pd.DataFrame]:
|
180
|
-
"""
|
181
|
-
Get institutions info.
|
182
|
-
|
183
|
-
Parameters
|
184
|
-
----------
|
185
|
-
as_dict: bool, default False
|
186
|
-
Returns available institutions as dictionary.
|
187
|
-
|
188
|
-
Returns
|
189
|
-
-------
|
190
|
-
inst: dictionary or pd.DataFrame
|
191
|
-
Dictionary or dataframe with info on available institutions.
|
192
|
-
"""
|
193
|
-
# req data
|
194
|
-
inst = self.req_meta(data_type='catalog_institutions')
|
195
|
-
# wrangle data resp
|
196
|
-
inst = WrangleInfo(inst).cm_inst_info(as_dict=as_dict)
|
197
|
-
|
198
|
-
return inst
|
199
|
-
|
200
170
|
def get_markets_info(self, as_list: bool = False) -> Union[List[str], pd.DataFrame]:
|
201
171
|
"""
|
202
172
|
Get markets info.
|
@@ -258,14 +228,11 @@ class CoinMetrics(DataVendor):
|
|
258
228
|
# req data
|
259
229
|
ohlcv_fields = ['price_open', 'price_close', 'price_high', 'price_low', 'vwap', 'volume', 'candle_usd_volume',
|
260
230
|
'candle_trades_count'] # get market fields
|
261
|
-
inst_fields = [v for k, v in self.get_inst_info(as_dict=True).items()][0] # inst fields
|
262
231
|
onchain_fields = self.get_onchain_fields_info() # get onchain fields
|
263
232
|
|
264
233
|
# fields df
|
265
234
|
if data_type == "market":
|
266
235
|
self.fields = onchain_fields[onchain_fields.category == "Market"]
|
267
|
-
elif data_type == "off-chain":
|
268
|
-
self.fields = inst_fields
|
269
236
|
else:
|
270
237
|
self.fields = onchain_fields
|
271
238
|
|
@@ -275,10 +242,8 @@ class CoinMetrics(DataVendor):
|
|
275
242
|
self.fields = ohlcv_fields + list(self.fields.index)
|
276
243
|
elif data_type == "on-chain":
|
277
244
|
self.fields = list(self.fields.index)
|
278
|
-
elif data_type == "off-chain":
|
279
|
-
self.fields = inst_fields
|
280
245
|
else:
|
281
|
-
self.fields = ohlcv_fields + list(self.fields.index)
|
246
|
+
self.fields = ohlcv_fields + list(self.fields.index)
|
282
247
|
|
283
248
|
return self.fields
|
284
249
|
|
@@ -297,15 +262,14 @@ class CoinMetrics(DataVendor):
|
|
297
262
|
List of available assets for selected fields.
|
298
263
|
"""
|
299
264
|
# convert data request parameters to Coin Metrics format
|
300
|
-
|
301
|
-
# fields param
|
302
|
-
fields = cm_data_req["fields"]
|
265
|
+
self.data_req = ConvertParams(data_req).to_coinmetrics()
|
303
266
|
|
304
267
|
# fields info
|
305
268
|
self.get_fields_info()
|
269
|
+
|
306
270
|
# fields dict
|
307
271
|
fields_dict = {}
|
308
|
-
for field in
|
272
|
+
for field in self.data_req.source_fields:
|
309
273
|
if field in self.fields.index:
|
310
274
|
df = self.fields.loc[field] # get fields metadata
|
311
275
|
# add to dict
|
@@ -341,12 +305,14 @@ class CoinMetrics(DataVendor):
|
|
341
305
|
if self.fields is None:
|
342
306
|
self.get_fields_info(as_list=True)
|
343
307
|
|
344
|
-
def req_data(self, data_type: str, params: Dict[str, Union[str, int]]) -> pd.DataFrame:
|
308
|
+
def req_data(self, data_req: DataRequest, data_type: str, params: Dict[str, Union[str, int]]) -> pd.DataFrame:
|
345
309
|
"""
|
346
310
|
Sends data request to Python client.
|
347
311
|
|
348
312
|
Parameters
|
349
313
|
----------
|
314
|
+
data_req: DataRequest
|
315
|
+
Parameters of data request in CryptoDataPy format.
|
350
316
|
data_type: str
|
351
317
|
Data type to retrieve.
|
352
318
|
params: dict
|
@@ -361,7 +327,7 @@ class CoinMetrics(DataVendor):
|
|
361
327
|
url = self.base_url + data_type
|
362
328
|
|
363
329
|
# data request
|
364
|
-
data_resp =
|
330
|
+
data_resp = data_req.get_req(url=url, params=params)
|
365
331
|
|
366
332
|
# raise error if data is None
|
367
333
|
if data_resp is None:
|
@@ -374,10 +340,10 @@ class CoinMetrics(DataVendor):
|
|
374
340
|
# while loop
|
375
341
|
while next_page_url:
|
376
342
|
# wait to avoid exceeding rate limit
|
377
|
-
sleep(
|
343
|
+
sleep(data_req.pause)
|
378
344
|
|
379
345
|
# request next page
|
380
|
-
next_page_data_resp =
|
346
|
+
next_page_data_resp = data_req.get_req(url=next_page_url, params=None)
|
381
347
|
next_page_data, next_page_url = next_page_data_resp.get('data', []), next_page_data_resp.get(
|
382
348
|
'next_page_url')
|
383
349
|
|
@@ -390,7 +356,7 @@ class CoinMetrics(DataVendor):
|
|
390
356
|
return df
|
391
357
|
|
392
358
|
@staticmethod
|
393
|
-
def wrangle_data_resp(data_req: DataRequest, data_resp: pd.DataFrame()):
|
359
|
+
def wrangle_data_resp(data_req: DataRequest, data_resp: pd.DataFrame()) -> pd.DataFrame():
|
394
360
|
"""
|
395
361
|
Wrangle data response.
|
396
362
|
|
@@ -433,13 +399,13 @@ class CoinMetrics(DataVendor):
|
|
433
399
|
Dataframe with DatetimeIndex (level 0), ticker (level 1) and values for fields/col, in tidy data format.
|
434
400
|
"""
|
435
401
|
# get entire data history
|
436
|
-
df = self.req_data(data_type, params)
|
402
|
+
df = self.req_data(data_req, data_type, params)
|
437
403
|
# wrangle df
|
438
404
|
df = self.wrangle_data_resp(data_req, df)
|
439
405
|
|
440
406
|
return df
|
441
407
|
|
442
|
-
def check_tickers(self, data_req: DataRequest, data_type: str) ->
|
408
|
+
def check_tickers(self, data_req: DataRequest, data_type: str) -> DataRequest:
|
443
409
|
"""
|
444
410
|
Checks tickers for data availability.
|
445
411
|
|
@@ -457,40 +423,41 @@ class CoinMetrics(DataVendor):
|
|
457
423
|
List of available tickers.
|
458
424
|
"""
|
459
425
|
# convert params
|
460
|
-
|
461
|
-
# tickers
|
462
|
-
tickers = []
|
426
|
+
self.data_req = ConvertParams(data_req).to_coinmetrics()
|
463
427
|
|
464
428
|
# check indexes
|
465
429
|
if data_type == 'indexes':
|
466
430
|
self.get_indexes_info(as_list=True)
|
467
431
|
# avail tickers
|
468
|
-
|
432
|
+
self.data_req.source_tickers = [ticker for ticker in self.data_req.source_tickers
|
433
|
+
if ticker.upper() in self.indexes]
|
469
434
|
|
470
435
|
# check markets
|
471
436
|
elif data_type == 'market_candles' or data_type == 'open_interest' or \
|
472
437
|
data_type == 'funding_rates' or data_type == 'trades' or data_type == 'quotes':
|
473
438
|
self.get_assets_info(as_list=True)
|
474
439
|
# avail tickers
|
475
|
-
|
476
|
-
|
440
|
+
self.data_req.source_markets = [market for ticker, market in
|
441
|
+
zip(self.data_req.source_tickers, self.data_req.source_markets)
|
442
|
+
if ticker in self.assets]
|
477
443
|
|
478
444
|
# check assets
|
479
445
|
elif data_type == 'asset_metrics':
|
480
446
|
self.get_assets_info(as_list=True)
|
481
447
|
# avail tickers
|
482
|
-
|
448
|
+
self.data_req.source_tickers = [ticker for ticker in self.data_req.source_tickers
|
449
|
+
if ticker in self.assets]
|
483
450
|
|
484
451
|
# raise error if no tickers available
|
485
|
-
if len(
|
452
|
+
if len(self.data_req.source_tickers) == 0:
|
486
453
|
raise ValueError(
|
487
454
|
f"{data_req.tickers} are not valid tickers for the requested data type."
|
488
455
|
f" Use get_metadata to get a list of available indexes and assets."
|
489
456
|
)
|
490
457
|
|
491
|
-
return
|
458
|
+
return self.data_req
|
492
459
|
|
493
|
-
def check_fields(self, data_req: DataRequest, data_type: str) ->
|
460
|
+
def check_fields(self, data_req: DataRequest, data_type: str) -> DataRequest:
|
494
461
|
"""
|
495
462
|
Checks fields for data availability.
|
496
463
|
|
@@ -508,33 +475,25 @@ class CoinMetrics(DataVendor):
|
|
508
475
|
List of avaialble fields.
|
509
476
|
"""
|
510
477
|
# convert params
|
511
|
-
|
512
|
-
# fields
|
513
|
-
fields = []
|
514
|
-
|
515
|
-
# check instution
|
516
|
-
if data_type == 'institutions':
|
517
|
-
# avail inst
|
518
|
-
inst_list = [val for key, val in self.get_inst_info(as_dict=True).items()][0]
|
519
|
-
fields = [field for field in cm_data_req["fields"] if field in inst_list]
|
478
|
+
self.data_req = ConvertParams(data_req).to_coinmetrics()
|
520
479
|
|
521
480
|
# check on-chain metrics
|
522
|
-
|
481
|
+
if data_type == 'asset_metrics':
|
523
482
|
self.get_fields_info(data_type='on-chain', as_list=True)
|
524
483
|
# avail fields
|
525
|
-
|
484
|
+
self.data_req.source_fields = [field for field in self.data_req.source_fields
|
485
|
+
if field in self.fields]
|
526
486
|
|
527
487
|
# raise error if fields is empty
|
528
|
-
if len(
|
488
|
+
if len(self.data_req.source_fields) == 0:
|
529
489
|
raise ValueError(
|
530
490
|
f"{data_req.fields} are not valid fields."
|
531
491
|
f" Use the get_fields_info or get_inst_info methods to get available source fields."
|
532
492
|
)
|
533
493
|
|
534
|
-
return
|
494
|
+
return self.data_req
|
535
495
|
|
536
|
-
|
537
|
-
def check_params(data_req: DataRequest, data_type: str) -> None:
|
496
|
+
def check_params(self, data_req: DataRequest, data_type: str) -> None:
|
538
497
|
"""
|
539
498
|
Checks data request parameters.
|
540
499
|
|
@@ -548,27 +507,19 @@ class CoinMetrics(DataVendor):
|
|
548
507
|
|
549
508
|
"""
|
550
509
|
# convert params
|
551
|
-
|
510
|
+
self.data_req = ConvertParams(data_req).to_coinmetrics()
|
552
511
|
|
553
512
|
# indexes
|
554
513
|
if data_type == 'indexes':
|
555
|
-
if
|
514
|
+
if self.data_req.source_freq not in ["1h", "1d"]:
|
556
515
|
raise ValueError(
|
557
516
|
f"Indexes data is only available for hourly, daily, weekly, monthly and quarterly"
|
558
517
|
f" frequencies. Change data request frequency and try again."
|
559
518
|
)
|
560
519
|
|
561
|
-
# institutions
|
562
|
-
elif data_type == 'institutions':
|
563
|
-
if cm_data_req["freq"] != "1d":
|
564
|
-
raise ValueError(
|
565
|
-
f"Institutions data is only available for daily frequency."
|
566
|
-
f" Change data request frequency and try again."
|
567
|
-
)
|
568
|
-
|
569
520
|
# ohlcv
|
570
521
|
elif data_type == 'market_candles':
|
571
|
-
if
|
522
|
+
if self.data_req.source_freq not in ["1m", "1h", "1d"]:
|
572
523
|
raise ValueError(
|
573
524
|
f"OHLCV data is only available for minute, hourly, daily, weekly, monthly and quarterly"
|
574
525
|
f" frequencies. Change data request frequency and try again."
|
@@ -576,7 +527,7 @@ class CoinMetrics(DataVendor):
|
|
576
527
|
|
577
528
|
# on-chain
|
578
529
|
elif data_type == 'asset_metrics':
|
579
|
-
if
|
530
|
+
if self.data_req.source_freq not in ["1b", "1d"]:
|
580
531
|
raise ValueError(
|
581
532
|
f"On-chain data is only available for 'block' and 'd' frequencies."
|
582
533
|
f" Change data request frequency and try again."
|
@@ -584,14 +535,15 @@ class CoinMetrics(DataVendor):
|
|
584
535
|
|
585
536
|
# funding rate
|
586
537
|
elif data_type == 'funding_rates':
|
587
|
-
if data_req.mkt_type not in ["perpetual_future", "future", "option"]:
|
538
|
+
if self.data_req.mkt_type not in ["perpetual_future", "future", "option"]:
|
588
539
|
raise ValueError(
|
589
540
|
f"Funding rates are only available for 'perpetual_future', 'future' and"
|
590
541
|
f" 'option' market types. Change 'mkt_type' in data request and try again."
|
591
542
|
)
|
592
543
|
|
544
|
+
# oi
|
593
545
|
elif data_type == 'open_interest':
|
594
|
-
if data_req.mkt_type not in ["perpetual_future", "future", "option"]:
|
546
|
+
if self.data_req.mkt_type not in ["perpetual_future", "future", "option"]:
|
595
547
|
raise ValueError(
|
596
548
|
f"Open interest is only available for 'perpetual_future', 'future' and"
|
597
549
|
f" 'option' market types. Change 'mkt_type' in data request and try again."
|
@@ -599,7 +551,7 @@ class CoinMetrics(DataVendor):
|
|
599
551
|
|
600
552
|
# trades
|
601
553
|
elif data_type == 'trades':
|
602
|
-
if
|
554
|
+
if self.data_req.source_freq != "raw":
|
603
555
|
raise ValueError(
|
604
556
|
f"{data_type} data is only available at the 'tick' frequency."
|
605
557
|
f" Change data request frequency and try again."
|
@@ -607,7 +559,7 @@ class CoinMetrics(DataVendor):
|
|
607
559
|
|
608
560
|
# quotes
|
609
561
|
elif data_type == 'quotes':
|
610
|
-
if
|
562
|
+
if self.data_req.source_freq not in ["raw", "1s", "1m", "1h", "1d"]:
|
611
563
|
raise ValueError(
|
612
564
|
f"{data_type} data is only available at the 'tick', '1s', '1m', '1h' and '1d' frequencies."
|
613
565
|
f" Change data request frequency and try again."
|
@@ -630,21 +582,21 @@ class CoinMetrics(DataVendor):
|
|
630
582
|
DataFrame with DatetimeIndex (level 0), tickers (level 1) and index values (cols).
|
631
583
|
"""
|
632
584
|
# convert data request parameters to Coin Metrics format
|
633
|
-
|
585
|
+
self.data_req = ConvertParams(data_req).to_coinmetrics()
|
634
586
|
|
635
587
|
# check params
|
636
588
|
self.check_params(data_req, data_type='indexes')
|
637
589
|
|
638
590
|
# check tickers
|
639
|
-
|
640
|
-
sleep(
|
591
|
+
self.check_tickers(data_req, data_type='indexes')
|
592
|
+
sleep(self.data_req.pause)
|
641
593
|
|
642
594
|
# params
|
643
595
|
params = {
|
644
|
-
'indexes': ','.join(
|
645
|
-
'frequency':
|
646
|
-
'start_time':
|
647
|
-
'end_time':
|
596
|
+
'indexes': ','.join(self.data_req.source_tickers),
|
597
|
+
'frequency': self.data_req.source_freq,
|
598
|
+
'start_time': self.data_req.source_start_date,
|
599
|
+
'end_time': self.data_req.source_end_date,
|
648
600
|
'pretty': True,
|
649
601
|
'page_size': 10000,
|
650
602
|
}
|
@@ -656,48 +608,6 @@ class CoinMetrics(DataVendor):
|
|
656
608
|
|
657
609
|
return df
|
658
610
|
|
659
|
-
def get_institutions(self, data_req: DataRequest) -> pd.DataFrame:
|
660
|
-
"""
|
661
|
-
Get institutions data.
|
662
|
-
|
663
|
-
Parameters
|
664
|
-
----------
|
665
|
-
data_req: DataRequest
|
666
|
-
Parameters of data request in CryptoDataPy format.
|
667
|
-
|
668
|
-
Returns
|
669
|
-
-------
|
670
|
-
df: pd.DataFrame
|
671
|
-
DataFrame with DatetimeIndex (level 0), tickers (level 1) and institution fields values (cols).
|
672
|
-
"""
|
673
|
-
# convert data request parameters to Coin Metrics format
|
674
|
-
cm_data_req = ConvertParams(data_req).to_coinmetrics()
|
675
|
-
|
676
|
-
# check params
|
677
|
-
self.check_params(data_req, data_type='institutions')
|
678
|
-
|
679
|
-
# check fields
|
680
|
-
fields = self.check_fields(data_req, data_type='institutions')
|
681
|
-
sleep(0.6)
|
682
|
-
|
683
|
-
# params
|
684
|
-
params = {
|
685
|
-
'institutions': cm_data_req["inst"],
|
686
|
-
'metrics': ','.join(fields),
|
687
|
-
'frequency': cm_data_req['freq'],
|
688
|
-
'start_time': cm_data_req["start_date"],
|
689
|
-
'end_time': cm_data_req["end_date"],
|
690
|
-
'pretty': True,
|
691
|
-
'page_size': 10000,
|
692
|
-
}
|
693
|
-
|
694
|
-
# get tidy data
|
695
|
-
df = self.get_tidy_data(data_req,
|
696
|
-
data_type='/timeseries/institution-metrics',
|
697
|
-
params=params)
|
698
|
-
|
699
|
-
return df
|
700
|
-
|
701
611
|
def get_ohlcv(self, data_req: DataRequest) -> pd.DataFrame:
|
702
612
|
"""
|
703
613
|
Get OHLCV (candles) data.
|
@@ -713,21 +623,21 @@ class CoinMetrics(DataVendor):
|
|
713
623
|
DataFrame with DatetimeIndex (level 0), ticker (level 1), and OHLCV values (cols).
|
714
624
|
"""
|
715
625
|
# convert data request parameters to Coin Metrics format
|
716
|
-
|
626
|
+
self.data_req = ConvertParams(data_req).to_coinmetrics()
|
717
627
|
|
718
628
|
# check freq
|
719
629
|
self.check_params(data_req, data_type='market_candles')
|
720
630
|
|
721
631
|
# check tickers
|
722
|
-
|
723
|
-
sleep(
|
632
|
+
self.check_tickers(data_req, data_type='market_candles')
|
633
|
+
sleep(self.data_req.pause)
|
724
634
|
|
725
635
|
# params
|
726
636
|
params = {
|
727
|
-
'markets': ','.join(
|
728
|
-
'frequency':
|
729
|
-
'start_time':
|
730
|
-
'end_time':
|
637
|
+
'markets': ','.join(self.data_req.source_markets),
|
638
|
+
'frequency': self.data_req.source_freq,
|
639
|
+
'start_time': self.data_req.source_start_date,
|
640
|
+
'end_time': self.data_req.source_end_date,
|
731
641
|
'pretty': True,
|
732
642
|
'page_size': 10000,
|
733
643
|
}
|
@@ -754,26 +664,26 @@ class CoinMetrics(DataVendor):
|
|
754
664
|
DataFrame with DatetimeIndex (level 0), ticker (level 1), and on-chain values (cols).
|
755
665
|
"""
|
756
666
|
# convert data request parameters to Coin Metrics format
|
757
|
-
|
667
|
+
self.data_req = ConvertParams(data_req).to_coinmetrics()
|
758
668
|
|
759
669
|
# check params
|
760
670
|
self.check_params(data_req, data_type='asset_metrics')
|
761
671
|
|
762
672
|
# check tickers
|
763
|
-
|
764
|
-
sleep(
|
673
|
+
self.check_tickers(data_req, data_type='asset_metrics')
|
674
|
+
sleep(self.data_req.pause)
|
765
675
|
|
766
676
|
# check fields
|
767
|
-
|
768
|
-
sleep(
|
677
|
+
self.check_fields(data_req, data_type='asset_metrics')
|
678
|
+
sleep(self.data_req.pause)
|
769
679
|
|
770
680
|
# params
|
771
681
|
params = {
|
772
|
-
'assets': ','.join(
|
773
|
-
'metrics': ','.join(
|
774
|
-
'frequency':
|
775
|
-
'start_time':
|
776
|
-
'end_time':
|
682
|
+
'assets': ','.join(self.data_req.source_tickers),
|
683
|
+
'metrics': ','.join(self.data_req.source_fields),
|
684
|
+
'frequency': self.data_req.source_freq,
|
685
|
+
'start_time': self.data_req.source_start_date,
|
686
|
+
'end_time': self.data_req.source_end_date,
|
777
687
|
'pretty': True,
|
778
688
|
'page_size': 10000,
|
779
689
|
'ignore_forbidden_errors': True,
|
@@ -804,20 +714,20 @@ class CoinMetrics(DataVendor):
|
|
804
714
|
DataFrame with DatetimeIndex (level 0), ticker (level 1), and open interest values (cols).
|
805
715
|
"""
|
806
716
|
# convert data request parameters to Coin Metrics format
|
807
|
-
|
717
|
+
self.data_req = ConvertParams(data_req).to_coinmetrics()
|
808
718
|
|
809
719
|
# check params
|
810
720
|
self.check_params(data_req, data_type='open_interest')
|
811
721
|
|
812
722
|
# check tickers
|
813
|
-
|
814
|
-
sleep(
|
723
|
+
self.check_tickers(data_req, data_type='open_interest')
|
724
|
+
sleep(self.data_req.pause)
|
815
725
|
|
816
726
|
# params
|
817
727
|
params = {
|
818
|
-
'markets': ','.join(
|
819
|
-
'start_time':
|
820
|
-
'end_time':
|
728
|
+
'markets': ','.join(self.data_req.source_markets),
|
729
|
+
'start_time': self.data_req.source_start_date,
|
730
|
+
'end_time': self.data_req.source_end_date,
|
821
731
|
'pretty': True,
|
822
732
|
'page_size': 10000,
|
823
733
|
}
|
@@ -845,20 +755,20 @@ class CoinMetrics(DataVendor):
|
|
845
755
|
DataFrame with DatetimeIndex (level 0), ticker (level 1), and funding rates values (cols).
|
846
756
|
"""
|
847
757
|
# convert data request parameters to Coin Metrics format
|
848
|
-
|
758
|
+
self.data_req = ConvertParams(data_req).to_coinmetrics()
|
849
759
|
|
850
760
|
# check params
|
851
761
|
self.check_params(data_req, data_type='funding_rates')
|
852
762
|
|
853
763
|
# check tickers
|
854
|
-
|
855
|
-
sleep(
|
764
|
+
self.check_tickers(data_req, data_type='funding_rates')
|
765
|
+
sleep(self.data_req.pause)
|
856
766
|
|
857
767
|
# params
|
858
768
|
params = {
|
859
|
-
'markets': ','.join(
|
860
|
-
'start_time':
|
861
|
-
'end_time':
|
769
|
+
'markets': ','.join(self.data_req.source_markets),
|
770
|
+
'start_time': self.data_req.source_start_date,
|
771
|
+
'end_time': self.data_req.source_end_date,
|
862
772
|
'pretty': True,
|
863
773
|
'page_size': 10000,
|
864
774
|
}
|
@@ -886,20 +796,20 @@ class CoinMetrics(DataVendor):
|
|
886
796
|
DataFrame with DatetimeIndex (level 0), ticker (level 1), and bid/ask price and size values (cols).
|
887
797
|
"""
|
888
798
|
# convert data request parameters to Coin Metrics format
|
889
|
-
|
799
|
+
self.data_req = ConvertParams(data_req).to_coinmetrics()
|
890
800
|
|
891
801
|
# check params
|
892
802
|
self.check_params(data_req, data_type='trades')
|
893
803
|
|
894
804
|
# check tickers
|
895
|
-
|
896
|
-
sleep(
|
805
|
+
self.check_tickers(data_req, data_type='trades')
|
806
|
+
sleep(self.data_req.pause)
|
897
807
|
|
898
808
|
# params
|
899
809
|
params = {
|
900
|
-
'markets': ','.join(
|
901
|
-
'start_time':
|
902
|
-
'end_time':
|
810
|
+
'markets': ','.join(self.data_req.source_markets),
|
811
|
+
'start_time': self.data_req.source_start_date,
|
812
|
+
'end_time': self.data_req.source_end_date,
|
903
813
|
'pretty': True,
|
904
814
|
'page_size': 10000,
|
905
815
|
}
|
@@ -927,21 +837,21 @@ class CoinMetrics(DataVendor):
|
|
927
837
|
DataFrame with DatetimeIndex (level 0), ticker (level 1), and bid/ask price and size values (cols).
|
928
838
|
"""
|
929
839
|
# convert data request parameters to Coin Metrics format
|
930
|
-
|
840
|
+
self.data_req = ConvertParams(data_req).to_coinmetrics()
|
931
841
|
|
932
842
|
# check params
|
933
843
|
self.check_params(data_req, data_type='quotes')
|
934
844
|
|
935
845
|
# check tickers
|
936
|
-
|
937
|
-
sleep(
|
846
|
+
self.check_tickers(data_req, data_type='quotes')
|
847
|
+
sleep(self.data_req.pause)
|
938
848
|
|
939
849
|
# params
|
940
850
|
params = {
|
941
|
-
'markets': ','.join(
|
942
|
-
'granularity':
|
943
|
-
'start_time':
|
944
|
-
'end_time':
|
851
|
+
'markets': ','.join(self.data_req.source_markets),
|
852
|
+
'granularity': self.data_req.source_freq,
|
853
|
+
'start_time': self.data_req.source_start_date,
|
854
|
+
'end_time': self.data_req.source_end_date,
|
945
855
|
'pretty': True,
|
946
856
|
'page_size': 10000,
|
947
857
|
}
|
@@ -970,16 +880,17 @@ class CoinMetrics(DataVendor):
|
|
970
880
|
fields (cols), in tidy format.
|
971
881
|
"""
|
972
882
|
# convert data request parameters to Coin Metrics format
|
973
|
-
|
883
|
+
self.data_req = ConvertParams(data_req).to_coinmetrics()
|
974
884
|
|
975
885
|
# check if fields available
|
976
886
|
self.get_fields_info(as_list=True)
|
977
|
-
sleep(
|
978
|
-
|
979
|
-
|
980
|
-
|
981
|
-
|
982
|
-
|
887
|
+
sleep(self.data_req.pause)
|
888
|
+
|
889
|
+
if not all([field in self.fields for field in self.data_req.source_fields]):
|
890
|
+
raise ValueError(
|
891
|
+
"Some selected fields are not available. Check available fields with"
|
892
|
+
" get_fields_info method and try again."
|
893
|
+
)
|
983
894
|
|
984
895
|
# field lists
|
985
896
|
ohlcv_list = ['price_open', 'price_close', 'price_high', 'price_low', 'vwap', 'volume',
|
@@ -991,9 +902,9 @@ class CoinMetrics(DataVendor):
|
|
991
902
|
|
992
903
|
# get indexes data
|
993
904
|
self.get_indexes_info(as_list=True)
|
994
|
-
sleep(
|
995
|
-
if any([ticker.upper() in self.indexes for ticker in
|
996
|
-
[field in ohlcv_list for field in
|
905
|
+
sleep(self.data_req.pause)
|
906
|
+
if any([ticker.upper() in self.indexes for ticker in self.data_req.source_tickers]) and any(
|
907
|
+
[field in ohlcv_list for field in self.data_req.source_fields]
|
997
908
|
):
|
998
909
|
df0 = self.get_indexes(data_req)
|
999
910
|
df = pd.concat([df, df0])
|
@@ -1001,15 +912,15 @@ class CoinMetrics(DataVendor):
|
|
1001
912
|
# get OHLCV data
|
1002
913
|
self.get_assets_info(as_list=True)
|
1003
914
|
sleep(0.6)
|
1004
|
-
if any([ticker in self.assets for ticker in
|
1005
|
-
[field in ohlcv_list for field in
|
915
|
+
if any([ticker in self.assets for ticker in self.data_req.source_tickers]) and any(
|
916
|
+
[field in ohlcv_list for field in self.data_req.source_fields]
|
1006
917
|
):
|
1007
918
|
df1 = self.get_ohlcv(data_req)
|
1008
919
|
df = pd.concat([df, df1])
|
1009
920
|
|
1010
921
|
# get on-chain data
|
1011
|
-
if any([ticker in self.assets for ticker in
|
1012
|
-
[field in oc_list for field in
|
922
|
+
if any([ticker in self.assets for ticker in self.data_req.source_tickers]) and any(
|
923
|
+
[field in oc_list for field in self.data_req.source_fields]
|
1013
924
|
):
|
1014
925
|
df2 = self.get_onchain(data_req)
|
1015
926
|
df = pd.concat([df, df2], axis=1)
|
@@ -549,7 +549,7 @@ class Tiingo(DataVendor):
|
|
549
549
|
df = pd.concat([df, df0])
|
550
550
|
|
551
551
|
else:
|
552
|
-
for ticker in self.data_req.
|
552
|
+
for ticker in self.data_req.source_tickers:
|
553
553
|
try:
|
554
554
|
df0 = self.get_tidy_data(self.data_req, data_type, ticker)
|
555
555
|
except Exception as e:
|
@@ -679,7 +679,7 @@ class Tiingo(DataVendor):
|
|
679
679
|
"""
|
680
680
|
self.data_req = ConvertParams(data_req).to_tiingo()
|
681
681
|
|
682
|
-
# get
|
682
|
+
# get metadata
|
683
683
|
self.get_assets_info(as_list=True)
|
684
684
|
self.get_fields_info()
|
685
685
|
|
@@ -690,11 +690,16 @@ class Tiingo(DataVendor):
|
|
690
690
|
)
|
691
691
|
|
692
692
|
# check assets
|
693
|
-
if
|
694
|
-
|
695
|
-
|
696
|
-
|
697
|
-
|
693
|
+
if self.data_req.cat == 'eqty':
|
694
|
+
if not any([ticker.upper() in self.assets[self.data_req.cat] for ticker in self.data_req.source_tickers]):
|
695
|
+
raise ValueError(
|
696
|
+
f"Selected eqty tickers are not available. Use assets attribute to see available eqty tickers."
|
697
|
+
)
|
698
|
+
elif self.data_req.cat == 'crypto':
|
699
|
+
if not any([ticker in self.assets[self.data_req.cat] for ticker in self.data_req.source_markets]):
|
700
|
+
raise ValueError(
|
701
|
+
f"Selected crypto tickers are not available. Use assets attribute to see available crypto tickers."
|
702
|
+
)
|
698
703
|
|
699
704
|
# check fields
|
700
705
|
if not any([field in self.fields[data_req.cat] for field in self.data_req.fields]):
|
@@ -26,7 +26,6 @@ class DataRequest:
|
|
26
26
|
end_date: Optional[Union[str, datetime, pd.Timestamp]] = None,
|
27
27
|
fields: Union[str, List[str]] = ["close"],
|
28
28
|
tz: Optional[str] = None,
|
29
|
-
inst: Optional[str] = None,
|
30
29
|
cat: Optional[str] = None,
|
31
30
|
trials: Optional[int] = 3,
|
32
31
|
pause: Optional[float] = 0.1,
|
@@ -69,8 +68,6 @@ class DataRequest:
|
|
69
68
|
Fields for data request. OHLC bars/fields are the most common fields for market data.
|
70
69
|
tz: str, optional, default None
|
71
70
|
Timezone for the start/end dates in tz database format.
|
72
|
-
inst: str, optional, default None
|
73
|
-
Name of institution from which to pull fund data, e.g. 'grayscale', 'purpose', etc.
|
74
71
|
cat: str, optional, {'crypto', 'fx', 'cmdty', 'eqty', 'rates', 'bonds', 'credit', 'macro', 'alt'}, default None
|
75
72
|
Category of data, e.g. crypto, fx, rates, or macro.
|
76
73
|
trials: int, optional, default 3
|
@@ -107,7 +104,6 @@ class DataRequest:
|
|
107
104
|
self.end_date = end_date # end date
|
108
105
|
self.fields = fields # fields
|
109
106
|
self.tz = tz # tz
|
110
|
-
self.inst = inst # institution
|
111
107
|
self.cat = cat # category of asset class or time series
|
112
108
|
self.trials = trials # number of times to try query request
|
113
109
|
self.pause = pause # number of seconds to pause between query request trials
|
@@ -412,25 +408,6 @@ class DataRequest:
|
|
412
408
|
else:
|
413
409
|
raise TypeError("Fields must be a string or list of strings.")
|
414
410
|
|
415
|
-
@property
|
416
|
-
def inst(self):
|
417
|
-
"""
|
418
|
-
Returns institution name for data request.
|
419
|
-
"""
|
420
|
-
return self._inst
|
421
|
-
|
422
|
-
@inst.setter
|
423
|
-
def inst(self, inst):
|
424
|
-
"""
|
425
|
-
Sets institution's name for data request.
|
426
|
-
"""
|
427
|
-
if inst is None:
|
428
|
-
self._inst = inst
|
429
|
-
elif isinstance(inst, str):
|
430
|
-
self._inst = inst
|
431
|
-
else:
|
432
|
-
raise TypeError("Institution must be a string.")
|
433
|
-
|
434
411
|
@property
|
435
412
|
def tz(self):
|
436
413
|
"""
|
@@ -1483,6 +1483,8 @@ class CCXT(Library):
|
|
1483
1483
|
df: pd.DataFrame - MultiIndex
|
1484
1484
|
DataFrame with DatetimeIndex (level 0), ticker (level 1), and values for selected fields (cols).
|
1485
1485
|
"""
|
1486
|
+
logging.info("Retrieving data request from CCXT...")
|
1487
|
+
|
1486
1488
|
# get OHLCV
|
1487
1489
|
if any([field in ["open", "high", "low", "close", "volume"] for field in data_req.fields]):
|
1488
1490
|
df = await self.fetch_tidy_ohlcv_async(data_req)
|
@@ -1523,6 +1525,8 @@ class CCXT(Library):
|
|
1523
1525
|
df: pd.DataFrame - MultiIndex
|
1524
1526
|
DataFrame with DatetimeIndex (level 0), ticker (level 1), and values for selected fields (cols).
|
1525
1527
|
"""
|
1528
|
+
logging.info("Retrieving data request from CCXT...")
|
1529
|
+
|
1526
1530
|
# get OHLCV
|
1527
1531
|
if any([field in ["open", "high", "low", "close", "volume"] for field in data_req.fields]):
|
1528
1532
|
df = self.fetch_tidy_ohlcv(data_req)
|
@@ -101,159 +101,134 @@ class ConvertParams:
|
|
101
101
|
"source_fields": self.data_req.source_fields,
|
102
102
|
}
|
103
103
|
|
104
|
-
def to_coinmetrics(self) ->
|
104
|
+
def to_coinmetrics(self) -> DataRequest:
|
105
105
|
"""
|
106
106
|
Convert tickers from CryptoDataPy to CoinMetrics format.
|
107
107
|
"""
|
108
|
-
#
|
109
|
-
if self.data_req.source_tickers is
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
# convert freq
|
115
|
-
if self.data_req.source_freq is not None:
|
116
|
-
freq = self.data_req.source_freq
|
117
|
-
self.data_req.freq = self.data_req.source_freq
|
118
|
-
else:
|
108
|
+
# tickers
|
109
|
+
if self.data_req.source_tickers is None:
|
110
|
+
self.data_req.source_tickers = [ticker.lower() for ticker in self.data_req.tickers]
|
111
|
+
|
112
|
+
# freq
|
113
|
+
if self.data_req.source_freq is None:
|
119
114
|
if self.data_req.freq is None:
|
120
|
-
|
115
|
+
self.data_req.source_freq = "1d"
|
121
116
|
elif self.data_req.freq == "block":
|
122
|
-
|
117
|
+
self.data_req.source_freq = "1b"
|
123
118
|
elif self.data_req.freq == "tick":
|
124
|
-
|
119
|
+
self.data_req.source_freq = "raw"
|
125
120
|
elif self.data_req.freq[-1] == "s":
|
126
|
-
|
121
|
+
self.data_req.source_freq = "1s"
|
127
122
|
elif self.data_req.freq[-3:] == "min":
|
128
|
-
|
123
|
+
self.data_req.source_freq = "1m"
|
129
124
|
elif self.data_req.freq[-1] == "h":
|
130
|
-
|
125
|
+
self.data_req.source_freq = "1h"
|
131
126
|
else:
|
132
|
-
|
133
|
-
|
127
|
+
self.data_req.source_freq = "1d"
|
128
|
+
|
129
|
+
# quote ccy
|
134
130
|
if self.data_req.quote_ccy is None:
|
135
|
-
quote_ccy = "usdt"
|
136
|
-
else:
|
137
|
-
quote_ccy = self.data_req.quote_ccy.lower()
|
138
|
-
# convert inst
|
139
|
-
if self.data_req.inst is None:
|
140
|
-
inst = "grayscale"
|
131
|
+
self.data_req.quote_ccy = "usdt"
|
141
132
|
else:
|
142
|
-
|
143
|
-
|
133
|
+
self.data_req.quote_ccy = self.data_req.quote_ccy.lower()
|
134
|
+
|
135
|
+
# exchange
|
144
136
|
if self.data_req.exch is None:
|
145
|
-
exch = "binance"
|
137
|
+
self.data_req.exch = "binance"
|
146
138
|
else:
|
147
|
-
exch = self.data_req.exch.lower()
|
139
|
+
self.data_req.exch = self.data_req.exch.lower()
|
140
|
+
|
148
141
|
# fields
|
149
|
-
if self.data_req.source_fields is
|
150
|
-
|
151
|
-
|
152
|
-
else:
|
153
|
-
fields = self.convert_fields(data_source='coinmetrics')
|
142
|
+
if self.data_req.source_fields is None:
|
143
|
+
self.data_req.source_fields = self.convert_fields(data_source='coinmetrics')
|
144
|
+
|
154
145
|
# convert tz
|
155
146
|
if self.data_req.tz is None:
|
156
|
-
tz = "UTC"
|
147
|
+
self.data_req.tz = "UTC"
|
157
148
|
else:
|
158
|
-
tz = self.data_req.tz
|
159
|
-
# convert tickers to markets
|
160
|
-
mkts_list = []
|
149
|
+
self.data_req.tz = self.data_req.tz
|
161
150
|
|
162
|
-
|
163
|
-
|
164
|
-
self.data_req.tickers = self.data_req.source_tickers
|
151
|
+
# markets
|
152
|
+
mkts_list = []
|
165
153
|
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
+ "-"
|
171
|
-
+ ticker.lower()
|
172
|
-
+ "-"
|
173
|
-
+ quote_ccy.lower()
|
174
|
-
+ "-"
|
175
|
-
+ self.data_req.mkt_type.lower()
|
176
|
-
)
|
177
|
-
elif self.data_req.mkt_type == "perpetual_future":
|
178
|
-
if exch == "binance" or exch == "bybit" or exch == "bitmex":
|
179
|
-
mkts_list.append(
|
180
|
-
exch
|
181
|
-
+ "-"
|
182
|
-
+ ticker.upper()
|
183
|
-
+ quote_ccy.upper()
|
184
|
-
+ "-"
|
185
|
-
+ "future"
|
186
|
-
)
|
187
|
-
elif exch == "ftx":
|
188
|
-
mkts_list.append(
|
189
|
-
exch + "-" + ticker.upper() + "-" + "PERP" + "-" + "future"
|
190
|
-
)
|
191
|
-
elif exch == "okex":
|
192
|
-
mkts_list.append(
|
193
|
-
exch
|
194
|
-
+ "-"
|
195
|
-
+ ticker.upper()
|
196
|
-
+ "-"
|
197
|
-
+ quote_ccy.upper()
|
198
|
-
+ "-"
|
199
|
-
+ "SWAP"
|
200
|
-
+ "-"
|
201
|
-
+ "future"
|
202
|
-
)
|
203
|
-
elif exch == "huobi":
|
154
|
+
if self.data_req.source_markets is None:
|
155
|
+
# loop through tickers
|
156
|
+
for ticker in self.data_req.tickers:
|
157
|
+
if self.data_req.mkt_type == "spot":
|
204
158
|
mkts_list.append(
|
205
|
-
exch
|
206
|
-
+ "-"
|
207
|
-
+ ticker.upper()
|
159
|
+
self.data_req.exch
|
208
160
|
+ "-"
|
209
|
-
+
|
210
|
-
+ "_"
|
211
|
-
+ "SWAP"
|
212
|
-
+ "-"
|
213
|
-
+ "future"
|
214
|
-
)
|
215
|
-
elif exch == "hitbtc":
|
216
|
-
mkts_list.append(
|
217
|
-
exch
|
161
|
+
+ ticker.lower()
|
218
162
|
+ "-"
|
219
|
-
+
|
220
|
-
+ quote_ccy.upper()
|
221
|
-
+ "_"
|
222
|
-
+ "PERP"
|
163
|
+
+ self.data_req.quote_ccy.lower()
|
223
164
|
+ "-"
|
224
|
-
+
|
165
|
+
+ self.data_req.mkt_type.lower()
|
225
166
|
)
|
167
|
+
elif self.data_req.mkt_type == "perpetual_future":
|
168
|
+
if (self.data_req.exch == "binance" or self.data_req.exch == "bybit" or
|
169
|
+
self.data_req.exch == "bitmex"):
|
170
|
+
mkts_list.append(
|
171
|
+
self.data_req.exch
|
172
|
+
+ "-"
|
173
|
+
+ ticker.upper()
|
174
|
+
+ self.data_req.quote_ccy.upper()
|
175
|
+
+ "-"
|
176
|
+
+ "future"
|
177
|
+
)
|
178
|
+
elif self.data_req.exch == "ftx":
|
179
|
+
mkts_list.append(
|
180
|
+
self.data_req.exch + "-" + ticker.upper() + "-" + "PERP" + "-" + "future"
|
181
|
+
)
|
182
|
+
elif self.data_req.exch == "okex":
|
183
|
+
mkts_list.append(
|
184
|
+
self.data_req.exch
|
185
|
+
+ "-"
|
186
|
+
+ ticker.upper()
|
187
|
+
+ "-"
|
188
|
+
+ self.data_req.quote_ccy.upper()
|
189
|
+
+ "-"
|
190
|
+
+ "SWAP"
|
191
|
+
+ "-"
|
192
|
+
+ "future"
|
193
|
+
)
|
194
|
+
elif self.data_req.exch == "huobi":
|
195
|
+
mkts_list.append(
|
196
|
+
self.data_req.exch
|
197
|
+
+ "-"
|
198
|
+
+ ticker.upper()
|
199
|
+
+ "-"
|
200
|
+
+ self.data_req.quote_ccy.upper()
|
201
|
+
+ "_"
|
202
|
+
+ "SWAP"
|
203
|
+
+ "-"
|
204
|
+
+ "future"
|
205
|
+
)
|
206
|
+
elif self.data_req.exch == "hitbtc":
|
207
|
+
mkts_list.append(
|
208
|
+
self.data_req.exch
|
209
|
+
+ "-"
|
210
|
+
+ ticker.upper()
|
211
|
+
+ self.data_req.quote_ccy.upper()
|
212
|
+
+ "_"
|
213
|
+
+ "PERP"
|
214
|
+
+ "-"
|
215
|
+
+ "future"
|
216
|
+
)
|
217
|
+
self.data_req.source_markets = mkts_list
|
218
|
+
|
226
219
|
# start date
|
227
220
|
if self.data_req.start_date is not None:
|
228
|
-
|
221
|
+
self.data_req.source_start_date = self.data_req.start_date.strftime('%Y-%m-%d')
|
229
222
|
else:
|
230
|
-
|
223
|
+
self.data_req.source_start_date = None
|
224
|
+
|
231
225
|
# end date
|
232
226
|
if self.data_req.end_date is not None:
|
233
|
-
|
227
|
+
self.data_req.source_end_date = self.data_req.end_date.strftime('%Y-%m-%d')
|
234
228
|
else:
|
235
|
-
|
229
|
+
self.data_req.source_end_date = None
|
236
230
|
|
237
|
-
return
|
238
|
-
"tickers": tickers,
|
239
|
-
"freq": freq,
|
240
|
-
"quote_ccy": quote_ccy,
|
241
|
-
"exch": exch,
|
242
|
-
"ctys": None,
|
243
|
-
"mkt_type": self.data_req.mkt_type,
|
244
|
-
"mkts": mkts_list,
|
245
|
-
"start_date": start_date,
|
246
|
-
"end_date": end_date,
|
247
|
-
"fields": fields,
|
248
|
-
"tz": tz,
|
249
|
-
"inst": inst,
|
250
|
-
"cat": 'crypto',
|
251
|
-
"trials": self.data_req.trials,
|
252
|
-
"pause": self.data_req.pause,
|
253
|
-
"source_tickers": self.data_req.source_tickers,
|
254
|
-
"source_freq": self.data_req.source_freq,
|
255
|
-
"source_fields": self.data_req.source_fields,
|
256
|
-
}
|
231
|
+
return self.data_req
|
257
232
|
|
258
233
|
def to_glassnode(self) -> Dict[str, Union[list, str, int, float, None]]:
|
259
234
|
"""
|
@@ -344,8 +319,20 @@ class ConvertParams:
|
|
344
319
|
Convert tickers from CryptoDataPy to Tiingo format.
|
345
320
|
"""
|
346
321
|
# tickers
|
347
|
-
|
348
|
-
|
322
|
+
with resources.path("cryptodatapy.conf", "tickers.csv") as f:
|
323
|
+
tickers_path = f
|
324
|
+
tickers_df = pd.read_csv(tickers_path, index_col=0, encoding="latin1")
|
325
|
+
|
326
|
+
if self.data_req.source_tickers is None and self.data_req.cat == 'eqty':
|
327
|
+
self.data_req.source_tickers = []
|
328
|
+
for ticker in self.data_req.tickers:
|
329
|
+
try:
|
330
|
+
self.data_req.source_tickers.append(tickers_df.loc[ticker, "tiingo_id"])
|
331
|
+
except KeyError:
|
332
|
+
logging.warning(
|
333
|
+
f"{ticker} not found for Tiingo source. Check tickers in"
|
334
|
+
f" data catalog and try again."
|
335
|
+
)
|
349
336
|
|
350
337
|
# freq
|
351
338
|
if self.data_req.source_freq is None:
|
cryptodatapy/transform/impute.py
CHANGED
@@ -79,11 +79,18 @@ class Impute:
|
|
79
79
|
order = 3
|
80
80
|
|
81
81
|
# interpolate
|
82
|
-
self.imputed_df =
|
83
|
-
|
82
|
+
self.imputed_df = (
|
83
|
+
self.filtered_df
|
84
|
+
.unstack()
|
85
|
+
.interpolate(method=method,
|
86
|
+
order=order,
|
87
|
+
axis=axis,
|
88
|
+
limit=limit)
|
89
|
+
.stack(future_stack=True)
|
90
|
+
.reindex(self.filtered_df.index))
|
84
91
|
|
85
92
|
# type conversion
|
86
|
-
self.imputed_df = self.imputed_df.
|
93
|
+
self.imputed_df = self.imputed_df.convert_dtypes()
|
87
94
|
|
88
95
|
# plot
|
89
96
|
if self.plot:
|
@@ -123,7 +130,6 @@ class Impute:
|
|
123
130
|
# type conversion
|
124
131
|
self.imputed_df = self.imputed_df.convert_dtypes()
|
125
132
|
|
126
|
-
|
127
133
|
# plot
|
128
134
|
if self.plot:
|
129
135
|
if not isinstance(self.plot_series, tuple):
|
cryptodatapy/transform/od.py
CHANGED
@@ -205,9 +205,9 @@ class OutlierDetection:
|
|
205
205
|
med = np.exp(med)
|
206
206
|
|
207
207
|
# type conversion
|
208
|
-
self.yhat = med.
|
209
|
-
self.outliers = out_df.
|
210
|
-
self.filtered_df = filt_df.
|
208
|
+
self.yhat = med.convert_dtypes().sort_index()
|
209
|
+
self.outliers = out_df.convert_dtypes().sort_index()
|
210
|
+
self.filtered_df = filt_df.convert_dtypes().sort_index()
|
211
211
|
|
212
212
|
# plot
|
213
213
|
if self.plot:
|
@@ -260,9 +260,9 @@ class OutlierDetection:
|
|
260
260
|
med = np.exp(med)
|
261
261
|
|
262
262
|
# type conversion
|
263
|
-
med = med.
|
264
|
-
out_df = out_df.
|
265
|
-
filt_df = filt_df.
|
263
|
+
med = med.convert_dtypes()
|
264
|
+
out_df = out_df.convert_dtypes()
|
265
|
+
filt_df = filt_df.convert_dtypes()
|
266
266
|
|
267
267
|
self.yhat = med.sort_index()
|
268
268
|
self.outliers = out_df.sort_index()
|
@@ -334,9 +334,9 @@ class OutlierDetection:
|
|
334
334
|
roll_mean = np.exp(roll_mean)
|
335
335
|
|
336
336
|
# type conversion
|
337
|
-
roll_mean = roll_mean.
|
338
|
-
out_df = out_df.
|
339
|
-
filt_df = filt_df.
|
337
|
+
roll_mean = roll_mean.convert_dtypes()
|
338
|
+
out_df = out_df.convert_dtypes()
|
339
|
+
filt_df = filt_df.convert_dtypes()
|
340
340
|
|
341
341
|
self.yhat = roll_mean.sort_index()
|
342
342
|
self.outliers = out_df.sort_index()
|
@@ -382,9 +382,9 @@ class OutlierDetection:
|
|
382
382
|
ewma = np.exp(ewma)
|
383
383
|
|
384
384
|
# type conversion
|
385
|
-
ewma = ewma.
|
386
|
-
out_df = out_df.
|
387
|
-
filt_df = filt_df.
|
385
|
+
ewma = ewma.convert_dtypes()
|
386
|
+
out_df = out_df.convert_dtypes()
|
387
|
+
filt_df = filt_df.convert_dtypes()
|
388
388
|
|
389
389
|
self.yhat = ewma.sort_index()
|
390
390
|
self.outliers = out_df.sort_index()
|
@@ -491,14 +491,14 @@ class OutlierDetection:
|
|
491
491
|
filt_df = self.df.unstack()[resid_df.abs() < self.thresh_val]
|
492
492
|
|
493
493
|
# stack and reindex
|
494
|
-
out_df = out_df.stack().reindex(mult_idx)
|
495
|
-
filt_df = filt_df.stack().reindex(mult_idx)
|
496
|
-
yhat_df = yhat_df.stack().reindex(mult_idx)
|
494
|
+
out_df = out_df.stack(future_stack=True).reindex(mult_idx)
|
495
|
+
filt_df = filt_df.stack(future_stack=True).reindex(mult_idx)
|
496
|
+
yhat_df = yhat_df.stack(future_stack=True).reindex(mult_idx)
|
497
497
|
|
498
498
|
# convert dtypes
|
499
|
-
yhat_df = yhat_df.
|
500
|
-
out_df = out_df.
|
501
|
-
filt_df = filt_df.
|
499
|
+
yhat_df = yhat_df.convert_dtypes()
|
500
|
+
out_df = out_df.convert_dtypes()
|
501
|
+
filt_df = filt_df.convert_dtypes()
|
502
502
|
|
503
503
|
self.yhat = yhat_df.sort_index()
|
504
504
|
self.outliers = out_df.sort_index()
|
@@ -634,14 +634,14 @@ class OutlierDetection:
|
|
634
634
|
filt_df = self.df.unstack()[resid_df.abs() < self.thresh_val]
|
635
635
|
|
636
636
|
# stack and reindex
|
637
|
-
out_df = out_df.stack().reindex(mult_idx)
|
638
|
-
filt_df = filt_df.stack().reindex(mult_idx)
|
639
|
-
yhat_df = yhat_df.stack().reindex(mult_idx)
|
637
|
+
out_df = out_df.stack(future_stack=True).reindex(mult_idx)
|
638
|
+
filt_df = filt_df.stack(future_stack=True).reindex(mult_idx)
|
639
|
+
yhat_df = yhat_df.stack(future_stack=True).reindex(mult_idx)
|
640
640
|
|
641
641
|
# convert dtypes
|
642
|
-
yhat_df = yhat_df.
|
643
|
-
out_df = out_df.
|
644
|
-
filt_df = filt_df.
|
642
|
+
yhat_df = yhat_df.convert_dtypes()
|
643
|
+
out_df = out_df.convert_dtypes()
|
644
|
+
filt_df = filt_df.convert_dtypes()
|
645
645
|
|
646
646
|
self.yhat = yhat_df.sort_index()
|
647
647
|
self.outliers = out_df.sort_index()
|
@@ -744,14 +744,14 @@ class OutlierDetection:
|
|
744
744
|
filt_df = self.df.unstack()[self.df.unstack().lt(yhat_upper) & self.df.unstack().gt(yhat_lower)]
|
745
745
|
|
746
746
|
# stack and reindex
|
747
|
-
yhat_df = yhat_df.stack().reindex(mult_idx)
|
748
|
-
out_df = out_df.stack().reindex(mult_idx)
|
749
|
-
filt_df = filt_df.stack().reindex(mult_idx)
|
747
|
+
yhat_df = yhat_df.stack(future_stack=True).reindex(mult_idx)
|
748
|
+
out_df = out_df.stack(future_stack=True).reindex(mult_idx)
|
749
|
+
filt_df = filt_df.stack(future_stack=True).reindex(mult_idx)
|
750
750
|
|
751
751
|
# convert dtypes
|
752
|
-
yhat_df = yhat_df.
|
753
|
-
out_df = out_df.
|
754
|
-
filt_df = filt_df.
|
752
|
+
yhat_df = yhat_df.convert_dtypes()
|
753
|
+
out_df = out_df.convert_dtypes()
|
754
|
+
filt_df = filt_df.convert_dtypes()
|
755
755
|
|
756
756
|
self.yhat = yhat_df.sort_index()
|
757
757
|
self.outliers = out_df.sort_index()
|
@@ -528,7 +528,6 @@ class WrangleData:
|
|
528
528
|
-------
|
529
529
|
pd.DataFrame
|
530
530
|
Wrangled dataframe into tidy data format.
|
531
|
-
|
532
531
|
"""
|
533
532
|
# convert fields to lib
|
534
533
|
self.convert_fields_to_lib(data_source='coinmetrics')
|
@@ -881,8 +880,17 @@ class WrangleData:
|
|
881
880
|
self.data_resp.columns = self.data_req.tickers # convert tickers to cryptodatapy format
|
882
881
|
|
883
882
|
# resample to match end of reporting period, not beginning
|
884
|
-
self.data_resp =
|
885
|
-
|
883
|
+
self.data_resp = (
|
884
|
+
self.data_resp
|
885
|
+
.resample('d')
|
886
|
+
.last()
|
887
|
+
.ffill()
|
888
|
+
.resample(self.data_req.freq)
|
889
|
+
.last()
|
890
|
+
.stack(future_stack=True)
|
891
|
+
.to_frame()
|
892
|
+
.reset_index()
|
893
|
+
)
|
886
894
|
|
887
895
|
# convert cols
|
888
896
|
if self.data_req.cat == 'macro':
|
@@ -983,7 +991,7 @@ class WrangleData:
|
|
983
991
|
|
984
992
|
# format index
|
985
993
|
self.data_resp.index.name = 'date' # rename
|
986
|
-
self.data_resp = self.data_resp.stack().to_frame('er')
|
994
|
+
self.data_resp = self.data_resp.stack(future_stack=True).to_frame('er')
|
987
995
|
self.data_resp.index.names = ['date', 'ticker']
|
988
996
|
|
989
997
|
# type and conversion to decimals
|
@@ -1010,7 +1018,7 @@ class WrangleData:
|
|
1010
1018
|
with resources.path("cryptodatapy.conf", "tickers.csv") as f:
|
1011
1019
|
tickers_path = f
|
1012
1020
|
tickers_df = pd.read_csv(tickers_path, index_col=0, encoding="latin1")
|
1013
|
-
self.data_resp = self.data_resp.stack().to_frame() # stack df
|
1021
|
+
self.data_resp = self.data_resp.stack(future_stack=True).to_frame() # stack df
|
1014
1022
|
# create list of tickers using tickers csv
|
1015
1023
|
tickers = []
|
1016
1024
|
for row in self.data_resp.iterrows():
|
@@ -1092,11 +1100,11 @@ class WrangleData:
|
|
1092
1100
|
self.data_resp = df
|
1093
1101
|
self.filter_dates()
|
1094
1102
|
# stack df
|
1095
|
-
self.data_resp = self.data_resp.stack().to_frame('er')
|
1103
|
+
self.data_resp = self.data_resp.stack(future_stack=True).to_frame('er')
|
1096
1104
|
# create multi index
|
1097
1105
|
self.data_resp.index.names = ['date', 'ticker']
|
1098
1106
|
# type and conversion to decimals
|
1099
|
-
self.data_resp = self.data_resp.
|
1107
|
+
self.data_resp = self.data_resp.convert_dtypes()
|
1100
1108
|
# remove bad data
|
1101
1109
|
self.data_resp = self.data_resp[~self.data_resp.index.duplicated()] # duplicate rows
|
1102
1110
|
self.data_resp = self.data_resp.dropna(how='all').dropna(how='all', axis=1) # entire row or col NaNs
|
@@ -27,18 +27,18 @@ cryptodatapy/extract/data_vendors/.ipynb_checkpoints/InvestPy-checkpoint.ipynb,s
|
|
27
27
|
cryptodatapy/extract/data_vendors/.ipynb_checkpoints/NasdaqDataLink-checkpoint.ipynb,sha256=hY2QkCcTiLgPnl8SQPsO8spio5-RBMGeBLYzAwgSWb4,147170
|
28
28
|
cryptodatapy/extract/data_vendors/.ipynb_checkpoints/PandasDataReader-checkpoint.ipynb,sha256=n7vzOV6AxC_Ti5CLWW2ABMEEcbbBpiBBs4qTUBQinIg,24171
|
29
29
|
cryptodatapy/extract/data_vendors/__init__.py,sha256=Nk6gcT43d0XOLfrlVA9r--5mvHCgHfq295IKL3Puu74,354
|
30
|
-
cryptodatapy/extract/data_vendors/coinmetrics_api.py,sha256=
|
30
|
+
cryptodatapy/extract/data_vendors/coinmetrics_api.py,sha256=VdUjTn0DPsXFmwzD_JHZNhGUoaEAv1nwMYpT0ZcY2fs,32972
|
31
31
|
cryptodatapy/extract/data_vendors/cryptocompare_api.py,sha256=3oBfQioBz1vrs9JNtwE0hBLI4BTtpFBBEEsDawmobE8,28872
|
32
32
|
cryptodatapy/extract/data_vendors/datavendor.py,sha256=kGKxHcPng6JiGGhcuPx87ij0DXl4E-OSqxlvxhJ1HQo,12642
|
33
33
|
cryptodatapy/extract/data_vendors/glassnode_api.py,sha256=PuuJOjHztoJyFijb5XU1zm1S_2NAj7MX-wC89DL_bWQ,13103
|
34
|
-
cryptodatapy/extract/data_vendors/tiingo_api.py,sha256=
|
35
|
-
cryptodatapy/extract/datarequest.py,sha256=
|
34
|
+
cryptodatapy/extract/data_vendors/tiingo_api.py,sha256=kcrDZ3ndG-E6M8L9qnXYShgCdXDLWnq4npX2QAGmcgM,25413
|
35
|
+
cryptodatapy/extract/datarequest.py,sha256=EPq_7itSjwoqLY1v3HLQRbiFbSMCOzFmA2xGpaM3PQc,25215
|
36
36
|
cryptodatapy/extract/exchanges/__init__.py,sha256=VKTNzrbe-wltGHWH9lK5RLZoXCGHp-UGGZ4gMVHJXrQ,113
|
37
37
|
cryptodatapy/extract/exchanges/dydx.py,sha256=Oifb4sKbPRKArdZBx9q5ob4yTFkd65n0zXiS7hga0mk,4199
|
38
38
|
cryptodatapy/extract/exchanges/exchange.py,sha256=Cicj3KS4zLbwmXX5fu89byXNwqqU4TH31GFv0zj3D4s,13010
|
39
39
|
cryptodatapy/extract/getdata.py,sha256=BQ-OCfoQqPTYN1GtSI8t93iPpCUbtwiXC0BDxbrdKKM,11477
|
40
40
|
cryptodatapy/extract/libraries/__init__.py,sha256=9rJ_hFHWlvkPwyIkNG5bqH6HTY2jQNPIKQjzYEsVSDo,319
|
41
|
-
cryptodatapy/extract/libraries/ccxt_api.py,sha256=
|
41
|
+
cryptodatapy/extract/libraries/ccxt_api.py,sha256=5aPGuZFQ1HwKCxE9Y5UlapNK77lMfnsWEi64HpSkK44,56766
|
42
42
|
cryptodatapy/extract/libraries/dbnomics_api.py,sha256=M6kPIH-hKqkmeBQb-g56dY9jatqLCtSl_MnvPblHtAc,9421
|
43
43
|
cryptodatapy/extract/libraries/investpy_api.py,sha256=qtGm3LDluXxJorvFv0w1bm1oBrcZIfE5cZSYzNYvttY,18409
|
44
44
|
cryptodatapy/extract/libraries/library.py,sha256=eU8NnQZ9luLGdIF5hms6j8VPCWc50evkREc4xdh-g1I,12301
|
@@ -48,15 +48,15 @@ cryptodatapy/extract/web/aqr.py,sha256=LS1D7QzG6UWkLUfDMgBFtiHpznnnAUOpec5Sx3vRG
|
|
48
48
|
cryptodatapy/extract/web/web.py,sha256=R1xEnHE1McxSWxp4vrTfgh9gW6FF6XDlp0gmp2NmWOM,12126
|
49
49
|
cryptodatapy/transform/__init__.py,sha256=Spb5cGJ3V_o8hgSWOSrF8J_vsSZpFk0uzW7RpkgfbFE,131
|
50
50
|
cryptodatapy/transform/clean.py,sha256=C9VypQOjdJ987TcD-qAHh7qYaoJBotvp3cWTr3ttSGM,12807
|
51
|
-
cryptodatapy/transform/convertparams.py,sha256=
|
51
|
+
cryptodatapy/transform/convertparams.py,sha256=VWQMk2w5SOyRGXENy-TlOjeVmtwlcxg-DOTTnM_MFsE,43442
|
52
52
|
cryptodatapy/transform/filter.py,sha256=KYYyta0uREAjBkYTVyvhOCCLSKR_qPSlqj5Nl7l4iBk,9064
|
53
|
-
cryptodatapy/transform/impute.py,sha256=
|
54
|
-
cryptodatapy/transform/od.py,sha256=
|
55
|
-
cryptodatapy/transform/wrangle.py,sha256=
|
53
|
+
cryptodatapy/transform/impute.py,sha256=_0-SX5nnPrYgJYT-HKwBGNkmWXRMy9-C2oeU6VqkQp0,5537
|
54
|
+
cryptodatapy/transform/od.py,sha256=4vxqxCKka8yzkcT9CJ1qthn8TSqo1-fnt0dIZrLl5_k,30386
|
55
|
+
cryptodatapy/transform/wrangle.py,sha256=dnaBi3zOXoQymkxN7r5Eneb9UccGhfp7UdCS31T41i8,42396
|
56
56
|
cryptodatapy/util/__init__.py,sha256=zSQ2HU2QIXzCuptJjknmrClwtQKCvIj4aNysZljIgrU,116
|
57
57
|
cryptodatapy/util/datacatalog.py,sha256=qCCX6srXvaAbVAKuA0M2y5IK_2OEx5xA3yRahDZlC-g,13157
|
58
58
|
cryptodatapy/util/datacredentials.py,sha256=fXuGgI2NKCLlcnK8M37CtdyAc3O_YCV23x3KTlfakjA,2160
|
59
|
-
cryptodatapy-0.2.
|
60
|
-
cryptodatapy-0.2.
|
61
|
-
cryptodatapy-0.2.
|
62
|
-
cryptodatapy-0.2.
|
59
|
+
cryptodatapy-0.2.14.dist-info/LICENSE,sha256=sw4oVq8bDjT3uMtaFebQ-xeIVP4H-bXldTs9q-Jjeks,11344
|
60
|
+
cryptodatapy-0.2.14.dist-info/METADATA,sha256=xqDhe1gQjohAbXUsIT9ami2dLsCX2f-D2zXFubcWtVk,6428
|
61
|
+
cryptodatapy-0.2.14.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
|
62
|
+
cryptodatapy-0.2.14.dist-info/RECORD,,
|
File without changes
|
File without changes
|