cryptodatapy 0.2.6__py3-none-any.whl → 0.2.7__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (16) hide show
  1. cryptodatapy/conf/fields.csv +1 -1
  2. cryptodatapy/extract/datarequest.py +142 -27
  3. cryptodatapy/extract/libraries/Untitled.ipynb +33 -0
  4. cryptodatapy/extract/libraries/ccxt.ipynb +747 -0
  5. cryptodatapy/extract/libraries/ccxt_api.py +629 -356
  6. cryptodatapy/extract/libraries/pandasdr_api.py +13 -12
  7. cryptodatapy/extract/libraries/yfinance_api.py +511 -0
  8. cryptodatapy/transform/clean_perp_futures_ohlcv.ipynb +226 -30
  9. cryptodatapy/transform/cmdty_data.ipynb +402 -0
  10. cryptodatapy/transform/convertparams.py +114 -166
  11. cryptodatapy/transform/eqty_data.ipynb +126 -99
  12. cryptodatapy/transform/wrangle.py +109 -20
  13. {cryptodatapy-0.2.6.dist-info → cryptodatapy-0.2.7.dist-info}/METADATA +9 -6
  14. {cryptodatapy-0.2.6.dist-info → cryptodatapy-0.2.7.dist-info}/RECORD +16 -12
  15. {cryptodatapy-0.2.6.dist-info → cryptodatapy-0.2.7.dist-info}/WHEEL +1 -1
  16. {cryptodatapy-0.2.6.dist-info → cryptodatapy-0.2.7.dist-info}/LICENSE +0 -0
cryptodatapy/transform/eqty_data.ipynb CHANGED
@@ -2,10 +2,19 @@
2
2
  "cells": [
3
3
  {
4
4
  "cell_type": "code",
5
- "execution_count": 44,
5
+ "execution_count": 1,
6
6
  "id": "9fea9fae",
7
7
  "metadata": {},
8
- "outputs": [],
8
+ "outputs": [
9
+ {
10
+ "name": "stderr",
11
+ "output_type": "stream",
12
+ "text": [
13
+ "fatal: bad revision 'HEAD'\n",
14
+ "Importing plotly failed. Interactive plots will not work.\n"
15
+ ]
16
+ }
17
+ ],
9
18
  "source": [
10
19
  "import pandas as pd\n",
11
20
  "import numpy as np\n",
@@ -30,7 +39,7 @@
30
39
  },
31
40
  {
32
41
  "cell_type": "code",
33
- "execution_count": 45,
42
+ "execution_count": 2,
34
43
  "id": "2ad72bc7-5fdd-4ae5-8d9e-e90118efcc26",
35
44
  "metadata": {},
36
45
  "outputs": [],
@@ -40,7 +49,7 @@
40
49
  },
41
50
  {
42
51
  "cell_type": "code",
43
- "execution_count": 46,
52
+ "execution_count": 3,
44
53
  "id": "0342bab7-a792-4be3-8d4d-44c4343d0e6a",
45
54
  "metadata": {},
46
55
  "outputs": [
@@ -76,7 +85,7 @@
76
85
  "Name: yahoo_id, dtype: object"
77
86
  ]
78
87
  },
79
- "execution_count": 46,
88
+ "execution_count": 3,
80
89
  "metadata": {},
81
90
  "output_type": "execute_result"
82
91
  }
@@ -95,7 +104,7 @@
95
104
  },
96
105
  {
97
106
  "cell_type": "code",
98
- "execution_count": 47,
107
+ "execution_count": 4,
99
108
  "id": "d875cd96-a29c-4e22-9806-a1b1c2513564",
100
109
  "metadata": {},
101
110
  "outputs": [],
@@ -105,7 +114,7 @@
105
114
  },
106
115
  {
107
116
  "cell_type": "code",
108
- "execution_count": 48,
117
+ "execution_count": 5,
109
118
  "id": "1bf0af0d-7ed7-4e07-9da1-5625b1f32bce",
110
119
  "metadata": {},
111
120
  "outputs": [
@@ -168,7 +177,7 @@
168
177
  " 'REET']"
169
178
  ]
170
179
  },
171
- "execution_count": 48,
180
+ "execution_count": 5,
172
181
  "metadata": {},
173
182
  "output_type": "execute_result"
174
183
  }
@@ -179,7 +188,7 @@
179
188
  },
180
189
  {
181
190
  "cell_type": "code",
182
- "execution_count": 49,
191
+ "execution_count": 6,
183
192
  "id": "0b1bc395-40dd-44c0-bac1-dd1e00f8a5c4",
184
193
  "metadata": {},
185
194
  "outputs": [],
@@ -189,7 +198,7 @@
189
198
  },
190
199
  {
191
200
  "cell_type": "code",
192
- "execution_count": 50,
201
+ "execution_count": 7,
193
202
  "id": "2674260a-56c7-40a4-9708-0eb335fa075d",
194
203
  "metadata": {},
195
204
  "outputs": [
@@ -207,7 +216,7 @@
207
216
  },
208
217
  {
209
218
  "cell_type": "code",
210
- "execution_count": 56,
219
+ "execution_count": 8,
211
220
  "id": "ee0523a8-c6ee-42b9-9410-36b26906f2de",
212
221
  "metadata": {},
213
222
  "outputs": [
@@ -425,78 +434,6 @@
425
434
  " <td>...</td>\n",
426
435
  " </tr>\n",
427
436
  " <tr>\n",
428
- " <th>2024-09-11</th>\n",
429
- " <td>64.93</td>\n",
430
- " <td>66.199997</td>\n",
431
- " <td>23.629999</td>\n",
432
- " <td>92.019997</td>\n",
433
- " <td>31.809999</td>\n",
434
- " <td>8.06</td>\n",
435
- " <td>69.559998</td>\n",
436
- " <td>50.709999</td>\n",
437
- " <td>25.52</td>\n",
438
- " <td>36.630001</td>\n",
439
- " <td>...</td>\n",
440
- " <td>61.66</td>\n",
441
- " <td>&lt;NA&gt;</td>\n",
442
- " <td>57.130001</td>\n",
443
- " <td>49.330002</td>\n",
444
- " <td>114.629997</td>\n",
445
- " <td>17.67</td>\n",
446
- " <td>26.59</td>\n",
447
- " <td>25.309999</td>\n",
448
- " <td>39.07</td>\n",
449
- " <td>15.0</td>\n",
450
- " </tr>\n",
451
- " <tr>\n",
452
- " <th>2024-09-12</th>\n",
453
- " <td>65.75</td>\n",
454
- " <td>67.93</td>\n",
455
- " <td>23.870001</td>\n",
456
- " <td>92.519997</td>\n",
457
- " <td>32.130001</td>\n",
458
- " <td>8.06</td>\n",
459
- " <td>70.25</td>\n",
460
- " <td>51.07</td>\n",
461
- " <td>25.450001</td>\n",
462
- " <td>36.790001</td>\n",
463
- " <td>...</td>\n",
464
- " <td>62.509998</td>\n",
465
- " <td>&lt;NA&gt;</td>\n",
466
- " <td>57.759998</td>\n",
467
- " <td>48.939999</td>\n",
468
- " <td>115.629997</td>\n",
469
- " <td>18.1</td>\n",
470
- " <td>26.780001</td>\n",
471
- " <td>25.85</td>\n",
472
- " <td>39.950001</td>\n",
473
- " <td>14.96</td>\n",
474
- " </tr>\n",
475
- " <tr>\n",
476
- " <th>2024-09-13</th>\n",
477
- " <td>66.139999</td>\n",
478
- " <td>68.529999</td>\n",
479
- " <td>24.120001</td>\n",
480
- " <td>93.5</td>\n",
481
- " <td>32.400002</td>\n",
482
- " <td>8.06</td>\n",
483
- " <td>69.870003</td>\n",
484
- " <td>51.389999</td>\n",
485
- " <td>25.93</td>\n",
486
- " <td>36.900002</td>\n",
487
- " <td>...</td>\n",
488
- " <td>63.299999</td>\n",
489
- " <td>&lt;NA&gt;</td>\n",
490
- " <td>57.849998</td>\n",
491
- " <td>48.790001</td>\n",
492
- " <td>116.209999</td>\n",
493
- " <td>18.139999</td>\n",
494
- " <td>27.01</td>\n",
495
- " <td>25.950001</td>\n",
496
- " <td>40.549999</td>\n",
497
- " <td>15.11</td>\n",
498
- " </tr>\n",
499
- " <tr>\n",
500
437
  " <th>2024-09-16</th>\n",
501
438
  " <td>66.760002</td>\n",
502
439
  " <td>68.5</td>\n",
@@ -544,9 +481,81 @@
544
481
  " <td>40.439999</td>\n",
545
482
  " <td>15.19</td>\n",
546
483
  " </tr>\n",
484
+ " <tr>\n",
485
+ " <th>2024-09-18</th>\n",
486
+ " <td>66.400002</td>\n",
487
+ " <td>68.769997</td>\n",
488
+ " <td>24.27</td>\n",
489
+ " <td>94.199997</td>\n",
490
+ " <td>32.580002</td>\n",
491
+ " <td>8.06</td>\n",
492
+ " <td>69.389999</td>\n",
493
+ " <td>51.16</td>\n",
494
+ " <td>26.42</td>\n",
495
+ " <td>36.990002</td>\n",
496
+ " <td>...</td>\n",
497
+ " <td>63.259998</td>\n",
498
+ " <td>&lt;NA&gt;</td>\n",
499
+ " <td>57.66</td>\n",
500
+ " <td>49.880001</td>\n",
501
+ " <td>116.160004</td>\n",
502
+ " <td>18.049999</td>\n",
503
+ " <td>26.889999</td>\n",
504
+ " <td>25.66</td>\n",
505
+ " <td>40.900002</td>\n",
506
+ " <td>15.07</td>\n",
507
+ " </tr>\n",
508
+ " <tr>\n",
509
+ " <th>2024-09-19</th>\n",
510
+ " <td>67.709999</td>\n",
511
+ " <td>70.150002</td>\n",
512
+ " <td>24.690001</td>\n",
513
+ " <td>95.419998</td>\n",
514
+ " <td>33.09</td>\n",
515
+ " <td>8.06</td>\n",
516
+ " <td>71.230003</td>\n",
517
+ " <td>51.509998</td>\n",
518
+ " <td>26.74</td>\n",
519
+ " <td>37.490002</td>\n",
520
+ " <td>...</td>\n",
521
+ " <td>63.91</td>\n",
522
+ " <td>&lt;NA&gt;</td>\n",
523
+ " <td>58.110001</td>\n",
524
+ " <td>47.720001</td>\n",
525
+ " <td>118.230003</td>\n",
526
+ " <td>18.219999</td>\n",
527
+ " <td>26.98</td>\n",
528
+ " <td>25.799999</td>\n",
529
+ " <td>41.400002</td>\n",
530
+ " <td>15.41</td>\n",
531
+ " </tr>\n",
532
+ " <tr>\n",
533
+ " <th>2024-09-20</th>\n",
534
+ " <td>68.089996</td>\n",
535
+ " <td>69.389999</td>\n",
536
+ " <td>24.549999</td>\n",
537
+ " <td>94.970001</td>\n",
538
+ " <td>32.689999</td>\n",
539
+ " <td>8.06</td>\n",
540
+ " <td>71.269997</td>\n",
541
+ " <td>50.66</td>\n",
542
+ " <td>26.940001</td>\n",
543
+ " <td>37.040001</td>\n",
544
+ " <td>...</td>\n",
545
+ " <td>63.32</td>\n",
546
+ " <td>&lt;NA&gt;</td>\n",
547
+ " <td>58.740002</td>\n",
548
+ " <td>47.43</td>\n",
549
+ " <td>117.760002</td>\n",
550
+ " <td>18.15</td>\n",
551
+ " <td>26.83</td>\n",
552
+ " <td>25.799999</td>\n",
553
+ " <td>41.349998</td>\n",
554
+ " <td>15.4</td>\n",
555
+ " </tr>\n",
547
556
  " </tbody>\n",
548
557
  "</table>\n",
549
- "<p>7965 rows × 54 columns</p>\n",
558
+ "<p>7968 rows × 54 columns</p>\n",
550
559
  "</div>"
551
560
  ],
552
561
  "text/plain": [
@@ -558,11 +567,11 @@
558
567
  "1993-02-03 <NA> <NA> <NA> <NA> <NA> <NA> \n",
559
568
  "1993-02-04 <NA> <NA> <NA> <NA> <NA> <NA> \n",
560
569
  "... ... ... ... ... ... ... \n",
561
- "2024-09-11 64.93 66.199997 23.629999 92.019997 31.809999 8.06 \n",
562
- "2024-09-12 65.75 67.93 23.870001 92.519997 32.130001 8.06 \n",
563
- "2024-09-13 66.139999 68.529999 24.120001 93.5 32.400002 8.06 \n",
564
570
  "2024-09-16 66.760002 68.5 24.48 94.160004 32.599998 8.06 \n",
565
571
  "2024-09-17 66.43 68.260002 24.15 94.330002 32.549999 8.06 \n",
572
+ "2024-09-18 66.400002 68.769997 24.27 94.199997 32.580002 8.06 \n",
573
+ "2024-09-19 67.709999 70.150002 24.690001 95.419998 33.09 8.06 \n",
574
+ "2024-09-20 68.089996 69.389999 24.549999 94.970001 32.689999 8.06 \n",
566
575
  "\n",
567
576
  "ticker EIDO EIRL EIS ENOR ... PAK PGAL \\\n",
568
577
  "date ... \n",
@@ -572,11 +581,11 @@
572
581
  "1993-02-03 <NA> <NA> <NA> <NA> ... <NA> <NA> \n",
573
582
  "1993-02-04 <NA> <NA> <NA> <NA> ... <NA> <NA> \n",
574
583
  "... ... ... ... ... ... ... ... \n",
575
- "2024-09-11 69.559998 50.709999 25.52 36.630001 ... 61.66 <NA> \n",
576
- "2024-09-12 70.25 51.07 25.450001 36.790001 ... 62.509998 <NA> \n",
577
- "2024-09-13 69.870003 51.389999 25.93 36.900002 ... 63.299999 <NA> \n",
578
584
  "2024-09-16 70.540001 51.580002 26.049999 37.299999 ... 63.52 <NA> \n",
579
585
  "2024-09-17 69.709999 51.400002 26.34 37.029999 ... 63.220001 <NA> \n",
586
+ "2024-09-18 69.389999 51.16 26.42 36.990002 ... 63.259998 <NA> \n",
587
+ "2024-09-19 71.230003 51.509998 26.74 37.490002 ... 63.91 <NA> \n",
588
+ "2024-09-20 71.269997 50.66 26.940001 37.040001 ... 63.32 <NA> \n",
580
589
  "\n",
581
590
  "ticker QAT REET SPY THD TUR UAE \\\n",
582
591
  "date \n",
@@ -586,11 +595,11 @@
586
595
  "1993-02-03 <NA> <NA> <NA> <NA> <NA> <NA> \n",
587
596
  "1993-02-04 <NA> <NA> <NA> <NA> <NA> <NA> \n",
588
597
  "... ... ... ... ... ... ... \n",
589
- "2024-09-11 57.130001 49.330002 114.629997 17.67 26.59 25.309999 \n",
590
- "2024-09-12 57.759998 48.939999 115.629997 18.1 26.780001 25.85 \n",
591
- "2024-09-13 57.849998 48.790001 116.209999 18.139999 27.01 25.950001 \n",
592
598
  "2024-09-16 58.130001 49.240002 116.610001 18.200001 27.17 26.07 \n",
593
599
  "2024-09-17 57.959999 50.299999 116.489998 18.049999 26.969999 25.82 \n",
600
+ "2024-09-18 57.66 49.880001 116.160004 18.049999 26.889999 25.66 \n",
601
+ "2024-09-19 58.110001 47.720001 118.230003 18.219999 26.98 25.799999 \n",
602
+ "2024-09-20 58.740002 47.43 117.760002 18.15 26.83 25.799999 \n",
594
603
  "\n",
595
604
  "ticker URTH VXX \n",
596
605
  "date \n",
@@ -600,16 +609,16 @@
600
609
  "1993-02-03 <NA> <NA> \n",
601
610
  "1993-02-04 <NA> <NA> \n",
602
611
  "... ... ... \n",
603
- "2024-09-11 39.07 15.0 \n",
604
- "2024-09-12 39.950001 14.96 \n",
605
- "2024-09-13 40.549999 15.11 \n",
606
612
  "2024-09-16 40.66 15.09 \n",
607
613
  "2024-09-17 40.439999 15.19 \n",
614
+ "2024-09-18 40.900002 15.07 \n",
615
+ "2024-09-19 41.400002 15.41 \n",
616
+ "2024-09-20 41.349998 15.4 \n",
608
617
  "\n",
609
- "[7965 rows x 54 columns]"
618
+ "[7968 rows x 54 columns]"
610
619
  ]
611
620
  },
612
- "execution_count": 56,
621
+ "execution_count": 8,
613
622
  "metadata": {},
614
623
  "output_type": "execute_result"
615
624
  }
@@ -620,10 +629,28 @@
620
629
  },
621
630
  {
622
631
  "cell_type": "code",
623
- "execution_count": null,
632
+ "execution_count": 10,
624
633
  "id": "83f30ee7-b686-4ee8-8c5f-9333d2be31c8",
625
634
  "metadata": {},
626
635
  "outputs": [],
636
+ "source": [
637
+ "msci_etf_df.to_parquet('s3://factorlab-data/global_msci_eqty_etf_data_daily.parquet')"
638
+ ]
639
+ },
640
+ {
641
+ "cell_type": "code",
642
+ "execution_count": null,
643
+ "id": "fa7fb5fc-e6f3-4a1e-bda5-365d09e2b3e0",
644
+ "metadata": {},
645
+ "outputs": [],
646
+ "source": []
647
+ },
648
+ {
649
+ "cell_type": "code",
650
+ "execution_count": null,
651
+ "id": "0ad35f12-5943-4640-96fb-07b7ea738abd",
652
+ "metadata": {},
653
+ "outputs": [],
627
654
  "source": []
628
655
  },
629
656
  {
cryptodatapy/transform/wrangle.py CHANGED
@@ -469,7 +469,6 @@ class WrangleData:
469
469
  """
470
470
  Wrangles time series data responses from various APIs into tidy data format.
471
471
  """
472
-
473
472
  def __init__(self, data_req: DataRequest, data_resp: Union[Dict[str, pd.DataFrame], pd.DataFrame]):
474
473
  """
475
474
  Constructor
@@ -484,6 +483,7 @@ class WrangleData:
484
483
  """
485
484
  self.data_req = data_req
486
485
  self.data_resp = data_resp
486
+ self.tidy_data = pd.DataFrame()
487
487
 
488
488
  def cryptocompare(self) -> pd.DataFrame:
489
489
  """
@@ -734,36 +734,125 @@ class WrangleData:
734
734
 
735
735
  return self.data_resp
736
736
 
737
- def ccxt(self) -> pd.DataFrame:
737
+ def ccxt_ohlcv(self) -> pd.DataFrame:
738
738
  """
739
- Wrangles CCXT data response to dataframe with tidy data format.
739
+ Wrangles CCXT OHLCV data response to dataframe with tidy data format.
740
740
 
741
741
  Returns
742
742
  -------
743
743
  pd.DataFrame
744
- Wrangled dataframe into tidy data format.
744
+ Dataframe with tidy data format.
745
+ """
746
+ # field cols
747
+ cols = ["date", "open", "high", "low", "close", "volume"]
748
+
749
+ # add tickers
750
+ for i in range(len(self.data_req.source_markets)):
751
+ df = pd.DataFrame(self.data_resp[i], columns=cols)
752
+ df['ticker'] = self.data_req.source_markets[i]
753
+ self.tidy_data = pd.concat([self.tidy_data, df])
754
+
755
+ # convert to datetime
756
+ self.tidy_data['date'] = pd.to_datetime(self.tidy_data['date'], unit='ms')
745
757
 
758
+ # set index
759
+ self.tidy_data = self.tidy_data.set_index(['date', 'ticker']).sort_index()
760
+
761
+ return self.tidy_data
762
+
763
+ def ccxt_funding_rates(self) -> pd.DataFrame:
746
764
  """
747
- # convert fields to lib
765
+ Wrangles CCXT funding rates data response to dataframe with tidy data format.
766
+
767
+ Returns
768
+ -------
769
+ pd.DataFrame
770
+ Dataframe with tidy data format.
771
+ """
772
+ # add tickers
773
+ for i in range(len(self.data_req.source_markets)):
774
+ df = pd.DataFrame(self.data_resp[i])
775
+ self.tidy_data = pd.concat([self.tidy_data, df])
776
+ self.tidy_data = self.tidy_data[['symbol', 'fundingRate', 'datetime']]
777
+ self.data_resp = self.tidy_data
778
+
779
+ # convert to lib fields
748
780
  self.convert_fields_to_lib(data_source='ccxt')
781
+ self.tidy_data = self.data_resp
782
+
749
783
  # convert to datetime
750
- if 'close' in self.data_resp.columns:
751
- self.data_resp['date'] = pd.to_datetime(self.data_resp.date, unit='ms')
752
- elif 'funding_rate' in self.data_resp.columns:
753
- self.data_resp['date'] = pd.to_datetime(self.data_resp.set_index('date').index).floor('s').tz_localize(None)
784
+ self.tidy_data['date'] = pd.to_datetime(self.tidy_data.set_index('date').index).floor('s').tz_localize(None)
785
+
754
786
  # set index
755
- self.data_resp = self.data_resp.set_index('date').sort_index()
787
+ self.tidy_data = self.tidy_data.set_index(['date', 'ticker']).sort_index()
788
+
756
789
  # resample
757
- if 'funding_rate' in self.data_resp.columns and self.data_req.freq in ['d', 'w', 'm', 'q', 'y']:
758
- self.data_resp = ((self.data_resp.funding_rate + 1).resample(self.data_req.freq).prod() - 1).to_frame()
790
+ if self.data_req.freq in ['d', 'w', 'm', 'q', 'y']:
791
+ self.tidy_data = (
792
+ (1 + self.tidy_data.funding_rate)
793
+ .groupby('ticker')
794
+ .resample('d', level='date')
795
+ .prod() - 1
796
+ ).to_frame().swaplevel('ticker', 'date').sort_index()
797
+
798
+ return self.tidy_data
799
+
800
+ def ccxt_open_interest(self) -> pd.DataFrame:
801
+ """
802
+ Wrangles CCXT open interest data response to dataframe with tidy data format.
803
+
804
+ Returns
805
+ -------
806
+ pd.DataFrame
807
+ Dataframe with tidy data format.
808
+ """
809
+ # add tickers
810
+ for i in range(len(self.data_req.source_markets)):
811
+ df = pd.DataFrame(self.data_resp[i])
812
+ self.tidy_data = pd.concat([self.tidy_data, df])
813
+ self.tidy_data = self.tidy_data[['symbol', 'openInterestAmount', 'datetime']]
814
+ self.data_resp = self.tidy_data
815
+
816
+ # convert to lib fields
817
+ self.convert_fields_to_lib(data_source='ccxt')
818
+ self.tidy_data = self.data_resp
819
+
820
+ # convert to datetime
821
+ self.tidy_data['date'] = pd.to_datetime(self.tidy_data.set_index('date').index).floor('s').tz_localize(None)
822
+
823
+ # set index
824
+ self.tidy_data = self.tidy_data.set_index(['date', 'ticker']).sort_index()
825
+
826
+ return self.tidy_data
827
+
828
+ def ccxt(self, data_type: str) -> pd.DataFrame:
829
+ """
830
+ Wrangles CCXT data response to dataframe with tidy data format.
831
+
832
+ Returns
833
+ -------
834
+ pd.DataFrame
835
+ Wrangled dataframe into tidy data format.
836
+
837
+ """
838
+ if data_type == 'ohlcv':
839
+ self.tidy_data = self.ccxt_ohlcv()
840
+ elif data_type == 'funding_rates':
841
+ self.tidy_data = self.ccxt_funding_rates()
842
+ elif data_type == 'open_interest':
843
+ self.tidy_data = self.ccxt_open_interest()
844
+ else:
845
+ raise ValueError(f"Data type {data_type} not supported.")
846
+
759
847
  # type conversion
760
- self.data_resp = self.data_resp.apply(pd.to_numeric, errors='coerce').convert_dtypes()
848
+ self.tidy_data = self.tidy_data.apply(pd.to_numeric, errors='coerce').convert_dtypes()
849
+
761
850
  # remove bad data
762
- self.data_resp = self.data_resp[self.data_resp != 0] # 0 values
763
- self.data_resp = self.data_resp[~self.data_resp.index.duplicated()] # duplicate rows
764
- self.data_resp = self.data_resp.dropna(how='all').dropna(how='all', axis=1) # entire row or col NaNs
851
+ self.tidy_data = self.tidy_data[self.tidy_data != 0] # 0 values
852
+ self.tidy_data = self.tidy_data[~self.tidy_data.index.duplicated()] # duplicate rows
853
+ self.tidy_data = self.tidy_data.dropna(how='all').dropna(how='all', axis=1) # entire row or col NaNs
765
854
 
766
- return self.data_resp
855
+ return self.tidy_data
767
856
 
768
857
  def fred(self) -> pd.DataFrame:
769
858
  """
@@ -1016,9 +1105,9 @@ class WrangleData:
1016
1105
 
1017
1106
  # loop through data resp cols
1018
1107
  for col in self.data_resp.columns:
1019
- if self.data_req.source_fields is not None and col in self.data_req.source_fields:
1020
- pass
1021
- elif col in fields_list or col.title() in fields_list or col.lower() in fields_list:
1108
+ # if self.data_req.source_fields is not None and col in self.data_req.source_fields:
1109
+ # pass
1110
+ if col in fields_list or col.title() in fields_list or col.lower() in fields_list:
1022
1111
  self.data_resp.rename(columns={col: fields_df[(fields_df[str(data_source) + '_id']
1023
1112
  == col.title()) |
1024
1113
  (fields_df[str(data_source) + '_id'] == col.lower()) |
{cryptodatapy-0.2.6.dist-info → cryptodatapy-0.2.7.dist-info}/METADATA RENAMED
@@ -1,18 +1,20 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: cryptodatapy
3
- Version: 0.2.6
3
+ Version: 0.2.7
4
4
  Summary: Cryptoasset data library
5
5
  License: Apache-2.0
6
6
  Author: Systamental
7
7
  Requires-Python: >=3.8,<4.0
8
8
  Classifier: License :: OSI Approved :: Apache Software License
9
9
  Classifier: Programming Language :: Python :: 3
10
- Classifier: Programming Language :: Python :: 3.10
11
10
  Classifier: Programming Language :: Python :: 3.8
12
11
  Classifier: Programming Language :: Python :: 3.9
12
+ Classifier: Programming Language :: Python :: 3.10
13
+ Classifier: Programming Language :: Python :: 3.11
14
+ Classifier: Programming Language :: Python :: 3.12
13
15
  Requires-Dist: DBnomics (>=1.2.3)
14
16
  Requires-Dist: ccxt (>=1.91.52)
15
- Requires-Dist: coinmetrics-api-client (>=2022.6.17); python_version >= "3.7"
17
+ Requires-Dist: coinmetrics-api-client (>=2022.6.17) ; python_version >= "3.7"
16
18
  Requires-Dist: fsspec (>=2024.6.1)
17
19
  Requires-Dist: investpy (>=1.0.8)
18
20
  Requires-Dist: matplotlib (>=3.5.2)
@@ -20,13 +22,14 @@ Requires-Dist: numpy (>=1.23.2)
20
22
  Requires-Dist: openpyxl (>=3.1.2)
21
23
  Requires-Dist: pandas (>=1.4.4)
22
24
  Requires-Dist: pandas-datareader (>=0.10.0)
23
- Requires-Dist: prophet (>=1.1); python_version >= "3.7"
25
+ Requires-Dist: prophet (>=1.1) ; python_version >= "3.7"
24
26
  Requires-Dist: pyarrow (>=17.0.0)
25
- Requires-Dist: requests (>=2.28.0); python_version >= "3.7"
27
+ Requires-Dist: requests (>=2.28.0) ; python_version >= "3.7"
26
28
  Requires-Dist: responses (>=0.21.0)
27
- Requires-Dist: s3fs (>=2024.6.1,<2025.0.0)
29
+ Requires-Dist: s3fs (>=2024.6.1)
28
30
  Requires-Dist: selenium (>=4.4.3)
29
31
  Requires-Dist: statsmodels (>=0.13.2)
32
+ Requires-Dist: tdqm (>=0.0.1)
30
33
  Requires-Dist: webdriver-manager (>=3.8.3)
31
34
  Requires-Dist: xlrd (>=2.0.1)
32
35
  Requires-Dist: yfinance (>=0.2.14)
{cryptodatapy-0.2.6.dist-info → cryptodatapy-0.2.7.dist-info}/RECORD RENAMED
@@ -1,6 +1,6 @@
1
1
  cryptodatapy/__init__.py,sha256=ee1UaINHZn1A_SZ96XM3hCguQEJgiPTvKlnYsk3mmS4,185
2
2
  cryptodatapy/conf/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
3
- cryptodatapy/conf/fields.csv,sha256=6gl7HSZyuNeDMEuaxn4XzMZdRfF2bjGFob3Th5olfko,25717
3
+ cryptodatapy/conf/fields.csv,sha256=Xjs_sWIY4DuhdHlWOPo0pgDC8sHYA6xzf6kb0PuUp9w,25735
4
4
  cryptodatapy/conf/fx_tickers.csv,sha256=vqbY93_6Zi4vXg8iu0veXZ-NDm_NV2rrmb5lNYRqNUA,288
5
5
  cryptodatapy/conf/tickers.csv,sha256=Bs9KfDKawoUPKIQZMN8CtoLYJuzOkwLh2il30c8CsqE,357066
6
6
  cryptodatapy/datasets/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -34,14 +34,17 @@ cryptodatapy/extract/data_vendors/cryptocompare_api.py,sha256=3oBfQioBz1vrs9JNtw
34
34
  cryptodatapy/extract/data_vendors/datavendor.py,sha256=kGKxHcPng6JiGGhcuPx87ij0DXl4E-OSqxlvxhJ1HQo,12642
35
35
  cryptodatapy/extract/data_vendors/glassnode_api.py,sha256=PuuJOjHztoJyFijb5XU1zm1S_2NAj7MX-wC89DL_bWQ,13103
36
36
  cryptodatapy/extract/data_vendors/tiingo_api.py,sha256=Bvj5nF8zCkpU3cf5ImUmCS1cd1w2UtjgQvRmQ9Wfg6g,26404
37
- cryptodatapy/extract/datarequest.py,sha256=omKY6zxFwMMUaOGubpsBsBFBuWb4qZhQYEqVQobTls4,20582
37
+ cryptodatapy/extract/datarequest.py,sha256=KMfrc4WeHivNeIKN-sbLFWD7sqPrwasPFiee6QUWRuA,25015
38
38
  cryptodatapy/extract/getdata.py,sha256=HzWQyacfmphms97LVKbx1gEgcgsQJViBT4BBxL9TBXk,8703
39
+ cryptodatapy/extract/libraries/Untitled.ipynb,sha256=4SGovpmy_kLijEhzBhVJoYNVPrwrERRO0DL6gwzeRvM,614
39
40
  cryptodatapy/extract/libraries/__init__.py,sha256=9rJ_hFHWlvkPwyIkNG5bqH6HTY2jQNPIKQjzYEsVSDo,319
40
- cryptodatapy/extract/libraries/ccxt_api.py,sha256=_1r6wV2Eep6KlnUO8zKgjejUjsLE6bAuQBX1mGqKHNs,25197
41
+ cryptodatapy/extract/libraries/ccxt.ipynb,sha256=O-xkr_jtRBY4kuKWek61UOLaU5AiyNRM7AnquNLUjFs,22865
42
+ cryptodatapy/extract/libraries/ccxt_api.py,sha256=E_jVIKZ5iDjuTK-JwJFENvuWG08B24nN-NcVruWSoeQ,35882
41
43
  cryptodatapy/extract/libraries/dbnomics_api.py,sha256=M6kPIH-hKqkmeBQb-g56dY9jatqLCtSl_MnvPblHtAc,9421
42
44
  cryptodatapy/extract/libraries/investpy_api.py,sha256=qtGm3LDluXxJorvFv0w1bm1oBrcZIfE5cZSYzNYvttY,18409
43
45
  cryptodatapy/extract/libraries/library.py,sha256=070YsO1RJzm4z_enhCjqe5hrj8qsk-Ni0Q_QKoAwQ6U,12316
44
- cryptodatapy/extract/libraries/pandasdr_api.py,sha256=GdNi7ybPqSD2ieuUaaIyhTSmZG-YchMs_P4WQIy5JBw,12943
46
+ cryptodatapy/extract/libraries/pandasdr_api.py,sha256=DFko-N_ZmINUnzmXIct_CgnwEeoTh1xkaXewcJXdR2k,13177
47
+ cryptodatapy/extract/libraries/yfinance_api.py,sha256=E4c8gIpDh5ta8ILsn9SBs3C1pOU1VP4OqwQb6TcOzCc,17311
45
48
  cryptodatapy/extract/web/__init__.py,sha256=8i0fweCeqSpdiPf-47jT240I4ca6SizCu9aD-qDS67w,89
46
49
  cryptodatapy/extract/web/aqr.py,sha256=LS1D7QzG6UWkLUfDMgBFtiHpznnnAUOpec5Sx3vRGME,11875
47
50
  cryptodatapy/extract/web/web.py,sha256=27cAzlIyYn6R29726J7p9NhSwHypas9EQSjHLILtcjk,9748
@@ -49,21 +52,22 @@ cryptodatapy/transform/__init__.py,sha256=Spb5cGJ3V_o8hgSWOSrF8J_vsSZpFk0uzW7Rpk
49
52
  cryptodatapy/transform/cc_onchain_data.csv,sha256=qA9u3hekHk_NueBlMYQ7IKATh7AlnY-EN9E9X-9kIsU,9544500
50
53
  cryptodatapy/transform/clean.py,sha256=C9VypQOjdJ987TcD-qAHh7qYaoJBotvp3cWTr3ttSGM,12807
51
54
  cryptodatapy/transform/clean_onchain_data.ipynb,sha256=WrVPs8_WVKEgL6XRvGUATzeinqGUDTbXv_CHivg0nXg,687176
52
- cryptodatapy/transform/clean_perp_futures_ohlcv.ipynb,sha256=oLdksjQgbLZ9DXx7b824TwD2EC0zdffVuiquapWiFis,67207
53
- cryptodatapy/transform/convertparams.py,sha256=nnpzNLmo8-pY6kveLF-nW3iOPAsGfDXv7WJDTRDo0NI,44831
55
+ cryptodatapy/transform/clean_perp_futures_ohlcv.ipynb,sha256=3TFTG6riUfu5f0uYvlMC44iUtQRd27sQPxBMXBXzp6A,72758
56
+ cryptodatapy/transform/cmdty_data.ipynb,sha256=McAMfzNDfrv61gSlzFOkw_DXaOGZE1qfqXc2E_KeSbs,1220371
57
+ cryptodatapy/transform/convertparams.py,sha256=RxAOgtmmFaffX4cs0drLlG-jn3erF5K8BQ8EzlviZDo,43261
54
58
  cryptodatapy/transform/credit_data.ipynb,sha256=Wvvnu9ejsmqCb0s3cTG8bLJaywWQCskgk6FBd5J5Vf8,1892822
55
- cryptodatapy/transform/eqty_data.ipynb,sha256=FvxNz77g86S8SnSXazmrHr_HkW9N9nA65Bs5scTvrlE,24997
59
+ cryptodatapy/transform/eqty_data.ipynb,sha256=A5cA13hOPrOe7Fra0HL4QPFkJGVfArigTR0GUUBpQ3A,25609
56
60
  cryptodatapy/transform/filter.py,sha256=iQDUXthEXVGcrZUZLjevhDqwf9oywEQHTIh6n_sxOhU,9056
57
61
  cryptodatapy/transform/global_credit_data_daily.parquet ,sha256=Dw27SX41AeSYcZyYlrGbwVe8KZM6c35TQ-gzCd2gU2I,745732
58
62
  cryptodatapy/transform/impute.py,sha256=c7qdgFg0qs_xuQnX0jazpt0wgASC0KElLZRuxTkeVKY,5519
59
63
  cryptodatapy/transform/od.py,sha256=z__CWiN70f1leqx12SS9pIvTggxpUPrg1falJIKMZCc,31031
60
64
  cryptodatapy/transform/rates_data.ipynb,sha256=olKY4t2j4sfjsCYlhupTgaviC6922HHGBr-y3f80qjQ,13358
61
65
  cryptodatapy/transform/us_rates_daily.csv,sha256=BIA4a6egQYrVsLk51IZ54ZXXWMwjrx_t5S4XMdvHg44,6434830
62
- cryptodatapy/transform/wrangle.py,sha256=cBPV2Ub4RmVCdFdx3UlOlI2s4Rc_zzoMAXWcJvqbehs,40157
66
+ cryptodatapy/transform/wrangle.py,sha256=uYOrV4PQ3I0z4C5K3x468w82U3uhGRTEIPVrabaSS04,42987
63
67
  cryptodatapy/util/__init__.py,sha256=zSQ2HU2QIXzCuptJjknmrClwtQKCvIj4aNysZljIgrU,116
64
68
  cryptodatapy/util/datacatalog.py,sha256=qCCX6srXvaAbVAKuA0M2y5IK_2OEx5xA3yRahDZlC-g,13157
65
69
  cryptodatapy/util/datacredentials.py,sha256=fXuGgI2NKCLlcnK8M37CtdyAc3O_YCV23x3KTlfakjA,2160
66
- cryptodatapy-0.2.6.dist-info/LICENSE,sha256=sw4oVq8bDjT3uMtaFebQ-xeIVP4H-bXldTs9q-Jjeks,11344
67
- cryptodatapy-0.2.6.dist-info/WHEEL,sha256=y3eDiaFVSNTPbgzfNn0nYn5tEn1cX6WrdetDlQM4xWw,83
68
- cryptodatapy-0.2.6.dist-info/METADATA,sha256=d-7_dl8LX9OwKRwxgcHYYZxptq6GXJAYZofiqoNAVa0,6301
69
- cryptodatapy-0.2.6.dist-info/RECORD,,
70
+ cryptodatapy-0.2.7.dist-info/LICENSE,sha256=sw4oVq8bDjT3uMtaFebQ-xeIVP4H-bXldTs9q-Jjeks,11344
71
+ cryptodatapy-0.2.7.dist-info/METADATA,sha256=lwX6PFxeEg4RAqQo7LAteOZ2vxnh-NefSslgZtaQNzA,6426
72
+ cryptodatapy-0.2.7.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
73
+ cryptodatapy-0.2.7.dist-info/RECORD,,
{cryptodatapy-0.2.6.dist-info → cryptodatapy-0.2.7.dist-info}/WHEEL RENAMED
@@ -1,4 +1,4 @@
1
1
  Wheel-Version: 1.0
2
- Generator: poetry 1.0.7
2
+ Generator: poetry-core 1.9.0
3
3
  Root-Is-Purelib: true
4
4
  Tag: py3-none-any