cryptodatapy 0.2.3__py3-none-any.whl → 0.2.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (7) hide show
  1. cryptodatapy/transform/clean.py +43 -7
  2. cryptodatapy/transform/clean_perp_futures_ohlcv.ipynb +194 -808
  3. cryptodatapy/transform/filter.py +32 -4
  4. {cryptodatapy-0.2.3.dist-info → cryptodatapy-0.2.4.dist-info}/METADATA +4 -1
  5. {cryptodatapy-0.2.3.dist-info → cryptodatapy-0.2.4.dist-info}/RECORD +7 -7
  6. {cryptodatapy-0.2.3.dist-info → cryptodatapy-0.2.4.dist-info}/LICENSE +0 -0
  7. {cryptodatapy-0.2.3.dist-info → cryptodatapy-0.2.4.dist-info}/WHEEL +0 -0
cryptodatapy/transform/clean_perp_futures_ohlcv.ipynb CHANGED
@@ -2,10 +2,19 @@
2
2
  "cells": [
3
3
  {
4
4
  "cell_type": "code",
5
- "execution_count": 86,
5
+ "execution_count": 1,
6
6
  "id": "9fea9fae",
7
7
  "metadata": {},
8
- "outputs": [],
8
+ "outputs": [
9
+ {
10
+ "name": "stderr",
11
+ "output_type": "stream",
12
+ "text": [
13
+ "fatal: bad revision 'HEAD'\n",
14
+ "Importing plotly failed. Interactive plots will not work.\n"
15
+ ]
16
+ }
17
+ ],
9
18
  "source": [
10
19
  "import pandas as pd\n",
11
20
  "import numpy as np\n",
@@ -19,179 +28,6 @@
19
28
  "from cryptodatapy.transform.impute import Impute"
20
29
  ]
21
30
  },
22
- {
23
- "cell_type": "code",
24
- "execution_count": 94,
25
- "id": "c9876f0b",
26
- "metadata": {},
27
- "outputs": [],
28
- "source": [
29
- "data_req = DataRequest(source='tiingo')"
30
- ]
31
- },
32
- {
33
- "cell_type": "code",
34
- "execution_count": 95,
35
- "id": "e23c9751",
36
- "metadata": {},
37
- "outputs": [
38
- {
39
- "ename": "AttributeError",
40
- "evalue": "'Tiingo' object has no attribute 'get_top_mkt_cap_info'",
41
- "output_type": "error",
42
- "traceback": [
43
- "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
44
- "\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)",
45
- "Cell \u001b[0;32mIn [95], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m GetData(data_req)\u001b[38;5;241m.\u001b[39mget_meta(method\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mget_top_mkt_cap_info\u001b[39m\u001b[38;5;124m'\u001b[39m)\n",
46
- "File \u001b[0;32m~/projects/systamental/cryptodatapy/src/cryptodatapy/extract/getdata.py:145\u001b[0m, in \u001b[0;36mGetData.get_meta\u001b[0;34m(self, attr, method, **kwargs)\u001b[0m\n\u001b[1;32m 143\u001b[0m meta \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mgetattr\u001b[39m(ds, attr)\n\u001b[1;32m 144\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m method \u001b[38;5;129;01min\u001b[39;00m valid_meth:\n\u001b[0;32m--> 145\u001b[0m meta \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mgetattr\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mds\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m)\u001b[49m(\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m 146\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 147\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mAttributeError\u001b[39;00m(\n\u001b[1;32m 148\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mSelect a valid attribute or method. Valid attributes: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mvalid_attr\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 149\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m Valid methods include: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mvalid_meth\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 150\u001b[0m )\n",
47
- "\u001b[0;31mAttributeError\u001b[0m: 'Tiingo' object has no attribute 'get_top_mkt_cap_info'"
48
- ]
49
- }
50
- ],
51
- "source": [
52
- "GetData(data_req).get_meta(method='get_top_mkt_cap_info')"
53
- ]
54
- },
55
- {
56
- "cell_type": "code",
57
- "execution_count": 89,
58
- "id": "b8facf81",
59
- "metadata": {},
60
- "outputs": [
61
- {
62
- "data": {
63
- "text/html": [
64
- "<div>\n",
65
- "<style scoped>\n",
66
- " .dataframe tbody tr th:only-of-type {\n",
67
- " vertical-align: middle;\n",
68
- " }\n",
69
- "\n",
70
- " .dataframe tbody tr th {\n",
71
- " vertical-align: top;\n",
72
- " }\n",
73
- "\n",
74
- " .dataframe thead th {\n",
75
- " text-align: right;\n",
76
- " }\n",
77
- "</style>\n",
78
- "<table border=\"1\" class=\"dataframe\">\n",
79
- " <thead>\n",
80
- " <tr style=\"text-align: right;\">\n",
81
- " <th></th>\n",
82
- " <th></th>\n",
83
- " <th>close</th>\n",
84
- " </tr>\n",
85
- " <tr>\n",
86
- " <th>date</th>\n",
87
- " <th>ticker</th>\n",
88
- " <th></th>\n",
89
- " </tr>\n",
90
- " </thead>\n",
91
- " <tbody>\n",
92
- " <tr>\n",
93
- " <th>2019-01-31</th>\n",
94
- " <th>EURUSD</th>\n",
95
- " <td>1.147579</td>\n",
96
- " </tr>\n",
97
- " <tr>\n",
98
- " <th>2019-02-28</th>\n",
99
- " <th>EURUSD</th>\n",
100
- " <td>1.138693</td>\n",
101
- " </tr>\n",
102
- " <tr>\n",
103
- " <th>2019-03-31</th>\n",
104
- " <th>EURUSD</th>\n",
105
- " <td>1.122355</td>\n",
106
- " </tr>\n",
107
- " <tr>\n",
108
- " <th>2019-04-30</th>\n",
109
- " <th>EURUSD</th>\n",
110
- " <td>1.120699</td>\n",
111
- " </tr>\n",
112
- " <tr>\n",
113
- " <th>2019-05-31</th>\n",
114
- " <th>EURUSD</th>\n",
115
- " <td>1.11433</td>\n",
116
- " </tr>\n",
117
- " <tr>\n",
118
- " <th>...</th>\n",
119
- " <th>...</th>\n",
120
- " <td>...</td>\n",
121
- " </tr>\n",
122
- " <tr>\n",
123
- " <th>2024-04-30</th>\n",
124
- " <th>EURUSD</th>\n",
125
- " <td>1.066605</td>\n",
126
- " </tr>\n",
127
- " <tr>\n",
128
- " <th>2024-05-31</th>\n",
129
- " <th>EURUSD</th>\n",
130
- " <td>1.084825</td>\n",
131
- " </tr>\n",
132
- " <tr>\n",
133
- " <th>2024-06-30</th>\n",
134
- " <th>EURUSD</th>\n",
135
- " <td>1.073515</td>\n",
136
- " </tr>\n",
137
- " <tr>\n",
138
- " <th>2024-07-31</th>\n",
139
- " <th>EURUSD</th>\n",
140
- " <td>1.082395</td>\n",
141
- " </tr>\n",
142
- " <tr>\n",
143
- " <th>2024-08-31</th>\n",
144
- " <th>EURUSD</th>\n",
145
- " <td>1.092375</td>\n",
146
- " </tr>\n",
147
- " </tbody>\n",
148
- "</table>\n",
149
- "<p>68 rows × 1 columns</p>\n",
150
- "</div>"
151
- ],
152
- "text/plain": [
153
- " close\n",
154
- "date ticker \n",
155
- "2019-01-31 EURUSD 1.147579\n",
156
- "2019-02-28 EURUSD 1.138693\n",
157
- "2019-03-31 EURUSD 1.122355\n",
158
- "2019-04-30 EURUSD 1.120699\n",
159
- "2019-05-31 EURUSD 1.11433\n",
160
- "... ...\n",
161
- "2024-04-30 EURUSD 1.066605\n",
162
- "2024-05-31 EURUSD 1.084825\n",
163
- "2024-06-30 EURUSD 1.073515\n",
164
- "2024-07-31 EURUSD 1.082395\n",
165
- "2024-08-31 EURUSD 1.092375\n",
166
- "\n",
167
- "[68 rows x 1 columns]"
168
- ]
169
- },
170
- "execution_count": 89,
171
- "metadata": {},
172
- "output_type": "execute_result"
173
- }
174
- ],
175
- "source": [
176
- "fx_df"
177
- ]
178
- },
179
- {
180
- "cell_type": "code",
181
- "execution_count": null,
182
- "id": "005404ee",
183
- "metadata": {},
184
- "outputs": [],
185
- "source": []
186
- },
187
- {
188
- "cell_type": "code",
189
- "execution_count": null,
190
- "id": "8e3dc50d",
191
- "metadata": {},
192
- "outputs": [],
193
- "source": []
194
- },
195
31
  {
196
32
  "cell_type": "code",
197
33
  "execution_count": 2,
@@ -257,7 +93,7 @@
257
93
  {
258
94
  "data": {
259
95
  "text/plain": [
260
- "284"
96
+ "314"
261
97
  ]
262
98
  },
263
99
  "execution_count": 6,
@@ -327,7 +163,7 @@
327
163
  {
328
164
  "data": {
329
165
  "text/plain": [
330
- "257"
166
+ "0"
331
167
  ]
332
168
  },
333
169
  "execution_count": 11,
@@ -718,7 +554,7 @@
718
554
  },
719
555
  {
720
556
  "cell_type": "code",
721
- "execution_count": 61,
557
+ "execution_count": 23,
722
558
  "id": "f5ee4f6d",
723
559
  "metadata": {},
724
560
  "outputs": [],
@@ -729,7 +565,7 @@
729
565
  },
730
566
  {
731
567
  "cell_type": "code",
732
- "execution_count": 62,
568
+ "execution_count": 24,
733
569
  "id": "cbe07c91",
734
570
  "metadata": {},
735
571
  "outputs": [
@@ -838,7 +674,7 @@
838
674
  "2010-07-21 BTC 0.07474 0.07921 0.06634 0.07921 575.00 0.0"
839
675
  ]
840
676
  },
841
- "execution_count": 62,
677
+ "execution_count": 24,
842
678
  "metadata": {},
843
679
  "output_type": "execute_result"
844
680
  }
@@ -849,24 +685,85 @@
849
685
  },
850
686
  {
851
687
  "cell_type": "code",
852
- "execution_count": 63,
853
- "id": "bcbd3394",
688
+ "execution_count": 25,
689
+ "id": "d4c497d1",
854
690
  "metadata": {},
855
691
  "outputs": [],
856
692
  "source": [
857
- "# Filter data\n",
858
- "clean_df = CleanData(df).filter_outliers(od_method='mad', excl_cols=['volume', 'funding_rate'], thresh_val=10).\\\n",
693
+ "delisted_tickers = ['AGIX', 'CTK', 'CVC', 'CVX', 'DGB', 'FTT', 'GLMR', 'IDEX', 'MDT',\n",
694
+ " 'OCEAN', 'RAD', 'RAY', 'SC', 'SLP', 'SNT', 'STPT', 'STRAX', 'WAVES']"
695
+ ]
696
+ },
697
+ {
698
+ "cell_type": "code",
699
+ "execution_count": 26,
700
+ "id": "a9b1764c",
701
+ "metadata": {},
702
+ "outputs": [
703
+ {
704
+ "name": "stdout",
705
+ "output_type": "stream",
706
+ "text": [
707
+ "Index(['AGIX', 'CTK', 'CVC', 'CVX', 'DGB', 'FTT', 'GLMR', 'IDEX', 'MDT',\n",
708
+ " 'OCEAN', 'RAD', 'RAY', 'SC', 'SLP', 'SNT', 'STPT', 'STRAX', 'WAVES'],\n",
709
+ " dtype='object', name='ticker')\n"
710
+ ]
711
+ }
712
+ ],
713
+ "source": [
714
+ "# clean data\n",
715
+ "clean_df = CleanData(df).filter_delisted_tickers().\\\n",
716
+ " filter_min_nobs(ts_obs=1500, cs_obs=10).\\\n",
717
+ " filter_outliers(od_method='mad', excl_cols=['volume', 'funding_rate'], thresh_val=10).\\\n",
859
718
  " repair_outliers(imp_method='fcst').\\\n",
860
719
  " filter_avg_trading_val(thresh_val=1000000).\\\n",
861
720
  " filter_missing_vals_gaps().\\\n",
862
- " filter_min_nobs(ts_obs=1000, cs_obs=3).\\\n",
863
721
  " get(attr='df').dropna(how='all')"
864
722
  ]
865
723
  },
866
724
  {
867
725
  "cell_type": "code",
868
- "execution_count": 64,
869
- "id": "66c762a3",
726
+ "execution_count": 27,
727
+ "id": "3d423e53",
728
+ "metadata": {},
729
+ "outputs": [
730
+ {
731
+ "ename": "KeyError",
732
+ "evalue": "'OCEAN'",
733
+ "output_type": "error",
734
+ "traceback": [
735
+ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
736
+ "\u001b[0;31mKeyError\u001b[0m Traceback (most recent call last)",
737
+ "File \u001b[0;32m~/opt/anaconda3/envs/cryptodatapy/lib/python3.9/site-packages/pandas/core/indexes/base.py:3800\u001b[0m, in \u001b[0;36mIndex.get_loc\u001b[0;34m(self, key, method, tolerance)\u001b[0m\n\u001b[1;32m 3799\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m-> 3800\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_engine\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_loc\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcasted_key\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 3801\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mKeyError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m err:\n",
738
+ "File \u001b[0;32m~/opt/anaconda3/envs/cryptodatapy/lib/python3.9/site-packages/pandas/_libs/index.pyx:138\u001b[0m, in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
739
+ "File \u001b[0;32m~/opt/anaconda3/envs/cryptodatapy/lib/python3.9/site-packages/pandas/_libs/index.pyx:165\u001b[0m, in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
740
+ "File \u001b[0;32mpandas/_libs/hashtable_class_helper.pxi:5745\u001b[0m, in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[0;34m()\u001b[0m\n",
741
+ "File \u001b[0;32mpandas/_libs/hashtable_class_helper.pxi:5753\u001b[0m, in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[0;34m()\u001b[0m\n",
742
+ "\u001b[0;31mKeyError\u001b[0m: 'OCEAN'",
743
+ "\nThe above exception was the direct cause of the following exception:\n",
744
+ "\u001b[0;31mKeyError\u001b[0m Traceback (most recent call last)",
745
+ "Cell \u001b[0;32mIn [27], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m clean_df\u001b[38;5;241m.\u001b[39mloc[:, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mOCEAN\u001b[39m\u001b[38;5;124m'\u001b[39m, :]\n",
746
+ "File \u001b[0;32m~/opt/anaconda3/envs/cryptodatapy/lib/python3.9/site-packages/pandas/core/indexing.py:1068\u001b[0m, in \u001b[0;36m_LocationIndexer.__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m 1066\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_is_scalar_access(key):\n\u001b[1;32m 1067\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mobj\u001b[38;5;241m.\u001b[39m_get_value(\u001b[38;5;241m*\u001b[39mkey, takeable\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_takeable)\n\u001b[0;32m-> 1068\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_getitem_tuple\u001b[49m\u001b[43m(\u001b[49m\u001b[43mkey\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1069\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 1070\u001b[0m \u001b[38;5;66;03m# we by definition only have the 0th axis\u001b[39;00m\n\u001b[1;32m 1071\u001b[0m axis \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39maxis \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;241m0\u001b[39m\n",
747
+ "File \u001b[0;32m~/opt/anaconda3/envs/cryptodatapy/lib/python3.9/site-packages/pandas/core/indexing.py:1248\u001b[0m, in \u001b[0;36m_LocIndexer._getitem_tuple\u001b[0;34m(self, tup)\u001b[0m\n\u001b[1;32m 1246\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m suppress(IndexingError):\n\u001b[1;32m 1247\u001b[0m tup \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_expand_ellipsis(tup)\n\u001b[0;32m-> 1248\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_getitem_lowerdim\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtup\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1250\u001b[0m \u001b[38;5;66;03m# no multi-index, so validate all of the indexers\u001b[39;00m\n\u001b[1;32m 1251\u001b[0m tup \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_validate_tuple_indexer(tup)\n",
748
+ "File \u001b[0;32m~/opt/anaconda3/envs/cryptodatapy/lib/python3.9/site-packages/pandas/core/indexing.py:942\u001b[0m, in \u001b[0;36m_LocationIndexer._getitem_lowerdim\u001b[0;34m(self, tup)\u001b[0m\n\u001b[1;32m 940\u001b[0m \u001b[38;5;66;03m# we may have a nested tuples indexer here\u001b[39;00m\n\u001b[1;32m 941\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_is_nested_tuple_indexer(tup):\n\u001b[0;32m--> 942\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_getitem_nested_tuple\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtup\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 944\u001b[0m \u001b[38;5;66;03m# we maybe be using a tuple to represent multiple dimensions here\u001b[39;00m\n\u001b[1;32m 945\u001b[0m ax0 \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mobj\u001b[38;5;241m.\u001b[39m_get_axis(\u001b[38;5;241m0\u001b[39m)\n",
749
+ "File \u001b[0;32m~/opt/anaconda3/envs/cryptodatapy/lib/python3.9/site-packages/pandas/core/indexing.py:1034\u001b[0m, in \u001b[0;36m_LocationIndexer._getitem_nested_tuple\u001b[0;34m(self, tup)\u001b[0m\n\u001b[1;32m 1031\u001b[0m \u001b[38;5;66;03m# this is a series with a multi-index specified a tuple of\u001b[39;00m\n\u001b[1;32m 1032\u001b[0m \u001b[38;5;66;03m# selectors\u001b[39;00m\n\u001b[1;32m 1033\u001b[0m axis \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39maxis \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;241m0\u001b[39m\n\u001b[0;32m-> 1034\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_getitem_axis\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtup\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43maxis\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43maxis\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1036\u001b[0m \u001b[38;5;66;03m# handle the multi-axis by taking sections and reducing\u001b[39;00m\n\u001b[1;32m 1037\u001b[0m \u001b[38;5;66;03m# this is iterative\u001b[39;00m\n\u001b[1;32m 1038\u001b[0m obj \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mobj\n",
750
+ "File \u001b[0;32m~/opt/anaconda3/envs/cryptodatapy/lib/python3.9/site-packages/pandas/core/indexing.py:1306\u001b[0m, in \u001b[0;36m_LocIndexer._getitem_axis\u001b[0;34m(self, key, axis)\u001b[0m\n\u001b[1;32m 1304\u001b[0m \u001b[38;5;66;03m# nested tuple slicing\u001b[39;00m\n\u001b[1;32m 1305\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m is_nested_tuple(key, labels):\n\u001b[0;32m-> 1306\u001b[0m locs \u001b[38;5;241m=\u001b[39m \u001b[43mlabels\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_locs\u001b[49m\u001b[43m(\u001b[49m\u001b[43mkey\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1307\u001b[0m indexer \u001b[38;5;241m=\u001b[39m [\u001b[38;5;28mslice\u001b[39m(\u001b[38;5;28;01mNone\u001b[39;00m)] \u001b[38;5;241m*\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mndim\n\u001b[1;32m 1308\u001b[0m indexer[axis] \u001b[38;5;241m=\u001b[39m locs\n",
751
+ "File \u001b[0;32m~/opt/anaconda3/envs/cryptodatapy/lib/python3.9/site-packages/pandas/core/indexes/multi.py:3422\u001b[0m, in \u001b[0;36mMultiIndex.get_locs\u001b[0;34m(self, seq)\u001b[0m\n\u001b[1;32m 3418\u001b[0m \u001b[38;5;28;01mcontinue\u001b[39;00m\n\u001b[1;32m 3420\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 3421\u001b[0m \u001b[38;5;66;03m# a slice or a single label\u001b[39;00m\n\u001b[0;32m-> 3422\u001b[0m lvl_indexer \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_get_level_indexer\u001b[49m\u001b[43m(\u001b[49m\u001b[43mk\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlevel\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mi\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mindexer\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mindexer\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 3424\u001b[0m \u001b[38;5;66;03m# update indexer\u001b[39;00m\n\u001b[1;32m 3425\u001b[0m lvl_indexer \u001b[38;5;241m=\u001b[39m _to_bool_indexer(lvl_indexer)\n",
752
+ "File \u001b[0;32m~/opt/anaconda3/envs/cryptodatapy/lib/python3.9/site-packages/pandas/core/indexes/multi.py:3262\u001b[0m, in \u001b[0;36mMultiIndex._get_level_indexer\u001b[0;34m(self, key, level, indexer)\u001b[0m\n\u001b[1;32m 3258\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mslice\u001b[39m(i, j, step)\n\u001b[1;32m 3260\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 3262\u001b[0m idx \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_get_loc_single_level_index\u001b[49m\u001b[43m(\u001b[49m\u001b[43mlevel_index\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkey\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 3264\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m level \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0\u001b[39m \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_lexsort_depth \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[1;32m 3265\u001b[0m \u001b[38;5;66;03m# Desired level is not sorted\u001b[39;00m\n\u001b[1;32m 3266\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(idx, \u001b[38;5;28mslice\u001b[39m):\n\u001b[1;32m 3267\u001b[0m \u001b[38;5;66;03m# test_get_loc_partial_timestamp_multiindex\u001b[39;00m\n",
753
+ "File \u001b[0;32m~/opt/anaconda3/envs/cryptodatapy/lib/python3.9/site-packages/pandas/core/indexes/multi.py:2848\u001b[0m, in \u001b[0;36mMultiIndex._get_loc_single_level_index\u001b[0;34m(self, level_index, key)\u001b[0m\n\u001b[1;32m 2846\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m\n\u001b[1;32m 2847\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 2848\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mlevel_index\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_loc\u001b[49m\u001b[43m(\u001b[49m\u001b[43mkey\u001b[49m\u001b[43m)\u001b[49m\n",
754
+ "File \u001b[0;32m~/opt/anaconda3/envs/cryptodatapy/lib/python3.9/site-packages/pandas/core/indexes/base.py:3802\u001b[0m, in \u001b[0;36mIndex.get_loc\u001b[0;34m(self, key, method, tolerance)\u001b[0m\n\u001b[1;32m 3800\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_engine\u001b[38;5;241m.\u001b[39mget_loc(casted_key)\n\u001b[1;32m 3801\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mKeyError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m err:\n\u001b[0;32m-> 3802\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mKeyError\u001b[39;00m(key) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01merr\u001b[39;00m\n\u001b[1;32m 3803\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m:\n\u001b[1;32m 3804\u001b[0m \u001b[38;5;66;03m# If we have a listlike key, _check_indexing_error will raise\u001b[39;00m\n\u001b[1;32m 3805\u001b[0m \u001b[38;5;66;03m# InvalidIndexError. Otherwise we fall through and re-raise\u001b[39;00m\n\u001b[1;32m 3806\u001b[0m \u001b[38;5;66;03m# the TypeError.\u001b[39;00m\n\u001b[1;32m 3807\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_check_indexing_error(key)\n",
755
+ "\u001b[0;31mKeyError\u001b[0m: 'OCEAN'"
756
+ ]
757
+ }
758
+ ],
759
+ "source": [
760
+ "clean_df.loc[:, 'OCEAN', :]"
761
+ ]
762
+ },
763
+ {
764
+ "cell_type": "code",
765
+ "execution_count": 28,
766
+ "id": "ad5a885c",
870
767
  "metadata": {},
871
768
  "outputs": [
872
769
  {
@@ -890,7 +787,6 @@
890
787
  " <thead>\n",
891
788
  " <tr style=\"text-align: right;\">\n",
892
789
  " <th></th>\n",
893
- " <th></th>\n",
894
790
  " <th>open</th>\n",
895
791
  " <th>high</th>\n",
896
792
  " <th>low</th>\n",
@@ -900,7 +796,6 @@
900
796
  " </tr>\n",
901
797
  " <tr>\n",
902
798
  " <th>date</th>\n",
903
- " <th>ticker</th>\n",
904
799
  " <th></th>\n",
905
800
  " <th></th>\n",
906
801
  " <th></th>\n",
@@ -911,55 +806,52 @@
911
806
  " </thead>\n",
912
807
  " <tbody>\n",
913
808
  " <tr>\n",
914
- " <th rowspan=\"4\" valign=\"top\">2017-03-21</th>\n",
915
- " <th>BTC</th>\n",
916
- " <td>1047.51</td>\n",
917
- " <td>1125.53</td>\n",
918
- " <td>1043.87</td>\n",
919
- " <td>1121.29</td>\n",
920
- " <td>9.259127e+04</td>\n",
809
+ " <th>2015-06-12</th>\n",
810
+ " <td>229.88</td>\n",
811
+ " <td>231.58</td>\n",
812
+ " <td>229.29</td>\n",
813
+ " <td>230.46</td>\n",
814
+ " <td>40744.820</td>\n",
921
815
  " <td>0.000000</td>\n",
922
816
  " </tr>\n",
923
817
  " <tr>\n",
924
- " <th>ETC</th>\n",
925
- " <td>1.867</td>\n",
926
- " <td>2.39</td>\n",
927
- " <td>1.867</td>\n",
928
- " <td>2.378</td>\n",
929
- " <td>2.151590e+06</td>\n",
818
+ " <th>2015-06-13</th>\n",
819
+ " <td>230.46</td>\n",
820
+ " <td>233.14</td>\n",
821
+ " <td>229.01</td>\n",
822
+ " <td>232.48</td>\n",
823
+ " <td>38008.730</td>\n",
930
824
  " <td>0.000000</td>\n",
931
825
  " </tr>\n",
932
826
  " <tr>\n",
933
- " <th>ETH</th>\n",
934
- " <td>42.51</td>\n",
935
- " <td>43.8</td>\n",
936
- " <td>41.68</td>\n",
937
- " <td>42.67</td>\n",
938
- " <td>4.843660e+05</td>\n",
827
+ " <th>2015-06-14</th>\n",
828
+ " <td>232.48</td>\n",
829
+ " <td>235.51</td>\n",
830
+ " <td>232.09</td>\n",
831
+ " <td>233.75</td>\n",
832
+ " <td>32894.870</td>\n",
939
833
  " <td>0.000000</td>\n",
940
834
  " </tr>\n",
941
835
  " <tr>\n",
942
- " <th>LTC</th>\n",
943
- " <td>4.121</td>\n",
944
- " <td>4.155</td>\n",
945
- " <td>4.014</td>\n",
946
- " <td>4.09</td>\n",
947
- " <td>1.932581e+05</td>\n",
836
+ " <th>2015-06-15</th>\n",
837
+ " <td>233.75</td>\n",
838
+ " <td>238.55</td>\n",
839
+ " <td>233.29</td>\n",
840
+ " <td>237.0</td>\n",
841
+ " <td>63467.090</td>\n",
948
842
  " <td>0.000000</td>\n",
949
843
  " </tr>\n",
950
844
  " <tr>\n",
951
- " <th>2017-03-22</th>\n",
952
- " <th>BTC</th>\n",
953
- " <td>1121.29</td>\n",
954
- " <td>1121.88</td>\n",
955
- " <td>997.78</td>\n",
956
- " <td>1044.72</td>\n",
957
- " <td>1.152861e+05</td>\n",
845
+ " <th>2015-06-16</th>\n",
846
+ " <td>237.0</td>\n",
847
+ " <td>254.15</td>\n",
848
+ " <td>235.7</td>\n",
849
+ " <td>249.82</td>\n",
850
+ " <td>122473.610</td>\n",
958
851
  " <td>0.000000</td>\n",
959
852
  " </tr>\n",
960
853
  " <tr>\n",
961
854
  " <th>...</th>\n",
962
- " <th>...</th>\n",
963
855
  " <td>...</td>\n",
964
856
  " <td>...</td>\n",
965
857
  " <td>...</td>\n",
@@ -968,648 +860,142 @@
968
860
  " <td>...</td>\n",
969
861
  " </tr>\n",
970
862
  " <tr>\n",
971
- " <th rowspan=\"5\" valign=\"top\">2024-08-04</th>\n",
972
- " <th>YFI</th>\n",
973
- " <td>5341.0</td>\n",
974
- " <td>5341.0</td>\n",
975
- " <td>5196.0</td>\n",
976
- " <td>5198.0</td>\n",
977
- " <td>4.281050e+02</td>\n",
978
- " <td>0.000169</td>\n",
863
+ " <th>2024-07-31</th>\n",
864
+ " <td>66159.3</td>\n",
865
+ " <td>66826.3</td>\n",
866
+ " <td>64500.4</td>\n",
867
+ " <td>64601.8</td>\n",
868
+ " <td>246389.446</td>\n",
869
+ " <td>0.000141</td>\n",
979
870
  " </tr>\n",
980
871
  " <tr>\n",
981
- " <th>ZEC</th>\n",
982
- " <td>31.76</td>\n",
983
- " <td>34.44</td>\n",
984
- " <td>31.28</td>\n",
985
- " <td>31.55</td>\n",
986
- " <td>1.348085e+06</td>\n",
987
- " <td>0.000191</td>\n",
872
+ " <th>2024-08-01</th>\n",
873
+ " <td>64601.8</td>\n",
874
+ " <td>65650.0</td>\n",
875
+ " <td>62271.2</td>\n",
876
+ " <td>65328.9</td>\n",
877
+ " <td>372654.590</td>\n",
878
+ " <td>0.000282</td>\n",
988
879
  " </tr>\n",
989
880
  " <tr>\n",
990
- " <th>ZEN</th>\n",
991
- " <td>9.657</td>\n",
992
- " <td>9.112</td>\n",
993
- " <td>8.285</td>\n",
994
- " <td>8.462</td>\n",
995
- " <td>2.071124e+06</td>\n",
996
- " <td>0.000194</td>\n",
881
+ " <th>2024-08-02</th>\n",
882
+ " <td>65329.0</td>\n",
883
+ " <td>65577.0</td>\n",
884
+ " <td>61200.2</td>\n",
885
+ " <td>61483.7</td>\n",
886
+ " <td>421628.420</td>\n",
887
+ " <td>0.000300</td>\n",
997
888
  " </tr>\n",
998
889
  " <tr>\n",
999
- " <th>ZIL</th>\n",
1000
- " <td>0.01422</td>\n",
1001
- " <td>0.01441</td>\n",
1002
- " <td>0.01392</td>\n",
1003
- " <td>0.01396</td>\n",
1004
- " <td>2.048626e+08</td>\n",
1005
- " <td>-0.000031</td>\n",
890
+ " <th>2024-08-03</th>\n",
891
+ " <td>61483.7</td>\n",
892
+ " <td>63871.5</td>\n",
893
+ " <td>59800.0</td>\n",
894
+ " <td>60684.6</td>\n",
895
+ " <td>290469.956</td>\n",
896
+ " <td>0.000240</td>\n",
1006
897
  " </tr>\n",
1007
898
  " <tr>\n",
1008
- " <th>ZRX</th>\n",
1009
- " <td>0.3055</td>\n",
1010
- " <td>0.3121</td>\n",
1011
- " <td>0.2983</td>\n",
1012
- " <td>0.2988</td>\n",
1013
- " <td>9.810764e+06</td>\n",
1014
- " <td>0.000186</td>\n",
899
+ " <th>2024-08-04</th>\n",
900
+ " <td>60684.5</td>\n",
901
+ " <td>61089.5</td>\n",
902
+ " <td>60080.5</td>\n",
903
+ " <td>60357.6</td>\n",
904
+ " <td>85220.266</td>\n",
905
+ " <td>0.000153</td>\n",
1015
906
  " </tr>\n",
1016
907
  " </tbody>\n",
1017
908
  "</table>\n",
1018
- "<p>193002 rows × 6 columns</p>\n",
909
+ "<p>3342 rows × 6 columns</p>\n",
1019
910
  "</div>"
1020
911
  ],
1021
912
  "text/plain": [
1022
- " open high low close volume \\\n",
1023
- "date ticker \n",
1024
- "2017-03-21 BTC 1047.51 1125.53 1043.87 1121.29 9.259127e+04 \n",
1025
- " ETC 1.867 2.39 1.867 2.378 2.151590e+06 \n",
1026
- " ETH 42.51 43.8 41.68 42.67 4.843660e+05 \n",
1027
- " LTC 4.121 4.155 4.014 4.09 1.932581e+05 \n",
1028
- "2017-03-22 BTC 1121.29 1121.88 997.78 1044.72 1.152861e+05 \n",
1029
- "... ... ... ... ... ... \n",
1030
- "2024-08-04 YFI 5341.0 5341.0 5196.0 5198.0 4.281050e+02 \n",
1031
- " ZEC 31.76 34.44 31.28 31.55 1.348085e+06 \n",
1032
- " ZEN 9.657 9.112 8.285 8.462 2.071124e+06 \n",
1033
- " ZIL 0.01422 0.01441 0.01392 0.01396 2.048626e+08 \n",
1034
- " ZRX 0.3055 0.3121 0.2983 0.2988 9.810764e+06 \n",
1035
- "\n",
1036
- " funding_rate \n",
1037
- "date ticker \n",
1038
- "2017-03-21 BTC 0.000000 \n",
1039
- " ETC 0.000000 \n",
1040
- " ETH 0.000000 \n",
1041
- " LTC 0.000000 \n",
1042
- "2017-03-22 BTC 0.000000 \n",
1043
- "... ... \n",
1044
- "2024-08-04 YFI 0.000169 \n",
1045
- " ZEC 0.000191 \n",
1046
- " ZEN 0.000194 \n",
1047
- " ZIL -0.000031 \n",
1048
- " ZRX 0.000186 \n",
913
+ " open high low close volume funding_rate\n",
914
+ "date \n",
915
+ "2015-06-12 229.88 231.58 229.29 230.46 40744.820 0.000000\n",
916
+ "2015-06-13 230.46 233.14 229.01 232.48 38008.730 0.000000\n",
917
+ "2015-06-14 232.48 235.51 232.09 233.75 32894.870 0.000000\n",
918
+ "2015-06-15 233.75 238.55 233.29 237.0 63467.090 0.000000\n",
919
+ "2015-06-16 237.0 254.15 235.7 249.82 122473.610 0.000000\n",
920
+ "... ... ... ... ... ... ...\n",
921
+ "2024-07-31 66159.3 66826.3 64500.4 64601.8 246389.446 0.000141\n",
922
+ "2024-08-01 64601.8 65650.0 62271.2 65328.9 372654.590 0.000282\n",
923
+ "2024-08-02 65329.0 65577.0 61200.2 61483.7 421628.420 0.000300\n",
924
+ "2024-08-03 61483.7 63871.5 59800.0 60684.6 290469.956 0.000240\n",
925
+ "2024-08-04 60684.5 61089.5 60080.5 60357.6 85220.266 0.000153\n",
1049
926
  "\n",
1050
- "[193002 rows x 6 columns]"
927
+ "[3342 rows x 6 columns]"
1051
928
  ]
1052
929
  },
1053
- "execution_count": 64,
930
+ "execution_count": 28,
1054
931
  "metadata": {},
1055
932
  "output_type": "execute_result"
1056
933
  }
1057
934
  ],
1058
935
  "source": [
1059
- "clean_df"
936
+ "clean_df.loc[:, 'BTC', :]"
1060
937
  ]
1061
938
  },
1062
939
  {
1063
940
  "cell_type": "code",
1064
- "execution_count": 65,
1065
- "id": "bec999ba",
941
+ "execution_count": 29,
942
+ "id": "a9c262fc",
1066
943
  "metadata": {},
1067
944
  "outputs": [],
1068
945
  "source": [
1069
- "# Filter data\n",
1070
- "clean = CleanData(df)"
946
+ "clean_df.to_parquet('s3://factorlab-data/binance_historical_ohlcv_daily.parquet')"
1071
947
  ]
1072
948
  },
1073
949
  {
1074
950
  "cell_type": "code",
1075
- "execution_count": 66,
1076
- "id": "34372e70",
951
+ "execution_count": 30,
952
+ "id": "893e3e38",
1077
953
  "metadata": {},
1078
- "outputs": [
1079
- {
1080
- "data": {
1081
- "text/plain": [
1082
- "<cryptodatapy.transform.clean.CleanData at 0x7face8e7eb50>"
1083
- ]
1084
- },
1085
- "execution_count": 66,
1086
- "metadata": {},
1087
- "output_type": "execute_result"
1088
- }
1089
- ],
954
+ "outputs": [],
1090
955
  "source": [
1091
- "clean.filter_outliers(od_method='mad', excl_cols=['volume', 'funding_rate'], thresh_val=10).\\\n",
1092
- " repair_outliers(imp_method='fcst').\\\n",
1093
- " filter_avg_trading_val(thresh_val=1000000).\\\n",
1094
- " filter_missing_vals_gaps().\\\n",
1095
- " filter_min_nobs(ts_obs=1000, cs_obs=3)"
956
+ "clean_df.to_parquet('../../../../factorlab/notebooks/binance_historical_ohlcv_daily.parquet')"
1096
957
  ]
1097
958
  },
1098
959
  {
1099
960
  "cell_type": "code",
1100
- "execution_count": 79,
1101
- "id": "ba73590b",
961
+ "execution_count": null,
962
+ "id": "8a962fa7",
1102
963
  "metadata": {},
1103
- "outputs": [
1104
- {
1105
- "data": {
1106
- "text/html": [
1107
- "<div>\n",
1108
- "<style scoped>\n",
1109
- " .dataframe tbody tr th:only-of-type {\n",
1110
- " vertical-align: middle;\n",
1111
- " }\n",
1112
- "\n",
1113
- " .dataframe tbody tr th {\n",
1114
- " vertical-align: top;\n",
1115
- " }\n",
1116
- "\n",
1117
- " .dataframe thead th {\n",
1118
- " text-align: right;\n",
1119
- " }\n",
1120
- "</style>\n",
1121
- "<table border=\"1\" class=\"dataframe\">\n",
1122
- " <thead>\n",
1123
- " <tr style=\"text-align: right;\">\n",
1124
- " <th></th>\n",
1125
- " <th>1000SATS</th>\n",
1126
- " <th>1INCH</th>\n",
1127
- " <th>AAVE</th>\n",
1128
- " <th>ACE</th>\n",
1129
- " <th>ACH</th>\n",
1130
- " <th>ADA</th>\n",
1131
- " <th>AEVO</th>\n",
1132
- " <th>AGIX</th>\n",
1133
- " <th>AGLD</th>\n",
1134
- " <th>AI</th>\n",
1135
- " <th>...</th>\n",
1136
- " <th>XVG</th>\n",
1137
- " <th>XVS</th>\n",
1138
- " <th>YFI</th>\n",
1139
- " <th>YGG</th>\n",
1140
- " <th>ZEC</th>\n",
1141
- " <th>ZEN</th>\n",
1142
- " <th>ZIL</th>\n",
1143
- " <th>ZK</th>\n",
1144
- " <th>ZRO</th>\n",
1145
- " <th>ZRX</th>\n",
1146
- " </tr>\n",
1147
- " </thead>\n",
1148
- " <tbody>\n",
1149
- " <tr>\n",
1150
- " <th>n_obs</th>\n",
1151
- " <td>237.000000</td>\n",
1152
- " <td>1319.000000</td>\n",
1153
- " <td>1395.000000</td>\n",
1154
- " <td>326.000000</td>\n",
1155
- " <td>1419.000000</td>\n",
1156
- " <td>2500.000000</td>\n",
1157
- " <td>278.000000</td>\n",
1158
- " <td>536.000000</td>\n",
1159
- " <td>1040.000000</td>\n",
1160
- " <td>524.000000</td>\n",
1161
- " <td>...</td>\n",
1162
- " <td>3101.000000</td>\n",
1163
- " <td>1399.000000</td>\n",
1164
- " <td>1466.000000</td>\n",
1165
- " <td>1067.000000</td>\n",
1166
- " <td>2838.000000</td>\n",
1167
- " <td>2617.000000</td>\n",
1168
- " <td>2379.000000</td>\n",
1169
- " <td>49.000000</td>\n",
1170
- " <td>46.000000</td>\n",
1171
- " <td>2551.000000</td>\n",
1172
- " </tr>\n",
1173
- " <tr>\n",
1174
- " <th>%_NaN_start</th>\n",
1175
- " <td>95.382817</td>\n",
1176
- " <td>74.303526</td>\n",
1177
- " <td>72.822911</td>\n",
1178
- " <td>93.648938</td>\n",
1179
- " <td>72.355348</td>\n",
1180
- " <td>51.295539</td>\n",
1181
- " <td>94.584064</td>\n",
1182
- " <td>89.557763</td>\n",
1183
- " <td>79.738944</td>\n",
1184
- " <td>89.791545</td>\n",
1185
- " <td>...</td>\n",
1186
- " <td>39.586986</td>\n",
1187
- " <td>72.744983</td>\n",
1188
- " <td>71.439704</td>\n",
1189
- " <td>79.212936</td>\n",
1190
- " <td>44.710695</td>\n",
1191
- " <td>49.016170</td>\n",
1192
- " <td>53.652835</td>\n",
1193
- " <td>99.045393</td>\n",
1194
- " <td>99.103838</td>\n",
1195
- " <td>50.301968</td>\n",
1196
- " </tr>\n",
1197
- " <tr>\n",
1198
- " <th>%_outliers</th>\n",
1199
- " <td>0.038964</td>\n",
1200
- " <td>0.642899</td>\n",
1201
- " <td>0.370154</td>\n",
1202
- " <td>0.253263</td>\n",
1203
- " <td>0.584454</td>\n",
1204
- " <td>0.876680</td>\n",
1205
- " <td>0.331190</td>\n",
1206
- " <td>0.253263</td>\n",
1207
- " <td>0.526008</td>\n",
1208
- " <td>0.253263</td>\n",
1209
- " <td>...</td>\n",
1210
- " <td>2.824859</td>\n",
1211
- " <td>0.642899</td>\n",
1212
- " <td>0.389636</td>\n",
1213
- " <td>0.467563</td>\n",
1214
- " <td>0.857199</td>\n",
1215
- " <td>1.207871</td>\n",
1216
- " <td>0.974089</td>\n",
1217
- " <td>0.000000</td>\n",
1218
- " <td>0.019482</td>\n",
1219
- " <td>0.993571</td>\n",
1220
- " </tr>\n",
1221
- " <tr>\n",
1222
- " <th>%_imputed</th>\n",
1223
- " <td>0.175336</td>\n",
1224
- " <td>1.052016</td>\n",
1225
- " <td>0.584454</td>\n",
1226
- " <td>0.370154</td>\n",
1227
- " <td>0.701344</td>\n",
1228
- " <td>1.246834</td>\n",
1229
- " <td>0.545490</td>\n",
1230
- " <td>1.188389</td>\n",
1231
- " <td>0.779271</td>\n",
1232
- " <td>0.662381</td>\n",
1233
- " <td>...</td>\n",
1234
- " <td>3.779466</td>\n",
1235
- " <td>0.779271</td>\n",
1236
- " <td>0.701344</td>\n",
1237
- " <td>0.740308</td>\n",
1238
- " <td>1.207871</td>\n",
1239
- " <td>1.461134</td>\n",
1240
- " <td>1.422170</td>\n",
1241
- " <td>0.116891</td>\n",
1242
- " <td>0.136372</td>\n",
1243
- " <td>1.363725</td>\n",
1244
- " </tr>\n",
1245
- " <tr>\n",
1246
- " <th>%_below_avg_trading_val</th>\n",
1247
- " <td>0.564972</td>\n",
1248
- " <td>0.564972</td>\n",
1249
- " <td>0.603935</td>\n",
1250
- " <td>1.753361</td>\n",
1251
- " <td>6.409507</td>\n",
1252
- " <td>0.564972</td>\n",
1253
- " <td>2.591077</td>\n",
1254
- " <td>1.110462</td>\n",
1255
- " <td>1.636470</td>\n",
1256
- " <td>6.019871</td>\n",
1257
- " <td>...</td>\n",
1258
- " <td>41.671537</td>\n",
1259
- " <td>4.227547</td>\n",
1260
- " <td>0.564972</td>\n",
1261
- " <td>1.402688</td>\n",
1262
- " <td>4.773037</td>\n",
1263
- " <td>17.786869</td>\n",
1264
- " <td>9.526593</td>\n",
1265
- " <td>0.564972</td>\n",
1266
- " <td>0.564972</td>\n",
1267
- " <td>12.195597</td>\n",
1268
- " </tr>\n",
1269
- " <tr>\n",
1270
- " <th>%_missing_vals_gaps</th>\n",
1271
- " <td>0.000000</td>\n",
1272
- " <td>0.000000</td>\n",
1273
- " <td>0.000000</td>\n",
1274
- " <td>0.019482</td>\n",
1275
- " <td>0.019482</td>\n",
1276
- " <td>0.000000</td>\n",
1277
- " <td>0.000000</td>\n",
1278
- " <td>0.000000</td>\n",
1279
- " <td>0.000000</td>\n",
1280
- " <td>0.000000</td>\n",
1281
- " <td>...</td>\n",
1282
- " <td>10.325346</td>\n",
1283
- " <td>13.598286</td>\n",
1284
- " <td>0.000000</td>\n",
1285
- " <td>11.903370</td>\n",
1286
- " <td>16.540035</td>\n",
1287
- " <td>6.916034</td>\n",
1288
- " <td>6.623807</td>\n",
1289
- " <td>0.000000</td>\n",
1290
- " <td>0.000000</td>\n",
1291
- " <td>7.305669</td>\n",
1292
- " </tr>\n",
1293
- " <tr>\n",
1294
- " <th>n_tickers_below_min_obs</th>\n",
1295
- " <td>129.000000</td>\n",
1296
- " <td>129.000000</td>\n",
1297
- " <td>129.000000</td>\n",
1298
- " <td>129.000000</td>\n",
1299
- " <td>129.000000</td>\n",
1300
- " <td>129.000000</td>\n",
1301
- " <td>129.000000</td>\n",
1302
- " <td>129.000000</td>\n",
1303
- " <td>129.000000</td>\n",
1304
- " <td>129.000000</td>\n",
1305
- " <td>...</td>\n",
1306
- " <td>129.000000</td>\n",
1307
- " <td>129.000000</td>\n",
1308
- " <td>129.000000</td>\n",
1309
- " <td>129.000000</td>\n",
1310
- " <td>129.000000</td>\n",
1311
- " <td>129.000000</td>\n",
1312
- " <td>129.000000</td>\n",
1313
- " <td>129.000000</td>\n",
1314
- " <td>129.000000</td>\n",
1315
- " <td>129.000000</td>\n",
1316
- " </tr>\n",
1317
- " </tbody>\n",
1318
- "</table>\n",
1319
- "<p>7 rows × 256 columns</p>\n",
1320
- "</div>"
1321
- ],
1322
- "text/plain": [
1323
- " 1000SATS 1INCH AAVE ACE \\\n",
1324
- "n_obs 237.000000 1319.000000 1395.000000 326.000000 \n",
1325
- "%_NaN_start 95.382817 74.303526 72.822911 93.648938 \n",
1326
- "%_outliers 0.038964 0.642899 0.370154 0.253263 \n",
1327
- "%_imputed 0.175336 1.052016 0.584454 0.370154 \n",
1328
- "%_below_avg_trading_val 0.564972 0.564972 0.603935 1.753361 \n",
1329
- "%_missing_vals_gaps 0.000000 0.000000 0.000000 0.019482 \n",
1330
- "n_tickers_below_min_obs 129.000000 129.000000 129.000000 129.000000 \n",
1331
- "\n",
1332
- " ACH ADA AEVO AGIX \\\n",
1333
- "n_obs 1419.000000 2500.000000 278.000000 536.000000 \n",
1334
- "%_NaN_start 72.355348 51.295539 94.584064 89.557763 \n",
1335
- "%_outliers 0.584454 0.876680 0.331190 0.253263 \n",
1336
- "%_imputed 0.701344 1.246834 0.545490 1.188389 \n",
1337
- "%_below_avg_trading_val 6.409507 0.564972 2.591077 1.110462 \n",
1338
- "%_missing_vals_gaps 0.019482 0.000000 0.000000 0.000000 \n",
1339
- "n_tickers_below_min_obs 129.000000 129.000000 129.000000 129.000000 \n",
1340
- "\n",
1341
- " AGLD AI ... XVG \\\n",
1342
- "n_obs 1040.000000 524.000000 ... 3101.000000 \n",
1343
- "%_NaN_start 79.738944 89.791545 ... 39.586986 \n",
1344
- "%_outliers 0.526008 0.253263 ... 2.824859 \n",
1345
- "%_imputed 0.779271 0.662381 ... 3.779466 \n",
1346
- "%_below_avg_trading_val 1.636470 6.019871 ... 41.671537 \n",
1347
- "%_missing_vals_gaps 0.000000 0.000000 ... 10.325346 \n",
1348
- "n_tickers_below_min_obs 129.000000 129.000000 ... 129.000000 \n",
1349
- "\n",
1350
- " XVS YFI YGG ZEC \\\n",
1351
- "n_obs 1399.000000 1466.000000 1067.000000 2838.000000 \n",
1352
- "%_NaN_start 72.744983 71.439704 79.212936 44.710695 \n",
1353
- "%_outliers 0.642899 0.389636 0.467563 0.857199 \n",
1354
- "%_imputed 0.779271 0.701344 0.740308 1.207871 \n",
1355
- "%_below_avg_trading_val 4.227547 0.564972 1.402688 4.773037 \n",
1356
- "%_missing_vals_gaps 13.598286 0.000000 11.903370 16.540035 \n",
1357
- "n_tickers_below_min_obs 129.000000 129.000000 129.000000 129.000000 \n",
1358
- "\n",
1359
- " ZEN ZIL ZK ZRO \\\n",
1360
- "n_obs 2617.000000 2379.000000 49.000000 46.000000 \n",
1361
- "%_NaN_start 49.016170 53.652835 99.045393 99.103838 \n",
1362
- "%_outliers 1.207871 0.974089 0.000000 0.019482 \n",
1363
- "%_imputed 1.461134 1.422170 0.116891 0.136372 \n",
1364
- "%_below_avg_trading_val 17.786869 9.526593 0.564972 0.564972 \n",
1365
- "%_missing_vals_gaps 6.916034 6.623807 0.000000 0.000000 \n",
1366
- "n_tickers_below_min_obs 129.000000 129.000000 129.000000 129.000000 \n",
1367
- "\n",
1368
- " ZRX \n",
1369
- "n_obs 2551.000000 \n",
1370
- "%_NaN_start 50.301968 \n",
1371
- "%_outliers 0.993571 \n",
1372
- "%_imputed 1.363725 \n",
1373
- "%_below_avg_trading_val 12.195597 \n",
1374
- "%_missing_vals_gaps 7.305669 \n",
1375
- "n_tickers_below_min_obs 129.000000 \n",
1376
- "\n",
1377
- "[7 rows x 256 columns]"
1378
- ]
1379
- },
1380
- "execution_count": 79,
1381
- "metadata": {},
1382
- "output_type": "execute_result"
1383
- }
1384
- ],
1385
- "source": [
1386
- "clean.summary.close"
1387
- ]
964
+ "outputs": [],
965
+ "source": []
1388
966
  },
1389
967
  {
1390
968
  "cell_type": "code",
1391
- "execution_count": 80,
1392
- "id": "f89c762c",
969
+ "execution_count": 31,
970
+ "id": "54b818cd",
1393
971
  "metadata": {},
1394
- "outputs": [
1395
- {
1396
- "data": {
1397
- "text/plain": [
1398
- "<cryptodatapy.transform.clean.CleanData at 0x7face8e7eb50>"
1399
- ]
1400
- },
1401
- "execution_count": 80,
1402
- "metadata": {},
1403
- "output_type": "execute_result"
1404
- }
1405
- ],
972
+ "outputs": [],
1406
973
  "source": [
1407
- "clean.filter_tickers(['BTC'])"
974
+ "clean_df.to_csv('../../../../factorlab/notebooks/binance_historical_ohlcv_daily.csv')"
1408
975
  ]
1409
976
  },
1410
977
  {
1411
978
  "cell_type": "code",
1412
- "execution_count": 85,
1413
- "id": "d621e6f3",
979
+ "execution_count": 32,
980
+ "id": "a1c49f01",
1414
981
  "metadata": {},
1415
- "outputs": [
1416
- {
1417
- "data": {
1418
- "text/html": [
1419
- "<div>\n",
1420
- "<style scoped>\n",
1421
- " .dataframe tbody tr th:only-of-type {\n",
1422
- " vertical-align: middle;\n",
1423
- " }\n",
1424
- "\n",
1425
- " .dataframe tbody tr th {\n",
1426
- " vertical-align: top;\n",
1427
- " }\n",
1428
- "\n",
1429
- " .dataframe thead th {\n",
1430
- " text-align: right;\n",
1431
- " }\n",
1432
- "</style>\n",
1433
- "<table border=\"1\" class=\"dataframe\">\n",
1434
- " <thead>\n",
1435
- " <tr style=\"text-align: right;\">\n",
1436
- " <th></th>\n",
1437
- " <th></th>\n",
1438
- " <th>open</th>\n",
1439
- " <th>high</th>\n",
1440
- " <th>low</th>\n",
1441
- " <th>close</th>\n",
1442
- " <th>volume</th>\n",
1443
- " <th>funding_rate</th>\n",
1444
- " </tr>\n",
1445
- " <tr>\n",
1446
- " <th>date</th>\n",
1447
- " <th>ticker</th>\n",
1448
- " <th></th>\n",
1449
- " <th></th>\n",
1450
- " <th></th>\n",
1451
- " <th></th>\n",
1452
- " <th></th>\n",
1453
- " <th></th>\n",
1454
- " </tr>\n",
1455
- " </thead>\n",
1456
- " <tbody>\n",
1457
- " <tr>\n",
1458
- " <th>2013-09-29</th>\n",
1459
- " <th>LTC</th>\n",
1460
- " <td>2.56500</td>\n",
1461
- " <td>2.59000</td>\n",
1462
- " <td>2.59000</td>\n",
1463
- " <td>2.59000</td>\n",
1464
- " <td>5.000000e+00</td>\n",
1465
- " <td>0.000000</td>\n",
1466
- " </tr>\n",
1467
- " <tr>\n",
1468
- " <th>2013-09-30</th>\n",
1469
- " <th>LTC</th>\n",
1470
- " <td>2.59000</td>\n",
1471
- " <td>2.55400</td>\n",
1472
- " <td>2.55400</td>\n",
1473
- " <td>2.55400</td>\n",
1474
- " <td>5.623000e+01</td>\n",
1475
- " <td>0.000000</td>\n",
1476
- " </tr>\n",
1477
- " <tr>\n",
1478
- " <th>2013-10-01</th>\n",
1479
- " <th>LTC</th>\n",
1480
- " <td>2.55400</td>\n",
1481
- " <td>2.66600</td>\n",
1482
- " <td>2.52500</td>\n",
1483
- " <td>2.66600</td>\n",
1484
- " <td>1.000000e+00</td>\n",
1485
- " <td>0.000000</td>\n",
1486
- " </tr>\n",
1487
- " <tr>\n",
1488
- " <th>2013-10-02</th>\n",
1489
- " <th>LTC</th>\n",
1490
- " <td>2.66600</td>\n",
1491
- " <td>2.33700</td>\n",
1492
- " <td>2.33700</td>\n",
1493
- " <td>2.33700</td>\n",
1494
- " <td>1.500000e+00</td>\n",
1495
- " <td>0.000000</td>\n",
1496
- " </tr>\n",
1497
- " <tr>\n",
1498
- " <th>2013-10-03</th>\n",
1499
- " <th>LTC</th>\n",
1500
- " <td>2.33700</td>\n",
1501
- " <td>2.50200</td>\n",
1502
- " <td>2.48900</td>\n",
1503
- " <td>2.50200</td>\n",
1504
- " <td>9.274000e+01</td>\n",
1505
- " <td>0.000000</td>\n",
1506
- " </tr>\n",
1507
- " <tr>\n",
1508
- " <th>...</th>\n",
1509
- " <th>...</th>\n",
1510
- " <td>...</td>\n",
1511
- " <td>...</td>\n",
1512
- " <td>...</td>\n",
1513
- " <td>...</td>\n",
1514
- " <td>...</td>\n",
1515
- " <td>...</td>\n",
1516
- " </tr>\n",
1517
- " <tr>\n",
1518
- " <th rowspan=\"5\" valign=\"top\">2024-08-04</th>\n",
1519
- " <th>ZEN</th>\n",
1520
- " <td>8.40600</td>\n",
1521
- " <td>9.11200</td>\n",
1522
- " <td>8.28500</td>\n",
1523
- " <td>8.46200</td>\n",
1524
- " <td>2.071124e+06</td>\n",
1525
- " <td>0.000194</td>\n",
1526
- " </tr>\n",
1527
- " <tr>\n",
1528
- " <th>ZIL</th>\n",
1529
- " <td>0.01422</td>\n",
1530
- " <td>0.01441</td>\n",
1531
- " <td>0.01392</td>\n",
1532
- " <td>0.01396</td>\n",
1533
- " <td>2.048626e+08</td>\n",
1534
- " <td>-0.000031</td>\n",
1535
- " </tr>\n",
1536
- " <tr>\n",
1537
- " <th>ZK</th>\n",
1538
- " <td>0.11451</td>\n",
1539
- " <td>0.11673</td>\n",
1540
- " <td>0.10857</td>\n",
1541
- " <td>0.10874</td>\n",
1542
- " <td>3.833253e+08</td>\n",
1543
- " <td>0.000200</td>\n",
1544
- " </tr>\n",
1545
- " <tr>\n",
1546
- " <th>ZRO</th>\n",
1547
- " <td>3.63100</td>\n",
1548
- " <td>3.64800</td>\n",
1549
- " <td>3.36200</td>\n",
1550
- " <td>3.36700</td>\n",
1551
- " <td>7.049472e+07</td>\n",
1552
- " <td>0.000184</td>\n",
1553
- " </tr>\n",
1554
- " <tr>\n",
1555
- " <th>ZRX</th>\n",
1556
- " <td>0.30550</td>\n",
1557
- " <td>0.31210</td>\n",
1558
- " <td>0.29830</td>\n",
1559
- " <td>0.29880</td>\n",
1560
- " <td>9.810764e+06</td>\n",
1561
- " <td>0.000186</td>\n",
1562
- " </tr>\n",
1563
- " </tbody>\n",
1564
- "</table>\n",
1565
- "<p>387494 rows × 6 columns</p>\n",
1566
- "</div>"
1567
- ],
1568
- "text/plain": [
1569
- " open high low close volume \\\n",
1570
- "date ticker \n",
1571
- "2013-09-29 LTC 2.56500 2.59000 2.59000 2.59000 5.000000e+00 \n",
1572
- "2013-09-30 LTC 2.59000 2.55400 2.55400 2.55400 5.623000e+01 \n",
1573
- "2013-10-01 LTC 2.55400 2.66600 2.52500 2.66600 1.000000e+00 \n",
1574
- "2013-10-02 LTC 2.66600 2.33700 2.33700 2.33700 1.500000e+00 \n",
1575
- "2013-10-03 LTC 2.33700 2.50200 2.48900 2.50200 9.274000e+01 \n",
1576
- "... ... ... ... ... ... \n",
1577
- "2024-08-04 ZEN 8.40600 9.11200 8.28500 8.46200 2.071124e+06 \n",
1578
- " ZIL 0.01422 0.01441 0.01392 0.01396 2.048626e+08 \n",
1579
- " ZK 0.11451 0.11673 0.10857 0.10874 3.833253e+08 \n",
1580
- " ZRO 3.63100 3.64800 3.36200 3.36700 7.049472e+07 \n",
1581
- " ZRX 0.30550 0.31210 0.29830 0.29880 9.810764e+06 \n",
1582
- "\n",
1583
- " funding_rate \n",
1584
- "date ticker \n",
1585
- "2013-09-29 LTC 0.000000 \n",
1586
- "2013-09-30 LTC 0.000000 \n",
1587
- "2013-10-01 LTC 0.000000 \n",
1588
- "2013-10-02 LTC 0.000000 \n",
1589
- "2013-10-03 LTC 0.000000 \n",
1590
- "... ... \n",
1591
- "2024-08-04 ZEN 0.000194 \n",
1592
- " ZIL -0.000031 \n",
1593
- " ZK 0.000200 \n",
1594
- " ZRO 0.000184 \n",
1595
- " ZRX 0.000186 \n",
1596
- "\n",
1597
- "[387494 rows x 6 columns]"
1598
- ]
1599
- },
1600
- "execution_count": 85,
1601
- "metadata": {},
1602
- "output_type": "execute_result"
1603
- }
1604
- ],
982
+ "outputs": [],
1605
983
  "source": [
1606
- "df.drop(['BTC'], level=1, axis=0)"
984
+ "clean_df.to_csv('s3://factorlab-data/binance_historical_ohlcv_daily.csv')"
1607
985
  ]
1608
986
  },
1609
987
  {
1610
988
  "cell_type": "code",
1611
989
  "execution_count": null,
1612
- "id": "fabfa152",
990
+ "id": "b8fa525d",
991
+ "metadata": {},
992
+ "outputs": [],
993
+ "source": []
994
+ },
995
+ {
996
+ "cell_type": "code",
997
+ "execution_count": null,
998
+ "id": "f9488eba",
1613
999
  "metadata": {},
1614
1000
  "outputs": [],
1615
1001
  "source": []