cryptodatapy 0.2.4__py3-none-any.whl → 0.2.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (9) hide show
  1. cryptodatapy/extract/libraries/ccxt.ipynb +873 -0
  2. cryptodatapy/extract/libraries/ccxt_api.py +40 -133
  3. cryptodatapy/transform/clean_perp_futures_ohlcv.ipynb +1426 -247
  4. cryptodatapy/transform/convertparams.py +0 -1
  5. cryptodatapy/transform/filter.py +0 -1
  6. {cryptodatapy-0.2.4.dist-info → cryptodatapy-0.2.5.dist-info}/METADATA +1 -1
  7. {cryptodatapy-0.2.4.dist-info → cryptodatapy-0.2.5.dist-info}/RECORD +9 -8
  8. {cryptodatapy-0.2.4.dist-info → cryptodatapy-0.2.5.dist-info}/LICENSE +0 -0
  9. {cryptodatapy-0.2.4.dist-info → cryptodatapy-0.2.5.dist-info}/WHEEL +0 -0
cryptodatapy/extract/libraries/ccxt.ipynb ADDED
@@ -0,0 +1,873 @@
1
+ {
2
+ "cells": [
3
+ {
4
+ "cell_type": "code",
5
+ "execution_count": 1,
6
+ "id": "ae661131",
7
+ "metadata": {},
8
+ "outputs": [
9
+ {
10
+ "name": "stderr",
11
+ "output_type": "stream",
12
+ "text": [
13
+ "fatal: bad revision 'HEAD'\n"
14
+ ]
15
+ }
16
+ ],
17
+ "source": [
18
+ "import logging\n",
19
+ "from time import sleep\n",
20
+ "from typing import Any, Dict, List, Optional, Union\n",
21
+ "\n",
22
+ "# import ccxt\n",
23
+ "# import dbnomics\n",
24
+ "import numpy as np\n",
25
+ "import pandas as pd\n",
26
+ "\n",
27
+ "from cryptodatapy.extract.datarequest import DataRequest\n",
28
+ "# from cryptodatapy.extract.libraries.library import Library\n",
29
+ "# from cryptodatapy.transform.convertparams import ConvertParams\n",
30
+ "# from cryptodatapy.transform.wrangle import WrangleData\n",
31
+ "# from cryptodatapy.util.datacredentials import DataCredentials\n",
32
+ "\n",
33
+ "# data credentials\n",
34
+ "# data_cred = DataCredentials()"
35
+ ]
36
+ },
37
+ {
38
+ "cell_type": "code",
39
+ "execution_count": null,
40
+ "id": "afa3031b",
41
+ "metadata": {},
42
+ "outputs": [],
43
+ "source": []
44
+ },
45
+ {
46
+ "cell_type": "code",
47
+ "execution_count": null,
48
+ "id": "ae3caa8d",
49
+ "metadata": {},
50
+ "outputs": [],
51
+ "source": []
52
+ },
53
+ {
54
+ "cell_type": "code",
55
+ "execution_count": null,
56
+ "id": "d33712c1",
57
+ "metadata": {},
58
+ "outputs": [],
59
+ "source": [
60
+ "class CCXT(Library):\n",
61
+ " \"\"\"\n",
62
+ " Retrieves data from CCXT API.\n",
63
+ " \"\"\"\n",
64
+ "\n",
65
+ " def __init__(\n",
66
+ " self,\n",
67
+ " categories: Union[str, List[str]] = \"crypto\",\n",
68
+ " exchanges: Optional[List[str]] = None,\n",
69
+ " indexes: Optional[List[str]] = None,\n",
70
+ " assets: Optional[Dict[str, List[str]]] = None,\n",
71
+ " markets: Optional[Dict[str, List[str]]] = None,\n",
72
+ " market_types: List[str] = [\"spot\", \"future\", \"perpetual_future\", \"option\"],\n",
73
+ " fields: Optional[List[str]] = None,\n",
74
+ " frequencies: Optional[Dict[str, List[str]]] = None,\n",
75
+ " base_url: Optional[str] = None,\n",
76
+ " api_key: Optional[str] = None,\n",
77
+ " max_obs_per_call: Optional[int] = 10000,\n",
78
+ " rate_limit: Optional[Any] = None,\n",
79
+ " ):\n",
80
+ " \"\"\"\n",
81
+ " Constructor\n",
82
+ "\n",
83
+ " Parameters\n",
84
+ " ----------\n",
85
+ " categories: list or str, {'crypto', 'fx', 'rates', 'eqty', 'commodities', 'credit', 'macro', 'alt'}\n",
86
+ " List or string of available categories, e.g. ['crypto', 'fx', 'alt'].\n",
87
+ " exchanges: list, optional, default None\n",
88
+ " List of available exchanges, e.g. ['Binance', 'Coinbase', 'Kraken', 'FTX', ...].\n",
89
+ " indexes: list, optional, default None\n",
90
+ " List of available indexes, e.g. ['mvda', 'bvin'].\n",
91
+ " assets: dictionary, optional, default None\n",
92
+ " Dictionary of available assets, by exchange-assets key-value pairs, e.g. {'ftx': 'btc', 'eth', ...}.\n",
93
+ " markets: dictionary, optional, default None\n",
94
+ " Dictionary of available markets as base asset/quote currency pairs, by exchange-markets key-value pairs,\n",
95
+ " e.g. {'kraken': btcusdt', 'ethbtc', ...}.\n",
96
+ " market_types: list\n",
97
+ " List of available market types, e.g. [spot', 'perpetual_future', 'future', 'option'].\n",
98
+ " fields: list, optional, default None\n",
99
+ " List of available fields, e.g. ['open', 'high', 'low', 'close', 'volume'].\n",
100
+ " frequencies: dict, optional, default None\n",
101
+ " Dictionary of available frequencies, by exchange-frequencies key-value pairs,\n",
102
+ " e.g. {'binance' : '5min', '10min', '20min', '30min', '1h', '2h', '4h', '8h', 'd', 'w', 'm'}.\n",
103
+ " base_url: str, optional, default None\n",
104
+ " Base url used for GET requests. If not provided, default is set to base_url stored in DataCredentials.\n",
105
+ " api_key: str, optional, default None\n",
106
+ " Api key, e.g. 'dcf13983adf7dfa79a0dfa35adf'. If not provided, default is set to\n",
107
+ " api_key stored in DataCredentials.\n",
108
+ " max_obs_per_call: int, optional, default 10,000\n",
109
+ " Maximum number of observations returned per API call. If not provided, default is set to\n",
110
+ " api_limit stored in DataCredentials.\n",
111
+ " rate_limit: Any, optional, Default None\n",
112
+ " Number of API calls made and left, by time frequency.\n",
113
+ " \"\"\"\n",
114
+ " Library.__init__(\n",
115
+ " self,\n",
116
+ " categories,\n",
117
+ " exchanges,\n",
118
+ " indexes,\n",
119
+ " assets,\n",
120
+ " markets,\n",
121
+ " market_types,\n",
122
+ " fields,\n",
123
+ " frequencies,\n",
124
+ " base_url,\n",
125
+ " api_key,\n",
126
+ " max_obs_per_call,\n",
127
+ " rate_limit,\n",
128
+ " )\n",
129
+ "\n",
130
+ " def get_exchanges_info(self) -> List[str]:\n",
131
+ " \"\"\"\n",
132
+ " Get exchanges info.\n",
133
+ "\n",
134
+ " Returns\n",
135
+ " -------\n",
136
+ " exch: list or pd.DataFrame\n",
137
+ " List or dataframe with info on supported exchanges.\n",
138
+ " \"\"\"\n",
139
+ " self.exchanges = ccxt.exchanges\n",
140
+ "\n",
141
+ " return self.exchanges\n",
142
+ "\n",
143
+ " def get_indexes_info(self) -> None:\n",
144
+ " \"\"\"\n",
145
+ " Get indexes info.\n",
146
+ " \"\"\"\n",
147
+ " return None\n",
148
+ "\n",
149
+ " def get_assets_info(\n",
150
+ " self,\n",
151
+ " exch: str = \"binance\",\n",
152
+ " as_list: bool = False\n",
153
+ " ) -> Union[pd.DataFrame, List[str]]:\n",
154
+ " \"\"\"\n",
155
+ " Get assets info.\n",
156
+ "\n",
157
+ " Parameters\n",
158
+ " ----------\n",
159
+ " exch: str, default 'binance'\n",
160
+ " Name of exchange.\n",
161
+ " as_list: bool, default False\n",
162
+ " Returns assets info for selected exchanges as list.\n",
163
+ "\n",
164
+ " Returns\n",
165
+ " -------\n",
166
+ " assets: list or pd.DataFrame\n",
167
+ " Dataframe with info on available assets or list of assets.\n",
168
+ " \"\"\"\n",
169
+ " # inst exch\n",
170
+ " exchange = getattr(ccxt, exch)()\n",
171
+ "\n",
172
+ " # get assets on exchange and create df\n",
173
+ " exchange.load_markets()\n",
174
+ " self.assets = pd.DataFrame(exchange.currencies).T\n",
175
+ " self.assets.index.name = \"ticker\"\n",
176
+ "\n",
177
+ " # as list of assets\n",
178
+ " if as_list:\n",
179
+ " self.assets = self.assets.index.to_list()\n",
180
+ "\n",
181
+ " return self.assets\n",
182
+ "\n",
183
+ " def get_markets_info(\n",
184
+ " self,\n",
185
+ " exch: str = \"binance\",\n",
186
+ " quote_ccy: Optional[str] = None,\n",
187
+ " mkt_type: Optional[str] = None,\n",
188
+ " as_list: bool = False,\n",
189
+ " ) -> Union[pd.DataFrame, List[str]]:\n",
190
+ " \"\"\"\n",
191
+ " Get markets info.\n",
192
+ "\n",
193
+ " Parameters\n",
194
+ " ----------\n",
195
+ " exch: str, default 'binance'\n",
196
+ " Name of exchange.\n",
197
+ " quote_ccy: str, optional, default None\n",
198
+ " Quote currency.\n",
199
+ " mkt_type: str, {'spot', 'future', 'perpetual_future', 'option'}, optional, default None\n",
200
+ " Market type.\n",
201
+ " as_list: bool, default False\n",
202
+ " Returns markets info as list for selected exchange.\n",
203
+ "\n",
204
+ " Returns\n",
205
+ " -------\n",
206
+ " markets: list or pd.DataFrame\n",
207
+ " List or dataframe with info on available markets, by exchange.\n",
208
+ " \"\"\"\n",
209
+ " # inst exch\n",
210
+ " exchange = getattr(ccxt, exch)()\n",
211
+ "\n",
212
+ " # get assets on exchange\n",
213
+ " self.markets = pd.DataFrame(exchange.load_markets()).T\n",
214
+ " self.markets.index.name = \"ticker\"\n",
215
+ "\n",
216
+ " # quote ccy\n",
217
+ " if quote_ccy is not None:\n",
218
+ " self.markets = self.markets[self.markets.quote == quote_ccy.upper()]\n",
219
+ "\n",
220
+ " # mkt type\n",
221
+ " if mkt_type == \"perpetual_future\":\n",
222
+ " if self.markets[self.markets.type == \"swap\"].empty:\n",
223
+ " self.markets = self.markets[self.markets.type == \"future\"]\n",
224
+ " else:\n",
225
+ " self.markets = self.markets[self.markets.type == \"swap\"]\n",
226
+ " elif mkt_type == \"spot\" or mkt_type == \"future\" or mkt_type == \"option\":\n",
227
+ " self.markets = self.markets[self.markets.type == mkt_type]\n",
228
+ "\n",
229
+ " # dict of assets\n",
230
+ " if as_list:\n",
231
+ " self.markets = self.markets.index.to_list()\n",
232
+ "\n",
233
+ " return self.markets\n",
234
+ "\n",
235
+ " def get_fields_info(self) -> List[str]:\n",
236
+ " \"\"\"\n",
237
+ " Get fields info.\n",
238
+ "\n",
239
+ " Returns\n",
240
+ " -------\n",
241
+ " fields: list\n",
242
+ " List of available fields.\n",
243
+ " \"\"\"\n",
244
+ " # list of fields\n",
245
+ " self.fields = [\"open\", \"high\", \"low\", \"close\", \"volume\", \"funding_rate\"]\n",
246
+ "\n",
247
+ " return self.fields\n",
248
+ "\n",
249
+ " def get_frequencies_info(self, exch: str = \"binance\") -> Dict[str, List[str]]:\n",
250
+ " \"\"\"\n",
251
+ " Get frequencies info.\n",
252
+ "\n",
253
+ " Parameters\n",
254
+ " ----------\n",
255
+ " exch: str, default 'binance'\n",
256
+ " Name of exchange for which to get available assets.\n",
257
+ "\n",
258
+ " Returns\n",
259
+ " -------\n",
260
+ " freq: dictionary\n",
261
+ " Dictionary with info on available frequencies.\n",
262
+ " \"\"\"\n",
263
+ " # inst exch and load mkts\n",
264
+ " exchange = getattr(ccxt, exch)()\n",
265
+ " exchange.load_markets()\n",
266
+ "\n",
267
+ " # freq dict\n",
268
+ " self.frequencies = exchange.timeframes\n",
269
+ "\n",
270
+ " return self.frequencies\n",
271
+ "\n",
272
+ " def get_rate_limit_info(self, exch: str = \"binance\") -> Dict[str, Union[str, int]]:\n",
273
+ " \"\"\"\n",
274
+ " Get rate limit info.\n",
275
+ "\n",
276
+ " Parameters\n",
277
+ " ----------\n",
278
+ " exch: str, default 'binance'\n",
279
+ " Name of exchange.\n",
280
+ "\n",
281
+ " Returns\n",
282
+ " -------\n",
283
+ " rate_limit: dictionary\n",
284
+ " Dictionary with exchange and required minimal delay between HTTP requests that exchange in milliseconds.\n",
285
+ " \"\"\"\n",
286
+ " # inst exch\n",
287
+ " exchange = getattr(ccxt, exch)()\n",
288
+ "\n",
289
+ " self.rate_limit = {\n",
290
+ " \"exchange rate limit\": \"delay in milliseconds between two consequent HTTP requests to the same exchange\",\n",
291
+ " exch: exchange.rateLimit\n",
292
+ " }\n",
293
+ " return self.rate_limit\n",
294
+ "\n",
295
+ " def get_metadata(self) -> None:\n",
296
+ " \"\"\"\n",
297
+ " Get CCXT metadata.\n",
298
+ " \"\"\"\n",
299
+ " if self.exchanges is None:\n",
300
+ " self.exchanges = self.get_exchanges_info()\n",
301
+ " if self.market_types is None:\n",
302
+ " self.market_types = [\"spot\", \"future\", \"perpetual_future\", \"option\"]\n",
303
+ " if self.assets is None:\n",
304
+ " self.assets = self.get_assets_info(as_list=True)\n",
305
+ " if self.markets is None:\n",
306
+ " self.markets = self.get_markets_info(as_list=True)\n",
307
+ " if self.fields is None:\n",
308
+ " self.fields = self.get_fields_info()\n",
309
+ " if self.frequencies is None:\n",
310
+ " self.frequencies = self.get_frequencies_info()\n",
311
+ " if self.rate_limit is None:\n",
312
+ " self.rate_limit = self.get_rate_limit_info()\n",
313
+ "\n",
314
+ " def req_data(self,\n",
315
+ " data_req: DataRequest,\n",
316
+ " data_type: str,\n",
317
+ " ticker: str,\n",
318
+ " start_date: str = None,\n",
319
+ " ) -> pd.DataFrame:\n",
320
+ " \"\"\"\n",
321
+ " Sends data request to Python client.\n",
322
+ "\n",
323
+ " Parameters\n",
324
+ " ----------\n",
325
+ " data_req: DataRequest\n",
326
+ " Parameters of data request in CryptoDataPy format.\n",
327
+ " data_type: str, {'ohlcv', 'funding_rates'},\n",
328
+ " Data type to retrieve.\n",
329
+ " ticker: str\n",
330
+ " Ticker symbol to request data for.\n",
331
+ " start_date: str\n",
332
+ " Start date in 'YYYY-MM-DD' format.\n",
333
+ "\n",
334
+ "\n",
335
+ " Returns\n",
336
+ " -------\n",
337
+ " df: pd.DataFrame\n",
338
+ " Dataframe with datetime, ticker/identifier, and field/col values.\n",
339
+ " \"\"\"\n",
340
+ " # convert data request parameters to CCXT format\n",
341
+ " cx_data_req = ConvertParams(data_req).to_ccxt()\n",
342
+ " if start_date is None:\n",
343
+ " start_date = cx_data_req['start_date']\n",
344
+ "\n",
345
+ " # data types\n",
346
+ " data_types = {'ohlcv': 'fetchOHLCV', 'funding_rates': 'fetchFundingRateHistory'}\n",
347
+ "\n",
348
+ " # inst exch\n",
349
+ " exch = getattr(ccxt, cx_data_req['exch'])()\n",
350
+ " data_resp = []\n",
351
+ "\n",
352
+ " try:\n",
353
+ " if data_type == 'ohlcv':\n",
354
+ " data_resp = getattr(exch, data_types[data_type])(\n",
355
+ " ticker,\n",
356
+ " cx_data_req[\"freq\"],\n",
357
+ " since=start_date,\n",
358
+ " limit=self.max_obs_per_call,\n",
359
+ " )\n",
360
+ " elif data_type == 'funding_rates':\n",
361
+ " data_resp = getattr(exch, data_types[data_type])(\n",
362
+ " ticker,\n",
363
+ " since=start_date,\n",
364
+ " limit=1000,\n",
365
+ " )\n",
366
+ " \n",
367
+ " return data_resp\n",
368
+ " \n",
369
+ " except Exception as e:\n",
370
+ " logging.warning(f\"Failed to get {data_type} data for {ticker}.\")\n",
371
+ " logging.warning(e)\n",
372
+ " \n",
373
+ " return None\n",
374
+ "\n",
375
+ "\n",
376
+ " def fetch_all_ohlcv_hist(self, data_req: DataRequest, ticker: str) -> pd.DataFrame:\n",
377
+ " \"\"\"\n",
378
+ " Submits get requests to API until entire OHLCV history has been collected. Only necessary when\n",
379
+ " number of observations is larger than the maximum number of observations per call.\n",
380
+ "\n",
381
+ " Parameters\n",
382
+ " ----------\n",
383
+ " data_req: DataRequest\n",
384
+ " Parameters of data request in CryptoDataPy format.\n",
385
+ " ticker: str\n",
386
+ " Ticker symbol.\n",
387
+ "\n",
388
+ " Returns\n",
389
+ " -------\n",
390
+ " df: pd.DataFrame\n",
391
+ " Dataframe with entire data history retrieved.\n",
392
+ " \"\"\"\n",
393
+ " # convert data request parameters to CCXT format and set start date\n",
394
+ " cx_data_req = ConvertParams(data_req).to_ccxt()\n",
395
+ " start_date = cx_data_req['start_date']\n",
396
+ "\n",
397
+ " # create empty df\n",
398
+ " df = pd.DataFrame()\n",
399
+ " \n",
400
+ " # while loop condition\n",
401
+ " missing_vals, attempts = True, 0\n",
402
+ "\n",
403
+ " # run a while loop until all data collected\n",
404
+ " while missing_vals and attempts < cx_data_req['trials']:\n",
405
+ " \n",
406
+ " \n",
407
+ " data_resp = self.req_data(data_req=data_req,\n",
408
+ " data_type='ohlcv',\n",
409
+ " ticker=ticker,\n",
410
+ " start_date=start_date)\n",
411
+ " \n",
412
+ " if data_resp is None:\n",
413
+ " attempts += 1\n",
414
+ " sleep(self.get_rate_limit_info(exch=cx_data_req['exch'])[cx_data_req['exch']] / 1000)\n",
415
+ " logging.warning(\n",
416
+ " f\"Failed to pull data on attempt #{attempts}.\"\n",
417
+ " )\n",
418
+ " if attempts == cx_data_req[\"trials\"]:\n",
419
+ " logging.warning(\n",
420
+ " f\"Failed to get OHLCV data from {cx_data_req['exch']} for {ticker} after many attempts.\"\n",
421
+ " )\n",
422
+ " return None\n",
423
+ " \n",
424
+ " else:\n",
425
+ " # name cols and create df\n",
426
+ " header = [\"datetime\", \"open\", \"high\", \"low\", \"close\", \"volume\"]\n",
427
+ " data = pd.DataFrame(data_resp, columns=header)\n",
428
+ " df = pd.concat([df, data])\n",
429
+ "\n",
430
+ " # check if all data has been extracted\n",
431
+ " time_diff = cx_data_req[\"end_date\"] - df.datetime.iloc[-1]\n",
432
+ " if pd.Timedelta(milliseconds=time_diff) < pd.Timedelta(cx_data_req[\"freq\"]):\n",
433
+ " missing_vals = False\n",
434
+ " # missing data, infinite loop\n",
435
+ " elif df.datetime.iloc[-1] == df.datetime.iloc[-2]:\n",
436
+ " missing_vals = False\n",
437
+ " logging.warning(f\"Missing recent OHLCV data for {ticker}.\")\n",
438
+ " # reset end date and pause before calling API\n",
439
+ " else:\n",
440
+ " # change end date\n",
441
+ " start_date = df.datetime.iloc[-1]\n",
442
+ "\n",
443
+ " # rate limit\n",
444
+ " sleep(self.get_rate_limit_info(exch=cx_data_req['exch'])[cx_data_req['exch']] / 1000)\n",
445
+ "\n",
446
+ " return df\n",
447
+ "\n",
448
+ "\n",
449
+ " def fetch_all_funding_hist(self, data_req: DataRequest, ticker: str) -> pd.DataFrame:\n",
450
+ " \"\"\"\n",
451
+ " Submits get requests to API until entire funding rate history has been collected. Only necessary when\n",
452
+ " number of observations is larger than the maximum number of observations per call.\n",
453
+ "\n",
454
+ " Parameters\n",
455
+ " ----------\n",
456
+ " data_req: DataRequest\n",
457
+ " Parameters of data request in CryptoDataPy format.\n",
458
+ " ticker: str\n",
459
+ " Ticker symbol.|\n",
460
+ "\n",
461
+ " Returns\n",
462
+ " -------\n",
463
+ " df: pd.DataFrame\n",
464
+ " Dataframe with entire data history retrieved.\n",
465
+ " \"\"\"\n",
466
+ " # convert data request parameters to CCXT format and set start date\n",
467
+ " cx_data_req = ConvertParams(data_req).to_ccxt()\n",
468
+ " start_date = cx_data_req['start_date']\n",
469
+ "\n",
470
+ " # create empty df\n",
471
+ " df = pd.DataFrame()\n",
472
+ " # while loop condition\n",
473
+ " missing_vals, attempts = True, 0\n",
474
+ "\n",
475
+ " # run a while loop until all data collected\n",
476
+ " while missing_vals and attempts < cx_data_req['trials']:\n",
477
+ " \n",
478
+ " # data req\n",
479
+ " data_resp = self.req_data(data_req=data_req,\n",
480
+ " data_type='funding_rates',\n",
481
+ " ticker=ticker,\n",
482
+ " start_date=start_date)\n",
483
+ " \n",
484
+ " if data_resp is None:\n",
485
+ " attempts += 1\n",
486
+ " sleep(self.get_rate_limit_info(exch=cx_data_req['exch'])[cx_data_req['exch']] / 1000)\n",
487
+ " logging.warning(\n",
488
+ " f\"Failed to pull data on attempt #{attempts}.\"\n",
489
+ " )\n",
490
+ " if attempts == cx_data_req[\"trials\"]:\n",
491
+ " logging.warning(\n",
492
+ " f\"Failed to get funding_rates from {cx_data_req['exch']} for {ticker} after many attempts.\"\n",
493
+ " )\n",
494
+ " return None\n",
495
+ "\n",
496
+ " else:\n",
497
+ " # add to df\n",
498
+ " data = pd.DataFrame(data_resp)\n",
499
+ " df = pd.concat([df, data])\n",
500
+ " # check if all data has been extracted\n",
501
+ " time_diff = pd.to_datetime(\n",
502
+ " cx_data_req[\"end_date\"], unit=\"ms\"\n",
503
+ " ) - pd.to_datetime(data.datetime.iloc[-1]).tz_localize(None)\n",
504
+ " if time_diff < pd.Timedelta(\"8h\"):\n",
505
+ " missing_vals = False\n",
506
+ " # missing data, infinite loop\n",
507
+ " elif df.datetime.iloc[-1] == df.datetime.iloc[-2]:\n",
508
+ " missing_vals = False\n",
509
+ " logging.warning(f\"Missing recent funding rate data for {ticker}.\")\n",
510
+ " # reset end date and pause before calling API\n",
511
+ " else:\n",
512
+ " # change end date\n",
513
+ " start_date = data.timestamp.iloc[-1]\n",
514
+ "\n",
515
+ " # rate limit\n",
516
+ " sleep(self.get_rate_limit_info(exch=cx_data_req['exch'])[cx_data_req['exch']] / 1000)\n",
517
+ "\n",
518
+ " return df\n",
519
+ "\n",
520
+ " @staticmethod\n",
521
+ " def wrangle_data_resp(data_req: DataRequest, data_resp: pd.DataFrame) -> pd.DataFrame:\n",
522
+ " \"\"\"\n",
523
+ " Wrangle data response.\n",
524
+ "\n",
525
+ " Parameters\n",
526
+ " ----------\n",
527
+ " data_req: DataRequest\n",
528
+ " Parameters of data request in CryptoDataPy format.\n",
529
+ " data_resp: pd.DataFrame\n",
530
+ " Data response from GET request.\n",
531
+ "\n",
532
+ " Returns\n",
533
+ " -------\n",
534
+ " df: pd.DataFrame\n",
535
+ " Wrangled dataframe with DatetimeIndex and values in tidy format.\n",
536
+ " \"\"\"\n",
537
+ "\n",
538
+ " return WrangleData(data_req, data_resp).ccxt()\n",
539
+ "\n",
540
+ " def fetch_tidy_ohlcv(self, data_req: DataRequest, ticker: str) -> pd.DataFrame:\n",
541
+ " \"\"\"\n",
542
+ " Gets entire OHLCV history and wrangles the data response into tidy data format.\n",
543
+ "\n",
544
+ " Parameters\n",
545
+ " ----------\n",
546
+ " data_req: DataRequest\n",
547
+ " Parameters of data request in CryptoDataPy format.\n",
548
+ " ticker: str\n",
549
+ " Ticker symbol.\n",
550
+ "\n",
551
+ " Returns\n",
552
+ " -------\n",
553
+ " df: pd.DataFrame\n",
554
+ " Dataframe with entire data history retrieved and wrangled into tidy data format.\n",
555
+ " \"\"\"\n",
556
+ " # get entire data history\n",
557
+ " df = self.fetch_all_ohlcv_hist(data_req, ticker)\n",
558
+ " \n",
559
+ " # wrangle df\n",
560
+ " if df is not None:\n",
561
+ " df = self.wrangle_data_resp(data_req, df)\n",
562
+ " \n",
563
+ " return df\n",
564
+ "\n",
565
+ "\n",
566
+ " def fetch_tidy_funding_rates(self, data_req: DataRequest, ticker: str) -> pd.DataFrame:\n",
567
+ " \"\"\"\n",
568
+ " Gets entire funding rates history and wrangles the data response into tidy data format.\n",
569
+ "\n",
570
+ " Parameters\n",
571
+ " ----------\n",
572
+ " data_req: DataRequest\n",
573
+ " Parameters of data request in CryptoDataPy format.\n",
574
+ " ticker: str\n",
575
+ " Ticker symbol.\n",
576
+ "\n",
577
+ " Returns\n",
578
+ " -------\n",
579
+ " df: pd.DataFrame\n",
580
+ " Dataframe with entire data history retrieved and wrangled into tidy data format.\n",
581
+ " \"\"\"\n",
582
+ " # get entire data history\n",
583
+ " df = self.fetch_all_funding_hist(data_req, ticker)\n",
584
+ " \n",
585
+ " # wrangle df\n",
586
+ " if df is not None:\n",
587
+ " df = self.wrangle_data_resp(data_req, df)\n",
588
+ " \n",
589
+ " return df\n",
590
+ "\n",
591
+ " def check_params(self, data_req) -> None:\n",
592
+ " \"\"\"\n",
593
+ " Checks if data request parameters are valid.\n",
594
+ "\n",
595
+ " \"\"\"\n",
596
+ " # convert data request parameters to CCXT format\n",
597
+ " cx_data_req = ConvertParams(data_req).to_ccxt()\n",
598
+ "\n",
599
+ " # inst exch\n",
600
+ " exch = getattr(ccxt, cx_data_req['exch'])()\n",
601
+ "\n",
602
+ " # check tickers\n",
603
+ " tickers = self.get_assets_info(exch=cx_data_req[\"exch\"], as_list=True)\n",
604
+ " if not any([ticker.upper() in tickers for ticker in cx_data_req[\"tickers\"]]):\n",
605
+ " raise ValueError(\n",
606
+ " f\"Assets are not available. Use assets attribute to check available assets for {cx_data_req['exch']}\")\n",
607
+ "\n",
608
+ " # check tickers\n",
609
+ " fields = self.get_fields_info()\n",
610
+ " if not any([field in fields for field in data_req.fields]):\n",
611
+ " raise ValueError(\n",
612
+ " f\"Fields are not available. Use fields attribute to check available fields.\"\n",
613
+ " )\n",
614
+ "\n",
615
+ " # check freq\n",
616
+ " if cx_data_req[\"freq\"] not in exch.timeframes:\n",
617
+ " raise ValueError(\n",
618
+ " f\"{data_req.freq} is not available for {cx_data_req['exch']}.\"\n",
619
+ " )\n",
620
+ "\n",
621
+ " # check if ohlcv avail on exch\n",
622
+ " if any([field in self.fields[:-1] for field in data_req.fields]) and \\\n",
623
+ " not exch.has[\"fetchOHLCV\"]:\n",
624
+ " raise ValueError(\n",
625
+ " f\"OHLCV data is not available for {cx_data_req['exch']}.\"\n",
626
+ " f\" Try another exchange or data request.\"\n",
627
+ " )\n",
628
+ "\n",
629
+ " # check if funding avail on exch\n",
630
+ " if any([field == 'funding_rate' for field in data_req.fields]) and \\\n",
631
+ " not exch.has[\"fetchFundingRateHistory\"]:\n",
632
+ " raise ValueError(\n",
633
+ " f\"Funding rates are not available for {cx_data_req['exch']}.\"\n",
634
+ " f\" Try another exchange or data request.\"\n",
635
+ " )\n",
636
+ "\n",
637
+ " # check if perp future\n",
638
+ " if any([field == 'funding_rate' for field in data_req.fields]) and \\\n",
639
+ " data_req.mkt_type == \"spot\":\n",
640
+ " raise ValueError(\n",
641
+ " f\"Funding rates are not available for spot markets.\"\n",
642
+ " f\" Market type must be perpetual futures.\"\n",
643
+ " )\n",
644
+ "\n",
645
+ " def fetch_ohlcv(self, data_req: DataRequest) -> pd.DataFrame:\n",
646
+ " \"\"\"\n",
647
+ " Loops list of tickers, retrieves OHLCV data for each ticker in tidy format and stores it in a\n",
648
+ " multiindex dataframe.\n",
649
+ "\n",
650
+ " Parameters\n",
651
+ " ----------\n",
652
+ " data_req: DataRequest\n",
653
+ " Parameters of data request in CryptoDataPy format.\n",
654
+ "\n",
655
+ " Returns\n",
656
+ " -------\n",
657
+ " df: pd.DataFrame - MultiIndex\n",
658
+ " Dataframe with DatetimeIndex (level 0), ticker (level 1) and OHLCV values (cols), in tidy data format.\n",
659
+ " \"\"\"\n",
660
+ " # convert data request parameters to CCXT format\n",
661
+ " cx_data_req = ConvertParams(data_req).to_ccxt()\n",
662
+ "\n",
663
+ " # check params\n",
664
+ " self.check_params(data_req)\n",
665
+ "\n",
666
+ " # empty df to add data\n",
667
+ " df = pd.DataFrame()\n",
668
+ "\n",
669
+ " # loop through tickers\n",
670
+ " for mkt, ticker in zip(cx_data_req['mkts'], data_req.tickers):\n",
671
+ " \n",
672
+ " df0 = self.fetch_tidy_ohlcv(data_req, mkt)\n",
673
+ " \n",
674
+ " if df0 is not None:\n",
675
+ " \n",
676
+ " # add ticker to index\n",
677
+ " df0['ticker'] = ticker.upper()\n",
678
+ " df0.set_index(['ticker'], append=True, inplace=True)\n",
679
+ " # concat df and df1\n",
680
+ " df = pd.concat([df, df0])\n",
681
+ " \n",
682
+ " return df\n",
683
+ "\n",
684
+ " def fetch_funding_rates(self, data_req: DataRequest) -> pd.DataFrame:\n",
685
+ " \"\"\"\n",
686
+ " Loops list of tickers, retrieves funding rates data for each ticker in tidy format and stores it in a\n",
687
+ " multiindex dataframe.\n",
688
+ "\n",
689
+ " Parameters\n",
690
+ " ----------\n",
691
+ " data_req: DataRequest\n",
692
+ " Parameters of data request in CryptoDataPy format.\n",
693
+ "\n",
694
+ " Returns\n",
695
+ " -------\n",
696
+ " df: pd.DataFrame - MultiIndex\n",
697
+ " Dataframe with DatetimeIndex (level 0), ticker (level 1) and OHLCV values (cols), in tidy data format.\n",
698
+ " \"\"\"\n",
699
+ " # convert data request parameters to CCXT format\n",
700
+ " cx_data_req = ConvertParams(data_req).to_ccxt()\n",
701
+ "\n",
702
+ " # check params\n",
703
+ " self.check_params(data_req)\n",
704
+ "\n",
705
+ " # empty df to add data\n",
706
+ " df = pd.DataFrame()\n",
707
+ " \n",
708
+ " # loop through tickers\n",
709
+ " for mkt, ticker in zip(cx_data_req['mkts'], data_req.tickers):\n",
710
+ " \n",
711
+ " df0 = self.fetch_tidy_funding_rates(data_req, mkt)\n",
712
+ " \n",
713
+ " if df0 is not None:\n",
714
+ " \n",
715
+ " # add ticker to index\n",
716
+ " df0['ticker'] = ticker.upper()\n",
717
+ " df0.set_index(['ticker'], append=True, inplace=True)\n",
718
+ " # concat df and df1\n",
719
+ " df = pd.concat([df, df0])\n",
720
+ " \n",
721
+ " return df\n",
722
+ "\n",
723
+ " def get_data(self, data_req: DataRequest) -> pd.DataFrame:\n",
724
+ " \"\"\"\n",
725
+ " Get data specified by data request.\n",
726
+ "\n",
727
+ " Parameters\n",
728
+ " data_req: DataRequest\n",
729
+ " Parameters of data request in CryptoDataPy format.\n",
730
+ "\n",
731
+ " Returns\n",
732
+ " -------\n",
733
+ " df: pd.DataFrame - MultiIndex\n",
734
+ " DataFrame with DatetimeIndex (level 0), ticker (level 1), and values for selected fields (cols).\n",
735
+ " \"\"\"\n",
736
+ " # empty df\n",
737
+ " df = pd.DataFrame()\n",
738
+ "\n",
739
+ " # get OHLCV data\n",
740
+ " ohlcv_list = [\"open\", \"high\", \"low\", \"close\", \"volume\"]\n",
741
+ " if any([field in ohlcv_list for field in data_req.fields]):\n",
742
+ " df0 = self.fetch_ohlcv(data_req)\n",
743
+ " df = pd.concat([df, df0])\n",
744
+ "\n",
745
+ " # get funding rate data\n",
746
+ " if any([field == \"funding_rate\" for field in data_req.fields]):\n",
747
+ " df1 = self.fetch_funding_rates(data_req)\n",
748
+ " df = pd.concat([df, df1], axis=1)\n",
749
+ "\n",
750
+ " # check if df empty\n",
751
+ " if df.empty:\n",
752
+ " raise Exception(\n",
753
+ " \"No data returned. Check data request parameters and try again.\"\n",
754
+ " )\n",
755
+ "\n",
756
+ " # filter df for desired fields and typecast\n",
757
+ " fields = [field for field in data_req.fields if field in df.columns]\n",
758
+ " df = df.loc[:, fields]\n",
759
+ "\n",
760
+ " return df.sort_index()"
761
+ ]
762
+ },
763
+ {
764
+ "cell_type": "code",
765
+ "execution_count": null,
766
+ "id": "bb2c3737",
767
+ "metadata": {},
768
+ "outputs": [],
769
+ "source": [
770
+ "cc = CCXT()"
771
+ ]
772
+ },
773
+ {
774
+ "cell_type": "code",
775
+ "execution_count": null,
776
+ "id": "4422a4b7",
777
+ "metadata": {},
778
+ "outputs": [],
779
+ "source": [
780
+ "dr = DataRequest(mkt_type='perpetual_future', tickers=['BTC', 'BTCST', 'ETH'])"
781
+ ]
782
+ },
783
+ {
784
+ "cell_type": "code",
785
+ "execution_count": null,
786
+ "id": "6d17ce2d",
787
+ "metadata": {},
788
+ "outputs": [],
789
+ "source": [
790
+ "# cc.get_all_ohlcv_hist(dr, 'BTCST/USDT:USDT')"
791
+ ]
792
+ },
793
+ {
794
+ "cell_type": "code",
795
+ "execution_count": null,
796
+ "id": "496f75fb",
797
+ "metadata": {},
798
+ "outputs": [],
799
+ "source": [
800
+ "cc.req_data(dr, 'ohlcv','BTCST/USDT:USDT')"
801
+ ]
802
+ },
803
+ {
804
+ "cell_type": "code",
805
+ "execution_count": null,
806
+ "id": "6e9fb0df",
807
+ "metadata": {},
808
+ "outputs": [],
809
+ "source": [
810
+ "cc.fetch_all_ohlcv_hist(dr, 'BTCST/USDT:USDT')"
811
+ ]
812
+ },
813
+ {
814
+ "cell_type": "code",
815
+ "execution_count": null,
816
+ "id": "5320902e",
817
+ "metadata": {},
818
+ "outputs": [],
819
+ "source": [
820
+ "cc.fetch_tidy_ohlcv(dr, 'BTCST/USDT:USDT')"
821
+ ]
822
+ },
823
+ {
824
+ "cell_type": "code",
825
+ "execution_count": null,
826
+ "id": "55526fa6",
827
+ "metadata": {},
828
+ "outputs": [],
829
+ "source": [
830
+ "cc.fetch_ohlcv(dr)"
831
+ ]
832
+ },
833
+ {
834
+ "cell_type": "code",
835
+ "execution_count": null,
836
+ "id": "a0fdd090",
837
+ "metadata": {},
838
+ "outputs": [],
839
+ "source": [
840
+ "cc.get_data(dr)"
841
+ ]
842
+ },
843
+ {
844
+ "cell_type": "code",
845
+ "execution_count": null,
846
+ "id": "30fccc7c",
847
+ "metadata": {},
848
+ "outputs": [],
849
+ "source": []
850
+ }
851
+ ],
852
+ "metadata": {
853
+ "kernelspec": {
854
+ "display_name": "cryptodatapy",
855
+ "language": "python",
856
+ "name": "cryptodatapy"
857
+ },
858
+ "language_info": {
859
+ "codemirror_mode": {
860
+ "name": "ipython",
861
+ "version": 3
862
+ },
863
+ "file_extension": ".py",
864
+ "mimetype": "text/x-python",
865
+ "name": "python",
866
+ "nbconvert_exporter": "python",
867
+ "pygments_lexer": "ipython3",
868
+ "version": "3.9.12"
869
+ }
870
+ },
871
+ "nbformat": 4,
872
+ "nbformat_minor": 5
873
+ }