cryptodatapy 0.2.4__py3-none-any.whl → 0.2.6__py3-none-any.whl

cryptodatapy/extract/data_vendors/cryptocompare_api.py

@@ -84,8 +84,9 @@ class CryptoCompare(DataVendor):
         if categories is None:
             self.categories = ['crypto']
         if api_key is None:
-            raise TypeError("Set your api key. We recommend setting your api key in environment variables as"
-                            "'CRYPTOCOMPARE_API_KEY', will allow DataCredentials to automatically load it.")
+            raise TypeError("Set your CryptoCompare api key in environment variables as 'CRYPTOCOMPARE_API_KEY' or "
+                            "add it as an argument when instantiating the class. To get an api key, visit: "
+                            "https://min-api.cryptocompare.com/")
         if exchanges is None:
             self.exchanges = self.get_exchanges_info(as_list=True)
         if indexes is None:
@@ -467,9 +468,6 @@ class CryptoCompare(DataVendor):
         urls_params = self.set_urls_params(data_req, data_type, ticker)
         url, params = urls_params['url'], urls_params['params']
 
-        # # data req
-        # data_resp = DataRequest().get_req(url=url, params=params)
-
         # data req
         data_resp = DataRequest().get_req(url=url, params=params)
 

cryptodatapy/extract/data_vendors/datavendor.py

@@ -92,7 +92,7 @@ class DataVendor(ABC):
 
     @exchanges.setter
     def exchanges(
-        self, exchanges: Optional[Union[str, List[str], Dict[str, List[str]]]]
+        self, exchanges: Optional[Union[str, List[str], Dict[str, List[str]], pd.DataFrame]]
     ):
         """
         Sets a list of available exchanges for the data vendor.
@@ -101,6 +101,7 @@ class DataVendor(ABC):
             exchanges is None
             or isinstance(exchanges, list)
             or isinstance(exchanges, dict)
+            or isinstance(exchanges, pd.DataFrame)
         ):
             self._exchanges = exchanges
         elif isinstance(exchanges, str):
@@ -126,11 +127,16 @@ class DataVendor(ABC):
         return self._indexes
 
     @indexes.setter
-    def indexes(self, indexes: Optional[Union[str, List[str], Dict[str, List[str]]]]):
+    def indexes(self, indexes: Optional[Union[str, List[str], Dict[str, List[str]], pd.DataFrame]]):
         """
         Sets a list of available indexes for the data vendor.
         """
-        if indexes is None or isinstance(indexes, list) or isinstance(indexes, dict):
+        if (
+            indexes is None
+            or isinstance(indexes, list)
+            or isinstance(indexes, dict)
+            or isinstance(indexes, pd.DataFrame)
+        ):
             self._indexes = indexes
         elif isinstance(indexes, str):
             self._indexes = [indexes]
@@ -155,11 +161,16 @@ class DataVendor(ABC):
         return self._assets
 
     @assets.setter
-    def assets(self, assets: Optional[Union[str, List[str], Dict[str, List[str]]]]):
+    def assets(self, assets: Optional[Union[str, List[str], Dict[str, List[str]], pd.DataFrame]]):
         """
         Sets a list of available assets for the data vendor.
         """
-        if assets is None or isinstance(assets, list) or isinstance(assets, dict):
+        if (
+            assets is None
+            or isinstance(assets, list)
+            or isinstance(assets, dict)
+            or isinstance(assets, pd.DataFrame)
+        ):
             self._assets = assets
         elif isinstance(assets, str):
             self._assets = [assets]
@@ -184,11 +195,16 @@ class DataVendor(ABC):
         return self._markets
 
     @markets.setter
-    def markets(self, markets: Optional[Union[str, List[str], Dict[str, List[str]]]]):
+    def markets(self, markets: Optional[Union[str, List[str], Dict[str, List[str]], pd.DataFrame]]):
         """
         Sets a list of available markets for the data vendor.
         """
-        if markets is None or isinstance(markets, list) or isinstance(markets, dict):
+        if (
+            markets is None
+            or isinstance(markets, list)
+            or isinstance(markets, dict)
+            or isinstance(markets, pd.DataFrame)
+        ):
             self._markets = markets
         elif isinstance(markets, str):
             self._markets = [markets]
@@ -251,11 +267,16 @@ class DataVendor(ABC):
         return self._fields
 
     @fields.setter
-    def fields(self, fields: Optional[Union[str, List[str], Dict[str, List[str]]]]):
+    def fields(self, fields: Optional[Union[str, List[str], Dict[str, List[str]], pd.DataFrame]]):
         """
         Sets a list of available fields for the data vendor.
         """
-        if fields is None or isinstance(fields, list) or isinstance(fields, dict):
+        if (
+            fields is None
+            or isinstance(fields, list)
+            or isinstance(fields, dict)
+            or isinstance(fields, pd.DataFrame)
+        ):
             self._fields = fields
         elif isinstance(fields, str):
             self._fields = [fields]
@@ -280,9 +301,7 @@ class DataVendor(ABC):
         return self._frequencies
 
     @frequencies.setter
-    def frequencies(
-        self, frequencies: Optional[Union[str, List[str], Dict[str, List[str]]]]
-    ):
+    def frequencies(self, frequencies: Optional[Union[str, List[str], Dict[str, List[str]], pd.DataFrame]]):
         """
         Sets a list of available data frequencies for the data vendor.
         """
@@ -290,6 +309,7 @@ class DataVendor(ABC):
             frequencies is None
             or isinstance(frequencies, list)
             or isinstance(frequencies, dict)
+            or isinstance(frequencies, pd.DataFrame)
         ):
             self._frequencies = frequencies
         elif isinstance(frequencies, str):

cryptodatapy/extract/data_vendors/glassnode_api.py

@@ -79,8 +79,9 @@ class Glassnode(DataVendor):
         if categories is None:
             self.categories = ['crypto']
         if api_key is None:
-            raise TypeError("Set your api key. We recommend setting your api key in environment variables as"
-                            "'GLASSNODE_API_KEY', will allow DataCredentials to automatically load it.")
+            raise TypeError("Set your Glassnode api key in environment variables as 'GLASSNODE_API_KEY' or "
+                            "add it as an argument when instantiating the class. To get an api key, visit: "
+                            "https://docs.glassnode.com/basic-api/api-key")
         if assets is None:
             self.assets = self.get_assets_info(as_list=True)
         if fields is None:

cryptodatapy/extract/data_vendors/tiingo_api.py

@@ -109,8 +109,9 @@ class Tiingo(DataVendor):
         if categories is None:
             self.categories = ["crypto", "fx", "eqty"]
         if api_key is None:
-            raise TypeError("Set your api key. We recommend setting your api key in environment variables as"
-                            "'TIINGO_API_KEY', will allow DataCredentials to automatically load it.")
+            raise TypeError("Set your Tiingo api key in environment variables as 'TIINGO_API_KEY' or "
+                            "add it as an argument when instantiating the class. To get an api key, visit: "
+                            "https://www.tiingo.com/")
         if exchanges is None:
             self.exchanges = self.get_exchanges_info()
         if assets is None:

cryptodatapy/extract/datarequest.py

@@ -197,7 +197,9 @@ class DataRequest:
             "y": "yearly",
         }
 
-        if frequency not in list(freq_dict.keys()):
+        if frequency is None:
+            self._frequency = frequency
+        elif frequency not in list(freq_dict.keys()):
             raise ValueError(
                 f"{frequency} is an invalid data frequency. Valid frequencies are: {freq_dict}"
             )
@@ -555,15 +557,59 @@ class DataRequest:
             # get request
             try:
                 resp = requests.get(url, params=params, headers=headers)
-                assert resp.status_code == 200
-            # exception
-            except AssertionError as e:
-                logging.warning(e)
+                # check for status code
+                resp.raise_for_status()
+
+                return resp.json()
+
+            # handle HTTP errors
+            except requests.exceptions.HTTPError as http_err:
+                status_code = resp.status_code
+
+                # Tailored handling for different status codes
+                if status_code == 400:
+                    logging.warning(f"Bad Request (400): {resp.text}")
+                elif status_code == 401:
+                    logging.warning("Unauthorized (401): Check the authentication credentials.")
+                elif status_code == 403:
+                    logging.warning("Forbidden (403): You do not have permission to access this resource.")
+                elif status_code == 404:
+                    logging.warning("Not Found (404): The requested resource could not be found.")
+                elif status_code == 500:
+                    logging.error("Internal Server Error (500): The server encountered an error.")
+                elif status_code == 503:
+                    logging.error("Service Unavailable (503): The server is temporarily unavailable.")
+                else:
+                    logging.error(f"HTTP error occurred: {http_err} (Status Code: {status_code})")
+                    logging.error(f"Response Content: {resp.text}")
+
+                # Increment attempts and log warning
+                attempts += 1
+                logging.warning(f"Attempt #{attempts}: Failed to get data due to: {http_err}")
+                sleep(self.pause)  # Pause before retrying
+                if attempts == self.trials:
+                    logging.error("Max attempts reached. Unable to fetch data.")
+                    break
+
+            # handle non-HTTP exceptions (e.g., network issues)
+            except requests.exceptions.RequestException as req_err:
                 attempts += 1
-                logging.warning(f"Failed to get data on attempt #{attempts}.")
+                logging.warning(f"Request error on attempt #{attempts}: {req_err}. "
+                                f"Retrying after {self.pause} seconds...")
                 sleep(self.pause)
-                if attempts == 3:
+                if attempts == self.trials:
+                    logging.error("Max attempts reached. Unable to fetch data due to request errors.")
                     break
 
-            else:
-                return resp.json()
+            # handle other exceptions
+            except Exception as e:
+                attempts += 1
+                logging.warning(f"An unexpected error occurred: {e}. "
+                                f"Retrying after {self.pause} seconds...")
+                sleep(self.pause)
+                if attempts == self.trials:
+                    logging.error("Max attempts reached. Unable to fetch data due to request errors.")
+                    break
+
+        # return None if the API call fails
+        return None

cryptodatapy/extract/libraries/ccxt_api.py

@@ -3,7 +3,6 @@ from time import sleep
 from typing import Any, Dict, List, Optional, Union
 
 import ccxt
-import numpy as np
 import pandas as pd
 
 from cryptodatapy.extract.datarequest import DataRequest
@@ -20,7 +19,6 @@ class CCXT(Library):
     """
     Retrieves data from CCXT API.
     """
-
     def __init__(
             self,
             categories: Union[str, List[str]] = "crypto",
@@ -86,97 +84,16 @@ class CCXT(Library):
             rate_limit,
         )
 
-    def get_exchanges_info(
-            self,
-            exch: Optional[str] = None,
-            as_list: bool = True
-    ) -> Union[List[str], pd.DataFrame]:
+    def get_exchanges_info(self) -> List[str]:
         """
         Get exchanges info.
 
-        Parameters
-        ----------
-        exch: str, default None
-            Name of exchange.
-        as_list: bool, default False
-            Returns exchanges info as list.
-
         Returns
         -------
         exch: list or pd.DataFrame
             List or dataframe with info on supported exchanges.
         """
-        # list
-        if as_list:
-            self.exchanges = ccxt.exchanges
-
-        # df
-        else:
-            if exch is not None:
-                self.exchanges = [exch]
-            else:
-                self.exchanges = ccxt.exchanges
-                print(
-                    "Getting metadata for all supported exchanges can take a few minutes."
-                    " For quick info on a specific exchange, provide the name of the exchange in the exch parameter."
-                )
-
-            # exch df
-            exch_df = pd.DataFrame(
-                index=self.exchanges,
-                columns=[
-                    "id",
-                    "name",
-                    "countries",
-                    "urls",
-                    "version",
-                    "api",
-                    "has",
-                    "timeframes",
-                    "timeout",
-                    "rateLimit",
-                    "userAgent",
-                    "verbose",
-                    "markets",
-                    "symbols",
-                    "currencies",
-                    "markets_by_id",
-                    "currencies_by_id",
-                    "api_key",
-                    "secret",
-                    "uid",
-                    "options",
-                ],
-            )
-
-            # Extract exchange info
-            for index, row in exch_df.iterrows():
-                try:
-                    exchange = getattr(ccxt, index)()
-                    exchange.load_markets()
-                except AttributeError as e:
-                    print(f"AttributeError: {e} for exchange {index}")
-                    exch_df.loc[index, :] = np.nan
-                except ccxt.BaseError as e:  # Catch specific ccxt exceptions
-                    print(f"CCXT Error: {e} for exchange {index}")
-                    exch_df.loc[index, :] = np.nan
-                except Exception as e:  # Fallback for any other exceptions
-                    print(f"Unexpected error: {e} for exchange {index}")
-                    exch_df.loc[index, :] = np.nan
-                else:
-                    for col in exch_df.columns:
-                        try:
-                            exch_df.loc[index, col] = str(getattr(exchange, col))
-                        except AttributeError as e:
-                            print(f"AttributeError: {e} for attribute {col} in exchange {index}")
-                            exch_df.loc[index, col] = np.nan
-                        except Exception as e:  # Fallback for any other exceptions
-                            print(f"Unexpected error: {e} for attribute {col} in exchange {index}")
-                            exch_df.loc[index, col] = np.nan
-
-            # set index name
-            exch_df.index.name = "exchange"
-            self.exchanges = exch_df
+        self.exchanges = ccxt.exchanges
 
         return self.exchanges
 
@@ -337,7 +254,7 @@ class CCXT(Library):
         Get CCXT metadata.
         """
         if self.exchanges is None:
-            self.exchanges = self.get_exchanges_info(as_list=True)
+            self.exchanges = self.get_exchanges_info()
         if self.market_types is None:
             self.market_types = ["spot", "future", "perpetual_future", "option"]
         if self.assets is None:
@@ -356,6 +273,7 @@ class CCXT(Library):
                  data_type: str,
                  ticker: str,
                  start_date: str = None,
+                 end_date: str = None,
                  ) -> pd.DataFrame:
         """
         Sends data request to Python client.
@@ -370,9 +288,8 @@ class CCXT(Library):
             Ticker symbol to request data for.
         start_date: str
             Start date in 'YYYY-MM-DD' format.
-
-        Other Parameters
-        ----------------
+        end_date: str
+            End date in 'YYYY-MM-DD' format.
 
 
         Returns
@@ -384,6 +301,8 @@ class CCXT(Library):
         cx_data_req = ConvertParams(data_req).to_ccxt()
         if start_date is None:
             start_date = cx_data_req['start_date']
+        if end_date is None:
+            end_date = cx_data_req['end_date']
 
         # data types
         data_types = {'ohlcv': 'fetchOHLCV', 'funding_rates': 'fetchFundingRateHistory'}
@@ -399,24 +318,25 @@ class CCXT(Library):
                     cx_data_req["freq"],
                     since=start_date,
                     limit=self.max_obs_per_call,
+                    params={'until': end_date}
                 )
             elif data_type == 'funding_rates':
                 data_resp = getattr(exch, data_types[data_type])(
                     ticker,
                     since=start_date,
                     limit=1000,
+                    params={'until': end_date}
                 )
 
-            assert data_resp != []
+            return data_resp
 
         except Exception as e:
             logging.warning(f"Failed to get {data_type} data for {ticker}.")
             logging.warning(e)
 
-        else:
-            return data_resp
+            return None
 
-    def get_all_ohlcv_hist(self, data_req: DataRequest, ticker: str) -> pd.DataFrame:
+    def fetch_all_ohlcv_hist(self, data_req: DataRequest, ticker: str) -> pd.DataFrame:
         """
         Submits get requests to API until entire OHLCV history has been collected. Only necessary when
         number of observations is larger than the maximum number of observations per call.
@@ -436,24 +356,24 @@ class CCXT(Library):
         # convert data request parameters to CCXT format and set start date
         cx_data_req = ConvertParams(data_req).to_ccxt()
         start_date = cx_data_req['start_date']
+        end_date = cx_data_req['end_date']
 
         # create empty df
         df = pd.DataFrame()
+
         # while loop condition
         missing_vals, attempts = True, 0
 
         # run a while loop until all data collected
         while missing_vals and attempts < cx_data_req['trials']:
 
-            try:
-                # data req
-                data_resp = self.req_data(data_req=data_req,
-                                          data_type='ohlcv',
-                                          ticker=ticker,
-                                          start_date=start_date)
+            data_resp = self.req_data(data_req=data_req,
+                                      data_type='ohlcv',
+                                      ticker=ticker,
+                                      start_date=start_date,
+                                      end_date=end_date)
 
-            except AssertionError as e:
-                logging.warning(e)
+            if data_resp is None:
                 attempts += 1
                 sleep(self.get_rate_limit_info(exch=cx_data_req['exch'])[cx_data_req['exch']] / 1000)
                 logging.warning(
@@ -463,6 +383,7 @@ class CCXT(Library):
                     logging.warning(
                         f"Failed to get OHLCV data from {cx_data_req['exch']} for {ticker} after many attempts."
                     )
+                    return None
 
             else:
                 # name cols and create df
@@ -488,7 +409,7 @@ class CCXT(Library):
 
         return df
 
-    def get_all_funding_hist(self, data_req: DataRequest, ticker: str) -> pd.DataFrame:
+    def fetch_all_funding_hist(self, data_req: DataRequest, ticker: str) -> pd.DataFrame:
         """
         Submits get requests to API until entire funding rate history has been collected. Only necessary when
         number of observations is larger than the maximum number of observations per call.
@@ -508,6 +429,7 @@ class CCXT(Library):
         # convert data request parameters to CCXT format and set start date
         cx_data_req = ConvertParams(data_req).to_ccxt()
         start_date = cx_data_req['start_date']
+        end_date = cx_data_req['end_date']
 
         # create empty df
         df = pd.DataFrame()
@@ -517,15 +439,14 @@ class CCXT(Library):
         # run a while loop until all data collected
         while missing_vals and attempts < cx_data_req['trials']:
 
-            try:
-                # data req
-                data_resp = self.req_data(data_req=data_req,
-                                          data_type='funding_rates',
-                                          ticker=ticker,
-                                          start_date=start_date)
+            # data req
+            data_resp = self.req_data(data_req=data_req,
+                                      data_type='funding_rates',
+                                      ticker=ticker,
+                                      start_date=start_date,
+                                      end_date=end_date)
 
-            except AssertionError as e:
-                logging.warning(e)
+            if data_resp is None:
                 attempts += 1
                 sleep(self.get_rate_limit_info(exch=cx_data_req['exch'])[cx_data_req['exch']] / 1000)
                 logging.warning(
@@ -533,8 +454,9 @@ class CCXT(Library):
                 )
                 if attempts == cx_data_req["trials"]:
                     logging.warning(
-                        f"Failed to get funding rates from {cx_data_req['exch']} for {ticker} after many attempts."
+                        f"Failed to get funding_rates from {cx_data_req['exch']} for {ticker} after many attempts."
                     )
+                    return None
 
             else:
                 # add to df
@@ -580,7 +502,7 @@ class CCXT(Library):
 
         return WrangleData(data_req, data_resp).ccxt()
 
-    def get_tidy_ohlcv(self, data_req: DataRequest, ticker: str) -> pd.DataFrame:
+    def fetch_tidy_ohlcv(self, data_req: DataRequest, ticker: str) -> pd.DataFrame:
         """
         Gets entire OHLCV history and wrangles the data response into tidy data format.
 
@@ -597,13 +519,15 @@ class CCXT(Library):
             Dataframe with entire data history retrieved and wrangled into tidy data format.
         """
         # get entire data history
-        df = self.get_all_ohlcv_hist(data_req, ticker)
+        df = self.fetch_all_ohlcv_hist(data_req, ticker)
+
         # wrangle df
-        df = self.wrangle_data_resp(data_req, df)
+        if df is not None:
+            df = self.wrangle_data_resp(data_req, df)
 
         return df
 
-    def get_tidy_funding_rates(self, data_req: DataRequest, ticker: str) -> pd.DataFrame:
+    def fetch_tidy_funding_rates(self, data_req: DataRequest, ticker: str) -> pd.DataFrame:
         """
         Gets entire funding rates history and wrangles the data response into tidy data format.
 
@@ -620,9 +544,11 @@ class CCXT(Library):
             Dataframe with entire data history retrieved and wrangled into tidy data format.
         """
         # get entire data history
-        df = self.get_all_funding_hist(data_req, ticker)
+        df = self.fetch_all_funding_hist(data_req, ticker)
+
         # wrangle df
-        df = self.wrangle_data_resp(data_req, df)
+        if df is not None:
+            df = self.wrangle_data_resp(data_req, df)
 
         return df
 
@@ -680,7 +606,7 @@ class CCXT(Library):
                 f" Market type must be perpetual futures."
             )
 
-    def get_ohlcv(self, data_req: DataRequest) -> pd.DataFrame:
+    def fetch_ohlcv(self, data_req: DataRequest) -> pd.DataFrame:
         """
         Loops list of tickers, retrieves OHLCV data for each ticker in tidy format and stores it in a
         multiindex dataframe.
@@ -707,13 +633,9 @@ class CCXT(Library):
         # loop through tickers
         for mkt, ticker in zip(cx_data_req['mkts'], data_req.tickers):
 
-            try:
-                df0 = self.get_tidy_ohlcv(data_req, mkt)
-
-            except AssertionError:
-                logging.info(f"Failed to get OHLCV data for {ticker} after many attempts.")
+            df0 = self.fetch_tidy_ohlcv(data_req, mkt)
 
-            else:
+            if df0 is not None:
                 # add ticker to index
                 df0['ticker'] = ticker.upper()
                 df0.set_index(['ticker'], append=True, inplace=True)
@@ -722,7 +644,7 @@ class CCXT(Library):
 
         return df
 
-    def get_funding_rates(self, data_req: DataRequest) -> pd.DataFrame:
+    def fetch_funding_rates(self, data_req: DataRequest) -> pd.DataFrame:
         """
         Loops list of tickers, retrieves funding rates data for each ticker in tidy format and stores it in a
         multiindex dataframe.
@@ -749,13 +671,9 @@ class CCXT(Library):
         # loop through tickers
         for mkt, ticker in zip(cx_data_req['mkts'], data_req.tickers):
 
-            try:
-                df0 = self.get_tidy_funding_rates(data_req, mkt)
-
-            except AssertionError:
-                logging.info(f"Failed to get funding rates for {ticker} after many attempts.")
+            df0 = self.fetch_tidy_funding_rates(data_req, mkt)
 
-            else:
+            if df0 is not None:
                 # add ticker to index
                 df0['ticker'] = ticker.upper()
                 df0.set_index(['ticker'], append=True, inplace=True)
@@ -783,12 +701,12 @@ class CCXT(Library):
         # get OHLCV data
         ohlcv_list = ["open", "high", "low", "close", "volume"]
         if any([field in ohlcv_list for field in data_req.fields]):
-            df0 = self.get_ohlcv(data_req)
+            df0 = self.fetch_ohlcv(data_req)
             df = pd.concat([df, df0])
 
         # get funding rate data
         if any([field == "funding_rate" for field in data_req.fields]):
-            df1 = self.get_funding_rates(data_req)
+            df1 = self.fetch_funding_rates(data_req)
             df = pd.concat([df, df1], axis=1)
 
         # check if df empty