cryptodatapy 0.2.9__tar.gz → 0.2.11__tar.gz

{cryptodatapy-0.2.9 → cryptodatapy-0.2.11}/PKG-INFO

@@ -1,17 +1,17 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.3
 Name: cryptodatapy
-Version: 0.2.9
+Version: 0.2.11
 Summary: Cryptoasset data library
 License: Apache-2.0
 Author: Systamental
-Requires-Python: >=3.8,<4.0
+Requires-Python: >=3.9,<4.0
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: DBnomics (>=1.2.3)
 Requires-Dist: ccxt (>=1.91.52)
 Requires-Dist: coinmetrics-api-client (>=2022.6.17) ; python_version >= "3.7"
@@ -20,7 +20,7 @@ Requires-Dist: investpy (>=1.0.8)
 Requires-Dist: matplotlib (>=3.5.2)
 Requires-Dist: numpy (>=1.23.2)
 Requires-Dist: openpyxl (>=3.1.2)
-Requires-Dist: pandas (>=1.4.4)
+Requires-Dist: pandas (>=2.2.3)
 Requires-Dist: pandas-datareader (>=0.10.0)
 Requires-Dist: prophet (>=1.1) ; python_version >= "3.7"
 Requires-Dist: pyarrow (>=17.0.0)

{cryptodatapy-0.2.9 → cryptodatapy-0.2.11}/pyproject.toml

@@ -1,13 +1,13 @@
 [tool.poetry]
 name = "cryptodatapy"
-version = "0.2.9"
+version = "0.2.11"
 description = "Cryptoasset data library"
 authors = ["Systamental"]
 license = "Apache License 2.0"
 readme = "README.md"
 
 [tool.poetry.dependencies]
-python = ">=3.8,<4.0"
+python = ">=3.9,<4.0"
 matplotlib = ">=3.5.2"
 requests = {version = ">=2.28.0", python = ">=3.7"}
 coinmetrics-api-client = {version = ">=2022.6.17", python = ">=3.7"}
@@ -19,7 +19,7 @@ statsmodels = ">=0.13.2"
 prophet = {version = ">=1.1", python = ">=3.7"}
 selenium = ">=4.4.3"
 numpy = ">=1.23.2"
-pandas = ">=1.4.4"
+pandas = ">=2.2.3"
 pandas-datareader = ">=0.10.0"
 responses = ">=0.21.0"
 yfinance = ">=0.2.14"

{cryptodatapy-0.2.9 → cryptodatapy-0.2.11}/src/cryptodatapy/extract/getdata.py

@@ -220,3 +220,71 @@ class GetData:
         df = getattr(ds, method)(self.data_req)
 
         return df
+
+    async def get_series_async(self, method: str = "get_data_async") -> pd.DataFrame:
+        """
+        Get requested data.
+
+        Parameters
+        ----------
+        method: str, default 'get_data'
+            Gets the specified method from the data source object.
+
+        Returns
+        -------
+        df: pd.DataFrame - MultiIndex
+            DataFrame with DatetimeIndex (level 0), ticker (level 1), and field (cols) values.
+
+        Examples
+        --------
+        >>> data_req = DataRequest(source='ccxt', tickers=['btc', 'eth'], fields=['open', 'high', 'low', 'close',
+                                   'volume'], freq='d', exch='ftx', start_date='2017-01-01')
+        >>> GetData(data_req).get_series()
+                                open        high        low         close       volume
+        date	    ticker
+        2020-03-28	BTC	        6243.25	    6298.5	    6028.0	    6237.5	    3888.9424
+                    ETH	        128.995	    133.0	    125.11	    131.04	    1751.65972
+        2020-03-29	BTC	        6233.5	    6262.5	    5869.5	    5876.5	    114076.5831
+                    ETH	        130.98	    131.84	    123.81	    124.33	    138449.60906
+        2020-03-30	BTC	        5876.0	    6609.0	    5856.0	    6396.5	    224231.1718
+
+
+        >>> data_req = DataRequest(source='glassnode', tickers=['btc', 'eth'],
+                                   fields=['add_act', 'tx_count', 'issuance'], freq='d', start_date='2016-01-01')
+        >>> GetData(data_req).get_series()
+                                add_act     tx_count    issuance
+        date        ticker
+        2016-01-01	BTC	        316489	    123957	    0.085386
+                    ETH	        2350	    8232	    0.133048
+        2016-01-02	BTC	        419389	    148893	    0.09197
+                    ETH	        2410	    9164	    0.140147
+        2016-01-03	BTC	        394047	    142463	    0.091947
+        """
+        # data source objects
+        data_source_dict = {
+            "cryptocompare": CryptoCompare,
+            "coinmetrics": CoinMetrics,
+            "ccxt": CCXT,
+            "glassnode": Glassnode,
+            "tiingo": Tiingo,
+            "investpy": InvestPy,
+            "dbnomics": DBnomics,
+            "yahoo": PandasDataReader,
+            "fred": PandasDataReader,
+            "av-daily": PandasDataReader,
+            "av-forex-daily": PandasDataReader,
+            "famafrench": PandasDataReader,
+            "aqr": AQR
+        }
+
+        # data source
+        ds = data_source_dict[self.data_req.source]
+        # instantiate ds obj
+        if self.api_key is not None:
+            ds = ds(api_key=self.api_key)
+        else:
+            ds = ds()
+        # get data
+        df = await getattr(ds, method)(self.data_req)
+
+        return df

{cryptodatapy-0.2.9 → cryptodatapy-0.2.11}/src/cryptodatapy/extract/libraries/ccxt_api.py

@@ -2,6 +2,7 @@ import logging
 from typing import Any, Dict, List, Optional, Union
 import pandas as pd
 import asyncio
+import random
 from time import sleep
 import ccxt
 import ccxt.async_support as ccxt_async
@@ -76,7 +77,7 @@ class CCXT(Library):
             fields, frequencies, base_url, api_key, max_obs_per_call, rate_limit
         )
         self.exchange = None
-        # self.exchange_async = None
+        self.exchange_async = None
         self.data_req = None
         self.data = pd.DataFrame()
 
@@ -311,13 +312,26 @@ class CCXT(Library):
         if self.rate_limit is None:
             self.rate_limit = self.exchange.rateLimit
 
+    @staticmethod
+    def exponential_backoff_with_jitter(base_delay: float, max_delay: int, attempts: int) -> None:
+        delay = min(max_delay, base_delay * (2 ** attempts))
+        delay_with_jitter = delay + random.uniform(0, delay * 0.5)
+        sleep(delay_with_jitter)
+
+    @staticmethod
+    async def exponential_backoff_with_jitter_async(base_delay: float, max_delay: int, attempts: int) -> None:
+        delay = min(max_delay, base_delay * (2 ** attempts))
+        delay_with_jitter = delay + random.uniform(0, delay * 0.5)
+        await asyncio.sleep(delay_with_jitter)
+
     async def _fetch_ohlcv_async(self,
                                  ticker: str,
                                  freq: str,
                                  start_date: str,
                                  end_date: str,
                                  exch: str,
-                                 trials: int = 3
+                                 trials: int = 3,
+                                 pause: int = 1
                                  ) -> List:
         """
         Fetches OHLCV data for a specific ticker.
@@ -336,6 +350,8 @@ class CCXT(Library):
             Name of exchange.
         trials: int, default 3
             Number of attempts to fetch data.
+        pause: int, default 60
+            Pause in seconds to respect the rate limit.
 
         Returns
         -------
@@ -346,16 +362,17 @@ class CCXT(Library):
         data = []
 
         # inst exch
-        self.exchange = getattr(ccxt_async, exch)()
+        if self.exchange_async is None:
+            self.exchange_async = getattr(ccxt_async, exch)()
 
         # fetch data
-        if self.exchange.has['fetchOHLCV']:
+        if self.exchange_async.has['fetchOHLCV']:
 
             # while loop to fetch all data
             while start_date < end_date and attempts < trials:
 
                 try:
-                    data_resp = await self.exchange.fetch_ohlcv(
+                    data_resp = await self.exchange_async.fetch_ohlcv(
                         ticker,
                         freq,
                         since=start_date,
@@ -368,29 +385,33 @@ class CCXT(Library):
                         start_date = data_resp[-1][0] + 1
                         data.extend(data_resp)
                     else:
+                        if not data:
+                            logging.warning(f"No OHLCV data available for {ticker}.")
                         break
 
                 except Exception as e:
                     logging.warning(
-                        f"Failed to get OHLCV data from {self.exchange.id} for {ticker} "
+                        f"Failed to get OHLCV data from {self.exchange_async.id} for {ticker} "
                         f"on attempt #{attempts + 1}: {e}."
                     )
                     attempts += 1
                     if attempts >= trials:
                         logging.warning(
-                            f"Failed to get OHLCV data from {self.exchange.id} "
+                            f"Failed to get OHLCV data from {self.exchange_async.id} "
                             f"for {ticker} after {trials} attempts."
                         )
                         break
 
                 finally:
-                    await asyncio.sleep(self.exchange.rateLimit / 1000)
+                    await self.exponential_backoff_with_jitter_async(self.exchange_async.rateLimit / 1000,
+                                                                     pause,
+                                                                     attempts)
 
-            await self.exchange.close()
+            await self.exchange_async.close()
             return data
 
         else:
-            logging.warning(f"OHLCV data is not available for {self.exchange.id}.")
+            logging.warning(f"OHLCV data is not available for {self.exchange_async.id}.")
             return None
 
     def _fetch_ohlcv(self,
@@ -399,7 +420,8 @@ class CCXT(Library):
                      start_date: str,
                      end_date: str,
                      exch: str,
-                     trials: int = 3
+                     trials: int = 3,
+                     pause: int = 1
                      ) -> List:
         """
         Fetches OHLCV data for a specific ticker.
@@ -418,6 +440,8 @@ class CCXT(Library):
             Name of exchange.
         trials: int, default 3
             Number of attempts to fetch data.
+        pause: int, default 60
+            Pause in seconds to respect the rate limit.
 
         Returns
         -------
@@ -428,7 +452,8 @@ class CCXT(Library):
         data = []
 
         # inst exch
-        self.exchange = getattr(ccxt, exch)()
+        if self.exchange is None:
+            self.exchange = getattr(ccxt, exch)()
 
         # fetch data
         if self.exchange.has['fetchOHLCV']:
@@ -442,7 +467,10 @@ class CCXT(Library):
                         freq,
                         since=start_date,
                         limit=self.max_obs_per_call,
-                        params={'until': end_date}
+                        params={
+                            'until': end_date,
+                            'paginate': True
+                        }
                     )
 
                     # add data to list
@@ -450,6 +478,8 @@ class CCXT(Library):
                         start_date = data_resp[-1][0] + 1
                         data.extend(data_resp)
                     else:
+                        if not data:
+                            logging.warning(f"No OHLCV data available for {ticker}.")
                         break
 
                 except Exception as e:
@@ -466,7 +496,7 @@ class CCXT(Library):
                         break
 
                 finally:
-                    sleep(self.exchange.rateLimit / 1000)
+                    self.exponential_backoff_with_jitter(self.exchange.rateLimit / 1000, pause, attempts)
 
             return data
 
@@ -481,7 +511,7 @@ class CCXT(Library):
                                      end_date: str,
                                      exch: str,
                                      trials: int = 3,
-                                     pause: int = 0.5
+                                     pause: int = 1
                                      ):
         """
         Fetches OHLCV data for a list of tickers.
@@ -509,7 +539,8 @@ class CCXT(Library):
             List of lists of timestamps and OHLCV data for each ticker.
         """
         # inst exch
-        self.exchange = getattr(ccxt_async, exch)()
+        if self.exchange_async is None:
+            self.exchange_async = getattr(ccxt_async, exch)()
 
         data = []
 
@@ -523,7 +554,7 @@ class CCXT(Library):
             data.append(data_resp)
             pbar.update(1)
 
-        await self.exchange.close()
+        await self.exchange_async.close()
 
         return data
 
@@ -534,7 +565,7 @@ class CCXT(Library):
                          end_date: str,
                          exch: str,
                          trials: int = 3,
-                         pause: int = 0.5
+                         pause: int = 1
                          ):
         """
         Fetches OHLCV data for a list of tickers.
@@ -557,7 +588,8 @@ class CCXT(Library):
             Pause in seconds to respect the rate limit.
         """
         # inst exch
-        self.exchange = getattr(ccxt, exch)()
+        if self.exchange is None:
+            self.exchange = getattr(ccxt, exch)()
 
         data = []
 
@@ -578,7 +610,8 @@ class CCXT(Library):
                                          start_date: str,
                                          end_date: str,
                                          exch: str,
-                                         trials: int = 3
+                                         trials: int = 3,
+                                         pause: int = 1
                                          ) -> List:
         """
         Fetches funding rates data for a specific ticker.
@@ -593,6 +626,8 @@ class CCXT(Library):
             End date in integers in milliseconds since Unix epoch.
         trials: int, default 3
             Number of attempts to fetch data.
+        pause: int, default 1
+            Pause in seconds to respect the rate limit.
 
         Returns
         -------
@@ -603,16 +638,17 @@ class CCXT(Library):
         data = []
 
         # inst exch
-        self.exchange = getattr(ccxt_async, exch)()
+        if self.exchange_async is None:
+            self.exchange_async = getattr(ccxt_async, exch)()
 
         # fetch data
-        if self.exchange.has['fetchFundingRateHistory']:
+        if self.exchange_async.has['fetchFundingRateHistory']:
 
             # while loop to get all data
             while start_date < end_date and attempts < trials:
 
                 try:
-                    data_resp = await self.exchange.fetch_funding_rate_history(
+                    data_resp = await self.exchange_async.fetch_funding_rate_history(
                         ticker,
                         since=start_date,
                         limit=self.max_obs_per_call,
@@ -624,29 +660,33 @@ class CCXT(Library):
                         start_date = data_resp[-1]['timestamp'] + 1
                         data.extend(data_resp)
                     else:
+                        if not data:
+                            logging.warning(f"No funding rates data available for {ticker}.")
                         break
 
                 except Exception as e:
                     logging.warning(
-                        f"Failed to get funding rates from {self.exchange.id} for {ticker} "
+                        f"Failed to get funding rates from {self.exchange_async.id} for {ticker} "
                         f"on attempt #{attempts + 1}: {e}."
                     )
                     attempts += 1
                     if attempts >= trials:
                         logging.warning(
-                            f"Failed to get funding rates from {self.exchange.id} "
+                            f"Failed to get funding rates from {self.exchange_async.id} "
                             f"for {ticker} after {trials} attempts."
                         )
                         break
 
                 finally:
-                    await asyncio.sleep(self.exchange.rateLimit / 1000)
+                    await self.exponential_backoff_with_jitter_async(self.exchange_async.rateLimit / 1000,
+                                                                     pause,
+                                                                     attempts)
 
-            await self.exchange.close()
+            await self.exchange_async.close()
             return data
 
         else:
-            logging.warning(f"Funding rates are not available for {self.exchange.id}.")
+            logging.warning(f"Funding rates are not available for {self.exchange_async.id}.")
             return None
 
     def _fetch_funding_rates(self,
@@ -654,7 +694,8 @@ class CCXT(Library):
                              start_date: str,
                              end_date: str,
                              exch: str,
-                             trials: int = 3
+                             trials: int = 3,
+                             pause: int = 1
                              ) -> List:
         """
         Fetches funding rates data for a specific ticker.
@@ -669,6 +710,8 @@ class CCXT(Library):
             End date in integers in milliseconds since Unix epoch.
         trials: int, default 3
             Number of attempts to fetch data.
+        pause: int, default 1
+            Pause in seconds to respect the rate limit.
 
         Returns
         -------
@@ -679,7 +722,8 @@ class CCXT(Library):
         data = []
 
         # inst exch
-        self.exchange = getattr(ccxt, exch)()
+        if self.exchange is None:
+            self.exchange = getattr(ccxt, exch)()
 
         # fetch data
         if self.exchange.has['fetchFundingRateHistory']:
@@ -700,6 +744,8 @@ class CCXT(Library):
                         start_date = data_resp[-1]['timestamp'] + 1
                         data.extend(data_resp)
                     else:
+                        if not data:
+                            logging.warning(f"No funding rates data available for {ticker}.")
                         break
 
                 except Exception as e:
@@ -716,7 +762,7 @@ class CCXT(Library):
                         break
 
                 finally:
-                    sleep(self.exchange.rateLimit / 1000)
+                    self.exponential_backoff_with_jitter(self.exchange.rateLimit / 1000, pause, attempts)
 
             return data
 
@@ -730,7 +776,7 @@ class CCXT(Library):
                                              end_date: str,
                                              exch: str,
                                              trials: int = 3,
-                                             pause: int = 0.5
+                                             pause: int = 1
                                              ):
         """
         Fetches funding rates data for a list of tickers.
@@ -756,7 +802,8 @@ class CCXT(Library):
             List of lists of dictionaries with timestamps and funding rates data for each ticker.
         """
         # inst exch
-        self.exchange = getattr(ccxt_async, exch)()
+        if self.exchange_async is None:
+            self.exchange_async = getattr(ccxt_async, exch)()
 
         data = []
 
@@ -770,7 +817,7 @@ class CCXT(Library):
             pbar.update(1)
             await asyncio.sleep(pause)
 
-        await self.exchange.close()
+        await self.exchange_async.close()
 
         return data
 
@@ -780,7 +827,7 @@ class CCXT(Library):
                                  end_date: str,
                                  exch: str,
                                  trials: int = 3,
-                                 pause: int = 0.5
+                                 pause: int = 1
                                  ):
         """
         Fetches funding rates data for a list of tickers.
@@ -807,7 +854,8 @@ class CCXT(Library):
         """
 
         # inst exch
-        self.exchange = getattr(ccxt, exch)()
+        if self.exchange is None:
+            self.exchange = getattr(ccxt, exch)()
 
         data = []
 
@@ -829,7 +877,8 @@ class CCXT(Library):
                                          start_date: str,
                                          end_date: str,
                                          exch: str,
-                                         trials: int = 3
+                                         trials: int = 3,
+                                         pause: int = 1
                                          ) -> List:
         """
         Fetches open interest data for a specific ticker.
@@ -848,6 +897,8 @@ class CCXT(Library):
             Name of exchange.
         trials: int, default 3
             Number of attempts to fetch data.
+        pause: int, default 1
+            Pause in seconds to respect the rate limit.
 
         Returns
         -------
@@ -859,16 +910,17 @@ class CCXT(Library):
         data = []
 
         # inst exch
-        self.exchange = getattr(ccxt_async, exch)()
+        if self.exchange_async is None:
+            self.exchange_async = getattr(ccxt_async, exch)()
 
         # fetch data
-        if self.exchange.has['fetchOpenInterestHistory']:
+        if self.exchange_async.has['fetchOpenInterestHistory']:
 
             # while loop to get all data
             while start_date < end_date and attempts < trials:
 
                 try:
-                    data_resp = await self.exchange.fetch_open_interest_history(
+                    data_resp = await self.exchange_async.fetch_open_interest_history(
                         ticker,
                         freq,
                         since=start_date,
@@ -881,29 +933,34 @@ class CCXT(Library):
                         start_date = data_resp[-1]['timestamp'] + 1
                         data.extend(data_resp)
                     else:
+                        if not data:
+                            logging.warning(f"No open interest data available for {ticker}.")
                         break
 
                 except Exception as e:
                     logging.warning(
-                        f"Failed to get open interest from {self.exchange.id} for {ticker} "
+                        f"Failed to get open interest from {self.exchange_async.id} for {ticker} "
                         f"on attempt #{attempts + 1}: {e}."
                     )
                     attempts += 1
                     if attempts >= trials:
                         logging.warning(
-                            f"Failed to get open interest from {self.exchange.id} "
+                            f"Failed to get open interest from {self.exchange_async.id} "
                             f"for {ticker} after {trials} attempts."
                         )
                         break
 
                 finally:
-                    await asyncio.sleep(self.exchange.rateLimit / 1000)
+                    await self.exponential_backoff_with_jitter_async(self.exchange_async.rateLimit / 1000,
+                                                                     pause,
+                                                                     attempts)
+                    # await asyncio.sleep(self.exchange_async.rateLimit / 1000)
 
-            await self.exchange.close()
+            await self.exchange_async.close()
             return data
 
         else:
-            logging.warning(f"Open interest is not available for {self.exchange.id}.")
+            logging.warning(f"Open interest is not available for {self.exchange_async.id}.")
             return None
 
     def _fetch_open_interest(self,
@@ -912,7 +969,8 @@ class CCXT(Library):
                              start_date: str,
                              end_date: str,
                              exch: str,
-                             trials: int = 3
+                             trials: int = 3,
+                             pause: int = 1
                              ) -> List:
         """
         Fetches open interest data for a specific ticker.
@@ -931,6 +989,8 @@ class CCXT(Library):
             Name of exchange.
         trials: int, default 3
             Number of attempts to fetch data.
+        pause: int, default 1
+            Pause in seconds to respect the rate limit.
 
         Returns
         -------
@@ -964,6 +1024,8 @@ class CCXT(Library):
                         start_date = data_resp[-1]['timestamp'] + 1
                         data.extend(data_resp)
                     else:
+                        if not data:
+                            logging.warning(f"No open interest data available for {ticker}.")
                         break
 
                 except Exception as e:
@@ -980,7 +1042,8 @@ class CCXT(Library):
                         break
 
                 finally:
-                    sleep(self.exchange.rateLimit / 1000)
+                    self.exponential_backoff_with_jitter(self.exchange.rateLimit / 1000, pause, attempts)
+                    # sleep(self.exchange.rateLimit / 1000)
 
             return data
 
@@ -995,7 +1058,7 @@ class CCXT(Library):
                                              end_date: str,
                                              exch: str,
                                              trials: int = 3,
-                                             pause: int = 0.5
+                                             pause: int = 1
                                              ):
         """
         Fetches open interest data for a list of tickers.
@@ -1023,7 +1086,8 @@ class CCXT(Library):
             List of lists of dictionaries with timestamps and open interest data for each ticker.
         """
         # inst exch
-        self.exchange = getattr(ccxt_async, exch)()
+        if self.exchange_async is None:
+            self.exchange_async = getattr(ccxt_async, exch)()
 
         data = []
 
@@ -1038,7 +1102,7 @@ class CCXT(Library):
             pbar.update(1)
             await asyncio.sleep(pause)
 
-        await self.exchange.close()
+        await self.exchange_async.close()
 
         return data
 
@@ -1049,7 +1113,7 @@ class CCXT(Library):
                                  end_date: str,
                                  exch: str,
                                  trials: int = 3,
-                                 pause: int = 0.5
+                                 pause: int = 1
                                  ):
         """
         Fetches open interest data for a list of tickers.
@@ -1077,7 +1141,8 @@ class CCXT(Library):
             List of lists of dictionaries with timestamps and open interest data for each ticker.
         """
         # inst exch
-        self.exchange = getattr(ccxt, exch)()
+        if self.exchange is None:
+            self.exchange = getattr(ccxt, exch)()
 
         data = []
 
@@ -1112,10 +1177,6 @@ class CCXT(Library):
         # get metadata
         self.get_metadata(self.data_req.exch)
 
-        print(self.data_req.exch)
-
-        print(self.exchange)
-
         # check markets
         if not any([market in self.markets for market in self.data_req.source_markets]):
             raise ValueError(

{cryptodatapy-0.2.9 → cryptodatapy-0.2.11}/src/cryptodatapy/transform/convertparams.py

@@ -57,17 +57,17 @@ class ConvertParams:
             exch = self.data_req.exch
         # convert start date
         if self.data_req.freq[-3:] == "min":  # limit to higher frequency data responses
-            start_date = round((datetime.now() - timedelta(days=7)).timestamp())
+            start_date = int((datetime.now() - timedelta(days=7)).timestamp())
         # no start date
         elif self.data_req.start_date is None:
-            start_date = round(pd.Timestamp("2009-01-03 00:00:00").timestamp())
+            start_date = int(pd.Timestamp("2009-01-03 00:00:00").timestamp())
         else:
-            start_date = round(pd.Timestamp(self.data_req.start_date).timestamp())
+            start_date = int(pd.Timestamp(self.data_req.start_date).timestamp())
         # convert end date
         if self.data_req.end_date is None:
-            end_date = round(pd.Timestamp.utcnow()).timestamp()
+            end_date = int(pd.Timestamp.utcnow().timestamp())
         else:
-            end_date = round(pd.Timestamp(self.data_req.end_date).timestamp())
+            end_date = int(pd.Timestamp(self.data_req.end_date).timestamp())
         # fields
         if self.data_req.source_fields is not None:
             fields = self.data_req.source_fields
@@ -986,7 +986,6 @@ class ConvertParams:
         -------
         quote_ccy: str
             Quote currency.
-
         """
         mkts = []  # fx pairs list
         # fx groups
@@ -1021,7 +1020,7 @@ class ConvertParams:
             List of fields in data source format.
 
         """
-        # get fields
+        # x fields
         with resources.path("cryptodatapy.conf", "fields.csv") as f:
             fields_dict_path = f
         fields_df, fields_list = (
@@ -1046,3 +1045,121 @@ class ConvertParams:
                     )
 
         return fields_list
+
+    def to_dydx_dict(self) -> Dict[str, Union[list, str, int, float, None]]:
+        """
+        Convert parameters from CryptoDataPy to dYdX format.
+        """
+        if self.data_req.source_tickers is not None:
+            tickers = self.data_req.source_tickers
+            self.data_req.tickers = self.data_req.source_tickers
+        else:
+            tickers = [ticker.upper() for ticker in self.data_req.tickers]
+        
+        # convert markets (if needed)
+        markets = [f"{ticker}-USD" for ticker in tickers]
+        
+        # convert freq
+        if self.data_req.source_freq is not None:
+            freq = self.data_req.source_freq
+            self.data_req.freq = self.data_req.source_freq
+        else:
+            if self.data_req.freq is None:
+                freq = "1DAY"
+            elif self.data_req.freq == "1min":
+                freq = "1MIN"
+            elif self.data_req.freq == "5min":
+                freq = "5MINS"
+            elif self.data_req.freq == "15min":
+                freq = "15MINS"
+            elif self.data_req.freq == "30min":
+                freq = "30MINS"
+            elif self.data_req.freq == "1h":
+                freq = "1HOUR"
+            elif self.data_req.freq == "4h":
+                freq = "4HOURS"
+            elif self.data_req.freq in ["1d", "d"]:
+                freq = "1DAY"
+            else:
+                freq = "1DAY"  # Default to daily
+            
+        # convert fields
+        if self.data_req.source_fields is not None:
+            fields = self.data_req.source_fields
+            self.data_req.fields = self.data_req.source_fields
+        else:
+            # Map our standard fields to dYdX field names
+            field_mapping = {
+                'open': 'open',
+                'high': 'high',
+                'low': 'low',
+                'close': 'close',
+                'volume': 'baseTokenVolume',
+                'funding_rate': 'nextFundingRate',
+                'oi': 'openInterest'
+            }
+            
+            fields = []
+            for field in self.data_req.fields:
+                if field in field_mapping:
+                    fields.append(field_mapping[field])
+                else:
+                    logging.warning(f"Field {field} not available in dYdX API")
+            return {
+            "tickers": tickers,  # List of market tickers
+            "freq": freq,        # Converted frequency
+            "quote_ccy": self.data_req.quote_ccy,
+            "exch": "dydx",
+            "mkt_type": self.data_req.mkt_type,
+            "mkts": markets,     # Market identifiers
+            "start_date": self.data_req.start_date,
+            "end_date": self.data_req.end_date,
+            "fields": fields,    # Converted field names
+            "tz": self.data_req.tz or "UTC",
+            "cat": "crypto",
+            "trials": self.data_req.trials,
+            "pause": self.data_req.pause,
+            "source_tickers": self.data_req.source_tickers,
+            "source_freq": self.data_req.source_freq,
+            "source_fields": self.data_req.source_fields,
+        }
+
+    def to_dydx(self) -> DataRequest:
+
+        # tickers
+        if self.data_req.source_tickers is None:
+            self.data_req.source_tickers = [ticker.upper() for ticker in self.data_req.tickers]
+
+        # markets 
+        if self.data_req.source_markets is None:
+            self.data_req.source_markets = [f"{ticker}-USD" 
+                                            for ticker in self.data_req.source_tickers]
+        
+        if self.data_req.source_freq is None:
+            if self.data_req.freq is None:
+                self.data_req.source_freq = "1DAY"
+            elif self.data_req.freq == "1min":
+                self.data_req.source_freq = "1MIN"
+            elif self.data_req.freq == "5min":
+                self.data_req.source_freq = "5MINS"
+            elif self.data_req.freq == "15min":
+                self.data_req.source_freq = "15MINS"
+            elif self.data_req.freq == "30min":
+                self.data_req.source_freq = "30MINS"
+            elif self.data_req.freq == "1h":
+                self.data_req.source_freq = "1HOUR"
+            elif self.data_req.freq == "4h":
+                self.data_req.source_freq = "4HOURS"
+            elif self.data_req.freq in ["1d", "d"]:
+                self.data_req.source_freq = "1DAY"
+
+        field_mapping = {
+                'open': 'open',
+                'high': 'high',
+                'low': 'low',
+                'close': 'close',
+                'volume': 'baseTokenVolume',
+                'funding_rate': 'nextFundingRate',
+                'oi': 'openInterest'
+            }
+        return self.data_req

{cryptodatapy-0.2.9 → cryptodatapy-0.2.11}/src/cryptodatapy/transform/filter.py

@@ -159,12 +159,12 @@ class Filter:
             and fields (cols).
         """
         # drop tickers with nobs < ts_obs
-        obs = self.df.groupby(level=1).count().min(axis=1)
+        obs = self.df.groupby(level=1).count().median(axis=1)
         drop_tickers_list = obs[obs < ts_obs].index.to_list()
         self.filtered_df = self.df.drop(drop_tickers_list, level=1, axis=0)
 
         # drop tickers with nobs < cs_obs
-        obs = self.filtered_df.groupby(level=0).count().min(axis=1)
+        obs = self.filtered_df.groupby(level=0).count().median(axis=1)
         idx_start = obs[obs > cs_obs].index[0]
         self.filtered_df = self.filtered_df.loc[idx_start:]
 
@@ -185,16 +185,15 @@ class Filter:
             Filtered dataFrame with DatetimeIndex (level 0), tickers (level 1) and fields (cols).
         """
         # unchanged rows
-        unch_rows = (self.df.subtract(self.df.iloc[:, :4].mean(axis=1), axis=0) == 0).any(axis=1)
+        unch_rows: object = (self.df.subtract(self.df.iloc[:, :4].mean(axis=1), axis=0) == 0).any(axis=1)
 
-        # delisted tickers
-        delisted_tickers = unch_rows.unstack().iloc[-1][unch_rows.unstack().iloc[-1]].index.to_list()
+        # replace delisted with NaNs
+        self.filtered_df = self.df.loc[~unch_rows].reindex(self.df.index)
 
         # repair
         if method == 'remove':
-            self.filtered_df = self.df.drop(delisted_tickers, level=1)
-        else:
-            self.filtered_df = self.df.loc[~unch_rows].reindex(self.df.index)
+            self.filtered_df = list(self.filtered_df.unstack().columns[self.filtered_df.unstack().iloc[-1].isna()].
+                                    droplevel(0).unique())
 
         return self.filtered_df
 

{cryptodatapy-0.2.9 → cryptodatapy-0.2.11}/src/cryptodatapy/transform/impute.py

@@ -121,7 +121,8 @@ class Impute:
         self.imputed_df = pd.DataFrame(imp_yhat, index=self.filtered_df.index, columns=self.filtered_df.columns)
 
         # type conversion
-        self.imputed_df = self.imputed_df.apply(pd.to_numeric, errors="ignore").convert_dtypes()
+        self.imputed_df = self.imputed_df.convert_dtypes()
+
 
         # plot
         if self.plot:

{cryptodatapy-0.2.9 → cryptodatapy-0.2.11}/src/cryptodatapy/transform/wrangle.py

@@ -854,7 +854,8 @@ class WrangleData:
         self.tidy_data = self.tidy_data.apply(pd.to_numeric, errors='coerce').convert_dtypes()
 
         # remove bad data
-        self.tidy_data = self.tidy_data[self.tidy_data != 0]  # 0 values
+        if data_type != 'funding_rates':
+            self.tidy_data = self.tidy_data[self.tidy_data != 0]  # 0 values
         self.tidy_data = self.tidy_data[~self.tidy_data.index.duplicated()]  # duplicate rows
         self.tidy_data = self.tidy_data.dropna(how='all').dropna(how='all', axis=1)  # entire row or col NaNs
 
@@ -910,13 +911,10 @@ class WrangleData:
         # tickers
         tickers_dict = {source_ticker: ticker for source_ticker, ticker in zip(self.data_req.source_tickers,
                         self.data_req.tickers)}
-        if len(self.data_req.tickers) == 1:
-            self.data_resp['Ticker'] = self.data_req.tickers[0]
-        else:
-            self.data_resp = self.data_resp.stack()
-            self.data_resp.index.names = ['Date', 'Ticker']
-            self.data_resp.index = self.data_resp.index.set_levels(self.data_resp.index.levels[1].map(tickers_dict),
-                                                                   level=1)
+        self.data_resp = self.data_resp.stack(future_stack=True)
+        self.data_resp.columns.name = None
+        self.data_resp.index = self.data_resp.index.set_levels(self.data_resp.index.levels[1].map(tickers_dict),
+                                                               level=1)
         self.data_resp.reset_index(inplace=True)
 
         #  fields
@@ -931,9 +929,6 @@ class WrangleData:
             resample(self.data_req.freq, level='date').\
             last().swaplevel('ticker', 'date').sort_index()
 
-        # re-order cols
-        self.data_resp = self.data_resp.loc[:, ['open', 'high', 'low', 'close', 'close_adj', 'volume']]
-
         # type conversion
         self.data_resp = self.data_resp.apply(pd.to_numeric, errors='coerce').convert_dtypes()