bbstrader 2.0.3__cp312-cp312-macosx_11_0_arm64.whl

bbstrader/metatrader/rates.py

@@ -0,0 +1,495 @@
+from datetime import datetime
+from typing import Optional, Union
+
+import pandas as pd
+from exchange_calendars import get_calendar, get_calendar_names
+
+from bbstrader.metatrader.account import Account
+from bbstrader.metatrader.broker import EXCHANGES
+from bbstrader.metatrader.utils import TIMEFRAMES, SymbolType, raise_mt5_error
+
+try:
+    import MetaTrader5 as Mt5
+except ImportError:
+    import bbstrader.compat  # noqa: F401
+
+
+__all__ = [
+    "Rates",
+    "download_historical_data",
+    "get_data_from_pos",
+    "get_data_from_date",
+]
+
+MAX_BARS = 10_000_000
+
+
+IDX_CALENDARS = {
+    "CAD": "XTSE",
+    "AUD": "XASX",
+    "GBP": "XLON",
+    "HKD": "XSHG",
+    "ZAR": "XJSE",
+    "CHF": "XSWX",
+    "NOK": "XOSL",
+    "EUR": "XETR",
+    "SGD": "XSES",
+    "USD": "us_futures",
+    "JPY": "us_futures",
+}
+
+
+CALENDARS = {
+    SymbolType.FOREX: "us_futures",
+    SymbolType.STOCKS: EXCHANGES,
+    SymbolType.ETFs: EXCHANGES,
+    SymbolType.INDICES: IDX_CALENDARS,
+    SymbolType.COMMODITIES: "us_futures",
+    SymbolType.CRYPTO: "24/7",
+    SymbolType.FUTURES: "us_futures",
+}
+
+SESSION_TIMEFRAMES = [
+    Mt5.TIMEFRAME_D1,
+    Mt5.TIMEFRAME_W1,
+    Mt5.TIMEFRAME_H12,
+    Mt5.TIMEFRAME_MN1,
+]
+
+
+class Rates(object):
+    """
+    Provides methods to retrieve historical financial data from MetaTrader 5.
+
+    This class encapsulates interactions with the MetaTrader 5 (MT5) terminal
+    to fetch historical price data for a given symbol and timeframe. It offers
+    flexibility in retrieving data either by specifying a starting position
+    and count of bars or by providing a specific date range .
+
+    Notes:
+        All data is rerturn as pandas.DataFrame
+
+        1. Befor using this class, ensure that the `Max bars in chart` in your terminal
+        is set to a value that is greater than the number of bars you want to retrieve
+        or just set it to Unlimited.
+        In your MT5 terminal, go to `Tools` -> `Options` -> `Charts` -> `Max bars in chart`.
+
+        2. The `open, high, low, close, adjclose, returns,
+        volume` properties returns data in  Broker's timezone by default.
+
+        See `bbstrader.metatrader.broker.check_mt5_connection()` for more details on how to connect to MT5 terminal.
+
+    Example:
+        >>> rates = Rates("EURUSD", "1h")
+        >>> df = rates.get_historical_data(
+        ...     date_from=datetime(2023, 1, 1),
+        ...     date_to=datetime(2023, 1, 10),
+        ... )
+        >>> print(df.head())
+    """
+
+    def __init__(
+        self,
+        symbol: str,
+        timeframe: str = "D1",
+        start_pos: int = 0,
+        count: Optional[int] = MAX_BARS,
+        **kwargs,
+    ):
+        """
+        Initializes a new Rates instance.
+
+        Args:
+            symbol (str): Financial instrument symbol (e.g., "EURUSD").
+            timeframe (str): Timeframe string (e.g., "D1", "1h", "5m").
+            start_pos (int): Starting index (int) for data retrieval.
+            count (int, optional): Number of bars to retrieve default is
+                the maximum bars availble in the MT5 terminal.
+        Raises:
+            ValueError: If the provided timeframe is invalid.
+        """
+        self.symbol = symbol
+        self.start_pos = start_pos
+        self.count = count
+        self.time_frame = self._validate_time_frame(timeframe)
+        self.__account = Account(**kwargs)
+        self.__data = self.get_rates_from_pos
+
+    @property
+    def open(self):
+        return self.__data()["Open"]
+
+    @property
+    def high(self):
+        return self.__data()["High"]
+
+    @property
+    def low(self):
+        return self.__data()["Low"]
+
+    @property
+    def close(self):
+        return self.__data()["Close"]
+
+    @property
+    def adjclose(self):
+        return self.__data()["Adj Close"]
+
+    @property
+    def returns(self):
+        """
+        Fractional change between the current and a prior element.
+
+        Computes the fractional change from the immediately previous row by default.
+        This is useful in comparing the fraction of change in a time series of elements.
+
+        Note
+        ----
+        It calculates fractional change (also known as `per unit change or relative change`)
+        and `not percentage change`. If you need the percentage change, multiply these values by 100.
+        """
+        data = self.__data()
+        data["Returns"] = data["Adj Close"].pct_change()
+        data = data.dropna()
+        return data["Returns"]
+
+    @property
+    def volume(self):
+        return self.__data()["Volume"]
+
+    def _validate_time_frame(self, time_frame: str) -> int:
+        """Validates and returns the MT5 timeframe code."""
+        if time_frame not in TIMEFRAMES:
+            raise ValueError(
+                f"Unsupported time frame '{time_frame}'. "
+                f"Possible values are: {list(TIMEFRAMES.keys())}"
+            )
+        return TIMEFRAMES[time_frame]
+
+    def _fetch_data(
+        self,
+        start: Union[int, datetime, pd.Timestamp],
+        count: Union[int, datetime, pd.Timestamp],
+        lower_colnames=False,
+        utc=False,
+    ) -> Union[pd.DataFrame, None]:
+        """Fetches data from MT5 and returns a DataFrame or None."""
+        try:
+            rates = None
+            if isinstance(start, int) and isinstance(count, int):
+                rates = Mt5.copy_rates_from_pos(
+                    self.symbol, self.time_frame, start, count
+                )
+            elif isinstance(start, (datetime, pd.Timestamp)) and isinstance(count, int):
+                rates = Mt5.copy_rates_from(self.symbol, self.time_frame, start, count)
+            elif isinstance(start, (datetime, pd.Timestamp)) and isinstance(
+                count, (datetime, pd.Timestamp)
+            ):
+                rates = Mt5.copy_rates_range(self.symbol, self.time_frame, start, count)
+            if rates is None:
+                return None
+
+            df = pd.DataFrame(rates)
+            return self._format_dataframe(df, lower_colnames=lower_colnames, utc=utc)
+        except Exception as e:
+            raise_mt5_error(e)
+
+    def _format_dataframe(
+        self, df: pd.DataFrame, lower_colnames=False, utc=False
+    ) -> pd.DataFrame:
+        """Formats the raw MT5 data into a standardized DataFrame."""
+
+        df = df.copy()
+
+        df = df[["time", "open", "high", "low", "close", "tick_volume"]]
+        df.columns = ["Date", "Open", "High", "Low", "Close", "Volume"]
+
+        df["Adj Close"] = df["Close"]
+        df = df[["Date", "Open", "High", "Low", "Close", "Adj Close", "Volume"]]
+
+        df["Date"] = pd.to_datetime(df["Date"], unit="s", utc=utc)
+        df.set_index("Date", inplace=True)
+
+        if lower_colnames:
+            df.columns = df.columns.str.lower().str.replace(" ", "_")
+            df.index.name = df.index.name.lower().replace(" ", "_")
+
+        return df
+
+    def _filter_data(
+        self, df: pd.DataFrame, date_from=None, date_to=None, fill_na=False
+    ) -> pd.DataFrame:
+        df = df.copy()
+        symbol_type = self.__account.get_symbol_type(self.symbol)
+        currencies = self.__account.get_currency_rates(self.symbol)
+        if symbol_type in CALENDARS:
+            if symbol_type == SymbolType.STOCKS or symbol_type == SymbolType.ETFs:
+                for exchange in CALENDARS[symbol_type]:
+                    if exchange in get_calendar_names():
+                        symbols = self.__account.get_stocks_from_exchange(
+                            exchange_code=exchange
+                        )
+                        if self.symbol in symbols:
+                            calendar = get_calendar(exchange, side="right")
+                            break
+            elif symbol_type == SymbolType.INDICES:
+                calendar = get_calendar(
+                    CALENDARS[symbol_type][currencies["mc"]], side="right"
+                )
+            else:
+                calendar = get_calendar(CALENDARS[symbol_type], side="right")
+            date_from = date_from or df.index[0]
+            date_to = date_to or df.index[-1]
+            if self.time_frame in SESSION_TIMEFRAMES:
+                valid_sessions = calendar.sessions_in_range(date_from, date_to)
+            else:
+                valid_sessions = calendar.minutes_in_range(date_from, date_to)
+            if self.time_frame in [Mt5.TIMEFRAME_M1, Mt5.TIMEFRAME_D1]:
+                # save the index name of the dataframe
+                index_name = df.index.name
+                if fill_na:
+                    if isinstance(fill_na, bool):
+                        method = "nearest"
+                    if isinstance(fill_na, str):
+                        method = fill_na
+                    df = df.reindex(valid_sessions, method=method)
+                else:
+                    df.reindex(valid_sessions, method=None)
+                df.index = df.index.rename(index_name)
+            else:
+                df = df[df.index.isin(valid_sessions)]
+        return df
+
+    def _check_filter(self, filter, utc):
+        if filter and self.time_frame not in SESSION_TIMEFRAMES and not utc:
+            utc = True
+        elif filter and self.time_frame in SESSION_TIMEFRAMES and utc:
+            utc = False
+        return utc
+
+    def get_rates_from_pos(
+        self, filter=False, fill_na=False, lower_colnames=False, utc=False
+    ) -> Union[pd.DataFrame, None]:
+        """
+        Retrieves historical data starting from a specific position.
+
+        Uses the `start_pos` and `count` attributes specified during
+        initialization to fetch data.
+
+        Args:
+            filter : See `Rates.get_historical_data` for more details.
+            fill_na : See `Rates.get_historical_data` for more details.
+            lower_colnames : If True, the column names will be converted to lowercase.
+            utc (bool, optional): If True, the data will be in UTC timezone.
+                Defaults to False.
+
+        Returns:
+            Union[pd.DataFrame, None]: A DataFrame containing historical
+            data if successful, otherwise None.
+
+        Raises:
+            ValueError: If `start_pos` or `count` is not provided during
+                initialization.
+
+        Notes:
+            The Datetime for this method is in Broker's timezone.
+        """
+        if self.start_pos is None or self.count is None:
+            raise ValueError(
+                "Both 'start_pos' and 'count' must be provided "
+                "when calling 'get_rates_from_pos'."
+            )
+        utc = self._check_filter(filter, utc)
+        df = self._fetch_data(
+            self.start_pos, self.count, lower_colnames=lower_colnames, utc=utc
+        )
+        if df is None:
+            return None
+        if filter:
+            return self._filter_data(df, fill_na=fill_na)
+        return df
+
+    def get_rates_from(
+        self,
+        date_from: datetime | pd.Timestamp,
+        count: int = MAX_BARS,
+        filter=False,
+        fill_na=False,
+        lower_colnames=False,
+        utc=False,
+    ) -> Union[pd.DataFrame, None]:
+        """
+        Retrieves historical data within a specified date range.
+
+        Args:
+            date_from : Starting date for data retrieval.
+                The data will be retrieved from this date going to the past.
+
+            count : Number of bars to retrieve.
+
+            filter : See `Rates.get_historical_data` for more details.
+            fill_na : See `Rates.get_historical_data` for more details.
+            lower_colnames : If True, the column names will be converted to lowercase.
+            utc (bool, optional): If True, the data will be in UTC timezone.
+                Defaults to False.
+
+        Returns:
+            Union[pd.DataFrame, None]: A DataFrame containing historical
+            data if successful, otherwise None.
+        """
+        utc = self._check_filter(filter, utc)
+        df = self._fetch_data(date_from, count, lower_colnames=lower_colnames, utc=utc)
+        if df is None:
+            return None
+        if filter:
+            return self._filter_data(df, fill_na=fill_na)
+        return df
+
+    def get_historical_data(
+        self,
+        date_from: datetime | pd.Timestamp,
+        date_to: datetime | pd.Timestamp = pd.Timestamp.now(),
+        utc: bool = False,
+        filter: Optional[bool] = False,
+        fill_na: Optional[bool | str] = False,
+        lower_colnames: Optional[bool] = True,
+        save_csv: Optional[bool] = False,
+    ) -> Union[pd.DataFrame, None]:
+        """
+        Retrieves historical data within a specified date range.
+
+        Args:
+            date_from : Starting date for data retrieval.
+
+            date_to : Ending date for data retrieval.
+                Defaults to the current time.
+
+            utc : If True, the data will be in UTC timezone.
+                Defaults to False.
+
+            filter : If True, the data will be filtered based
+                on the trading sessions for the symbol.
+                This is use when we want to use the data for backtesting using Zipline.
+
+            fill_na : If True, the data will be filled with the nearest value.
+                This is use only when `filter` is True and time frame is "1m" or "D1",
+                this is because we use ``calendar.minutes_in_range`` or ``calendar.sessions_in_range``
+                where calendar is the ``ExchangeCalendar`` from `exchange_calendars` package.
+                So, for "1m" or "D1" time frame, the data will be filled with the nearest value
+                because the data from MT5 will have approximately the same number of rows as the
+                number of trading days or minute in the exchange calendar, so we can fill the missing
+                data with the nearest value.
+
+                But for other time frames, the data will be reindexed with the exchange calendar
+                because the data from MT5 will have more rows than the number of trading days or minute
+                in the exchange calendar. So we only take the data that is in the range of the exchange
+                calendar sessions or minutes.
+
+            lower_colnames : If True, the column names will be converted to lowercase.
+
+            save_csv : File path to save the data as a CSV.
+                If None, the data won't be saved.
+
+        Returns:
+            Union[pd.DataFrame, None]: A DataFrame containing historical data
+                if successful, otherwise None.
+
+        Raises:
+            ValueError: If the starting date is greater than the ending date.
+
+        Notes:
+            The `filter` for this method can be use only for Admira Markets Group (AMG) symbols.
+            The Datetime for this method is in Local timezone by default.
+            All STK symbols are filtered based on the the exchange calendar.
+            All FX symbols are filtered based on the ``us_futures`` calendar.
+            All IDX symbols are filtered based on the exchange calendar of margin currency.
+            All COMD symbols are filtered based on the exchange calendar of the commodity.
+        """
+        utc = self._check_filter(filter, utc)
+        df = self._fetch_data(
+            date_from, date_to, lower_colnames=lower_colnames, utc=utc
+        )
+        if df is None:
+            return None
+        if filter:
+            df = self._filter_data(
+                df, date_from=date_from, date_to=date_to, fill_na=fill_na
+            )
+        if save_csv:
+            df.to_csv(f"{self.symbol}.csv")
+        return df
+
+
+def download_historical_data(
+    symbol,
+    timeframe,
+    date_from,
+    date_to=pd.Timestamp.now(),
+    lower_colnames=True,
+    utc=False,
+    filter=False,
+    fill_na=False,
+    save_csv=False,
+    **kwargs,
+):
+    """Download historical data from MetaTrader 5 terminal.
+    See `Rates.get_historical_data` for more details.
+    """
+    rates = Rates(symbol, timeframe, **kwargs)
+    data = rates.get_historical_data(
+        date_from=date_from,
+        date_to=date_to,
+        save_csv=save_csv,
+        utc=utc,
+        filter=filter,
+        lower_colnames=lower_colnames,
+    )
+    return data
+
+
+def get_data_from_pos(
+    symbol,
+    timeframe,
+    start_pos=0,
+    fill_na=False,
+    count=MAX_BARS,
+    lower_colnames=False,
+    utc=False,
+    filter=False,
+    session_duration=23.0,
+    **kwargs,
+):
+    """Get historical data from a specific position.
+    See `Rates.get_rates_from_pos` for more details.
+    """
+    rates = Rates(symbol, timeframe, start_pos, count, session_duration, **kwargs)
+    data = rates.get_rates_from_pos(
+        filter=filter, fill_na=fill_na, lower_colnames=lower_colnames, utc=utc
+    )
+    return data
+
+
+def get_data_from_date(
+    symbol,
+    timeframe,
+    date_from,
+    count=MAX_BARS,
+    fill_na=False,
+    lower_colnames=False,
+    utc=False,
+    filter=False,
+    **kwargs,
+):
+    """Get historical data from a specific date.
+    See `Rates.get_rates_from` for more details.
+    """
+    rates = Rates(symbol, timeframe, **kwargs)
+    data = rates.get_rates_from(
+        date_from,
+        count,
+        filter=filter,
+        fill_na=fill_na,
+        lower_colnames=lower_colnames,
+        utc=utc,
+    )
+    return data