bbstrader 0.0.1__py3-none-any.whl

bbstrader/btengine/data.py

@@ -0,0 +1,374 @@
+import os.path
+import numpy as np
+import pandas as pd
+import yfinance as yf
+from pathlib import Path
+from typing import List
+from queue import Queue
+from abc import ABCMeta, abstractmethod
+from bbstrader.metatrader.rates import Rates
+from bbstrader.btengine.event import MarketEvent
+
+
+__all__ = [
+    "DataHandler",
+    "BaseCSVDataHandler",
+    "HistoricCSVDataHandler",
+    "MT5HistoricDataHandler",
+    "YFHistoricDataHandler"
+]
+
+
+class DataHandler(metaclass=ABCMeta):
+    """
+    One of the goals of an event-driven trading system is to minimise 
+    duplication of code between the backtesting element and the live execution 
+    element. Ideally it would be optimal to utilise the same signal generation 
+    methodology and portfolio management components for both historical testing 
+    and live trading. In order for this to work the Strategy object which generates
+    the Signals, and the `Portfolio` object which provides Orders based on them, 
+    must utilise an identical interface to a market feed for both historic and live 
+    running.
+
+    This motivates the concept of a class hierarchy based on a `DataHandler` object,
+    which givesall subclasses an interface for providing market data to the remaining 
+    components within thesystem. In this way any subclass data handler can be "swapped out", 
+    without affecting strategy or portfolio calculation.
+
+    Specific example subclasses could include `HistoricCSVDataHandler`, 
+    `YFinanceDataHandler`, `FMPDataHandler`, `IBMarketFeedDataHandler` etc.
+    """
+
+    @abstractmethod
+    def get_latest_bar(self, symbol):
+        """
+        Returns the last bar updated.
+        """
+        raise NotImplementedError(
+            "Should implement get_latest_bar()"
+        )
+
+    @abstractmethod
+    def get_latest_bars(self, symbol, N=1):
+        """
+        Returns the last N bars updated.
+        """
+        raise NotImplementedError(
+            "Should implement get_latest_bars()"
+        )
+
+    @abstractmethod
+    def get_latest_bar_datetime(self, symbol):
+        """
+        Returns a Python datetime object for the last bar.
+        """
+        raise NotImplementedError(
+            "Should implement get_latest_bar_datetime()"
+        )
+
+    @abstractmethod
+    def get_latest_bar_value(self, symbol, val_type):
+        """
+        Returns one of the Open, High, Low, Close, Adj Close, Volume or Returns
+        from the last bar.
+        """
+        raise NotImplementedError(
+            "Should implement get_latest_bar_value()"
+        )
+
+    @abstractmethod
+    def get_latest_bars_values(self, symbol, val_type, N=1):
+        """
+        Returns the last N bar values from the
+        latest_symbol list, or N-k if less available.
+        """
+        raise NotImplementedError(
+            "Should implement get_latest_bars_values()"
+        )
+
+    @abstractmethod
+    def update_bars(self):
+        """
+        Pushes the latest bars to the bars_queue for each symbol
+        in a tuple OHLCVI format: (datetime, Open, High, Low,
+        Close, Adj Close, Volume, Retruns).
+        """
+        raise NotImplementedError(
+            "Should implement update_bars()"
+        )
+
+
+class BaseCSVDataHandler(DataHandler):
+    """
+    Base class for handling data loaded from CSV files.
+    """
+
+    def __init__(self, events: Queue, symbol_list: List[str], csv_dir: str):
+        self.events = events
+        self.symbol_list = symbol_list
+        self.csv_dir = csv_dir
+        self.symbol_data = {}
+        self.latest_symbol_data = {}
+        self.continue_backtest = True
+        self._load_and_process_data()
+
+    def _load_and_process_data(self):
+        """
+        Opens the CSV files from the data directory, converting
+        them into pandas DataFrames within a symbol dictionary.
+        """
+        comb_index = None
+        for s in self.symbol_list:
+            # Load the CSV file with no header information,
+            # indexed on date
+            self.symbol_data[s] = pd.read_csv(
+                os.path.join(self.csv_dir, f'{s}.csv'),
+                header=0, index_col=0, parse_dates=True,
+                names=[
+                    'Datetime', 'Open', 'High',
+                    'Low', 'Close', 'Adj Close', 'Volume'
+                ]
+            )
+            self.symbol_data[s].sort_index(inplace=True)
+            # Combine the index to pad forward values
+            if comb_index is None:
+                comb_index = self.symbol_data[s].index
+            else:
+                comb_index.union(self.symbol_data[s].index)
+            # Set the latest symbol_data to None
+            self.latest_symbol_data[s] = []
+
+        # Reindex the dataframes
+        for s in self.symbol_list:
+            self.symbol_data[s] = self.symbol_data[s].reindex(
+                index=comb_index, method='pad'
+            )
+            self.symbol_data[s]["Returns"] = self.symbol_data[s][
+                "Adj Close"
+            ].pct_change().dropna()
+            self.symbol_data[s] = self.symbol_data[s].iterrows()
+
+    def _get_new_bar(self, symbol: str):
+        """
+        Returns the latest bar from the data feed.
+        """
+        for b in self.symbol_data[symbol]:
+            yield b
+
+    def get_latest_bar(self, symbol: str):
+        """
+        Returns the last bar from the latest_symbol list.
+        """
+        try:
+            bars_list = self.latest_symbol_data[symbol]
+        except KeyError:
+            print("Symbol not available in the historical data set.")
+            raise
+        else:
+            return bars_list[-1]
+
+    def get_latest_bars(self, symbol: str, N=1):
+        """
+        Returns the last N bars from the latest_symbol list,
+        or N-k if less available.
+        """
+        try:
+            bars_list = self.latest_symbol_data[symbol]
+        except KeyError:
+            print("Symbol not available in the historical data set.")
+            raise
+        else:
+            return bars_list[-N:]
+
+    def get_latest_bar_datetime(self, symbol: str):
+        """
+        Returns a Python datetime object for the last bar.
+        """
+        try:
+            bars_list = self.latest_symbol_data[symbol]
+        except KeyError:
+            print("Symbol not available in the historical data set.")
+            raise
+        else:
+            return bars_list[-1][0]
+
+    def get_latest_bar_value(self, symbol: str, val_type: str):
+        """
+        Returns one of the Open, High, Low, Close, Volume or OI
+        values from the pandas Bar series object.
+        """
+        try:
+            bars_list = self.latest_symbol_data[symbol]
+        except KeyError:
+            print("Symbol not available in the historical data set.")
+            raise
+        else:
+            return getattr(bars_list[-1][1], val_type)
+
+    def get_latest_bars_values(self, symbol: str, val_type: str, N=1):
+        """
+        Returns the last N bar values from the
+        latest_symbol list, or N-k if less available.
+        """
+        try:
+            bars_list = self.get_latest_bars(symbol, N)
+        except KeyError:
+            print("That symbol is not available in the historical data set.")
+            raise
+        else:
+            return np.array([getattr(b[1], val_type) for b in bars_list])
+
+    def update_bars(self):
+        """
+        Pushes the latest bar to the latest_symbol_data structure
+        for all symbols in the symbol list.
+        """
+        for s in self.symbol_list:
+            try:
+                bar = next(self._get_new_bar(s))
+            except StopIteration:
+                self.continue_backtest = False
+            else:
+                if bar is not None:
+                    self.latest_symbol_data[s].append(bar)
+        self.events.put(MarketEvent())
+
+
+class HistoricCSVDataHandler(BaseCSVDataHandler):
+    """
+    `HistoricCSVDataHandler` is designed to read CSV files for
+    each requested symbol from disk and provide an interface
+    to obtain the "latest" bar in a manner identical to a live
+    trading interface.
+
+    This class is useful when you have your own data or you want 
+    to cutomize specific data in some form based on your `Strategy()` .   
+    """
+
+    def __init__(self, events: Queue, symbol_list: List[str], **kwargs):
+        """
+        Initialises the historic data handler by requesting
+        the location of the CSV files and a list of symbols.
+        It will be assumed that all files are of the form
+        `symbol.csv`, where `symbol` is a string in the list.
+
+        Args:
+            events (Queue): The Event Queue.
+            csv_dir (str): Absolute directory path to the CSV files.
+            symbol_list (List[str]): A list of symbol strings.
+        """
+        csv_dir = kwargs.get("csv_dir")
+        super().__init__(events, symbol_list, csv_dir)
+
+MAX_BARS = 10_000_000
+class MT5HistoricDataHandler(BaseCSVDataHandler):
+    """
+    Downloads historical data from MetaTrader 5 (MT5) and provides 
+    an interface for accessing this data bar-by-bar, simulating 
+    a live market feed for backtesting.
+
+    Data is downloaded from MT5, saved as CSV files, and then loaded
+    using the functionality inherited from `BaseCSVDataHandler`. 
+
+    This class is useful when you need to get data from specific broker
+    for different time frames. 
+    """
+
+    def __init__(self, events: Queue, symbol_list: List[str], **kwargs):
+        """
+        Args:
+            events (Queue): The Event Queue for passing market events.
+            symbol_list (List[str]): A list of symbol strings to download data for.
+            **kwargs: Keyword arguments for data retrieval:
+                time_frame (str): MT5 time frame (e.g., 'D1' for daily).
+                max_bars (int): Maximum number of bars to download per symbol.
+                start_pos (int | str, optional): Starting bar position (default: 0).
+                    If it set to `str`, it must be in 'YYYY-MM-DD' format and
+                session_duration (int | float): Number of trading hours per day.
+                mt5_data (str): Directory for storing data (default: 'mt5_data').
+
+        Note:
+            Requires a working connection to an MT5 terminal.
+        """
+        self.tf = kwargs.get('time_frame', 'D1')
+        self.start_pos = kwargs.get('start_pos', 0)
+        self.max_bars = kwargs.get('max_bars', MAX_BARS)
+        self.sd = kwargs.get('session_duration', 6.5)
+        self.data_dir = kwargs.get('mt5_data', 'mt5_data')
+        self.symbol_list = symbol_list
+        csv_dir = self._download_data(self.data_dir)
+        super().__init__(events, symbol_list, csv_dir)
+
+    def _download_data(self, cache_dir: str):
+        data_dir = Path() / cache_dir
+        data_dir.mkdir(parents=True, exist_ok=True)
+        for symbol in self.symbol_list:
+            try:
+                rate = Rates(symbol, self.tf, self.start_pos, self.max_bars, self.sd)
+                data = rate.get_rates_from_pos()
+                if data is None:
+                    raise ValueError(f"No data found for {symbol}")
+                data.to_csv(data_dir / f'{symbol}.csv')
+            except Exception as e:
+                raise ValueError(f"Error downloading {symbol}: {e}")
+        return data_dir
+
+
+class YFHistoricDataHandler(BaseCSVDataHandler):
+    """
+    Downloads historical data from Yahoo Finance and provides 
+    an interface for accessing this data bar-by-bar, simulating
+    a live market feed for backtesting.
+
+    Data is fetched using the `yfinance` library and optionally cached
+    to disk to speed up subsequent runs. 
+
+    This class is useful when working with historical daily prices.
+    """
+
+    def __init__(self, events: Queue, symbol_list: List[str], **kwargs):
+        """
+        Args:
+            events (Queue): The Event Queue for passing market events.
+            symbol_list (list[str]): List of symbols to download data for.
+            yf_start (str): Start date for historical data (YYYY-MM-DD).
+            yf_end (str): End date for historical data (YYYY-MM-DD).
+            cache_dir (str, optional): Directory for caching data (default: 'yf_cache').
+        """
+        self.symbol_list = symbol_list
+        self.start_date = kwargs.get('yf_start')
+        self.end_date = kwargs.get('yf_end')
+        self.cache_dir = kwargs.get('yf_cache', 'yf_cache')
+        csv_dir = self._download_and_cache_data(self.cache_dir)
+        super().__init__(events, symbol_list, csv_dir)
+
+    def _download_and_cache_data(self, cache_dir: str):
+        """Downloads and caches historical data as CSV files."""
+        os.makedirs(cache_dir, exist_ok=True)
+        for symbol in self.symbol_list:
+            filepath = os.path.join(cache_dir, f"{symbol}.csv")
+            if not os.path.exists(filepath):
+                try:
+                    data = yf.download(
+                        symbol, start=self.start_date, end=self.end_date)
+                    if data.empty:
+                        raise ValueError(f"No data found for {symbol}")
+                    data.to_csv(filepath)  # Cache the data
+                except Exception as e:
+                    raise ValueError(f"Error downloading {symbol}: {e}")
+        return cache_dir
+
+
+# TODO # Get data from FinancialModelingPrep ()
+class FMPHistoricDataHandler(BaseCSVDataHandler):
+    ...
+
+
+class BaseFMPDataHanler(object):
+    ...
+
+
+class FMPFundamentalDataHandler(BaseFMPDataHanler):
+    ...
+
+# TODO Add other Handlers

bbstrader/btengine/event.py

@@ -0,0 +1,201 @@
+from datetime import datetime
+
+__all__ = [
+    "Event",
+    "MarketEvent",
+    "SignalEvent",
+    "OrderEvent",
+    "FillEvent"
+]
+
+
+class Event(object):
+    """
+    Event is base class providing an interface for all subsequent
+    (inherited) events, that will trigger further events in the
+    trading infrastructure.
+    Since in many implementations the Event objects will likely develop greater 
+    complexity, it is thus being "future-proofed" by creating a class hierarchy. 
+    The Event class is simply a way to ensure that all events have a common interface
+    and can be handled in a consistent manner.
+    """
+    ...
+
+
+class MarketEvent(Event):
+    """
+    Market Events are triggered when the outer while loop of the backtesting 
+    system begins a new `"heartbeat"`. It occurs when the `DataHandler` object 
+    receives a new update of market data for any symbols which are currently 
+    being tracked. It is used to `trigger the Strategy object` generating 
+    new `trading signals`. The event object simply contains an identification 
+    that it is a market event, with no other structure.
+    """
+
+    def __init__(self):
+        """
+        Initialises the MarketEvent.
+        """
+        self.type = 'MARKET'
+
+
+class SignalEvent(Event):
+    """
+    The `Strategy object` utilises market data to create new `SignalEvents`. 
+    The SignalEvent contains a `strategy ID`, a `ticker symbol`, a `timestamp` 
+    for when it was generated, a `direction` (long or short) and a `"strength"` 
+    indicator (this is useful for mean reversion strategies) and the `quantiy` 
+    to buy or sell. The `SignalEvents` are utilised by the `Portfolio object` 
+    as advice for how to trade.
+    """
+
+    def __init__(self,
+                 strategy_id: int,
+                 symbol: str,
+                 datetime: datetime,
+                 signal_type: str,
+                 quantity: int | float = 100,
+                 strength: int | float = 1.0
+                 ):
+        """
+        Initialises the SignalEvent.
+
+        Args:
+            strategy_id (int): The unique identifier for the strategy that
+                generated the signal.
+
+            symbol (str): The ticker symbol, e.g. 'GOOG'.
+            datetime (datetime): The timestamp at which the signal was generated.
+            signal_type (str): 'LONG' or 'SHORT'.
+            quantity (int | float): An optional integer (or float) representing the order size.
+            strength (int | float): An adjustment factor "suggestion" used to scale
+                quantity at the portfolio level. Useful for pairs strategies.
+        """
+        self.type = 'SIGNAL'
+        self.strategy_id = strategy_id
+        self.symbol = symbol
+        self.datetime = datetime
+        self.signal_type = signal_type
+        self.quantity = quantity
+        self.strength = strength
+
+
+class OrderEvent(Event):
+    """
+    When a Portfolio object receives `SignalEvents` it assesses them 
+    in the wider context of the portfolio, in terms of risk and position sizing. 
+    This ultimately leads to `OrderEvents` that will be sent to an `ExecutionHandler`.
+
+    The `OrderEvents` is slightly more complex than a `SignalEvents` since 
+    it contains a quantity field in addition to the aforementioned properties 
+    of SignalEvent. The quantity is determined by the Portfolio constraints. 
+    In addition the OrderEvent has a `print_order()` method, used to output the 
+    information to the console if necessary.
+    """
+
+    def __init__(self,
+                 symbol: str,
+                 order_type: str,
+                 quantity: int | float,
+                 direction: str
+                 ):
+        """
+        Initialises the order type, setting whether it is
+        a Market order ('MKT') or Limit order ('LMT'), has
+        a quantity (integral or float) and its direction ('BUY' or 'SELL').
+
+        Args:
+            symbol (str): The instrument to trade.
+            order_type (str): 'MKT' or 'LMT' for Market or Limit. 
+            quantity (int | float): Non-negative number for quantity.            
+            direction (str): 'BUY' or 'SELL' for long or short.
+        """
+        self.type = 'ORDER'
+        self.symbol = symbol
+        self.order_type = order_type
+        self.quantity = quantity
+        self.direction = direction
+
+        def print_order(self):
+            """
+            Outputs the values within the Order.
+            """
+            print(
+                "Order: Symbol=%s, Type=%s, Quantity=%s, Direction=%s" %
+                (self.symbol, self.order_type, self.quantity, self.direction)
+            )
+
+
+class FillEvent(Event):
+    """
+    When an `ExecutionHandler` receives an `OrderEvent` it must transact the order.
+    Once an order has been transacted it generates a `FillEvent`, which describes 
+    the cost of purchase or sale as well as the transaction costs, such as fees 
+    or slippage.
+
+    The `FillEvent` is the Event with the greatest complexity. 
+    It contains a `timestamp` for when an order was filled, the `symbol` 
+    of the order and the `exchange` it was executed on, the `quantity` 
+    of shares transacted, the `actual price of the purchase` and the `commission 
+    incurred`.
+
+    The commission is calculated using the Interactive Brokers commissions. 
+    For US API orders this commission is `1.30 USD` minimum per order, with a flat 
+    rate of either 0.013 USD or 0.08 USD per share depending upon whether 
+    the trade size is below or above `500 units` of stock.
+    """
+
+    def __init__(self,
+                 timeindex: datetime,
+                 symbol: str,
+                 exchange: str,
+                 quantity: int | float,
+                 direction: str,
+                 fill_cost: int | float,
+                 commission: float | None = None
+                 ):
+        """
+        Initialises the FillEvent object. Sets the symbol, exchange,
+        quantity, direction, cost of fill and an optional
+        commission.
+
+        If commission is not provided, the Fill object will
+        calculate it based on the trade size and Interactive
+        Brokers fees.
+
+        Args:
+            timeindex (datetime): The bar-resolution when the order was filled.
+            symbol (str): The instrument which was filled.
+            exchange (str): The exchange where the order was filled.
+            quantity (int | float): The filled quantity.
+            direction (str): The direction of fill ('BUY' or 'SELL')
+            fill_cost (int | float): The holdings value in dollars.
+            commission (float | None): An optional commission sent from IB.
+        """
+        self.type = 'FILL'
+        self.timeindex = timeindex
+        self.symbol = symbol
+        self.exchange = exchange
+        self.quantity = quantity
+        self.direction = direction
+        self.fill_cost = fill_cost
+        # Calculate commission
+        if commission is None:
+            self.commission = self.calculate_ib_commission()
+        else:
+            self.commission = commission
+
+    def calculate_ib_commission(self):
+        """
+        Calculates the fees of trading based on an Interactive
+        Brokers fee structure for API, in USD.
+        This does not include exchange or ECN fees.
+        Based on "US API Directed Orders":
+        https://www.interactivebrokers.com/en/index.php?f=commission&p=stocks2
+        """
+        full_cost = 1.3
+        if self.quantity <= 500:
+            full_cost = max(1.3, 0.013 * self.quantity)
+        else:
+            full_cost = max(1.3, 0.008 * self.quantity)
+        return full_cost

bbstrader/btengine/execution.py

@@ -0,0 +1,83 @@
+import datetime
+from queue import Queue
+from abc import ABCMeta, abstractmethod
+from bbstrader.btengine.event import FillEvent, OrderEvent
+
+__all__ = [
+    "ExecutionHandler",
+    "SimulatedExecutionHandler"
+]
+
+
+class ExecutionHandler(metaclass=ABCMeta):
+    """
+    The ExecutionHandler abstract class handles the interaction
+    between a set of order objects generated by a Portfolio and
+    the ultimate set of Fill objects that actually occur in the
+    market.
+
+    The handlers can be used to subclass simulated brokerages
+    or live brokerages, with identical interfaces. This allows
+    strategies to be backtested in a very similar manner to the
+    live trading engine.
+
+    The ExecutionHandler described here is exceedingly simple, 
+    since it fills all orders at the current market price. 
+    This is highly unrealistic, but serves as a good baseline for improvement.
+    """
+
+    @abstractmethod
+    def execute_order(self, event: OrderEvent):
+        """
+        Takes an Order event and executes it, producing
+        a Fill event that gets placed onto the Events queue.
+
+        Args:
+            event (OrderEvent): Contains an Event object with order information.
+        """
+        raise NotImplementedError(
+            "Should implement execute_order()"
+        )
+
+
+class SimulatedExecutionHandler(ExecutionHandler):
+    """
+    The simulated execution handler simply converts all order
+    objects into their equivalent fill objects automatically
+    without latency, slippage or fill-ratio issues.
+
+    This allows a straightforward "first go" test of any strategy,
+    before implementation with a more sophisticated execution
+    handler.
+    """
+
+    def __init__(self, events: Queue):
+        """
+        Initialises the handler, setting the event queues
+        up internally.
+
+        Args:
+            events (Queue): The Queue of Event objects.
+        """
+        self.events = events
+
+    def execute_order(self, event: OrderEvent):
+        """
+        Simply converts Order objects into Fill objects naively,
+        i.e. without any latency, slippage or fill ratio problems.
+
+        Args:
+            event (OrderEvent): Contains an Event object with order information.
+        """
+        if event.type == 'ORDER':
+            fill_event = FillEvent(
+                datetime.datetime.now(), event.symbol,
+                'ARCA', event.quantity, event.direction, None
+            )
+            self.events.put(fill_event)
+
+# TODO # Use in live execution
+
+
+class MT5ExecutionHandler(ExecutionHandler):
+    ...