lumibot 4.0.23__py3-none-any.whl → 4.1.0__py3-none-any.whl

lumibot/backtesting/__init__.py

@@ -8,8 +8,9 @@ from .polygon_backtesting import PolygonDataBacktesting
 from .thetadata_backtesting import ThetaDataBacktesting
 from .yahoo_backtesting import YahooDataBacktesting
 
-# Import DataBento backtesting - use pandas version (polars version is slow)
-try:
-    from .databento_backtesting import DataBentoDataBacktesting
-except ImportError:
-    from .databento_backtesting_polars import DataBentoDataBacktestingPolars as DataBentoDataBacktesting
+# Import DataBento backtesting
+# Polars version (NEW DEFAULT - faster performance)
+from lumibot.data_sources.databento_data_polars_backtesting import DataBentoDataPolarsBacktesting as DataBentoDataBacktesting
+
+# Pandas version (stable fallback, kept for compatibility)
+from .databento_backtesting import DataBentoDataBacktesting as DataBentoDataBacktestingPandas

lumibot/backtesting/backtesting_broker.py

@@ -17,6 +17,91 @@ from lumibot.trading_builtins import CustomStream
 logger = get_logger(__name__)
 
 
+# Typical initial margin requirements for common futures contracts
+# Used for backtesting to simulate margin deduction/release
+TYPICAL_FUTURES_MARGINS = {
+    # CME Micro E-mini Futures
+    "MES": 1300,      # Micro E-mini S&P 500 (~$1,300)
+    "MNQ": 1700,      # Micro E-mini Nasdaq-100 (~$1,700)
+    "MYM": 1100,      # Micro E-mini Dow (~$1,100)
+    "M2K": 800,       # Micro E-mini Russell 2000 (~$800)
+    "MCL": 1500,      # Micro Crude Oil (~$1,500)
+    "MGC": 1200,      # Micro Gold (~$1,200)
+
+    # CME Standard E-mini Futures
+    "ES": 13000,      # E-mini S&P 500 (~$13,000)
+    "NQ": 17000,      # E-mini Nasdaq-100 (~$17,000)
+    "YM": 11000,      # E-mini Dow (~$11,000)
+    "RTY": 8000,      # E-mini Russell 2000 (~$8,000)
+
+    # CME Full-Size Futures
+    "CL": 8000,       # Crude Oil (~$8,000)
+    "GC": 10000,      # Gold (~$10,000)
+    "SI": 14000,      # Silver (~$14,000)
+    "NG": 3000,       # Natural Gas (~$3,000)
+    "HG": 4000,       # Copper (~$4,000)
+
+    # CME Currency Futures
+    "6E": 2500,       # Euro FX (~$2,500)
+    "6J": 3000,       # Japanese Yen (~$3,000)
+    "6B": 2800,       # British Pound (~$2,800)
+    "6C": 2000,       # Canadian Dollar (~$2,000)
+
+    # CME Interest Rate Futures
+    "ZB": 4000,       # 30-Year T-Bond (~$4,000)
+    "ZN": 2000,       # 10-Year T-Note (~$2,000)
+    "ZF": 1500,       # 5-Year T-Note (~$1,500)
+    "ZT": 800,        # 2-Year T-Note (~$800)
+
+    # CME Agricultural Futures
+    "ZC": 2000,       # Corn (~$2,000)
+    "ZS": 3000,       # Soybeans (~$3,000)
+    "ZW": 2500,       # Wheat (~$2,500)
+    "ZL": 1500,       # Soybean Oil (~$1,500)
+
+    # Default for unknown futures
+    "DEFAULT": 5000,  # Conservative default
+}
+
+
+def get_futures_margin_requirement(asset: Asset) -> float:
+    """
+    Get the initial margin requirement for a futures contract.
+
+    This is used in backtesting to simulate the margin deduction when opening
+    a futures position and margin release when closing.
+
+    Args:
+        asset: The futures Asset object
+
+    Returns:
+        float: Initial margin requirement in dollars
+
+    Note:
+        These are TYPICAL values and may not match current broker requirements.
+        For live trading, brokers handle margin internally.
+    """
+    symbol = asset.symbol.upper()
+
+    # Try exact match first
+    if symbol in TYPICAL_FUTURES_MARGINS:
+        return TYPICAL_FUTURES_MARGINS[symbol]
+
+    # Try base symbol (remove month/year codes like "ESH4" -> "ES")
+    # Most futures symbols are 2-3 characters followed by month/year
+    base_symbol = ''.join(c for c in symbol if c.isalpha())
+    if base_symbol in TYPICAL_FUTURES_MARGINS:
+        return TYPICAL_FUTURES_MARGINS[base_symbol]
+
+    # Unknown contract - use conservative default
+    logger.warning(
+        f"Unknown futures contract '{symbol}'. Using default margin of "
+        f"${TYPICAL_FUTURES_MARGINS['DEFAULT']:.2f}. "
+        f"Consider adding this contract to TYPICAL_FUTURES_MARGINS."
+    )
+    return TYPICAL_FUTURES_MARGINS["DEFAULT"]
+
+
 class BacktestingBroker(Broker):
     # Metainfo
     IS_BACKTESTING_BROKER = True
@@ -215,17 +300,30 @@ class BacktestingBroker(Broker):
         trading_day = search.iloc[0]
         open_time = trading_day.market_open
 
+        # DEBUG: Log what's happening
+        print(f"[BROKER DEBUG] get_time_to_open: now={now}, next_trading_day={trading_day.name}, open_time={open_time}")
+
         # For Backtesting, sometimes the user can just pass in dates (i.e. 2023-08-01) and not datetimes
         # In this case the "now" variable is starting at midnight, so we need to adjust the open_time to be actual
-        # market open time.  In the case where the user passes in a time inside a valid trading day, use that time
+        # market open time.  In the case where the user passes in a valid trading day, use that time
         # as the start of trading instead of market open.
+        # BUT: Only do this if the current day (now.date()) is actually a trading day
         if self.IS_BACKTESTING_BROKER and now > open_time:
-            open_time = self.data_source.datetime_start
+            # Check if now.date() is in trading days before overriding
+            now_date = now.date() if hasattr(now, 'date') else now
+            trading_day_dates = self._trading_days.index.date
+            if now_date in trading_day_dates:
+                print(f"[BROKER DEBUG] Overriding open_time to datetime_start because now ({now}) is on a trading day but after market open")
+                open_time = self.data_source.datetime_start
+            else:
+                print(f"[BROKER DEBUG] NOT overriding open_time because now ({now}) is NOT a trading day")
 
         if now >= open_time:
+            print(f"[BROKER DEBUG] Market already open: now={now} >= open_time={open_time}, returning 0")
             return 0
 
         delta = open_time - now
+        print(f"[BROKER DEBUG] Market opens in {delta.total_seconds()} seconds")
         return delta.total_seconds()
 
     def get_time_to_close(self):
@@ -262,24 +360,30 @@ class BacktestingBroker(Broker):
     def _await_market_to_open(self, timedelta=None, strategy=None):
         # Process outstanding orders first before waiting for market to open
         # or else they don't get processed until the next day
+        print(f"[BROKER DEBUG] _await_market_to_open called, current datetime={self.datetime}, timedelta={timedelta}")
         self.process_pending_orders(strategy=strategy)
 
         time_to_open = self.get_time_to_open()
+        print(f"[BROKER DEBUG] get_time_to_open returned: {time_to_open}")
 
         # If None is returned, it means we've reached the end of available trading days
         if time_to_open is None:
             logger.info("Backtesting reached end of available trading days data")
+            print(f"[BROKER DEBUG] time_to_open is None, returning early")
             return
 
         # Allow the caller to specify a buffer (in minutes) before the actual open
         if timedelta:
             time_to_open -= 60 * timedelta
+            print(f"[BROKER DEBUG] Adjusted time_to_open for timedelta buffer: {time_to_open}")
 
         # Only advance time if there is something positive to advance;
         # prevents zero or negative time updates.
         if time_to_open <= 0:
+            print(f"[BROKER DEBUG] time_to_open <= 0 ({time_to_open}), returning without advancing time")
             return
 
+        print(f"[BROKER DEBUG] Advancing time by {time_to_open} seconds")
         self._update_datetime(time_to_open)
 
     def _await_market_to_close(self, timedelta=None, strategy=None):
@@ -499,8 +603,16 @@ class BacktestingBroker(Broker):
         def _cancel_inline(order: Order):
             if order.identifier in canceled_identifiers:
                 return
-            canceled_identifiers.add(order.identifier)
-            self._process_trade_event(order, self.CANCELED_ORDER)
+
+            # BUGFIX: Only process CANCELED event if the order is actually active
+            # Don't try to cancel orders that are already filled or canceled
+            if order.is_active():
+                canceled_identifiers.add(order.identifier)
+                self._process_trade_event(order, self.CANCELED_ORDER)
+            else:
+                logger.debug(f"Order {order.identifier} not active (status={order.status}), skipping cancel event")
+                canceled_identifiers.add(order.identifier)
+
             for child in order.child_orders:
                 _cancel_inline(child)
 
@@ -920,9 +1032,92 @@ class BacktestingBroker(Broker):
         asset_type = getattr(order.asset, "asset_type", None)
         quote_asset_type = getattr(order.quote, "asset_type", None) if hasattr(order, "quote") and order.quote else None
 
+        # For futures, use margin-based cash management (not full notional value)
+        # Futures don't tie up full contract value - only margin requirement
+        if (
+            not is_multileg_parent
+            and asset_type in (Asset.AssetType.FUTURE, Asset.AssetType.CONT_FUTURE)
+        ):
+            # Reconstruct position state BEFORE this order to determine if opening/closing
+            futures_qty_before = 0
+            futures_entry_price = None
+
+            # Look through filled_orders to find position before this order
+            for filled_order in self._filled_orders.get_list():
+                if (filled_order.asset == order.asset
+                    and filled_order.strategy == order.strategy
+                    and filled_order != order):  # Don't count the current order
+
+                    if filled_order.side in (Order.OrderSide.BUY, "buy", "buy_to_open"):
+                        futures_qty_before += filled_order.quantity
+                        # Track most recent BUY entry price (for long positions)
+                        if filled_order.avg_fill_price:
+                            futures_entry_price = float(filled_order.avg_fill_price)
+                    elif filled_order.side in (Order.OrderSide.SELL, Order.OrderSide.SELL_TO_CLOSE, "sell", "sell_to_close"):
+                        futures_qty_before -= filled_order.quantity
+                        # Track most recent SELL entry price (for short positions)
+                        # Note: This gets overwritten by SELL_TO_CLOSE, which is correct
+                        # We want the opening SELL price, not closing prices
+                        if (filled_order.side in (Order.OrderSide.SELL, "sell")  # Opening short
+                            and filled_order.avg_fill_price):
+                            futures_entry_price = float(filled_order.avg_fill_price)
+
+            # Determine if this order is opening or closing a position
+            is_opening = (futures_qty_before == 0)
+            is_closing_long = (
+                futures_qty_before > 0
+                and order.side in (Order.OrderSide.SELL, Order.OrderSide.SELL_TO_CLOSE, "sell", "sell_to_close")
+            )
+            is_closing_short = (
+                futures_qty_before < 0
+                and order.side in (Order.OrderSide.BUY, Order.OrderSide.BUY_TO_OPEN, "buy", "buy_to_open")
+            )
+            is_closing = is_closing_long or is_closing_short
+
+            # Get margin requirement and multiplier
+            margin_per_contract = get_futures_margin_requirement(order.asset)
+            multiplier = getattr(order.asset, "multiplier", 1)
+            total_margin = margin_per_contract * float(filled_quantity)
+
+            current_cash = strategy.cash
+
+            if is_opening:
+                # ENTRY (long or short): Deduct initial margin from cash
+                new_cash = current_cash - total_margin
+                strategy._set_cash_position(new_cash)
+
+            elif is_closing:
+                # EXIT (close long or cover short): Release margin and apply realized P&L
+                if futures_entry_price:
+                    exit_price = float(price)
+
+                    # For shorts, P&L is inverted: profit when price goes down
+                    if futures_qty_before < 0:
+                        # Closing short: P&L = (entry - exit) × qty × multiplier
+                        realized_pnl = (futures_entry_price - exit_price) * float(filled_quantity) * float(multiplier)
+                    else:
+                        # Closing long: P&L = (exit - entry) × qty × multiplier
+                        realized_pnl = (exit_price - futures_entry_price) * float(filled_quantity) * float(multiplier)
+
+                    # Update cash: release margin + add realized P&L
+                    new_cash = current_cash + total_margin + realized_pnl
+                    strategy._set_cash_position(new_cash)
+                else:
+                    # No entry price found - just release margin (shouldn't happen normally)
+                    logger.warning(
+                        f"No entry price found for futures exit: {order.asset.symbol}. "
+                        f"Only releasing margin, no P&L applied."
+                    )
+                    new_cash = current_cash + total_margin
+                    strategy._set_cash_position(new_cash)
+            else:
+                # Adding to existing position: deduct margin for additional contracts
+                new_cash = current_cash - total_margin
+                strategy._set_cash_position(new_cash)
+
         # For crypto base with forex quote (like BTC/USD where USD is forex), use cash
         # For crypto base with crypto quote (like BTC/USDT where both are crypto), use positions
-        if (
+        elif (
             not is_multileg_parent
             and asset_type == Asset.AssetType.CRYPTO
             and quote_asset_type == Asset.AssetType.FOREX
@@ -964,16 +1159,21 @@ class BacktestingBroker(Broker):
             self._apply_trade_cost(strategy, trade_cost)
 
     def _process_crypto_quote(self, order, quantity, price):
-        """Override to skip crypto quote processing for crypto+forex trades that are handled with direct cash updates."""
-        # Check if this is a crypto+forex trade
+        """Override to skip quote processing for assets that use direct cash updates or margin-based trading."""
+        # Check asset types
         asset_type = getattr(order.asset, "asset_type", None)
         quote_asset_type = getattr(order.quote, "asset_type", None) if hasattr(order, "quote") and order.quote else None
 
-        # For crypto+forex trades, skip position-based quote processing since we handle cash directly
+        # Skip position-based quote processing for:
+        # 1. Crypto+forex trades (handled with direct cash updates)
+        # 2. Futures contracts (use margin, only realize P&L on close, not full notional)
         if asset_type == Asset.AssetType.CRYPTO and quote_asset_type == Asset.AssetType.FOREX:
             return
 
-        # For crypto+crypto trades, use the original position-based processing
+        if asset_type in (Asset.AssetType.FUTURE, Asset.AssetType.CONT_FUTURE):
+            return
+
+        # For other asset types (crypto+crypto, stocks, etc.), use the original position-based processing
         super()._process_crypto_quote(order, quantity, price)
 
     def calculate_trade_cost(self, order: Order, strategy, price: float):

lumibot/backtesting/databento_backtesting.py

@@ -71,6 +71,15 @@ class DataBentoDataBacktesting(PandasData):
         # Track data requests to avoid repeated log messages
         self._logged_requests = set()
 
+        # OPTIMIZATION: Iteration-level caching to avoid redundant filtering
+        # Cache filtered DataFrames per iteration (datetime)
+        self._filtered_bars_cache = {}  # {(asset_key, length, timestep, timeshift, dt): DataFrame}
+        self._last_price_cache = {}     # {(asset_key, dt): price}
+        self._cache_datetime = None     # Track when to invalidate cache
+
+        # Track which futures assets we've fetched multipliers for (to avoid redundant API calls)
+        self._multiplier_fetched_assets = set()
+
         # Verify DataBento availability
         if not databento_helper.DATABENTO_AVAILABLE:
             logger.error("DataBento package not available. Please install with: pip install databento")
@@ -78,6 +87,81 @@ class DataBentoDataBacktesting(PandasData):
 
         logger.info(f"DataBento backtesting initialized for period: {datetime_start} to {datetime_end}")
 
+    def _check_and_clear_cache(self):
+        """
+        OPTIMIZATION: Clear iteration caches when datetime changes.
+        This ensures fresh filtering for each new iteration while reusing
+        results within the same iteration.
+        """
+        current_dt = self.get_datetime()
+        if self._cache_datetime != current_dt:
+            self._filtered_bars_cache.clear()
+            self._last_price_cache.clear()
+            self._cache_datetime = current_dt
+
+    def _ensure_futures_multiplier(self, asset):
+        """
+        Ensure futures asset has correct multiplier set.
+
+        This method is idempotent and cached - safe to call multiple times.
+        Only fetches multiplier once per unique asset.
+
+        Design rationale:
+        - Futures multipliers must be fetched from data provider (e.g., DataBento)
+        - Asset class defaults to multiplier=1
+        - Data source is responsible for updating multiplier on first use
+        - Lazy fetching is more efficient than prefetching all possible assets
+
+        Parameters
+        ----------
+        asset : Asset
+            The asset to ensure has correct multiplier
+        """
+        # Skip if not a futures asset
+        if asset.asset_type not in (Asset.AssetType.FUTURE, Asset.AssetType.CONT_FUTURE):
+            return
+
+        # Skip if multiplier already set to non-default value
+        if asset.multiplier != 1:
+            return
+
+        # Create cache key to track which assets we've already processed
+        # Use symbol + asset_type + expiration to handle different contracts
+        cache_key = (asset.symbol, asset.asset_type, getattr(asset, 'expiration', None))
+
+        # Check if we already tried to fetch for this asset
+        if cache_key in self._multiplier_fetched_assets:
+            return  # Already attempted (even if failed, don't retry every time)
+
+        # Mark as attempted to avoid redundant API calls
+        self._multiplier_fetched_assets.add(cache_key)
+
+        # Fetch and set multiplier from DataBento
+        try:
+            client = databento_helper.DataBentoClient(self._api_key)
+
+            # Resolve symbol based on asset type
+            if asset.asset_type == Asset.AssetType.CONT_FUTURE:
+                resolved_symbol = databento_helper._format_futures_symbol_for_databento(
+                    asset, reference_date=self.datetime_start
+                )
+            else:
+                resolved_symbol = databento_helper._format_futures_symbol_for_databento(asset)
+
+            # Fetch multiplier from DataBento instrument definition
+            databento_helper._fetch_and_update_futures_multiplier(
+                client=client,
+                asset=asset,
+                resolved_symbol=resolved_symbol,
+                dataset="GLBX.MDP3",
+                reference_date=self.datetime_start
+            )
+
+            logger.info(f"Successfully set multiplier for {asset.symbol}: {asset.multiplier}")
+
+        except Exception as e:
+            logger.warning(f"Could not fetch multiplier for {asset.symbol}: {e}")
+
     def prefetch_data(self, assets, timestep="minute"):
         """
         Prefetch all required data for the specified assets for the entire backtest period.
@@ -186,6 +270,9 @@ class DataBentoDataBacktesting(PandasData):
         else:
             search_asset = (search_asset, quote_asset)
 
+        # Ensure futures have correct multiplier set
+        self._ensure_futures_multiplier(asset_separated)
+
         # If this asset was already prefetched, we don't need to do anything
         if search_asset in self._prefetched_assets:
             return
@@ -293,7 +380,7 @@ class DataBentoDataBacktesting(PandasData):
     def get_last_price(self, asset, quote=None, exchange=None):
         """
         Get the last price for an asset at the current backtest time
-        
+
         Parameters
         ----------
         asset : Asset
@@ -302,30 +389,39 @@ class DataBentoDataBacktesting(PandasData):
             Quote asset (not typically used with DataBento)
         exchange : str, optional
             Exchange filter
-            
+
         Returns
         -------
         float, Decimal, or None
             Last price at current backtest time
         """
         try:
-            # For backtesting, we get the price at the current simulation time
+            # OPTIMIZATION: Check cache first
+            self._check_and_clear_cache()
             current_dt = self.get_datetime()
-            
+
             # Try to get data from our cached pandas_data first
             search_asset = asset
             quote_asset = quote if quote is not None else Asset("USD", "forex")
-            
+
             if isinstance(search_asset, tuple):
                 asset_separated, quote_asset = search_asset
             else:
                 search_asset = (search_asset, quote_asset)
                 asset_separated = asset
-            
+
+            # Ensure futures have correct multiplier set
+            self._ensure_futures_multiplier(asset_separated)
+
+            # OPTIMIZATION: Check iteration cache
+            cache_key = (search_asset, current_dt)
+            if cache_key in self._last_price_cache:
+                return self._last_price_cache[cache_key]
+
             if search_asset in self.pandas_data:
                 asset_data = self.pandas_data[search_asset]
                 df = asset_data.df
-                
+
                 if not df.empty and 'close' in df.columns:
                         # Ensure current_dt is timezone-aware for comparison
                         current_dt_aware = to_datetime_aware(current_dt)
@@ -341,11 +437,14 @@ class DataBentoDataBacktesting(PandasData):
 
                         # Filter to data up to current backtest time (exclude current bar unless broker overrides)
                         filtered_df = df[df.index <= cutoff_dt]
-                        
+
                         if not filtered_df.empty:
                             last_price = filtered_df['close'].iloc[-1]
                             if not pd.isna(last_price):
-                                return float(last_price)
+                                price = float(last_price)
+                                # OPTIMIZATION: Cache the result
+                                self._last_price_cache[cache_key] = price
+                                return price
             
             # If no cached data, try to get recent data
             logger.warning(f"No cached data for {asset.symbol}, attempting direct fetch")
@@ -470,34 +569,50 @@ class DataBentoDataBacktesting(PandasData):
     ):
         """
         Override parent method to fetch data from DataBento instead of pre-loaded data store
-        
+
         This method is called by get_historical_prices and is responsible for actually
         fetching the data from the DataBento API.
         """
         timestep = timestep if timestep else "minute"
-        
-        # Check if we need to fetch data by calling _update_pandas_data first
-        # This will only fetch if data is not already cached or prefetched
-        self._update_pandas_data(asset, quote, length, timestep)
-        
+
+        # OPTIMIZATION: Check iteration cache first
+        self._check_and_clear_cache()
+        current_dt = self.get_datetime()
+
         # Get data from our cached pandas_data
         search_asset = asset
         quote_asset = quote if quote is not None else Asset("USD", "forex")
-        
+
         if isinstance(search_asset, tuple):
             asset_separated, quote_asset = search_asset
         else:
             search_asset = (search_asset, quote_asset)
             asset_separated = asset
-        
+
+        # OPTIMIZATION: Build cache key and check cache
+        # Convert timeshift to consistent format for caching
+        timeshift_key = 0
+        if timeshift:
+            if isinstance(timeshift, int):
+                timeshift_key = timeshift
+            else:
+                timeshift_key = int(timeshift.total_seconds() / 60)
+
+        cache_key = (search_asset, length, timestep, timeshift_key, current_dt)
+        if cache_key in self._filtered_bars_cache:
+            return self._filtered_bars_cache[cache_key]
+
+        # Check if we need to fetch data by calling _update_pandas_data first
+        # This will only fetch if data is not already cached or prefetched
+        self._update_pandas_data(asset, quote, length, timestep)
+
         # Check if we have data in pandas_data cache
         if search_asset in self.pandas_data:
             asset_data = self.pandas_data[search_asset]
             df = asset_data.df
-            
+
             if not df.empty:
                 # Apply timeshift if specified
-                current_dt = self.get_datetime()
                 shift_seconds = 0
                 if timeshift:
                     if isinstance(timeshift, int):
@@ -506,10 +621,10 @@ class DataBentoDataBacktesting(PandasData):
                     else:
                         shift_seconds = timeshift.total_seconds()
                         current_dt = current_dt - timeshift
-                
+
                 # Ensure current_dt is timezone-aware for comparison
                 current_dt_aware = to_datetime_aware(current_dt)
-                
+
                 # Step back one bar to avoid exposing the in-progress bar
                 bar_delta = timedelta(minutes=1)
                 if asset_data.timestep == "hour":
@@ -521,14 +636,16 @@ class DataBentoDataBacktesting(PandasData):
 
                 # Filter data up to current backtest time (exclude current bar unless broker overrides)
                 filtered_df = df[df.index <= cutoff_dt] if shift_seconds > 0 else df[df.index < current_dt_aware]
-                
+
                 # Take the last 'length' bars
                 result_df = filtered_df.tail(length)
-                
+
+                # OPTIMIZATION: Cache the result before returning
                 if not result_df.empty:
-                    # Return DataFrame directly like other data sources
+                    self._filtered_bars_cache[cache_key] = result_df
                     return result_df
                 else:
+                    self._filtered_bars_cache[cache_key] = None
                     return None
             else:
                 return None