lumibot 4.2.5__py3-none-any.whl → 4.2.9__py3-none-any.whl

lumibot/backtesting/databento_backtesting_pandas.py

@@ -410,6 +410,7 @@ class DataBentoDataBacktestingPandas(PandasData):
             # OPTIMIZATION: Check cache first
             self._check_and_clear_cache()
             current_dt = self.get_datetime()
+            current_dt_aware = to_datetime_aware(current_dt)
 
             # Try to get data from our cached pandas_data first
             search_asset = asset
@@ -435,8 +436,6 @@ class DataBentoDataBacktestingPandas(PandasData):
 
                 if not df.empty and 'close' in df.columns:
                         # Ensure current_dt is timezone-aware for comparison
-                        current_dt_aware = to_datetime_aware(current_dt)
-
                         # Step back one bar so only fully closed bars are visible
                         bar_delta = timedelta(minutes=1)
                         if asset_data.timestep == "hour":
@@ -454,19 +453,45 @@ class DataBentoDataBacktestingPandas(PandasData):
                             filtered_df = df[df.index <= current_dt_aware]
 
                         if not filtered_df.empty:
-                            last_price = filtered_df['close'].iloc[-1]
-                            if not pd.isna(last_price):
-                                price = float(last_price)
+                            valid_closes = filtered_df['close'].dropna()
+                            if not valid_closes.empty:
+                                price = float(valid_closes.iloc[-1])
                                 # OPTIMIZATION: Cache the result
                                 self._last_price_cache[cache_key] = price
                                 return price
             
-            # If no cached data, try to get recent data
+            # If no cached data, try to load it for the backtest window
+            try:
+                fetched_bars = self.get_historical_prices(
+                    asset_separated,
+                    length=1,
+                    quote=quote_asset,
+                    timestep="minute",
+                )
+                if fetched_bars is not None:
+                    asset_data = self.pandas_data.get(search_asset)
+                    if asset_data is not None:
+                        df = asset_data.df
+                        if not df.empty and 'close' in df.columns:
+                            valid_closes = df[df.index <= current_dt_aware]['close'].dropna()
+                            if not valid_closes.empty:
+                                price = float(valid_closes.iloc[-1])
+                                self._last_price_cache[cache_key] = price
+                                return price
+            except Exception as exc:
+                logger.debug(
+                    "Attempted to hydrate Databento cache for %s but hit error: %s",
+                    asset.symbol,
+                    exc,
+                )
+
+            # If still no data, fall back to direct fetch (live-style)
             logger.warning(f"No cached data for {asset.symbol}, attempting direct fetch")
             return databento_helper.get_last_price_from_databento(
                 api_key=self._api_key,
                 asset=asset_separated,
-                venue=exchange
+                venue=exchange,
+                reference_date=current_dt_aware
             )
             
         except DataBentoAuthenticationError as e:

lumibot/backtesting/thetadata_backtesting_pandas.py

@@ -771,7 +771,7 @@ class ThetaDataBacktestingPandas(PandasData):
                 quote_columns = ['bid', 'ask', 'bid_size', 'ask_size', 'bid_condition', 'ask_condition', 'bid_exchange', 'ask_exchange']
                 existing_quote_cols = [col for col in quote_columns if col in df.columns]
                 if existing_quote_cols:
-                    df[existing_quote_cols] = df[existing_quote_cols].fillna(method='ffill')
+                    df[existing_quote_cols] = df[existing_quote_cols].ffill()
 
                     # Log how much forward filling occurred
                     if 'bid' in df.columns and 'ask' in df.columns:

lumibot/components/options_helper.py

@@ -1,5 +1,7 @@
 from dataclasses import dataclass
 from datetime import date, datetime, timedelta
+from decimal import Decimal
+import math
 from typing import Any, Dict, List, Optional, Tuple, Union
 import warnings
 
@@ -23,6 +25,7 @@ class OptionMarketEvaluation:
     sell_price: Optional[float]
     used_last_price_fallback: bool
     max_spread_pct: Optional[float]
+    data_quality_flags: List[str]
 
 
 class OptionsHelper:
@@ -58,6 +61,54 @@ class OptionsHelper:
         self._liquidity_deprecation_warned = False
         self.strategy.log_message("OptionsHelper initialized.", color="blue")
 
+    @staticmethod
+    def _coerce_price(value: Any, field_name: str, flags: List[str], notes: List[str]) -> Optional[float]:
+        """Normalize quote values and record data quality issues."""
+        raw_value = value
+
+        if value is None:
+            flags.append(f"{field_name}_missing")
+            return None
+
+        try:
+            if isinstance(value, Decimal):
+                value = float(value)
+            else:
+                value = float(value)  # type: ignore[arg-type]
+        except (TypeError, ValueError):
+            flags.append(f"{field_name}_non_numeric")
+            notes.append(f"{field_name} value {raw_value!r} is non-numeric; dropping.")
+            return None
+
+        if math.isnan(value) or math.isinf(value):
+            flags.append(f"{field_name}_non_finite")
+            notes.append(f"{field_name} value {value!r} is not finite; dropping.")
+            return None
+
+        if value <= 0:
+            flags.append(f"{field_name}_non_positive")
+            notes.append(f"{field_name} value {value!r} is non-positive; dropping.")
+            return None
+
+        return value
+
+    @staticmethod
+    def has_actionable_price(evaluation: Optional["OptionMarketEvaluation"]) -> bool:
+        """Return True when the evaluation contains a usable buy price."""
+        if evaluation is None:
+            return False
+
+        price = evaluation.buy_price
+        if price is None:
+            return False
+
+        try:
+            price = float(price)
+        except (TypeError, ValueError):
+            return False
+
+        return math.isfinite(price) and price > 0 and not evaluation.spread_too_wide
+
     # ============================================================
     # Basic Utility Functions
     # ============================================================
@@ -467,6 +518,9 @@ class OptionsHelper:
         buy_price: Optional[float] = None
         sell_price: Optional[float] = None
 
+        data_quality_flags: List[str] = []
+        sanitization_notes: List[str] = []
+
         # Attempt to get quotes first
         quote = None
         try:
@@ -478,24 +532,20 @@ class OptionsHelper:
             )
 
         if quote and quote.bid is not None and quote.ask is not None:
-            try:
-                bid = float(quote.bid)
-                ask = float(quote.ask)
-            except (TypeError, ValueError):
-                bid = quote.bid
-                ask = quote.ask
+            bid = self._coerce_price(quote.bid, "bid", data_quality_flags, sanitization_notes)
+            ask = self._coerce_price(quote.ask, "ask", data_quality_flags, sanitization_notes)
             has_bid_ask = bid is not None and ask is not None
 
         if has_bid_ask and bid is not None and ask is not None:
             buy_price = ask
             sell_price = bid
-            mid = (ask + bid) / 2 if (ask is not None and bid is not None) else None
-            if mid and mid > 0:
+            mid = (ask + bid) / 2
+            if not math.isfinite(mid) or mid <= 0:
+                spread_pct = None
+            else:
                 spread_pct = (ask - bid) / mid
                 if max_spread_pct is not None:
                     spread_too_wide = spread_pct > max_spread_pct
-            else:
-                spread_pct = None
         else:
             missing_bid_ask = True
 
@@ -510,6 +560,10 @@ class OptionsHelper:
 
         if last_price is None:
             missing_last_price = True
+        else:
+            last_price = self._coerce_price(last_price, "last_price", data_quality_flags, sanitization_notes)
+            if last_price is None:
+                missing_last_price = True
 
         if not has_bid_ask and allow_fallback and last_price is not None:
             buy_price = last_price
@@ -519,6 +573,14 @@ class OptionsHelper:
                 f"Using last-price fallback for {option_asset} due to missing bid/ask quotes.",
                 color="yellow",
             )
+        elif not has_bid_ask and allow_fallback and last_price is None:
+            data_quality_flags.append("last_price_unusable")
+
+        if buy_price is not None and (not math.isfinite(buy_price) or buy_price <= 0):
+            sanitization_notes.append(f"buy_price {buy_price!r} is not actionable; clearing.")
+            data_quality_flags.append("buy_price_non_finite")
+            buy_price = None
+            sell_price = None
 
         # Compose log message
         spread_str = f"{spread_pct:.2%}" if spread_pct is not None else "None"
@@ -526,6 +588,12 @@ class OptionsHelper:
         log_color = "red" if spread_too_wide else (
             "yellow" if (missing_bid_ask or missing_last_price or used_last_price_fallback) else "blue"
         )
+        if sanitization_notes:
+            note_summary = "; ".join(sanitization_notes)
+            self.strategy.log_message(
+                f"Option data sanitization for {option_asset}: {note_summary}",
+                color="yellow",
+            )
         self.strategy.log_message(
             (
                 f"Option market evaluation for {option_asset}: "
@@ -533,7 +601,8 @@ class OptionsHelper:
                 f"max_spread={max_spread_str}, missing_bid_ask={missing_bid_ask}, "
                 f"missing_last_price={missing_last_price}, spread_too_wide={spread_too_wide}, "
                 f"used_last_price_fallback={used_last_price_fallback}, "
-                f"buy_price={buy_price}, sell_price={sell_price}"
+                f"buy_price={buy_price}, sell_price={sell_price}, "
+                f"data_quality_flags={data_quality_flags}"
             ),
             color=log_color,
         )
@@ -551,6 +620,7 @@ class OptionsHelper:
             sell_price=sell_price,
             used_last_price_fallback=used_last_price_fallback,
             max_spread_pct=max_spread_pct,
+            data_quality_flags=data_quality_flags,
         )
 
     def check_option_liquidity(self, option_asset: Asset, max_spread_pct: float) -> bool:
@@ -721,18 +791,11 @@ class OptionsHelper:
                     self.strategy.log_message(f"Cannot validate data without underlying symbol, returning {exp_date}", color="yellow")
                     return exp_date
 
-        # No future expirations with valid data; log and check last available
-        if expiration_dates:
-            # Check the last available expiry for data
-            for exp_str, exp_date in reversed(expiration_dates):
-                strikes = specific_chain.get(exp_str)
-                if strikes and len(strikes) > 0:
-                    self.strategy.log_message(
-                        f"No valid expirations on or after {dt}; using latest available {exp_date} for {call_or_put_caps}.",
-                        color="yellow",
-                    )
-                    return exp_date
-
+        # No future expirations with tradeable data; let the caller skip entries gracefully.
+        self.strategy.log_message(
+            f"No valid expirations on or after {dt} with tradeable data for {call_or_put_caps}; skipping.",
+            color="yellow",
+        )
         return None
 
     # ============================================================

lumibot/strategies/_strategy.py

@@ -126,14 +126,20 @@ class Vars:
 class _Strategy:
     @staticmethod
     def _normalize_backtest_datetime(value):
-        """Convert backtest boundary datetimes to the LumiBot default timezone."""
+        """Ensure backtest boundary datetimes are timezone-aware.
+
+        Naive datetimes are localized to the LumiBot default timezone; timezone-aware
+        inputs are returned unchanged so their original offsets are preserved.
+        """
         if value is None:
             return None
-        aware = to_datetime_aware(value)
-        tzinfo = getattr(aware, "tzinfo", None)
-        if tzinfo is not None and tzinfo != LUMIBOT_DEFAULT_PYTZ:
-            return aware.astimezone(LUMIBOT_DEFAULT_PYTZ)
-        return aware
+        if isinstance(value, datetime.datetime):
+            tzinfo = value.tzinfo
+            if tzinfo is None or tzinfo.utcoffset(value) is None:
+                return to_datetime_aware(value)
+            if not hasattr(tzinfo, "zone"):
+                return value.astimezone(LUMIBOT_DEFAULT_PYTZ)
+        return value
 
     @property
     def is_backtesting(self) -> bool:

lumibot/tools/ccxt_data_store.py

@@ -445,7 +445,7 @@ class CcxtCacheDB:
         if freq == "1d":
             dt_range = pd.date_range(start=df.index.min(), end=df.index.max(), freq="D")
         else:
-            dt_range = pd.date_range(start=df.index.min(), end=df.index.max(), freq="T")
+            dt_range = pd.date_range(start=df.index.min(), end=df.index.max(), freq="min")
 
         df_complete = df.reindex(dt_range).ffill()
         df_complete['missing'] = np.where(df_complete.index.isin(df.index), 0, 1)

lumibot/tools/databento_helper.py

@@ -947,6 +947,7 @@ def get_last_price_from_databento(
     api_key: str,
     asset: Asset,
     venue: Optional[str] = None,
+    reference_date: Optional[datetime] = None,
     **kwargs
 ) -> Optional[Union[float, Decimal]]:
     """
@@ -978,12 +979,14 @@ def get_last_price_from_databento(
         
         # For continuous futures, resolve to the current active contract
         if asset.asset_type == Asset.AssetType.CONT_FUTURE:
-            # Use Asset class method to resolve continuous futures to actual contract (returns string)
-            resolved_symbol = asset.resolve_continuous_futures_contract(year_digits=1)
+            # Resolve based on reference date when backtesting so we match the contract in use
+            resolved_symbol = _format_futures_symbol_for_databento(
+                asset,
+                reference_date=reference_date,
+            )
             if resolved_symbol is None:
                 logger.error(f"Could not resolve continuous futures contract for {asset.symbol}")
                 return None
-            # Generate the correct DataBento symbol format (should be single result)
             symbols_to_try = _generate_databento_symbol_alternatives(asset.symbol, resolved_symbol)
             logger.info(f"Resolved continuous future {asset.symbol} to specific contract: {resolved_symbol}")
             logger.info(f"DataBento symbol format for last price: {symbols_to_try[0]}")
@@ -1000,12 +1003,17 @@ def get_last_price_from_databento(
             if hasattr(range_result, 'end') and range_result.end:
                 if hasattr(range_result.end, 'tz_localize'):
                     # Already a pandas Timestamp
-                    available_end = range_result.end if range_result.end.tz else range_result.end.tz_localize('UTC')
+                    if range_result.end.tz is not None:
+                        available_end = range_result.end.tz_convert('UTC')
+                    else:
+                        available_end = range_result.end.tz_localize('UTC')
                 else:
                     # Convert to pandas Timestamp
-                    available_end = pd.to_datetime(range_result.end).tz_localize('UTC')
+                    ts = pd.to_datetime(range_result.end)
+                    available_end = ts if ts.tz is not None else ts.tz_localize('UTC')
             elif isinstance(range_result, dict) and 'end' in range_result:
-                available_end = pd.to_datetime(range_result['end']).tz_localize('UTC')
+                ts = pd.to_datetime(range_result['end'])
+                available_end = ts if ts.tz is not None else ts.tz_localize('UTC')
             else:
                 logger.warning(f"Could not parse dataset range for {dataset}: {range_result}")
                 # Fallback: use a recent date that's likely to have data
@@ -1047,10 +1055,10 @@ def get_last_price_from_databento(
                         df = pd.DataFrame(data)
                     
                     if not df.empty:
-                        # Get the last available price (close price of most recent bar)
                         if 'close' in df.columns:
-                            price = df['close'].iloc[-1]
-                            if pd.notna(price):
+                            closes = df['close'].dropna()
+                            if not closes.empty:
+                                price = closes.iloc[-1]
                                 logger.info(f"✓ SUCCESS: Got last price for {symbol_to_use}: {price}")
                                 return float(price)