lumibot 4.1.3__tar.gz → 4.2.0__tar.gz

{lumibot-4.1.3/lumibot.egg-info → lumibot-4.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lumibot
-Version: 4.1.3
+Version: 4.2.0
 Summary: Backtesting and Trading Library, Made by Lumiwealth
 Home-page: https://github.com/Lumiwealth/lumibot
 Author: Robert Grzesik
@@ -51,6 +51,7 @@ Requires-Dist: schwab-py>=1.5.0
 Requires-Dist: Flask>=2.3
 Requires-Dist: free-proxy
 Requires-Dist: requests-oauthlib
+Requires-Dist: boto3>=1.28.0
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier
@@ -146,6 +147,13 @@ To run an individual test file, you can run the following command:
 pytest tests/test_asset.py
 ```
 
+## Remote Cache Configuration
+
+Lumibot can mirror its local parquet caches to AWS S3 when you enable the new
+backtest cache manager. The feature is optional and defaults to local storage.
+To configure the environment variables, understand the key naming convention,
+and follow the manual validation checklist, review `docs/remote_cache.md`.
+
 ### Showing Code Coverage
 
 To show code coverage, you can run the following command:

{lumibot-4.1.3 → lumibot-4.2.0}/README.md

@@ -81,6 +81,13 @@ To run an individual test file, you can run the following command:
 pytest tests/test_asset.py
 ```
 
+## Remote Cache Configuration
+
+Lumibot can mirror its local parquet caches to AWS S3 when you enable the new
+backtest cache manager. The feature is optional and defaults to local storage.
+To configure the environment variables, understand the key naming convention,
+and follow the manual validation checklist, review `docs/remote_cache.md`.
+
 ### Showing Code Coverage
 
 To show code coverage, you can run the following command:

lumibot-4.2.0/lumibot/backtesting/__init__.py

@@ -0,0 +1,30 @@
+from .alpaca_backtesting import AlpacaBacktesting
+from .alpha_vantage_backtesting import AlphaVantageBacktesting
+from .backtesting_broker import BacktestingBroker
+from .ccxt_backtesting import CcxtBacktesting
+from .interactive_brokers_rest_backtesting import InteractiveBrokersRESTBacktesting
+from .pandas_backtesting import PandasDataBacktesting
+from .polygon_backtesting import PolygonDataBacktesting
+from .thetadata_backtesting import ThetaDataBacktesting
+from .thetadata_backtesting_pandas import ThetaDataBacktestingPandas
+from .yahoo_backtesting import YahooDataBacktesting
+
+from .databento_backtesting import DataBentoDataBacktesting
+from .databento_backtesting_pandas import DataBentoDataBacktestingPandas
+from .databento_backtesting_polars import DataBentoDataBacktestingPolars
+
+__all__ = [
+    "AlpacaBacktesting",
+    "AlphaVantageBacktesting",
+    "BacktestingBroker",
+    "CcxtBacktesting",
+    "InteractiveBrokersRESTBacktesting",
+    "PandasDataBacktesting",
+    "PolygonDataBacktesting",
+    "ThetaDataBacktesting",
+    "ThetaDataBacktestingPandas",
+    "YahooDataBacktesting",
+    "DataBentoDataBacktesting",
+    "DataBentoDataBacktestingPandas",
+    "DataBentoDataBacktestingPolars",
+]

{lumibot-4.1.3 → lumibot-4.2.0}/lumibot/backtesting/backtesting_broker.py

@@ -1,3 +1,4 @@
+import math
 import traceback
 import threading
 from collections import OrderedDict
@@ -1353,17 +1354,21 @@ class BacktestingBroker(Broker):
 
             # Get the OHLCV data for the asset if we're using the YAHOO, CCXT data source
             data_source_name = self.data_source.SOURCE.upper()
-            if data_source_name in ["CCXT", "YAHOO", "ALPACA", "DATABENTO"]:
-                # Default to backing up one minute so fills use the next bar, consistent with other sources.
+            if data_source_name in ["CCXT", "YAHOO", "ALPACA", "DATABENTO", "DATABENTO_POLARS"]:
+                # Negative deltas here are intentional: _pull_source_symbol_bars subtracts the offset, so
+                # passing -1 minute yields an effective +1 minute guard that keeps us on the previously
+                # completed bar. See tests/*_lookahead for regression coverage.
                 timeshift = timedelta(minutes=-1)
-                if data_source_name == "DATABENTO":
-                    # DataBento mimics Polygon by requesting two bars to guard against gaps.
+                if data_source_name in {"DATABENTO", "DATABENTO_POLARS"}:
+                    # DataBento feeds can skip minutes around maintenance windows. Giving it a two-minute
+                    # cushion mirrors the legacy Polygon behaviour and avoids falling through gaps.
                     timeshift = timedelta(minutes=-2)
                 elif data_source_name == "YAHOO":
-                    # Yahoo uses day bars; shift one day instead to mirror legacy behavior.
+                    # Yahoo daily bars are stamped at the close (16:00). A one-day backstep keeps fills on
+                    # the previous session so we never peek at the in-progress bar.
                     timeshift = timedelta(days=-1)
                 elif data_source_name == "ALPACA":
-                    # Alpaca minute bars are aligned to the current iteration already.
+                    # Alpaca minute bars line up with our clock already; no offset needed.
                     timeshift = None
 
                 ohlc = self.data_source.get_historical_prices(
@@ -1373,6 +1378,23 @@ class BacktestingBroker(Broker):
                     timeshift=timeshift,
                 )
 
+                if (
+                    ohlc is None
+                    or getattr(ohlc, "df", None) is None
+                    or (hasattr(ohlc.df, "empty") and ohlc.df.empty)
+                ):
+                    if strategy is not None:
+                        display_symbol = getattr(order.asset, "symbol", order.asset)
+                        order_identifier = getattr(order, "identifier", None)
+                        if order_identifier is None:
+                            order_identifier = getattr(order, "id", "<unknown>")
+                        strategy.log_message(
+                            f"[DIAG] No historical bars returned for {display_symbol} at {self.datetime}; "
+                            f"pending {order.order_type} id={order_identifier}",
+                            color="yellow",
+                        )
+                    continue
+
                 # Handle both pandas and polars DataFrames
                 if hasattr(ohlc.df, 'index'):  # pandas
                     dt = ohlc.df.index[-1]
@@ -1406,6 +1428,16 @@ class BacktestingBroker(Broker):
                 )
                 # Check if we got any ohlc data
                 if ohlc is None or ohlc.empty:
+                    if strategy is not None:
+                        display_symbol = getattr(order.asset, "symbol", order.asset)
+                        order_identifier = getattr(order, "identifier", None)
+                        if order_identifier is None:
+                            order_identifier = getattr(order, "id", "<unknown>")
+                        strategy.log_message(
+                            f"[DIAG] No pandas bars for {display_symbol} at {self.datetime}; "
+                            f"canceling {order.order_type} id={order_identifier}",
+                            color="yellow",
+                        )
                     self.cancel_order(order)
                     continue
 
@@ -1502,41 +1534,89 @@ class BacktestingBroker(Broker):
                     strategy=strategy,
                 )
             else:
+                if strategy is not None:
+                    display_symbol = getattr(order.asset, "symbol", order.asset)
+                    order_identifier = getattr(order, "identifier", None)
+                    if order_identifier is None:
+                        order_identifier = getattr(order, "id", "<unknown>")
+                    detail = (
+                        f"limit={order.limit_price}, high={high}, low={low}"
+                        if order.order_type == Order.OrderType.LIMIT
+                        else f"type={order.order_type}, high={high}, low={low}, stop={getattr(order, 'stop_price', None)}"
+                    )
+                    strategy.log_message(
+                        f"[DIAG] Order remained open for {display_symbol} ({detail}) "
+                        f"id={order_identifier} at {self.datetime}",
+                        color="yellow",
+                    )
                 continue
 
         # After handling all pending orders, cash settle any residual expired contracts.
         self.process_expired_option_contracts(strategy)
 
+    def _coerce_price(self, value):
+        """Convert numeric inputs to float when possible for safe comparisons."""
+        if value is None:
+            return None
+        try:
+            return float(value)
+        except (TypeError, ValueError):
+            return value
+
+    def _is_invalid_price(self, value):
+        """Determine whether a price is unusable (None or NaN)."""
+        if value is None:
+            return True
+        if isinstance(value, float) and math.isnan(value):
+            return True
+        return False
+
     def limit_order(self, limit_price, side, open_, high, low):
         """Limit order logic."""
+        open_val = self._coerce_price(open_)
+        high_val = self._coerce_price(high)
+        low_val = self._coerce_price(low)
+        limit_val = self._coerce_price(limit_price)
+
+        if any(self._is_invalid_price(val) for val in (open_val, high_val, low_val, limit_val)):
+            return None
+
         # Gap Up case: Limit wasn't triggered by previous candle but current candle opens higher, fill it now
-        if side == "sell" and limit_price <= open_:
-            return open_
+        if side == "sell" and limit_val <= open_val:
+            return open_val
 
         # Gap Down case: Limit wasn't triggered by previous candle but current candle opens lower, fill it now
-        if side == "buy" and limit_price >= open_:
-            return open_
+        if side == "buy" and limit_val >= open_val:
+            return open_val
 
         # Current candle triggered limit normally
-        if low <= limit_price <= high:
-            return limit_price
+        if low_val <= limit_val <= high_val:
+            return limit_val
 
         # Limit has not been met
         return None
 
     def stop_order(self, stop_price, side, open_, high, low):
         """Stop order logic."""
+        open_val = self._coerce_price(open_)
+        high_val = self._coerce_price(high)
+        low_val = self._coerce_price(low)
+        stop_val = self._coerce_price(stop_price)
+
+        if any(self._is_invalid_price(val) for val in (open_val, high_val, low_val, stop_val)):
+            return None
+
         # Gap Down case: Stop wasn't triggered by previous candle but current candle opens lower, fill it now
-        if side == "sell" and stop_price >= open_:
-            return open_
+        if side == "sell" and stop_val >= open_val:
+            return open_val
 
         # Gap Up case: Stop wasn't triggered by previous candle but current candle opens higher, fill it now
-        if side == "buy" and stop_price <= open_:
-            return open_
+        if side == "buy" and stop_val <= open_val:
+            return open_val
 
         # Current candle triggered stop normally
-        if low <= stop_price <= high:
-            return stop_price
+        if low_val <= stop_val <= high_val:
+            return stop_val
 
         # Stop has not been met
         return None

lumibot-4.2.0/lumibot/backtesting/databento_backtesting.py

@@ -0,0 +1,7 @@
+"""Canonical DataBento backtesting aliasing the Polars implementation."""
+
+from .databento_backtesting_polars import DataBentoDataBacktestingPolars as DataBentoDataBacktesting
+from .databento_backtesting_pandas import DataBentoDataBacktestingPandas
+from .databento_backtesting_polars import DataBentoDataBacktestingPolars
+
+__all__ = ["DataBentoDataBacktesting", "DataBentoDataBacktestingPandas", "DataBentoDataBacktestingPolars"]

lumibot-4.1.3/lumibot/backtesting/databento_backtesting.py → lumibot-4.2.0/lumibot/backtesting/databento_backtesting_pandas.py

@@ -7,7 +7,9 @@ from lumibot import LUMIBOT_DEFAULT_PYTZ
 from lumibot.data_sources import PandasData
 from lumibot.entities import Asset, Data
 from lumibot.tools import databento_helper
+from lumibot.tools.databento_helper import DataBentoAuthenticationError
 from lumibot.tools.helpers import to_datetime_aware
+from termcolor import colored
 
 from lumibot.tools.lumibot_logger import get_logger
 logger = get_logger(__name__)
@@ -15,7 +17,7 @@ logger = get_logger(__name__)
 START_BUFFER = timedelta(days=5)
 
 
-class DataBentoDataBacktesting(PandasData):
+class DataBentoDataBacktestingPandas(PandasData):
     """
     Backtesting implementation of DataBento data source
     
@@ -85,7 +87,7 @@ class DataBentoDataBacktesting(PandasData):
             logger.error("DataBento package not available. Please install with: pip install databento")
             raise ImportError("DataBento package not available")
 
-        logger.info(f"DataBento backtesting initialized for period: {datetime_start} to {datetime_end}")
+        logger.debug(f"DataBento backtesting initialized for period: {datetime_start} to {datetime_end}")
 
     def _check_and_clear_cache(self):
         """
@@ -157,8 +159,11 @@ class DataBentoDataBacktesting(PandasData):
                 reference_date=self.datetime_start
             )
 
-            logger.info(f"Successfully set multiplier for {asset.symbol}: {asset.multiplier}")
+            logger.debug(f"Successfully set multiplier for {asset.symbol}: {asset.multiplier}")
 
+        except DataBentoAuthenticationError as e:
+            logger.error(colored(f"DataBento authentication failed while fetching multiplier for {asset.symbol}: {e}", "red"))
+            raise
         except Exception as e:
             logger.warning(f"Could not fetch multiplier for {asset.symbol}: {e}")
 
@@ -177,7 +182,7 @@ class DataBentoDataBacktesting(PandasData):
         if not assets:
             return
             
-        logger.info(f"Prefetching DataBento data for {len(assets)} assets...")
+        logger.debug(f"Prefetching DataBento data for {len(assets)} assets...")
         
         for asset in assets:
             # Create search key for the asset
@@ -193,7 +198,7 @@ class DataBentoDataBacktesting(PandasData):
                 start_datetime = self.datetime_start - START_BUFFER
                 end_datetime = self.datetime_end + timedelta(days=1)
                 
-                logger.info(f"Fetching {asset.symbol} data from {start_datetime.date()} to {end_datetime.date()}")
+                logger.debug(f"Fetching {asset.symbol} data from {start_datetime.date()} to {end_datetime.date()}")
                 
                 # Get data from DataBento for entire period
                 df = databento_helper.get_price_data_from_databento(
@@ -231,11 +236,14 @@ class DataBentoDataBacktesting(PandasData):
                         quote=quote_asset,
                     )
                     self.pandas_data[search_asset] = data_obj
-                    logger.info(f"Cached {len(df)} rows for {asset.symbol}")
+                    logger.debug(f"Cached {len(df)} rows for {asset.symbol}")
                 
                 # Mark as prefetched
                 self._prefetched_assets.add(search_asset)
                 
+            except DataBentoAuthenticationError as e:
+                logger.error(colored(f"DataBento authentication failed while prefetching {asset.symbol}: {e}", "red"))
+                raise
             except Exception as e:
                 logger.error(f"Error prefetching data for {asset.symbol}: {str(e)}")
                 logger.error(traceback.format_exc())
@@ -317,7 +325,7 @@ class DataBentoDataBacktesting(PandasData):
         try:
             # Only log fetch message once per asset/timestep combination
             if log_key not in self._logged_requests:
-                logger.info(f"Fetching {timestep} data for {asset_separated.symbol}")
+                logger.debug(f"Fetching {timestep} data for {asset_separated.symbol}")
                 self._logged_requests.add(log_key)
             
             # Get the start datetime and timestep unit
@@ -373,6 +381,9 @@ class DataBentoDataBacktesting(PandasData):
             
             self.pandas_data[search_asset] = data_obj
 
+        except DataBentoAuthenticationError as e:
+            logger.error(colored(f"DataBento authentication failed for {asset_separated.symbol}: {e}", "red"))
+            raise
         except Exception as e:
             logger.error(f"Error updating pandas data for {asset_separated.symbol}: {str(e)}")
             logger.error(traceback.format_exc())
@@ -458,6 +469,9 @@ class DataBentoDataBacktesting(PandasData):
                 venue=exchange
             )
             
+        except DataBentoAuthenticationError as e:
+            logger.error(colored(f"DataBento authentication failed while getting last price for {asset.symbol}: {e}", "red"))
+            raise
         except Exception as e:
             logger.error(f"Error getting last price for {asset.symbol}: {e}")
             return None
@@ -555,6 +569,9 @@ class DataBentoDataBacktesting(PandasData):
                     logger.warning(f"No data found for {asset.symbol}")
                     result[asset] = None
                     
+            except DataBentoAuthenticationError as e:
+                logger.error(colored(f"DataBento authentication failed while getting bars for {asset}: {e}", "red"))
+                raise
             except Exception as e:
                 logger.error(f"Error getting bars for {asset}: {e}")
                 result[asset] = None
@@ -616,7 +633,28 @@ class DataBentoDataBacktesting(PandasData):
             df = asset_data.df
 
             if not df.empty:
-                # Apply timeshift if specified
+                # ========================================================================
+                # CRITICAL: NEGATIVE TIMESHIFT ARITHMETIC FOR LOOKAHEAD
+                # ========================================================================
+                # Negative timeshift allows broker to "peek ahead" for realistic fills.
+                #
+                # Example with timeshift=-2 at broker_dt=09:30:
+                #   - Arithmetic: current_dt - timeshift
+                #                = 09:30 - timedelta(minutes=-2)
+                #                = 09:30 - (-2 minutes)
+                #                = 09:30 + 2 minutes
+                #                = 09:32
+                #   - Data source returns bars up to 09:32: [..., 09:29, 09:30, 09:31, 09:32]
+                #   - Broker filters to future bars (>= 09:30): [09:30, 09:31, 09:32]
+                #   - Broker uses FIRST future bar (09:31) and its OPEN price for fills
+                #
+                # Why this is necessary:
+                #   - Real world: Order placed at 09:30:30 fills at 09:31:00 open
+                #   - Backtesting: Broker at 09:30 needs to see 09:31 bar for realistic fills
+                #
+                # DO NOT change this arithmetic! "current_dt - timeshift" with negative
+                # timeshift is CORRECT and INTENTIONAL.
+                # ========================================================================
                 shift_seconds = 0
                 if timeshift:
                     if isinstance(timeshift, int):
@@ -638,9 +676,21 @@ class DataBentoDataBacktesting(PandasData):
 
                 cutoff_dt = current_dt_aware - bar_delta
 
+                # INSTRUMENTATION: Log timeshift application and filtering
+                broker_dt_orig = self.get_datetime()
+                filter_branch = "shift_seconds > 0" if shift_seconds > 0 else "shift_seconds <= 0"
+
                 # Filter data up to current backtest time (exclude current bar unless broker overrides)
                 filtered_df = df[df.index <= cutoff_dt] if shift_seconds > 0 else df[df.index < current_dt_aware]
 
+                # Log what bar we're returning
+                if not filtered_df.empty:
+                    returned_bar_dt = filtered_df.index[-1]
+                    logger.debug(f"[TIMESHIFT_PANDAS] asset={asset_separated.symbol} broker_dt={broker_dt_orig} "
+                               f"timeshift={timeshift} shift_seconds={shift_seconds} "
+                               f"shifted_dt={current_dt_aware} cutoff_dt={cutoff_dt} "
+                               f"filter={filter_branch} returned_bar={returned_bar_dt}")
+
                 # Take the last 'length' bars
                 result_df = filtered_df.tail(length)
 
@@ -685,4 +735,4 @@ class DataBentoDataBacktesting(PandasData):
         # Prefetch data for all assets
         self.prefetch_data(assets, timestep)
         
-        logger.info(f"Initialized DataBento backtesting with prefetched data for {len(assets)} assets")
+        logger.debug(f"Initialized DataBento backtesting with prefetched data for {len(assets)} assets")