lumibot 4.1.3__py3-none-any.whl → 4.2.0__py3-none-any.whl

lumibot/brokers/alpaca.py

@@ -8,7 +8,7 @@ from decimal import Decimal
 
 import pandas_market_calendars as mcal
 from alpaca.trading.client import TradingClient
-from alpaca.trading.enums import QueryOrderStatus
+from alpaca.trading.enums import QueryOrderStatus, PositionSide
 from alpaca.trading.requests import GetOrdersRequest, ReplaceOrderRequest
 from alpaca.trading.stream import TradingStream
 from dateutil import tz
@@ -404,6 +404,13 @@ class Alpaca(Broker):
             avg_fill_price = None
 
         position = Position(strategy, asset, quantity, orders=orders, avg_fill_price=avg_fill_price)
+
+        position.pnl = float(broker_position.unrealized_pl) if broker_position.unrealized_pl else None
+        position.current_price = float(broker_position.current_price) if broker_position.current_price else None
+        position.side = Position.PositionSide.LONG if broker_position.side == PositionSide.LONG else Position.PositionSide.SHORT
+        position.market_value = float(broker_position.market_value) if broker_position.market_value else None
+        
+
         return position
 
     def _pull_broker_position(self, asset):

lumibot/brokers/schwab.py

@@ -87,7 +87,7 @@ class Schwab(Broker):
         # Initialize Schwab specific attributes
         self._subscribers = []
         # Use standard logging module's logger
-        self.logger = get_logger(__name__)
+        # self.logger = get_logger(__name__)
         self.extended_trading_minutes = 0
         # self.schwab_authorization_error = False # Moved earlier
         self.client = None
@@ -526,6 +526,8 @@ class Schwab(Broker):
 
                 # Extract position-specific details
                 average_price = schwab_position.get('averagePrice', 0.0)
+                pnl = schwab_position.get('longOpenProfitLoss') or schwab_position.get('shortOpenProfitLoss') or None
+                market_value = schwab_position.get('marketValue', None)
 
                 # Only create position object if we have a valid asset
                 if asset is not None:
@@ -537,7 +539,12 @@ class Schwab(Broker):
 
                     # If we already have this asset in our dict, update the quantity
                     if key in pos_dict:
-                        pos_dict[key].quantity += net_quantity
+                        existing_position = pos_dict[key]
+                        existing_position.quantity += net_quantity
+                        if pnl is not None:
+                           existing_position.pnl += pnl
+                        if market_value is not None:
+                            existing_position.market_value += market_value
                     else:
                         # Create a new Position object
                         pos_dict[key] = Position(
@@ -546,6 +553,9 @@ class Schwab(Broker):
                             quantity=net_quantity,
                             avg_fill_price=average_price,
                         )
+                        
+                        pos_dict[key].pnl = pnl
+                        pos_dict[key].market_value = market_value
 
             # Log the number of positions found
             logger.debug(f"Pulled {len(pos_dict)} unique positions from Schwab")

lumibot/credentials.py

@@ -182,6 +182,19 @@ DATABENTO_CONFIG = {
     "MAX_RETRIES": int(os.environ.get("DATABENTO_MAX_RETRIES", "3")),
 }
 
+# Remote cache configuration (disabled by default)
+CACHE_REMOTE_CONFIG = {
+    "backend": os.environ.get("LUMIBOT_CACHE_BACKEND", "local"),
+    "mode": os.environ.get("LUMIBOT_CACHE_MODE", "disabled"),
+    "s3_bucket": os.environ.get("LUMIBOT_CACHE_S3_BUCKET"),
+    "s3_prefix": os.environ.get("LUMIBOT_CACHE_S3_PREFIX", ""),
+    "s3_region": os.environ.get("LUMIBOT_CACHE_S3_REGION"),
+    "s3_access_key_id": os.environ.get("LUMIBOT_CACHE_S3_ACCESS_KEY_ID"),
+    "s3_secret_access_key": os.environ.get("LUMIBOT_CACHE_S3_SECRET_ACCESS_KEY"),
+    "s3_session_token": os.environ.get("LUMIBOT_CACHE_S3_SESSION_TOKEN"),
+    "s3_version": os.environ.get("LUMIBOT_CACHE_S3_VERSION", "v1"),
+}
+
 # Alpaca Configuration
 ALPACA_CONFIG = {
     # Add ALPACA_API_KEY, ALPACA_API_SECRET, ALPACA_OAUTH_TOKEN, and ALPACA_IS_PAPER to your .env file or set them as secrets

lumibot/data_sources/__init__.py

@@ -6,20 +6,17 @@ from .data_source_backtesting import DataSourceBacktesting
 from .exceptions import NoDataFound, UnavailabeTimestep
 from .interactive_brokers_data import InteractiveBrokersData
 from .pandas_data import PandasData
+from .polars_data import PolarsData
 from .tradier_data import TradierData
-
-from .yahoo_data_polars import YahooDataPolars as YahooData
-
-
-from .polygon_data_polars import PolygonDataPolars as PolygonDataBacktesting
-
 from .bitunix_data import BitunixData
 from .ccxt_backtesting_data import CcxtBacktestingData
 from .example_broker_data import ExampleBrokerData
 from .interactive_brokers_rest_data import InteractiveBrokersRESTData
 from .schwab_data import SchwabData
 from .tradovate_data import TradovateData
+from .yahoo_data import YahooData
 
-from .databento_data_polars_live import DataBentoDataPolarsLive as DataBentoData
-from .databento_data_polars_backtesting import DataBentoDataPolarsBacktesting as DataBentoDataBacktesting
+from .databento_data import DataBentoData, DataBentoDataPandas, DataBentoDataPolars
 from .projectx_data import ProjectXData
+
+from ..backtesting.polygon_backtesting import PolygonDataBacktesting

lumibot/data_sources/data_source.py

@@ -154,12 +154,16 @@ class DataSource(ABC):
         include_after_hours : bool
             Whether to include after hours data.
         return_polars : bool
-            If True, return Bars with Polars DataFrame for better performance. Default is False (returns pandas).
+            If True, returns Polars DataFrame via bars.df (2-3x faster for indicator calculations).
+            All data sources support this parameter. The Bars class automatically converts
+            pandas→polars when needed. Default is False for backward compatibility (returns pandas).
 
         Returns
         -------
         Bars
-            The bars for the asset.
+            The bars for the asset. Access via bars.df which returns:
+            - Polars DataFrame if return_polars=True (recommended for performance)
+            - Pandas DataFrame if return_polars=False (default, backward compatible)
         """
         pass
 

lumibot/data_sources/data_source_backtesting.py

@@ -77,6 +77,36 @@ class DataSourceBacktesting(DataSource, ABC):
         self._last_logging_time = None
         self._portfolio_value = None
 
+    @staticmethod
+    def estimate_requested_length(length=None, start_date=None, end_date=None, timestep="minute"):
+        """
+        Infer the number of rows required to satisfy a backtest data request.
+        """
+        if length is not None:
+            try:
+                return max(int(length), 1)
+            except (TypeError, ValueError):
+                pass
+
+        if start_date is None or end_date is None:
+            return 1
+
+        try:
+            td, unit = DataSource.convert_timestep_str_to_timedelta(timestep)
+        except Exception:
+            return 1
+
+        if end_date < start_date:
+            start_date, end_date = end_date, start_date
+
+        if unit == "day":
+            delta_days = (end_date.date() - start_date.date()).days
+            return max(delta_days + 1, 1)
+
+        interval_seconds = max(td.total_seconds(), 1)
+        total_seconds = max((end_date - start_date).total_seconds(), 0)
+        return max(int(total_seconds // interval_seconds) + 1, 1)
+
     def get_datetime(self, adjust_for_delay=False):
         """
         Get the current datetime of the backtest.

lumibot/data_sources/databento_data.py

@@ -1,392 +1,7 @@
-from datetime import datetime, timedelta
-from decimal import Decimal
-from typing import Union, Optional
+"""Canonical DataBento data source aliasing the Polars implementation."""
 
-from lumibot.data_sources import DataSource
-from lumibot.entities import Asset, Bars, Quote
-from lumibot.tools import databento_helper
-from lumibot.tools.lumibot_logger import get_logger
+from .databento_data_polars import DataBentoDataPolars as DataBentoData
+from .databento_data_pandas import DataBentoDataPandas
+from .databento_data_polars import DataBentoDataPolars
 
-try:
-    from .databento_data_polars_live import DataBentoDataPolarsLive
-except Exception:  # pragma: no cover - optional dependency path
-    DataBentoDataPolarsLive = None
-
-logger = get_logger(__name__)
-
-
-class DataBentoData(DataSource):
-    """
-    DataBento data source for historical market data
-    
-    This data source provides access to DataBento's institutional-grade market data,
-    with a focus on futures data and support for multiple asset types.
-    """
-
-    SOURCE = "DATABENTO"
-    MIN_TIMESTEP = "minute"
-    TIMESTEP_MAPPING = [
-        {"timestep": "minute", "representations": ["1m", "minute", "1 minute"]},
-        {"timestep": "hour", "representations": ["1h", "hour", "1 hour"]},
-        {"timestep": "day", "representations": ["1d", "day", "1 day"]},
-    ]
-
-    def __init__(
-        self,
-        api_key: str,
-        timeout: int = 30,
-        max_retries: int = 3,
-        **kwargs
-    ):
-        """
-        Initialize DataBento data source
-        
-        Parameters
-        ----------
-        api_key : str
-            DataBento API key
-        timeout : int, optional
-            API request timeout in seconds, default 30
-        max_retries : int, optional
-            Maximum number of API retry attempts, default 3
-        **kwargs
-            Additional parameters passed to parent class
-        """
-        # Initialize parent class
-        super().__init__(api_key=api_key, **kwargs)
-
-        self.name = "databento"
-        self._api_key = api_key
-        self._timeout = timeout
-        self._max_retries = max_retries
-        self._data_store = {}
-        self._live_delegate = None
-
-        # For live trading, this is a live data source
-        self.is_backtesting_mode = False
-
-        # Verify DataBento availability
-        if not databento_helper.DATABENTO_AVAILABLE:
-            logger.error("DataBento package not available. Please install with: pip install databento")
-            raise ImportError("DataBento package not available")
-
-    def get_historical_prices(
-        self,
-        asset: Asset,
-        length: int,
-        timestep: str = "minute",
-        timeshift: timedelta = None,
-        quote: Asset = None,
-        exchange: str = None,
-        include_after_hours: bool = True
-    ) -> Bars:
-        """
-        Get historical price data for an asset
-        
-        Parameters
-        ----------
-        asset : Asset
-            The asset to get historical prices for
-        length : int
-            Number of bars to retrieve
-        timestep : str, optional
-            Timestep for the data ('minute', 'hour', 'day'), default 'minute'
-        timeshift : timedelta, optional
-            Time shift to apply to the data retrieval
-        quote : Asset, optional
-            Quote asset (not used for DataBento)
-        exchange : str, optional
-            Exchange/venue filter
-        include_after_hours : bool, optional
-            Whether to include after-hours data, default True
-            
-        Returns
-        -------
-        Bars
-            Historical price data as Bars object
-        """
-        logger.info(f"Getting historical prices for {asset.symbol}, length={length}, timestep={timestep}")
-
-        # Validate asset type - DataBento primarily supports futures
-        supported_asset_types = [Asset.AssetType.FUTURE, Asset.AssetType.CONT_FUTURE]
-        if asset.asset_type not in supported_asset_types:
-            error_msg = f"DataBento data source only supports futures assets. Received asset type '{asset.asset_type}' for symbol '{asset.symbol}'. Supported types: {[t.value for t in supported_asset_types]}"
-            logger.error(error_msg)
-            raise ValueError(error_msg)
-
-        # Additional logging for debugging
-        logger.info(f"DataBento request - Asset: {asset.symbol}, Type: {asset.asset_type}, Length: {length}, Timestep: {timestep}")
-        logger.info(f"DataBento live trading mode: Requesting data for futures asset {asset.symbol}")
-
-        # Calculate the date range for data retrieval
-        # Use timezone-naive datetime for consistency
-        current_dt = datetime.now()
-        if current_dt.tzinfo is not None:
-            current_dt = current_dt.replace(tzinfo=None)
-
-        logger.info(f"Using current datetime for live trading: {current_dt}")
-
-        # Apply timeshift if specified
-        if timeshift:
-            current_dt = current_dt - timeshift
-
-        # Calculate start date based on length and timestep
-        if timestep == "day":
-            buffer_days = max(10, length // 2)  # Buffer for live trading
-            start_dt = current_dt - timedelta(days=length + buffer_days)
-            # For live trading, end should be current time (no future data available)
-            end_dt = current_dt
-        elif timestep == "hour":
-            buffer_hours = max(24, length // 2)  # Buffer for live trading
-            start_dt = current_dt - timedelta(hours=length + buffer_hours)
-            # For live trading, end should be current time (no future data available)
-            end_dt = current_dt
-        else:  # minute or other
-            buffer_minutes = max(1440, length)  # Buffer for live trading
-            start_dt = current_dt - timedelta(minutes=length + buffer_minutes)
-            # For live trading, end should be current time (no future data available)
-            end_dt = current_dt
-
-        # Ensure both dates are timezone-naive for consistency
-        if start_dt.tzinfo is not None:
-            start_dt = start_dt.replace(tzinfo=None)
-        if end_dt.tzinfo is not None:
-            end_dt = end_dt.replace(tzinfo=None)
-
-        # Ensure we always have a valid date range (start < end)
-        if start_dt >= end_dt:
-            # If dates are equal or start is after end, adjust end date
-            if timestep == "day":
-                end_dt = start_dt + timedelta(days=max(1, length))
-            elif timestep == "hour":
-                end_dt = start_dt + timedelta(hours=max(1, length))
-            else:  # minute or other
-                end_dt = start_dt + timedelta(minutes=max(1, length))
-
-        # Final safety check: ensure end is always after start
-        if start_dt >= end_dt:
-            logger.error(f"Invalid date range after adjustment: start={start_dt}, end={end_dt}")
-            if timestep == "day":
-                end_dt = start_dt + timedelta(days=1)
-            elif timestep == "hour":
-                end_dt = start_dt + timedelta(hours=1)
-            else:
-                end_dt = start_dt + timedelta(minutes=1)
-
-        # Get data from DataBento
-        logger.info(f"Requesting DataBento data for asset: {asset} (type: {asset.asset_type})")
-        logger.info(f"Date range: {start_dt} to {end_dt}")
-
-        try:
-            df = databento_helper.get_price_data_from_databento(
-                api_key=self._api_key,
-                asset=asset,
-                start=start_dt,
-                end=end_dt,
-                timestep=timestep,
-                venue=exchange
-            )
-        except Exception as e:
-            logger.error(f"Error getting data from DataBento for {asset.symbol}: {e}")
-            return None
-
-        if df is None or df.empty:
-            logger.error(f"No data returned from DataBento for {asset.symbol}. This could be due to:")
-            logger.error("1. Incorrect symbol format")
-            logger.error("2. Wrong dataset selection")
-            logger.error("3. Data not available for the requested time range")
-            logger.error("4. API authentication issues")
-            return None
-
-        # Filter data to the current time (for live trading)
-        # Handle timezone consistency for comparison
-        if hasattr(df.index, 'tz') and df.index.tz is not None:
-            # DataFrame has timezone-aware index, convert current_dt to match
-            if current_dt.tzinfo is None:
-                import pytz
-                current_dt = current_dt.replace(tzinfo=pytz.UTC)
-        else:
-            # DataFrame has timezone-naive index, ensure current_dt is also naive
-            if current_dt.tzinfo is not None:
-                current_dt = current_dt.replace(tzinfo=None)
-
-        df_filtered = df[df.index <= current_dt]
-
-        # Take the last 'length' bars
-        df_result = df_filtered.tail(length)
-
-        if df_result.empty:
-            logger.warning(f"No data available for {asset.symbol} up to {current_dt}")
-            return None
-
-        # Create and return Bars object
-        bars = Bars(
-            df=df_result,
-            source=self.SOURCE,
-            asset=asset,
-            quote=quote
-        )
-
-        logger.info(f"Retrieved {len(df_result)} bars for {asset.symbol}")
-        return bars
-
-    def get_last_price(
-        self,
-        asset: Asset,
-        quote: Asset = None,
-        exchange: str = None
-    ) -> Union[float, Decimal, None]:
-        """
-        Get the last known price for an asset
-        
-        Parameters
-        ----------
-        asset : Asset
-            The asset to get the last price for
-        quote : Asset, optional
-            Quote asset (not used for DataBento)
-        exchange : str, optional
-            Exchange/venue filter
-            
-        Returns
-        -------
-        float, Decimal, or None
-            Last known price of the asset
-        """
-        logger.info(f"Getting last price for {asset.symbol}")
-
-        # Prefer live delegate when available
-        delegate = self._ensure_live_delegate()
-        if delegate:
-            price = delegate.get_last_price(asset, quote=quote, exchange=exchange)
-            if price is not None:
-                return price
-
-        try:
-            last_price = databento_helper.get_last_price_from_databento(
-                api_key=self._api_key,
-                asset=asset,
-                venue=exchange
-            )
-
-            if last_price is not None:
-                logger.info(f"Last price for {asset.symbol}: {last_price}")
-                return last_price
-            else:
-                logger.warning(f"No last price available for {asset.symbol}")
-                return None
-
-        except Exception as e:
-            logger.error(f"Error getting last price for {asset.symbol}: {e}")
-            return None
-
-    def get_chains(self, asset: Asset, quote: Asset = None) -> dict:
-        """
-        Get option chains for an asset
-        
-        Note: DataBento primarily focuses on market data rather than options chains.
-        This method returns an empty dict as DataBento doesn't provide options chain data.
-        
-        Parameters
-        ----------
-        asset : Asset
-            The asset to get option chains for
-        quote : Asset, optional
-            Quote asset
-            
-        Returns
-        -------
-        dict
-            Empty dictionary as DataBento doesn't provide options chains
-        """
-        logger.warning("DataBento does not provide options chain data")
-        return {}
-
-    def get_quote(self, asset: Asset, quote: Asset = None) -> Union[float, Decimal, None]:
-        """
-        Get current quote for an asset
-        
-        For DataBento, this returns the last known price since real-time quotes
-        may not be available for all assets.
-        
-        Parameters
-        ----------
-        asset : Asset
-            The asset to get the quote for
-        quote : Asset, optional
-            Quote asset (not used for DataBento)
-            
-        Returns
-        -------
-        float, Decimal, or None
-            Current quote/last price of the asset
-        """
-        delegate = self._ensure_live_delegate()
-        if delegate:
-            quote_obj = delegate.get_quote(asset, quote=quote, exchange=None)
-            if quote_obj:
-                return quote_obj
-
-        price = self.get_last_price(asset, quote=quote)
-        return Quote(asset=asset, price=price)
-
-    def _ensure_live_delegate(self) -> Optional['DataBentoDataPolarsLive']:
-        if DataBentoDataPolarsLive is None or self.is_backtesting_mode:
-            return None
-
-        if self._live_delegate is None:
-            try:
-                self._live_delegate = DataBentoDataPolarsLive(
-                    api_key=self._api_key,
-                    has_paid_subscription=True,
-                    enable_cache=False,
-                    cache_duration_minutes=0,
-                    enable_live_stream=True,
-                )
-            except Exception as e:
-                logger.error(f"Failed to initialize live DataBento delegate: {e}")
-                self._live_delegate = None
-
-        return self._live_delegate
-
-    def _parse_source_symbol_bars(self, response, asset, quote=None):
-        """
-        Parse source data for a single asset into Bars format
-        
-        Parameters
-        ----------
-        response : pd.DataFrame
-            Raw data from DataBento API
-        asset : Asset
-            The asset the data is for
-        quote : Asset, optional
-            Quote asset (not used for DataBento)
-            
-        Returns
-        -------
-        Bars or None
-            Parsed bars data or None if parsing fails
-        """
-        try:
-            if response is None or response.empty:
-                return None
-
-            # Check if required columns exist
-            required_columns = ['open', 'high', 'low', 'close', 'volume']
-            if not all(col in response.columns for col in required_columns):
-                logger.warning(f"Missing required columns in DataBento data for {asset.symbol}")
-                return None
-
-            # Create Bars object
-            bars = Bars(
-                df=response,
-                source=self.SOURCE,
-                asset=asset,
-                quote=quote
-            )
-
-            return bars
-
-        except Exception as e:
-            logger.error(f"Error parsing DataBento data for {asset.symbol}: {e}")
-            return None
+__all__ = ["DataBentoData", "DataBentoDataPandas", "DataBentoDataPolars"]