lumibot 4.1.3__py3-none-any.whl → 4.2.1__py3-none-any.whl

lumibot/data_sources/databento_data_pandas.py

@@ -0,0 +1,440 @@
+from datetime import datetime, timedelta
+from decimal import Decimal
+from typing import Union, Optional
+
+import pandas as pd
+import polars as pl
+
+from .data_source import DataSource
+from lumibot.entities import Asset, Bars, Quote
+from lumibot.tools import databento_helper, databento_helper_polars
+from lumibot.tools.lumibot_logger import get_logger
+
+try:
+    from .databento_data_polars import DataBentoDataPolars
+except Exception:  # pragma: no cover - optional dependency path
+    DataBentoDataPolars = None
+
+logger = get_logger(__name__)
+
+
+class DataBentoDataPandas(DataSource):
+    """
+    DataBento data source for historical market data
+    
+    This data source provides access to DataBento's institutional-grade market data,
+    with a focus on futures data and support for multiple asset types.
+    """
+
+    SOURCE = "DATABENTO"
+    MIN_TIMESTEP = "minute"
+    TIMESTEP_MAPPING = [
+        {"timestep": "minute", "representations": ["1m", "minute", "1 minute"]},
+        {"timestep": "hour", "representations": ["1h", "hour", "1 hour"]},
+        {"timestep": "day", "representations": ["1d", "day", "1 day"]},
+    ]
+
+    def __init__(
+        self,
+        api_key: str,
+        timeout: int = 30,
+        max_retries: int = 3,
+        **kwargs
+    ):
+        """
+        Initialize DataBento data source
+        
+        Parameters
+        ----------
+        api_key : str
+            DataBento API key
+        timeout : int, optional
+            API request timeout in seconds, default 30
+        max_retries : int, optional
+            Maximum number of API retry attempts, default 3
+        **kwargs
+            Additional parameters passed to parent class
+        """
+        enable_live_stream = kwargs.pop("enable_live_stream", False)
+
+        # Initialize parent class
+        super().__init__(api_key=api_key, **kwargs)
+
+        self._api_key = api_key
+        self._timeout = timeout
+        self._max_retries = max_retries
+        self._data_store = {}
+        self._live_delegate = None
+        self._default_quote_asset = Asset("USD", "forex")
+
+        # For live trading, this is a live data source
+        self.is_backtesting_mode = False
+        self.enable_live_stream = enable_live_stream
+
+        # Verify DataBento availability
+        if not databento_helper.DATABENTO_AVAILABLE:
+            logger.error("DataBento package not available. Please install with: pip install databento")
+            raise ImportError("DataBento package not available")
+
+    def get_historical_prices(
+        self,
+        asset: Asset,
+        length: int,
+        timestep: str = "minute",
+        timeshift: timedelta = None,
+        quote: Asset = None,
+        exchange: str = None,
+        include_after_hours: bool = True,
+        return_polars: bool = False
+    ) -> Bars:
+        """
+        Get historical price data for an asset
+        
+        Parameters
+        ----------
+        asset : Asset
+            The asset to get historical prices for
+        length : int
+            Number of bars to retrieve
+        timestep : str, optional
+            Timestep for the data ('minute', 'hour', 'day'), default 'minute'
+        timeshift : timedelta, optional
+            Time shift to apply to the data retrieval
+        quote : Asset, optional
+            Quote asset (not used for DataBento)
+        exchange : str, optional
+            Exchange/venue filter
+        include_after_hours : bool, optional
+            Whether to include after-hours data, default True
+            
+        Returns
+        -------
+        Bars
+            Historical price data as Bars object
+        """
+        logger.debug(f"Getting historical prices for {asset.symbol}, length={length}, timestep={timestep}")
+
+        # Validate asset type - DataBento primarily supports futures
+        supported_asset_types = [Asset.AssetType.FUTURE, Asset.AssetType.CONT_FUTURE]
+        if asset.asset_type not in supported_asset_types:
+            error_msg = (
+                "DataBento data source only supports futures assets. "
+                f"Received asset type '{asset.asset_type}' for symbol '{asset.symbol}'. "
+                f"Supported types: {[t.value for t in supported_asset_types]}"
+            )
+            logger.error(error_msg)
+            return None
+
+        # Additional logging for debugging
+        logger.debug(f"DataBento request - Asset: {asset.symbol}, Type: {asset.asset_type}, Length: {length}, Timestep: {timestep}")
+        logger.debug(f"DataBento live trading mode: Requesting data for futures asset {asset.symbol}")
+
+        # Calculate the date range for data retrieval
+        # Use timezone-naive datetime for consistency
+        current_dt = datetime.now()
+        if current_dt.tzinfo is not None:
+            current_dt = current_dt.replace(tzinfo=None)
+
+        logger.debug(f"Using current datetime for live trading: {current_dt}")
+
+        # Apply timeshift if specified
+        if timeshift:
+            current_dt = current_dt - timeshift
+
+        # Calculate start date based on length and timestep
+        if timestep == "day":
+            buffer_days = max(10, length // 2)  # Buffer for live trading
+            start_dt = current_dt - timedelta(days=length + buffer_days)
+            # For live trading, end should be current time (no future data available)
+            end_dt = current_dt
+        elif timestep == "hour":
+            buffer_hours = max(24, length // 2)  # Buffer for live trading
+            start_dt = current_dt - timedelta(hours=length + buffer_hours)
+            # For live trading, end should be current time (no future data available)
+            end_dt = current_dt
+        else:  # minute or other
+            buffer_minutes = max(1440, length)  # Buffer for live trading
+            start_dt = current_dt - timedelta(minutes=length + buffer_minutes)
+            # For live trading, end should be current time (no future data available)
+            end_dt = current_dt
+
+        # Ensure both dates are timezone-naive for consistency
+        if start_dt.tzinfo is not None:
+            start_dt = start_dt.replace(tzinfo=None)
+        if end_dt.tzinfo is not None:
+            end_dt = end_dt.replace(tzinfo=None)
+
+        # Ensure we always have a valid date range (start < end)
+        if start_dt >= end_dt:
+            # If dates are equal or start is after end, adjust end date
+            if timestep == "day":
+                end_dt = start_dt + timedelta(days=max(1, length))
+            elif timestep == "hour":
+                end_dt = start_dt + timedelta(hours=max(1, length))
+            else:  # minute or other
+                end_dt = start_dt + timedelta(minutes=max(1, length))
+
+        # Final safety check: ensure end is always after start
+        if start_dt >= end_dt:
+            logger.error(f"Invalid date range after adjustment: start={start_dt}, end={end_dt}")
+            if timestep == "day":
+                end_dt = start_dt + timedelta(days=1)
+            elif timestep == "hour":
+                end_dt = start_dt + timedelta(hours=1)
+            else:
+                end_dt = start_dt + timedelta(minutes=1)
+
+        # Get data from DataBento
+        logger.debug(f"Requesting DataBento data for asset: {asset} (type: {asset.asset_type})")
+        logger.debug(f"Date range: {start_dt} to {end_dt}")
+
+        try:
+            df = databento_helper_polars.get_price_data_from_databento_polars(
+                api_key=self._api_key,
+                asset=asset,
+                start=start_dt,
+                end=end_dt,
+                timestep=timestep,
+                venue=exchange
+            )
+        except Exception as e:
+            logger.error(f"Error getting data from DataBento for {asset.symbol}: {e}")
+            raise
+
+        if df is None:
+            logger.error(f"No data returned from DataBento for {asset.symbol}. This could be due to:")
+            logger.error("1. Incorrect symbol format")
+            logger.error("2. Wrong dataset selection")
+            logger.error("3. Data not available for the requested time range")
+            logger.error("4. API authentication issues")
+            return None
+
+        if not isinstance(df, pd.DataFrame):
+            df = df.to_pandas()
+            if "datetime" in df.columns:
+                df = df.set_index(pd.to_datetime(df["datetime"], utc=True))
+                df.index.name = "datetime"
+
+        if df.empty:
+            logger.error(f"No data returned from DataBento for {asset.symbol}. This could be due to:")
+            logger.error("1. Incorrect symbol format")
+            logger.error("2. Wrong dataset selection")
+            logger.error("3. Data not available for the requested time range")
+            logger.error("4. API authentication issues")
+            return None
+
+        # Filter data to the current time (for live trading)
+        # Handle timezone consistency for comparison
+        if hasattr(df.index, 'tz') and df.index.tz is not None:
+            # DataFrame has timezone-aware index, convert current_dt to match
+            if current_dt.tzinfo is None:
+                import pytz
+                current_dt = current_dt.replace(tzinfo=pytz.UTC)
+        else:
+            # DataFrame has timezone-naive index, ensure current_dt is also naive
+            if current_dt.tzinfo is not None:
+                current_dt = current_dt.replace(tzinfo=None)
+
+        df_filtered = df[df.index <= current_dt]
+        if df_filtered.empty:
+            df_filtered = df
+
+        # Take the last 'length' bars
+        df_result = df_filtered.tail(length)
+
+        if df_result.empty:
+            logger.warning(f"No data available for {asset.symbol} up to {current_dt}")
+            return None
+
+        # Create and return Bars object
+        bars = Bars(
+            df=df_result,
+            source=self.SOURCE,
+            asset=asset,
+            quote=quote,
+            return_polars=return_polars
+        )
+        quote_asset = quote if quote is not None else self._default_quote_asset
+        self._data_store[(asset, quote_asset)] = bars
+
+        logger.debug(f"Retrieved {len(df_result)} bars for {asset.symbol}")
+        return bars
+
+    def get_last_price(
+        self,
+        asset: Asset,
+        quote: Asset = None,
+        exchange: str = None
+    ) -> Union[float, Decimal, None]:
+        """
+        Get the last known price for an asset
+        
+        Parameters
+        ----------
+        asset : Asset
+            The asset to get the last price for
+        quote : Asset, optional
+            Quote asset (not used for DataBento)
+        exchange : str, optional
+            Exchange/venue filter
+            
+        Returns
+        -------
+        float, Decimal, or None
+            Last known price of the asset
+        """
+        logger.debug(f"Getting last price for {asset.symbol}")
+
+        # Prefer live delegate when available
+        delegate = self._ensure_live_delegate()
+        if delegate:
+            price = delegate.get_last_price(asset, quote=quote, exchange=exchange)
+            if price is not None:
+                return price
+
+        quote_asset = quote if quote is not None else self._default_quote_asset
+        cached_bars = self._data_store.get((asset, quote_asset))
+        if cached_bars is None:
+            try:
+                self.get_historical_prices(asset, length=1, timestep=self.MIN_TIMESTEP, quote=quote, return_polars=False)
+            except Exception:
+                pass
+            cached_bars = self._data_store.get((asset, quote_asset))
+        if cached_bars is not None:
+            df = cached_bars.df if hasattr(cached_bars, "df") else None
+            if df is not None and not df.empty and "close" in df.columns:
+                return float(df["close"].iloc[-1])
+
+        try:
+            last_price = databento_helper.get_last_price_from_databento(
+                api_key=self._api_key,
+                asset=asset,
+                venue=exchange
+            )
+
+            if last_price is not None:
+                logger.debug(f"Last price for {asset.symbol}: {last_price}")
+                return last_price
+            else:
+                logger.warning(f"No last price available for {asset.symbol}")
+                return None
+
+        except Exception as e:
+            logger.error(f"Error getting last price for {asset.symbol}: {e}")
+            return None
+
+    def get_chains(self, asset: Asset, quote: Asset = None) -> dict:
+        """
+        Get option chains for an asset
+        
+        Note: DataBento primarily focuses on market data rather than options chains.
+        This method returns an empty dict as DataBento doesn't provide options chain data.
+        
+        Parameters
+        ----------
+        asset : Asset
+            The asset to get option chains for
+        quote : Asset, optional
+            Quote asset
+            
+        Returns
+        -------
+        dict
+            Empty dictionary as DataBento doesn't provide options chains
+        """
+        logger.warning("DataBento does not provide options chain data")
+        return {}
+
+    def get_quote(self, asset: Asset, quote: Asset = None) -> Union[float, Decimal, None]:
+        """
+        Get current quote for an asset
+        
+        For DataBento, this returns the last known price since real-time quotes
+        may not be available for all assets.
+        
+        Parameters
+        ----------
+        asset : Asset
+            The asset to get the quote for
+        quote : Asset, optional
+            Quote asset (not used for DataBento)
+            
+        Returns
+        -------
+        float, Decimal, or None
+            Current quote/last price of the asset
+        """
+        delegate = self._ensure_live_delegate()
+        if delegate:
+            quote_obj = delegate.get_quote(asset, quote=quote, exchange=None)
+            if quote_obj:
+                return quote_obj
+
+        price = self.get_last_price(asset, quote=quote)
+        return Quote(asset=asset, price=price, bid=price, ask=price)
+
+    def _ensure_live_delegate(self) -> Optional['DataBentoDataPolars']:
+        if not self.enable_live_stream:
+            return None
+        if DataBentoDataPolars is None or self.is_backtesting_mode:
+            return None
+
+        if self._live_delegate is None:
+            try:
+                self._live_delegate = DataBentoDataPolars(
+                    api_key=self._api_key,
+                    has_paid_subscription=True,
+                    enable_cache=False,
+                    cache_duration_minutes=0,
+                    enable_live_stream=True,
+                )
+            except Exception as e:
+                logger.error(f"Failed to initialize live DataBento delegate: {e}")
+                self._live_delegate = None
+
+        return self._live_delegate
+
+    def _parse_source_symbol_bars(self, response, asset, quote=None, return_polars: bool = False):
+        """
+        Parse source data for a single asset into Bars format
+
+        Parameters
+        ----------
+        response : pd.DataFrame
+            Raw data from DataBento API
+        asset : Asset
+            The asset the data is for
+        quote : Asset, optional
+            Quote asset (not used for DataBento)
+        return_polars : bool, optional
+            Whether to return a Polars DataFrame instead of pandas, default False
+
+        Returns
+        -------
+        Bars or None
+            Parsed bars data or None if parsing fails
+        """
+        try:
+            if response is None or response.empty:
+                return None
+
+            # Check if required columns exist
+            required_columns = ['open', 'high', 'low', 'close', 'volume']
+            if not all(col in response.columns for col in required_columns):
+                logger.warning(f"Missing required columns in DataBento data for {asset.symbol}")
+                return None
+
+            # Create Bars object
+            bars = Bars(
+                df=response,
+                source=self.SOURCE,
+                asset=asset,
+                quote=quote,
+                return_polars=return_polars
+            )
+
+            return bars
+
+        except Exception as e:
+            logger.error(f"Error parsing DataBento data for {asset.symbol}: {e}")
+            return None

lumibot/data_sources/databento_data_polars.py

@@ -16,10 +16,13 @@ import queue
 from collections import defaultdict
 
 import polars as pl
-import databento as db
+try:
+    import databento as db
+except ImportError:  # pragma: no cover - optional dependency
+    db = None
 
-from lumibot.data_sources import DataSource
-from lumibot.data_sources.polars_mixin import PolarsMixin
+from .data_source import DataSource
+from .polars_mixin import PolarsMixin
 from lumibot.entities import Asset, Bars, Quote
 from lumibot.tools import databento_helper_polars
 from lumibot.tools.databento_helper_polars import _ensure_polars_datetime_timezone as _ensure_polars_tz
@@ -31,7 +34,7 @@ logger = get_logger(__name__)
 class DataBentoDataPolars(PolarsMixin, DataSource):
     """
     DataBento data source optimized with Polars and proper Live API usage.
-    
+
     Uses Live API for real-time trade streaming to achieve <1 minute lag.
     Falls back to Historical API for older data.
     """
@@ -57,6 +60,9 @@ class DataBentoDataPolars(PolarsMixin, DataSource):
         """Initialize DataBento data source with Live API support"""
         super().__init__(api_key=api_key, has_paid_subscription=has_paid_subscription)
 
+        if db is None:
+            raise ImportError("DataBento package not available. Please install with: pip install databento")
+
         # Core configuration
         self._api_key = api_key
         self.has_paid_subscription = has_paid_subscription
@@ -212,7 +218,7 @@ class DataBentoDataPolars(PolarsMixin, DataSource):
                         logger.warning(f"[DATABENTO][PRODUCER] Queue full, dropping record")
                 
                 # Clean exit
-                logger.info(f"[DATABENTO][PRODUCER] {symbol} stopped after {record_count} records")
+                logger.debug(f"[DATABENTO][PRODUCER] {symbol} stopped after {record_count} records")
                 break  # Successful completion
                 
             except Exception as e:
@@ -221,7 +227,7 @@ class DataBentoDataPolars(PolarsMixin, DataSource):
                 
                 if reconnect_attempts < max_reconnect_attempts:
                     sleep_time = backoff_seconds * (2 ** reconnect_attempts)
-                    logger.info(f"[DATABENTO][PRODUCER] Reconnecting {symbol} in {sleep_time}s (attempt {reconnect_attempts})")
+                    logger.debug(f"[DATABENTO][PRODUCER] Reconnecting {symbol} in {sleep_time}s (attempt {reconnect_attempts})")
                     time.sleep(sleep_time)
                     
                     # Update start time for reconnection to avoid duplicate data
@@ -230,7 +236,7 @@ class DataBentoDataPolars(PolarsMixin, DataSource):
                         ts_ns = self._last_ts_event[symbol]
                         if ts_ns > 0:
                             start_time = datetime.fromtimestamp(ts_ns / 1e9, tz=timezone.utc)
-                        logger.info(f"[DATABENTO][PRODUCER] Resuming from last event: {start_time.isoformat()}")
+                        logger.debug(f"[DATABENTO][PRODUCER] Resuming from last event: {start_time.isoformat()}")
                 else:
                     logger.error(f"[DATABENTO][PRODUCER] {symbol} max reconnection attempts reached")
     
@@ -342,7 +348,7 @@ class DataBentoDataPolars(PolarsMixin, DataSource):
             except Exception as e:
                 logger.error(f"[DATABENTO][CONSUMER] Error processing record: {e}")
         
-        logger.info(f"[DATABENTO][CONSUMER] Stopped after {trade_count} trades")
+        logger.debug(f"[DATABENTO][CONSUMER] Stopped after {trade_count} trades")
     
     def _finalizer_loop(self):
         """Finalizer thread that marks old bars as complete"""
@@ -378,7 +384,7 @@ class DataBentoDataPolars(PolarsMixin, DataSource):
             except Exception as e:
                 logger.error(f"[DATABENTO][FINALIZER] Error: {e}")
         
-        logger.info("[DATABENTO][FINALIZER] Stopped")
+        logger.debug("[DATABENTO][FINALIZER] Stopped")
     
     def _aggregate_trade(self, symbol: str, price: float, size: float, trade_time: datetime):
         """Aggregate a trade into minute bars"""

lumibot/data_sources/pandas_data.py

@@ -304,17 +304,26 @@ class PandasData(DataSourceBacktesting):
             result[asset] = self.get_last_price(asset, quote=quote, exchange=exchange)
         return result
 
-    def find_asset_in_data_store(self, asset, quote=None):
-        if asset in self._data_store:
-            return asset
-        elif quote is not None:
-            asset = (asset, quote)
-            if asset in self._data_store:
-                return asset
-        elif isinstance(asset, Asset) and asset.asset_type in ["option", "future", "stock", "index"]:
-            asset = (asset, Asset("USD", "forex"))
-            if asset in self._data_store:
-                return asset
+    def find_asset_in_data_store(self, asset, quote=None, timestep=None):
+        candidates = []
+
+        if timestep is not None:
+            base_quote = quote if quote is not None else Asset("USD", "forex")
+            candidates.append((asset, base_quote, timestep))
+            if quote is not None:
+                candidates.append((asset, Asset("USD", "forex"), timestep))
+
+        if quote is not None:
+            candidates.append((asset, quote))
+
+        if isinstance(asset, Asset) and asset.asset_type in ["option", "future", "stock", "index"]:
+            candidates.append((asset, Asset("USD", "forex")))
+
+        candidates.append(asset)
+
+        for key in candidates:
+            if key in self._data_store:
+                return key
         return None
 
     def _pull_source_symbol_bars(
@@ -336,7 +345,7 @@ class PandasData(DataSourceBacktesting):
         if not timeshift:
             timeshift = 0
 
-        asset_to_find = self.find_asset_in_data_store(asset, quote)
+        asset_to_find = self.find_asset_in_data_store(asset, quote, timestep)
 
         if asset_to_find in self._data_store:
             data = self._data_store[asset_to_find]
@@ -369,7 +378,7 @@ class PandasData(DataSourceBacktesting):
     ):
         """Pull all bars for an asset"""
         timestep = timestep if timestep else self.MIN_TIMESTEP
-        asset_to_find = self.find_asset_in_data_store(asset, quote)
+        asset_to_find = self.find_asset_in_data_store(asset, quote, timestep)
 
         if asset_to_find in self._data_store:
             data = self._data_store[asset_to_find]
@@ -412,13 +421,17 @@ class PandasData(DataSourceBacktesting):
 
         return result
 
-    def _parse_source_symbol_bars(self, response, asset, quote=None, length=None):
-        """parse broker response for a single asset"""
+    def _parse_source_symbol_bars(self, response, asset, quote=None, length=None, return_polars: bool = False):
+        """parse broker response for a single asset
+
+        CRITICAL: return_polars defaults to False for backwards compatibility.
+        PandasData always returns pandas-backed Bars for consistency.
+        """
         asset1 = asset
         asset2 = quote
         if isinstance(asset, tuple):
             asset1, asset2 = asset
-        bars = Bars(response, self.SOURCE, asset1, quote=asset2, raw=response)
+        bars = Bars(response, self.SOURCE, asset1, quote=asset2, raw=response, return_polars=return_polars)
         return bars
 
     def get_yesterday_dividend(self, asset, quote=None):
@@ -541,5 +554,5 @@ class PandasData(DataSourceBacktesting):
         elif response is None:
             return None
 
-        bars = self._parse_source_symbol_bars(response, asset, quote=quote, length=length)
+        bars = self._parse_source_symbol_bars(response, asset, quote=quote, length=length, return_polars=return_polars)
         return bars