lumibot 4.0.23__py3-none-any.whl → 4.1.0__py3-none-any.whl

lumibot/tools/databento_roll.py

@@ -0,0 +1,216 @@
+"""
+Shared utilities for handling DataBento continuous futures roll logic.
+
+This module centralizes symbol resolution and roll schedule computation so that
+both the pandas and polars implementations stay in sync.
+"""
+
+from __future__ import annotations
+
+import os
+from datetime import datetime, timedelta, timezone
+from typing import Callable, Dict, Iterable, List, Tuple
+
+import pytz
+
+from lumibot.constants import LUMIBOT_DEFAULT_PYTZ
+from lumibot.entities import Asset
+
+# Number of calendar days before expiration to roll into the next contract.
+# This defaults to 7 (~5 business days) but can be overridden with an env var.
+ROLL_DAYS_BEFORE_EXPIRATION = int(os.getenv("LUMIBOT_FUTURES_ROLL_DAYS", "7"))
+
+# Caches used for symbol resolution so repeated lookups are cheap.
+_DATETIME_NORMALIZATION_CACHE: Dict[float, datetime] = {}
+_SYMBOL_RESOLUTION_CACHE: Dict[Tuple[str, str, float], str] = {}
+
+NY_TZ = pytz.timezone("America/New_York")
+UTC = timezone.utc
+
+
+def _ensure_tz(dt: datetime) -> datetime:
+    """Ensure a datetime is timezone-aware, defaulting to the platform TZ."""
+    if dt.tzinfo is None:
+        return LUMIBOT_DEFAULT_PYTZ.localize(dt)
+    return dt
+
+
+def _normalize_reference_datetime(dt: datetime) -> datetime:
+    """Normalize datetimes for use in caches when resolving symbols."""
+    if dt is None:
+        return dt
+
+    cache_key = dt.timestamp() if hasattr(dt, "timestamp") else None
+    if cache_key is not None and cache_key in _DATETIME_NORMALIZATION_CACHE:
+        return _DATETIME_NORMALIZATION_CACHE[cache_key]
+
+    if dt.tzinfo is not None:
+        normalized = dt.astimezone(LUMIBOT_DEFAULT_PYTZ).replace(tzinfo=None)
+    else:
+        normalized = dt
+
+    if cache_key is not None:
+        _DATETIME_NORMALIZATION_CACHE[cache_key] = normalized
+
+    return normalized
+
+
+def resolve_symbol_for_datetime(asset: Asset, dt: datetime) -> str:
+    """
+    Resolve the continuous futures symbol for a specific datetime using the
+    asset's roll rules.
+    """
+    dt_norm = _normalize_reference_datetime(dt)
+    cache_key = (
+        asset.symbol,
+        asset.asset_type,
+        dt_norm.timestamp() if dt_norm is not None else float("inf"),
+    )
+
+    if cache_key in _SYMBOL_RESOLUTION_CACHE:
+        return _SYMBOL_RESOLUTION_CACHE[cache_key]
+
+    variants = asset.resolve_continuous_futures_contract_variants(reference_date=dt_norm)
+    contract = variants[2]  # two-digit year variant
+
+    # DataBento prefers the short year format (single digit); reuse helper.
+    month_code = contract[len(asset.symbol)]
+    year_char = contract[-1]
+    resolved_symbol = f"{asset.symbol}{month_code}{year_char}"
+
+    _SYMBOL_RESOLUTION_CACHE[cache_key] = resolved_symbol
+    return resolved_symbol
+
+
+def resolve_symbols_for_range(asset: Asset, start: datetime, end: datetime) -> List[str]:
+    """
+    Resolve the list of DataBento contract symbols required to cover a datetime range.
+    """
+    if start is None or end is None:
+        return []
+
+    start_ref = _normalize_reference_datetime(start)
+    end_ref = _normalize_reference_datetime(end)
+
+    if start_ref is None or end_ref is None:
+        return [
+            resolve_symbol_for_datetime(asset, _ensure_tz(start)),
+        ]
+
+    symbols: List[str] = []
+    seen = set()
+    cursor = start_ref
+    step = timedelta(days=45)  # ensures we hop across quarter rolls
+
+    while cursor <= end_ref + timedelta(days=45):
+        symbol = resolve_symbol_for_datetime(asset, cursor)
+        if symbol not in seen:
+            seen.add(symbol)
+            symbols.append(symbol)
+        cursor += step
+
+    # Ensure the last contract covers the end reference.
+    final_symbol = resolve_symbol_for_datetime(asset, end_ref)
+    if final_symbol not in seen:
+        symbols.append(final_symbol)
+
+    return symbols
+
+
+def _parse_expiration(definition: Dict) -> datetime:
+    """
+    Parse the expiration field from a DataBento instrument definition.
+    """
+    expiration = (
+        definition.get("expiration")
+        or definition.get("maturity_date")
+        or definition.get("last_trade_date")
+    )
+    if expiration is None:
+        raise ValueError("Instrument definition missing expiration information")
+
+    if isinstance(expiration, datetime):
+        dt_local = expiration
+    else:
+        expiration_str = str(expiration)
+        # Handle ISO strings with optional timezone offset.
+        if "T" in expiration_str:
+            expiration_str = expiration_str.replace("Z", "+00:00")
+            dt_local = datetime.fromisoformat(expiration_str)
+        else:
+            dt_local = datetime.strptime(expiration_str, "%Y-%m-%d")
+
+    if dt_local.tzinfo is None:
+        dt_local = NY_TZ.localize(dt_local)
+    else:
+        dt_local = dt_local.astimezone(NY_TZ)
+
+    # Futures generally stop trading in the afternoon; rolling on midnight is fine.
+    return dt_local
+
+
+def build_roll_schedule(
+    asset: Asset,
+    start: datetime,
+    end: datetime,
+    definition_provider: Callable[[str], Dict],
+    roll_days: int = ROLL_DAYS_BEFORE_EXPIRATION,
+) -> List[Tuple[str, datetime, datetime]]:
+    """
+    Build a list of (symbol, start_utc, end_utc) windows indicating which contract
+    should be used at each point in time.
+    """
+    if roll_days < 0:
+        raise ValueError("roll_days must be non-negative")
+
+    start = _ensure_tz(start)
+    end = _ensure_tz(end)
+    symbols = resolve_symbols_for_range(asset, start, end)
+
+    if not symbols:
+        return []
+
+    schedule: List[Tuple[str, datetime, datetime]] = []
+    current_start = datetime.min.replace(tzinfo=UTC)
+
+    for idx, symbol in enumerate(symbols):
+        definition = definition_provider(symbol)
+        if not definition:
+            continue
+
+        expiration_local = _parse_expiration(definition)
+        roll_local = expiration_local - timedelta(days=roll_days)
+        roll_local = max(roll_local, start)
+        roll_utc = roll_local.astimezone(UTC)
+
+        if idx < len(symbols) - 1:
+            end_utc = roll_utc
+        else:
+            end_utc = datetime.max.replace(tzinfo=UTC)
+
+        schedule.append((symbol, current_start, end_utc))
+        current_start = roll_utc
+
+    if not schedule:
+        return []
+
+    start_utc = start.astimezone(UTC)
+    end_utc = end.astimezone(UTC)
+
+    clipped: List[Tuple[str, datetime, datetime]] = []
+    for symbol, window_start, window_end in schedule:
+        s = max(window_start, start_utc)
+        e = min(window_end, end_utc)
+        if e <= s:
+            continue
+        clipped.append((symbol, s, e))
+
+    if not clipped:
+        clipped.append((schedule[-1][0], start_utc, end_utc))
+    else:
+        last_symbol, s, e = clipped[-1]
+        if e < end_utc:
+            clipped[-1] = (last_symbol, s, end_utc)
+
+    return clipped
+

lumibot/tools/lumibot_logger.py

@@ -656,11 +656,18 @@ def _ensure_handlers_configured():
 
     # Determine the effective file (root) log level and console level
     if is_backtesting:
-        console_level = logging.ERROR
         backtesting_quiet = os.environ.get("BACKTESTING_QUIET_LOGS")
         if backtesting_quiet is None:
             backtesting_quiet = "true"
-        effective_log_level = logging.ERROR if backtesting_quiet.lower() == "true" else log_level
+
+        if backtesting_quiet.lower() == "true":
+            # Quiet mode: only ERROR+ messages to console and file
+            console_level = logging.ERROR
+            effective_log_level = logging.ERROR
+        else:
+            # Verbose mode: respect LUMIBOT_LOG_LEVEL for both console and file
+            console_level = log_level
+            effective_log_level = log_level
     else:
         console_level = log_level
         effective_log_level = log_level
@@ -851,27 +858,35 @@ def set_log_level(level: str):
         # Get the actual lumibot root logger
         root_logger = logging.getLogger("lumibot")
         root_logger.setLevel(log_level)
-        
-        # Update handlers but respect console handler's ERROR level during backtesting
+
+        # Update handlers with respect to backtesting quiet logs setting
         is_backtesting = os.environ.get("IS_BACKTESTING", "").lower() == "true"
-        
+
         if is_backtesting:
-            # During backtesting, we need special handling to ensure console stays quiet
-            # Set root logger to allow messages through (for file logging)
-            root_logger.setLevel(log_level)
-            
-            # But ensure ALL console handlers stay at ERROR level
-            for handler in root_logger.handlers:
-                if isinstance(handler, logging.StreamHandler):
-                    handler.setLevel(logging.ERROR)
-            
-            # Don't update individual logger levels - this would bypass handler filtering
+            # Check if quiet logs are enabled
+            backtesting_quiet = os.environ.get("BACKTESTING_QUIET_LOGS")
+            if backtesting_quiet is None:
+                backtesting_quiet = "true"
+
+            if backtesting_quiet.lower() == "true":
+                # Quiet mode: console stays at ERROR, but allow file handlers to use requested level
+                root_logger.setLevel(log_level)
+                for handler in root_logger.handlers:
+                    if isinstance(handler, logging.StreamHandler) and not isinstance(handler, logging.FileHandler):
+                        handler.setLevel(logging.ERROR)  # Console: quiet
+                    else:
+                        handler.setLevel(log_level)  # File handlers: verbose
+            else:
+                # Verbose mode: respect requested level for all handlers
+                root_logger.setLevel(log_level)
+                for handler in root_logger.handlers:
+                    handler.setLevel(log_level)
         else:
-            # For live trading, set everything normally
+            # Live trading: set everything normally
             root_logger.setLevel(log_level)
             for handler in root_logger.handlers:
                 handler.setLevel(log_level)
-            
+
             # Update all existing loggers in our registry
             for logger in _logger_registry.values():
                 logger.setLevel(log_level)

lumibot/tools/polygon_helper.py

@@ -142,6 +142,35 @@ def get_price_data_from_polygon(
     if not missing_dates:
         if df_all is not None:
             df_all = df_all.dropna(how="all")
+            # Filter cached data to requested date range before returning
+            if not df_all.empty:
+                # For daily data, use date-based filtering (timestamps vary by provider)
+                # For intraday data, use precise datetime filtering
+                if timespan == "day":
+                    # Convert index to dates for comparison
+                    import pandas as pd
+                    df_dates = pd.to_datetime(df_all.index).date
+                    start_date = start.date() if hasattr(start, 'date') else start
+                    end_date = end.date() if hasattr(end, 'date') else end
+                    mask = (df_dates >= start_date) & (df_dates <= end_date)
+                    df_all = df_all[mask]
+                else:
+                    # Intraday: use precise datetime filtering
+                    import datetime as dt
+                    import pytz
+                    from lumibot import LUMIBOT_DEFAULT_PYTZ
+
+                    # Convert date to datetime if needed
+                    if isinstance(start, dt.date) and not isinstance(start, dt.datetime):
+                        start = dt.datetime.combine(start, dt.time.min)
+                    if isinstance(end, dt.date) and not isinstance(end, dt.datetime):
+                        end = dt.datetime.combine(end, dt.time.max)
+
+                    if start.tzinfo is None:
+                        start = LUMIBOT_DEFAULT_PYTZ.localize(start).astimezone(pytz.UTC)
+                    if end.tzinfo is None:
+                        end = LUMIBOT_DEFAULT_PYTZ.localize(end).astimezone(pytz.UTC)
+                    df_all = df_all[(df_all.index >= start) & (df_all.index <= end)]
         return df_all
 
     # Create a PolygonClient and get the symbol for the asset.
@@ -209,6 +238,42 @@ def get_price_data_from_polygon(
     else:
         df_all_output = df_all_full.copy()
     df_all_output = df_all_output.dropna(how="all")
+
+    # Filter cached data to requested date range before returning
+    if not df_all_output.empty:
+        # For daily data, use date-based filtering (timestamps vary by provider)
+        # For intraday data, use precise datetime filtering
+        if timespan == "day":
+            # Convert index to dates for comparison
+            import pandas as pd
+            df_dates = pd.to_datetime(df_all_output.index).date
+            start_date = start.date() if hasattr(start, 'date') else start
+            end_date = end.date() if hasattr(end, 'date') else end
+            mask = (df_dates >= start_date) & (df_dates <= end_date)
+            df_all_output = df_all_output[mask]
+        else:
+            # Intraday: use precise datetime filtering
+            import datetime as dt
+            import pytz
+            from lumibot import LUMIBOT_DEFAULT_PYTZ
+
+            # Convert date to datetime if needed
+            if isinstance(start, dt.date) and not isinstance(start, dt.datetime):
+                start = dt.datetime.combine(start, dt.time.min)
+            if isinstance(end, dt.date) and not isinstance(end, dt.datetime):
+                end = dt.datetime.combine(end, dt.time.max)
+
+            # Handle datetime objects with midnight time (users often pass datetime(YYYY, MM, DD))
+            if isinstance(end, dt.datetime) and end.time() == dt.time.min:
+                # Convert end-of-period midnight to end-of-day
+                end = dt.datetime.combine(end.date(), dt.time.max)
+
+            if start.tzinfo is None:
+                start = LUMIBOT_DEFAULT_PYTZ.localize(start).astimezone(pytz.UTC)
+            if end.tzinfo is None:
+                end = LUMIBOT_DEFAULT_PYTZ.localize(end).astimezone(pytz.UTC)
+            df_all_output = df_all_output[(df_all_output.index >= start) & (df_all_output.index <= end)]
+
     return df_all_output