lumibot 4.1.3__py3-none-any.whl → 4.2.1__py3-none-any.whl

lumibot/entities/position.py

@@ -1,6 +1,7 @@
 from decimal import Decimal
 
 import lumibot.entities as entities
+from lumibot.entities.asset import StrEnum #todo: this should be centralized, and not repeated in Asset and Position
 
 
 class Position:
@@ -26,8 +27,36 @@ class Position:
         The assets that are free in the portfolio. (Crypto: only)
     avg_fill_price : float
         The average fill price of the position.
+    current_price : float
+        The current price of the asset.
+    market_value : float
+        The market value of the position.
+    pnl : float
+        The profit and loss of the position.
+    pnl_percent : float
+        The profit and loss of the position as a percentage of the average fill price.
+    asset_type : str
+        The type of the asset.
+    exchange : str
+        The exchange that the position is on.
+    currency : str
+        The currency that the position is denominated in.
+    multiplier : float
+        The multiplier of the asset.
+    expiration : datetime.date
+        The expiration of the asset. (Options and futures: only). Probably better to use on position.asset
+    strike : float
+        The strike price of the asset. (Options: only). Probably better to use on position.asset
+    option_type : str
+        The type of the option. (Options: only). Probably better to use on position.asset
+    side : PositionSide
+        The side of the position (LONG or SHORT)
     """
 
+    class PositionSide(StrEnum):
+        LONG = "LONG"
+        SHORT = "SHORT"
+
     def __init__(
             self,
             strategy,
@@ -38,6 +67,11 @@ class Position:
             available=0,
             avg_fill_price=None
         ):
+        """Creates a position.
+
+        NOTE: There are some properties that can be assigned to a position entity outside of the constructor (pnl, current_price, etc)
+
+        """
         self.strategy = strategy
         self.asset = asset
         self.symbol = self.asset.symbol
@@ -223,13 +257,13 @@ class Position:
             result['currency'] = self.currency
         if hasattr(self, 'multiplier'):
             result['multiplier'] = self.multiplier
-        if hasattr(self, 'expiration'):
+        if hasattr(self, 'expiration'):  #should probably use position.asset instead
             result['expiration'] = str(self.expiration) if self.expiration else None
-        if hasattr(self, 'strike'):
+        if hasattr(self, 'strike'): #should probably use position.asset instead
             result['strike'] = float(self.strike) if self.strike else None
-        if hasattr(self, 'option_type'):
+        if hasattr(self, 'option_type'): #should probably use position.asset instead
             result['option_type'] = self.option_type
-        if hasattr(self, 'underlying_symbol'):
+        if hasattr(self, 'underlying_symbol'): #should probably use position.asset instead
             result['underlying_symbol'] = self.underlying_symbol
 
         # Handle orders carefully - ensure to_dict() is called properly

lumibot/strategies/_strategy.py

@@ -819,7 +819,7 @@ class _Strategy:
 
             assets = []
             for position in positions:
-                if position.asset != self._quote_asset:
+                if position.asset != self._quote_asset and position.asset.asset_type != "option":
                     assets.append(position.asset)
 
             # Early return if no assets - avoid expensive dividend API calls
@@ -827,6 +827,7 @@ class _Strategy:
                 return self.cash
 
             dividends_per_share = self.get_yesterday_dividends(assets)
+
             for position in positions:
                 asset = position.asset
                 quantity = position.quantity

lumibot/strategies/strategy.py

@@ -19,6 +19,7 @@ from termcolor import colored, COLORS
 from ..data_sources import DataSource
 from ..entities import Asset, Data, Order, Position, Quote, TradingFee
 from ..tools import get_risk_free_rate
+from ..tools.polars_utils import PolarsResampleError, resample_polars_ohlc
 from ..traders import Trader
 from ._strategy import _Strategy
 
@@ -3353,7 +3354,7 @@ class Strategy(_Strategy):
         asset: Union[Asset, str],
         length: int,
         timestep: str = "",
-        timeshift: datetime.timedelta = None,
+        timeshift: Union[int, datetime.timedelta, None] = None,
         quote: Asset = None,
         exchange: str = None,
         include_after_hours: bool = True,
@@ -3388,10 +3389,12 @@ class Strategy(_Strategy):
             When using multi-timeframe formats, the method automatically fetches the
             underlying minute or day data and resamples it to your desired timeframe.
             Default value depends on the data_source (minute for alpaca, day for yahoo, ...)
-        timeshift : timedelta
-            ``None`` by default. If specified indicates the time shift from
-            the present. If  backtesting in Pandas, use integer representing
-            number of bars.
+        timeshift : int, timedelta, or None
+            ``None`` by default. When provided it shifts the data window relative to
+            the current backtest time.
+
+            - Passing an ``int`` shifts by bars (positive = past, negative = future).
+            - Passing a ``timedelta`` shifts by wall-clock time (e.g. ``timedelta(hours=-1)``).
         quote : Asset
             The quote currency for crypto currencies (e.g. USD, USDT, EUR, ...).
             Default is the quote asset for the strategy.
@@ -3517,6 +3520,7 @@ class Strategy(_Strategy):
         asset = self.crypto_assets_to_tuple(asset, quote)
         if not actual_timestep:
             actual_timestep = self.broker.data_source.get_timestep()
+        effective_return_polars = return_polars
         # Call through to the appropriate data source. Only pass `return_polars` if supported
         # to maintain compatibility with live data sources that don't yet accept it.
         import inspect
@@ -3540,7 +3544,7 @@ class Strategy(_Strategy):
                 return fn(
                     asset,
                     actual_length,  # Use the actual length for fetching
-                    return_polars=return_polars,
+                    return_polars=effective_return_polars,
                     **common_kwargs,
                 )
             else:
@@ -3557,49 +3561,57 @@ class Strategy(_Strategy):
             bars = _call_get_hist(self.broker.data_source)
 
         # If we need to resample the data
-        if needs_resampling and bars and bars.df is not None and not bars.df.empty:
-            try:
-                # Import pandas for resampling
-                import pandas as pd
-                from lumibot.entities import Bars
-
-                # Get the dataframe
-                df = bars.df.copy()
-
-                # Ensure datetime index
-                if not isinstance(df.index, pd.DatetimeIndex):
-                    # Try to convert the index to datetime
-                    df.index = pd.to_datetime(df.index)
-
-                # Determine resampling rule
-                if base_unit == "minute":
-                    resample_rule = f'{multiplier}min'  # 'min' for minutes (pandas 2.0+ compatible)
-                elif base_unit == "day":
-                    resample_rule = f'{multiplier}D'  # D for days
-                else:
-                    # Fallback to original if we can't determine the rule
-                    return bars
-
-                # Perform resampling with proper aggregation
-                resampled = df.resample(resample_rule, label='left', closed='left').agg({
-                    'open': 'first',
-                    'high': 'max',
-                    'low': 'min',
-                    'close': 'last',
-                    'volume': 'sum'
-                }).dropna()
-
-                # Limit to requested length
-                if len(resampled) > length:
-                    resampled = resampled.iloc[-length:]
-
-                # Create new Bars object with resampled data
-                bars.df = resampled
-                bars.raw_data = resampled  # Update raw_data as well if it exists
-
-            except Exception as e:
-                # If resampling fails, log warning and return original data
-                self.logger.warning(f"Failed to resample data from {actual_timestep} to {original_timestep}: {e}")
+        if needs_resampling and bars and len(bars) > 0:
+            resampled_with_polars = False
+            if return_polars:
+                try:
+                    polars_frame = bars.polars_df
+                    resampled_frame = resample_polars_ohlc(polars_frame, multiplier, base_unit, length)
+                    if resampled_frame is not None and not resampled_frame.is_empty():
+                        bars.df = resampled_frame
+                        resampled_with_polars = True
+                except PolarsResampleError as exc:
+                    self.logger.debug(
+                        "Unsupported polars resample for %s (%s); falling back to pandas.",
+                        asset,
+                        exc,
+                    )
+                except Exception as exc:
+                    self.logger.warning(
+                        "Polars resample failed for %s; falling back to pandas. Error: %s",
+                        asset,
+                        exc,
+                    )
+            if not resampled_with_polars:
+                try:
+                    import pandas as pd
+
+                    df = bars.pandas_df.copy()
+                    if not isinstance(df.index, pd.DatetimeIndex):
+                        df.index = pd.to_datetime(df.index)
+
+                    if base_unit == "minute":
+                        resample_rule = f"{multiplier}min"
+                    elif base_unit == "day":
+                        resample_rule = f"{multiplier}D"
+                    else:
+                        return bars
+
+                    resampled_df = df.resample(resample_rule, label='left', closed='left').agg({
+                        'open': 'first',
+                        'high': 'max',
+                        'low': 'min',
+                        'close': 'last',
+                        'volume': 'sum'
+                    }).dropna()
+
+                    if len(resampled_df) > length:
+                        resampled_df = resampled_df.iloc[-length:]
+
+                    bars.df = resampled_df
+                except Exception as e:
+                    # If resampling fails, log warning and return original data
+                    self.logger.warning(f"Failed to resample data from {actual_timestep} to {original_timestep}: {e}")
 
         return bars
 

lumibot/tools/backtest_cache.py

@@ -0,0 +1,284 @@
+from __future__ import annotations
+
+import os
+import threading
+from dataclasses import dataclass
+from enum import Enum
+from pathlib import Path
+from typing import Callable, Dict, Optional
+
+from lumibot.constants import LUMIBOT_CACHE_FOLDER
+from lumibot.credentials import CACHE_REMOTE_CONFIG
+from lumibot.tools.lumibot_logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class CacheMode(str, Enum):
+    DISABLED = "disabled"
+    S3_READWRITE = "s3_readwrite"
+    S3_READONLY = "s3_readonly"
+
+
+@dataclass(frozen=True)
+class BacktestCacheSettings:
+    backend: str
+    mode: CacheMode
+    bucket: Optional[str] = None
+    prefix: str = ""
+    region: Optional[str] = None
+    access_key_id: Optional[str] = None
+    secret_access_key: Optional[str] = None
+    session_token: Optional[str] = None
+    version: str = "v1"
+
+    @staticmethod
+    def from_env(env: Dict[str, Optional[str]]) -> Optional["BacktestCacheSettings"]:
+        backend = (env.get("backend") or "local").strip().lower()
+        mode_raw = (env.get("mode") or "disabled").strip().lower()
+
+        if backend != "s3":
+            return None
+
+        if mode_raw in ("disabled", "off", "local"):
+            return None
+
+        if mode_raw in ("readwrite", "rw", "s3_readwrite"):
+            mode = CacheMode.S3_READWRITE
+        elif mode_raw in ("readonly", "ro", "s3_readonly"):
+            mode = CacheMode.S3_READONLY
+        else:
+            raise ValueError(
+                f"Unsupported LUMIBOT_CACHE_MODE '{mode_raw}'. "
+                "Expected one of: disabled, readwrite, readonly."
+            )
+
+        bucket = (env.get("s3_bucket") or "").strip()
+        if not bucket:
+            raise ValueError("LUMIBOT_CACHE_S3_BUCKET must be set when using the S3 cache backend.")
+
+        prefix = (env.get("s3_prefix") or "").strip().strip("/")
+        region = (env.get("s3_region") or "").strip() or None
+        access_key_id = (env.get("s3_access_key_id") or "").strip() or None
+        secret_access_key = (env.get("s3_secret_access_key") or "").strip() or None
+        session_token = (env.get("s3_session_token") or "").strip() or None
+        version = (env.get("s3_version") or "v1").strip().strip("/")
+
+        if not version:
+            version = "v1"
+
+        return BacktestCacheSettings(
+            backend=backend,
+            mode=mode,
+            bucket=bucket,
+            prefix=prefix,
+            region=region,
+            access_key_id=access_key_id,
+            secret_access_key=secret_access_key,
+            session_token=session_token,
+            version=version,
+        )
+
+
+class _StubbedS3ErrorCodes:
+    NOT_FOUND = {"404", "400", "NoSuchKey", "NotFound"}
+
+
+class BacktestCacheManager:
+    def __init__(
+        self,
+        settings: Optional[BacktestCacheSettings],
+        client_factory: Optional[Callable[[BacktestCacheSettings], object]] = None,
+    ) -> None:
+        self._settings = settings
+        self._client_factory = client_factory
+        self._client = None
+        self._client_lock = threading.Lock()
+
+    @property
+    def enabled(self) -> bool:
+        return bool(self._settings and self._settings.mode != CacheMode.DISABLED)
+
+    @property
+    def mode(self) -> CacheMode:
+        if not self.enabled:
+            return CacheMode.DISABLED
+        return self._settings.mode  # type: ignore[return-value]
+
+    def ensure_local_file(
+        self,
+        local_path: Path,
+        payload: Optional[Dict[str, object]] = None,
+        force_download: bool = False,
+    ) -> bool:
+        if not self.enabled:
+            return False
+
+        if not isinstance(local_path, Path):
+            local_path = Path(local_path)
+
+        if local_path.exists() and not force_download:
+            return False
+
+        remote_key = self.remote_key_for(local_path, payload)
+        if remote_key is None:
+            return False
+
+        client = self._get_client()
+        tmp_path = local_path.with_suffix(local_path.suffix + ".s3tmp")
+        local_path.parent.mkdir(parents=True, exist_ok=True)
+
+        try:
+            client.download_file(self._settings.bucket, remote_key, str(tmp_path))
+            os.replace(tmp_path, local_path)
+            logger.debug(
+                "[REMOTE_CACHE][DOWNLOAD] %s -> %s", remote_key, local_path.as_posix()
+            )
+            return True
+        except Exception as exc:  # pragma: no cover - narrow in helper
+            if tmp_path.exists():
+                tmp_path.unlink(missing_ok=True)  # type: ignore[attr-defined]
+            if self._is_not_found_error(exc):
+                logger.debug(
+                    "[REMOTE_CACHE][MISS] %s (reason=%s)", remote_key, self._describe_error(exc)
+                )
+                return False
+            raise
+
+    def on_local_update(
+        self,
+        local_path: Path,
+        payload: Optional[Dict[str, object]] = None,
+    ) -> bool:
+        if not self.enabled or self.mode != CacheMode.S3_READWRITE:
+            return False
+
+        if not isinstance(local_path, Path):
+            local_path = Path(local_path)
+
+        if not local_path.exists():
+            logger.warning(
+                "[REMOTE_CACHE][UPLOAD_SKIP] Local file %s does not exist.", local_path.as_posix()
+            )
+            return False
+
+        remote_key = self.remote_key_for(local_path, payload)
+        if remote_key is None:
+            return False
+
+        client = self._get_client()
+        client.upload_file(str(local_path), self._settings.bucket, remote_key)
+        logger.debug(
+            "[REMOTE_CACHE][UPLOAD] %s <- %s", remote_key, local_path.as_posix()
+        )
+        return True
+
+    def remote_key_for(
+        self,
+        local_path: Path,
+        payload: Optional[Dict[str, object]] = None,
+    ) -> Optional[str]:
+        if not self.enabled:
+            return None
+
+        if not isinstance(local_path, Path):
+            local_path = Path(local_path)
+
+        try:
+            relative_path = local_path.resolve().relative_to(Path(LUMIBOT_CACHE_FOLDER).resolve())
+        except ValueError:
+            logger.debug(
+                "[REMOTE_CACHE][SKIP] %s is outside the cache root.", local_path.as_posix()
+            )
+            return None
+
+        components = [
+            self._settings.prefix if self._settings and self._settings.prefix else None,
+            self._settings.version if self._settings else None,
+            relative_path.as_posix(),
+        ]
+        sanitized = [c.strip("/") for c in components if c]
+        remote_key = "/".join(sanitized)
+        return remote_key
+
+    def _get_client(self):
+        if not self.enabled:
+            raise RuntimeError("Remote cache manager is disabled.")
+
+        if self._client is None:
+            with self._client_lock:
+                if self._client is None:
+                    if self._client_factory:
+                        self._client = self._client_factory(self._settings)
+                    else:
+                        self._client = self._create_s3_client()
+        return self._client
+
+    def _create_s3_client(self):
+        try:
+            import boto3  # type: ignore
+        except ImportError as exc:  # pragma: no cover - exercised when boto3 missing
+            raise RuntimeError(
+                "S3 cache backend requires boto3. Install it or disable the remote cache."
+            ) from exc
+
+        session = boto3.session.Session(
+            aws_access_key_id=self._settings.access_key_id,
+            aws_secret_access_key=self._settings.secret_access_key,
+            aws_session_token=self._settings.session_token,
+            region_name=self._settings.region,
+        )
+        return session.client("s3")
+
+    @staticmethod
+    def _is_not_found_error(exc: Exception) -> bool:
+        # Prefer botocore error codes if available
+        response = getattr(exc, "response", None)
+        if isinstance(response, dict):
+            error = response.get("Error") or {}
+            code = error.get("Code")
+            if isinstance(code, str) and code in _StubbedS3ErrorCodes.NOT_FOUND:
+                return True
+
+        # Handle stubbed errors (FileNotFoundError or message-based)
+        if isinstance(exc, FileNotFoundError):
+            return True
+
+        message = str(exc)
+        for token in _StubbedS3ErrorCodes.NOT_FOUND:
+            if token in message:
+                return True
+        return False
+
+    @staticmethod
+    def _describe_error(exc: Exception) -> str:
+        response = getattr(exc, "response", None)
+        if isinstance(response, dict):
+            error = response.get("Error") or {}
+            code = error.get("Code")
+            message = error.get("Message")
+            return f"{code}: {message}" if code or message else "unknown"
+        return str(exc)
+
+
+_MANAGER_LOCK = threading.Lock()
+_MANAGER_INSTANCE: Optional[BacktestCacheManager] = None
+
+
+def get_backtest_cache() -> BacktestCacheManager:
+    global _MANAGER_INSTANCE
+    if _MANAGER_INSTANCE is None:
+        with _MANAGER_LOCK:
+            if _MANAGER_INSTANCE is None:
+                settings = BacktestCacheSettings.from_env(CACHE_REMOTE_CONFIG)
+                _MANAGER_INSTANCE = BacktestCacheManager(settings)
+    return _MANAGER_INSTANCE
+
+
+def reset_backtest_cache_manager(for_testing: bool = False) -> None:
+    """Reset the cached manager instance (intended for unit tests)."""
+    global _MANAGER_INSTANCE
+    with _MANAGER_LOCK:
+        _MANAGER_INSTANCE = None
+        if not for_testing:
+            logger.debug("[REMOTE_CACHE] Manager reset requested.")

lumibot/tools/databento_helper.py

@@ -9,12 +9,18 @@ from decimal import Decimal
 import pandas as pd
 from lumibot import LUMIBOT_CACHE_FOLDER
 from lumibot.entities import Asset
-from lumibot.tools import databento_roll
+from lumibot.tools import futures_roll
+from termcolor import colored
 
 # Set up module-specific logger
 from lumibot.tools.lumibot_logger import get_logger
 logger = get_logger(__name__)
 
+
+class DataBentoAuthenticationError(RuntimeError):
+    """Raised when DataBento rejects authentication credentials."""
+    pass
+
 # DataBento imports (will be installed as dependency)
 try:
     import databento as db
@@ -161,11 +167,16 @@ class DataBentoClient:
                         continue
                     else:
                         logger.error(f"DataBento authentication failed after {self.max_retries} retries")
+                        raise DataBentoAuthenticationError(
+                            f"DataBento authentication failed after {self.max_retries} retries: {str(e)}"
+                        ) from e
                         
                 # For non-auth errors, don't retry - fail fast
-                logger.error("DATABENTO_API_ERROR: DataBento API error: %s | Symbols: %s, Start: %s, End: %s", 
-                            str(e), symbols, start, end)
-                raise e
+                logger.error(
+                    "DATABENTO_API_ERROR: DataBento API error: %s | Symbols: %s, Start: %s, End: %s",
+                    str(e), symbols, start, end
+                )
+                raise
         
         # This should never be reached, but just in case
         raise Exception(f"DataBento request failed after {self.max_retries} retries")
@@ -805,8 +816,17 @@ def get_price_data_from_databento(
 
     if roll_asset.asset_type == Asset.AssetType.CONT_FUTURE:
         schedule_start = start
-        symbols = databento_roll.resolve_symbols_for_range(roll_asset, schedule_start, end)
-        front_symbol = databento_roll.resolve_symbol_for_datetime(roll_asset, reference_date or start)
+        symbols = futures_roll.resolve_symbols_for_range(
+            roll_asset,
+            schedule_start,
+            end,
+            year_digits=1,
+        )
+        front_symbol = futures_roll.resolve_symbol_for_datetime(
+            roll_asset,
+            reference_date or start,
+            year_digits=1,
+        )
         if front_symbol not in symbols:
             symbols.insert(0, front_symbol)
     else:
@@ -879,6 +899,13 @@ def get_price_data_from_databento(
                     end=end_naive,
                     **kwargs,
                 )
+            except DataBentoAuthenticationError as exc:
+                auth_msg = colored(
+                    f"❌ DataBento authentication failed while requesting {symbol}: {exc}",
+                    "red"
+                )
+                logger.error(auth_msg)
+                raise
             except Exception as exc:
                 logger.warning(f"Error fetching {symbol} from DataBento: {exc}")
                 continue
@@ -900,38 +927,11 @@ def get_price_data_from_databento(
     combined = pd.concat(frames, axis=0)
     combined.sort_index(inplace=True)
 
-    definition_client: Optional[DataBentoClient] = None
-
-    def get_definition(symbol_code: str) -> Optional[Dict]:
-        nonlocal definition_client
-        cache_key = (symbol_code, dataset)
-        if cache_key in _INSTRUMENT_DEFINITION_CACHE:
-            return _INSTRUMENT_DEFINITION_CACHE[cache_key]
-        if definition_client is None:
-            try:
-                definition_client = DataBentoClient(api_key=api_key)
-            except Exception as exc:
-                logger.warning(f"Unable to create DataBento definition client: {exc}")
-                return None
-        try:
-            definition = definition_client.get_instrument_definition(
-                dataset=dataset,
-                symbol=symbol_code,
-                reference_date=reference_date or start,
-            )
-        except Exception as exc:
-            logger.warning(f"Failed to fetch definition for {symbol_code}: {exc}")
-            return None
-        if definition:
-            _INSTRUMENT_DEFINITION_CACHE[cache_key] = definition
-        return definition
-
-    schedule = databento_roll.build_roll_schedule(
+    schedule = futures_roll.build_roll_schedule(
         roll_asset,
         schedule_start,
         end,
-        definition_provider=get_definition,
-        roll_days=databento_roll.ROLL_DAYS_BEFORE_EXPIRATION,
+        year_digits=1,
     )
 
     if schedule: