lumibot 4.1.2__py3-none-any.whl → 4.2.0__py3-none-any.whl

lumibot/entities/position.py

@@ -1,6 +1,7 @@
 from decimal import Decimal
 
 import lumibot.entities as entities
+from lumibot.entities.asset import StrEnum #todo: this should be centralized, and not repeated in Asset and Position
 
 
 class Position:
@@ -26,8 +27,36 @@ class Position:
         The assets that are free in the portfolio. (Crypto: only)
     avg_fill_price : float
         The average fill price of the position.
+    current_price : float
+        The current price of the asset.
+    market_value : float
+        The market value of the position.
+    pnl : float
+        The profit and loss of the position.
+    pnl_percent : float
+        The profit and loss of the position as a percentage of the average fill price.
+    asset_type : str
+        The type of the asset.
+    exchange : str
+        The exchange that the position is on.
+    currency : str
+        The currency that the position is denominated in.
+    multiplier : float
+        The multiplier of the asset.
+    expiration : datetime.date
+        The expiration of the asset. (Options and futures: only). Probably better to use on position.asset
+    strike : float
+        The strike price of the asset. (Options: only). Probably better to use on position.asset
+    option_type : str
+        The type of the option. (Options: only). Probably better to use on position.asset
+    side : PositionSide
+        The side of the position (LONG or SHORT)
     """
 
+    class PositionSide(StrEnum):
+        LONG = "LONG"
+        SHORT = "SHORT"
+
     def __init__(
             self,
             strategy,
@@ -38,6 +67,11 @@ class Position:
             available=0,
             avg_fill_price=None
         ):
+        """Creates a position.
+
+        NOTE: There are some properties that can be assigned to a position entity outside of the constructor (pnl, current_price, etc)
+
+        """
         self.strategy = strategy
         self.asset = asset
         self.symbol = self.asset.symbol
@@ -223,13 +257,13 @@ class Position:
             result['currency'] = self.currency
         if hasattr(self, 'multiplier'):
             result['multiplier'] = self.multiplier
-        if hasattr(self, 'expiration'):
+        if hasattr(self, 'expiration'):  #should probably use position.asset instead
             result['expiration'] = str(self.expiration) if self.expiration else None
-        if hasattr(self, 'strike'):
+        if hasattr(self, 'strike'): #should probably use position.asset instead
             result['strike'] = float(self.strike) if self.strike else None
-        if hasattr(self, 'option_type'):
+        if hasattr(self, 'option_type'): #should probably use position.asset instead
             result['option_type'] = self.option_type
-        if hasattr(self, 'underlying_symbol'):
+        if hasattr(self, 'underlying_symbol'): #should probably use position.asset instead
             result['underlying_symbol'] = self.underlying_symbol
 
         # Handle orders carefully - ensure to_dict() is called properly

lumibot/strategies/_strategy.py

@@ -819,7 +819,7 @@ class _Strategy:
 
             assets = []
             for position in positions:
-                if position.asset != self._quote_asset:
+                if position.asset != self._quote_asset and position.asset.asset_type != "option":
                     assets.append(position.asset)
 
             # Early return if no assets - avoid expensive dividend API calls
@@ -827,6 +827,7 @@ class _Strategy:
                 return self.cash
 
             dividends_per_share = self.get_yesterday_dividends(assets)
+
             for position in positions:
                 asset = position.asset
                 quantity = position.quantity
@@ -1264,9 +1265,24 @@ class _Strategy:
         if show_indicators is None:
             show_indicators = SHOW_INDICATORS
 
-        # Auto-select datasource from environment variable if None
-        if datasource_class is None:
-            from lumibot.credentials import BACKTESTING_DATA_SOURCE
+        from lumibot.credentials import BACKTESTING_DATA_SOURCE as _DEFAULT_BACKTESTING_DATA_SOURCE
+
+        # Determine whether an environment override exists. When BACKTESTING_DATA_SOURCE
+        # is set (and not blank/\"none\"), it should take precedence even if a
+        # datasource_class argument was provided.
+        env_override_raw = os.environ.get("BACKTESTING_DATA_SOURCE")
+        env_override_name = None
+
+        if env_override_raw is not None:
+            trimmed = env_override_raw.strip()
+            if trimmed and trimmed.lower() != "none":
+                env_override_name = trimmed.lower()
+        elif datasource_class is None:
+            # No override provided and no class in code – fall back to the default
+            # configured in credentials (ThetaData unless the project overrides it).
+            env_override_name = _DEFAULT_BACKTESTING_DATA_SOURCE.lower()
+
+        if env_override_name is not None:
             from lumibot.backtesting import (
                 PolygonDataBacktesting,
                 ThetaDataBacktesting,
@@ -1285,18 +1301,24 @@ class _Strategy:
                 "databento": DataBentoDataBacktesting,
             }
 
-            datasource_name = BACKTESTING_DATA_SOURCE.lower()
-            if datasource_name not in datasource_map:
+            if env_override_name not in datasource_map:
+                label = env_override_raw or _DEFAULT_BACKTESTING_DATA_SOURCE
                 raise ValueError(
-                    f"Unknown BACKTESTING_DATA_SOURCE: '{BACKTESTING_DATA_SOURCE}'. "
+                    f"Unknown BACKTESTING_DATA_SOURCE: '{label}'. "
                     f"Valid options: {list(datasource_map.keys())}"
                 )
 
-            datasource_class = datasource_map[datasource_name]
+            datasource_class = datasource_map[env_override_name]
+            label = env_override_raw or _DEFAULT_BACKTESTING_DATA_SOURCE
             get_logger(__name__).info(colored(
-                f"Auto-selected backtesting data source from BACKTESTING_DATA_SOURCE env var: {BACKTESTING_DATA_SOURCE}",
+                f"Using BACKTESTING_DATA_SOURCE setting for backtest data: {label}",
                 "green"
             ))
+        elif datasource_class is None:
+            raise ValueError(
+                "No backtesting data source provided. Set BACKTESTING_DATA_SOURCE in the environment "
+                "or pass datasource_class when calling backtest()."
+            )
 
         # Make sure polygon_api_key is set if using PolygonDataBacktesting
         polygon_api_key = polygon_api_key if polygon_api_key is not None else POLYGON_API_KEY

lumibot/strategies/strategy.py

@@ -19,6 +19,7 @@ from termcolor import colored, COLORS
 from ..data_sources import DataSource
 from ..entities import Asset, Data, Order, Position, Quote, TradingFee
 from ..tools import get_risk_free_rate
+from ..tools.polars_utils import PolarsResampleError, resample_polars_ohlc
 from ..traders import Trader
 from ._strategy import _Strategy
 
@@ -3353,7 +3354,7 @@ class Strategy(_Strategy):
         asset: Union[Asset, str],
         length: int,
         timestep: str = "",
-        timeshift: datetime.timedelta = None,
+        timeshift: Union[int, datetime.timedelta, None] = None,
         quote: Asset = None,
         exchange: str = None,
         include_after_hours: bool = True,
@@ -3388,10 +3389,12 @@ class Strategy(_Strategy):
             When using multi-timeframe formats, the method automatically fetches the
             underlying minute or day data and resamples it to your desired timeframe.
             Default value depends on the data_source (minute for alpaca, day for yahoo, ...)
-        timeshift : timedelta
-            ``None`` by default. If specified indicates the time shift from
-            the present. If  backtesting in Pandas, use integer representing
-            number of bars.
+        timeshift : int, timedelta, or None
+            ``None`` by default. When provided it shifts the data window relative to
+            the current backtest time.
+
+            - Passing an ``int`` shifts by bars (positive = past, negative = future).
+            - Passing a ``timedelta`` shifts by wall-clock time (e.g. ``timedelta(hours=-1)``).
         quote : Asset
             The quote currency for crypto currencies (e.g. USD, USDT, EUR, ...).
             Default is the quote asset for the strategy.
@@ -3517,6 +3520,7 @@ class Strategy(_Strategy):
         asset = self.crypto_assets_to_tuple(asset, quote)
         if not actual_timestep:
             actual_timestep = self.broker.data_source.get_timestep()
+        effective_return_polars = return_polars
         # Call through to the appropriate data source. Only pass `return_polars` if supported
         # to maintain compatibility with live data sources that don't yet accept it.
         import inspect
@@ -3540,7 +3544,7 @@ class Strategy(_Strategy):
                 return fn(
                     asset,
                     actual_length,  # Use the actual length for fetching
-                    return_polars=return_polars,
+                    return_polars=effective_return_polars,
                     **common_kwargs,
                 )
             else:
@@ -3557,49 +3561,57 @@ class Strategy(_Strategy):
             bars = _call_get_hist(self.broker.data_source)
 
         # If we need to resample the data
-        if needs_resampling and bars and bars.df is not None and not bars.df.empty:
-            try:
-                # Import pandas for resampling
-                import pandas as pd
-                from lumibot.entities import Bars
-
-                # Get the dataframe
-                df = bars.df.copy()
-
-                # Ensure datetime index
-                if not isinstance(df.index, pd.DatetimeIndex):
-                    # Try to convert the index to datetime
-                    df.index = pd.to_datetime(df.index)
-
-                # Determine resampling rule
-                if base_unit == "minute":
-                    resample_rule = f'{multiplier}min'  # 'min' for minutes (pandas 2.0+ compatible)
-                elif base_unit == "day":
-                    resample_rule = f'{multiplier}D'  # D for days
-                else:
-                    # Fallback to original if we can't determine the rule
-                    return bars
-
-                # Perform resampling with proper aggregation
-                resampled = df.resample(resample_rule, label='left', closed='left').agg({
-                    'open': 'first',
-                    'high': 'max',
-                    'low': 'min',
-                    'close': 'last',
-                    'volume': 'sum'
-                }).dropna()
-
-                # Limit to requested length
-                if len(resampled) > length:
-                    resampled = resampled.iloc[-length:]
-
-                # Create new Bars object with resampled data
-                bars.df = resampled
-                bars.raw_data = resampled  # Update raw_data as well if it exists
-
-            except Exception as e:
-                # If resampling fails, log warning and return original data
-                self.logger.warning(f"Failed to resample data from {actual_timestep} to {original_timestep}: {e}")
+        if needs_resampling and bars and len(bars) > 0:
+            resampled_with_polars = False
+            if return_polars:
+                try:
+                    polars_frame = bars.polars_df
+                    resampled_frame = resample_polars_ohlc(polars_frame, multiplier, base_unit, length)
+                    if resampled_frame is not None and not resampled_frame.is_empty():
+                        bars.df = resampled_frame
+                        resampled_with_polars = True
+                except PolarsResampleError as exc:
+                    self.logger.debug(
+                        "Unsupported polars resample for %s (%s); falling back to pandas.",
+                        asset,
+                        exc,
+                    )
+                except Exception as exc:
+                    self.logger.warning(
+                        "Polars resample failed for %s; falling back to pandas. Error: %s",
+                        asset,
+                        exc,
+                    )
+            if not resampled_with_polars:
+                try:
+                    import pandas as pd
+
+                    df = bars.pandas_df.copy()
+                    if not isinstance(df.index, pd.DatetimeIndex):
+                        df.index = pd.to_datetime(df.index)
+
+                    if base_unit == "minute":
+                        resample_rule = f"{multiplier}min"
+                    elif base_unit == "day":
+                        resample_rule = f"{multiplier}D"
+                    else:
+                        return bars
+
+                    resampled_df = df.resample(resample_rule, label='left', closed='left').agg({
+                        'open': 'first',
+                        'high': 'max',
+                        'low': 'min',
+                        'close': 'last',
+                        'volume': 'sum'
+                    }).dropna()
+
+                    if len(resampled_df) > length:
+                        resampled_df = resampled_df.iloc[-length:]
+
+                    bars.df = resampled_df
+                except Exception as e:
+                    # If resampling fails, log warning and return original data
+                    self.logger.warning(f"Failed to resample data from {actual_timestep} to {original_timestep}: {e}")
 
         return bars
 

lumibot/tools/backtest_cache.py

@@ -0,0 +1,284 @@
+from __future__ import annotations
+
+import os
+import threading
+from dataclasses import dataclass
+from enum import Enum
+from pathlib import Path
+from typing import Callable, Dict, Optional
+
+from lumibot.constants import LUMIBOT_CACHE_FOLDER
+from lumibot.credentials import CACHE_REMOTE_CONFIG
+from lumibot.tools.lumibot_logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class CacheMode(str, Enum):
+    DISABLED = "disabled"
+    S3_READWRITE = "s3_readwrite"
+    S3_READONLY = "s3_readonly"
+
+
+@dataclass(frozen=True)
+class BacktestCacheSettings:
+    backend: str
+    mode: CacheMode
+    bucket: Optional[str] = None
+    prefix: str = ""
+    region: Optional[str] = None
+    access_key_id: Optional[str] = None
+    secret_access_key: Optional[str] = None
+    session_token: Optional[str] = None
+    version: str = "v1"
+
+    @staticmethod
+    def from_env(env: Dict[str, Optional[str]]) -> Optional["BacktestCacheSettings"]:
+        backend = (env.get("backend") or "local").strip().lower()
+        mode_raw = (env.get("mode") or "disabled").strip().lower()
+
+        if backend != "s3":
+            return None
+
+        if mode_raw in ("disabled", "off", "local"):
+            return None
+
+        if mode_raw in ("readwrite", "rw", "s3_readwrite"):
+            mode = CacheMode.S3_READWRITE
+        elif mode_raw in ("readonly", "ro", "s3_readonly"):
+            mode = CacheMode.S3_READONLY
+        else:
+            raise ValueError(
+                f"Unsupported LUMIBOT_CACHE_MODE '{mode_raw}'. "
+                "Expected one of: disabled, readwrite, readonly."
+            )
+
+        bucket = (env.get("s3_bucket") or "").strip()
+        if not bucket:
+            raise ValueError("LUMIBOT_CACHE_S3_BUCKET must be set when using the S3 cache backend.")
+
+        prefix = (env.get("s3_prefix") or "").strip().strip("/")
+        region = (env.get("s3_region") or "").strip() or None
+        access_key_id = (env.get("s3_access_key_id") or "").strip() or None
+        secret_access_key = (env.get("s3_secret_access_key") or "").strip() or None
+        session_token = (env.get("s3_session_token") or "").strip() or None
+        version = (env.get("s3_version") or "v1").strip().strip("/")
+
+        if not version:
+            version = "v1"
+
+        return BacktestCacheSettings(
+            backend=backend,
+            mode=mode,
+            bucket=bucket,
+            prefix=prefix,
+            region=region,
+            access_key_id=access_key_id,
+            secret_access_key=secret_access_key,
+            session_token=session_token,
+            version=version,
+        )
+
+
+class _StubbedS3ErrorCodes:
+    NOT_FOUND = {"404", "400", "NoSuchKey", "NotFound"}
+
+
+class BacktestCacheManager:
+    def __init__(
+        self,
+        settings: Optional[BacktestCacheSettings],
+        client_factory: Optional[Callable[[BacktestCacheSettings], object]] = None,
+    ) -> None:
+        self._settings = settings
+        self._client_factory = client_factory
+        self._client = None
+        self._client_lock = threading.Lock()
+
+    @property
+    def enabled(self) -> bool:
+        return bool(self._settings and self._settings.mode != CacheMode.DISABLED)
+
+    @property
+    def mode(self) -> CacheMode:
+        if not self.enabled:
+            return CacheMode.DISABLED
+        return self._settings.mode  # type: ignore[return-value]
+
+    def ensure_local_file(
+        self,
+        local_path: Path,
+        payload: Optional[Dict[str, object]] = None,
+        force_download: bool = False,
+    ) -> bool:
+        if not self.enabled:
+            return False
+
+        if not isinstance(local_path, Path):
+            local_path = Path(local_path)
+
+        if local_path.exists() and not force_download:
+            return False
+
+        remote_key = self.remote_key_for(local_path, payload)
+        if remote_key is None:
+            return False
+
+        client = self._get_client()
+        tmp_path = local_path.with_suffix(local_path.suffix + ".s3tmp")
+        local_path.parent.mkdir(parents=True, exist_ok=True)
+
+        try:
+            client.download_file(self._settings.bucket, remote_key, str(tmp_path))
+            os.replace(tmp_path, local_path)
+            logger.debug(
+                "[REMOTE_CACHE][DOWNLOAD] %s -> %s", remote_key, local_path.as_posix()
+            )
+            return True
+        except Exception as exc:  # pragma: no cover - narrow in helper
+            if tmp_path.exists():
+                tmp_path.unlink(missing_ok=True)  # type: ignore[attr-defined]
+            if self._is_not_found_error(exc):
+                logger.debug(
+                    "[REMOTE_CACHE][MISS] %s (reason=%s)", remote_key, self._describe_error(exc)
+                )
+                return False
+            raise
+
+    def on_local_update(
+        self,
+        local_path: Path,
+        payload: Optional[Dict[str, object]] = None,
+    ) -> bool:
+        if not self.enabled or self.mode != CacheMode.S3_READWRITE:
+            return False
+
+        if not isinstance(local_path, Path):
+            local_path = Path(local_path)
+
+        if not local_path.exists():
+            logger.warning(
+                "[REMOTE_CACHE][UPLOAD_SKIP] Local file %s does not exist.", local_path.as_posix()
+            )
+            return False
+
+        remote_key = self.remote_key_for(local_path, payload)
+        if remote_key is None:
+            return False
+
+        client = self._get_client()
+        client.upload_file(str(local_path), self._settings.bucket, remote_key)
+        logger.debug(
+            "[REMOTE_CACHE][UPLOAD] %s <- %s", remote_key, local_path.as_posix()
+        )
+        return True
+
+    def remote_key_for(
+        self,
+        local_path: Path,
+        payload: Optional[Dict[str, object]] = None,
+    ) -> Optional[str]:
+        if not self.enabled:
+            return None
+
+        if not isinstance(local_path, Path):
+            local_path = Path(local_path)
+
+        try:
+            relative_path = local_path.resolve().relative_to(Path(LUMIBOT_CACHE_FOLDER).resolve())
+        except ValueError:
+            logger.debug(
+                "[REMOTE_CACHE][SKIP] %s is outside the cache root.", local_path.as_posix()
+            )
+            return None
+
+        components = [
+            self._settings.prefix if self._settings and self._settings.prefix else None,
+            self._settings.version if self._settings else None,
+            relative_path.as_posix(),
+        ]
+        sanitized = [c.strip("/") for c in components if c]
+        remote_key = "/".join(sanitized)
+        return remote_key
+
+    def _get_client(self):
+        if not self.enabled:
+            raise RuntimeError("Remote cache manager is disabled.")
+
+        if self._client is None:
+            with self._client_lock:
+                if self._client is None:
+                    if self._client_factory:
+                        self._client = self._client_factory(self._settings)
+                    else:
+                        self._client = self._create_s3_client()
+        return self._client
+
+    def _create_s3_client(self):
+        try:
+            import boto3  # type: ignore
+        except ImportError as exc:  # pragma: no cover - exercised when boto3 missing
+            raise RuntimeError(
+                "S3 cache backend requires boto3. Install it or disable the remote cache."
+            ) from exc
+
+        session = boto3.session.Session(
+            aws_access_key_id=self._settings.access_key_id,
+            aws_secret_access_key=self._settings.secret_access_key,
+            aws_session_token=self._settings.session_token,
+            region_name=self._settings.region,
+        )
+        return session.client("s3")
+
+    @staticmethod
+    def _is_not_found_error(exc: Exception) -> bool:
+        # Prefer botocore error codes if available
+        response = getattr(exc, "response", None)
+        if isinstance(response, dict):
+            error = response.get("Error") or {}
+            code = error.get("Code")
+            if isinstance(code, str) and code in _StubbedS3ErrorCodes.NOT_FOUND:
+                return True
+
+        # Handle stubbed errors (FileNotFoundError or message-based)
+        if isinstance(exc, FileNotFoundError):
+            return True
+
+        message = str(exc)
+        for token in _StubbedS3ErrorCodes.NOT_FOUND:
+            if token in message:
+                return True
+        return False
+
+    @staticmethod
+    def _describe_error(exc: Exception) -> str:
+        response = getattr(exc, "response", None)
+        if isinstance(response, dict):
+            error = response.get("Error") or {}
+            code = error.get("Code")
+            message = error.get("Message")
+            return f"{code}: {message}" if code or message else "unknown"
+        return str(exc)
+
+
+_MANAGER_LOCK = threading.Lock()
+_MANAGER_INSTANCE: Optional[BacktestCacheManager] = None
+
+
+def get_backtest_cache() -> BacktestCacheManager:
+    global _MANAGER_INSTANCE
+    if _MANAGER_INSTANCE is None:
+        with _MANAGER_LOCK:
+            if _MANAGER_INSTANCE is None:
+                settings = BacktestCacheSettings.from_env(CACHE_REMOTE_CONFIG)
+                _MANAGER_INSTANCE = BacktestCacheManager(settings)
+    return _MANAGER_INSTANCE
+
+
+def reset_backtest_cache_manager(for_testing: bool = False) -> None:
+    """Reset the cached manager instance (intended for unit tests)."""
+    global _MANAGER_INSTANCE
+    with _MANAGER_LOCK:
+        _MANAGER_INSTANCE = None
+        if not for_testing:
+            logger.debug("[REMOTE_CACHE] Manager reset requested.")