investing-algorithm-framework 7.19.14__py3-none-any.whl

investing_algorithm_framework/domain/backtesting/backtest_permutation_test.py

@@ -0,0 +1,275 @@
+import os
+import json
+
+from dataclasses import dataclass, field
+from typing import List, Dict
+import numpy as np
+import pandas as pd
+from datetime import timezone
+
+from .backtest_metrics import BacktestMetrics
+
+
+@dataclass
+class BacktestPermutationTest:
+    """
+    Represents the result of a permutation test on backtest metrics.
+
+    Attributes:
+        real_metrics (BacktestMetrics): The real backtest metrics.
+        permutated_metrics (List[BacktestMetrics]): A list of backtest
+            metrics objects from permuted backtests.
+        p_values (Dict[str, float]): A dictionary mapping metric names
+            to their permutation test p-values.
+    """
+
+    # Default set of metrics for permutation testing
+    DEFAULT_METRICS: List[str] = field(default_factory=lambda: [
+        "cagr",
+        "sharpe_ratio",
+        "sortino_ratio",
+        "calmar_ratio",
+        "profit_factor",
+        "annual_volatility",
+        "max_drawdown",
+        "win_rate",
+        "win_loss_ratio",
+        "average_monthly_return"
+    ])
+    real_metrics: BacktestMetrics = None
+    permutated_metrics: List[BacktestMetrics] = field(default_factory=list)
+    p_values: Dict[str, float] = field(default_factory=dict)
+    ohlcv_permutated_datasets: Dict[str, List[pd.DataFrame]] = \
+        field(default_factory=dict)
+    ohlcv_original_datasets: Dict[str, pd.DataFrame] = \
+        field(default_factory=dict)
+    backtest_start_date: pd.Timestamp = None
+    backtest_end_date: pd.Timestamp = None
+    backtest_date_range_name: str = None
+
+    def compute_p_values(
+        self, metrics: List[str] = None, one_sided: bool = True
+    ) -> None:
+        """
+        Compute p-values for the selected metrics based on the
+        permutation distribution.
+
+        Args:
+            metrics (List[str]): List of metric names to compute p-values for.
+                If None, uses DEFAULT_METRICS.
+            one_sided (bool): Whether to compute a one-sided
+                test (default: True).
+        """
+        if metrics is None:
+            metrics = self.DEFAULT_METRICS
+
+        self.p_values = {}
+
+        for metric in metrics:
+            real_value = getattr(self.real_metrics, metric, None)
+            if real_value is None:
+                continue
+
+            # Collect metric values across all permuted backtests
+            dist = np.array([
+                getattr(pm, metric, None)
+                for pm in self.permutated_metrics
+                if getattr(pm, metric, None) is not None
+            ])
+
+            if len(dist) == 0:
+                continue
+
+            if one_sided:
+                p = np.mean(dist >= real_value)
+            else:
+                p = np.mean(np.abs(dist) >= abs(real_value))
+
+            self.p_values[metric] = float(p)
+
+    def summary(
+        self, metrics: List[str] = None
+    ) -> Dict[str, Dict[str, float]]:
+        """
+        Return a summary of real values, mean permuted values, and p-values.
+
+        Args:
+            metrics (List[str]): List of metric names to include
+                in the summary. If None, uses DEFAULT_METRICS.
+
+        Returns:
+            Dict[str, Dict[str, float]]: A dictionary where each key
+                is a metric name and the value is another dictionary
+                with keys 'real', 'permuted_mean', and 'p_value'.
+        """
+
+        if metrics is None:
+            metrics = self.DEFAULT_METRICS
+
+        if not self.p_values:  # lazy compute
+            self.compute_p_values(metrics=metrics)
+
+        summary_dict = {}
+        for metric in metrics:
+            real_value = getattr(self.real_metrics, metric, None)
+            if real_value is None:
+                continue
+
+            dist = np.array([
+                getattr(pm, metric, None)
+                for pm in self.permutated_metrics
+                if getattr(pm, metric, None) is not None
+            ])
+
+            # Filter out inf / nan
+            dist = dist[np.isfinite(dist)]
+
+            real_value = getattr(self.real_metrics, metric, None)
+
+            if real_value is None or not np.isfinite(real_value):
+                continue
+
+            if len(dist) == 0:
+                continue
+
+            summary_dict[metric] = {
+                "real": float(real_value),
+                "permuted_mean": float(np.mean(dist)),
+                "p_value": self.p_values.get(metric, None),
+            }
+
+        return summary_dict
+
+    def save(self, path: str) -> None:
+        """
+        Save the permutation test results to disk (JSON + Parquet).
+
+        Args:
+            path (str): The directory path where to save the results.
+
+        Returns:
+            None
+        """
+        def ensure_iso(value):
+            if hasattr(value, "isoformat"):
+                if value.tzinfo is None:
+                    value = value.replace(tzinfo=timezone.utc)
+                return value.isoformat()
+            return value
+
+        os.makedirs(path, exist_ok=True)
+
+        # Save the real metrics
+        self.real_metrics.save(os.path.join(path, "original_metrics.json"))
+
+        permuted_dir = os.path.join(path, "permuted_metrics")
+        os.makedirs(permuted_dir, exist_ok=True)
+
+        for i, pm in enumerate(self.permutated_metrics):
+            pm.save(os.path.join(permuted_dir, f"permuted_{i}.json"))
+
+        # Save the P-values
+        with open(os.path.join(path, "p_values.json"), "w") as f:
+            json.dump(self.p_values, f)
+
+        # Create a metadata file to store additional info such as
+        # date range name, start and end dates
+        metadata = {
+            "backtest_start_date": ensure_iso(self.backtest_start_date),
+            "backtest_date_range_name": self.backtest_date_range_name,
+            "backtest_end_date": ensure_iso(self.backtest_end_date),
+        }
+
+        with open(os.path.join(path, "metadata.json"), "w") as f:
+            json.dump(metadata, f)
+
+    @staticmethod
+    def open(path: str) -> "BacktestPermutationTest":
+        """
+        Load the permutation test results from disk (JSON + Parquet).
+
+        Args:
+            path (str): The directory path where the results are saved.
+
+        Returns:
+            BacktestPermutationTest: The loaded permutation test results.
+        """
+        original_metrics = os.path.join(path, "original_metrics.json")
+
+        # Rehydrate BacktestMetrics
+        real_metrics = BacktestMetrics.open(original_metrics)
+
+        permuted_dir = os.path.join(path, "permuted_metrics")
+
+        permutated_metrics = []
+        if os.path.exists(permuted_dir):
+            for fname in os.listdir(permuted_dir):
+                if fname.startswith("permuted_"):
+                    pm = BacktestMetrics.open(
+                        os.path.join(permuted_dir, fname)
+                    )
+                    permutated_metrics.append(pm)
+
+        p_values_path = os.path.join(path, "p_values.json")
+        p_values = {}
+
+        if os.path.exists(p_values_path):
+            with open(p_values_path, "r") as f:
+                p_values = json.load(f)
+
+        # Load metadata
+        metadata_path = os.path.join(path, "metadata.json")
+        backtest_start_date = None
+        backtest_end_date = None
+        backtest_date_range_name = None
+
+        if os.path.exists(metadata_path):
+            with open(metadata_path, "r") as f:
+                metadata = json.load(f)
+
+            backtest_start_date = pd.to_datetime(
+                metadata.get("backtest_start_date"), utc=True
+            )
+            backtest_end_date = pd.to_datetime(
+                metadata.get("backtest_end_date"), utc=True
+            )
+            backtest_date_range_name = metadata.get(
+                "backtest_date_range_name"
+            )
+
+        return BacktestPermutationTest(
+            real_metrics=real_metrics,
+            permutated_metrics=permutated_metrics,
+            p_values=p_values,
+            backtest_start_date=backtest_start_date,
+            backtest_end_date=backtest_end_date,
+            backtest_date_range_name=backtest_date_range_name
+        )
+
+    def create_directory_name(self) -> str:
+        """
+        Create a directory name for the backtest run based on its attributes.
+
+        Returns:
+            str: A string representing the directory name.
+        """
+        start_str = self.real_metrics.backtest_start_date.strftime("%Y%m%d")
+        end_str = self.real_metrics.backtest_end_date.strftime("%Y%m%d")
+        dir_name = f"permutation_test_{start_str}_{end_str}"
+        return dir_name
+
+    def to_dict(self) -> Dict:
+        """
+        Convert the permutation test results to a dictionary.
+
+        Returns:
+            dict: A dictionary representation of the permutation test results.
+        """
+        return {
+            "real_metrics": self.real_metrics.to_dict(),
+            "permutated_metrics": [
+                pm.to_dict() for pm in self.permutated_metrics
+            ],
+            "p_values": self.p_values,
+            # Note: DataFrames are not included in the dict representation
+        }

investing_algorithm_framework/domain/backtesting/backtest_run.py

@@ -0,0 +1,435 @@
+import json
+import os
+from typing import Dict
+from pathlib import Path
+from datetime import datetime, timezone
+from dataclasses import dataclass, field
+from logging import getLogger
+from typing import Union, List, Optional
+
+from investing_algorithm_framework.domain.exceptions \
+    import OperationalException
+from investing_algorithm_framework.domain.models.order import Order, \
+    OrderSide, OrderStatus
+from investing_algorithm_framework.domain.models.position import Position
+from investing_algorithm_framework.domain.models.trade import Trade
+from investing_algorithm_framework.domain.models.portfolio import \
+    PortfolioSnapshot
+from investing_algorithm_framework.domain.models.trade.trade_status import \
+    TradeStatus
+
+
+from .backtest_metrics import BacktestMetrics
+
+
+logger = getLogger(__name__)
+
+
+@dataclass
+class BacktestRun:
+    """
+    Represents a backtest of an algorithm. It contains the backtest metrics,
+    backtest results, and paths to strategy and data files.
+
+    Attributes:
+        backtest_metrics (Optional[List[BacktestMetrics]]): A list of
+            backtest metrics objects, each representing the performance
+            metrics of a single backtest run.
+        backtest_start_date (datetime): The start date of the backtest.
+        backtest_end_date (datetime): The end date of the backtest.
+        backtest_date_range_name (str): The name of the date range used for
+            the backtest.
+        trading_symbol (str): The trading symbol used in the backtest.
+        initial_unallocated (float): The initial unallocated amount in the
+            backtest.
+        number_of_runs (int): The number of runs in the backtest.
+        portfolio_snapshots (List[PortfolioSnapshot]): A list of portfolio
+            snapshots taken during the backtest.
+        trades (List[Trade]): A list of trades executed during the backtest.
+        orders (List[Order]): A list of orders placed during the backtest.
+        positions (List[Position]): A list of positions held during the
+            backtest.
+        created_at (datetime): The date and time when the backtest was created.
+        symbols (List[str]): A list of trading symbols involved in
+            the backtest.
+        number_of_days (int): The total number of days the backtest ran.
+        number_of_trades (int): The total number of trades executed during
+            the backtest.
+        number_of_trades_closed (int): The total number of trades that were
+            closed during the backtest.
+        number_of_trades_open (int): The total number of trades that are
+            still open at the end of the backtest.
+        number_of_orders (int): The total number of orders placed during
+            the backtest.
+        number_of_positions (int): The total number of positions held
+            during the backtest.
+    """
+    backtest_start_date: datetime
+    backtest_end_date: datetime
+    trading_symbol: str
+    initial_unallocated: float = 0.0
+    number_of_runs: int = 0
+    portfolio_snapshots: List[PortfolioSnapshot] = field(default_factory=list)
+    trades: List[Trade] = field(default_factory=list)
+    orders: List[Order] = field(default_factory=list)
+    positions: List[Position] = field(default_factory=list)
+    created_at: datetime = None,
+    symbols: List[str] = field(default_factory=list)
+    number_of_days: int = 0
+    number_of_trades: int = 0
+    number_of_trades_closed: int = 0
+    number_of_trades_open: int = 0
+    number_of_orders: int = 0
+    number_of_positions: int = 0
+    backtest_metrics: BacktestMetrics = None
+    backtest_date_range_name: str = None
+    data_sources: List[Dict] = field(default_factory=list)
+    metadata: Dict[str, str] = field(default_factory=dict)
+
+    def to_dict(self) -> dict:
+        """
+        Convert the Backtest instance to a dictionary with all
+        date/datetime fields as ISO strings (always UTC).
+        """
+        def ensure_iso(value):
+            if hasattr(value, "isoformat"):
+                if value.tzinfo is None:
+                    value = value.replace(tzinfo=timezone.utc)
+                return value.isoformat()
+            return value
+
+        backtest_metrics = self.backtest_metrics.to_dict() \
+            if self.backtest_metrics else None
+        return {
+            "backtest_metrics": backtest_metrics,
+            "backtest_start_date": ensure_iso(self.backtest_start_date),
+            "backtest_date_range_name": self.backtest_date_range_name,
+            "backtest_end_date": ensure_iso(self.backtest_end_date),
+            "trading_symbol": self.trading_symbol,
+            "initial_unallocated": self.initial_unallocated,
+            "number_of_runs": self.number_of_runs,
+            "portfolio_snapshots": [
+                ps.to_dict() for ps in self.portfolio_snapshots
+            ],
+            "trades": [trade.to_dict() for trade in self.trades],
+            "orders": [order.to_dict() for order in self.orders],
+            "positions": [position.to_dict() for position in self.positions],
+            "created_at": ensure_iso(self.created_at),
+            "symbols": self.symbols,
+            "number_of_days": self.number_of_days,
+            "number_of_trades": self.number_of_trades,
+            "number_of_trades_closed": self.number_of_trades_closed,
+            "number_of_trades_open": self.number_of_trades_open,
+            "number_of_orders": self.number_of_orders,
+            "number_of_positions": self.number_of_positions,
+            "metadata": self.metadata,
+        }
+
+    @staticmethod
+    def open(directory_path: Union[str, Path]) -> 'BacktestRun':
+        """
+        Open a backtest report from a directory and return a Backtest instance.
+
+        Args:
+            directory_path (str): The path to the directory containing the
+                backtest report files.
+
+        Returns:
+            Backtest: An instance of Backtest with the loaded metrics
+                and results.
+
+        Raises:
+            OperationalException: If the directory does not exist or if
+            there is an error loading the files.
+        """
+        backtest_metrics = None
+
+        if not os.path.exists(directory_path):
+            raise OperationalException(
+                f"The directory {directory_path} does not exist."
+            )
+
+        # Load combined backtest metrics
+        metrics_file = os.path.join(directory_path, "metrics.json")
+
+        if os.path.isfile(metrics_file):
+            backtest_metrics = BacktestMetrics.open(metrics_file)
+
+        # Load backtest results
+        run_file = os.path.join(directory_path, "run.json")
+
+        if os.path.isfile(run_file):
+            data = json.load(open(run_file, 'r'))
+        else:
+            raise OperationalException(
+                f"The run file {run_file} does not exist."
+            )
+
+        # Parse datetime fields
+        data["backtest_start_date"] = datetime.strptime(
+            data["backtest_start_date"], "%Y-%m-%d %H:%M:%S"
+        )
+        data["backtest_end_date"] = datetime.strptime(
+            data["backtest_end_date"], "%Y-%m-%d %H:%M:%S"
+        )
+        data["created_at"] = datetime.strptime(
+            data["created_at"], "%Y-%m-%d %H:%M:%S"
+        )
+        # Convert all to utc timezone
+        data["backtest_start_date"] = data[
+            "backtest_start_date"].replace(tzinfo=timezone.utc)
+        data["backtest_end_date"] = data[
+            "backtest_end_date"].replace(tzinfo=timezone.utc)
+        data["created_at"] = data["created_at"].replace(tzinfo=timezone.utc)
+
+        # Parse orders
+        data["orders"] = [
+            Order.from_dict(order) for order in data.get("orders", [])
+        ]
+
+        # Parse positions
+        data["positions"] = [
+            Position.from_dict(position)
+            for position in data.get("positions", [])
+        ]
+
+        # Parse trades
+        data["trades"] = [
+            Trade.from_dict(trade) for trade in data.get("trades", [])
+        ]
+
+        # Parse portfolio snapshots
+        data["portfolio_snapshots"] = [
+            PortfolioSnapshot.from_dict(ps)
+            for ps in data.get("portfolio_snapshots", [])
+        ]
+
+        return BacktestRun(
+            backtest_metrics=backtest_metrics,
+            **data
+        )
+
+    def create_directory_name(self) -> str:
+        """
+        Create a directory name for the backtest run based on its attributes.
+
+        Returns:
+            str: A string representing the directory name.
+        """
+        start_str = self.backtest_start_date.strftime("%Y%m%d")
+        end_str = self.backtest_end_date.strftime("%Y%m%d")
+        dir_name = f"backtest_{self.trading_symbol}_{start_str}_{end_str}"
+        return dir_name
+
+    def save(self, directory_path: Union[str, Path]) -> None:
+        """
+        Save the backtest run to a directory.
+
+        Args:
+            directory_path (str): The directory where the metrics
+            file will be saved.
+
+        Raises:
+            OperationalException: If the directory does not exist or if
+            there is an error saving the files.
+
+        Returns:
+            None: This method does not return anything, it saves the
+            metrics to a file.
+        """
+
+        metrics_path = os.path.join(directory_path, "metrics.json")
+        run_path = os.path.join(directory_path, "run.json")
+
+        if not os.path.exists(directory_path):
+            os.makedirs(directory_path)
+
+        # Call the save method of BacktestMetrics
+        if self.backtest_metrics:
+            self.backtest_metrics.save(metrics_path)
+
+        # Save the run data
+        with open(run_path, 'w') as f:
+            # string format datetime objects
+            data = self.to_dict()
+
+            # Remove backtest_metrics to avoid redundancy
+            data.pop("backtest_metrics", None)
+
+            # Ensure datetime objects are in UTC before formatting
+            backtest_start_date = self.backtest_start_date
+
+            if backtest_start_date.tzinfo is None:
+                # Naive datetime - treat as UTC
+                backtest_start_date = backtest_start_date.replace(
+                    tzinfo=timezone.utc
+                )
+            else:
+                # Timezone-aware - convert to UTC
+                backtest_start_date = backtest_start_date.astimezone(
+                    timezone.utc
+                )
+
+            backtest_end_date = self.backtest_end_date
+            if backtest_end_date.tzinfo is None:
+                backtest_end_date = backtest_end_date.replace(
+                    tzinfo=timezone.utc
+                )
+            else:
+                backtest_end_date = backtest_end_date.astimezone(timezone.utc)
+
+            created_at = self.created_at
+            if created_at.tzinfo is None:
+                created_at = created_at.replace(tzinfo=timezone.utc)
+            else:
+                created_at = created_at.astimezone(timezone.utc)
+
+            data["backtest_start_date"] = backtest_start_date.strftime(
+                "%Y-%m-%d %H:%M:%S"
+            )
+            data["backtest_end_date"] = backtest_end_date.strftime(
+                "%Y-%m-%d %H:%M:%S"
+            )
+            data["created_at"] = created_at.strftime(
+                "%Y-%m-%d %H:%M:%S"
+            )
+            json.dump(data, f, default=str)
+
+    def get_trades(self, target_symbol=None, trade_status=None) -> List[Trade]:
+        """
+        Get the trades of a backtest report
+
+        Args:
+            target_symbol (str): The target_symbol
+            trade_status: The trade_status
+
+        Returns:
+            list: The trades of the backtest report
+        """
+        selection = self.trades
+
+        if target_symbol is not None:
+            selection = [
+                trade for trade in selection
+                if trade.target_symbol.lower() == target_symbol.lower()
+            ]
+
+        if trade_status is not None:
+            trade_status = TradeStatus.from_value(trade_status)
+            selection = [
+                trade for trade in selection
+                if trade.status == trade_status.value
+            ]
+
+        return selection
+
+    def get_portfolio_snapshots(
+        self,
+        created_at_lt: Optional[datetime] = None,
+        created_at_lte: Optional[datetime] = None,
+        created_at_gt: Optional[datetime] = None,
+        created_at_gte: Optional[datetime] = None
+    ) -> List[PortfolioSnapshot]:
+        """
+        Get the portfolio snapshots of the backtest report
+
+        Args:
+            created_at_lt (datetime): The created_at date to filter
+                the snapshots
+            created_at_lte (datetime): The created_at date to filter
+                the snapshots
+            created_at_gt (datetime): The created_at date to filter
+                the snapshots
+            created_at_gte (datetime): The created_at date to filter
+                the snapshots
+
+        Returns:
+            list: The portfolio snapshots of the backtest report
+        """
+        selection = self.portfolio_snapshots
+
+        if created_at_lt is not None:
+            selection = [
+                snapshot for snapshot in selection
+                if snapshot.created_at < created_at_lt
+            ]
+
+        if created_at_lte is not None:
+            selection = [
+                snapshot for snapshot in selection
+                if snapshot.created_at <= created_at_lte
+            ]
+
+        if created_at_gt is not None:
+            selection = [
+                snapshot for snapshot in selection
+                if snapshot.created_at > created_at_gt
+            ]
+
+        if created_at_gte is not None:
+            selection = [
+                snapshot for snapshot in selection
+                if snapshot.created_at >= created_at_gte
+            ]
+
+        return selection
+
+    def get_orders(
+        self,
+        target_symbol=None,
+        order_side=None,
+        order_status=None,
+        created_at_lt=None,
+    ) -> List[Order]:
+        """
+        Get the orders of a backtest report
+
+        Args:
+            target_symbol (str): The target_symbol
+            order_side (str): The order side
+            order_status (str): The order status
+            created_at_lt (datetime): The created_at date to filter the orders
+
+        Returns:
+            list: The orders of the backtest report
+        """
+        selection = self.orders
+
+        if created_at_lt is not None:
+            selection = [
+                order for order in selection
+                if order.created_at < created_at_lt
+            ]
+
+        if target_symbol is not None:
+            selection = [
+                order for order in selection
+                if order.target_symbol == target_symbol
+            ]
+
+        if order_side is not None:
+            order_side = OrderSide.from_value(order_side)
+            selection = [
+                order for order in selection
+                if order.order_side == order_side.value
+            ]
+
+        if order_status is not None:
+            status = OrderStatus.from_value(order_status)
+            selection = [
+                order for order in selection
+                if order.status == status.value
+            ]
+
+        return selection
+
+    def __repr__(self):
+        """
+        Return a string representation of the Backtest instance.
+
+        Returns:
+            str: A string representation of the Backtest instance.
+        """
+        return json.dumps(
+            self.to_dict(), indent=4, sort_keys=True, default=str
+        )