investing-algorithm-framework 7.19.2__tar.gz → 7.19.4__tar.gz

{investing_algorithm_framework-7.19.2 → investing_algorithm_framework-7.19.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: investing-algorithm-framework
-Version: 7.19.2
+Version: 7.19.4
 Summary: A framework for creating trading bots
 Author: MDUYN
 Requires-Python: >=3.10

{investing_algorithm_framework-7.19.2 → investing_algorithm_framework-7.19.4}/investing_algorithm_framework/app/app.py

@@ -2,10 +2,9 @@ import inspect
 import logging
 import os
 import threading
-from datetime import datetime, timezone
+from datetime import datetime, timezone, timedelta
 from typing import List, Optional, Any, Dict, Tuple
 
-import pandas as pd
 from flask import Flask
 
 from investing_algorithm_framework.app.algorithm import Algorithm
@@ -19,7 +18,7 @@ from investing_algorithm_framework.domain import DATABASE_NAME, TimeUnit, \
     AppMode, BacktestDateRange, DATABASE_DIRECTORY_NAME, DataSource, \
     BACKTESTING_INITIAL_AMOUNT, SNAPSHOT_INTERVAL, Backtest, DataError, \
     PortfolioConfiguration, SnapshotInterval, DataType, combine_backtests, \
-    PortfolioProvider, OrderExecutor, ImproperlyConfigured, \
+    PortfolioProvider, OrderExecutor, ImproperlyConfigured, TimeFrame, \
     DataProvider, INDEX_DATETIME, tqdm, BacktestPermutationTest, \
     LAST_SNAPSHOT_DATETIME, BACKTESTING_FLAG, generate_backtest_summary_metrics
 from investing_algorithm_framework.infrastructure import setup_sqlalchemy, \
@@ -794,8 +793,9 @@ class App:
     def check_data_completeness(
         self,
         strategies: List[TradingStrategy],
-        backtest_date_range: BacktestDateRange
-    ) -> None:
+        backtest_date_range: BacktestDateRange,
+        show_progress: bool = True
+    ) -> Tuple[bool, Dict[str, Any]]:
         """
         Function to check the data completeness for a set of strategies
         over a given backtest date range. This method checks if all data
@@ -807,10 +807,15 @@ class App:
                 to check data completeness for.
             backtest_date_range (BacktestDateRange): The date range to
                 check data completeness for.
+            show_progress (bool): Whether to show a progress bar when
+                checking data completeness.
         Returns:
-            None
+            Tuple[bool, Dict[str, Any]]: A tuple containing a boolean
+                indicating if the data is complete and a dictionary
+                with information about missing data for each data source.
         """
         data_sources = []
+        missing_data_info = {}
 
         for strategy in strategies:
             data_sources.extend(strategy.data_sources)
@@ -818,7 +823,7 @@ class App:
         self.initialize_data_sources_backtest(
             data_sources,
             backtest_date_range,
-            show_progress=True
+            show_progress=show_progress
         )
         data_provider_service = self.container.data_provider_service()
 
@@ -827,36 +832,60 @@ class App:
             for data_source in strategy.data_sources:
 
                 if DataType.OHLCV.equals(data_source.data_type):
-                    df = data_provider_service.get_ohlcv_data(
-                        symbol=data_source.symbol,
-                        start_date=backtest_date_range.start_date,
-                        end_date=backtest_date_range.end_date,
-                        pandas=True,
-                        add_pandas_index=False,
-                        time_frame=data_source.time_frame
-                    )
-                    df = df.copy()
-                    df['Datetime'] = pd.to_datetime(df['Datetime'])
-                    df = df.sort_values('Datetime')\
-                        .tail(data_source.window_size)
-                    start = df['Datetime'].iloc[0]
-                    end = df['Datetime'].iloc[-1]
-                    freq = pd.to_timedelta(data_source.time_frame.value)
-                    expected = pd.date_range(start, end, freq=freq)
-                    actual = df['Datetime']
-                    missing = expected.difference(actual)
-
-                    # Calculate the percentage completeness
-                    completeness = len(actual) / len(expected) * 100
-
-                    if completeness < 100:
+                    required_start_date = backtest_date_range.start_date - \
+                        timedelta(
+                            minutes=TimeFrame.from_value(
+                                data_source.time_frame
+                            ).amount_of_minutes * data_source.window_size
+                        )
+                    number_of_required_data_points = \
+                        data_source.get_number_of_required_data_points(
+                            backtest_date_range.start_date,
+                            backtest_date_range.end_date
+                        )
+
+                    try:
+                        data_provider = data_provider_service.get(data_source)
+                        number_of_available_data_points = \
+                            data_provider.get_number_of_data_points(
+                                backtest_date_range.start_date,
+                                backtest_date_range.end_date
+                            )
+
+                        missing_dates = \
+                            data_provider.get_missing_data_dates(
+                                required_start_date,
+                                backtest_date_range.end_date
+                            )
+                        if number_of_available_data_points > 0:
+                            missing_data_info[data_source.identifier] = {
+                                "data_source_id": data_source.identifier,
+                                "completeness_percentage": (
+                                    (
+                                        number_of_available_data_points /
+                                        number_of_required_data_points
+                                    ) * 100
+                                ),
+                                "missing_data_points": len(
+                                    missing_dates
+                                ),
+                                "missing_dates": missing_dates,
+                                "data_source_file_path":
+                                    data_provider.get_data_source_file_path()
+                            }
+
+                    except Exception as e:
                         raise DataError(
-                            f"Data completeness for data source "
+                            f"Error getting data provider for data source "
                             f"{data_source.identifier} "
-                            f"({data_source.symbol}) is {completeness:.2f}% "
-                            f"complete. Missing data points: {len(missing)}"
+                            f"({data_source.symbol}): {str(e)}"
                         )
 
+                    if len(missing_data_info.keys()) > 0:
+                        return False, missing_data_info
+
+        return True, missing_data_info
+
     def run_vector_backtests(
         self,
         initial_amount,
@@ -1070,13 +1099,10 @@ class App:
                 progress bar when initializing data sources.
             market (str): The market to use for the backtest. This is used
                 to create a portfolio configuration if no portfolio
-                configuration is found for the strategy. If not provided,
-                the first portfolio configuration found will be used.
+                configuration is provided in the strategy.
             trading_symbol (str): The trading symbol to use for the backtest.
                 This is used to create a portfolio configuration if no
-                portfolio configuration is found for the strategy. If not
-                provided, the first trading symbol found in the portfolio
-                configuration will be used.
+                portfolio configuration is provided in the strategy.
             initial_amount (float): The initial amount to start the
                 backtest with. This will be the amount of trading currency
                 that the portfolio will start with. If not provided,
@@ -1180,7 +1206,7 @@ class App:
             backtest_date_ranges (List[BacktestDateRange]): List of date ranges
             initial_amount (float): The initial amount to start the
                 backtest with. This will be the amount of trading currency
-                that the portfolio will start with.
+                that the backtest portfolio will start with.
             snapshot_interval (SnapshotInterval): The snapshot interval to use
                 for the backtest. This is used to determine how often the
                 portfolio snapshot should be taken during the backtest.
@@ -1405,11 +1431,11 @@ class App:
                 the risk-free rate from the US Treasury website.
             market (str): The market to use for the backtest. This is used
                 to create a portfolio configuration if no portfolio
-                configuration is found for the strategy. If not provided,
+                configuration is provided in the strategy. If not provided,
                 the first portfolio configuration found will be used.
             trading_symbol (str): The trading symbol to use for the backtest.
                 This is used to create a portfolio configuration if no
-                portfolio configuration is found for the strategy. If not
+                portfolio configuration is provided in the strategy. If not
                 provided, the first trading symbol found in the portfolio
                 configuration will be used.
 

{investing_algorithm_framework-7.19.2 → investing_algorithm_framework-7.19.4}/investing_algorithm_framework/domain/data_provider.py

@@ -1,4 +1,4 @@
-from typing import List, Any
+from typing import List, Any, Union
 from abc import ABC, abstractmethod
 from datetime import datetime
 from investing_algorithm_framework.domain.exceptions import \
@@ -282,3 +282,53 @@ class DataProvider(ABC):
                 configuration.
         """
         raise NotImplementedError("Subclasses should implement this method.")
+
+    @abstractmethod
+    def get_number_of_data_points(
+        self,
+        start_date: datetime,
+        end_date: datetime,
+    ) -> int:
+        """
+        Returns the number of data points available between the
+        given start and end dates.
+
+        Args:
+            start_date (datetime): The start date for the data points.
+            end_date (datetime): The end date for the data points.
+        Returns:
+            int: The number of data points available between the
+                given start and end dates.
+        """
+        raise NotImplementedError("Subclasses should implement this method.")
+
+    @abstractmethod
+    def get_missing_data_dates(
+        self,
+        start_date: datetime,
+        end_date: datetime,
+    ) -> List[datetime]:
+        """
+        Returns a list of dates for which data is missing between the
+        given start and end dates.
+
+        Args:
+            start_date (datetime): The start date for checking missing data.
+            end_date (datetime): The end date for checking missing data.
+
+        Returns:
+            List[datetime]: A list of dates for which data is missing
+                between the given start and end dates.
+        """
+        raise NotImplementedError("Subclasses should implement this method.")
+
+    @abstractmethod
+    def get_data_source_file_path(self) -> Union[str, None]:
+        """
+        Returns the file path for the given data source if applicable.
+
+        Returns:
+            Union[str, None]: The file path for the data source or None
+                if not applicable.
+        """
+        raise NotImplementedError("Subclasses should implement this method.")

{investing_algorithm_framework-7.19.2 → investing_algorithm_framework-7.19.4}/investing_algorithm_framework/domain/exceptions.py

@@ -92,9 +92,18 @@ class DataError(Exception):
     during data retrieval or processing
     """
 
-    def __init__(self, message) -> None:
+    def __init__(
+        self,
+        message,
+        data_source_file_path: str = None,
+        number_of_missing_data_points: int = None,
+        total_number_of_data_points: int = None,
+    ) -> None:
         super(DataError, self).__init__(message)
         self.error_message = message
+        self.data_source_file_path = data_source_file_path
+        self.number_of_missing_data_points = number_of_missing_data_points
+        self.total_number_of_data_points = total_number_of_data_points
 
     def to_response(self):
         return {

{investing_algorithm_framework-7.19.2 → investing_algorithm_framework-7.19.4}/investing_algorithm_framework/domain/models/data/data_source.py

@@ -179,3 +179,36 @@ class DataSource:
             (self.window_size * timedelta(
                 minutes=self.time_frame.amount_of_minutes
             ))
+
+    def get_number_of_required_data_points(
+        self, start_date: datetime, end_date: datetime
+    ) -> int:
+        """
+        Returns the number of data points required based on the given
+        attributes of the data source. If the required number of data points
+        can't be determined, it returns None.
+
+        E.g., for OHLCV data source, it
+        calculates the number of data points needed between the
+        start_date and end_date based on the time frame.
+
+        Args:
+            start_date (datetime): The start date for the data points.
+            end_date (datetime): The end date for the data points.
+
+        Returns:
+            int: The number of required data points, or None if it can't
+            be determined.
+        """
+
+        if self.time_frame is None:
+            return None
+
+        delta = end_date - start_date
+        total_minutes = delta.total_seconds() / 60
+        data_points = total_minutes / self.time_frame.amount_of_minutes
+
+        if self.window_size is not None:
+            data_points += self.window_size
+
+        return int(data_points)

{investing_algorithm_framework-7.19.2 → investing_algorithm_framework-7.19.4}/investing_algorithm_framework/domain/utils/polars.py

@@ -6,6 +6,7 @@ def convert_polars_to_pandas(
     data: PolarsDataFrame,
     remove_duplicates=True,
     add_index=True,
+    add_datetime_column=True,
     datetime_column_name="Datetime"
 ):
     """
@@ -21,6 +22,8 @@ def convert_polars_to_pandas(
         dates will be removed from the dataframe
         add_index: Boolean - If set to true, an index will
             be added to the dataframe
+        add_datetime_column: Boolean - If set to true, a datetime
+            column will be added to the dataframe
         datetime_column_name: String - the column name that has the
             datetime object. By default this is set to column name Datetime
             This is only used if add_index is set to True
@@ -35,6 +38,9 @@ def convert_polars_to_pandas(
 
     df = data.to_pandas().copy()
 
+    if add_datetime_column and datetime_column_name not in df.columns:
+        df[datetime_column_name] = pd.to_datetime(df.index)
+
     # Ensure datetime column is datetime type
     df[datetime_column_name] = pd.to_datetime(df[datetime_column_name])
 

{investing_algorithm_framework-7.19.2 → investing_algorithm_framework-7.19.4}/investing_algorithm_framework/infrastructure/data_providers/ccxt.py

@@ -2,7 +2,7 @@ import logging
 import os.path
 from datetime import datetime, timedelta, timezone
 from time import sleep
-from typing import Union
+from typing import Union, List
 
 import ccxt
 import pandas as pd
@@ -99,6 +99,9 @@ class CCXTOHLCVDataProvider(DataProvider):
         self.pandas = pandas
         self.window_cache = {}
         self.data = None
+        self.total_number_of_data_points = 0
+        self.missing_data_point_dates = []
+        self.data_file_path = None
 
     def has_data(
         self,
@@ -235,6 +238,14 @@ class CCXTOHLCVDataProvider(DataProvider):
         self.data = data
         self._start_date_data_source = self.data["Datetime"].min()
         self._end_date_data_source = self.data["Datetime"].max()
+        self.total_number_of_data_points = len(self.data)
+
+        if required_start_date < self._start_date_data_source:
+            self.number_of_missing_data_points = (
+                self._start_date_data_source - required_start_date
+            ).total_seconds() / (
+                TimeFrame.from_value(self.time_frame).amount_of_minutes * 60
+            )
 
         if self.window_size is not None:
             # Create cache with sliding windows
@@ -246,6 +257,22 @@ class CCXTOHLCVDataProvider(DataProvider):
                 end_date=backtest_end_date
             )
 
+        n_min = TimeFrame.from_value(self.time_frame).amount_of_minutes
+        # Assume self.data is a Polars DataFrame with a "Datetime" column
+        expected_dates = pl.datetime_range(
+            start=required_start_date,
+            end=backtest_end_date,
+            interval=f"{n_min}m",
+            eager=True
+        ).to_list()
+
+        actual_dates = self.data["Datetime"].to_list()
+
+        # Find missing dates
+        self.missing_data_point_dates = sorted(
+            set(expected_dates) - set(actual_dates)
+        )
+
     def get_data(
         self,
         date: datetime = None,
@@ -418,7 +445,7 @@ class CCXTOHLCVDataProvider(DataProvider):
                         f"the available data "
                         f"{self._start_date_data_source} "
                         f"- {self._end_date_data_source}."
-                        f" for data source {data_source.identifier}."
+                        f" for data source {data_source.identifier}.",
                     )
 
                 raise OperationalException(
@@ -426,7 +453,7 @@ class CCXTOHLCVDataProvider(DataProvider):
                     f"is before the range of "
                     f"the available data "
                     f"{self._start_date_data_source} "
-                    f"- {self._end_date_data_source}."
+                    f"- {self._end_date_data_source}.",
                 )
 
             if backtest_end_date > self._end_date_data_source:
@@ -820,6 +847,7 @@ class CCXTOHLCVDataProvider(DataProvider):
                             # If the data source specification matches,
                             # read the file
                             file_path = os.path.join(storage_path, file_name)
+                            self.data_file_path = file_path
                             data = pl.read_csv(
                                 file_path,
                                 schema_overrides={"Datetime": pl.Datetime},
@@ -1025,3 +1053,60 @@ class CCXTOHLCVDataProvider(DataProvider):
             config=self.config,
             pandas=data_source.pandas,
         )
+
+    def get_number_of_data_points(
+        self,
+        start_date: datetime,
+        end_date: datetime
+    ) -> int:
+
+        """
+        Returns the number of data points available between the given
+        start and end dates.
+
+        Args:
+            start_date (datetime): The start date for checking missing data.
+            end_date (datetime): The end date for checking missing data.
+
+        Returns:
+            int: The number of available data points between the given
+                start and end dates.
+        """
+        available_dates = [
+            date for date in self.data["Datetime"].to_list()
+            if start_date <= date <= end_date
+        ]
+        return len(available_dates)
+
+    def get_missing_data_dates(
+        self,
+        start_date: datetime,
+        end_date: datetime,
+    ) -> List[datetime]:
+        """
+        Returns a list of dates for which data is missing between the
+        given start and end dates.
+
+        Args:
+            start_date (datetime): The start date for checking missing data.
+            end_date (datetime): The end date for checking missing data.
+
+        Returns:
+            List[datetime]: A list of dates for which data is missing
+                between the given start and end dates.
+        """
+        missing_dates = [
+            date for date in self.missing_data_point_dates
+            if start_date < date < end_date
+        ]
+        return missing_dates
+
+    def get_data_source_file_path(self) -> Union[str, None]:
+        """
+        Get the file path of the data source if stored in local storage.
+
+        Returns:
+            Union[str, None]: The file path of the data source if stored
+                locally, otherwise None.
+        """
+        return self.data_file_path

{investing_algorithm_framework-7.19.2 → investing_algorithm_framework-7.19.4}/investing_algorithm_framework/infrastructure/data_providers/csv.py

@@ -1,3 +1,4 @@
+from typing import List, Union
 from datetime import datetime, timezone, timedelta
 
 import polars as pl
@@ -67,6 +68,8 @@ class CSVOHLCVDataProvider(DataProvider):
         self.window_cache = {}
         self._load_data(self.storage_path)
         self.pandas = pandas
+        self.number_of_missing_data_points = 0
+        self.missing_data_point_dates: List[datetime] = []
 
     def has_data(
         self,
@@ -212,8 +215,6 @@ class CSVOHLCVDataProvider(DataProvider):
         Prepares backtest data for a given symbol and date range.
 
         Args:
-            data_source (DataSource): The data source specification that
-                matches a data provider.
             backtest_start_date (datetime): The start date for the
                 backtest data.
             backtest_end_date (datetime): The end date for the
@@ -248,14 +249,6 @@ class CSVOHLCVDataProvider(DataProvider):
                 .amount_of_minutes * self.window_size
             )
 
-        if required_start_date < self._start_date_data_source:
-            raise OperationalException(
-                f"Not enough data available for backtest. "
-                f"Data earlier then {required_start_date} is "
-                "required, but only "
-                f"{self._start_date_data_source} is available."
-            )
-
         # Create cache with sliding windows
         self._precompute_sliding_windows(
             window_size=self.window_size,
@@ -263,6 +256,30 @@ class CSVOHLCVDataProvider(DataProvider):
             end_date=backtest_end_date
         )
 
+        if required_start_date < self._start_date_data_source:
+            self.number_of_missing_data_points = (
+                self._start_date_data_source - required_start_date
+            ).total_seconds() / (
+                TimeFrame.from_value(self.time_frame).amount_of_minutes * 60
+            )
+
+        n_min = TimeFrame.from_value(self.time_frame).amount_of_minutes
+
+        # Assume self.data is a Polars DataFrame with a "Datetime" column
+        expected_dates = pl.datetime_range(
+            start=required_start_date,
+            end=backtest_end_date,
+            interval=f"{n_min}m",
+            eager=True
+        ).to_list()
+
+        actual_dates = self.data["Datetime"].to_list()
+
+        # Find missing dates
+        self.missing_data_point_dates = sorted(
+            set(expected_dates) - set(actual_dates)
+        )
+
     def get_backtest_data(
         self,
         backtest_index_date: datetime,
@@ -492,3 +509,60 @@ class CSVOHLCVDataProvider(DataProvider):
             window_size=data_source.window_size,
             data_provider_identifier=self.data_provider_identifier
         )
+
+    def get_number_of_data_points(
+        self,
+        start_date: datetime,
+        end_date: datetime
+    ) -> int:
+
+        """
+        Returns the number of data points available between the given
+        start and end dates.
+
+        Args:
+            start_date (datetime): The start date for checking missing data.
+            end_date (datetime): The end date for checking missing data.
+
+        Returns:
+            int: The number of available data points between the given
+                start and end dates.
+        """
+        available_dates = [
+            date for date in self.data["Datetime"].to_list()
+            if start_date <= date <= end_date
+        ]
+        return len(available_dates)
+
+    def get_missing_data_dates(
+        self,
+        start_date: datetime,
+        end_date: datetime,
+    ) -> List[datetime]:
+        """
+        Returns a list of dates for which data is missing between the
+        given start and end dates.
+
+        Args:
+            start_date (datetime): The start date for checking missing data.
+            end_date (datetime): The end date for checking missing data.
+
+        Returns:
+            List[datetime]: A list of dates for which data is missing
+                between the given start and end dates.
+        """
+        missing_dates = [
+            date for date in self.missing_data_point_dates
+            if start_date < date < end_date
+        ]
+        return missing_dates
+
+    def get_data_source_file_path(self) -> Union[str, None]:
+        """
+        Get the file path of the data source if stored in local storage.
+
+        Returns:
+            Union[str, None]: The file path of the data source if stored
+                locally, otherwise None.
+        """
+        return self.storage_path