investing-algorithm-framework 7.18.0__py3-none-any.whl → 7.19.3__py3-none-any.whl

investing_algorithm_framework/__init__.py

@@ -10,7 +10,7 @@ from .app import App, Algorithm, \
     get_ohlcv_data_completeness_chart, get_equity_curve_chart
 from .domain import ApiException, combine_backtests, PositionSize, \
     OrderType, OperationalException, OrderStatus, OrderSide, \
-    TimeUnit, TimeInterval, Order, Portfolio, Backtest, \
+    TimeUnit, TimeInterval, Order, Portfolio, Backtest, DataError, \
     Position, TimeFrame, INDEX_DATETIME, MarketCredential, \
     PortfolioConfiguration, RESOURCE_DIRECTORY, AWS_LAMBDA_LOGGING_CONFIG, \
     Trade, SYMBOLS, RESERVED_BALANCES, APP_MODE, AppMode, DATETIME_FORMAT, \
@@ -191,5 +191,6 @@ __all__ = [
     "get_number_of_trades",
     "BacktestRun",
     "load_backtests_from_directory",
-    "save_backtests_to_directory"
+    "save_backtests_to_directory",
+    "DataError"
 ]

investing_algorithm_framework/app/app.py

@@ -5,6 +5,7 @@ import threading
 from datetime import datetime, timezone
 from typing import List, Optional, Any, Dict, Tuple
 
+import pandas as pd
 from flask import Flask
 
 from investing_algorithm_framework.app.algorithm import Algorithm
@@ -16,7 +17,7 @@ from investing_algorithm_framework.domain import DATABASE_NAME, TimeUnit, \
     SQLALCHEMY_DATABASE_URI, OperationalException, StateHandler, \
     BACKTESTING_START_DATE, BACKTESTING_END_DATE, APP_MODE, MarketCredential, \
     AppMode, BacktestDateRange, DATABASE_DIRECTORY_NAME, DataSource, \
-    BACKTESTING_INITIAL_AMOUNT, SNAPSHOT_INTERVAL, Backtest, \
+    BACKTESTING_INITIAL_AMOUNT, SNAPSHOT_INTERVAL, Backtest, DataError, \
     PortfolioConfiguration, SnapshotInterval, DataType, combine_backtests, \
     PortfolioProvider, OrderExecutor, ImproperlyConfigured, \
     DataProvider, INDEX_DATETIME, tqdm, BacktestPermutationTest, \
@@ -32,6 +33,7 @@ from .app_hook import AppHook
 from .eventloop import EventLoopService
 from .analysis import create_ohlcv_permutation
 
+
 logger = logging.getLogger("investing_algorithm_framework")
 COLOR_RESET = '\033[0m'
 COLOR_GREEN = '\033[92m'
@@ -789,6 +791,73 @@ class App:
             .market_credential_service()
         return market_credential_service.get_all()
 
+    def check_data_completeness(
+        self,
+        strategies: List[TradingStrategy],
+        backtest_date_range: BacktestDateRange
+    ) -> None:
+        """
+        Function to check the data completeness for a set of strategies
+        over a given backtest date range. This method checks if all data
+        sources required by the strategies have complete data for the
+        specified date range.
+
+        Args:
+            strategies (List[TradingStrategy]): List of strategy objects
+                to check data completeness for.
+            backtest_date_range (BacktestDateRange): The date range to
+                check data completeness for.
+        Returns:
+            None
+        """
+        data_sources = []
+
+        for strategy in strategies:
+            data_sources.extend(strategy.data_sources)
+
+        self.initialize_data_sources_backtest(
+            data_sources,
+            backtest_date_range,
+            show_progress=True
+        )
+        data_provider_service = self.container.data_provider_service()
+
+        for strategy in strategies:
+
+            for data_source in strategy.data_sources:
+
+                if DataType.OHLCV.equals(data_source.data_type):
+                    df = data_provider_service.get_ohlcv_data(
+                        symbol=data_source.symbol,
+                        start_date=backtest_date_range.start_date,
+                        end_date=backtest_date_range.end_date,
+                        pandas=True,
+                        add_pandas_index=False,
+                        add_datetime_column=True,
+                        time_frame=data_source.time_frame
+                    )
+                    df = df.copy()
+                    df['Datetime'] = pd.to_datetime(df['Datetime'])
+                    df = df.sort_values('Datetime')\
+                        .tail(data_source.window_size)
+                    start = df['Datetime'].iloc[0]
+                    end = df['Datetime'].iloc[-1]
+                    freq = pd.to_timedelta(data_source.time_frame.value)
+                    expected = pd.date_range(start, end, freq=freq)
+                    actual = df['Datetime']
+                    missing = expected.difference(actual)
+
+                    # Calculate the percentage completeness
+                    completeness = len(actual) / len(expected) * 100
+
+                    if completeness < 100:
+                        raise DataError(
+                            f"Data completeness for data source "
+                            f"{data_source.identifier} "
+                            f"({data_source.symbol}) is {completeness:.2f}% "
+                            f"complete. Missing data points: {len(missing)}"
+                        )
+
     def run_vector_backtests(
         self,
         initial_amount,

investing_algorithm_framework/domain/__init__.py

@@ -15,7 +15,7 @@ from .constants import ITEMIZE, ITEMIZED, PER_PAGE, PAGE, ENVIRONMENT, \
 from .data_provider import DataProvider
 from .data_structures import PeekableQueue
 from .decimal_parsing import parse_decimal_to_string, parse_string_to_decimal
-from .exceptions import OperationalException, ApiException, \
+from .exceptions import OperationalException, ApiException, DataError, \
     PermissionDeniedApiException, ImproperlyConfigured, NetworkError
 from .models import OrderStatus, OrderSide, OrderType, TimeInterval, \
     TimeUnit, TimeFrame, PortfolioConfiguration, Portfolio, Position, \
@@ -143,5 +143,6 @@ __all__ = [
     "BacktestEvaluationFocus",
     'combine_backtests',
     'PositionSize',
-    'generate_backtest_summary_metrics'
+    'generate_backtest_summary_metrics',
+    'DataError'
 ]

investing_algorithm_framework/domain/exceptions.py

@@ -84,3 +84,20 @@ class NetworkError(Exception):
             "status": "error",
             "message": self.error_message
         }
+
+
+class DataError(Exception):
+    """
+    Class DataError: Exception class indicating a problem occurred
+    during data retrieval or processing
+    """
+
+    def __init__(self, message) -> None:
+        super(DataError, self).__init__(message)
+        self.error_message = message
+
+    def to_response(self):
+        return {
+            "status": "error",
+            "message": self.error_message
+        }

investing_algorithm_framework/domain/utils/polars.py

@@ -6,6 +6,7 @@ def convert_polars_to_pandas(
     data: PolarsDataFrame,
     remove_duplicates=True,
     add_index=True,
+    add_datetime_column=True,
     datetime_column_name="Datetime"
 ):
     """
@@ -21,6 +22,8 @@ def convert_polars_to_pandas(
         dates will be removed from the dataframe
         add_index: Boolean - If set to true, an index will
             be added to the dataframe
+        add_datetime_column: Boolean - If set to true, a datetime
+            column will be added to the dataframe
         datetime_column_name: String - the column name that has the
             datetime object. By default this is set to column name Datetime
             This is only used if add_index is set to True
@@ -35,6 +38,9 @@ def convert_polars_to_pandas(
 
     df = data.to_pandas().copy()
 
+    if add_datetime_column and datetime_column_name not in df.columns:
+        df[datetime_column_name] = pd.to_datetime(df.index)
+
     # Ensure datetime column is datetime type
     df[datetime_column_name] = pd.to_datetime(df[datetime_column_name])
 

investing_algorithm_framework/services/data_providers/data_provider_service.py

@@ -7,7 +7,7 @@ from typing import List, Tuple, Optional, Dict, Any
 
 from investing_algorithm_framework.domain import DataProvider, \
     OperationalException, ImproperlyConfigured, DataSource, DataType, \
-    BacktestDateRange, tqdm, convert_polars_to_pandas
+    BacktestDateRange, tqdm, convert_polars_to_pandas, TimeFrame
 
 logger = logging.getLogger("investing_algorithm_framework")
 
@@ -26,6 +26,7 @@ class DataProviderIndex:
         self.data_providers_lookup = defaultdict()
         self.ohlcv_data_providers = defaultdict()
         self.ohlcv_data_providers_no_market = defaultdict()
+        self.ohlcv_data_providers_with_timeframe = defaultdict()
         self.ticker_data_providers = defaultdict()
 
     def add(self, data_provider: DataProvider):
@@ -80,11 +81,15 @@ class DataProviderIndex:
 
         symbol = data_source.symbol
         market = data_source.market
+        time_frame = data_source.time_frame
 
         if DataType.OHLCV.equals(data_source.data_type):
             if symbol not in self.ohlcv_data_providers:
                 self.ohlcv_data_providers[(symbol, market)] = best_provider
                 self.ohlcv_data_providers_no_market[symbol] = best_provider
+                self.ohlcv_data_providers_with_timeframe[
+                    (symbol, market, time_frame)
+                ] = best_provider
             else:
                 try:
                     # If the symbol already exists, we can update the provider
@@ -105,6 +110,11 @@ class DataProviderIndex:
                         self.ohlcv_data_providers_no_market[symbol] =\
                             best_provider
 
+                    time_frame_key = (symbol, market, time_frame)
+                    self.ohlcv_data_providers_with_timeframe[
+                        time_frame_key
+                    ] = best_provider
+
                 except Exception:
                     # If the existing provider does not have a time_frame
                     # attribute, we can safely ignore this
@@ -165,12 +175,16 @@ class DataProviderIndex:
 
         symbol = data_source.symbol
         market = data_source.market
+        time_frame = data_source.time_frame
 
         if DataType.OHLCV.equals(data_source.data_type):
 
             if symbol not in self.ohlcv_data_providers:
                 self.ohlcv_data_providers[(symbol, market)] = best_provider
                 self.ohlcv_data_providers_no_market[symbol] = best_provider
+                self.ohlcv_data_providers_with_timeframe[
+                    (symbol, market, time_frame)
+                ] = best_provider
             else:
                 try:
                     # If the symbol already exists, we can update the provider
@@ -192,6 +206,11 @@ class DataProviderIndex:
                         self.ohlcv_data_providers_no_market[symbol] = \
                             best_provider
 
+                    time_frame_key = (symbol, market, data_source.time_frame)
+                    self.ohlcv_data_providers_with_timeframe[
+                        time_frame_key
+                    ] = best_provider
+
                 except Exception:
                     # If the existing provider does not have a time_frame
                     # attribute, we can safely ignore this
@@ -264,7 +283,10 @@ class DataProviderIndex:
         return len(self.data_providers_lookup)
 
     def get_ohlcv_data_provider(
-        self, symbol: str, market: Optional[str] = None
+        self,
+        symbol: str,
+        market: Optional[str] = None,
+        time_frame: Optional[str] = None
     ) -> Optional[DataProvider]:
         """
         Get the OHLCV data provider for a given symbol and market.
@@ -272,12 +294,20 @@ class DataProviderIndex:
         Args:
             symbol (str): The symbol to get the data provider for.
             market (Optional[str]): The market to get the data provider for.
+            time_frame (Optional[str]): The time frame to get the
+                data provider for.
 
         Returns:
             DataProvider: The OHLCV data provider for the symbol and market,
                 or None if no provider is found.
         """
 
+        if market is not None and time_frame is not None:
+            time_frame = TimeFrame.from_value(time_frame)
+            return self.ohlcv_data_providers_with_timeframe.get(
+                (symbol, market, time_frame), None
+            )
+
         if market is None:
             # If no market is specified
             return self.ohlcv_data_providers_no_market.get(symbol, None)
@@ -462,7 +492,9 @@ class DataProviderService:
         start_date: Optional[datetime] = None,
         end_date: Optional[datetime] = None,
         window_size: Optional[int] = None,
-        pandas: bool = False
+        pandas: bool = False,
+        add_pandas_index: bool = True,
+        add_datetime_column: bool = False,
     ):
         """
         Function to get OHLCV data from the data provider.
@@ -476,6 +508,10 @@ class DataProviderService:
             end_date (datetime): The end date for the OHLCV data.
             window_size (int): The window size for the OHLCV data.
             pandas (bool): Whether to return the data as a pandas DataFrame.
+            add_pandas_index (bool): Whether to add a pandas index to
+                the DataFrame if pandas is True.
+            add_datetime_column (bool): Whether to add a datetime column
+                to the DataFrame if pandas is True.
 
         Returns:
             DataFrame: The OHLCV data for the given symbol and market.
@@ -483,6 +519,7 @@ class DataProviderService:
         data_provider = self.data_provider_index.get_ohlcv_data_provider(
             symbol=symbol,
             market=market,
+            time_frame=time_frame
         )
 
         if data_provider is None:
@@ -511,7 +548,11 @@ class DataProviderService:
 
         if pandas:
             if isinstance(data, pl.DataFrame):
-                return convert_polars_to_pandas(data)
+                return convert_polars_to_pandas(
+                    data,
+                    add_index=add_pandas_index,
+                    add_datetime_column=add_datetime_column
+                )
             else:
                 return data
 

{investing_algorithm_framework-7.18.0.dist-info → investing_algorithm_framework-7.19.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: investing-algorithm-framework
-Version: 7.18.0
+Version: 7.19.3
 Summary: A framework for creating trading bots
 Author: MDUYN
 Requires-Python: >=3.10

{investing_algorithm_framework-7.18.0.dist-info → investing_algorithm_framework-7.19.3.dist-info}/RECORD

@@ -1,4 +1,4 @@
-investing_algorithm_framework/__init__.py,sha256=ACNFUkjEf2UxGoBoci9JHeT6hGFISFyy4U-rLnZW8_E,6989
+investing_algorithm_framework/__init__.py,sha256=e5i9FAiyvS1KwuvSkrFC0dPOgKCix9mGoGy-51Mqe1s,7017
 investing_algorithm_framework/app/__init__.py,sha256=BjVkBQvVuI7ovTpR2Bn8YOHL2p5vsX-LRqpe-aS3ImM,1671
 investing_algorithm_framework/app/algorithm/__init__.py,sha256=-a9o9bTfAhW9qSW-bKvlLQuMCf-YXxIztudo2TxMjCI,136
 investing_algorithm_framework/app/algorithm/algorithm.py,sha256=v8AZZ7hr5ZKJbavk242xCUpGHv3mKZ4sqfGV7BwPgdU,6854
@@ -8,7 +8,7 @@ investing_algorithm_framework/app/analysis/backtest_data_ranges.py,sha256=pt8vUj
 investing_algorithm_framework/app/analysis/backtest_utils.py,sha256=W_X1T8f1UOllUUkaO4ZiQlQ7dSEGi9TqVo_ROyv7XNE,2121
 investing_algorithm_framework/app/analysis/permutation.py,sha256=NHzyMQ9aCqLiLXyw1CpRZfITLzsRwHmMn8Uj8oV5_1E,3941
 investing_algorithm_framework/app/analysis/ranking.py,sha256=-XEWmU3rxLvkC9GOW6Zci7E3Y7H2xKwU3id8ljf0n9k,10888
-investing_algorithm_framework/app/app.py,sha256=saAfkfWJg-in7qKCOIpN1HljMU2YCuvdqSTWi4a-Ub0,83223
+investing_algorithm_framework/app/app.py,sha256=VeVyiQKImJCd4uR5kH62kPCUE-wpVKeM_2nnX46yAf8,85991
 investing_algorithm_framework/app/app_hook.py,sha256=cDiY4x2n06tljpx-fcbIM1oPnjTnEthibvqxUvfEppo,834
 investing_algorithm_framework/app/context.py,sha256=kWOBZq7E45xoAPbMfn9HPhDQmEcyCqBYWi-NJK5nMXk,58874
 investing_algorithm_framework/app/eventloop.py,sha256=w9zufVpiHrgsxuh4_AW2DXxOfI4LVcScNHTLfL9-Tws,21736
@@ -83,7 +83,7 @@ investing_algorithm_framework/cli/templates/run_backtest.py.template,sha256=mHwK
 investing_algorithm_framework/cli/templates/strategy.py.template,sha256=Ox8IZ6XlPDCCgq9T8KO3NazR2UXJs3FgkjbgfPqoad8,4603
 investing_algorithm_framework/create_app.py,sha256=HN_45Bza5Ro3o-v364m2mjSVjjZnyESOGHn3dcPDfQ4,1372
 investing_algorithm_framework/dependency_container.py,sha256=LIqK4fH2OdIKpvixzI2eSCXp6EtvEcWAJxJo-7a9qp4,6699
-investing_algorithm_framework/domain/__init__.py,sha256=PASQlRGcU_MDIwnLppanXGo-BUXdkhajvpRvcwqxdL0,4903
+investing_algorithm_framework/domain/__init__.py,sha256=o2a3lAaCs4Jir88lSa8ZlwZ8Ghjwy1--S_9KDwny5MU,4931
 investing_algorithm_framework/domain/backtesting/__init__.py,sha256=q-NejGHzE233w5jXPhSsuLpBZ_yl3m-qb2g6FnxZaps,699
 investing_algorithm_framework/domain/backtesting/backtest.py,sha256=eRSaujcoggYdX3jmGO8KL4a2gOdYUeWH7Jx5vcLkeYg,17514
 investing_algorithm_framework/domain/backtesting/backtest_date_range.py,sha256=e_V7HMdtln4uu87jwwa_Yr7ZesgrpFqsEqtr0e0DTto,3186
@@ -98,7 +98,7 @@ investing_algorithm_framework/domain/constants.py,sha256=-ZU0z1DSgJaUQvkp_ELZslq
 investing_algorithm_framework/domain/data_provider.py,sha256=yMr7RWxHh6I9kSHsrwwukXi0k6UHLwmntIQGv0V0Q1w,11018
 investing_algorithm_framework/domain/data_structures.py,sha256=ePtdGhVaB16QLUvKQn5MiWM_TBOcBBTj5M0llW8tGEE,989
 investing_algorithm_framework/domain/decimal_parsing.py,sha256=NtMNkxZyWrFHxGKd6gLIDmWF88BYcTl8tYaAaKO1tlg,823
-investing_algorithm_framework/domain/exceptions.py,sha256=CjMPVqA5_hPoyeThH0vP-grGpmWXrKU_Tb19wWUJGqM,2359
+investing_algorithm_framework/domain/exceptions.py,sha256=2BmMmpu3jFW7vJ4KVJw0AAPTS70yQ1NH9OMc-exEBuc,2768
 investing_algorithm_framework/domain/models/__init__.py,sha256=5SN_d8MTbQzgLIYpt0ySkG2UZ3MsUz_30w40-vxUmk0,1116
 investing_algorithm_framework/domain/models/app_mode.py,sha256=V1p9QMSNLlz6KjPib8d1J2EaSZh7jmXjZ1V2DFOrIaU,812
 investing_algorithm_framework/domain/models/base_model.py,sha256=1_DMaDR13gc6l5yWpMgKaAK7OJstc4lmvZV5DhwYM6o,659
@@ -150,7 +150,7 @@ investing_algorithm_framework/domain/utils/csv.py,sha256=QK7EC_jY4u9s97OFOcoxwZs
 investing_algorithm_framework/domain/utils/custom_tqdm.py,sha256=LwPiuYedzujtV-kp4Uq_Pe8CqjH6zFvvCuSWCharOP8,583
 investing_algorithm_framework/domain/utils/dates.py,sha256=T3ZwN6UXKZmcqSjrdH5j2Ielq2PB4fm12QXSy-wdrZw,1541
 investing_algorithm_framework/domain/utils/jupyter_notebook_detection.py,sha256=Al0T72bhvmyYFGEYDVPbIBOpXtIfgmn8_Tq1KKJYyig,624
-investing_algorithm_framework/domain/utils/polars.py,sha256=nO8OdTsdnq6Q3TQJ13AKJeE4tzvGHySEyQ_ojST9Kmk,1550
+investing_algorithm_framework/domain/utils/polars.py,sha256=1oqlZbNgOvIUyoJTRuwxdzZhlSqYF7R1SDjdhgb0Reg,1828
 investing_algorithm_framework/domain/utils/random.py,sha256=25PQSDuTmMd32sN7H3LQejUO73hQW5FCHGOpKYfdXI0,945
 investing_algorithm_framework/domain/utils/signatures.py,sha256=MO2hCrrnQjbsT6guX4AdeJfcAP55pRXtjJ73uX1cjaI,417
 investing_algorithm_framework/domain/utils/stoppable_thread.py,sha256=z3-OGZ-wrygfYn4zhB3COHdB43qQYa65TeShyVokSp0,608
@@ -206,7 +206,7 @@ investing_algorithm_framework/services/backtesting/__init__.py,sha256=sD6JMQVuUT
 investing_algorithm_framework/services/backtesting/backtest_service.py,sha256=rWxOxAXhuywb0jx_j0XVChKBPU3CUV4vYF0T7oTtG2E,24841
 investing_algorithm_framework/services/configuration_service.py,sha256=BCgiBlrLjMjfU4afmjYaHu9gOWNmgaxhf6RBN2XJkw0,2853
 investing_algorithm_framework/services/data_providers/__init__.py,sha256=OHVccpIYGc-1B2AkCI_2Nhsb9KMaAUrng4DHhIbFD8Y,96
-investing_algorithm_framework/services/data_providers/data_provider_service.py,sha256=KOhxQEdU1VxE5MXl713Lo4lzv3J8kuSxCa025Ywyr20,28455
+investing_algorithm_framework/services/data_providers/data_provider_service.py,sha256=7gUzwGD2E1e4IEAFRBoSnksAyMzVof6rZwDLsNm3tZE,30239
 investing_algorithm_framework/services/market_credential_service.py,sha256=syitQ61sECzK0i0Wd-Hc8xaTv4tpRYRFbCjyw9pWMeA,1197
 investing_algorithm_framework/services/metrics/__init__.py,sha256=kQaAw5r-hXCsaEY9fQ5Fch8CfjltdjKrGFn6gNSWWdA,4322
 investing_algorithm_framework/services/metrics/alpha.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -253,8 +253,8 @@ investing_algorithm_framework/services/trade_order_evaluator/default_trade_order
 investing_algorithm_framework/services/trade_order_evaluator/trade_order_evaluator.py,sha256=pNnmgaKMR9RY6Kxq7xS0nURKoaQDe2ehrP5GfElkkcI,1328
 investing_algorithm_framework/services/trade_service/__init__.py,sha256=AcwPyJjDRdiREnl_MWMkDSc-V-ZjXtvpHD6eQT9mc1o,68
 investing_algorithm_framework/services/trade_service/trade_service.py,sha256=OtzIS5EebByGcqDvV2AFeBjXSarvrgubMXDaVKg6Rbw,41193
-investing_algorithm_framework-7.18.0.dist-info/LICENSE,sha256=wbVEDvoZiMPHufRY3sLEffvAr7GH5hOIngHF8y4HFQg,11343
-investing_algorithm_framework-7.18.0.dist-info/METADATA,sha256=_8KvJKynhbGR8VmlZzsG6tY6D2u_EzA8iApv5zCkTjE,19635
-investing_algorithm_framework-7.18.0.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
-investing_algorithm_framework-7.18.0.dist-info/entry_points.txt,sha256=jrPF0YksDs27vYzEvj3tXLe3OGWU24EJA05z5xHqmq8,91
-investing_algorithm_framework-7.18.0.dist-info/RECORD,,
+investing_algorithm_framework-7.19.3.dist-info/LICENSE,sha256=wbVEDvoZiMPHufRY3sLEffvAr7GH5hOIngHF8y4HFQg,11343
+investing_algorithm_framework-7.19.3.dist-info/METADATA,sha256=fuIfBz-8G1y2uDWrEynERnrI0pM2xek0NxI8-DfexD4,19635
+investing_algorithm_framework-7.19.3.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
+investing_algorithm_framework-7.19.3.dist-info/entry_points.txt,sha256=jrPF0YksDs27vYzEvj3tXLe3OGWU24EJA05z5xHqmq8,91
+investing_algorithm_framework-7.19.3.dist-info/RECORD,,