investing-algorithm-framework 7.25.2__tar.gz → 7.25.4__tar.gz

{investing_algorithm_framework-7.25.2 → investing_algorithm_framework-7.25.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: investing-algorithm-framework
-Version: 7.25.2
+Version: 7.25.4
 Summary: A framework for creating trading bots
 Author: MDUYN
 Requires-Python: >=3.10

{investing_algorithm_framework-7.25.2 → investing_algorithm_framework-7.25.4}/investing_algorithm_framework/__init__.py

@@ -1,15 +1,18 @@
+from .analysis import generate_rolling_backtest_windows, \
+    select_backtest_date_ranges, rank_results, create_weights, \
+    get_missing_timeseries_data_entries, fill_missing_timeseries_data, \
+    create_markdown_table
 from .app import App, Algorithm, \
     TradingStrategy, StatelessAction, Task, AppHook, Context, \
-    add_html_report, BacktestReport, generate_rolling_backtest_windows, \
+    add_html_report, BacktestReport, \
     pretty_print_trades, pretty_print_positions, \
-    pretty_print_orders, pretty_print_backtest, select_backtest_date_ranges, \
+    pretty_print_orders, pretty_print_backtest, \
     get_equity_curve_with_drawdown_chart, \
-    get_rolling_sharpe_ratio_chart, rank_results, \
-    get_monthly_returns_heatmap_chart, create_weights, \
+    get_rolling_sharpe_ratio_chart, get_monthly_returns_heatmap_chart, \
     get_yearly_returns_bar_chart, get_entry_and_exit_signals, \
     get_ohlcv_data_completeness_chart, get_equity_curve_chart
 from .domain import ApiException, combine_backtests, PositionSize, \
-    OrderType, OperationalException, OrderStatus, OrderSide, \
+    OrderType, OperationalException, OrderStatus, OrderSide, tqdm, \
     TimeUnit, TimeInterval, Order, Portfolio, Backtest, DataError, \
     Position, TimeFrame, INDEX_DATETIME, MarketCredential, TakeProfitRule, \
     PortfolioConfiguration, RESOURCE_DIRECTORY, AWS_LAMBDA_LOGGING_CONFIG, \
@@ -24,7 +27,8 @@ from .infrastructure import AzureBlobStorageStateHandler, \
     CSVOHLCVDataProvider, CCXTOHLCVDataProvider, PandasOHLCVDataProvider, \
     AWSS3StorageStateHandler
 from .create_app import create_app
-from .download_data import download
+from .download_data import download, download_v2, DownloadResult, \
+    create_data_storage_path
 from .services import get_annual_volatility, get_sortino_ratio, \
     get_drawdown_series, get_max_drawdown, get_equity_curve, \
     get_price_efficiency_ratio, get_sharpe_ratio, \
@@ -199,5 +203,9 @@ __all__ = [
     "TradeTakeProfitService",
     "generate_algorithm_id",
     "BacktestMetrics",
-    "generate_rolling_backtest_windows"
+    "generate_rolling_backtest_windows",
+    "tqdm",
+    "get_missing_timeseries_data_entries",
+    "fill_missing_timeseries_data",
+    "create_markdown_table"
 ]

investing_algorithm_framework-7.25.4/investing_algorithm_framework/analysis/__init__.py

@@ -0,0 +1,16 @@
+from .markdown import create_markdown_table
+from .backtest_data_ranges import select_backtest_date_ranges, \
+    generate_rolling_backtest_windows
+from .ranking import create_weights, rank_results
+from .data import fill_missing_timeseries_data, \
+    get_missing_timeseries_data_entries
+
+__all__ = [
+    "create_markdown_table",
+    "select_backtest_date_ranges",
+    "generate_rolling_backtest_windows",
+    "create_weights",
+    "rank_results",
+    "fill_missing_timeseries_data",
+    "get_missing_timeseries_data_entries",
+]

{investing_algorithm_framework-7.25.2/investing_algorithm_framework/app → investing_algorithm_framework-7.25.4/investing_algorithm_framework}/analysis/backtest_data_ranges.py

@@ -126,68 +126,47 @@ def select_backtest_date_ranges(
 
 
 def generate_rolling_backtest_windows(
-    data: pd.DataFrame,
-    window_type: str = "rolling",  # or "expanding"
-    start_date: datetime = None,
-    end_date: datetime = None,
+    start_date: datetime,
+    end_date: datetime,
     train_days: int = 365,
     test_days: int = 90,
     step_days: int = 90,
     gap_days: int = 0,
-    min_data_threshold: float = 0.8
-) -> List[Dict[str, any]]:
+) -> List[Dict[str, BacktestDateRange]]:
     """
-    Generate rolling or expanding windows for walk-forward backtesting.
+    Generate rolling windows for walk-forward backtesting.
 
-    This function creates training and testing windows for
-    time-series backtesting,
-    avoiding look-ahead bias and providing realistic out-of-sample
-    performance estimates.
+    This function creates training and testing date ranges for
+    time-series backtesting, avoiding look-ahead bias and providing
+    realistic out-of-sample performance estimates.
 
     Args:
-        window_type (str): Type of window to generate:
-            - "rolling": Fixed train_days
-            window (tests with consistent history)
-            - "expanding": Growing train window (uses all data from start_date)
-        start_date (datetime, optional): The starting date for the first
-            training window. Defaults to the earliest date in the data.
-        end_date (datetime, optional): The ending date for the last
-            testing window. Defaults to the latest date in the data.
+        start_date (datetime): The starting date for the first
+            training window.
+        end_date (datetime): The ending date for the last
+            testing window.
         train_days (int): Number of days in the training window.
         test_days (int): Number of days in the testing window.
         step_days (int): Number of days to step forward for the next window.
         gap_days (int): Number of days to skip between train and test windows.
             Useful to avoid look-ahead bias in indicators
             with lag (e.g., 26 for MACD). Default is 0 (no gap).
-        min_data_threshold (float): Minimum fraction of expected data required.
-            Windows with less data than (train_days * threshold)
-            or (test_days * threshold) will be skipped. Default is 0.8 (80%).
 
     Returns:
-        List[Dict[str, any]]: A list of dictionaries containing:
+        List[Dict[str, BacktestDateRange]]: A list of dictionaries containing:
             - "train_range": BacktestDateRange for training period
             - "test_range": BacktestDateRange for testing period
-            - "train_data": DataFrame for training period
-            - "test_data": DataFrame for testing period
-            - "metadata": Dictionary with window statistics
 
     Example:
         >>> windows = generate_rolling_backtest_windows(
-        ...     data=btc_data,
-        ...     start_date=datetime(2021, 1, 1),
-        ...     end_date=datetime(2024, 12, 31),
+        ...     start_date=datetime(2021, 1, 1, tzinfo=timezone.utc),
+        ...     end_date=datetime(2024, 12, 31, tzinfo=timezone.utc),
         ...     train_days=365,
         ...     test_days=90,
         ...     step_days=90,
-        ...     window_type="expanding",
         ...     gap_days=30
         ... )
     """
-    if start_date is None:
-        start_date = data.index.min()
-    if end_date is None:
-        end_date = data.index.max()
-
     windows = []
     current_start = start_date
     max_iterations = 10000  # Safety limit to prevent infinite loops
@@ -203,89 +182,19 @@ def generate_rolling_backtest_windows(
         if test_end > end_date:
             break
 
-        if window_type == "expanding":
-            # Train on ALL data from start_date to train_end (growing window)
-            train_data = data[(data.index >= start_date)
-                              & (data.index < train_end)]
-        else:
-            # Current implementation (fixed window)
-            train_data = data[(data.index >= train_start)
-                              & (data.index < train_end)]
-
-        test_data = data[(data.index >= test_start) & (data.index < test_end)]
-
-        # Validate minimum data threshold
-        min_train_records = int(train_days * min_data_threshold)
-        min_test_records = int(test_days * min_data_threshold)
-
-        if len(train_data) < min_train_records:
-            logger.warning(
-                f"Skipping window starting {train_start.date()}: "
-                f"train data ({len(train_data)} records) "
-                f"below threshold ({min_train_records})"
-            )
-            current_start += pd.Timedelta(days=step_days)
-            continue
-
-        if len(test_data) < min_test_records:
-            logger.warning(
-                f"Skipping window starting {train_start.date()}: "
-                f"test data ({len(test_data)} records) "
-                f"below threshold ({min_test_records})"
-            )
-            current_start += pd.Timedelta(days=step_days)
-            continue
-
-        # Calculate metadata
-        train_actual_start = train_data.index.min() \
-            if len(train_data) > 0 else train_start
-        train_actual_end = train_data.index.max() \
-            if len(train_data) > 0 else train_end
-        test_actual_start = test_data.index.min() \
-            if len(test_data) > 0 else test_start
-        test_actual_end = test_data.index.max() \
-            if len(test_data) > 0 else test_end
-
-        metadata = {
-            "train_days_actual": len(train_data),
-            "test_days_actual": len(test_data),
-            "train_days_expected": train_days,
-            "test_days_expected": test_days,
-            "gap_days": gap_days,
-            "train_actual_start": train_actual_start,
-            "train_actual_end": train_actual_end,
-            "test_actual_start": test_actual_start,
-            "test_actual_end": test_actual_end,
-            "window_type": window_type
-        }
-
-        # Calculate returns if close price exists
-        if 'close' in train_data.columns and len(train_data) > 1:
-            metadata["train_return"] = (train_data['close'].iloc[-1]
-                                        / train_data['close'].iloc[0]) - 1
-        else:
-            metadata["train_return"] = None
-
-        if 'close' in test_data.columns and len(test_data) > 1:
-            metadata["test_return"] = (test_data['close'].iloc[-1]
-                                       / test_data['close'].iloc[0]) - 1
-        else:
-            metadata["test_return"] = None
-
         train_backtest_date_range = BacktestDateRange(
-            name=f"train_range_{iteration}",
+            name=f"train_window_{iteration}",
             start_date=train_start,
             end_date=train_end
         )
         test_backtest_date_range = BacktestDateRange(
-            name=f"test_range_{iteration}",
+            name=f"test_window_{iteration}",
             start_date=test_start,
             end_date=test_end
         )
         windows.append({
             "train_range": train_backtest_date_range,
             "test_range": test_backtest_date_range,
-            "metadata": metadata
         })
 
         current_start += pd.Timedelta(days=step_days)

investing_algorithm_framework-7.25.4/investing_algorithm_framework/analysis/data.py

@@ -0,0 +1,161 @@
+import pandas as pd
+import polars as pl
+from typing import Union
+
+
+def fill_missing_timeseries_data(
+    data: Union[pd.DataFrame, pl.DataFrame, str],
+    missing_dates: list = None,
+    start_date=None,
+    end_date=None,
+    save_to_file: bool = False,
+    file_path: str = None
+) -> Union[pd.DataFrame, pl.DataFrame]:
+    """
+    Fill missing dates in time-series data using a hybrid approach:
+    forward-fill by default, backward-fill only when missing dates
+    are at the start of the data.
+
+    This function handles missing rows (dates) in time-series data,
+    not just missing values within existing rows. All columns are
+    duplicated from the adjacent row.
+
+    Args:
+        data (pd.DataFrame | pl.DataFrame | str): Time-series data as a
+            DataFrame or path to a CSV file. Index should be datetime.
+        missing_dates (list, optional): List of datetime objects representing
+            specific missing dates to fill. If None and start_date/end_date
+            are provided, missing dates will be auto-detected.
+        start_date (datetime, optional): Start date to check for missing dates.
+        end_date (datetime, optional): End date to check for missing dates.
+        save_to_file (bool): If True, save the result back to the CSV file.
+        file_path (str, optional): Path to save/load the CSV file.
+
+    Returns:
+        pd.DataFrame | pl.DataFrame: DataFrame with missing dates filled.
+    """
+    is_polars = False
+
+    # Load data if file path is provided
+    if isinstance(data, str):
+        file_path = data
+        df = pd.read_csv(file_path, index_col=0, parse_dates=True)
+    elif isinstance(data, pl.DataFrame):
+        is_polars = True
+        # Convert polars to pandas for processing
+        df = data.to_pandas()
+        if 'Datetime' in df.columns:
+            df = df.set_index('Datetime')
+        df.index = pd.to_datetime(df.index)
+    else:
+        df = data.copy()
+
+    # Store the index name
+    index_name = df.index.name if df.index.name else 'Datetime'
+
+    rows_to_add = []
+
+    # Determine which dates need to be filled
+    if missing_dates is not None and len(missing_dates) > 0:
+        dates_to_fill = [d for d in missing_dates if d not in df.index]
+    elif start_date is not None and end_date is not None:
+        dates_to_fill = get_missing_timeseries_data_entries(
+            df, start=start_date, end=end_date
+        )
+    else:
+        dates_to_fill = []
+
+    # Fill missing dates
+    for missing_timestamp in dates_to_fill:
+        # Find the index position where this timestamp should be inserted
+        position = df[df.index < missing_timestamp].shape[0]
+
+        # Edge case: missing timestamp is BEFORE all existing data
+        if position == 0:
+            # Use the FIRST row (backward fill)
+            if len(df) > 0:
+                next_row = df.iloc[0]
+                new_row = next_row.to_dict()
+            else:
+                continue  # No data to fill from
+        else:
+            # Normal case: use previous row (forward fill)
+            prev_row = df.iloc[position - 1]
+            new_row = prev_row.to_dict()
+
+        rows_to_add.append({index_name: missing_timestamp, **new_row})
+
+    # Add new rows if any
+    if rows_to_add:
+        new_df = pd.DataFrame(rows_to_add).set_index(index_name)
+        df = pd.concat([df, new_df]).sort_index()
+
+    # Save to file if requested
+    if save_to_file and file_path:
+        df.to_csv(file_path)
+
+    # Convert back to polars if input was polars
+    if is_polars:
+        df = df.reset_index()
+        return pl.from_pandas(df)
+
+    return df
+
+
+def get_missing_timeseries_data_entries(
+    data: Union[pd.DataFrame, pl.DataFrame, str],
+    start=None,
+    end=None,
+    freq: str = None
+):
+    """
+    Identify missing timestamps in a time series.
+
+    Args:
+        data (pd.DataFrame | pl.DataFrame | str): Time-series data as a
+            DataFrame or path to a CSV file. Index should be datetime.
+        start (datetime, optional): The start datetime for the expected range.
+            If None, uses the first timestamp in the data.
+        end (datetime, optional): The end datetime for the expected range.
+            If None, uses the last timestamp in the data.
+        freq (str, optional): Frequency string (e.g., 'D' for daily, 'H' for
+            hourly). If None, will be inferred from the data.
+
+    Returns:
+        list: A list of missing timestamps within the specified range.
+    """
+    # Load data if file path is provided
+    if isinstance(data, str):
+        df = pd.read_csv(data, index_col=0, parse_dates=True)
+    elif isinstance(data, pl.DataFrame):
+        df = data.to_pandas()
+        if 'Datetime' in df.columns:
+            df = df.set_index('Datetime')
+        df.index = pd.to_datetime(df.index)
+    else:
+        df = data
+
+    # Get existing timestamps from the index
+    existing_timestamps = pd.to_datetime(df.index)
+
+    # Use data bounds if start/end not provided
+    if start is None:
+        start = existing_timestamps.min()
+    if end is None:
+        end = existing_timestamps.max()
+
+    # Infer frequency if not provided
+    if freq is None:
+        freq = df.index.inferred_freq
+        if freq is None and len(df) >= 2:
+            diff = df.index[1] - df.index[0]
+            freq = diff
+        elif freq is None:
+            freq = 'D'  # Default to daily
+
+    expected_timestamps = pd.date_range(start=start, end=end, freq=freq)
+
+    # Find missing by checking which expected timestamps are not in existing
+    missing_mask = ~expected_timestamps.isin(existing_timestamps)
+    missing_timestamps = expected_timestamps[missing_mask].tolist()
+    return missing_timestamps

investing_algorithm_framework-7.25.4/investing_algorithm_framework/analysis/markdown.py

@@ -0,0 +1,91 @@
+from typing import Dict, List, Any, Union
+
+
+def create_markdown_table(data: List[Union[Dict[str, Any], Any]]):
+    """
+    Create a markdown table with evenly spaced columns for nice display
+    in notebook output cells.
+
+    Args:
+        data (List[dict] or List[object]): List of dictionaries or objects
+            containing data.
+
+    Returns:
+        str: Markdown formatted table with evenly spaced columns.
+    """
+    if not data or len(data) == 0:
+        return ("| No Data Available |\n"
+                "|-------------------|\n"
+                "| No records found  |\n")
+
+    # Determine if data contains dicts or objects
+    is_dict = isinstance(data[0], dict)
+
+    # Get columns from data
+    if is_dict:
+        columns = list(data[0].keys())
+    else:
+        # For objects, get all attributes (excluding private ones)
+        columns = [
+            attr for attr in dir(data[0])
+            if not attr.startswith('_')
+            and not callable(getattr(data[0], attr))
+        ]
+
+    # Generate header titles
+    header_titles = [col.replace("_", " ").title() for col in columns]
+
+    # Collect and format all row data
+    all_rows_data = []
+    for item in data:
+        row_values = []
+        for col in columns:
+            # Get value
+            if is_dict:
+                value = item.get(col)
+            else:
+                value = getattr(item, col, None)
+
+            # Format value
+            if value is None:
+                formatted_value = "N/A"
+            elif isinstance(value, float):
+                formatted_value = f"{value:.2f}"
+            else:
+                formatted_value = str(value)
+
+            row_values.append(formatted_value)
+        all_rows_data.append(row_values)
+
+    # Calculate column widths based on both headers and data
+    col_widths = []
+    for i, header in enumerate(header_titles):
+        max_width = len(header)
+        for row in all_rows_data:
+            if i < len(row):
+                max_width = max(max_width, len(row[i]))
+        col_widths.append(max_width)
+
+    # Build markdown table
+    markdown = ""
+
+    # Header with padding
+    header_parts = [
+        title.ljust(width)
+        for title, width in zip(header_titles, col_widths)
+    ]
+    markdown += "| " + " | ".join(header_parts) + " |\n"
+
+    # Separator
+    separator_parts = ["-" * width for width in col_widths]
+    markdown += "| " + " | ".join(separator_parts) + " |\n"
+
+    # Data rows
+    for row_values in all_rows_data:
+        row_parts = [
+            value.ljust(width)
+            for value, width in zip(row_values, col_widths)
+        ]
+        markdown += "| " + " | ".join(row_parts) + " |\n"
+
+    return markdown

{investing_algorithm_framework-7.25.2 → investing_algorithm_framework-7.25.4}/investing_algorithm_framework/app/__init__.py

@@ -13,8 +13,6 @@ from .reporting import add_html_report, \
     get_monthly_returns_heatmap_chart, \
     get_yearly_returns_bar_chart, get_equity_curve_chart, \
     get_ohlcv_data_completeness_chart, get_entry_and_exit_signals
-from .analysis import select_backtest_date_ranges, rank_results, \
-    create_weights, generate_rolling_backtest_windows
 
 
 __all__ = [
@@ -32,15 +30,11 @@ __all__ = [
     "pretty_print_trades",
     "pretty_print_positions",
     "pretty_print_orders",
-    "select_backtest_date_ranges",
     "get_equity_curve_with_drawdown_chart",
     "get_rolling_sharpe_ratio_chart",
     "get_monthly_returns_heatmap_chart",
     "get_yearly_returns_bar_chart",
     "get_ohlcv_data_completeness_chart",
-    "rank_results",
-    "create_weights",
     "get_entry_and_exit_signals",
     "get_equity_curve_chart",
-    "generate_rolling_backtest_windows"
 ]

{investing_algorithm_framework-7.25.2 → investing_algorithm_framework-7.25.4}/investing_algorithm_framework/app/app.py

@@ -25,12 +25,11 @@ from investing_algorithm_framework.domain import DATABASE_NAME, TimeUnit, \
 from investing_algorithm_framework.infrastructure import setup_sqlalchemy, \
     create_all_tables, CCXTOrderExecutor, CCXTPortfolioProvider, \
     BacktestOrderExecutor, CCXTOHLCVDataProvider, clear_db, \
-    PandasOHLCVDataProvider
+    PandasOHLCVDataProvider, BacktestService
 from investing_algorithm_framework.services import OrderBacktestService, \
     BacktestPortfolioService, DefaultTradeOrderEvaluator, get_risk_free_rate_us
 from .app_hook import AppHook
 from .eventloop import EventLoopService
-from .analysis import create_ohlcv_permutation
 
 logger = logging.getLogger("investing_algorithm_framework")
 COLOR_RESET = '\033[0m'
@@ -1638,6 +1637,7 @@ class App:
                     "connection"
                 )
 
+        backtest_service = self.container.backtest_service()
         data_provider_service = self.container.data_provider_service()
         backtest = self.run_vector_backtest(
             backtest_date_range=backtest_date_range,
@@ -1690,7 +1690,8 @@ class App:
 
             for combi in original_data_combinations:
                 # Permute the data for the data source
-                permutated_data = create_ohlcv_permutation(data=combi[1])
+                permutated_data = backtest_service\
+                    .create_ohlcv_permutation(data=combi[1])
                 permutated_datasets.append((combi[0], permutated_data))
 
                 if combi[0].symbol not in permuted_datasets_ordered_by_symbol: