investing-algorithm-framework 6.9.1__py3-none-any.whl → 7.19.15__py3-none-any.whl

investing_algorithm_framework/infrastructure/data_providers/csv.py

@@ -1,147 +1,232 @@
-import polars
-from datetime import datetime
+from typing import List, Union
+from datetime import datetime, timezone, timedelta
+
+import polars as pl
+
 from investing_algorithm_framework.domain import DataProvider, \
-    TradingDataType, OperationalException
+    OperationalException, DataSource, DataType, TimeFrame, \
+    convert_polars_to_pandas
 
 
 class CSVOHLCVDataProvider(DataProvider):
     """
-    Implementation of Data Provider for OHLCV data.
+    Implementation of Data Provider for OHLCV data. OHLCV data
+    will be loaded from a CSV file. The CSV file should contain
+    the following columns: Datetime, Open, High, Low, Close, Volume.
+    The Datetime column should be in UTC timezone and in milliseconds.
+    The data will be loaded into a Polars DataFrame and will be kept in memory.
+
+    Attributes:
+        data_type (DataType): The type of data provided by this provider,
+            which is OHLCV.
+        data_provider_identifier (str): Identifier for the CSV OHLCV data
+            provider.
+        _start_date_data_source (datetime): The start date of the data
+            source, determined from the first row of the data.
+        _end_date_data_source (datetime): The end date of the data
+            source, determined from the last row of the data.
+        data (polars.DataFrame): The OHLCV data loaded from the CSV file.
     """
+    data_type = DataType.OHLCV
+    data_provider_identifier = "csv_ohlcv_data_provider"
+
     def __init__(
         self,
-        file_path: str,
+        storage_path: str,
         symbol: str,
         time_frame: str,
-        market: str = None,
-        priority: int = 0,
+        market: str,
         window_size=None,
-        storage_path=None,
+        data_provider_identifier: str = None,
+        pandas: bool = False,
     ):
         """
         Initialize the CSV Data Provider.
 
         Args:
-            file_path (str): Path to the CSV file.
+            storage_path (str): Path to the CSV file.
+            symbol (str): The symbol for which the data is provided.
+            time_frame (str): The time frame for the data.
+            market (str, optional): The market for the data. Defaults to None.
+            window_size (int, optional): The window size for the data.
+                Defaults to None.
         """
-
+        if data_provider_identifier is None:
+            data_provider_identifier = self.data_provider_identifier
         super().__init__(
-            data_type=TradingDataType.OHLCV.value,
             symbol=symbol,
             market=market,
-            markets=[],
-            priority=priority,
             time_frame=time_frame,
             window_size=window_size,
             storage_path=storage_path,
+            data_provider_identifier=data_provider_identifier,
+            data_type=DataType.OHLCV.value
         )
-        self.file_path = file_path
         self._start_date_data_source = None
         self._end_date_data_source = None
-        self.data = None
+        self._columns = ["Datetime", "Open", "High", "Low", "Close", "Volume"]
+        self.window_cache = {}
+        self._load_data(self.storage_path)
+        self.pandas = pandas
+        self.number_of_missing_data_points = 0
+        self.missing_data_point_dates: List[datetime] = []
 
     def has_data(
         self,
-        data_type: str = None,
-        symbol: str = None,
-        market: str = None,
-        time_frame: str = None,
+        data_source: DataSource,
         start_date: datetime = None,
-        end_date: datetime = None,
-        window_size=None
+        end_date: datetime = None
     ) -> bool:
+        """
+        Implementation of the has_data method to check if
+        the data provider has data for the given data source.
+
+        Args:
+            data_source (DataSource): The data source to check.
+            start_date (datetime, optional): The start date for the data.
+                Defaults to None.
+            end_date (datetime, optional): The end date for the data.
+                Defaults to None.
+
+        Returns:
+            bool: True if the data provider has data for the given data source,
+                False otherwise.
+        """
+        if start_date is None and end_date is None:
+            return False
+
+        if DataType.OHLCV.equals(data_source.data_type) and \
+                data_source.symbol == self.symbol and \
+                data_source.time_frame.equals(self.time_frame) and \
+                data_source.market == self.market:
+
+            if end_date > self._end_date_data_source:
+                return False
+
+            if data_source.window_size is not None:
+                minutes = TimeFrame.from_value(
+                    data_source.time_frame
+                ).amount_of_minutes * data_source.window_size
+                required_start_date = end_date - timedelta(
+                    minutes=minutes
+                )
+
+                if required_start_date < self._start_date_data_source:
+                    return False
+            else:
+                required_start_date = start_date
+                if required_start_date < self._start_date_data_source:
+                    return False
 
-        if symbol == self.symbol and market == self.market and \
-                data_type == self.data_type and time_frame == self.time_frame:
             return True
 
         return False
 
     def get_data(
         self,
-        data_type: str = None,
         date: datetime = None,
-        symbol: str = None,
-        market: str = None,
-        time_frame: str = None,
         start_date: datetime = None,
         end_date: datetime = None,
-        storage_path=None,
-        window_size=None,
-        pandas=False
+        save: bool = False,
     ):
+        """
+        Fetches OHLCV data for a given symbol and date range.
+        If no date range is provided, it returns the entire dataset.
 
-        if self.data is None:
-            self._load_data(self.file_path)
+        Args:
+            date (datetime, optional): A specific date to fetch data for.
+                Defaults to None.
+            start_date (datetime, optional): The start date for the data.
+                Defaults to None.
+            end_date (datetime, optional): The end date for the data.
+                Defaults to None.
+            save (bool, optional): Whether to save the data to a file.
+
+        Returns:
+            polars.DataFrame: A DataFrame containing the OHLCV data for the
+                specified symbol and date range.
+        """
+        windows_size = self.window_size
 
         if start_date is None and end_date is None:
-            return self.data
-
-        if end_date is not None and start_date is not None:
-
-            if end_date < start_date:
-                raise OperationalException(
-                    f"End date {end_date} is before the start date "
-                    f"{start_date}"
-                )
-
-            if start_date > self._end_date_data_source:
-                return polars.DataFrame()
-
+            end_date = datetime.now(tz=timezone.utc)
+            time_frame = TimeFrame.from_value(self.time_frame)
+            start_date = end_date - timedelta(
+                minutes=time_frame.amount_of_minutes() * windows_size
+            )
+        elif start_date is None and end_date is not None:
+            start_date = end_date - timedelta(
+                minutes=TimeFrame.from_value(
+                    self.time_frame
+                ).amount_of_minutes * windows_size
+            )
             df = self.data
             df = df.filter(
-                (df['Datetime'] >= start_date)
-                & (df['Datetime'] <= end_date)
+                (df['Datetime'] >= start_date) & (df['Datetime'] <= end_date)
             )
             return df
 
         if start_date is not None:
+            end_date = start_date + timedelta(
+                minutes=TimeFrame.from_value(self.time_frame)
+                .amount_of_minutes * windows_size
+            )
 
             if start_date < self._start_date_data_source:
-                return polars.DataFrame()
+                return pl.DataFrame()
 
             if start_date > self._end_date_data_source:
-                return polars.DataFrame()
+                return pl.DataFrame()
 
             df = self.data
             df = df.filter(
-                (df['Datetime'] >= start_date)
+                (df['Datetime'] >= start_date) & (df['Datetime'] <= end_date)
             )
-            df = df.head(self.window_size)
             return df
 
         if end_date is not None:
+            start_date = end_date - timedelta(
+                minutes=TimeFrame.from_value(
+                    self.time_frame
+                ).amount_of_minutes * windows_size
+            )
 
             if end_date < self._start_date_data_source:
-                return polars.DataFrame()
+                return pl.DataFrame()
 
             if end_date > self._end_date_data_source:
-                return polars.DataFrame()
+                return pl.DataFrame()
 
             df = self.data
             df = df.filter(
-                (df['Datetime'] <= end_date)
+                (df['Datetime'] >= start_date) & (df['Datetime'] <= end_date)
             )
-            df = df.tail(self.window_size)
             return df
 
         return self.data
 
-    def pre_pare_backtest_data(
+    def prepare_backtest_data(
         self,
         backtest_start_date,
-        backtest_end_date,
-        symbol: str = None,
-        market: str = None,
-        time_frame: str = None,
-        window_size=None
+        backtest_end_date
     ) -> None:
+        """
+        Prepares backtest data for a given symbol and date range.
 
-        if symbol is not None:
-            return
-
-        if self.data is None:
-            self._load_data(self.file_path)
+        Args:
+            backtest_start_date (datetime): The start date for the
+                backtest data.
+            backtest_end_date (datetime): The end date for the
+                backtest data.
+
+        Raises:
+            OperationalException: If the backtest start date is before the
+                start date of the data source or if the backtest end date is
+                after the end date of the data source.
+
+        Returns:
+            None
+        """
 
         if backtest_start_date < self._start_date_data_source:
             raise OperationalException(
@@ -155,80 +240,173 @@ class CSVOHLCVDataProvider(DataProvider):
                 f"end date {self._end_date_data_source}"
             )
 
+        # There must be at least backtest_start_date - window_size * time_frame
+        # data available to create a sliding window.
+        required_start_date = backtest_start_date - \
+            timedelta(
+                minutes=TimeFrame.from_value(self.time_frame)
+                .amount_of_minutes * self.window_size
+            )
+
+        # Create cache with sliding windows
+        self._precompute_sliding_windows(
+            window_size=self.window_size,
+            start_date=backtest_start_date,
+            end_date=backtest_end_date
+        )
+
+        if required_start_date < self._start_date_data_source:
+            self.number_of_missing_data_points = (
+                self._start_date_data_source - required_start_date
+            ).total_seconds() / (
+                TimeFrame.from_value(self.time_frame).amount_of_minutes * 60
+            )
+
+        n_min = TimeFrame.from_value(self.time_frame).amount_of_minutes
+
+        # Assume self.data is a Polars DataFrame with a "Datetime" column
+        expected_dates = pl.datetime_range(
+            start=required_start_date,
+            end=backtest_end_date,
+            interval=f"{n_min}m",
+            eager=True
+        ).to_list()
+
+        actual_dates = self.data["Datetime"].to_list()
+
+        # Find missing dates
+        self.missing_data_point_dates = sorted(
+            set(expected_dates) - set(actual_dates)
+        )
+
     def get_backtest_data(
         self,
-        date: datetime = None,
-        symbol: str = None,
-        market: str = None,
-        time_frame: str = None,
+        backtest_index_date: datetime,
         backtest_start_date: datetime = None,
         backtest_end_date: datetime = None,
-        window_size=None,
-        pandas=False
+        data_source: DataSource = None
     ) -> None:
+        """
+        Fetches backtest data for a given datasource
 
-        if self.data is None:
-            self._load_data(self.file_path)
+        Args:
+            backtest_index_date (datetime): The date for which to fetch
+               backtest data.
+            backtest_start_date (datetime): The start date for the
+               backtest data.
+            backtest_end_date (datetime): The end date for the
+               backtest data.
+            data_source (Optional[DataSource]): The data source specification
+               that matches a data provider.
+
+        Raises:
+            OperationalException: If the requested backtest date range
+                is outside the available data range.
+
+        Returns:
+           pl.DataFrame: The backtest data for the given datasource.
+        """
+        if backtest_start_date is not None and \
+                backtest_end_date is not None:
 
-        if backtest_start_date is None and backtest_end_date is None:
-            return self.data
+            if backtest_start_date < self._start_date_data_source:
 
-        if backtest_start_date is not None and backtest_end_date is not None:
+                if data_source is not None:
+                    raise OperationalException(
+                        f"Request data date {backtest_end_date} "
+                        f"is after the range of "
+                        f"the available data "
+                        f"{self._start_date_data_source} "
+                        f"- {self._end_date_data_source}."
+                        f" for data source {data_source.identifier}."
+                    )
 
-            if backtest_end_date < backtest_start_date:
                 raise OperationalException(
-                    f"Backtest end date {backtest_end_date} is before the "
-                    f"start date {backtest_start_date}"
+                    f"Request data date {backtest_start_date} "
+                    f"is before the range of "
+                    f"the available data "
+                    f"{self._start_date_data_source} "
+                    f"- {self._end_date_data_source}."
                 )
 
-            if backtest_start_date > self._end_date_data_source:
-                return polars.DataFrame()
-
-            df = self.data
-            df = df.filter(
-                (df['Datetime'] >= backtest_start_date)
-                & (df['Datetime'] <= backtest_end_date)
-            )
-            return df
+            if backtest_end_date > self._end_date_data_source:
 
-        if backtest_start_date is not None:
+                if data_source is not None:
+                    raise OperationalException(
+                        f"Request data date {backtest_end_date} "
+                        f"is after the range of "
+                        f"the available data "
+                        f"{self._start_date_data_source} "
+                        f"- {self._end_date_data_source}."
+                        f" for data source {data_source.identifier}."
+                    )
 
-            if backtest_start_date < self._start_date_data_source:
-                return polars.DataFrame()
-
-            if backtest_start_date > self._end_date_data_source:
-                return polars.DataFrame()
+                raise OperationalException(
+                    f"Request data date {backtest_end_date} "
+                    f"is after the range of "
+                    f"the available data "
+                    f"{self._start_date_data_source} "
+                    f"- {self._end_date_data_source}."
+                )
 
-            df = self.data
-            df = df.filter(
-                (df['Datetime'] >= backtest_start_date)
+            data = self.data.filter(
+                (pl.col("Datetime") >= backtest_start_date) &
+                (pl.col("Datetime") <= backtest_end_date)
             )
-            df = df.head(self.window_size)
-            return df
-
-        if backtest_end_date is not None:
+        else:
+            try:
+                data = self.window_cache[backtest_index_date]
+            except KeyError:
+
+                try:
+                    # Return the key in the cache that is closest to the
+                    # backtest_index_date but not after it.
+                    closest_key = min(
+                        [k for k in self.window_cache.keys()
+                         if k >= backtest_index_date]
+                    )
+                    data = self.window_cache[closest_key]
+                except ValueError:
+
+                    if data_source is not None:
+                        raise OperationalException(
+                            "No data available for the "
+                            f"date: {backtest_index_date} "
+                            "within the prepared backtest data "
+                            f"for data source {data_source.identifier}."
+                        )
+
+                    raise OperationalException(
+                        "No data available for the "
+                        f"date: {backtest_index_date} "
+                        "within the prepared backtest data."
+                    )
+
+        if self.pandas:
+            data = convert_polars_to_pandas(data)
+
+        return data
 
-            if backtest_end_date < self._start_date_data_source:
-                return polars.DataFrame()
+    def _load_data(self, storage_path):
+        """
+        Load OHLCV data from a CSV file into a Polars DataFrame.
+        The CSV file should contain the following columns:
 
-            if backtest_end_date > self._end_date_data_source:
-                return polars.DataFrame()
+        Datetime, Open, High, Low, Close, Volume.
 
-            df = self.data
-            df = df.filter(
-                (df['Datetime'] <= backtest_end_date)
-            )
-            df = df.tail(self.window_size)
-            return df
+        The Datetime column should be in UTC timezone and in milliseconds.
 
-        return self.data
+        Args:
+            storage_path (str): The path to the CSV file containing OHLCV data.
 
-    def _load_data(self, storage_path):
-        self._columns = [
-            "Datetime", "Open", "High", "Low", "Close", "Volume"
-        ]
+        Raises:
+            OperationalException: If the CSV file does not contain all
+                required OHLCV columns.
 
-        df = polars.read_csv(storage_path)
+        Returns:
+            None
+        """
+        df = pl.read_csv(storage_path)
 
         # Check if all column names are in the csv file
         if not all(column in df.columns for column in self._columns):
@@ -241,13 +419,13 @@ class CSVOHLCVDataProvider(DataProvider):
                 f"Missing columns: {missing_columns}"
             )
 
-        self.data = polars.read_csv(
+        self.data = pl.read_csv(
             storage_path,
-            schema_overrides={"Datetime": polars.Datetime},
+            schema_overrides={"Datetime": pl.Datetime},
             low_memory=True
         ).with_columns(
-            polars.col("Datetime").cast(
-                polars.Datetime(time_unit="ms", time_zone="UTC")
+            pl.col("Datetime").cast(
+                pl.Datetime(time_unit="ms", time_zone="UTC")
             )
         )
 
@@ -255,3 +433,136 @@ class CSVOHLCVDataProvider(DataProvider):
         last_row = self.data.tail(1)
         self._start_date_data_source = first_row["Datetime"][0]
         self._end_date_data_source = last_row["Datetime"][0]
+
+    def _precompute_sliding_windows(
+        self,
+        window_size: int,
+        start_date: datetime,
+        end_date: datetime
+    ) -> None:
+        """
+        Precompute all sliding windows for fast retrieval in backtest mode.
+
+        A sliding window is calculated as a subset of the data. It will
+        take for each timestamp in the data a window of size `window_size`
+        and stores it in a cache with the last timestamp of the window.
+
+        So if the window size is 200, the first window will be
+        the first 200 rows of the data, the second window will be
+        the rows 1 to 200, the third window will be the rows
+        2 to 201, and so on until the last window which will be
+        the last 200 rows of the data.
+
+        Args:
+            window_size (int): The size of the sliding window to precompute.
+            start_date (datetime, optional): The start date for the sliding
+                windows.
+            end_date (datetime, optional): The end date for the sliding
+                windows.
+
+        Returns:
+            None
+        """
+        self.window_cache = {}
+        timestamps = self.data["Datetime"].to_list()
+
+        # Only select the entries after the start date
+        timestamps = [
+            ts for ts in timestamps
+            if start_date <= ts <= end_date
+        ]
+
+        # Create sliding windows of size <window_size> for each timestamp
+        # in the data with the given the time frame and window size
+        for timestamp in timestamps:
+            # Use timestamp as key
+            self.window_cache[timestamp] = self.data.filter(
+                (self.data["Datetime"] <= timestamp) &
+                (self.data["Datetime"] >= timestamp - timedelta(
+                    minutes=self.time_frame.amount_of_minutes * window_size
+                ))
+            )
+
+    def copy(self, data_source: DataSource) -> "DataProvider":
+        """
+        Create a copy of the data provider with the given data source.
+
+        Args:
+            data_source (DataSource): The data source to copy.
+
+        Returns:
+            DataProvider: A new instance of the data provider with the
+                specified data source.
+        """
+
+        storage_path = data_source.storage_path
+
+        if storage_path is None:
+            storage_path = self.storage_path
+
+        return CSVOHLCVDataProvider(
+            storage_path=storage_path,
+            symbol=data_source.symbol,
+            time_frame=data_source.time_frame,
+            market=data_source.market,
+            window_size=data_source.window_size,
+            data_provider_identifier=self.data_provider_identifier,
+            pandas=data_source.pandas
+        )
+
+    def get_number_of_data_points(
+        self,
+        start_date: datetime,
+        end_date: datetime
+    ) -> int:
+
+        """
+        Returns the number of data points available between the given
+        start and end dates.
+
+        Args:
+            start_date (datetime): The start date for checking missing data.
+            end_date (datetime): The end date for checking missing data.
+
+        Returns:
+            int: The number of available data points between the given
+                start and end dates.
+        """
+        available_dates = [
+            date for date in self.data["Datetime"].to_list()
+            if start_date <= date <= end_date
+        ]
+        return len(available_dates)
+
+    def get_missing_data_dates(
+        self,
+        start_date: datetime,
+        end_date: datetime,
+    ) -> List[datetime]:
+        """
+        Returns a list of dates for which data is missing between the
+        given start and end dates.
+
+        Args:
+            start_date (datetime): The start date for checking missing data.
+            end_date (datetime): The end date for checking missing data.
+
+        Returns:
+            List[datetime]: A list of dates for which data is missing
+                between the given start and end dates.
+        """
+        missing_dates = [
+            date for date in self.missing_data_point_dates
+            if start_date <= date <= end_date
+        ]
+        return missing_dates
+
+    def get_data_source_file_path(self) -> Union[str, None]:
+        """
+        Get the file path of the data source if stored in local storage.
+
+        Returns:
+            Union[str, None]: The file path of the data source if stored
+                locally, otherwise None.
+        """
+        return self.storage_path