dijkies 0.0.3__py3-none-any.whl

dijkies/__init__.py

@@ -0,0 +1,4 @@
+from dijkies.executors import ExchangeAssetClient, Order, State
+from dijkies.strategy import Strategy
+
+__all__ = ["Strategy", "State", "Order", "ExchangeAssetClient"]

dijkies/backtest.py

@@ -0,0 +1,98 @@
+from datetime import datetime, timedelta
+from typing import Optional
+
+import pandas as pd
+from pandas.core.frame import DataFrame as PandasDataFrame
+
+from dijkies.evaluate import EvaluationFramework
+from dijkies.executors import BacktestExchangeAssetClient
+from dijkies.performance import PerformanceInformationRow
+from dijkies.exceptions import (
+    DataTimeWindowShorterThanSuggestedAnalysisWindowError,
+    InvalidExchangeAssetClientError,
+    InvalidTypeForTimeColumnError,
+    MissingOHLCVColumnsError,
+    TimeColumnNotDefinedError,
+)
+from dijkies.strategy import Strategy
+
+
+class Backtester:
+    def __init__(
+        self,
+        evaluation: Optional[EvaluationFramework] = None,
+    ):
+        self.evaluation = evaluation
+
+    @staticmethod
+    def get_analysis_df(
+        data: PandasDataFrame, current_time: datetime, look_back_in_min: int
+    ) -> PandasDataFrame:
+        start_analysis_df = current_time - timedelta(minutes=look_back_in_min)
+
+        analysis_df = data.loc[
+            (data.time >= start_analysis_df) & (data.time <= current_time)
+        ]
+
+        return analysis_df
+
+    def simulate(
+        self,
+        data: PandasDataFrame,
+        strategy: Strategy,
+    ) -> PandasDataFrame:
+        """
+        This method runs the backtest. It expects data, this should have the following properties:
+        """
+
+        # validate args
+
+        if "time" not in data.columns:
+            raise TimeColumnNotDefinedError()
+
+        if not pd.api.types.is_datetime64_any_dtype(data.time):
+            raise InvalidTypeForTimeColumnError()
+
+        lookback_in_min = strategy.analysis_dataframe_size_in_minutes
+        timespan_data_in_min = (data.time.max() - data.time.min()).total_seconds() / 60
+
+        if lookback_in_min > timespan_data_in_min:
+            raise DataTimeWindowShorterThanSuggestedAnalysisWindowError()
+
+        if not {"open", "high", "low", "close", "volume"}.issubset(data.columns):
+            raise MissingOHLCVColumnsError()
+
+        if not isinstance(strategy.executor, BacktestExchangeAssetClient):
+            raise InvalidExchangeAssetClientError()
+
+        start_time = data.iloc[0].time + timedelta(minutes=lookback_in_min)
+        simulation_df: PandasDataFrame = data.loc[data.time >= start_time]
+        start_candle = simulation_df.iloc[0]
+        start_value_in_quote = strategy.state.total_value_in_quote(start_candle.open)
+        result = []
+
+        for _, candle in simulation_df.iterrows():
+            analysis_df = self.get_analysis_df(data, candle.time, lookback_in_min)
+            strategy.executor.update_current_candle(candle)
+
+            strategy.run(analysis_df)
+
+            result.append(
+                PerformanceInformationRow.from_objects(
+                    candle, start_candle, strategy.state, start_value_in_quote
+                )
+            )
+
+        return pd.DataFrame([r.dict() for r in result])
+
+    def run(
+        self,
+        candle_df: PandasDataFrame,
+        strategy: Strategy,
+    ) -> PandasDataFrame:
+
+        results = self.simulate(candle_df, strategy)
+        if isinstance(self.evaluation, EvaluationFramework):
+            self.evaluation.evaluate(results)
+
+        return results

dijkies/credentials.py

@@ -0,0 +1,6 @@
+from pydantic import BaseModel
+
+
+class Credentials(BaseModel):
+    api_key: str
+    api_secret_key: str

dijkies/data_pipeline.py

@@ -0,0 +1,15 @@
+from abc import ABC, abstractmethod
+
+import pandas as pd
+from pandas.core.frame import DataFrame as PandasDataFrame
+
+
+class DataPipeline(ABC):
+    @abstractmethod
+    def run(self) -> PandasDataFrame:
+        pass
+
+
+class NoDataPipeline(ABC):
+    def run(self) -> PandasDataFrame:
+        return pd.DataFrame({})

dijkies/deployment.py

@@ -0,0 +1,127 @@
+import os
+import json
+import pickle
+
+from abc import ABC, abstractmethod
+
+from dijkies.strategy import Strategy
+from dijkies.executors import State
+from dijkies.credentials import Credentials
+from dijkies.executors import BitvavoExchangeAssetClient
+from dijkies.logger import get_logger
+
+
+class StrategyRepository(ABC):
+    @abstractmethod
+    def store(self, strategy: Strategy, id: str) -> None:
+        pass
+
+    @abstractmethod
+    def store_state(self, state: State, id: str) -> None:
+        pass
+
+    @abstractmethod
+    def read(self, id: str) -> Strategy:
+        pass
+
+    @abstractmethod
+    def read_state(self, id: str) -> State:
+        pass
+
+
+class LocalStrategyRepository(StrategyRepository):
+    def __init__(self, root_directory: str) -> None:
+        self.root_directory = root_directory
+
+    def store(self, strategy: Strategy, id: str):
+        path = os.path.join(self.root_directory, id + ".pkl")
+        with open(path, "wb") as file:
+            pickle.dump(strategy, file)
+
+    def store_state(self, state: State, id: str) -> None:
+        path = os.path.join(self.root_directory, "state_" + id, ".json")
+        with open(path, "w") as file:
+            json.dump(state, file)
+
+    def read(self, id: str) -> Strategy:
+        path = os.path.join(self.root_directory, id + ".pkl")
+        with open(path, "rb") as file:
+            strategy = pickle.load(file)
+        return strategy
+
+    def read_state(self, id: str) -> State:
+        path = os.path.join(self.root_directory, "state_" + id, ".json")
+        with open(path, "r") as file:
+            state = json.load(file)
+        return state
+
+
+class CredentialsRepository(ABC):
+    @abstractmethod
+    def get_api_key(self, id: str) -> str:
+        pass
+
+    @abstractmethod
+    def store_api_key(self, id: str, api_key: str) -> None:
+        pass
+
+    @abstractmethod
+    def get_api_secret_key(self, id: str) -> str:
+        pass
+
+    @abstractmethod
+    def store_api_secret_key(self, id: str, api_secret_key: str) -> None:
+        pass
+
+    def get_credentials(self, id: str) -> Credentials:
+        return Credentials(
+            api_key=self.get_api_key(id),
+            api_secret_key=self.get_api_secret_key(id)
+        )
+
+
+class LocalCredentialRepository(CredentialsRepository):
+    def get_api_key(self, id: str) -> str:
+        return os.environ.get(f"{id}_api_key")
+
+    def store_api_key(self, id: str, api_key: str) -> None:
+        pass
+
+    def get_api_secret_key(self, id: str) -> str:
+        return os.environ.get(f"{id}_api_secret_key")
+
+    def store_api_secret_key(self, id: str, api_secret_key: str) -> None:
+        pass
+
+
+class Bot:
+    def __init__(
+        self,
+        strategy_repository: StrategyRepository,
+        credential_repository: CredentialsRepository,
+    ) -> None:
+        self.strategy_repository = strategy_repository
+        self.credential_repository = credential_repository
+
+    def set_executor(self, strategy: Strategy):
+        api_key = self.credential_repository.get_api_key(id)
+        api_secret_key = self.credential_repository.get_api_secret_key(id)
+
+        if strategy.exchange == "bitvavo":
+            strategy.executor = BitvavoExchangeAssetClient(
+                strategy.state,
+                api_key,
+                api_secret_key,
+                1,
+                get_logger()
+            )
+
+    def run(self, id: str) -> None:
+        strategy = self.strategy_repository.read(id)
+        self.set_executor(strategy)
+
+        data_pipeline = strategy.get_data_pipeline()
+        data = data_pipeline.run()
+        strategy.run(data)
+
+        self.strategy_repository.store(strategy)

dijkies/evaluate.py

@@ -0,0 +1,235 @@
+import logging
+import os
+import tempfile
+from abc import ABC, abstractmethod
+from datetime import datetime, timezone
+from typing import Optional, Union
+
+import matplotlib.pyplot as plt
+import mlflow
+import pandas as pd
+from pandas.core.frame import DataFrame as PandasDataFrame
+
+from dijkies.performance import Metric
+
+
+class EvaluationFramework(ABC):
+    @abstractmethod
+    def evaluate(self, performance_results: PandasDataFrame) -> None:
+        pass
+
+
+class MLFlowEvaluator(ABC):
+    @abstractmethod
+    def evaluate(self, performance_results: PandasDataFrame) -> None:
+        pass
+
+
+class MLFlowEvaluationFramework(EvaluationFramework):
+    def __init__(
+        self,
+        evaluators: list[MLFlowEvaluator],
+        experiment_name: str,
+        logger: logging.Logger,
+        strategy_parameters: Optional[dict[str, Union[int, str, float, bool]]],
+        log_dataset: bool = False,
+    ) -> None:
+        self.evaluators = evaluators
+        self.logger = logger
+        self.experiment_name = experiment_name
+        self.log_dataset = log_dataset
+        self.strategy_parameters = strategy_parameters
+
+    def evaluate(self, performance_results: PandasDataFrame) -> None:
+        mlflow.set_experiment(self.experiment_name)
+        # for results:
+        # poetry run mlflow server --host 127.0.0.1 --port 8080
+
+        run_name = "run__" + datetime.now(tz=timezone.utc).strftime("%Y_%m_%d_%H_%M%Z")
+
+        with mlflow.start_run(run_name=run_name) as run:
+            run_id = run.info.run_id
+            self.logger.info("Run created: " + run_id)
+            if self.log_dataset:
+                dataset = mlflow.data.from_pandas(
+                    performance_results, source="local", name="training_data"
+                )
+                mlflow.log_input(dataset, context="training")
+                mlflow.log_params(self.strategy_parameters)
+                with tempfile.TemporaryDirectory() as tmpdir:
+                    file_path = os.path.join(tmpdir, "data.csv")
+                    performance_results.to_csv(file_path, index=False)
+                    mlflow.log_artifact(file_path)
+
+            [
+                evaluator.evaluate(performance_results) for evaluator in self.evaluators
+            ]  # type: ignore
+
+
+class MLFlowOverallEvaluator(MLFlowEvaluator):
+    def __init__(self, metrics: list[Metric], logger: logging.Logger) -> None:
+        self.metrics = metrics
+        self.logger = logger
+
+    def log_metrics(self, performance_results: PandasDataFrame) -> None:
+        for metric in self.metrics:
+            mlflow.log_metric(
+                "strategy_" + metric.metric_name,
+                round(metric.calculate(performance_results.total_value_strategy), 2),
+            )
+            mlflow.log_metric(
+                "hodl_" + metric.metric_name,
+                round(metric.calculate(performance_results.total_value_hodl), 2),
+            )
+
+    @staticmethod
+    def plot_fee(performance_results: PandasDataFrame) -> None:
+        plt.figure(figsize=(8, 5))
+        plt.plot(
+            performance_results.candle_time,
+            performance_results.total_fee_paid,
+            color="blue",
+            label="fee",
+        )
+        plt.xlabel("Time")
+        plt.xticks(rotation=45)
+        plt.ylabel("fee paid in €")
+        plt.title("total transaction fee paid to Exchange")
+        plt.grid(True)
+        plt.legend()
+
+        # Log figure directly to MLflow
+        mlflow.log_figure(plt.gcf(), "total_fee_paid.png")
+
+        plt.close()  # free memory
+
+    @staticmethod
+    def plot_balance_fractions(performance_results: PandasDataFrame) -> None:
+        perc_in_quote = (
+            performance_results.balance_total_quote
+            / performance_results.total_value_strategy
+        )
+        perc_in_base = (
+            performance_results.balance_total_base
+            / performance_results.total_value_strategy
+            * performance_results.candle_close
+        )
+
+        plt.figure(figsize=(8, 5))
+        plt.plot(
+            performance_results.candle_time,
+            perc_in_quote,
+            color="blue",
+            label="percentage value in quote",
+        )
+        plt.plot(
+            performance_results.candle_time,
+            perc_in_base,
+            color="red",
+            label="percentage value in base",
+        )
+        plt.xlabel("Time")
+        plt.xticks(rotation=45)
+        plt.ylabel("fraction")
+        plt.title("fraction of total value in quote")
+        plt.grid(True)
+        plt.legend()
+
+        # Log figure directly to MLflow
+        mlflow.log_figure(plt.gcf(), "balance_fractions.png")
+
+        plt.close()  # free memory
+
+    @staticmethod
+    def plot_strategy_vs_hodl(performance_results: PandasDataFrame) -> None:
+        plt.figure(figsize=(8, 5))
+        plt.plot(
+            performance_results.candle_time,
+            performance_results.total_value_strategy,
+            color="blue",
+            label="strategy",
+        )
+        plt.plot(
+            performance_results.candle_time,
+            performance_results.total_value_hodl,
+            color="red",
+            label="hodl",
+        )
+        plt.xlabel("Time")
+        plt.xticks(rotation=45)
+        plt.ylabel("Value")
+        plt.title("strategy vs. hodl")
+        plt.grid(True)
+        plt.legend()
+
+        # Log figure directly to MLflow
+        mlflow.log_figure(plt.gcf(), "overal_result.png")
+
+        plt.close()  # free memory
+
+    def plot_results(self, performance_results: PandasDataFrame) -> None:
+        self.plot_strategy_vs_hodl(performance_results)
+        self.plot_fee(performance_results)
+        self.plot_balance_fractions(performance_results)
+
+    def evaluate(self, performance_results: PandasDataFrame) -> None:
+        self.log_metrics(performance_results)
+        self.plot_results(performance_results)
+
+
+class MLFlowSliceEvaluator(MLFlowEvaluator):
+    def __init__(
+        self, window_size_in_min: int, metrics: list[Metric], logger: logging.Logger
+    ) -> None:
+        self.window_size_in_min = window_size_in_min
+        self.metrics = metrics
+        self.logger = logger
+
+    def results_window_slicer(self, results: PandasDataFrame) -> PandasDataFrame:
+        candle_interval_in_minutes = (
+            results.iloc[1].candle_time - results.iloc[0].candle_time
+        ).total_seconds() / 60
+        window_size = self.window_size_in_min / candle_interval_in_minutes
+
+        evaluation = []
+
+        for sub_result in [
+            results.loc[i : i + window_size]
+            for i in range(len(results) - (int(window_size) + 1))
+        ]:
+            row = {}
+            for metric in self.metrics:
+                row["strategy_" + metric.metric_name] = metric.calculate(
+                    sub_result.total_value_strategy
+                )
+                row["hodl_" + metric.metric_name] = metric.calculate(
+                    sub_result.total_value_hodl
+                )
+            evaluation.append(row)
+
+        return pd.DataFrame(evaluation)
+
+    def plot_results(self, slicer_results: PandasDataFrame) -> None:
+        for col in slicer_results.columns:
+            self.logger.info(f"create plot {col}")
+            plt.figure(figsize=(8, 5))
+            plt.hist(slicer_results[col])
+            plt.title(f"Column: {col}")
+            plt.xlabel(col)
+            plt.grid(True)
+
+            # Log figure directly to MLflow
+            mlflow.log_figure(plt.gcf(), f"{col}.png")
+
+            plt.close()  # free memory
+
+    def log_metrics(self, slicer_results: PandasDataFrame) -> None:
+        for col in slicer_results.columns:
+            self.logger.info(f"compute metrics for {col}")
+            mlflow.log_metric(f"{col}_mean", round(slicer_results[col].mean(), 3))
+            mlflow.log_metric(f"{col}_std", round(slicer_results[col].std(), 3))
+
+    def evaluate(self, performance_results: PandasDataFrame) -> None:
+        slicer_results = self.results_window_slicer(performance_results)
+        self.log_metrics(slicer_results)
+        self.plot_results(slicer_results)

dijkies/exceptions.py

@@ -0,0 +1,60 @@
+class TimeColumnNotDefinedError(Exception):
+    def __init__(self):
+        super().__init__("the provided data should have a 'time' column")
+
+
+class InvalidTypeForTimeColumnError(Exception):
+    def __init__(self):
+        super().__init__("'time' column has not the right dtype")
+
+
+class DataTimeWindowShorterThanSuggestedAnalysisWindowError(Exception):
+    def __init__(self):
+        super().__init__(
+            "the timespan of provided data is shorter than the analysis window, so no backtest can be executed."
+        )
+
+
+class MissingOHLCVColumnsError(Exception):
+    def __init__(self):
+        super().__init__("one of the OHLCV columns is missing in the provided data")
+
+
+class DataTimeSpanDifferentFromAlgorithmSetting(Exception):
+    def __init__(self):
+        super().__init__("one of the OHLCV columns is missing in the provided data")
+
+
+class InvalidExchangeAssetClientError(Exception):
+    def __init__(self):
+        super().__init__("use BacktestExchangeAssetExecutor! glad that I saved you :)")
+
+
+class NoOrderFoundError(Exception):
+    def __init__(self, order_id):
+        super().__init__(f"order with order_id {order_id} not found.")
+
+
+class MultipleOrdersFoundError(Exception):
+    def __init__(self, order_id):
+        super().__init__(f"multiple orders found with order_id {order_id}.")
+
+
+class PlaceOrderError(Exception):
+    def __init__(self, message: str):
+        super().__init__(f"an error occured during order creation: {message}")
+
+
+class GetOrderInfoError(Exception):
+    def __init__(self, message: str):
+        super().__init__(f"an error occured during order info retrieval: {message}")
+
+
+class CancelOrderError(Exception):
+    def __init__(self, message: str):
+        super().__init__(f"an error occured during cancelling order: {message}")
+
+
+class MethodNotDefinedError(Exception):
+    def __init__(self):
+        super().__init__("method not implemented...")