Qubx 0.6.37__cp312-cp312-manylinux_2_39_x86_64.whl → 0.6.40__cp312-cp312-manylinux_2_39_x86_64.whl

qubx/backtester/runner.py

@@ -20,8 +20,9 @@ from qubx.core.interfaces import (
     ITimeProvider,
     StrategyState,
 )
-from qubx.core.loggers import InMemoryLogsWriter, StrategyLogging
+from qubx.core.loggers import StrategyLogging
 from qubx.core.lookups import lookup
+from qubx.loggers.inmemory import InMemoryLogsWriter
 from qubx.pandaz.utils import _frame_to_str
 
 from .account import SimulatedAccountProcessor

qubx/cli/deploy.py

@@ -182,6 +182,29 @@ def setup_poetry_environment(output_dir: str) -> bool:
         return False
 
 
+def create_strategy_runners(output_dir: str):
+    """
+    Creates a strategy runner script in the output_dir
+    """
+    import sys
+
+    if sys.platform == "win32":
+        _pfx = ""
+        _f_name = os.path.join(output_dir, "run_paper.bat")
+    else:
+        _pfx = "#!/bin/bash\n"
+        _f_name = os.path.join(output_dir, "run_paper.sh")
+
+    logger.info(f"Creating strategy paper runner script: {_f_name}")
+
+    try:
+        with open(_f_name, "w") as f:
+            f.write(f"{_pfx}poetry run qubx run config.yml --paper -j")
+        os.chmod(_f_name, 0o755)
+    except Exception as e:
+        logger.error(f"Failed to create strategy paper runner script: {e}")
+
+
 def deploy_strategy(zip_file: str, output_dir: str | None, force: bool) -> bool:
     """
     Deploys a strategy from a zip file created by the release command.
@@ -222,9 +245,10 @@ def deploy_strategy(zip_file: str, output_dir: str | None, force: bool) -> bool:
     if not setup_poetry_environment(resolved_output_dir):
         return False
 
+    # Create the strategy runners
+    create_strategy_runners(resolved_output_dir)
+
     # Success messages
     logger.info(f"Strategy deployed successfully to {resolved_output_dir}")
-    logger.info(
-        f"To run the strategy (paper mode): <cyan>cd {resolved_output_dir} && poetry run qubx run config.yml --paper</cyan>"
-    )
+    logger.info(f" -> To run the strategy (in paper mode): <cyan>cd {resolved_output_dir} && ./run_paper.sh</cyan>")
     return True

qubx/core/loggers.py

@@ -1,10 +1,6 @@
-import csv
-import os
-from multiprocessing.pool import ThreadPool
 from typing import Any, Dict, List, Tuple
 
 import numpy as np
-import pandas as pd
 
 from qubx import logger
 from qubx.core.basics import (
@@ -14,11 +10,11 @@ from qubx.core.basics import (
     Position,
     TargetPosition,
 )
-from qubx.core.metrics import split_cumulative_pnl
+
 from qubx.core.series import time_as_nsec
 from qubx.core.utils import recognize_timeframe
-from qubx.pandaz.utils import scols
-from qubx.utils.misc import Stopwatch, makedirs
+
+from qubx.utils.misc import Stopwatch
 from qubx.utils.time import convert_tf_str_td64, floor_t64
 
 _SW = Stopwatch()
@@ -48,159 +44,6 @@ class LogsWriter:
         pass
 
 
-class InMemoryLogsWriter(LogsWriter):
-    _portfolio: List
-    _execs: List
-    _signals: List
-
-    def __init__(self, account_id: str, strategy_id: str, run_id: str) -> None:
-        super().__init__(account_id, strategy_id, run_id)
-        self._portfolio = []
-        self._execs = []
-        self._signals = []
-
-    def write_data(self, log_type: str, data: List[Dict[str, Any]]):
-        if len(data) > 0:
-            if log_type == "portfolio":
-                self._portfolio.extend(data)
-            elif log_type == "executions":
-                self._execs.extend(data)
-            elif log_type == "signals":
-                self._signals.extend(data)
-
-    def get_portfolio(self, as_plain_dataframe=True) -> pd.DataFrame:
-        pfl = pd.DataFrame.from_records(self._portfolio, index="timestamp")
-        pfl.index = pd.DatetimeIndex(pfl.index)
-        if as_plain_dataframe:
-            # - convert to Qube presentation (TODO: temporary)
-            pis = []
-            for s in set(pfl["symbol"]):
-                pi = pfl[pfl["symbol"] == s]
-                pi = pi.drop(columns=["symbol", "realized_pnl_quoted", "current_price", "exchange_time"])
-                pi = pi.rename(
-                    {
-                        "pnl_quoted": "PnL",
-                        "quantity": "Pos",
-                        "avg_position_price": "Price",
-                        "market_value_quoted": "Value",
-                        "commissions_quoted": "Commissions",
-                    },
-                    axis=1,
-                )
-                # We want to convert the value to just price * quantity
-                # in reality value of perps is just the unrealized pnl but
-                # it's not important after simulation for metric calculations
-                pi["Value"] = pi["Pos"] * pi["Price"] + pi["Value"]
-                pis.append(pi.rename(lambda x: s + "_" + x, axis=1))
-            return split_cumulative_pnl(scols(*pis))
-        return pfl
-
-    def get_executions(self) -> pd.DataFrame:
-        p = pd.DataFrame()
-        if self._execs:
-            p = pd.DataFrame.from_records(self._execs, index="timestamp")
-            p.index = pd.DatetimeIndex(p.index)
-        return p
-
-    def get_signals(self) -> pd.DataFrame:
-        p = pd.DataFrame()
-        if self._signals:
-            p = pd.DataFrame.from_records(self._signals, index="timestamp")
-            p.index = pd.DatetimeIndex(p.index)
-        return p
-
-
-class CsvFileLogsWriter(LogsWriter):
-    """
-    Simple CSV strategy log data writer. It does data writing in separate thread.
-    """
-
-    def __init__(self, account_id: str, strategy_id: str, run_id: str, log_folder="logs") -> None:
-        super().__init__(account_id, strategy_id, run_id)
-
-        path = makedirs(log_folder)
-        # - it rewrites positions every time
-        self._pos_file_path = f"{path}/{self.strategy_id}_{self.account_id}_positions.csv"
-        self._balance_file_path = f"{path}/{self.strategy_id}_{self.account_id}_balance.csv"
-
-        _pfl_path = f"{path}/{strategy_id}_{account_id}_portfolio.csv"
-        _exe_path = f"{path}/{strategy_id}_{account_id}_executions.csv"
-        _sig_path = f"{path}/{strategy_id}_{account_id}_signals.csv"
-        self._hdr_pfl = not os.path.exists(_pfl_path)
-        self._hdr_exe = not os.path.exists(_exe_path)
-        self._hdr_sig = not os.path.exists(_sig_path)
-
-        self._pfl_file_ = open(_pfl_path, "+a", newline="")
-        self._execs_file_ = open(_exe_path, "+a", newline="")
-        self._sig_file_ = open(_sig_path, "+a", newline="")
-        self._pfl_writer = csv.writer(self._pfl_file_)
-        self._exe_writer = csv.writer(self._execs_file_)
-        self._sig_writer = csv.writer(self._sig_file_)
-        self.pool = ThreadPool(3)
-
-    @staticmethod
-    def _header(d: dict) -> List[str]:
-        return list(d.keys()) + ["run_id"]
-
-    def _values(self, data: List[Dict[str, Any]]) -> List[List[str]]:
-        # - attach run_id (last column)
-        return [list((d | {"run_id": self.run_id}).values()) for d in data]
-
-    def _do_write(self, log_type, data):
-        match log_type:
-            case "positions":
-                with open(self._pos_file_path, "w", newline="") as f:
-                    w = csv.writer(f)
-                    w.writerow(self._header(data[0]))
-                    w.writerows(self._values(data))
-
-            case "portfolio":
-                if self._hdr_pfl:
-                    self._pfl_writer.writerow(self._header(data[0]))
-                    self._hdr_pfl = False
-                self._pfl_writer.writerows(self._values(data))
-                self._pfl_file_.flush()
-
-            case "executions":
-                if self._hdr_exe:
-                    self._exe_writer.writerow(self._header(data[0]))
-                    self._hdr_exe = False
-                self._exe_writer.writerows(self._values(data))
-                self._execs_file_.flush()
-
-            case "signals":
-                if self._hdr_sig:
-                    self._sig_writer.writerow(self._header(data[0]))
-                    self._hdr_sig = False
-                self._sig_writer.writerows(self._values(data))
-                self._sig_file_.flush()
-
-            case "balance":
-                with open(self._balance_file_path, "w", newline="") as f:
-                    w = csv.writer(f)
-                    w.writerow(self._header(data[0]))
-                    w.writerows(self._values(data))
-
-    def write_data(self, log_type: str, data: List[Dict[str, Any]]):
-        if len(data) > 0:
-            self.pool.apply_async(self._do_write, (log_type, data))
-
-    def flush_data(self):
-        try:
-            self._pfl_file_.flush()
-            self._execs_file_.flush()
-            self._sig_file_.flush()
-        except Exception as e:
-            logger.warning(f"Error flushing log writer: {str(e)}")
-
-    def close(self):
-        self._pfl_file_.close()
-        self._execs_file_.close()
-        self._sig_file_.close()
-        self.pool.close()
-        self.pool.join()
-
-
 class _BaseIntervalDumper:
     """
     Basic functionality for all interval based dumpers

qubx/core/metrics.py

@@ -884,7 +884,7 @@ class TradingSessionResult:
         _perf = info.pop("performance", None)
         info["instruments"] = info.pop("symbols")
         # - fix for old versions
-        _exch = info.pop("exchange")
+        _exch = info.pop("exchange") if "exchange" in info else info.pop("exchanges")
         info["exchanges"] = _exch if isinstance(_exch, list) else [_exch]
         tsr = TradingSessionResult(**info, portfolio_log=portfolio, executions_log=executions, signals_log=signals)
         tsr.qubx_version = _qbx_version

qubx/loggers/__init__.py

@@ -0,0 +1,17 @@
+"""
+Loggers module for qubx.
+
+This module provides implementations for logs writing, like csv writer or mongodb writer.
+"""
+
+from qubx.loggers.csv import CsvFileLogsWriter
+from qubx.loggers.inmemory import InMemoryLogsWriter
+from qubx.loggers.mongo import MongoDBLogsWriter
+from qubx.loggers.factory import create_logs_writer
+
+__all__ = [
+    "CsvFileLogsWriter",
+    "InMemoryLogsWriter",
+    "MongoDBLogsWriter",
+    "create_logs_writer",
+]

qubx/loggers/csv.py

@@ -0,0 +1,100 @@
+import csv
+import os
+
+from typing import Any, Dict, List
+from multiprocessing.pool import ThreadPool
+
+from qubx import logger
+from qubx.core.loggers import LogsWriter
+from qubx.utils.misc import  makedirs
+
+class CsvFileLogsWriter(LogsWriter):
+    """
+    Simple CSV strategy log data writer. It does data writing in separate thread.
+    """
+
+    def __init__(self, account_id: str, strategy_id: str, run_id: str, log_folder="logs") -> None:
+        super().__init__(account_id, strategy_id, run_id)
+
+        path = makedirs(log_folder)
+        # - it rewrites positions every time
+        self._pos_file_path = f"{path}/{self.strategy_id}_{self.account_id}_positions.csv"
+        self._balance_file_path = f"{path}/{self.strategy_id}_{self.account_id}_balance.csv"
+
+        _pfl_path = f"{path}/{strategy_id}_{account_id}_portfolio.csv"
+        _exe_path = f"{path}/{strategy_id}_{account_id}_executions.csv"
+        _sig_path = f"{path}/{strategy_id}_{account_id}_signals.csv"
+        self._hdr_pfl = not os.path.exists(_pfl_path)
+        self._hdr_exe = not os.path.exists(_exe_path)
+        self._hdr_sig = not os.path.exists(_sig_path)
+
+        self._pfl_file_ = open(_pfl_path, "+a", newline="")
+        self._execs_file_ = open(_exe_path, "+a", newline="")
+        self._sig_file_ = open(_sig_path, "+a", newline="")
+        self._pfl_writer = csv.writer(self._pfl_file_)
+        self._exe_writer = csv.writer(self._execs_file_)
+        self._sig_writer = csv.writer(self._sig_file_)
+        self.pool = ThreadPool(3)
+
+    @staticmethod
+    def _header(d: dict) -> List[str]:
+        return list(d.keys()) + ["run_id"]
+
+    def _values(self, data: List[Dict[str, Any]]) -> List[List[str]]:
+        # - attach run_id (last column)
+        return [list((d | {"run_id": self.run_id}).values()) for d in data]
+
+    def _do_write(self, log_type, data):
+        match log_type:
+            case "positions":
+                with open(self._pos_file_path, "w", newline="") as f:
+                    w = csv.writer(f)
+                    w.writerow(self._header(data[0]))
+                    w.writerows(self._values(data))
+
+            case "portfolio":
+                if self._hdr_pfl:
+                    self._pfl_writer.writerow(self._header(data[0]))
+                    self._hdr_pfl = False
+                self._pfl_writer.writerows(self._values(data))
+                self._pfl_file_.flush()
+
+            case "executions":
+                if self._hdr_exe:
+                    self._exe_writer.writerow(self._header(data[0]))
+                    self._hdr_exe = False
+                self._exe_writer.writerows(self._values(data))
+                self._execs_file_.flush()
+
+            case "signals":
+                if self._hdr_sig:
+                    self._sig_writer.writerow(self._header(data[0]))
+                    self._hdr_sig = False
+                self._sig_writer.writerows(self._values(data))
+                self._sig_file_.flush()
+
+            case "balance":
+                with open(self._balance_file_path, "w", newline="") as f:
+                    w = csv.writer(f)
+                    w.writerow(self._header(data[0]))
+                    w.writerows(self._values(data))
+
+    def write_data(self, log_type: str, data: List[Dict[str, Any]]):
+        if len(data) > 0:
+            self.pool.apply_async(self._do_write, (log_type, data))
+
+    def flush_data(self):
+        try:
+            self._pfl_file_.flush()
+            self._execs_file_.flush()
+            self._sig_file_.flush()
+        except Exception as e:
+            logger.warning(f"Error flushing log writer: {str(e)}")
+
+    def close(self):
+        self._pfl_file_.close()
+        self._execs_file_.close()
+        self._sig_file_.close()
+        self.pool.close()
+        self.pool.join()
+

qubx/loggers/factory.py

@@ -0,0 +1,55 @@
+import inspect
+
+from typing import Type
+
+from qubx.core.loggers import LogsWriter
+from qubx.loggers.csv import CsvFileLogsWriter
+from qubx.loggers.mongo import MongoDBLogsWriter
+from qubx.loggers.inmemory import InMemoryLogsWriter
+
+# Registry of logs writer types
+LOGS_WRITER_REGISTRY: dict[str, Type[LogsWriter]] = {
+    "CsvFileLogsWriter": CsvFileLogsWriter,
+    "MongoDBLogsWriter": MongoDBLogsWriter,
+    "InMemoryLogsWriter": InMemoryLogsWriter
+}
+
+def create_logs_writer(log_writer_type: str, parameters: dict | None = None) -> LogsWriter:
+    """
+    Create a logs writer based on configuration.
+
+    Args:
+        log_wirter_type: The type of logs writer to create.
+        parameters: Parameters to pass to the logs writer constructor.
+
+    Returns:
+        An instance of the specified logs writer.
+
+    Raises:
+        ValueError: If the specified logs writer type is not registered.
+    """
+    if log_writer_type not in LOGS_WRITER_REGISTRY:
+        raise ValueError(
+            f"Unknown logs writer type: {log_writer_type}. "
+            f"Available types: {', '.join(LOGS_WRITER_REGISTRY.keys())}"
+        )
+
+    logs_writer_class = LOGS_WRITER_REGISTRY[log_writer_type]
+    params = parameters.copy() if parameters else {}
+
+    sig = inspect.signature(logs_writer_class)
+    accepted_params = set(sig.parameters.keys())
+    filtered_params = {k: v for k, v in params.items() if k in accepted_params}
+
+    return logs_writer_class(**filtered_params)
+
+
+def register_logs_writer(log_writer_type: str, logs_witer_class: Type[LogsWriter]) -> None:
+    """
+    Register a new logs writer type.
+
+    Args:
+        log_writer_type: The name of the logs writer type.
+        logs_witer_class: The logs writer class to register.
+    """
+    LOGS_WRITER_REGISTRY[log_writer_type] = logs_witer_class

qubx/loggers/inmemory.py

@@ -0,0 +1,68 @@
+import pandas as pd
+
+from typing import Any, Dict, List
+
+from qubx.core.loggers import LogsWriter
+from qubx.core.metrics import split_cumulative_pnl
+from qubx.pandaz.utils import scols
+
+class InMemoryLogsWriter(LogsWriter):
+    _portfolio: List
+    _execs: List
+    _signals: List
+
+    def __init__(self, account_id: str, strategy_id: str, run_id: str) -> None:
+        super().__init__(account_id, strategy_id, run_id)
+        self._portfolio = []
+        self._execs = []
+        self._signals = []
+
+    def write_data(self, log_type: str, data: List[Dict[str, Any]]):
+        if len(data) > 0:
+            if log_type == "portfolio":
+                self._portfolio.extend(data)
+            elif log_type == "executions":
+                self._execs.extend(data)
+            elif log_type == "signals":
+                self._signals.extend(data)
+
+    def get_portfolio(self, as_plain_dataframe=True) -> pd.DataFrame:
+        pfl = pd.DataFrame.from_records(self._portfolio, index="timestamp")
+        pfl.index = pd.DatetimeIndex(pfl.index)
+        if as_plain_dataframe:
+            # - convert to Qube presentation (TODO: temporary)
+            pis = []
+            for s in set(pfl["symbol"]):
+                pi = pfl[pfl["symbol"] == s]
+                pi = pi.drop(columns=["symbol", "realized_pnl_quoted", "current_price", "exchange_time"])
+                pi = pi.rename(
+                    {
+                        "pnl_quoted": "PnL",
+                        "quantity": "Pos",
+                        "avg_position_price": "Price",
+                        "market_value_quoted": "Value",
+                        "commissions_quoted": "Commissions",
+                    },
+                    axis=1,
+                )
+                # We want to convert the value to just price * quantity
+                # in reality value of perps is just the unrealized pnl but
+                # it's not important after simulation for metric calculations
+                pi["Value"] = pi["Pos"] * pi["Price"] + pi["Value"]
+                pis.append(pi.rename(lambda x: s + "_" + x, axis=1))
+            return split_cumulative_pnl(scols(*pis))
+        return pfl
+
+    def get_executions(self) -> pd.DataFrame:
+        p = pd.DataFrame()
+        if self._execs:
+            p = pd.DataFrame.from_records(self._execs, index="timestamp")
+            p.index = pd.DatetimeIndex(p.index)
+        return p
+
+    def get_signals(self) -> pd.DataFrame:
+        p = pd.DataFrame()
+        if self._signals:
+            p = pd.DataFrame.from_records(self._signals, index="timestamp")
+            p.index = pd.DatetimeIndex(p.index)
+        return p

qubx/loggers/mongo.py

@@ -0,0 +1,80 @@
+from datetime import datetime
+from multiprocessing.pool import ThreadPool
+from pymongo import MongoClient
+from typing import Any, Dict, List
+
+from qubx.core.loggers import LogsWriter
+
+
+class MongoDBLogsWriter(LogsWriter):
+    """
+    MongoDB implementation of LogsWriter interface.
+    Writes log data to a single MongoDB collection asynchronously.
+    Supports TTL expiration via index on 'timestamp' field.
+    """
+
+    def __init__(
+        self,
+        account_id: str,
+        strategy_id: str,
+        run_id: str,
+        mongo_uri: str = "mongodb://localhost:27017/",
+        db_name: str = "default_logs_db",
+        collection_name_prefix: str = "qubx_logs",
+        pool_size: int = 3,
+        ttl_seconds: int = 86400,
+    ) -> None:
+        super().__init__(account_id, strategy_id, run_id)
+        self.client = MongoClient(mongo_uri)
+        self.db = self.client[db_name]
+        self.pool = ThreadPool(pool_size)
+        self.collection_name_prefix = collection_name_prefix
+
+        # Ensure TTL index exists on the 'timestamp' field
+        self.db[f"{collection_name_prefix}_positions"].create_index(
+            "timestamp", expireAfterSeconds=ttl_seconds
+        )
+        self.db[f"{collection_name_prefix}_portfolio"].create_index(
+            "timestamp", expireAfterSeconds=ttl_seconds
+        )
+        self.db[f"{collection_name_prefix}_executions"].create_index(
+            "timestamp", expireAfterSeconds=ttl_seconds
+        )
+        self.db[f"{collection_name_prefix}_signals"].create_index(
+            "timestamp", expireAfterSeconds=ttl_seconds
+        )
+        self.db[f"{collection_name_prefix}_balance"].create_index(
+            "timestamp", expireAfterSeconds=ttl_seconds
+        )
+    
+    def _attach_metadata(
+        self, data: List[Dict[str, Any]], log_type: str
+    ) -> List[Dict[str, Any]]:
+        now = datetime.utcnow()
+        return [
+            {
+                **d,
+                "run_id": self.run_id,
+                "account_id": self.account_id,
+                "strategy_name": self.strategy_id,
+                "log_type": log_type,
+                "timestamp": now,
+            }
+            for d in data
+        ]
+    
+    def _do_write(self, log_type: str, data: List[Dict[str, Any]]):
+        docs = self._attach_metadata(data, log_type)
+        self.db[f"{self.collection_name_prefix}_{log_type}"].insert_many(docs)
+
+    def write_data(self, log_type: str, data: List[Dict[str, Any]]):
+        if len(data) > 0:
+            self.pool.apply_async(self._do_write, (log_type, data,))
+    
+    def flush_data(self):
+        pass
+    
+    def close(self):
+        self.pool.close()
+        self.pool.join()
+        self.client.close()

qubx/restorers/balance.py

@@ -7,6 +7,7 @@ from various sources.
 
 import os
 from pathlib import Path
+from pymongo import MongoClient
 
 import pandas as pd
 
@@ -118,3 +119,78 @@ class CsvBalanceRestorer(IBalanceRestorer):
             balances[currency] = balance
 
         return balances
+
+
+class MongoDBBalanceRestorer(IBalanceRestorer):
+    """
+    Balance restorer that reads account balances from a MongoDB collection.
+
+    This restorer queries the most recent balance entries stored using MongoDBLogsWriter.
+    It restores data only from the most recent run_id for the given bot_id.
+    """
+
+    def __init__(
+        self,
+        strategy_name: str,
+        mongo_client: MongoClient,
+        db_name: str = "default_logs_db",
+        collection_name: str = "qubx_logs",
+    ):
+        self.mongo_client = mongo_client
+        self.db_name = db_name
+        self.collection_name = collection_name
+        self.strategy_name = strategy_name
+
+        self.collection = self.mongo_client[db_name][collection_name]
+
+    def restore_balances(self) -> dict[str, AssetBalance]:
+        """
+        Restore account balances from the most recent run.
+
+        Returns:
+            A dictionary mapping currency codes to AssetBalance objects.
+            Example: {'USDT': AssetBalance(total=100000.0, locked=0.0)}
+        """
+        try:
+            match_query = {
+                "log_type": "balance",
+                "strategy_name": self.strategy_name,
+            }
+
+            latest_run_doc = (
+                self.collection.find(match_query, {"run_id": 1, "timestamp": 1})
+                .sort("timestamp", -1)
+                .limit(1)
+            )
+
+            latest_run = next(latest_run_doc, None)
+            if not latest_run:
+                logger.warning("No balance logs found for given filters.")
+                return {}
+
+            latest_run_id = latest_run["run_id"]
+
+            logger.info(f"Restoring balances from MongoDB for run_id: {latest_run_id}")
+
+            query = {**match_query, "run_id": latest_run_id}
+            logs = self.collection.find(query).sort("timestamp", 1)
+
+            balances = {}
+
+            for log in logs:
+                currency = log.get("currency")
+                if currency:
+                    total = log.get("total", 0.0)
+                    locked = log.get("locked", 0.0)
+
+                    balance = AssetBalance(
+                        total=total,
+                        locked=locked,
+                    )
+                    balance.free = total - locked
+                    balances[currency] = balance
+
+            return balances
+        except Exception as e:
+            logger.error(f"Error restoring balances from MongoDB: {e}")
+            return {}