Qubx 0.6.36__cp312-cp312-manylinux_2_39_x86_64.whl → 0.6.38__cp312-cp312-manylinux_2_39_x86_64.whl

qubx/backtester/runner.py

@@ -20,8 +20,9 @@ from qubx.core.interfaces import (
     ITimeProvider,
     StrategyState,
 )
-from qubx.core.loggers import InMemoryLogsWriter, StrategyLogging
+from qubx.core.loggers import StrategyLogging
 from qubx.core.lookups import lookup
+from qubx.loggers.inmemory import InMemoryLogsWriter
 from qubx.pandaz.utils import _frame_to_str
 
 from .account import SimulatedAccountProcessor

qubx/connectors/ccxt/broker.py

@@ -14,7 +14,7 @@ from qubx.core.basics import (
     Order,
     OrderSide,
 )
-from qubx.core.errors import OrderCancellationError, OrderCreationError, create_error_event
+from qubx.core.errors import ErrorLevel, OrderCancellationError, OrderCreationError, create_error_event
 from qubx.core.exceptions import BadRequest, InvalidOrderParameters
 from qubx.core.interfaces import (
     IAccountProcessor,
@@ -61,6 +61,57 @@ class CcxtBroker(IBroker):
     def is_simulated_trading(self) -> bool:
         return False
 
+    def _post_order_error_to_databus(
+        self,
+        error: Exception,
+        instrument: Instrument,
+        order_side: OrderSide,
+        order_type: str,
+        amount: float,
+        price: float | None,
+        client_id: str | None,
+        time_in_force: str,
+        **options,
+    ):
+        level = ErrorLevel.LOW
+        match error:
+            case ccxt.InsufficientFunds():
+                level = ErrorLevel.HIGH
+                logger.error(
+                    f"(::create_order) INSUFFICIENT FUNDS for {order_side} {amount} {order_type} for {instrument.symbol} : {error}"
+                )
+            case ccxt.OrderNotFillable():
+                level = ErrorLevel.LOW
+                logger.error(
+                    f"(::create_order) ORDER NOT FILLEABLE for {order_side} {amount} {order_type} for [{instrument.symbol}] : {error}"
+                )
+            case ccxt.InvalidOrder():
+                level = ErrorLevel.LOW
+                logger.error(
+                    f"(::create_order) INVALID ORDER for {order_side} {amount} {order_type} for {instrument.symbol} : {error}"
+                )
+            case ccxt.BadRequest():
+                level = ErrorLevel.LOW
+                logger.error(
+                    f"(::create_order) BAD REQUEST for {order_side} {amount} {order_type} for {instrument.symbol} : {error}"
+                )
+            case _:
+                level = ErrorLevel.MEDIUM
+                logger.error(f"(::create_order) Unexpected error: {error}")
+
+        error_event = OrderCreationError(
+            timestamp=self.time_provider.time(),
+            message=f"Error message: {str(error)}",
+            level=level,
+            instrument=instrument,
+            amount=amount,
+            price=price,
+            order_type=order_type,
+            side=order_side,
+            error=error,
+        )
+        self.channel.send(create_error_event(error_event))
+
     def send_order_async(
         self,
         instrument: Instrument,
@@ -93,33 +144,20 @@ class CcxtBroker(IBroker):
                 )
 
                 if error:
-                    # Create and send an error event through the channel
-                    error_event = OrderCreationError(
-                        timestamp=self.time_provider.time(),
-                        message=str(error),
-                        instrument=instrument,
-                        amount=amount,
-                        price=price,
-                        order_type=order_type,
-                        side=order_side,
+                    self._post_order_error_to_databus(
+                        error, instrument, order_side, order_type, amount, price, client_id, time_in_force, **options
                     )
-                    self.channel.send(create_error_event(error_event))
-                    return None
+                    order = None
+
                 return order
+
             except Exception as err:
                 # Catch any unexpected errors and send them through the channel as well
-                logger.error(f"Unexpected error in async order creation: {err}")
+                logger.error(f"{self.__class__.__name__} :: Unexpected error in async order creation: {err}")
                 logger.error(traceback.format_exc())
-                error_event = OrderCreationError(
-                    timestamp=self.time_provider.time(),
-                    message=f"Unexpected error: {str(err)}",
-                    instrument=instrument,
-                    amount=amount,
-                    price=price,
-                    order_type=order_type,
-                    side=order_side,
+                self._post_order_error_to_databus(
+                    err, instrument, order_side, order_type, amount, price, client_id, time_in_force, **options
                 )
-                self.channel.send(create_error_event(error_event))
                 return None
 
         # Submit the task to the async loop
@@ -135,7 +173,7 @@ class CcxtBroker(IBroker):
         client_id: str | None = None,
         time_in_force: str = "gtc",
         **options,
-    ) -> Order:
+    ) -> Order | None:
         """
         Submit an order and wait for the result. Exceptions will be raised on errors.
 
@@ -169,14 +207,16 @@ class CcxtBroker(IBroker):
 
             # If there was no error but also no order, something went wrong
             if not order and not self.enable_create_order_ws:
-                raise ExchangeError("Order creation failed with no specific error")
+                raise ExchangeError(f"{self.__class__.__name__} :: Order creation failed with no specific error")
 
             return order
 
         except Exception as err:
             # This will catch any errors from future.result() or if we explicitly raise an error
-            logger.error(f"Error in send_order: {err}")
-            raise
+            self._post_order_error_to_databus(
+                err, instrument, order_side, order_type, amount, price, client_id, time_in_force, **options
+            )
+            return None
 
     def cancel_order(self, order_id: str) -> Order | None:
         orders = self.account.get_orders()
@@ -231,25 +271,7 @@ class CcxtBroker(IBroker):
             logger.info(f"New order {order}")
             return order, None
 
-        except ccxt.OrderNotFillable as exc:
-            logger.error(
-                f"(::_create_order) [{instrument.symbol}] ORDER NOT FILLEABLE for {order_side} {amount} {order_type} : {exc}"
-            )
-            return None, exc
-        except ccxt.InvalidOrder as exc:
-            logger.error(
-                f"(::_create_order) INVALID ORDER for {order_side} {amount} {order_type} for {instrument.symbol} : {exc}"
-            )
-            return None, exc
-        except ccxt.BadRequest as exc:
-            logger.error(
-                f"(::_create_order) BAD REQUEST for {order_side} {amount} {order_type} for {instrument.symbol} : {exc}"
-            )
-            return None, exc
         except Exception as err:
-            logger.error(
-                f"(::_create_order) {order_side} {amount} {order_type} for {instrument.symbol} exception : {err}"
-            )
             return None, err
 
     def _prepare_order_payload(
@@ -371,6 +393,8 @@ class CcxtBroker(IBroker):
                             order_id=order_id,
                             message=f"Timeout reached for canceling order {order_id}",
                             instrument=instrument,
+                            level=ErrorLevel.LOW,
+                            error=None,
                         )
                     )
                 )

qubx/connectors/ccxt/exchanges/__init__.py

@@ -25,7 +25,7 @@ EXCHANGE_ALIASES = {
 
 CUSTOM_BROKERS = {
     "binance": partial(BinanceCcxtBroker, enable_create_order_ws=True, enable_cancel_order_ws=False),
-    "binance.um": partial(BinanceCcxtBroker, enable_create_order_ws=True, enable_cancel_order_ws=False),
+    "binance.um": partial(BinanceCcxtBroker, enable_create_order_ws=True, enable_cancel_order_ws=True),
     "binance.cm": partial(BinanceCcxtBroker, enable_create_order_ws=True, enable_cancel_order_ws=False),
     "binance.pm": partial(BinanceCcxtBroker, enable_create_order_ws=False, enable_cancel_order_ws=False),
     "bitfinex.f": partial(CcxtBroker, enable_create_order_ws=True, enable_cancel_order_ws=True),

qubx/connectors/ccxt/exchanges/binance/exchange.py

@@ -3,7 +3,7 @@ from typing import Dict, List
 import ccxt.pro as cxp
 from ccxt.async_support.base.ws.cache import ArrayCache, ArrayCacheByTimestamp
 from ccxt.async_support.base.ws.client import Client
-from ccxt.base.errors import ArgumentsRequired, BadRequest, NotSupported
+from ccxt.base.errors import ArgumentsRequired, BadRequest, InsufficientFunds, NotSupported
 from ccxt.base.precise import Precise
 from ccxt.base.types import (
     Any,
@@ -34,7 +34,12 @@ class BinanceQV(cxp.binance):
                         "name": "aggTrade",
                     },
                     "localOrderBookLimit": 10_000,  # set a large limit to avoid cutting off the orderbook
-                }
+                },
+                "exceptions": {
+                    "exact": {
+                        "-2019": InsufficientFunds,  # ccxt doesn't have this code for some weird reason !!
+                    },
+                },
             },
         )
 

qubx/core/context.py

@@ -16,6 +16,7 @@ from qubx.core.basics import (
     Timestamped,
     dt_64,
 )
+from qubx.core.errors import BaseErrorEvent, ErrorLevel
 from qubx.core.exceptions import StrategyExceededMaxNumberOfRuntimeFailuresError
 from qubx.core.helpers import (
     BasicScheduler,
@@ -286,7 +287,7 @@ class StrategyContext(IStrategyContext):
 
         # - invoke strategy's stop code
         try:
-            if not self._strategy_state.is_warmup_in_progress:
+            if not self.is_warmup_in_progress:
                 self.strategy.on_stop(self)
         except Exception as strat_error:
             logger.error(
@@ -327,7 +328,7 @@ class StrategyContext(IStrategyContext):
         return self._data_providers[0].is_simulation
 
     @property
-    def is_simulated_trading(self) -> bool:
+    def is_paper_trading(self) -> bool:
         return self._brokers[0].is_simulated_trading
 
     # IAccountViewer delegation
@@ -536,6 +537,16 @@ class StrategyContext(IStrategyContext):
                 if _should_record:
                     self._health_monitor.record_start_processing(d_type, dt_64(data.time, "ns"))
 
+                # - notify error if error level is medium or higher
+                if (
+                    self._lifecycle_notifier
+                    and isinstance(data, BaseErrorEvent)
+                    and data.level.value >= ErrorLevel.MEDIUM.value
+                ):
+                    self._lifecycle_notifier.notify_error(
+                        self._strategy_name, data.error or Exception("Unknown error"), {"message": str(data)}
+                    )
+
                 if self.process_data(instrument, d_type, data, hist):
                     channel.stop()
                     break

qubx/core/errors.py

@@ -3,14 +3,27 @@ Error types that are sent through the event channel.
 """
 
 from dataclasses import dataclass
+from enum import Enum
 
 from qubx.core.basics import Instrument, dt_64
 
 
+class ErrorLevel(Enum):
+    LOW = 1  # continue trading
+    MEDIUM = 2  # send notifications and continue trading
+    HIGH = 3  # send notification and cancel orders and close positions
+    CRITICAL = 4  # send notification and shutdown strategy
+
+
 @dataclass
 class BaseErrorEvent:
     timestamp: dt_64
     message: str
+    level: ErrorLevel
+    error: Exception | None
+
+    def __str__(self):
+        return f"[{self.level}] : {self.timestamp} : {self.message} / {self.error}"
 
 
 def create_error_event(error: BaseErrorEvent) -> tuple[None, str, BaseErrorEvent, bool]:
@@ -25,8 +38,14 @@ class OrderCreationError(BaseErrorEvent):
     order_type: str
     side: str
 
+    def __str__(self):
+        return f"[{self.level}] : {self.timestamp} : {self.message} / {self.error} ||| Order creation error for {self.order_type} {self.side} {self.instrument} {self.amount}"
+
 
 @dataclass
 class OrderCancellationError(BaseErrorEvent):
     instrument: Instrument
     order_id: str
+
+    def __str__(self):
+        return f"[{self.level}] : {self.timestamp} : {self.message} / {self.error} ||| Order cancellation error for {self.order_id} {self.instrument}"

qubx/core/interfaces.py

@@ -1068,7 +1068,6 @@ class IStrategyContext(
     IProcessingManager,
     IAccountViewer,
     IWarmupStateSaver,
-    StrategyState,
 ):
     strategy: "IStrategy"
     initializer: "IStrategyInitializer"
@@ -1086,17 +1085,27 @@ class IStrategyContext(
         """Stop the strategy context."""
         pass
 
+    @property
+    def state(self) -> StrategyState:
+        """Get the strategy state."""
+        return StrategyState(**self._strategy_state.__dict__)
+
     def is_running(self) -> bool:
         """Check if the strategy context is running."""
         return False
 
+    @property
+    def is_warmup_in_progress(self) -> bool:
+        """Check if the warmup is in progress."""
+        return self._strategy_state.is_warmup_in_progress
+
     @property
     def is_simulation(self) -> bool:
         """Check if the strategy context is running in simulation mode."""
         return False
 
     @property
-    def is_simulated_trading(self) -> bool:
+    def is_paper_trading(self) -> bool:
         """Check if the strategy context is running in simulated trading mode."""
         return False
 

qubx/core/loggers.py

@@ -1,10 +1,6 @@
-import csv
-import os
-from multiprocessing.pool import ThreadPool
 from typing import Any, Dict, List, Tuple
 
 import numpy as np
-import pandas as pd
 
 from qubx import logger
 from qubx.core.basics import (
@@ -14,11 +10,11 @@ from qubx.core.basics import (
     Position,
     TargetPosition,
 )
-from qubx.core.metrics import split_cumulative_pnl
+
 from qubx.core.series import time_as_nsec
 from qubx.core.utils import recognize_timeframe
-from qubx.pandaz.utils import scols
-from qubx.utils.misc import Stopwatch, makedirs
+
+from qubx.utils.misc import Stopwatch
 from qubx.utils.time import convert_tf_str_td64, floor_t64
 
 _SW = Stopwatch()
@@ -48,159 +44,6 @@ class LogsWriter:
         pass
 
 
-class InMemoryLogsWriter(LogsWriter):
-    _portfolio: List
-    _execs: List
-    _signals: List
-
-    def __init__(self, account_id: str, strategy_id: str, run_id: str) -> None:
-        super().__init__(account_id, strategy_id, run_id)
-        self._portfolio = []
-        self._execs = []
-        self._signals = []
-
-    def write_data(self, log_type: str, data: List[Dict[str, Any]]):
-        if len(data) > 0:
-            if log_type == "portfolio":
-                self._portfolio.extend(data)
-            elif log_type == "executions":
-                self._execs.extend(data)
-            elif log_type == "signals":
-                self._signals.extend(data)
-
-    def get_portfolio(self, as_plain_dataframe=True) -> pd.DataFrame:
-        pfl = pd.DataFrame.from_records(self._portfolio, index="timestamp")
-        pfl.index = pd.DatetimeIndex(pfl.index)
-        if as_plain_dataframe:
-            # - convert to Qube presentation (TODO: temporary)
-            pis = []
-            for s in set(pfl["symbol"]):
-                pi = pfl[pfl["symbol"] == s]
-                pi = pi.drop(columns=["symbol", "realized_pnl_quoted", "current_price", "exchange_time"])
-                pi = pi.rename(
-                    {
-                        "pnl_quoted": "PnL",
-                        "quantity": "Pos",
-                        "avg_position_price": "Price",
-                        "market_value_quoted": "Value",
-                        "commissions_quoted": "Commissions",
-                    },
-                    axis=1,
-                )
-                # We want to convert the value to just price * quantity
-                # in reality value of perps is just the unrealized pnl but
-                # it's not important after simulation for metric calculations
-                pi["Value"] = pi["Pos"] * pi["Price"] + pi["Value"]
-                pis.append(pi.rename(lambda x: s + "_" + x, axis=1))
-            return split_cumulative_pnl(scols(*pis))
-        return pfl
-
-    def get_executions(self) -> pd.DataFrame:
-        p = pd.DataFrame()
-        if self._execs:
-            p = pd.DataFrame.from_records(self._execs, index="timestamp")
-            p.index = pd.DatetimeIndex(p.index)
-        return p
-
-    def get_signals(self) -> pd.DataFrame:
-        p = pd.DataFrame()
-        if self._signals:
-            p = pd.DataFrame.from_records(self._signals, index="timestamp")
-            p.index = pd.DatetimeIndex(p.index)
-        return p
-
-
-class CsvFileLogsWriter(LogsWriter):
-    """
-    Simple CSV strategy log data writer. It does data writing in separate thread.
-    """
-
-    def __init__(self, account_id: str, strategy_id: str, run_id: str, log_folder="logs") -> None:
-        super().__init__(account_id, strategy_id, run_id)
-
-        path = makedirs(log_folder)
-        # - it rewrites positions every time
-        self._pos_file_path = f"{path}/{self.strategy_id}_{self.account_id}_positions.csv"
-        self._balance_file_path = f"{path}/{self.strategy_id}_{self.account_id}_balance.csv"
-
-        _pfl_path = f"{path}/{strategy_id}_{account_id}_portfolio.csv"
-        _exe_path = f"{path}/{strategy_id}_{account_id}_executions.csv"
-        _sig_path = f"{path}/{strategy_id}_{account_id}_signals.csv"
-        self._hdr_pfl = not os.path.exists(_pfl_path)
-        self._hdr_exe = not os.path.exists(_exe_path)
-        self._hdr_sig = not os.path.exists(_sig_path)
-
-        self._pfl_file_ = open(_pfl_path, "+a", newline="")
-        self._execs_file_ = open(_exe_path, "+a", newline="")
-        self._sig_file_ = open(_sig_path, "+a", newline="")
-        self._pfl_writer = csv.writer(self._pfl_file_)
-        self._exe_writer = csv.writer(self._execs_file_)
-        self._sig_writer = csv.writer(self._sig_file_)
-        self.pool = ThreadPool(3)
-
-    @staticmethod
-    def _header(d: dict) -> List[str]:
-        return list(d.keys()) + ["run_id"]
-
-    def _values(self, data: List[Dict[str, Any]]) -> List[List[str]]:
-        # - attach run_id (last column)
-        return [list((d | {"run_id": self.run_id}).values()) for d in data]
-
-    def _do_write(self, log_type, data):
-        match log_type:
-            case "positions":
-                with open(self._pos_file_path, "w", newline="") as f:
-                    w = csv.writer(f)
-                    w.writerow(self._header(data[0]))
-                    w.writerows(self._values(data))
-
-            case "portfolio":
-                if self._hdr_pfl:
-                    self._pfl_writer.writerow(self._header(data[0]))
-                    self._hdr_pfl = False
-                self._pfl_writer.writerows(self._values(data))
-                self._pfl_file_.flush()
-
-            case "executions":
-                if self._hdr_exe:
-                    self._exe_writer.writerow(self._header(data[0]))
-                    self._hdr_exe = False
-                self._exe_writer.writerows(self._values(data))
-                self._execs_file_.flush()
-
-            case "signals":
-                if self._hdr_sig:
-                    self._sig_writer.writerow(self._header(data[0]))
-                    self._hdr_sig = False
-                self._sig_writer.writerows(self._values(data))
-                self._sig_file_.flush()
-
-            case "balance":
-                with open(self._balance_file_path, "w", newline="") as f:
-                    w = csv.writer(f)
-                    w.writerow(self._header(data[0]))
-                    w.writerows(self._values(data))
-
-    def write_data(self, log_type: str, data: List[Dict[str, Any]]):
-        if len(data) > 0:
-            self.pool.apply_async(self._do_write, (log_type, data))
-
-    def flush_data(self):
-        try:
-            self._pfl_file_.flush()
-            self._execs_file_.flush()
-            self._sig_file_.flush()
-        except Exception as e:
-            logger.warning(f"Error flushing log writer: {str(e)}")
-
-    def close(self):
-        self._pfl_file_.close()
-        self._execs_file_.close()
-        self._sig_file_.close()
-        self.pool.close()
-        self.pool.join()
-
-
 class _BaseIntervalDumper:
     """
     Basic functionality for all interval based dumpers

qubx/core/metrics.py

@@ -717,7 +717,7 @@ class TradingSessionResult:
             "name": self.name,
             "start": pd.Timestamp(self.start).isoformat(),
             "stop": pd.Timestamp(self.stop).isoformat(),
-            "exchange": self.exchanges,
+            "exchanges": self.exchanges,
             "capital": self.capital,
             "base_currency": self.base_currency,
             "commissions": self.commissions,
@@ -824,6 +824,12 @@ class TradingSessionResult:
             info = self.info()
             if description:
                 info["description"] = description
+            # - set name if not specified
+            if info.get("name") is None:
+                info["name"] = name
+
+            # - add numpy array representer
+            yaml.SafeDumper.add_representer(np.ndarray, lambda dumper, data: dumper.represent_list(data.tolist()))
             yaml.safe_dump(info, f, sort_keys=False, indent=4)
 
         # - save logs
@@ -855,15 +861,31 @@ class TradingSessionResult:
 
         with zipfile.ZipFile(path, "r") as zip_ref:
             info = yaml.safe_load(zip_ref.read("info.yml"))
-            portfolio = pd.read_csv(zip_ref.open("portfolio.csv"), index_col=["timestamp"], parse_dates=["timestamp"])
-            executions = pd.read_csv(zip_ref.open("executions.csv"), index_col=["timestamp"], parse_dates=["timestamp"])
-            signals = pd.read_csv(zip_ref.open("signals.csv"), index_col=["timestamp"], parse_dates=["timestamp"])
+            try:
+                portfolio = pd.read_csv(
+                    zip_ref.open("portfolio.csv"), index_col=["timestamp"], parse_dates=["timestamp"]
+                )
+            except:
+                portfolio = pd.DataFrame()
+            try:
+                executions = pd.read_csv(
+                    zip_ref.open("executions.csv"), index_col=["timestamp"], parse_dates=["timestamp"]
+                )
+            except:
+                executions = pd.DataFrame()
+            try:
+                signals = pd.read_csv(zip_ref.open("signals.csv"), index_col=["timestamp"], parse_dates=["timestamp"])
+            except:
+                signals = pd.DataFrame()
 
         # load result
         _qbx_version = info.pop("qubx_version")
         _decr = info.pop("description", None)
         _perf = info.pop("performance", None)
         info["instruments"] = info.pop("symbols")
+        # - fix for old versions
+        _exch = info.pop("exchange")
+        info["exchanges"] = _exch if isinstance(_exch, list) else [_exch]
         tsr = TradingSessionResult(**info, portfolio_log=portfolio, executions_log=executions, signals_log=signals)
         tsr.qubx_version = _qbx_version
         tsr._metrics = _perf