Qubx 0.6.37__cp312-cp312-manylinux_2_39_x86_64.whl → 0.6.38__cp312-cp312-manylinux_2_39_x86_64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of Qubx might be problematic. Click here for more details.
- qubx/backtester/runner.py +2 -1
- qubx/core/loggers.py +3 -160
- qubx/core/series.cpython-312-x86_64-linux-gnu.so +0 -0
- qubx/core/utils.cpython-312-x86_64-linux-gnu.so +0 -0
- qubx/loggers/__init__.py +17 -0
- qubx/loggers/csv.py +100 -0
- qubx/loggers/factory.py +55 -0
- qubx/loggers/inmemory.py +68 -0
- qubx/loggers/mongo.py +80 -0
- qubx/restorers/balance.py +76 -0
- qubx/restorers/factory.py +8 -4
- qubx/restorers/position.py +95 -0
- qubx/restorers/signal.py +115 -0
- qubx/restorers/state.py +89 -3
- qubx/ta/indicators.cpython-312-x86_64-linux-gnu.so +0 -0
- qubx/utils/runner/runner.py +6 -8
- {qubx-0.6.37.dist-info → qubx-0.6.38.dist-info}/METADATA +1 -1
- {qubx-0.6.37.dist-info → qubx-0.6.38.dist-info}/RECORD +21 -16
- {qubx-0.6.37.dist-info → qubx-0.6.38.dist-info}/LICENSE +0 -0
- {qubx-0.6.37.dist-info → qubx-0.6.38.dist-info}/WHEEL +0 -0
- {qubx-0.6.37.dist-info → qubx-0.6.38.dist-info}/entry_points.txt +0 -0
qubx/backtester/runner.py
CHANGED
|
@@ -20,8 +20,9 @@ from qubx.core.interfaces import (
|
|
|
20
20
|
ITimeProvider,
|
|
21
21
|
StrategyState,
|
|
22
22
|
)
|
|
23
|
-
from qubx.core.loggers import
|
|
23
|
+
from qubx.core.loggers import StrategyLogging
|
|
24
24
|
from qubx.core.lookups import lookup
|
|
25
|
+
from qubx.loggers.inmemory import InMemoryLogsWriter
|
|
25
26
|
from qubx.pandaz.utils import _frame_to_str
|
|
26
27
|
|
|
27
28
|
from .account import SimulatedAccountProcessor
|
qubx/core/loggers.py
CHANGED
|
@@ -1,10 +1,6 @@
|
|
|
1
|
-
import csv
|
|
2
|
-
import os
|
|
3
|
-
from multiprocessing.pool import ThreadPool
|
|
4
1
|
from typing import Any, Dict, List, Tuple
|
|
5
2
|
|
|
6
3
|
import numpy as np
|
|
7
|
-
import pandas as pd
|
|
8
4
|
|
|
9
5
|
from qubx import logger
|
|
10
6
|
from qubx.core.basics import (
|
|
@@ -14,11 +10,11 @@ from qubx.core.basics import (
|
|
|
14
10
|
Position,
|
|
15
11
|
TargetPosition,
|
|
16
12
|
)
|
|
17
|
-
|
|
13
|
+
|
|
18
14
|
from qubx.core.series import time_as_nsec
|
|
19
15
|
from qubx.core.utils import recognize_timeframe
|
|
20
|
-
|
|
21
|
-
from qubx.utils.misc import Stopwatch
|
|
16
|
+
|
|
17
|
+
from qubx.utils.misc import Stopwatch
|
|
22
18
|
from qubx.utils.time import convert_tf_str_td64, floor_t64
|
|
23
19
|
|
|
24
20
|
_SW = Stopwatch()
|
|
@@ -48,159 +44,6 @@ class LogsWriter:
|
|
|
48
44
|
pass
|
|
49
45
|
|
|
50
46
|
|
|
51
|
-
class InMemoryLogsWriter(LogsWriter):
|
|
52
|
-
_portfolio: List
|
|
53
|
-
_execs: List
|
|
54
|
-
_signals: List
|
|
55
|
-
|
|
56
|
-
def __init__(self, account_id: str, strategy_id: str, run_id: str) -> None:
|
|
57
|
-
super().__init__(account_id, strategy_id, run_id)
|
|
58
|
-
self._portfolio = []
|
|
59
|
-
self._execs = []
|
|
60
|
-
self._signals = []
|
|
61
|
-
|
|
62
|
-
def write_data(self, log_type: str, data: List[Dict[str, Any]]):
|
|
63
|
-
if len(data) > 0:
|
|
64
|
-
if log_type == "portfolio":
|
|
65
|
-
self._portfolio.extend(data)
|
|
66
|
-
elif log_type == "executions":
|
|
67
|
-
self._execs.extend(data)
|
|
68
|
-
elif log_type == "signals":
|
|
69
|
-
self._signals.extend(data)
|
|
70
|
-
|
|
71
|
-
def get_portfolio(self, as_plain_dataframe=True) -> pd.DataFrame:
|
|
72
|
-
pfl = pd.DataFrame.from_records(self._portfolio, index="timestamp")
|
|
73
|
-
pfl.index = pd.DatetimeIndex(pfl.index)
|
|
74
|
-
if as_plain_dataframe:
|
|
75
|
-
# - convert to Qube presentation (TODO: temporary)
|
|
76
|
-
pis = []
|
|
77
|
-
for s in set(pfl["symbol"]):
|
|
78
|
-
pi = pfl[pfl["symbol"] == s]
|
|
79
|
-
pi = pi.drop(columns=["symbol", "realized_pnl_quoted", "current_price", "exchange_time"])
|
|
80
|
-
pi = pi.rename(
|
|
81
|
-
{
|
|
82
|
-
"pnl_quoted": "PnL",
|
|
83
|
-
"quantity": "Pos",
|
|
84
|
-
"avg_position_price": "Price",
|
|
85
|
-
"market_value_quoted": "Value",
|
|
86
|
-
"commissions_quoted": "Commissions",
|
|
87
|
-
},
|
|
88
|
-
axis=1,
|
|
89
|
-
)
|
|
90
|
-
# We want to convert the value to just price * quantity
|
|
91
|
-
# in reality value of perps is just the unrealized pnl but
|
|
92
|
-
# it's not important after simulation for metric calculations
|
|
93
|
-
pi["Value"] = pi["Pos"] * pi["Price"] + pi["Value"]
|
|
94
|
-
pis.append(pi.rename(lambda x: s + "_" + x, axis=1))
|
|
95
|
-
return split_cumulative_pnl(scols(*pis))
|
|
96
|
-
return pfl
|
|
97
|
-
|
|
98
|
-
def get_executions(self) -> pd.DataFrame:
|
|
99
|
-
p = pd.DataFrame()
|
|
100
|
-
if self._execs:
|
|
101
|
-
p = pd.DataFrame.from_records(self._execs, index="timestamp")
|
|
102
|
-
p.index = pd.DatetimeIndex(p.index)
|
|
103
|
-
return p
|
|
104
|
-
|
|
105
|
-
def get_signals(self) -> pd.DataFrame:
|
|
106
|
-
p = pd.DataFrame()
|
|
107
|
-
if self._signals:
|
|
108
|
-
p = pd.DataFrame.from_records(self._signals, index="timestamp")
|
|
109
|
-
p.index = pd.DatetimeIndex(p.index)
|
|
110
|
-
return p
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
class CsvFileLogsWriter(LogsWriter):
|
|
114
|
-
"""
|
|
115
|
-
Simple CSV strategy log data writer. It does data writing in separate thread.
|
|
116
|
-
"""
|
|
117
|
-
|
|
118
|
-
def __init__(self, account_id: str, strategy_id: str, run_id: str, log_folder="logs") -> None:
|
|
119
|
-
super().__init__(account_id, strategy_id, run_id)
|
|
120
|
-
|
|
121
|
-
path = makedirs(log_folder)
|
|
122
|
-
# - it rewrites positions every time
|
|
123
|
-
self._pos_file_path = f"{path}/{self.strategy_id}_{self.account_id}_positions.csv"
|
|
124
|
-
self._balance_file_path = f"{path}/{self.strategy_id}_{self.account_id}_balance.csv"
|
|
125
|
-
|
|
126
|
-
_pfl_path = f"{path}/{strategy_id}_{account_id}_portfolio.csv"
|
|
127
|
-
_exe_path = f"{path}/{strategy_id}_{account_id}_executions.csv"
|
|
128
|
-
_sig_path = f"{path}/{strategy_id}_{account_id}_signals.csv"
|
|
129
|
-
self._hdr_pfl = not os.path.exists(_pfl_path)
|
|
130
|
-
self._hdr_exe = not os.path.exists(_exe_path)
|
|
131
|
-
self._hdr_sig = not os.path.exists(_sig_path)
|
|
132
|
-
|
|
133
|
-
self._pfl_file_ = open(_pfl_path, "+a", newline="")
|
|
134
|
-
self._execs_file_ = open(_exe_path, "+a", newline="")
|
|
135
|
-
self._sig_file_ = open(_sig_path, "+a", newline="")
|
|
136
|
-
self._pfl_writer = csv.writer(self._pfl_file_)
|
|
137
|
-
self._exe_writer = csv.writer(self._execs_file_)
|
|
138
|
-
self._sig_writer = csv.writer(self._sig_file_)
|
|
139
|
-
self.pool = ThreadPool(3)
|
|
140
|
-
|
|
141
|
-
@staticmethod
|
|
142
|
-
def _header(d: dict) -> List[str]:
|
|
143
|
-
return list(d.keys()) + ["run_id"]
|
|
144
|
-
|
|
145
|
-
def _values(self, data: List[Dict[str, Any]]) -> List[List[str]]:
|
|
146
|
-
# - attach run_id (last column)
|
|
147
|
-
return [list((d | {"run_id": self.run_id}).values()) for d in data]
|
|
148
|
-
|
|
149
|
-
def _do_write(self, log_type, data):
|
|
150
|
-
match log_type:
|
|
151
|
-
case "positions":
|
|
152
|
-
with open(self._pos_file_path, "w", newline="") as f:
|
|
153
|
-
w = csv.writer(f)
|
|
154
|
-
w.writerow(self._header(data[0]))
|
|
155
|
-
w.writerows(self._values(data))
|
|
156
|
-
|
|
157
|
-
case "portfolio":
|
|
158
|
-
if self._hdr_pfl:
|
|
159
|
-
self._pfl_writer.writerow(self._header(data[0]))
|
|
160
|
-
self._hdr_pfl = False
|
|
161
|
-
self._pfl_writer.writerows(self._values(data))
|
|
162
|
-
self._pfl_file_.flush()
|
|
163
|
-
|
|
164
|
-
case "executions":
|
|
165
|
-
if self._hdr_exe:
|
|
166
|
-
self._exe_writer.writerow(self._header(data[0]))
|
|
167
|
-
self._hdr_exe = False
|
|
168
|
-
self._exe_writer.writerows(self._values(data))
|
|
169
|
-
self._execs_file_.flush()
|
|
170
|
-
|
|
171
|
-
case "signals":
|
|
172
|
-
if self._hdr_sig:
|
|
173
|
-
self._sig_writer.writerow(self._header(data[0]))
|
|
174
|
-
self._hdr_sig = False
|
|
175
|
-
self._sig_writer.writerows(self._values(data))
|
|
176
|
-
self._sig_file_.flush()
|
|
177
|
-
|
|
178
|
-
case "balance":
|
|
179
|
-
with open(self._balance_file_path, "w", newline="") as f:
|
|
180
|
-
w = csv.writer(f)
|
|
181
|
-
w.writerow(self._header(data[0]))
|
|
182
|
-
w.writerows(self._values(data))
|
|
183
|
-
|
|
184
|
-
def write_data(self, log_type: str, data: List[Dict[str, Any]]):
|
|
185
|
-
if len(data) > 0:
|
|
186
|
-
self.pool.apply_async(self._do_write, (log_type, data))
|
|
187
|
-
|
|
188
|
-
def flush_data(self):
|
|
189
|
-
try:
|
|
190
|
-
self._pfl_file_.flush()
|
|
191
|
-
self._execs_file_.flush()
|
|
192
|
-
self._sig_file_.flush()
|
|
193
|
-
except Exception as e:
|
|
194
|
-
logger.warning(f"Error flushing log writer: {str(e)}")
|
|
195
|
-
|
|
196
|
-
def close(self):
|
|
197
|
-
self._pfl_file_.close()
|
|
198
|
-
self._execs_file_.close()
|
|
199
|
-
self._sig_file_.close()
|
|
200
|
-
self.pool.close()
|
|
201
|
-
self.pool.join()
|
|
202
|
-
|
|
203
|
-
|
|
204
47
|
class _BaseIntervalDumper:
|
|
205
48
|
"""
|
|
206
49
|
Basic functionality for all interval based dumpers
|
|
Binary file
|
|
Binary file
|
qubx/loggers/__init__.py
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Loggers module for qubx.
|
|
3
|
+
|
|
4
|
+
This module provides implementations for logs writing, like csv writer or mongodb writer.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from qubx.loggers.csv import CsvFileLogsWriter
|
|
8
|
+
from qubx.loggers.inmemory import InMemoryLogsWriter
|
|
9
|
+
from qubx.loggers.mongo import MongoDBLogsWriter
|
|
10
|
+
from qubx.loggers.factory import create_logs_writer
|
|
11
|
+
|
|
12
|
+
__all__ = [
|
|
13
|
+
"CsvFileLogsWriter",
|
|
14
|
+
"InMemoryLogsWriter",
|
|
15
|
+
"MongoDBLogsWriter",
|
|
16
|
+
"create_logs_writer",
|
|
17
|
+
]
|
qubx/loggers/csv.py
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
import csv
|
|
2
|
+
import os
|
|
3
|
+
|
|
4
|
+
from typing import Any, Dict, List
|
|
5
|
+
from multiprocessing.pool import ThreadPool
|
|
6
|
+
|
|
7
|
+
from qubx import logger
|
|
8
|
+
from qubx.core.loggers import LogsWriter
|
|
9
|
+
from qubx.utils.misc import makedirs
|
|
10
|
+
|
|
11
|
+
class CsvFileLogsWriter(LogsWriter):
|
|
12
|
+
"""
|
|
13
|
+
Simple CSV strategy log data writer. It does data writing in separate thread.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
def __init__(self, account_id: str, strategy_id: str, run_id: str, log_folder="logs") -> None:
|
|
17
|
+
super().__init__(account_id, strategy_id, run_id)
|
|
18
|
+
|
|
19
|
+
path = makedirs(log_folder)
|
|
20
|
+
# - it rewrites positions every time
|
|
21
|
+
self._pos_file_path = f"{path}/{self.strategy_id}_{self.account_id}_positions.csv"
|
|
22
|
+
self._balance_file_path = f"{path}/{self.strategy_id}_{self.account_id}_balance.csv"
|
|
23
|
+
|
|
24
|
+
_pfl_path = f"{path}/{strategy_id}_{account_id}_portfolio.csv"
|
|
25
|
+
_exe_path = f"{path}/{strategy_id}_{account_id}_executions.csv"
|
|
26
|
+
_sig_path = f"{path}/{strategy_id}_{account_id}_signals.csv"
|
|
27
|
+
self._hdr_pfl = not os.path.exists(_pfl_path)
|
|
28
|
+
self._hdr_exe = not os.path.exists(_exe_path)
|
|
29
|
+
self._hdr_sig = not os.path.exists(_sig_path)
|
|
30
|
+
|
|
31
|
+
self._pfl_file_ = open(_pfl_path, "+a", newline="")
|
|
32
|
+
self._execs_file_ = open(_exe_path, "+a", newline="")
|
|
33
|
+
self._sig_file_ = open(_sig_path, "+a", newline="")
|
|
34
|
+
self._pfl_writer = csv.writer(self._pfl_file_)
|
|
35
|
+
self._exe_writer = csv.writer(self._execs_file_)
|
|
36
|
+
self._sig_writer = csv.writer(self._sig_file_)
|
|
37
|
+
self.pool = ThreadPool(3)
|
|
38
|
+
|
|
39
|
+
@staticmethod
|
|
40
|
+
def _header(d: dict) -> List[str]:
|
|
41
|
+
return list(d.keys()) + ["run_id"]
|
|
42
|
+
|
|
43
|
+
def _values(self, data: List[Dict[str, Any]]) -> List[List[str]]:
|
|
44
|
+
# - attach run_id (last column)
|
|
45
|
+
return [list((d | {"run_id": self.run_id}).values()) for d in data]
|
|
46
|
+
|
|
47
|
+
def _do_write(self, log_type, data):
|
|
48
|
+
match log_type:
|
|
49
|
+
case "positions":
|
|
50
|
+
with open(self._pos_file_path, "w", newline="") as f:
|
|
51
|
+
w = csv.writer(f)
|
|
52
|
+
w.writerow(self._header(data[0]))
|
|
53
|
+
w.writerows(self._values(data))
|
|
54
|
+
|
|
55
|
+
case "portfolio":
|
|
56
|
+
if self._hdr_pfl:
|
|
57
|
+
self._pfl_writer.writerow(self._header(data[0]))
|
|
58
|
+
self._hdr_pfl = False
|
|
59
|
+
self._pfl_writer.writerows(self._values(data))
|
|
60
|
+
self._pfl_file_.flush()
|
|
61
|
+
|
|
62
|
+
case "executions":
|
|
63
|
+
if self._hdr_exe:
|
|
64
|
+
self._exe_writer.writerow(self._header(data[0]))
|
|
65
|
+
self._hdr_exe = False
|
|
66
|
+
self._exe_writer.writerows(self._values(data))
|
|
67
|
+
self._execs_file_.flush()
|
|
68
|
+
|
|
69
|
+
case "signals":
|
|
70
|
+
if self._hdr_sig:
|
|
71
|
+
self._sig_writer.writerow(self._header(data[0]))
|
|
72
|
+
self._hdr_sig = False
|
|
73
|
+
self._sig_writer.writerows(self._values(data))
|
|
74
|
+
self._sig_file_.flush()
|
|
75
|
+
|
|
76
|
+
case "balance":
|
|
77
|
+
with open(self._balance_file_path, "w", newline="") as f:
|
|
78
|
+
w = csv.writer(f)
|
|
79
|
+
w.writerow(self._header(data[0]))
|
|
80
|
+
w.writerows(self._values(data))
|
|
81
|
+
|
|
82
|
+
def write_data(self, log_type: str, data: List[Dict[str, Any]]):
|
|
83
|
+
if len(data) > 0:
|
|
84
|
+
self.pool.apply_async(self._do_write, (log_type, data))
|
|
85
|
+
|
|
86
|
+
def flush_data(self):
|
|
87
|
+
try:
|
|
88
|
+
self._pfl_file_.flush()
|
|
89
|
+
self._execs_file_.flush()
|
|
90
|
+
self._sig_file_.flush()
|
|
91
|
+
except Exception as e:
|
|
92
|
+
logger.warning(f"Error flushing log writer: {str(e)}")
|
|
93
|
+
|
|
94
|
+
def close(self):
|
|
95
|
+
self._pfl_file_.close()
|
|
96
|
+
self._execs_file_.close()
|
|
97
|
+
self._sig_file_.close()
|
|
98
|
+
self.pool.close()
|
|
99
|
+
self.pool.join()
|
|
100
|
+
|
qubx/loggers/factory.py
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import inspect
|
|
2
|
+
|
|
3
|
+
from typing import Type
|
|
4
|
+
|
|
5
|
+
from qubx.core.loggers import LogsWriter
|
|
6
|
+
from qubx.loggers.csv import CsvFileLogsWriter
|
|
7
|
+
from qubx.loggers.mongo import MongoDBLogsWriter
|
|
8
|
+
from qubx.loggers.inmemory import InMemoryLogsWriter
|
|
9
|
+
|
|
10
|
+
# Registry of logs writer types
|
|
11
|
+
LOGS_WRITER_REGISTRY: dict[str, Type[LogsWriter]] = {
|
|
12
|
+
"CsvFileLogsWriter": CsvFileLogsWriter,
|
|
13
|
+
"MongoDBLogsWriter": MongoDBLogsWriter,
|
|
14
|
+
"InMemoryLogsWriter": InMemoryLogsWriter
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
def create_logs_writer(log_writer_type: str, parameters: dict | None = None) -> LogsWriter:
|
|
18
|
+
"""
|
|
19
|
+
Create a logs writer based on configuration.
|
|
20
|
+
|
|
21
|
+
Args:
|
|
22
|
+
log_wirter_type: The type of logs writer to create.
|
|
23
|
+
parameters: Parameters to pass to the logs writer constructor.
|
|
24
|
+
|
|
25
|
+
Returns:
|
|
26
|
+
An instance of the specified logs writer.
|
|
27
|
+
|
|
28
|
+
Raises:
|
|
29
|
+
ValueError: If the specified logs writer type is not registered.
|
|
30
|
+
"""
|
|
31
|
+
if log_writer_type not in LOGS_WRITER_REGISTRY:
|
|
32
|
+
raise ValueError(
|
|
33
|
+
f"Unknown logs writer type: {log_writer_type}. "
|
|
34
|
+
f"Available types: {', '.join(LOGS_WRITER_REGISTRY.keys())}"
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
logs_writer_class = LOGS_WRITER_REGISTRY[log_writer_type]
|
|
38
|
+
params = parameters.copy() if parameters else {}
|
|
39
|
+
|
|
40
|
+
sig = inspect.signature(logs_writer_class)
|
|
41
|
+
accepted_params = set(sig.parameters.keys())
|
|
42
|
+
filtered_params = {k: v for k, v in params.items() if k in accepted_params}
|
|
43
|
+
|
|
44
|
+
return logs_writer_class(**filtered_params)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def register_logs_writer(log_writer_type: str, logs_witer_class: Type[LogsWriter]) -> None:
|
|
48
|
+
"""
|
|
49
|
+
Register a new logs writer type.
|
|
50
|
+
|
|
51
|
+
Args:
|
|
52
|
+
log_writer_type: The name of the logs writer type.
|
|
53
|
+
logs_witer_class: The logs writer class to register.
|
|
54
|
+
"""
|
|
55
|
+
LOGS_WRITER_REGISTRY[log_writer_type] = logs_witer_class
|
qubx/loggers/inmemory.py
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import pandas as pd
|
|
2
|
+
|
|
3
|
+
from typing import Any, Dict, List
|
|
4
|
+
|
|
5
|
+
from qubx.core.loggers import LogsWriter
|
|
6
|
+
from qubx.core.metrics import split_cumulative_pnl
|
|
7
|
+
from qubx.pandaz.utils import scols
|
|
8
|
+
|
|
9
|
+
class InMemoryLogsWriter(LogsWriter):
|
|
10
|
+
_portfolio: List
|
|
11
|
+
_execs: List
|
|
12
|
+
_signals: List
|
|
13
|
+
|
|
14
|
+
def __init__(self, account_id: str, strategy_id: str, run_id: str) -> None:
|
|
15
|
+
super().__init__(account_id, strategy_id, run_id)
|
|
16
|
+
self._portfolio = []
|
|
17
|
+
self._execs = []
|
|
18
|
+
self._signals = []
|
|
19
|
+
|
|
20
|
+
def write_data(self, log_type: str, data: List[Dict[str, Any]]):
|
|
21
|
+
if len(data) > 0:
|
|
22
|
+
if log_type == "portfolio":
|
|
23
|
+
self._portfolio.extend(data)
|
|
24
|
+
elif log_type == "executions":
|
|
25
|
+
self._execs.extend(data)
|
|
26
|
+
elif log_type == "signals":
|
|
27
|
+
self._signals.extend(data)
|
|
28
|
+
|
|
29
|
+
def get_portfolio(self, as_plain_dataframe=True) -> pd.DataFrame:
|
|
30
|
+
pfl = pd.DataFrame.from_records(self._portfolio, index="timestamp")
|
|
31
|
+
pfl.index = pd.DatetimeIndex(pfl.index)
|
|
32
|
+
if as_plain_dataframe:
|
|
33
|
+
# - convert to Qube presentation (TODO: temporary)
|
|
34
|
+
pis = []
|
|
35
|
+
for s in set(pfl["symbol"]):
|
|
36
|
+
pi = pfl[pfl["symbol"] == s]
|
|
37
|
+
pi = pi.drop(columns=["symbol", "realized_pnl_quoted", "current_price", "exchange_time"])
|
|
38
|
+
pi = pi.rename(
|
|
39
|
+
{
|
|
40
|
+
"pnl_quoted": "PnL",
|
|
41
|
+
"quantity": "Pos",
|
|
42
|
+
"avg_position_price": "Price",
|
|
43
|
+
"market_value_quoted": "Value",
|
|
44
|
+
"commissions_quoted": "Commissions",
|
|
45
|
+
},
|
|
46
|
+
axis=1,
|
|
47
|
+
)
|
|
48
|
+
# We want to convert the value to just price * quantity
|
|
49
|
+
# in reality value of perps is just the unrealized pnl but
|
|
50
|
+
# it's not important after simulation for metric calculations
|
|
51
|
+
pi["Value"] = pi["Pos"] * pi["Price"] + pi["Value"]
|
|
52
|
+
pis.append(pi.rename(lambda x: s + "_" + x, axis=1))
|
|
53
|
+
return split_cumulative_pnl(scols(*pis))
|
|
54
|
+
return pfl
|
|
55
|
+
|
|
56
|
+
def get_executions(self) -> pd.DataFrame:
|
|
57
|
+
p = pd.DataFrame()
|
|
58
|
+
if self._execs:
|
|
59
|
+
p = pd.DataFrame.from_records(self._execs, index="timestamp")
|
|
60
|
+
p.index = pd.DatetimeIndex(p.index)
|
|
61
|
+
return p
|
|
62
|
+
|
|
63
|
+
def get_signals(self) -> pd.DataFrame:
|
|
64
|
+
p = pd.DataFrame()
|
|
65
|
+
if self._signals:
|
|
66
|
+
p = pd.DataFrame.from_records(self._signals, index="timestamp")
|
|
67
|
+
p.index = pd.DatetimeIndex(p.index)
|
|
68
|
+
return p
|
qubx/loggers/mongo.py
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
from datetime import datetime
|
|
2
|
+
from multiprocessing.pool import ThreadPool
|
|
3
|
+
from pymongo import MongoClient
|
|
4
|
+
from typing import Any, Dict, List
|
|
5
|
+
|
|
6
|
+
from qubx.core.loggers import LogsWriter
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class MongoDBLogsWriter(LogsWriter):
|
|
10
|
+
"""
|
|
11
|
+
MongoDB implementation of LogsWriter interface.
|
|
12
|
+
Writes log data to a single MongoDB collection asynchronously.
|
|
13
|
+
Supports TTL expiration via index on 'timestamp' field.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
def __init__(
|
|
17
|
+
self,
|
|
18
|
+
account_id: str,
|
|
19
|
+
strategy_id: str,
|
|
20
|
+
run_id: str,
|
|
21
|
+
mongo_uri: str = "mongodb://localhost:27017/",
|
|
22
|
+
db_name: str = "default_logs_db",
|
|
23
|
+
collection_name_prefix: str = "qubx_logs",
|
|
24
|
+
pool_size: int = 3,
|
|
25
|
+
ttl_seconds: int = 86400,
|
|
26
|
+
) -> None:
|
|
27
|
+
super().__init__(account_id, strategy_id, run_id)
|
|
28
|
+
self.client = MongoClient(mongo_uri)
|
|
29
|
+
self.db = self.client[db_name]
|
|
30
|
+
self.pool = ThreadPool(pool_size)
|
|
31
|
+
self.collection_name_prefix = collection_name_prefix
|
|
32
|
+
|
|
33
|
+
# Ensure TTL index exists on the 'timestamp' field
|
|
34
|
+
self.db[f"{collection_name_prefix}_positions"].create_index(
|
|
35
|
+
"timestamp", expireAfterSeconds=ttl_seconds
|
|
36
|
+
)
|
|
37
|
+
self.db[f"{collection_name_prefix}_portfolio"].create_index(
|
|
38
|
+
"timestamp", expireAfterSeconds=ttl_seconds
|
|
39
|
+
)
|
|
40
|
+
self.db[f"{collection_name_prefix}_executions"].create_index(
|
|
41
|
+
"timestamp", expireAfterSeconds=ttl_seconds
|
|
42
|
+
)
|
|
43
|
+
self.db[f"{collection_name_prefix}_signals"].create_index(
|
|
44
|
+
"timestamp", expireAfterSeconds=ttl_seconds
|
|
45
|
+
)
|
|
46
|
+
self.db[f"{collection_name_prefix}_balance"].create_index(
|
|
47
|
+
"timestamp", expireAfterSeconds=ttl_seconds
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
def _attach_metadata(
|
|
51
|
+
self, data: List[Dict[str, Any]], log_type: str
|
|
52
|
+
) -> List[Dict[str, Any]]:
|
|
53
|
+
now = datetime.utcnow()
|
|
54
|
+
return [
|
|
55
|
+
{
|
|
56
|
+
**d,
|
|
57
|
+
"run_id": self.run_id,
|
|
58
|
+
"account_id": self.account_id,
|
|
59
|
+
"strategy_name": self.strategy_id,
|
|
60
|
+
"log_type": log_type,
|
|
61
|
+
"timestamp": now,
|
|
62
|
+
}
|
|
63
|
+
for d in data
|
|
64
|
+
]
|
|
65
|
+
|
|
66
|
+
def _do_write(self, log_type: str, data: List[Dict[str, Any]]):
|
|
67
|
+
docs = self._attach_metadata(data, log_type)
|
|
68
|
+
self.db[f"{self.collection_name_prefix}_{log_type}"].insert_many(docs)
|
|
69
|
+
|
|
70
|
+
def write_data(self, log_type: str, data: List[Dict[str, Any]]):
|
|
71
|
+
if len(data) > 0:
|
|
72
|
+
self.pool.apply_async(self._do_write, (log_type, data,))
|
|
73
|
+
|
|
74
|
+
def flush_data(self):
|
|
75
|
+
pass
|
|
76
|
+
|
|
77
|
+
def close(self):
|
|
78
|
+
self.pool.close()
|
|
79
|
+
self.pool.join()
|
|
80
|
+
self.client.close()
|
qubx/restorers/balance.py
CHANGED
|
@@ -7,6 +7,7 @@ from various sources.
|
|
|
7
7
|
|
|
8
8
|
import os
|
|
9
9
|
from pathlib import Path
|
|
10
|
+
from pymongo import MongoClient
|
|
10
11
|
|
|
11
12
|
import pandas as pd
|
|
12
13
|
|
|
@@ -118,3 +119,78 @@ class CsvBalanceRestorer(IBalanceRestorer):
|
|
|
118
119
|
balances[currency] = balance
|
|
119
120
|
|
|
120
121
|
return balances
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
class MongoDBBalanceRestorer(IBalanceRestorer):
|
|
125
|
+
"""
|
|
126
|
+
Balance restorer that reads account balances from a MongoDB collection.
|
|
127
|
+
|
|
128
|
+
This restorer queries the most recent balance entries stored using MongoDBLogsWriter.
|
|
129
|
+
It restores data only from the most recent run_id for the given bot_id.
|
|
130
|
+
"""
|
|
131
|
+
|
|
132
|
+
def __init__(
|
|
133
|
+
self,
|
|
134
|
+
strategy_name: str,
|
|
135
|
+
mongo_client: MongoClient,
|
|
136
|
+
db_name: str = "default_logs_db",
|
|
137
|
+
collection_name: str = "qubx_logs",
|
|
138
|
+
):
|
|
139
|
+
self.mongo_client = mongo_client
|
|
140
|
+
self.db_name = db_name
|
|
141
|
+
self.collection_name = collection_name
|
|
142
|
+
self.strategy_name = strategy_name
|
|
143
|
+
|
|
144
|
+
self.collection = self.mongo_client[db_name][collection_name]
|
|
145
|
+
|
|
146
|
+
def restore_balances(self) -> dict[str, AssetBalance]:
|
|
147
|
+
"""
|
|
148
|
+
Restore account balances from the most recent run.
|
|
149
|
+
|
|
150
|
+
Returns:
|
|
151
|
+
A dictionary mapping currency codes to AssetBalance objects.
|
|
152
|
+
Example: {'USDT': AssetBalance(total=100000.0, locked=0.0)}
|
|
153
|
+
"""
|
|
154
|
+
try:
|
|
155
|
+
match_query = {
|
|
156
|
+
"log_type": "balance",
|
|
157
|
+
"strategy_name": self.strategy_name,
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
latest_run_doc = (
|
|
161
|
+
self.collection.find(match_query, {"run_id": 1, "timestamp": 1})
|
|
162
|
+
.sort("timestamp", -1)
|
|
163
|
+
.limit(1)
|
|
164
|
+
)
|
|
165
|
+
|
|
166
|
+
latest_run = next(latest_run_doc, None)
|
|
167
|
+
if not latest_run:
|
|
168
|
+
logger.warning("No balance logs found for given filters.")
|
|
169
|
+
return {}
|
|
170
|
+
|
|
171
|
+
latest_run_id = latest_run["run_id"]
|
|
172
|
+
|
|
173
|
+
logger.info(f"Restoring balances from MongoDB for run_id: {latest_run_id}")
|
|
174
|
+
|
|
175
|
+
query = {**match_query, "run_id": latest_run_id}
|
|
176
|
+
logs = self.collection.find(query).sort("timestamp", 1)
|
|
177
|
+
|
|
178
|
+
balances = {}
|
|
179
|
+
|
|
180
|
+
for log in logs:
|
|
181
|
+
currency = log.get("currency")
|
|
182
|
+
if currency:
|
|
183
|
+
total = log.get("total", 0.0)
|
|
184
|
+
locked = log.get("locked", 0.0)
|
|
185
|
+
|
|
186
|
+
balance = AssetBalance(
|
|
187
|
+
total=total,
|
|
188
|
+
locked=locked,
|
|
189
|
+
)
|
|
190
|
+
balance.free = total - locked
|
|
191
|
+
balances[currency] = balance
|
|
192
|
+
|
|
193
|
+
return balances
|
|
194
|
+
except Exception as e:
|
|
195
|
+
logger.error(f"Error restoring balances from MongoDB: {e}")
|
|
196
|
+
return {}
|
qubx/restorers/factory.py
CHANGED
|
@@ -8,33 +8,37 @@ based on configuration.
|
|
|
8
8
|
from typing import Type
|
|
9
9
|
|
|
10
10
|
from qubx.core.lookups import GlobalLookup
|
|
11
|
-
from qubx.restorers.balance import CsvBalanceRestorer
|
|
11
|
+
from qubx.restorers.balance import CsvBalanceRestorer, MongoDBBalanceRestorer
|
|
12
12
|
from qubx.restorers.interfaces import IBalanceRestorer, IPositionRestorer, ISignalRestorer, IStateRestorer
|
|
13
|
-
from qubx.restorers.position import CsvPositionRestorer
|
|
14
|
-
from qubx.restorers.signal import CsvSignalRestorer
|
|
15
|
-
from qubx.restorers.state import CsvStateRestorer
|
|
13
|
+
from qubx.restorers.position import CsvPositionRestorer, MongoDBPositionRestorer
|
|
14
|
+
from qubx.restorers.signal import CsvSignalRestorer, MongoDBSignalRestorer
|
|
15
|
+
from qubx.restorers.state import CsvStateRestorer, MongoDBStateRestorer
|
|
16
16
|
|
|
17
17
|
# Registry of position restorer types
|
|
18
18
|
POSITION_RESTORER_REGISTRY: dict[str, Type[IPositionRestorer]] = {
|
|
19
19
|
"CsvPositionRestorer": CsvPositionRestorer,
|
|
20
|
+
"MongoDBPositionRestorer": MongoDBPositionRestorer,
|
|
20
21
|
}
|
|
21
22
|
|
|
22
23
|
|
|
23
24
|
# Registry of signal restorer types
|
|
24
25
|
SIGNAL_RESTORER_REGISTRY: dict[str, Type[ISignalRestorer]] = {
|
|
25
26
|
"CsvSignalRestorer": CsvSignalRestorer,
|
|
27
|
+
"MongoDBSignalRestorer": MongoDBSignalRestorer,
|
|
26
28
|
}
|
|
27
29
|
|
|
28
30
|
|
|
29
31
|
# Registry of balance restorer types
|
|
30
32
|
BALANCE_RESTORER_REGISTRY: dict[str, Type[IBalanceRestorer]] = {
|
|
31
33
|
"CsvBalanceRestorer": CsvBalanceRestorer,
|
|
34
|
+
"MongoDBBalanceRestorer": MongoDBBalanceRestorer,
|
|
32
35
|
}
|
|
33
36
|
|
|
34
37
|
|
|
35
38
|
# Registry of state restorer types
|
|
36
39
|
STATE_RESTORER_REGISTRY: dict[str, Type[IStateRestorer]] = {
|
|
37
40
|
"CsvStateRestorer": CsvStateRestorer,
|
|
41
|
+
"MongoDBStateRestorer": MongoDBStateRestorer,
|
|
38
42
|
}
|
|
39
43
|
|
|
40
44
|
|
qubx/restorers/position.py
CHANGED
|
@@ -7,6 +7,7 @@ for restoring positions from various sources.
|
|
|
7
7
|
|
|
8
8
|
import os
|
|
9
9
|
from pathlib import Path
|
|
10
|
+
from pymongo import MongoClient
|
|
10
11
|
|
|
11
12
|
import pandas as pd
|
|
12
13
|
|
|
@@ -135,3 +136,97 @@ class CsvPositionRestorer(IPositionRestorer):
|
|
|
135
136
|
positions[instrument] = position
|
|
136
137
|
|
|
137
138
|
return positions
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
class MongoDBPositionRestorer(IPositionRestorer):
|
|
142
|
+
"""
|
|
143
|
+
Position restorer that reads positions from a MongoDB collection.
|
|
144
|
+
|
|
145
|
+
This restorer queries the most recent position entries stored using MongoDBLogsWriter,
|
|
146
|
+
and restores only from the latest run_id for the provided identifiers.
|
|
147
|
+
"""
|
|
148
|
+
|
|
149
|
+
def __init__(
|
|
150
|
+
self,
|
|
151
|
+
strategy_name: str,
|
|
152
|
+
mongo_client: MongoClient,
|
|
153
|
+
db_name: str = "default_logs_db",
|
|
154
|
+
collection_name: str = "qubx_logs",
|
|
155
|
+
):
|
|
156
|
+
self.mongo_client = mongo_client
|
|
157
|
+
self.db_name = db_name
|
|
158
|
+
self.collection_name = collection_name
|
|
159
|
+
self.strategy_name = strategy_name
|
|
160
|
+
|
|
161
|
+
self.collection = self.mongo_client[db_name][collection_name]
|
|
162
|
+
|
|
163
|
+
def restore_positions(self) -> dict[Instrument, Position]:
|
|
164
|
+
"""
|
|
165
|
+
Restore the latest positions grouped by instrument from the most recent run.
|
|
166
|
+
|
|
167
|
+
Returns:
|
|
168
|
+
A dictionary mapping instruments to positions.
|
|
169
|
+
"""
|
|
170
|
+
try:
|
|
171
|
+
match_query = {
|
|
172
|
+
"log_type": "positions",
|
|
173
|
+
"strategy_name": self.strategy_name,
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
latest_run_doc = (
|
|
177
|
+
self.collection.find(match_query, {"run_id": 1, "timestamp": 1})
|
|
178
|
+
.sort("timestamp", -1)
|
|
179
|
+
.limit(1)
|
|
180
|
+
)
|
|
181
|
+
|
|
182
|
+
latest_run = next(latest_run_doc, None)
|
|
183
|
+
if not latest_run:
|
|
184
|
+
logger.warning("No position logs found for given filters.")
|
|
185
|
+
return {}
|
|
186
|
+
|
|
187
|
+
latest_run_id = latest_run["run_id"]
|
|
188
|
+
|
|
189
|
+
logger.info(f"Restoring positions from MongoDB for run_id: {latest_run_id}")
|
|
190
|
+
|
|
191
|
+
query = {**match_query, "run_id": latest_run_id}
|
|
192
|
+
logs = self.collection.find(query).sort("timestamp", 1)
|
|
193
|
+
|
|
194
|
+
positions = {}
|
|
195
|
+
seen_keys = set()
|
|
196
|
+
|
|
197
|
+
for log in logs:
|
|
198
|
+
key = (log.get("symbol"), log.get("exchange"), log.get("market_type"))
|
|
199
|
+
if None in key or key in seen_keys:
|
|
200
|
+
continue
|
|
201
|
+
seen_keys.add(key)
|
|
202
|
+
|
|
203
|
+
symbol = log["symbol"]
|
|
204
|
+
exchange = log["exchange"]
|
|
205
|
+
|
|
206
|
+
instrument = lookup.find_symbol(exchange, symbol)
|
|
207
|
+
if instrument is None:
|
|
208
|
+
logger.warning(f"Instrument not found for {symbol} on {exchange}")
|
|
209
|
+
continue
|
|
210
|
+
|
|
211
|
+
quantity = log.get("quantity") or log.get("size", 0.0)
|
|
212
|
+
avg_price = log.get("avg_position_price") or log.get("avg_price", 0.0)
|
|
213
|
+
r_pnl = log.get("realized_pnl_quoted") or log.get("realized_pnl", 0.0)
|
|
214
|
+
current_price = log.get("current_price")
|
|
215
|
+
|
|
216
|
+
position = Position(
|
|
217
|
+
instrument=instrument,
|
|
218
|
+
quantity=quantity,
|
|
219
|
+
pos_average_price=avg_price,
|
|
220
|
+
r_pnl=r_pnl,
|
|
221
|
+
)
|
|
222
|
+
|
|
223
|
+
if current_price is not None:
|
|
224
|
+
timestamp = recognize_time(log.get("timestamp"))
|
|
225
|
+
position.update_market_price(timestamp, current_price, 1.0)
|
|
226
|
+
|
|
227
|
+
positions[instrument] = position
|
|
228
|
+
|
|
229
|
+
return positions
|
|
230
|
+
except Exception as e:
|
|
231
|
+
logger.error(f"Error restoring positions from MongoDB: {e}")
|
|
232
|
+
return {}
|
qubx/restorers/signal.py
CHANGED
|
@@ -8,6 +8,7 @@ for restoring signals from various sources.
|
|
|
8
8
|
import os
|
|
9
9
|
from datetime import datetime, timedelta
|
|
10
10
|
from pathlib import Path
|
|
11
|
+
from pymongo import MongoClient
|
|
11
12
|
|
|
12
13
|
import pandas as pd
|
|
13
14
|
|
|
@@ -174,3 +175,117 @@ class CsvSignalRestorer(ISignalRestorer):
|
|
|
174
175
|
targets_by_instrument[instrument] = target_positions
|
|
175
176
|
|
|
176
177
|
return targets_by_instrument
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
class MongoDBSignalRestorer(ISignalRestorer):
|
|
182
|
+
"""
|
|
183
|
+
Signal restorer that reads historical signals from MongoDB.
|
|
184
|
+
|
|
185
|
+
This restorer reads signals written by the MongoDBLogsWriter
|
|
186
|
+
for the most recent run_id associated with a given bot.
|
|
187
|
+
"""
|
|
188
|
+
|
|
189
|
+
def __init__(
|
|
190
|
+
self,
|
|
191
|
+
strategy_name: str,
|
|
192
|
+
mongo_client: MongoClient,
|
|
193
|
+
db_name: str = "default_logs_db",
|
|
194
|
+
collection_name: str = "qubx_logs",
|
|
195
|
+
):
|
|
196
|
+
self.mongo_client = mongo_client
|
|
197
|
+
self.db_name = db_name
|
|
198
|
+
self.collection_name = collection_name
|
|
199
|
+
self.strategy_name = strategy_name
|
|
200
|
+
|
|
201
|
+
self.collection = self.mongo_client[db_name][collection_name]
|
|
202
|
+
|
|
203
|
+
def restore_signals(self) -> dict[Instrument, list[TargetPosition]]:
|
|
204
|
+
"""
|
|
205
|
+
Restore signals from MongoDB for the latest run_id.
|
|
206
|
+
|
|
207
|
+
Returns:
|
|
208
|
+
A dictionary mapping instruments to lists of signals.
|
|
209
|
+
"""
|
|
210
|
+
try:
|
|
211
|
+
match_query = {
|
|
212
|
+
"log_type": "signals",
|
|
213
|
+
"strategy_name": self.strategy_name,
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
latest_run_doc = (
|
|
217
|
+
self.collection.find(match_query, {"run_id": 1, "timestamp": 1})
|
|
218
|
+
.sort("timestamp", -1)
|
|
219
|
+
.limit(1)
|
|
220
|
+
)
|
|
221
|
+
|
|
222
|
+
latest_run = next(latest_run_doc, None)
|
|
223
|
+
if not latest_run:
|
|
224
|
+
logger.warning("No signal logs found for given filters.")
|
|
225
|
+
return {}
|
|
226
|
+
|
|
227
|
+
latest_run_id = latest_run["run_id"]
|
|
228
|
+
|
|
229
|
+
logger.info(f"Restoring signals from MongoDB for run_id: {latest_run_id}")
|
|
230
|
+
|
|
231
|
+
query = {**match_query, "run_id": latest_run_id}
|
|
232
|
+
logs = self.collection.find(query).sort("timestamp", 1)
|
|
233
|
+
|
|
234
|
+
result: dict[Instrument, list[TargetPosition]] = {}
|
|
235
|
+
|
|
236
|
+
for log in logs:
|
|
237
|
+
try:
|
|
238
|
+
instrument = lookup.find_symbol(log["exchange"], log["symbol"])
|
|
239
|
+
if instrument is None:
|
|
240
|
+
logger.warning(f"Instrument not found for {log['symbol']} on {log['exchange']}")
|
|
241
|
+
continue
|
|
242
|
+
|
|
243
|
+
timestamp = recognize_time(log["timestamp"])
|
|
244
|
+
target_position_size = log.get("target_position")
|
|
245
|
+
|
|
246
|
+
if target_position_size is None:
|
|
247
|
+
logger.warning(f"No target_position in signal log: {log}")
|
|
248
|
+
continue
|
|
249
|
+
|
|
250
|
+
signal_value = log.get("signal")
|
|
251
|
+
if signal_value is None and "side" in log:
|
|
252
|
+
signal_value = 1.0 if str(log["side"]).lower() == "buy" else -1.0
|
|
253
|
+
|
|
254
|
+
if signal_value is None:
|
|
255
|
+
logger.warning(f"Missing signal or side for log: {log}")
|
|
256
|
+
continue
|
|
257
|
+
|
|
258
|
+
price = log.get("price") or log.get("reference_price")
|
|
259
|
+
|
|
260
|
+
options = {}
|
|
261
|
+
for key in ["target_position", "comment", "size", "meta"]:
|
|
262
|
+
if key in log:
|
|
263
|
+
options[key] = log[key]
|
|
264
|
+
|
|
265
|
+
signal = Signal(
|
|
266
|
+
instrument=instrument,
|
|
267
|
+
signal=signal_value,
|
|
268
|
+
price=price,
|
|
269
|
+
stop=None,
|
|
270
|
+
take=None,
|
|
271
|
+
reference_price=log.get("reference_price"),
|
|
272
|
+
group=log.get("group", ""),
|
|
273
|
+
comment=log.get("comment", ""),
|
|
274
|
+
options=options,
|
|
275
|
+
)
|
|
276
|
+
|
|
277
|
+
target_position = TargetPosition(
|
|
278
|
+
time=timestamp,
|
|
279
|
+
target_position_size=target_position_size,
|
|
280
|
+
signal=signal,
|
|
281
|
+
)
|
|
282
|
+
|
|
283
|
+
result.setdefault(instrument, []).append(target_position)
|
|
284
|
+
|
|
285
|
+
except Exception as e:
|
|
286
|
+
logger.exception(f"Failed to process signal document: {e}")
|
|
287
|
+
|
|
288
|
+
return result
|
|
289
|
+
except Exception as e:
|
|
290
|
+
logger.error(f"Error restoring signals from MongoDB: {e}")
|
|
291
|
+
return {}
|
qubx/restorers/state.py
CHANGED
|
@@ -7,16 +7,17 @@ from various sources.
|
|
|
7
7
|
|
|
8
8
|
import os
|
|
9
9
|
from pathlib import Path
|
|
10
|
+
from pymongo import MongoClient
|
|
10
11
|
|
|
11
12
|
import numpy as np
|
|
12
13
|
|
|
13
14
|
from qubx import logger
|
|
14
15
|
from qubx.core.basics import RestoredState
|
|
15
16
|
from qubx.core.utils import recognize_time
|
|
16
|
-
from qubx.restorers.balance import CsvBalanceRestorer
|
|
17
|
+
from qubx.restorers.balance import CsvBalanceRestorer, MongoDBBalanceRestorer
|
|
17
18
|
from qubx.restorers.interfaces import IStateRestorer
|
|
18
|
-
from qubx.restorers.position import CsvPositionRestorer
|
|
19
|
-
from qubx.restorers.signal import CsvSignalRestorer
|
|
19
|
+
from qubx.restorers.position import CsvPositionRestorer, MongoDBPositionRestorer
|
|
20
|
+
from qubx.restorers.signal import CsvSignalRestorer, MongoDBSignalRestorer
|
|
20
21
|
from qubx.restorers.utils import find_latest_run_folder
|
|
21
22
|
|
|
22
23
|
|
|
@@ -112,3 +113,88 @@ class CsvStateRestorer(IStateRestorer):
|
|
|
112
113
|
instrument_to_target_positions=target_positions,
|
|
113
114
|
balances=balances,
|
|
114
115
|
)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
class MongoDBStateRestorer(IStateRestorer):
|
|
119
|
+
"""
|
|
120
|
+
State restorer that reads strategy state from MongoDB.
|
|
121
|
+
|
|
122
|
+
This restorer combines the functionality of MongoDBPositionRestorer,
|
|
123
|
+
MongoDBSignalRestorer, and MongoDBBalanceRestorer to create a complete RestartState.
|
|
124
|
+
"""
|
|
125
|
+
|
|
126
|
+
def __init__(
|
|
127
|
+
self,
|
|
128
|
+
strategy_name: str,
|
|
129
|
+
mongo_uri: str = "mongodb://localhost:27017/",
|
|
130
|
+
db_name: str = "default_logs_db",
|
|
131
|
+
collection_name_prefix: str = "qubx_logs",
|
|
132
|
+
):
|
|
133
|
+
self.mongo_uri = mongo_uri
|
|
134
|
+
self.db_name = db_name
|
|
135
|
+
self.collection_name_prefix = collection_name_prefix
|
|
136
|
+
self.strategy_name = strategy_name
|
|
137
|
+
|
|
138
|
+
self.client = MongoClient(mongo_uri)
|
|
139
|
+
|
|
140
|
+
# Create individual restorers
|
|
141
|
+
self.position_restorer = MongoDBPositionRestorer(
|
|
142
|
+
strategy_name=strategy_name,
|
|
143
|
+
mongo_client=self.client,
|
|
144
|
+
db_name=db_name,
|
|
145
|
+
collection_name=f"{collection_name_prefix}_positions",
|
|
146
|
+
)
|
|
147
|
+
|
|
148
|
+
self.signal_restorer = MongoDBSignalRestorer(
|
|
149
|
+
strategy_name=strategy_name,
|
|
150
|
+
mongo_client=self.client,
|
|
151
|
+
db_name=db_name,
|
|
152
|
+
collection_name=f"{collection_name_prefix}_signals",
|
|
153
|
+
)
|
|
154
|
+
|
|
155
|
+
self.balance_restorer = MongoDBBalanceRestorer(
|
|
156
|
+
strategy_name=strategy_name,
|
|
157
|
+
mongo_client=self.client,
|
|
158
|
+
db_name=db_name,
|
|
159
|
+
collection_name=f"{collection_name_prefix}_balance",
|
|
160
|
+
)
|
|
161
|
+
|
|
162
|
+
def restore_state(self) -> RestoredState:
|
|
163
|
+
"""
|
|
164
|
+
Restore the complete strategy state from MongoDB.
|
|
165
|
+
|
|
166
|
+
Returns:
|
|
167
|
+
A RestoredState object containing positions, target positions, and balances.
|
|
168
|
+
"""
|
|
169
|
+
mongo_collections = self.client[self.db_name].list_collection_names()
|
|
170
|
+
required_suffixes = ["positions", "signals", "balance"]
|
|
171
|
+
|
|
172
|
+
if not any(f"{self.collection_name_prefix}_{suffix}" in mongo_collections for suffix in required_suffixes):
|
|
173
|
+
logger.warning(f"No logs collections found in MongodDB {self.db_name}.")
|
|
174
|
+
self.client.close()
|
|
175
|
+
return RestoredState(
|
|
176
|
+
time=np.datetime64("now"),
|
|
177
|
+
positions={},
|
|
178
|
+
instrument_to_target_positions={},
|
|
179
|
+
balances={},
|
|
180
|
+
)
|
|
181
|
+
|
|
182
|
+
logger.info(f"Restoring state from MongoDB {self.db_name}")
|
|
183
|
+
|
|
184
|
+
positions = self.position_restorer.restore_positions()
|
|
185
|
+
target_positions = self.signal_restorer.restore_signals()
|
|
186
|
+
balances = self.balance_restorer.restore_balances()
|
|
187
|
+
|
|
188
|
+
latest_position_timestamp = (
|
|
189
|
+
max(position.last_update_time for position in positions.values()) if positions else np.datetime64("now")
|
|
190
|
+
)
|
|
191
|
+
if np.isnan(latest_position_timestamp):
|
|
192
|
+
latest_position_timestamp = np.datetime64("now")
|
|
193
|
+
|
|
194
|
+
self.client.close()
|
|
195
|
+
return RestoredState(
|
|
196
|
+
time=recognize_time(latest_position_timestamp),
|
|
197
|
+
positions=positions,
|
|
198
|
+
instrument_to_target_positions=target_positions,
|
|
199
|
+
balances=balances,
|
|
200
|
+
)
|
|
Binary file
|
qubx/utils/runner/runner.py
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import inspect
|
|
2
1
|
import socket
|
|
3
2
|
import time
|
|
4
3
|
from collections import defaultdict
|
|
@@ -46,6 +45,7 @@ from qubx.core.interfaces import (
|
|
|
46
45
|
from qubx.core.loggers import StrategyLogging
|
|
47
46
|
from qubx.core.lookups import lookup
|
|
48
47
|
from qubx.health import BaseHealthMonitor
|
|
48
|
+
from qubx.loggers import create_logs_writer
|
|
49
49
|
from qubx.restarts.state_resolvers import StateResolver
|
|
50
50
|
from qubx.restarts.time_finders import TimeFinder
|
|
51
51
|
from qubx.restorers import create_state_restorer
|
|
@@ -388,20 +388,18 @@ def _setup_strategy_logging(
|
|
|
388
388
|
run_id = f"{socket.gethostname()}-{str(int(time.time() * 10**9))}"
|
|
389
389
|
|
|
390
390
|
_log_writer_name = log_config.logger
|
|
391
|
-
if "." not in _log_writer_name:
|
|
392
|
-
_log_writer_name = f"qubx.core.loggers.{_log_writer_name}"
|
|
393
391
|
|
|
394
392
|
logger.debug(f"Setup <g>{_log_writer_name}</g> logger...")
|
|
395
|
-
|
|
396
|
-
|
|
393
|
+
|
|
394
|
+
override_params = {
|
|
397
395
|
"account_id": "account",
|
|
398
396
|
"strategy_id": stg_name,
|
|
399
397
|
"run_id": run_id,
|
|
400
398
|
"log_folder": run_folder,
|
|
401
399
|
}
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
_log_writer =
|
|
400
|
+
_log_writer_params = {**override_params, **log_config.args}
|
|
401
|
+
|
|
402
|
+
_log_writer = create_logs_writer(_log_writer_name, _log_writer_params)
|
|
405
403
|
stg_logging = StrategyLogging(
|
|
406
404
|
logs_writer=_log_writer,
|
|
407
405
|
positions_log_freq=log_config.position_interval,
|
|
@@ -7,7 +7,7 @@ qubx/backtester/data.py,sha256=De2ioNv3Zh-cCGakzK0igb2caDcqbibZ_tsYmF7sTTQ,6601
|
|
|
7
7
|
qubx/backtester/management.py,sha256=HuyzFsBPgR7j-ei78Ngcx34CeSn65c9atmaii1aTsYg,14900
|
|
8
8
|
qubx/backtester/ome.py,sha256=Uf3wqyVjUEpm1jrDQ4PE77E3R3B7wJJy1vaOOmvQqWg,15610
|
|
9
9
|
qubx/backtester/optimization.py,sha256=HHUIYA6Y66rcOXoePWFOuOVX9iaHGKV0bGt_4d5e6FM,7619
|
|
10
|
-
qubx/backtester/runner.py,sha256=
|
|
10
|
+
qubx/backtester/runner.py,sha256=TnNM0t8PgBE_gnCOZZTIOc28a3RqtXmp2Xj4Gq5j6bo,20504
|
|
11
11
|
qubx/backtester/simulated_data.py,sha256=niujaMRj__jf4IyzCZrSBR5ZoH1VUbvsZHSewHftdmI,17240
|
|
12
12
|
qubx/backtester/simulated_exchange.py,sha256=ATGcJXnKdD47kUwgbc5tvPVL0tq4_-6jpgsTTAMxW3c,8124
|
|
13
13
|
qubx/backtester/simulator.py,sha256=cSbW42X-YlAutZlOQ3Y4mAJWXr_1WomYprtWZVMe3Uk,9225
|
|
@@ -43,7 +43,7 @@ qubx/core/exceptions.py,sha256=11wQC3nnNLsl80zBqbE6xiKCqm31kctqo6W_gdnZkg8,581
|
|
|
43
43
|
qubx/core/helpers.py,sha256=qzRsttt4sMYMarDWMzWvc3b2W-Qp9qAQwFiQBljAsA0,19722
|
|
44
44
|
qubx/core/initializer.py,sha256=PUiD_cIjvGpuPjYyRpUjpwm3xNQ2Kipa8bAhbtxCQRo,3935
|
|
45
45
|
qubx/core/interfaces.py,sha256=CzIl8tB6ImQkDcZEmhpstwHPOCY8NhZxXmBHLQUAieI,58253
|
|
46
|
-
qubx/core/loggers.py,sha256=
|
|
46
|
+
qubx/core/loggers.py,sha256=0g33jfipGFShSMrXBoYVzL0GfTzI36mwBJqHNUHmhdo,13342
|
|
47
47
|
qubx/core/lookups.py,sha256=n5ZjjEhhRvmidCB-Cubr1b0Opm6lf_QVZNEWa_BOQG0,19376
|
|
48
48
|
qubx/core/metrics.py,sha256=Gq3Ultwn5meICfyauBUJrBS4nffSxFVH3OF6N1Y0xgo,58664
|
|
49
49
|
qubx/core/mixins/__init__.py,sha256=AMCLvfNuIb1kkQl3bhCj9jIOEl2eKcVPJeyLgrkB-rk,329
|
|
@@ -52,11 +52,11 @@ qubx/core/mixins/processing.py,sha256=dqehukrfqcLy5BeILKnkpHCvva4SbLKj1ZbQdnByu1
|
|
|
52
52
|
qubx/core/mixins/subscription.py,sha256=V_g9wCPQ8S5SHkU-qOZ84cV5nReAUrV7DoSNAGG0LPY,10372
|
|
53
53
|
qubx/core/mixins/trading.py,sha256=idfRPaqrvkfMxzu9mXr9i_xfqLee-ZAOrERxkxv6Ruo,7256
|
|
54
54
|
qubx/core/mixins/universe.py,sha256=L3s2Jw46_J1iDh4622Gk_LvCjol4W7mflBwEHrLfZEw,9899
|
|
55
|
-
qubx/core/series.cpython-312-x86_64-linux-gnu.so,sha256=
|
|
55
|
+
qubx/core/series.cpython-312-x86_64-linux-gnu.so,sha256=oCjBv31hRr3bJ47EucFuzDzwStG05dUWTNyAXBg2HwQ,978280
|
|
56
56
|
qubx/core/series.pxd,sha256=jBdMwgO8J4Zrue0e_xQ5RlqTXqihpzQNu6V3ckZvvpY,3978
|
|
57
57
|
qubx/core/series.pyi,sha256=RaHm_oHHiWiNUMJqVfx5FXAXniGLsHxUFOUpacn7GC0,4604
|
|
58
58
|
qubx/core/series.pyx,sha256=7cM3zZThW59waHiYcZmMxvYj-HYD7Ej_l7nKA4emPjE,46477
|
|
59
|
-
qubx/core/utils.cpython-312-x86_64-linux-gnu.so,sha256=
|
|
59
|
+
qubx/core/utils.cpython-312-x86_64-linux-gnu.so,sha256=jpJmqz2ebCWD6_Wb8-IkGvGLhhgT2Gk0f2EmL8bkUC8,86568
|
|
60
60
|
qubx/core/utils.pyi,sha256=a-wS13V2p_dM1CnGq40JVulmiAhixTwVwt0ah5By0Hc,348
|
|
61
61
|
qubx/core/utils.pyx,sha256=k5QHfEFvqhqWfCob89ANiJDKNG8gGbOh-O4CVoneZ8M,1696
|
|
62
62
|
qubx/data/__init__.py,sha256=ELZykvpPGWc5rX7QoNyNQwMLgdKMG8MACOByA4pM5hA,549
|
|
@@ -89,6 +89,11 @@ qubx/features/utils.py,sha256=5wMlfH4x1dUh00dxvtnHhSiHeRaiod4VMTcmgm-o_wA,264
|
|
|
89
89
|
qubx/gathering/simplest.py,sha256=24SIjsCfutuTinSW5zSkPHGJvl-vnyhe3FAX3quUx4E,4011
|
|
90
90
|
qubx/health/__init__.py,sha256=ThJTgf-CPD5tMU_emqANpnE6oXfUmzyyugfbDfzeVB0,111
|
|
91
91
|
qubx/health/base.py,sha256=FmpZ7l_L-wJ8JCQ42uRjng3hAbhfmAf3nIUc4vSs9cI,27622
|
|
92
|
+
qubx/loggers/__init__.py,sha256=HY26weGv_qI-i5NrvXepGCwXENOIeORW9uhvyFMVMcM,443
|
|
93
|
+
qubx/loggers/csv.py,sha256=95JLFz2yxo_OjSG21cmWVYZP1XuN3V0FXNPUB_q6ucY,3791
|
|
94
|
+
qubx/loggers/factory.py,sha256=09ahIIQU_59vI4OO1rKYLAHgBbZ2506wv0FqYie_Rb0,1805
|
|
95
|
+
qubx/loggers/inmemory.py,sha256=49Y-jsRxDzBLWQdQMIKjVTvnx_79EbjFpHwj3v8Mgno,2642
|
|
96
|
+
qubx/loggers/mongo.py,sha256=dOEpCcIxT6O9MgpK2srpzxyuto6QaQgTxMK0WcEIR70,2703
|
|
92
97
|
qubx/math/__init__.py,sha256=ltHSQj40sCBm3owcvtoZp34h6ws7pZCFcSZgUkTsUCY,114
|
|
93
98
|
qubx/math/stats.py,sha256=uXm4NpBRxuHFTjXERv8rjM0MAJof8zr1Cklyra4CcBA,4056
|
|
94
99
|
qubx/notifications/__init__.py,sha256=2xsk3kPykiNZDZv0y4YMDbgnFvKy14SkNeg7StHk4bI,340
|
|
@@ -109,15 +114,15 @@ qubx/restarts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
|
109
114
|
qubx/restarts/state_resolvers.py,sha256=GJ617qwpulqMp_-WhpmsLozQobxgt5lU4ZGOIUaUzas,5606
|
|
110
115
|
qubx/restarts/time_finders.py,sha256=r7yyRhJByV2uqdgamDRX2XClwpWWI9BNpc80t9nk6c0,2448
|
|
111
116
|
qubx/restorers/__init__.py,sha256=vrnZBPJHR0-6knAccj4bK0tkjUPNRl32qiLr5Mv4aR0,911
|
|
112
|
-
qubx/restorers/balance.py,sha256=
|
|
113
|
-
qubx/restorers/factory.py,sha256=
|
|
117
|
+
qubx/restorers/balance.py,sha256=Oz-LY7s8tdSIJl83z9rII-DpX5xa8-qiqjcDYfW-KiE,6456
|
|
118
|
+
qubx/restorers/factory.py,sha256=eoijcUHDaBVPHSfkjyo1AHvWTvvs0kj7jJbF_NE30aw,6737
|
|
114
119
|
qubx/restorers/interfaces.py,sha256=CcjBWavKq8_GIMKTSPodMa-n3wJQwcQTwyvYyNo_J3c,1776
|
|
115
|
-
qubx/restorers/position.py,sha256=
|
|
116
|
-
qubx/restorers/signal.py,sha256=
|
|
117
|
-
qubx/restorers/state.py,sha256=
|
|
120
|
+
qubx/restorers/position.py,sha256=xmaqaQQrmzF0VEHAZK35cXb6WKYRnDDapaN-EgZcH8U,8090
|
|
121
|
+
qubx/restorers/signal.py,sha256=9TAaJOEKPjZXuciFFVn6Z8a-Z8CfVSjRGFRcwEgbPLY,10745
|
|
122
|
+
qubx/restorers/state.py,sha256=dLaVnUwRCNRkUqbYyi0RfZs3Q3AdglkI_qTtQ8GDD5Y,7289
|
|
118
123
|
qubx/restorers/utils.py,sha256=We2gfqwQKWziUYhuUnjb-xo-5tSlbuHWpPQn0CEMTn0,1155
|
|
119
124
|
qubx/ta/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
120
|
-
qubx/ta/indicators.cpython-312-x86_64-linux-gnu.so,sha256=
|
|
125
|
+
qubx/ta/indicators.cpython-312-x86_64-linux-gnu.so,sha256=EAm2KZhw7wkpqvoEQRH9a3Z2enYqYQkk5EdVVOPuB6s,654440
|
|
121
126
|
qubx/ta/indicators.pxd,sha256=Goo0_N0Xnju8XGo3Xs-3pyg2qr_0Nh5C-_26DK8U_IE,4224
|
|
122
127
|
qubx/ta/indicators.pyi,sha256=19W0uERft49In5bf9jkJHkzJYEyE9gzudN7_DJ5Vdv8,1963
|
|
123
128
|
qubx/ta/indicators.pyx,sha256=Xgpew46ZxSXsdfSEWYn3A0Q35MLsopB9n7iyCsXTufs,25969
|
|
@@ -151,11 +156,11 @@ qubx/utils/runner/_jupyter_runner.pyt,sha256=fDj4AUs25jsdGmY9DDeSFufH1JkVhLFwy0B
|
|
|
151
156
|
qubx/utils/runner/accounts.py,sha256=mpiv6oxr5z97zWt7STYyARMhWQIpc_XFKungb_pX38U,3270
|
|
152
157
|
qubx/utils/runner/configs.py,sha256=4lonQgksh4wDygsN67lIidVRIUksskWuhL25A2IZwho,3694
|
|
153
158
|
qubx/utils/runner/factory.py,sha256=vQ2dBTbrQE9YH__-TvuFzGF-E1li-vt_qQum9GHa11g,11666
|
|
154
|
-
qubx/utils/runner/runner.py,sha256=
|
|
159
|
+
qubx/utils/runner/runner.py,sha256=dyrwFiVmU3nkgGiDRIg6cxhikf3KJ4ylEdojUdf9WaQ,29070
|
|
155
160
|
qubx/utils/time.py,sha256=J0ZFGjzFL5T6GA8RPAel8hKG0sg2LZXeQ5YfDCfcMHA,10055
|
|
156
161
|
qubx/utils/version.py,sha256=e52fIHyxzCiIuH7svCF6pkHuDlqL64rklqz-2XjWons,5309
|
|
157
|
-
qubx-0.6.
|
|
158
|
-
qubx-0.6.
|
|
159
|
-
qubx-0.6.
|
|
160
|
-
qubx-0.6.
|
|
161
|
-
qubx-0.6.
|
|
162
|
+
qubx-0.6.38.dist-info/LICENSE,sha256=qwMHOSJ2TD0nx6VUJvFhu1ynJdBfNozRMt6tnSul-Ts,35140
|
|
163
|
+
qubx-0.6.38.dist-info/METADATA,sha256=_71dUs78KDAbSy2GWSHBg_oTYYZnugqV8e8oACi9t-4,4492
|
|
164
|
+
qubx-0.6.38.dist-info/WHEEL,sha256=XjdW4AGUgFDhpG9b3b2KPhtR_JLZvHyfemLgJJwcqOI,110
|
|
165
|
+
qubx-0.6.38.dist-info/entry_points.txt,sha256=VqilDTe8mVuV9SbR-yVlZJBTjbkHIL2JBgXfQw076HY,47
|
|
166
|
+
qubx-0.6.38.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|