kaq-quant-common 0.2.7__py3-none-any.whl → 0.2.8__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kaq_quant_common/api/common/__init__.py +1 -1
- kaq_quant_common/api/common/api_interface.py +38 -38
- kaq_quant_common/api/rest/api_client_base.py +42 -42
- kaq_quant_common/api/rest/instruction/helper/order_helper.py +342 -342
- kaq_quant_common/api/rest/instruction/models/__init__.py +17 -17
- kaq_quant_common/api/rest/instruction/models/transfer.py +32 -32
- kaq_quant_common/api/ws/exchange/models.py +23 -23
- kaq_quant_common/api/ws/exchange/ws_exchange_server.py +440 -440
- kaq_quant_common/common/ddb_table_monitor.py +106 -106
- kaq_quant_common/common/http_monitor.py +69 -69
- kaq_quant_common/common/modules/funding_rate_helper.py +36 -21
- kaq_quant_common/common/modules/limit_order_helper.py +60 -23
- kaq_quant_common/common/monitor_base.py +84 -84
- kaq_quant_common/common/monitor_group.py +97 -97
- kaq_quant_common/common/ws_wrapper.py +21 -21
- kaq_quant_common/resources/kaq_ddb_stream_write_resources.py +172 -96
- kaq_quant_common/utils/logger_utils.py +5 -5
- kaq_quant_common/utils/signal_utils.py +23 -23
- kaq_quant_common/utils/uuid_utils.py +5 -5
- {kaq_quant_common-0.2.7.dist-info → kaq_quant_common-0.2.8.dist-info}/METADATA +1 -1
- {kaq_quant_common-0.2.7.dist-info → kaq_quant_common-0.2.8.dist-info}/RECORD +22 -22
- {kaq_quant_common-0.2.7.dist-info → kaq_quant_common-0.2.8.dist-info}/WHEEL +1 -1
|
@@ -1,440 +1,440 @@
|
|
|
1
|
-
import threading
|
|
2
|
-
import time
|
|
3
|
-
from datetime import datetime, timedelta
|
|
4
|
-
from typing import List, Optional, Tuple, Union
|
|
5
|
-
|
|
6
|
-
import pandas as pd
|
|
7
|
-
|
|
8
|
-
from kaq_quant_common.api.ws.exchange.models import FundingRateEvent
|
|
9
|
-
from kaq_quant_common.api.ws.ws_server_base import WsServerBase
|
|
10
|
-
from kaq_quant_common.resources.kaq_mysql_resources import KaqQuantMysqlRepository
|
|
11
|
-
from kaq_quant_common.utils import logger_utils
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
class WsExchangeServer(WsServerBase):
|
|
15
|
-
"""
|
|
16
|
-
模拟加密货币平台 WS 服务器:
|
|
17
|
-
- 从 MySQL 按 exchange 读取资金费率历史数据
|
|
18
|
-
- 从指定 start_time 开始,按事件时间差和加速倍数推送
|
|
19
|
-
- 支持将推送事件的 event_time 重写为实时时间
|
|
20
|
-
- 主题:
|
|
21
|
-
* funding_rate.all
|
|
22
|
-
* funding_rate.<symbol>
|
|
23
|
-
"""
|
|
24
|
-
|
|
25
|
-
def __init__(
|
|
26
|
-
self,
|
|
27
|
-
exchange: str,
|
|
28
|
-
mysql_host: str,
|
|
29
|
-
mysql_port: int,
|
|
30
|
-
mysql_user: str,
|
|
31
|
-
mysql_passwd: str,
|
|
32
|
-
mysql_db: str,
|
|
33
|
-
charset: str = "utf8mb4",
|
|
34
|
-
start_time: Union[int, float, datetime] = None,
|
|
35
|
-
speed_multiplier: float = 1.0,
|
|
36
|
-
use_realtime_event_time: bool = False,
|
|
37
|
-
inject_sample_on_empty: bool = True,
|
|
38
|
-
host: str = "0.0.0.0",
|
|
39
|
-
port: int = 8767,
|
|
40
|
-
):
|
|
41
|
-
super().__init__(self, host, port)
|
|
42
|
-
self._logger = logger_utils.get_logger(self)
|
|
43
|
-
# 平台
|
|
44
|
-
self._exchange = exchange
|
|
45
|
-
# 加速设置
|
|
46
|
-
self._speed_multiplier = max(speed_multiplier if speed_multiplier and speed_multiplier > 0 else 1.0, 1e-6)
|
|
47
|
-
# 是否使用实时时间
|
|
48
|
-
self._use_realtime_event_time = use_realtime_event_time
|
|
49
|
-
# 示例数据注入开关
|
|
50
|
-
self._inject_sample_on_empty = inject_sample_on_empty
|
|
51
|
-
|
|
52
|
-
# 单次增量加载的窗口大小
|
|
53
|
-
self._window_size: timedelta = timedelta(minutes=10)
|
|
54
|
-
# 触发预加载的阈值
|
|
55
|
-
self._preload_horizon: timedelta = timedelta(minutes=5)
|
|
56
|
-
# 丢弃数据的阈值
|
|
57
|
-
self._retain_horizon: timedelta = timedelta(minutes=10)
|
|
58
|
-
|
|
59
|
-
# 数据开始游标(时间) 支持毫秒时间戳或 datetime
|
|
60
|
-
if isinstance(start_time, (int, float)):
|
|
61
|
-
self._start_dt = datetime.fromtimestamp(start_time / 1000.0)
|
|
62
|
-
elif isinstance(start_time, datetime):
|
|
63
|
-
self._start_dt = start_time
|
|
64
|
-
else:
|
|
65
|
-
# 默认很早的时间,确保可以从最早记录开始
|
|
66
|
-
self._start_dt = datetime.fromtimestamp(0)
|
|
67
|
-
|
|
68
|
-
# DB
|
|
69
|
-
self._repo = KaqQuantMysqlRepository(mysql_host, mysql_port, mysql_user, mysql_passwd, mysql_db, charset)
|
|
70
|
-
|
|
71
|
-
# 推送控制
|
|
72
|
-
self._push_thread: Optional[threading.Thread] = None
|
|
73
|
-
self._stop_push = threading.Event()
|
|
74
|
-
|
|
75
|
-
# 预拉取首批数据,开始的时候加载多一点
|
|
76
|
-
initial_end = self._start_dt + self._preload_horizon * 2
|
|
77
|
-
self._events: List[Tuple[datetime, FundingRateEvent]] = self._load_events_window(self._start_dt, initial_end)
|
|
78
|
-
self._last_fetched_dt = initial_end
|
|
79
|
-
self._logger.info(f"资金费率事件已加载: {len(self._events)} 条")
|
|
80
|
-
# 添加测试数据
|
|
81
|
-
if not self._events and self._inject_sample_on_empty:
|
|
82
|
-
base_dt = self._start_dt if self._start_dt else datetime.fromtimestamp(0)
|
|
83
|
-
sample: List[Tuple[datetime, FundingRateEvent]] = []
|
|
84
|
-
for i in range(5):
|
|
85
|
-
dt = base_dt if i == 0 else base_dt + pd.Timedelta(minutes=i)
|
|
86
|
-
evt = FundingRateEvent(
|
|
87
|
-
event_time=int(dt.timestamp() * 1000),
|
|
88
|
-
exchange_symbol=f"{self._exchange}:BTCUSDT",
|
|
89
|
-
exchange=self._exchange,
|
|
90
|
-
symbol="BTCUSDT",
|
|
91
|
-
open_rate=0.0001,
|
|
92
|
-
close_rate=0.0001,
|
|
93
|
-
high_rate=0.0002,
|
|
94
|
-
low_rate=0.0000,
|
|
95
|
-
)
|
|
96
|
-
sample.append((dt, evt))
|
|
97
|
-
self._events = sample
|
|
98
|
-
self._logger.warning("资金费率历史为空,已注入示例数据用于推送测试")
|
|
99
|
-
|
|
100
|
-
# 把开始时间调整为第一条数据的时间
|
|
101
|
-
if self._events:
|
|
102
|
-
self._start_dt = self._events[0][0]
|
|
103
|
-
# 最后对齐整分钟
|
|
104
|
-
self._start_dt = self._start_dt.replace(second=0, microsecond=0)
|
|
105
|
-
|
|
106
|
-
# 实际启动时间,用于计算虚拟时间,需要对齐整分钟
|
|
107
|
-
self._real_start_dt: datetime = datetime.now()
|
|
108
|
-
self._real_start_dt = self._real_start_dt.replace(second=0, microsecond=0)
|
|
109
|
-
|
|
110
|
-
# 原始全量拉取(保留以兼容旧逻辑)
|
|
111
|
-
def _load_events(self) -> List[Tuple[datetime, FundingRateEvent]]:
|
|
112
|
-
try:
|
|
113
|
-
query = (
|
|
114
|
-
"SELECT event_time, exchange_symbol, exchange, symbol, open_rate, close_rate, high_rate, low_rate, close_time, next_event_time, id, ctimestamp "
|
|
115
|
-
"FROM kaq_future_fr_klines_1m "
|
|
116
|
-
f"WHERE exchange = '{self._exchange}' AND event_time >= '{self._start_dt.strftime('%Y-%m-%d %H:%M:%S')}' "
|
|
117
|
-
"ORDER BY event_time ASC"
|
|
118
|
-
)
|
|
119
|
-
df: pd.DataFrame = self._repo.fetch_data(query)
|
|
120
|
-
if df is None or df.empty:
|
|
121
|
-
self._logger.warning("资金费率表无数据或查询为空")
|
|
122
|
-
return []
|
|
123
|
-
|
|
124
|
-
events: List[Tuple[datetime, FundingRateEvent]] = []
|
|
125
|
-
for _, row in df.iterrows():
|
|
126
|
-
evt_dt: datetime = row.get("event_time")
|
|
127
|
-
if not isinstance(evt_dt, datetime):
|
|
128
|
-
try:
|
|
129
|
-
evt_dt = pd.to_datetime(evt_dt).to_pydatetime()
|
|
130
|
-
except Exception:
|
|
131
|
-
continue
|
|
132
|
-
# 转为毫秒时间戳
|
|
133
|
-
close_time_val = row.get("close_time")
|
|
134
|
-
next_event_time_val = row.get("next_event_time")
|
|
135
|
-
try:
|
|
136
|
-
close_time_ms = int(pd.to_datetime(close_time_val).timestamp() * 1000) if close_time_val is not None else None
|
|
137
|
-
except Exception:
|
|
138
|
-
close_time_ms = None
|
|
139
|
-
try:
|
|
140
|
-
next_event_time_ms = int(pd.to_datetime(next_event_time_val).timestamp() * 1000) if next_event_time_val is not None else None
|
|
141
|
-
except Exception:
|
|
142
|
-
next_event_time_ms = None
|
|
143
|
-
evt_ms = int(evt_dt.timestamp() * 1000)
|
|
144
|
-
event = FundingRateEvent(
|
|
145
|
-
event_time=evt_ms,
|
|
146
|
-
exchange_symbol=row.get("exchange_symbol"),
|
|
147
|
-
exchange=row.get("exchange"),
|
|
148
|
-
symbol=row.get("symbol"),
|
|
149
|
-
open_rate=row.get("open_rate"),
|
|
150
|
-
close_rate=row.get("close_rate"),
|
|
151
|
-
high_rate=row.get("high_rate"),
|
|
152
|
-
low_rate=row.get("low_rate"),
|
|
153
|
-
close_time=close_time_ms,
|
|
154
|
-
next_event_time=next_event_time_ms,
|
|
155
|
-
#
|
|
156
|
-
id=row.get("id"),
|
|
157
|
-
ctimestamp=row.get("ctimestamp"),
|
|
158
|
-
)
|
|
159
|
-
events.append((evt_dt, event))
|
|
160
|
-
return events
|
|
161
|
-
except Exception as e:
|
|
162
|
-
self._logger.error(f"加载资金费率事件失败: {e}")
|
|
163
|
-
return []
|
|
164
|
-
|
|
165
|
-
# 增量批量拉取(保留以兼容旧逻辑)
|
|
166
|
-
def _load_events_batch(self, start_from: datetime, limit: int) -> List[Tuple[datetime, FundingRateEvent]]:
|
|
167
|
-
try:
|
|
168
|
-
query = (
|
|
169
|
-
"SELECT event_time, exchange_symbol, exchange, symbol, open_rate, close_rate, high_rate, low_rate, close_time, next_event_time, id, ctimestamp "
|
|
170
|
-
"FROM kaq_future_fr_klines_1m "
|
|
171
|
-
f"WHERE exchange = '{self._exchange}' AND event_time > '{start_from.strftime('%Y-%m-%d %H:%M:%S')}' "
|
|
172
|
-
"ORDER BY event_time ASC "
|
|
173
|
-
f"LIMIT {int(limit)}"
|
|
174
|
-
)
|
|
175
|
-
df: pd.DataFrame = self._repo.fetch_data(query)
|
|
176
|
-
if df is None or df.empty:
|
|
177
|
-
return []
|
|
178
|
-
events: List[Tuple[datetime, FundingRateEvent]] = []
|
|
179
|
-
for _, row in df.iterrows():
|
|
180
|
-
evt_dt: datetime = row.get("event_time")
|
|
181
|
-
if not isinstance(evt_dt, datetime):
|
|
182
|
-
try:
|
|
183
|
-
evt_dt = pd.to_datetime(evt_dt).to_pydatetime()
|
|
184
|
-
except Exception:
|
|
185
|
-
continue
|
|
186
|
-
# 转为毫秒时间戳
|
|
187
|
-
close_time_val = row.get("close_time")
|
|
188
|
-
next_event_time_val = row.get("next_event_time")
|
|
189
|
-
try:
|
|
190
|
-
close_time_ms = int(pd.to_datetime(close_time_val).timestamp() * 1000) if close_time_val is not None else None
|
|
191
|
-
except Exception:
|
|
192
|
-
close_time_ms = None
|
|
193
|
-
try:
|
|
194
|
-
next_event_time_ms = int(pd.to_datetime(next_event_time_val).timestamp() * 1000) if next_event_time_val is not None else None
|
|
195
|
-
except Exception:
|
|
196
|
-
next_event_time_ms = None
|
|
197
|
-
evt_ms = int(evt_dt.timestamp() * 1000)
|
|
198
|
-
event = FundingRateEvent(
|
|
199
|
-
event_time=evt_ms,
|
|
200
|
-
exchange_symbol=row.get("exchange_symbol"),
|
|
201
|
-
exchange=row.get("exchange"),
|
|
202
|
-
symbol=row.get("symbol"),
|
|
203
|
-
open_rate=row.get("open_rate"),
|
|
204
|
-
close_rate=row.get("close_rate"),
|
|
205
|
-
high_rate=row.get("high_rate"),
|
|
206
|
-
low_rate=row.get("low_rate"),
|
|
207
|
-
close_time=close_time_ms,
|
|
208
|
-
next_event_time=next_event_time_ms,
|
|
209
|
-
#
|
|
210
|
-
id=row.get("id"),
|
|
211
|
-
ctimestamp=row.get("ctimestamp"),
|
|
212
|
-
)
|
|
213
|
-
events.append((evt_dt, event))
|
|
214
|
-
return events
|
|
215
|
-
except Exception as e:
|
|
216
|
-
self._logger.error(f"增量加载资金费率事件失败: {e}")
|
|
217
|
-
return []
|
|
218
|
-
|
|
219
|
-
# 按照时间窗口拉取
|
|
220
|
-
def _load_events_window(self, start_from: datetime, end_until: datetime) -> List[Tuple[datetime, FundingRateEvent]]:
|
|
221
|
-
try:
|
|
222
|
-
query = (
|
|
223
|
-
"SELECT event_time, exchange_symbol, exchange, symbol, open_rate, close_rate, high_rate, low_rate, close_time, next_event_time, id, ctimestamp "
|
|
224
|
-
"FROM kaq_future_fr_klines_1m "
|
|
225
|
-
f"WHERE exchange = '{self._exchange}' AND event_time > '{start_from.strftime('%Y-%m-%d %H:%M:%S')}' AND event_time <= '{end_until.strftime('%Y-%m-%d %H:%M:%S')}' "
|
|
226
|
-
"ORDER BY event_time ASC"
|
|
227
|
-
)
|
|
228
|
-
df: pd.DataFrame = self._repo.fetch_data(query)
|
|
229
|
-
if df is None or df.empty:
|
|
230
|
-
return []
|
|
231
|
-
events: List[Tuple[datetime, FundingRateEvent]] = []
|
|
232
|
-
for _, row in df.iterrows():
|
|
233
|
-
evt_dt: datetime = row.get("event_time")
|
|
234
|
-
if not isinstance(evt_dt, datetime):
|
|
235
|
-
try:
|
|
236
|
-
evt_dt = pd.to_datetime(evt_dt).to_pydatetime()
|
|
237
|
-
except Exception:
|
|
238
|
-
continue
|
|
239
|
-
close_time_val = row.get("close_time")
|
|
240
|
-
next_event_time_val = row.get("next_event_time")
|
|
241
|
-
try:
|
|
242
|
-
close_time_ms = int(pd.to_datetime(close_time_val).timestamp() * 1000) if close_time_val is not None else None
|
|
243
|
-
except Exception:
|
|
244
|
-
close_time_ms = None
|
|
245
|
-
try:
|
|
246
|
-
next_event_time_ms = int(pd.to_datetime(next_event_time_val).timestamp() * 1000) if next_event_time_val is not None else None
|
|
247
|
-
except Exception:
|
|
248
|
-
next_event_time_ms = None
|
|
249
|
-
evt_ms = int(evt_dt.timestamp() * 1000)
|
|
250
|
-
event = FundingRateEvent(
|
|
251
|
-
event_time=evt_ms,
|
|
252
|
-
exchange_symbol=row.get("exchange_symbol"),
|
|
253
|
-
exchange=row.get("exchange"),
|
|
254
|
-
symbol=row.get("symbol"),
|
|
255
|
-
open_rate=row.get("open_rate"),
|
|
256
|
-
close_rate=row.get("close_rate"),
|
|
257
|
-
high_rate=row.get("high_rate"),
|
|
258
|
-
low_rate=row.get("low_rate"),
|
|
259
|
-
close_time=close_time_ms,
|
|
260
|
-
next_event_time=next_event_time_ms,
|
|
261
|
-
#
|
|
262
|
-
id=row.get("id"),
|
|
263
|
-
ctimestamp=row.get("ctimestamp"),
|
|
264
|
-
)
|
|
265
|
-
events.append((evt_dt, event))
|
|
266
|
-
return events
|
|
267
|
-
except Exception as e:
|
|
268
|
-
self._logger.error(f"时间窗加载资金费率事件失败: {e}")
|
|
269
|
-
return []
|
|
270
|
-
|
|
271
|
-
# 从数据库拉取最新的记录
|
|
272
|
-
def _get_latest_event_dt(self) -> Optional[datetime]:
|
|
273
|
-
try:
|
|
274
|
-
query = "SELECT MAX(event_time) AS max_event_time " "FROM kaq_future_fr_klines_1m " f"WHERE exchange = '{self._exchange}'"
|
|
275
|
-
df: pd.DataFrame = self._repo.fetch_data(query)
|
|
276
|
-
if df is None or df.empty:
|
|
277
|
-
return None
|
|
278
|
-
val = df.iloc[0].get("max_event_time")
|
|
279
|
-
|
|
280
|
-
if isinstance(val, datetime):
|
|
281
|
-
return val
|
|
282
|
-
try:
|
|
283
|
-
return pd.to_datetime(val).to_pydatetime()
|
|
284
|
-
except Exception:
|
|
285
|
-
return None
|
|
286
|
-
except Exception as e:
|
|
287
|
-
self._logger.error(f"查询最新资金费率事件时间失败: {e}")
|
|
288
|
-
return None
|
|
289
|
-
|
|
290
|
-
def _push_loop(self):
|
|
291
|
-
# 初始缓冲检查
|
|
292
|
-
if not self._events:
|
|
293
|
-
# 获取一次最新记录
|
|
294
|
-
latest_dt = self._get_latest_event_dt()
|
|
295
|
-
initial_end = self._last_fetched_dt + self._window_size
|
|
296
|
-
# 防止超过
|
|
297
|
-
if latest_dt is not None and initial_end > latest_dt:
|
|
298
|
-
initial_end = latest_dt
|
|
299
|
-
self._events = self._load_events_window(self._last_fetched_dt, initial_end)
|
|
300
|
-
# 主要是为了记录这个最新拉取的时间点,下一次增量加载从这里开始
|
|
301
|
-
self._last_fetched_dt = initial_end
|
|
302
|
-
if not self._events:
|
|
303
|
-
self._logger.warning("无可推送的资金费率历史事件")
|
|
304
|
-
return
|
|
305
|
-
|
|
306
|
-
prev_dt = None
|
|
307
|
-
prev_counter = 0
|
|
308
|
-
discard_dt = None
|
|
309
|
-
idx = 0
|
|
310
|
-
while not self._stop_push.is_set():
|
|
311
|
-
# 虚拟时钟,考虑加速比
|
|
312
|
-
now_real = datetime.now()
|
|
313
|
-
# 计算出虚拟时间点
|
|
314
|
-
now_virtual = self._start_dt + (now_real - self._real_start_dt) * self._speed_multiplier
|
|
315
|
-
|
|
316
|
-
# 确保缓冲区覆盖到虚拟时间的预加载阈值
|
|
317
|
-
# 获取缓冲区最新的记录时间
|
|
318
|
-
buffer_max_dt = self._events[-1][0] if self._events else self._last_fetched_dt
|
|
319
|
-
# 根据虚拟时间点计算出需要覆盖的时间点
|
|
320
|
-
target_cover_dt = now_virtual + self._preload_horizon
|
|
321
|
-
# 最新的记录时间
|
|
322
|
-
latest_dt: datetime = None
|
|
323
|
-
while buffer_max_dt < target_cover_dt and not self._stop_push.is_set():
|
|
324
|
-
# 每次进入循环,先拉取一次最新数据时间
|
|
325
|
-
if latest_dt is None:
|
|
326
|
-
latest_dt = self._get_latest_event_dt()
|
|
327
|
-
# 若数据库已无更晚数据,停止增量加载
|
|
328
|
-
if latest_dt is not None and self._last_fetched_dt >= latest_dt:
|
|
329
|
-
buffer_max_dt = latest_dt
|
|
330
|
-
self._logger.debug("已到达数据库最新事件时间,停止增量加载")
|
|
331
|
-
break
|
|
332
|
-
# 计算出增量加载的窗口时间点
|
|
333
|
-
window_end = self._last_fetched_dt + self._window_size
|
|
334
|
-
# 限制窗口终点不超过数据库最新时间
|
|
335
|
-
if latest_dt is not None and window_end > latest_dt:
|
|
336
|
-
window_end = latest_dt
|
|
337
|
-
# 增量加载
|
|
338
|
-
new_events = self._load_events_window(self._last_fetched_dt, window_end)
|
|
339
|
-
self._last_fetched_dt = window_end
|
|
340
|
-
if new_events:
|
|
341
|
-
self._logger.debug(f"增量获取{len(new_events)}条数据,截止时间{window_end}")
|
|
342
|
-
self._events.extend(new_events)
|
|
343
|
-
buffer_max_dt = self._events[-1][0]
|
|
344
|
-
else:
|
|
345
|
-
# 即便当前窗无数据,推进游标,避免卡住
|
|
346
|
-
buffer_max_dt = max(buffer_max_dt, window_end)
|
|
347
|
-
self._logger.debug(f"增量获取{0}条数据,截止时间{window_end}")
|
|
348
|
-
latest_dt = None
|
|
349
|
-
|
|
350
|
-
# 如果缓冲区已消费完,退出(或等待下一轮覆盖)
|
|
351
|
-
if idx >= len(self._events):
|
|
352
|
-
time.sleep(0.05)
|
|
353
|
-
continue
|
|
354
|
-
|
|
355
|
-
# 取出一条数据
|
|
356
|
-
evt_dt, evt = self._events[idx]
|
|
357
|
-
|
|
358
|
-
# 计算等待时间,按加速比缩放
|
|
359
|
-
try:
|
|
360
|
-
# 数据时间 - 当前虚拟时间
|
|
361
|
-
delta = (evt_dt - now_virtual).total_seconds()
|
|
362
|
-
# 等待秒数
|
|
363
|
-
wait_s = max(delta, 0.0)
|
|
364
|
-
if wait_s > 0:
|
|
365
|
-
time.sleep(0.05)
|
|
366
|
-
continue
|
|
367
|
-
|
|
368
|
-
if prev_dt != evt_dt:
|
|
369
|
-
# 输出一下时间
|
|
370
|
-
if prev_dt is not None:
|
|
371
|
-
self._logger.debug(f"完成处理{prev_dt}的资金费率{prev_counter}条")
|
|
372
|
-
self._logger.debug(f"开始处理{evt_dt}的资金费率")
|
|
373
|
-
prev_counter = 0
|
|
374
|
-
prev_dt = evt_dt
|
|
375
|
-
#
|
|
376
|
-
if discard_dt is None:
|
|
377
|
-
discard_dt = prev_dt
|
|
378
|
-
except Exception:
|
|
379
|
-
pass
|
|
380
|
-
|
|
381
|
-
#
|
|
382
|
-
idx += 1
|
|
383
|
-
prev_counter += 1
|
|
384
|
-
|
|
385
|
-
# 构造推送数据
|
|
386
|
-
payload = evt.model_dump()
|
|
387
|
-
if self._use_realtime_event_time:
|
|
388
|
-
# 调整为实时时间
|
|
389
|
-
payload["event_time"] = int(time.time() * 1000)
|
|
390
|
-
# 广播到主题:all 和 symbol
|
|
391
|
-
try:
|
|
392
|
-
self.broadcast("funding_rate.all", payload)
|
|
393
|
-
sym = payload.get("symbol") or ""
|
|
394
|
-
if sym:
|
|
395
|
-
self.broadcast(f"funding_rate.{sym}", payload)
|
|
396
|
-
except Exception as e:
|
|
397
|
-
self._logger.warning(f"广播资金费率事件失败: {e}")
|
|
398
|
-
|
|
399
|
-
# 时间型丢弃:移除过旧数据以控制内存
|
|
400
|
-
if prev_dt - discard_dt > self._retain_horizon:
|
|
401
|
-
discard_dt = prev_dt
|
|
402
|
-
cut_idx = -1
|
|
403
|
-
for i in range(len(self._events)):
|
|
404
|
-
if self._events[i][0] < discard_dt:
|
|
405
|
-
cut_idx = i
|
|
406
|
-
elif self._events[i][0] >= discard_dt:
|
|
407
|
-
break
|
|
408
|
-
# 裁剪
|
|
409
|
-
if cut_idx > 0:
|
|
410
|
-
cut_count = cut_idx + 1
|
|
411
|
-
self._logger.debug(f"丢弃{cut_count}条过期数据")
|
|
412
|
-
self._events = self._events[cut_count:]
|
|
413
|
-
# 修正idx
|
|
414
|
-
idx = max(0, idx - cut_count)
|
|
415
|
-
|
|
416
|
-
def run_with_thread(self, block: bool = True):
|
|
417
|
-
# 启动推送线程
|
|
418
|
-
self._stop_push.clear()
|
|
419
|
-
self._push_thread = threading.Thread(target=self._push_loop, name="WsExchangePush", daemon=True)
|
|
420
|
-
self._push_thread.start()
|
|
421
|
-
# 启动WS服务器线程
|
|
422
|
-
super().run_with_thread(block)
|
|
423
|
-
|
|
424
|
-
def shutdown_with_thread(self):
|
|
425
|
-
# 先停止推送线程
|
|
426
|
-
try:
|
|
427
|
-
self._stop_push.set()
|
|
428
|
-
if self._push_thread and self._push_thread.is_alive():
|
|
429
|
-
self._push_thread.join(timeout=3)
|
|
430
|
-
except Exception:
|
|
431
|
-
pass
|
|
432
|
-
# 再关闭WS服务器
|
|
433
|
-
super().shutdown_with_thread()
|
|
434
|
-
|
|
435
|
-
async def _ws_handler(self, ws):
|
|
436
|
-
# 覆盖基类鉴权,模拟平台测试环境不启用鉴权
|
|
437
|
-
try:
|
|
438
|
-
await self._handle_connection(ws)
|
|
439
|
-
except Exception:
|
|
440
|
-
pass
|
|
1
|
+
import threading
|
|
2
|
+
import time
|
|
3
|
+
from datetime import datetime, timedelta
|
|
4
|
+
from typing import List, Optional, Tuple, Union
|
|
5
|
+
|
|
6
|
+
import pandas as pd
|
|
7
|
+
|
|
8
|
+
from kaq_quant_common.api.ws.exchange.models import FundingRateEvent
|
|
9
|
+
from kaq_quant_common.api.ws.ws_server_base import WsServerBase
|
|
10
|
+
from kaq_quant_common.resources.kaq_mysql_resources import KaqQuantMysqlRepository
|
|
11
|
+
from kaq_quant_common.utils import logger_utils
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class WsExchangeServer(WsServerBase):
|
|
15
|
+
"""
|
|
16
|
+
模拟加密货币平台 WS 服务器:
|
|
17
|
+
- 从 MySQL 按 exchange 读取资金费率历史数据
|
|
18
|
+
- 从指定 start_time 开始,按事件时间差和加速倍数推送
|
|
19
|
+
- 支持将推送事件的 event_time 重写为实时时间
|
|
20
|
+
- 主题:
|
|
21
|
+
* funding_rate.all
|
|
22
|
+
* funding_rate.<symbol>
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
def __init__(
|
|
26
|
+
self,
|
|
27
|
+
exchange: str,
|
|
28
|
+
mysql_host: str,
|
|
29
|
+
mysql_port: int,
|
|
30
|
+
mysql_user: str,
|
|
31
|
+
mysql_passwd: str,
|
|
32
|
+
mysql_db: str,
|
|
33
|
+
charset: str = "utf8mb4",
|
|
34
|
+
start_time: Union[int, float, datetime] = None,
|
|
35
|
+
speed_multiplier: float = 1.0,
|
|
36
|
+
use_realtime_event_time: bool = False,
|
|
37
|
+
inject_sample_on_empty: bool = True,
|
|
38
|
+
host: str = "0.0.0.0",
|
|
39
|
+
port: int = 8767,
|
|
40
|
+
):
|
|
41
|
+
super().__init__(self, host, port)
|
|
42
|
+
self._logger = logger_utils.get_logger(self)
|
|
43
|
+
# 平台
|
|
44
|
+
self._exchange = exchange
|
|
45
|
+
# 加速设置
|
|
46
|
+
self._speed_multiplier = max(speed_multiplier if speed_multiplier and speed_multiplier > 0 else 1.0, 1e-6)
|
|
47
|
+
# 是否使用实时时间
|
|
48
|
+
self._use_realtime_event_time = use_realtime_event_time
|
|
49
|
+
# 示例数据注入开关
|
|
50
|
+
self._inject_sample_on_empty = inject_sample_on_empty
|
|
51
|
+
|
|
52
|
+
# 单次增量加载的窗口大小
|
|
53
|
+
self._window_size: timedelta = timedelta(minutes=10)
|
|
54
|
+
# 触发预加载的阈值
|
|
55
|
+
self._preload_horizon: timedelta = timedelta(minutes=5)
|
|
56
|
+
# 丢弃数据的阈值
|
|
57
|
+
self._retain_horizon: timedelta = timedelta(minutes=10)
|
|
58
|
+
|
|
59
|
+
# 数据开始游标(时间) 支持毫秒时间戳或 datetime
|
|
60
|
+
if isinstance(start_time, (int, float)):
|
|
61
|
+
self._start_dt = datetime.fromtimestamp(start_time / 1000.0)
|
|
62
|
+
elif isinstance(start_time, datetime):
|
|
63
|
+
self._start_dt = start_time
|
|
64
|
+
else:
|
|
65
|
+
# 默认很早的时间,确保可以从最早记录开始
|
|
66
|
+
self._start_dt = datetime.fromtimestamp(0)
|
|
67
|
+
|
|
68
|
+
# DB
|
|
69
|
+
self._repo = KaqQuantMysqlRepository(mysql_host, mysql_port, mysql_user, mysql_passwd, mysql_db, charset)
|
|
70
|
+
|
|
71
|
+
# 推送控制
|
|
72
|
+
self._push_thread: Optional[threading.Thread] = None
|
|
73
|
+
self._stop_push = threading.Event()
|
|
74
|
+
|
|
75
|
+
# 预拉取首批数据,开始的时候加载多一点
|
|
76
|
+
initial_end = self._start_dt + self._preload_horizon * 2
|
|
77
|
+
self._events: List[Tuple[datetime, FundingRateEvent]] = self._load_events_window(self._start_dt, initial_end)
|
|
78
|
+
self._last_fetched_dt = initial_end
|
|
79
|
+
self._logger.info(f"资金费率事件已加载: {len(self._events)} 条")
|
|
80
|
+
# 添加测试数据
|
|
81
|
+
if not self._events and self._inject_sample_on_empty:
|
|
82
|
+
base_dt = self._start_dt if self._start_dt else datetime.fromtimestamp(0)
|
|
83
|
+
sample: List[Tuple[datetime, FundingRateEvent]] = []
|
|
84
|
+
for i in range(5):
|
|
85
|
+
dt = base_dt if i == 0 else base_dt + pd.Timedelta(minutes=i)
|
|
86
|
+
evt = FundingRateEvent(
|
|
87
|
+
event_time=int(dt.timestamp() * 1000),
|
|
88
|
+
exchange_symbol=f"{self._exchange}:BTCUSDT",
|
|
89
|
+
exchange=self._exchange,
|
|
90
|
+
symbol="BTCUSDT",
|
|
91
|
+
open_rate=0.0001,
|
|
92
|
+
close_rate=0.0001,
|
|
93
|
+
high_rate=0.0002,
|
|
94
|
+
low_rate=0.0000,
|
|
95
|
+
)
|
|
96
|
+
sample.append((dt, evt))
|
|
97
|
+
self._events = sample
|
|
98
|
+
self._logger.warning("资金费率历史为空,已注入示例数据用于推送测试")
|
|
99
|
+
|
|
100
|
+
# 把开始时间调整为第一条数据的时间
|
|
101
|
+
if self._events:
|
|
102
|
+
self._start_dt = self._events[0][0]
|
|
103
|
+
# 最后对齐整分钟
|
|
104
|
+
self._start_dt = self._start_dt.replace(second=0, microsecond=0)
|
|
105
|
+
|
|
106
|
+
# 实际启动时间,用于计算虚拟时间,需要对齐整分钟
|
|
107
|
+
self._real_start_dt: datetime = datetime.now()
|
|
108
|
+
self._real_start_dt = self._real_start_dt.replace(second=0, microsecond=0)
|
|
109
|
+
|
|
110
|
+
# 原始全量拉取(保留以兼容旧逻辑)
|
|
111
|
+
def _load_events(self) -> List[Tuple[datetime, FundingRateEvent]]:
|
|
112
|
+
try:
|
|
113
|
+
query = (
|
|
114
|
+
"SELECT event_time, exchange_symbol, exchange, symbol, open_rate, close_rate, high_rate, low_rate, close_time, next_event_time, id, ctimestamp "
|
|
115
|
+
"FROM kaq_future_fr_klines_1m "
|
|
116
|
+
f"WHERE exchange = '{self._exchange}' AND event_time >= '{self._start_dt.strftime('%Y-%m-%d %H:%M:%S')}' "
|
|
117
|
+
"ORDER BY event_time ASC"
|
|
118
|
+
)
|
|
119
|
+
df: pd.DataFrame = self._repo.fetch_data(query)
|
|
120
|
+
if df is None or df.empty:
|
|
121
|
+
self._logger.warning("资金费率表无数据或查询为空")
|
|
122
|
+
return []
|
|
123
|
+
|
|
124
|
+
events: List[Tuple[datetime, FundingRateEvent]] = []
|
|
125
|
+
for _, row in df.iterrows():
|
|
126
|
+
evt_dt: datetime = row.get("event_time")
|
|
127
|
+
if not isinstance(evt_dt, datetime):
|
|
128
|
+
try:
|
|
129
|
+
evt_dt = pd.to_datetime(evt_dt).to_pydatetime()
|
|
130
|
+
except Exception:
|
|
131
|
+
continue
|
|
132
|
+
# 转为毫秒时间戳
|
|
133
|
+
close_time_val = row.get("close_time")
|
|
134
|
+
next_event_time_val = row.get("next_event_time")
|
|
135
|
+
try:
|
|
136
|
+
close_time_ms = int(pd.to_datetime(close_time_val).timestamp() * 1000) if close_time_val is not None else None
|
|
137
|
+
except Exception:
|
|
138
|
+
close_time_ms = None
|
|
139
|
+
try:
|
|
140
|
+
next_event_time_ms = int(pd.to_datetime(next_event_time_val).timestamp() * 1000) if next_event_time_val is not None else None
|
|
141
|
+
except Exception:
|
|
142
|
+
next_event_time_ms = None
|
|
143
|
+
evt_ms = int(evt_dt.timestamp() * 1000)
|
|
144
|
+
event = FundingRateEvent(
|
|
145
|
+
event_time=evt_ms,
|
|
146
|
+
exchange_symbol=row.get("exchange_symbol"),
|
|
147
|
+
exchange=row.get("exchange"),
|
|
148
|
+
symbol=row.get("symbol"),
|
|
149
|
+
open_rate=row.get("open_rate"),
|
|
150
|
+
close_rate=row.get("close_rate"),
|
|
151
|
+
high_rate=row.get("high_rate"),
|
|
152
|
+
low_rate=row.get("low_rate"),
|
|
153
|
+
close_time=close_time_ms,
|
|
154
|
+
next_event_time=next_event_time_ms,
|
|
155
|
+
#
|
|
156
|
+
id=row.get("id"),
|
|
157
|
+
ctimestamp=row.get("ctimestamp"),
|
|
158
|
+
)
|
|
159
|
+
events.append((evt_dt, event))
|
|
160
|
+
return events
|
|
161
|
+
except Exception as e:
|
|
162
|
+
self._logger.error(f"加载资金费率事件失败: {e}")
|
|
163
|
+
return []
|
|
164
|
+
|
|
165
|
+
# 增量批量拉取(保留以兼容旧逻辑)
|
|
166
|
+
def _load_events_batch(self, start_from: datetime, limit: int) -> List[Tuple[datetime, FundingRateEvent]]:
|
|
167
|
+
try:
|
|
168
|
+
query = (
|
|
169
|
+
"SELECT event_time, exchange_symbol, exchange, symbol, open_rate, close_rate, high_rate, low_rate, close_time, next_event_time, id, ctimestamp "
|
|
170
|
+
"FROM kaq_future_fr_klines_1m "
|
|
171
|
+
f"WHERE exchange = '{self._exchange}' AND event_time > '{start_from.strftime('%Y-%m-%d %H:%M:%S')}' "
|
|
172
|
+
"ORDER BY event_time ASC "
|
|
173
|
+
f"LIMIT {int(limit)}"
|
|
174
|
+
)
|
|
175
|
+
df: pd.DataFrame = self._repo.fetch_data(query)
|
|
176
|
+
if df is None or df.empty:
|
|
177
|
+
return []
|
|
178
|
+
events: List[Tuple[datetime, FundingRateEvent]] = []
|
|
179
|
+
for _, row in df.iterrows():
|
|
180
|
+
evt_dt: datetime = row.get("event_time")
|
|
181
|
+
if not isinstance(evt_dt, datetime):
|
|
182
|
+
try:
|
|
183
|
+
evt_dt = pd.to_datetime(evt_dt).to_pydatetime()
|
|
184
|
+
except Exception:
|
|
185
|
+
continue
|
|
186
|
+
# 转为毫秒时间戳
|
|
187
|
+
close_time_val = row.get("close_time")
|
|
188
|
+
next_event_time_val = row.get("next_event_time")
|
|
189
|
+
try:
|
|
190
|
+
close_time_ms = int(pd.to_datetime(close_time_val).timestamp() * 1000) if close_time_val is not None else None
|
|
191
|
+
except Exception:
|
|
192
|
+
close_time_ms = None
|
|
193
|
+
try:
|
|
194
|
+
next_event_time_ms = int(pd.to_datetime(next_event_time_val).timestamp() * 1000) if next_event_time_val is not None else None
|
|
195
|
+
except Exception:
|
|
196
|
+
next_event_time_ms = None
|
|
197
|
+
evt_ms = int(evt_dt.timestamp() * 1000)
|
|
198
|
+
event = FundingRateEvent(
|
|
199
|
+
event_time=evt_ms,
|
|
200
|
+
exchange_symbol=row.get("exchange_symbol"),
|
|
201
|
+
exchange=row.get("exchange"),
|
|
202
|
+
symbol=row.get("symbol"),
|
|
203
|
+
open_rate=row.get("open_rate"),
|
|
204
|
+
close_rate=row.get("close_rate"),
|
|
205
|
+
high_rate=row.get("high_rate"),
|
|
206
|
+
low_rate=row.get("low_rate"),
|
|
207
|
+
close_time=close_time_ms,
|
|
208
|
+
next_event_time=next_event_time_ms,
|
|
209
|
+
#
|
|
210
|
+
id=row.get("id"),
|
|
211
|
+
ctimestamp=row.get("ctimestamp"),
|
|
212
|
+
)
|
|
213
|
+
events.append((evt_dt, event))
|
|
214
|
+
return events
|
|
215
|
+
except Exception as e:
|
|
216
|
+
self._logger.error(f"增量加载资金费率事件失败: {e}")
|
|
217
|
+
return []
|
|
218
|
+
|
|
219
|
+
# 按照时间窗口拉取
|
|
220
|
+
def _load_events_window(self, start_from: datetime, end_until: datetime) -> List[Tuple[datetime, FundingRateEvent]]:
|
|
221
|
+
try:
|
|
222
|
+
query = (
|
|
223
|
+
"SELECT event_time, exchange_symbol, exchange, symbol, open_rate, close_rate, high_rate, low_rate, close_time, next_event_time, id, ctimestamp "
|
|
224
|
+
"FROM kaq_future_fr_klines_1m "
|
|
225
|
+
f"WHERE exchange = '{self._exchange}' AND event_time > '{start_from.strftime('%Y-%m-%d %H:%M:%S')}' AND event_time <= '{end_until.strftime('%Y-%m-%d %H:%M:%S')}' "
|
|
226
|
+
"ORDER BY event_time ASC"
|
|
227
|
+
)
|
|
228
|
+
df: pd.DataFrame = self._repo.fetch_data(query)
|
|
229
|
+
if df is None or df.empty:
|
|
230
|
+
return []
|
|
231
|
+
events: List[Tuple[datetime, FundingRateEvent]] = []
|
|
232
|
+
for _, row in df.iterrows():
|
|
233
|
+
evt_dt: datetime = row.get("event_time")
|
|
234
|
+
if not isinstance(evt_dt, datetime):
|
|
235
|
+
try:
|
|
236
|
+
evt_dt = pd.to_datetime(evt_dt).to_pydatetime()
|
|
237
|
+
except Exception:
|
|
238
|
+
continue
|
|
239
|
+
close_time_val = row.get("close_time")
|
|
240
|
+
next_event_time_val = row.get("next_event_time")
|
|
241
|
+
try:
|
|
242
|
+
close_time_ms = int(pd.to_datetime(close_time_val).timestamp() * 1000) if close_time_val is not None else None
|
|
243
|
+
except Exception:
|
|
244
|
+
close_time_ms = None
|
|
245
|
+
try:
|
|
246
|
+
next_event_time_ms = int(pd.to_datetime(next_event_time_val).timestamp() * 1000) if next_event_time_val is not None else None
|
|
247
|
+
except Exception:
|
|
248
|
+
next_event_time_ms = None
|
|
249
|
+
evt_ms = int(evt_dt.timestamp() * 1000)
|
|
250
|
+
event = FundingRateEvent(
|
|
251
|
+
event_time=evt_ms,
|
|
252
|
+
exchange_symbol=row.get("exchange_symbol"),
|
|
253
|
+
exchange=row.get("exchange"),
|
|
254
|
+
symbol=row.get("symbol"),
|
|
255
|
+
open_rate=row.get("open_rate"),
|
|
256
|
+
close_rate=row.get("close_rate"),
|
|
257
|
+
high_rate=row.get("high_rate"),
|
|
258
|
+
low_rate=row.get("low_rate"),
|
|
259
|
+
close_time=close_time_ms,
|
|
260
|
+
next_event_time=next_event_time_ms,
|
|
261
|
+
#
|
|
262
|
+
id=row.get("id"),
|
|
263
|
+
ctimestamp=row.get("ctimestamp"),
|
|
264
|
+
)
|
|
265
|
+
events.append((evt_dt, event))
|
|
266
|
+
return events
|
|
267
|
+
except Exception as e:
|
|
268
|
+
self._logger.error(f"时间窗加载资金费率事件失败: {e}")
|
|
269
|
+
return []
|
|
270
|
+
|
|
271
|
+
# 从数据库拉取最新的记录
|
|
272
|
+
def _get_latest_event_dt(self) -> Optional[datetime]:
|
|
273
|
+
try:
|
|
274
|
+
query = "SELECT MAX(event_time) AS max_event_time " "FROM kaq_future_fr_klines_1m " f"WHERE exchange = '{self._exchange}'"
|
|
275
|
+
df: pd.DataFrame = self._repo.fetch_data(query)
|
|
276
|
+
if df is None or df.empty:
|
|
277
|
+
return None
|
|
278
|
+
val = df.iloc[0].get("max_event_time")
|
|
279
|
+
|
|
280
|
+
if isinstance(val, datetime):
|
|
281
|
+
return val
|
|
282
|
+
try:
|
|
283
|
+
return pd.to_datetime(val).to_pydatetime()
|
|
284
|
+
except Exception:
|
|
285
|
+
return None
|
|
286
|
+
except Exception as e:
|
|
287
|
+
self._logger.error(f"查询最新资金费率事件时间失败: {e}")
|
|
288
|
+
return None
|
|
289
|
+
|
|
290
|
+
def _push_loop(self):
|
|
291
|
+
# 初始缓冲检查
|
|
292
|
+
if not self._events:
|
|
293
|
+
# 获取一次最新记录
|
|
294
|
+
latest_dt = self._get_latest_event_dt()
|
|
295
|
+
initial_end = self._last_fetched_dt + self._window_size
|
|
296
|
+
# 防止超过
|
|
297
|
+
if latest_dt is not None and initial_end > latest_dt:
|
|
298
|
+
initial_end = latest_dt
|
|
299
|
+
self._events = self._load_events_window(self._last_fetched_dt, initial_end)
|
|
300
|
+
# 主要是为了记录这个最新拉取的时间点,下一次增量加载从这里开始
|
|
301
|
+
self._last_fetched_dt = initial_end
|
|
302
|
+
if not self._events:
|
|
303
|
+
self._logger.warning("无可推送的资金费率历史事件")
|
|
304
|
+
return
|
|
305
|
+
|
|
306
|
+
prev_dt = None
|
|
307
|
+
prev_counter = 0
|
|
308
|
+
discard_dt = None
|
|
309
|
+
idx = 0
|
|
310
|
+
while not self._stop_push.is_set():
|
|
311
|
+
# 虚拟时钟,考虑加速比
|
|
312
|
+
now_real = datetime.now()
|
|
313
|
+
# 计算出虚拟时间点
|
|
314
|
+
now_virtual = self._start_dt + (now_real - self._real_start_dt) * self._speed_multiplier
|
|
315
|
+
|
|
316
|
+
# 确保缓冲区覆盖到虚拟时间的预加载阈值
|
|
317
|
+
# 获取缓冲区最新的记录时间
|
|
318
|
+
buffer_max_dt = self._events[-1][0] if self._events else self._last_fetched_dt
|
|
319
|
+
# 根据虚拟时间点计算出需要覆盖的时间点
|
|
320
|
+
target_cover_dt = now_virtual + self._preload_horizon
|
|
321
|
+
# 最新的记录时间
|
|
322
|
+
latest_dt: datetime = None
|
|
323
|
+
while buffer_max_dt < target_cover_dt and not self._stop_push.is_set():
|
|
324
|
+
# 每次进入循环,先拉取一次最新数据时间
|
|
325
|
+
if latest_dt is None:
|
|
326
|
+
latest_dt = self._get_latest_event_dt()
|
|
327
|
+
# 若数据库已无更晚数据,停止增量加载
|
|
328
|
+
if latest_dt is not None and self._last_fetched_dt >= latest_dt:
|
|
329
|
+
buffer_max_dt = latest_dt
|
|
330
|
+
self._logger.debug("已到达数据库最新事件时间,停止增量加载")
|
|
331
|
+
break
|
|
332
|
+
# 计算出增量加载的窗口时间点
|
|
333
|
+
window_end = self._last_fetched_dt + self._window_size
|
|
334
|
+
# 限制窗口终点不超过数据库最新时间
|
|
335
|
+
if latest_dt is not None and window_end > latest_dt:
|
|
336
|
+
window_end = latest_dt
|
|
337
|
+
# 增量加载
|
|
338
|
+
new_events = self._load_events_window(self._last_fetched_dt, window_end)
|
|
339
|
+
self._last_fetched_dt = window_end
|
|
340
|
+
if new_events:
|
|
341
|
+
self._logger.debug(f"增量获取{len(new_events)}条数据,截止时间{window_end}")
|
|
342
|
+
self._events.extend(new_events)
|
|
343
|
+
buffer_max_dt = self._events[-1][0]
|
|
344
|
+
else:
|
|
345
|
+
# 即便当前窗无数据,推进游标,避免卡住
|
|
346
|
+
buffer_max_dt = max(buffer_max_dt, window_end)
|
|
347
|
+
self._logger.debug(f"增量获取{0}条数据,截止时间{window_end}")
|
|
348
|
+
latest_dt = None
|
|
349
|
+
|
|
350
|
+
# 如果缓冲区已消费完,退出(或等待下一轮覆盖)
|
|
351
|
+
if idx >= len(self._events):
|
|
352
|
+
time.sleep(0.05)
|
|
353
|
+
continue
|
|
354
|
+
|
|
355
|
+
# 取出一条数据
|
|
356
|
+
evt_dt, evt = self._events[idx]
|
|
357
|
+
|
|
358
|
+
# 计算等待时间,按加速比缩放
|
|
359
|
+
try:
|
|
360
|
+
# 数据时间 - 当前虚拟时间
|
|
361
|
+
delta = (evt_dt - now_virtual).total_seconds()
|
|
362
|
+
# 等待秒数
|
|
363
|
+
wait_s = max(delta, 0.0)
|
|
364
|
+
if wait_s > 0:
|
|
365
|
+
time.sleep(0.05)
|
|
366
|
+
continue
|
|
367
|
+
|
|
368
|
+
if prev_dt != evt_dt:
|
|
369
|
+
# 输出一下时间
|
|
370
|
+
if prev_dt is not None:
|
|
371
|
+
self._logger.debug(f"完成处理{prev_dt}的资金费率{prev_counter}条")
|
|
372
|
+
self._logger.debug(f"开始处理{evt_dt}的资金费率")
|
|
373
|
+
prev_counter = 0
|
|
374
|
+
prev_dt = evt_dt
|
|
375
|
+
#
|
|
376
|
+
if discard_dt is None:
|
|
377
|
+
discard_dt = prev_dt
|
|
378
|
+
except Exception:
|
|
379
|
+
pass
|
|
380
|
+
|
|
381
|
+
#
|
|
382
|
+
idx += 1
|
|
383
|
+
prev_counter += 1
|
|
384
|
+
|
|
385
|
+
# 构造推送数据
|
|
386
|
+
payload = evt.model_dump()
|
|
387
|
+
if self._use_realtime_event_time:
|
|
388
|
+
# 调整为实时时间
|
|
389
|
+
payload["event_time"] = int(time.time() * 1000)
|
|
390
|
+
# 广播到主题:all 和 symbol
|
|
391
|
+
try:
|
|
392
|
+
self.broadcast("funding_rate.all", payload)
|
|
393
|
+
sym = payload.get("symbol") or ""
|
|
394
|
+
if sym:
|
|
395
|
+
self.broadcast(f"funding_rate.{sym}", payload)
|
|
396
|
+
except Exception as e:
|
|
397
|
+
self._logger.warning(f"广播资金费率事件失败: {e}")
|
|
398
|
+
|
|
399
|
+
# 时间型丢弃:移除过旧数据以控制内存
|
|
400
|
+
if prev_dt - discard_dt > self._retain_horizon:
|
|
401
|
+
discard_dt = prev_dt
|
|
402
|
+
cut_idx = -1
|
|
403
|
+
for i in range(len(self._events)):
|
|
404
|
+
if self._events[i][0] < discard_dt:
|
|
405
|
+
cut_idx = i
|
|
406
|
+
elif self._events[i][0] >= discard_dt:
|
|
407
|
+
break
|
|
408
|
+
# 裁剪
|
|
409
|
+
if cut_idx > 0:
|
|
410
|
+
cut_count = cut_idx + 1
|
|
411
|
+
self._logger.debug(f"丢弃{cut_count}条过期数据")
|
|
412
|
+
self._events = self._events[cut_count:]
|
|
413
|
+
# 修正idx
|
|
414
|
+
idx = max(0, idx - cut_count)
|
|
415
|
+
|
|
416
|
+
def run_with_thread(self, block: bool = True):
|
|
417
|
+
# 启动推送线程
|
|
418
|
+
self._stop_push.clear()
|
|
419
|
+
self._push_thread = threading.Thread(target=self._push_loop, name="WsExchangePush", daemon=True)
|
|
420
|
+
self._push_thread.start()
|
|
421
|
+
# 启动WS服务器线程
|
|
422
|
+
super().run_with_thread(block)
|
|
423
|
+
|
|
424
|
+
def shutdown_with_thread(self):
|
|
425
|
+
# 先停止推送线程
|
|
426
|
+
try:
|
|
427
|
+
self._stop_push.set()
|
|
428
|
+
if self._push_thread and self._push_thread.is_alive():
|
|
429
|
+
self._push_thread.join(timeout=3)
|
|
430
|
+
except Exception:
|
|
431
|
+
pass
|
|
432
|
+
# 再关闭WS服务器
|
|
433
|
+
super().shutdown_with_thread()
|
|
434
|
+
|
|
435
|
+
async def _ws_handler(self, ws):
|
|
436
|
+
# 覆盖基类鉴权,模拟平台测试环境不启用鉴权
|
|
437
|
+
try:
|
|
438
|
+
await self._handle_connection(ws)
|
|
439
|
+
except Exception:
|
|
440
|
+
pass
|