kaq-quant-common 0.2.16__py3-none-any.whl → 0.2.18__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (25) hide show
  1. kaq_quant_common/api/common/__init__.py +1 -1
  2. kaq_quant_common/api/common/api_interface.py +38 -38
  3. kaq_quant_common/api/rest/api_client_base.py +187 -187
  4. kaq_quant_common/api/rest/instruction/helper/commission_helper.py +141 -141
  5. kaq_quant_common/api/rest/instruction/helper/mock_order_helper.py +346 -346
  6. kaq_quant_common/api/rest/instruction/helper/order_helper.py +362 -362
  7. kaq_quant_common/api/rest/instruction/instruction_client.py +40 -2
  8. kaq_quant_common/api/rest/instruction/instruction_server_base.py +62 -53
  9. kaq_quant_common/api/rest/instruction/models/__init__.py +17 -17
  10. kaq_quant_common/api/rest/instruction/models/kline.py +60 -0
  11. kaq_quant_common/api/rest/instruction/models/transfer.py +32 -32
  12. kaq_quant_common/api/ws/exchange/models.py +23 -23
  13. kaq_quant_common/api/ws/exchange/ws_exchange_server.py +440 -440
  14. kaq_quant_common/common/ddb_table_monitor.py +106 -106
  15. kaq_quant_common/common/http_monitor.py +69 -69
  16. kaq_quant_common/common/modules/limit_order_helper.py +12 -0
  17. kaq_quant_common/common/monitor_base.py +84 -84
  18. kaq_quant_common/common/monitor_group.py +97 -97
  19. kaq_quant_common/common/ws_wrapper.py +21 -21
  20. kaq_quant_common/utils/logger_utils.py +4 -4
  21. kaq_quant_common/utils/signal_utils.py +23 -23
  22. kaq_quant_common/utils/uuid_utils.py +5 -5
  23. {kaq_quant_common-0.2.16.dist-info → kaq_quant_common-0.2.18.dist-info}/METADATA +1 -1
  24. {kaq_quant_common-0.2.16.dist-info → kaq_quant_common-0.2.18.dist-info}/RECORD +25 -24
  25. {kaq_quant_common-0.2.16.dist-info → kaq_quant_common-0.2.18.dist-info}/WHEEL +1 -1
@@ -1,440 +1,440 @@
1
- import threading
2
- import time
3
- from datetime import datetime, timedelta
4
- from typing import List, Optional, Tuple, Union
5
-
6
- import pandas as pd
7
-
8
- from kaq_quant_common.api.ws.exchange.models import FundingRateEvent
9
- from kaq_quant_common.api.ws.ws_server_base import WsServerBase
10
- from kaq_quant_common.resources.kaq_mysql_resources import KaqQuantMysqlRepository
11
- from kaq_quant_common.utils import logger_utils
12
-
13
-
14
- class WsExchangeServer(WsServerBase):
15
- """
16
- 模拟加密货币平台 WS 服务器:
17
- - 从 MySQL 按 exchange 读取资金费率历史数据
18
- - 从指定 start_time 开始,按事件时间差和加速倍数推送
19
- - 支持将推送事件的 event_time 重写为实时时间
20
- - 主题:
21
- * funding_rate.all
22
- * funding_rate.<symbol>
23
- """
24
-
25
- def __init__(
26
- self,
27
- exchange: str,
28
- mysql_host: str,
29
- mysql_port: int,
30
- mysql_user: str,
31
- mysql_passwd: str,
32
- mysql_db: str,
33
- charset: str = "utf8mb4",
34
- start_time: Union[int, float, datetime] = None,
35
- speed_multiplier: float = 1.0,
36
- use_realtime_event_time: bool = False,
37
- inject_sample_on_empty: bool = True,
38
- host: str = "0.0.0.0",
39
- port: int = 8767,
40
- ):
41
- super().__init__(self, host, port)
42
- self._logger = logger_utils.get_logger(self)
43
- # 平台
44
- self._exchange = exchange
45
- # 加速设置
46
- self._speed_multiplier = max(speed_multiplier if speed_multiplier and speed_multiplier > 0 else 1.0, 1e-6)
47
- # 是否使用实时时间
48
- self._use_realtime_event_time = use_realtime_event_time
49
- # 示例数据注入开关
50
- self._inject_sample_on_empty = inject_sample_on_empty
51
-
52
- # 单次增量加载的窗口大小
53
- self._window_size: timedelta = timedelta(minutes=10)
54
- # 触发预加载的阈值
55
- self._preload_horizon: timedelta = timedelta(minutes=5)
56
- # 丢弃数据的阈值
57
- self._retain_horizon: timedelta = timedelta(minutes=10)
58
-
59
- # 数据开始游标(时间) 支持毫秒时间戳或 datetime
60
- if isinstance(start_time, (int, float)):
61
- self._start_dt = datetime.fromtimestamp(start_time / 1000.0)
62
- elif isinstance(start_time, datetime):
63
- self._start_dt = start_time
64
- else:
65
- # 默认很早的时间,确保可以从最早记录开始
66
- self._start_dt = datetime.fromtimestamp(0)
67
-
68
- # DB
69
- self._repo = KaqQuantMysqlRepository(mysql_host, mysql_port, mysql_user, mysql_passwd, mysql_db, charset)
70
-
71
- # 推送控制
72
- self._push_thread: Optional[threading.Thread] = None
73
- self._stop_push = threading.Event()
74
-
75
- # 预拉取首批数据,开始的时候加载多一点
76
- initial_end = self._start_dt + self._preload_horizon * 2
77
- self._events: List[Tuple[datetime, FundingRateEvent]] = self._load_events_window(self._start_dt, initial_end)
78
- self._last_fetched_dt = initial_end
79
- self._logger.info(f"资金费率事件已加载: {len(self._events)} 条")
80
- # 添加测试数据
81
- if not self._events and self._inject_sample_on_empty:
82
- base_dt = self._start_dt if self._start_dt else datetime.fromtimestamp(0)
83
- sample: List[Tuple[datetime, FundingRateEvent]] = []
84
- for i in range(5):
85
- dt = base_dt if i == 0 else base_dt + pd.Timedelta(minutes=i)
86
- evt = FundingRateEvent(
87
- event_time=int(dt.timestamp() * 1000),
88
- exchange_symbol=f"{self._exchange}:BTCUSDT",
89
- exchange=self._exchange,
90
- symbol="BTCUSDT",
91
- open_rate=0.0001,
92
- close_rate=0.0001,
93
- high_rate=0.0002,
94
- low_rate=0.0000,
95
- )
96
- sample.append((dt, evt))
97
- self._events = sample
98
- self._logger.warning("资金费率历史为空,已注入示例数据用于推送测试")
99
-
100
- # 把开始时间调整为第一条数据的时间
101
- if self._events:
102
- self._start_dt = self._events[0][0]
103
- # 最后对齐整分钟
104
- self._start_dt = self._start_dt.replace(second=0, microsecond=0)
105
-
106
- # 实际启动时间,用于计算虚拟时间,需要对齐整分钟
107
- self._real_start_dt: datetime = datetime.now()
108
- self._real_start_dt = self._real_start_dt.replace(second=0, microsecond=0)
109
-
110
- # 原始全量拉取(保留以兼容旧逻辑)
111
- def _load_events(self) -> List[Tuple[datetime, FundingRateEvent]]:
112
- try:
113
- query = (
114
- "SELECT event_time, exchange_symbol, exchange, symbol, open_rate, close_rate, high_rate, low_rate, close_time, next_event_time, id, ctimestamp "
115
- "FROM kaq_future_fr_klines_1m "
116
- f"WHERE exchange = '{self._exchange}' AND event_time >= '{self._start_dt.strftime('%Y-%m-%d %H:%M:%S')}' "
117
- "ORDER BY event_time ASC"
118
- )
119
- df: pd.DataFrame = self._repo.fetch_data(query)
120
- if df is None or df.empty:
121
- self._logger.warning("资金费率表无数据或查询为空")
122
- return []
123
-
124
- events: List[Tuple[datetime, FundingRateEvent]] = []
125
- for _, row in df.iterrows():
126
- evt_dt: datetime = row.get("event_time")
127
- if not isinstance(evt_dt, datetime):
128
- try:
129
- evt_dt = pd.to_datetime(evt_dt).to_pydatetime()
130
- except Exception:
131
- continue
132
- # 转为毫秒时间戳
133
- close_time_val = row.get("close_time")
134
- next_event_time_val = row.get("next_event_time")
135
- try:
136
- close_time_ms = int(pd.to_datetime(close_time_val).timestamp() * 1000) if close_time_val is not None else None
137
- except Exception:
138
- close_time_ms = None
139
- try:
140
- next_event_time_ms = int(pd.to_datetime(next_event_time_val).timestamp() * 1000) if next_event_time_val is not None else None
141
- except Exception:
142
- next_event_time_ms = None
143
- evt_ms = int(evt_dt.timestamp() * 1000)
144
- event = FundingRateEvent(
145
- event_time=evt_ms,
146
- exchange_symbol=row.get("exchange_symbol"),
147
- exchange=row.get("exchange"),
148
- symbol=row.get("symbol"),
149
- open_rate=row.get("open_rate"),
150
- close_rate=row.get("close_rate"),
151
- high_rate=row.get("high_rate"),
152
- low_rate=row.get("low_rate"),
153
- close_time=close_time_ms,
154
- next_event_time=next_event_time_ms,
155
- #
156
- id=row.get("id"),
157
- ctimestamp=row.get("ctimestamp"),
158
- )
159
- events.append((evt_dt, event))
160
- return events
161
- except Exception as e:
162
- self._logger.error(f"加载资金费率事件失败: {e}")
163
- return []
164
-
165
- # 增量批量拉取(保留以兼容旧逻辑)
166
- def _load_events_batch(self, start_from: datetime, limit: int) -> List[Tuple[datetime, FundingRateEvent]]:
167
- try:
168
- query = (
169
- "SELECT event_time, exchange_symbol, exchange, symbol, open_rate, close_rate, high_rate, low_rate, close_time, next_event_time, id, ctimestamp "
170
- "FROM kaq_future_fr_klines_1m "
171
- f"WHERE exchange = '{self._exchange}' AND event_time > '{start_from.strftime('%Y-%m-%d %H:%M:%S')}' "
172
- "ORDER BY event_time ASC "
173
- f"LIMIT {int(limit)}"
174
- )
175
- df: pd.DataFrame = self._repo.fetch_data(query)
176
- if df is None or df.empty:
177
- return []
178
- events: List[Tuple[datetime, FundingRateEvent]] = []
179
- for _, row in df.iterrows():
180
- evt_dt: datetime = row.get("event_time")
181
- if not isinstance(evt_dt, datetime):
182
- try:
183
- evt_dt = pd.to_datetime(evt_dt).to_pydatetime()
184
- except Exception:
185
- continue
186
- # 转为毫秒时间戳
187
- close_time_val = row.get("close_time")
188
- next_event_time_val = row.get("next_event_time")
189
- try:
190
- close_time_ms = int(pd.to_datetime(close_time_val).timestamp() * 1000) if close_time_val is not None else None
191
- except Exception:
192
- close_time_ms = None
193
- try:
194
- next_event_time_ms = int(pd.to_datetime(next_event_time_val).timestamp() * 1000) if next_event_time_val is not None else None
195
- except Exception:
196
- next_event_time_ms = None
197
- evt_ms = int(evt_dt.timestamp() * 1000)
198
- event = FundingRateEvent(
199
- event_time=evt_ms,
200
- exchange_symbol=row.get("exchange_symbol"),
201
- exchange=row.get("exchange"),
202
- symbol=row.get("symbol"),
203
- open_rate=row.get("open_rate"),
204
- close_rate=row.get("close_rate"),
205
- high_rate=row.get("high_rate"),
206
- low_rate=row.get("low_rate"),
207
- close_time=close_time_ms,
208
- next_event_time=next_event_time_ms,
209
- #
210
- id=row.get("id"),
211
- ctimestamp=row.get("ctimestamp"),
212
- )
213
- events.append((evt_dt, event))
214
- return events
215
- except Exception as e:
216
- self._logger.error(f"增量加载资金费率事件失败: {e}")
217
- return []
218
-
219
- # 按照时间窗口拉取
220
- def _load_events_window(self, start_from: datetime, end_until: datetime) -> List[Tuple[datetime, FundingRateEvent]]:
221
- try:
222
- query = (
223
- "SELECT event_time, exchange_symbol, exchange, symbol, open_rate, close_rate, high_rate, low_rate, close_time, next_event_time, id, ctimestamp "
224
- "FROM kaq_future_fr_klines_1m "
225
- f"WHERE exchange = '{self._exchange}' AND event_time > '{start_from.strftime('%Y-%m-%d %H:%M:%S')}' AND event_time <= '{end_until.strftime('%Y-%m-%d %H:%M:%S')}' "
226
- "ORDER BY event_time ASC"
227
- )
228
- df: pd.DataFrame = self._repo.fetch_data(query)
229
- if df is None or df.empty:
230
- return []
231
- events: List[Tuple[datetime, FundingRateEvent]] = []
232
- for _, row in df.iterrows():
233
- evt_dt: datetime = row.get("event_time")
234
- if not isinstance(evt_dt, datetime):
235
- try:
236
- evt_dt = pd.to_datetime(evt_dt).to_pydatetime()
237
- except Exception:
238
- continue
239
- close_time_val = row.get("close_time")
240
- next_event_time_val = row.get("next_event_time")
241
- try:
242
- close_time_ms = int(pd.to_datetime(close_time_val).timestamp() * 1000) if close_time_val is not None else None
243
- except Exception:
244
- close_time_ms = None
245
- try:
246
- next_event_time_ms = int(pd.to_datetime(next_event_time_val).timestamp() * 1000) if next_event_time_val is not None else None
247
- except Exception:
248
- next_event_time_ms = None
249
- evt_ms = int(evt_dt.timestamp() * 1000)
250
- event = FundingRateEvent(
251
- event_time=evt_ms,
252
- exchange_symbol=row.get("exchange_symbol"),
253
- exchange=row.get("exchange"),
254
- symbol=row.get("symbol"),
255
- open_rate=row.get("open_rate"),
256
- close_rate=row.get("close_rate"),
257
- high_rate=row.get("high_rate"),
258
- low_rate=row.get("low_rate"),
259
- close_time=close_time_ms,
260
- next_event_time=next_event_time_ms,
261
- #
262
- id=row.get("id"),
263
- ctimestamp=row.get("ctimestamp"),
264
- )
265
- events.append((evt_dt, event))
266
- return events
267
- except Exception as e:
268
- self._logger.error(f"时间窗加载资金费率事件失败: {e}")
269
- return []
270
-
271
- # 从数据库拉取最新的记录
272
- def _get_latest_event_dt(self) -> Optional[datetime]:
273
- try:
274
- query = "SELECT MAX(event_time) AS max_event_time " "FROM kaq_future_fr_klines_1m " f"WHERE exchange = '{self._exchange}'"
275
- df: pd.DataFrame = self._repo.fetch_data(query)
276
- if df is None or df.empty:
277
- return None
278
- val = df.iloc[0].get("max_event_time")
279
-
280
- if isinstance(val, datetime):
281
- return val
282
- try:
283
- return pd.to_datetime(val).to_pydatetime()
284
- except Exception:
285
- return None
286
- except Exception as e:
287
- self._logger.error(f"查询最新资金费率事件时间失败: {e}")
288
- return None
289
-
290
- def _push_loop(self):
291
- # 初始缓冲检查
292
- if not self._events:
293
- # 获取一次最新记录
294
- latest_dt = self._get_latest_event_dt()
295
- initial_end = self._last_fetched_dt + self._window_size
296
- # 防止超过
297
- if latest_dt is not None and initial_end > latest_dt:
298
- initial_end = latest_dt
299
- self._events = self._load_events_window(self._last_fetched_dt, initial_end)
300
- # 主要是为了记录这个最新拉取的时间点,下一次增量加载从这里开始
301
- self._last_fetched_dt = initial_end
302
- if not self._events:
303
- self._logger.warning("无可推送的资金费率历史事件")
304
- return
305
-
306
- prev_dt = None
307
- prev_counter = 0
308
- discard_dt = None
309
- idx = 0
310
- while not self._stop_push.is_set():
311
- # 虚拟时钟,考虑加速比
312
- now_real = datetime.now()
313
- # 计算出虚拟时间点
314
- now_virtual = self._start_dt + (now_real - self._real_start_dt) * self._speed_multiplier
315
-
316
- # 确保缓冲区覆盖到虚拟时间的预加载阈值
317
- # 获取缓冲区最新的记录时间
318
- buffer_max_dt = self._events[-1][0] if self._events else self._last_fetched_dt
319
- # 根据虚拟时间点计算出需要覆盖的时间点
320
- target_cover_dt = now_virtual + self._preload_horizon
321
- # 最新的记录时间
322
- latest_dt: datetime = None
323
- while buffer_max_dt < target_cover_dt and not self._stop_push.is_set():
324
- # 每次进入循环,先拉取一次最新数据时间
325
- if latest_dt is None:
326
- latest_dt = self._get_latest_event_dt()
327
- # 若数据库已无更晚数据,停止增量加载
328
- if latest_dt is not None and self._last_fetched_dt >= latest_dt:
329
- buffer_max_dt = latest_dt
330
- self._logger.debug("已到达数据库最新事件时间,停止增量加载")
331
- break
332
- # 计算出增量加载的窗口时间点
333
- window_end = self._last_fetched_dt + self._window_size
334
- # 限制窗口终点不超过数据库最新时间
335
- if latest_dt is not None and window_end > latest_dt:
336
- window_end = latest_dt
337
- # 增量加载
338
- new_events = self._load_events_window(self._last_fetched_dt, window_end)
339
- self._last_fetched_dt = window_end
340
- if new_events:
341
- self._logger.debug(f"增量获取{len(new_events)}条数据,截止时间{window_end}")
342
- self._events.extend(new_events)
343
- buffer_max_dt = self._events[-1][0]
344
- else:
345
- # 即便当前窗无数据,推进游标,避免卡住
346
- buffer_max_dt = max(buffer_max_dt, window_end)
347
- self._logger.debug(f"增量获取{0}条数据,截止时间{window_end}")
348
- latest_dt = None
349
-
350
- # 如果缓冲区已消费完,退出(或等待下一轮覆盖)
351
- if idx >= len(self._events):
352
- time.sleep(0.05)
353
- continue
354
-
355
- # 取出一条数据
356
- evt_dt, evt = self._events[idx]
357
-
358
- # 计算等待时间,按加速比缩放
359
- try:
360
- # 数据时间 - 当前虚拟时间
361
- delta = (evt_dt - now_virtual).total_seconds()
362
- # 等待秒数
363
- wait_s = max(delta, 0.0)
364
- if wait_s > 0:
365
- time.sleep(0.05)
366
- continue
367
-
368
- if prev_dt != evt_dt:
369
- # 输出一下时间
370
- if prev_dt is not None:
371
- self._logger.debug(f"完成处理{prev_dt}的资金费率{prev_counter}条")
372
- self._logger.debug(f"开始处理{evt_dt}的资金费率")
373
- prev_counter = 0
374
- prev_dt = evt_dt
375
- #
376
- if discard_dt is None:
377
- discard_dt = prev_dt
378
- except Exception:
379
- pass
380
-
381
- #
382
- idx += 1
383
- prev_counter += 1
384
-
385
- # 构造推送数据
386
- payload = evt.model_dump()
387
- if self._use_realtime_event_time:
388
- # 调整为实时时间
389
- payload["event_time"] = int(time.time() * 1000)
390
- # 广播到主题:all 和 symbol
391
- try:
392
- self.broadcast("funding_rate.all", payload)
393
- sym = payload.get("symbol") or ""
394
- if sym:
395
- self.broadcast(f"funding_rate.{sym}", payload)
396
- except Exception as e:
397
- self._logger.warning(f"广播资金费率事件失败: {e}")
398
-
399
- # 时间型丢弃:移除过旧数据以控制内存
400
- if prev_dt - discard_dt > self._retain_horizon:
401
- discard_dt = prev_dt
402
- cut_idx = -1
403
- for i in range(len(self._events)):
404
- if self._events[i][0] < discard_dt:
405
- cut_idx = i
406
- elif self._events[i][0] >= discard_dt:
407
- break
408
- # 裁剪
409
- if cut_idx > 0:
410
- cut_count = cut_idx + 1
411
- self._logger.debug(f"丢弃{cut_count}条过期数据")
412
- self._events = self._events[cut_count:]
413
- # 修正idx
414
- idx = max(0, idx - cut_count)
415
-
416
- def run_with_thread(self, block: bool = True):
417
- # 启动推送线程
418
- self._stop_push.clear()
419
- self._push_thread = threading.Thread(target=self._push_loop, name="WsExchangePush", daemon=True)
420
- self._push_thread.start()
421
- # 启动WS服务器线程
422
- super().run_with_thread(block)
423
-
424
- def shutdown_with_thread(self):
425
- # 先停止推送线程
426
- try:
427
- self._stop_push.set()
428
- if self._push_thread and self._push_thread.is_alive():
429
- self._push_thread.join(timeout=3)
430
- except Exception:
431
- pass
432
- # 再关闭WS服务器
433
- super().shutdown_with_thread()
434
-
435
- async def _ws_handler(self, ws):
436
- # 覆盖基类鉴权,模拟平台测试环境不启用鉴权
437
- try:
438
- await self._handle_connection(ws)
439
- except Exception:
440
- pass
1
+ import threading
2
+ import time
3
+ from datetime import datetime, timedelta
4
+ from typing import List, Optional, Tuple, Union
5
+
6
+ import pandas as pd
7
+
8
+ from kaq_quant_common.api.ws.exchange.models import FundingRateEvent
9
+ from kaq_quant_common.api.ws.ws_server_base import WsServerBase
10
+ from kaq_quant_common.resources.kaq_mysql_resources import KaqQuantMysqlRepository
11
+ from kaq_quant_common.utils import logger_utils
12
+
13
+
14
+ class WsExchangeServer(WsServerBase):
15
+ """
16
+ 模拟加密货币平台 WS 服务器:
17
+ - 从 MySQL 按 exchange 读取资金费率历史数据
18
+ - 从指定 start_time 开始,按事件时间差和加速倍数推送
19
+ - 支持将推送事件的 event_time 重写为实时时间
20
+ - 主题:
21
+ * funding_rate.all
22
+ * funding_rate.<symbol>
23
+ """
24
+
25
+ def __init__(
26
+ self,
27
+ exchange: str,
28
+ mysql_host: str,
29
+ mysql_port: int,
30
+ mysql_user: str,
31
+ mysql_passwd: str,
32
+ mysql_db: str,
33
+ charset: str = "utf8mb4",
34
+ start_time: Union[int, float, datetime] = None,
35
+ speed_multiplier: float = 1.0,
36
+ use_realtime_event_time: bool = False,
37
+ inject_sample_on_empty: bool = True,
38
+ host: str = "0.0.0.0",
39
+ port: int = 8767,
40
+ ):
41
+ super().__init__(self, host, port)
42
+ self._logger = logger_utils.get_logger(self)
43
+ # 平台
44
+ self._exchange = exchange
45
+ # 加速设置
46
+ self._speed_multiplier = max(speed_multiplier if speed_multiplier and speed_multiplier > 0 else 1.0, 1e-6)
47
+ # 是否使用实时时间
48
+ self._use_realtime_event_time = use_realtime_event_time
49
+ # 示例数据注入开关
50
+ self._inject_sample_on_empty = inject_sample_on_empty
51
+
52
+ # 单次增量加载的窗口大小
53
+ self._window_size: timedelta = timedelta(minutes=10)
54
+ # 触发预加载的阈值
55
+ self._preload_horizon: timedelta = timedelta(minutes=5)
56
+ # 丢弃数据的阈值
57
+ self._retain_horizon: timedelta = timedelta(minutes=10)
58
+
59
+ # 数据开始游标(时间) 支持毫秒时间戳或 datetime
60
+ if isinstance(start_time, (int, float)):
61
+ self._start_dt = datetime.fromtimestamp(start_time / 1000.0)
62
+ elif isinstance(start_time, datetime):
63
+ self._start_dt = start_time
64
+ else:
65
+ # 默认很早的时间,确保可以从最早记录开始
66
+ self._start_dt = datetime.fromtimestamp(0)
67
+
68
+ # DB
69
+ self._repo = KaqQuantMysqlRepository(mysql_host, mysql_port, mysql_user, mysql_passwd, mysql_db, charset)
70
+
71
+ # 推送控制
72
+ self._push_thread: Optional[threading.Thread] = None
73
+ self._stop_push = threading.Event()
74
+
75
+ # 预拉取首批数据,开始的时候加载多一点
76
+ initial_end = self._start_dt + self._preload_horizon * 2
77
+ self._events: List[Tuple[datetime, FundingRateEvent]] = self._load_events_window(self._start_dt, initial_end)
78
+ self._last_fetched_dt = initial_end
79
+ self._logger.info(f"资金费率事件已加载: {len(self._events)} 条")
80
+ # 添加测试数据
81
+ if not self._events and self._inject_sample_on_empty:
82
+ base_dt = self._start_dt if self._start_dt else datetime.fromtimestamp(0)
83
+ sample: List[Tuple[datetime, FundingRateEvent]] = []
84
+ for i in range(5):
85
+ dt = base_dt if i == 0 else base_dt + pd.Timedelta(minutes=i)
86
+ evt = FundingRateEvent(
87
+ event_time=int(dt.timestamp() * 1000),
88
+ exchange_symbol=f"{self._exchange}:BTCUSDT",
89
+ exchange=self._exchange,
90
+ symbol="BTCUSDT",
91
+ open_rate=0.0001,
92
+ close_rate=0.0001,
93
+ high_rate=0.0002,
94
+ low_rate=0.0000,
95
+ )
96
+ sample.append((dt, evt))
97
+ self._events = sample
98
+ self._logger.warning("资金费率历史为空,已注入示例数据用于推送测试")
99
+
100
+ # 把开始时间调整为第一条数据的时间
101
+ if self._events:
102
+ self._start_dt = self._events[0][0]
103
+ # 最后对齐整分钟
104
+ self._start_dt = self._start_dt.replace(second=0, microsecond=0)
105
+
106
+ # 实际启动时间,用于计算虚拟时间,需要对齐整分钟
107
+ self._real_start_dt: datetime = datetime.now()
108
+ self._real_start_dt = self._real_start_dt.replace(second=0, microsecond=0)
109
+
110
+ # 原始全量拉取(保留以兼容旧逻辑)
111
+ def _load_events(self) -> List[Tuple[datetime, FundingRateEvent]]:
112
+ try:
113
+ query = (
114
+ "SELECT event_time, exchange_symbol, exchange, symbol, open_rate, close_rate, high_rate, low_rate, close_time, next_event_time, id, ctimestamp "
115
+ "FROM kaq_future_fr_klines_1m "
116
+ f"WHERE exchange = '{self._exchange}' AND event_time >= '{self._start_dt.strftime('%Y-%m-%d %H:%M:%S')}' "
117
+ "ORDER BY event_time ASC"
118
+ )
119
+ df: pd.DataFrame = self._repo.fetch_data(query)
120
+ if df is None or df.empty:
121
+ self._logger.warning("资金费率表无数据或查询为空")
122
+ return []
123
+
124
+ events: List[Tuple[datetime, FundingRateEvent]] = []
125
+ for _, row in df.iterrows():
126
+ evt_dt: datetime = row.get("event_time")
127
+ if not isinstance(evt_dt, datetime):
128
+ try:
129
+ evt_dt = pd.to_datetime(evt_dt).to_pydatetime()
130
+ except Exception:
131
+ continue
132
+ # 转为毫秒时间戳
133
+ close_time_val = row.get("close_time")
134
+ next_event_time_val = row.get("next_event_time")
135
+ try:
136
+ close_time_ms = int(pd.to_datetime(close_time_val).timestamp() * 1000) if close_time_val is not None else None
137
+ except Exception:
138
+ close_time_ms = None
139
+ try:
140
+ next_event_time_ms = int(pd.to_datetime(next_event_time_val).timestamp() * 1000) if next_event_time_val is not None else None
141
+ except Exception:
142
+ next_event_time_ms = None
143
+ evt_ms = int(evt_dt.timestamp() * 1000)
144
+ event = FundingRateEvent(
145
+ event_time=evt_ms,
146
+ exchange_symbol=row.get("exchange_symbol"),
147
+ exchange=row.get("exchange"),
148
+ symbol=row.get("symbol"),
149
+ open_rate=row.get("open_rate"),
150
+ close_rate=row.get("close_rate"),
151
+ high_rate=row.get("high_rate"),
152
+ low_rate=row.get("low_rate"),
153
+ close_time=close_time_ms,
154
+ next_event_time=next_event_time_ms,
155
+ #
156
+ id=row.get("id"),
157
+ ctimestamp=row.get("ctimestamp"),
158
+ )
159
+ events.append((evt_dt, event))
160
+ return events
161
+ except Exception as e:
162
+ self._logger.error(f"加载资金费率事件失败: {e}")
163
+ return []
164
+
165
+ # 增量批量拉取(保留以兼容旧逻辑)
166
+ def _load_events_batch(self, start_from: datetime, limit: int) -> List[Tuple[datetime, FundingRateEvent]]:
167
+ try:
168
+ query = (
169
+ "SELECT event_time, exchange_symbol, exchange, symbol, open_rate, close_rate, high_rate, low_rate, close_time, next_event_time, id, ctimestamp "
170
+ "FROM kaq_future_fr_klines_1m "
171
+ f"WHERE exchange = '{self._exchange}' AND event_time > '{start_from.strftime('%Y-%m-%d %H:%M:%S')}' "
172
+ "ORDER BY event_time ASC "
173
+ f"LIMIT {int(limit)}"
174
+ )
175
+ df: pd.DataFrame = self._repo.fetch_data(query)
176
+ if df is None or df.empty:
177
+ return []
178
+ events: List[Tuple[datetime, FundingRateEvent]] = []
179
+ for _, row in df.iterrows():
180
+ evt_dt: datetime = row.get("event_time")
181
+ if not isinstance(evt_dt, datetime):
182
+ try:
183
+ evt_dt = pd.to_datetime(evt_dt).to_pydatetime()
184
+ except Exception:
185
+ continue
186
+ # 转为毫秒时间戳
187
+ close_time_val = row.get("close_time")
188
+ next_event_time_val = row.get("next_event_time")
189
+ try:
190
+ close_time_ms = int(pd.to_datetime(close_time_val).timestamp() * 1000) if close_time_val is not None else None
191
+ except Exception:
192
+ close_time_ms = None
193
+ try:
194
+ next_event_time_ms = int(pd.to_datetime(next_event_time_val).timestamp() * 1000) if next_event_time_val is not None else None
195
+ except Exception:
196
+ next_event_time_ms = None
197
+ evt_ms = int(evt_dt.timestamp() * 1000)
198
+ event = FundingRateEvent(
199
+ event_time=evt_ms,
200
+ exchange_symbol=row.get("exchange_symbol"),
201
+ exchange=row.get("exchange"),
202
+ symbol=row.get("symbol"),
203
+ open_rate=row.get("open_rate"),
204
+ close_rate=row.get("close_rate"),
205
+ high_rate=row.get("high_rate"),
206
+ low_rate=row.get("low_rate"),
207
+ close_time=close_time_ms,
208
+ next_event_time=next_event_time_ms,
209
+ #
210
+ id=row.get("id"),
211
+ ctimestamp=row.get("ctimestamp"),
212
+ )
213
+ events.append((evt_dt, event))
214
+ return events
215
+ except Exception as e:
216
+ self._logger.error(f"增量加载资金费率事件失败: {e}")
217
+ return []
218
+
219
+ # 按照时间窗口拉取
220
+ def _load_events_window(self, start_from: datetime, end_until: datetime) -> List[Tuple[datetime, FundingRateEvent]]:
221
+ try:
222
+ query = (
223
+ "SELECT event_time, exchange_symbol, exchange, symbol, open_rate, close_rate, high_rate, low_rate, close_time, next_event_time, id, ctimestamp "
224
+ "FROM kaq_future_fr_klines_1m "
225
+ f"WHERE exchange = '{self._exchange}' AND event_time > '{start_from.strftime('%Y-%m-%d %H:%M:%S')}' AND event_time <= '{end_until.strftime('%Y-%m-%d %H:%M:%S')}' "
226
+ "ORDER BY event_time ASC"
227
+ )
228
+ df: pd.DataFrame = self._repo.fetch_data(query)
229
+ if df is None or df.empty:
230
+ return []
231
+ events: List[Tuple[datetime, FundingRateEvent]] = []
232
+ for _, row in df.iterrows():
233
+ evt_dt: datetime = row.get("event_time")
234
+ if not isinstance(evt_dt, datetime):
235
+ try:
236
+ evt_dt = pd.to_datetime(evt_dt).to_pydatetime()
237
+ except Exception:
238
+ continue
239
+ close_time_val = row.get("close_time")
240
+ next_event_time_val = row.get("next_event_time")
241
+ try:
242
+ close_time_ms = int(pd.to_datetime(close_time_val).timestamp() * 1000) if close_time_val is not None else None
243
+ except Exception:
244
+ close_time_ms = None
245
+ try:
246
+ next_event_time_ms = int(pd.to_datetime(next_event_time_val).timestamp() * 1000) if next_event_time_val is not None else None
247
+ except Exception:
248
+ next_event_time_ms = None
249
+ evt_ms = int(evt_dt.timestamp() * 1000)
250
+ event = FundingRateEvent(
251
+ event_time=evt_ms,
252
+ exchange_symbol=row.get("exchange_symbol"),
253
+ exchange=row.get("exchange"),
254
+ symbol=row.get("symbol"),
255
+ open_rate=row.get("open_rate"),
256
+ close_rate=row.get("close_rate"),
257
+ high_rate=row.get("high_rate"),
258
+ low_rate=row.get("low_rate"),
259
+ close_time=close_time_ms,
260
+ next_event_time=next_event_time_ms,
261
+ #
262
+ id=row.get("id"),
263
+ ctimestamp=row.get("ctimestamp"),
264
+ )
265
+ events.append((evt_dt, event))
266
+ return events
267
+ except Exception as e:
268
+ self._logger.error(f"时间窗加载资金费率事件失败: {e}")
269
+ return []
270
+
271
+ # 从数据库拉取最新的记录
272
+ def _get_latest_event_dt(self) -> Optional[datetime]:
273
+ try:
274
+ query = "SELECT MAX(event_time) AS max_event_time " "FROM kaq_future_fr_klines_1m " f"WHERE exchange = '{self._exchange}'"
275
+ df: pd.DataFrame = self._repo.fetch_data(query)
276
+ if df is None or df.empty:
277
+ return None
278
+ val = df.iloc[0].get("max_event_time")
279
+
280
+ if isinstance(val, datetime):
281
+ return val
282
+ try:
283
+ return pd.to_datetime(val).to_pydatetime()
284
+ except Exception:
285
+ return None
286
+ except Exception as e:
287
+ self._logger.error(f"查询最新资金费率事件时间失败: {e}")
288
+ return None
289
+
290
+ def _push_loop(self):
291
+ # 初始缓冲检查
292
+ if not self._events:
293
+ # 获取一次最新记录
294
+ latest_dt = self._get_latest_event_dt()
295
+ initial_end = self._last_fetched_dt + self._window_size
296
+ # 防止超过
297
+ if latest_dt is not None and initial_end > latest_dt:
298
+ initial_end = latest_dt
299
+ self._events = self._load_events_window(self._last_fetched_dt, initial_end)
300
+ # 主要是为了记录这个最新拉取的时间点,下一次增量加载从这里开始
301
+ self._last_fetched_dt = initial_end
302
+ if not self._events:
303
+ self._logger.warning("无可推送的资金费率历史事件")
304
+ return
305
+
306
+ prev_dt = None
307
+ prev_counter = 0
308
+ discard_dt = None
309
+ idx = 0
310
+ while not self._stop_push.is_set():
311
+ # 虚拟时钟,考虑加速比
312
+ now_real = datetime.now()
313
+ # 计算出虚拟时间点
314
+ now_virtual = self._start_dt + (now_real - self._real_start_dt) * self._speed_multiplier
315
+
316
+ # 确保缓冲区覆盖到虚拟时间的预加载阈值
317
+ # 获取缓冲区最新的记录时间
318
+ buffer_max_dt = self._events[-1][0] if self._events else self._last_fetched_dt
319
+ # 根据虚拟时间点计算出需要覆盖的时间点
320
+ target_cover_dt = now_virtual + self._preload_horizon
321
+ # 最新的记录时间
322
+ latest_dt: datetime = None
323
+ while buffer_max_dt < target_cover_dt and not self._stop_push.is_set():
324
+ # 每次进入循环,先拉取一次最新数据时间
325
+ if latest_dt is None:
326
+ latest_dt = self._get_latest_event_dt()
327
+ # 若数据库已无更晚数据,停止增量加载
328
+ if latest_dt is not None and self._last_fetched_dt >= latest_dt:
329
+ buffer_max_dt = latest_dt
330
+ self._logger.debug("已到达数据库最新事件时间,停止增量加载")
331
+ break
332
+ # 计算出增量加载的窗口时间点
333
+ window_end = self._last_fetched_dt + self._window_size
334
+ # 限制窗口终点不超过数据库最新时间
335
+ if latest_dt is not None and window_end > latest_dt:
336
+ window_end = latest_dt
337
+ # 增量加载
338
+ new_events = self._load_events_window(self._last_fetched_dt, window_end)
339
+ self._last_fetched_dt = window_end
340
+ if new_events:
341
+ self._logger.debug(f"增量获取{len(new_events)}条数据,截止时间{window_end}")
342
+ self._events.extend(new_events)
343
+ buffer_max_dt = self._events[-1][0]
344
+ else:
345
+ # 即便当前窗无数据,推进游标,避免卡住
346
+ buffer_max_dt = max(buffer_max_dt, window_end)
347
+ self._logger.debug(f"增量获取{0}条数据,截止时间{window_end}")
348
+ latest_dt = None
349
+
350
+ # 如果缓冲区已消费完,退出(或等待下一轮覆盖)
351
+ if idx >= len(self._events):
352
+ time.sleep(0.05)
353
+ continue
354
+
355
+ # 取出一条数据
356
+ evt_dt, evt = self._events[idx]
357
+
358
+ # 计算等待时间,按加速比缩放
359
+ try:
360
+ # 数据时间 - 当前虚拟时间
361
+ delta = (evt_dt - now_virtual).total_seconds()
362
+ # 等待秒数
363
+ wait_s = max(delta, 0.0)
364
+ if wait_s > 0:
365
+ time.sleep(0.05)
366
+ continue
367
+
368
+ if prev_dt != evt_dt:
369
+ # 输出一下时间
370
+ if prev_dt is not None:
371
+ self._logger.debug(f"完成处理{prev_dt}的资金费率{prev_counter}条")
372
+ self._logger.debug(f"开始处理{evt_dt}的资金费率")
373
+ prev_counter = 0
374
+ prev_dt = evt_dt
375
+ #
376
+ if discard_dt is None:
377
+ discard_dt = prev_dt
378
+ except Exception:
379
+ pass
380
+
381
+ #
382
+ idx += 1
383
+ prev_counter += 1
384
+
385
+ # 构造推送数据
386
+ payload = evt.model_dump()
387
+ if self._use_realtime_event_time:
388
+ # 调整为实时时间
389
+ payload["event_time"] = int(time.time() * 1000)
390
+ # 广播到主题:all 和 symbol
391
+ try:
392
+ self.broadcast("funding_rate.all", payload)
393
+ sym = payload.get("symbol") or ""
394
+ if sym:
395
+ self.broadcast(f"funding_rate.{sym}", payload)
396
+ except Exception as e:
397
+ self._logger.warning(f"广播资金费率事件失败: {e}")
398
+
399
+ # 时间型丢弃:移除过旧数据以控制内存
400
+ if prev_dt - discard_dt > self._retain_horizon:
401
+ discard_dt = prev_dt
402
+ cut_idx = -1
403
+ for i in range(len(self._events)):
404
+ if self._events[i][0] < discard_dt:
405
+ cut_idx = i
406
+ elif self._events[i][0] >= discard_dt:
407
+ break
408
+ # 裁剪
409
+ if cut_idx > 0:
410
+ cut_count = cut_idx + 1
411
+ self._logger.debug(f"丢弃{cut_count}条过期数据")
412
+ self._events = self._events[cut_count:]
413
+ # 修正idx
414
+ idx = max(0, idx - cut_count)
415
+
416
+ def run_with_thread(self, block: bool = True):
417
+ # 启动推送线程
418
+ self._stop_push.clear()
419
+ self._push_thread = threading.Thread(target=self._push_loop, name="WsExchangePush", daemon=True)
420
+ self._push_thread.start()
421
+ # 启动WS服务器线程
422
+ super().run_with_thread(block)
423
+
424
+ def shutdown_with_thread(self):
425
+ # 先停止推送线程
426
+ try:
427
+ self._stop_push.set()
428
+ if self._push_thread and self._push_thread.is_alive():
429
+ self._push_thread.join(timeout=3)
430
+ except Exception:
431
+ pass
432
+ # 再关闭WS服务器
433
+ super().shutdown_with_thread()
434
+
435
+ async def _ws_handler(self, ws):
436
+ # 覆盖基类鉴权,模拟平台测试环境不启用鉴权
437
+ try:
438
+ await self._handle_connection(ws)
439
+ except Exception:
440
+ pass