kaq-quant-common 0.2.13__tar.gz → 0.2.15__tar.gz

{kaq_quant_common-0.2.13 → kaq_quant_common-0.2.15}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kaq_quant_common
-Version: 0.2.13
+Version: 0.2.15
 Summary: 
 Author: kevinfuture
 Author-email: liuenbofuture@foxmail.com

{kaq_quant_common-0.2.13 → kaq_quant_common-0.2.15}/kaq_quant_common/api/rest/instruction/helper/mock_order_helper.py

@@ -37,6 +37,7 @@ class MockOrderHelper:
         exchange: str,
         symbol: str,
         position_side,
+        lever: int,
         coin_quantity: float,
         usdt_quantity: float,
         open_ins_id: str,
@@ -44,6 +45,7 @@ class MockOrderHelper:
         open_fee: float,
         open_fee_rate: float,
         open_time: int,
+        is_spot: bool,
     ):
         redis = self._server._redis
         if redis is None:
@@ -53,6 +55,7 @@ class MockOrderHelper:
             "exchange": exchange,
             "symbol": symbol,
             "position_side": position_side.value,
+            "lever": lever,
             "coin_quantity": coin_quantity,
             "usdt_quantity": usdt_quantity,
             "open_ins_id": open_ins_id,
@@ -64,6 +67,7 @@ class MockOrderHelper:
             "close_price": 0,
             "close_time": 0,
             "status": PositionStatus.OPEN.value,
+            "is_spot": is_spot,
             # 标识模拟
             "is_mock": True,
         }
@@ -75,6 +79,7 @@ class MockOrderHelper:
         exchange: str,
         symbol: str,
         position_side,
+        lever: int,
         coin_quantity: float,
         usdt_quantity: float,
         open_ins_id: str,
@@ -87,6 +92,7 @@ class MockOrderHelper:
         close_fee: float,
         close_fee_rate: float,
         close_time: int,
+        is_spot: bool,
     ):
         redis = self._server._redis
         if redis is None:
@@ -95,26 +101,21 @@ class MockOrderHelper:
         # 先从 redis 读取现有的 position 数据，获取 funding_rate_records 字段
         funding_rate_records = None
         try:
-            existing_position_json = redis.client.hget(
-                self._redis_key_position, position_id
-            )
+            existing_position_json = redis.client.hget(self._redis_key_position, position_id)
             if existing_position_json:
                 existing_position = json.loads(existing_position_json)
                 if existing_position and "funding_rate_records" in existing_position:
-                    funding_rate_records = existing_position.get(
-                        "funding_rate_records"
-                    )
+                    funding_rate_records = existing_position.get("funding_rate_records")
         except Exception as e:
             # 读取失败不影响后续流程，记录日志
-            self._logger.warning(
-                f"Failed to get funding_rate_records for position {position_id}: {e}"
-            )
+            self._logger.warning(f"Failed to get funding_rate_records for position {position_id}: {e}")
 
         data = {
             "id": position_id,
             "exchange": exchange,
             "symbol": symbol,
             "position_side": position_side.value,
+            "lever": lever,
             "coin_quantity": coin_quantity,
             "usdt_quantity": usdt_quantity,
             "open_ins_id": open_ins_id,
@@ -128,6 +129,7 @@ class MockOrderHelper:
             "close_fee_rate": close_fee_rate,
             "close_time": close_time,
             "status": PositionStatus.CLOSE.value,
+            "is_spot": is_spot,
             # 标识模拟
             "is_mock": True,
         }
@@ -178,6 +180,9 @@ class MockOrderHelper:
         symbol = order.symbol
         side = order.side
         position_side = order.position_side
+        lever = order.level
+        # TODO
+        is_spot = False
 
         is_open = True
         side_str = "开仓"
@@ -198,9 +203,7 @@ class MockOrderHelper:
                 side_str = "平仓"
                 is_open = False
 
-        self._logger.info(
-            f"[MOCK] {ins_id}_{exchange}_{symbol} step 1. {side_str}模拟挂单 {order_id}"
-        )
+        self._logger.info(f"[MOCK] {ins_id}_{exchange}_{symbol} step 1. {"现货" if is_spot else "合约"}{side_str}模拟挂单 {order_id}")
 
         # 步骤1.挂单成功 插入到订单记录
         current_time = int(time.time() * 1000)
@@ -208,8 +211,8 @@ class MockOrderHelper:
         if mysql is not None:
             status = OrderStatus.CREATE
             sql = f"""
-            INSERT INTO {self._mysql_table_name_order} (ins_id, exchange, symbol, side, position_side, orig_price, orig_coin_quantity, order_id, status, create_time, last_update_time, is_mock) 
-            VALUES ( '{ins_id}', '{exchange}', '{symbol}', '{side.value}', '{order.position_side.value}', {order.current_price or order.target_price}, {order.quantity}, '{order_id}', '{status.value}', {current_time}, {current_time}, 1 );
+            INSERT INTO {self._mysql_table_name_order} (ins_id, exchange, symbol, side, position_side, lever, orig_price, orig_coin_quantity, order_id, status, create_time, last_update_time, is_spot, is_mock) 
+            VALUES ( '{ins_id}', '{exchange}', '{symbol}', '{side.value}', '{order.position_side.value}', '{lever}', {order.current_price or order.target_price}, {order.quantity}, '{order_id}', '{status.value}', {current_time}, {current_time}, '{1 if is_spot else 0}', 1 );
             """
             execute_ret = mysql.execute_sql(sql, True)
 
@@ -229,17 +232,13 @@ class MockOrderHelper:
         end_time = time.time()
         cost_time = end_time - start_time
 
-        self._logger.info(
-            f"[MOCK] {ins_id}_{exchange}_{symbol} step 2. {side_str}模拟订单 {order_id} 成交 耗时 {int(cost_time * 1000)}ms"
-        )
+        self._logger.info(f"[MOCK] {ins_id}_{exchange}_{symbol} step 2. {side_str}模拟订单 {order_id} 成交 耗时 {int(cost_time * 1000)}ms")
 
         # 步骤3.把最终持仓写进去
         current_time = int(time.time() * 1000)
 
         if mysql is None:
-            self._logger.warning(
-                f"[MOCK] {ins_id}_{exchange}_{symbol} 仅操作，没有入库，请设置 mysql!!"
-            )
+            self._logger.warning(f"[MOCK] {ins_id}_{exchange}_{symbol} 仅操作，没有入库，请设置 mysql!!")
             return
 
         status = OrderStatus.FINISH
@@ -252,34 +251,34 @@ class MockOrderHelper:
         execute_ret = mysql.execute_sql(sql, True)
 
         self._logger.info(
-            f"[MOCK] {ins_id}_{exchange}_{symbol} step 2. 模拟订单成交 {order_id}, {side_str}价格 {avg_price}, {side_str}数量 {executed_qty}, {side_str}usdt {executed_usdt}"
+            f"[MOCK] {ins_id}_{exchange}_{symbol} step 2. 模拟订单成交 {order_id}, {side_str}价格 {avg_price}, {side_str}数量 {executed_qty}, {side_str}usdt {executed_usdt} 杠杆 {lever}"
         )
 
         if is_open:
             # 同时插入持仓表
             position_id = uuid_utils.generate_uuid()
             sql = f"""
-            INSERT INTO {self._mysql_table_name_position} (id, exchange, symbol, position_side, coin_quantity, usdt_quantity, open_ins_id, open_price, open_fee, open_fee_rate, open_time, status, is_mock) 
-            VALUES ( '{position_id}', '{exchange}', '{symbol}', '{position_side.value}', '{executed_qty}', '{executed_usdt}', '{ins_id}', '{avg_price}', '{fee}', '{fee_rate}', {current_time}, '{PositionStatus.OPEN.value}', 1 );
+            INSERT INTO {self._mysql_table_name_position} (id, exchange, symbol, position_side, lever, coin_quantity, usdt_quantity, open_ins_id, open_price, open_fee, open_fee_rate, open_time, status, is_spot, is_mock) 
+            VALUES ( '{position_id}', '{exchange}', '{symbol}', '{position_side.value}', '{lever}', '{executed_qty}', '{executed_usdt}', '{ins_id}', '{avg_price}', '{fee}', '{fee_rate}', {current_time}, '{PositionStatus.OPEN.value}', '{1 if is_spot else 0}', 1 );
             """
             execute_ret = mysql.execute_sql(sql, True)
 
-            self._logger.info(
-                f"[MOCK] {ins_id}_{exchange}_{symbol} step 3. 创建持仓记录 {position_id}"
-            )
+            self._logger.info(f"[MOCK] {ins_id}_{exchange}_{symbol} step 3. 创建持仓记录 {position_id}")
             try:
                 self._write_position_open_to_redis(
-                    position_id,
-                    exchange,
-                    symbol,
-                    position_side,
-                    executed_qty,
-                    executed_usdt,
-                    ins_id,
-                    avg_price,
-                    fee,
-                    fee_rate,
-                    current_time,
+                    position_id=position_id,
+                    exchange=exchange,
+                    symbol=symbol,
+                    position_side=position_side,
+                    lever=lever,
+                    coin_quantity=executed_qty,
+                    usdt_quantity=executed_usdt,
+                    open_ins_id=ins_id,
+                    open_price=avg_price,
+                    open_fee=fee,
+                    open_fee_rate=fee_rate,
+                    open_time=current_time,
+                    is_spot=is_spot,
                 )
             except:
                 pass
@@ -287,7 +286,12 @@ class MockOrderHelper:
             # 需要找到对应的持仓记录
             sql = f"""
             SELECT * FROM {self._mysql_table_name_position}
-            WHERE exchange = '{exchange}' AND symbol = '{symbol}' AND position_side = '{position_side.value}' AND status = '{PositionStatus.OPEN.value}' AND is_mock = 1 
+            WHERE exchange = '{exchange}' 
+            AND symbol = '{symbol}' 
+            AND position_side = '{position_side.value}' 
+            AND status = '{PositionStatus.OPEN.value}' 
+            AND is_spot = '{1 if is_spot else 0}'
+            AND is_mock = 1 
             ORDER BY open_time ASC;
             """
 
@@ -297,9 +301,7 @@ class MockOrderHelper:
                 SELECT * FROM {self._mysql_table_name_position}
                 WHERE id = '{order.position_id}' AND status = '{PositionStatus.OPEN.value}' AND is_mock = 1
                 """
-                self._logger.info(
-                    f"[MOCK] {ins_id}_{exchange}_{symbol} get position by id {order.position_id}"
-                )
+                self._logger.info(f"[MOCK] {ins_id}_{exchange}_{symbol} get position by id {order.position_id}")
 
             execute_ret = mysql.execute_sql(sql)
             try:
@@ -314,27 +316,27 @@ class MockOrderHelper:
                     """
                     execute_ret = mysql.execute_sql(sql, True)
 
-                    self._logger.info(
-                        f"[MOCK] {ins_id}_{exchange}_{symbol} step 3. 更新持仓记录 {position_id}"
-                    )
+                    self._logger.info(f"[MOCK] {ins_id}_{exchange}_{symbol} step 3. 更新持仓记录 {position_id}")
                     try:
                         self._write_position_close_to_redis(
-                            position_id,
-                            exchange,
-                            symbol,
-                            position_side,
-                            float(row.coin_quantity),
-                            float(row.usdt_quantity),
-                            row.open_ins_id,
-                            float(row.open_price),
-                            float(row.open_fee),
-                            float(row.open_fee_rate),
-                            int(row.open_time),
-                            ins_id,
-                            avg_price,
-                            fee,
-                            fee_rate,
-                            current_time,
+                            position_id=position_id,
+                            exchange=exchange,
+                            symbol=symbol,
+                            position_side=position_side,
+                            lever=row.lever,
+                            coin_quantity=float(row.coin_quantity),
+                            usdt_quantity=float(row.usdt_quantity),
+                            open_ins_id=row.open_ins_id,
+                            open_price=float(row.open_price),
+                            open_fee=float(row.open_fee),
+                            open_fee_rate=float(row.open_fee_rate),
+                            open_time=int(row.open_time),
+                            close_ins_id=ins_id,
+                            close_price=avg_price,
+                            close_fee=fee,
+                            close_fee_rate=fee_rate,
+                            close_time=current_time,
+                            is_spot=row.is_spot,
                         )
                     except:
                         pass

{kaq_quant_common-0.2.13 → kaq_quant_common-0.2.15}/kaq_quant_common/api/rest/instruction/helper/order_helper.py

@@ -36,6 +36,7 @@ class OrderHelper:
         exchange: str,
         symbol: str,
         position_side,
+        lever: int,
         coin_quantity: float,
         usdt_quantity: float,
         open_ins_id: str,
@@ -43,6 +44,7 @@ class OrderHelper:
         open_fee: float,
         open_fee_rate: float,
         open_time: int,
+        is_spot: bool,
     ):
         redis = self._server._redis
         if redis is None:
@@ -52,6 +54,7 @@ class OrderHelper:
             "exchange": exchange,
             "symbol": symbol,
             "position_side": position_side.value,
+            "lever": lever,
             "coin_quantity": coin_quantity,
             "usdt_quantity": usdt_quantity,
             "open_ins_id": open_ins_id,
@@ -63,6 +66,7 @@ class OrderHelper:
             "close_price": 0,
             "close_time": 0,
             "status": PositionStatus.OPEN.value,
+            "is_spot": is_spot,
         }
         redis.client.hset(self._redis_key_position, position_id, json.dumps(data))
 
@@ -72,6 +76,7 @@ class OrderHelper:
         exchange: str,
         symbol: str,
         position_side,
+        lever: int,
         coin_quantity: float,
         usdt_quantity: float,
         open_ins_id: str,
@@ -84,11 +89,12 @@ class OrderHelper:
         close_fee: float,
         close_fee_rate: float,
         close_time: int,
+        is_spot: bool,
     ):
         redis = self._server._redis
         if redis is None:
             return
-        
+
         # 先从 redis 读取现有的 position 数据，获取 funding_rate_records 字段
         funding_rate_records = None
         try:
@@ -100,12 +106,13 @@ class OrderHelper:
         except Exception as e:
             # 读取失败不影响后续流程，记录日志
             self._logger.warning(f"Failed to get funding_rate_records for position {position_id}: {e}")
-        
+
         data = {
             "id": position_id,
             "exchange": exchange,
             "symbol": symbol,
             "position_side": position_side.value,
+            "lever": lever,
             "coin_quantity": coin_quantity,
             "usdt_quantity": usdt_quantity,
             "open_ins_id": open_ins_id,
@@ -119,12 +126,13 @@ class OrderHelper:
             "close_fee_rate": close_fee_rate,
             "close_time": close_time,
             "status": PositionStatus.CLOSE.value,
+            "is_spot": is_spot,
         }
-        
+
         # 如果存在 funding_rate_records，添加到 data 中
         if funding_rate_records is not None:
             data["funding_rate_records"] = funding_rate_records
-        
+
         redis.client.hdel(self._redis_key_position, position_id)
         redis.client.rpush(self._redis_key_position_history, json.dumps(data))
 
@@ -163,6 +171,9 @@ class OrderHelper:
         symbol = order.symbol
         side = order.side
         position_side = order.position_side
+        lever = order.level
+        # TODO
+        is_spot = False
 
         is_open = True
         side_str = "开仓"
@@ -184,7 +195,7 @@ class OrderHelper:
                 is_open = False
 
         if first:
-            self._logger.info(f"{ins_id}_{exchange}_{symbol} step 1. {side_str}挂单成功 {order_id}")
+            self._logger.info(f"{ins_id}_{exchange}_{symbol} step 1. {"现货" if is_spot else "合约"}{side_str}挂单成功 {order_id}")
 
             # 步骤1.挂单成功 插入到订单记录
             # 获取当前时间-ms
@@ -193,8 +204,8 @@ class OrderHelper:
             if mysql is not None:
                 status = OrderStatus.CREATE
                 sql = f"""
-                INSERT INTO {self._mysql_table_name_order} (ins_id, exchange, symbol, side, position_side, orig_price, orig_coin_quantity, order_id, status, create_time, last_update_time) 
-                VALUES ( '{ins_id}', '{exchange}', '{symbol}', '{side.value}', '{order.position_side.value}', {order.current_price or order.target_price}, {order.quantity}, '{order_id}', '{status.value}', {current_time}, {current_time} );
+                INSERT INTO {self._mysql_table_name_order} (ins_id, exchange, symbol, side, position_side, lever, orig_price, orig_coin_quantity, order_id, status, create_time, last_update_time, is_spot) 
+                VALUES ( '{ins_id}', '{exchange}', '{symbol}', '{side.value}', '{order.position_side.value}', '{lever}', {order.current_price or order.target_price}, {order.quantity}, '{order_id}', '{status.value}', {current_time}, {current_time}, '{1 if is_spot else 0}' );
                 """
                 execute_ret = mysql.execute_sql(sql, True)
 
@@ -257,31 +268,33 @@ class OrderHelper:
         execute_ret = mysql.execute_sql(sql, True)
 
         self._logger.info(
-            f"{ins_id}_{exchange}_{symbol} step 2. 订单成交 {order_id}, {side_str}价格 {avg_price}, {side_str}数量 {executed_qty}, {side_str}usdt {executed_usdt}"
+            f"{ins_id}_{exchange}_{symbol} step 2. 订单成交 {order_id}, {side_str}价格 {avg_price}, {side_str}数量 {executed_qty}, {side_str}usdt {executed_usdt} 杠杆 {lever}"
         )
         if is_open:
             # 同时插入持仓表
             position_id = uuid_utils.generate_uuid()
             sql = f"""
-            INSERT INTO {self._mysql_table_name_position} (id, exchange, symbol, position_side, coin_quantity, usdt_quantity, open_ins_id, open_price, open_fee, open_fee_rate, open_time, status) 
-            VALUES ( '{position_id}', '{exchange}', '{symbol}', '{position_side.value}', '{executed_qty}', '{executed_usdt}', '{ins_id}', '{avg_price}', '{fee}', '{fee_rate}', {current_time}, '{PositionStatus.OPEN.value}' );
+            INSERT INTO {self._mysql_table_name_position} (id, exchange, symbol, position_side, lever, coin_quantity, usdt_quantity, open_ins_id, open_price, open_fee, open_fee_rate, open_time, status, is_spot) 
+            VALUES ( '{position_id}', '{exchange}', '{symbol}', '{position_side.value}', '{lever}', '{executed_qty}', '{executed_usdt}', '{ins_id}', '{avg_price}', '{fee}', '{fee_rate}', {current_time}, '{PositionStatus.OPEN.value}', '{1 if is_spot else 0}' );
             """
             execute_ret = mysql.execute_sql(sql, True)
 
             self._logger.info(f"{ins_id}_{exchange}_{symbol} step 3. 创建持仓记录 {position_id}")
             try:
                 self._write_position_open_to_redis(
-                    position_id,
-                    exchange,
-                    symbol,
-                    position_side,
-                    executed_qty,
-                    executed_usdt,
-                    ins_id,
-                    avg_price,
-                    fee,
-                    fee_rate,
-                    current_time,
+                    position_id=position_id,
+                    exchange=exchange,
+                    symbol=symbol,
+                    position_side=position_side,
+                    lever=lever,
+                    coin_quantity=executed_qty,
+                    usdt_quantity=executed_usdt,
+                    open_ins_id=ins_id,
+                    open_price=avg_price,
+                    open_fee=fee,
+                    open_fee_rate=fee_rate,
+                    open_time=current_time,
+                    is_spot=is_spot,
                 )
             except:
                 pass
@@ -289,7 +302,12 @@ class OrderHelper:
             # 需要找到对应的持仓记录
             sql = f"""
             SELECT * FROM {self._mysql_table_name_position}
-            WHERE exchange = '{exchange}' AND symbol = '{symbol}' AND position_side = '{position_side.value}' AND status = '{PositionStatus.OPEN.value}' 
+            WHERE exchange = '{exchange}' 
+            AND symbol = '{symbol}' 
+            AND position_side = '{position_side.value}' 
+            AND status = '{PositionStatus.OPEN.value}' 
+            AND is_spot = '{1 if is_spot else 0}' 
+            AND is_mock = 0
             ORDER BY open_time ASC;
             """
 
@@ -317,22 +335,24 @@ class OrderHelper:
                     self._logger.info(f"{ins_id}_{exchange}_{symbol} step 3. 更新持仓记录 {position_id}")
                     try:
                         self._write_position_close_to_redis(
-                            position_id,
-                            exchange,
-                            symbol,
-                            position_side,
-                            float(row.coin_quantity),
-                            float(row.usdt_quantity),
-                            row.open_ins_id,
-                            float(row.open_price),
-                            float(row.open_fee),
-                            float(row.open_fee_rate),
-                            int(row.open_time),
-                            ins_id,
-                            avg_price,
-                            fee,
-                            fee_rate,
-                            current_time,
+                            position_id=position_id,
+                            exchange=exchange,
+                            symbol=symbol,
+                            position_side=position_side,
+                            lever=row.lever,
+                            coin_quantity=float(row.coin_quantity),
+                            usdt_quantity=float(row.usdt_quantity),
+                            open_ins_id=row.open_ins_id,
+                            open_price=float(row.open_price),
+                            open_fee=float(row.open_fee),
+                            open_fee_rate=float(row.open_fee_rate),
+                            open_time=int(row.open_time),
+                            close_ins_id=ins_id,
+                            close_price=avg_price,
+                            close_fee=fee,
+                            close_fee_rate=fee_rate,
+                            close_time=current_time,
+                            is_spot=row.is_spot,
                         )
                     except:
                         pass

kaq_quant_common-0.2.15/kaq_quant_common/api/rest/instruction/models/loan.py

@@ -0,0 +1,22 @@
+from typing import Optional
+
+from kaq_quant_common.api.rest.instruction.models import (
+    InstructionRequestBase, InstructionResponseBase)
+
+# -----------------------------借贷-------------------
+
+
+# ↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓ 活期借币
+class QueryFlexibleLoanAssetRequest(InstructionRequestBase):
+    # 币种代码
+    coin: str
+
+class QueryFlexibleLoanAssetResponse(InstructionResponseBase):
+    # 币种代码
+    coin: str
+    # 活期借币利率
+    interestRate: Optional[float] = None
+    # 最低可借数量
+    minLimit: Optional[float] = None
+    # 最高可借数量
+    maxLimit: Optional[float] = None

kaq_quant_common-0.2.15/kaq_quant_common/common/modules/limit_order_helper.py

@@ -0,0 +1,308 @@
+"""
+LimitOrderHelper - 高性能数据批量写入助手
+
+使用独立事件循环的 asyncio 实现，消除 GIL 影响，提升性能。
+
+特性：
+- 外部同步接口，内部 asyncio 实现
+- 独立线程运行事件循环，无 GIL 竞争
+- 适配所有交易所 SDK（同步/异步）
+- 线程安全的跨线程调用
+"""
+
+import asyncio
+import datetime
+import threading
+from typing import Callable, Optional
+
+import pandas as pd
+from kaq_quant_common.resources.kaq_ddb_stream_write_resources import (
+    KaqQuantDdbStreamMTWWriteRepository,
+    KaqQuantDdbStreamWriteRepository,
+)
+from kaq_quant_common.utils import logger_utils
+
+
+class LimitOrderHelper:
+    """
+    使用独立事件循环的高性能数据批量写入助手
+
+    架构说明：
+    1. 外部通过同步方法 push_data() 推送数据（适配任意 SDK）
+    2. 内部使用独立线程运行 asyncio 事件循环
+    3. 数据通过 asyncio.Queue 缓冲，定时批量写入数据库
+    4. 消除 GIL 锁竞争，提升高频场景性能
+    """
+
+    def __init__(
+        self,
+        ddb: KaqQuantDdbStreamWriteRepository | KaqQuantDdbStreamMTWWriteRepository,
+        ddb_table_name: str,
+        _flush_interval_ms: int = 100,
+        max_queue_size: int = 10000,
+    ):
+        """
+        初始化 LimitOrderHelper
+
+        Args:
+            ddb: DDB 数据库连接（支持普通或 MTW 版本）
+            ddb_table_name: 数据库表名
+            _flush_interval_ms: 刷新间隔（毫秒），默认 100ms
+            max_queue_size: 队列最大容量，默认 10000
+        """
+        # DDB 相关
+        self._ddb = ddb
+        self._isMtwDdb = isinstance(self._ddb, KaqQuantDdbStreamMTWWriteRepository)
+        self._ddb_table_name = ddb_table_name
+
+        # 配置参数
+        self._flush_interval_ms = _flush_interval_ms
+        self._max_queue_size = max_queue_size
+
+        # 异步组件（将在后台线程中创建）
+        self._loop: Optional[asyncio.AbstractEventLoop] = None
+        self._data_queue: Optional[asyncio.Queue] = None
+        self._flusher_task: Optional[asyncio.Task] = None
+
+        # 线程控制
+        self._event_loop_thread: Optional[threading.Thread] = None
+        self._stop_event = threading.Event()
+        self._started = threading.Event()
+
+        # 日志和回调
+        self._logger = logger_utils.get_logger(self)
+        self._build_data: Optional[Callable] = None
+
+    def set_build_data(self, build_data: Callable):
+        """
+        设置数据构建函数
+
+        Args:
+            build_data: 回调函数，签名为 (symbol, data, arg) -> list | DataFrame
+        """
+        self._build_data = build_data
+
+    def push_data(self, symbol: str, data: dict, arg: dict = None):
+        """
+        同步接口：推送数据到队列
+
+        该方法是线程安全的，可从任何线程调用。
+        内部使用 call_soon_threadsafe 将数据投递到异步队列。
+
+        Args:
+            symbol: 交易对符号
+            data: 数据字典
+            arg: 可选参数
+        """
+        if not self._started.is_set():
+            self._logger.warning("Helper 未启动，数据被丢弃")
+            return
+
+        # 使用 call_soon_threadsafe 将数据投递到异步队列
+        self._loop.call_soon_threadsafe(self._async_push_data, symbol, data, arg)
+
+    def _async_push_data(self, symbol: str, data: dict, arg: dict):
+        """
+        内部方法：异步推送数据（在事件循环线程中执行）
+
+        Args:
+            symbol: 交易对符号
+            data: 数据字典
+            arg: 可选参数
+        """
+        try:
+            # 非阻塞推送
+            self._data_queue.put_nowait((symbol, data, arg))
+        except asyncio.QueueFull:
+            self._logger.warning(f"队列已满 ({self._max_queue_size})，丢弃 {symbol} 数据")
+
+    def start(self):
+        """启动后台线程和事件循环"""
+        if self._event_loop_thread is not None:
+            self._logger.warning("Helper 已经启动")
+            return
+
+        self._stop_event.clear()
+
+        # 启动后台线程
+        self._event_loop_thread = threading.Thread(target=self._run_event_loop, daemon=True, name="LimitOrderHelperAsyncThread")
+        self._event_loop_thread.start()
+
+        # 等待事件循环就绪
+        self._started.wait(timeout=5.0)
+
+        if not self._started.is_set():
+            raise RuntimeError("事件循环启动超时")
+
+        self._logger.info("LimitOrderHelper 启动成功（asyncio 模式）")
+
+    def stop(self):
+        """优雅停止"""
+        if not self._started.is_set():
+            return
+
+        self._logger.info("正在停止 LimitOrderHelper...")
+
+        # 设置停止标志
+        self._stop_event.set()
+
+        # 如果事件循环正在运行，停止它
+        if self._loop and self._loop.is_running():
+            self._loop.call_soon_threadsafe(self._loop.stop)
+
+        # 等待线程结束
+        if self._event_loop_thread:
+            self._event_loop_thread.join(timeout=10.0)
+            if self._event_loop_thread.is_alive():
+                self._logger.error("事件循环线程未能正常退出")
+
+        self._logger.info("LimitOrderHelper 已停止")
+
+    def _run_event_loop(self):
+        """
+        在独立线程中运行事件循环
+        """
+        try:
+            # 创建新的事件循环
+            self._loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(self._loop)
+
+            # 创建异步队列
+            self._data_queue = asyncio.Queue(maxsize=self._max_queue_size)
+
+            # 创建后台刷新任务
+            self._flusher_task = self._loop.create_task(self._flush_loop())
+
+            # 标记已启动
+            self._started.set()
+
+            # 运行事件循环
+            self._loop.run_forever()
+
+        except Exception as e:
+            self._logger.error(f"事件循环异常: {e}", exc_info=True)
+        finally:
+            # 清理
+            try:
+                # 取消所有任务
+                pending = asyncio.all_tasks(self._loop)
+                for task in pending:
+                    task.cancel()
+
+                # 等待任务完成
+                if pending:
+                    self._loop.run_until_complete(asyncio.gather(*pending, return_exceptions=True))
+
+                # 关闭事件循环
+                self._loop.close()
+            except Exception as e:
+                self._logger.error(f"清理事件循环时出错: {e}")
+
+    async def _flush_loop(self):
+        """
+        异步刷新循环（在事件循环线程中执行）
+
+        定时从队列批量获取数据并写入数据库。
+        """
+        cum_count = 0
+        cum_total_use_time = 0
+
+        try:
+            while not self._stop_event.is_set():
+                # 批量收集数据
+                batch = {}
+                deadline = asyncio.get_event_loop().time() + self._flush_interval_ms / 1000.0
+
+                # 在时间窗口内尽可能多地收集数据
+                while asyncio.get_event_loop().time() < deadline:
+                    try:
+                        remaining_time = deadline - asyncio.get_event_loop().time()
+                        if remaining_time <= 0:
+                            break
+
+                        # 从队列获取数据（带超时）
+                        symbol, data, arg = await asyncio.wait_for(self._data_queue.get(), timeout=remaining_time)
+
+                        # 只保留每个 symbol 的最新数据（去重）
+                        batch[symbol] = (data, arg)
+
+                    except asyncio.TimeoutError:
+                        break
+
+                # 如果有数据，批量写入
+                if batch:
+                    start_time = datetime.datetime.now()
+
+                    try:
+                        await self._write_to_db(batch)
+                    except Exception as e:
+                        self._logger.error(f"批量写入失败: {e}", exc_info=True)
+
+                    # 统计
+                    end_time = datetime.datetime.now()
+                    total_use_time = (end_time - start_time).total_seconds() * 1000
+
+                    cum_count += len(batch)
+                    cum_total_use_time += total_use_time
+
+                    # if total_use_time > 500:
+                    #     self._logger.debug(
+                    #         f"批量写入 {len(batch)} 条数据耗时 {total_use_time:.2f}ms "
+                    #         f"(avg {cum_total_use_time / cum_count:.2f}ms)"
+                    #     )
+                else:
+                    # 没有数据时短暂休眠
+                    await asyncio.sleep(0.01)
+
+        except asyncio.CancelledError:
+            self._logger.info("刷新循环被取消")
+        except Exception as e:
+            self._logger.error(f"刷新循环异常: {e}", exc_info=True)
+
+    async def _write_to_db(self, batch: dict):
+        """
+        写入数据库（异步）
+
+        Args:
+            batch: 批量数据字典，格式为 {symbol: (data, arg)}
+        """
+        # 转换数据
+        df: Optional[pd.DataFrame] = None
+        list_data: list = []
+
+        for symbol, (data, arg) in batch.items():
+            sub_data = self._build_data(symbol, data, arg)
+
+            if sub_data is None or len(sub_data) == 0:
+                continue
+
+            if not self._isMtwDdb:
+                # DataFrame 方式
+                if isinstance(sub_data, pd.DataFrame):
+                    df = sub_data if df is None else pd.concat([df, sub_data], ignore_index=True)
+                # List 方式
+                else:
+                    if isinstance(sub_data[0], list):
+                        list_data.extend(sub_data)
+                    else:
+                        list_data.append(sub_data)
+            else:
+                # MTW 直接写入（使用 to_thread 包装同步调用）
+                try:
+                    await asyncio.to_thread(self._ddb.save2stream_list, sub_data)
+                except Exception as e:
+                    self._logger.error(f"MTW 写入失败: {e}")
+
+        # 批量写入（使用 to_thread 包装同步 DDB 调用）
+        if not self._isMtwDdb:
+            if df is not None and not df.empty:
+                try:
+                    await asyncio.to_thread(self._ddb.save2stream_batch, self._ddb_table_name, df=df)
+                except Exception as e:
+                    self._logger.error(f"批量写入 DataFrame 失败: {e}")
+
+            if list_data:
+                try:
+                    await asyncio.to_thread(self._ddb.save2stream_batch_list, self._ddb_table_name, data=list_data)
+                except Exception as e:
+                    self._logger.error(f"批量写入 List 失败: {e}")

{kaq_quant_common-0.2.13 → kaq_quant_common-0.2.15}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "kaq_quant_common"
-version = "0.2.13"
+version = "0.2.15"
 description = ""
 authors = [
     {name = "kevinfuture",email = "liuenbofuture@foxmail.com"}

kaq_quant_common-0.2.13/kaq_quant_common/common/modules/limit_order_helper.py

@@ -1,160 +0,0 @@
-# 避免写入导致阻塞
-import datetime
-import threading
-import time
-
-import pandas as pd
-from kaq_quant_common.resources.kaq_ddb_stream_write_resources import (
-    KaqQuantDdbStreamMTWWriteRepository,
-    KaqQuantDdbStreamWriteRepository,
-)
-from kaq_quant_common.utils import logger_utils
-
-
-class LimitOrderHelper:
-
-    def __init__(
-        self, ddb: KaqQuantDdbStreamWriteRepository | KaqQuantDdbStreamMTWWriteRepository, ddb_table_name: str, _flush_interval_ms: int = 100
-    ):
-        # 最新快照缓存与刷库线程控制
-        self._latest_snapshots: dict[str, tuple] = {}
-        self._latest_lock = threading.Lock()
-        # 写入到ddb的频率，默认100ms
-        self._flush_interval_ms = _flush_interval_ms
-        self._stop_event = threading.Event()
-        self._flusher_thread = threading.Thread(target=self._flush_loop, daemon=True)
-        self._flusher_thread.name = "LimitOrderHelperFlusherThread"
-
-        #
-        self._ddb = ddb
-        self._isMtwDdb = isinstance(self._ddb, KaqQuantDdbStreamMTWWriteRepository)
-        self._ddb_table_name = ddb_table_name
-
-        #
-        self._logger = logger_utils.get_logger(self)
-
-        self._build_data: callable = None
-
-    def set_build_data(self, build_data: callable):
-        self._build_data = build_data
-
-    def push_data(self, symbol: str, data: dict, arg: dict = None):
-        with self._latest_lock:
-            self._latest_snapshots[symbol] = (data, arg)
-
-    def start(self):
-        self._flusher_thread.start()
-
-    def stop(self):
-        self._stop_event.set()
-        self._flusher_thread.join()
-
-    def _flush_loop(self):
-        cum_count = 0
-        cum_convert_time = 0
-        cum_write_ddb_time = 0
-        cum_total_use_time = 0
-        # 周期性地将每个symbol的最新快照批量入库
-        while not self._stop_event.is_set():
-            to_process = None
-            with self._latest_lock:
-                if self._latest_snapshots:
-                    to_process = list(self._latest_snapshots.items())
-                    self._latest_snapshots.clear()
-
-            if to_process:
-                df: pd.DataFrame = None
-                list_data: list = []
-                now = int(datetime.datetime.now().timestamp() * 1000)
-
-                for symbol, (data, arg) in to_process:
-                    sub_data = self._build_data(symbol, data, arg)
-
-                    if sub_data is None:
-                        continue
-
-                    if len(sub_data) == 0:
-                        continue
-
-                    if not self._isMtwDdb:
-                        # 可以是数组，可以是dataFrame
-                        is_df = type(sub_data) is pd.DataFrame
-
-                        if is_df:
-                            # df就用df的方式写入
-                            data_first_now = int(sub_data["create_time"].iloc[0])
-                            # if now - data_first_now > 2000:
-                            #     self._logger.debug(f"数据时间{data_first_now} 与当前时间{now} 差值{now - data_first_now} 超过2000ms")
-                            #     pass
-
-                            if df is None:
-                                df = sub_data
-                            else:
-                                df = pd.concat([df, sub_data], ignore_index=True)
-                        else:
-                            # 数组就用数组的方式写入
-                            # 子元素是否数组
-                            is_sub_list = type(sub_data[0]) is list
-                            if is_sub_list:
-                                # 多条数据
-                                data_first_now = int(sub_data[0][0])
-                                # if now - data_first_now > 2000:
-                                #     self._logger.debug(f"数据时间{data_first_now} 与当前时间{now} 差值{now - data_first_now} 超过2000ms")
-                                #     pass
-                                list_data.extend(sub_data)
-                            else:
-                                # 单条数据
-                                data_first_now = int(sub_data[0])
-                                # if now - data_first_now > 2000:
-                                #     self._logger.debug(f"数据时间{data_first_now} 与当前时间{now} 差值{now - data_first_now} 超过2000ms")
-                                #     pass
-                                list_data.append(sub_data)
-
-                    else:
-                        # 直接调用 save2stream_list 写入
-                        try:
-                            self._ddb.save2stream_list(sub_data)
-                        except Exception as e:
-                            # 避免刷库异常导致线程退出
-                            self._logger.error(f"批量写入数组失败: {e}")
-
-                convert_time = int(datetime.datetime.now().timestamp() * 1000)
-
-                # 入库
-                if not self._isMtwDdb:
-                    # 兼容df和数组
-                    if df is not None and not df.empty:
-                        try:
-                            self._ddb.save2stream_batch(self._ddb_table_name, df=df)
-                        except Exception as e:
-                            # 避免刷库异常导致线程退出
-                            self._logger.error(f"批量写入df失败: {e}")
-                    if len(list_data) > 0:
-                        try:
-                            self._ddb.save2stream_batch_list(self._ddb_table_name, data=list_data)
-                        except Exception as e:
-                            # 避免刷库异常导致线程退出
-                            self._logger.error(f"批量写入list失败: {e}")
-
-                # 统计一下
-                end = int(datetime.datetime.now().timestamp() * 1000)
-                total_use_time = end - now
-                convert_use = convert_time - now
-                write_ddb_use = total_use_time - convert_use
-
-                #
-                cum_count += len(to_process)
-                cum_convert_time += convert_use
-                cum_write_ddb_time += write_ddb_use
-                cum_total_use_time += total_use_time
-
-                # if total_use_time > 500 and cum_count > 0:
-                #     self._logger.debug(
-                #         f"批量写入{len(to_process)}条数据耗时{total_use_time}ms(avg {cum_total_use_time / cum_count:.2f}ms) 转换耗时{convert_use}ms(avg {cum_convert_time / cum_count:.2f}ms) 写入ddb耗时{write_ddb_use}ms(avg {cum_write_ddb_time / cum_count:.2f}ms)"
-                #     )
-
-            if not self._isMtwDdb:
-                time.sleep(self._flush_interval_ms / 1000.0)
-            else:
-                # mtw交由ddb自己控制节奏
-                time.sleep(50 / 1000.0)