kaq-quant-common 0.2.3__py3-none-any.whl → 0.2.5__py3-none-any.whl

kaq_quant_common/common/modules/funding_rate_helper.py

@@ -0,0 +1,103 @@
+# 避免写入导致阻塞
+import datetime
+import threading
+import time
+
+import pandas as pd
+
+from kaq_quant_common.resources.kaq_ddb_stream_write_resources import (
+    KaqQuantDdbStreamMTWWriteRepository, KaqQuantDdbStreamWriteRepository)
+from kaq_quant_common.utils import logger_utils
+
+
+class FundingRateHelper:
+
+    def __init__(
+        self,
+        ddb: KaqQuantDdbStreamWriteRepository | KaqQuantDdbStreamMTWWriteRepository,
+        ddb_table_name: str,
+    ):
+        # 最新快照缓存与刷库线程控制
+        self._latest_snapshots: dict[str, tuple] = {}
+        self._latest_lock = threading.Lock()
+        # 写入到ddb的频率，默认1s
+        self._flush_interval_ms = 1000
+        self._stop_event = threading.Event()
+        self._flusher_thread = threading.Thread(target=self._flush_loop, daemon=True)
+        self._flusher_thread.name = "FundingRateHelperFlusherThread"
+        self._is_df = False
+
+        #
+        self._ddb = ddb
+        self._ddb_table_name = ddb_table_name
+
+        #
+        self._logger = logger_utils.get_logger(self)
+
+        self._build_data: callable = None
+
+    def set_build_data(self, build_data: callable):
+        self._build_data = build_data
+
+    def push_data(self, symbol: str, data: dict, arg: dict = None):
+        with self._latest_lock:
+            self._latest_snapshots[symbol] = (data, arg)
+
+    def start(self):
+        self._flusher_thread.start()
+
+    def stop(self):
+        self._stop_event.set()
+        self._flusher_thread.join()
+
+    def _flush_loop(self):
+        # 周期性地将每个symbol的最新快照批量入库
+        while not self._stop_event.is_set():
+            to_process = None
+            with self._latest_lock:
+                if self._latest_snapshots:
+                    to_process = list(self._latest_snapshots.items())
+                    self._latest_snapshots.clear()
+
+            if to_process:
+                df: pd.DataFrame = None
+                list_data: list = []
+                now = int(datetime.datetime.now().timestamp() * 1000)
+
+                for symbol, (data, arg) in to_process:
+                    sub_df = self._build_data(symbol, data, arg)
+                    if isinstance(sub_df, pd.DataFrame):
+                        self._is_df = True
+                        # 输出一下
+                        data_first_now = int(sub_df["create_time"].iloc[0])
+                        if now - data_first_now > 2000:
+                            self._logger.debug(
+                                f"数据时间{data_first_now} 与当前时间{now} 差值{now - data_first_now} 超过2000ms"
+                            )
+
+                        if df is None:
+                            df = sub_df
+                        else:
+                            df = pd.concat([df, sub_df], ignore_index=True)
+                    else:
+                        list_data = sub_df
+
+                # 入库
+                if self._is_df:
+                    if df is not None and not df.empty:
+                        try:
+                            self._ddb.save2stream_batch(self._ddb_table_name, df=df)
+                        except Exception as e:
+                            # 避免刷库异常导致线程退出
+                            self._logger.error(f"批量写入df失败: {e}")
+                else:
+                    if len(list_data) > 0:
+                        try:
+                            self._ddb.save2stream_list(list_data)
+                        except Exception as e:
+                            # 避免刷库异常导致线程退出
+                            self._logger.error(f"批量写入数组失败: {e}")
+
+            # 是dataFrame的才睡眠，数组处理交由ddb自己控制节奏
+            if self._is_df:
+                time.sleep(self._flush_interval_ms / 1000.0)

kaq_quant_common/common/modules/limit_order_helper.py

@@ -4,15 +4,19 @@ import threading
 import time
 
 import pandas as pd
+
 from kaq_quant_common.resources.kaq_ddb_stream_write_resources import (
-    KaqQuantDdbStreamWriteRepository,
-)
+    KaqQuantDdbStreamMTWWriteRepository, KaqQuantDdbStreamWriteRepository)
 from kaq_quant_common.utils import logger_utils
 
 
 class LimitOrderHelper:
 
-    def __init__(self, ddb: KaqQuantDdbStreamWriteRepository, ddb_table_name: str):
+    def __init__(
+        self,
+        ddb: KaqQuantDdbStreamWriteRepository | KaqQuantDdbStreamMTWWriteRepository,
+        ddb_table_name: str,
+    ):
         # 最新快照缓存与刷库线程控制
         self._latest_snapshots: dict[str, tuple] = {}
         self._latest_lock = threading.Lock()
@@ -21,6 +25,7 @@ class LimitOrderHelper:
         self._stop_event = threading.Event()
         self._flusher_thread = threading.Thread(target=self._flush_loop, daemon=True)
         self._flusher_thread.name = "LimitOrderHelperFlusherThread"
+        self._is_df = False
 
         #
         self._ddb = ddb
@@ -56,26 +61,43 @@ class LimitOrderHelper:
 
             if to_process:
                 df: pd.DataFrame = None
+                list_data: list = []
                 now = int(datetime.datetime.now().timestamp() * 1000)
 
                 for symbol, (data, arg) in to_process:
                     sub_df = self._build_data(symbol, data, arg)
-                    # 输出一下
-                    data_first_now = int(sub_df["create_time"].iloc[0])
-                    if now - data_first_now > 2000:
-                        self._logger.debug(f"数据时间{data_first_now} 与当前时间{now} 差值{now - data_first_now} 超过2000ms")
-
-                    if df is None:
-                        df = sub_df
+                    if isinstance(sub_df, pd.DataFrame):
+                        self._is_df = True
+                        # 输出一下
+                        data_first_now = int(sub_df["create_time"].iloc[0])
+                        if now - data_first_now > 2000:
+                            self._logger.debug(
+                                f"数据时间{data_first_now} 与当前时间{now} 差值{now - data_first_now} 超过2000ms"
+                            )
+
+                        if df is None:
+                            df = sub_df
+                        else:
+                            df = pd.concat([df, sub_df], ignore_index=True)
                     else:
-                        df = pd.concat([df, sub_df], ignore_index=True)
+                        list_data = sub_df
 
                 # 入库
-                if df is not None and not df.empty:
-                    try:
-                        self._ddb.save2stream_batch(self._ddb_table_name, df=df)
-                    except Exception as e:
-                        # 避免刷库异常导致线程退出
-                        self._logger.error(f"批量写入失败: {e}")
-
-            time.sleep(self._flush_interval_ms / 1000.0)
+                if self._is_df:
+                    if df is not None and not df.empty:
+                        try:
+                            self._ddb.save2stream_batch(self._ddb_table_name, df=df)
+                        except Exception as e:
+                            # 避免刷库异常导致线程退出
+                            self._logger.error(f"批量写入df失败: {e}")
+                else:
+                    if len(list_data) > 0:
+                        try:
+                            self._ddb.save2stream_list(list_data)
+                        except Exception as e:
+                            # 避免刷库异常导致线程退出
+                            self._logger.error(f"批量写入数组失败: {e}")
+
+            # 是dataFrame的才睡眠，数组处理交由ddb自己控制节奏
+            if self._is_df:
+                time.sleep(self._flush_interval_ms / 1000.0)

kaq_quant_common/resources/kaq_ddb_pool_stream_read_resources.py

@@ -22,7 +22,7 @@ class KaqQuantDdbPoolStreamReadRepository:
             
             # 连接地址为localhost，端口为8848的DolphinDB，登录用户名为admin，密码为123456的账户，连接数为8
             self.pool = ddb.DBConnectionPool(host, port, pool_size, user, passwd, 
-                                             loadBalance=True, 
+                                             loadBalance=False, 
                                              compress=True, 
                                              protocol=protocal,
                                              reConnect=True, 

kaq_quant_common/resources/kaq_ddb_stream_write_resources.py

@@ -29,7 +29,7 @@ class KaqQuantDdbStreamWriteRepository:
             self.session.connect(host, port, user, passwd, tryReconnectNums=10, reconnect=True, keepAliveTime=1000, readTimeout=10, writeTimeout=5)
             # 流表订阅用的
             # self.session.enableStreaming(threadCount=5)
-            # self.pool = ddb.DBConnectionPool(host, port, userid=user, password=passwd, loadBalance=True, reConnect=True, tryReconnectNums=5, sqlStd=SqlStd.MySQL)
+            # self.pool = ddb.DBConnectionPool(host, port, userid=user, password=passwd, loadBalance=False, reConnect=True, tryReconnectNums=5, sqlStd=SqlStd.MySQL)
 
             # 需要注意的是 fetchSize 取值不能小于 8192 （记录条数）
             self.size = 8192

{kaq_quant_common-0.2.3.dist-info → kaq_quant_common-0.2.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kaq_quant_common
-Version: 0.2.3
+Version: 0.2.5
 Summary: 
 Author: kevinfuture
 Author-email: liuenbofuture@foxmail.com

{kaq_quant_common-0.2.3.dist-info → kaq_quant_common-0.2.5.dist-info}/RECORD

@@ -27,7 +27,8 @@ kaq_quant_common/api/ws/ws_server_base.py,sha256=-JFA5fnYHXPYBZ09aZmhYuhgDHFfJbk
 kaq_quant_common/common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 kaq_quant_common/common/ddb_table_monitor.py,sha256=7Yihz_uGGujo_QqqPl45Gp8fwUMMw1auXx5egbzyYlE,3662
 kaq_quant_common/common/http_monitor.py,sha256=_yChiwfVv1c5g_lKgYUjWY40fX61BWVK8SL4kXwRfwk,2375
-kaq_quant_common/common/modules/limit_order_helper.py,sha256=Pbh1RHapr9D0Uacup5oQqeXgfI72zLAG2vZHEjPgoPY,2894
+kaq_quant_common/common/modules/funding_rate_helper.py,sha256=BVGNS6E7aVPITOz5u4g7vBUHOrm1S1Uri8psgWxp-BQ,3853
+kaq_quant_common/common/modules/limit_order_helper.py,sha256=Py9DPBXq1-QnjEh6mGQOA7vD3TVNUabnS_rMf0SQx5U,3853
 kaq_quant_common/common/modules/limit_order_symbol_monitor.py,sha256=TBK48qyeCSQvkfDMv3J_0UM7f3OuBRKRFYDcL9kG6Cs,2876
 kaq_quant_common/common/modules/limit_order_symbol_monitor_group.py,sha256=oEqHIwxhqAzckmluHJHZHiHUNmAyaS2JyK2nXO58UhY,2394
 kaq_quant_common/common/monitor_base.py,sha256=E4EUMsO3adNltCDNRgxkvUSbTTfKOL9S1zzN3WkZvpU,2467
@@ -38,10 +39,10 @@ kaq_quant_common/common/statistics/kline_history_statistics.py,sha256=pQgWkP7Z0n
 kaq_quant_common/common/ws_wrapper.py,sha256=JNJ0CIjDXgCsRjOLSbCi7ysYDHw7tT_aK7V4ADqw3vA,452
 kaq_quant_common/config/config.yaml,sha256=ST_QBLo7kwVaoNOvuN3mpeSF7LPNSWdD7EjxrBYZYBs,230
 kaq_quant_common/resources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-kaq_quant_common/resources/kaq_ddb_pool_stream_read_resources.py,sha256=Y-rtGpBF3bNh2mPZ2Lze6D9JZnpkqh24OEh7okikh1w,2536
+kaq_quant_common/resources/kaq_ddb_pool_stream_read_resources.py,sha256=q4P96rSrEcWn9ki09UD0vw00iFq_bpgOFTrRzVG7uCA,2537
 kaq_quant_common/resources/kaq_ddb_stream_init_resources.py,sha256=0MxxbajocTFzcRD344rfZQPKOwwuqCbyoT6dQpIox-o,3349
 kaq_quant_common/resources/kaq_ddb_stream_read_resources.py,sha256=WShsXMoL8o-JZvrtAd7H2Cg-vrE47QbsdGgURaQwiZs,3165
-kaq_quant_common/resources/kaq_ddb_stream_write_resources.py,sha256=_-bNnxnG5I98JOlV46FYXlYoJRkzlMzWREHUonornF4,12638
+kaq_quant_common/resources/kaq_ddb_stream_write_resources.py,sha256=Ql8pqZDzbC_4pZCVgfiW09Vbsnc8qC_iTJpjBE9xjD8,12639
 kaq_quant_common/resources/kaq_mysql_init_resources.py,sha256=UcqWey6LgoMqvLq1SxK33nS6-rkViGYhzUPxcrucOks,827
 kaq_quant_common/resources/kaq_mysql_resources.py,sha256=282jpXvYlEQNx-hicYTNBHDii85KYgN7BQQSMS9aPFM,13211
 kaq_quant_common/resources/kaq_postgresql_resources.py,sha256=iG1eYkciI0xUIBdEpGqKGOLBFxvVrfbBoTuaOmhQ0v0,1762
@@ -61,6 +62,6 @@ kaq_quant_common/utils/signal_utils.py,sha256=zBSyEltNTKqkQCsrETd47kEBb3Q_OWUBUn
 kaq_quant_common/utils/sqlite_utils.py,sha256=UDDFKfwL0N-jFifl40HdyOCENh2YQfW5so6hRaSJpv0,5722
 kaq_quant_common/utils/uuid_utils.py,sha256=pm_pnXpd8n9CI66x3A20cOEUiriJyqHaKGCeLrgkBxU,71
 kaq_quant_common/utils/yml_utils.py,sha256=gcKjb_-uuUajBGAl5QBPIZTg2wXm7qeeJvtHflj_zOE,4513
-kaq_quant_common-0.2.3.dist-info/METADATA,sha256=wk6Okdv4qKVJxUnM54tR6sTtw4zMR2Qicwv5NZOXsXk,1970
-kaq_quant_common-0.2.3.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
-kaq_quant_common-0.2.3.dist-info/RECORD,,
+kaq_quant_common-0.2.5.dist-info/METADATA,sha256=fW-ErJTmmEIBIGdLBabB7xZNNaUrQDyRlonF6-pkdEI,1970
+kaq_quant_common-0.2.5.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
+kaq_quant_common-0.2.5.dist-info/RECORD,,