kaq-quant-common 0.2.3__py3-none-any.whl → 0.2.4__py3-none-any.whl

kaq_quant_common/common/modules/funding_rate_helper.py

@@ -0,0 +1,103 @@
+# 避免写入导致阻塞
+import datetime
+import threading
+import time
+
+import pandas as pd
+
+from kaq_quant_common.resources.kaq_ddb_stream_write_resources import (
+    KaqQuantDdbStreamMTWWriteRepository, KaqQuantDdbStreamWriteRepository)
+from kaq_quant_common.utils import logger_utils
+
+
+class FundingRateHelper:
+
+    def __init__(
+        self,
+        ddb: KaqQuantDdbStreamWriteRepository | KaqQuantDdbStreamMTWWriteRepository,
+        ddb_table_name: str,
+    ):
+        # 最新快照缓存与刷库线程控制
+        self._latest_snapshots: dict[str, tuple] = {}
+        self._latest_lock = threading.Lock()
+        # 写入到ddb的频率，默认1s
+        self._flush_interval_ms = 1000
+        self._stop_event = threading.Event()
+        self._flusher_thread = threading.Thread(target=self._flush_loop, daemon=True)
+        self._flusher_thread.name = "FundingRateHelperFlusherThread"
+        self._is_df = False
+
+        #
+        self._ddb = ddb
+        self._ddb_table_name = ddb_table_name
+
+        #
+        self._logger = logger_utils.get_logger(self)
+
+        self._build_data: callable = None
+
+    def set_build_data(self, build_data: callable):
+        self._build_data = build_data
+
+    def push_data(self, symbol: str, data: dict, arg: dict = None):
+        with self._latest_lock:
+            self._latest_snapshots[symbol] = (data, arg)
+
+    def start(self):
+        self._flusher_thread.start()
+
+    def stop(self):
+        self._stop_event.set()
+        self._flusher_thread.join()
+
+    def _flush_loop(self):
+        # 周期性地将每个symbol的最新快照批量入库
+        while not self._stop_event.is_set():
+            to_process = None
+            with self._latest_lock:
+                if self._latest_snapshots:
+                    to_process = list(self._latest_snapshots.items())
+                    self._latest_snapshots.clear()
+
+            if to_process:
+                df: pd.DataFrame = None
+                list_data: list = []
+                now = int(datetime.datetime.now().timestamp() * 1000)
+
+                for symbol, (data, arg) in to_process:
+                    sub_df = self._build_data(symbol, data, arg)
+                    if isinstance(sub_df, pd.DataFrame):
+                        self._is_df = True
+                        # 输出一下
+                        data_first_now = int(sub_df["create_time"].iloc[0])
+                        if now - data_first_now > 2000:
+                            self._logger.debug(
+                                f"数据时间{data_first_now} 与当前时间{now} 差值{now - data_first_now} 超过2000ms"
+                            )
+
+                        if df is None:
+                            df = sub_df
+                        else:
+                            df = pd.concat([df, sub_df], ignore_index=True)
+                    else:
+                        list_data = sub_df
+
+                # 入库
+                if self._is_df:
+                    if df is not None and not df.empty:
+                        try:
+                            self._ddb.save2stream_batch(self._ddb_table_name, df=df)
+                        except Exception as e:
+                            # 避免刷库异常导致线程退出
+                            self._logger.error(f"批量写入df失败: {e}")
+                else:
+                    if len(list_data) > 0:
+                        try:
+                            self._ddb.save2stream_list(list_data)
+                        except Exception as e:
+                            # 避免刷库异常导致线程退出
+                            self._logger.error(f"批量写入数组失败: {e}")
+
+            # 是dataFrame的才睡眠，数组处理交由ddb自己控制节奏
+            if self._is_df:
+                time.sleep(self._flush_interval_ms / 1000.0)

kaq_quant_common/common/modules/limit_order_helper.py

@@ -4,15 +4,19 @@ import threading
 import time
 
 import pandas as pd
+
 from kaq_quant_common.resources.kaq_ddb_stream_write_resources import (
-    KaqQuantDdbStreamWriteRepository,
-)
+    KaqQuantDdbStreamMTWWriteRepository, KaqQuantDdbStreamWriteRepository)
 from kaq_quant_common.utils import logger_utils
 
 
 class LimitOrderHelper:
 
-    def __init__(self, ddb: KaqQuantDdbStreamWriteRepository, ddb_table_name: str):
+    def __init__(
+        self,
+        ddb: KaqQuantDdbStreamWriteRepository | KaqQuantDdbStreamMTWWriteRepository,
+        ddb_table_name: str,
+    ):
         # 最新快照缓存与刷库线程控制
         self._latest_snapshots: dict[str, tuple] = {}
         self._latest_lock = threading.Lock()
@@ -21,6 +25,7 @@ class LimitOrderHelper:
         self._stop_event = threading.Event()
         self._flusher_thread = threading.Thread(target=self._flush_loop, daemon=True)
         self._flusher_thread.name = "LimitOrderHelperFlusherThread"
+        self._is_df = False
 
         #
         self._ddb = ddb
@@ -56,26 +61,43 @@ class LimitOrderHelper:
 
             if to_process:
                 df: pd.DataFrame = None
+                list_data: list = []
                 now = int(datetime.datetime.now().timestamp() * 1000)
 
                 for symbol, (data, arg) in to_process:
                     sub_df = self._build_data(symbol, data, arg)
-                    # 输出一下
-                    data_first_now = int(sub_df["create_time"].iloc[0])
-                    if now - data_first_now > 2000:
-                        self._logger.debug(f"数据时间{data_first_now} 与当前时间{now} 差值{now - data_first_now} 超过2000ms")
-
-                    if df is None:
-                        df = sub_df
+                    if isinstance(sub_df, pd.DataFrame):
+                        self._is_df = True
+                        # 输出一下
+                        data_first_now = int(sub_df["create_time"].iloc[0])
+                        if now - data_first_now > 2000:
+                            self._logger.debug(
+                                f"数据时间{data_first_now} 与当前时间{now} 差值{now - data_first_now} 超过2000ms"
+                            )
+
+                        if df is None:
+                            df = sub_df
+                        else:
+                            df = pd.concat([df, sub_df], ignore_index=True)
                     else:
-                        df = pd.concat([df, sub_df], ignore_index=True)
+                        list_data = sub_df
 
                 # 入库
-                if df is not None and not df.empty:
-                    try:
-                        self._ddb.save2stream_batch(self._ddb_table_name, df=df)
-                    except Exception as e:
-                        # 避免刷库异常导致线程退出
-                        self._logger.error(f"批量写入失败: {e}")
-
-            time.sleep(self._flush_interval_ms / 1000.0)
+                if self._is_df:
+                    if df is not None and not df.empty:
+                        try:
+                            self._ddb.save2stream_batch(self._ddb_table_name, df=df)
+                        except Exception as e:
+                            # 避免刷库异常导致线程退出
+                            self._logger.error(f"批量写入df失败: {e}")
+                else:
+                    if len(list_data) > 0:
+                        try:
+                            self._ddb.save2stream_list(list_data)
+                        except Exception as e:
+                            # 避免刷库异常导致线程退出
+                            self._logger.error(f"批量写入数组失败: {e}")
+
+            # 是dataFrame的才睡眠，数组处理交由ddb自己控制节奏
+            if self._is_df:
+                time.sleep(self._flush_interval_ms / 1000.0)

{kaq_quant_common-0.2.3.dist-info → kaq_quant_common-0.2.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kaq_quant_common
-Version: 0.2.3
+Version: 0.2.4
 Summary: 
 Author: kevinfuture
 Author-email: liuenbofuture@foxmail.com

{kaq_quant_common-0.2.3.dist-info → kaq_quant_common-0.2.4.dist-info}/RECORD

@@ -27,7 +27,8 @@ kaq_quant_common/api/ws/ws_server_base.py,sha256=-JFA5fnYHXPYBZ09aZmhYuhgDHFfJbk
 kaq_quant_common/common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 kaq_quant_common/common/ddb_table_monitor.py,sha256=7Yihz_uGGujo_QqqPl45Gp8fwUMMw1auXx5egbzyYlE,3662
 kaq_quant_common/common/http_monitor.py,sha256=_yChiwfVv1c5g_lKgYUjWY40fX61BWVK8SL4kXwRfwk,2375
-kaq_quant_common/common/modules/limit_order_helper.py,sha256=Pbh1RHapr9D0Uacup5oQqeXgfI72zLAG2vZHEjPgoPY,2894
+kaq_quant_common/common/modules/funding_rate_helper.py,sha256=BVGNS6E7aVPITOz5u4g7vBUHOrm1S1Uri8psgWxp-BQ,3853
+kaq_quant_common/common/modules/limit_order_helper.py,sha256=Py9DPBXq1-QnjEh6mGQOA7vD3TVNUabnS_rMf0SQx5U,3853
 kaq_quant_common/common/modules/limit_order_symbol_monitor.py,sha256=TBK48qyeCSQvkfDMv3J_0UM7f3OuBRKRFYDcL9kG6Cs,2876
 kaq_quant_common/common/modules/limit_order_symbol_monitor_group.py,sha256=oEqHIwxhqAzckmluHJHZHiHUNmAyaS2JyK2nXO58UhY,2394
 kaq_quant_common/common/monitor_base.py,sha256=E4EUMsO3adNltCDNRgxkvUSbTTfKOL9S1zzN3WkZvpU,2467
@@ -61,6 +62,6 @@ kaq_quant_common/utils/signal_utils.py,sha256=zBSyEltNTKqkQCsrETd47kEBb3Q_OWUBUn
 kaq_quant_common/utils/sqlite_utils.py,sha256=UDDFKfwL0N-jFifl40HdyOCENh2YQfW5so6hRaSJpv0,5722
 kaq_quant_common/utils/uuid_utils.py,sha256=pm_pnXpd8n9CI66x3A20cOEUiriJyqHaKGCeLrgkBxU,71
 kaq_quant_common/utils/yml_utils.py,sha256=gcKjb_-uuUajBGAl5QBPIZTg2wXm7qeeJvtHflj_zOE,4513
-kaq_quant_common-0.2.3.dist-info/METADATA,sha256=wk6Okdv4qKVJxUnM54tR6sTtw4zMR2Qicwv5NZOXsXk,1970
-kaq_quant_common-0.2.3.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
-kaq_quant_common-0.2.3.dist-info/RECORD,,
+kaq_quant_common-0.2.4.dist-info/METADATA,sha256=J-fMkZRZdJJSSOkTumujjx2t9HhwgxgKtLKtvNqwaTk,1970
+kaq_quant_common-0.2.4.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
+kaq_quant_common-0.2.4.dist-info/RECORD,,