kaq-quant-common 0.2.2__tar.gz → 0.2.4__tar.gz

{kaq_quant_common-0.2.2 → kaq_quant_common-0.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kaq_quant_common
-Version: 0.2.2
+Version: 0.2.4
 Summary: 
 Author: kevinfuture
 Author-email: liuenbofuture@foxmail.com

kaq_quant_common-0.2.4/kaq_quant_common/common/modules/funding_rate_helper.py

@@ -0,0 +1,103 @@
+# 避免写入导致阻塞
+import datetime
+import threading
+import time
+
+import pandas as pd
+
+from kaq_quant_common.resources.kaq_ddb_stream_write_resources import (
+    KaqQuantDdbStreamMTWWriteRepository, KaqQuantDdbStreamWriteRepository)
+from kaq_quant_common.utils import logger_utils
+
+
+class FundingRateHelper:
+
+    def __init__(
+        self,
+        ddb: KaqQuantDdbStreamWriteRepository | KaqQuantDdbStreamMTWWriteRepository,
+        ddb_table_name: str,
+    ):
+        # 最新快照缓存与刷库线程控制
+        self._latest_snapshots: dict[str, tuple] = {}
+        self._latest_lock = threading.Lock()
+        # 写入到ddb的频率，默认1s
+        self._flush_interval_ms = 1000
+        self._stop_event = threading.Event()
+        self._flusher_thread = threading.Thread(target=self._flush_loop, daemon=True)
+        self._flusher_thread.name = "FundingRateHelperFlusherThread"
+        self._is_df = False
+
+        #
+        self._ddb = ddb
+        self._ddb_table_name = ddb_table_name
+
+        #
+        self._logger = logger_utils.get_logger(self)
+
+        self._build_data: callable = None
+
+    def set_build_data(self, build_data: callable):
+        self._build_data = build_data
+
+    def push_data(self, symbol: str, data: dict, arg: dict = None):
+        with self._latest_lock:
+            self._latest_snapshots[symbol] = (data, arg)
+
+    def start(self):
+        self._flusher_thread.start()
+
+    def stop(self):
+        self._stop_event.set()
+        self._flusher_thread.join()
+
+    def _flush_loop(self):
+        # 周期性地将每个symbol的最新快照批量入库
+        while not self._stop_event.is_set():
+            to_process = None
+            with self._latest_lock:
+                if self._latest_snapshots:
+                    to_process = list(self._latest_snapshots.items())
+                    self._latest_snapshots.clear()
+
+            if to_process:
+                df: pd.DataFrame = None
+                list_data: list = []
+                now = int(datetime.datetime.now().timestamp() * 1000)
+
+                for symbol, (data, arg) in to_process:
+                    sub_df = self._build_data(symbol, data, arg)
+                    if isinstance(sub_df, pd.DataFrame):
+                        self._is_df = True
+                        # 输出一下
+                        data_first_now = int(sub_df["create_time"].iloc[0])
+                        if now - data_first_now > 2000:
+                            self._logger.debug(
+                                f"数据时间{data_first_now} 与当前时间{now} 差值{now - data_first_now} 超过2000ms"
+                            )
+
+                        if df is None:
+                            df = sub_df
+                        else:
+                            df = pd.concat([df, sub_df], ignore_index=True)
+                    else:
+                        list_data = sub_df
+
+                # 入库
+                if self._is_df:
+                    if df is not None and not df.empty:
+                        try:
+                            self._ddb.save2stream_batch(self._ddb_table_name, df=df)
+                        except Exception as e:
+                            # 避免刷库异常导致线程退出
+                            self._logger.error(f"批量写入df失败: {e}")
+                else:
+                    if len(list_data) > 0:
+                        try:
+                            self._ddb.save2stream_list(list_data)
+                        except Exception as e:
+                            # 避免刷库异常导致线程退出
+                            self._logger.error(f"批量写入数组失败: {e}")
+
+            # 是dataFrame的才睡眠，数组处理交由ddb自己控制节奏
+            if self._is_df:
+                time.sleep(self._flush_interval_ms / 1000.0)

{kaq_quant_common-0.2.2 → kaq_quant_common-0.2.4}/kaq_quant_common/common/modules/limit_order_helper.py

@@ -4,15 +4,19 @@ import threading
 import time
 
 import pandas as pd
+
 from kaq_quant_common.resources.kaq_ddb_stream_write_resources import (
-    KaqQuantDdbStreamWriteRepository,
-)
+    KaqQuantDdbStreamMTWWriteRepository, KaqQuantDdbStreamWriteRepository)
 from kaq_quant_common.utils import logger_utils
 
 
 class LimitOrderHelper:
 
-    def __init__(self, ddb: KaqQuantDdbStreamWriteRepository, ddb_table_name: str):
+    def __init__(
+        self,
+        ddb: KaqQuantDdbStreamWriteRepository | KaqQuantDdbStreamMTWWriteRepository,
+        ddb_table_name: str,
+    ):
         # 最新快照缓存与刷库线程控制
         self._latest_snapshots: dict[str, tuple] = {}
         self._latest_lock = threading.Lock()
@@ -21,6 +25,7 @@ class LimitOrderHelper:
         self._stop_event = threading.Event()
         self._flusher_thread = threading.Thread(target=self._flush_loop, daemon=True)
         self._flusher_thread.name = "LimitOrderHelperFlusherThread"
+        self._is_df = False
 
         #
         self._ddb = ddb
@@ -56,26 +61,43 @@ class LimitOrderHelper:
 
             if to_process:
                 df: pd.DataFrame = None
+                list_data: list = []
                 now = int(datetime.datetime.now().timestamp() * 1000)
 
                 for symbol, (data, arg) in to_process:
                     sub_df = self._build_data(symbol, data, arg)
-                    # 输出一下
-                    data_first_now = int(sub_df["create_time"].iloc[0])
-                    if now - data_first_now > 2000:
-                        self._logger.debug(f"数据时间{data_first_now} 与当前时间{now} 差值{now - data_first_now} 超过2000ms")
-
-                    if df is None:
-                        df = sub_df
+                    if isinstance(sub_df, pd.DataFrame):
+                        self._is_df = True
+                        # 输出一下
+                        data_first_now = int(sub_df["create_time"].iloc[0])
+                        if now - data_first_now > 2000:
+                            self._logger.debug(
+                                f"数据时间{data_first_now} 与当前时间{now} 差值{now - data_first_now} 超过2000ms"
+                            )
+
+                        if df is None:
+                            df = sub_df
+                        else:
+                            df = pd.concat([df, sub_df], ignore_index=True)
                     else:
-                        df = pd.concat([df, sub_df], ignore_index=True)
+                        list_data = sub_df
 
                 # 入库
-                if df is not None and not df.empty:
-                    try:
-                        self._ddb.save2stream_batch(self._ddb_table_name, df=df)
-                    except Exception as e:
-                        # 避免刷库异常导致线程退出
-                        self._logger.error(f"批量写入失败: {e}")
-
-            time.sleep(self._flush_interval_ms / 1000.0)
+                if self._is_df:
+                    if df is not None and not df.empty:
+                        try:
+                            self._ddb.save2stream_batch(self._ddb_table_name, df=df)
+                        except Exception as e:
+                            # 避免刷库异常导致线程退出
+                            self._logger.error(f"批量写入df失败: {e}")
+                else:
+                    if len(list_data) > 0:
+                        try:
+                            self._ddb.save2stream_list(list_data)
+                        except Exception as e:
+                            # 避免刷库异常导致线程退出
+                            self._logger.error(f"批量写入数组失败: {e}")
+
+            # 是dataFrame的才睡眠，数组处理交由ddb自己控制节奏
+            if self._is_df:
+                time.sleep(self._flush_interval_ms / 1000.0)

{kaq_quant_common-0.2.2 → kaq_quant_common-0.2.4}/kaq_quant_common/resources/kaq_ddb_stream_write_resources.py

@@ -94,7 +94,9 @@ class KaqQuantDdbStreamWriteRepository:
         for row in arr:
             parts = []
             for i, v in enumerate(row):
-                if i in str_idx:
+                if v is None or pd.isna(v):
+                    parts.append("NULL")
+                elif i in str_idx:
                     parts.append(f"'{v}'")   # 直接拼接最快
                 else:
                     parts.append(str(v))

{kaq_quant_common-0.2.2 → kaq_quant_common-0.2.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "kaq_quant_common"
-version = "0.2.2"
+version = "0.2.4"
 description = ""
 authors = [
     {name = "kevinfuture",email = "liuenbofuture@foxmail.com"}