kaq-quant-common 0.2.7__tar.gz → 0.2.9__tar.gz

{kaq_quant_common-0.2.7 → kaq_quant_common-0.2.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kaq_quant_common
-Version: 0.2.7
+Version: 0.2.9
 Summary: 
 Author: kevinfuture
 Author-email: liuenbofuture@foxmail.com

{kaq_quant_common-0.2.7 → kaq_quant_common-0.2.9}/kaq_quant_common/common/modules/funding_rate_helper.py

@@ -26,10 +26,10 @@ class FundingRateHelper:
         self._stop_event = threading.Event()
         self._flusher_thread = threading.Thread(target=self._flush_loop, daemon=True)
         self._flusher_thread.name = "FundingRateHelperFlusherThread"
-        self._is_df = False
 
         #
         self._ddb = ddb
+        self._isMtwDdb = isinstance(self._ddb, KaqQuantDdbStreamMTWWriteRepository)
         self._ddb_table_name = ddb_table_name
 
         #
@@ -66,39 +66,57 @@ class FundingRateHelper:
                 now = int(datetime.datetime.now().timestamp() * 1000)
 
                 for symbol, (data, arg) in to_process:
-                    sub_df = self._build_data(symbol, data, arg)
-                    if isinstance(sub_df, pd.DataFrame):
-                        self._is_df = True
-                        # 输出一下
-                        # data_first_now = int(sub_df["create_time"].iloc[0])
-                        # if now - data_first_now > 2000:
-                        #     self._logger.debug(
-                        #         f"数据时间{data_first_now} 与当前时间{now} 差值{now - data_first_now} 超过2000ms"
-                        #     )
-
-                        if df is None:
-                            df = sub_df
+                    sub_data = self._build_data(symbol, data, arg)
+
+                    if not self._isMtwDdb:
+                        # 可以是数组，可以是dataFrame
+                        is_df = type(sub_data) is pd.DataFrame
+                        
+                        if is_df:
+                            # df就用df的方式写入
+                            # data_first_now = int(sub_data["create_time"].iloc[0])
+                            # if now - data_first_now > 2000:
+                            #     self._logger.warning(f"数据时间{data_first_now} 与当前时间{now} 差值{now - data_first_now} 超过2000ms")
+                            #     pass
+
+                            if df is None:
+                                df = sub_data
+                            else:
+                                df = pd.concat([df, sub_data], ignore_index=True)
                         else:
-                            df = pd.concat([df, sub_df], ignore_index=True)
+                            # 数组就用数组的方式写入
+                            # data_first_now = int(sub_data[0])
+                            # if now - data_first_now > 2000:
+                            #     self._logger.warning(f"数据时间{data_first_now} 与当前时间{now} 差值{now - data_first_now} 超过2000ms")
+                            #     pass
+
+                            list_data.append(sub_data)
                     else:
-                        list_data = sub_df
+                        # 只能是数组
+                        if len(sub_data) > 0:
+                            # 直接调用 save2stream_list 写入
+                            try:
+                                self._ddb.save2stream_list(sub_data)
+                            except Exception as e:
+                                # 避免刷库异常导致线程退出
+                                self._logger.error(f"批量写入数组失败: {e}")
 
                 # 入库
-                if self._is_df:
+                if not self._isMtwDdb:
+                    # 兼容df和数组
                     if df is not None and not df.empty:
                         try:
                             self._ddb.save2stream_batch(self._ddb_table_name, df=df)
                         except Exception as e:
                             # 避免刷库异常导致线程退出
                             self._logger.error(f"批量写入df失败: {e}")
-                else:
                     if len(list_data) > 0:
                         try:
-                            self._ddb.save2stream_list(list_data)
+                            self._ddb.save2stream_batch_list(self._ddb_table_name, data=list_data)
                         except Exception as e:
                             # 避免刷库异常导致线程退出
-                            self._logger.error(f"批量写入数组失败: {e}")
+                            self._logger.error(f"批量写入list失败: {e}")
 
-            # 是dataFrame的才睡眠，数组处理交由ddb自己控制节奏
-            if self._is_df:
+            # mtw交由ddb自己控制节奏
+            if self._isMtwDdb:
                 time.sleep(self._flush_interval_ms / 1000.0)

kaq_quant_common-0.2.9/kaq_quant_common/common/modules/limit_order_helper.py

@@ -0,0 +1,143 @@
+# 避免写入导致阻塞
+import datetime
+import threading
+import time
+
+import pandas as pd
+from kaq_quant_common.resources.kaq_ddb_stream_write_resources import (
+    KaqQuantDdbStreamMTWWriteRepository,
+    KaqQuantDdbStreamWriteRepository,
+)
+from kaq_quant_common.utils import logger_utils
+
+
+class LimitOrderHelper:
+
+    def __init__(
+        self, ddb: KaqQuantDdbStreamWriteRepository | KaqQuantDdbStreamMTWWriteRepository, ddb_table_name: str, _flush_interval_ms: int = 100
+    ):
+        # 最新快照缓存与刷库线程控制
+        self._latest_snapshots: dict[str, tuple] = {}
+        self._latest_lock = threading.Lock()
+        # 写入到ddb的频率，默认100ms
+        self._flush_interval_ms = _flush_interval_ms
+        self._stop_event = threading.Event()
+        self._flusher_thread = threading.Thread(target=self._flush_loop, daemon=True)
+        self._flusher_thread.name = "LimitOrderHelperFlusherThread"
+
+        #
+        self._ddb = ddb
+        self._isMtwDdb = isinstance(self._ddb, KaqQuantDdbStreamMTWWriteRepository)
+        self._ddb_table_name = ddb_table_name
+
+        #
+        self._logger = logger_utils.get_logger(self)
+
+        self._build_data: callable = None
+
+    def set_build_data(self, build_data: callable):
+        self._build_data = build_data
+
+    def push_data(self, symbol: str, data: dict, arg: dict = None):
+        with self._latest_lock:
+            self._latest_snapshots[symbol] = (data, arg)
+
+    def start(self):
+        self._flusher_thread.start()
+
+    def stop(self):
+        self._stop_event.set()
+        self._flusher_thread.join()
+
+    def _flush_loop(self):
+        cum_count = 0
+        cum_convert_time = 0
+        cum_write_ddb_time = 0
+        cum_total_use_time = 0
+        # 周期性地将每个symbol的最新快照批量入库
+        while not self._stop_event.is_set():
+            to_process = None
+            with self._latest_lock:
+                if self._latest_snapshots:
+                    to_process = list(self._latest_snapshots.items())
+                    self._latest_snapshots.clear()
+
+            if to_process:
+                df: pd.DataFrame = None
+                list_data: list = []
+                now = int(datetime.datetime.now().timestamp() * 1000)
+
+                for symbol, (data, arg) in to_process:
+                    sub_data = self._build_data(symbol, data, arg)
+
+                    if not self._isMtwDdb:
+                        # 可以是数组，可以是dataFrame
+                        is_df = type(sub_data) is pd.DataFrame
+
+                        if is_df:
+                            # df就用df的方式写入
+                            data_first_now = int(sub_data["create_time"].iloc[0])
+                            if now - data_first_now > 2000:
+                                self._logger.debug(f"数据时间{data_first_now} 与当前时间{now} 差值{now - data_first_now} 超过2000ms")
+                                pass
+
+                            if df is None:
+                                df = sub_data
+                            else:
+                                df = pd.concat([df, sub_data], ignore_index=True)
+                        else:
+                            # 数组就用数组的方式写入
+                            data_first_now = int(sub_data[0])
+                            if now - data_first_now > 2000:
+                                self._logger.debug(f"数据时间{data_first_now} 与当前时间{now} 差值{now - data_first_now} 超过2000ms")
+                                pass
+
+                            list_data.append(sub_data)
+                    else:
+                        # 只能是数组
+                        if len(sub_data) > 0:
+                            # 直接调用 save2stream_list 写入
+                            try:
+                                self._ddb.save2stream_list(sub_data)
+                            except Exception as e:
+                                # 避免刷库异常导致线程退出
+                                self._logger.error(f"批量写入数组失败: {e}")
+
+                convert_time = int(datetime.datetime.now().timestamp() * 1000)
+
+                # 入库
+                if not self._isMtwDdb:
+                    # 兼容df和数组
+                    if df is not None and not df.empty:
+                        try:
+                            self._ddb.save2stream_batch(self._ddb_table_name, df=df)
+                        except Exception as e:
+                            # 避免刷库异常导致线程退出
+                            self._logger.error(f"批量写入df失败: {e}")
+                    if len(list_data) > 0:
+                        try:
+                            self._ddb.save2stream_batch_list(self._ddb_table_name, data=list_data)
+                        except Exception as e:
+                            # 避免刷库异常导致线程退出
+                            self._logger.error(f"批量写入list失败: {e}")
+
+                # 统计一下
+                end = int(datetime.datetime.now().timestamp() * 1000)
+                total_use_time = end - now
+                convert_use = convert_time - now
+                write_ddb_use = total_use_time - convert_use
+
+                #
+                cum_count += len(to_process)
+                cum_convert_time += convert_use
+                cum_write_ddb_time += write_ddb_use
+                cum_total_use_time += total_use_time
+
+                if total_use_time > 500 and cum_count > 0:
+                    self._logger.debug(
+                        f"批量写入{len(to_process)}条数据耗时{total_use_time}ms(avg {cum_total_use_time / cum_count:.2f}ms) 转换耗时{convert_use}ms(avg {cum_convert_time / cum_count:.2f}ms) 写入ddb耗时{write_ddb_use}ms(avg {cum_write_ddb_time / cum_count:.2f}ms)"
+                    )
+
+            # mtw交由ddb自己控制节奏
+            if self._isMtwDdb:
+                time.sleep(self._flush_interval_ms / 1000.0)

{kaq_quant_common-0.2.7 → kaq_quant_common-0.2.9}/kaq_quant_common/resources/kaq_ddb_stream_write_resources.py

@@ -1,28 +1,31 @@
 import os
 import queue
+import threading
 import time
+import traceback
 from typing import Optional, Union
+
 import dolphindb as ddb
-from kaq_quant_common.utils import yml_utils
 import numpy as np
 import pandas as pd
-import threading
+from kaq_quant_common.utils import yml_utils
 from kaq_quant_common.utils.logger_utils import get_logger
-import traceback
 from typing_extensions import deprecated
 
 mutex = threading.Lock()
 
+
 # 方式一: 异步调用
 class KaqQuantDdbStreamWriteRepository:
-    '''
+    """
     异步不等待写入
-    '''
+    """
+
     def __init__(self, host, port, user, passwd):
         self.logger = get_logger(self)
-        '''
+        """
         创建ddb连接 && 添加ddb流数据表支持
-        '''
+        """
         try:
             mutex.acquire()
             self.session = ddb.session(enableASYNC=True)
@@ -34,14 +37,14 @@ class KaqQuantDdbStreamWriteRepository:
             # 需要注意的是 fetchSize 取值不能小于 8192 （记录条数）
             self.size = 8192
         except Exception as e:
-            self.logger.error(f'KaqQuantDdbStreamWriteRepository.__init__ is occured error: {str(e)} - {str(traceback.format_exc())}')
+            self.logger.error(f"KaqQuantDdbStreamWriteRepository.__init__ is occured error: {str(e)} - {str(traceback.format_exc())}")
         finally:
             mutex.release()
-            
+
     def save_rows(self, ddb_table_name: str, rows: Optional[Union[np.ndarray, list]] = None):
-        '''
+        """
         调用此方法之前, 需要将dataframe中的字符串类型的值 ，添加引号
-        '''
+        """
         if rows is None:
             return
         # 获取维度
@@ -51,77 +54,139 @@ class KaqQuantDdbStreamWriteRepository:
                 formatted_values = []
                 for row in rows:
                     # 这里的 row 是 np.array([1767708564161, "BTC", 92500.1])
-                    row = [f'\'{v}\'' if isinstance(v, str) else str(v) for v in row]
-                    formatted_values.append(f'({', '.join(row)})')
+                    row = [f"'{v}'" if isinstance(v, str) else str(v) for v in row]
+                    formatted_values.append(f"({', '.join(row)})")
                 script = f"insert into {ddb_table_name} values {', '.join(str(x) for x in formatted_values)}"
-            
+
                 self.session.run(script, clearMemory=True)
             except Exception as e:
-                self.logger.error(f'KaqQuantDdbStreamWriteRepository.save_rows is occured error: tableName is {ddb_table_name} - {str(e)} - {str(traceback.format_exc())}')
+                self.logger.error(
+                    f"KaqQuantDdbStreamWriteRepository.save_rows is occured error: tableName is {ddb_table_name} - {str(e)} - {str(traceback.format_exc())}"
+                )
         else:
             # 是一行数据，调用 insert
-            formatted_values = [f'\'{v}\'' if isinstance(v, str) else str(v) for v in rows]
+            formatted_values = [f"'{v}'" if isinstance(v, str) else str(v) for v in rows]
             script = f"insert into {ddb_table_name} values({', '.join(str(x) for x in formatted_values)})"
             try:
                 self.session.run(script, clearMemory=True)
             except Exception as e:
-                self.logger.error(f'KaqQuantDdbStreamWriteRepository.save_rows is occured error: tableName is {ddb_table_name} - {str(e)} - {str(traceback.format_exc())}')
-        
-    def save2stream(self, ddb_table_name: str, df : pd.DataFrame):
-        '''
+                self.logger.error(
+                    f"KaqQuantDdbStreamWriteRepository.save_rows is occured error: tableName is {ddb_table_name} - {str(e)} - {str(traceback.format_exc())}"
+                )
+
+    def save2stream(self, ddb_table_name: str, df: pd.DataFrame):
+        """
         调用此方法之前, 需要将dataframe中的字符串类型的值 ，添加引号
-        '''
+        """
         # 遍历每列的数据类型
         for column, dtype in df.dtypes.items():
-            if dtype == 'object' or dtype == 'str':
-                df[column] = '\'' + df[column] + '\''
+            if dtype == "object" or dtype == "str":
+                df[column] = "'" + df[column] + "'"
         for index, row in df.iterrows():
             script = f"insert into {ddb_table_name} values({', '.join(str(x) for x in row.values)})"
             try:
                 self.session.run(script, clearMemory=True)
             except Exception as e:
-                self.logger.error(f'KaqQuantDdbStreamWriteRepository.save2stream is occured error: tableName is {ddb_table_name} - {str(e)} - {str(traceback.format_exc())}')
-                
-    def build_insert_values_fast(self, df):
-        if df.empty:
+                self.logger.error(
+                    f"KaqQuantDdbStreamWriteRepository.save2stream is occured error: tableName is {ddb_table_name} - {str(e)} - {str(traceback.format_exc())}"
+                )
+
+    def build_insert_values_fast(self, data: pd.DataFrame | list):
+        if data.empty:
             return []
-        dtypes = df.dtypes.tolist()
+        dtypes = data.dtypes.tolist()
         # 提前确定哪些列需要加引号
-        str_idx = {i for i, dt in enumerate(dtypes) if dt == object or dt == 'object' or dt == 'str'}
+        str_idx = {i for i, dt in enumerate(dtypes) if dt == object or dt == "object" or dt == "str"}
         # 转成 ndarray，减少 pandas 参与
-        arr = df.to_numpy()
-        rows = []
-        for row in arr:
-            parts = []
-            for i, v in enumerate(row):
-                if v is None or pd.isna(v):
-                    parts.append("NULL")
-                elif i in str_idx:
-                    parts.append(f"'{v}'")   # 直接拼接最快
-                else:
-                    parts.append(str(v))
-            rows.append("(" + ", ".join(parts) + ")")
-        return rows            
-    
-    def save2stream_batch(self, ddb_table_name: str, df : pd.DataFrame):
-        '''
+        arr = data.to_numpy()
+
+        # 使用内部函数避免lambda闭包问题，提升性能
+        def format_value(i, v):
+            if v is None or (isinstance(v, float) and np.isnan(v)):
+                return "NULL"
+            elif i in str_idx:
+                return f"'{v}'"
+            else:
+                return str(v)
+
+        # 使用列表推导式一次性生成所有行，避免多次append
+        return ["(" + ", ".join(format_value(i, v) for i, v in enumerate(row)) + ")" for row in arr]
+
+    def build_insert_values_fast_list(self, data: list):
+        if not data:
+            return []
+        #
+        first_row = data[0]
+        str_idx = {i for i, v in enumerate(first_row) if type(v) is str}
+
+        # 优化：使用 type() 代替 isinstance() 减少函数调用开销
+        def format_value(i, v):
+            if v is None:
+                return "NULL"
+            # 如果在字符串索引中，直接格式化（避免 isinstance）
+            if i in str_idx:
+                return f"'{v}'"
+
+            # 使用 type() 比 isinstance() 更快
+            v_type = type(v)
+            if v_type is float:
+                if np.isnan(v):
+                    return "NULL"
+                return str(v)
+            if v_type is str:
+                return f"'{v}'"  # 第一行可能是 None 导致误判
+            return str(v)
+
+        # 使用列表推导式一次性生成所有行
+        return ["(" + ", ".join(format_value(i, v) for i, v in enumerate(row)) + ")" for row in data]
+
+    def save2stream_batch(self, ddb_table_name: str, df: pd.DataFrame):
+        """
         调用此方法之前, 需要将dataframe中的字符串类型的值 ，添加引号
-        '''
+        """
         try:
-            start1 = time.monotonic_ns()
+            # start1 = time.monotonic_ns()
             row = self.build_insert_values_fast(df)
-            values = ', '.join(row)
+            values = ", ".join(row)
             script = f"insert into {ddb_table_name} values {values}"
-            start2 = time.monotonic_ns()
+            # start2 = time.monotonic_ns()
             self.session.run(script, clearMemory=True)
-            end = time.monotonic_ns()
-            if "KAQ_QUANT_LOG" in os.environ:
-                diff = end - start2
-                if diff > 1_000_000_0:  # 超过1毫秒
-                    self.logger.warning(f'KaqQuantDdbStreamWriteRepository.save2stream cost time is only write : {end - start2} ns, save2stream_batch :{end - start1} ns, batch size is {len(df)}, tableName is {ddb_table_name}')
+            # end = time.monotonic_ns()
+            # if "KAQ_QUANT_LOG" in os.environ:
+            #     diff = end - start2
+            #     if diff > 1_000_000_0:  # 超过1毫秒
+            #         self.logger.warning(
+            #             f"KaqQuantDdbStreamWriteRepository.save2stream_batch cost time is only write : {end - start2} ns, save2stream_batch :{end - start1} ns, batch size is {len(df)}, tableName is {ddb_table_name}"
+            #         )
         except Exception as e:
-            self.logger.error(f'KaqQuantDdbStreamWriteRepository.save2stream_batch is occured error: tableName is {ddb_table_name} - {str(e)} - {str(traceback.format_exc())}')
-  
+            self.logger.error(
+                f"KaqQuantDdbStreamWriteRepository.save2stream_batch is occured error: tableName is {ddb_table_name} - {str(e)} - {str(traceback.format_exc())}"
+            )
+
+    def save2stream_batch_list(self, ddb_table_name: str, data: list):
+        """
+        调用此方法之前, 需要将数组中的字符串类型的值 ，添加引号
+        """
+        try:
+            # start1 = time.monotonic_ns()
+            row = self.build_insert_values_fast_list(data)
+            values = ", ".join(row)
+            script = f"insert into {ddb_table_name} values {values}"
+            # start2 = time.monotonic_ns()
+            self.session.run(script, clearMemory=True)
+            # end = time.monotonic_ns()
+            # if "KAQ_QUANT_LOG" in os.environ:
+            #     diff = end - start2
+            #     if diff > 1_000_000_0:  # 超过1毫秒
+            #         self.logger.warning(
+            #             f"KaqQuantDdbStreamWriteRepository.save2stream_batch_list cost time is only write : {end - start2} ns, save2stream_batch_list :{end - start1} ns, batch size is {len(data)}, tableName is {ddb_table_name}"
+            #         )
+        except Exception as e:
+            self.logger.error(
+                f"KaqQuantDdbStreamWriteRepository.save2stream_batch_list is occured error: tableName is {ddb_table_name} - {str(e)} - {str(traceback.format_exc())}"
+            )
+
+
 # 方式二: 同步调用,但有python端的队列等待
 class DDBAsyncDFWriter:
     def __init__(self, appender, batch_size=1000, flush_interval_ms=80):
@@ -129,10 +194,10 @@ class DDBAsyncDFWriter:
         self.appender = appender
         self.batch_size = batch_size
         self.flush_interval = flush_interval_ms / 1000.0
-        
+
         self.queue = queue.Queue(maxsize=10000)
         self.running = True
-        
+
         self.thread = threading.Thread(target=self._worker, daemon=True)
         self.thread.start()
 
@@ -165,9 +230,7 @@ class DDBAsyncDFWriter:
 
             now = time.time()
             # 触发条件：行数够了，或时间到了，或者程序正在停止
-            if buffer and (current_rows >= self.batch_size or 
-                           (now - last_flush_time) >= self.flush_interval or
-                           not self.running):
+            if buffer and (current_rows >= self.batch_size or (now - last_flush_time) >= self.flush_interval or not self.running):
                 self._do_flush(buffer)
                 buffer = []
                 current_rows = 0
@@ -190,67 +253,80 @@ class DDBAsyncDFWriter:
         """
         self.logger.warning("正在停止 DDBAsyncDFWriter 并清空残留数据...")
         self.running = False
-        self.thread.join() # 等待工作线程处理完最后一批 buffer
+        self.thread.join()  # 等待工作线程处理完最后一批 buffer
         self.logger.info("DDBAsyncDFWriter 已安全停止。")
-            
+
+
 class KaqQuantDdbStreamWriteSyncRepository:
-    '''
+    """
     使用appender直接写入的方式
-    '''
+    """
+
     def __init__(self, host, port, user, passwd, tableName=None, batch_size=1000, flush_interval_ms=80):
         if tableName is None:
-            raise ValueError(f'Error tableName, please set. tableName={tableName}')
+            raise ValueError(f"Error tableName, please set. tableName={tableName}")
         self.tableName = tableName
         self.logger = get_logger(self)
-        '''
+        """
         创建ddb连接 && 添加ddb流数据表支持
-        '''
+        """
         try:
             mutex.acquire()
             self.session = ddb.session()
             self.session.connect(host, port, user, passwd, tryReconnectNums=100, reconnect=True, keepAliveTime=1000, readTimeout=10, writeTimeout=5)
-            
-            self.batch_writer = DDBAsyncDFWriter(ddb.TableAppender(table_name=self.tableName, conn=self.session), batch_size=batch_size, flush_interval_ms=flush_interval_ms)
+
+            self.batch_writer = DDBAsyncDFWriter(
+                ddb.TableAppender(table_name=self.tableName, conn=self.session), batch_size=batch_size, flush_interval_ms=flush_interval_ms
+            )
             # 需要注意的是 fetchSize 取值不能小于 8192 （记录条数）
             self.size = 8192
         except Exception as e:
-            self.logger.error(f'KaqQuantDdbTableStreamWriteRepository.__init__ is occured error: {str(e)} - {str(traceback.format_exc())}')
+            self.logger.error(f"KaqQuantDdbTableStreamWriteRepository.__init__ is occured error: {str(e)} - {str(traceback.format_exc())}")
         finally:
             mutex.release()
-       
+
     # @deprecated("请确保pandas数据与ddb表的数据类型一致.")
-    def insert(self, df : pd.DataFrame):
-        '''
+    def insert(self, df: pd.DataFrame):
+        """
         dataframe中日期等类型与ddb流表中一致,例如：
             df['create_time'] = pd.to_datetime(df['create_time'], unit='ms')
             df['event_time'] = pd.to_datetime(df['event_time'], unit='ms')
-        '''
+        """
         try:
             self.batch_writer.add_df(df)
         except Exception as e:
-            self.logger.error(f'KaqQuantDdbTableStreamWriteRepository.insert is occured error:  {str(e)} - {str(traceback.format_exc())}')
+            self.logger.error(f"KaqQuantDdbTableStreamWriteRepository.insert is occured error:  {str(e)} - {str(traceback.format_exc())}")
+
 
 # 方式三: 异步调用, 但属于ddb的client内部的c++多线程解析与写入，适合一条条写入
-class KaqQuantDdbStreamMTWWriteRepository: 
-    def __init__(self, host, port, user, passwd, tableName=None, batch_size=1000, throttle=50, partitionCol='', threadCount=1):
+class KaqQuantDdbStreamMTWWriteRepository:
+    def __init__(self, host, port, user, passwd, tableName=None, batch_size=1000, throttle=50, partitionCol="", threadCount=1):
         self.logger = get_logger(self)
-        '''
+        """
         创建ddb连接 && 添加ddb流数据表支持
-        '''
+        """
         try:
             mutex.acquire()
             self.session = ddb.session(enableASYNC=True)
             self.session.connect(host, port, user, passwd, tryReconnectNums=100, reconnect=True, keepAliveTime=1000, readTimeout=10, writeTimeout=5)
             self.batch_writer = ddb.MultithreadedTableWriter(
-                host, port, user, passwd, tableName=tableName, dbPath='',
-                batchSize=batch_size, throttle=throttle, threadCount=threadCount, partitionCol=partitionCol
+                host,
+                port,
+                user,
+                passwd,
+                tableName=tableName,
+                dbPath="",
+                batchSize=batch_size,
+                throttle=throttle,
+                threadCount=threadCount,
+                partitionCol=partitionCol,
             )
         except Exception as e:
-            self.logger.error(f'KaqQuantDdbStreamMTWWriteRepository.__init__ is occured error: {str(e)} - {str(traceback.format_exc())}')
+            self.logger.error(f"KaqQuantDdbStreamMTWWriteRepository.__init__ is occured error: {str(e)} - {str(traceback.format_exc())}")
         finally:
             mutex.release()
-            
-    def save2stream_batch(self, df:pd.DataFrame=pd.DataFrame(), cols:list=[]):
+
+    def save2stream_batch(self, df: pd.DataFrame = pd.DataFrame(), cols: list = []):
         try:
             if cols is None or len(cols) <= 0:
                 for _, row in df.iterrows():
@@ -261,23 +337,23 @@ class KaqQuantDdbStreamMTWWriteRepository:
                     _args = [row[i] for i in cols]
                     self.batch_writer.insert(*_args)
         except Exception as e:
-            self.logger.error(f'KaqQuantDdbStreamMTWWriteRepository.insert is occured error: {str(e)} - {str(traceback.format_exc())}')
-            
-    def save2stream_list(self, row:list=[]):
+            self.logger.error(f"KaqQuantDdbStreamMTWWriteRepository.insert is occured error: {str(e)} - {str(traceback.format_exc())}")
+
+    def save2stream_list(self, row: list = []):
         try:
             if row is None or len(row) <= 0:
                 return
             self.batch_writer.insert(*row)
         except Exception as e:
-            self.logger.error(f'KaqQuantDdbStreamMTWWriteRepository.insert is occured error: {str(e)} - {str(traceback.format_exc())}')
-            
+            self.logger.error(f"KaqQuantDdbStreamMTWWriteRepository.insert is occured error: {str(e)} - {str(traceback.format_exc())}")
+
     def stop(self):
-        '''
+        """
         结束调用
-        '''
+        """
         self.batch_writer.waitForThreadCompletion()
-    
-    
-if __name__ == '__main__':
+
+
+if __name__ == "__main__":
     host, port, user, passwd = yml_utils.get_ddb_info(os.getcwd())
-    kaq = KaqQuantDdbStreamWriteRepository(host, port, user, passwd)
+    kaq = KaqQuantDdbStreamWriteRepository(host, port, user, passwd)

{kaq_quant_common-0.2.7 → kaq_quant_common-0.2.9}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "kaq_quant_common"
-version = "0.2.7"
+version = "0.2.9"
 description = ""
 authors = [
     {name = "kevinfuture",email = "liuenbofuture@foxmail.com"}

kaq_quant_common-0.2.7/kaq_quant_common/common/modules/limit_order_helper.py

@@ -1,106 +0,0 @@
-# 避免写入导致阻塞
-import datetime
-import threading
-import time
-
-import pandas as pd
-
-from kaq_quant_common.resources.kaq_ddb_stream_write_resources import (
-    KaqQuantDdbStreamMTWWriteRepository,
-    KaqQuantDdbStreamWriteRepository,
-)
-from kaq_quant_common.utils import logger_utils
-
-
-class LimitOrderHelper:
-
-    def __init__(
-        self, ddb: KaqQuantDdbStreamWriteRepository | KaqQuantDdbStreamMTWWriteRepository, ddb_table_name: str, _flush_interval_ms: int = 100
-    ):
-        # 最新快照缓存与刷库线程控制
-        self._latest_snapshots: dict[str, tuple] = {}
-        self._latest_lock = threading.Lock()
-        # 写入到ddb的频率，默认100ms
-        self._flush_interval_ms = _flush_interval_ms
-        self._stop_event = threading.Event()
-        self._flusher_thread = threading.Thread(target=self._flush_loop, daemon=True)
-        self._flusher_thread.name = "LimitOrderHelperFlusherThread"
-        self._is_df = False
-
-        #
-        self._ddb = ddb
-        self._ddb_table_name = ddb_table_name
-
-        #
-        self._logger = logger_utils.get_logger(self)
-
-        self._build_data: callable = None
-
-    def set_build_data(self, build_data: callable):
-        self._build_data = build_data
-
-    def push_data(self, symbol: str, data: dict, arg: dict = None):
-        with self._latest_lock:
-            self._latest_snapshots[symbol] = (data, arg)
-
-    def start(self):
-        self._flusher_thread.start()
-
-    def stop(self):
-        self._stop_event.set()
-        self._flusher_thread.join()
-
-    def _flush_loop(self):
-        # 周期性地将每个symbol的最新快照批量入库
-        while not self._stop_event.is_set():
-            to_process = None
-            with self._latest_lock:
-                if self._latest_snapshots:
-                    to_process = list(self._latest_snapshots.items())
-                    self._latest_snapshots.clear()
-
-            if to_process:
-                df: pd.DataFrame = None
-                list_data: list = []
-                now = int(datetime.datetime.now().timestamp() * 1000)
-
-                for symbol, (data, arg) in to_process:
-                    sub_df = self._build_data(symbol, data, arg)
-                    if isinstance(sub_df, pd.DataFrame):
-                        self._is_df = True
-                        # 输出一下
-                        data_first_now = int(sub_df["create_time"].iloc[0])
-                        if now - data_first_now > 2000:
-                            self._logger.warning(f"数据时间{data_first_now} 与当前时间{now} 差值{now - data_first_now} 超过2000ms")
-
-                        if df is None:
-                            df = sub_df
-                        else:
-                            df = pd.concat([df, sub_df], ignore_index=True)
-                    else:
-                        list_data = sub_df
-
-                # 入库
-                if self._is_df:
-                    if df is not None and not df.empty:
-                        try:
-                            self._ddb.save2stream_batch(self._ddb_table_name, df=df)
-                        except Exception as e:
-                            # 避免刷库异常导致线程退出
-                            self._logger.error(f"批量写入df失败: {e}")
-                else:
-                    if len(list_data) > 0:
-                        try:
-                            self._ddb.save2stream_list(list_data)
-                        except Exception as e:
-                            # 避免刷库异常导致线程退出
-                            self._logger.error(f"批量写入数组失败: {e}")
-
-                # 统计一下
-                end = int(datetime.datetime.now().timestamp() * 1000)
-                if (end - now) > self._flush_interval_ms:
-                    self._logger.warning(f"批量写入{len(to_process)}条数据耗时{end - now}ms")
-
-            # 是dataFrame的才睡眠，数组处理交由ddb自己控制节奏
-            if self._is_df:
-                time.sleep(self._flush_interval_ms / 1000.0)