kaq-quant-common 0.2.7__py3-none-any.whl → 0.2.8__py3-none-any.whl

kaq_quant_common/common/ddb_table_monitor.py

@@ -1,106 +1,106 @@
-import threading
-import traceback
-from abc import abstractmethod
-
-import dolphindb as ddb
-import numpy as np
-from kaq_quant_common.common.monitor_base import MonitorBase
-from kaq_quant_common.utils import logger_utils
-
-mutex = threading.Lock()
-
-
-# ddb表订阅监听器
-class DdbTableMonitor(MonitorBase):
-
-    def __init__(self, table_name: str, action_name: str, batch_size=1000, filter=[]):
-        # 表名
-        self._table_name = table_name
-        #
-        self._action_name = action_name
-        #
-        self._batch_size = batch_size
-        #
-        self._filter = filter
-
-        # logger
-        self._logger = logger_utils.get_logger(self)
-        #
-        super().__init__()
-
-    # ↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓ abstract methods
-    def _do_init(self):
-        # 初始化ddb
-        self._init_ddb()
-
-    def _do_start(self):
-        # 开启ddb订阅
-        self._start_subscribe()
-
-    def _do_stop(self):
-        # 关闭订阅
-        self._stop_subscribe()
-
-    # ↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓
-    # 初始化ddb
-    def _init_ddb(self):
-        '''
-        创建ddb连接 && 添加ddb流数据表支持
-        '''
-        try:
-            ddb_config = self._on_get_ddb_config()
-            host, port, user, passwd = ddb_config['host'], ddb_config['port'], ddb_config['user'], ddb_config['passwd']
-            mutex.acquire()
-            self._session = ddb.session(enableASYNC=True)
-            self._host = host
-            self._port = port
-            self._user = user
-            self._passwd = passwd
-            self._session.connect(host, port, user, passwd)
-            self._session.enableStreaming()
-        except Exception as e:
-            self._logger.error(f'DdbTableMonitor._init_ddb error: {str(e)} - {str(traceback.format_exc())}')
-        finally:
-            mutex.release()
-
-    # 开启订阅
-    def _start_subscribe(self):
-        '''
-        订阅ddb表
-        '''
-        self._session.subscribe(
-            self._host,
-            self._port,
-            self._handle,
-            tableName=self._table_name,
-            actionName=self._action_name,
-            filter=np.array(self._filter),
-            offset=-1,
-            batchSize=self._batch_size,
-            throttle=5,
-            msgAsTable=True,
-        )
-        self._logger.info(f'开始订阅 {self._host}:{self._port} {self._table_name} - {self._action_name}')
-
-    def _stop_subscribe(self):
-        # TODO
-        # script = f"""
-        # existsSubscriptionTopic(,`{self._table_name},`{self._action_name})
-        # """
-        # exitsTopic = self._session.run(script)
-        exitsTopic = True
-        if exitsTopic is True:
-            self._session.unsubscribe(self._host, self._port, self._table_name, self._action_name)
-            self._logger.info(f'取消订阅 {self._table_name} - {self._action_name}')
-        if not self._session.isClosed():
-            self._session.close()
-
-    # ↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓ abstract methods
-    # 需要返回ddb配置，包含host, port, user, passwd 添加类型提示
-    @abstractmethod
-    def _on_get_ddb_config(self, data) -> dict:
-        pass
-
-    @abstractmethod
-    def _handle(self, data):
-        pass
+import threading
+import traceback
+from abc import abstractmethod
+
+import dolphindb as ddb
+import numpy as np
+from kaq_quant_common.common.monitor_base import MonitorBase
+from kaq_quant_common.utils import logger_utils
+
+mutex = threading.Lock()
+
+
+# ddb表订阅监听器
+class DdbTableMonitor(MonitorBase):
+
+    def __init__(self, table_name: str, action_name: str, batch_size=1000, filter=[]):
+        # 表名
+        self._table_name = table_name
+        #
+        self._action_name = action_name
+        #
+        self._batch_size = batch_size
+        #
+        self._filter = filter
+
+        # logger
+        self._logger = logger_utils.get_logger(self)
+        #
+        super().__init__()
+
+    # ↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓ abstract methods
+    def _do_init(self):
+        # 初始化ddb
+        self._init_ddb()
+
+    def _do_start(self):
+        # 开启ddb订阅
+        self._start_subscribe()
+
+    def _do_stop(self):
+        # 关闭订阅
+        self._stop_subscribe()
+
+    # ↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓
+    # 初始化ddb
+    def _init_ddb(self):
+        '''
+        创建ddb连接 && 添加ddb流数据表支持
+        '''
+        try:
+            ddb_config = self._on_get_ddb_config()
+            host, port, user, passwd = ddb_config['host'], ddb_config['port'], ddb_config['user'], ddb_config['passwd']
+            mutex.acquire()
+            self._session = ddb.session(enableASYNC=True)
+            self._host = host
+            self._port = port
+            self._user = user
+            self._passwd = passwd
+            self._session.connect(host, port, user, passwd)
+            self._session.enableStreaming()
+        except Exception as e:
+            self._logger.error(f'DdbTableMonitor._init_ddb error: {str(e)} - {str(traceback.format_exc())}')
+        finally:
+            mutex.release()
+
+    # 开启订阅
+    def _start_subscribe(self):
+        '''
+        订阅ddb表
+        '''
+        self._session.subscribe(
+            self._host,
+            self._port,
+            self._handle,
+            tableName=self._table_name,
+            actionName=self._action_name,
+            filter=np.array(self._filter),
+            offset=-1,
+            batchSize=self._batch_size,
+            throttle=5,
+            msgAsTable=True,
+        )
+        self._logger.info(f'开始订阅 {self._host}:{self._port} {self._table_name} - {self._action_name}')
+
+    def _stop_subscribe(self):
+        # TODO
+        # script = f"""
+        # existsSubscriptionTopic(,`{self._table_name},`{self._action_name})
+        # """
+        # exitsTopic = self._session.run(script)
+        exitsTopic = True
+        if exitsTopic is True:
+            self._session.unsubscribe(self._host, self._port, self._table_name, self._action_name)
+            self._logger.info(f'取消订阅 {self._table_name} - {self._action_name}')
+        if not self._session.isClosed():
+            self._session.close()
+
+    # ↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓ abstract methods
+    # 需要返回ddb配置，包含host, port, user, passwd 添加类型提示
+    @abstractmethod
+    def _on_get_ddb_config(self, data) -> dict:
+        pass
+
+    @abstractmethod
+    def _handle(self, data):
+        pass

kaq_quant_common/common/http_monitor.py

@@ -1,69 +1,69 @@
-import threading
-import time
-from abc import abstractmethod
-
-from kaq_quant_common.common.monitor_base import MonitorBase
-from kaq_quant_common.common.ws_wrapper import WsWrapper
-from kaq_quant_common.utils import logger_utils
-
-
-# 封装http定时请求
-class HttpMonitor(MonitorBase):
-    def __init__(self, interval=5):
-        super().__init__()
-        # 执行间隔
-        self._interval = interval
-        self._logger = logger_utils.get_logger()
-
-    def _do_start(self):
-        # 开启一条线程，定时执行http请求
-        self._ticker_thread_event = threading.Event()
-
-        def http_request():
-            # 上次请求时间
-            last_request_time = 0
-            while True:
-                # 检查是否需要退出
-                if self._ticker_thread_event.is_set():
-                    self._logger.info("ticker thread exit")
-                    break
-
-                # 当前时间
-                current_time = time.time()
-                # 如果上次请求时间距离当前时间不足，等待
-                if current_time - last_request_time < self._interval:
-                    time.sleep(0.1)
-                    continue
-
-                #
-                last_request_time = time.time()
-
-                # self._logger.debug('tick start')
-                try:
-                    self._do_request()
-                except Exception as e:
-                    self._logger.error(f"http request error: {e}")
-                # self._logger.debug('tick finish')
-                #
-                time.sleep(0.1)
-
-        # 开启线程
-        self._ticker_thread = threading.Thread(target=http_request)
-        # 设置为守护线程
-        self._ticker_thread.daemon = True
-        self._ticker_thread.start()
-
-    def _do_stop(self):
-        if self._ticker_thread_event is not None:
-            self._ticker_thread_event.set()
-
-    # ↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓
-
-    @abstractmethod
-    def _do_request(self):
-        """
-        子类实现
-        """
-        pass
-
-    # ↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓
+import threading
+import time
+from abc import abstractmethod
+
+from kaq_quant_common.common.monitor_base import MonitorBase
+from kaq_quant_common.common.ws_wrapper import WsWrapper
+from kaq_quant_common.utils import logger_utils
+
+
+# 封装http定时请求
+class HttpMonitor(MonitorBase):
+    def __init__(self, interval=5):
+        super().__init__()
+        # 执行间隔
+        self._interval = interval
+        self._logger = logger_utils.get_logger()
+
+    def _do_start(self):
+        # 开启一条线程，定时执行http请求
+        self._ticker_thread_event = threading.Event()
+
+        def http_request():
+            # 上次请求时间
+            last_request_time = 0
+            while True:
+                # 检查是否需要退出
+                if self._ticker_thread_event.is_set():
+                    self._logger.info("ticker thread exit")
+                    break
+
+                # 当前时间
+                current_time = time.time()
+                # 如果上次请求时间距离当前时间不足，等待
+                if current_time - last_request_time < self._interval:
+                    time.sleep(0.1)
+                    continue
+
+                #
+                last_request_time = time.time()
+
+                # self._logger.debug('tick start')
+                try:
+                    self._do_request()
+                except Exception as e:
+                    self._logger.error(f"http request error: {e}")
+                # self._logger.debug('tick finish')
+                #
+                time.sleep(0.1)
+
+        # 开启线程
+        self._ticker_thread = threading.Thread(target=http_request)
+        # 设置为守护线程
+        self._ticker_thread.daemon = True
+        self._ticker_thread.start()
+
+    def _do_stop(self):
+        if self._ticker_thread_event is not None:
+            self._ticker_thread_event.set()
+
+    # ↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓
+
+    @abstractmethod
+    def _do_request(self):
+        """
+        子类实现
+        """
+        pass
+
+    # ↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓

kaq_quant_common/common/modules/funding_rate_helper.py

@@ -26,10 +26,10 @@ class FundingRateHelper:
         self._stop_event = threading.Event()
         self._flusher_thread = threading.Thread(target=self._flush_loop, daemon=True)
         self._flusher_thread.name = "FundingRateHelperFlusherThread"
-        self._is_df = False
 
         #
         self._ddb = ddb
+        self._isMtwDdb = isinstance(self._ddb, KaqQuantDdbStreamMTWWriteRepository)
         self._ddb_table_name = ddb_table_name
 
         #
@@ -66,39 +66,54 @@ class FundingRateHelper:
                 now = int(datetime.datetime.now().timestamp() * 1000)
 
                 for symbol, (data, arg) in to_process:
-                    sub_df = self._build_data(symbol, data, arg)
-                    if isinstance(sub_df, pd.DataFrame):
-                        self._is_df = True
-                        # 输出一下
-                        # data_first_now = int(sub_df["create_time"].iloc[0])
-                        # if now - data_first_now > 2000:
-                        #     self._logger.debug(
-                        #         f"数据时间{data_first_now} 与当前时间{now} 差值{now - data_first_now} 超过2000ms"
-                        #     )
-
-                        if df is None:
-                            df = sub_df
+                    sub_data = self._build_data(symbol, data, arg)
+
+                    if not self._isMtwDdb:
+                        if is_df:
+                            # df就用df的方式写入
+                            # data_first_now = int(sub_data["create_time"].iloc[0])
+                            # if now - data_first_now > 2000:
+                            #     self._logger.warning(f"数据时间{data_first_now} 与当前时间{now} 差值{now - data_first_now} 超过2000ms")
+                            #     pass
+
+                            if df is None:
+                                df = sub_data
+                            else:
+                                df = pd.concat([df, sub_data], ignore_index=True)
                         else:
-                            df = pd.concat([df, sub_df], ignore_index=True)
+                            # 数组就用数组的方式写入
+                            # data_first_now = int(sub_data[0])
+                            # if now - data_first_now > 2000:
+                            #     self._logger.warning(f"数据时间{data_first_now} 与当前时间{now} 差值{now - data_first_now} 超过2000ms")
+                            #     pass
+
+                            list_data.append(sub_data)
                     else:
-                        list_data = sub_df
+                        # 只能是数组
+                        if len(sub_data) > 0:
+                            # 直接调用 save2stream_list 写入
+                            try:
+                                self._ddb.save2stream_list(sub_data)
+                            except Exception as e:
+                                # 避免刷库异常导致线程退出
+                                self._logger.error(f"批量写入数组失败: {e}")
 
                 # 入库
-                if self._is_df:
+                if not self._isMtwDdb:
+                    # 兼容df和数组
                     if df is not None and not df.empty:
                         try:
                             self._ddb.save2stream_batch(self._ddb_table_name, df=df)
                         except Exception as e:
                             # 避免刷库异常导致线程退出
                             self._logger.error(f"批量写入df失败: {e}")
-                else:
                     if len(list_data) > 0:
                         try:
-                            self._ddb.save2stream_list(list_data)
+                            self._ddb.save2stream_batch_list(self._ddb_table_name, data=list_data)
                         except Exception as e:
                             # 避免刷库异常导致线程退出
-                            self._logger.error(f"批量写入数组失败: {e}")
+                            self._logger.error(f"批量写入list失败: {e}")
 
-            # 是dataFrame的才睡眠，数组处理交由ddb自己控制节奏
-            if self._is_df:
+            # mtw交由ddb自己控制节奏
+            if self._isMtwDdb:
                 time.sleep(self._flush_interval_ms / 1000.0)

kaq_quant_common/common/modules/limit_order_helper.py

@@ -4,7 +4,6 @@ import threading
 import time
 
 import pandas as pd
-
 from kaq_quant_common.resources.kaq_ddb_stream_write_resources import (
     KaqQuantDdbStreamMTWWriteRepository,
     KaqQuantDdbStreamWriteRepository,
@@ -25,10 +24,10 @@ class LimitOrderHelper:
         self._stop_event = threading.Event()
         self._flusher_thread = threading.Thread(target=self._flush_loop, daemon=True)
         self._flusher_thread.name = "LimitOrderHelperFlusherThread"
-        self._is_df = False
 
         #
         self._ddb = ddb
+        self._isMtwDdb = isinstance(self._ddb, KaqQuantDdbStreamMTWWriteRepository)
         self._ddb_table_name = ddb_table_name
 
         #
@@ -51,6 +50,10 @@ class LimitOrderHelper:
         self._flusher_thread.join()
 
     def _flush_loop(self):
+        cum_count = 0
+        cum_convert_time = 0
+        cum_write_ddb_time = 0
+        cum_total_use_time = 0
         # 周期性地将每个symbol的最新快照批量入库
         while not self._stop_event.is_set():
             to_process = None
@@ -65,42 +68,76 @@ class LimitOrderHelper:
                 now = int(datetime.datetime.now().timestamp() * 1000)
 
                 for symbol, (data, arg) in to_process:
-                    sub_df = self._build_data(symbol, data, arg)
-                    if isinstance(sub_df, pd.DataFrame):
-                        self._is_df = True
-                        # 输出一下
-                        data_first_now = int(sub_df["create_time"].iloc[0])
-                        if now - data_first_now > 2000:
-                            self._logger.warning(f"数据时间{data_first_now} 与当前时间{now} 差值{now - data_first_now} 超过2000ms")
-
-                        if df is None:
-                            df = sub_df
+                    sub_data = self._build_data(symbol, data, arg)
+
+                    if not self._isMtwDdb:
+                        # 可以是数组，可以是dataFrame
+                        is_df = type(sub_data) is pd.DataFrame
+
+                        if is_df:
+                            # df就用df的方式写入
+                            data_first_now = int(sub_data["create_time"].iloc[0])
+                            if now - data_first_now > 2000:
+                                self._logger.debug(f"数据时间{data_first_now} 与当前时间{now} 差值{now - data_first_now} 超过2000ms")
+                                pass
+
+                            if df is None:
+                                df = sub_data
+                            else:
+                                df = pd.concat([df, sub_data], ignore_index=True)
                         else:
-                            df = pd.concat([df, sub_df], ignore_index=True)
+                            # 数组就用数组的方式写入
+                            data_first_now = int(sub_data[0])
+                            if now - data_first_now > 2000:
+                                self._logger.debug(f"数据时间{data_first_now} 与当前时间{now} 差值{now - data_first_now} 超过2000ms")
+                                pass
+
+                            list_data.append(sub_data)
                     else:
-                        list_data = sub_df
+                        # 只能是数组
+                        if len(sub_data) > 0:
+                            # 直接调用 save2stream_list 写入
+                            try:
+                                self._ddb.save2stream_list(sub_data)
+                            except Exception as e:
+                                # 避免刷库异常导致线程退出
+                                self._logger.error(f"批量写入数组失败: {e}")
+
+                convert_time = int(datetime.datetime.now().timestamp() * 1000)
 
                 # 入库
-                if self._is_df:
+                if not self._isMtwDdb:
+                    # 兼容df和数组
                     if df is not None and not df.empty:
                         try:
                             self._ddb.save2stream_batch(self._ddb_table_name, df=df)
                         except Exception as e:
                             # 避免刷库异常导致线程退出
                             self._logger.error(f"批量写入df失败: {e}")
-                else:
                     if len(list_data) > 0:
                         try:
-                            self._ddb.save2stream_list(list_data)
+                            self._ddb.save2stream_batch_list(self._ddb_table_name, data=list_data)
                         except Exception as e:
                             # 避免刷库异常导致线程退出
-                            self._logger.error(f"批量写入数组失败: {e}")
+                            self._logger.error(f"批量写入list失败: {e}")
 
                 # 统计一下
                 end = int(datetime.datetime.now().timestamp() * 1000)
-                if (end - now) > self._flush_interval_ms:
-                    self._logger.warning(f"批量写入{len(to_process)}条数据耗时{end - now}ms")
-
-            # 是dataFrame的才睡眠，数组处理交由ddb自己控制节奏
-            if self._is_df:
+                total_use_time = end - now
+                convert_use = convert_time - now
+                write_ddb_use = total_use_time - convert_use
+
+                #
+                cum_count += len(to_process)
+                cum_convert_time += convert_use
+                cum_write_ddb_time += write_ddb_use
+                cum_total_use_time += total_use_time
+
+                if total_use_time > 500 and cum_count > 0:
+                    self._logger.debug(
+                        f"批量写入{len(to_process)}条数据耗时{total_use_time}ms(avg {cum_total_use_time / cum_count:.2f}ms) 转换耗时{convert_use}ms(avg {cum_convert_time / cum_count:.2f}ms) 写入ddb耗时{write_ddb_use}ms(avg {cum_write_ddb_time / cum_count:.2f}ms)"
+                    )
+
+            # mtw交由ddb自己控制节奏
+            if self._isMtwDdb:
                 time.sleep(self._flush_interval_ms / 1000.0)