kaq-quant-common 0.2.9__py3-none-any.whl → 0.2.11__py3-none-any.whl

kaq_quant_common/common/ddb_table_monitor.py

@@ -1,106 +1,106 @@
-import threading
-import traceback
-from abc import abstractmethod
-
-import dolphindb as ddb
-import numpy as np
-from kaq_quant_common.common.monitor_base import MonitorBase
-from kaq_quant_common.utils import logger_utils
-
-mutex = threading.Lock()
-
-
-# ddb表订阅监听器
-class DdbTableMonitor(MonitorBase):
-
-    def __init__(self, table_name: str, action_name: str, batch_size=1000, filter=[]):
-        # 表名
-        self._table_name = table_name
-        #
-        self._action_name = action_name
-        #
-        self._batch_size = batch_size
-        #
-        self._filter = filter
-
-        # logger
-        self._logger = logger_utils.get_logger(self)
-        #
-        super().__init__()
-
-    # ↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓ abstract methods
-    def _do_init(self):
-        # 初始化ddb
-        self._init_ddb()
-
-    def _do_start(self):
-        # 开启ddb订阅
-        self._start_subscribe()
-
-    def _do_stop(self):
-        # 关闭订阅
-        self._stop_subscribe()
-
-    # ↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓
-    # 初始化ddb
-    def _init_ddb(self):
-        '''
-        创建ddb连接 && 添加ddb流数据表支持
-        '''
-        try:
-            ddb_config = self._on_get_ddb_config()
-            host, port, user, passwd = ddb_config['host'], ddb_config['port'], ddb_config['user'], ddb_config['passwd']
-            mutex.acquire()
-            self._session = ddb.session(enableASYNC=True)
-            self._host = host
-            self._port = port
-            self._user = user
-            self._passwd = passwd
-            self._session.connect(host, port, user, passwd)
-            self._session.enableStreaming()
-        except Exception as e:
-            self._logger.error(f'DdbTableMonitor._init_ddb error: {str(e)} - {str(traceback.format_exc())}')
-        finally:
-            mutex.release()
-
-    # 开启订阅
-    def _start_subscribe(self):
-        '''
-        订阅ddb表
-        '''
-        self._session.subscribe(
-            self._host,
-            self._port,
-            self._handle,
-            tableName=self._table_name,
-            actionName=self._action_name,
-            filter=np.array(self._filter),
-            offset=-1,
-            batchSize=self._batch_size,
-            throttle=5,
-            msgAsTable=True,
-        )
-        self._logger.info(f'开始订阅 {self._host}:{self._port} {self._table_name} - {self._action_name}')
-
-    def _stop_subscribe(self):
-        # TODO
-        # script = f"""
-        # existsSubscriptionTopic(,`{self._table_name},`{self._action_name})
-        # """
-        # exitsTopic = self._session.run(script)
-        exitsTopic = True
-        if exitsTopic is True:
-            self._session.unsubscribe(self._host, self._port, self._table_name, self._action_name)
-            self._logger.info(f'取消订阅 {self._table_name} - {self._action_name}')
-        if not self._session.isClosed():
-            self._session.close()
-
-    # ↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓ abstract methods
-    # 需要返回ddb配置，包含host, port, user, passwd 添加类型提示
-    @abstractmethod
-    def _on_get_ddb_config(self, data) -> dict:
-        pass
-
-    @abstractmethod
-    def _handle(self, data):
-        pass
+import threading
+import traceback
+from abc import abstractmethod
+
+import dolphindb as ddb
+import numpy as np
+from kaq_quant_common.common.monitor_base import MonitorBase
+from kaq_quant_common.utils import logger_utils
+
+mutex = threading.Lock()
+
+
+# ddb表订阅监听器
+class DdbTableMonitor(MonitorBase):
+
+    def __init__(self, table_name: str, action_name: str, batch_size=1000, filter=[]):
+        # 表名
+        self._table_name = table_name
+        #
+        self._action_name = action_name
+        #
+        self._batch_size = batch_size
+        #
+        self._filter = filter
+
+        # logger
+        self._logger = logger_utils.get_logger(self)
+        #
+        super().__init__()
+
+    # ↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓ abstract methods
+    def _do_init(self):
+        # 初始化ddb
+        self._init_ddb()
+
+    def _do_start(self):
+        # 开启ddb订阅
+        self._start_subscribe()
+
+    def _do_stop(self):
+        # 关闭订阅
+        self._stop_subscribe()
+
+    # ↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓
+    # 初始化ddb
+    def _init_ddb(self):
+        '''
+        创建ddb连接 && 添加ddb流数据表支持
+        '''
+        try:
+            ddb_config = self._on_get_ddb_config()
+            host, port, user, passwd = ddb_config['host'], ddb_config['port'], ddb_config['user'], ddb_config['passwd']
+            mutex.acquire()
+            self._session = ddb.session(enableASYNC=True)
+            self._host = host
+            self._port = port
+            self._user = user
+            self._passwd = passwd
+            self._session.connect(host, port, user, passwd)
+            self._session.enableStreaming()
+        except Exception as e:
+            self._logger.error(f'DdbTableMonitor._init_ddb error: {str(e)} - {str(traceback.format_exc())}')
+        finally:
+            mutex.release()
+
+    # 开启订阅
+    def _start_subscribe(self):
+        '''
+        订阅ddb表
+        '''
+        self._session.subscribe(
+            self._host,
+            self._port,
+            self._handle,
+            tableName=self._table_name,
+            actionName=self._action_name,
+            filter=np.array(self._filter),
+            offset=-1,
+            batchSize=self._batch_size,
+            throttle=5,
+            msgAsTable=True,
+        )
+        self._logger.info(f'开始订阅 {self._host}:{self._port} {self._table_name} - {self._action_name}')
+
+    def _stop_subscribe(self):
+        # TODO
+        # script = f"""
+        # existsSubscriptionTopic(,`{self._table_name},`{self._action_name})
+        # """
+        # exitsTopic = self._session.run(script)
+        exitsTopic = True
+        if exitsTopic is True:
+            self._session.unsubscribe(self._host, self._port, self._table_name, self._action_name)
+            self._logger.info(f'取消订阅 {self._table_name} - {self._action_name}')
+        if not self._session.isClosed():
+            self._session.close()
+
+    # ↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓ abstract methods
+    # 需要返回ddb配置，包含host, port, user, passwd 添加类型提示
+    @abstractmethod
+    def _on_get_ddb_config(self, data) -> dict:
+        pass
+
+    @abstractmethod
+    def _handle(self, data):
+        pass

kaq_quant_common/common/http_monitor.py

@@ -1,69 +1,69 @@
-import threading
-import time
-from abc import abstractmethod
-
-from kaq_quant_common.common.monitor_base import MonitorBase
-from kaq_quant_common.common.ws_wrapper import WsWrapper
-from kaq_quant_common.utils import logger_utils
-
-
-# 封装http定时请求
-class HttpMonitor(MonitorBase):
-    def __init__(self, interval=5):
-        super().__init__()
-        # 执行间隔
-        self._interval = interval
-        self._logger = logger_utils.get_logger()
-
-    def _do_start(self):
-        # 开启一条线程，定时执行http请求
-        self._ticker_thread_event = threading.Event()
-
-        def http_request():
-            # 上次请求时间
-            last_request_time = 0
-            while True:
-                # 检查是否需要退出
-                if self._ticker_thread_event.is_set():
-                    self._logger.info("ticker thread exit")
-                    break
-
-                # 当前时间
-                current_time = time.time()
-                # 如果上次请求时间距离当前时间不足，等待
-                if current_time - last_request_time < self._interval:
-                    time.sleep(0.1)
-                    continue
-
-                #
-                last_request_time = time.time()
-
-                # self._logger.debug('tick start')
-                try:
-                    self._do_request()
-                except Exception as e:
-                    self._logger.error(f"http request error: {e}")
-                # self._logger.debug('tick finish')
-                #
-                time.sleep(0.1)
-
-        # 开启线程
-        self._ticker_thread = threading.Thread(target=http_request)
-        # 设置为守护线程
-        self._ticker_thread.daemon = True
-        self._ticker_thread.start()
-
-    def _do_stop(self):
-        if self._ticker_thread_event is not None:
-            self._ticker_thread_event.set()
-
-    # ↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓
-
-    @abstractmethod
-    def _do_request(self):
-        """
-        子类实现
-        """
-        pass
-
-    # ↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓
+import threading
+import time
+from abc import abstractmethod
+
+from kaq_quant_common.common.monitor_base import MonitorBase
+from kaq_quant_common.common.ws_wrapper import WsWrapper
+from kaq_quant_common.utils import logger_utils
+
+
+# 封装http定时请求
+class HttpMonitor(MonitorBase):
+    def __init__(self, interval=5):
+        super().__init__()
+        # 执行间隔
+        self._interval = interval
+        self._logger = logger_utils.get_logger()
+
+    def _do_start(self):
+        # 开启一条线程，定时执行http请求
+        self._ticker_thread_event = threading.Event()
+
+        def http_request():
+            # 上次请求时间
+            last_request_time = 0
+            while True:
+                # 检查是否需要退出
+                if self._ticker_thread_event.is_set():
+                    self._logger.info("ticker thread exit")
+                    break
+
+                # 当前时间
+                current_time = time.time()
+                # 如果上次请求时间距离当前时间不足，等待
+                if current_time - last_request_time < self._interval:
+                    time.sleep(0.1)
+                    continue
+
+                #
+                last_request_time = time.time()
+
+                # self._logger.debug('tick start')
+                try:
+                    self._do_request()
+                except Exception as e:
+                    self._logger.error(f"http request error: {e}")
+                # self._logger.debug('tick finish')
+                #
+                time.sleep(0.1)
+
+        # 开启线程
+        self._ticker_thread = threading.Thread(target=http_request)
+        # 设置为守护线程
+        self._ticker_thread.daemon = True
+        self._ticker_thread.start()
+
+    def _do_stop(self):
+        if self._ticker_thread_event is not None:
+            self._ticker_thread_event.set()
+
+    # ↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓
+
+    @abstractmethod
+    def _do_request(self):
+        """
+        子类实现
+        """
+        pass
+
+    # ↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓

kaq_quant_common/common/modules/funding_rate_helper.py

@@ -68,10 +68,16 @@ class FundingRateHelper:
                 for symbol, (data, arg) in to_process:
                     sub_data = self._build_data(symbol, data, arg)
 
+                    if sub_data is None:
+                        continue
+
+                    if len(sub_data) == 0:
+                        continue
+
                     if not self._isMtwDdb:
                         # 可以是数组，可以是dataFrame
                         is_df = type(sub_data) is pd.DataFrame
-                        
+
                         if is_df:
                             # df就用df的方式写入
                             # data_first_now = int(sub_data["create_time"].iloc[0])
@@ -85,21 +91,30 @@ class FundingRateHelper:
                                 df = pd.concat([df, sub_data], ignore_index=True)
                         else:
                             # 数组就用数组的方式写入
-                            # data_first_now = int(sub_data[0])
-                            # if now - data_first_now > 2000:
-                            #     self._logger.warning(f"数据时间{data_first_now} 与当前时间{now} 差值{now - data_first_now} 超过2000ms")
-                            #     pass
-
-                            list_data.append(sub_data)
+                            # 子元素是否数组
+                            is_sub_list = type(sub_data) is list
+                            if is_sub_list:
+                                # 多条数据
+                                # data_first_now = int(sub_data[0][0])
+                                # if now - data_first_now > 2000:
+                                #     self._logger.debug(f"数据时间{data_first_now} 与当前时间{now} 差值{now - data_first_now} 超过2000ms")
+                                #     pass
+                                list_data.extend(sub_data)
+                            else:
+                                # 单条数据
+                                # data_first_now = int(sub_data[0])
+                                # if now - data_first_now > 2000:
+                                #     self._logger.debug(f"数据时间{data_first_now} 与当前时间{now} 差值{now - data_first_now} 超过2000ms")
+                                #     pass
+                                list_data.append(sub_data)
                     else:
-                        # 只能是数组
-                        if len(sub_data) > 0:
-                            # 直接调用 save2stream_list 写入
-                            try:
-                                self._ddb.save2stream_list(sub_data)
-                            except Exception as e:
-                                # 避免刷库异常导致线程退出
-                                self._logger.error(f"批量写入数组失败: {e}")
+                        # 直接调用 save2stream_list 写入
+                        try:
+                            self._ddb.save2stream_list(sub_data)
+                        except Exception as e:
+                            # 避免刷库异常导致线程退出
+                            self._logger.error(f"批量写入数组失败: {e}")
+                            
 
                 # 入库
                 if not self._isMtwDdb:

kaq_quant_common/common/modules/limit_order_helper.py

@@ -70,6 +70,12 @@ class LimitOrderHelper:
                 for symbol, (data, arg) in to_process:
                     sub_data = self._build_data(symbol, data, arg)
 
+                    if sub_data is None:
+                        continue
+
+                    if len(sub_data) == 0:
+                        continue
+
                     if not self._isMtwDdb:
                         # 可以是数组，可以是dataFrame
                         is_df = type(sub_data) is pd.DataFrame
@@ -77,9 +83,9 @@ class LimitOrderHelper:
                         if is_df:
                             # df就用df的方式写入
                             data_first_now = int(sub_data["create_time"].iloc[0])
-                            if now - data_first_now > 2000:
-                                self._logger.debug(f"数据时间{data_first_now} 与当前时间{now} 差值{now - data_first_now} 超过2000ms")
-                                pass
+                            # if now - data_first_now > 2000:
+                            #     self._logger.debug(f"数据时间{data_first_now} 与当前时间{now} 差值{now - data_first_now} 超过2000ms")
+                            #     pass
 
                             if df is None:
                                 df = sub_data
@@ -87,21 +93,30 @@ class LimitOrderHelper:
                                 df = pd.concat([df, sub_data], ignore_index=True)
                         else:
                             # 数组就用数组的方式写入
-                            data_first_now = int(sub_data[0])
-                            if now - data_first_now > 2000:
-                                self._logger.debug(f"数据时间{data_first_now} 与当前时间{now} 差值{now - data_first_now} 超过2000ms")
-                                pass
+                            # 子元素是否数组
+                            is_sub_list = type(sub_data[0]) is list
+                            if is_sub_list:
+                                # 多条数据
+                                data_first_now = int(sub_data[0][0])
+                                # if now - data_first_now > 2000:
+                                #     self._logger.debug(f"数据时间{data_first_now} 与当前时间{now} 差值{now - data_first_now} 超过2000ms")
+                                #     pass
+                                list_data.extend(sub_data)
+                            else:
+                                # 单条数据
+                                data_first_now = int(sub_data[0])
+                                # if now - data_first_now > 2000:
+                                #     self._logger.debug(f"数据时间{data_first_now} 与当前时间{now} 差值{now - data_first_now} 超过2000ms")
+                                #     pass
+                                list_data.append(sub_data)
 
-                            list_data.append(sub_data)
                     else:
-                        # 只能是数组
-                        if len(sub_data) > 0:
-                            # 直接调用 save2stream_list 写入
-                            try:
-                                self._ddb.save2stream_list(sub_data)
-                            except Exception as e:
-                                # 避免刷库异常导致线程退出
-                                self._logger.error(f"批量写入数组失败: {e}")
+                        # 直接调用 save2stream_list 写入
+                        try:
+                            self._ddb.save2stream_list(sub_data)
+                        except Exception as e:
+                            # 避免刷库异常导致线程退出
+                            self._logger.error(f"批量写入数组失败: {e}")
 
                 convert_time = int(datetime.datetime.now().timestamp() * 1000)
 
@@ -133,10 +148,10 @@ class LimitOrderHelper:
                 cum_write_ddb_time += write_ddb_use
                 cum_total_use_time += total_use_time
 
-                if total_use_time > 500 and cum_count > 0:
-                    self._logger.debug(
-                        f"批量写入{len(to_process)}条数据耗时{total_use_time}ms(avg {cum_total_use_time / cum_count:.2f}ms) 转换耗时{convert_use}ms(avg {cum_convert_time / cum_count:.2f}ms) 写入ddb耗时{write_ddb_use}ms(avg {cum_write_ddb_time / cum_count:.2f}ms)"
-                    )
+                # if total_use_time > 500 and cum_count > 0:
+                #     self._logger.debug(
+                #         f"批量写入{len(to_process)}条数据耗时{total_use_time}ms(avg {cum_total_use_time / cum_count:.2f}ms) 转换耗时{convert_use}ms(avg {cum_convert_time / cum_count:.2f}ms) 写入ddb耗时{write_ddb_use}ms(avg {cum_write_ddb_time / cum_count:.2f}ms)"
+                #     )
 
             # mtw交由ddb自己控制节奏
             if self._isMtwDdb: