kaq-quant-common 0.1.83__py3-none-any.whl → 0.1.85__py3-none-any.whl

kaq_quant_common/common/ddb_table_monitor.py

@@ -1,106 +1,106 @@
-import threading
-import traceback
-from abc import abstractmethod
-
-import dolphindb as ddb
-import numpy as np
-from kaq_quant_common.common.monitor_base import MonitorBase
-from kaq_quant_common.utils import logger_utils
-
-mutex = threading.Lock()
-
-
-# ddb表订阅监听器
-class DdbTableMonitor(MonitorBase):
-
-    def __init__(self, table_name: str, action_name: str, batch_size=1000, filter=[]):
-        # 表名
-        self._table_name = table_name
-        #
-        self._action_name = action_name
-        #
-        self._batch_size = batch_size
-        #
-        self._filter = filter
-
-        # logger
-        self._logger = logger_utils.get_logger(self)
-        #
-        super().__init__()
-
-    # ↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓ abstract methods
-    def _do_init(self):
-        # 初始化ddb
-        self._init_ddb()
-
-    def _do_start(self):
-        # 开启ddb订阅
-        self._start_subscribe()
-
-    def _do_stop(self):
-        # 关闭订阅
-        self._stop_subscribe()
-
-    # ↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓
-    # 初始化ddb
-    def _init_ddb(self):
-        '''
-        创建ddb连接 && 添加ddb流数据表支持
-        '''
-        try:
-            ddb_config = self._on_get_ddb_config()
-            host, port, user, passwd = ddb_config['host'], ddb_config['port'], ddb_config['user'], ddb_config['passwd']
-            mutex.acquire()
-            self._session = ddb.session(enableASYNC=True)
-            self._host = host
-            self._port = port
-            self._user = user
-            self._passwd = passwd
-            self._session.connect(host, port, user, passwd)
-            self._session.enableStreaming()
-        except Exception as e:
-            self._logger.error(f'DdbTableMonitor._init_ddb error: {str(e)} - {str(traceback.format_exc())}')
-        finally:
-            mutex.release()
-
-    # 开启订阅
-    def _start_subscribe(self):
-        '''
-        订阅ddb表
-        '''
-        self._session.subscribe(
-            self._host,
-            self._port,
-            self._handle,
-            tableName=self._table_name,
-            actionName=self._action_name,
-            filter=np.array(self._filter),
-            offset=-1,
-            batchSize=self._batch_size,
-            throttle=5,
-            msgAsTable=True,
-        )
-        self._logger.info(f'开始订阅 {self._host}:{self._port} {self._table_name} - {self._action_name}')
-
-    def _stop_subscribe(self):
-        # TODO
-        # script = f"""
-        # existsSubscriptionTopic(,`{self._table_name},`{self._action_name})
-        # """
-        # exitsTopic = self._session.run(script)
-        exitsTopic = True
-        if exitsTopic is True:
-            self._session.unsubscribe(self._host, self._port, self._table_name, self._action_name)
-            self._logger.info(f'取消订阅 {self._table_name} - {self._action_name}')
-        if not self._session.isClosed():
-            self._session.close()
-
-    # ↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓ abstract methods
-    # 需要返回ddb配置，包含host, port, user, passwd 添加类型提示
-    @abstractmethod
-    def _on_get_ddb_config(self, data) -> dict:
-        pass
-
-    @abstractmethod
-    def _handle(self, data):
-        pass
+import threading
+import traceback
+from abc import abstractmethod
+
+import dolphindb as ddb
+import numpy as np
+from kaq_quant_common.common.monitor_base import MonitorBase
+from kaq_quant_common.utils import logger_utils
+
+mutex = threading.Lock()
+
+
+# ddb表订阅监听器
+class DdbTableMonitor(MonitorBase):
+
+    def __init__(self, table_name: str, action_name: str, batch_size=1000, filter=[]):
+        # 表名
+        self._table_name = table_name
+        #
+        self._action_name = action_name
+        #
+        self._batch_size = batch_size
+        #
+        self._filter = filter
+
+        # logger
+        self._logger = logger_utils.get_logger(self)
+        #
+        super().__init__()
+
+    # ↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓ abstract methods
+    def _do_init(self):
+        # 初始化ddb
+        self._init_ddb()
+
+    def _do_start(self):
+        # 开启ddb订阅
+        self._start_subscribe()
+
+    def _do_stop(self):
+        # 关闭订阅
+        self._stop_subscribe()
+
+    # ↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓
+    # 初始化ddb
+    def _init_ddb(self):
+        '''
+        创建ddb连接 && 添加ddb流数据表支持
+        '''
+        try:
+            ddb_config = self._on_get_ddb_config()
+            host, port, user, passwd = ddb_config['host'], ddb_config['port'], ddb_config['user'], ddb_config['passwd']
+            mutex.acquire()
+            self._session = ddb.session(enableASYNC=True)
+            self._host = host
+            self._port = port
+            self._user = user
+            self._passwd = passwd
+            self._session.connect(host, port, user, passwd)
+            self._session.enableStreaming()
+        except Exception as e:
+            self._logger.error(f'DdbTableMonitor._init_ddb error: {str(e)} - {str(traceback.format_exc())}')
+        finally:
+            mutex.release()
+
+    # 开启订阅
+    def _start_subscribe(self):
+        '''
+        订阅ddb表
+        '''
+        self._session.subscribe(
+            self._host,
+            self._port,
+            self._handle,
+            tableName=self._table_name,
+            actionName=self._action_name,
+            filter=np.array(self._filter),
+            offset=-1,
+            batchSize=self._batch_size,
+            throttle=5,
+            msgAsTable=True,
+        )
+        self._logger.info(f'开始订阅 {self._host}:{self._port} {self._table_name} - {self._action_name}')
+
+    def _stop_subscribe(self):
+        # TODO
+        # script = f"""
+        # existsSubscriptionTopic(,`{self._table_name},`{self._action_name})
+        # """
+        # exitsTopic = self._session.run(script)
+        exitsTopic = True
+        if exitsTopic is True:
+            self._session.unsubscribe(self._host, self._port, self._table_name, self._action_name)
+            self._logger.info(f'取消订阅 {self._table_name} - {self._action_name}')
+        if not self._session.isClosed():
+            self._session.close()
+
+    # ↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓ abstract methods
+    # 需要返回ddb配置，包含host, port, user, passwd 添加类型提示
+    @abstractmethod
+    def _on_get_ddb_config(self, data) -> dict:
+        pass
+
+    @abstractmethod
+    def _handle(self, data):
+        pass

kaq_quant_common/common/http_monitor.py

@@ -1,69 +1,69 @@
-import threading
-import time
-from abc import abstractmethod
-
-from kaq_quant_common.common.monitor_base import MonitorBase
-from kaq_quant_common.common.ws_wrapper import WsWrapper
-from kaq_quant_common.utils import logger_utils
-
-
-# 封装http定时请求
-class HttpMonitor(MonitorBase):
-    def __init__(self, interval=5):
-        super().__init__()
-        # 执行间隔
-        self._interval = interval
-        self._logger = logger_utils.get_logger()
-
-    def _do_start(self):
-        # 开启一条线程，定时执行http请求
-        self._ticker_thread_event = threading.Event()
-
-        def http_request():
-            # 上次请求时间
-            last_request_time = 0
-            while True:
-                # 检查是否需要退出
-                if self._ticker_thread_event.is_set():
-                    self._logger.info("ticker thread exit")
-                    break
-
-                # 当前时间
-                current_time = time.time()
-                # 如果上次请求时间距离当前时间不足，等待
-                if current_time - last_request_time < self._interval:
-                    time.sleep(0.1)
-                    continue
-
-                #
-                last_request_time = time.time()
-
-                # self._logger.debug('tick start')
-                try:
-                    self._do_request()
-                except Exception as e:
-                    self._logger.error(f"http request error: {e}")
-                # self._logger.debug('tick finish')
-                #
-                time.sleep(0.1)
-
-        # 开启线程
-        self._ticker_thread = threading.Thread(target=http_request)
-        # 设置为守护线程
-        self._ticker_thread.daemon = True
-        self._ticker_thread.start()
-
-    def _do_stop(self):
-        if self._ticker_thread_event is not None:
-            self._ticker_thread_event.set()
-
-    # ↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓
-
-    @abstractmethod
-    def _do_request(self):
-        """
-        子类实现
-        """
-        pass
-
-    # ↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓
+import threading
+import time
+from abc import abstractmethod
+
+from kaq_quant_common.common.monitor_base import MonitorBase
+from kaq_quant_common.common.ws_wrapper import WsWrapper
+from kaq_quant_common.utils import logger_utils
+
+
+# 封装http定时请求
+class HttpMonitor(MonitorBase):
+    def __init__(self, interval=5):
+        super().__init__()
+        # 执行间隔
+        self._interval = interval
+        self._logger = logger_utils.get_logger()
+
+    def _do_start(self):
+        # 开启一条线程，定时执行http请求
+        self._ticker_thread_event = threading.Event()
+
+        def http_request():
+            # 上次请求时间
+            last_request_time = 0
+            while True:
+                # 检查是否需要退出
+                if self._ticker_thread_event.is_set():
+                    self._logger.info("ticker thread exit")
+                    break
+
+                # 当前时间
+                current_time = time.time()
+                # 如果上次请求时间距离当前时间不足，等待
+                if current_time - last_request_time < self._interval:
+                    time.sleep(0.1)
+                    continue
+
+                #
+                last_request_time = time.time()
+
+                # self._logger.debug('tick start')
+                try:
+                    self._do_request()
+                except Exception as e:
+                    self._logger.error(f"http request error: {e}")
+                # self._logger.debug('tick finish')
+                #
+                time.sleep(0.1)
+
+        # 开启线程
+        self._ticker_thread = threading.Thread(target=http_request)
+        # 设置为守护线程
+        self._ticker_thread.daemon = True
+        self._ticker_thread.start()
+
+    def _do_stop(self):
+        if self._ticker_thread_event is not None:
+            self._ticker_thread_event.set()
+
+    # ↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓
+
+    @abstractmethod
+    def _do_request(self):
+        """
+        子类实现
+        """
+        pass
+
+    # ↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓

kaq_quant_common/common/modules/limit_order_helper.py

@@ -1,81 +1,81 @@
-# 避免写入导致阻塞
-import datetime
-import threading
-import time
-
-import pandas as pd
-from kaq_quant_common.resources.kaq_ddb_stream_write_resources import (
-    KaqQuantDdbStreamWriteRepository,
-)
-from kaq_quant_common.utils import logger_utils
-
-
-class LimitOrderHelper:
-
-    def __init__(self, ddb: KaqQuantDdbStreamWriteRepository, ddb_table_name: str):
-        # 最新快照缓存与刷库线程控制
-        self._latest_snapshots: dict[str, tuple] = {}
-        self._latest_lock = threading.Lock()
-        # 写入到ddb的频率，默认100ms
-        self._flush_interval_ms = 100
-        self._stop_event = threading.Event()
-        self._flusher_thread = threading.Thread(target=self._flush_loop, daemon=True)
-        self._flusher_thread.name = "LimitOrderHelperFlusherThread"
-
-        #
-        self._ddb = ddb
-        self._ddb_table_name = ddb_table_name
-
-        #
-        self._logger = logger_utils.get_logger(self)
-
-        self._build_data: callable = None
-
-    def set_build_data(self, build_data: callable):
-        self._build_data = build_data
-
-    def push_data(self, symbol: str, data: dict, arg: dict = None):
-        with self._latest_lock:
-            self._latest_snapshots[symbol] = (data, arg)
-
-    def start(self):
-        self._flusher_thread.start()
-
-    def stop(self):
-        self._stop_event.set()
-        self._flusher_thread.join()
-
-    def _flush_loop(self):
-        # 周期性地将每个symbol的最新快照批量入库
-        while not self._stop_event.is_set():
-            to_process = None
-            with self._latest_lock:
-                if self._latest_snapshots:
-                    to_process = list(self._latest_snapshots.items())
-                    self._latest_snapshots.clear()
-
-            if to_process:
-                df: pd.DataFrame = None
-                now = int(datetime.datetime.now().timestamp() * 1000)
-
-                for symbol, (data, arg) in to_process:
-                    sub_df = self._build_data(symbol, data, arg)
-                    # 输出一下
-                    data_first_now = int(sub_df["create_time"].iloc[0])
-                    if now - data_first_now > 2000:
-                        self._logger.debug(f"数据时间{data_first_now} 与当前时间{now} 差值{now - data_first_now} 超过2000ms")
-
-                    if df is None:
-                        df = sub_df
-                    else:
-                        df = pd.concat([df, sub_df], ignore_index=True)
-
-                # 入库
-                if df is not None and not df.empty:
-                    try:
-                        self._ddb.save2stream_batch(self._ddb_table_name, df=df)
-                    except Exception as e:
-                        # 避免刷库异常导致线程退出
-                        self._logger.error(f"批量写入失败: {e}")
-
-            time.sleep(self._flush_interval_ms / 1000.0)
+# 避免写入导致阻塞
+import datetime
+import threading
+import time
+
+import pandas as pd
+from kaq_quant_common.resources.kaq_ddb_stream_write_resources import (
+    KaqQuantDdbStreamWriteRepository,
+)
+from kaq_quant_common.utils import logger_utils
+
+
+class LimitOrderHelper:
+
+    def __init__(self, ddb: KaqQuantDdbStreamWriteRepository, ddb_table_name: str):
+        # 最新快照缓存与刷库线程控制
+        self._latest_snapshots: dict[str, tuple] = {}
+        self._latest_lock = threading.Lock()
+        # 写入到ddb的频率，默认100ms
+        self._flush_interval_ms = 100
+        self._stop_event = threading.Event()
+        self._flusher_thread = threading.Thread(target=self._flush_loop, daemon=True)
+        self._flusher_thread.name = "LimitOrderHelperFlusherThread"
+
+        #
+        self._ddb = ddb
+        self._ddb_table_name = ddb_table_name
+
+        #
+        self._logger = logger_utils.get_logger(self)
+
+        self._build_data: callable = None
+
+    def set_build_data(self, build_data: callable):
+        self._build_data = build_data
+
+    def push_data(self, symbol: str, data: dict, arg: dict = None):
+        with self._latest_lock:
+            self._latest_snapshots[symbol] = (data, arg)
+
+    def start(self):
+        self._flusher_thread.start()
+
+    def stop(self):
+        self._stop_event.set()
+        self._flusher_thread.join()
+
+    def _flush_loop(self):
+        # 周期性地将每个symbol的最新快照批量入库
+        while not self._stop_event.is_set():
+            to_process = None
+            with self._latest_lock:
+                if self._latest_snapshots:
+                    to_process = list(self._latest_snapshots.items())
+                    self._latest_snapshots.clear()
+
+            if to_process:
+                df: pd.DataFrame = None
+                now = int(datetime.datetime.now().timestamp() * 1000)
+
+                for symbol, (data, arg) in to_process:
+                    sub_df = self._build_data(symbol, data, arg)
+                    # 输出一下
+                    data_first_now = int(sub_df["create_time"].iloc[0])
+                    if now - data_first_now > 2000:
+                        self._logger.debug(f"数据时间{data_first_now} 与当前时间{now} 差值{now - data_first_now} 超过2000ms")
+
+                    if df is None:
+                        df = sub_df
+                    else:
+                        df = pd.concat([df, sub_df], ignore_index=True)
+
+                # 入库
+                if df is not None and not df.empty:
+                    try:
+                        self._ddb.save2stream_batch(self._ddb_table_name, df=df)
+                    except Exception as e:
+                        # 避免刷库异常导致线程退出
+                        self._logger.error(f"批量写入失败: {e}")
+
+            time.sleep(self._flush_interval_ms / 1000.0)

kaq_quant_common/common/modules/limit_order_symbol_monitor.py

@@ -1,76 +1,76 @@
-import json
-
-from kaq_quant_common.common.redis_table_monitor import RedisTableMonitor
-from kaq_quant_common.utils import yml_utils
-
-
-# 订单簿交易对监听器
-class LimitOrderSymbolMonitor(RedisTableMonitor):
-    def __init__(self, exchange: str, handler: callable = None, default_symbols=["BTCUSDT", "ETHUSDT"]):
-        #
-        self._exchange = exchange
-        # 回调
-        self._handler = handler
-        #
-        self._support_symbols = default_symbols
-        # 记录一下之前的字符串
-        self._str_value = None
-
-        # 间隔1秒
-        super().__init__(table_name="kaq_all_futures_limit_order_symbols_config", interval=1)
-
-        # 输出一下支持的交易对
-        # self._logger.info(f"init {self._exchange} limit order support symbols: {self._support_symbols}")
-
-    def _on_get_redis_config(self):
-        # 获取redis配置
-        host, port, passwd = yml_utils.get(f"kaq_{self._exchange}_quant", "redis", ["host", "port", "passwd"])
-        return {"host": host, "port": int(port), "passwd": passwd}
-
-    def _do_query(self) -> str:
-        # 获取字符串
-        str = super()._do_query()
-        try:
-            # 解析为json
-            json_obj = json.loads(str)
-            # 只要平台的值
-            symbols = json_obj.get(self._exchange, self._support_symbols)
-
-            #
-            tmp_symbols = []
-            tmp_symbols.extend(symbols)
-
-            if self._support_symbols is not None and len(self._support_symbols) > 0: 
-                # !! 注意，这里需要过滤一下，只保留支持的交易对
-                tmp_symbols = [symbol for symbol in symbols if symbol in self._support_symbols]
-            # df_symbols = pd.Series(symbols)
-            # df_symbols = df_symbols[df_symbols.isin(self._support_symbols)]
-            # symbols = df_symbols.tolist()
-            if tmp_symbols != symbols and self._str_value != str:
-                # 输出一下日志
-                self._logger.warning(f"{self._exchange} limit order symbols contain unsupported symbols, before: {symbols} filtered: {tmp_symbols}")
-
-            symbols = tmp_symbols
-        except Exception as e:
-            symbols = self._support_symbols
-
-        self._str_value = str
-
-        return symbols
-
-    def _do_compare(self, value1: list[str], value2: list[str]) -> bool:
-        # 判断数组是否一样，数组也是这样判断也可以
-        return value1 == value2
-
-    def _on_value_change(self, value: list[str]):
-        # 输出一下变化的交易对
-        self._logger.info(f"{self._exchange} limit order symbols changed: {value}")
-        if self._handler is not None:
-            self._handler(value)
-
-    # get, set
-    def set_handler(self, handler: callable):
-        self._handler = handler
-
-    def get_symbols(self) -> list[str]:
-        return self._value
+import json
+
+from kaq_quant_common.common.redis_table_monitor import RedisTableMonitor
+from kaq_quant_common.utils import yml_utils
+
+
+# 订单簿交易对监听器
+class LimitOrderSymbolMonitor(RedisTableMonitor):
+    def __init__(self, exchange: str, handler: callable = None, default_symbols=["BTCUSDT", "ETHUSDT"]):
+        #
+        self._exchange = exchange
+        # 回调
+        self._handler = handler
+        #
+        self._support_symbols = default_symbols
+        # 记录一下之前的字符串
+        self._str_value = None
+
+        # 间隔1秒
+        super().__init__(table_name="kaq_all_futures_limit_order_symbols_config", interval=1)
+
+        # 输出一下支持的交易对
+        # self._logger.info(f"init {self._exchange} limit order support symbols: {self._support_symbols}")
+
+    def _on_get_redis_config(self):
+        # 获取redis配置
+        host, port, passwd = yml_utils.get(f"kaq_{self._exchange}_quant", "redis", ["host", "port", "passwd"])
+        return {"host": host, "port": int(port), "passwd": passwd}
+
+    def _do_query(self) -> str:
+        # 获取字符串
+        str = super()._do_query()
+        try:
+            # 解析为json
+            json_obj = json.loads(str)
+            # 只要平台的值
+            symbols = json_obj.get(self._exchange, self._support_symbols)
+
+            #
+            tmp_symbols = []
+            tmp_symbols.extend(symbols)
+
+            if self._support_symbols is not None and len(self._support_symbols) > 0: 
+                # !! 注意，这里需要过滤一下，只保留支持的交易对
+                tmp_symbols = [symbol for symbol in symbols if symbol in self._support_symbols]
+            # df_symbols = pd.Series(symbols)
+            # df_symbols = df_symbols[df_symbols.isin(self._support_symbols)]
+            # symbols = df_symbols.tolist()
+            if tmp_symbols != symbols and self._str_value != str:
+                # 输出一下日志
+                self._logger.warning(f"{self._exchange} limit order symbols contain unsupported symbols, before: {symbols} filtered: {tmp_symbols}")
+
+            symbols = tmp_symbols
+        except Exception as e:
+            symbols = self._support_symbols
+
+        self._str_value = str
+
+        return symbols
+
+    def _do_compare(self, value1: list[str], value2: list[str]) -> bool:
+        # 判断数组是否一样，数组也是这样判断也可以
+        return value1 == value2
+
+    def _on_value_change(self, value: list[str]):
+        # 输出一下变化的交易对
+        self._logger.info(f"{self._exchange} limit order symbols changed: {value}")
+        if self._handler is not None:
+            self._handler(value)
+
+    # get, set
+    def set_handler(self, handler: callable):
+        self._handler = handler
+
+    def get_symbols(self) -> list[str]:
+        return self._value