kaq-quant-common 0.1.99__py3-none-any.whl → 0.2.2__py3-none-any.whl

kaq_quant_common/common/monitor_base.py

@@ -1,84 +1,84 @@
-import threading
-import time
-from abc import ABC, abstractmethod
-
-from kaq_quant_common.utils import logger_utils, signal_utils
-
-
-# 通用的抽象类
-class MonitorBase(ABC):
-    def __init__(self):
-        # 初始化
-        self.init()
-        # 用来标记最后一次收到数据的时间，超过指定时间没有收到数据，认为连接断开
-        self._keep_alive_deadline = 0
-
-    # 初始化
-    def init(self):
-        # 执行初始化
-        self._do_init()
-
-    @abstractmethod
-    def _do_init(self):
-        pass
-
-    # 开始，需要确保这个方法在op线程执行，执行这个方法会阻塞当前线程！
-    def start(self, block=True):
-        # 全局退出事件，用于传递终止信号
-        exit_event = threading.Event()
-        self._exit_event = exit_event
-
-        logger = logger_utils.get_logger(self)
-
-        #
-        def handle_terminate_signal(signum, frame=None):
-            """信号处理函数：捕获终止信号并触发退出事件"""
-            logger.info(f"收到终止信号 {signum}")
-            exit_event.set()
-            # 优雅地停止
-            try:
-                self.stop()
-            except Exception as e:
-                pass
-
-        signal_utils.register_signal_handler(handle_terminate_signal)
-
-        self._do_start()
-
-        # 需要阻塞
-        if block:
-            # 监听退出事件
-            while not exit_event.is_set():
-                time.sleep(1)
-                if self._check_exit():
-                    # 假装发送信号
-                    handle_terminate_signal("exit")
-                if self._keep_alive_deadline > 0 and time.time() > self._keep_alive_deadline:
-                    logger.warning("MonitorBase 收据接收超时，需要停止 MonitorBase")
-                    # 假装发送信号
-                    handle_terminate_signal("recv time out")
-
-            logger.warning("MonitorBase 线程退出")
-
-    # 子类实现，判断连接是否断开，断开返回True
-    def _check_exit(self):
-        return False
-
-    @abstractmethod
-    def _do_start(self):
-        pass
-
-    # 停止
-    def stop(self):
-        self._do_stop()
-
-    @abstractmethod
-    def _do_stop(self):
-        pass
-
-    def keep_alive(self, deadline: int = 60):
-        """
-        保持连接alive，超过指定时间不活跃，认为需要停止
-        """
-        cur_time = time.time()
-        self._keep_alive_deadline = cur_time + deadline
+import threading
+import time
+from abc import ABC, abstractmethod
+
+from kaq_quant_common.utils import logger_utils, signal_utils
+
+
+# 通用的抽象类
+class MonitorBase(ABC):
+    def __init__(self):
+        # 初始化
+        self.init()
+        # 用来标记最后一次收到数据的时间，超过指定时间没有收到数据，认为连接断开
+        self._keep_alive_deadline = 0
+
+    # 初始化
+    def init(self):
+        # 执行初始化
+        self._do_init()
+
+    @abstractmethod
+    def _do_init(self):
+        pass
+
+    # 开始，需要确保这个方法在op线程执行，执行这个方法会阻塞当前线程！
+    def start(self, block=True):
+        # 全局退出事件，用于传递终止信号
+        exit_event = threading.Event()
+        self._exit_event = exit_event
+
+        logger = logger_utils.get_logger(self)
+
+        #
+        def handle_terminate_signal(signum, frame=None):
+            """信号处理函数：捕获终止信号并触发退出事件"""
+            logger.info(f"收到终止信号 {signum}")
+            exit_event.set()
+            # 优雅地停止
+            try:
+                self.stop()
+            except Exception as e:
+                pass
+
+        signal_utils.register_signal_handler(handle_terminate_signal)
+
+        self._do_start()
+
+        # 需要阻塞
+        if block:
+            # 监听退出事件
+            while not exit_event.is_set():
+                time.sleep(1)
+                if self._check_exit():
+                    # 假装发送信号
+                    handle_terminate_signal("exit")
+                if self._keep_alive_deadline > 0 and time.time() > self._keep_alive_deadline:
+                    logger.warning("MonitorBase 收据接收超时，需要停止 MonitorBase")
+                    # 假装发送信号
+                    handle_terminate_signal("recv time out")
+
+            logger.warning("MonitorBase 线程退出")
+
+    # 子类实现，判断连接是否断开，断开返回True
+    def _check_exit(self):
+        return False
+
+    @abstractmethod
+    def _do_start(self):
+        pass
+
+    # 停止
+    def stop(self):
+        self._do_stop()
+
+    @abstractmethod
+    def _do_stop(self):
+        pass
+
+    def keep_alive(self, deadline: int = 60):
+        """
+        保持连接alive，超过指定时间不活跃，认为需要停止
+        """
+        cur_time = time.time()
+        self._keep_alive_deadline = cur_time + deadline

kaq_quant_common/common/monitor_group.py

@@ -1,97 +1,97 @@
-import threading
-import time
-
-from kaq_quant_common.common.monitor_base import MonitorBase
-from kaq_quant_common.utils import logger_utils, signal_utils
-
-
-class MonitorGroup:
-
-    def __init__(self):
-        #
-        self._monitors: list[MonitorBase] = []
-        self._logger = logger_utils.get_logger(self)
-
-    def get_monitors(self) -> list[MonitorBase]:
-        return self._monitors
-
-    # 开始，需要确保这个方法在op线程执行，执行这个方法会阻塞当前线程！
-    def start(self, block=True):
-        # 全局退出事件，用于传递终止信号
-        exit_event = threading.Event()
-        self._exit_event = exit_event
-
-        logger = self._logger
-
-        #
-        def handle_terminate_signal(signum, frame=None):
-            """信号处理函数：捕获终止信号并触发退出事件"""
-            logger.info(f"收到终止信号 {signum}")
-            exit_event.set()
-            # 优雅地停止
-            try:
-                self.stop()
-            except Exception as e:
-                pass
-
-        signal_utils.register_signal_handler(handle_terminate_signal)
-
-        # 需要阻塞
-        if block:
-            # 监听退出事件
-            while not exit_event.is_set():
-                time.sleep(1)
-                if self._check_exit():
-                    # 假装发送信号
-                    handle_terminate_signal("exit")
-
-            logger.warning("MonitorGroup 线程退出")
-
-    # 停止
-    def stop(self):
-        #
-        self.stop_monitors()
-        #
-        self._exit_event.set()
-
-    def _check_exit(self):
-        # 避免还没有链接monitor
-        if len(self._monitors) == 0:
-            return False
-
-        # 标识是否全部都已经退出
-        all_exit = True
-        one_exit = False
-
-        # 遍历检测是否全部都退出了
-        for monitor in self._monitors:
-            if not monitor._check_exit():
-                all_exit = False
-            else:
-                one_exit = True
-
-        return one_exit
-
-    # 只停止 monitor
-    def stop_monitors(self, clear=False):
-        self._logger.info(f"停止 {len(self._monitors)} 个 monitor")
-        tmp_monitors = self._monitors.copy()
-        if clear:
-            # 清空数组
-            self._monitors.clear()
-        for monitor in tmp_monitors:
-            monitor.stop()
-
-    #
-    def link(self, monitors: list[MonitorBase], clear=False):
-        # 如果需要clear,先清空之前的
-        if clear:
-            self.stop_monitors(clear=True)
-        self._logger.info(f"开启 {len(monitors)} 个 monitor")
-        # 遍历调用start
-        for monitor in monitors:
-            monitor.start(block=False)
-            # 延迟一下，防止同时创建多个连接有问题
-            time.sleep(1)
-        # 追加新的
-        self._monitors.extend(monitors)
+import threading
+import time
+
+from kaq_quant_common.common.monitor_base import MonitorBase
+from kaq_quant_common.utils import logger_utils, signal_utils
+
+
+class MonitorGroup:
+
+    def __init__(self):
+        #
+        self._monitors: list[MonitorBase] = []
+        self._logger = logger_utils.get_logger(self)
+
+    def get_monitors(self) -> list[MonitorBase]:
+        return self._monitors
+
+    # 开始，需要确保这个方法在op线程执行，执行这个方法会阻塞当前线程！
+    def start(self, block=True):
+        # 全局退出事件，用于传递终止信号
+        exit_event = threading.Event()
+        self._exit_event = exit_event
+
+        logger = self._logger
+
+        #
+        def handle_terminate_signal(signum, frame=None):
+            """信号处理函数：捕获终止信号并触发退出事件"""
+            logger.info(f"收到终止信号 {signum}")
+            exit_event.set()
+            # 优雅地停止
+            try:
+                self.stop()
+            except Exception as e:
+                pass
+
+        signal_utils.register_signal_handler(handle_terminate_signal)
+
+        # 需要阻塞
+        if block:
+            # 监听退出事件
+            while not exit_event.is_set():
+                time.sleep(1)
+                if self._check_exit():
+                    # 假装发送信号
+                    handle_terminate_signal("exit")
+
+            logger.warning("MonitorGroup 线程退出")
+
+    # 停止
+    def stop(self):
+        #
+        self.stop_monitors()
+        #
+        self._exit_event.set()
+
+    def _check_exit(self):
+        # 避免还没有链接monitor
+        if len(self._monitors) == 0:
+            return False
+
+        # 标识是否全部都已经退出
+        all_exit = True
+        one_exit = False
+
+        # 遍历检测是否全部都退出了
+        for monitor in self._monitors:
+            if not monitor._check_exit():
+                all_exit = False
+            else:
+                one_exit = True
+
+        return one_exit
+
+    # 只停止 monitor
+    def stop_monitors(self, clear=False):
+        self._logger.info(f"停止 {len(self._monitors)} 个 monitor")
+        tmp_monitors = self._monitors.copy()
+        if clear:
+            # 清空数组
+            self._monitors.clear()
+        for monitor in tmp_monitors:
+            monitor.stop()
+
+    #
+    def link(self, monitors: list[MonitorBase], clear=False):
+        # 如果需要clear,先清空之前的
+        if clear:
+            self.stop_monitors(clear=True)
+        self._logger.info(f"开启 {len(monitors)} 个 monitor")
+        # 遍历调用start
+        for monitor in monitors:
+            monitor.start(block=False)
+            # 延迟一下，防止同时创建多个连接有问题
+            time.sleep(1)
+        # 追加新的
+        self._monitors.extend(monitors)

kaq_quant_common/common/ws_wrapper.py

@@ -1,21 +1,21 @@
-from abc import abstractmethod
-
-from kaq_quant_common.common.monitor_base import MonitorBase
-
-
-# 通用的抽象类，包装一层ws操作
-class WsWrapper(MonitorBase):
-    def __init__(self):
-        super().__init__()
-
-    # stop 就是调用close，ws 更好理解
-    def _do_stop(self):
-        self._do_close()
-
-    # 断开连接，主动关闭
-    def close(self):
-        self.stop()
-
-    @abstractmethod
-    def _do_close(self):
-        pass
+from abc import abstractmethod
+
+from kaq_quant_common.common.monitor_base import MonitorBase
+
+
+# 通用的抽象类，包装一层ws操作
+class WsWrapper(MonitorBase):
+    def __init__(self):
+        super().__init__()
+
+    # stop 就是调用close，ws 更好理解
+    def _do_stop(self):
+        self._do_close()
+
+    # 断开连接，主动关闭
+    def close(self):
+        self.stop()
+
+    @abstractmethod
+    def _do_close(self):
+        pass

kaq_quant_common/resources/kaq_ddb_stream_write_resources.py

@@ -1,8 +1,10 @@
 import os
 import queue
 import time
+from typing import Optional, Union
 import dolphindb as ddb
 from kaq_quant_common.utils import yml_utils
+import numpy as np
 import pandas as pd
 import threading
 from kaq_quant_common.utils.logger_utils import get_logger
@@ -35,19 +37,35 @@ class KaqQuantDdbStreamWriteRepository:
             self.logger.error(f'KaqQuantDdbStreamWriteRepository.__init__ is occured error: {str(e)} - {str(traceback.format_exc())}')
         finally:
             mutex.release()
-    
-    def save_row(self, ddb_table_name: str, row:list=[]):
+            
+    def save_rows(self, ddb_table_name: str, rows: Optional[Union[np.ndarray, list]] = None):
         '''
         调用此方法之前, 需要将dataframe中的字符串类型的值 ，添加引号
         '''
-        if row is None or len(row) <= 0:
+        if rows is None:
             return
-        formatted_values = [f'\'{v}\'' if isinstance(v, str) else str(v) for v in row]
-        script = f"insert into {ddb_table_name} values({', '.join(str(x) for x in formatted_values)})"
-        try:
-            self.session.run(script, clearMemory=True)
-        except Exception as e:
-            self.logger.error(f'KaqQuantDdbStreamWriteRepository.save2stream is occured error: tableName is {ddb_table_name} - {str(e)} - {str(traceback.format_exc())}')
+        # 获取维度
+        if isinstance(rows[0], (list, np.ndarray, tuple)):
+            # 多行数据
+            try:
+                formatted_values = []
+                for row in rows:
+                    # 这里的 row 是 np.array([1767708564161, "BTC", 92500.1])
+                    row = [f'\'{v}\'' if isinstance(v, str) else str(v) for v in row]
+                    formatted_values.append(f'({', '.join(row)})')
+                script = f"insert into {ddb_table_name} values {', '.join(str(x) for x in formatted_values)}"
+            
+                self.session.run(script, clearMemory=True)
+            except Exception as e:
+                self.logger.error(f'KaqQuantDdbStreamWriteRepository.save_rows is occured error: tableName is {ddb_table_name} - {str(e)} - {str(traceback.format_exc())}')
+        else:
+            # 是一行数据，调用 insert
+            formatted_values = [f'\'{v}\'' if isinstance(v, str) else str(v) for v in rows]
+            script = f"insert into {ddb_table_name} values({', '.join(str(x) for x in formatted_values)})"
+            try:
+                self.session.run(script, clearMemory=True)
+            except Exception as e:
+                self.logger.error(f'KaqQuantDdbStreamWriteRepository.save_rows is occured error: tableName is {ddb_table_name} - {str(e)} - {str(traceback.format_exc())}')
         
     def save2stream(self, ddb_table_name: str, df : pd.DataFrame):
         '''
@@ -230,7 +248,7 @@ class KaqQuantDdbStreamMTWWriteRepository:
         finally:
             mutex.release()
             
-    def insert(self, df:pd.DataFrame=pd.DataFrame(), cols:list=[]):
+    def save2stream_batch(self, df:pd.DataFrame=pd.DataFrame(), cols:list=[]):
         try:
             if cols is None or len(cols) <= 0:
                 for _, row in df.iterrows():
@@ -243,7 +261,7 @@ class KaqQuantDdbStreamMTWWriteRepository:
         except Exception as e:
             self.logger.error(f'KaqQuantDdbStreamMTWWriteRepository.insert is occured error: {str(e)} - {str(traceback.format_exc())}')
             
-    def save2stream(self, row:list=[]):
+    def save2stream_list(self, row:list=[]):
         try:
             if row is None or len(row) <= 0:
                 return

kaq_quant_common/resources/kaq_mysql_resources.py

@@ -97,18 +97,13 @@ class KaqQuantMysqlRepository:
             session.close()
         return ret
 
-    # 判断表是否存在
+     # 判断表是否存在
     def rename_table(self, table_name):
         # 从连接池获取连接
         session = self.session_maker()
         try:
-            # 1. 计算上周一的日期字符串
-            today = datetime.date.today()
-            # 2. 先计算本周一，再减去 7 天
-            last_monday = today - datetime.timedelta(days=today.weekday(), weeks=1)
-            # 3. 格式化
-            last_monday_str = last_monday.strftime('%Y%m%d')
-            sql = f"RENAME TABLE {table_name} TO {table_name}_{last_monday_str};"
+            time_str = time.strftime("%Y_%m_%d_%H_%M", time.localtime(time.time()))
+            sql = f"RENAME TABLE {table_name} TO {table_name}_{time_str};"
             session.execute(text(sql))
             return True
         except Exception as e:

kaq_quant_common/utils/logger_utils.py

@@ -13,7 +13,7 @@ logger.add(
     # 输出到标准输出（控制台）
     sink=sys.stdout,
     # TODO 配置
-    level="DEBUG",
+    level="INFO",
     #
     # format="{time} {level} [{extra[module]}] {message}",  # 显示绑定的module
     format=(
@@ -36,9 +36,9 @@ def _is_dagster_env():
 
 
 def get_logger(obj: Union[str, object] = None):
-    """
+    '''
     获取logger
-    """
+    '''
 
     is_dagster_env = _is_dagster_env()
 
@@ -46,9 +46,9 @@ def get_logger(obj: Union[str, object] = None):
     if isinstance(obj, str):
         # do nothing
         name = obj
-    elif hasattr(obj, "__class__"):
+    elif hasattr(obj, '__class__'):
         name = obj.__class__.__name__
-    elif hasattr(obj, "__name__"):
+    elif hasattr(obj, '__name__'):
         name = obj.__name__
     else:
         name = ""

kaq_quant_common/utils/signal_utils.py

@@ -1,23 +1,23 @@
-import signal
-import sys
-
-# 1. 为目标信号（例如 SIGINT，即 Ctrl+C 触发）维护一个处理函数列表
-signal_handlers = []
-
-
-# 2. 定义「总处理函数」：触发时依次执行列表中的所有处理逻辑
-def total_handler(signalnum, frame):
-    # 依次调用所有注册的处理函数
-    for handler in signal_handlers:
-        handler(signalnum, frame)
-
-
-# 3. 注册「总处理函数」到目标信号（例如 SIGINT）
-# SIGTERM：Dagster通常发送此信号进行终止
-# SIGINT：对应Ctrl+C，用于本地测试
-signal.signal(signal.SIGTERM, total_handler)
-signal.signal(signal.SIGINT, total_handler)
-
-
-def register_signal_handler(handler):
-    signal_handlers.append(handler)
+import signal
+import sys
+
+# 1. 为目标信号（例如 SIGINT，即 Ctrl+C 触发）维护一个处理函数列表
+signal_handlers = []
+
+
+# 2. 定义「总处理函数」：触发时依次执行列表中的所有处理逻辑
+def total_handler(signalnum, frame):
+    # 依次调用所有注册的处理函数
+    for handler in signal_handlers:
+        handler(signalnum, frame)
+
+
+# 3. 注册「总处理函数」到目标信号（例如 SIGINT）
+# SIGTERM：Dagster通常发送此信号进行终止
+# SIGINT：对应Ctrl+C，用于本地测试
+signal.signal(signal.SIGTERM, total_handler)
+signal.signal(signal.SIGINT, total_handler)
+
+
+def register_signal_handler(handler):
+    signal_handlers.append(handler)

kaq_quant_common/utils/uuid_utils.py

@@ -1,5 +1,5 @@
-import uuid
-
-
-def generate_uuid() -> str:
-    return str(uuid.uuid4())
+import uuid
+
+
+def generate_uuid() -> str:
+    return str(uuid.uuid4())

{kaq_quant_common-0.1.99.dist-info → kaq_quant_common-0.2.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kaq_quant_common
-Version: 0.1.99
+Version: 0.2.2
 Summary: 
 Author: kevinfuture
 Author-email: liuenbofuture@foxmail.com