kaq-quant-common 0.1.91__tar.gz → 0.1.92__tar.gz

{kaq_quant_common-0.1.91 → kaq_quant_common-0.1.92}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kaq_quant_common
-Version: 0.1.91
+Version: 0.1.92
 Summary: 
 Author: kevinfuture
 Author-email: liuenbofuture@foxmail.com

{kaq_quant_common-0.1.91 → kaq_quant_common-0.1.92}/kaq_quant_common/resources/kaq_ddb_stream_init_resources.py

@@ -27,7 +27,7 @@ class KaqQuantDdbStreamInitRepository:
         '''
         self.session = ddb.session(enableASYNC=True)
         self.session.connect(host, port, user, passwd)
-        self.session.enableStreaming()
+        # self.session.enableStreaming()
 
         '''
         创建流数据表 && 创建引擎

{kaq_quant_common-0.1.91 → kaq_quant_common-0.1.92}/kaq_quant_common/resources/kaq_ddb_stream_read_resources.py

@@ -23,7 +23,7 @@ class KaqQuantDdbStreamReadRepository:
             self.session = ddb.session(enableASYNC=False)
             self.session.setTimeout(3600)
             self.session.connect(host, port, user, passwd, keepAliveTime=240, reconnect=True, tryReconnectNums=10)
-            self.session.enableStreaming()
+            # self.session.enableStreaming()
 
             # 需要注意的是 fetchSize 取值不能小于 8192 （记录条数）
             self.size = 8192

kaq_quant_common-0.1.92/kaq_quant_common/resources/kaq_ddb_stream_write_resources.py

@@ -0,0 +1,202 @@
+import os
+import queue
+import time
+import dolphindb as ddb
+from kaq_quant_common.utils import yml_utils
+import pandas as pd
+import threading
+from kaq_quant_common.utils.logger_utils import get_logger
+import traceback
+from typing_extensions import deprecated
+
+mutex = threading.Lock()
+
+class KaqQuantDdbStreamWriteRepository:
+    '''
+    定义 asof_join的级联方式, 合并数据到一起, 然后可以订阅判断
+    '''
+    def __init__(self, host, port, user, passwd):
+        self.logger = get_logger(self)
+        '''
+        创建ddb连接 && 添加ddb流数据表支持
+        '''
+        try:
+            mutex.acquire()
+            self.session = ddb.session(enableASYNC=True)
+            self.session.connect(host, port, user, passwd, tryReconnectNums=10, reconnect=True, keepAliveTime=1000, readTimeout=10, writeTimeout=5)
+            # 流表订阅用的
+            # self.session.enableStreaming(threadCount=5)
+            # self.pool = ddb.DBConnectionPool(host, port, userid=user, password=passwd, loadBalance=True, reConnect=True, tryReconnectNums=5, sqlStd=SqlStd.MySQL)
+
+            # 需要注意的是 fetchSize 取值不能小于 8192 （记录条数）
+            self.size = 8192
+        except Exception as e:
+            self.logger.error(f'KaqQuantDdbStreamWriteRepository.__init__ is occured error: {str(e)} - {str(traceback.format_exc())}')
+        finally:
+            mutex.release()
+        
+    def save2stream(self, ddb_table_name: str, df : pd.DataFrame):
+        '''
+        调用此方法之前, 需要将dataframe中的字符串类型的值 ，添加引号
+        '''
+        # 遍历每列的数据类型
+        for column, dtype in df.dtypes.items():
+            if dtype == 'object' or dtype == 'str':
+                df[column] = '\'' + df[column] + '\''
+        for index, row in df.iterrows():
+            script = f"insert into {ddb_table_name} values({', '.join(str(x) for x in row.values)})"
+            try:
+                self.session.run(script, clearMemory=True)
+            except Exception as e:
+                self.logger.error(f'KaqQuantDdbStreamWriteRepository.save2stream is occured error: tableName is {ddb_table_name} - {str(e)} - {str(traceback.format_exc())}')
+                
+    def build_insert_values_fast(self, df):
+        if df.empty:
+            return []
+        dtypes = df.dtypes.tolist()
+        # 提前确定哪些列需要加引号
+        str_idx = {i for i, dt in enumerate(dtypes) if dt == object or dt == 'object' or dt == 'str'}
+        # 转成 ndarray，减少 pandas 参与
+        arr = df.to_numpy()
+        rows = []
+        for row in arr:
+            parts = []
+            for i, v in enumerate(row):
+                if i in str_idx:
+                    parts.append(f"'{v}'")   # 直接拼接最快
+                else:
+                    parts.append(str(v))
+            rows.append("(" + ", ".join(parts) + ")")
+        return rows            
+    
+    def save2stream_batch(self, ddb_table_name: str, df : pd.DataFrame):
+        '''
+        调用此方法之前, 需要将dataframe中的字符串类型的值 ，添加引号
+        '''
+        try:
+            start1 = time.monotonic_ns()
+            row = self.build_insert_values_fast(df)
+            values = ', '.join(row)
+            script = f"insert into {ddb_table_name} values {values}"
+            start2 = time.monotonic_ns()
+            self.session.run(script, clearMemory=True)
+            end = time.monotonic_ns()
+            if "KAQ_QUANT_LOG" in os.environ:
+                diff = end - start2
+                if diff > 1_000_000_0:  # 超过1毫秒
+                    self.logger.warning(f'KaqQuantDdbStreamWriteRepository.save2stream cost time is only write : {end - start2} ns, save2stream_batch :{end - start1} ns, batch size is {len(df)}, tableName is {ddb_table_name}')
+        except Exception as e:
+            self.logger.error(f'KaqQuantDdbStreamWriteRepository.save2stream_batch is occured error: tableName is {ddb_table_name} - {str(e)} - {str(traceback.format_exc())}')
+  
+            
+class DDBAsyncDFWriter:
+    def __init__(self, appender, batch_size=1000, flush_interval_ms=80):
+        self.logger = get_logger()
+        self.appender = appender
+        self.batch_size = batch_size
+        self.flush_interval = flush_interval_ms / 1000.0
+        
+        self.queue = queue.Queue(maxsize=10000)
+        self.running = True
+        
+        self.thread = threading.Thread(target=self._worker, daemon=True)
+        self.thread.start()
+
+    def add_df(self, df):
+        """直接传入一个 DataFrame"""
+        if not self.running:
+            return
+        if df is None or df.empty:
+            return
+        try:
+            self.queue.put(df, block=False)
+        except queue.Full:
+            self.logger.error("Warning: DDBAsyncDFWriter queue is full!")
+
+    def _worker(self):
+        buffer = []
+        current_rows = 0
+        last_flush_time = time.time()
+
+        while self.running or not self.queue.empty():
+            try:
+                # 使用较短的 timeout 以便能快速响应 running=False 状态
+                df = self.queue.get(timeout=0.01)
+                buffer.append(df)
+                current_rows += len(df)
+            except queue.Empty:
+                # 即使没有新数据，如果已经进入停止流程且 buffer 还有数，也要处理
+                if not self.running and not buffer:
+                    break
+
+            now = time.time()
+            # 触发条件：行数够了，或时间到了，或者程序正在停止
+            if buffer and (current_rows >= self.batch_size or 
+                           (now - last_flush_time) >= self.flush_interval or
+                           not self.running):
+                self._do_flush(buffer)
+                buffer = []
+                current_rows = 0
+                last_flush_time = now
+
+    def _do_flush(self, buffer):
+        try:
+            if not buffer:
+                return
+            final_df = pd.concat(buffer, ignore_index=True)
+            self.appender.append(final_df)
+        except Exception as e:
+            self.logger.error(f"DolphinDB 写入异常: {e}")
+
+    def stop(self):
+        """
+        优雅停止：
+        1. 设置 running 为 False
+        2. 等待后台线程把队列里剩余的数据全部 flush 完
+        """
+        self.logger.warning("正在停止 DDBAsyncDFWriter 并清空残留数据...")
+        self.running = False
+        self.thread.join() # 等待工作线程处理完最后一批 buffer
+        self.logger.info("DDBAsyncDFWriter 已安全停止。")
+            
+class KaqQuantDdbStreamWriteSyncRepository:
+    '''
+    使用appender直接写入的方式
+    '''
+    def __init__(self, host, port, user, passwd, tableName=None):
+        if tableName is None:
+            raise ValueError(f'Error tableName, please set. tableName={tableName}')
+        self.tableName = tableName
+        self.logger = get_logger(self)
+        '''
+        创建ddb连接 && 添加ddb流数据表支持
+        '''
+        try:
+            mutex.acquire()
+            self.session = ddb.session()
+            self.session.connect(host, port, user, passwd, tryReconnectNums=10, reconnect=True, keepAliveTime=1000, readTimeout=10, writeTimeout=5)
+            
+            self.batch_writer = DDBAsyncDFWriter(ddb.TableAppender(table_name=self.tableName, conn=self.session))
+            # 需要注意的是 fetchSize 取值不能小于 8192 （记录条数）
+            self.size = 8192
+        except Exception as e:
+            self.logger.error(f'KaqQuantDdbTableStreamWriteRepository.__init__ is occured error: {str(e)} - {str(traceback.format_exc())}')
+        finally:
+            mutex.release()
+       
+    @deprecated("请确保pandas数据与ddb表的数据类型一致.")
+    def insert(self, df : pd.DataFrame):
+        '''
+        dataframe中日期等类型与ddb流表中一致,例如：
+            df['create_time'] = pd.to_datetime(df['create_time'], unit='ms')
+            df['event_time'] = pd.to_datetime(df['event_time'], unit='ms')
+        '''
+        try:
+            self.batch_writer.add_df(df)
+        except Exception as e:
+            self.logger.error(f'KaqQuantDdbTableStreamWriteRepository.insert is occured error:  {str(e)} - {str(traceback.format_exc())}')
+
+
+if __name__ == '__main__':
+    host, port, user, passwd = yml_utils.get_ddb_info(os.getcwd())
+    kaq = KaqQuantDdbStreamWriteRepository(host, port, user, passwd)

{kaq_quant_common-0.1.91 → kaq_quant_common-0.1.92}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "kaq_quant_common"
-version = "0.1.91"
+version = "0.1.92"
 description = ""
 authors = [
     {name = "kevinfuture",email = "liuenbofuture@foxmail.com"}

kaq_quant_common-0.1.91/kaq_quant_common/resources/kaq_ddb_stream_write_resources.py

@@ -1,91 +0,0 @@
-import os
-import time
-import dolphindb as ddb
-from kaq_quant_common.utils import yml_utils
-import pandas as pd
-import threading
-from kaq_quant_common.utils.logger_utils import get_logger
-import traceback
-
-mutex = threading.Lock()
-
-class KaqQuantDdbStreamWriteRepository:
-    '''
-    定义 asof_join的级联方式, 合并数据到一起, 然后可以订阅判断
-    '''
-    def __init__(self, host, port, user, passwd):
-        self.logger = get_logger(self)
-        '''
-        创建ddb连接 && 添加ddb流数据表支持
-        '''
-        try:
-            mutex.acquire()
-            self.session = ddb.session(enableASYNC=True)
-            self.session.connect(host, port, user, passwd, tryReconnectNums=10, reconnect=True, keepAliveTime=1000, readTimeout=10, writeTimeout=5)
-            self.session.enableStreaming(threadCount=5)
-            # self.pool = ddb.DBConnectionPool(host, port, userid=user, password=passwd, loadBalance=True, reConnect=True, tryReconnectNums=5, sqlStd=SqlStd.MySQL)
-
-            # 需要注意的是 fetchSize 取值不能小于 8192 （记录条数）
-            self.size = 8192
-        except Exception as e:
-            self.logger.error(f'KaqQuantDdbStreamWriteRepository.__init__ is occured error: {str(e)} - {str(traceback.format_exc())}')
-        finally:
-            mutex.release()
-        
-    def save2stream(self, ddb_table_name: str, df : pd.DataFrame):
-        '''
-        调用此方法之前, 需要将dataframe中的字符串类型的值 ，添加引号
-        '''
-        # 遍历每列的数据类型
-        for column, dtype in df.dtypes.items():
-            if dtype == 'object' or dtype == 'str':
-                df[column] = '\'' + df[column] + '\''
-        for index, row in df.iterrows():
-            script = f"insert into {ddb_table_name} values({', '.join(str(x) for x in row.values)})"
-            try:
-                self.session.run(script, clearMemory=True)
-            except Exception as e:
-                self.logger.error(f'KaqQuantDdbStreamWriteRepository.save2stream is occured error: tableName is {ddb_table_name} - {str(e)} - {str(traceback.format_exc())}')
-                
-    def build_insert_values_fast(self, df):
-        if df.empty:
-            return []
-        dtypes = df.dtypes.tolist()
-        # 提前确定哪些列需要加引号
-        str_idx = {i for i, dt in enumerate(dtypes) if dt == object or dt == 'object' or dt == 'str'}
-        # 转成 ndarray，减少 pandas 参与
-        arr = df.to_numpy()
-        rows = []
-        for row in arr:
-            parts = []
-            for i, v in enumerate(row):
-                if i in str_idx:
-                    parts.append(f"'{v}'")   # 直接拼接最快
-                else:
-                    parts.append(str(v))
-            rows.append("(" + ", ".join(parts) + ")")
-        return rows            
-    
-    def save2stream_batch(self, ddb_table_name: str, df : pd.DataFrame):
-        '''
-        调用此方法之前, 需要将dataframe中的字符串类型的值 ，添加引号
-        '''
-        try:
-            start1 = time.monotonic_ns()
-            row = self.build_insert_values_fast(df)
-            values = ', '.join(row)
-            script = f"insert into {ddb_table_name} values {values}"
-            start2 = time.monotonic_ns()
-            self.session.run(script, clearMemory=True)
-            end = time.monotonic_ns()
-            if "KAQ_QUANT_LOG" in os.environ:
-                diff = end - start2
-                if diff > 1_000_000_0:  # 超过1毫秒
-                    self.logger.warning(f'KaqQuantDdbStreamWriteRepository.save2stream cost time is only write : {end - start2} ns, save2stream_batch :{end - start1} ns, batch size is {len(df)}, tableName is {ddb_table_name}')
-        except Exception as e:
-            self.logger.error(f'KaqQuantDdbStreamWriteRepository.save2stream_batch is occured error: tableName is {ddb_table_name} - {str(e)} - {str(traceback.format_exc())}')
-
-
-if __name__ == '__main__':
-    host, port, user, passwd = yml_utils.get_ddb_info(os.getcwd())
-    kaq = KaqQuantDdbStreamWriteRepository(host, port, user, passwd)