kaq-quant-common 0.1.91__py3-none-any.whl → 0.1.92__py3-none-any.whl

kaq_quant_common/resources/kaq_ddb_stream_init_resources.py

@@ -27,7 +27,7 @@ class KaqQuantDdbStreamInitRepository:
         '''
         self.session = ddb.session(enableASYNC=True)
         self.session.connect(host, port, user, passwd)
-        self.session.enableStreaming()
+        # self.session.enableStreaming()
 
         '''
         创建流数据表 && 创建引擎

kaq_quant_common/resources/kaq_ddb_stream_read_resources.py

@@ -23,7 +23,7 @@ class KaqQuantDdbStreamReadRepository:
             self.session = ddb.session(enableASYNC=False)
             self.session.setTimeout(3600)
             self.session.connect(host, port, user, passwd, keepAliveTime=240, reconnect=True, tryReconnectNums=10)
-            self.session.enableStreaming()
+            # self.session.enableStreaming()
 
             # 需要注意的是 fetchSize 取值不能小于 8192 （记录条数）
             self.size = 8192

kaq_quant_common/resources/kaq_ddb_stream_write_resources.py

@@ -1,4 +1,5 @@
 import os
+import queue
 import time
 import dolphindb as ddb
 from kaq_quant_common.utils import yml_utils
@@ -6,6 +7,7 @@ import pandas as pd
 import threading
 from kaq_quant_common.utils.logger_utils import get_logger
 import traceback
+from typing_extensions import deprecated
 
 mutex = threading.Lock()
 
@@ -22,7 +24,8 @@ class KaqQuantDdbStreamWriteRepository:
             mutex.acquire()
             self.session = ddb.session(enableASYNC=True)
             self.session.connect(host, port, user, passwd, tryReconnectNums=10, reconnect=True, keepAliveTime=1000, readTimeout=10, writeTimeout=5)
-            self.session.enableStreaming(threadCount=5)
+            # 流表订阅用的
+            # self.session.enableStreaming(threadCount=5)
             # self.pool = ddb.DBConnectionPool(host, port, userid=user, password=passwd, loadBalance=True, reConnect=True, tryReconnectNums=5, sqlStd=SqlStd.MySQL)
 
             # 需要注意的是 fetchSize 取值不能小于 8192 （记录条数）
@@ -84,6 +87,114 @@ class KaqQuantDdbStreamWriteRepository:
                     self.logger.warning(f'KaqQuantDdbStreamWriteRepository.save2stream cost time is only write : {end - start2} ns, save2stream_batch :{end - start1} ns, batch size is {len(df)}, tableName is {ddb_table_name}')
         except Exception as e:
             self.logger.error(f'KaqQuantDdbStreamWriteRepository.save2stream_batch is occured error: tableName is {ddb_table_name} - {str(e)} - {str(traceback.format_exc())}')
+  
+            
+class DDBAsyncDFWriter:
+    def __init__(self, appender, batch_size=1000, flush_interval_ms=80):
+        self.logger = get_logger()
+        self.appender = appender
+        self.batch_size = batch_size
+        self.flush_interval = flush_interval_ms / 1000.0
+        
+        self.queue = queue.Queue(maxsize=10000)
+        self.running = True
+        
+        self.thread = threading.Thread(target=self._worker, daemon=True)
+        self.thread.start()
+
+    def add_df(self, df):
+        """直接传入一个 DataFrame"""
+        if not self.running:
+            return
+        if df is None or df.empty:
+            return
+        try:
+            self.queue.put(df, block=False)
+        except queue.Full:
+            self.logger.error("Warning: DDBAsyncDFWriter queue is full!")
+
+    def _worker(self):
+        buffer = []
+        current_rows = 0
+        last_flush_time = time.time()
+
+        while self.running or not self.queue.empty():
+            try:
+                # 使用较短的 timeout 以便能快速响应 running=False 状态
+                df = self.queue.get(timeout=0.01)
+                buffer.append(df)
+                current_rows += len(df)
+            except queue.Empty:
+                # 即使没有新数据，如果已经进入停止流程且 buffer 还有数，也要处理
+                if not self.running and not buffer:
+                    break
+
+            now = time.time()
+            # 触发条件：行数够了，或时间到了，或者程序正在停止
+            if buffer and (current_rows >= self.batch_size or 
+                           (now - last_flush_time) >= self.flush_interval or
+                           not self.running):
+                self._do_flush(buffer)
+                buffer = []
+                current_rows = 0
+                last_flush_time = now
+
+    def _do_flush(self, buffer):
+        try:
+            if not buffer:
+                return
+            final_df = pd.concat(buffer, ignore_index=True)
+            self.appender.append(final_df)
+        except Exception as e:
+            self.logger.error(f"DolphinDB 写入异常: {e}")
+
+    def stop(self):
+        """
+        优雅停止：
+        1. 设置 running 为 False
+        2. 等待后台线程把队列里剩余的数据全部 flush 完
+        """
+        self.logger.warning("正在停止 DDBAsyncDFWriter 并清空残留数据...")
+        self.running = False
+        self.thread.join() # 等待工作线程处理完最后一批 buffer
+        self.logger.info("DDBAsyncDFWriter 已安全停止。")
+            
+class KaqQuantDdbStreamWriteSyncRepository:
+    '''
+    使用appender直接写入的方式
+    '''
+    def __init__(self, host, port, user, passwd, tableName=None):
+        if tableName is None:
+            raise ValueError(f'Error tableName, please set. tableName={tableName}')
+        self.tableName = tableName
+        self.logger = get_logger(self)
+        '''
+        创建ddb连接 && 添加ddb流数据表支持
+        '''
+        try:
+            mutex.acquire()
+            self.session = ddb.session()
+            self.session.connect(host, port, user, passwd, tryReconnectNums=10, reconnect=True, keepAliveTime=1000, readTimeout=10, writeTimeout=5)
+            
+            self.batch_writer = DDBAsyncDFWriter(ddb.TableAppender(table_name=self.tableName, conn=self.session))
+            # 需要注意的是 fetchSize 取值不能小于 8192 （记录条数）
+            self.size = 8192
+        except Exception as e:
+            self.logger.error(f'KaqQuantDdbTableStreamWriteRepository.__init__ is occured error: {str(e)} - {str(traceback.format_exc())}')
+        finally:
+            mutex.release()
+       
+    @deprecated("请确保pandas数据与ddb表的数据类型一致.")
+    def insert(self, df : pd.DataFrame):
+        '''
+        dataframe中日期等类型与ddb流表中一致,例如：
+            df['create_time'] = pd.to_datetime(df['create_time'], unit='ms')
+            df['event_time'] = pd.to_datetime(df['event_time'], unit='ms')
+        '''
+        try:
+            self.batch_writer.add_df(df)
+        except Exception as e:
+            self.logger.error(f'KaqQuantDdbTableStreamWriteRepository.insert is occured error:  {str(e)} - {str(traceback.format_exc())}')
 
 
 if __name__ == '__main__':

{kaq_quant_common-0.1.91.dist-info → kaq_quant_common-0.1.92.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kaq_quant_common
-Version: 0.1.91
+Version: 0.1.92
 Summary: 
 Author: kevinfuture
 Author-email: liuenbofuture@foxmail.com

{kaq_quant_common-0.1.91.dist-info → kaq_quant_common-0.1.92.dist-info}/RECORD

@@ -39,9 +39,9 @@ kaq_quant_common/common/ws_wrapper.py,sha256=JNJ0CIjDXgCsRjOLSbCi7ysYDHw7tT_aK7V
 kaq_quant_common/config/config.yaml,sha256=ST_QBLo7kwVaoNOvuN3mpeSF7LPNSWdD7EjxrBYZYBs,230
 kaq_quant_common/resources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 kaq_quant_common/resources/kaq_ddb_pool_stream_read_resources.py,sha256=Y-rtGpBF3bNh2mPZ2Lze6D9JZnpkqh24OEh7okikh1w,2536
-kaq_quant_common/resources/kaq_ddb_stream_init_resources.py,sha256=0FY0cmqedrLSYoaHJah2QiO0L7l3lsQJzIfXXpd9WSQ,3347
-kaq_quant_common/resources/kaq_ddb_stream_read_resources.py,sha256=yqeruZELmxAKjVsdnZ7ieAeV3Wt7KpFYJdt8Vn_Y_qA,3163
-kaq_quant_common/resources/kaq_ddb_stream_write_resources.py,sha256=pebBaDypkY6Mn--Od5Wh1_ZfHewbQH3ce_JsfiWsyl4,4139
+kaq_quant_common/resources/kaq_ddb_stream_init_resources.py,sha256=0MxxbajocTFzcRD344rfZQPKOwwuqCbyoT6dQpIox-o,3349
+kaq_quant_common/resources/kaq_ddb_stream_read_resources.py,sha256=WShsXMoL8o-JZvrtAd7H2Cg-vrE47QbsdGgURaQwiZs,3165
+kaq_quant_common/resources/kaq_ddb_stream_write_resources.py,sha256=RUWaonAnTSNQlSXnrdhrD4VHEueT0UN2kgG6dKoec04,8524
 kaq_quant_common/resources/kaq_mysql_init_resources.py,sha256=UcqWey6LgoMqvLq1SxK33nS6-rkViGYhzUPxcrucOks,827
 kaq_quant_common/resources/kaq_mysql_resources.py,sha256=UYBFn-o_fFaNwG6bT2E7CYdsns82EmR4ve2XgC9wFJw,13452
 kaq_quant_common/resources/kaq_postgresql_resources.py,sha256=iG1eYkciI0xUIBdEpGqKGOLBFxvVrfbBoTuaOmhQ0v0,1762
@@ -61,6 +61,6 @@ kaq_quant_common/utils/signal_utils.py,sha256=zBSyEltNTKqkQCsrETd47kEBb3Q_OWUBUn
 kaq_quant_common/utils/sqlite_utils.py,sha256=UDDFKfwL0N-jFifl40HdyOCENh2YQfW5so6hRaSJpv0,5722
 kaq_quant_common/utils/uuid_utils.py,sha256=pm_pnXpd8n9CI66x3A20cOEUiriJyqHaKGCeLrgkBxU,71
 kaq_quant_common/utils/yml_utils.py,sha256=gcKjb_-uuUajBGAl5QBPIZTg2wXm7qeeJvtHflj_zOE,4513
-kaq_quant_common-0.1.91.dist-info/METADATA,sha256=9ojNBXlJU6gHrFNcTNlpsrjjyfwXAaUu4OL1NJ0hlvM,1971
-kaq_quant_common-0.1.91.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
-kaq_quant_common-0.1.91.dist-info/RECORD,,
+kaq_quant_common-0.1.92.dist-info/METADATA,sha256=_LDEPTc7fW8A3pP2WmwrFjD86MF_yZSR1zbrATVLmUo,1971
+kaq_quant_common-0.1.92.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
+kaq_quant_common-0.1.92.dist-info/RECORD,,