kaq-quant-common 0.1.92__tar.gz → 0.1.94__tar.gz

{kaq_quant_common-0.1.92 → kaq_quant_common-0.1.94}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kaq_quant_common
-Version: 0.1.92
+Version: 0.1.94
 Summary: 
 Author: kevinfuture
 Author-email: liuenbofuture@foxmail.com

{kaq_quant_common-0.1.92 → kaq_quant_common-0.1.94}/kaq_quant_common/resources/kaq_ddb_stream_write_resources.py

@@ -11,9 +11,10 @@ from typing_extensions import deprecated
 
 mutex = threading.Lock()
 
+# 方式一: 异步调用
 class KaqQuantDdbStreamWriteRepository:
     '''
-    定义 asof_join的级联方式, 合并数据到一起, 然后可以订阅判断
+    异步不等待写入
     '''
     def __init__(self, host, port, user, passwd):
         self.logger = get_logger(self)
@@ -88,7 +89,7 @@ class KaqQuantDdbStreamWriteRepository:
         except Exception as e:
             self.logger.error(f'KaqQuantDdbStreamWriteRepository.save2stream_batch is occured error: tableName is {ddb_table_name} - {str(e)} - {str(traceback.format_exc())}')
   
-            
+# 方式二: 同步调用,但有python端的队列等待
 class DDBAsyncDFWriter:
     def __init__(self, appender, batch_size=1000, flush_interval_ms=80):
         self.logger = get_logger()
@@ -163,7 +164,7 @@ class KaqQuantDdbStreamWriteSyncRepository:
     '''
     使用appender直接写入的方式
     '''
-    def __init__(self, host, port, user, passwd, tableName=None):
+    def __init__(self, host, port, user, passwd, tableName=None, batch_size=1000, flush_interval_ms=80):
         if tableName is None:
             raise ValueError(f'Error tableName, please set. tableName={tableName}')
         self.tableName = tableName
@@ -176,7 +177,7 @@ class KaqQuantDdbStreamWriteSyncRepository:
             self.session = ddb.session()
             self.session.connect(host, port, user, passwd, tryReconnectNums=10, reconnect=True, keepAliveTime=1000, readTimeout=10, writeTimeout=5)
             
-            self.batch_writer = DDBAsyncDFWriter(ddb.TableAppender(table_name=self.tableName, conn=self.session))
+            self.batch_writer = DDBAsyncDFWriter(ddb.TableAppender(table_name=self.tableName, conn=self.session), batch_size=batch_size, flush_interval_ms=flush_interval_ms)
             # 需要注意的是 fetchSize 取值不能小于 8192 （记录条数）
             self.size = 8192
         except Exception as e:
@@ -184,7 +185,7 @@ class KaqQuantDdbStreamWriteSyncRepository:
         finally:
             mutex.release()
        
-    @deprecated("请确保pandas数据与ddb表的数据类型一致.")
+    # @deprecated("请确保pandas数据与ddb表的数据类型一致.")
     def insert(self, df : pd.DataFrame):
         '''
         dataframe中日期等类型与ddb流表中一致,例如：
@@ -196,7 +197,41 @@ class KaqQuantDdbStreamWriteSyncRepository:
         except Exception as e:
             self.logger.error(f'KaqQuantDdbTableStreamWriteRepository.insert is occured error:  {str(e)} - {str(traceback.format_exc())}')
 
-
+# 方式三: 异步调用, 但属于ddb的client内部的c++多线程解析与写入，适合一条条写入
+class KaqQuantDdbStreamMTWWriteRepository: 
+    def __init__(self, host, port, user, passwd, tableName=None, batch_size=1000, throttle=50, partitionCol=''):
+        self.logger = get_logger(self)
+        '''
+        创建ddb连接 && 添加ddb流数据表支持
+        '''
+        try:
+            mutex.acquire()
+            self.session = ddb.session(enableASYNC=True)
+            self.session.connect(host, port, user, passwd, tryReconnectNums=10, reconnect=True, keepAliveTime=1000, readTimeout=10, writeTimeout=5)
+            self.batch_writer = ddb.MultithreadedTableWriter(
+                host, port, user, passwd, tableName=tableName, dbPath='',
+                batchSize=batch_size, throttle=throttle, threadCount=1, partitionCol=partitionCol
+            )
+        except Exception as e:
+            self.logger.error(f'KaqQuantDdbStreamMTWWriteRepository.__init__ is occured error: {str(e)} - {str(traceback.format_exc())}')
+        finally:
+            mutex.release()
+            
+    def insert(self, df:pd.DataFrame=pd.DataFrame()):
+        try:
+            for _, row in df.iterrows():
+                _args = row.tolist()
+                self.batch_writer.insert(*_args)
+        except Exception as e:
+            self.logger.error(f'KaqQuantDdbStreamMTWWriteRepository.insert is occured error: {str(e)} - {str(traceback.format_exc())}')
+            
+    def stop(self):
+        '''
+        结束调用
+        '''
+        self.batch_writer.waitForThreadCompletion()
+    
+    
 if __name__ == '__main__':
     host, port, user, passwd = yml_utils.get_ddb_info(os.getcwd())
     kaq = KaqQuantDdbStreamWriteRepository(host, port, user, passwd)

{kaq_quant_common-0.1.92 → kaq_quant_common-0.1.94}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "kaq_quant_common"
-version = "0.1.92"
+version = "0.1.94"
 description = ""
 authors = [
     {name = "kevinfuture",email = "liuenbofuture@foxmail.com"}