kaq-quant-common 0.1.93__tar.gz → 0.1.94__tar.gz

{kaq_quant_common-0.1.93 → kaq_quant_common-0.1.94}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kaq_quant_common
-Version: 0.1.93
+Version: 0.1.94
 Summary: 
 Author: kevinfuture
 Author-email: liuenbofuture@foxmail.com

{kaq_quant_common-0.1.93 → kaq_quant_common-0.1.94}/kaq_quant_common/resources/kaq_ddb_stream_write_resources.py

@@ -11,9 +11,10 @@ from typing_extensions import deprecated
 
 mutex = threading.Lock()
 
+# 方式一: 异步调用
 class KaqQuantDdbStreamWriteRepository:
     '''
-    定义 asof_join的级联方式, 合并数据到一起, 然后可以订阅判断
+    异步不等待写入
     '''
     def __init__(self, host, port, user, passwd):
         self.logger = get_logger(self)
@@ -88,7 +89,7 @@ class KaqQuantDdbStreamWriteRepository:
         except Exception as e:
             self.logger.error(f'KaqQuantDdbStreamWriteRepository.save2stream_batch is occured error: tableName is {ddb_table_name} - {str(e)} - {str(traceback.format_exc())}')
   
-            
+# 方式二: 同步调用,但有python端的队列等待
 class DDBAsyncDFWriter:
     def __init__(self, appender, batch_size=1000, flush_interval_ms=80):
         self.logger = get_logger()
@@ -196,7 +197,41 @@ class KaqQuantDdbStreamWriteSyncRepository:
         except Exception as e:
             self.logger.error(f'KaqQuantDdbTableStreamWriteRepository.insert is occured error:  {str(e)} - {str(traceback.format_exc())}')
 
-
+# 方式三: 异步调用, 但属于ddb的client内部的c++多线程解析与写入，适合一条条写入
+class KaqQuantDdbStreamMTWWriteRepository: 
+    def __init__(self, host, port, user, passwd, tableName=None, batch_size=1000, throttle=50, partitionCol=''):
+        self.logger = get_logger(self)
+        '''
+        创建ddb连接 && 添加ddb流数据表支持
+        '''
+        try:
+            mutex.acquire()
+            self.session = ddb.session(enableASYNC=True)
+            self.session.connect(host, port, user, passwd, tryReconnectNums=10, reconnect=True, keepAliveTime=1000, readTimeout=10, writeTimeout=5)
+            self.batch_writer = ddb.MultithreadedTableWriter(
+                host, port, user, passwd, tableName=tableName, dbPath='',
+                batchSize=batch_size, throttle=throttle, threadCount=1, partitionCol=partitionCol
+            )
+        except Exception as e:
+            self.logger.error(f'KaqQuantDdbStreamMTWWriteRepository.__init__ is occured error: {str(e)} - {str(traceback.format_exc())}')
+        finally:
+            mutex.release()
+            
+    def insert(self, df:pd.DataFrame=pd.DataFrame()):
+        try:
+            for _, row in df.iterrows():
+                _args = row.tolist()
+                self.batch_writer.insert(*_args)
+        except Exception as e:
+            self.logger.error(f'KaqQuantDdbStreamMTWWriteRepository.insert is occured error: {str(e)} - {str(traceback.format_exc())}')
+            
+    def stop(self):
+        '''
+        结束调用
+        '''
+        self.batch_writer.waitForThreadCompletion()
+    
+    
 if __name__ == '__main__':
     host, port, user, passwd = yml_utils.get_ddb_info(os.getcwd())
     kaq = KaqQuantDdbStreamWriteRepository(host, port, user, passwd)

{kaq_quant_common-0.1.93 → kaq_quant_common-0.1.94}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "kaq_quant_common"
-version = "0.1.93"
+version = "0.1.94"
 description = ""
 authors = [
     {name = "kevinfuture",email = "liuenbofuture@foxmail.com"}