re-common 10.0.31__py3-none-any.whl → 10.0.33__py3-none-any.whl

re_common/v2/baselibrary/tools/hdfs_base_processor.py

@@ -1,5 +1,6 @@
 import abc
 import asyncio
+import sys
 from concurrent.futures import ProcessPoolExecutor
 import gzip
 
@@ -17,6 +18,7 @@ from re_common.v2.baselibrary.tools.resume_tracker import ResumeTracker
 
 
 _pool = None
+_loop = None
 
 
 class HDFSBaseProcessor(abc.ABC):
@@ -169,8 +171,15 @@ class HDFSBaseProcessor(abc.ABC):
     def _process_file_wrapper(self, args):
         """为多进程执行准备的同步包装函数"""
         hdfs_file_path, process_func = args
-        loop = asyncio.get_event_loop()
-        return loop.run_until_complete(self._retry_process_file(hdfs_file_path, process_func))
+        if sys.platform == "win32":
+            loop = asyncio.get_event_loop()
+            return loop.run_until_complete(self._retry_process_file(hdfs_file_path, process_func))
+        else:
+            global _loop
+            if _loop is None:
+                _loop = asyncio.new_event_loop()
+                asyncio.set_event_loop(_loop)
+            return _loop.run_until_complete(self._retry_process_file(hdfs_file_path, process_func))
 
     async def _run_multi_process(self, gz_files, process_func):
         """多进程并发运行文件处理任务"""

re_common/v2/baselibrary/utils/db.py

@@ -1,38 +1,40 @@
-import atexit
 import os
+import aiomysql
+import asyncio
 from contextlib import asynccontextmanager
 from typing import AsyncGenerator, Tuple
 
 from aiomysql import Pool, Connection, Cursor
 
 DB_CONFIG = {
-    'host': '192.168.98.55',
-    'port': 4000,
-    'user': 'dataware_house_baseUser',
-    'password': 'FF19AF831AEBD580B450B16BF9264200',
-    'db': 'dataware_house_base',
-    'charset': 'utf8mb4',
-    'minsize': 16,  # 最小连接数
-    'maxsize': 128,  # 最大连接数
-    'autocommit': False,  # 自动提交事务
-    'pool_recycle': 3600,  # 每个连接的回收时间（秒），超过此时间后连接将被关闭并重新创建，避免失效连接
-    'echo': False,  # 打印SQL语句
+    "host": "192.168.98.64",
+    "port": 4000,
+    "user": "dataware_house_baseUser",
+    "password": "FF19AF831AEBD580B450B16BF9264200",
+    "db": "dataware_house_base",
+    "charset": "utf8mb4",
+    "minsize": 16,  # 最小连接数
+    "maxsize": 128,  # 最大连接数
+    "autocommit": False,  # 自动提交事务
+    "pool_recycle": 3600,  # 每个连接的回收时间（秒），超过此时间后连接将被关闭并重新创建，避免失效连接
+    "echo": False,  # 打印SQL语句
 }
 
 DB_CONFIG1 = {
-    'host': '192.168.98.55',
-    'port': 4000,
-    'user': 'foreign_fulltextUser',
-    'password': 'i4hIeasw1qpmhGN2nwL7',
-    'db': 'foreign_fulltext',
-    'charset': 'utf8mb4',
-    'minsize': 16,  # 最小连接数
-    'maxsize': 128,  # 最大连接数
-    'autocommit': False,  # 自动提交事务
-    'pool_recycle': 3600,  # 每个连接的回收时间（秒），超过此时间后连接将被关闭并重新创建，避免失效连接
-    'echo': False,  # 打印SQL语句
+    "host": "192.168.98.64",
+    "port": 4000,
+    "user": "foreign_fulltextUser",
+    "password": "i4hIeasw1qpmhGN2nwL7",
+    "db": "foreign_fulltext",
+    "charset": "utf8mb4",
+    "minsize": 16,  # 最小连接数
+    "maxsize": 128,  # 最大连接数
+    "autocommit": False,  # 自动提交事务
+    "pool_recycle": 3600,  # 每个连接的回收时间（秒），超过此时间后连接将被关闭并重新创建，避免失效连接
+    "echo": False,  # 打印SQL语句
 }
 
+
 async def get_pool_only(_DB_CONFIG: dict = None):
     global DB_CONFIG
     if _DB_CONFIG is not None:
@@ -40,6 +42,7 @@ async def get_pool_only(_DB_CONFIG: dict = None):
     pool: Pool = await aiomysql.create_pool(**DB_CONFIG)
     return pool
 
+
 @asynccontextmanager
 async def get_db_pool(_DB_CONFIG: dict = None):
     """异步数据库连接池管理工具"""
@@ -63,8 +66,7 @@ async def get_session(pool: Pool) -> AsyncGenerator[Tuple[Connection, Cursor], N
 
 
 # main.py
-import aiomysql
-import asyncio
+
 
 aiomysql_pool = None
 pool_lock = asyncio.Lock()  # 全局异步锁

{re_common-10.0.31.dist-info → re_common-10.0.33.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: re_common
-Version: 10.0.31
+Version: 10.0.33
 Summary: a library about all python projects
 Home-page: https://gitee.com/xujiangios/re-common
 Author: vic

{re_common-10.0.31.dist-info → re_common-10.0.33.dist-info}/RECORD

@@ -179,7 +179,7 @@ re_common/v2/baselibrary/tools/ac_ahocorasick.py,sha256=c63y5RtKVLD37nyPCnBqfNyg
 re_common/v2/baselibrary/tools/concurrency.py,sha256=ctKBoeEbq1SGmhPp7oVR_QSXTKVAGLMByAHQKI6dAqU,981
 re_common/v2/baselibrary/tools/dict_tools.py,sha256=eSMwPTLp3oSjuviC_wlXg0I-dnkkmZfUfCRLX5djWV8,1365
 re_common/v2/baselibrary/tools/dolphinscheduler.py,sha256=1m7UGYDiuvJUCI6ik6CGM2fO8U5XteJzn55VRbwB9ts,7978
-re_common/v2/baselibrary/tools/hdfs_base_processor.py,sha256=WVYBwQ3gUm5jrDV_Ar9Rv0VwJCMODkGByY19HP6fZCA,8405
+re_common/v2/baselibrary/tools/hdfs_base_processor.py,sha256=PFTyhth9RBoMTxsOzUxpyEfQegRi_Sd-C0lvIEOCxCg,8746
 re_common/v2/baselibrary/tools/hdfs_bulk_processor.py,sha256=8FjuZbcBXC_27zEQonLKvb0fMGwcIH9MPRTOmdMAuLU,2396
 re_common/v2/baselibrary/tools/hdfs_data_processer.py,sha256=g0DaNjXM1hIUblFQ6YBwnwEBKIXn48X8Y9Eiok4dVlQ,14824
 re_common/v2/baselibrary/tools/hdfs_line_processor.py,sha256=h1J_mOPoNvsjw7zYMsD7rr0Q6bXvVzo9tRJZVAbei1s,2732
@@ -202,7 +202,7 @@ re_common/v2/baselibrary/utils/basedict.py,sha256=sH3_RZ8u4649-jX2V1uKNNkjJVUijZ
 re_common/v2/baselibrary/utils/basehdfs.py,sha256=TPwFct_-UrmO1KCbo4gpV77rsnlCQDumNBbQKL0ZI9o,5953
 re_common/v2/baselibrary/utils/basepika.py,sha256=ifOb3UsGj79k40aD9UK6-5BMPw43ZAo0SO3AYD4q4vw,7332
 re_common/v2/baselibrary/utils/basetime.py,sha256=b7U_ho6nE3fjYBxSkdMHXUOd3ClH6KkW_7p7l2Gs4gA,3038
-re_common/v2/baselibrary/utils/db.py,sha256=ceXTLGI0JfZQm06gl-hnQww6Lw4IuwkgeehYFo7bVlA,2509
+re_common/v2/baselibrary/utils/db.py,sha256=SdrLIhijgXUsXQL2aStQS-NgnAfWMvfn8RHEyqdoezI,2500
 re_common/v2/baselibrary/utils/json_cls.py,sha256=M93piYtmgm_wP8E57culTrd_AhHLoGg6PqeAJYdW2SM,438
 re_common/v2/baselibrary/utils/mq.py,sha256=UHpO8iNIHs91Tgp-BgnSUpZwjWquxrGLdpr3FMMv2zw,2858
 re_common/v2/baselibrary/utils/n_ary_expression_tree.py,sha256=-05kO6G2Rth7CEK-5lfFrthFZ1Q0-0a7cni7mWZ-2gg,9172
@@ -236,8 +236,8 @@ re_common/vip/title/transform/TransformRegulationTitleToZt.py,sha256=LKRdIsWKues
 re_common/vip/title/transform/TransformStandardTitleToZt.py,sha256=-fCKAbSBzXVyQDCE61CalvR9E_QzQMA08QOO_NePFNI,5563
 re_common/vip/title/transform/TransformThesisTitleToZt.py,sha256=QS-uV0cQrpUFAcKucuJQ9Ue2VRQH-inmfn_X3IplfRo,5488
 re_common/vip/title/transform/__init__.py,sha256=m83-CWyRq_VHPYHaALEQlmXrkTdrZ3e4B_kCfBYE-uc,239
-re_common-10.0.31.dist-info/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
-re_common-10.0.31.dist-info/METADATA,sha256=72SoNHoqsmiwIFYenIt366QhT1U3saTuy_V7JTqG4_Y,764
-re_common-10.0.31.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-re_common-10.0.31.dist-info/top_level.txt,sha256=_H9H23zoLIalm1AIY_KYTVh_H0ZnmjxQIxsvXtLv45o,10
-re_common-10.0.31.dist-info/RECORD,,
+re_common-10.0.33.dist-info/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
+re_common-10.0.33.dist-info/METADATA,sha256=bmraxgArGm6vGU0k3cC6EfoOY_T0SYZnuL-XVAIhI90,764
+re_common-10.0.33.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+re_common-10.0.33.dist-info/top_level.txt,sha256=_H9H23zoLIalm1AIY_KYTVh_H0ZnmjxQIxsvXtLv45o,10
+re_common-10.0.33.dist-info/RECORD,,