re-common 10.0.31__tar.gz → 10.0.33__tar.gz

{re_common-10.0.31/re_common.egg-info → re_common-10.0.33}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: re_common
-Version: 10.0.31
+Version: 10.0.33
 Summary: a library about all python projects
 Home-page: https://gitee.com/xujiangios/re-common
 Author: vic

{re_common-10.0.31 → re_common-10.0.33}/re_common/v2/baselibrary/tools/hdfs_base_processor.py

@@ -1,5 +1,6 @@
 import abc
 import asyncio
+import sys
 from concurrent.futures import ProcessPoolExecutor
 import gzip
 
@@ -17,6 +18,7 @@ from re_common.v2.baselibrary.tools.resume_tracker import ResumeTracker
 
 
 _pool = None
+_loop = None
 
 
 class HDFSBaseProcessor(abc.ABC):
@@ -169,8 +171,15 @@ class HDFSBaseProcessor(abc.ABC):
     def _process_file_wrapper(self, args):
         """为多进程执行准备的同步包装函数"""
         hdfs_file_path, process_func = args
-        loop = asyncio.get_event_loop()
-        return loop.run_until_complete(self._retry_process_file(hdfs_file_path, process_func))
+        if sys.platform == "win32":
+            loop = asyncio.get_event_loop()
+            return loop.run_until_complete(self._retry_process_file(hdfs_file_path, process_func))
+        else:
+            global _loop
+            if _loop is None:
+                _loop = asyncio.new_event_loop()
+                asyncio.set_event_loop(_loop)
+            return _loop.run_until_complete(self._retry_process_file(hdfs_file_path, process_func))
 
     async def _run_multi_process(self, gz_files, process_func):
         """多进程并发运行文件处理任务"""

{re_common-10.0.31 → re_common-10.0.33}/re_common/v2/baselibrary/utils/db.py

@@ -1,38 +1,40 @@
-import atexit
 import os
+import aiomysql
+import asyncio
 from contextlib import asynccontextmanager
 from typing import AsyncGenerator, Tuple
 
 from aiomysql import Pool, Connection, Cursor
 
 DB_CONFIG = {
-    'host': '192.168.98.55',
-    'port': 4000,
-    'user': 'dataware_house_baseUser',
-    'password': 'FF19AF831AEBD580B450B16BF9264200',
-    'db': 'dataware_house_base',
-    'charset': 'utf8mb4',
-    'minsize': 16,  # 最小连接数
-    'maxsize': 128,  # 最大连接数
-    'autocommit': False,  # 自动提交事务
-    'pool_recycle': 3600,  # 每个连接的回收时间（秒），超过此时间后连接将被关闭并重新创建，避免失效连接
-    'echo': False,  # 打印SQL语句
+    "host": "192.168.98.64",
+    "port": 4000,
+    "user": "dataware_house_baseUser",
+    "password": "FF19AF831AEBD580B450B16BF9264200",
+    "db": "dataware_house_base",
+    "charset": "utf8mb4",
+    "minsize": 16,  # 最小连接数
+    "maxsize": 128,  # 最大连接数
+    "autocommit": False,  # 自动提交事务
+    "pool_recycle": 3600,  # 每个连接的回收时间（秒），超过此时间后连接将被关闭并重新创建，避免失效连接
+    "echo": False,  # 打印SQL语句
 }
 
 DB_CONFIG1 = {
-    'host': '192.168.98.55',
-    'port': 4000,
-    'user': 'foreign_fulltextUser',
-    'password': 'i4hIeasw1qpmhGN2nwL7',
-    'db': 'foreign_fulltext',
-    'charset': 'utf8mb4',
-    'minsize': 16,  # 最小连接数
-    'maxsize': 128,  # 最大连接数
-    'autocommit': False,  # 自动提交事务
-    'pool_recycle': 3600,  # 每个连接的回收时间（秒），超过此时间后连接将被关闭并重新创建，避免失效连接
-    'echo': False,  # 打印SQL语句
+    "host": "192.168.98.64",
+    "port": 4000,
+    "user": "foreign_fulltextUser",
+    "password": "i4hIeasw1qpmhGN2nwL7",
+    "db": "foreign_fulltext",
+    "charset": "utf8mb4",
+    "minsize": 16,  # 最小连接数
+    "maxsize": 128,  # 最大连接数
+    "autocommit": False,  # 自动提交事务
+    "pool_recycle": 3600,  # 每个连接的回收时间（秒），超过此时间后连接将被关闭并重新创建，避免失效连接
+    "echo": False,  # 打印SQL语句
 }
 
+
 async def get_pool_only(_DB_CONFIG: dict = None):
     global DB_CONFIG
     if _DB_CONFIG is not None:
@@ -40,6 +42,7 @@ async def get_pool_only(_DB_CONFIG: dict = None):
     pool: Pool = await aiomysql.create_pool(**DB_CONFIG)
     return pool
 
+
 @asynccontextmanager
 async def get_db_pool(_DB_CONFIG: dict = None):
     """异步数据库连接池管理工具"""
@@ -63,8 +66,7 @@ async def get_session(pool: Pool) -> AsyncGenerator[Tuple[Connection, Cursor], N
 
 
 # main.py
-import aiomysql
-import asyncio
+
 
 aiomysql_pool = None
 pool_lock = asyncio.Lock()  # 全局异步锁

{re_common-10.0.31 → re_common-10.0.33/re_common.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: re_common
-Version: 10.0.31
+Version: 10.0.33
 Summary: a library about all python projects
 Home-page: https://gitee.com/xujiangios/re-common
 Author: vic

{re_common-10.0.31 → re_common-10.0.33}/setup.py

@@ -34,7 +34,7 @@ long_description = """
 """
 setuptools.setup(
     name="re_common",
-    version="10.0.31",
+    version="10.0.33",
     author="vic",
     author_email="xujiang5@163.com",
     description="a library about all python projects",