re-common 10.0.30__py3-none-any.whl → 10.0.32__py3-none-any.whl

re_common/v2/baselibrary/tools/hdfs_base_processor.py

@@ -1,5 +1,6 @@
 import abc
 import asyncio
+import sys
 from concurrent.futures import ProcessPoolExecutor
 import gzip
 
@@ -17,6 +18,7 @@ from re_common.v2.baselibrary.tools.resume_tracker import ResumeTracker
 
 
 _pool = None
+_loop = None
 
 
 class HDFSBaseProcessor(abc.ABC):
@@ -169,8 +171,15 @@ class HDFSBaseProcessor(abc.ABC):
     def _process_file_wrapper(self, args):
         """为多进程执行准备的同步包装函数"""
         hdfs_file_path, process_func = args
-        loop = asyncio.get_event_loop()
-        return loop.run_until_complete(self._retry_process_file(hdfs_file_path, process_func))
+        if sys.platform == "win32":
+            loop = asyncio.get_event_loop()
+            return loop.run_until_complete(self._retry_process_file(hdfs_file_path, process_func))
+        else:
+            global _loop
+            if _loop is None:
+                _loop = asyncio.new_event_loop()
+                asyncio.set_event_loop(_loop)
+            return _loop.run_until_complete(self._retry_process_file(hdfs_file_path, process_func))
 
     async def _run_multi_process(self, gz_files, process_func):
         """多进程并发运行文件处理任务"""

re_common/v2/baselibrary/tools/resume_tracker.py

@@ -11,7 +11,7 @@ class ResumeTracker:
         self,
         db_path: Union[str, Path] = "processed.db",
         timeout: float = 10.0,
-        isolation_level: Literal["DEFERRED", "EXCLUSIVE", "IMMEDIATE"] | None = "DEFERRED",
+        isolation_level: Union[Literal["DEFERRED", "EXCLUSIVE", "IMMEDIATE"], None] = "DEFERRED",
     ):
         self.db_path = Path(db_path)
         self.timeout = timeout

{re_common-10.0.30.dist-info → re_common-10.0.32.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: re_common
-Version: 10.0.30
+Version: 10.0.32
 Summary: a library about all python projects
 Home-page: https://gitee.com/xujiangios/re-common
 Author: vic

{re_common-10.0.30.dist-info → re_common-10.0.32.dist-info}/RECORD

@@ -179,12 +179,12 @@ re_common/v2/baselibrary/tools/ac_ahocorasick.py,sha256=c63y5RtKVLD37nyPCnBqfNyg
 re_common/v2/baselibrary/tools/concurrency.py,sha256=ctKBoeEbq1SGmhPp7oVR_QSXTKVAGLMByAHQKI6dAqU,981
 re_common/v2/baselibrary/tools/dict_tools.py,sha256=eSMwPTLp3oSjuviC_wlXg0I-dnkkmZfUfCRLX5djWV8,1365
 re_common/v2/baselibrary/tools/dolphinscheduler.py,sha256=1m7UGYDiuvJUCI6ik6CGM2fO8U5XteJzn55VRbwB9ts,7978
-re_common/v2/baselibrary/tools/hdfs_base_processor.py,sha256=WVYBwQ3gUm5jrDV_Ar9Rv0VwJCMODkGByY19HP6fZCA,8405
+re_common/v2/baselibrary/tools/hdfs_base_processor.py,sha256=PFTyhth9RBoMTxsOzUxpyEfQegRi_Sd-C0lvIEOCxCg,8746
 re_common/v2/baselibrary/tools/hdfs_bulk_processor.py,sha256=8FjuZbcBXC_27zEQonLKvb0fMGwcIH9MPRTOmdMAuLU,2396
 re_common/v2/baselibrary/tools/hdfs_data_processer.py,sha256=g0DaNjXM1hIUblFQ6YBwnwEBKIXn48X8Y9Eiok4dVlQ,14824
 re_common/v2/baselibrary/tools/hdfs_line_processor.py,sha256=h1J_mOPoNvsjw7zYMsD7rr0Q6bXvVzo9tRJZVAbei1s,2732
 re_common/v2/baselibrary/tools/list_tools.py,sha256=1NxGVM4EytSXh4IGAEfZQnvq0Ev-UOF-PGZBg2EQbOg,2132
-re_common/v2/baselibrary/tools/resume_tracker.py,sha256=h3WyeIX0L2Q-O9AkCvKWJg48HP8Z8qILTowXzbRrB7c,3246
+re_common/v2/baselibrary/tools/resume_tracker.py,sha256=rVvkd3-eF5lX0VesIHc3YjVRE6bsgAd1VT0G8Uq5DVM,3252
 re_common/v2/baselibrary/tools/search_hash_tools.py,sha256=2ENLtZE8opRsfkwRtTNMzITmpTsjO7wZ1ZkfkqpOH9U,1937
 re_common/v2/baselibrary/tools/text_matcher.py,sha256=cPMoFxaA0-ce3tLRxVSs8_3pTYS1oVIHDnNy_AlPU-4,10756
 re_common/v2/baselibrary/tools/unionfind_tools.py,sha256=VYHZZPXwBYljsm7TjV1B6iCgDn3O3btzNf9hMvQySVU,2965
@@ -236,8 +236,8 @@ re_common/vip/title/transform/TransformRegulationTitleToZt.py,sha256=LKRdIsWKues
 re_common/vip/title/transform/TransformStandardTitleToZt.py,sha256=-fCKAbSBzXVyQDCE61CalvR9E_QzQMA08QOO_NePFNI,5563
 re_common/vip/title/transform/TransformThesisTitleToZt.py,sha256=QS-uV0cQrpUFAcKucuJQ9Ue2VRQH-inmfn_X3IplfRo,5488
 re_common/vip/title/transform/__init__.py,sha256=m83-CWyRq_VHPYHaALEQlmXrkTdrZ3e4B_kCfBYE-uc,239
-re_common-10.0.30.dist-info/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
-re_common-10.0.30.dist-info/METADATA,sha256=DiZvXFGwxsVgDiegqZsCWgUpx1r-KZg51ajXGUso-E4,764
-re_common-10.0.30.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-re_common-10.0.30.dist-info/top_level.txt,sha256=_H9H23zoLIalm1AIY_KYTVh_H0ZnmjxQIxsvXtLv45o,10
-re_common-10.0.30.dist-info/RECORD,,
+re_common-10.0.32.dist-info/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
+re_common-10.0.32.dist-info/METADATA,sha256=dhCvbocMDtJ_7KmIPeQFMypp6LOSS-HQ6YovmjRwvI8,764
+re_common-10.0.32.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+re_common-10.0.32.dist-info/top_level.txt,sha256=_H9H23zoLIalm1AIY_KYTVh_H0ZnmjxQIxsvXtLv45o,10
+re_common-10.0.32.dist-info/RECORD,,