re-common 10.0.40__tar.gz → 10.0.41__tar.gz

{re_common-10.0.40/re_common.egg-info → re_common-10.0.41}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.1
 Name: re_common
-Version: 10.0.40
+Version: 10.0.41
 Summary: a library about all python projects
 Home-page: https://gitee.com/xujiangios/re-common
 Author: vic
@@ -11,14 +11,6 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.6
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Dynamic: author
-Dynamic: author-email
-Dynamic: classifier
-Dynamic: description
-Dynamic: description-content-type
-Dynamic: home-page
-Dynamic: requires-python
-Dynamic: summary
 
 
     这是一个基础类，依赖很多的第三方包，是一个用得到的第三方库的封装，可以在此基础上迅速构建项目

{re_common-10.0.40 → re_common-10.0.41}/re_common/v2/baselibrary/business_utils/BusinessStringUtil.py

@@ -1,6 +1,9 @@
 # 某些业务中的字符串处理 算是特定场景的工具 不算通用工具
+import itertools
 import re
 
+from rapidfuzz.fuzz import partial_token_set_ratio
+
 from re_common.v2.baselibrary.utils.author_smi import AuthorRatio
 from re_common.v2.baselibrary.utils.string_bool import is_all_symbols
 from re_common.v2.baselibrary.utils.string_clear import rel_clear
@@ -199,7 +202,7 @@ def deal_num(num_str):
     return num_str.lower().strip()
 
 
-def clear_author_1st(author_str:str):
+def clear_author_1st(author_str: str):
     # 清理括号 防止前面流程没有清理干净
     author_str = re.sub("\\[.*?]", "", author_str)
     author_str = re.sub("\\(.*?\\)", "", author_str)
@@ -209,6 +212,7 @@ def clear_author_1st(author_str:str):
 
     return author_str
 
+
 def is_same_author(a1, a2):
     if get_alphabetic_ratio(a1.strip()) > 0.7 and get_alphabetic_ratio(a2.strip()) > 0.7:
         author_similar_ = AuthorRatio(a1.strip(), a2.strip())
@@ -217,4 +221,15 @@ def is_same_author(a1, a2):
     else:
         if rel_clear(a1.strip()) == rel_clear(a2.strip()):
             return True
-    return False
+    return False
+
+
+def abs_smi(abs_l1, abs_l2):
+    abs_l1 = [cleared for cleared in map(rel_clear, abs_l1) if cleared]
+    abs_l2 = [cleared for cleared in map(rel_clear, abs_l2) if cleared]
+    lists_max = []
+    for abs1, abs2 in list(itertools.product(abs_l1, abs_l2)):
+        max_smi = partial_token_set_ratio(abs1, abs2, processor=rel_clear)
+        lists_max.append(max_smi)
+
+    return max(lists_max)

{re_common-10.0.40 → re_common-10.0.41}/re_common/v2/baselibrary/tools/data_processer/data_processer.py

@@ -43,6 +43,15 @@ class DatabaseHandler:
                 )
                 conn.commit()
 
+    def get_processed_files_count(self):
+        """查看db3存储了多少成功的记录"""
+        with FileLock(self.lock_file):
+            with sqlite3.connect(self.db_file) as conn:
+                cursor = conn.cursor()
+                cursor.execute("SELECT COUNT(*) FROM processed_files")
+                count = cursor.fetchone()[0]
+            return count
+
     def save_processed_files_many(self, file_paths):
         """批量保存处理过的文件路径"""
         if not file_paths:
@@ -110,6 +119,21 @@ class DatabaseHandler:
             print(f"伪造处理记录时出错: {str(e)}")
 
 
+def on_retry(retry_state):
+    # 每次抛错进入该函数打印消息
+    exc = retry_state.outcome.exception()
+    tb = ''.join(traceback.format_exception(type(exc), exc, exc.__traceback__))
+    print(tb)
+    print(
+        f"处理文件 {retry_state.args[0]} 时发生错误: {exc}，正在重试 {retry_state.attempt_number}")
+
+
+def on_retry_error(retry_state):
+    # 最后抛错后调用
+    print(f"处理文件 {retry_state.args[0]} 失败，达到重试上限")
+    return False
+
+
 class DataProcessor:
     def __init__(
             self,
@@ -249,38 +273,6 @@ class DataProcessor:
         for file_path in all_files:
             yield file_path
 
-    @retry(stop=stop_after_attempt(3),
-           wait=wait_random(min=10, max=30),
-           retry=retry_if_result(lambda result: not result),  # 如果返回值是 False（失败），则重试 最后会抛出一个默认错误tenacity.RetryError:
-           reraise=True)
-    async def _batch_process_file(self, hdfs_file_path: str, process_func: Callable[[str], Any],
-                                  write_dir: str = None):
-        """批量更新所有 gz 文件"""
-        # all_succeed = True
-        # for hdfs_file_path in self.get_file_list(hdfs_dir):
-        #     if self.db_handler.is_file_processed(hdfs_file_path):
-        #         print(f"跳过已处理文件: {hdfs_file_path}")
-        #         continue  # 如果文件已处理，跳过
-        #     succeed = await self.retry_process_file(hdfs_file_path, process_func, write_dir)  # 处理文件
-        #     if succeed is False:
-        #         all_succeed = False
-        #
-        # if all_succeed:
-        #     # 处理完成后删除数据库文件
-        #     try:
-        #         if os.path.exists(self.db_file):
-        #             os.remove(self.db_file)
-        #             print(f"已删除断点重试文件: {self.db_file}")
-        #             return True
-        #     except Exception as e:
-        #         print(f"删除断点重试文件失败: {e}")
-        # return False
-        if self.db_handler.is_file_processed(hdfs_file_path):
-            print(f"跳过已处理文件: {hdfs_file_path}")
-            return True  # 如果文件已处理，跳过
-        succeed = await self.retry_process_file(hdfs_file_path, process_func, write_dir)  # 处理文件
-        return succeed
-
     async def process_file_bulk(self, hdfs_file_path, process_func, write_dir):
         """按批次处理单个文件，批量数据传递给处理函数"""
         # 获取文件的数据总量
@@ -391,36 +383,6 @@ class DataProcessor:
                 succeed = await self._batch_process_file(hdfs_file_path, process_func, write_dir)
             if succeed is False:
                 all_succeed = False
-        if all_succeed:
-            # 处理完成后删除数据库文件
-            try:
-                if os.path.exists(self.db_file):
-                    os.remove(self.db_file)
-                    print(f"已删除断点重试文件: {self.db_file}")
-                    return True
-            except Exception as e:
-                print(f"删除断点重试文件失败: {e}")
-        return False
-
-    @retry(stop=stop_after_attempt(3),
-           wait=wait_random(min=10, max=30),
-           retry=retry_if_result(lambda result: not result),  # 如果返回值是 False（失败），则重试 最后会抛出一个默认错误tenacity.RetryError:
-           reraise=True)
-    async def _batch_process_file_bulk(self, hdfs_file_path: str, process_func: Callable[[List[str]], Any],
-                                       write_dir: str = None):
-        """批量处理 gz 文件中的数据"""
-        # 获取所有文件
-        # all_succeed = True
-        # for hdfs_file_path in self.get_file_list(hdfs_dir):
-        #     # 查看是否跳过文件
-        #     if self.db_handler.is_file_processed(hdfs_file_path):
-        #         print(f"跳过已处理文件: {hdfs_file_path}")
-        #         continue  # 跳过已处理文件
-        #     # 开始批量处理文件
-        #     succeed = await self.retry_process_file_bulk(hdfs_file_path, process_func, write_dir)
-        #     if succeed is False:
-        #         all_succeed = False
-        #
         # if all_succeed:
         #     # 处理完成后删除数据库文件
         #     try:
@@ -430,7 +392,17 @@ class DataProcessor:
         #             return True
         #     except Exception as e:
         #         print(f"删除断点重试文件失败: {e}")
-        # return False
+        return all_succeed
+
+    @retry(stop=stop_after_attempt(3),
+           wait=wait_random(min=10, max=30),
+           # retry=retry_if_result(lambda result: not result),  # 如果返回值是 False（失败），则重试 最后会抛出一个默认错误tenacity.RetryError:
+           before_sleep=on_retry,  # 每次抛错后使用
+           retry_error_callback=on_retry_error,  # 如果最后没成功 返回 False
+           reraise=True)  # 如果函数一直失败，重试结束时会 重新抛出最后一次调用时的原始异常。
+    async def _batch_process_file_bulk(self, hdfs_file_path: str, process_func: Callable[[List[str]], Any],
+                                       write_dir: str = None):
+        """批量处理 gz 文件中的数据"""
         # 查看是否跳过文件
         if self.db_handler.is_file_processed(hdfs_file_path):
             print(f"跳过已处理文件: {hdfs_file_path}")
@@ -439,6 +411,21 @@ class DataProcessor:
         succeed = await self.retry_process_file_bulk(hdfs_file_path, process_func, write_dir)
         return succeed
 
+    @retry(stop=stop_after_attempt(3),
+           wait=wait_random(min=10, max=30),
+           # retry=retry_if_result(lambda result: not result),  # 如果返回值是 False（失败），则重试 最后会抛出一个默认错误tenacity.RetryError:
+           before_sleep=on_retry,  # 每次抛错后使用
+           retry_error_callback=on_retry_error,  # 如果最后没成功 返回 False
+           reraise=True)
+    async def _batch_process_file(self, hdfs_file_path: str, process_func: Callable[[str], Any],
+                                  write_dir: str = None):
+        """批量更新所有 gz 文件"""
+        if self.db_handler.is_file_processed(hdfs_file_path):
+            print(f"跳过已处理文件: {hdfs_file_path}")
+            return True  # 如果文件已处理，跳过
+        succeed = await self.retry_process_file(hdfs_file_path, process_func, write_dir)  # 处理文件
+        return succeed
+
 
 # 全局变量，每个进程独立持有
 _processor: DataProcessor | None = None
@@ -506,3 +493,5 @@ def run_worker_many(hdfs_dir: str, process_func: Callable[[List[str]], Any] | Ca
     for result in results:
         if result:
             print(result)
+    db3_count = processor.db_handler.get_processed_files_count()
+    print(f"db3文件数据量{db3_count}，文件实际数据量{len(all_file)}，是否完成全部转移: {db3_count == len(all_file)}")

re_common-10.0.41/re_common/v2/baselibrary/tools/tree_processor/builder.py

@@ -0,0 +1,25 @@
+from re_common.v2.baselibrary.tools.tree_processor.node import TreeNode
+
+
+def build_forest(node_list):
+    nodes = {}  # cid -> TreeNode
+    has_parent = set()
+
+    # 第一步：创建所有节点
+    for cid, pid, count in node_list:
+        node = TreeNode(cid, count)
+        nodes[cid] = node
+        if pid is not None:
+            has_parent.add(cid)
+
+    # 第二步：连接 parent-child
+    for cid, pid, _ in node_list:
+        if pid is not None and pid in nodes:
+            parent = nodes[pid]
+            child = nodes[cid]
+            parent.children.append(child)
+            child.parent = parent
+
+    # 第三步：找所有根节点（即没有 parent 的）
+    roots = [node for cid, node in nodes.items() if node.parent is None]
+    return roots  # 返回多棵树的根节点列表

re_common-10.0.41/re_common/v2/baselibrary/tools/tree_processor/node.py

@@ -0,0 +1,13 @@
+class TreeNode:
+    def __init__(self, cid, count):
+        self.id = cid
+        self.count = count
+        self.children = []
+        self.parent = None
+
+    def add_child(self, child):
+        self.children.append(child)
+        child.parent = self
+
+    def is_leaf(self):
+        return len(self.children) == 0

{re_common-10.0.40 → re_common-10.0.41}/re_common/v2/baselibrary/utils/basetime.py

@@ -75,3 +75,20 @@ class BaseTime(object):
         current_time = BaseTime.get_current_beijing_time()
         last_time = BaseTime.parse_beijing_time(last_time_str)
         return current_time.hour != last_time.hour
+
+    @staticmethod
+    def is_weekday(num_weekday: int) -> bool:
+        """
+        判断当前日期是否为指定星期。
+
+        参数:
+            num_weekday (int): 表示星期的数字（1=星期一, 2=星期二, ..., 7=星期日）。
+
+        返回:
+            bool: 如果当前日期不是指定的星期，则返回 True；否则返回 False。
+
+        示例:
+            如果 num_weekday=6（星期六），而今天是星期五（weekday()=4），则返回 True。
+        """
+        current_weekday = datetime.now().weekday()  # 获取当前星期（0=星期一, 1=星期二, ..., 6=星期日）
+        return current_weekday != num_weekday - 1

{re_common-10.0.40 → re_common-10.0.41}/re_common/v2/baselibrary/utils/db.py

@@ -94,7 +94,6 @@ aiomysql_pool = None
 pool_lock = asyncio.Lock()  # 全局异步锁
 
 
-
 async def init_aiomysql_pool_async():
     global aiomysql_pool
     if aiomysql_pool is None:
@@ -110,6 +109,7 @@ client = None
 motor_fs_lock = asyncio.Lock()  # 全局异步锁
 _loop_id_mongo = None
 
+
 async def check_connection(client):
     try:
         print("check mongodb client ping")
@@ -142,6 +142,7 @@ async def init_motor_async(uri, db_name, bucket_name, is_reload=False):
                 _loop_id_mongo = id(asyncio.get_running_loop())
     return motor_fs, client
 
+
 # async def run_main():
 #     while True:
 #         uri = "mongodb://192.168.98.80:27001/wpdc"
@@ -154,3 +155,20 @@ async def init_motor_async(uri, db_name, bucket_name, is_reload=False):
 #
 # if __name__ == "__main__":
 #     asyncio.run(run_main())
+
+
+def get_connection(autocommit: bool = True) -> Connection:
+    from pymysql import Connection
+    from pymysql.cursors import DictCursor
+    import pymysql
+    db_conf = {
+        "host": "192.168.98.55",
+        "port": 4000,
+        "user": "dataware_house_baseUser",
+        "password": "FF19AF831AEBD580B450B16BF9264200",
+        "database": "dataware_house_base",
+        "autocommit": autocommit,
+        "cursorclass": DictCursor,
+    }
+    conn: Connection = pymysql.connect(**db_conf)
+    return conn

{re_common-10.0.40 → re_common-10.0.41}/re_common/v2/baselibrary/utils/string_bool.py

@@ -1,5 +1,6 @@
 import re
 
+import regex
 import unicodedata
 
 
@@ -14,7 +15,7 @@ def is_all_english_chars(s):
 
 
 def contains_chinese_chars(s):
-    return bool(re.search(r'[\u3400-\u9fff]', s))
+    return bool(regex.search(r"[\p{IsHan}]", s))
 
 
 def is_empty(value):

{re_common-10.0.40 → re_common-10.0.41}/re_common/v2/baselibrary/utils/stringutils.py

@@ -128,6 +128,47 @@ def get_alphabetic_ratio(text: str) -> float:
     return len(alphabetic_chars) / len(clean_text)
 
 
+def get_chinese_ratio(text: str, mode: str = "letters_only") -> float:
+    """
+    计算中文字符在文本中的比例。
+
+    参数:
+    - text: 原始文本
+    - mode:
+        - "letters_only": 只保留所有语言的字母（默认）
+        - "letters_numbers": 保留字母 + 所有 Unicode 数字（包括全角数字、罗马数字等）
+        - "letters_arabic_numbers": 保留字母 + 阿拉伯数字（0-9）
+        - "no_numbers": 只保留字母，排除所有数字
+
+        区别
+        letters_only: 删除非字母（数字自动被删除，因为不是字母）
+        no_numbers: 删除非字母 + 显式再删除所有数字（即使你前面想保留） 这里会一些删除而额外的数字表达 比如罗马数字
+
+
+    返回:
+    - 中文字符占清洗后文本的比例（float）
+    """
+    if not text:
+        return 0.0
+
+    if mode == "letters_only":
+        clean_text = regex.sub(r"[^\p{L}]", "", text)
+    elif mode == "letters_numbers":
+        clean_text = regex.sub(r"[^\p{L}\p{N}]", "", text)
+    elif mode == "letters_arabic_numbers":
+        clean_text = regex.sub(r"[^\p{L}0-9]", "", text)
+    elif mode == "no_numbers":
+        clean_text = regex.sub(r"[^\p{L}]|\p{N}", "", text)  # 去掉数字
+    else:
+        raise ValueError(f"Unsupported mode: {mode}")
+
+    if len(clean_text) == 0:
+        return 0.0
+
+    chinese_chars = regex.findall(r"[\p{IsHan}]", clean_text)
+    return len(chinese_chars) / len(clean_text)
+
+
 class HTMLTextExtractor(HTMLParser):
     _thread_local = threading.local()  # 线程局部存储
 

{re_common-10.0.40 → re_common-10.0.41/re_common.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.1
 Name: re_common
-Version: 10.0.40
+Version: 10.0.41
 Summary: a library about all python projects
 Home-page: https://gitee.com/xujiangios/re-common
 Author: vic
@@ -11,14 +11,6 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.6
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Dynamic: author
-Dynamic: author-email
-Dynamic: classifier
-Dynamic: description
-Dynamic: description-content-type
-Dynamic: home-page
-Dynamic: requires-python
-Dynamic: summary
 
 
     这是一个基础类，依赖很多的第三方包，是一个用得到的第三方库的封装，可以在此基础上迅速构建项目

{re_common-10.0.40 → re_common-10.0.41}/re_common.egg-info/SOURCES.txt

@@ -206,6 +206,9 @@ re_common/v2/baselibrary/tools/data_processer/base.py
 re_common/v2/baselibrary/tools/data_processer/data_processer.py
 re_common/v2/baselibrary/tools/data_processer/data_reader.py
 re_common/v2/baselibrary/tools/data_processer/data_writer.py
+re_common/v2/baselibrary/tools/tree_processor/__init__.py
+re_common/v2/baselibrary/tools/tree_processor/builder.py
+re_common/v2/baselibrary/tools/tree_processor/node.py
 re_common/v2/baselibrary/utils/BusinessStringUtil.py
 re_common/v2/baselibrary/utils/__init__.py
 re_common/v2/baselibrary/utils/api_net_utils.py

{re_common-10.0.40 → re_common-10.0.41}/setup.py

@@ -34,7 +34,7 @@ long_description = """
 """
 setuptools.setup(
     name="re_common",
-    version="10.0.40",
+    version="10.0.41",
     author="vic",
     author_email="xujiang5@163.com",
     description="a library about all python projects",