re-common 10.0.16__tar.gz → 10.0.17__tar.gz

{re_common-10.0.16 → re_common-10.0.17}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: re_common
-Version: 10.0.16
+Version: 10.0.17
 Summary: a library about all python projects
 Home-page: https://gitee.com/xujiangios/re-common
 Author: vic

{re_common-10.0.16 → re_common-10.0.17}/re_common/v2/baselibrary/tools/hdfs_data_processer.py

@@ -1,6 +1,7 @@
 import asyncio
 import gzip
 import json
+from pathlib import Path
 import sqlite3
 import time
 import os
@@ -12,12 +13,12 @@ from hdfs import InsecureClient
 
 class HDFSDataProcessor:
     def __init__(
-            self,
-            hdfs_url="http://VIP-DC-MASTER-2:9870",
-            hdfs_user="root",
-            db_file="processed_files.db",
-            batch_size=50,
-            retry_limit=3,
+        self,
+        hdfs_url="http://VIP-DC-MASTER-2:9870",
+        hdfs_user="root",
+        db_file="processed_files.db",
+        batch_size=50,
+        retry_limit=3,
     ):
         self.hdfs_url = hdfs_url
         self.hdfs_user = hdfs_user
@@ -44,14 +45,20 @@ class HDFSDataProcessor:
         """保存处理过的文件"""
         with sqlite3.connect(self.db_file) as conn:
             cursor = conn.cursor()
-            cursor.execute("INSERT OR IGNORE INTO processed_files (file_path) VALUES (?)", (file_path,))
+            cursor.execute(
+                "INSERT OR IGNORE INTO processed_files (file_path) VALUES (?)",
+                (file_path,),
+            )
             conn.commit()
 
     def is_file_processed(self, file_path):
         """检查文件是否已处理"""
         with sqlite3.connect(self.db_file) as conn:
             cursor = conn.cursor()
-            cursor.execute("SELECT file_path FROM processed_files WHERE file_path = ?", (file_path,))
+            cursor.execute(
+                "SELECT file_path FROM processed_files WHERE file_path = ?",
+                (file_path,),
+            )
             result = cursor.fetchone()
         return result is not None
 
@@ -81,7 +88,7 @@ class HDFSDataProcessor:
                         break
                     yield lines
 
-    def all_read_gz(self, gz_file_path: str, encoding='utf-8'):
+    def all_read_gz(self, gz_file_path: str, encoding="utf-8"):
         """
         读取 HDFS 上的 .gz 文件内容。
         :param hdfs_path: HDFS 文件路径（必须以 .gz 结尾）
@@ -94,7 +101,7 @@ class HDFSDataProcessor:
                 content = gz_file.read().decode(encoding)  # 解码为字符串
                 print(f"文件读取成功: {gz_file_path}")
                 lines = [i for i in content.split("\n") if i.strip()]
-                result = [lines[i:i + self.batch_size] for i in range(0, len(lines), self.batch_size)]
+                result = [lines[i : i + self.batch_size] for i in range(0, len(lines), self.batch_size)]
                 return result
 
     async def process_data(self, data, process_func):
@@ -107,14 +114,15 @@ class HDFSDataProcessor:
             except Exception as e:
                 retry_count += 1
                 print(f"处理数据时发生错误: {e}, 正在重试 {retry_count}/{self.retry_limit}, data: {data}")
-                await asyncio.sleep(2 ** retry_count)
+                await asyncio.sleep(2**retry_count)
         print(f"处理数据失败, 达到重试上限, data: {data}")
 
-    async def process_file(self, hdfs_file_path, process_func):
+    async def process_file(self, hdfs_file_path, process_func, write_dir: str):
         """处理单个 gz 文件"""
         total_lines = self.count_total_lines(hdfs_file_path)
         processed_lines = 0
         start_time = time.time()
+        results = []
         #   # 这里根据不同的配置选用不同的读取文件的方法
         for lines in self.read_hdfs_fanc[self.read_hdfs_model](hdfs_file_path):
             processing_start_time = time.time()  # 记录本批处理开始时间
@@ -128,7 +136,7 @@ class HDFSDataProcessor:
                     print(f"解析JSON失败: {e}, 行内容: {line.strip()}")
 
             # await AsyncTaskPool(self.batch_size).run(tasks) # AsyncTaskPool 适用于一次提交所有任务, 限制并发数执行
-            await asyncio.gather(*tasks)
+            results.extend(await asyncio.gather(*tasks))
 
             processed_lines += len(lines)
 
@@ -152,6 +160,18 @@ class HDFSDataProcessor:
                 f"预估剩余时间: {remaining_time:.2f}秒 | 平均每条处理时间: {avg_processing_time:.2f}毫秒"
             )
 
+        def generate_write_data(results):
+            for res in results:
+                yield str(res) + "\n"
+
+        if write_dir is not None:
+            self.client.write(
+                write_dir.rstrip("/") + f"/{Path(hdfs_file_path).stem}",
+                data=generate_write_data(results),
+                overwrite=True,
+                encoding="utf-8",
+            )
+
         # 最终进度显示
         final_elapsed_time = time.time() - start_time  # 最终已用时间
         print(
@@ -164,22 +184,22 @@ class HDFSDataProcessor:
 
         self.save_processed_file(hdfs_file_path)  # 保存处理过的文件
 
-    async def retry_process_file(self, hdfs_file_path, process_func):
+    async def retry_process_file(self, hdfs_file_path, process_func, write_dir):
         """带重试机制的文件处理"""
         retry_count = 0
         while retry_count < self.retry_limit:
             try:
-                await self.process_file(hdfs_file_path, process_func)
+                await self.process_file(hdfs_file_path, process_func, write_dir)
                 return True  # 成功处理后退出
             except Exception as e:
                 retry_count += 1
                 print(f"处理文件 {hdfs_file_path} 时发生错误: {e}，正在重试 {retry_count}/{self.retry_limit}")
-                await asyncio.sleep(2 ** retry_count)
+                await asyncio.sleep(2**retry_count)
         print(f"处理文件 {hdfs_file_path} 失败，达到重试上限")
         return False
         # raise
 
-    async def batch_process_file(self, hdfs_dir: str, process_func: Callable[[dict], Any]):
+    async def batch_process_file(self, hdfs_dir: str, process_func: Callable[[dict], Any], write_dir: str = None):
         """批量更新所有 gz 文件"""
         gz_files = self.list_gz_files(hdfs_dir)
         all_succeed = True
@@ -187,7 +207,7 @@ class HDFSDataProcessor:
             if self.is_file_processed(hdfs_file_path):
                 print(f"跳过已处理文件: {hdfs_file_path}")
                 continue  # 如果文件已处理，跳过
-            succeed = await self.retry_process_file(hdfs_file_path, process_func)  # 处理文件
+            succeed = await self.retry_process_file(hdfs_file_path, process_func, write_dir)  # 处理文件
             if succeed is False:
                 all_succeed = False
 
@@ -276,7 +296,7 @@ class HDFSDataProcessor:
             except Exception as e:
                 retry_count += 1
                 print(f"处理文件 {hdfs_file_path} 时发生错误: {e}，正在重试 {retry_count}/{self.retry_limit}")
-                await asyncio.sleep(2 ** retry_count)
+                await asyncio.sleep(2**retry_count)
         print(f"处理文件 {hdfs_file_path} 失败，达到重试上限")
         return False
 
@@ -301,6 +321,7 @@ class HDFSDataProcessor:
             except Exception as e:
                 print(f"删除断点重试文件失败: {e}")
 
+
 # # 使用示例
 # async def update_refer(data: dict):
 #     ref_id = data["ref_id"]

{re_common-10.0.16 → re_common-10.0.17}/re_common/v2/baselibrary/tools/list_tools.py

@@ -50,3 +50,17 @@ def filter_and_sort_by_smi(all_list, top_n=1000):
 
     # 3. 取前 top_n 个
     return unique_list[:top_n]
+
+
+def list_to_dict(list_data,key_name):
+    # 使用 defaultdict 来处理重复 id
+    from collections import defaultdict
+
+    dict_data = defaultdict(list)
+
+    for item in list_data:
+        dict_data[item[key_name]].append(item)
+
+    # 将 defaultdict 转换成普通字典
+    dict_data = dict(dict_data)
+    return dict_data

{re_common-10.0.16 → re_common-10.0.17}/re_common/v2/baselibrary/utils/BusinessStringUtil.py

@@ -177,7 +177,14 @@ def deal_num(num_str):
     if num_str.lower().startswith("n "):
         num_str = num_str.lower().replace("n ", "").strip()
 
+    num_str = num_str.lower().replace("special_issue_", '').replace("_special_issue", '').replace("issue", "")
+    num_str = num_str.replace("spec.", "").replace("iss.", "").replace("spl.", "").replace("special.", "").replace(
+        "specialissue.", "")
+    num_str = num_str.replace("spec", "").replace("iss", "").replace("spl", "").replace("special", "").replace(
+        "specialissue", '')
+
     num_str = num_str.replace("-", "_").replace(".", "_").upper()
+    num_str = num_str.lstrip("_").rstrip("_")
     if num_str.find("_") > -1:
         start, end = num_str.split("_")
         start = deal_num_strs(start)
@@ -186,4 +193,4 @@ def deal_num(num_str):
     else:
         num_str = deal_num_strs(num_str)
 
-    return num_str
+    return num_str.lower().strip()

re_common-10.0.17/re_common/v2/baselibrary/utils/base_string_similarity.py

@@ -0,0 +1,158 @@
+import Levenshtein
+import jellyfish
+from rapidfuzz.distance import DamerauLevenshtein, Hamming, Indel, LCSseq, OSA
+
+
+class BaseStringSimilarity(object):
+
+    @classmethod
+    def levenshtein_similarity(cls, str1, str2) -> float:
+        """
+        返回 两个字字符串之间的编辑距离 分数
+        """
+        # 编辑距离长度
+        distance = Levenshtein.distance(str1, str2)
+        # 以最长字符串为除数算分
+        similarity = 1 - (distance / max(len(str1), len(str2)))
+        return similarity
+
+    @classmethod
+    def damerau_normalized_distance_similarity(cls, str1, str2) -> float:
+        """
+          # 计算 归一化的编辑距离，取值范围 [0, 1]，值越小表示越相似。 一般不以小评估分 所以不用
+        similarity = DamerauLevenshtein.normalized_distance(str1, str2)
+        作用：计算 相似度得分，取值范围 [0, max_len]，值越大表示越相似。
+        print(DamerauLevenshtein.similarity(str1, str2))
+        """
+        # 该算法与 cls.levenshtein_similarity 算法一致 只是 编辑距离的得值不一样
+        similarity = DamerauLevenshtein.normalized_similarity(str1, str2)
+        return similarity
+
+    @classmethod
+    def indel_levenshtein_similarity(cls, str1, str2) -> float:
+        """
+        本质上使用的 是 Indel.normalized_similarity(str1,str2) 方法
+
+        计算 str1 和 str2 之间的 Indel 距离（插入和删除操作的最小次数）
+        Indel.distance(str1, str2)
+        计算 标准化后的 Indel 距离，取值范围在 [0, 1] 之间，其中 0 表示完全相同，1 表示完全不同。 ``distance / (len1 + len2)``.
+        Indel.normalized_distance(str1, str2)
+        计算 [max, 0] 范围内的 Indel 相似度。计算公式为“(len1 + len2) - distance”
+        Indel.similarity(str1, str2)
+        计算 [0, 1] 范围内的归一化插入/缺失相似度。计算公式为“1 - normalized_distance”
+        Indel.normalized_similarity(str1, str2)
+
+        """
+        # 计算相似度（0到1之间的值，1表示完全相同）
+        similarity = Levenshtein.ratio(str1, str2)
+        return similarity
+
+    @classmethod
+    def jaro_similarity(cls, str1, str2) -> float:
+        """
+        Jaro 相似度是一种用于测量两个字符串相似度的算法，主要考虑：
+        匹配的字符
+        字符顺序
+        字符转置（位置交换）
+
+        与 Jaro.normalized_similarity(str1,str2) 一致
+        """
+        return jellyfish.jaro_similarity(str1, str2)
+
+    @classmethod
+    def jaro_winkler_similarity(cls, str1, str2) -> float:
+        """
+        Jaro-Winkler 是 Jaro 的改进版，对前缀匹配给予更多权重
+
+        与 JaroWinkler.normalized_similarity(str1,str2) 结果一致
+
+        print(JaroWinkler.distance(str1, str2))
+        与 print(JaroWinkler.normalized_distance(str1, str2)) 结果一致
+
+        print(JaroWinkler.similarity(str1, str2))
+        与 print(JaroWinkler.normalized_similarity(str1,str2)) 结果一致
+        """
+        return jellyfish.jaro_winkler_similarity(str1, str2)
+
+    @classmethod
+    def osa_similarity(cls, str1, str2) -> float:
+        """
+        计算 [0, 1] 范围内的归一化最佳字符串比对 (OSA) 相似度。
+
+        计算公式为“1 - normalized_distance”
+        """
+        return OSA.normalized_similarity(str1, str2)
+
+    @classmethod
+    def lcs_seq_similarity(cls, str1, str2) -> float:
+        """
+        计算 [0, 1] 范围内的归一化 LCS 相似度。
+        计算公式为“1 - normalized_distance”
+        """
+        return LCSseq.normalized_similarity(str1, str2)
+
+    @classmethod
+    def lcs_seq_distance(cls, str1, str2) -> int:
+        """
+        LCSseq.distance 是 RapidFuzz 库中的一个方法，用于计算两个字符串之间的 最长公共子序列（Longest Common Subsequence, LCS）距离。
+        LCS 是指两个字符串中 按顺序出现但不一定连续 的最长子序列。例如：
+        "abcde" 和 "ace" 的 LCS 是 "ace"（长度 3）。
+        "Druitt, Robert" 和 "Druitt R." 的 LCS 可能是 "Druitt R"（长度 8）。
+        计算 [0, max] 范围内的 LCS 距离。
+        计算公式为“max(len1, len2) - 相似度”。
+        """
+        return LCSseq.distance(str1, str2)
+
+    @classmethod
+    def osa_distance(cls, str1, str2) -> int:
+        """
+        OSA.distance（Optimal String Alignment，最优字符串对齐距离）是 RapidFuzz 库中的一个方法，用于计算两个字符串之间的 编辑距离（Edit Distance），但比标准的 Levenshtein 距离 限制更严格。
+
+        OSA 额外允许 相邻字符交换（Transposition），但限制比 Damerau-Levenshtein 更严格（Damerau 允许多次交换，而 OSA 仅限一次）。
+        """
+        return OSA.distance(str1, str2)
+
+    @classmethod
+    def levenshtein_distance(cls, str1, str2) -> int:
+        """
+        返回 两个字字符串之间的编辑距离 分数
+        标准 Levenshtein 距离 允许 插入、删除、替换 三种操作，但不允许 相邻字符交换（transposition）
+
+        jellyfish.levenshtein_distance(str1,str2) 该方法结果与 本方法一致
+
+        print(Jaro.distance(str1, str2))
+        与 print(Jaro.normalized_distance(str1, str2)) 结果一致
+
+        print(Jaro.similarity(str1, str2))
+        与 print(Jaro.normalized_similarity(str1,str2)) 结果一致
+        """
+        # 编辑距离长度
+        distance = Levenshtein.distance(str1, str2)
+        print(jellyfish.levenshtein_distance(str1, str2))
+        return distance
+
+    @classmethod
+    def indel_distance(cls, str1, str2) -> int:
+        """
+        Indel（Insertion + Deletion）距离是 仅考虑插入和删除操作 的编辑距离，不考虑替换操作。
+        """
+        return Indel.distance(str1, str2)
+
+    @classmethod
+    def damerau_levenshtein_distance(cls, str1, str2) -> int:
+        """
+        Damerau-Levenshtein 距离是 Levenshtein 距离的修改，它将换位（例如将 ifsh 表示为 fish）计为一次编辑
+        """
+        # 编辑距离长度
+        distance = jellyfish.damerau_levenshtein_distance(str1, str2)
+        print(DamerauLevenshtein.distance(str1, str2))
+        return distance
+
+    @classmethod
+    def hamming_distance(cls, str1, str2) -> int:
+        return Hamming.distance(str1, str2)
+
+# str1 = "primulina elegant ladyis a new culitvar developed by crossing seed parent primulina medica and pollen parent primulina longii it has fresh and elegant flowershigh ornamental value and strong shade tolerance it is easy to cultivate and propagate"
+# str2 = "primulinaelegant labyis a new cultivar developed by crossing seed parent primulina medica and pollen parent primulina longii it has fresh and elegant flowershigh ornamental value and strong shade tolerance it is easy to cultivate and propagate 2019 editorial office of acta horticulturae sinica all rights reserved"
+# # str1 = "primulina elegant ladyis a new cultivar developed by crossing seed parent primulina medica and pollen parent primulina longii it has fresh and elegant flowershigh ornamental value and strong shade tolerance it is easy to cultivate and propagate"
+# # str2 = "primulinaelegant ladyis a new cultivar developed by crossing seed parent primulina medica and pollen parent primulina longii it has fresh and elegant flowershigh ornamental value and strong shade tolerance it is easy to cultivate and propagate 2019 editorial office of acta horticulturae sinica all rights reserved"

{re_common-10.0.16 → re_common-10.0.17}/re_common/v2/baselibrary/utils/string_clear.py

@@ -6,6 +6,11 @@ import regex
 from re_common.v2.baselibrary.utils.stringutils import qj2bj, bj2qj, get_diacritic_variant, clean_html, \
     remove_spaces_between_chinese_characters
 
+from opencc import OpenCC
+
+# pip install opencc-python-reimplemented
+cc = OpenCC("t2s")  # t2s是繁体转简体
+
 
 class StringClear(object):
 
@@ -31,6 +36,11 @@ class StringClear(object):
         self.obj_str = bj2qj(self.obj_str)
         return self
 
+    def convert_to_simplified(self):
+        # 繁体转简体
+        self.obj_str = cc.convert(self.obj_str)
+        return self
+
     def lower(self):
         self.obj_str = self.obj_str.lower()
         return self

{re_common-10.0.16 → re_common-10.0.17}/re_common.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: re_common
-Version: 10.0.16
+Version: 10.0.17
 Summary: a library about all python projects
 Home-page: https://gitee.com/xujiangios/re-common
 Author: vic

{re_common-10.0.16 → re_common-10.0.17}/re_common.egg-info/SOURCES.txt

@@ -188,6 +188,7 @@ re_common/v2/baselibrary/tools/unionfind_tools.py
 re_common/v2/baselibrary/utils/BusinessStringUtil.py
 re_common/v2/baselibrary/utils/__init__.py
 re_common/v2/baselibrary/utils/author_smi.py
+re_common/v2/baselibrary/utils/base_string_similarity.py
 re_common/v2/baselibrary/utils/basedict.py
 re_common/v2/baselibrary/utils/basehdfs.py
 re_common/v2/baselibrary/utils/basepika.py

{re_common-10.0.16 → re_common-10.0.17}/setup.py

@@ -34,7 +34,7 @@ long_description = """
 """
 setuptools.setup(
     name="re_common",
-    version="10.0.16",
+    version="10.0.17",
     author="vic",
     author_email="xujiang5@163.com",
     description="a library about all python projects",