re-common 10.0.10__py3-none-any.whl → 10.0.12__py3-none-any.whl

re_common/v2/baselibrary/tools/list_tools.py

@@ -0,0 +1,9 @@
+
+def check_no_duplicates_2d(lst_2d):
+    """检查二维列表的每一行是否无重复"""
+    for row in lst_2d:
+        # 将行转为集合，比较长度
+        if len(row) != len(set(row)):
+            return False
+    return True
+

re_common/v2/baselibrary/utils/BusinessStringUtil.py

@@ -114,3 +114,62 @@ def get_wos_author_abbr(author_row: str):
     abbr_list = [author.strip() for author in abbr_list if
                  author.strip() and author.strip().lower() not in ("*", "and")]
     return ";".join(abbr_list)
+
+
+def deal_rel_vol(vol_str: str):
+    """
+    处理 期刊融合时的卷处理逻辑
+    """
+    if vol_str.replace(".", "").isdigit():
+        try:
+            float_num = float(vol_str)
+            if int(float_num) == float_num:
+                return str(int(float_num))
+        except:
+            pass
+
+    if vol_str.lower().startswith("v "):
+        vol_str = vol_str.lower().replace("v ", "").strip()
+        return vol_str
+    if vol_str.lower().startswith("volume "):
+        vol_str = vol_str.lower().replace("volume ", "").strip()
+        return vol_str
+    if vol_str.lower().startswith("vol. "):
+        vol_str = vol_str.lower().replace("vol. ", "").strip()
+        return vol_str
+    if vol_str.lower().startswith("vol "):
+        vol_str = vol_str.lower().replace("vol ", "").strip()
+        return vol_str
+    return vol_str
+
+
+def deal_num_strs(input_str):
+    """
+    int后在str 防止有浮点型的表达方式
+    """
+    number_list = re.findall(r'\d+', input_str)
+    transformed_numbers = [str(int(num)) for num in number_list]
+
+    # 替换原字符串中的数字为转换后的数字
+    for num, transformed_num in zip(number_list, transformed_numbers):
+        input_str = input_str.replace(num, transformed_num)
+    return input_str
+
+
+def deal_num(strs):
+    """
+    将 期格式化 方便 group尤其是有横杆的数据
+    该方法 为融合二次分割时使用，如果场景合适也可以用于其他地方
+    :param strs:
+    :return:
+    """
+    strs = strs.replace("-", "_").replace(".", "_").upper()
+    if strs.find("_") > -1:
+        start, end = strs.split("_")
+        start = deal_num_strs(start)
+        end = deal_num_strs(end)
+        strs = start + "_" + end
+    else:
+        strs = deal_num_strs(strs)
+
+    return strs

re_common/v2/baselibrary/utils/basedict.py

@@ -1,11 +1,12 @@
 import logging
+from itertools import groupby
 
 logger = logging.getLogger(__name__)  # 创建 logger 实例
 
 
 class BaseDict(object):
     @classmethod
-    def flip_dict(cls, original_dict, raise_on_conflict=False):
+    def flip_dict(cls, original_dict, raise_on_conflict=True):
         """
         翻转字典：将 key 是字符串、value 是列表的字典，转换为 key 是原 value 列表中的元素、value 是原 key 的字典。
         :param original_dict: 原始字典
@@ -24,3 +25,14 @@ class BaseDict(object):
                             f"Warning: Key conflict detected for {value}. Overwriting with new value: {key}.")
                 flipped_dict[value] = key
         return flipped_dict
+
+    @classmethod
+    def get_temp_gid_dicts(cls,lists,key_name):
+        """
+        对 列表字典 分组 组成 分组id的字典
+        """
+        dicts = {}
+        for group_id, group_tmp in groupby(sorted(lists, key=lambda x: x[key_name]),
+                                           key=lambda x: x[key_name]):
+            dicts[group_id] = group_tmp
+        return dicts

re_common/v2/baselibrary/utils/string_clear.py

@@ -1,7 +1,8 @@
 import re
 import regex
 
-from re_common.v2.baselibrary.utils.stringutils import qj2bj, bj2qj, get_diacritic_variant, clean_html
+from re_common.v2.baselibrary.utils.stringutils import qj2bj, bj2qj, get_diacritic_variant, clean_html, \
+    remove_spaces_between_chinese_characters
 
 
 class StringClear(object):
@@ -101,6 +102,7 @@ class StringClear(object):
         return self
 
     def remove_html_tag(self):
+        # 去除 html 标签
         import html
 
         self.obj_str = html.unescape(self.obj_str)
@@ -109,6 +111,11 @@ class StringClear(object):
 
         return self
 
+    def remove_spaces_in_chinese_characters(self):
+        # 匹配中文间的空格并替换为空字符串
+        self.obj_str = remove_spaces_between_chinese_characters(self.obj_str)
+        return self
+
     def get_str(self):
         return self.obj_str
 
@@ -122,6 +129,7 @@ def rel_clear(str_obj):
             .remove_html_tag()  # html标签清理
             .remove_special_chars()  # 移除特殊字符，仅保留字母、数字、空格和汉字 \w 已经包括所有 Unicode 字母 下划线 _ 会被保留
             .collapse_spaces()  # 移除多余空格,连续多个空格变一个
+            .remove_spaces_in_chinese_characters()  # 匹配中文间的空格并替换为空字符串
             .lower()  # 小写
             .get_str()  # 获取str
             .strip())  # 去掉空格

re_common/v2/baselibrary/utils/string_smi.py

@@ -0,0 +1,18 @@
+import jellyfish
+from rapidfuzz.distance import DamerauLevenshtein
+
+
+class JaroDamerauLevenshteinMaxSim(object):
+    """
+    jaro_similarity 有缺陷 以下样例数据会导致分很低
+    s1 = "in situ monitoring of semiconductor wafer temperature using infrared interfe rometry"
+    s2 = "insitu monitoring of semiconductor wafer temperature using infrared interferometry"
+    """
+
+    def get_sim(self, str1: str, str2: str) -> float:
+        similarity1 = jellyfish.jaro_similarity(str1, str2)
+        if str1.strip() == "" and str2.strip() == "":
+            similarity2 = 0
+        else:
+            similarity2 = 1 - DamerauLevenshtein.normalized_distance(str1, str2)
+        return max(similarity1, similarity2)

re_common/v2/baselibrary/utils/stringutils.py

@@ -1,10 +1,14 @@
 import re
 import threading
+from itertools import combinations
 
 import regex
 import unicodedata
 from html.parser import HTMLParser
 
+from re_common.v2.baselibrary.utils.string_clear import rel_clear
+from re_common.v2.baselibrary.utils.string_smi import JaroDamerauLevenshteinMaxSim
+
 
 def bj2qj(src):
     if src is None:
@@ -143,4 +147,66 @@ def clean_html(html):
     return parser.get_text()
 
 
+def remove_spaces_between_chinese_characters(text):
+    """
+    匹配中文间的空格并替换为空字符串
+
+    这里没有选取 后面的一些扩展分区 是那些分区比较分散 都写进来消耗性能,
+    认为只包含这些也够用了
+    """
+    pattern = r'(?<=[\u3400-\u9fff])\s+(?=[\u3400-\u9fff])'
+    return re.sub(pattern, '', text)
+
+
+
+sim_utils = JaroDamerauLevenshteinMaxSim()
+
+def group_similar_texts(texts, threshold=0.9):
+    """根据相似度对文本进行分组"""
+    n = len(texts)
+    # 创建邻接表表示图
+    graph = [[] for _ in range(n)]
+    # 计算所有文本对的相似度并构建图
+    for i, j in combinations(range(n), 2):
+        similarity = sim_utils.get_sim(rel_clear(texts[i]), rel_clear(texts[j]))
+        if similarity >= threshold:
+            graph[i].append(j)
+            graph[j].append(i)
 
+    visited = [False] * n
+    groups = []
+
+    # 使用DFS找到连通分量
+    def dfs(node, group):
+        visited[node] = True
+        group.append(node)
+        for neighbor in graph[node]:
+            if not visited[neighbor]:
+                dfs(neighbor, group)
+
+    # 找到所有连通分量
+    for i in range(n):
+        if not visited[i]:
+            current_group = []
+            dfs(i, current_group)
+            groups.append(current_group)
+
+    return groups
+
+
+def get_group_abstract(lists):
+    """
+    这是一个 分组程序 ，会根据简单的连通图分组
+    lists: [(id,txt),...]
+    return: all_list 返回一个二维列表 每个列表里面是id 每个列表为一个分组
+    """
+    abstract_list = [i[1] for i in lists]
+    keyid_list = [i[0] for i in lists]
+    groups = group_similar_texts(abstract_list, threshold=0.9)
+    all_list = []
+    for group in groups:
+        t_list = []
+        for text_idx in group:
+            t_list.append(keyid_list[text_idx])
+        all_list.append(t_list)
+    return all_list

{re_common-10.0.10.dist-info → re_common-10.0.12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: re_common
-Version: 10.0.10
+Version: 10.0.12
 Summary: a library about all python projects
 Home-page: https://gitee.com/xujiangios/re-common
 Author: vic

{re_common-10.0.10.dist-info → re_common-10.0.12.dist-info}/RECORD

@@ -170,20 +170,21 @@ re_common/v2/baselibrary/tools/WeChatRobot.py,sha256=EaQgNncROAhU5-psYRGWAshIV5a
 re_common/v2/baselibrary/tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 re_common/v2/baselibrary/tools/dict_tools.py,sha256=BTh7oJuJ619IZgxiYlim0ltrXBclDtb7WzyFGr7wVf0,1246
 re_common/v2/baselibrary/tools/dolphinscheduler.py,sha256=1m7UGYDiuvJUCI6ik6CGM2fO8U5XteJzn55VRbwB9ts,7978
-re_common/v2/baselibrary/tools/list_tools.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+re_common/v2/baselibrary/tools/list_tools.py,sha256=qYxdLccRbrULOBbaPdJ_MyFFmVJGVMdW5E36nJ3ejr8,249
 re_common/v2/baselibrary/tools/search_hash_tools.py,sha256=d_h9j7VxiXpcn1GHZ7L2tpx9_LDQshcl58tlKvSxZPg,1691
 re_common/v2/baselibrary/tools/text_matcher.py,sha256=F4WtLO-b7H6V9TIvOntCD9ZXSQP_KijPuLLYcLPtrKQ,7021
 re_common/v2/baselibrary/tools/unionfind_tools.py,sha256=VYHZZPXwBYljsm7TjV1B6iCgDn3O3btzNf9hMvQySVU,2965
-re_common/v2/baselibrary/utils/BusinessStringUtil.py,sha256=tzjVr_-6iPAKTt14hR-BhRshdRgeT_MPJpUQkxcTXns,4084
+re_common/v2/baselibrary/utils/BusinessStringUtil.py,sha256=__9MECbdrMnYc-ksYn2liM8vEbqF9uR4hZKqw86kW1Q,5924
 re_common/v2/baselibrary/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 re_common/v2/baselibrary/utils/author_smi.py,sha256=1ebH3AHv19jtJWdlqNdwu6t58HNVLCotuCB6ff1SWiw,13666
-re_common/v2/baselibrary/utils/basedict.py,sha256=tSV85pARe8ZQDY77_h_heS81EWwcgJW076DcA9WQyjY,1161
+re_common/v2/baselibrary/utils/basedict.py,sha256=sH3_RZ8u4649-jX2V1uKNNkjJVUijZBDp6SdqncOZ88,1583
 re_common/v2/baselibrary/utils/basehdfs.py,sha256=NVV5Q0OMPlM_zTrs9ZDoPJv29GQv5wi9-AP1us5dBrQ,4651
 re_common/v2/baselibrary/utils/json_cls.py,sha256=dHOkWafG9lbQDoub9cbDwT2fDjMKtblQnjFLeA4hECA,286
 re_common/v2/baselibrary/utils/n_ary_expression_tree.py,sha256=-05kO6G2Rth7CEK-5lfFrthFZ1Q0-0a7cni7mWZ-2gg,9172
 re_common/v2/baselibrary/utils/string_bool.py,sha256=EJnkSck4ofcIeJ6nLzAOVtlt6o1WBgvgVwIqJKj5Suc,2993
-re_common/v2/baselibrary/utils/string_clear.py,sha256=LDIf-3Czq1sXp-54aifXdXbdGUX7hpFBKqQa5Azj_lo,5861
-re_common/v2/baselibrary/utils/stringutils.py,sha256=GLXHAm8IulC_8hWrN2aiFQjsoOpjczvcVozmTJj86-A,3864
+re_common/v2/baselibrary/utils/string_clear.py,sha256=pGxL9PlzQDM06sC0j6U0zYRemvsJ7-OOpfzS5ETCxAs,6258
+re_common/v2/baselibrary/utils/string_smi.py,sha256=cU0WAWHRGnGoVQx3eCEKeM_q_olFNzRTJe7rSe586SY,741
+re_common/v2/baselibrary/utils/stringutils.py,sha256=ISheMydPZeNmqsffGDT4Ut_UGpK3r6k8STR78Ere8Wg,6033
 re_common/vip/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 re_common/vip/base_step_process.py,sha256=VXXiNj0I5CpzXIMCgOPU86bzDJkSBkUS-9CpZIl_GOk,205
 re_common/vip/baseencodeid.py,sha256=nERoe89ueFM52bG7xwJdflcZHk6T2RQQKbc5uUZc3RM,3272
@@ -210,8 +211,8 @@ re_common/vip/title/transform/TransformRegulationTitleToZt.py,sha256=LKRdIsWKues
 re_common/vip/title/transform/TransformStandardTitleToZt.py,sha256=-fCKAbSBzXVyQDCE61CalvR9E_QzQMA08QOO_NePFNI,5563
 re_common/vip/title/transform/TransformThesisTitleToZt.py,sha256=QS-uV0cQrpUFAcKucuJQ9Ue2VRQH-inmfn_X3IplfRo,5488
 re_common/vip/title/transform/__init__.py,sha256=m83-CWyRq_VHPYHaALEQlmXrkTdrZ3e4B_kCfBYE-uc,239
-re_common-10.0.10.dist-info/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
-re_common-10.0.10.dist-info/METADATA,sha256=mOarqqiMSzMjAcu1sV0OxUGdwfANLray_3ZpjkAPxFg,582
-re_common-10.0.10.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-re_common-10.0.10.dist-info/top_level.txt,sha256=_H9H23zoLIalm1AIY_KYTVh_H0ZnmjxQIxsvXtLv45o,10
-re_common-10.0.10.dist-info/RECORD,,
+re_common-10.0.12.dist-info/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
+re_common-10.0.12.dist-info/METADATA,sha256=N9MX7TnI7lhFQyhFaV0n0wr5XWT5prKsFX9gsM-X4T4,582
+re_common-10.0.12.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+re_common-10.0.12.dist-info/top_level.txt,sha256=_H9H23zoLIalm1AIY_KYTVh_H0ZnmjxQIxsvXtLv45o,10
+re_common-10.0.12.dist-info/RECORD,,