re-common 10.0.10__tar.gz → 10.0.12__tar.gz

{re_common-10.0.10 → re_common-10.0.12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: re_common
-Version: 10.0.10
+Version: 10.0.12
 Summary: a library about all python projects
 Home-page: https://gitee.com/xujiangios/re-common
 Author: vic

re_common-10.0.12/re_common/v2/baselibrary/tools/list_tools.py

@@ -0,0 +1,9 @@
+
+def check_no_duplicates_2d(lst_2d):
+    """检查二维列表的每一行是否无重复"""
+    for row in lst_2d:
+        # 将行转为集合，比较长度
+        if len(row) != len(set(row)):
+            return False
+    return True
+

{re_common-10.0.10 → re_common-10.0.12}/re_common/v2/baselibrary/utils/BusinessStringUtil.py

@@ -114,3 +114,62 @@ def get_wos_author_abbr(author_row: str):
     abbr_list = [author.strip() for author in abbr_list if
                  author.strip() and author.strip().lower() not in ("*", "and")]
     return ";".join(abbr_list)
+
+
+def deal_rel_vol(vol_str: str):
+    """
+    处理 期刊融合时的卷处理逻辑
+    """
+    if vol_str.replace(".", "").isdigit():
+        try:
+            float_num = float(vol_str)
+            if int(float_num) == float_num:
+                return str(int(float_num))
+        except:
+            pass
+
+    if vol_str.lower().startswith("v "):
+        vol_str = vol_str.lower().replace("v ", "").strip()
+        return vol_str
+    if vol_str.lower().startswith("volume "):
+        vol_str = vol_str.lower().replace("volume ", "").strip()
+        return vol_str
+    if vol_str.lower().startswith("vol. "):
+        vol_str = vol_str.lower().replace("vol. ", "").strip()
+        return vol_str
+    if vol_str.lower().startswith("vol "):
+        vol_str = vol_str.lower().replace("vol ", "").strip()
+        return vol_str
+    return vol_str
+
+
+def deal_num_strs(input_str):
+    """
+    int后在str 防止有浮点型的表达方式
+    """
+    number_list = re.findall(r'\d+', input_str)
+    transformed_numbers = [str(int(num)) for num in number_list]
+
+    # 替换原字符串中的数字为转换后的数字
+    for num, transformed_num in zip(number_list, transformed_numbers):
+        input_str = input_str.replace(num, transformed_num)
+    return input_str
+
+
+def deal_num(strs):
+    """
+    将 期格式化 方便 group尤其是有横杆的数据
+    该方法 为融合二次分割时使用，如果场景合适也可以用于其他地方
+    :param strs:
+    :return:
+    """
+    strs = strs.replace("-", "_").replace(".", "_").upper()
+    if strs.find("_") > -1:
+        start, end = strs.split("_")
+        start = deal_num_strs(start)
+        end = deal_num_strs(end)
+        strs = start + "_" + end
+    else:
+        strs = deal_num_strs(strs)
+
+    return strs

{re_common-10.0.10 → re_common-10.0.12}/re_common/v2/baselibrary/utils/basedict.py

@@ -1,11 +1,12 @@
 import logging
+from itertools import groupby
 
 logger = logging.getLogger(__name__)  # 创建 logger 实例
 
 
 class BaseDict(object):
     @classmethod
-    def flip_dict(cls, original_dict, raise_on_conflict=False):
+    def flip_dict(cls, original_dict, raise_on_conflict=True):
         """
         翻转字典：将 key 是字符串、value 是列表的字典，转换为 key 是原 value 列表中的元素、value 是原 key 的字典。
         :param original_dict: 原始字典
@@ -24,3 +25,14 @@ class BaseDict(object):
                             f"Warning: Key conflict detected for {value}. Overwriting with new value: {key}.")
                 flipped_dict[value] = key
         return flipped_dict
+
+    @classmethod
+    def get_temp_gid_dicts(cls,lists,key_name):
+        """
+        对 列表字典 分组 组成 分组id的字典
+        """
+        dicts = {}
+        for group_id, group_tmp in groupby(sorted(lists, key=lambda x: x[key_name]),
+                                           key=lambda x: x[key_name]):
+            dicts[group_id] = group_tmp
+        return dicts

{re_common-10.0.10 → re_common-10.0.12}/re_common/v2/baselibrary/utils/string_clear.py

@@ -1,7 +1,8 @@
 import re
 import regex
 
-from re_common.v2.baselibrary.utils.stringutils import qj2bj, bj2qj, get_diacritic_variant, clean_html
+from re_common.v2.baselibrary.utils.stringutils import qj2bj, bj2qj, get_diacritic_variant, clean_html, \
+    remove_spaces_between_chinese_characters
 
 
 class StringClear(object):
@@ -101,6 +102,7 @@ class StringClear(object):
         return self
 
     def remove_html_tag(self):
+        # 去除 html 标签
         import html
 
         self.obj_str = html.unescape(self.obj_str)
@@ -109,6 +111,11 @@ class StringClear(object):
 
         return self
 
+    def remove_spaces_in_chinese_characters(self):
+        # 匹配中文间的空格并替换为空字符串
+        self.obj_str = remove_spaces_between_chinese_characters(self.obj_str)
+        return self
+
     def get_str(self):
         return self.obj_str
 
@@ -122,6 +129,7 @@ def rel_clear(str_obj):
             .remove_html_tag()  # html标签清理
             .remove_special_chars()  # 移除特殊字符，仅保留字母、数字、空格和汉字 \w 已经包括所有 Unicode 字母 下划线 _ 会被保留
             .collapse_spaces()  # 移除多余空格,连续多个空格变一个
+            .remove_spaces_in_chinese_characters()  # 匹配中文间的空格并替换为空字符串
             .lower()  # 小写
             .get_str()  # 获取str
             .strip())  # 去掉空格

re_common-10.0.12/re_common/v2/baselibrary/utils/string_smi.py

@@ -0,0 +1,18 @@
+import jellyfish
+from rapidfuzz.distance import DamerauLevenshtein
+
+
+class JaroDamerauLevenshteinMaxSim(object):
+    """
+    jaro_similarity 有缺陷 以下样例数据会导致分很低
+    s1 = "in situ monitoring of semiconductor wafer temperature using infrared interfe rometry"
+    s2 = "insitu monitoring of semiconductor wafer temperature using infrared interferometry"
+    """
+
+    def get_sim(self, str1: str, str2: str) -> float:
+        similarity1 = jellyfish.jaro_similarity(str1, str2)
+        if str1.strip() == "" and str2.strip() == "":
+            similarity2 = 0
+        else:
+            similarity2 = 1 - DamerauLevenshtein.normalized_distance(str1, str2)
+        return max(similarity1, similarity2)

{re_common-10.0.10 → re_common-10.0.12}/re_common/v2/baselibrary/utils/stringutils.py

@@ -1,10 +1,14 @@
 import re
 import threading
+from itertools import combinations
 
 import regex
 import unicodedata
 from html.parser import HTMLParser
 
+from re_common.v2.baselibrary.utils.string_clear import rel_clear
+from re_common.v2.baselibrary.utils.string_smi import JaroDamerauLevenshteinMaxSim
+
 
 def bj2qj(src):
     if src is None:
@@ -143,4 +147,66 @@ def clean_html(html):
     return parser.get_text()
 
 
+def remove_spaces_between_chinese_characters(text):
+    """
+    匹配中文间的空格并替换为空字符串
+
+    这里没有选取 后面的一些扩展分区 是那些分区比较分散 都写进来消耗性能,
+    认为只包含这些也够用了
+    """
+    pattern = r'(?<=[\u3400-\u9fff])\s+(?=[\u3400-\u9fff])'
+    return re.sub(pattern, '', text)
+
+
+
+sim_utils = JaroDamerauLevenshteinMaxSim()
+
+def group_similar_texts(texts, threshold=0.9):
+    """根据相似度对文本进行分组"""
+    n = len(texts)
+    # 创建邻接表表示图
+    graph = [[] for _ in range(n)]
+    # 计算所有文本对的相似度并构建图
+    for i, j in combinations(range(n), 2):
+        similarity = sim_utils.get_sim(rel_clear(texts[i]), rel_clear(texts[j]))
+        if similarity >= threshold:
+            graph[i].append(j)
+            graph[j].append(i)
 
+    visited = [False] * n
+    groups = []
+
+    # 使用DFS找到连通分量
+    def dfs(node, group):
+        visited[node] = True
+        group.append(node)
+        for neighbor in graph[node]:
+            if not visited[neighbor]:
+                dfs(neighbor, group)
+
+    # 找到所有连通分量
+    for i in range(n):
+        if not visited[i]:
+            current_group = []
+            dfs(i, current_group)
+            groups.append(current_group)
+
+    return groups
+
+
+def get_group_abstract(lists):
+    """
+    这是一个 分组程序 ，会根据简单的连通图分组
+    lists: [(id,txt),...]
+    return: all_list 返回一个二维列表 每个列表里面是id 每个列表为一个分组
+    """
+    abstract_list = [i[1] for i in lists]
+    keyid_list = [i[0] for i in lists]
+    groups = group_similar_texts(abstract_list, threshold=0.9)
+    all_list = []
+    for group in groups:
+        t_list = []
+        for text_idx in group:
+            t_list.append(keyid_list[text_idx])
+        all_list.append(t_list)
+    return all_list

{re_common-10.0.10 → re_common-10.0.12}/re_common.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: re_common
-Version: 10.0.10
+Version: 10.0.12
 Summary: a library about all python projects
 Home-page: https://gitee.com/xujiangios/re-common
 Author: vic

{re_common-10.0.10 → re_common-10.0.12}/re_common.egg-info/SOURCES.txt

@@ -190,6 +190,7 @@ re_common/v2/baselibrary/utils/json_cls.py
 re_common/v2/baselibrary/utils/n_ary_expression_tree.py
 re_common/v2/baselibrary/utils/string_bool.py
 re_common/v2/baselibrary/utils/string_clear.py
+re_common/v2/baselibrary/utils/string_smi.py
 re_common/v2/baselibrary/utils/stringutils.py
 re_common/vip/__init__.py
 re_common/vip/base_step_process.py

{re_common-10.0.10 → re_common-10.0.12}/setup.py

@@ -34,7 +34,7 @@ long_description = """
 """
 setuptools.setup(
     name="re_common",
-    version="10.0.10",
+    version="10.0.12",
     author="vic",
     author_email="xujiang5@163.com",
     description="a library about all python projects",