re-common 10.0.4__py3-none-any.whl → 10.0.6__py3-none-any.whl

re_common/v2/baselibrary/utils/author_smi.py

@@ -3,9 +3,12 @@ import re
 import string
 
 import regex
-import unicodedata
+from jellyfish import damerau_levenshtein_distance
 from rapidfuzz._utils import setupPandas, is_none
 from rapidfuzz.distance import Jaro
+from unidecode import unidecode
+
+from re_common.v2.baselibrary.utils.stringutils import get_diacritic_variant
 
 """
 作者比率分布 大部分在 1和 2 
@@ -23,17 +26,6 @@ additional_chars = '‑–‐’·．—'
 extended_punctuation = string.punctuation + additional_chars
 
 
-def get_diacritic_variant(char1):
-    # 将字符转换为标准的 Unicode 形式
-    normalized_char1 = unicodedata.normalize('NFD', char1)
-
-    # 获取基本字符（去掉变音符号）
-    base_char1 = ''.join(c for c in normalized_char1 if unicodedata.category(c) != 'Mn')
-
-    # 判断基本字符是否相同
-    return base_char1
-
-
 def detect_other_languages(text):
     # 匹配所有非中文、非英文、非数字字符
     pattern = r'[^\u4E00-\u9FFFa-zA-Z0-9\s.,!?;:\'\"()‑\-–—‐’·˜．]'
@@ -127,6 +119,49 @@ def custom_rstrip(s):
     return s
 
 
+# 分割中文拼音，如"Xiaohong" ————> ['Xiao', 'hong']
+def chinese_pinyin_split_by_rules(input_str):
+    # 声母列表（含复合声母）
+    initials = {
+        'b', 'p', 'm', 'f', 'd', 't', 'n', 'l', 'g', 'k', 'h',
+        'j', 'q', 'x', 'zh', 'ch', 'sh', 'r', 'z', 'c', 's', 'y', 'w'
+    }
+    # 韵母列表（部分示例）
+    finals = {
+        'a', 'o', 'e', 'ai', 'ei', 'ao', 'ou', 'an', 'en', 'ang', 'eng', 'ong',
+        'i', 'ia', 'ie', 'iao', 'iu', 'ian', 'in', 'iang', 'ing', 'iong',
+        'u', 'ua', 'uo', 'uai', 'ui', 'uan', 'un', 'uang', 'ueng',
+        'v', 've', 'van', 'vn'
+    }
+    result = []
+    while input_str:
+        # 尝试匹配最长声母
+        max_initial_len = 2  # 最长声母如 'zh'
+        matched_initial = ""
+        for length in range(max_initial_len, 0, -1):
+            candidate = input_str[:length]
+            if candidate.lower() in initials:
+                matched_initial = candidate
+                break
+        # 切分声母后的剩余部分
+        remaining = input_str[len(matched_initial):]
+        # 匹配韵母
+        max_final_len = min(4, len(remaining))  # 最长韵母如 'iong'
+        matched_final = ""
+        for length in range(max_final_len, 0, -1):
+            candidate = remaining[:length]
+            if candidate.lower() in finals:
+                matched_final = candidate
+                break
+        if matched_final:
+            # 合并声母和韵母
+            syllable = matched_initial + matched_final
+            result.append(syllable)
+            input_str = input_str[len(syllable):]
+        else:
+            return []  # 无法切分
+    return result
+
 
 def AuthorRatio(
         s1,
@@ -151,8 +186,8 @@ def AuthorRatio(
         if not s1 or not s2:
             return 0
     # 处理音标问题
-    s1 = get_diacritic_variant(s1)
-    s2 = get_diacritic_variant(s2)
+    s1 = get_diacritic_variant(unidecode(s1))
+    s2 = get_diacritic_variant(unidecode(s2))
     # 这里提出来是为了少计算 但后期需要平衡内存和算力
     # 移除指定符号 这里做了小写化处理
     s1_punc = remove_punctuation(s1)
@@ -218,6 +253,10 @@ def AuthorRatio(
             # 如果两个字符忽略大小写后相同，继续比较下一个字符
             if i1.lower() == i2.lower():
                 continue
+            # 在作者中 有可能错误字母 当单词大于3 且只有一个字母错误或者位置交换时 可以认为这两个单词相同
+            # 样例 "De Gusmio, Ana Paula Henriques","De Gusmão, Ana Paula Henriques"
+            if len(i1) > 3 and damerau_levenshtein_distance(i1, i2) <= 1:
+                continue
 
             # 如果其中一个字符的长度为1（即是单个字母），检查它们的首字母是否匹配
             if len(i1) == 1 or len(i2) == 1:

re_common/v2/baselibrary/utils/n_ary_expression_tree.py

@@ -0,0 +1,244 @@
+class Node:
+    def __init__(self, value, children=None):
+        self.value = value
+        self.children = children if children is not None else []
+
+    def __repr__(self):
+        return f"Node(value={self.value}, children={self.children})"
+
+
+def tokenize(expression):
+    """将表达式分解为标记（token），仅以 and, or, not 和括号作为分界符
+
+    Args:
+        expression (str): 输入的字符串表达式，例如 'a = 3 + (b > 2) and c'
+
+    Returns:
+        list: 分解后的标记列表，例如 ['a = 3 + (b > 2)', 'and', 'c']
+    """
+    tokens = []  # 存储最终的标记列表
+    current = ""  # 当前正在构建的标记字符串
+    i = 0  # 当前字符的索引
+    length = len(expression)  # 输入表达式的长度
+
+    def is_delimiter_match(expression, i, delimiter_len=3, delimiter="and"):
+        """检查当前位置是否匹配指定的分隔符（and, or, not）
+
+        Args:
+            expression (str): 输入的表达式字符串
+            i (int): 当前检查的起始索引
+            delimiter_len (int): 分隔符的长度，默认为 3（适用于 'and' 和 'not'）
+            delimiter (str): 要检查的分隔符，默认为 'and'
+
+        Returns:
+            bool: 如果当前位置匹配分隔符且前后有空格，返回 True，否则返回 False
+        """
+        # 检查索引是否超出范围
+        if not i + delimiter_len <= length:
+            return False
+        # 检查当前位置是否匹配指定分隔符（忽略大小写）
+        if not expression[i:i + delimiter_len].lower() == delimiter:
+            return False
+
+        # 检查分隔符前是否有一个空格（如果不是字符串开头）
+        if i - 1 >= 0:
+            if not expression[i - 1].lower() == ' ':
+                return False
+
+        # 检查分隔符后是否有一个空格（如果不是字符串结尾）
+        if i + delimiter_len + 1 <= length:
+            if not expression[i + delimiter_len].lower() == ' ':
+                return False
+        return True
+
+    # 遍历表达式的每个字符
+    while i < length:
+        char = expression[i]  # 当前处理的字符
+
+        # 处理括号
+        if char in "()":
+            if current.strip():  # 如果当前标记有内容，先将其添加到 tokens
+                tokens.append(current.strip())
+                current = ""  # 重置当前标记
+            tokens.append(char)  # 将括号作为独立标记添加
+            i += 1  # 移动到下一个字符
+            continue
+
+        # 检查是否遇到 and, or, not 分隔符
+        if is_delimiter_match(expression, i, delimiter_len=3, delimiter="and"):
+            if current.strip():  # 如果当前标记有内容，先添加
+                tokens.append(current.strip())
+                current = ""  # 重置当前标记
+            tokens.append("and")  # 添加 'and' 标记
+            i += 3  # 跳过 'and' 的长度
+            continue
+        elif is_delimiter_match(expression, i, delimiter_len=2, delimiter="or"):
+            if current.strip():  # 如果当前标记有内容，先添加
+                tokens.append(current.strip())
+                current = ""  # 重置当前标记
+            tokens.append("or")  # 添加 'or' 标记
+            i += 2  # 跳过 'or' 的长度
+            continue
+        elif is_delimiter_match(expression, i, delimiter_len=3, delimiter="not"):
+            if current.strip():  # 如果当前标记有内容，先添加
+                tokens.append(current.strip())
+                current = ""  # 重置当前标记
+            tokens.append("not")  # 添加 'not' 标记
+            i += 3  # 跳过 'not' 的长度
+            continue
+
+        # 将非分隔符字符追加到当前标记中，包括空格
+        current += char
+        i += 1  # 移动到下一个字符
+
+    # 处理最后一个标记（如果有内容）
+    if current.strip():
+        tokens.append(current.strip())
+
+    return tokens  # 返回标记列表
+
+
+
+
+def parse_expression(tokens):
+    """递归下降解析表达式"""
+
+    def parse_or(tokens, pos):
+        """解析 OR 级别（最低优先级）"""
+        left, pos = parse_and(tokens, pos)
+        while pos < len(tokens) and tokens[pos] == 'or':
+            pos += 1
+            if pos >= len(tokens):
+                raise ValueError("Incomplete expression after 'or'")
+            right, pos = parse_and(tokens, pos)
+            left = Node('or', [left, right])
+        return left, pos
+
+    def parse_and(tokens, pos):
+        """解析 AND 级别（次高优先级）"""
+        left, pos = parse_not(tokens, pos)
+        while pos < len(tokens) and tokens[pos] == 'and':
+            pos += 1
+            if pos >= len(tokens):
+                raise ValueError("Incomplete expression after 'and'")
+            right, pos = parse_not(tokens, pos)
+            left = Node('and', [left, right])
+        return left, pos
+
+    def parse_not(tokens, pos):
+        """解析 NOT 级别（最高优先级）"""
+        if pos < len(tokens) and tokens[pos] == 'not':
+            pos += 1
+            if pos >= len(tokens):
+                raise ValueError("Incomplete expression after 'not'")
+            child, pos = parse_primary(tokens, pos)
+            return Node('not', [child]), pos
+        return parse_primary(tokens, pos)
+
+    def parse_primary(tokens, pos):
+        """解析基本单元（条件或括号表达式）"""
+        if pos >= len(tokens):
+            raise ValueError("Unexpected end of expression")
+
+        if tokens[pos] == '(':
+            pos += 1
+            subtree, pos = parse_or(tokens, pos)
+            if pos >= len(tokens) or tokens[pos] != ')':
+                raise ValueError("Missing closing parenthesis")
+            return subtree, pos + 1
+        else:
+            # 假设这是一个条件（如 A=1）
+            return Node(tokens[pos]), pos + 1
+
+    # 从头开始解析
+    tree, pos = parse_or(tokens, 0)
+    if pos < len(tokens):
+        raise ValueError(f"Extra tokens after expression: {tokens[pos:]}")
+    return tree
+
+
+def flatten_tree(node):
+    """清理语法树，将嵌套的同级 and/or 节点展平。
+
+    Args:
+        node (Node): 输入的语法树节点
+
+    Returns:
+        Node: 清理后的新语法树节点
+    """
+    # 如果没有子节点，直接返回原节点（条件节点）
+    if not node.children:
+        return Node(value=node.value, children=[])
+
+    # 递归清理所有子节点
+    cleaned_children = [flatten_tree(child) for child in node.children]
+
+    # 如果当前节点是 'and' 或 'or'，展平嵌套的同类节点
+    if node.value in ('and', 'or'):
+        flattened_children = []
+        for child in cleaned_children:
+            # 如果子节点的值与当前节点相同（例如 'or' 下的 'or'），将其子节点提升
+            if child.value == node.value:
+                flattened_children.extend(child.children)
+            else:
+                flattened_children.append(child)
+        return Node(value=node.value, children=flattened_children)
+
+    # 对于其他节点（例如 'not'），保持结构不变，只更新子节点
+    return Node(value=node.value, children=cleaned_children)
+
+
+def pretty_print_tree(node, indent=0, prefix=""):
+    """生成语法树的格式化字符串表示，带有层次缩进。
+
+    Args:
+        node (Node): 要格式化的语法树节点
+        indent (int): 当前缩进级别（空格数），默认从 0 开始
+        prefix (str): 前缀字符串，用于表示当前行的开头
+
+    Returns:
+        str: 格式化后的树形字符串
+    """
+    # 基本缩进单位
+    spaces = " " * indent
+
+    # 如果没有子节点，返回单行表示
+    if not node.children:
+        return f"{spaces}{prefix}Node(value='{node.value}', children=[])"
+
+    # 构建当前节点的字符串
+    result = [f"{spaces}{prefix}Node(value='{node.value}', children=["]
+
+    # 递归处理每个子节点
+    for i, child in enumerate(node.children):
+        is_last = i == len(node.children) - 1
+        child_prefix = " " if is_last else " "
+        result.append(pretty_print_tree(child, indent + 4, child_prefix))
+
+    # 添加结束括号
+    result.append(f"{spaces}])")
+
+    # 将所有行连接成一个字符串
+    return "\n".join(result)
+
+# 测试代码
+expressions = [
+    "not A=1 and B= 2",
+    "A=1 and (not B=2 or (C=3 or D=4))",
+    "A=1 and not (B=2 or C=3 and D=4 or E=5)",
+    "(A=1 and not (B=2 or C=3 or D=4))",
+    "A=1 and",  # 不完整表达式
+    "and A=1",  # 不完整表达式
+]
+
+for expr in expressions:
+    try:
+        print(f"\nExpression: {expr}")
+        tokens = tokenize(expr)
+        print("Tokens:", tokens)
+        tree = parse_expression(tokens)
+        tree = flatten_tree(tree)
+        tree = pretty_print_tree(tree)
+        print("Tree:", tree)
+    except ValueError as e:
+        print(f"Error: {e}")

re_common/v2/baselibrary/utils/string_clear.py

@@ -67,7 +67,6 @@ class StringClear(object):
         return self
 
     def replace_dash_with_space(self):
-        # 横线换成空格 比 去除符号有时更有用
         self.obj_str = self.obj_str.replace("-", " ")
         return self
 
@@ -106,7 +105,6 @@ def rel_clear(str_obj):
             .to_str()  # 防止其他类型传入 比如 int double
             .qj_to_bj()  # 全角转半角
             .remove_html_tag()  # html标签清理
-            .replace_dash_with_space()  # 横线转空格 在 英文 title 中更有用
             .remove_special_chars()  # 移除特殊字符，仅保留字母、数字、空格和汉字 \w 已经包括所有 Unicode 字母 下划线 _ 会被保留
             .collapse_spaces()  # 移除多余空格,连续多个空格变一个
             .lower()  # 小写

{re_common-10.0.4.dist-info → re_common-10.0.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: re_common
-Version: 10.0.4
+Version: 10.0.6
 Summary: a library about all python projects
 Home-page: https://gitee.com/xujiangios/re-common
 Author: vic

{re_common-10.0.4.dist-info → re_common-10.0.6.dist-info}/RECORD

@@ -173,12 +173,13 @@ re_common/v2/baselibrary/tools/text_matcher.py,sha256=F4WtLO-b7H6V9TIvOntCD9ZXSQ
 re_common/v2/baselibrary/tools/unionfind_tools.py,sha256=VYHZZPXwBYljsm7TjV1B6iCgDn3O3btzNf9hMvQySVU,2965
 re_common/v2/baselibrary/utils/BusinessStringUtil.py,sha256=tzjVr_-6iPAKTt14hR-BhRshdRgeT_MPJpUQkxcTXns,4084
 re_common/v2/baselibrary/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-re_common/v2/baselibrary/utils/author_smi.py,sha256=Mjl0GYH9e0TP48yxnxC7qgMP2bZW04pa8TQezpKo9L0,11796
+re_common/v2/baselibrary/utils/author_smi.py,sha256=1ebH3AHv19jtJWdlqNdwu6t58HNVLCotuCB6ff1SWiw,13666
 re_common/v2/baselibrary/utils/basedict.py,sha256=tSV85pARe8ZQDY77_h_heS81EWwcgJW076DcA9WQyjY,1161
 re_common/v2/baselibrary/utils/basehdfs.py,sha256=NVV5Q0OMPlM_zTrs9ZDoPJv29GQv5wi9-AP1us5dBrQ,4651
 re_common/v2/baselibrary/utils/json_cls.py,sha256=dHOkWafG9lbQDoub9cbDwT2fDjMKtblQnjFLeA4hECA,286
+re_common/v2/baselibrary/utils/n_ary_expression_tree.py,sha256=-05kO6G2Rth7CEK-5lfFrthFZ1Q0-0a7cni7mWZ-2gg,9172
 re_common/v2/baselibrary/utils/string_bool.py,sha256=4VCr1g8pX5YnzZSKctQgQfmhSQ0aw7a8ruhWdiRmBFU,641
-re_common/v2/baselibrary/utils/string_clear.py,sha256=6mkBAZUNh5-JTPmB9lj_i4eLT9C6ZW1nH4tZiGveIE4,3778
+re_common/v2/baselibrary/utils/string_clear.py,sha256=sKKXEqCtItbJxsjgrBXBeubXaiAYuoc0301EOVFzXbk,3627
 re_common/v2/baselibrary/utils/stringutils.py,sha256=GLXHAm8IulC_8hWrN2aiFQjsoOpjczvcVozmTJj86-A,3864
 re_common/vip/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 re_common/vip/base_step_process.py,sha256=VXXiNj0I5CpzXIMCgOPU86bzDJkSBkUS-9CpZIl_GOk,205
@@ -206,8 +207,8 @@ re_common/vip/title/transform/TransformRegulationTitleToZt.py,sha256=LKRdIsWKues
 re_common/vip/title/transform/TransformStandardTitleToZt.py,sha256=-fCKAbSBzXVyQDCE61CalvR9E_QzQMA08QOO_NePFNI,5563
 re_common/vip/title/transform/TransformThesisTitleToZt.py,sha256=QS-uV0cQrpUFAcKucuJQ9Ue2VRQH-inmfn_X3IplfRo,5488
 re_common/vip/title/transform/__init__.py,sha256=m83-CWyRq_VHPYHaALEQlmXrkTdrZ3e4B_kCfBYE-uc,239
-re_common-10.0.4.dist-info/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
-re_common-10.0.4.dist-info/METADATA,sha256=SNPQXc5koTrhSxu9yAPRPN42uItn6onNvmG7GHTMdcE,581
-re_common-10.0.4.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-re_common-10.0.4.dist-info/top_level.txt,sha256=_H9H23zoLIalm1AIY_KYTVh_H0ZnmjxQIxsvXtLv45o,10
-re_common-10.0.4.dist-info/RECORD,,
+re_common-10.0.6.dist-info/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
+re_common-10.0.6.dist-info/METADATA,sha256=4gcNYlu46W2s5D1IRTzBtM_Sp3DtETT51Xxv-RkX7Ns,581
+re_common-10.0.6.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+re_common-10.0.6.dist-info/top_level.txt,sha256=_H9H23zoLIalm1AIY_KYTVh_H0ZnmjxQIxsvXtLv45o,10
+re_common-10.0.6.dist-info/RECORD,,