re-common 10.0.3__tar.gz → 10.0.5__tar.gz

{re_common-10.0.3 → re_common-10.0.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: re_common
-Version: 10.0.3
+Version: 10.0.5
 Summary: a library about all python projects
 Home-page: https://gitee.com/xujiangios/re-common
 Author: vic

{re_common-10.0.3 → re_common-10.0.5}/re_common/v2/baselibrary/utils/author_smi.py

@@ -3,9 +3,12 @@ import re
 import string
 
 import regex
-import unicodedata
+from jellyfish import damerau_levenshtein_distance
 from rapidfuzz._utils import setupPandas, is_none
 from rapidfuzz.distance import Jaro
+from unidecode import unidecode
+
+from re_common.v2.baselibrary.utils.stringutils import get_diacritic_variant
 
 """
 作者比率分布 大部分在 1和 2 
@@ -23,17 +26,6 @@ additional_chars = '‑–‐’·．—'
 extended_punctuation = string.punctuation + additional_chars
 
 
-def get_diacritic_variant(char1):
-    # 将字符转换为标准的 Unicode 形式
-    normalized_char1 = unicodedata.normalize('NFD', char1)
-
-    # 获取基本字符（去掉变音符号）
-    base_char1 = ''.join(c for c in normalized_char1 if unicodedata.category(c) != 'Mn')
-
-    # 判断基本字符是否相同
-    return base_char1
-
-
 def detect_other_languages(text):
     # 匹配所有非中文、非英文、非数字字符
     pattern = r'[^\u4E00-\u9FFFa-zA-Z0-9\s.,!?;:\'\"()‑\-–—‐’·˜．]'
@@ -127,6 +119,49 @@ def custom_rstrip(s):
     return s
 
 
+# 分割中文拼音，如"Xiaohong" ————> ['Xiao', 'hong']
+def chinese_pinyin_split_by_rules(input_str):
+    # 声母列表（含复合声母）
+    initials = {
+        'b', 'p', 'm', 'f', 'd', 't', 'n', 'l', 'g', 'k', 'h',
+        'j', 'q', 'x', 'zh', 'ch', 'sh', 'r', 'z', 'c', 's', 'y', 'w'
+    }
+    # 韵母列表（部分示例）
+    finals = {
+        'a', 'o', 'e', 'ai', 'ei', 'ao', 'ou', 'an', 'en', 'ang', 'eng', 'ong',
+        'i', 'ia', 'ie', 'iao', 'iu', 'ian', 'in', 'iang', 'ing', 'iong',
+        'u', 'ua', 'uo', 'uai', 'ui', 'uan', 'un', 'uang', 'ueng',
+        'v', 've', 'van', 'vn'
+    }
+    result = []
+    while input_str:
+        # 尝试匹配最长声母
+        max_initial_len = 2  # 最长声母如 'zh'
+        matched_initial = ""
+        for length in range(max_initial_len, 0, -1):
+            candidate = input_str[:length]
+            if candidate.lower() in initials:
+                matched_initial = candidate
+                break
+        # 切分声母后的剩余部分
+        remaining = input_str[len(matched_initial):]
+        # 匹配韵母
+        max_final_len = min(4, len(remaining))  # 最长韵母如 'iong'
+        matched_final = ""
+        for length in range(max_final_len, 0, -1):
+            candidate = remaining[:length]
+            if candidate.lower() in finals:
+                matched_final = candidate
+                break
+        if matched_final:
+            # 合并声母和韵母
+            syllable = matched_initial + matched_final
+            result.append(syllable)
+            input_str = input_str[len(syllable):]
+        else:
+            return []  # 无法切分
+    return result
+
 
 def AuthorRatio(
         s1,
@@ -151,8 +186,8 @@ def AuthorRatio(
         if not s1 or not s2:
             return 0
     # 处理音标问题
-    s1 = get_diacritic_variant(s1)
-    s2 = get_diacritic_variant(s2)
+    s1 = get_diacritic_variant(unidecode(s1))
+    s2 = get_diacritic_variant(unidecode(s2))
     # 这里提出来是为了少计算 但后期需要平衡内存和算力
     # 移除指定符号 这里做了小写化处理
     s1_punc = remove_punctuation(s1)
@@ -218,6 +253,10 @@ def AuthorRatio(
             # 如果两个字符忽略大小写后相同，继续比较下一个字符
             if i1.lower() == i2.lower():
                 continue
+            # 在作者中 有可能错误字母 当单词大于3 且只有一个字母错误或者位置交换时 可以认为这两个单词相同
+            # 样例 "De Gusmio, Ana Paula Henriques","De Gusmão, Ana Paula Henriques"
+            if len(i1) > 3 and damerau_levenshtein_distance(i1, i2) <= 1:
+                continue
 
             # 如果其中一个字符的长度为1（即是单个字母），检查它们的首字母是否匹配
             if len(i1) == 1 or len(i2) == 1:
@@ -260,7 +299,6 @@ def AuthorRatio(
     if len(sort_l1) == len(sort_l2) and (is_same_or_initials_match(sort_l1, sort_l2) or set(sort_l1) == set(sort_l2)):
         return 0.99
 
-
     ##############################################################
     # 以上为情况穷举情况，以下为其他情况的相似率计算
     ##############################################################

re_common-10.0.5/re_common/v2/baselibrary/utils/n_ary_expression_tree.py

@@ -0,0 +1,178 @@
+class Node:
+    def __init__(self, value, children=None):
+        self.value = value
+        self.children = children if children is not None else []
+
+    def __repr__(self):
+        return f"Node(value={self.value}, children={self.children})"
+
+
+def tokenize(expression):
+    """将表达式分解为标记（token），仅以 and, or, not 和括号作为分界符
+
+    Args:
+        expression (str): 输入的字符串表达式，例如 'a = 3 + (b > 2) and c'
+
+    Returns:
+        list: 分解后的标记列表，例如 ['a = 3 + (b > 2)', 'and', 'c']
+    """
+    tokens = []  # 存储最终的标记列表
+    current = ""  # 当前正在构建的标记字符串
+    i = 0  # 当前字符的索引
+    length = len(expression)  # 输入表达式的长度
+
+    def is_delimiter_match(expression, i, delimiter_len=3, delimiter="and"):
+        """检查当前位置是否匹配指定的分隔符（and, or, not）
+
+        Args:
+            expression (str): 输入的表达式字符串
+            i (int): 当前检查的起始索引
+            delimiter_len (int): 分隔符的长度，默认为 3（适用于 'and' 和 'not'）
+            delimiter (str): 要检查的分隔符，默认为 'and'
+
+        Returns:
+            bool: 如果当前位置匹配分隔符且前后有空格，返回 True，否则返回 False
+        """
+        # 检查索引是否超出范围
+        if not i + delimiter_len <= length:
+            return False
+        # 检查当前位置是否匹配指定分隔符（忽略大小写）
+        if not expression[i:i + delimiter_len].lower() == delimiter:
+            return False
+
+        # 检查分隔符前是否有一个空格（如果不是字符串开头）
+        if i - 1 >= 0:
+            if not expression[i - 1].lower() == ' ':
+                return False
+
+        # 检查分隔符后是否有一个空格（如果不是字符串结尾）
+        if i + delimiter_len + 1 <= length:
+            if not expression[i + delimiter_len].lower() == ' ':
+                return False
+        return True
+
+    # 遍历表达式的每个字符
+    while i < length:
+        char = expression[i]  # 当前处理的字符
+
+        # 处理括号
+        if char in "()":
+            if current.strip():  # 如果当前标记有内容，先将其添加到 tokens
+                tokens.append(current.strip())
+                current = ""  # 重置当前标记
+            tokens.append(char)  # 将括号作为独立标记添加
+            i += 1  # 移动到下一个字符
+            continue
+
+        # 检查是否遇到 and, or, not 分隔符
+        if is_delimiter_match(expression, i, delimiter_len=3, delimiter="and"):
+            if current.strip():  # 如果当前标记有内容，先添加
+                tokens.append(current.strip())
+                current = ""  # 重置当前标记
+            tokens.append("and")  # 添加 'and' 标记
+            i += 3  # 跳过 'and' 的长度
+            continue
+        elif is_delimiter_match(expression, i, delimiter_len=2, delimiter="or"):
+            if current.strip():  # 如果当前标记有内容，先添加
+                tokens.append(current.strip())
+                current = ""  # 重置当前标记
+            tokens.append("or")  # 添加 'or' 标记
+            i += 2  # 跳过 'or' 的长度
+            continue
+        elif is_delimiter_match(expression, i, delimiter_len=3, delimiter="not"):
+            if current.strip():  # 如果当前标记有内容，先添加
+                tokens.append(current.strip())
+                current = ""  # 重置当前标记
+            tokens.append("not")  # 添加 'not' 标记
+            i += 3  # 跳过 'not' 的长度
+            continue
+
+        # 将非分隔符字符追加到当前标记中，包括空格
+        current += char
+        i += 1  # 移动到下一个字符
+
+    # 处理最后一个标记（如果有内容）
+    if current.strip():
+        tokens.append(current.strip())
+
+    return tokens  # 返回标记列表
+
+
+
+
+def parse_expression(tokens):
+    """递归下降解析表达式"""
+
+    def parse_or(tokens, pos):
+        """解析 OR 级别（最低优先级）"""
+        left, pos = parse_and(tokens, pos)
+        while pos < len(tokens) and tokens[pos] == 'or':
+            pos += 1
+            if pos >= len(tokens):
+                raise ValueError("Incomplete expression after 'or'")
+            right, pos = parse_and(tokens, pos)
+            left = Node('or', [left, right])
+        return left, pos
+
+    def parse_and(tokens, pos):
+        """解析 AND 级别（次高优先级）"""
+        left, pos = parse_not(tokens, pos)
+        while pos < len(tokens) and tokens[pos] == 'and':
+            pos += 1
+            if pos >= len(tokens):
+                raise ValueError("Incomplete expression after 'and'")
+            right, pos = parse_not(tokens, pos)
+            left = Node('and', [left, right])
+        return left, pos
+
+    def parse_not(tokens, pos):
+        """解析 NOT 级别（最高优先级）"""
+        if pos < len(tokens) and tokens[pos] == 'not':
+            pos += 1
+            if pos >= len(tokens):
+                raise ValueError("Incomplete expression after 'not'")
+            child, pos = parse_primary(tokens, pos)
+            return Node('not', [child]), pos
+        return parse_primary(tokens, pos)
+
+    def parse_primary(tokens, pos):
+        """解析基本单元（条件或括号表达式）"""
+        if pos >= len(tokens):
+            raise ValueError("Unexpected end of expression")
+
+        if tokens[pos] == '(':
+            pos += 1
+            subtree, pos = parse_or(tokens, pos)
+            if pos >= len(tokens) or tokens[pos] != ')':
+                raise ValueError("Missing closing parenthesis")
+            return subtree, pos + 1
+        else:
+            # 假设这是一个条件（如 A=1）
+            return Node(tokens[pos]), pos + 1
+
+    # 从头开始解析
+    tree, pos = parse_or(tokens, 0)
+    if pos < len(tokens):
+        raise ValueError(f"Extra tokens after expression: {tokens[pos:]}")
+    return tree
+
+
+# 测试代码
+expressions = [
+    "not A=1 and B= 2",
+    "A=1 and (not B=2 or (C=3 or D=4))",
+    "A=1 and not (B=2 or C=3 and D=4 or E=5)",
+    "(A=1 and not (B=2 or C=3 or D=4))",
+    "A=1 and",  # 不完整表达式
+    "and A=1",  # 不完整表达式
+]
+
+for expr in expressions:
+    try:
+        print(f"\nExpression: {expr}")
+        tokens = tokenize(expr)
+        print("Tokens:", tokens)
+        tree = parse_expression(tokens)
+        print("Tree:", tree)
+    except ValueError as e:
+        print(f"Error: {e}")

{re_common-10.0.3 → re_common-10.0.5}/re_common/v2/baselibrary/utils/string_clear.py

@@ -1,7 +1,7 @@
 import re
 import regex
 
-from re_common.v2.baselibrary.utils.stringutils import qj2bj, bj2qj, get_diacritic_variant
+from re_common.v2.baselibrary.utils.stringutils import qj2bj, bj2qj, get_diacritic_variant, clean_html
 
 
 class StringClear(object):
@@ -67,7 +67,6 @@ class StringClear(object):
         return self
 
     def replace_dash_with_space(self):
-        # 横线换成空格 比 去除符号有时更有用
         self.obj_str = self.obj_str.replace("-", " ")
         return self
 
@@ -88,20 +87,9 @@ class StringClear(object):
 
     def remove_html_tag(self):
         import html
-        from parsel import Selector
 
         self.obj_str = html.unescape(self.obj_str)
 
-        def clean_html(html):
-            sel = Selector(text=html, type='html')
-            # 移除脚本和样式
-            sel.xpath('//script').remove()
-            sel.xpath('//style').remove()
-
-            # 提取文本
-            text_nodes = sel.xpath('//text()').getall()
-            return ''.join(t.strip() for t in text_nodes if t.strip())
-
         self.obj_str = clean_html(self.obj_str)
 
         return self
@@ -117,7 +105,6 @@ def rel_clear(str_obj):
             .to_str()  # 防止其他类型传入 比如 int double
             .qj_to_bj()  # 全角转半角
             .remove_html_tag()  # html标签清理
-            .replace_dash_with_space()  # 横线转空格 在 英文 title 中更有用
             .remove_special_chars()  # 移除特殊字符，仅保留字母、数字、空格和汉字 \w 已经包括所有 Unicode 字母 下划线 _ 会被保留
             .collapse_spaces()  # 移除多余空格,连续多个空格变一个
             .lower()  # 小写

{re_common-10.0.3 → re_common-10.0.5}/re_common/v2/baselibrary/utils/stringutils.py

@@ -1,7 +1,9 @@
 import re
+import threading
 
 import regex
 import unicodedata
+from html.parser import HTMLParser
 
 
 def bj2qj(src):
@@ -25,6 +27,7 @@ def bj2qj(src):
 
     return ''.join(buf)
 
+
 def qj2bj(src):
     """
     全角转半角
@@ -62,6 +65,7 @@ def get_diacritic_variant(char1):
     # 判断基本字符是否相同
     return base_char1
 
+
 def get_alphabetic_ratio(text: str) -> float:
     # 返回字母型字符所占比例
     if not text:
@@ -93,4 +97,50 @@ def get_alphabetic_ratio(text: str) -> float:
     alphabetic_chars = re.findall(alphabetic_pattern, clean_text)
 
     # 返回字母型字符所占比例
-    return len(alphabetic_chars) / len(clean_text)
+    return len(alphabetic_chars) / len(clean_text)
+
+
+class HTMLTextExtractor(HTMLParser):
+    _thread_local = threading.local()  # 线程局部存储
+
+    def __init__(self):
+        super().__init__()
+        self.reset_state()
+
+    def handle_starttag(self, tag, attrs):
+        if tag in ('script', 'style'):
+            self.skip = True
+
+    def handle_endtag(self, tag):
+        if tag in ('script', 'style'):
+            self.skip = False
+
+    def handle_data(self, data):
+        if not self.skip and data.strip():
+            self.text.append(data)
+
+    def reset_state(self):
+        self.reset()
+        self.text = []
+        self.skip = False
+
+    def get_text(self):
+        return ''.join(self.text).strip()
+
+    @classmethod
+    def get_parser(cls):
+        # 每个线程获取独立实例
+        if not hasattr(cls._thread_local, 'parser'):
+            cls._thread_local.parser = cls()
+        return cls._thread_local.parser
+
+
+def clean_html(html):
+    parser = HTMLTextExtractor.get_parser()
+    parser.reset_state()
+    parser.feed(html)
+    parser.close()
+    return parser.get_text()
+
+
+

{re_common-10.0.3 → re_common-10.0.5}/re_common.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: re_common
-Version: 10.0.3
+Version: 10.0.5
 Summary: a library about all python projects
 Home-page: https://gitee.com/xujiangios/re-common
 Author: vic

{re_common-10.0.3 → re_common-10.0.5}/re_common.egg-info/SOURCES.txt

@@ -184,6 +184,7 @@ re_common/v2/baselibrary/utils/author_smi.py
 re_common/v2/baselibrary/utils/basedict.py
 re_common/v2/baselibrary/utils/basehdfs.py
 re_common/v2/baselibrary/utils/json_cls.py
+re_common/v2/baselibrary/utils/n_ary_expression_tree.py
 re_common/v2/baselibrary/utils/string_bool.py
 re_common/v2/baselibrary/utils/string_clear.py
 re_common/v2/baselibrary/utils/stringutils.py

{re_common-10.0.3 → re_common-10.0.5}/setup.py

@@ -24,8 +24,7 @@ re_common
   # new 20250224
   password_new = pypi-AgEIcHlwaS5vcmcCJDJkMzJhNWZkLTBkOTktNGE4Yi1iN2Q5LWU0ZjNjMGNiNzhjYwACEVsxLFsicmUtY29tbW9uIl1dAAIsWzIsWyIwYWIzYjIzMy0zZTlmLTQ1ZjAtYWQ5OC03NjJhNDM5NjNiZmUiXV0AAAYg0Pu6BqqupvhX7ad69d8e6XS8BmnLav6x6D9HJ-U3gko
   
-  pypi-AgEIcHlwaS5vcmcCJDUzNWEzYmFkLTcyMTUtNDE5OS05MGY4LWZlNTZkMDMzM2U2MQACKlszLCJhMjk4YTg3ZS02YjU2LTQzOTMtYjA5YS1kMzEzMzhjZmNiMDYiXQAABiD00KOVAf9T8AyK7F9K1G9LECXl0MdYa3yyODA7ox5qDg
-  
+ pypi-AgEIcHlwaS5vcmcCJDBlMzc1OTU1LWVkOGItNDVjZS1iMGNhLTQwYmNmYmU3NmViMwACEVsxLFsicmUtY29tbW9uIl1dAAIsWzIsWyIwYWIzYjIzMy0zZTlmLTQ1ZjAtYWQ5OC03NjJhNDM5NjNiZmUiXV0AAAYgisva56cDT5J3a9EBb3paplusibupSRSfoZc_L8PUGDY
 python setup.py sdist bdist_wheel
 python -m twine upload dist/*
 
@@ -35,7 +34,7 @@ long_description = """
 """
 setuptools.setup(
     name="re_common",
-    version="10.0.3",
+    version="10.0.5",
     author="vic",
     author_email="xujiang5@163.com",
     description="a library about all python projects",