re-common 10.0.8__py3-none-any.whl → 10.0.10__py3-none-any.whl

re_common/v2/baselibrary/tools/dict_tools.py

@@ -2,6 +2,7 @@ class DotDict(dict):
     """
     让字典成为对象 既可以用字典方式访问 也可以用点访问key
     """
+
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         # 递归地将嵌套字典转换为 DotDict
@@ -21,4 +22,16 @@ class DotDict(dict):
     def __setattr__(self, key, value):
         if isinstance(value, dict):  # 如果值是字典，转换为 DotDict
             value = DotDict(value)
-        self[key] = value
+        self[key] = value
+
+    def to_dict(self):
+        """
+        将 DotDict 实例转换为普通字典
+        """
+        result = {}
+        for key, value in self.items():
+            if isinstance(value, DotDict):
+                result[key] = value.to_dict()
+            else:
+                result[key] = value
+        return result

re_common/v2/baselibrary/tools/search_hash_tools.py

@@ -1,7 +1,9 @@
 from typing import List
 
 import jieba
-from datasketch import MinHash
+from datasketch import MinHash, minhash
+
+from re_common.v2.baselibrary.utils.string_bool import is_single_cjk_char
 
 
 def tokenize(text: str, stopwords=None) -> List[str]:
@@ -11,8 +13,16 @@ def tokenize(text: str, stopwords=None) -> List[str]:
     if stopwords is None:
         stopwords = []
     words = jieba.lcut(text)
+
     # 统计单字符数据 长度，防止结巴分词分不了的单词 将数据分为单个字符
-    one_char_size = len([i for i in words if len(i) == 1])
+
+    # 这里为什么使用函数 而不是在推导式中兼容，主要是在一些 spark中 推导式的if 条件不遵循最短路径原则会将表达式当做一个整体算子
+    def is_singel_en(i):
+        if len(i) == 1 and not is_single_cjk_char(i):
+            return True
+        return False
+
+    one_char_size = len([i for i in words if is_singel_en(i)])
     all_size = len(words)
     # 如果单字符个数超过一定比例 就直接用空格分词
     if all_size != 0 and one_char_size / all_size > 0.6:
@@ -31,3 +41,13 @@ def create_minhash(words: List[str], num_perm=128) -> MinHash:
     for word in words:
         minhash.update(word.encode("utf-8"))
     return minhash
+
+
+def get_str_minhash(title):
+    from re_common.v2.baselibrary.utils.string_clear import rel_clear
+    rel_title = rel_clear(title)
+    if not rel_title:
+        return ""
+    words = tokenize(rel_title)
+    minhash = create_minhash(words)
+    return minhash

re_common/v2/baselibrary/utils/string_bool.py

@@ -10,17 +10,81 @@ def contains_chinese_chars(s):
 
 
 def is_empty(value):
+    """
+    判断一个值是否为空。
+
+    支持的类型：
+    - None
+    - 空字符串（去除空白后）
+    - pandas 的 NaN
+    - 其他可迭代类型（如列表、字典等）的长度为 0
+    - 其他情况返回 False
+    """
     # 如果是 None，直接返回 True
     if value is None:
         return True
 
+    # 尝试处理 pandas 的 NaN
+    try:
+        import pandas as pd
+        if pd.isna(value):
+            return True
+    except ImportError:
+        pass  # 如果没有安装 pandas，跳过
+
     # 如果是字符串，检查去除空白后是否为空
     if isinstance(value, str):
         return value.strip() == ""
 
-    # 可选：处理其他可迭代类型（如列表、字典等）
+    # 处理其他可迭代类型（如列表、字典等）
     if hasattr(value, "__len__"):
         return len(value) == 0
 
     # 默认情况下，非 None、非空类型返回 False
-    return False
+    return False
+
+
+class InvalidCharLengthError(Exception):
+    """自定义异常类，用于处理输入字符长度不为 1 的情况"""
+    pass
+
+
+def is_single_cjk_char(char):
+    """
+    判断单个字符是否为中日韩字符
+    :param char: 要判断的单个字符
+    :return: 如果是中日韩字符返回 True，否则返回 False
+    """
+    # 检查输入字符的长度
+    if len(char) != 1:
+        raise InvalidCharLengthError("输入的字符串长度必须为 1，请提供单个字符进行判断。")
+    code_point = ord(char)
+    # 中日韩统一表意文字
+    ranges = [
+        (0x4E00, 0x9FFF),  # CJK 统一表意符号
+        (0x3400, 0x4DBF),  # CJK 统一表意符号扩展 A
+        (0x20000, 0x2A6DF),  # CJK 统一表意符号扩展 B
+        (0x2A700, 0x2B73F),  # CJK 统一表意符号扩展 C
+        (0x2B740, 0x2B81F),  # CJK 统一表意符号扩展 D
+        (0x2B820, 0x2CEAF),  # CJK 统一表意符号扩展 E
+        (0x2CEB0, 0x2EBEF),  # CJK 统一表意符号扩展 F
+        (0x30000, 0x3134F),  # CJK 统一表意符号扩展 G
+        (0x31350, 0x323AF),  # CJK 统一表意符号扩展 H
+        (0x3300, 0x33FF),  # CJK 兼容符号
+        (0xFE30, 0xFE4F),  # CJK 兼容形式
+        (0xF900, 0xFAFF),  # CJK 兼容表意符号
+        (0x2F800, 0x2FA1F),  # CJK 兼容表意符号补充
+        (0x3105, 0x3129),  # 注音字母
+        (0x31A0, 0x31BF),  # 注音字母扩展
+        (0x3040, 0x309F),  # 平假名
+        (0x30A0, 0x30FF),  # 片假名
+        (0x31F0, 0x31FF),  # 片假名扩展
+        (0xAC00, 0xD7AF),  # 韩文音节
+        (0x1100, 0x11FF),  # 韩文字母
+        (0xA960, 0xA97F),  # 韩文字母扩展 A
+        (0xD7B0, 0xD7FF),  # 韩文字母扩展 B
+    ]
+    for start, end in ranges:
+        if start <= code_point <= end:
+            return True
+    return False

re_common/v2/baselibrary/utils/string_clear.py

@@ -165,10 +165,11 @@ def ref_clear(str_obj):
 def clear_obj(str_obj):
     # 为对象化定制的清理
     str_obj = clear_au_organ(str_obj)
-    str_obj = str_obj.replace("ß", "SS")
-    return (StringClear(str_obj)
-            .remove_diacritics()  # 清理音标
-            .upper()
-            .get_str()  # 获取str
-            .strip()  # 去掉空格
-            )
+    # str_obj = str_obj.replace("ß", "SS") # "ß" 的 大写就是 "SS"
+    result = (StringClear(str_obj)
+              .remove_diacritics()  # 清理音标
+              .upper()
+              .get_str()  # 获取str
+              .strip()  # 去掉空格
+              )
+    return result

{re_common-10.0.8.dist-info → re_common-10.0.10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: re_common
-Version: 10.0.8
+Version: 10.0.10
 Summary: a library about all python projects
 Home-page: https://gitee.com/xujiangios/re-common
 Author: vic

{re_common-10.0.8.dist-info → re_common-10.0.10.dist-info}/RECORD

@@ -168,10 +168,10 @@ re_common/v2/baselibrary/s3object/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeR
 re_common/v2/baselibrary/s3object/baseboto3.py,sha256=mXuIFx99pnrPGQ4LJCZwlN1HLbaU-OWLwck0cVzW6hc,11203
 re_common/v2/baselibrary/tools/WeChatRobot.py,sha256=EaQgNncROAhU5-psYRGWAshIV5aEw-p2u1kYLpvr7RA,2796
 re_common/v2/baselibrary/tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-re_common/v2/baselibrary/tools/dict_tools.py,sha256=HW-YZOUhv5GMzFsF-ArLfDoszui1K3_M7IiRIe4VEXA,909
+re_common/v2/baselibrary/tools/dict_tools.py,sha256=BTh7oJuJ619IZgxiYlim0ltrXBclDtb7WzyFGr7wVf0,1246
 re_common/v2/baselibrary/tools/dolphinscheduler.py,sha256=1m7UGYDiuvJUCI6ik6CGM2fO8U5XteJzn55VRbwB9ts,7978
 re_common/v2/baselibrary/tools/list_tools.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-re_common/v2/baselibrary/tools/search_hash_tools.py,sha256=o-PNjmcYDkfyiR75Jci_9sSn4cGi_F9jPCIrwYdnb1U,1013
+re_common/v2/baselibrary/tools/search_hash_tools.py,sha256=d_h9j7VxiXpcn1GHZ7L2tpx9_LDQshcl58tlKvSxZPg,1691
 re_common/v2/baselibrary/tools/text_matcher.py,sha256=F4WtLO-b7H6V9TIvOntCD9ZXSQP_KijPuLLYcLPtrKQ,7021
 re_common/v2/baselibrary/tools/unionfind_tools.py,sha256=VYHZZPXwBYljsm7TjV1B6iCgDn3O3btzNf9hMvQySVU,2965
 re_common/v2/baselibrary/utils/BusinessStringUtil.py,sha256=tzjVr_-6iPAKTt14hR-BhRshdRgeT_MPJpUQkxcTXns,4084
@@ -181,8 +181,8 @@ re_common/v2/baselibrary/utils/basedict.py,sha256=tSV85pARe8ZQDY77_h_heS81EWwcgJ
 re_common/v2/baselibrary/utils/basehdfs.py,sha256=NVV5Q0OMPlM_zTrs9ZDoPJv29GQv5wi9-AP1us5dBrQ,4651
 re_common/v2/baselibrary/utils/json_cls.py,sha256=dHOkWafG9lbQDoub9cbDwT2fDjMKtblQnjFLeA4hECA,286
 re_common/v2/baselibrary/utils/n_ary_expression_tree.py,sha256=-05kO6G2Rth7CEK-5lfFrthFZ1Q0-0a7cni7mWZ-2gg,9172
-re_common/v2/baselibrary/utils/string_bool.py,sha256=4VCr1g8pX5YnzZSKctQgQfmhSQ0aw7a8ruhWdiRmBFU,641
-re_common/v2/baselibrary/utils/string_clear.py,sha256=g_2s2C4yY0C5AvuANjn02g7e_VM_uNY1lxoQg5HtLrk,5799
+re_common/v2/baselibrary/utils/string_bool.py,sha256=EJnkSck4ofcIeJ6nLzAOVtlt6o1WBgvgVwIqJKj5Suc,2993
+re_common/v2/baselibrary/utils/string_clear.py,sha256=LDIf-3Czq1sXp-54aifXdXbdGUX7hpFBKqQa5Azj_lo,5861
 re_common/v2/baselibrary/utils/stringutils.py,sha256=GLXHAm8IulC_8hWrN2aiFQjsoOpjczvcVozmTJj86-A,3864
 re_common/vip/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 re_common/vip/base_step_process.py,sha256=VXXiNj0I5CpzXIMCgOPU86bzDJkSBkUS-9CpZIl_GOk,205
@@ -210,8 +210,8 @@ re_common/vip/title/transform/TransformRegulationTitleToZt.py,sha256=LKRdIsWKues
 re_common/vip/title/transform/TransformStandardTitleToZt.py,sha256=-fCKAbSBzXVyQDCE61CalvR9E_QzQMA08QOO_NePFNI,5563
 re_common/vip/title/transform/TransformThesisTitleToZt.py,sha256=QS-uV0cQrpUFAcKucuJQ9Ue2VRQH-inmfn_X3IplfRo,5488
 re_common/vip/title/transform/__init__.py,sha256=m83-CWyRq_VHPYHaALEQlmXrkTdrZ3e4B_kCfBYE-uc,239
-re_common-10.0.8.dist-info/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
-re_common-10.0.8.dist-info/METADATA,sha256=687IQ2myx3vDwQca9JMzKTf8KHCR6qSst65ykG1VZ9Y,581
-re_common-10.0.8.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-re_common-10.0.8.dist-info/top_level.txt,sha256=_H9H23zoLIalm1AIY_KYTVh_H0ZnmjxQIxsvXtLv45o,10
-re_common-10.0.8.dist-info/RECORD,,
+re_common-10.0.10.dist-info/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
+re_common-10.0.10.dist-info/METADATA,sha256=mOarqqiMSzMjAcu1sV0OxUGdwfANLray_3ZpjkAPxFg,582
+re_common-10.0.10.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+re_common-10.0.10.dist-info/top_level.txt,sha256=_H9H23zoLIalm1AIY_KYTVh_H0ZnmjxQIxsvXtLv45o,10
+re_common-10.0.10.dist-info/RECORD,,