re-common 10.0.27__py3-none-any.whl → 10.0.28__py3-none-any.whl

re_common/v2/baselibrary/business_utils/baseencodeid.py

@@ -76,11 +76,12 @@ class BaseLngid(object):
 
         return lngid
 
-    def getDoiid(self, doi):
-        doi_upper = doi.upper()
-        limited_id = self.BaseEncodeID(doi_upper)
+    def getDoiid(self, doi, case_insensitive=False):
+        if case_insensitive is False:
+            doi = doi.upper()
+        limited_id = self.BaseEncodeID(doi)
         if len(limited_id) > 240:
-            limited_id = hashlib.md5(doi_upper.encode('utf8')).hexdigest().upper()
+            limited_id = hashlib.md5(doi.encode('utf8')).hexdigest().upper()
         return limited_id
 
     def GetRawid(self, limited_id, case_insensitive=False):
@@ -97,4 +98,3 @@ class BaseLngid(object):
             raise Exception("长度超过20，不可逆")
 
         return uppercase_rawid
-

re_common/v2/baselibrary/utils/author_smi.py

@@ -185,6 +185,7 @@ def AuthorRatio(
         # 处理后是否为空字符串，如果有 返回0
         if not s1 or not s2:
             return 0
+    # get_diacritic_variant(unidecode(strs)) 更激进，会丢失非拉丁字符和原文信息，适合需要把多语言文本转换成 ASCII 拼音的场景。
     # 处理音标问题
     s1 = get_diacritic_variant(unidecode(s1))
     s2 = get_diacritic_variant(unidecode(s2))

re_common/v2/baselibrary/utils/string_clear.py

@@ -9,7 +9,7 @@ from re_common.v2.baselibrary.utils.stringutils import (
     bj2qj,
     get_diacritic_variant,
     clean_html,
-    remove_spaces_between_chinese_characters, clean_unicode_alnum,
+    remove_spaces_between_chinese_characters, clean_unicode_alnum, normalize_nfkc,
 )
 
 
@@ -112,10 +112,18 @@ class StringClear(object):
         return self
 
     def remove_diacritics(self):
+        """
+        和 clear_nkfc的关键区别 不去除连字
+        """
         # 去除音标 转换成字母
         self.obj_str = get_diacritic_variant(self.obj_str)
         return self
 
+    def clear_nkfc(self):
+        self.obj_str = normalize_nfkc(self.obj_str)
+        return self
+
+
     def remove_brackets(self):
         # 移除 方括号里面的内容
         self.obj_str = re.sub("\\[.*?]", "", self.obj_str)
@@ -148,6 +156,12 @@ class StringClear(object):
         self.obj_str = unquote(self.obj_str)
         return self
 
+    def ascii_text(self):
+        # 只保留 ASCII 范围内的可见字符：空格(32) 到 ~ (126)
+        self.obj_str = ''.join(c for c in self.obj_str if 32 <= ord(c) <= 126)
+        return self
+
+
     def get_str(self):
         return self.obj_str
 

re_common/v2/baselibrary/utils/stringutils.py

@@ -6,6 +6,7 @@ import regex
 import unicodedata
 from html.parser import HTMLParser
 
+from unidecode import unidecode
 
 from re_common.v2.baselibrary.utils.string_smi import JaroDamerauLevenshteinMaxSim
 
@@ -59,7 +60,27 @@ def qj2bj(src):
     return ''.join(buf)
 
 
+"""
+总结对比表
+规范名	处理步骤	组合方式	兼容性归一化	主要用途
+NFC	规范分解 → 规范组合	组合	否	保留预组合字符，文本呈现和存储
+NFD	规范分解	不组合	否	拆解字符，便于逐字符处理
+NFKC	兼容性分解 → 规范组合	组合	是	消除兼容差异，文本比较和索引
+NFKD	兼容性分解 → 规范分解	不组合	是	最大程度拆解，文本分析和预处理
+"""
+
+
 def get_diacritic_variant(char1):
+    """
+    NFD: 规范分解（Normalization Form D）
+    把字符拆分为基本字符 + 变音符号
+
+    但不处理兼容字符（如连字）
+
+    print(unicodedata.normalize('NFD', 'é'))  # 输出: 'é'（e + 组合符号） # 这里看起来是1个字符 len 其实是2
+    print(unicodedata.normalize('NFD', 'fl'))  # 输出: 'fl'（不变化）
+
+    """
     # 将字符转换为标准的 Unicode 形式
     normalized_char1 = unicodedata.normalize('NFD', char1)
 
@@ -70,6 +91,19 @@ def get_diacritic_variant(char1):
     return base_char1
 
 
+def normalize_nfkc(strs: str) -> str:
+    """
+    NFKC: 兼容字符归一化 + 组合（Normalization Form Compatibility Composition）
+    把 连字、圈数字、全角字符 等兼容字符转换为标准形式
+
+    同时做字符合并（例如 é 不再是 e+´，而是一个字符）
+    print(unicodedata.normalize('NFKC', 'fl'))   # 输出: 'fl'
+    print(unicodedata.normalize('NFKC', '①'))   # 输出: '1'
+    print(unicodedata.normalize('NFKC', 'Ａ'))  # 输出: 'A'
+    """
+    return unicodedata.normalize('NFKC', strs.strip())
+
+
 def get_alphabetic_ratio(text: str) -> float:
     # 返回字母型字符所占比例
     if not text:
@@ -158,9 +192,9 @@ def remove_spaces_between_chinese_characters(text):
     return re.sub(pattern, '', text)
 
 
-
 sim_utils = JaroDamerauLevenshteinMaxSim()
 
+
 def group_similar_texts(texts, threshold=0.9):
     """根据相似度对文本进行分组"""
     from re_common.v2.baselibrary.utils.string_clear import rel_clear

{re_common-10.0.27.dist-info → re_common-10.0.28.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: re_common
-Version: 10.0.27
+Version: 10.0.28
 Summary: a library about all python projects
 Home-page: https://gitee.com/xujiangios/re-common
 Author: vic

{re_common-10.0.27.dist-info → re_common-10.0.28.dist-info}/RECORD

@@ -165,7 +165,7 @@ re_common/v2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 re_common/v2/baselibrary/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 re_common/v2/baselibrary/business_utils/BusinessStringUtil.py,sha256=njPcRgeBWpnZr5u2cPAO4qdWBq-CgTn99rJuvWFcChk,6788
 re_common/v2/baselibrary/business_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-re_common/v2/baselibrary/business_utils/baseencodeid.py,sha256=J2Gs_F3-aAwIYjO06T1IIKHE6jen3mFfQVoVW3HywyI,3548
+re_common/v2/baselibrary/business_utils/baseencodeid.py,sha256=3f52e0jtgCFzPEyReia8TupwiE64t_VyBT-a7uQCXAY,3595
 re_common/v2/baselibrary/business_utils/full_doi_path.py,sha256=PaMIrgDWWt_fzSFyvvDD-8CcYZJTNo6Pj-uR0WafNbY,3319
 re_common/v2/baselibrary/business_utils/rel_tools.py,sha256=LfnGFCkUSxg1SHvOMOQdP1PiHxIKqk7Syuk5YYpjJag,295
 re_common/v2/baselibrary/decorators/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -191,7 +191,7 @@ re_common/v2/baselibrary/tools/data_processer/data_writer.py,sha256=OgKZ06zRJYNx
 re_common/v2/baselibrary/utils/BusinessStringUtil.py,sha256=njPcRgeBWpnZr5u2cPAO4qdWBq-CgTn99rJuvWFcChk,6788
 re_common/v2/baselibrary/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 re_common/v2/baselibrary/utils/api_net_utils.py,sha256=22q3WMWiKVg1IVGr4y2D1JrjhnbQtlChRDJm2S8rGlc,9868
-re_common/v2/baselibrary/utils/author_smi.py,sha256=1ebH3AHv19jtJWdlqNdwu6t58HNVLCotuCB6ff1SWiw,13666
+re_common/v2/baselibrary/utils/author_smi.py,sha256=Fh276u7za-GF_tK9xpBFYF5q1E3_tX22ZouWC8U7w8o,13831
 re_common/v2/baselibrary/utils/base_string_similarity.py,sha256=a40a79ttwoG_gC_hxMNB-sMXXecgICoRDWrj0DW8iEE,7749
 re_common/v2/baselibrary/utils/basedict.py,sha256=sH3_RZ8u4649-jX2V1uKNNkjJVUijZBDp6SdqncOZ88,1583
 re_common/v2/baselibrary/utils/basehdfs.py,sha256=TPwFct_-UrmO1KCbo4gpV77rsnlCQDumNBbQKL0ZI9o,5953
@@ -202,9 +202,9 @@ re_common/v2/baselibrary/utils/json_cls.py,sha256=M93piYtmgm_wP8E57culTrd_AhHLoG
 re_common/v2/baselibrary/utils/mq.py,sha256=UHpO8iNIHs91Tgp-BgnSUpZwjWquxrGLdpr3FMMv2zw,2858
 re_common/v2/baselibrary/utils/n_ary_expression_tree.py,sha256=-05kO6G2Rth7CEK-5lfFrthFZ1Q0-0a7cni7mWZ-2gg,9172
 re_common/v2/baselibrary/utils/string_bool.py,sha256=vxnjSFOfuHWGxkqaIbUNn21opx5tfV1uCXSahFfp1mU,6197
-re_common/v2/baselibrary/utils/string_clear.py,sha256=ywYR1KrKQyeM-zJgvTmORlfgbLdRSjWWKPe7K8oRx_8,7450
+re_common/v2/baselibrary/utils/string_clear.py,sha256=Ympa0Cs2y_72QeeyMS8de8y_QgtEFJJQ0AgHnylbMUc,7861
 re_common/v2/baselibrary/utils/string_smi.py,sha256=cU0WAWHRGnGoVQx3eCEKeM_q_olFNzRTJe7rSe586SY,741
-re_common/v2/baselibrary/utils/stringutils.py,sha256=TI6fw3km1l25ufXrnG6ha8dSBDtRh-MF4nWRt9u8Xbo,6452
+re_common/v2/baselibrary/utils/stringutils.py,sha256=eeuQYgXkWJ9apvyrYPcCCU3biTY9nD1KHos4_1ESNJE,7883
 re_common/vip/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 re_common/vip/base_step_process.py,sha256=VXXiNj0I5CpzXIMCgOPU86bzDJkSBkUS-9CpZIl_GOk,205
 re_common/vip/baseencodeid.py,sha256=nERoe89ueFM52bG7xwJdflcZHk6T2RQQKbc5uUZc3RM,3272
@@ -231,8 +231,8 @@ re_common/vip/title/transform/TransformRegulationTitleToZt.py,sha256=LKRdIsWKues
 re_common/vip/title/transform/TransformStandardTitleToZt.py,sha256=-fCKAbSBzXVyQDCE61CalvR9E_QzQMA08QOO_NePFNI,5563
 re_common/vip/title/transform/TransformThesisTitleToZt.py,sha256=QS-uV0cQrpUFAcKucuJQ9Ue2VRQH-inmfn_X3IplfRo,5488
 re_common/vip/title/transform/__init__.py,sha256=m83-CWyRq_VHPYHaALEQlmXrkTdrZ3e4B_kCfBYE-uc,239
-re_common-10.0.27.dist-info/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
-re_common-10.0.27.dist-info/METADATA,sha256=P-xatKifINopvPpQFudSbxCKEpgqzLyw5Pesn-7_VvU,582
-re_common-10.0.27.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-re_common-10.0.27.dist-info/top_level.txt,sha256=_H9H23zoLIalm1AIY_KYTVh_H0ZnmjxQIxsvXtLv45o,10
-re_common-10.0.27.dist-info/RECORD,,
+re_common-10.0.28.dist-info/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
+re_common-10.0.28.dist-info/METADATA,sha256=6LyRvl5fLSmKd4qNyZFc72DkO8_hyJ6FQ27dba4PEvc,582
+re_common-10.0.28.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+re_common-10.0.28.dist-info/top_level.txt,sha256=_H9H23zoLIalm1AIY_KYTVh_H0ZnmjxQIxsvXtLv45o,10
+re_common-10.0.28.dist-info/RECORD,,