re-common 10.0.26__py3-none-any.whl → 10.0.28__py3-none-any.whl

re_common/v2/baselibrary/business_utils/baseencodeid.py

@@ -0,0 +1,100 @@
+import base64
+import hashlib
+
+"""
+VIP编码lngid生成
+"""
+
+
+class BaseLngid(object):
+    def __int__(self):
+        pass
+
+    def BaseEncodeID(self, strRaw):
+        r""" 自定义base编码 """
+
+        strEncode = base64.b32encode(strRaw.encode('utf8')).decode('utf8')
+
+        if strEncode.endswith('======'):
+            strEncode = '%s%s' % (strEncode[0:-6], '0')
+        elif strEncode.endswith('===='):
+            strEncode = '%s%s' % (strEncode[0:-4], '1')
+        elif strEncode.endswith('==='):
+            strEncode = '%s%s' % (strEncode[0:-3], '8')
+        elif strEncode.endswith('='):
+            strEncode = '%s%s' % (strEncode[0:-1], '9')
+
+        table = str.maketrans('0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ', 'ZYXWVUTSRQPONMLKJIHGFEDCBA9876543210')
+        strEncode = strEncode.translate(table)
+
+        return strEncode
+
+    def BaseDecodeID(self, strEncode):
+        r""" 自定义base解码 """
+
+        table = str.maketrans('ZYXWVUTSRQPONMLKJIHGFEDCBA9876543210', '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ')
+        strEncode = strEncode.translate(table)
+
+        if strEncode.endswith('0'):
+            strEncode = '%s%s' % (strEncode[0:-1], '======')
+        elif strEncode.endswith('1'):
+            strEncode = '%s%s' % (strEncode[0:-1], '====')
+        elif strEncode.endswith('8'):
+            strEncode = '%s%s' % (strEncode[0:-1], '===')
+        elif strEncode.endswith('9'):
+            strEncode = '%s%s' % (strEncode[0:-1], '=')
+
+        strRaw = base64.b32decode(strEncode.encode('utf8')).decode('utf8')
+
+        return strRaw
+
+    def GetLngid(self, sub_db_id, rawid, case_insensitive=False):
+        """
+        :param sub_db_id:
+        :param rawid:
+        由 sub_db_id 和 rawid 得到 lngid。
+        :param case_insensitive: 标识源网站的 rawid 是否区分大小写
+        :return: lngid
+        """
+        uppercase_rawid = ''  # 大写版 rawid
+        if case_insensitive:  # 源网站的 rawid 区分大小写
+            for ch in rawid:
+                if ch.upper() == ch:
+                    uppercase_rawid += ch
+                else:
+                    uppercase_rawid += ch.upper() + '_'
+        else:
+            uppercase_rawid = rawid.upper()
+
+        limited_id = uppercase_rawid  # 限长ID
+        if len(uppercase_rawid) > 20:
+            limited_id = hashlib.md5(uppercase_rawid.encode('utf8')).hexdigest().upper()
+        else:
+            limited_id = self.BaseEncodeID(uppercase_rawid)
+
+        lngid = sub_db_id + limited_id
+
+        return lngid
+
+    def getDoiid(self, doi, case_insensitive=False):
+        if case_insensitive is False:
+            doi = doi.upper()
+        limited_id = self.BaseEncodeID(doi)
+        if len(limited_id) > 240:
+            limited_id = hashlib.md5(doi.encode('utf8')).hexdigest().upper()
+        return limited_id
+
+    def GetRawid(self, limited_id, case_insensitive=False):
+        try:
+            uppercase_rawid = self.BaseDecodeID(limited_id)
+            if case_insensitive:
+                str_ = "_"
+                uppercase_rawid_list = list(uppercase_rawid)
+                for num, li in enumerate(uppercase_rawid_list):
+                    if li == str_:
+                        old_str = "".join(uppercase_rawid_list[num - 1:num + 1])
+                        uppercase_rawid = uppercase_rawid.replace(old_str, uppercase_rawid_list[num - 1].lower())
+        except Exception as e:
+            raise Exception("长度超过20，不可逆")
+
+        return uppercase_rawid

re_common/v2/baselibrary/business_utils/full_doi_path.py

@@ -0,0 +1,116 @@
+import base64
+import hashlib
+import os
+
+from re_common.v2.baselibrary.business_utils.baseencodeid import BaseLngid
+
+import os
+import base64
+import hashlib
+
+"""
+DOI-文件路径 转换工具
+
+设计目标：
+1. 将任意DOI字符串转换为可逆、稳定的文件路径
+2. 提供高效的目录分散方案（65,536个子目录）
+3. 支持带文件扩展名的存储
+4. 完全可逆转换
+
+工作原理：
+1. DOI编码：
+   - 使用URL安全的Base64编码（RFC 3548）
+   - 移除Base64填充的'='字符
+   - 文件名长度 ≈ 原始DOI长度 × 4/3
+
+2. 目录分散：
+   - 使用MD5哈希创建两级目录结构
+   - 目录层级：/MD5[0:2]/MD5[2:4]/
+   - 支持65,536个目录（256×256），每目录约1,525个文件（假设10亿文件）
+
+3. 扩展名处理：
+   - 保持原始扩展名不变
+   - 解码时自动忽略扩展名
+
+典型转换示例：
+DOI: "10.1000/xyz123" -> 路径: "a1/b2/QTMuMTAwMC94eXoxMjM.pdf"
+路径: "a1/b2/QTMuMTAwMC94eXoxMjM.pdf" -> DOI: "10.1000/xyz123"
+"""
+
+base_lngid = BaseLngid()
+
+
+# 以后需要启用
+def doi_to_path(doi: str, ext: str = "") -> str:
+    """
+    将 DOI 转换为可逆的存储路径：
+    1. 对 DOI 进行 URL 安全的 Base64 编码（可逆）
+    2. 生成 DOI 的 MD5 哈希用于目录分散
+    3. 目录结构：MD5前2字符/次2字符/
+    4. 文件名：Base64编码的DOI + 扩展名
+
+    Args:
+        doi: 文件 DOI 标识符
+        ext: 文件扩展名（如 '.pdf'）
+
+    Returns:
+        相对文件路径（如 'a1/b2/QTMuMTAwMC94eXoxMjM=.pdf'）
+    """
+    # URL安全的Base64编码（可逆）
+    doi_b64 = base64.urlsafe_b64encode(doi.encode("utf-8")).decode("ascii").rstrip("=")
+
+    # 生成MD5哈希用于目录分配
+    hash_md5 = hashlib.md5(doi.encode("utf-8")).hexdigest()
+    dir_level1 = hash_md5[0:2]
+    dir_level2 = hash_md5[2:4]
+
+    return os.path.join(dir_level1, dir_level2, f"{doi_b64}{ext}")
+
+
+# 以后需要启用
+def path_to_doi(path: str) -> str:
+    """
+    从文件路径反推原始DOI
+    Args:
+        path: 文件路径（如 'a1/b2/QTMuMTAwMC94eXoxMjM=.pdf'）
+
+    Returns:
+        原始DOI字符串
+    """
+    # 提取文件名并移除扩展名
+    filename = os.path.basename(path)
+    base_name = os.path.splitext(filename)[0]
+
+    # 补齐Base64填充字符
+    padding = 4 - (len(base_name) % 4)
+    if padding != 4:  # 不需要补齐
+        base_name += "=" * padding
+
+    # Base64解码还原DOI
+    return base64.urlsafe_b64decode(base_name.encode("ascii")).decode("utf-8")
+
+
+def doi_to_dir(doi):
+    """生成文件的存储路径和可解码的文件名
+
+    Args:
+        doi (str): 文件的唯一DOI标识
+
+    Returns:
+        str: 文件相对路径，如 "ab/cd/Base64EncodedFileName"
+    """
+    # 计算DOI的MD5哈希
+    hash_md5 = hashlib.md5(doi.encode('utf-8')).hexdigest().lower()
+
+    # 提取目录层级：前2位作为一级目录，3-4位作为二级目录
+    first_dir = hash_md5[0:2].upper()
+    second_dir = hash_md5[2:4].upper()
+
+    return first_dir + "/" + second_dir
+
+
+def get_doi_path(doi):
+    # 目前使用
+    dir_path = doi_to_dir(doi)
+    file_name = base_lngid.getDoiid(doi) + ".pdf"
+    return dir_path + "/" + file_name

re_common/v2/baselibrary/utils/author_smi.py

@@ -185,6 +185,7 @@ def AuthorRatio(
         # 处理后是否为空字符串，如果有 返回0
         if not s1 or not s2:
             return 0
+    # get_diacritic_variant(unidecode(strs)) 更激进，会丢失非拉丁字符和原文信息，适合需要把多语言文本转换成 ASCII 拼音的场景。
     # 处理音标问题
     s1 = get_diacritic_variant(unidecode(s1))
     s2 = get_diacritic_variant(unidecode(s2))

re_common/v2/baselibrary/utils/string_clear.py

@@ -9,7 +9,7 @@ from re_common.v2.baselibrary.utils.stringutils import (
     bj2qj,
     get_diacritic_variant,
     clean_html,
-    remove_spaces_between_chinese_characters, clean_unicode_alnum,
+    remove_spaces_between_chinese_characters, clean_unicode_alnum, normalize_nfkc,
 )
 
 
@@ -112,10 +112,18 @@ class StringClear(object):
         return self
 
     def remove_diacritics(self):
+        """
+        和 clear_nkfc的关键区别 不去除连字
+        """
         # 去除音标 转换成字母
         self.obj_str = get_diacritic_variant(self.obj_str)
         return self
 
+    def clear_nkfc(self):
+        self.obj_str = normalize_nfkc(self.obj_str)
+        return self
+
+
     def remove_brackets(self):
         # 移除 方括号里面的内容
         self.obj_str = re.sub("\\[.*?]", "", self.obj_str)
@@ -148,6 +156,12 @@ class StringClear(object):
         self.obj_str = unquote(self.obj_str)
         return self
 
+    def ascii_text(self):
+        # 只保留 ASCII 范围内的可见字符：空格(32) 到 ~ (126)
+        self.obj_str = ''.join(c for c in self.obj_str if 32 <= ord(c) <= 126)
+        return self
+
+
     def get_str(self):
         return self.obj_str
 

re_common/v2/baselibrary/utils/stringutils.py

@@ -6,6 +6,7 @@ import regex
 import unicodedata
 from html.parser import HTMLParser
 
+from unidecode import unidecode
 
 from re_common.v2.baselibrary.utils.string_smi import JaroDamerauLevenshteinMaxSim
 
@@ -59,7 +60,27 @@ def qj2bj(src):
     return ''.join(buf)
 
 
+"""
+总结对比表
+规范名	处理步骤	组合方式	兼容性归一化	主要用途
+NFC	规范分解 → 规范组合	组合	否	保留预组合字符，文本呈现和存储
+NFD	规范分解	不组合	否	拆解字符，便于逐字符处理
+NFKC	兼容性分解 → 规范组合	组合	是	消除兼容差异，文本比较和索引
+NFKD	兼容性分解 → 规范分解	不组合	是	最大程度拆解，文本分析和预处理
+"""
+
+
 def get_diacritic_variant(char1):
+    """
+    NFD: 规范分解（Normalization Form D）
+    把字符拆分为基本字符 + 变音符号
+
+    但不处理兼容字符（如连字）
+
+    print(unicodedata.normalize('NFD', 'é'))  # 输出: 'é'（e + 组合符号） # 这里看起来是1个字符 len 其实是2
+    print(unicodedata.normalize('NFD', 'fl'))  # 输出: 'fl'（不变化）
+
+    """
     # 将字符转换为标准的 Unicode 形式
     normalized_char1 = unicodedata.normalize('NFD', char1)
 
@@ -70,6 +91,19 @@ def get_diacritic_variant(char1):
     return base_char1
 
 
+def normalize_nfkc(strs: str) -> str:
+    """
+    NFKC: 兼容字符归一化 + 组合（Normalization Form Compatibility Composition）
+    把 连字、圈数字、全角字符 等兼容字符转换为标准形式
+
+    同时做字符合并（例如 é 不再是 e+´，而是一个字符）
+    print(unicodedata.normalize('NFKC', 'fl'))   # 输出: 'fl'
+    print(unicodedata.normalize('NFKC', '①'))   # 输出: '1'
+    print(unicodedata.normalize('NFKC', 'Ａ'))  # 输出: 'A'
+    """
+    return unicodedata.normalize('NFKC', strs.strip())
+
+
 def get_alphabetic_ratio(text: str) -> float:
     # 返回字母型字符所占比例
     if not text:
@@ -158,9 +192,9 @@ def remove_spaces_between_chinese_characters(text):
     return re.sub(pattern, '', text)
 
 
-
 sim_utils = JaroDamerauLevenshteinMaxSim()
 
+
 def group_similar_texts(texts, threshold=0.9):
     """根据相似度对文本进行分组"""
     from re_common.v2.baselibrary.utils.string_clear import rel_clear

{re_common-10.0.26.dist-info → re_common-10.0.28.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: re_common
-Version: 10.0.26
+Version: 10.0.28
 Summary: a library about all python projects
 Home-page: https://gitee.com/xujiangios/re-common
 Author: vic

{re_common-10.0.26.dist-info → re_common-10.0.28.dist-info}/RECORD

@@ -165,6 +165,8 @@ re_common/v2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 re_common/v2/baselibrary/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 re_common/v2/baselibrary/business_utils/BusinessStringUtil.py,sha256=njPcRgeBWpnZr5u2cPAO4qdWBq-CgTn99rJuvWFcChk,6788
 re_common/v2/baselibrary/business_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+re_common/v2/baselibrary/business_utils/baseencodeid.py,sha256=3f52e0jtgCFzPEyReia8TupwiE64t_VyBT-a7uQCXAY,3595
+re_common/v2/baselibrary/business_utils/full_doi_path.py,sha256=PaMIrgDWWt_fzSFyvvDD-8CcYZJTNo6Pj-uR0WafNbY,3319
 re_common/v2/baselibrary/business_utils/rel_tools.py,sha256=LfnGFCkUSxg1SHvOMOQdP1PiHxIKqk7Syuk5YYpjJag,295
 re_common/v2/baselibrary/decorators/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 re_common/v2/baselibrary/decorators/utils.py,sha256=Q4D6KKCQxvNBXZkPQQn14keKKJpGtg8TUSakjJU40s0,2056
@@ -189,7 +191,7 @@ re_common/v2/baselibrary/tools/data_processer/data_writer.py,sha256=OgKZ06zRJYNx
 re_common/v2/baselibrary/utils/BusinessStringUtil.py,sha256=njPcRgeBWpnZr5u2cPAO4qdWBq-CgTn99rJuvWFcChk,6788
 re_common/v2/baselibrary/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 re_common/v2/baselibrary/utils/api_net_utils.py,sha256=22q3WMWiKVg1IVGr4y2D1JrjhnbQtlChRDJm2S8rGlc,9868
-re_common/v2/baselibrary/utils/author_smi.py,sha256=1ebH3AHv19jtJWdlqNdwu6t58HNVLCotuCB6ff1SWiw,13666
+re_common/v2/baselibrary/utils/author_smi.py,sha256=Fh276u7za-GF_tK9xpBFYF5q1E3_tX22ZouWC8U7w8o,13831
 re_common/v2/baselibrary/utils/base_string_similarity.py,sha256=a40a79ttwoG_gC_hxMNB-sMXXecgICoRDWrj0DW8iEE,7749
 re_common/v2/baselibrary/utils/basedict.py,sha256=sH3_RZ8u4649-jX2V1uKNNkjJVUijZBDp6SdqncOZ88,1583
 re_common/v2/baselibrary/utils/basehdfs.py,sha256=TPwFct_-UrmO1KCbo4gpV77rsnlCQDumNBbQKL0ZI9o,5953
@@ -200,9 +202,9 @@ re_common/v2/baselibrary/utils/json_cls.py,sha256=M93piYtmgm_wP8E57culTrd_AhHLoG
 re_common/v2/baselibrary/utils/mq.py,sha256=UHpO8iNIHs91Tgp-BgnSUpZwjWquxrGLdpr3FMMv2zw,2858
 re_common/v2/baselibrary/utils/n_ary_expression_tree.py,sha256=-05kO6G2Rth7CEK-5lfFrthFZ1Q0-0a7cni7mWZ-2gg,9172
 re_common/v2/baselibrary/utils/string_bool.py,sha256=vxnjSFOfuHWGxkqaIbUNn21opx5tfV1uCXSahFfp1mU,6197
-re_common/v2/baselibrary/utils/string_clear.py,sha256=ywYR1KrKQyeM-zJgvTmORlfgbLdRSjWWKPe7K8oRx_8,7450
+re_common/v2/baselibrary/utils/string_clear.py,sha256=Ympa0Cs2y_72QeeyMS8de8y_QgtEFJJQ0AgHnylbMUc,7861
 re_common/v2/baselibrary/utils/string_smi.py,sha256=cU0WAWHRGnGoVQx3eCEKeM_q_olFNzRTJe7rSe586SY,741
-re_common/v2/baselibrary/utils/stringutils.py,sha256=TI6fw3km1l25ufXrnG6ha8dSBDtRh-MF4nWRt9u8Xbo,6452
+re_common/v2/baselibrary/utils/stringutils.py,sha256=eeuQYgXkWJ9apvyrYPcCCU3biTY9nD1KHos4_1ESNJE,7883
 re_common/vip/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 re_common/vip/base_step_process.py,sha256=VXXiNj0I5CpzXIMCgOPU86bzDJkSBkUS-9CpZIl_GOk,205
 re_common/vip/baseencodeid.py,sha256=nERoe89ueFM52bG7xwJdflcZHk6T2RQQKbc5uUZc3RM,3272
@@ -229,8 +231,8 @@ re_common/vip/title/transform/TransformRegulationTitleToZt.py,sha256=LKRdIsWKues
 re_common/vip/title/transform/TransformStandardTitleToZt.py,sha256=-fCKAbSBzXVyQDCE61CalvR9E_QzQMA08QOO_NePFNI,5563
 re_common/vip/title/transform/TransformThesisTitleToZt.py,sha256=QS-uV0cQrpUFAcKucuJQ9Ue2VRQH-inmfn_X3IplfRo,5488
 re_common/vip/title/transform/__init__.py,sha256=m83-CWyRq_VHPYHaALEQlmXrkTdrZ3e4B_kCfBYE-uc,239
-re_common-10.0.26.dist-info/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
-re_common-10.0.26.dist-info/METADATA,sha256=kHLVPF-e0PjpnUL7dN9pAMqK_pw4yHwZGKxbJ_zlAY0,582
-re_common-10.0.26.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-re_common-10.0.26.dist-info/top_level.txt,sha256=_H9H23zoLIalm1AIY_KYTVh_H0ZnmjxQIxsvXtLv45o,10
-re_common-10.0.26.dist-info/RECORD,,
+re_common-10.0.28.dist-info/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
+re_common-10.0.28.dist-info/METADATA,sha256=6LyRvl5fLSmKd4qNyZFc72DkO8_hyJ6FQ27dba4PEvc,582
+re_common-10.0.28.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+re_common-10.0.28.dist-info/top_level.txt,sha256=_H9H23zoLIalm1AIY_KYTVh_H0ZnmjxQIxsvXtLv45o,10
+re_common-10.0.28.dist-info/RECORD,,