re-common 10.0.26__py3-none-any.whl → 10.0.27__py3-none-any.whl

re_common/v2/baselibrary/business_utils/baseencodeid.py

@@ -0,0 +1,100 @@
+import base64
+import hashlib
+
+"""
+VIP编码lngid生成
+"""
+
+
+class BaseLngid(object):
+    def __int__(self):
+        pass
+
+    def BaseEncodeID(self, strRaw):
+        r""" 自定义base编码 """
+
+        strEncode = base64.b32encode(strRaw.encode('utf8')).decode('utf8')
+
+        if strEncode.endswith('======'):
+            strEncode = '%s%s' % (strEncode[0:-6], '0')
+        elif strEncode.endswith('===='):
+            strEncode = '%s%s' % (strEncode[0:-4], '1')
+        elif strEncode.endswith('==='):
+            strEncode = '%s%s' % (strEncode[0:-3], '8')
+        elif strEncode.endswith('='):
+            strEncode = '%s%s' % (strEncode[0:-1], '9')
+
+        table = str.maketrans('0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ', 'ZYXWVUTSRQPONMLKJIHGFEDCBA9876543210')
+        strEncode = strEncode.translate(table)
+
+        return strEncode
+
+    def BaseDecodeID(self, strEncode):
+        r""" 自定义base解码 """
+
+        table = str.maketrans('ZYXWVUTSRQPONMLKJIHGFEDCBA9876543210', '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ')
+        strEncode = strEncode.translate(table)
+
+        if strEncode.endswith('0'):
+            strEncode = '%s%s' % (strEncode[0:-1], '======')
+        elif strEncode.endswith('1'):
+            strEncode = '%s%s' % (strEncode[0:-1], '====')
+        elif strEncode.endswith('8'):
+            strEncode = '%s%s' % (strEncode[0:-1], '===')
+        elif strEncode.endswith('9'):
+            strEncode = '%s%s' % (strEncode[0:-1], '=')
+
+        strRaw = base64.b32decode(strEncode.encode('utf8')).decode('utf8')
+
+        return strRaw
+
+    def GetLngid(self, sub_db_id, rawid, case_insensitive=False):
+        """
+        :param sub_db_id:
+        :param rawid:
+        由 sub_db_id 和 rawid 得到 lngid。
+        :param case_insensitive: 标识源网站的 rawid 是否区分大小写
+        :return: lngid
+        """
+        uppercase_rawid = ''  # 大写版 rawid
+        if case_insensitive:  # 源网站的 rawid 区分大小写
+            for ch in rawid:
+                if ch.upper() == ch:
+                    uppercase_rawid += ch
+                else:
+                    uppercase_rawid += ch.upper() + '_'
+        else:
+            uppercase_rawid = rawid.upper()
+
+        limited_id = uppercase_rawid  # 限长ID
+        if len(uppercase_rawid) > 20:
+            limited_id = hashlib.md5(uppercase_rawid.encode('utf8')).hexdigest().upper()
+        else:
+            limited_id = self.BaseEncodeID(uppercase_rawid)
+
+        lngid = sub_db_id + limited_id
+
+        return lngid
+
+    def getDoiid(self, doi):
+        doi_upper = doi.upper()
+        limited_id = self.BaseEncodeID(doi_upper)
+        if len(limited_id) > 240:
+            limited_id = hashlib.md5(doi_upper.encode('utf8')).hexdigest().upper()
+        return limited_id
+
+    def GetRawid(self, limited_id, case_insensitive=False):
+        try:
+            uppercase_rawid = self.BaseDecodeID(limited_id)
+            if case_insensitive:
+                str_ = "_"
+                uppercase_rawid_list = list(uppercase_rawid)
+                for num, li in enumerate(uppercase_rawid_list):
+                    if li == str_:
+                        old_str = "".join(uppercase_rawid_list[num - 1:num + 1])
+                        uppercase_rawid = uppercase_rawid.replace(old_str, uppercase_rawid_list[num - 1].lower())
+        except Exception as e:
+            raise Exception("长度超过20，不可逆")
+
+        return uppercase_rawid
+

re_common/v2/baselibrary/business_utils/full_doi_path.py

@@ -0,0 +1,116 @@
+import base64
+import hashlib
+import os
+
+from re_common.v2.baselibrary.business_utils.baseencodeid import BaseLngid
+
+import os
+import base64
+import hashlib
+
+"""
+DOI-文件路径 转换工具
+
+设计目标：
+1. 将任意DOI字符串转换为可逆、稳定的文件路径
+2. 提供高效的目录分散方案（65,536个子目录）
+3. 支持带文件扩展名的存储
+4. 完全可逆转换
+
+工作原理：
+1. DOI编码：
+   - 使用URL安全的Base64编码（RFC 3548）
+   - 移除Base64填充的'='字符
+   - 文件名长度 ≈ 原始DOI长度 × 4/3
+
+2. 目录分散：
+   - 使用MD5哈希创建两级目录结构
+   - 目录层级：/MD5[0:2]/MD5[2:4]/
+   - 支持65,536个目录（256×256），每目录约1,525个文件（假设10亿文件）
+
+3. 扩展名处理：
+   - 保持原始扩展名不变
+   - 解码时自动忽略扩展名
+
+典型转换示例：
+DOI: "10.1000/xyz123" -> 路径: "a1/b2/QTMuMTAwMC94eXoxMjM.pdf"
+路径: "a1/b2/QTMuMTAwMC94eXoxMjM.pdf" -> DOI: "10.1000/xyz123"
+"""
+
+base_lngid = BaseLngid()
+
+
+# 以后需要启用
+def doi_to_path(doi: str, ext: str = "") -> str:
+    """
+    将 DOI 转换为可逆的存储路径：
+    1. 对 DOI 进行 URL 安全的 Base64 编码（可逆）
+    2. 生成 DOI 的 MD5 哈希用于目录分散
+    3. 目录结构：MD5前2字符/次2字符/
+    4. 文件名：Base64编码的DOI + 扩展名
+
+    Args:
+        doi: 文件 DOI 标识符
+        ext: 文件扩展名（如 '.pdf'）
+
+    Returns:
+        相对文件路径（如 'a1/b2/QTMuMTAwMC94eXoxMjM=.pdf'）
+    """
+    # URL安全的Base64编码（可逆）
+    doi_b64 = base64.urlsafe_b64encode(doi.encode("utf-8")).decode("ascii").rstrip("=")
+
+    # 生成MD5哈希用于目录分配
+    hash_md5 = hashlib.md5(doi.encode("utf-8")).hexdigest()
+    dir_level1 = hash_md5[0:2]
+    dir_level2 = hash_md5[2:4]
+
+    return os.path.join(dir_level1, dir_level2, f"{doi_b64}{ext}")
+
+
+# 以后需要启用
+def path_to_doi(path: str) -> str:
+    """
+    从文件路径反推原始DOI
+    Args:
+        path: 文件路径（如 'a1/b2/QTMuMTAwMC94eXoxMjM=.pdf'）
+
+    Returns:
+        原始DOI字符串
+    """
+    # 提取文件名并移除扩展名
+    filename = os.path.basename(path)
+    base_name = os.path.splitext(filename)[0]
+
+    # 补齐Base64填充字符
+    padding = 4 - (len(base_name) % 4)
+    if padding != 4:  # 不需要补齐
+        base_name += "=" * padding
+
+    # Base64解码还原DOI
+    return base64.urlsafe_b64decode(base_name.encode("ascii")).decode("utf-8")
+
+
+def doi_to_dir(doi):
+    """生成文件的存储路径和可解码的文件名
+
+    Args:
+        doi (str): 文件的唯一DOI标识
+
+    Returns:
+        str: 文件相对路径，如 "ab/cd/Base64EncodedFileName"
+    """
+    # 计算DOI的MD5哈希
+    hash_md5 = hashlib.md5(doi.encode('utf-8')).hexdigest().lower()
+
+    # 提取目录层级：前2位作为一级目录，3-4位作为二级目录
+    first_dir = hash_md5[0:2].upper()
+    second_dir = hash_md5[2:4].upper()
+
+    return first_dir + "/" + second_dir
+
+
+def get_doi_path(doi):
+    # 目前使用
+    dir_path = doi_to_dir(doi)
+    file_name = base_lngid.getDoiid(doi) + ".pdf"
+    return dir_path + "/" + file_name

{re_common-10.0.26.dist-info → re_common-10.0.27.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: re_common
-Version: 10.0.26
+Version: 10.0.27
 Summary: a library about all python projects
 Home-page: https://gitee.com/xujiangios/re-common
 Author: vic

{re_common-10.0.26.dist-info → re_common-10.0.27.dist-info}/RECORD

@@ -165,6 +165,8 @@ re_common/v2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 re_common/v2/baselibrary/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 re_common/v2/baselibrary/business_utils/BusinessStringUtil.py,sha256=njPcRgeBWpnZr5u2cPAO4qdWBq-CgTn99rJuvWFcChk,6788
 re_common/v2/baselibrary/business_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+re_common/v2/baselibrary/business_utils/baseencodeid.py,sha256=J2Gs_F3-aAwIYjO06T1IIKHE6jen3mFfQVoVW3HywyI,3548
+re_common/v2/baselibrary/business_utils/full_doi_path.py,sha256=PaMIrgDWWt_fzSFyvvDD-8CcYZJTNo6Pj-uR0WafNbY,3319
 re_common/v2/baselibrary/business_utils/rel_tools.py,sha256=LfnGFCkUSxg1SHvOMOQdP1PiHxIKqk7Syuk5YYpjJag,295
 re_common/v2/baselibrary/decorators/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 re_common/v2/baselibrary/decorators/utils.py,sha256=Q4D6KKCQxvNBXZkPQQn14keKKJpGtg8TUSakjJU40s0,2056
@@ -229,8 +231,8 @@ re_common/vip/title/transform/TransformRegulationTitleToZt.py,sha256=LKRdIsWKues
 re_common/vip/title/transform/TransformStandardTitleToZt.py,sha256=-fCKAbSBzXVyQDCE61CalvR9E_QzQMA08QOO_NePFNI,5563
 re_common/vip/title/transform/TransformThesisTitleToZt.py,sha256=QS-uV0cQrpUFAcKucuJQ9Ue2VRQH-inmfn_X3IplfRo,5488
 re_common/vip/title/transform/__init__.py,sha256=m83-CWyRq_VHPYHaALEQlmXrkTdrZ3e4B_kCfBYE-uc,239
-re_common-10.0.26.dist-info/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
-re_common-10.0.26.dist-info/METADATA,sha256=kHLVPF-e0PjpnUL7dN9pAMqK_pw4yHwZGKxbJ_zlAY0,582
-re_common-10.0.26.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-re_common-10.0.26.dist-info/top_level.txt,sha256=_H9H23zoLIalm1AIY_KYTVh_H0ZnmjxQIxsvXtLv45o,10
-re_common-10.0.26.dist-info/RECORD,,
+re_common-10.0.27.dist-info/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
+re_common-10.0.27.dist-info/METADATA,sha256=P-xatKifINopvPpQFudSbxCKEpgqzLyw5Pesn-7_VvU,582
+re_common-10.0.27.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+re_common-10.0.27.dist-info/top_level.txt,sha256=_H9H23zoLIalm1AIY_KYTVh_H0ZnmjxQIxsvXtLv45o,10
+re_common-10.0.27.dist-info/RECORD,,