re-common 10.0.37__py3-none-any.whl → 10.0.39__py3-none-any.whl

re_common/v2/baselibrary/business_utils/full_doi_path.py

@@ -1,116 +1,116 @@
-import base64
-import hashlib
-import os
-
-from re_common.v2.baselibrary.business_utils.baseencodeid import BaseLngid
-
-import os
-import base64
-import hashlib
-
-"""
-DOI-文件路径 转换工具
-
-设计目标：
-1. 将任意DOI字符串转换为可逆、稳定的文件路径
-2. 提供高效的目录分散方案（65,536个子目录）
-3. 支持带文件扩展名的存储
-4. 完全可逆转换
-
-工作原理：
-1. DOI编码：
-   - 使用URL安全的Base64编码（RFC 3548）
-   - 移除Base64填充的'='字符
-   - 文件名长度 ≈ 原始DOI长度 × 4/3
-
-2. 目录分散：
-   - 使用MD5哈希创建两级目录结构
-   - 目录层级：/MD5[0:2]/MD5[2:4]/
-   - 支持65,536个目录（256×256），每目录约1,525个文件（假设10亿文件）
-
-3. 扩展名处理：
-   - 保持原始扩展名不变
-   - 解码时自动忽略扩展名
-
-典型转换示例：
-DOI: "10.1000/xyz123" -> 路径: "a1/b2/QTMuMTAwMC94eXoxMjM.pdf"
-路径: "a1/b2/QTMuMTAwMC94eXoxMjM.pdf" -> DOI: "10.1000/xyz123"
-"""
-
-base_lngid = BaseLngid()
-
-
-# 以后需要启用
-def doi_to_path(doi: str, ext: str = "") -> str:
-    """
-    将 DOI 转换为可逆的存储路径：
-    1. 对 DOI 进行 URL 安全的 Base64 编码（可逆）
-    2. 生成 DOI 的 MD5 哈希用于目录分散
-    3. 目录结构：MD5前2字符/次2字符/
-    4. 文件名：Base64编码的DOI + 扩展名
-
-    Args:
-        doi: 文件 DOI 标识符
-        ext: 文件扩展名（如 '.pdf'）
-
-    Returns:
-        相对文件路径（如 'a1/b2/QTMuMTAwMC94eXoxMjM=.pdf'）
-    """
-    # URL安全的Base64编码（可逆）
-    doi_b64 = base64.urlsafe_b64encode(doi.encode("utf-8")).decode("ascii").rstrip("=")
-
-    # 生成MD5哈希用于目录分配
-    hash_md5 = hashlib.md5(doi.encode("utf-8")).hexdigest()
-    dir_level1 = hash_md5[0:2]
-    dir_level2 = hash_md5[2:4]
-
-    return os.path.join(dir_level1, dir_level2, f"{doi_b64}{ext}")
-
-
-# 以后需要启用
-def path_to_doi(path: str) -> str:
-    """
-    从文件路径反推原始DOI
-    Args:
-        path: 文件路径（如 'a1/b2/QTMuMTAwMC94eXoxMjM=.pdf'）
-
-    Returns:
-        原始DOI字符串
-    """
-    # 提取文件名并移除扩展名
-    filename = os.path.basename(path)
-    base_name = os.path.splitext(filename)[0]
-
-    # 补齐Base64填充字符
-    padding = 4 - (len(base_name) % 4)
-    if padding != 4:  # 不需要补齐
-        base_name += "=" * padding
-
-    # Base64解码还原DOI
-    return base64.urlsafe_b64decode(base_name.encode("ascii")).decode("utf-8")
-
-
-def doi_to_dir(doi):
-    """生成文件的存储路径和可解码的文件名
-
-    Args:
-        doi (str): 文件的唯一DOI标识
-
-    Returns:
-        str: 文件相对路径，如 "ab/cd/Base64EncodedFileName"
-    """
-    # 计算DOI的MD5哈希
-    hash_md5 = hashlib.md5(doi.encode('utf-8')).hexdigest().lower()
-
-    # 提取目录层级：前2位作为一级目录，3-4位作为二级目录
-    first_dir = hash_md5[0:2].upper()
-    second_dir = hash_md5[2:4].upper()
-
-    return first_dir + "/" + second_dir
-
-
-def get_doi_path(doi, case_insensitive=False):
-    # 目前使用
-    dir_path = doi_to_dir(doi)
-    file_name = base_lngid.getDoiid(doi, case_insensitive=case_insensitive) + ".pdf"
-    return dir_path + "/" + file_name
+import base64
+import hashlib
+import os
+
+from re_common.v2.baselibrary.business_utils.baseencodeid import BaseLngid
+
+import os
+import base64
+import hashlib
+
+"""
+DOI-文件路径 转换工具
+
+设计目标：
+1. 将任意DOI字符串转换为可逆、稳定的文件路径
+2. 提供高效的目录分散方案（65,536个子目录）
+3. 支持带文件扩展名的存储
+4. 完全可逆转换
+
+工作原理：
+1. DOI编码：
+   - 使用URL安全的Base64编码（RFC 3548）
+   - 移除Base64填充的'='字符
+   - 文件名长度 ≈ 原始DOI长度 × 4/3
+
+2. 目录分散：
+   - 使用MD5哈希创建两级目录结构
+   - 目录层级：/MD5[0:2]/MD5[2:4]/
+   - 支持65,536个目录（256×256），每目录约1,525个文件（假设10亿文件）
+
+3. 扩展名处理：
+   - 保持原始扩展名不变
+   - 解码时自动忽略扩展名
+
+典型转换示例：
+DOI: "10.1000/xyz123" -> 路径: "a1/b2/QTMuMTAwMC94eXoxMjM.pdf"
+路径: "a1/b2/QTMuMTAwMC94eXoxMjM.pdf" -> DOI: "10.1000/xyz123"
+"""
+
+base_lngid = BaseLngid()
+
+
+# 以后需要启用
+def doi_to_path(doi: str, ext: str = "") -> str:
+    """
+    将 DOI 转换为可逆的存储路径：
+    1. 对 DOI 进行 URL 安全的 Base64 编码（可逆）
+    2. 生成 DOI 的 MD5 哈希用于目录分散
+    3. 目录结构：MD5前2字符/次2字符/
+    4. 文件名：Base64编码的DOI + 扩展名
+
+    Args:
+        doi: 文件 DOI 标识符
+        ext: 文件扩展名（如 '.pdf'）
+
+    Returns:
+        相对文件路径（如 'a1/b2/QTMuMTAwMC94eXoxMjM=.pdf'）
+    """
+    # URL安全的Base64编码（可逆）
+    doi_b64 = base64.urlsafe_b64encode(doi.encode("utf-8")).decode("ascii").rstrip("=")
+
+    # 生成MD5哈希用于目录分配
+    hash_md5 = hashlib.md5(doi.encode("utf-8")).hexdigest()
+    dir_level1 = hash_md5[0:2]
+    dir_level2 = hash_md5[2:4]
+
+    return os.path.join(dir_level1, dir_level2, f"{doi_b64}{ext}")
+
+
+# 以后需要启用
+def path_to_doi(path: str) -> str:
+    """
+    从文件路径反推原始DOI
+    Args:
+        path: 文件路径（如 'a1/b2/QTMuMTAwMC94eXoxMjM=.pdf'）
+
+    Returns:
+        原始DOI字符串
+    """
+    # 提取文件名并移除扩展名
+    filename = os.path.basename(path)
+    base_name = os.path.splitext(filename)[0]
+
+    # 补齐Base64填充字符
+    padding = 4 - (len(base_name) % 4)
+    if padding != 4:  # 不需要补齐
+        base_name += "=" * padding
+
+    # Base64解码还原DOI
+    return base64.urlsafe_b64decode(base_name.encode("ascii")).decode("utf-8")
+
+
+def doi_to_dir(doi):
+    """生成文件的存储路径和可解码的文件名
+
+    Args:
+        doi (str): 文件的唯一DOI标识
+
+    Returns:
+        str: 文件相对路径，如 "ab/cd/Base64EncodedFileName"
+    """
+    # 计算DOI的MD5哈希
+    hash_md5 = hashlib.md5(doi.encode('utf-8')).hexdigest().lower()
+
+    # 提取目录层级：前2位作为一级目录，3-4位作为二级目录
+    first_dir = hash_md5[0:2].upper()
+    second_dir = hash_md5[2:4].upper()
+
+    return first_dir + "/" + second_dir
+
+
+def get_doi_path(doi, case_insensitive=False):
+    # 目前使用
+    dir_path = doi_to_dir(doi)
+    file_name = base_lngid.getDoiid(doi, case_insensitive=case_insensitive) + ".pdf"
+    return dir_path + "/" + file_name

re_common/v2/baselibrary/business_utils/rel_tools.py

@@ -1,6 +1,6 @@
-def assign_group_id(rows: list, sub_db_order: list):
-    subdb_keyid_map = {row.sub_db_id: row.keyid for row in rows}
-    for sub_db_id in sub_db_order:
-        if keyid := subdb_keyid_map.get(sub_db_id):
-            return keyid, len(rows), rows
-    return rows[0].keyid, len(rows), rows
+def assign_group_id(rows: list, sub_db_order: list):
+    subdb_keyid_map = {row.sub_db_id: row.keyid for row in rows}
+    for sub_db_id in sub_db_order:
+        if keyid := subdb_keyid_map.get(sub_db_id):
+            return keyid, len(rows), rows
+    return rows[0].keyid, len(rows), rows

re_common/v2/baselibrary/decorators/utils.py

@@ -1,59 +1,59 @@
-import warnings
-import functools
-
-# 全局集合，用于记录已警告的函数或类
-_warned_once = set()
-
-
-def deprecated(message=None):
-    """
-    装饰器：标记函数或类为已废弃，整个进程只发出一次警告。
-
-    Args:
-        message (str): 自定义警告信息，默认为 None。
-    """
-
-    def decorator(obj):
-        # 如果是函数
-        if isinstance(obj, type(lambda: None)):
-            @functools.wraps(obj)
-            def wrapper(*args, **kwargs):
-                obj_id = id(obj)  # 使用对象的内存地址作为唯一标识
-                if obj_id not in _warned_once:
-                    default_msg = f"函数 {obj.__name__} 已不建议使用。"
-                    warn_msg = f"{default_msg} {message}" if message else default_msg
-                    warnings.warn(
-                        warn_msg,
-                        category=DeprecationWarning,
-                        stacklevel=2
-                    )
-                    _warned_once.add(obj_id)  # 记录已警告
-                return obj(*args, **kwargs)
-
-            return wrapper
-
-        # 如果是类
-        elif isinstance(obj, type):
-            orig_init = obj.__init__
-
-            @functools.wraps(orig_init)
-            def new_init(self, *args, **kwargs):
-                obj_id = id(obj)
-                if obj_id not in _warned_once:
-                    default_msg = f"类 {obj.__name__} 已不建议使用。"
-                    warn_msg = f"{default_msg} {message}" if message else default_msg
-                    warnings.warn(
-                        warn_msg,
-                        category=DeprecationWarning,
-                        stacklevel=2
-                    )
-                    _warned_once.add(obj_id)  # 记录已警告
-                orig_init(self, *args, **kwargs)
-
-            obj.__init__ = new_init
-            return obj
-
-        else:
-            raise TypeError("此装饰器仅适用于函数和类")
-
-    return decorator
+import warnings
+import functools
+
+# 全局集合，用于记录已警告的函数或类
+_warned_once = set()
+
+
+def deprecated(message=None):
+    """
+    装饰器：标记函数或类为已废弃，整个进程只发出一次警告。
+
+    Args:
+        message (str): 自定义警告信息，默认为 None。
+    """
+
+    def decorator(obj):
+        # 如果是函数
+        if isinstance(obj, type(lambda: None)):
+            @functools.wraps(obj)
+            def wrapper(*args, **kwargs):
+                obj_id = id(obj)  # 使用对象的内存地址作为唯一标识
+                if obj_id not in _warned_once:
+                    default_msg = f"函数 {obj.__name__} 已不建议使用。"
+                    warn_msg = f"{default_msg} {message}" if message else default_msg
+                    warnings.warn(
+                        warn_msg,
+                        category=DeprecationWarning,
+                        stacklevel=2
+                    )
+                    _warned_once.add(obj_id)  # 记录已警告
+                return obj(*args, **kwargs)
+
+            return wrapper
+
+        # 如果是类
+        elif isinstance(obj, type):
+            orig_init = obj.__init__
+
+            @functools.wraps(orig_init)
+            def new_init(self, *args, **kwargs):
+                obj_id = id(obj)
+                if obj_id not in _warned_once:
+                    default_msg = f"类 {obj.__name__} 已不建议使用。"
+                    warn_msg = f"{default_msg} {message}" if message else default_msg
+                    warnings.warn(
+                        warn_msg,
+                        category=DeprecationWarning,
+                        stacklevel=2
+                    )
+                    _warned_once.add(obj_id)  # 记录已警告
+                orig_init(self, *args, **kwargs)
+
+            obj.__init__ = new_init
+            return obj
+
+        else:
+            raise TypeError("此装饰器仅适用于函数和类")
+
+    return decorator

re_common/v2/baselibrary/helpers/search_packge/NearestNeighbors_test.py

@@ -1,105 +1,105 @@
-import datetime
-import gzip
-import io
-import time
-
-import joblib
-from hdfs import InsecureClient
-
-from re_common.v2.baselibrary.helpers.search_packge.fit_text_match import FitTextMatcher
-
-if __name__ == '__main__':
-    matcher = FitTextMatcher(
-        algorithm='auto',  # 对于小数据集，暴力搜索足够快 brute
-        n_jobs=2  # 使用所有CPU核心
-
-    )
-
-    client = InsecureClient("http://VIP-DC-MASTER-2:9870", user="xujiang")
-
-    lists = []
-    for i in ["vectorizer", "nn_model", "corpus", "idx"]:
-        fit_file_path = f"/b_task_data/class_smi/fit_file/t_23600_{i}.joblib.gz"
-        with client.read(fit_file_path) as reader:
-            tp = io.BytesIO(reader.read())
-            tp.seek(0)
-            lists.append(tp)
-
-    with gzip.GzipFile(fileobj=lists[2], mode='rb') as gz:
-        matcher.corpus = joblib.load(gz)
-
-    with gzip.GzipFile(fileobj=lists[3], mode='rb') as gz:
-        matcher.idx = joblib.load(gz)
-    matcher.corpus_size = max(len(matcher.corpus), len(matcher.idx))
-    print(f"加载bytes完成，共 {matcher.corpus_size} 篇文献")
-
-    matcher.fit(matcher.corpus)
-
-    print(matcher.nn._fit_method)
-
-    print("fit 训练完成")
-
-    count = 0
-    bacth_list = []
-    n = min(100, matcher.corpus_size)
-    for i in matcher.corpus:
-        count = count + 1
-        bacth_list.append(i)
-        if count % 10000 == 0:
-            t1 = time.time()
-            index, similarities = matcher.batch_search(bacth_list, n=n)
-            for rank, (idxs, sims) in enumerate(zip(index, similarities)):
-                print({"keyid": matcher.idx[rank],
-                       "search_list": [(matcher.idx[idx], sim) for idx, sim in zip(idxs, sims)]})
-
-            t2 = time.time()
-            now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S,%f")[:-3]
-            print(now, count, t2 - t1)
-            bacth_list.clear()
-
-
-
-
-
-
-
-    # with open("t_8220_corpus.joblib","rb") as f:
-    #     buf = io.BytesIO(f.read())
-    #     buf.seek(0)
-    #     corpus = buf
-    #
-    # matcher.corpus = joblib.load(corpus)
-    # print(len(matcher.corpus))
-
-    # matcher.load_bytes(vec, nn, corpus)
-
-    # with open(r"C:\Users\Administrator\Desktop\update\part-02440\part-02440_1", "r", encoding="utf-8") as f:
-    #     lists = [line.strip() for line in f if line]
-    #
-    # matcher.fit(lists)
-
-    # matcher.load("./","test")
-
-    # query = r"herbdrug interaction in the protective effect of alpinia officinarum against gastric injury induced by indomethacin based on pharmacokinetic tissue distribution and excretion studies in rats"
-    # result = matcher.search(query, n=100)
-    # print("query", query)
-    # for rank, (idx, sim) in enumerate(result):
-    #     print(f"\nTop {rank + 1} [相似度: {sim:.4f}]:")
-    #     print(f"文献 #{idx}: {lists[idx]}")
-    # print(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
-    # time.sleep(100)
-
-    # for query in lists[:1000]:
-    #     indices, similarities = matcher.search(query, n=100)
-    #     print("query", query)
-    #     for rank, (idx, sim) in enumerate(zip(indices, similarities)):
-    #         print(f"\nTop {rank + 1} [相似度: {sim:.4f}]:")
-    #         print(f"文献 #{idx}: {lists[idx]}")
-    #     print(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
-    #     time.sleep(100)
-    #
-    # idx,sim = matcher.batch_search(lists[:1000], n=100)
-    # for rank, (idxs, sims) in enumerate(zip(idx,sim)):
-    #     tp = (lists[rank],[(lists[idx], sim) for idx,sim in zip(idxs,sims)])
-    #     print(tp)
-    # time.sleep(100)
+import datetime
+import gzip
+import io
+import time
+
+import joblib
+from hdfs import InsecureClient
+
+from re_common.v2.baselibrary.helpers.search_packge.fit_text_match import FitTextMatcher
+
+if __name__ == '__main__':
+    matcher = FitTextMatcher(
+        algorithm='auto',  # 对于小数据集，暴力搜索足够快 brute
+        n_jobs=2  # 使用所有CPU核心
+
+    )
+
+    client = InsecureClient("http://VIP-DC-MASTER-2:9870", user="xujiang")
+
+    lists = []
+    for i in ["vectorizer", "nn_model", "corpus", "idx"]:
+        fit_file_path = f"/b_task_data/class_smi/fit_file/t_23600_{i}.joblib.gz"
+        with client.read(fit_file_path) as reader:
+            tp = io.BytesIO(reader.read())
+            tp.seek(0)
+            lists.append(tp)
+
+    with gzip.GzipFile(fileobj=lists[2], mode='rb') as gz:
+        matcher.corpus = joblib.load(gz)
+
+    with gzip.GzipFile(fileobj=lists[3], mode='rb') as gz:
+        matcher.idx = joblib.load(gz)
+    matcher.corpus_size = max(len(matcher.corpus), len(matcher.idx))
+    print(f"加载bytes完成，共 {matcher.corpus_size} 篇文献")
+
+    matcher.fit(matcher.corpus)
+
+    print(matcher.nn._fit_method)
+
+    print("fit 训练完成")
+
+    count = 0
+    bacth_list = []
+    n = min(100, matcher.corpus_size)
+    for i in matcher.corpus:
+        count = count + 1
+        bacth_list.append(i)
+        if count % 10000 == 0:
+            t1 = time.time()
+            index, similarities = matcher.batch_search(bacth_list, n=n)
+            for rank, (idxs, sims) in enumerate(zip(index, similarities)):
+                print({"keyid": matcher.idx[rank],
+                       "search_list": [(matcher.idx[idx], sim) for idx, sim in zip(idxs, sims)]})
+
+            t2 = time.time()
+            now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S,%f")[:-3]
+            print(now, count, t2 - t1)
+            bacth_list.clear()
+
+
+
+
+
+
+
+    # with open("t_8220_corpus.joblib","rb") as f:
+    #     buf = io.BytesIO(f.read())
+    #     buf.seek(0)
+    #     corpus = buf
+    #
+    # matcher.corpus = joblib.load(corpus)
+    # print(len(matcher.corpus))
+
+    # matcher.load_bytes(vec, nn, corpus)
+
+    # with open(r"C:\Users\Administrator\Desktop\update\part-02440\part-02440_1", "r", encoding="utf-8") as f:
+    #     lists = [line.strip() for line in f if line]
+    #
+    # matcher.fit(lists)
+
+    # matcher.load("./","test")
+
+    # query = r"herbdrug interaction in the protective effect of alpinia officinarum against gastric injury induced by indomethacin based on pharmacokinetic tissue distribution and excretion studies in rats"
+    # result = matcher.search(query, n=100)
+    # print("query", query)
+    # for rank, (idx, sim) in enumerate(result):
+    #     print(f"\nTop {rank + 1} [相似度: {sim:.4f}]:")
+    #     print(f"文献 #{idx}: {lists[idx]}")
+    # print(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
+    # time.sleep(100)
+
+    # for query in lists[:1000]:
+    #     indices, similarities = matcher.search(query, n=100)
+    #     print("query", query)
+    #     for rank, (idx, sim) in enumerate(zip(indices, similarities)):
+    #         print(f"\nTop {rank + 1} [相似度: {sim:.4f}]:")
+    #         print(f"文献 #{idx}: {lists[idx]}")
+    #     print(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
+    #     time.sleep(100)
+    #
+    # idx,sim = matcher.batch_search(lists[:1000], n=100)
+    # for rank, (idxs, sims) in enumerate(zip(idx,sim)):
+    #     tp = (lists[rank],[(lists[idx], sim) for idx,sim in zip(idxs,sims)])
+    #     print(tp)
+    # time.sleep(100)