re-common 10.0.34__py3-none-any.whl → 10.0.36__py3-none-any.whl

re_common/v2/baselibrary/business_utils/BusinessStringUtil.py

@@ -1,7 +1,10 @@
 # 某些业务中的字符串处理 算是特定场景的工具 不算通用工具
 import re
 
+from re_common.v2.baselibrary.utils.author_smi import AuthorRatio
 from re_common.v2.baselibrary.utils.string_bool import is_all_symbols
+from re_common.v2.baselibrary.utils.string_clear import rel_clear
+from re_common.v2.baselibrary.utils.stringutils import clean_unicode_alnum, get_alphabetic_ratio
 
 
 def clean_organ_postcode(organ):
@@ -194,3 +197,24 @@ def deal_num(num_str):
         num_str = deal_num_strs(num_str)
 
     return num_str.lower().strip()
+
+
+def clear_author_1st(author_str:str):
+    # 清理括号 防止前面流程没有清理干净
+    author_str = re.sub("\\[.*?]", "", author_str)
+    author_str = re.sub("\\(.*?\\)", "", author_str)
+
+    # 清理符号
+    author_str = clean_unicode_alnum(author_str)
+
+    return author_str
+
+def is_same_author(a1, a2):
+    if get_alphabetic_ratio(a1.strip()) > 0.7 and get_alphabetic_ratio(a2.strip()) > 0.7:
+        author_similar_ = AuthorRatio(a1.strip(), a2.strip())
+        if author_similar_ > 0.95:
+            return True
+    else:
+        if rel_clear(a1.strip()) == rel_clear(a2.strip()):
+            return True
+    return False

re_common/v2/baselibrary/business_utils/baseencodeid.py

@@ -10,7 +10,7 @@ class BaseLngid(object):
     def __int__(self):
         pass
 
-    def BaseEncodeID(self, strRaw):
+    def basBaseEncodeID(self, strRaw):
         r""" 自定义base编码 """
 
         strEncode = base64.b32encode(strRaw.encode('utf8')).decode('utf8')

re_common/v2/baselibrary/helpers/search_packge/NearestNeighbors_test.py

@@ -0,0 +1,105 @@
+import datetime
+import gzip
+import io
+import time
+
+import joblib
+from hdfs import InsecureClient
+
+from re_common.v2.baselibrary.helpers.search_packge.fit_text_match import FitTextMatcher
+
+if __name__ == '__main__':
+    matcher = FitTextMatcher(
+        algorithm='auto',  # 对于小数据集，暴力搜索足够快 brute
+        n_jobs=2  # 使用所有CPU核心
+
+    )
+
+    client = InsecureClient("http://VIP-DC-MASTER-2:9870", user="xujiang")
+
+    lists = []
+    for i in ["vectorizer", "nn_model", "corpus", "idx"]:
+        fit_file_path = f"/b_task_data/class_smi/fit_file/t_23600_{i}.joblib.gz"
+        with client.read(fit_file_path) as reader:
+            tp = io.BytesIO(reader.read())
+            tp.seek(0)
+            lists.append(tp)
+
+    with gzip.GzipFile(fileobj=lists[2], mode='rb') as gz:
+        matcher.corpus = joblib.load(gz)
+
+    with gzip.GzipFile(fileobj=lists[3], mode='rb') as gz:
+        matcher.idx = joblib.load(gz)
+    matcher.corpus_size = max(len(matcher.corpus), len(matcher.idx))
+    print(f"加载bytes完成，共 {matcher.corpus_size} 篇文献")
+
+    matcher.fit(matcher.corpus)
+
+    print(matcher.nn._fit_method)
+
+    print("fit 训练完成")
+
+    count = 0
+    bacth_list = []
+    n = min(100, matcher.corpus_size)
+    for i in matcher.corpus:
+        count = count + 1
+        bacth_list.append(i)
+        if count % 10000 == 0:
+            t1 = time.time()
+            index, similarities = matcher.batch_search(bacth_list, n=n)
+            for rank, (idxs, sims) in enumerate(zip(index, similarities)):
+                print({"keyid": matcher.idx[rank],
+                       "search_list": [(matcher.idx[idx], sim) for idx, sim in zip(idxs, sims)]})
+
+            t2 = time.time()
+            now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S,%f")[:-3]
+            print(now, count, t2 - t1)
+            bacth_list.clear()
+
+
+
+
+
+
+
+    # with open("t_8220_corpus.joblib","rb") as f:
+    #     buf = io.BytesIO(f.read())
+    #     buf.seek(0)
+    #     corpus = buf
+    #
+    # matcher.corpus = joblib.load(corpus)
+    # print(len(matcher.corpus))
+
+    # matcher.load_bytes(vec, nn, corpus)
+
+    # with open(r"C:\Users\Administrator\Desktop\update\part-02440\part-02440_1", "r", encoding="utf-8") as f:
+    #     lists = [line.strip() for line in f if line]
+    #
+    # matcher.fit(lists)
+
+    # matcher.load("./","test")
+
+    # query = r"herbdrug interaction in the protective effect of alpinia officinarum against gastric injury induced by indomethacin based on pharmacokinetic tissue distribution and excretion studies in rats"
+    # result = matcher.search(query, n=100)
+    # print("query", query)
+    # for rank, (idx, sim) in enumerate(result):
+    #     print(f"\nTop {rank + 1} [相似度: {sim:.4f}]:")
+    #     print(f"文献 #{idx}: {lists[idx]}")
+    # print(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
+    # time.sleep(100)
+
+    # for query in lists[:1000]:
+    #     indices, similarities = matcher.search(query, n=100)
+    #     print("query", query)
+    #     for rank, (idx, sim) in enumerate(zip(indices, similarities)):
+    #         print(f"\nTop {rank + 1} [相似度: {sim:.4f}]:")
+    #         print(f"文献 #{idx}: {lists[idx]}")
+    #     print(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
+    #     time.sleep(100)
+    #
+    # idx,sim = matcher.batch_search(lists[:1000], n=100)
+    # for rank, (idxs, sims) in enumerate(zip(idx,sim)):
+    #     tp = (lists[rank],[(lists[idx], sim) for idx,sim in zip(idxs,sims)])
+    #     print(tp)
+    # time.sleep(100)

re_common/v2/baselibrary/helpers/search_packge/fit_text_match.py

@@ -0,0 +1,254 @@
+import gzip
+import io
+import multiprocessing
+import os
+import time
+
+import jieba
+import joblib
+
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.neighbors import NearestNeighbors
+
+
+def create_gzip_joblib(obj):
+    temp_io = io.BytesIO()
+    with gzip.GzipFile(fileobj=temp_io, mode='wb') as f:
+        joblib.dump(obj, f)
+    temp_io.seek(0)
+    return temp_io
+
+
+def get_gzip_joblib(temp_io):
+    with gzip.GzipFile(fileobj=temp_io, mode='rb') as f:
+        loaded_obj = joblib.load(f)
+    return loaded_obj
+
+
+class JiebaTokenizer:
+    def __call__(self, doc):
+        return [tok for tok in jieba.cut(doc) if tok.strip()]
+
+
+class SplitTokenizer:
+    def __call__(self, doc):
+        return str.split(doc)
+
+
+def get_auto_n_jobs(fraction=0.5, max_jobs=16):
+    """
+    智能分配 CPU 核心数，用于设置 sklearn 的 n_jobs 参数。
+
+    参数:
+    fraction: 使用总核数的比例（如 0.5 表示一半）
+    max_jobs: 最大允许使用的核心数（防止过多）
+
+    返回:
+    合理的 n_jobs 整数值
+    """
+    total_cores = multiprocessing.cpu_count()
+    suggested = int(total_cores * fraction)
+    n_jobs = min(max(1, suggested), max_jobs)
+    return n_jobs
+
+
+class FitTextMatcher:
+    """
+    高性能文本匹配器
+    基于 TF-IDF + 最近邻搜索实现相似文献查找
+    """
+
+    def __init__(self, algorithm='brute', metric='cosine', n_jobs=-1, tokenizer=JiebaTokenizer()):
+        """
+        初始化文本匹配器
+
+        参数:
+        algorithm: 搜索算法 ('brute', 'kd_tree', 'ball_tree', 'lshf')
+        metric: 距离度量 ('cosine', 'euclidean', 'manhattan')
+        n_jobs: 并行作业数 (-1 表示使用所有CPU核心)
+        """
+        self.vectorizer = TfidfVectorizer(
+            max_features=None,  # 限制特征数量以提高性能
+            tokenizer=tokenizer,
+            stop_words=None,  # 中文不适用 'english'
+            ngram_range=(1, 2)  # 使用单字和双字组合
+        )
+
+        self.nn = NearestNeighbors(
+            algorithm=algorithm,
+            metric=metric,
+            n_jobs=n_jobs  # 并行处理加速搜索
+        )
+
+        self.corpus = None
+        self.corpus_size = 0
+
+    def fit(self, corpus):
+        """
+        训练匹配器
+        """
+        self.corpus = corpus
+        self.corpus_size = len(corpus)
+        print(f"处理 {self.corpus_size} 篇文献...")
+
+        # 向量化文本
+        start_time = time.time()
+        X = self.vectorizer.fit_transform(corpus)
+        vectorization_time = time.time() - start_time
+        print(f"TF-IDF 向量化完成, 耗时: {vectorization_time:.4f}秒")
+        print(f"特征维度: {X.shape[1]}")
+
+        # 训练最近邻模型
+        start_time = time.time()
+        self.nn.fit(X)
+        training_time = time.time() - start_time
+        print(f"最近邻模型训练完成, 耗时: {training_time:.4f}秒")
+
+        return self
+
+    def save(self, path, name):
+        """
+        保存模型和向量器
+        """
+        os.makedirs(path, exist_ok=True)
+        joblib.dump(self.vectorizer, os.path.join(path, name + "_vectorizer.joblib"))
+        joblib.dump(self.nn, os.path.join(path, name + "_nn_model.joblib"))
+        joblib.dump(self.corpus, os.path.join(path, name + "_corpus.joblib"))
+        print(f"模型保存至 {path}")
+        return self
+
+    def get_save_bytes_io(self, idx_list=None):
+        """
+        保存模型和向量器
+        """
+        if idx_list is None:
+            idx_list = []
+
+        result_list = []
+        for i in [self.vectorizer, self.nn, self.corpus, idx_list]:
+            temp_io = create_gzip_joblib(i)
+            result_list.append(temp_io)
+        print(f"获取模型字节码成功")
+        return result_list
+
+    def load(self, path, name):
+        """
+        从文件加载模型
+        """
+        self.vectorizer = joblib.load(os.path.join(path, name + "_vectorizer.joblib"))
+        self.nn = joblib.load(os.path.join(path, name + "_nn_model.joblib"))
+        self.corpus = joblib.load(os.path.join(path, name + "_corpus.joblib"))
+        self.corpus_size = len(self.corpus)
+        print(f"模型从 {path} 加载完成，共 {self.corpus_size} 篇文献")
+        return self
+
+    def load_bytes(self, vec, nn, corpus, idx):
+        # 解压并加载对象
+        with gzip.GzipFile(fileobj=vec, mode='rb') as gz:
+            self.vectorizer = joblib.load(gz)
+        with gzip.GzipFile(fileobj=nn, mode='rb') as gz:
+            self.nn = joblib.load(gz)
+        with gzip.GzipFile(fileobj=corpus, mode='rb') as gz:
+            self.corpus = joblib.load(gz)
+        with gzip.GzipFile(fileobj=idx, mode='rb') as gz:
+            self.idx = joblib.load(gz)
+        self.corpus_size = max(len(self.corpus), len(self.idx))
+        print(f"加载bytes完成，共 {self.corpus_size} 篇文献")
+        return self
+
+    def search(self, query, n=5, return_scores=True):
+        """
+        查找相似文献
+
+        参数:
+        query: 查询文本
+        n: 返回最相似文献的数量
+        return_scores: 是否返回相似度分数
+
+        返回:
+        匹配的文献索引和相似度分数
+        """
+        if self.corpus is None:
+            raise ValueError("请先使用 fit() 方法训练模型")
+
+        # 向量化查询文本
+        query_vec = self.vectorizer.transform([query])
+
+        # 查找最近邻
+        start_time = time.time()
+        distances, indices = self.nn.kneighbors(query_vec, n_neighbors=n)
+        search_time = time.time() - start_time
+
+        # print(f"搜索完成, 耗时: {search_time:.6f}秒")
+
+        # 将距离转换为相似度 (余弦距离 = 1 - 余弦相似度)
+        similarities = 1 - distances
+
+        # 返回结果
+        if return_scores:
+            return indices[0], similarities[0]
+        return indices[0]
+
+    def batch_search(self, queries, n=5, return_scores=True):
+        """
+        批量查找相似文献（一次处理多条 query）
+
+        参数:
+        queries: 查询文本列表
+        n: 每条 query 返回多少条相似文献
+        return_scores: 是否返回相似度分数
+
+        返回:
+        一个列表，包含每条 query 的匹配索引和相似度 [(indices1, sims1), (indices2, sims2), ...]
+        """
+        if self.corpus is None:
+            raise ValueError("请先使用 fit() 方法训练模型")
+
+        start_time = time.time()
+
+        # 向量化所有 query，一次性
+        query_vecs = self.vectorizer.transform(queries)
+
+        # 查找最近邻
+        distances, indices = self.nn.kneighbors(query_vecs, n_neighbors=n)
+        search_time = time.time() - start_time
+        # print(f"批量搜索完成，共 {len(queries)} 条，耗时: {search_time:.4f}秒")
+
+        if return_scores:
+            similarities = 1 - distances
+            return indices, similarities
+        return indices
+
+    def explain_match(self, query, index):
+        """
+        解释匹配结果 - 显示查询和匹配文献的关键词
+        """
+        # 获取TF-IDF特征名
+        feature_names = self.vectorizer.get_feature_names_out()
+
+        # 向量化查询和匹配文献
+        query_vec = self.vectorizer.transform([query])
+        doc_vec = self.vectorizer.transform([self.corpus[index]])
+
+        # 获取重要特征
+        query_data = zip(feature_names, query_vec.toarray()[0])
+        doc_data = zip(feature_names, doc_vec.toarray()[0])
+
+        # 筛选非零特征
+        query_keywords = [(word, score) for word, score in query_data if score > 0]
+        doc_keywords = [(word, score) for word, score in doc_data if score > 0]
+
+        # 按重要性排序
+        query_keywords.sort(key=lambda x: x[1], reverse=True)
+        doc_keywords.sort(key=lambda x: x[1], reverse=True)
+
+        # 打印结果
+        print(f"\n匹配文献 #{index} 解释:")
+        print(f"查询关键词: {[word for word, _ in query_keywords[:10]]}")
+        print(f"文献关键词: {[word for word, _ in doc_keywords[:10]]}")
+
+        # 计算共同关键词
+        common_keywords = set([word for word, _ in query_keywords[:20]]) & set([word for word, _ in doc_keywords[:20]])
+        print(f"共同关键词: {list(common_keywords)}")
+
+        return common_keywords

re_common/v2/baselibrary/helpers/search_packge/scikit_learn_text_matcher.py

@@ -0,0 +1,261 @@
+import numpy as np
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.neighbors import NearestNeighbors
+import time
+import matplotlib.pyplot as plt
+import seaborn as sns
+import pandas as pd
+from sklearn.metrics.pairwise import cosine_similarity
+
+class TextMatcher:
+    """
+    高性能文本匹配器
+    基于 TF-IDF + 最近邻搜索实现相似文献查找
+    """
+    
+    def __init__(self, algorithm='brute', metric='cosine', n_jobs=-1):
+        """
+        初始化文本匹配器
+        
+        参数:
+        algorithm: 搜索算法 ('brute', 'kd_tree', 'ball_tree', 'lshf')
+        metric: 距离度量 ('cosine', 'euclidean', 'manhattan')
+        n_jobs: 并行作业数 (-1 表示使用所有CPU核心)
+        """
+        self.vectorizer = TfidfVectorizer(
+            max_features=10000,  # 限制特征数量以提高性能
+            stop_words='english',  # 移除英文停用词
+            ngram_range=(1, 2)  # 使用单字和双字组合
+        )
+        
+        self.nn = NearestNeighbors(
+            algorithm=algorithm,
+            metric=metric,
+            n_jobs=n_jobs  # 并行处理加速搜索
+        )
+        
+        self.corpus = None
+        self.corpus_size = 0
+    
+    def fit(self, corpus):
+        """
+        训练匹配器
+        """
+        self.corpus = corpus
+        self.corpus_size = len(corpus)
+        print(f"处理 {self.corpus_size} 篇文献...")
+        
+        # 向量化文本
+        start_time = time.time()
+        X = self.vectorizer.fit_transform(corpus)
+        vectorization_time = time.time() - start_time
+        print(f"TF-IDF 向量化完成, 耗时: {vectorization_time:.4f}秒")
+        print(f"特征维度: {X.shape[1]}")
+        
+        # 训练最近邻模型
+        start_time = time.time()
+        self.nn.fit(X)
+        training_time = time.time() - start_time
+        print(f"最近邻模型训练完成, 耗时: {training_time:.4f}秒")
+        
+        return self
+    
+    def search(self, query, n=5, return_scores=True):
+        """
+        查找相似文献
+        
+        参数:
+        query: 查询文本
+        n: 返回最相似文献的数量
+        return_scores: 是否返回相似度分数
+        
+        返回:
+        匹配的文献索引和相似度分数
+        """
+        if self.corpus is None:
+            raise ValueError("请先使用 fit() 方法训练模型")
+        
+        # 向量化查询文本
+        query_vec = self.vectorizer.transform([query])
+        
+        # 查找最近邻
+        start_time = time.time()
+        distances, indices = self.nn.kneighbors(query_vec, n_neighbors=n)
+        search_time = time.time() - start_time
+        
+        print(f"搜索完成, 耗时: {search_time:.6f}秒")
+        
+        # 将距离转换为相似度 (余弦距离 = 1 - 余弦相似度)
+        similarities = 1 - distances
+        
+        # 返回结果
+        if return_scores:
+            return indices[0], similarities[0]
+        return indices[0]
+    
+    def explain_match(self, query, index):
+        """
+        解释匹配结果 - 显示查询和匹配文献的关键词
+        """
+        # 获取TF-IDF特征名
+        feature_names = self.vectorizer.get_feature_names_out()
+        
+        # 向量化查询和匹配文献
+        query_vec = self.vectorizer.transform([query])
+        doc_vec = self.vectorizer.transform([self.corpus[index]])
+        
+        # 获取重要特征
+        query_data = zip(feature_names, query_vec.toarray()[0])
+        doc_data = zip(feature_names, doc_vec.toarray()[0])
+        
+        # 筛选非零特征
+        query_keywords = [(word, score) for word, score in query_data if score > 0]
+        doc_keywords = [(word, score) for word, score in doc_data if score > 0]
+        
+        # 按重要性排序
+        query_keywords.sort(key=lambda x: x[1], reverse=True)
+        doc_keywords.sort(key=lambda x: x[1], reverse=True)
+        
+        # 打印结果
+        print(f"\n匹配文献 #{index} 解释:")
+        print(f"查询关键词: {[word for word, _ in query_keywords[:10]]}")
+        print(f"文献关键词: {[word for word, _ in doc_keywords[:10]]}")
+        
+        # 计算共同关键词
+        common_keywords = set([word for word, _ in query_keywords[:20]]) & set([word for word, _ in doc_keywords[:20]])
+        print(f"共同关键词: {list(common_keywords)}")
+        
+        return common_keywords
+
+# ======================
+# 演示使用
+# ======================
+
+if __name__ == "__main__":
+    # 1. 准备文献库 (实际应用中可从文件/数据库加载)
+    corpus = [
+        "机器学习是人工智能的一个分支，专注于开发算法让计算机从数据中学习",
+        "深度学习是机器学习的一个子领域，使用多层神经网络处理复杂模式",
+        "自然语言处理(NLP)使计算机能够理解、解释和生成人类语言",
+        "计算机视觉关注如何让计算机从图像和视频中获得高层次的理解",
+        "强化学习是一种机器学习方法，智能体通过与环境互动学习最优行为策略",
+        "监督学习使用标记数据训练模型，无监督学习则处理未标记数据",
+        "神经网络是受人脑启发的计算模型，由相互连接的节点层组成",
+        "卷积神经网络(CNN)特别适合处理图像识别任务",
+        "循环神经网络(RNN)设计用于处理序列数据，如文本和时间序列",
+        "Transformer模型通过自注意力机制处理序列数据，成为NLP的主流架构",
+        "生成对抗网络(GAN)由生成器和判别器组成，用于生成新数据样本",
+        "迁移学习允许将在一个任务上学到的知识应用到另一个相关任务",
+        "数据挖掘是从大型数据集中发现模式、关联和异常的过程",
+        "特征工程是创建更好的输入特征以提高模型性能的过程",
+        "过拟合发生在模型过于复杂，过度记忆训练数据而泛化能力差",
+        "正则化技术如L1/L2正则化用于防止过拟合",
+        "梯度下降是优化神经网络权重的主要算法",
+        "反向传播是训练神经网络的关键算法，用于计算梯度",
+        "激活函数如ReLU引入非线性，使神经网络能够学习复杂模式",
+        "批量归一化通过标准化层输入加速训练并提高稳定性"
+    ]
+    
+    # 2. 创建文本匹配器
+    print("="*50)
+    print("创建文本匹配器")
+    print("="*50)
+    matcher = TextMatcher(
+        algorithm='brute',  # 对于小数据集，暴力搜索足够快
+        n_jobs=-1           # 使用所有CPU核心
+    )
+    
+    # 3. 训练模型
+    matcher.fit(corpus)
+    
+    # 4. 执行查询
+    print("\n" + "="*50)
+    print("执行查询: '神经网络在人工智能中的应用'")
+    print("="*50)
+    query = "神经网络在人工智能中的应用"
+    indices, similarities = matcher.search(query, n=3)
+    
+    # 5. 显示结果
+    print("\n最相似的文献:")
+    for rank, (idx, sim) in enumerate(zip(indices, similarities)):
+        print(f"\nTop {rank+1} [相似度: {sim:.4f}]:")
+        print(f"文献 #{idx}: {corpus[idx]}")
+        
+        # 解释匹配
+        matcher.explain_match(query, idx)
+    
+    # 6. 性能测试 (可选)
+    print("\n" + "="*50)
+    print("性能测试")
+    print("="*50)
+    
+    # 测试不同文献库大小的性能
+    corpus_sizes = [100, 500, 1000, 5000]
+    times = []
+    
+    for size in corpus_sizes:
+        # 创建更大的文献库
+        large_corpus = corpus * (size // len(corpus) + 1)
+        large_corpus = large_corpus[:size]
+        
+        # 创建新的匹配器
+        test_matcher = TextMatcher(algorithm='brute', n_jobs=-1)
+        
+        # 测量训练时间
+        start_time = time.time()
+        test_matcher.fit(large_corpus)
+        train_time = time.time() - start_time
+        
+        # 测量查询时间
+        start_time = time.time()
+        test_matcher.search(query, n=5)
+        search_time = time.time() - start_time
+        
+        times.append((size, train_time, search_time))
+        print(f"文献库大小: {size} | 训练时间: {train_time:.4f}s | 查询时间: {search_time:.6f}s")
+    
+    # 可视化性能结果
+    sizes, train_times, search_times = zip(*times)
+    
+    plt.figure(figsize=(12, 6))
+    
+    plt.subplot(1, 2, 1)
+    plt.plot(sizes, train_times, 'o-')
+    plt.title('训练时间 vs 文献库大小')
+    plt.xlabel('文献数量')
+    plt.ylabel('时间 (秒)')
+    plt.grid(True)
+    
+    plt.subplot(1, 2, 2)
+    plt.plot(sizes, search_times, 'o-')
+    plt.title('查询时间 vs 文献库大小')
+    plt.xlabel('文献数量')
+    plt.ylabel('时间 (秒)')
+    plt.grid(True)
+    
+    plt.tight_layout()
+    plt.savefig('performance.png')
+    print("\n性能图表已保存为 'performance.png'")
+    
+    # 7. 相似度矩阵可视化 (可选)
+    print("\n" + "="*50)
+    print("文献相似度矩阵")
+    print("="*50)
+    
+    # 计算所有文献的TF-IDF向量
+    vectors = matcher.vectorizer.transform(corpus)
+    
+    # 计算余弦相似度矩阵
+    sim_matrix = cosine_similarity(vectors)
+    
+    # 创建DataFrame用于可视化
+    df = pd.DataFrame(sim_matrix, 
+                     columns=[f"Doc{i}" for i in range(len(corpus))],
+                     index=[f"Doc{i}" for i in range(len(corpus))])
+    
+    plt.figure(figsize=(12, 10))
+    sns.heatmap(df, cmap="YlGnBu", annot=False)
+    plt.title("文献相似度矩阵")
+    plt.tight_layout()
+    plt.savefig('similarity_matrix.png')
+    print("相似度矩阵已保存为 'similarity_matrix.png'")

re_common/v2/baselibrary/helpers/search_packge/test.py

@@ -0,0 +1,2 @@
+aa ="肿瘤 学 研究 进展"
+print(str.split(aa))

re_common/v2/baselibrary/utils/basehdfs.py

@@ -7,6 +7,8 @@ from hdfs import InsecureClient
 class HDFSUtils(object):
     """
     HDFS 工具类，封装常见的 HDFS 操作。
+
+    InsecureClient: 缺陷 写大文件数据时无法写入不报错
     """
 
     def __init__(self, hdfs_url, hdfs_user):

re_common/v2/baselibrary/utils/db.py

@@ -1,4 +1,6 @@
 import os
+import time
+
 import aiomysql
 import asyncio
 from contextlib import asynccontextmanager
@@ -92,6 +94,7 @@ aiomysql_pool = None
 pool_lock = asyncio.Lock()  # 全局异步锁
 
 
+
 async def init_aiomysql_pool_async():
     global aiomysql_pool
     if aiomysql_pool is None:
@@ -100,3 +103,54 @@ async def init_aiomysql_pool_async():
                 print(f"[{os.getpid()}] Initializing aiomysql pool...")
                 aiomysql_pool = await aiomysql.create_pool(**DB_CONFIG)
     return aiomysql_pool
+
+
+motor_fs = None
+client = None
+motor_fs_lock = asyncio.Lock()  # 全局异步锁
+_loop_id_mongo = None
+
+async def check_connection(client):
+    try:
+        print("check mongodb client ping")
+        await client.admin.command("ping")
+        return True
+    except Exception:
+        return False
+
+
+async def init_motor_async(uri, db_name, bucket_name, is_reload=False):
+    global motor_fs, client, _loop_id_mongo
+    is_ping = True
+
+    if _loop_id_mongo is not None:
+        loop_id = id(asyncio.get_running_loop())
+        if loop_id != _loop_id_mongo:
+            is_reload = True
+
+    # 防止 每次都检查 只有 is_reload 时才检查连接
+    if is_reload:
+        is_ping = await check_connection(client)
+    if motor_fs is None or not is_ping:
+        async with motor_fs_lock:
+            if motor_fs is None or not is_ping:
+                print(f"[{os.getpid()}] Initializing motor_fs...")
+                from motor.motor_asyncio import AsyncIOMotorClient, AsyncIOMotorGridFSBucket
+                client = AsyncIOMotorClient(uri)
+                db = client[db_name]
+                motor_fs = AsyncIOMotorGridFSBucket(database=db, bucket_name=bucket_name)
+                _loop_id_mongo = id(asyncio.get_running_loop())
+    return motor_fs, client
+
+# async def run_main():
+#     while True:
+#         uri = "mongodb://192.168.98.80:27001/wpdc"
+#         db_name = "wpdc"
+#         bucket_name = "sci_doc"
+#         motor_fs, client = await init_motor_async(uri, db_name, bucket_name,is_reload=True)
+#         # print(await check_connection(client))
+#         time.sleep(3)
+#
+#
+# if __name__ == "__main__":
+#     asyncio.run(run_main())

re_common/v2/baselibrary/utils/stringutils.py

@@ -1,12 +1,11 @@
 import re
 import threading
+from html.parser import HTMLParser
 from itertools import combinations
 
 import regex
 import unicodedata
-from html.parser import HTMLParser
-
-from unidecode import unidecode
+from parsel import Selector
 
 from re_common.v2.baselibrary.utils.string_smi import JaroDamerauLevenshteinMaxSim
 
@@ -173,12 +172,19 @@ class HTMLTextExtractor(HTMLParser):
         return cls._thread_local.parser
 
 
+# def clean_html(html):
+#     parser = HTMLTextExtractor.get_parser()
+#     parser.reset_state()
+#     parser.feed(html)
+#     parser.close()
+#     return parser.get_text()
+
 def clean_html(html):
-    parser = HTMLTextExtractor.get_parser()
-    parser.reset_state()
-    parser.feed(html)
-    parser.close()
-    return parser.get_text()
+    """使用 Parsel 提取 HTML 中的纯文本"""
+    sel = Selector(text=html, type='html')
+    # 提取所有文本（包括子元素的文本）
+    text = sel.xpath("string()").getall()
+    return "".join(text).strip()
 
 
 def remove_spaces_between_chinese_characters(text):

{re_common-10.0.34.dist-info → re_common-10.0.36.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.1
 Name: re_common
-Version: 10.0.34
+Version: 10.0.36
 Summary: a library about all python projects
 Home-page: https://gitee.com/xujiangios/re-common
 Author: vic
@@ -11,14 +11,6 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.6
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Dynamic: author
-Dynamic: author-email
-Dynamic: classifier
-Dynamic: description
-Dynamic: description-content-type
-Dynamic: home-page
-Dynamic: requires-python
-Dynamic: summary
 
 
     这是一个基础类，依赖很多的第三方包，是一个用得到的第三方库的封装，可以在此基础上迅速构建项目

{re_common-10.0.34.dist-info → re_common-10.0.36.dist-info}/RECORD

@@ -163,14 +163,19 @@ re_common/studio/streamlitstudio/first_app.py,sha256=t7Fw8YDlub7G9q99GgVo_3sPZXU
 re_common/studio/streamlitstudio/uber_pickups.py,sha256=cvrV5e8vRBM2_CpVDBE-f3V4mGFK9SqpRPZK8TEqr6U,785
 re_common/v2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 re_common/v2/baselibrary/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-re_common/v2/baselibrary/business_utils/BusinessStringUtil.py,sha256=njPcRgeBWpnZr5u2cPAO4qdWBq-CgTn99rJuvWFcChk,6788
+re_common/v2/baselibrary/business_utils/BusinessStringUtil.py,sha256=PaYg_5fLV2FM-Hp8r08RkLorOilXHvCtBLfRUT2MDJk,7681
 re_common/v2/baselibrary/business_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-re_common/v2/baselibrary/business_utils/baseencodeid.py,sha256=3f52e0jtgCFzPEyReia8TupwiE64t_VyBT-a7uQCXAY,3595
+re_common/v2/baselibrary/business_utils/baseencodeid.py,sha256=9opWNmDKg0ezSJKERHKaPF48W9PuChiUDNe64Xv08is,3598
 re_common/v2/baselibrary/business_utils/full_doi_path.py,sha256=vsoS1ZGyNzeORon_z1sHt1M41sS22pvJHMgWJH3xZ-M,3378
 re_common/v2/baselibrary/business_utils/rel_tools.py,sha256=LfnGFCkUSxg1SHvOMOQdP1PiHxIKqk7Syuk5YYpjJag,295
 re_common/v2/baselibrary/decorators/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 re_common/v2/baselibrary/decorators/utils.py,sha256=Q4D6KKCQxvNBXZkPQQn14keKKJpGtg8TUSakjJU40s0,2056
 re_common/v2/baselibrary/helpers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+re_common/v2/baselibrary/helpers/search_packge/NearestNeighbors_test.py,sha256=ygN_OzO6XDR-XcyY8hYfFNjfP2luek81lmV-bCbkb7A,3686
+re_common/v2/baselibrary/helpers/search_packge/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+re_common/v2/baselibrary/helpers/search_packge/fit_text_match.py,sha256=cMN4W7xuL6xzHp4U7SJnkE5i4FU2pMmiDivcDvD8zgA,8543
+re_common/v2/baselibrary/helpers/search_packge/scikit_learn_text_matcher.py,sha256=Ri8Ul2_URq1TVvlXwG0OvqBo9_LSpivvdvjQM7xr01I,9947
+re_common/v2/baselibrary/helpers/search_packge/test.py,sha256=jYDa6s66jqiz6xEhXMPLqmONFbmfv-EgxaVpdHbGk4U,52
 re_common/v2/baselibrary/s3object/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 re_common/v2/baselibrary/s3object/baseboto3.py,sha256=mXuIFx99pnrPGQ4LJCZwlN1HLbaU-OWLwck0cVzW6hc,11203
 re_common/v2/baselibrary/tools/WeChatRobot.py,sha256=sKBt2gPsfj0gzV6KaLSAhIhL-j3qNfHfqE-lII1LVwM,3537
@@ -199,17 +204,17 @@ re_common/v2/baselibrary/utils/api_net_utils.py,sha256=22q3WMWiKVg1IVGr4y2D1Jrjh
 re_common/v2/baselibrary/utils/author_smi.py,sha256=Fh276u7za-GF_tK9xpBFYF5q1E3_tX22ZouWC8U7w8o,13831
 re_common/v2/baselibrary/utils/base_string_similarity.py,sha256=a40a79ttwoG_gC_hxMNB-sMXXecgICoRDWrj0DW8iEE,7749
 re_common/v2/baselibrary/utils/basedict.py,sha256=sH3_RZ8u4649-jX2V1uKNNkjJVUijZBDp6SdqncOZ88,1583
-re_common/v2/baselibrary/utils/basehdfs.py,sha256=TPwFct_-UrmO1KCbo4gpV77rsnlCQDumNBbQKL0ZI9o,5953
+re_common/v2/baselibrary/utils/basehdfs.py,sha256=wwvk4kvipD-AVWCF0WRzBTM2q2wAvQtanLK-Hjp8FOk,6026
 re_common/v2/baselibrary/utils/basepika.py,sha256=ifOb3UsGj79k40aD9UK6-5BMPw43ZAo0SO3AYD4q4vw,7332
 re_common/v2/baselibrary/utils/basetime.py,sha256=b7U_ho6nE3fjYBxSkdMHXUOd3ClH6KkW_7p7l2Gs4gA,3038
-re_common/v2/baselibrary/utils/db.py,sha256=JktBem1IAsg7m-Wnsa-AT0Hj4bIkHyoKGBEDI1rL5Fc,3135
+re_common/v2/baselibrary/utils/db.py,sha256=5IOYOOGe6mNKhOeaCAq0iif5yJdDotBmrsMZmT1syjs,4862
 re_common/v2/baselibrary/utils/json_cls.py,sha256=M93piYtmgm_wP8E57culTrd_AhHLoGg6PqeAJYdW2SM,438
 re_common/v2/baselibrary/utils/mq.py,sha256=UHpO8iNIHs91Tgp-BgnSUpZwjWquxrGLdpr3FMMv2zw,2858
 re_common/v2/baselibrary/utils/n_ary_expression_tree.py,sha256=-05kO6G2Rth7CEK-5lfFrthFZ1Q0-0a7cni7mWZ-2gg,9172
 re_common/v2/baselibrary/utils/string_bool.py,sha256=vxnjSFOfuHWGxkqaIbUNn21opx5tfV1uCXSahFfp1mU,6197
 re_common/v2/baselibrary/utils/string_clear.py,sha256=Ympa0Cs2y_72QeeyMS8de8y_QgtEFJJQ0AgHnylbMUc,7861
 re_common/v2/baselibrary/utils/string_smi.py,sha256=cU0WAWHRGnGoVQx3eCEKeM_q_olFNzRTJe7rSe586SY,741
-re_common/v2/baselibrary/utils/stringutils.py,sha256=eeuQYgXkWJ9apvyrYPcCCU3biTY9nD1KHos4_1ESNJE,7883
+re_common/v2/baselibrary/utils/stringutils.py,sha256=F1JZ9vfSWM0TEffiNUGVE40yrzXz0fuzrYyys-PgDqw,8144
 re_common/vip/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 re_common/vip/base_step_process.py,sha256=VXXiNj0I5CpzXIMCgOPU86bzDJkSBkUS-9CpZIl_GOk,205
 re_common/vip/baseencodeid.py,sha256=nERoe89ueFM52bG7xwJdflcZHk6T2RQQKbc5uUZc3RM,3272
@@ -236,8 +241,8 @@ re_common/vip/title/transform/TransformRegulationTitleToZt.py,sha256=LKRdIsWKues
 re_common/vip/title/transform/TransformStandardTitleToZt.py,sha256=-fCKAbSBzXVyQDCE61CalvR9E_QzQMA08QOO_NePFNI,5563
 re_common/vip/title/transform/TransformThesisTitleToZt.py,sha256=QS-uV0cQrpUFAcKucuJQ9Ue2VRQH-inmfn_X3IplfRo,5488
 re_common/vip/title/transform/__init__.py,sha256=m83-CWyRq_VHPYHaALEQlmXrkTdrZ3e4B_kCfBYE-uc,239
-re_common-10.0.34.dist-info/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
-re_common-10.0.34.dist-info/METADATA,sha256=R_h_bdwzyfsnmkXt3-sV9JKg9-hWlpCkWb0iw7pp3-c,764
-re_common-10.0.34.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-re_common-10.0.34.dist-info/top_level.txt,sha256=_H9H23zoLIalm1AIY_KYTVh_H0ZnmjxQIxsvXtLv45o,10
-re_common-10.0.34.dist-info/RECORD,,
+re_common-10.0.36.dist-info/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
+re_common-10.0.36.dist-info/METADATA,sha256=2BGTDBrd17-eWFaKfrQbE9xZdyOe3i-NwME64lIRjvs,582
+re_common-10.0.36.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+re_common-10.0.36.dist-info/top_level.txt,sha256=_H9H23zoLIalm1AIY_KYTVh_H0ZnmjxQIxsvXtLv45o,10
+re_common-10.0.36.dist-info/RECORD,,

{re_common-10.0.34.dist-info → re_common-10.0.36.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.8.0)
+Generator: bdist_wheel (0.43.0)
 Root-Is-Purelib: true
 Tag: py3-none-any