re-common 10.0.13__py3-none-any.whl → 10.0.15__py3-none-any.whl

re_common/v2/baselibrary/decorators/utils.py

@@ -0,0 +1,59 @@
+import warnings
+import functools
+
+# 全局集合，用于记录已警告的函数或类
+_warned_once = set()
+
+
+def deprecated(message=None):
+    """
+    装饰器：标记函数或类为已废弃，整个进程只发出一次警告。
+
+    Args:
+        message (str): 自定义警告信息，默认为 None。
+    """
+
+    def decorator(obj):
+        # 如果是函数
+        if isinstance(obj, type(lambda: None)):
+            @functools.wraps(obj)
+            def wrapper(*args, **kwargs):
+                obj_id = id(obj)  # 使用对象的内存地址作为唯一标识
+                if obj_id not in _warned_once:
+                    default_msg = f"函数 {obj.__name__} 已不建议使用。"
+                    warn_msg = f"{default_msg} {message}" if message else default_msg
+                    warnings.warn(
+                        warn_msg,
+                        category=DeprecationWarning,
+                        stacklevel=2
+                    )
+                    _warned_once.add(obj_id)  # 记录已警告
+                return obj(*args, **kwargs)
+
+            return wrapper
+
+        # 如果是类
+        elif isinstance(obj, type):
+            orig_init = obj.__init__
+
+            @functools.wraps(orig_init)
+            def new_init(self, *args, **kwargs):
+                obj_id = id(obj)
+                if obj_id not in _warned_once:
+                    default_msg = f"类 {obj.__name__} 已不建议使用。"
+                    warn_msg = f"{default_msg} {message}" if message else default_msg
+                    warnings.warn(
+                        warn_msg,
+                        category=DeprecationWarning,
+                        stacklevel=2
+                    )
+                    _warned_once.add(obj_id)  # 记录已警告
+                orig_init(self, *args, **kwargs)
+
+            obj.__init__ = new_init
+            return obj
+
+        else:
+            raise TypeError("此装饰器仅适用于函数和类")
+
+    return decorator

re_common/v2/baselibrary/tools/ac_ahocorasick.py

@@ -0,0 +1,76 @@
+import pickle
+
+import ahocorasick
+
+
+class ACTool(object):
+
+    def __init__(self):
+        self.automaton = ahocorasick.Automaton()
+
+    def add_word(self, key, value, overwrite=True) -> bool:
+        """
+        为 AC 机添加数据,默认情况下 key重复直接覆盖
+        :param key: 要添加的关键字
+        :param value: 对应的值
+        :param overwrite: 是否覆盖已有的 key，默认为 True
+        :return: 是否成功添加或覆盖
+        """
+        if key in self.automaton:  # 检查 key 是否已存在
+            if overwrite:  # 如果允许覆盖
+                self.automaton.add_word(key, value)
+                return True
+            else:  # 不允许覆盖，跳过
+                return False
+        else:  # key 不存在，直接添加
+            self.automaton.add_word(key, value)
+            return True
+
+    def is_exists_key(self, key) -> bool:
+        # 是否存在key
+        if self.automaton.exists(key):
+            return True
+        else:
+            return False
+
+    def make_automaton(self):
+        """
+        添加完词后需要构建
+        """
+        self.automaton.make_automaton()
+
+    def iter(self, key):
+        """
+        结果为可迭代对象 可通过list 转换 [(end_index, value)]
+        tool.add_word("he", "word1")
+        tool.add_word("hello", "word2")
+
+        # 在字符串中查找匹配
+        input_string = "hello world"
+        matches = list(tool.automaton.iter(input_string))
+        print(matches)  # [(1, 'word1'), (4, 'word2')]
+
+        (1, 'word1'):
+        end_index = 1: 表示匹配的关键字 "he" 在 input_string = "hello world" 中的结束位置是索引 1（即字符串 "he" 的最后一个字符 'e' 的位置）。
+        "hello world" 的索引：h(0)e(1)l(2)l(3)o(4) (5)w(6)o(7)r(8)l(9)d(10)。
+        value = 'word1': 表示匹配的关键字 "he" 对应的值是 "word1"。
+        (4, 'word2'):
+        end_index = 4: 表示匹配的关键字 "hello" 在 input_string = "hello world" 中的结束位置是索引 4（即字符串 "hello" 的最后一个字符 'o' 的位置）。
+        value = 'word2': 表示匹配的关键字 "hello" 对应的值是 "word2"。
+
+        注意: 结果只会返回 value 不会返回 key，如果需要key  请将key 组合到结果中
+        """
+
+        result_iter = self.automaton.iter(key)  # ahocorasick.AutomatonSearchIter
+        return result_iter
+    def save(self,local_temp_path):
+        """
+        将构建好的ac自动机保存到本地
+        """
+        self.automaton.save(local_temp_path,pickle.dumps)
+
+    def load(self,local_temp_path):
+        """
+        加载已经构建好的ac自动机
+        """
+        self.automaton=ahocorasick.load(local_temp_path, pickle.loads)

re_common/v2/baselibrary/tools/hdfs_data_processer.py

@@ -0,0 +1,318 @@
+import asyncio
+import gzip
+import json
+import sqlite3
+import time
+import os
+from io import BytesIO
+from typing import Callable, Any, List
+
+from hdfs import InsecureClient
+
+
+class HDFSDataProcessor:
+    def __init__(
+            self,
+            hdfs_url="http://VIP-DC-MASTER-2:9870",
+            hdfs_user="root",
+            db_file="processed_files.db",
+            batch_size=50,
+            retry_limit=3,
+    ):
+        self.hdfs_url = hdfs_url
+        self.hdfs_user = hdfs_user
+        self.db_file = db_file
+        self.batch_size = batch_size
+        self.retry_limit = retry_limit
+        self.client = InsecureClient(self.hdfs_url, user=self.hdfs_user)
+        self.read_hdfs_fanc = {"all": self.all_read_gz, "batch": self.batch_read_gz}
+        self.read_hdfs_model = "all"
+        self.init_db()
+
+    def init_db(self):
+        """初始化 SQLite 数据库"""
+        with sqlite3.connect(self.db_file) as conn:
+            cursor = conn.cursor()
+            cursor.execute("""
+                    CREATE TABLE IF NOT EXISTS processed_files (
+                        file_path TEXT PRIMARY KEY
+                    )
+                    """)
+            conn.commit()
+
+    def save_processed_file(self, file_path):
+        """保存处理过的文件"""
+        with sqlite3.connect(self.db_file) as conn:
+            cursor = conn.cursor()
+            cursor.execute("INSERT OR IGNORE INTO processed_files (file_path) VALUES (?)", (file_path,))
+            conn.commit()
+
+    def is_file_processed(self, file_path):
+        """检查文件是否已处理"""
+        with sqlite3.connect(self.db_file) as conn:
+            cursor = conn.cursor()
+            cursor.execute("SELECT file_path FROM processed_files WHERE file_path = ?", (file_path,))
+            result = cursor.fetchone()
+        return result is not None
+
+    def list_gz_files(self, hdfs_dir):
+        """列出 HDFS 目录中的所有 gzip 文件"""
+        return [f"{hdfs_dir}/{file[0]}" for file in self.client.list(hdfs_dir, status=True) if file[0].endswith(".gz")]
+
+    def count_total_lines(self, gz_file_path: str):
+        with self.client.read(gz_file_path) as hdfs_file:
+            with gzip.GzipFile(fileobj=hdfs_file) as gz:
+                return sum(1 for _ in gz)
+
+    def batch_read_gz(self, gz_file_path: str):
+        """分批读取 gz 文件"""
+        with self.client.read(gz_file_path) as hdfs_file:
+            with gzip.GzipFile(fileobj=hdfs_file) as gz:
+                while True:
+                    lines = []
+                    for _ in range(self.batch_size):
+                        try:
+                            line = next(gz)
+                            if line.strip():  # 移除空行
+                                lines.append(line.decode("utf-8"))  # 解码
+                        except StopIteration:  # 文件已读完
+                            break
+                    if not lines:
+                        break
+                    yield lines
+
+    def all_read_gz(self, gz_file_path: str, encoding='utf-8'):
+        """
+        读取 HDFS 上的 .gz 文件内容。
+        :param hdfs_path: HDFS 文件路径（必须以 .gz 结尾）
+        :param encoding: 文件编码格式（默认 utf-8）
+        :return: 文件内容
+        """
+        with self.client.read(gz_file_path) as reader:  # 以二进制模式读取
+            compressed_data = reader.read()  # 读取压缩数据
+            with gzip.GzipFile(fileobj=BytesIO(compressed_data)) as gz_file:  # 解压缩
+                content = gz_file.read().decode(encoding)  # 解码为字符串
+                print(f"文件读取成功: {gz_file_path}")
+                lines = [i for i in content.splitlines() if i.strip()]
+                result = [lines[i:i + self.batch_size] for i in range(0, len(lines), self.batch_size)]
+                return result
+
+    async def process_data(self, data, process_func):
+        """处理数据并执行处理函数"""
+        retry_count = 0
+        while retry_count < self.retry_limit:
+            try:
+                await process_func(data)
+                return  # 成功处理后退出
+            except Exception as e:
+                retry_count += 1
+                print(f"处理数据时发生错误: {e}, 正在重试 {retry_count}/{self.retry_limit}, data: {data}")
+                await asyncio.sleep(2 ** retry_count)
+        print(f"处理数据失败, 达到重试上限, data: {data}")
+
+    async def process_file(self, hdfs_file_path, process_func):
+        """处理单个 gz 文件"""
+        total_lines = self.count_total_lines(hdfs_file_path)
+        processed_lines = 0
+        start_time = time.time()
+        #   # 这里根据不同的配置选用不同的读取文件的方法
+        for lines in self.read_hdfs_fanc[self.read_hdfs_model](hdfs_file_path):
+            processing_start_time = time.time()  # 记录本批处理开始时间
+
+            tasks = []
+            for line in lines:
+                try:
+                    data = json.loads(line)
+                    tasks.append(self.process_data(data, process_func))
+                except json.JSONDecodeError as e:
+                    print(f"解析JSON失败: {e}, 行内容: {line.strip()}")
+
+            # await AsyncTaskPool(self.batch_size).run(tasks) # AsyncTaskPool 适用于一次提交所有任务, 限制并发数执行
+            await asyncio.gather(*tasks)
+
+            processed_lines += len(lines)
+
+            elapsed_time = time.time() - start_time  # 已用时间
+            processing_time = time.time() - processing_start_time  # 本次处理时间
+            avg_processing_time = (
+                (elapsed_time * 1000) / processed_lines if processed_lines > 0 else float("inf")
+            )  # 平均每条数据的处理时间（毫秒）
+
+            # 估算剩余时间
+            remaining_time = (
+                ((avg_processing_time / 1000) * (total_lines - processed_lines))
+                if processed_lines > 0
+                else float("inf")
+            )
+
+            # 显示总进度信息
+            print(
+                f"文件: {hdfs_file_path} 总进度: {processed_lines}/{total_lines} 行 | "
+                f"已用时间: {elapsed_time:.2f}秒 | 本次处理时间: {processing_time:.2f}秒 | "
+                f"预估剩余时间: {remaining_time:.2f}秒 | 平均每条处理时间: {avg_processing_time:.2f}毫秒"
+            )
+
+        # 最终进度显示
+        final_elapsed_time = time.time() - start_time  # 最终已用时间
+        print(
+            f"文件: {hdfs_file_path} 处理完成 | 总进度: {processed_lines}/{total_lines} 行 | "
+            f"总已用时间: {final_elapsed_time:.2f}秒 | "
+            f"平均每条处理时间: {(final_elapsed_time * 1000) / processed_lines:.2f}毫秒"
+            if processed_lines > 0
+            else "处理无数据"
+        )
+
+        self.save_processed_file(hdfs_file_path)  # 保存处理过的文件
+
+    async def retry_process_file(self, hdfs_file_path, process_func):
+        """带重试机制的文件处理"""
+        retry_count = 0
+        while retry_count < self.retry_limit:
+            try:
+                await self.process_file(hdfs_file_path, process_func)
+                return True  # 成功处理后退出
+            except Exception as e:
+                retry_count += 1
+                print(f"处理文件 {hdfs_file_path} 时发生错误: {e}，正在重试 {retry_count}/{self.retry_limit}")
+                await asyncio.sleep(2 ** retry_count)
+        print(f"处理文件 {hdfs_file_path} 失败，达到重试上限")
+        return False
+        # raise
+
+    async def batch_process_file(self, hdfs_dir: str, process_func: Callable[[dict], Any]):
+        """批量更新所有 gz 文件"""
+        gz_files = self.list_gz_files(hdfs_dir)
+        all_succeed = True
+        for hdfs_file_path in gz_files:
+            if self.is_file_processed(hdfs_file_path):
+                print(f"跳过已处理文件: {hdfs_file_path}")
+                continue  # 如果文件已处理，跳过
+            succeed = await self.retry_process_file(hdfs_file_path, process_func)  # 处理文件
+            if succeed is False:
+                all_succeed = False
+
+        if all_succeed:
+            # 处理完成后删除数据库文件
+            try:
+                if os.path.exists(self.db_file):
+                    os.remove(self.db_file)
+                    print(f"已删除断点重试文件: {self.db_file}")
+            except Exception as e:
+                print(f"删除断点重试文件失败: {e}")
+
+    async def process_file_bulk(self, hdfs_file_path, process_func):
+        """按批次处理单个文件，批量数据传递给处理函数"""
+        total_lines = self.count_total_lines(hdfs_file_path)
+        processed_lines = 0
+        start_time = time.time()
+
+        tasks = []
+        # 这里根据不同的配置选用不同的读取文件的方法
+        for lines in self.read_hdfs_fanc[self.read_hdfs_model](hdfs_file_path):
+            processing_start_time = time.time()  # 记录本批处理开始时间
+
+            batch_data = []
+            for line in lines:
+                try:
+                    data = json.loads(line)
+                    batch_data.append(data)
+                except json.JSONDecodeError as e:
+                    print(f"解析JSON失败: {e}, 行内容: {line.strip()}")
+
+            # 处理读取到的批次数据
+            if batch_data:
+                tasks.append(process_func(batch_data))  # 将批次数据传递给处理函数并收集任务
+                processed_lines += len(batch_data)  # 更新已处理行数
+
+            # 当积累的任务数量达到 batch_size 时并发处理所有任务
+            if len(tasks) >= self.batch_size:
+                await asyncio.gather(*tasks)  # 同时处理多个批次
+
+                elapsed_time = time.time() - start_time  # 已用时间
+                processing_time = time.time() - processing_start_time  # 本次处理时间
+                avg_processing_time = (
+                    (elapsed_time * 1000) / processed_lines if processed_lines > 0 else float("inf")
+                )  # 平均每条数据的处理时间（毫秒）
+
+                # 估算剩余时间
+                remaining_time = (
+                    ((avg_processing_time / 1000) * (total_lines - processed_lines))
+                    if processed_lines > 0
+                    else float("inf")
+                )
+
+                # 显示总进度信息
+                print(
+                    f"文件: {hdfs_file_path} 总进度: {processed_lines}/{total_lines} 行 | "
+                    f"已用时间: {elapsed_time:.2f}秒 | 本次处理时间: {processing_time:.2f}秒 | "
+                    f"预估剩余时间: {remaining_time:.2f}秒 | 平均每条处理时间: {avg_processing_time:.2f}毫秒"
+                )
+
+                # 清空任务列表，准备下一批处理
+                tasks.clear()
+            # 处理剩余的任务
+        if tasks:
+            await asyncio.gather(*tasks)  # 处理未达到 batch_size 的剩余任务
+
+        # 最终进度显示
+        final_elapsed_time = time.time() - start_time  # 最终已用时间
+        print(
+            f"文件: {hdfs_file_path} 处理完成 | 总进度: {processed_lines}/{total_lines} 行 | "
+            f"总已用时间: {final_elapsed_time:.2f}秒 | "
+            f"平均每条处理时间: {(final_elapsed_time * 1000) / processed_lines:.2f}毫秒"
+            if processed_lines > 0
+            else "处理无数据"
+        )
+
+        self.save_processed_file(hdfs_file_path)
+
+    async def retry_process_file_bulk(self, hdfs_file_path, process_func):
+        """带重试机制的批量文件处理"""
+        retry_count = 0
+        while retry_count < self.retry_limit:
+            try:
+                await self.process_file_bulk(hdfs_file_path, process_func)
+                return True  # 成功处理后退出
+            except Exception as e:
+                retry_count += 1
+                print(f"处理文件 {hdfs_file_path} 时发生错误: {e}，正在重试 {retry_count}/{self.retry_limit}")
+                await asyncio.sleep(2 ** retry_count)
+        print(f"处理文件 {hdfs_file_path} 失败，达到重试上限")
+        return False
+
+    async def batch_process_file_bulk(self, hdfs_dir: str, process_func: Callable[[List[dict]], Any]):
+        """批量处理 gz 文件中的数据"""
+        gz_files = self.list_gz_files(hdfs_dir)
+        all_succeed = True
+        for hdfs_file_path in gz_files:
+            if self.is_file_processed(hdfs_file_path):
+                print(f"跳过已处理文件: {hdfs_file_path}")
+                continue  # 跳过已处理文件
+            succeed = await self.retry_process_file_bulk(hdfs_file_path, process_func)
+            if succeed is False:
+                all_succeed = False
+
+        if all_succeed:
+            # 处理完成后删除数据库文件
+            try:
+                if os.path.exists(self.db_file):
+                    os.remove(self.db_file)
+                    print(f"已删除断点重试文件: {self.db_file}")
+            except Exception as e:
+                print(f"删除断点重试文件失败: {e}")
+
+# # 使用示例
+# async def update_refer(data: dict):
+#     ref_id = data["ref_id"]
+#     url = f"http://192.168.98.79:8150/v1/fact_refer/update/{ref_id}"
+#     update_data = data["update_data"]
+#     if not update_data:
+#         return
+#
+#     # 此处为实际处理逻辑
+#     await ApiNetUtils.fetch_post(url=url, payload=update_data)
+#
+#
+# if __name__ == "__main__":
+#     processor = HDFSDataProcessor()  # 实例化数据处理类
+#     asyncio.run(processor.batch_process_file("/user/libaiyun/output/confidence", update_refer))

re_common/v2/baselibrary/tools/search_hash_tools.py

@@ -3,9 +3,10 @@ from typing import List
 import jieba
 from datasketch import MinHash, minhash
 
+from re_common.v2.baselibrary.decorators.utils import deprecated
 from re_common.v2.baselibrary.utils.string_bool import is_single_cjk_char
 
-
+@deprecated("请使用 TextMatcherV2 中的方法代替。")
 def tokenize(text: str, stopwords=None) -> List[str]:
     """
     分词并移除停用词
@@ -32,7 +33,7 @@ def tokenize(text: str, stopwords=None) -> List[str]:
     words = [w for w in words if w not in stopwords and w.strip()]
     return words
 
-
+@deprecated("请使用 TextMatcherV2 中的方法代替。")
 def create_minhash(words: List[str], num_perm=128) -> MinHash:
     """
     为分词结果创建 MinHash
@@ -42,7 +43,7 @@ def create_minhash(words: List[str], num_perm=128) -> MinHash:
         minhash.update(word.encode("utf-8"))
     return minhash
 
-
+@deprecated("请使用 TextMatcherV2 中的方法代替。")
 def get_str_minhash(title):
     from re_common.v2.baselibrary.utils.string_clear import rel_clear
     rel_title = rel_clear(title)

re_common/v2/baselibrary/tools/text_matcher.py

@@ -1,10 +1,16 @@
+import pickle
+
 import jieba
 import re
-from typing import List, Dict, Tuple, Set, Optional, Union
+from typing import List, Dict, Tuple, Set, Optional, Union, Hashable, Protocol
 from datasketch import MinHash, MinHashLSH
 
+from re_common.v2.baselibrary.decorators.utils import deprecated
+from re_common.v2.baselibrary.utils.string_bool import is_single_cjk_char
+
 
-class TextMatcher:
+@deprecated("请使用 TextMatcherV2 代替。")
+class TextMatcher(object):
     def __init__(
             self,
             threshold: float = 0.5,
@@ -188,36 +194,133 @@ class TextMatcher:
         self.doc_counter = 0
 
 
-if __name__ == "__main__":
-    # 创建匹配器实例
-    matcher = TextMatcher(
-        threshold=0.1,  # 相似度阈值
-        num_perm=128,  # MinHash 排列数
-    )
+# 定义一个协议，描述“像鸭子一样”的行为
+class TokenizeDuckLike(Protocol):
+    def get_words(self, text) -> List:
+        pass
+
+
+class JiebaTokenize(object):
+
+    def __init__(self, stopwords=None):
+        self.stopwords = stopwords
+
+    def get_words(self, text) -> List:
+
+        if self.stopwords is None:
+            stopwords = []
+        words = jieba.lcut(text)
+
+        # 统计单字符数据 长度，防止结巴分词分不了的单词 将数据分为单个字符
+
+        # 这里为什么使用函数 而不是在推导式中兼容，主要是在一些 spark中 推导式的if 条件不遵循最短路径原则会将表达式当做一个整体算子
+        def is_singel_en(i):
+            if len(i) == 1 and not is_single_cjk_char(i):
+                return True
+            return False
+
+        one_char_size = len([i for i in words if is_singel_en(i)])
+        all_size = len(words)
+        # 如果单字符个数超过一定比例 就直接用空格分词
+        if all_size != 0 and one_char_size / all_size > 0.6:
+            words = [i for i in text.split() if i.strip()]
+
+        # 过滤停用词和空字符
+        words = [w for w in words if w not in stopwords and w.strip()]
+        return words
+
+
+class TextMatcherV2(object):
+
+    def __init__(
+            self,
+            threshold: float = 0.5,
+            num_perm: int = 128,
+            tdk: TokenizeDuckLike = None
+    ):
+        """
+        初始化文本匹配器
+
+        Args:
+            threshold: LSH 相似度阈值
+            num_perm: MinHash 排列数
+            stopwords_path: 停用词文件路径
+            user_dict_path: 用户自定义词典路径
+        """
+        self.threshold = threshold
+        self.num_perm = num_perm
+        self.lsh = MinHashLSH(threshold=threshold, num_perm=num_perm)
+        self.tdk = tdk
+
+    def add_document(self, doc_id: str, minhash: Union[MinHash, str], tdk: TokenizeDuckLike = None):
+        if isinstance(minhash, str):
+            minhash = self.str_to_minihash(minhash, tdk)
 
-    # 添加单个文档
-    doc_id = matcher.add_document(
-        "北京是中国的首都"
-    )
+        self.lsh.insert(doc_id, minhash)
 
-    # 批量添加文档
-    docs = {"doc1": "北京是一座现代化的大都市", "doc2": "上海是中国最大的城市", "doc3": "中国的首都是北京"}
-    matcher.batch_add_documents(docs)
+    def batch_add_documents(self, betch_data: Union[list, dict], tdk: TokenizeDuckLike = None):
+        def _add_document(minhash_or_str, tdk):
+            if isinstance(minhash_or_str, str):
+                minhash_or_str = self.str_to_minihash(minhash_or_str, tdk)
+            self.add_document(docid, minhash_or_str, tdk)
+
+        if isinstance(betch_data, list):
+            # 必须是可解包的2个数据的元组或list
+            for docid, minhash_or_str in betch_data:
+                _add_document(minhash_or_str, tdk)
+        elif isinstance(betch_data, dict):
+            for docid, minhash_or_str in betch_data.items():
+                _add_document(minhash_or_str, tdk)
+        else:
+            raise Exception("数据类型错误")
+
+    def find_similar(self, query_minhash: Union[MinHash, str], tdk: TokenizeDuckLike = None) -> List[Hashable]:
+        # 使用 LSH 查找候选集
+        if isinstance(query_minhash, str):
+            query_minhash = self.str_to_minihash(query_minhash, tdk)
+        similar_docs = self.lsh.query(query_minhash)
+        return similar_docs
 
-    # 查找相似文档（不返回相似度分数）
-    similar_docs = matcher.find_similar("北京首都")
-    print("相似文档ID:", similar_docs)
+    def create_minhash(self, words: List[str], num_perm=None) -> MinHash:
+        """
+        为分词结果创建 MinHash
+        """
+        if num_perm is None:
+            num_perm = self.num_perm
+        minhash = MinHash(num_perm=num_perm)
+        for word in words:
+            minhash.update(word.encode("utf-8"))
+        return minhash
 
-    # 查找相似文档（返回相似度分数）
-    similar_docs_with_scores = matcher.find_similar("北京首都", return_similarities=True)
-    print("相似文档ID和分数:", similar_docs_with_scores)
+    def create_words(self, text: str, tdk: TokenizeDuckLike = None):
+        if tdk is None:
+            tdk = self.tdk
+        worlds = tdk.get_words(text)
+        return worlds
+
+    def str_to_minihash(self, text: str, tdk: TokenizeDuckLike = None):
+        if tdk is None:
+            tdk = self.tdk
+        words = self.create_words(text, tdk)
+        minhash = self.create_minhash(words, self.num_perm)
+        return minhash
 
-    # 获取原始文本
-    for doc_id, score in similar_docs_with_scores:
-        print(f"文档 {doc_id}: {matcher.get_text(doc_id)} (相似度: {score:.2f})")
+    def minhash_dumps(self, minhash) -> bytes:
+        """
+        序列化
+        """
+        serialized_minhash = pickle.dumps(minhash)
+        return serialized_minhash
 
-    # 删除文档
-    matcher.remove_document("doc1")
+    def minhash_loads(self, serialized_minhash) -> MinHash:
+        """
+        反序列化
+        """
+        minhash = pickle.loads(serialized_minhash)
+        return minhash
 
-    # 清空所有数据
-    matcher.clear()
+    def merge_other_minhashlsh(self, other_minhashlsh: MinHashLSH):
+        """
+        在其他地方创建好的lsh 合并进来
+        """
+        self.lsh.merge(other_minhashlsh)

re_common/v2/baselibrary/utils/BusinessStringUtil.py

@@ -1,6 +1,8 @@
 # 某些业务中的字符串处理 算是特定场景的工具 不算通用工具
 import re
 
+from re_common.v2.baselibrary.utils.string_bool import is_all_symbols
+
 
 def clean_organ_postcode(organ):
     """
@@ -120,6 +122,11 @@ def deal_rel_vol(vol_str: str):
     """
     处理 期刊融合时的卷处理逻辑
     """
+
+    # 如果卷是全符号 清理掉
+    if is_all_symbols(vol_str):
+        vol_str = ""
+
     if vol_str.replace(".", "").isdigit():
         try:
             float_num = float(vol_str)
@@ -156,20 +163,27 @@ def deal_num_strs(input_str):
     return input_str
 
 
-def deal_num(strs):
+def deal_num(num_str):
     """
     将 期格式化 方便 group尤其是有横杆的数据
     该方法 为融合二次分割时使用，如果场景合适也可以用于其他地方
     :param strs:
     :return:
     """
-    strs = strs.replace("-", "_").replace(".", "_").upper()
-    if strs.find("_") > -1:
-        start, end = strs.split("_")
+    # 如果期是全符号清理掉
+    if is_all_symbols(num_str):
+        num_str = ""
+
+    if num_str.lower().startswith("n "):
+        num_str = num_str.lower().replace("n ", "").strip()
+
+    num_str = num_str.replace("-", "_").replace(".", "_").upper()
+    if num_str.find("_") > -1:
+        start, end = num_str.split("_")
         start = deal_num_strs(start)
         end = deal_num_strs(end)
-        strs = start + "_" + end
+        num_str = start + "_" + end
     else:
-        strs = deal_num_strs(strs)
+        num_str = deal_num_strs(num_str)
 
-    return strs
+    return num_str

re_common/v2/baselibrary/utils/basepika.py

@@ -0,0 +1,180 @@
+import pika
+
+
+# https://blog.csdn.net/songfreeman/article/details/50943603
+class BasePika(object):
+
+    def __init__(self, username=None, password=None, mqhost=None, virtual_host=None):
+        self.username = username
+        self.password = password
+        self.conn = None
+        self.host = mqhost
+        self.virtual_host = virtual_host
+        self.auto_ack = True
+
+    def set_default(self):
+        self.host = "192.168.31.79"
+        self.virtual_host = "vhost_NetDataGather"
+        self.username = "vip"
+        self.password = "piv$*123"
+
+    def connect_str(self,amqp_str):
+        parameters = pika.URLParameters(amqp_str)
+        self.conn = pika.BlockingConnection(parameters)
+
+    def connect(self):
+        """
+        设置用户名 密码 进行连接
+        :return:
+        """
+        credentials = pika.PlainCredentials(self.username, self.password)
+        # parameters = pika.URLParameters('amqp://guest:guest@rabbit-server1:5672/%2F')
+        # 可以通过将 heartbeat 设为 0，关闭 rabbitmq 的心跳检测
+        parameters = pika.ConnectionParameters(host=self.host,
+                                               virtual_host=self.virtual_host,
+                                               credentials=credentials,
+                                               heartbeat=0)
+        self.conn = pika.BlockingConnection(parameters)
+
+    def close(self):
+        # 关闭消息队列
+        self.conn.close()
+
+    def create_channel(self):
+        self.channel = self.conn.channel()
+
+    def __del__(self):
+        self.channel.close()
+        self.conn.close()
+
+    def random_queue_declare(self):
+        """
+        这样, result.method.queue 包含一个随机的队列名, 比如：看起来像 amq.gen-JzTY20BRgKO-HjmUJj0wLg.
+        其次：
+        一旦我们断开consumer连接，这个队列名将自动删除。这里有一个标识设置：
+        :return:
+        """
+        return self.channel.queue_declare("", exclusive=True)
+
+    def queue_declare(self, queue="hello", durable=False):
+        """
+        创建目的地队列hello 取消息时也可以调用
+        取消息和发送消息都调用 保证队列存在，也保证了不管服务端还是客户端先启动都有队列
+        durable True 为持久化
+        :return:
+        """
+        return self.channel.queue_declare(queue=queue, durable=durable)
+
+    def get_queue_size(self, queue="hello"):
+        """
+        获取某个队列的长度
+        :param queue:
+        :return:
+        """
+        queue = self.queue_declare(queue=queue, durable=True)
+        return queue.method.message_count
+
+    def get_properties(self):
+        """
+        与 queue_declare里的 durable = True 配合使用，
+        设置给 easy_send_msg的properties
+        :return:
+        """
+        return pika.BasicProperties(
+            delivery_mode=2,  # 设置消息为持久化的
+        )
+
+    def easy_send_msg(self, exchange="", routing_key="hello", body="hello world", properties=None):
+        """
+        空字符串标识默认的或者匿名的exchange,如果存在routing_key, 消息路由到routing_key指定的队列中。
+        routing_key 标识发送到哪个队列,就是服务器上的队列名
+        body 发送的消息
+
+        basic_publish 如果 exchange 不是"" 但没有绑定队列 消息会消失
+        :return:
+        """
+        self.channel.basic_publish(exchange=exchange,
+                                   routing_key=routing_key,
+                                   body=body,
+                                   properties=properties)
+
+    def basic_ack(self, ch, method):
+        """
+        callback的消息确认
+        :param ch:
+        :param method:
+        :return:
+        """
+        ch.basic_ack(delivery_tag=method.delivery_tag)
+
+    def callback(self, ch, method, properties, body):
+        """
+        从队列接收消息要更复杂一些，它需要为队列订阅一个 callback 函数来进行接收。
+        当我们接收一个消息后，这个 callback 函数将会被 pika函数库自动调用，
+        在我们的这个实例里面这个函数将用来打印接收的消息内容到屏幕
+        :param method:
+        :param properties:
+        :param body:
+        :return:
+        """
+        print(type(body))
+        print(" [x] Received %r" % body)
+        if self.auto_ack is False:
+            self.basic_ack(ch, method)
+
+    def set_get_msg_callback(self, routing_key="hello", callback=None, auto_ack=True):
+        """
+        设置取消息的callback
+        no_ack 如果设置为True，将使用自动确认模式
+        no_ack 如果设置为False，在callback中确认
+        :return:
+        """
+        self.auto_ack = auto_ack
+        if callback is None:
+            callback = self.callback
+        self.channel.basic_consume(routing_key,
+                                   callback,
+                                   auto_ack=auto_ack)
+
+    def start_get_msg(self):
+        """
+        开始取消息,会循环不停的取消息
+        :return:
+        """
+        self.channel.start_consuming()
+
+    def basic_qos(self, prefetch_count=1):
+        """
+        可以提前发送几个消息来，当auto_ack=True时无效
+        prefetch_count==1 消息未处理完前不要发送信息的消息
+        :return:
+        """
+        self.channel.basic_qos(prefetch_count=prefetch_count)
+
+    def exchange_declare(self, exchangename="logs", type="fanout"):
+        """
+        fanout exchange非常简单，你从这个名字中就能猜出来，它将从Producer方收到的消息广播给所有他知道的receiver方。而这正是我们的logger记录所需要的消息。
+        交换的类型
+        直接交换（direct exchange）的路由算法很简单 --  消息发送到绑定键值(binding key) 刚好完全符合路由键值( routing key) 的消息队列中。
+
+        消息发送到一个 topic交换不能是一个任意的 routing_key -- 它必须是一个用小数点 分割的单词列表。 这个字符可以是任何单词，但是通常是指定一些连接特定消息的功能。一些有效的路由键(routing key）比如：“stock.usd.nyse",
+        topic 是 直接交换的升级版
+
+        headers Exchange ：headers交换器允许你匹配AMQP消息的header而非路由键。除此之外，headers交换器和direct交换器完全一致，但性能会差很多。因此它并不太实用，而且几乎再也用不到了。
+        exchangename接下来会与队列绑定
+         direct  ,  topic ,  headers  和 fanout
+        :return:
+        """
+        return self.channel.exchange_declare(exchange=exchangename,
+                                             exchange_type=type)
+
+    def queue_bind(self, exchange="logs", queue="", routing_key=""):
+        """
+        queue 临时队列获取 self.random_queue_declare().method.queue
+        :param exchange:
+        :param queue:
+        :return:
+        """
+        self.channel.queue_bind(exchange=exchange,
+                                queue=queue,
+                                routing_key=routing_key)

re_common/v2/baselibrary/utils/db.py

@@ -0,0 +1,38 @@
+from contextlib import asynccontextmanager
+from typing import AsyncGenerator, Tuple
+
+import aiomysql
+from aiomysql import Pool, Connection, Cursor
+
+DB_CONFIG = {
+    'host': '192.168.98.55',
+    'port': 4000,
+    'user': 'dataware_house_baseUser',
+    'password': 'FF19AF831AEBD580B450B16BF9264200',
+    'db': 'dataware_house_base',
+    'charset': 'utf8mb4',
+    'minsize': 16,  # 最小连接数
+    'maxsize': 128,  # 最大连接数
+    'autocommit': False,  # 自动提交事务
+    'pool_recycle': 3600,  # 每个连接的回收时间（秒），超过此时间后连接将被关闭并重新创建，避免失效连接
+    'echo': False,  # 打印SQL语句
+}
+
+
+@asynccontextmanager
+async def get_db_pool():
+    """异步数据库连接池管理工具"""
+    pool: Pool = await aiomysql.create_pool(**DB_CONFIG)
+    try:
+        yield pool
+    finally:
+        pool.close()
+        await pool.wait_closed()
+
+
+@asynccontextmanager
+async def get_session(pool: Pool) -> AsyncGenerator[Tuple[Connection, Cursor], None]:
+    """获取数据库会话"""
+    async with pool.acquire() as conn:
+        async with conn.cursor() as cursor:
+            yield conn, cursor

re_common/v2/baselibrary/utils/mq.py

@@ -0,0 +1,83 @@
+import logging
+import traceback
+
+from re_common.v2.baselibrary.utils.basepika import BasePika
+from retry import retry
+
+logging_logger = logging.getLogger(__name__)
+
+
+class UseMq(object):
+
+    def __init__(self, queue, qos=1):
+        self.queue = queue
+        self.qos = qos
+        self.basepika = BasePika()
+        self.basepika.set_default()
+        self.basepika.connect()
+        self.basepika.create_channel()
+        self.basepika.queue_declare(queue=queue, durable=True)
+        self.basepika.basic_qos(qos)
+        self.properties = self.basepika.get_properties()
+
+    def re_conn(self):
+        """
+        重新连接
+        :return:
+        """
+        self.basepika.connect()
+        self.basepika.create_channel()
+        self.basepika.queue_declare(queue=self.queue, durable=True)
+        self.basepika.basic_qos(self.qos)
+
+    @retry(delay=5, backoff=2, max_delay=60 * 3, logger=logging_logger)
+    def get_mq(self):
+        try:
+            if self.basepika.channel.is_closed:
+                logging_logger.info("重连中......")
+                self.re_conn()
+                logging_logger.info("重连完成......")
+            self.basepika.set_get_msg_callback(routing_key=self.queue, callback=self.callback, auto_ack=False)
+            self.basepika.start_get_msg()
+        except:
+            traceback.print_exc()
+            logging_logger.info("重连中......")
+            self.re_conn()
+
+    def callback(self, ch, method, properties, body):
+        # print(type(body))
+        # print(" [x] Received %r" % body)
+        # body = body.decode()
+        self.callback2(ch, method, properties, body)
+        if self.basepika.auto_ack is False:
+            self.basepika.basic_ack(ch, method)
+
+    def callback2(self, ch, method, properties, body):
+        pass
+
+    @retry(delay=5, backoff=2, max_delay=60 * 3, logger=logging_logger)
+    def send_mq(self, body, num=100):
+        try:
+            if self.basepika.get_queue_size(self.queue) < num:
+                self.basepika.easy_send_msg(routing_key=self.queue,
+                                            body=body,
+                                            properties=self.properties)
+                return True
+            else:
+                return False
+        except:
+            traceback.print_exc()
+            logging_logger.info("重连中......")
+            self.re_conn()
+            return False
+
+    def get_server_mq_num(self, num=100):
+        if self.basepika.get_queue_size(self.queue) < num:
+            return True
+        else:
+            return False
+
+    def easy_send_mq(self, body):
+        self.basepika.easy_send_msg(routing_key=self.queue,
+                                    body=body,
+                                    properties=self.properties)

re_common/v2/baselibrary/utils/string_bool.py

@@ -1,5 +1,7 @@
 import re
 
+import unicodedata
+
 
 def is_all_english_chars(s):
     return bool(re.match(r'^[A-Za-z]+$', s))
@@ -29,7 +31,7 @@ def is_empty(value):
         import pandas as pd
         if pd.isna(value):
             return True
-    except ImportError:
+    except:
         pass  # 如果没有安装 pandas，跳过
 
     # 如果是字符串，检查去除空白后是否为空
@@ -88,3 +90,13 @@ def is_single_cjk_char(char):
         if start <= code_point <= end:
             return True
     return False
+
+
+def is_all_symbols(text):
+    # 是否全是符号
+    # 如果字符串为空，返回 False
+    if not text:
+        return False
+
+    # 检查每个字符是否属于符号类别
+    return all(unicodedata.category(char).startswith(('P', 'S')) for char in text)

re_common/v2/baselibrary/utils/string_clear.py

@@ -1,4 +1,6 @@
 import re
+from urllib.parse import unquote
+
 import regex
 
 from re_common.v2.baselibrary.utils.stringutils import qj2bj, bj2qj, get_diacritic_variant, clean_html, \
@@ -116,6 +118,13 @@ class StringClear(object):
         self.obj_str = remove_spaces_between_chinese_characters(self.obj_str)
         return self
 
+    def url_to_str(self):
+        """
+        url 编码转字符
+        """
+        self.obj_str = unquote(self.obj_str)
+        return self
+
     def get_str(self):
         return self.obj_str
 

{re_common-10.0.13.dist-info → re_common-10.0.15.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: re_common
-Version: 10.0.13
+Version: 10.0.15
 Summary: a library about all python projects
 Home-page: https://gitee.com/xujiangios/re-common
 Author: vic

{re_common-10.0.13.dist-info → re_common-10.0.15.dist-info}/RECORD

@@ -163,26 +163,33 @@ re_common/studio/streamlitstudio/first_app.py,sha256=t7Fw8YDlub7G9q99GgVo_3sPZXU
 re_common/studio/streamlitstudio/uber_pickups.py,sha256=cvrV5e8vRBM2_CpVDBE-f3V4mGFK9SqpRPZK8TEqr6U,785
 re_common/v2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 re_common/v2/baselibrary/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+re_common/v2/baselibrary/decorators/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+re_common/v2/baselibrary/decorators/utils.py,sha256=Q4D6KKCQxvNBXZkPQQn14keKKJpGtg8TUSakjJU40s0,2056
 re_common/v2/baselibrary/helpers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 re_common/v2/baselibrary/s3object/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 re_common/v2/baselibrary/s3object/baseboto3.py,sha256=mXuIFx99pnrPGQ4LJCZwlN1HLbaU-OWLwck0cVzW6hc,11203
 re_common/v2/baselibrary/tools/WeChatRobot.py,sha256=EaQgNncROAhU5-psYRGWAshIV5aEw-p2u1kYLpvr7RA,2796
 re_common/v2/baselibrary/tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+re_common/v2/baselibrary/tools/ac_ahocorasick.py,sha256=c63y5RtKVLD37nyPCnBqfNygwRj4gTQqyIdDOrC65G0,2847
 re_common/v2/baselibrary/tools/dict_tools.py,sha256=BTh7oJuJ619IZgxiYlim0ltrXBclDtb7WzyFGr7wVf0,1246
 re_common/v2/baselibrary/tools/dolphinscheduler.py,sha256=1m7UGYDiuvJUCI6ik6CGM2fO8U5XteJzn55VRbwB9ts,7978
+re_common/v2/baselibrary/tools/hdfs_data_processer.py,sha256=cChy6vhK8uSVIf3bRMGWpjociIbkiV-0j29WlZqQXHM,14207
 re_common/v2/baselibrary/tools/list_tools.py,sha256=qYxdLccRbrULOBbaPdJ_MyFFmVJGVMdW5E36nJ3ejr8,249
-re_common/v2/baselibrary/tools/search_hash_tools.py,sha256=d_h9j7VxiXpcn1GHZ7L2tpx9_LDQshcl58tlKvSxZPg,1691
-re_common/v2/baselibrary/tools/text_matcher.py,sha256=F4WtLO-b7H6V9TIvOntCD9ZXSQP_KijPuLLYcLPtrKQ,7021
+re_common/v2/baselibrary/tools/search_hash_tools.py,sha256=2ENLtZE8opRsfkwRtTNMzITmpTsjO7wZ1ZkfkqpOH9U,1937
+re_common/v2/baselibrary/tools/text_matcher.py,sha256=cPMoFxaA0-ce3tLRxVSs8_3pTYS1oVIHDnNy_AlPU-4,10756
 re_common/v2/baselibrary/tools/unionfind_tools.py,sha256=VYHZZPXwBYljsm7TjV1B6iCgDn3O3btzNf9hMvQySVU,2965
-re_common/v2/baselibrary/utils/BusinessStringUtil.py,sha256=__9MECbdrMnYc-ksYn2liM8vEbqF9uR4hZKqw86kW1Q,5924
+re_common/v2/baselibrary/utils/BusinessStringUtil.py,sha256=qY6bWcucZIU7e4yiD5-x46iCdp4HFNg_32utsysCKkc,6322
 re_common/v2/baselibrary/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 re_common/v2/baselibrary/utils/author_smi.py,sha256=1ebH3AHv19jtJWdlqNdwu6t58HNVLCotuCB6ff1SWiw,13666
 re_common/v2/baselibrary/utils/basedict.py,sha256=sH3_RZ8u4649-jX2V1uKNNkjJVUijZBDp6SdqncOZ88,1583
 re_common/v2/baselibrary/utils/basehdfs.py,sha256=NVV5Q0OMPlM_zTrs9ZDoPJv29GQv5wi9-AP1us5dBrQ,4651
+re_common/v2/baselibrary/utils/basepika.py,sha256=ifOb3UsGj79k40aD9UK6-5BMPw43ZAo0SO3AYD4q4vw,7332
+re_common/v2/baselibrary/utils/db.py,sha256=6HfmQHAtDm-pFFoe-ouNQggkfGRdN8Do2pN4B0ev_WU,1204
 re_common/v2/baselibrary/utils/json_cls.py,sha256=dHOkWafG9lbQDoub9cbDwT2fDjMKtblQnjFLeA4hECA,286
+re_common/v2/baselibrary/utils/mq.py,sha256=UHpO8iNIHs91Tgp-BgnSUpZwjWquxrGLdpr3FMMv2zw,2858
 re_common/v2/baselibrary/utils/n_ary_expression_tree.py,sha256=-05kO6G2Rth7CEK-5lfFrthFZ1Q0-0a7cni7mWZ-2gg,9172
-re_common/v2/baselibrary/utils/string_bool.py,sha256=EJnkSck4ofcIeJ6nLzAOVtlt6o1WBgvgVwIqJKj5Suc,2993
-re_common/v2/baselibrary/utils/string_clear.py,sha256=pGxL9PlzQDM06sC0j6U0zYRemvsJ7-OOpfzS5ETCxAs,6258
+re_common/v2/baselibrary/utils/string_bool.py,sha256=0JxzftuL61UAF-2Vp9F1Og8kXp_y647KJC5jXus9QwM,3278
+re_common/v2/baselibrary/utils/string_clear.py,sha256=1QAb_IC8FoVL5KzXhPicz4stsYD7LyASh5sXaXfs084,6445
 re_common/v2/baselibrary/utils/string_smi.py,sha256=cU0WAWHRGnGoVQx3eCEKeM_q_olFNzRTJe7rSe586SY,741
 re_common/v2/baselibrary/utils/stringutils.py,sha256=WuxhXJVU6xuGfgHiSjxrn7Go1eobpa8DMR3Icoey4vo,6039
 re_common/vip/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -211,8 +218,8 @@ re_common/vip/title/transform/TransformRegulationTitleToZt.py,sha256=LKRdIsWKues
 re_common/vip/title/transform/TransformStandardTitleToZt.py,sha256=-fCKAbSBzXVyQDCE61CalvR9E_QzQMA08QOO_NePFNI,5563
 re_common/vip/title/transform/TransformThesisTitleToZt.py,sha256=QS-uV0cQrpUFAcKucuJQ9Ue2VRQH-inmfn_X3IplfRo,5488
 re_common/vip/title/transform/__init__.py,sha256=m83-CWyRq_VHPYHaALEQlmXrkTdrZ3e4B_kCfBYE-uc,239
-re_common-10.0.13.dist-info/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
-re_common-10.0.13.dist-info/METADATA,sha256=QdkCM_LoID9na9D2qL4Zamec2QOdKtXgOxCWBSZQO8k,582
-re_common-10.0.13.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-re_common-10.0.13.dist-info/top_level.txt,sha256=_H9H23zoLIalm1AIY_KYTVh_H0ZnmjxQIxsvXtLv45o,10
-re_common-10.0.13.dist-info/RECORD,,
+re_common-10.0.15.dist-info/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
+re_common-10.0.15.dist-info/METADATA,sha256=IhfGSUxRXpHVDZv-ZwqvSxD6yiI_WbeDEHVDpn-RvyU,582
+re_common-10.0.15.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+re_common-10.0.15.dist-info/top_level.txt,sha256=_H9H23zoLIalm1AIY_KYTVh_H0ZnmjxQIxsvXtLv45o,10
+re_common-10.0.15.dist-info/RECORD,,