re-common 10.0.24__py3-none-any.whl → 10.0.26__py3-none-any.whl

re_common/v2/baselibrary/business_utils/BusinessStringUtil.py

@@ -75,6 +75,7 @@ def get_first_author(author: str) -> str:
             return au
     return ""
 
+
 def get_author_list(author: str):
     lists = []
     if not author:

re_common/v2/baselibrary/tools/data_processer/base.py

@@ -0,0 +1,53 @@
+from abc import ABC, abstractmethod
+from typing import List, Generator
+
+
+class BaseFileReader(ABC):
+
+    def __init__(self, batch_size: int = 10000):
+        self.batch_size = batch_size
+        self.read_model = 1
+
+    @abstractmethod
+    def list_files(self, path: str) -> List[str]:
+        """列出路径下所有目标文件"""
+        pass
+
+    @abstractmethod
+    def count_lines(self, file_path: str) -> int:
+        """统计文件行数"""
+        pass
+
+    @abstractmethod
+    def read_lines(self, file_path: str) -> Generator[List[str], None, None]:
+        """读取文件内容，返回批量数据"""
+        pass
+
+    @abstractmethod
+    def read_all(self, file_path: str) -> List[List[str]]:
+        """读取整个文件，默认按1000行分批"""
+        return [line for line in self.read_lines(file_path)]
+
+    def read_select(self, file_path: str) -> Generator[List[str], None, None]:
+        if self.read_model == 1:
+            for batch_data in self.read_lines(file_path):
+                yield batch_data
+        elif self.read_model == 2:
+            for batch_data in self.read_all(file_path):
+                yield batch_data
+        else:
+            raise Exception("模式选择错误")
+
+
+class BaseFileWriter(ABC):
+
+    def __init__(self, file_path: str, compress: bool = True, overwrite: bool = True, encoding: str = "utf-8"):
+        self.file_path = file_path
+        self.compress = compress
+        self.encoding = encoding
+        self.overwrite = overwrite
+
+    @abstractmethod
+    def write_lines(self, lines: List[str], file_path: str):
+        """写入多行文本到文件，支持压缩"""
+        pass

re_common/v2/baselibrary/tools/data_processer/data_processer.py

@@ -0,0 +1,508 @@
+import asyncio
+import os
+import re
+import sqlite3
+import time
+import traceback
+from concurrent.futures import ProcessPoolExecutor
+from pathlib import Path
+from typing import List, Callable, Any
+
+from filelock import FileLock
+from tenacity import retry, stop_after_attempt, wait_exponential, wait_random, retry_if_result
+
+from re_common.v2.baselibrary.tools.data_processer.base import BaseFileReader, BaseFileWriter
+
+
+class DatabaseHandler:
+    def __init__(self, db_file="processed_files.db"):
+        self.db_file = db_file
+        self.lock_file = f"{self.db_file}.lock"
+        self._init_db()
+
+    def _init_db(self):
+        with FileLock(self.lock_file):
+            """初始化 SQLite 数据库"""
+            with sqlite3.connect(self.db_file) as conn:
+                cursor = conn.cursor()
+                cursor.execute("""
+                    CREATE TABLE IF NOT EXISTS processed_files (
+                        file_path TEXT PRIMARY KEY
+                    )
+                """)
+                conn.commit()
+
+    def save_processed_file(self, file_path):
+        """保存处理过的文件"""
+        with FileLock(self.lock_file):
+            with sqlite3.connect(self.db_file) as conn:
+                cursor = conn.cursor()
+                cursor.execute(
+                    "INSERT OR IGNORE INTO processed_files (file_path) VALUES (?)",
+                    (file_path,)
+                )
+                conn.commit()
+
+    def save_processed_files_many(self, file_paths):
+        """批量保存处理过的文件路径"""
+        if not file_paths:
+            return
+        with FileLock(self.lock_file):
+            with sqlite3.connect(self.db_file) as conn:
+                cursor = conn.cursor()
+                cursor.executemany(
+                    "INSERT OR IGNORE INTO processed_files (file_path) VALUES (?)",
+                    ((fp,) for fp in file_paths)
+                )
+                conn.commit()
+
+    def is_file_processed(self, file_path):
+        """检查文件是否已处理"""
+        with FileLock(self.lock_file):
+            with sqlite3.connect(self.db_file) as conn:
+                cursor = conn.cursor()
+                cursor.execute(
+                    "SELECT file_path FROM processed_files WHERE file_path = ?",
+                    (file_path,)
+                )
+                result = cursor.fetchone()
+            return result is not None
+
+    def fake_processed_files(self, start_index, end_index, file_list):
+        try:
+            # 将字符串序号转换为整数
+            start = int(start_index)
+            end = int(end_index)
+
+            # 验证序号范围
+            if start >= end:
+                raise ValueError(f"起始序号 {start_index} 必须小于结束序号 {end_index}")
+
+            list_formatted_num = []
+            # 为范围内的每个序号生成文件名
+            for num in range(start, end):
+                # 将序号格式化为5位字符串 (00120, 00121,...)
+                formatted_num = f"{num:05d}"
+                list_formatted_num.append(formatted_num)
+
+            skip_path_list = []
+            skip_formatted_num = []
+            for file_path in file_list:
+                re_f_num = re.findall(r'(?<!\d)\d{5}(?!\d)', str(Path(file_path).stem))
+                if re_f_num:
+                    if re_f_num[0] in list_formatted_num:
+                        skip_path_list.append(file_path)
+                        skip_formatted_num.append(re_f_num[0])
+
+            for item_list in [skip_path_list[i:i + 2000] for i in range(0, len(skip_path_list), 2000)]:
+                self.save_processed_files_many(item_list)
+                for file_path in item_list:
+                    print(f"伪造处理记录: {file_path}")
+
+            no_fil_num = set(list_formatted_num) - set(skip_formatted_num)
+            if len(no_fil_num) > 0:
+                print(f"没有对应num的文件,伪造失败数量为{len(no_fil_num)},样例:{list(no_fil_num)[:10]}")
+            print(f"成功伪造处理记录：序号 {start_index} 到 {end_index}（不含）的文件")
+
+        except ValueError as e:
+            print(f"错误: 序号格式无效 - {str(e)}")
+        except Exception as e:
+            print(f"伪造处理记录时出错: {str(e)}")
+
+
+class DataProcessor:
+    def __init__(
+            self,
+            reader: BaseFileReader,
+            writer: BaseFileWriter = None,
+            db_handler: DatabaseHandler = None,
+            db_file="processed_files.db",
+            batch_size=50,
+            retry_limit=3,
+    ):
+        self.reader = reader
+        self.writer = writer
+        self.db_file = db_file
+        self.batch_size = batch_size
+        self.retry_limit = retry_limit
+        self.db_handler = db_handler if db_handler else DatabaseHandler(db_file=db_file)
+
+    async def retry_process_data(self, data, process_func):
+        """处理数据并执行处理函数"""
+
+        def on_retry(retry_state):
+            # 每次抛错进入该函数打印消息
+            print(
+                f"重试次数: {retry_state.attempt_number}/{self.retry_limit},数据内容: {retry_state.args[0]}\n"
+                f"异常信息: {retry_state.outcome.exception()}"
+            )
+
+        def on_retry_error(retry_state):
+            # 最后抛错后调用
+            original_exc = retry_state.outcome.exception()
+            raise RuntimeError(
+                f"处理数据失败，达到重试上限。data: {retry_state.args[0]}") from original_exc  # 抛出的自定义异常中 保留 __process_func() 里的原始错误堆栈信息（traceback）
+
+        @retry(stop=stop_after_attempt(3),
+               wait=wait_exponential(multiplier=1, min=2, max=20),
+               before_sleep=on_retry,  # 每次抛错后使用
+               retry_error_callback=on_retry_error,  # 如果到最后都没有成功 抛错
+               reraise=True)
+        async def __process_func(_data):
+            return await process_func(_data)
+
+        return await __process_func(data)
+
+    async def process_file(self, hdfs_file_path, process_func, write_dir):
+        """处理单个 gz 文件"""
+        total_lines = self.reader.count_lines(hdfs_file_path)
+        processed_lines = 0
+        start_time = time.time()
+        results = []
+        #   # 这里根据不同的配置选用不同的读取文件的方法
+        for lines in self.reader.read_select(hdfs_file_path):
+            processing_start_time = time.time()  # 记录本批处理开始时间
+
+            tasks = []
+            for line in lines:
+                # try:
+                #     data = json.loads(line)
+                #     tasks.append(self.retry_process_data(data, process_func))
+                # except json.JSONDecodeError as e:
+                #     raise Exception(f"解析JSON失败: {e}, 行内容: {line.strip()}")
+                tasks.append(self.retry_process_data(line, process_func))
+
+            # await AsyncTaskPool(self.batch_size).run(tasks) # AsyncTaskPool 适用于一次提交所有任务, 限制并发数执行
+            results.extend(await asyncio.gather(*tasks))
+
+            processed_lines += len(lines)
+
+            elapsed_time = time.time() - start_time  # 已用时间
+            processing_time = time.time() - processing_start_time  # 本次处理时间
+            avg_processing_time = (
+                (elapsed_time * 1000) / processed_lines if processed_lines > 0 else float("inf")
+            )  # 平均每条数据的处理时间（毫秒）
+
+            # 估算剩余时间
+            remaining_time = (
+                ((avg_processing_time / 1000) * (total_lines - processed_lines))
+                if processed_lines > 0
+                else float("inf")
+            )
+
+            # 显示总进度信息
+            print(
+                f"文件: {hdfs_file_path} 总进度: {processed_lines}/{total_lines} 行 | "
+                f"已用时间: {elapsed_time:.2f}秒 | 本次处理时间: {processing_time:.2f}秒 | "
+                f"预估剩余时间: {remaining_time:.2f}秒 | 平均每条处理时间: {avg_processing_time:.2f}毫秒"
+            )
+
+        if write_dir is not None:
+            if not self.writer:
+                raise Exception("没有配置写数据的对象")
+            write_path = write_dir.rstrip("/") + f"/{Path(hdfs_file_path).stem}"
+            self.writer.write_lines([str(item) for item in results], write_path)
+
+        # 最终进度显示
+        final_elapsed_time = time.time() - start_time  # 最终已用时间
+        print(
+            f"文件: {hdfs_file_path} 处理完成 | 总进度: {processed_lines}/{total_lines} 行 | "
+            f"总已用时间: {final_elapsed_time:.2f}秒 | "
+            f"平均每条处理时间: {(final_elapsed_time * 1000) / processed_lines:.2f}毫秒"
+            if processed_lines > 0
+            else "处理无数据"
+        )
+
+        self.db_handler.save_processed_file(hdfs_file_path)  # 保存处理过的文件
+
+    async def retry_process_file(self, hdfs_file_path, process_func, write_dir):
+        """带重试机制的文件处理"""
+
+        def on_retry(retry_state):
+            # 每次抛错进入该函数打印消息
+            exc = retry_state.outcome.exception()
+            tb = ''.join(traceback.format_exception(type(exc), exc, exc.__traceback__))
+            print(tb)
+
+            print(
+                f"处理文件 {retry_state.args[0]} 时发生错误: {exc}，正在重试 {retry_state.attempt_number}/{self.retry_limit}")
+
+        def on_retry_error(retry_state):
+            # 最后抛错后调用
+            print(f"处理文件 {retry_state.args[0]} 失败，达到重试上限")
+            return False
+
+        @retry(stop=stop_after_attempt(3),
+               wait=wait_exponential(multiplier=1, min=2, max=20),
+               before_sleep=on_retry,  # 每次抛错后使用
+               retry_error_callback=on_retry_error,  # 如果最后没成功 返回 False
+               reraise=True)
+        async def __process_func(_hdfs_file_path, _process_func, _write_dir):
+            await self.process_file(_hdfs_file_path, _process_func, _write_dir)
+            return True  # 成功处理后退出
+
+        return await __process_func(hdfs_file_path, process_func, write_dir)
+
+    def get_file_list(self, hdfs_dir):
+        # 获取所有任务文件
+        all_files = self.reader.list_files(hdfs_dir)
+        for file_path in all_files:
+            yield file_path
+
+    @retry(stop=stop_after_attempt(3),
+           wait=wait_random(min=10, max=30),
+           retry=retry_if_result(lambda result: not result),  # 如果返回值是 False（失败），则重试 最后会抛出一个默认错误tenacity.RetryError:
+           reraise=True)
+    async def _batch_process_file(self, hdfs_file_path: str, process_func: Callable[[str], Any],
+                                  write_dir: str = None):
+        """批量更新所有 gz 文件"""
+        # all_succeed = True
+        # for hdfs_file_path in self.get_file_list(hdfs_dir):
+        #     if self.db_handler.is_file_processed(hdfs_file_path):
+        #         print(f"跳过已处理文件: {hdfs_file_path}")
+        #         continue  # 如果文件已处理，跳过
+        #     succeed = await self.retry_process_file(hdfs_file_path, process_func, write_dir)  # 处理文件
+        #     if succeed is False:
+        #         all_succeed = False
+        #
+        # if all_succeed:
+        #     # 处理完成后删除数据库文件
+        #     try:
+        #         if os.path.exists(self.db_file):
+        #             os.remove(self.db_file)
+        #             print(f"已删除断点重试文件: {self.db_file}")
+        #             return True
+        #     except Exception as e:
+        #         print(f"删除断点重试文件失败: {e}")
+        # return False
+        if self.db_handler.is_file_processed(hdfs_file_path):
+            print(f"跳过已处理文件: {hdfs_file_path}")
+            return True  # 如果文件已处理，跳过
+        succeed = await self.retry_process_file(hdfs_file_path, process_func, write_dir)  # 处理文件
+        return succeed
+
+    async def process_file_bulk(self, hdfs_file_path, process_func, write_dir):
+        """按批次处理单个文件，批量数据传递给处理函数"""
+        # 获取文件的数据总量
+        total_lines = self.reader.count_lines(hdfs_file_path)
+        processed_lines = 0
+        start_time = time.time()
+
+        results = []
+        tasks = []
+        # 这里根据不同的配置选用不同的读取文件的方法
+        for lines in self.reader.read_select(hdfs_file_path):
+            processing_start_time = time.time()  # 记录本批处理开始时间
+
+            # batch_data = []
+            # for line in lines:
+            #     try:
+            #         data = json.loads(line)
+            #         batch_data.append(data)
+            #     except json.JSONDecodeError as e:
+            #         raise Exception(f"解析JSON失败: {e}, 行内容: {line.strip()}")
+
+            # 处理读取到的批次数据
+            if lines:
+                tasks.append(process_func(lines))  # 将批次数据传递给处理函数并收集任务
+                processed_lines += len(lines)  # 更新已处理行数
+
+            # 当积累的任务数量达到 batch_size 时并发处理所有任务
+            if len(tasks) >= self.batch_size:
+                results.extend(await asyncio.gather(*tasks))
+                elapsed_time = time.time() - start_time  # 已用时间
+                processing_time = time.time() - processing_start_time  # 本次处理时间
+                avg_processing_time = (
+                    (elapsed_time * 1000) / processed_lines if processed_lines > 0 else float("inf")
+                )  # 平均每条数据的处理时间（毫秒）
+
+                # 估算剩余时间
+                remaining_time = (
+                    ((avg_processing_time / 1000) * (total_lines - processed_lines))
+                    if processed_lines > 0
+                    else float("inf")
+                )
+
+                # 显示总进度信息
+                print(
+                    f"文件: {hdfs_file_path} 总进度: {processed_lines}/{total_lines} 行 | "
+                    f"已用时间: {elapsed_time:.2f}秒 | 本次处理时间: {processing_time:.2f}秒 | "
+                    f"预估剩余时间: {remaining_time:.2f}秒 | 平均每条处理时间: {avg_processing_time:.2f}毫秒"
+                )
+
+                # 清空任务列表，准备下一批处理
+                tasks.clear()
+            # 处理剩余的任务
+        if tasks:
+            results.extend(await asyncio.gather(*tasks))  # 处理未达到 batch_size 的剩余任务
+
+        if write_dir is not None:
+            if not self.writer:
+                raise Exception("没有配置写数据的对象")
+            write_path = write_dir.rstrip("/") + f"/{Path(hdfs_file_path).stem}"
+            self.writer.write_lines([str(item) for items in results for item in items], write_path)
+
+        # 最终进度显示
+        final_elapsed_time = time.time() - start_time  # 最终已用时间
+        print(
+            f"文件: {hdfs_file_path} 处理完成 | 总进度: {processed_lines}/{total_lines} 行 | "
+            f"总已用时间: {final_elapsed_time:.2f}秒 | "
+            f"平均每条处理时间: {(final_elapsed_time * 1000) / processed_lines:.2f}毫秒"
+            if processed_lines > 0
+            else "处理无数据"
+        )
+
+        self.db_handler.save_processed_file(hdfs_file_path)
+
+    async def retry_process_file_bulk(self, hdfs_file_path, process_func, write_dir):
+        """带重试机制的批量文件处理"""
+
+        def on_retry(retry_state):
+            # 每次抛错进入该函数打印消息
+            exc = retry_state.outcome.exception()
+            tb = ''.join(traceback.format_exception(type(exc), exc, exc.__traceback__))
+            print(tb)
+            print(
+                f"处理文件 {retry_state.args[0]} 时发生错误: {exc}，正在重试 {retry_state.attempt_number}/{self.retry_limit}")
+
+        def on_retry_error(retry_state):
+            # 最后抛错后调用
+            print(f"处理文件 {retry_state.args[0]} 失败，达到重试上限")
+            return False
+
+        @retry(stop=stop_after_attempt(3),
+               wait=wait_exponential(multiplier=1, min=2, max=20),
+               before_sleep=on_retry,  # 每次抛错后使用
+               retry_error_callback=on_retry_error,  # 如果最后没成功 返回 False
+               reraise=True)
+        async def __process_func(_hdfs_file_path, _process_func, write_dir):
+            await self.process_file_bulk(_hdfs_file_path, _process_func, write_dir)
+            return True  # 成功处理后退出
+
+        return await __process_func(hdfs_file_path, process_func, write_dir)
+
+    async def batch_process_file(self, hdfs_dir: str, process_func: Callable[[List[str]], Any] | Callable[[str], Any],
+                                 write_dir: str = None, is_bulk: bool = False):
+        all_succeed = True
+        for hdfs_file_path in self.get_file_list(hdfs_dir):
+            if is_bulk:
+                succeed = await self._batch_process_file_bulk(hdfs_file_path, process_func, write_dir)
+            else:
+                succeed = await self._batch_process_file(hdfs_file_path, process_func, write_dir)
+            if succeed is False:
+                all_succeed = False
+        if all_succeed:
+            # 处理完成后删除数据库文件
+            try:
+                if os.path.exists(self.db_file):
+                    os.remove(self.db_file)
+                    print(f"已删除断点重试文件: {self.db_file}")
+                    return True
+            except Exception as e:
+                print(f"删除断点重试文件失败: {e}")
+        return False
+
+    @retry(stop=stop_after_attempt(3),
+           wait=wait_random(min=10, max=30),
+           retry=retry_if_result(lambda result: not result),  # 如果返回值是 False（失败），则重试 最后会抛出一个默认错误tenacity.RetryError:
+           reraise=True)
+    async def _batch_process_file_bulk(self, hdfs_file_path: str, process_func: Callable[[List[str]], Any],
+                                       write_dir: str = None):
+        """批量处理 gz 文件中的数据"""
+        # 获取所有文件
+        # all_succeed = True
+        # for hdfs_file_path in self.get_file_list(hdfs_dir):
+        #     # 查看是否跳过文件
+        #     if self.db_handler.is_file_processed(hdfs_file_path):
+        #         print(f"跳过已处理文件: {hdfs_file_path}")
+        #         continue  # 跳过已处理文件
+        #     # 开始批量处理文件
+        #     succeed = await self.retry_process_file_bulk(hdfs_file_path, process_func, write_dir)
+        #     if succeed is False:
+        #         all_succeed = False
+        #
+        # if all_succeed:
+        #     # 处理完成后删除数据库文件
+        #     try:
+        #         if os.path.exists(self.db_file):
+        #             os.remove(self.db_file)
+        #             print(f"已删除断点重试文件: {self.db_file}")
+        #             return True
+        #     except Exception as e:
+        #         print(f"删除断点重试文件失败: {e}")
+        # return False
+        # 查看是否跳过文件
+        if self.db_handler.is_file_processed(hdfs_file_path):
+            print(f"跳过已处理文件: {hdfs_file_path}")
+            return True  # 跳过已处理文件
+        # 开始批量处理文件
+        succeed = await self.retry_process_file_bulk(hdfs_file_path, process_func, write_dir)
+        return succeed
+
+
+# 全局变量，每个进程独立持有
+_processor: DataProcessor | None = None
+_process_func: Callable[[List[str]], Any] | Callable[[str], Any] | None = None
+_process_args: dict
+
+
+def get_data_processor_func(process_args):
+    _func_reader = process_args["reader_func"]
+    _reader_args = process_args["reader_kwargs"]
+    reader = _func_reader(**_reader_args)
+    writer = None
+    if process_args["is_writer"]:
+        _func_writer = process_args["writer_func"]
+        _writer_args = process_args["writer_kwargs"]
+        writer = _func_writer(**_writer_args)
+
+    data_kwargs = {
+        "reader": reader,
+        "writer": writer,
+        "db_file": process_args["db_file"]
+    }
+    if process_args.get("batch_size"):
+        data_kwargs["batch_size"] = process_args["batch_size"]
+    if process_args.get("retry_limit"):
+        data_kwargs["retry_limit"] = process_args["retry_limit"]
+
+    return DataProcessor(**data_kwargs)
+
+
+def init_worker(process_func, process_args):
+    global _processor, _process_func, _process_args
+    _processor = get_data_processor_func(process_args)
+    _process_func = process_func
+    _process_args = process_args
+
+    _init_func = _process_args.get("init_work", None)
+    if _init_func:
+        _init_func()
+
+    _async_init_work = _process_args.get("async_init_work", None)
+    if _init_func:
+        asyncio.run(_async_init_work())
+
+
+def worker(path_file):
+    if _process_args["is_bulk"]:
+        return asyncio.run(_processor._batch_process_file_bulk(path_file, _process_func, _process_args["write_dir"]))
+    else:
+        return asyncio.run(_processor._batch_process_file(path_file, _process_func, _process_args["write_dir"]))
+
+
+def run_worker_many(hdfs_dir: str, process_func: Callable[[List[str]], Any] | Callable[[str], Any],
+                    data_process_args: dict, max_workers=4):
+    processor = get_data_processor_func(data_process_args)
+    all_file = list(processor.get_file_list(hdfs_dir))
+    with ProcessPoolExecutor(
+            max_workers=max_workers,
+            initializer=init_worker,
+            initargs=(process_func, data_process_args)
+    ) as executor:
+        # 提交任务并等待结果
+        results = executor.map(worker, all_file)
+    # 输出结果
+    for result in results:
+        if result:
+            print(result)

re_common/v2/baselibrary/tools/data_processer/data_reader.py

@@ -0,0 +1,187 @@
+import gzip
+import io
+import json
+from io import BytesIO
+from pathlib import Path
+from typing import List, Generator
+
+import pandas as pd
+from hdfs import InsecureClient
+
+from re_common.v2.baselibrary.tools.data_processer.base import BaseFileReader
+
+
+class HDFSFileReader(BaseFileReader):
+    def __init__(self, batch_size: int = 1000, hdfs_url: str = "http://VIP-DC-MASTER-2:9870", hdfs_user: str = "root"):
+        super().__init__(batch_size)
+        self.client = InsecureClient(hdfs_url, user=hdfs_user)
+
+    def list_files(self, path: str) -> List[str]:
+        return [f"{path}/{f[0]}" for f in self.client.list(path, status=True) if f[0] != '_SUCCESS']
+
+    def count_lines(self, file_path: str) -> int:
+        with self.client.read(file_path) as f:
+            return sum(1 for _ in f)
+
+    def read_lines(self, file_path: str) -> Generator[List[str], None, None]:
+        # 批量读取后 处理 缺点 连接可能会断
+        with self.client.read(file_path) as f:
+            while True:
+                batch = []
+                for _ in range(self.batch_size):
+                    try:
+                        line = next(f)
+                        line = line.decode('utf-8')
+                        if line.strip():
+                            batch.append(line.strip())
+                    except StopIteration:
+                        break
+                if not batch:
+                    break
+                yield batch
+
+    def read_all(self, file_path: str) -> List[List[str]]:
+        # 一次读取返回所有后批量处理缺点 内存占用
+        with self.client.read(file_path) as f:
+            lines = [line.decode('utf-8').strip() for line in f if line.decode('utf-8').strip()]
+            return [lines[i: i + self.batch_size] for i in range(0, len(lines), self.batch_size)]
+
+
+class HDFSGZFileReader(BaseFileReader):
+    def __init__(self, batch_size: int = 1000, hdfs_url: str = "http://VIP-DC-MASTER-2:9870", hdfs_user: str = "root"):
+        super().__init__(batch_size)
+        self.hdfs_url = hdfs_url
+        self.hdfs_user = hdfs_user
+        self.client = None
+
+    def _init_client(self):
+        if self.client is None:
+            self.client = InsecureClient(self.hdfs_url, user=self.hdfs_user)
+        return self
+
+    def list_files(self, path: str) -> List[str]:
+        self._init_client()
+        return [f"{path}/{f[0]}" for f in self.client.list(path, status=True) if f[0].endswith(".gz")]
+
+    def count_lines(self, file_path: str) -> int:
+        self._init_client()
+        with self.client.read(file_path) as f:
+            with gzip.GzipFile(fileobj=f) as gz:
+                return sum(1 for _ in gz)
+
+    def read_lines(self, file_path: str) -> Generator[List[str], None, None]:
+        self._init_client()
+        # 批量读取后 处理 缺点 连接可能会断
+        with self.client.read(file_path) as f:
+            with gzip.GzipFile(fileobj=f) as gz:
+                while True:
+                    batch = []
+                    for _ in range(self.batch_size):
+                        try:
+                            line = next(gz)
+                            if line.strip():
+                                batch.append(line.decode("utf-8"))
+                        except StopIteration:
+                            break
+                    if not batch:
+                        break
+                    yield batch
+
+    def read_all(self, file_path: str) -> List[List[str]]:
+        self._init_client()
+        # 一次读取返回所有后批量处理缺点 内存占用
+        with self.client.read(file_path) as reader:
+            compressed_data = reader.read()
+            with gzip.GzipFile(fileobj=BytesIO(compressed_data)) as gz_file:
+                content = gz_file.read().decode("utf-8")
+                lines = [i for i in content.split("\n") if i.strip()]
+                return [lines[i: i + self.batch_size] for i in range(0, len(lines), self.batch_size)]
+
+
+class HDFSParquetFileReader(BaseFileReader):
+    def __init__(self, batch_size: int = 1000, hdfs_url: str = "http://VIP-DC-MASTER-2:9870", hdfs_user: str = "root"):
+        super().__init__(batch_size)
+        self.client = InsecureClient(hdfs_url, user=hdfs_user)
+
+    def list_files(self, path: str) -> List[str]:
+        return [f"{path}/{f[0]}" for f in self.client.list(path, status=True) if f[0].endswith(".parquet")]
+
+    def count_lines(self, file_path: str) -> int:
+        with self.client.read(file_path) as f:
+            data = f.read()
+            df = pd.read_parquet(io.BytesIO(data))
+        count = len(df)
+        return count
+
+    def read_lines(self, file_path: str) -> Generator[List[str], None, None]:
+        # 批量读取后 处理 缺点 连接可能会断
+        with self.client.read(file_path) as f:
+            data = f.read()
+            df = pd.read_parquet(io.BytesIO(data))
+            records = [json.dumps(row, ensure_ascii=False) for row in df.to_dict(orient='records')]
+            for i in range(0, len(records), self.batch_size):
+                yield records[i: i + self.batch_size]
+
+    def read_all(self, file_path: str) -> List[List[str]]:
+        # 一次读取返回所有后批量处理缺点 内存占用
+        with self.client.read(file_path) as f:
+            data = f.read()
+            df = pd.read_parquet(io.BytesIO(data))
+            records = [json.dumps(row, ensure_ascii=False) for row in df.to_dict(orient='records')]
+            return [records[i: i + self.batch_size] for i in range(0, len(records), self.batch_size)]
+
+
+class LocalGZFileReader(BaseFileReader):
+    def list_files(self, path: str) -> List[str]:
+        return [str(p) for p in Path(path).rglob("*.gz")]
+
+    def count_lines(self, file_path: str) -> int:
+        with gzip.open(file_path, 'rt', encoding='utf-8') as f:
+            return sum(1 for _ in f)
+
+    def read_lines(self, file_path: str) -> Generator[List[str], None, None]:
+        with gzip.open(file_path, 'rt', encoding='utf-8') as f:
+            while True:
+                batch = []
+                for _ in range(self.batch_size):
+                    line = f.readline()
+                    if not line:
+                        break
+                    if line.strip():
+                        batch.append(line.strip())
+                if not batch:
+                    break
+                yield batch
+
+    def read_all(self, file_path: str) -> List[List[str]]:
+        with gzip.open(file_path, 'rt', encoding='utf-8') as f:
+            lines = [line.strip() for line in f if line.strip()]
+            return [lines[i: i + self.batch_size] for i in range(0, len(lines), self.batch_size)]
+
+
+class LocalFileReader(BaseFileReader):
+    def list_files(self, path: str) -> List[str]:
+        return [str(p) for p in Path(path).rglob("*") if p.is_file()]
+
+    def count_lines(self, file_path: str) -> int:
+        with open(file_path, 'r', encoding='utf-8') as f:
+            return sum(1 for _ in f)
+
+    def read_lines(self, file_path: str) -> Generator[List[str], None, None]:
+        with open(file_path, 'r', encoding='utf-8') as f:
+            while True:
+                batch = []
+                for _ in range(self.batch_size):
+                    line = f.readline()
+                    if not line:
+                        break
+                    if line.strip():
+                        batch.append(line.strip())
+                if not batch:
+                    break
+                yield batch
+
+    def read_all(self, file_path: str) -> List[List[str]]:
+        with open(file_path, 'r', encoding='utf-8') as f:
+            lines = [line.strip() for line in f if line.strip()]
+            return [lines[i: i + self.batch_size] for i in range(0, len(lines), self.batch_size)]

re_common/v2/baselibrary/tools/data_processer/data_writer.py

@@ -0,0 +1,38 @@
+import gzip
+from io import BytesIO
+from typing import List
+
+from hdfs import InsecureClient
+
+from re_common.v2.baselibrary.tools.data_processer.base import BaseFileWriter
+
+
+class HDFSFileWriter(BaseFileWriter):
+    def __init__(self, file_path: str, hdfs_url: str, hdfs_user: str, *args, **kwargs):
+        super().__init__(file_path, *args, **kwargs)
+        self.client = InsecureClient(hdfs_url, user=hdfs_user)
+
+    def write_lines(self, lines: List[str], file_path: str = None):
+        if file_path is None:
+            file_path = self.file_path
+        data = "\n".join(lines).encode(self.encoding)
+        if self.compress:
+            buf = BytesIO()
+            with gzip.GzipFile(fileobj=buf, mode="wb") as gz:
+                gz.write(data)
+            buf.seek(0)
+            self.client.write(file_path, data=buf, overwrite=self.overwrite)
+        else:
+            self.client.write(file_path, data=data, overwrite=self.overwrite)
+
+
+class LocalFileWriter(BaseFileWriter):
+    def write_lines(self, lines: List[str], file_path: str, compress: bool = True, encoding="utf-8"):
+        if compress:
+            with gzip.open(file_path, 'wt', encoding=encoding) as f:
+                for line in lines:
+                    f.write(f"{line}\n")
+        else:
+            with open(file_path, 'w', encoding=encoding) as f:
+                for line in lines:
+                    f.write(f"{line}\n")

re_common/v2/baselibrary/tools/dict_tools.py

@@ -35,3 +35,10 @@ class DotDict(dict):
             else:
                 result[key] = value
         return result
+
+
+def none_to_empty_str(d):
+    for k, v in d.items():
+        if v is None:
+            d[k] = ""
+    return d

re_common/v2/baselibrary/tools/list_tools.py

@@ -63,4 +63,8 @@ def list_to_dict(list_data,key_name):
 
     # 将 defaultdict 转换成普通字典
     dict_data = dict(dict_data)
-    return dict_data
+    return dict_data
+
+def split_list_by_step(lst, step=100):
+    # 一维列表按照步长转换成二维列表
+    return [lst[i:i + step] for i in range(0, len(lst), step)]

re_common/v2/baselibrary/utils/api_net_utils.py

@@ -0,0 +1,270 @@
+import atexit
+import sys
+import asyncio
+import aiohttp
+from typing import Optional
+
+from tenacity import retry, stop_after_attempt, wait_random
+
+g_headers = {
+    'accept': 'application/json',
+    'Content-Type': 'application/json',
+    'Authorization': 'Bearer eyJhbGciOiJub25lIiwidHlwIjoiSldUIn0.eyJ1c2VyX2lkIjotMSwidXNlcl9uYW1lIjoiXHU1ZTk0XHU3NTI4XHU0ZTJkXHU1ZmMzQ2xpZW50In0.'
+}
+
+"""
+cls._conn = aiohttp.TCPConnector(
+        limit=50,  # 最大连接数
+        ssl=False,  # 禁用SSL验证（按需开启）
+        force_close=True,  # 保持连接活跃
+        enable_cleanup_closed=True  # 自动清理关闭的连接 
+    )
+# 由于网络上有重名，没有连接。如果加入域，请转到“控制面板”中的“系统”更改计算机名，然后重试。如果加入工作组，请选择其他工作组名。
+有可能是 
+force_close=True,  # 保持连接活跃
+enable_cleanup_closed=True  # 自动清理关闭的连接 
+照成的
+"""
+
+
+class HttpError(Exception):
+    code = 0
+    message = ""
+    headers = None
+
+    def __init__(
+            self,
+            *,
+            code: Optional[int] = None,
+            message: str = "",
+            headers: Optional[dict] = None,
+    ) -> None:
+        if code is not None:
+            self.code = code
+        self.headers = headers
+        self.message = message
+
+    def __str__(self) -> str:
+        return f"code: {self.code}, message:{self.message}"
+
+    def __repr__(self) -> str:
+        return f"<{self.__class__.__name__}: code={self.code}, message={self.message!r}>"
+
+
+def on_retry_error(retry_state):
+    # 最后抛错后调用
+    original_exc = retry_state.outcome.exception()
+    print(f"[HTTP 请求重试所有重试失败.] 错误消息{original_exc}")
+
+    raise HttpError(code=getattr(original_exc, 'code', 455),
+                    message=f"错误消息:{str(original_exc)}") from original_exc
+
+
+def on_retry(retry_state):
+    # 每次抛错进入该函数打印消息
+    print(
+        f"[HTTP 请求重试]"
+        f"当前重试 : 第 {retry_state.attempt_number} 次"
+        f"睡眠时间 : {retry_state.next_action.sleep:.2f} 秒"
+        f"\n异常原因 : {retry_state.outcome.exception()}"
+    )
+
+
+class ApiNetUtils:
+    """
+    HTTP请求工具类（异步版），提供GET/POST/PATCH请求方法
+    特性：
+    1. 自动复用TCP连接池
+    2. 自动重试机制（通过async_retry装饰器）
+    3. 进程退出时自动清理资源
+    4. 线程安全的延迟初始化
+    """
+
+    # 类属性使用Optional类型注解，初始化为None实现延迟初始化
+    _conn: Optional[aiohttp.TCPConnector] = None
+    _session: Optional[aiohttp.ClientSession] = None
+    _close_registered: bool = False  # 确保清理函数只注册一次
+
+    @classmethod
+    async def _get_connector(cls) -> aiohttp.TCPConnector:
+        """
+        获取TCP连接器（延迟初始化）
+        解决模块加载时没有事件循环的问题
+        """
+        if cls._conn is None or cls._conn.closed or cls.is_loop_closed(cls._session):
+            # 只有在首次使用时才创建连接器
+            cls._conn = aiohttp.TCPConnector(
+                limit=50,  # 最大连接数
+                ssl=False,  # 禁用SSL验证（按需开启）
+                force_close=True,  # 保持连接活跃
+                # enable_cleanup_closed=True,  # 自动清理关闭的连接 #
+                # keepalive_timeout=4.99  # 比服务器的5s 小一点
+            )
+        return cls._conn
+
+    @classmethod
+    async def _get_session(cls) -> aiohttp.ClientSession:
+        """
+        获取共享会话（线程安全的延迟初始化）
+        包含自动注册清理机制
+        """
+        if cls._session is None or cls._session.closed or cls.is_loop_closed(cls._session):
+            if cls._session:
+                await cls.close()
+            # 获取连接器（会自动初始化）
+            connector = await cls._get_connector()
+
+            # 强制获取新的事件循环
+            loop = asyncio.get_event_loop()
+
+            # 创建新会话
+            cls._session = aiohttp.ClientSession(
+                connector=connector,
+                timeout=aiohttp.ClientTimeout(total=30),  # 默认30秒超时
+                loop=loop)  # 显式指定事件循环
+
+            # # 注册退出时的清理钩子
+            cls._register_cleanup()
+
+        return cls._session
+
+    @staticmethod
+    def is_loop_closed(session: aiohttp.ClientSession) -> bool:
+        """
+        检查会话绑定的事件循环是否已关闭
+        """
+        loop = session._loop  # 获取会话绑定的事件循环
+        if loop.is_closed():
+            # print("Event loop is closed")
+            return True
+        # print("Event loop not is closed")
+        return False
+
+    @classmethod
+    def _register_cleanup(cls):
+        """
+        注册进程退出时的资源清理函数
+        包含正常退出和异常退出两种情况
+        """
+        if not cls._close_registered:
+            # 1. 正常退出处理
+            atexit.register(lambda: asyncio.run(cls.close()))
+
+            # 2. 异常退出处理
+            original_excepthook = sys.excepthook
+
+            def custom_excepthook(exctype, value, traceback):
+                """自定义异常钩子，确保资源被清理"""
+                # 先执行原始异常处理（打印堆栈等）
+                original_excepthook(exctype, value, traceback)
+                # 然后执行资源清理
+                try:
+                    asyncio.run(cls.close())
+                except RuntimeError:
+                    # 如果已经没有事件循环，则同步执行
+                    loop = asyncio.new_event_loop()
+                    loop.run_until_complete(cls.close())
+                    loop.close()
+
+            sys.excepthook = custom_excepthook
+            cls._close_registered = True
+
+    @classmethod
+    async def close(cls):
+        """
+        安全关闭所有网络资源
+        会自动在程序退出时调用，也可手动调用
+        """
+        if cls._session and not cls._session.closed:
+            await cls._session.close()
+            cls._session = None
+
+        if cls._conn and not cls._conn.closed:
+            await cls._conn.close()
+            cls._conn = None
+
+        # print("[ApiNetUtils] 网络资源已安全释放")
+
+    # -------------------- 公共API方法 -------------------- #
+
+    @classmethod
+    @retry(stop=stop_after_attempt(4),  # 本质上执行4次 但重试3次
+           wait=wait_random(min=5, max=15),
+           before_sleep=on_retry,  # 每次抛错后使用
+           retry_error_callback=on_retry_error,
+           reraise=True)
+    async def fetch_get(cls, url: str, headers=None, params=None):
+        """
+        GET请求封装
+        :param url: 请求URL
+        :param headers: 可选请求头（默认使用全局g_headers）
+        :param params: 查询参数（字典）
+        :return: 解析后的JSON数据
+        :raises HttpError: 当状态码非200时抛出
+        """
+        headers = headers or g_headers
+        session = await cls._get_session()
+
+        async with session.get(url, headers=headers, params=params) as response:
+            if response.status != 200:
+                error_text = await response.text()
+                raise HttpError(
+                    code=response.status,
+                    message=f"请求失败: url={url}, status={response.status}, 错误详情={error_text}"
+                )
+            return await response.json()
+
+    @classmethod
+    @retry(stop=stop_after_attempt(4),
+           wait=wait_random(min=5, max=15),
+           before_sleep=on_retry,  # 每次抛错后使用
+           retry_error_callback=on_retry_error,
+           reraise=True)
+    async def fetch_post(cls, url: str, payload: dict, headers=None):
+        """
+        POST请求封装（JSON格式）
+        """
+        headers = headers or g_headers
+        session = await cls._get_session()
+
+        async with session.post(url, json=payload, headers=headers) as response:
+            if response.status != 200:
+                error_text = await response.text()
+                raise HttpError(
+                    code=response.status,
+                    message=f"请求失败: url={url}, status={response.status}, 错误详情={error_text}"
+                )
+            return await response.json()
+
+    @classmethod
+    @retry(stop=stop_after_attempt(4),
+           wait=wait_random(min=5, max=15),
+           before_sleep=on_retry,  # 每次抛错后使用
+           retry_error_callback=on_retry_error,
+           reraise=True)
+    async def fetch_patch(cls, url: str, payload: dict, headers=None):
+        """
+        PATCH请求封装（JSON格式）
+        """
+        headers = headers or g_headers
+        session = await cls._get_session()
+
+        async with session.patch(url, json=payload, headers=headers) as response:
+            if response.status != 200:
+                error_text = await response.text()
+                raise HttpError(
+                    code=response.status,
+                    message=f"请求失败: url={url}, status={response.status}, 错误详情={error_text}"
+                )
+            return await response.json()
+
+    @classmethod
+    async def __aenter__(cls):
+        """支持async with语法"""
+        await cls._get_session()
+        return cls
+
+    @classmethod
+    async def __aexit__(cls, exc_type, exc, tb):
+        """async with退出时自动关闭"""
+        await cls.close()

re_common/v2/baselibrary/utils/db.py

@@ -1,8 +1,9 @@
+import atexit
+import os
 from contextlib import asynccontextmanager
 from typing import AsyncGenerator, Tuple
 
-import aiomysql
-from aiomysql import Pool, Connection, Cursor, DictCursor
+from aiomysql import Pool, Connection, Cursor
 
 DB_CONFIG = {
     'host': '192.168.98.55',
@@ -18,10 +19,27 @@ DB_CONFIG = {
     'echo': False,  # 打印SQL语句
 }
 
+DB_CONFIG1 = {
+    'host': '192.168.98.55',
+    'port': 4000,
+    'user': 'foreign_fulltextUser',
+    'password': 'i4hIeasw1qpmhGN2nwL7',
+    'db': 'foreign_fulltext',
+    'charset': 'utf8mb4',
+    'minsize': 16,  # 最小连接数
+    'maxsize': 128,  # 最大连接数
+    'autocommit': False,  # 自动提交事务
+    'pool_recycle': 3600,  # 每个连接的回收时间（秒），超过此时间后连接将被关闭并重新创建，避免失效连接
+    'echo': False,  # 打印SQL语句
+}
+
 
 @asynccontextmanager
-async def get_db_pool():
+async def get_db_pool(_DB_CONFIG: dict = None):
     """异步数据库连接池管理工具"""
+    global DB_CONFIG
+    if _DB_CONFIG is not None:
+        DB_CONFIG = _DB_CONFIG
     pool: Pool = await aiomysql.create_pool(**DB_CONFIG)
     try:
         yield pool
@@ -36,3 +54,21 @@ async def get_session(pool: Pool) -> AsyncGenerator[Tuple[Connection, Cursor], N
     async with pool.acquire() as conn:
         async with conn.cursor() as cursor:
             yield conn, cursor
+
+
+# main.py
+import aiomysql
+import asyncio
+
+aiomysql_pool = None
+pool_lock = asyncio.Lock()  # 全局异步锁
+
+
+async def init_aiomysql_pool_async():
+    global aiomysql_pool
+    if aiomysql_pool is None:
+        async with pool_lock:
+            if aiomysql_pool is None:
+                print(f"[{os.getpid()}] Initializing aiomysql pool...")
+                aiomysql_pool = await aiomysql.create_pool(**DB_CONFIG)
+    return aiomysql_pool

re_common/v2/baselibrary/utils/string_clear.py

@@ -9,7 +9,7 @@ from re_common.v2.baselibrary.utils.stringutils import (
     bj2qj,
     get_diacritic_variant,
     clean_html,
-    remove_spaces_between_chinese_characters,
+    remove_spaces_between_chinese_characters, clean_unicode_alnum,
 )
 
 
@@ -91,6 +91,11 @@ class StringClear(object):
         self.obj_str = re.sub(r"[^\w\s]", "", self.obj_str)
         return self
 
+    def remove_all_symbols(self):
+        # 一种更加强力的符号清理 只保留各个国家的字符 和各个国家的数字
+        self.obj_str = clean_unicode_alnum(self.obj_str)
+        return self
+
     def remove_underline(self):
         # 下划线在 \w 中 所以这里独立封装
         self.obj_str = re.sub("[_]", "", self.obj_str)

re_common/v2/baselibrary/utils/stringutils.py

@@ -211,3 +211,18 @@ def get_group_abstract(lists):
             t_list.append(keyid_list[text_idx])
         all_list.append(t_list)
     return all_list
+
+
+def clean_unicode_alnum(text: str) -> str:
+    """
+    清除所有非 Unicode 字母或数字的字符。
+
+    参数:
+        text (str): 输入文本。
+
+    返回:
+        str: 只包含 Unicode 字母和数字的文本。
+    \p{N} 匹配所有 Unicode 数字字符 包括非阿拉伯数字字符
+    \p{L} 匹配所有语言字符
+    """
+    return regex.sub(r"[^\p{L}\p{N}]+", "", text)

{re_common-10.0.24.dist-info → re_common-10.0.26.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: re_common
-Version: 10.0.24
+Version: 10.0.26
 Summary: a library about all python projects
 Home-page: https://gitee.com/xujiangios/re-common
 Author: vic

{re_common-10.0.24.dist-info → re_common-10.0.26.dist-info}/RECORD

@@ -163,7 +163,7 @@ re_common/studio/streamlitstudio/first_app.py,sha256=t7Fw8YDlub7G9q99GgVo_3sPZXU
 re_common/studio/streamlitstudio/uber_pickups.py,sha256=cvrV5e8vRBM2_CpVDBE-f3V4mGFK9SqpRPZK8TEqr6U,785
 re_common/v2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 re_common/v2/baselibrary/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-re_common/v2/baselibrary/business_utils/BusinessStringUtil.py,sha256=M3Pb9TB9QRePa9phjqx6JZV_igM-xRRa554iZ-WgyOo,6786
+re_common/v2/baselibrary/business_utils/BusinessStringUtil.py,sha256=njPcRgeBWpnZr5u2cPAO4qdWBq-CgTn99rJuvWFcChk,6788
 re_common/v2/baselibrary/business_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 re_common/v2/baselibrary/business_utils/rel_tools.py,sha256=LfnGFCkUSxg1SHvOMOQdP1PiHxIKqk7Syuk5YYpjJag,295
 re_common/v2/baselibrary/decorators/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -174,29 +174,35 @@ re_common/v2/baselibrary/s3object/baseboto3.py,sha256=mXuIFx99pnrPGQ4LJCZwlN1HLb
 re_common/v2/baselibrary/tools/WeChatRobot.py,sha256=sKBt2gPsfj0gzV6KaLSAhIhL-j3qNfHfqE-lII1LVwM,3537
 re_common/v2/baselibrary/tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 re_common/v2/baselibrary/tools/ac_ahocorasick.py,sha256=c63y5RtKVLD37nyPCnBqfNygwRj4gTQqyIdDOrC65G0,2847
-re_common/v2/baselibrary/tools/dict_tools.py,sha256=BTh7oJuJ619IZgxiYlim0ltrXBclDtb7WzyFGr7wVf0,1246
+re_common/v2/baselibrary/tools/dict_tools.py,sha256=eSMwPTLp3oSjuviC_wlXg0I-dnkkmZfUfCRLX5djWV8,1365
 re_common/v2/baselibrary/tools/dolphinscheduler.py,sha256=1m7UGYDiuvJUCI6ik6CGM2fO8U5XteJzn55VRbwB9ts,7978
 re_common/v2/baselibrary/tools/hdfs_data_processer.py,sha256=g0DaNjXM1hIUblFQ6YBwnwEBKIXn48X8Y9Eiok4dVlQ,14824
-re_common/v2/baselibrary/tools/list_tools.py,sha256=mZyrOGdW6tuany0lKQOD4P739xikvmeKm1VSzo37Byc,1973
+re_common/v2/baselibrary/tools/list_tools.py,sha256=1NxGVM4EytSXh4IGAEfZQnvq0Ev-UOF-PGZBg2EQbOg,2132
 re_common/v2/baselibrary/tools/search_hash_tools.py,sha256=2ENLtZE8opRsfkwRtTNMzITmpTsjO7wZ1ZkfkqpOH9U,1937
 re_common/v2/baselibrary/tools/text_matcher.py,sha256=cPMoFxaA0-ce3tLRxVSs8_3pTYS1oVIHDnNy_AlPU-4,10756
 re_common/v2/baselibrary/tools/unionfind_tools.py,sha256=VYHZZPXwBYljsm7TjV1B6iCgDn3O3btzNf9hMvQySVU,2965
+re_common/v2/baselibrary/tools/data_processer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+re_common/v2/baselibrary/tools/data_processer/base.py,sha256=i6HA2UQsSRZaKxW1wJMpiC9LAy3wYaI2BVxUAiFoRZ4,1704
+re_common/v2/baselibrary/tools/data_processer/data_processer.py,sha256=R7zHQG8eo3mfckYr-Pp53fyyQj6zd8fuweSxwzvDgN0,22683
+re_common/v2/baselibrary/tools/data_processer/data_reader.py,sha256=LWLbom7W2L0T6q38crA1_Gcvxkzk9Lm0btJjrmtMHMU,7945
+re_common/v2/baselibrary/tools/data_processer/data_writer.py,sha256=OgKZ06zRJYNx758rbjxZG_KNgkLuVLlyB1AvyRsJtS4,1447
 re_common/v2/baselibrary/utils/BusinessStringUtil.py,sha256=njPcRgeBWpnZr5u2cPAO4qdWBq-CgTn99rJuvWFcChk,6788
 re_common/v2/baselibrary/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+re_common/v2/baselibrary/utils/api_net_utils.py,sha256=22q3WMWiKVg1IVGr4y2D1JrjhnbQtlChRDJm2S8rGlc,9868
 re_common/v2/baselibrary/utils/author_smi.py,sha256=1ebH3AHv19jtJWdlqNdwu6t58HNVLCotuCB6ff1SWiw,13666
 re_common/v2/baselibrary/utils/base_string_similarity.py,sha256=a40a79ttwoG_gC_hxMNB-sMXXecgICoRDWrj0DW8iEE,7749
 re_common/v2/baselibrary/utils/basedict.py,sha256=sH3_RZ8u4649-jX2V1uKNNkjJVUijZBDp6SdqncOZ88,1583
 re_common/v2/baselibrary/utils/basehdfs.py,sha256=TPwFct_-UrmO1KCbo4gpV77rsnlCQDumNBbQKL0ZI9o,5953
 re_common/v2/baselibrary/utils/basepika.py,sha256=ifOb3UsGj79k40aD9UK6-5BMPw43ZAo0SO3AYD4q4vw,7332
 re_common/v2/baselibrary/utils/basetime.py,sha256=b7U_ho6nE3fjYBxSkdMHXUOd3ClH6KkW_7p7l2Gs4gA,3038
-re_common/v2/baselibrary/utils/db.py,sha256=lr6SI1Bk7tA0nw_4vHbmz-OaisPlb_p9ziC1Oz74tcA,1216
+re_common/v2/baselibrary/utils/db.py,sha256=ouDagXqqY9h4ucK4LDGrYVY-31rOiBQFxXLIlio9AJA,2297
 re_common/v2/baselibrary/utils/json_cls.py,sha256=M93piYtmgm_wP8E57culTrd_AhHLoGg6PqeAJYdW2SM,438
 re_common/v2/baselibrary/utils/mq.py,sha256=UHpO8iNIHs91Tgp-BgnSUpZwjWquxrGLdpr3FMMv2zw,2858
 re_common/v2/baselibrary/utils/n_ary_expression_tree.py,sha256=-05kO6G2Rth7CEK-5lfFrthFZ1Q0-0a7cni7mWZ-2gg,9172
 re_common/v2/baselibrary/utils/string_bool.py,sha256=vxnjSFOfuHWGxkqaIbUNn21opx5tfV1uCXSahFfp1mU,6197
-re_common/v2/baselibrary/utils/string_clear.py,sha256=_bK8oi7t34JfFh8IVbe0fFexBzafFWow0r3QJ45HsyE,7212
+re_common/v2/baselibrary/utils/string_clear.py,sha256=ywYR1KrKQyeM-zJgvTmORlfgbLdRSjWWKPe7K8oRx_8,7450
 re_common/v2/baselibrary/utils/string_smi.py,sha256=cU0WAWHRGnGoVQx3eCEKeM_q_olFNzRTJe7rSe586SY,741
-re_common/v2/baselibrary/utils/stringutils.py,sha256=WuxhXJVU6xuGfgHiSjxrn7Go1eobpa8DMR3Icoey4vo,6039
+re_common/v2/baselibrary/utils/stringutils.py,sha256=TI6fw3km1l25ufXrnG6ha8dSBDtRh-MF4nWRt9u8Xbo,6452
 re_common/vip/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 re_common/vip/base_step_process.py,sha256=VXXiNj0I5CpzXIMCgOPU86bzDJkSBkUS-9CpZIl_GOk,205
 re_common/vip/baseencodeid.py,sha256=nERoe89ueFM52bG7xwJdflcZHk6T2RQQKbc5uUZc3RM,3272
@@ -223,8 +229,8 @@ re_common/vip/title/transform/TransformRegulationTitleToZt.py,sha256=LKRdIsWKues
 re_common/vip/title/transform/TransformStandardTitleToZt.py,sha256=-fCKAbSBzXVyQDCE61CalvR9E_QzQMA08QOO_NePFNI,5563
 re_common/vip/title/transform/TransformThesisTitleToZt.py,sha256=QS-uV0cQrpUFAcKucuJQ9Ue2VRQH-inmfn_X3IplfRo,5488
 re_common/vip/title/transform/__init__.py,sha256=m83-CWyRq_VHPYHaALEQlmXrkTdrZ3e4B_kCfBYE-uc,239
-re_common-10.0.24.dist-info/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
-re_common-10.0.24.dist-info/METADATA,sha256=R1kT5A-yNJHQ3lvpchpqc3dX8ONJK1kuTfoGUQ08018,582
-re_common-10.0.24.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-re_common-10.0.24.dist-info/top_level.txt,sha256=_H9H23zoLIalm1AIY_KYTVh_H0ZnmjxQIxsvXtLv45o,10
-re_common-10.0.24.dist-info/RECORD,,
+re_common-10.0.26.dist-info/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
+re_common-10.0.26.dist-info/METADATA,sha256=kHLVPF-e0PjpnUL7dN9pAMqK_pw4yHwZGKxbJ_zlAY0,582
+re_common-10.0.26.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+re_common-10.0.26.dist-info/top_level.txt,sha256=_H9H23zoLIalm1AIY_KYTVh_H0ZnmjxQIxsvXtLv45o,10
+re_common-10.0.26.dist-info/RECORD,,