re-common 10.0.41__tar.gz → 10.0.43__tar.gz

{re_common-10.0.41/re_common.egg-info → re_common-10.0.43}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: re_common
-Version: 10.0.41
+Version: 10.0.43
 Summary: a library about all python projects
 Home-page: https://gitee.com/xujiangios/re-common
 Author: vic
@@ -11,6 +11,15 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.6
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license-file
+Dynamic: requires-python
+Dynamic: summary
 
 
     这是一个基础类，依赖很多的第三方包，是一个用得到的第三方库的封装，可以在此基础上迅速构建项目

re_common-10.0.43/re_common/v2/baselibrary/s3object/baseaioboto3.py

@@ -0,0 +1,122 @@
+import gzip
+import io
+from typing import Protocol
+
+import aioboto3
+import aiofiles
+from aiobotocore.config import AioConfig
+
+
+class AsyncReadable(Protocol):
+    async def read(self, n: int = -1) -> bytes:
+        ...
+
+
+# config = AioConfig(connect_timeout=600000, read_timeout=600000, retries={'max_attempts': 3},
+#                    max_pool_connections=10)
+
+class BaseAioBoto3(object):
+
+    def __init__(self, aws_access_key_id, aws_secret_access_key, endpoint_url,
+                 config=AioConfig(max_pool_connections=10)):
+        """
+          初始化华为云 OBS 客户端
+
+          Args:
+              access_key: 华为云 Access Key
+              secret_key: 华为云 Secret Key
+              region: 区域，如 'cn-north-4'
+              endpoint: 华为云 OBS 端点，可选
+          """
+        self.aws_access_key_id = aws_access_key_id
+        self.aws_secret_access_key = aws_secret_access_key
+        self.endpoint_url = endpoint_url
+        self.config = config
+        self.boto_session = None
+
+    async def initialize_class_variable(self):
+        if self.boto_session is None:
+            self.boto_session = aioboto3.Session(
+                aws_access_key_id=self.aws_access_key_id,
+                aws_secret_access_key=self.aws_secret_access_key
+            )
+
+    async def read_minio_data(self, bucket, key):
+        await self.initialize_class_variable()
+        async with self.boto_session.client("s3", endpoint_url=self.endpoint_url, config=self.config) as s3:
+            s3_ob = await s3.get_object(Bucket=bucket, Key=key)
+            result = await s3_ob["Body"].read()
+            return result
+
+    def ungzip(self, raw_bytes, encoding="utf-8"):
+        with gzip.GzipFile(fileobj=io.BytesIO(raw_bytes)) as gz:
+            return gz.read().decode(encoding)
+
+    # 异步下载大文件
+    async def download_file(self, bucket: str, key: str, local_path: str):
+        await self.initialize_class_variable()
+        async with self.boto_session.client("s3", endpoint_url=self.endpoint_url, config=self.config) as s3:
+            response = await s3.get_object(Bucket=bucket, Key=key)
+            body = response["Body"]
+
+            # 用异步方式写入本地
+            async with aiofiles.open(local_path, "wb") as f:
+                while True:
+                    chunk = await body.read(10 * 1024 * 1024)  # 每次读 10MB
+                    if not chunk:
+                        break
+                    await f.write(chunk)
+
+        return local_path
+
+    async def list_files(self, bucket: str, prefix: str, recursive: bool = True):
+        """
+        获取 bucket 下某个“目录”(prefix) 的文件列表
+
+        单文件返回样例 ['server_data/api-title-roc/py_full_organ_dic/', 'server_data/api-title-roc/py_full_organ_dic/part-00000.gz']
+
+        :param bucket: bucket 名
+        :param prefix: 目录前缀，如 'server_data/api-title-roc/'
+        :param recursive: 是否递归子目录
+        :return: List[str] 文件 key 列表
+        """
+        await self.initialize_class_variable()
+        keys = []
+
+        # 非递归时，用 delimiter 模拟目录
+        extra_args = {}
+        if not recursive:
+            extra_args["Delimiter"] = "/"
+
+        async with self.boto_session.client(
+                "s3",
+                endpoint_url=self.endpoint_url,
+                config=self.config
+        ) as s3:
+
+            continuation_token = None
+
+            while True:
+                kwargs = {
+                    "Bucket": bucket,
+                    "Prefix": prefix,
+                    **extra_args
+                }
+                # 下一页的“游标”
+                if continuation_token:
+                    kwargs["ContinuationToken"] = continuation_token
+
+                resp = await s3.list_objects_v2(**kwargs)
+
+                # 文件
+                for obj in resp.get("Contents", []):
+                    keys.append(obj["Key"])
+
+                # 是否还有下一页
+                if resp.get("IsTruncated"):  # 说明还有下一页
+                    # 下一页从哪里继续查
+                    continuation_token = resp.get("NextContinuationToken")
+                else:
+                    break
+
+        return keys

re_common-10.0.43/re_common/v2/baselibrary/s3object/huaweiobs.py

@@ -0,0 +1,80 @@
+# pip install pycryptodome==3.10.1
+# pip install esdk-obs-python
+# 引入模块
+import os
+
+from obs import ObsClient, GetObjectHeader
+
+
+class BaseObsClient(object):
+
+    def __init__(self, aws_access_key_id="", aws_secret_access_key="", endpoint_url=""):
+        self.aws_access_key_id = aws_access_key_id
+        self.aws_secret_access_key = aws_secret_access_key
+        self.endpoint_url = endpoint_url
+        self.client = None
+        if self.aws_access_key_id and self.aws_secret_access_key and self.endpoint_url:
+            self.get_client()
+
+    def get_client(self):
+        self.client = ObsClient(access_key_id=self.aws_access_key_id,
+                                secret_access_key=self.aws_secret_access_key,
+                                server=self.endpoint_url)
+        return self
+
+    def close(self):
+        self.client.close()
+
+    def put_object(self, bucket_name, objectKey, body):
+        """
+        直接写内容到文件
+        Args:
+            bucket_name:
+            key:
+            body: 需要
+
+        Returns:
+        """
+        # 上传文本对象
+        resp = self.client.putContent(bucket_name, objectKey, body)
+        # 返回码为2xx时，接口调用成功，否则接口调用失败
+        if resp.status < 300:
+            return True, resp
+        else:
+            return False, resp
+
+
+    def download_memobj(self, bucket_name, objectKey):
+        """
+        return: None
+        """
+        # 指定loadStreamInMemory为True忽略downloadpath路径，将文件的二进制流下载到内存
+        # 二进制下载对象
+        resp = self.client.getObject(bucketName=bucket_name, objectKey=objectKey, loadStreamInMemory=True)
+        # 返回码为2xx时，接口调用成功，否则接口调用失败
+        if resp.status < 300:
+            return True, resp
+        else:
+            return False, resp
+
+    def download_file(self, bucket_name, objectKey,downloadPath):
+        """
+        return: None
+        """
+        headers = GetObjectHeader()
+        resp = self.client.getObject(bucket_name, objectKey, downloadPath, headers=headers)
+        # 返回码为2xx时，接口调用成功，否则接口调用失败
+        if resp.status < 300:
+            return True, resp
+        else:
+            return False, resp
+
+
+    def list_prefixes(self,bucket_name,prefix, max_keys = 100):
+        # 列举桶内对象
+        resp = self.client.listObjects(bucket_name, prefix, max_keys=max_keys, encoding_type='url')
+        # 返回码为2xx时，接口调用成功，否则接口调用失败
+        if resp.status < 300:
+            return True, resp
+        else:
+            return False, resp

{re_common-10.0.41 → re_common-10.0.43}/re_common/v2/baselibrary/tools/data_processer/base.py

@@ -4,9 +4,9 @@ from typing import List, Generator
 
 class BaseFileReader(ABC):
 
-    def __init__(self, batch_size: int = 10000):
+    def __init__(self, batch_size: int = 10000, read_model: int = 1):
         self.batch_size = batch_size
-        self.read_model = 1
+        self.read_model = read_model
 
     @abstractmethod
     def list_files(self, path: str) -> List[str]:

{re_common-10.0.41 → re_common-10.0.43}/re_common/v2/baselibrary/tools/data_processer/data_reader.py

@@ -12,8 +12,9 @@ from re_common.v2.baselibrary.tools.data_processer.base import BaseFileReader
 
 
 class HDFSFileReader(BaseFileReader):
-    def __init__(self, batch_size: int = 1000, hdfs_url: str = "http://VIP-DC-MASTER-2:9870", hdfs_user: str = "root"):
-        super().__init__(batch_size)
+    def __init__(self, batch_size: int = 1000, read_model: int = 1, hdfs_url: str = "http://VIP-DC-MASTER-2:9870",
+                 hdfs_user: str = "root"):
+        super().__init__(batch_size, read_model)
         self.client = InsecureClient(hdfs_url, user=hdfs_user)
 
     def list_files(self, path: str) -> List[str]:
@@ -48,8 +49,9 @@ class HDFSFileReader(BaseFileReader):
 
 
 class HDFSGZFileReader(BaseFileReader):
-    def __init__(self, batch_size: int = 1000, hdfs_url: str = "http://VIP-DC-MASTER-2:9870", hdfs_user: str = "root"):
-        super().__init__(batch_size)
+    def __init__(self, batch_size: int = 1000, read_model: int = 1, hdfs_url: str = "http://VIP-DC-MASTER-2:9870",
+                 hdfs_user: str = "root"):
+        super().__init__(batch_size, read_model)
         self.hdfs_url = hdfs_url
         self.hdfs_user = hdfs_user
         self.client = None
@@ -99,8 +101,9 @@ class HDFSGZFileReader(BaseFileReader):
 
 
 class HDFSParquetFileReader(BaseFileReader):
-    def __init__(self, batch_size: int = 1000, hdfs_url: str = "http://VIP-DC-MASTER-2:9870", hdfs_user: str = "root"):
-        super().__init__(batch_size)
+    def __init__(self, batch_size: int = 1000, read_model: int = 1, hdfs_url: str = "http://VIP-DC-MASTER-2:9870",
+                 hdfs_user: str = "root"):
+        super().__init__(batch_size, read_model)
         self.client = InsecureClient(hdfs_url, user=hdfs_user)
 
     def list_files(self, path: str) -> List[str]:

re_common-10.0.43/re_common/v2/baselibrary/tools/dir_file_tools.py

@@ -0,0 +1,27 @@
+import json
+import os
+
+
+def scan_dir_fast(path):
+    file_infos = []
+    with os.scandir(path) as entries:
+        for entry in entries:
+            if entry.is_file():
+                info = entry.stat()
+                file_infos.append({
+                    "path": entry.path,
+                    "size": info.st_size
+                })
+    return file_infos
+
+
+def scan_dir(dir_name, result_file):
+    # dir_name r"/share/fulltext/errors"
+    # result_file "file_info_errors.txt"
+    for root, dirs, files in os.walk(dir_name):
+        print(root)
+        lists = scan_dir_fast(root)
+        with open(result_file, "a", encoding="utf-8") as file:
+            for i in lists:
+                if i:
+                    file.write(json.dumps(i, ensure_ascii=False) + "\n")

{re_common-10.0.41 → re_common-10.0.43}/re_common/v2/baselibrary/tools/list_tools.py

@@ -1,4 +1,5 @@
 import itertools
+from collections import Counter
 from typing import List, Any, Tuple
 
 
@@ -67,4 +68,20 @@ def list_to_dict(list_data,key_name):
 
 def split_list_by_step(lst, step=100):
     # 一维列表按照步长转换成二维列表
-    return [lst[i:i + step] for i in range(0, len(lst), step)]
+    return [lst[i:i + step] for i in range(0, len(lst), step)]
+
+
+def list_diff(l1, l2):
+    """
+    非去重差异比较
+    Counter 虽然长得像字典，但它在运算符 & 和 - 上有特殊的定义。
+    这样 能获取重复差集
+    """
+    c1, c2 = Counter(l1), Counter(l2)
+    # 共同部分
+    common = list((c1 & c2).elements())
+    # l1 多余的部分
+    extra1 = list((c1 - c2).elements())
+    # l2 多余的部分
+    extra2 = list((c2 - c1).elements())
+    return common, extra1, extra2

{re_common-10.0.41 → re_common-10.0.43}/re_common/v2/baselibrary/utils/api_net_utils.py

@@ -1,8 +1,11 @@
 import atexit
+import os
 import sys
 import asyncio
+import traceback
+
 import aiohttp
-from typing import Optional
+from typing import Optional, Union
 
 from tenacity import retry, stop_after_attempt, wait_random
 
@@ -62,6 +65,13 @@ def on_retry_error(retry_state):
 
 def on_retry(retry_state):
     # 每次抛错进入该函数打印消息
+
+    # # 获取函数调用参数
+    # args = retry_state.args
+    # kwargs = retry_state.kwargs
+    #
+    # print(id(args[0]._get_session()))
+
     print(
         f"[HTTP 请求重试]"
         f"当前重试 : 第 {retry_state.attempt_number} 次"
@@ -84,6 +94,8 @@ class ApiNetUtils:
     _conn: Optional[aiohttp.TCPConnector] = None
     _session: Optional[aiohttp.ClientSession] = None
     _close_registered: bool = False  # 确保清理函数只注册一次
+    _pid: Optional[int] = None  # 当前进程的 PID
+    lock = asyncio.Lock()
 
     @classmethod
     async def _get_connector(cls) -> aiohttp.TCPConnector:
@@ -96,9 +108,9 @@ class ApiNetUtils:
             cls._conn = aiohttp.TCPConnector(
                 limit=50,  # 最大连接数
                 ssl=False,  # 禁用SSL验证（按需开启）
-                force_close=True,  # 保持连接活跃
-                # enable_cleanup_closed=True,  # 自动清理关闭的连接 #
-                # keepalive_timeout=4.99  # 比服务器的5s 小一点
+                force_close=False,  # 保持连接活跃
+                enable_cleanup_closed=True,  # 自动清理关闭的连接 #
+                keepalive_timeout=4.99  # 比服务器的5s 小一点
             )
         return cls._conn
 
@@ -108,25 +120,41 @@ class ApiNetUtils:
         获取共享会话（线程安全的延迟初始化）
         包含自动注册清理机制
         """
-        if cls._session is None or cls._session.closed or cls.is_loop_closed(cls._session):
-            if cls._session:
-                await cls.close()
-            # 获取连接器（会自动初始化）
-            connector = await cls._get_connector()
-
-            # 强制获取新的事件循环
-            loop = asyncio.get_event_loop()
+        async with cls.lock:
+            current_pid = os.getpid()
+            if cls._pid != current_pid:
+                # 新进程，重新初始化
+                if cls._session:
+                    await cls.close()
+                cls._pid = current_pid
+
+            if cls._session is None or cls._session.closed or cls.is_loop_closed(cls._session):
+                if cls._session:
+                    await cls.close()
+                # 获取连接器（会自动初始化）
+                connector = await cls._get_connector()
+
+                # 强制获取新的事件循环
+                loop = asyncio.get_event_loop()
+
+                timeout = aiohttp.ClientTimeout(
+                    total=120,  # 整个请求最多 30 秒
+                    connect=10,  # 最多 5 秒连接
+                    sock_connect=10,
+                    sock_read=110,  # 最多 20 秒读取响应数据
+                )
 
-            # 创建新会话
-            cls._session = aiohttp.ClientSession(
-                connector=connector,
-                timeout=aiohttp.ClientTimeout(total=30),  # 默认30秒超时
-                loop=loop)  # 显式指定事件循环
+                # 创建新会话
+                cls._session = aiohttp.ClientSession(
+                    connector=connector,
+                    timeout=timeout,  # 默认30秒超时
+                    loop=loop,
+                )  # 显式指定事件循环
 
-            # # 注册退出时的清理钩子
-            cls._register_cleanup()
+                # # 注册退出时的清理钩子
+                cls._register_cleanup()
 
-        return cls._session
+            return cls._session
 
     @staticmethod
     def is_loop_closed(session: aiohttp.ClientSession) -> bool:
@@ -135,7 +163,7 @@ class ApiNetUtils:
         """
         loop = session._loop  # 获取会话绑定的事件循环
         if loop.is_closed():
-            # print("Event loop is closed")
+            print("Event loop is closed")
             return True
         # print("Event loop not is closed")
         return False

re_common-10.0.43/re_common/v2/baselibrary/utils/pinyin_utils.py

@@ -0,0 +1,178 @@
+import itertools
+from typing import List, Any
+
+from pypinyin import pinyin, Style
+
+
+class PinyinUtils:
+    @staticmethod
+    def get_pinyin_or_char(text: str) -> str:
+        """拼音 + 非中文原样返回 对应 toPinyin"""
+        result = []
+        for char in text:
+            if PinyinUtils.is_chinese(char):
+                py_one = pinyin(char, style=Style.NORMAL, v_to_u=True, heteronym=True)
+                result.append(py_one[0][0] if py_one else char)
+            else:
+                result.append(char)
+        return "".join(result)
+
+    @staticmethod
+    def get_pinyin_or_char_ex(text: str) -> str:
+        """拼音 + 非中文原样返回 对应 toPinyin"""
+        result = []
+        for char in text:
+            if PinyinUtils.is_chinese(char):
+                py_one = pinyin(char, style=Style.NORMAL, v_to_u=True, heteronym=True)
+                result.append("|".join(py_one[0]) if py_one else char)
+            else:
+                result.append(char)
+        return "".join(result)
+
+    @staticmethod
+    def is_chinese(char: str) -> bool:
+        """判断是否为汉字"""
+        return "\u4e00" <= char <= "\u9fff"
+
+    @staticmethod
+    def combine(lists: List[List[Any]]) -> list:
+        # lists = [[1, 2],['a', 'b'],[True, False]]
+        # 计算笛卡尔积
+        cartesian_product = list(itertools.product(*lists))
+
+        return [" ".join([str(ii) for ii in list(i)]) for i in cartesian_product]
+
+    @staticmethod
+    def to_pinyin_names(name: str) -> List[str]:
+        """
+        将中文姓名转换为所有可能的拼音组合
+
+        Args:
+            name: 中文姓名字符串
+
+        Returns:
+            所有可能的拼音组合列表
+        """
+        if not name:
+            return []
+
+        pinyin_options = []
+
+        for char in name:
+            py_one = PinyinUtils.get_pinyin_or_char_ex(char)
+            variants = []
+
+            if "ü" in py_one:
+                # 处理ü的特殊情况，生成u和v两种变体
+                for replacement in ["u", "v"]:
+                    replaced = py_one.replace("ü", replacement)
+                    variants.extend(replaced.split("|"))
+                variants = list(set(variants))
+            else:
+                variants = py_one.split("|")
+
+            pinyin_options.append(variants)
+
+        return PinyinUtils.combine(pinyin_options)
+
+    @staticmethod
+    def to_pinyin_name(name: str) -> str:
+        """
+        将中文姓名转换为拼音字符串
+
+        Args:
+            name: 中文字符串
+
+        Returns:
+            拼音字符串，用空格分隔每个字的拼音
+        """
+        result = []
+        for char in name:
+            py_one = PinyinUtils.get_pinyin_or_char(char).replace("ü", "v")
+            result.append(py_one)
+        return " ".join(result)
+
+
+VALID_PINYINS = {
+    "a", "o", "e", "ai", "ei", "ao", "ou", "an", "en", "ang", "eng", "er",
+    "ba", "bai", "ban", "bang", "bao", "bei", "ben", "beng", "bi", "bian", "biao", "bie", "bin", "bing", "bo", "bu",
+    "pa", "pai", "pan", "pang", "pao", "pei", "pen", "peng", "pi", "pian", "piao", "pie", "pin", "ping", "po", "pu",
+    "ma", "mai", "man", "mang", "mao", "mei", "men", "meng", "mi", "mian", "miao", "mie", "min", "ming", "mo",
+    "mou",
+    "mu",
+    "fa", "fan", "fang", "fei", "fen", "feng", "fo", "fou", "fu",
+    "da", "dai", "dan", "dang", "dao", "dei", "den", "deng", "di", "dian", "diao", "die", "ding", "diu", "dong",
+    "dou",
+    "du", "duan", "dui", "dun", "duo",
+    "ta", "tai", "tan", "tang", "tao", "tei", "teng", "ti", "tian", "tiao", "tie", "ting", "tong", "tou", "tu",
+    "tuan",
+    "tui", "tun", "tuo",
+    "na", "nai", "nan", "nang", "nao", "nei", "nen", "neng", "ni", "nian", "niang", "niao", "nie", "nin", "ning",
+    "niu",
+    "nong", "nou", "nu", "nuan", "nue", "nv",
+    "la", "lai", "lan", "lang", "lao", "lei", "leng", "li", "lia", "lian", "liang", "liao", "lie", "lin", "ling",
+    "liu",
+    "lo", "long", "lou", "lu", "luan", "lue", "lv",
+    "ga", "gai", "gan", "gang", "gao", "ge", "gei", "gen", "geng", "gong", "gou", "gu", "gua", "guai", "guan",
+    "guang",
+    "gui", "gun", "guo",
+    "ka", "kai", "kan", "kang", "kao", "ke", "ken", "keng", "kong", "kou", "ku", "kua", "kuai", "kuan", "kuang",
+    "kui",
+    "kun", "kuo",
+    "ha", "hai", "han", "hang", "hao", "he", "hei", "hen", "heng", "hong", "hou", "hu", "hua", "huai", "huan",
+    "huang",
+    "hui", "hun", "huo",
+    "ji", "jia", "jian", "jiang", "jiao", "jie", "jin", "jing", "jiong", "jiu", "ju", "juan", "jue", "jun",
+    "qi", "qia", "qian", "qiang", "qiao", "qie", "qin", "qing", "qiong", "qiu", "qu", "quan", "que", "qun",
+    "xi", "xia", "xian", "xiang", "xiao", "xie", "xin", "xing", "xiong", "xiu", "xu", "xuan", "xue", "xun",
+    "zha", "zhai", "zhan", "zhang", "zhao", "zhe", "zhei", "zhen", "zheng", "zhong", "zhou", "zhu", "zhua", "zhuai",
+    "zhuan", "zhuang", "zhui", "zhun", "zhuo",
+    "cha", "chai", "chan", "chang", "chao", "che", "chen", "cheng", "chi", "chong", "chou", "chu", "chua", "chuai",
+    "chuan", "chuang", "chui", "chun", "chuo",
+    "sha", "shai", "shan", "shang", "shao", "she", "shei", "shen", "sheng", "shi", "shou", "shu", "shua", "shuai",
+    "shuan", "shuang", "shui", "shun", "shuo",
+    "ra", "ran", "rang", "rao", "re", "ren", "reng", "ri", "rong", "rou", "ru", "rua", "ruan", "rui", "run", "ruo",
+    "za", "zai", "zan", "zang", "zao", "ze", "zei", "zen", "zeng", "zong", "zou", "zu", "zuan", "zui", "zun", "zuo",
+    "ca", "cai", "can", "cang", "cao", "ce", "cen", "ceng", "cong", "cou", "cu", "cuan", "cui", "cun", "cuo",
+    "sa", "sai", "san", "sang", "sao", "se", "sen", "seng", "song", "sou", "su", "suan", "sui", "sun", "suo",
+    "ya", "yan", "yang", "yao", "ye", "yi", "yin", "ying", "yo", "yong", "you", "yu", "yuan", "yue", "yun",
+    "wa", "wai", "wan", "wang", "wei", "wen", "weng", "wo", "wu"
+}
+
+
+def split_pinyin_all(s):
+    s = s.lower()
+    results = []
+
+    def dfs(start, path):
+        if start == len(s):
+            results.append(path[:])
+            return
+        for end in range(start + 1, min(len(s), start + 6) + 1):
+            part = s[start:end]
+            if part in VALID_PINYINS:
+                path.append(part)
+                dfs(end, path)
+                path.pop()
+
+    dfs(0, [])
+    return results
+
+
+def is_pinyin(word):
+    """检测一个字符串是否是合法拼音（无歧义）"""
+    try:
+        # 尝试转换为拼音，如果原词是拼音，转换结果应该和原词相近
+        pinyin_list = pinyin(word, style=Style.NORMAL)
+        reconstructed = "".join([p[0] for p in pinyin_list])
+        # 检查转换后的拼音是否与原词相似（忽略大小写）
+        return word.lower() == reconstructed.lower()
+    except:
+        return False
+
+def split_pinyin(word):
+    lists = split_pinyin_all(word)
+    lists = [" ".join(i) for i in lists]
+    if not lists:
+        return [word]
+    return lists