re-common 10.0.42__tar.gz → 10.0.44__tar.gz

{re_common-10.0.42/re_common.egg-info → re_common-10.0.44}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: re_common
-Version: 10.0.42
+Version: 10.0.44
 Summary: a library about all python projects
 Home-page: https://gitee.com/xujiangios/re-common
 Author: vic
@@ -11,6 +11,15 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.6
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license-file
+Dynamic: requires-python
+Dynamic: summary
 
 
     这是一个基础类，依赖很多的第三方包，是一个用得到的第三方库的封装，可以在此基础上迅速构建项目

re_common-10.0.44/re_common/v2/baselibrary/s3object/baseaioboto3.py

@@ -0,0 +1,122 @@
+import gzip
+import io
+from typing import Protocol
+
+import aioboto3
+import aiofiles
+from aiobotocore.config import AioConfig
+
+
+class AsyncReadable(Protocol):
+    async def read(self, n: int = -1) -> bytes:
+        ...
+
+
+# config = AioConfig(connect_timeout=600000, read_timeout=600000, retries={'max_attempts': 3},
+#                    max_pool_connections=10)
+
+class BaseAioBoto3(object):
+
+    def __init__(self, aws_access_key_id, aws_secret_access_key, endpoint_url,
+                 config=AioConfig(max_pool_connections=10)):
+        """
+          初始化华为云 OBS 客户端
+
+          Args:
+              access_key: 华为云 Access Key
+              secret_key: 华为云 Secret Key
+              region: 区域，如 'cn-north-4'
+              endpoint: 华为云 OBS 端点，可选
+          """
+        self.aws_access_key_id = aws_access_key_id
+        self.aws_secret_access_key = aws_secret_access_key
+        self.endpoint_url = endpoint_url
+        self.config = config
+        self.boto_session = None
+
+    async def initialize_class_variable(self):
+        if self.boto_session is None:
+            self.boto_session = aioboto3.Session(
+                aws_access_key_id=self.aws_access_key_id,
+                aws_secret_access_key=self.aws_secret_access_key
+            )
+
+    async def read_minio_data(self, bucket, key):
+        await self.initialize_class_variable()
+        async with self.boto_session.client("s3", endpoint_url=self.endpoint_url, config=self.config) as s3:
+            s3_ob = await s3.get_object(Bucket=bucket, Key=key)
+            result = await s3_ob["Body"].read()
+            return result
+
+    def ungzip(self, raw_bytes, encoding="utf-8"):
+        with gzip.GzipFile(fileobj=io.BytesIO(raw_bytes)) as gz:
+            return gz.read().decode(encoding)
+
+    # 异步下载大文件
+    async def download_file(self, bucket: str, key: str, local_path: str):
+        await self.initialize_class_variable()
+        async with self.boto_session.client("s3", endpoint_url=self.endpoint_url, config=self.config) as s3:
+            response = await s3.get_object(Bucket=bucket, Key=key)
+            body = response["Body"]
+
+            # 用异步方式写入本地
+            async with aiofiles.open(local_path, "wb") as f:
+                while True:
+                    chunk = await body.read(10 * 1024 * 1024)  # 每次读 10MB
+                    if not chunk:
+                        break
+                    await f.write(chunk)
+
+        return local_path
+
+    async def list_files(self, bucket: str, prefix: str, recursive: bool = True):
+        """
+        获取 bucket 下某个“目录”(prefix) 的文件列表
+
+        单文件返回样例 ['server_data/api-title-roc/py_full_organ_dic/', 'server_data/api-title-roc/py_full_organ_dic/part-00000.gz']
+
+        :param bucket: bucket 名
+        :param prefix: 目录前缀，如 'server_data/api-title-roc/'
+        :param recursive: 是否递归子目录
+        :return: List[str] 文件 key 列表
+        """
+        await self.initialize_class_variable()
+        keys = []
+
+        # 非递归时，用 delimiter 模拟目录
+        extra_args = {}
+        if not recursive:
+            extra_args["Delimiter"] = "/"
+
+        async with self.boto_session.client(
+                "s3",
+                endpoint_url=self.endpoint_url,
+                config=self.config
+        ) as s3:
+
+            continuation_token = None
+
+            while True:
+                kwargs = {
+                    "Bucket": bucket,
+                    "Prefix": prefix,
+                    **extra_args
+                }
+                # 下一页的“游标”
+                if continuation_token:
+                    kwargs["ContinuationToken"] = continuation_token
+
+                resp = await s3.list_objects_v2(**kwargs)
+
+                # 文件
+                for obj in resp.get("Contents", []):
+                    keys.append(obj["Key"])
+
+                # 是否还有下一页
+                if resp.get("IsTruncated"):  # 说明还有下一页
+                    # 下一页从哪里继续查
+                    continuation_token = resp.get("NextContinuationToken")
+                else:
+                    break
+
+        return keys

re_common-10.0.44/re_common/v2/baselibrary/s3object/huaweiobs.py

@@ -0,0 +1,80 @@
+# pip install pycryptodome==3.10.1
+# pip install esdk-obs-python
+# 引入模块
+import os
+
+from obs import ObsClient, GetObjectHeader
+
+
+class BaseObsClient(object):
+
+    def __init__(self, aws_access_key_id="", aws_secret_access_key="", endpoint_url=""):
+        self.aws_access_key_id = aws_access_key_id
+        self.aws_secret_access_key = aws_secret_access_key
+        self.endpoint_url = endpoint_url
+        self.client = None
+        if self.aws_access_key_id and self.aws_secret_access_key and self.endpoint_url:
+            self.get_client()
+
+    def get_client(self):
+        self.client = ObsClient(access_key_id=self.aws_access_key_id,
+                                secret_access_key=self.aws_secret_access_key,
+                                server=self.endpoint_url)
+        return self
+
+    def close(self):
+        self.client.close()
+
+    def put_object(self, bucket_name, objectKey, body):
+        """
+        直接写内容到文件
+        Args:
+            bucket_name:
+            key:
+            body: 需要
+
+        Returns:
+        """
+        # 上传文本对象
+        resp = self.client.putContent(bucket_name, objectKey, body)
+        # 返回码为2xx时，接口调用成功，否则接口调用失败
+        if resp.status < 300:
+            return True, resp
+        else:
+            return False, resp
+
+
+    def download_memobj(self, bucket_name, objectKey):
+        """
+        return: None
+        """
+        # 指定loadStreamInMemory为True忽略downloadpath路径，将文件的二进制流下载到内存
+        # 二进制下载对象
+        resp = self.client.getObject(bucketName=bucket_name, objectKey=objectKey, loadStreamInMemory=True)
+        # 返回码为2xx时，接口调用成功，否则接口调用失败
+        if resp.status < 300:
+            return True, resp
+        else:
+            return False, resp
+
+    def download_file(self, bucket_name, objectKey,downloadPath):
+        """
+        return: None
+        """
+        headers = GetObjectHeader()
+        resp = self.client.getObject(bucket_name, objectKey, downloadPath, headers=headers)
+        # 返回码为2xx时，接口调用成功，否则接口调用失败
+        if resp.status < 300:
+            return True, resp
+        else:
+            return False, resp
+
+
+    def list_prefixes(self,bucket_name,prefix, max_keys = 100):
+        # 列举桶内对象
+        resp = self.client.listObjects(bucket_name, prefix, max_keys=max_keys, encoding_type='url')
+        # 返回码为2xx时，接口调用成功，否则接口调用失败
+        if resp.status < 300:
+            return True, resp
+        else:
+            return False, resp

{re_common-10.0.42 → re_common-10.0.44}/re_common/v2/baselibrary/tools/data_processer/data_processer.py

@@ -467,7 +467,7 @@ def init_worker(process_func, process_args):
         _init_func()
 
     _async_init_work = _process_args.get("async_init_work", None)
-    if _init_func:
+    if _async_init_work:
         asyncio.run(_async_init_work())
 
 

{re_common-10.0.42 → re_common-10.0.44}/re_common/v2/baselibrary/tools/hdfs_base_processor.py

@@ -34,6 +34,7 @@ class HDFSBaseProcessor(abc.ABC):
         read_mode: Literal["all", "stream"] = "all",
         retries: int = 3,
         pool_factory: Callable[[], Awaitable[Any]] = None,
+        pool_factory_kwargs: dict = None,
         max_processes: int = None,  # 添加多进程支持
         result_dir: str = None,
     ):
@@ -48,6 +49,7 @@ class HDFSBaseProcessor(abc.ABC):
         self.retries = retries
         self.result_dir = result_dir
         self.pool_factory = pool_factory
+        self.pool_factory_kwargs = pool_factory_kwargs
         self.max_processes = max_processes or max(multiprocessing.cpu_count() - 1, 1)  # 默认使用CPU核心数-1
         self._client = None
 
@@ -64,7 +66,7 @@ class HDFSBaseProcessor(abc.ABC):
             return None
         global _pool
         if _pool is None:
-            _pool = await self.pool_factory()
+            _pool = await self.pool_factory(self.pool_factory_kwargs)
         return _pool
 
     def _list_gz_files(self) -> List[str]:

{re_common-10.0.42 → re_common-10.0.44}/re_common/v2/baselibrary/utils/db.py

@@ -9,33 +9,7 @@ from collections import namedtuple
 
 from aiomysql import Pool, Connection, Cursor
 
-DB_CONFIG = {
-    "host": "192.168.98.64",
-    "port": 4000,
-    "user": "dataware_house_baseUser",
-    "password": "FF19AF831AEBD580B450B16BF9264200",
-    "db": "dataware_house_base",
-    "charset": "utf8mb4",
-    "minsize": 16,  # 最小连接数
-    "maxsize": 128,  # 最大连接数
-    "autocommit": False,  # 自动提交事务
-    "pool_recycle": 3600,  # 每个连接的回收时间（秒），超过此时间后连接将被关闭并重新创建，避免失效连接
-    "echo": False,  # 打印SQL语句
-}
-
-DB_CONFIG1 = {
-    "host": "192.168.98.64",
-    "port": 4000,
-    "user": "foreign_fulltextUser",
-    "password": "i4hIeasw1qpmhGN2nwL7",
-    "db": "foreign_fulltext",
-    "charset": "utf8mb4",
-    "minsize": 16,  # 最小连接数
-    "maxsize": 128,  # 最大连接数
-    "autocommit": False,  # 自动提交事务
-    "pool_recycle": 3600,  # 每个连接的回收时间（秒），超过此时间后连接将被关闭并重新创建，避免失效连接
-    "echo": False,  # 打印SQL语句
-}
+
 
 
 async def get_pool_only(_DB_CONFIG: dict = None):

{re_common-10.0.42 → re_common-10.0.44}/re_common/v2/baselibrary/utils/string_bool.py

@@ -3,6 +3,8 @@ import re
 import regex
 import unicodedata
 
+from re_common.v2.baselibrary.decorators.utils import deprecated
+
 
 def is_ascii_alnum(char: str) -> bool:
     #  精准判断是否为英文或数字字符（ASCII 范围）
@@ -15,6 +17,20 @@ def is_all_english_chars(s):
 
 
 def contains_chinese_chars(s):
+    """
+    判断字符串中是否包含中文（汉字）字符
+
+    使用 regex 库（不是内置 re），通过 Unicode 属性 IsHan
+    来匹配任意一个汉字。
+
+    参数:
+        s (str): 待检测的字符串
+
+    返回:
+        bool:
+            True  - 字符串中至少包含一个中文字符
+            False - 字符串中不包含任何中文字符
+    """
     return bool(regex.search(r"[\p{IsHan}]", s))
 
 
@@ -108,56 +124,50 @@ def is_all_symbols(text):
     # 检查每个字符是否属于符号类别
     return all(unicodedata.category(char).startswith(('P', 'S')) for char in text)
 
-
-def is_whole_word_en(sub_str: str, long_str: str) -> bool:
+def is_whole_word_en_re(sub_str: str, long_str: str) -> bool:
+    """
+    与 is_whole_word_en 效果一致
     """
-    判断 sub_str 是否作为 long_str 中的一个完整英文单词（不被其他单词嵌套）。
+    if not sub_str or not long_str:
+        return False
 
-    参数:
-        sub_str: 要搜索的英文子串
-        long_str: 被搜索的字符串
+    pattern = rf"(^|[^a-z0-9-]){re.escape(sub_str)}([^a-z0-9-]|$)"
+    return re.search(pattern, long_str) is not None
 
-    返回:
-        True 表示 sub_str 是一个完整单词；False 表示是部分单词或不匹配。
-    """
-    # 用于 忽略大小写 进行匹配
-    regex_pattern = re.compile(r"[^a-z0-9]", re.IGNORECASE)  # 用于判断非字母数字字符
+@deprecated("请使用 is_whole_word_en_re 中的方法代替。")
+def is_whole_word_en(sub_str: str, long_str: str) -> bool:
+    regex_pattern = re.compile(r"[^a-z0-9-]", re.IGNORECASE)
 
     if not sub_str or not long_str:
         return False
 
-    # 检查整段是否完全等于 sub_str
-    if long_str == sub_str:
+    # 使用 startsWith 和 endsWith 检查边界
+    if long_str.startswith(sub_str) and long_str.endswith(sub_str) and len(sub_str) == len(long_str):
         return True
 
-    # 遍历所有 sub_str 的出现位置
-    index = 0
-    while index < len(long_str):
-        # 从字符串 long_str 的第 index 个位置开始，查找子串 sub_str 第一次出现的位置，并把它赋值给 index。
-        index = long_str.find(sub_str, index)
-        if index == -1:
-            break
-
-        # 检查 sub_str 前一个字符（如果有）是否为非字母数字
-        if index == 0:
-            is_start = True
+    # 检查是否在中间位置，且前后有非字母数字字符
+    # index = long_str.find(sub_str)
+    index_list = [m.start() for m in re.finditer(re.escape(sub_str), long_str)]
+
+    def get_bools(index):
+        if index >= 0:
+            is_start = False
+            is_end = False
+            if index == 0:
+                is_start = True
+            else:
+                is_start = bool(regex_pattern.match(long_str[index - 1]))
+
+            if len(long_str) == len(sub_str) + index:
+                is_end = True
+            else:
+                is_end = bool(regex_pattern.match(long_str[index + len(sub_str)]))
+
+            return is_start and is_end
         else:
-            is_start = bool(regex_pattern.match(long_str[index - 1]))
+            return False
 
-        # 检查 sub_str 后一个字符（如果有）是否为非字母数字
-        end_index = index + len(sub_str)
-        if end_index == len(long_str):
-            is_end = True
-        else:
-            is_end = bool(regex_pattern.match(long_str[end_index]))
-
-        if is_start and is_end:
-            return True
-
-        # 移动索引继续查找
-        index += 1
-
-    return False
+    return any([get_bools(index) for index in index_list])
 
 
 def is_whole_word(sub_str: str, long_str: str) -> bool:
@@ -178,10 +188,10 @@ def is_whole_word(sub_str: str, long_str: str) -> bool:
         # 是否是字母数字
         if is_ascii_alnum(sub_str[0]) or is_ascii_alnum(sub_str[-1]):
             # 表示中英文混合 看是否是截断单词即可
-            return is_whole_word_en(sub_str, long_str)
+            return is_whole_word_en_re(sub_str, long_str)
         else:
             # 中文子串只要被包含即可视为“完整词”
             return is_contain
     else:
         # 英文使用完整单词判断逻辑
-        return is_whole_word_en(sub_str, long_str)
+        return is_whole_word_en_re(sub_str, long_str)

{re_common-10.0.42 → re_common-10.0.44/re_common.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: re_common
-Version: 10.0.42
+Version: 10.0.44
 Summary: a library about all python projects
 Home-page: https://gitee.com/xujiangios/re-common
 Author: vic
@@ -11,6 +11,15 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.6
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license-file
+Dynamic: requires-python
+Dynamic: summary
 
 
     这是一个基础类，依赖很多的第三方包，是一个用得到的第三方库的封装，可以在此基础上迅速构建项目

{re_common-10.0.42 → re_common-10.0.44}/re_common.egg-info/SOURCES.txt

@@ -187,6 +187,7 @@ re_common/v2/baselibrary/helpers/search_packge/test.py
 re_common/v2/baselibrary/s3object/__init__.py
 re_common/v2/baselibrary/s3object/baseaioboto3.py
 re_common/v2/baselibrary/s3object/baseboto3.py
+re_common/v2/baselibrary/s3object/huaweiobs.py
 re_common/v2/baselibrary/tools/WeChatRobot.py
 re_common/v2/baselibrary/tools/__init__.py
 re_common/v2/baselibrary/tools/ac_ahocorasick.py

{re_common-10.0.42 → re_common-10.0.44}/setup.py

@@ -34,7 +34,7 @@ long_description = """
 """
 setuptools.setup(
     name="re_common",
-    version="10.0.42",
+    version="10.0.44",
     author="vic",
     author_email="xujiang5@163.com",
     description="a library about all python projects",

re_common-10.0.42/re_common/v2/baselibrary/s3object/baseaioboto3.py

@@ -1,48 +0,0 @@
-import aioboto3
-import aiofiles
-from aiobotocore.config import AioConfig
-
-
-# config = AioConfig(connect_timeout=600000, read_timeout=600000, retries={'max_attempts': 3},
-#                    max_pool_connections=10)
-
-class BaseAioBoto3(object):
-
-    def __init__(self, aws_access_key_id, aws_secret_access_key, endpoint_url,
-                 config=AioConfig(max_pool_connections=10)):
-        self.aws_access_key_id = aws_access_key_id
-        self.aws_secret_access_key = aws_secret_access_key
-        self.endpoint_url = endpoint_url
-        self.config = config
-        self.boto_session = None
-
-    async def initialize_class_variable(self):
-        if self.boto_session is None:
-            self.boto_session = aioboto3.Session(
-                aws_access_key_id=self.aws_access_key_id,
-                aws_secret_access_key=self.aws_secret_access_key,
-            )
-
-    async def read_minio_data(self, bucket, key):
-        await self.initialize_class_variable()
-        async with self.boto_session.client("s3", endpoint_url=self.endpoint_url, config=self.config) as s3:
-            s3_ob = await s3.get_object(Bucket=bucket, Key=key)
-            result = await s3_ob["Body"].read()
-            return result
-
-    # 异步下载大文件
-    async def download_file(self, bucket: str, key: str, local_path: str):
-        await self.initialize_class_variable()
-        async with self.boto_session.client("s3", endpoint_url=self.endpoint_url, config=self.config) as s3:
-            response = await s3.get_object(Bucket=bucket, Key=key)
-            body = response["Body"]
-
-            # 用异步方式写入本地
-            async with aiofiles.open(local_path, "wb") as f:
-                while True:
-                    chunk = await body.read(10 * 1024 * 1024)  # 每次读 10MB
-                    if not chunk:
-                        break
-                    await f.write(chunk)
-
-        return local_path