re-common 10.0.37__py3-none-any.whl → 10.0.39__py3-none-any.whl

re_common/v2/baselibrary/tools/WeChatRobot.py

@@ -1,95 +1,95 @@
-import sqlite3
-import requests
-import pandas as pd
-import os
-import json
-import traceback
-
-
-# c1d3a814-1a02-4bbd-b5c2-f756fef92cb8: b层机器人消息群-非聊天 的 pythonspark
-# 013547da-3d78-4a7f-b4a7-e668b192c293: b层机器人消息群-非聊天 的 数仓B层服务端部署通知
-
-# 发送消息到企业微信机器人
-# vx_key: string类型，自己的企业微信机器人的key
-# s:string类型，要发送的消息
-def send_vx(vx_key, s, i=0):
-    vx_url = 'https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=' + vx_key
-    headers = {"Content-Type": "text/plain"}
-    data = {
-        "msgtype": "text",
-        "text": {
-            "content": s,
-        }
-    }
-    if i > 3:
-        raise Exception(str(traceback.format_exc()))
-    try:
-        requests.post(url=vx_url, headers=headers, json=data, timeout=30)
-    except:
-        i = i + 1
-        send_vx(vx_key, str(traceback.format_exc()), i)
-
-
-# 发送文件到企业微信机器人
-# vx_key: string类型，自己的企业微信机器人的key
-# file_path: string类型，文件地址
-def post_file(vx_key, file_path):
-    id_url = 'https://qyapi.weixin.qq.com/cgi-bin/webhook/upload_media?key=' + vx_key + '&type=file'
-    wx_url = 'https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=' + vx_key
-    data = {'file': open(file_path, 'rb')}
-    response = requests.post(url=id_url, files=data)
-    json_res = response.json()
-    media_id = json_res['media_id']
-    data = {"msgtype": "file",
-            "file": {"media_id": media_id}
-            }
-    try:
-        requests.post(url=wx_url, json=data)
-    except:
-        send_vx(send_vx, str(traceback.format_exc()))
-
-
-# data: dataframe|string|dict|list|tuple|array
-# file_name: 带后缀的完整文件名
-# file_type: 文件类型，包括csv、excel、txt、json、sql
-def file_to_vx(vx_key, data, file_name):
-    file_type = file_name.split('.')[-1]
-    if file_type == "xls" or file_type == "xlsx":
-        file_type = "excel"
-    current_dir = os.getcwd()
-    temp_dir = os.path.join(os.getcwd(), "tmp")
-    if os.path.exists(temp_dir):
-        pass
-    else:
-        os.makedirs(temp_dir)
-    file_path = current_dir + "/" + file_name
-    try:
-        if isinstance(data, pd.DataFrame) and file_type != "txt":
-            if file_type == "db3":
-                conn = sqlite3.connect(file_path)
-                data.to_sql('base_table', conn, if_exists='replace', index=False)
-                post_file(vx_key, file_path)
-            else:
-                code_str = "data.to_" + file_type + "(file_path,index=False)"
-                eval(code_str)
-                post_file(vx_key, file_path)
-            os.system('rm ' + file_path + '')
-        else:
-            if isinstance(data, dict):
-                data_str = json.dumps(data, ensure_ascii=False)
-            elif isinstance(data, list):
-                data_str = ""
-                for i in data:
-                    if isinstance(i, dict):
-                        data_str = data_str + json.dumps(i, ensure_ascii=False) + "\n"
-                    else:
-                        data_str = data_str + str(i) + "\n"
-            else:
-                data_str = str(data)
-            print(data_str[:100])
-            with open(file_path, 'w', encoding='utf-8') as f:
-                f.write(data_str)
-            post_file(vx_key, file_path)
-    except:
-        send_vx(vx_key, str(traceback.format_exc()))
-    os.system('rm -r' + temp_dir + '')
+import sqlite3
+import requests
+import pandas as pd
+import os
+import json
+import traceback
+
+
+# c1d3a814-1a02-4bbd-b5c2-f756fef92cb8: b层机器人消息群-非聊天 的 pythonspark
+# 013547da-3d78-4a7f-b4a7-e668b192c293: b层机器人消息群-非聊天 的 数仓B层服务端部署通知
+
+# 发送消息到企业微信机器人
+# vx_key: string类型，自己的企业微信机器人的key
+# s:string类型，要发送的消息
+def send_vx(vx_key, s, i=0):
+    vx_url = 'https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=' + vx_key
+    headers = {"Content-Type": "text/plain"}
+    data = {
+        "msgtype": "text",
+        "text": {
+            "content": s,
+        }
+    }
+    if i > 3:
+        raise Exception(str(traceback.format_exc()))
+    try:
+        requests.post(url=vx_url, headers=headers, json=data, timeout=30)
+    except:
+        i = i + 1
+        send_vx(vx_key, str(traceback.format_exc()), i)
+
+
+# 发送文件到企业微信机器人
+# vx_key: string类型，自己的企业微信机器人的key
+# file_path: string类型，文件地址
+def post_file(vx_key, file_path):
+    id_url = 'https://qyapi.weixin.qq.com/cgi-bin/webhook/upload_media?key=' + vx_key + '&type=file'
+    wx_url = 'https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=' + vx_key
+    data = {'file': open(file_path, 'rb')}
+    response = requests.post(url=id_url, files=data)
+    json_res = response.json()
+    media_id = json_res['media_id']
+    data = {"msgtype": "file",
+            "file": {"media_id": media_id}
+            }
+    try:
+        requests.post(url=wx_url, json=data)
+    except:
+        send_vx(send_vx, str(traceback.format_exc()))
+
+
+# data: dataframe|string|dict|list|tuple|array
+# file_name: 带后缀的完整文件名
+# file_type: 文件类型，包括csv、excel、txt、json、sql
+def file_to_vx(vx_key, data, file_name):
+    file_type = file_name.split('.')[-1]
+    if file_type == "xls" or file_type == "xlsx":
+        file_type = "excel"
+    current_dir = os.getcwd()
+    temp_dir = os.path.join(os.getcwd(), "tmp")
+    if os.path.exists(temp_dir):
+        pass
+    else:
+        os.makedirs(temp_dir)
+    file_path = current_dir + "/" + file_name
+    try:
+        if isinstance(data, pd.DataFrame) and file_type != "txt":
+            if file_type == "db3":
+                conn = sqlite3.connect(file_path)
+                data.to_sql('base_table', conn, if_exists='replace', index=False)
+                post_file(vx_key, file_path)
+            else:
+                code_str = "data.to_" + file_type + "(file_path,index=False)"
+                eval(code_str)
+                post_file(vx_key, file_path)
+            os.system('rm ' + file_path + '')
+        else:
+            if isinstance(data, dict):
+                data_str = json.dumps(data, ensure_ascii=False)
+            elif isinstance(data, list):
+                data_str = ""
+                for i in data:
+                    if isinstance(i, dict):
+                        data_str = data_str + json.dumps(i, ensure_ascii=False) + "\n"
+                    else:
+                        data_str = data_str + str(i) + "\n"
+            else:
+                data_str = str(data)
+            print(data_str[:100])
+            with open(file_path, 'w', encoding='utf-8') as f:
+                f.write(data_str)
+            post_file(vx_key, file_path)
+    except:
+        send_vx(vx_key, str(traceback.format_exc()))
+    os.system('rm -r' + temp_dir + '')

re_common/v2/baselibrary/tools/ac_ahocorasick.py

@@ -1,76 +1,76 @@
-import pickle
-
-import ahocorasick
-
-
-class ACTool(object):
-
-    def __init__(self):
-        self.automaton = ahocorasick.Automaton()
-
-    def add_word(self, key, value, overwrite=True) -> bool:
-        """
-        为 AC 机添加数据,默认情况下 key重复直接覆盖
-        :param key: 要添加的关键字
-        :param value: 对应的值
-        :param overwrite: 是否覆盖已有的 key，默认为 True
-        :return: 是否成功添加或覆盖
-        """
-        if key in self.automaton:  # 检查 key 是否已存在
-            if overwrite:  # 如果允许覆盖
-                self.automaton.add_word(key, value)
-                return True
-            else:  # 不允许覆盖，跳过
-                return False
-        else:  # key 不存在，直接添加
-            self.automaton.add_word(key, value)
-            return True
-
-    def is_exists_key(self, key) -> bool:
-        # 是否存在key
-        if self.automaton.exists(key):
-            return True
-        else:
-            return False
-
-    def make_automaton(self):
-        """
-        添加完词后需要构建
-        """
-        self.automaton.make_automaton()
-
-    def iter(self, key):
-        """
-        结果为可迭代对象 可通过list 转换 [(end_index, value)]
-        tool.add_word("he", "word1")
-        tool.add_word("hello", "word2")
-
-        # 在字符串中查找匹配
-        input_string = "hello world"
-        matches = list(tool.automaton.iter(input_string))
-        print(matches)  # [(1, 'word1'), (4, 'word2')]
-
-        (1, 'word1'):
-        end_index = 1: 表示匹配的关键字 "he" 在 input_string = "hello world" 中的结束位置是索引 1（即字符串 "he" 的最后一个字符 'e' 的位置）。
-        "hello world" 的索引：h(0)e(1)l(2)l(3)o(4) (5)w(6)o(7)r(8)l(9)d(10)。
-        value = 'word1': 表示匹配的关键字 "he" 对应的值是 "word1"。
-        (4, 'word2'):
-        end_index = 4: 表示匹配的关键字 "hello" 在 input_string = "hello world" 中的结束位置是索引 4（即字符串 "hello" 的最后一个字符 'o' 的位置）。
-        value = 'word2': 表示匹配的关键字 "hello" 对应的值是 "word2"。
-
-        注意: 结果只会返回 value 不会返回 key，如果需要key  请将key 组合到结果中
-        """
-
-        result_iter = self.automaton.iter(key)  # ahocorasick.AutomatonSearchIter
-        return result_iter
-    def save(self,local_temp_path):
-        """
-        将构建好的ac自动机保存到本地
-        """
-        self.automaton.save(local_temp_path,pickle.dumps)
-
-    def load(self,local_temp_path):
-        """
-        加载已经构建好的ac自动机
-        """
+import pickle
+
+import ahocorasick
+
+
+class ACTool(object):
+
+    def __init__(self):
+        self.automaton = ahocorasick.Automaton()
+
+    def add_word(self, key, value, overwrite=True) -> bool:
+        """
+        为 AC 机添加数据,默认情况下 key重复直接覆盖
+        :param key: 要添加的关键字
+        :param value: 对应的值
+        :param overwrite: 是否覆盖已有的 key，默认为 True
+        :return: 是否成功添加或覆盖
+        """
+        if key in self.automaton:  # 检查 key 是否已存在
+            if overwrite:  # 如果允许覆盖
+                self.automaton.add_word(key, value)
+                return True
+            else:  # 不允许覆盖，跳过
+                return False
+        else:  # key 不存在，直接添加
+            self.automaton.add_word(key, value)
+            return True
+
+    def is_exists_key(self, key) -> bool:
+        # 是否存在key
+        if self.automaton.exists(key):
+            return True
+        else:
+            return False
+
+    def make_automaton(self):
+        """
+        添加完词后需要构建
+        """
+        self.automaton.make_automaton()
+
+    def iter(self, key):
+        """
+        结果为可迭代对象 可通过list 转换 [(end_index, value)]
+        tool.add_word("he", "word1")
+        tool.add_word("hello", "word2")
+
+        # 在字符串中查找匹配
+        input_string = "hello world"
+        matches = list(tool.automaton.iter(input_string))
+        print(matches)  # [(1, 'word1'), (4, 'word2')]
+
+        (1, 'word1'):
+        end_index = 1: 表示匹配的关键字 "he" 在 input_string = "hello world" 中的结束位置是索引 1（即字符串 "he" 的最后一个字符 'e' 的位置）。
+        "hello world" 的索引：h(0)e(1)l(2)l(3)o(4) (5)w(6)o(7)r(8)l(9)d(10)。
+        value = 'word1': 表示匹配的关键字 "he" 对应的值是 "word1"。
+        (4, 'word2'):
+        end_index = 4: 表示匹配的关键字 "hello" 在 input_string = "hello world" 中的结束位置是索引 4（即字符串 "hello" 的最后一个字符 'o' 的位置）。
+        value = 'word2': 表示匹配的关键字 "hello" 对应的值是 "word2"。
+
+        注意: 结果只会返回 value 不会返回 key，如果需要key  请将key 组合到结果中
+        """
+
+        result_iter = self.automaton.iter(key)  # ahocorasick.AutomatonSearchIter
+        return result_iter
+    def save(self,local_temp_path):
+        """
+        将构建好的ac自动机保存到本地
+        """
+        self.automaton.save(local_temp_path,pickle.dumps)
+
+    def load(self,local_temp_path):
+        """
+        加载已经构建好的ac自动机
+        """
         self.automaton=ahocorasick.load(local_temp_path, pickle.loads)

re_common/v2/baselibrary/tools/concurrency.py

@@ -1,35 +1,35 @@
-import asyncio
-from asyncio import Semaphore
-from typing import Awaitable, List, Iterable, Callable
-
-
-class AsyncTaskPool:
-    def __init__(self, max_workers: int = 10):
-        """
-
-        Args:
-            max_workers: 任务最大并发数
-        """
-        self.semaphore = Semaphore(max_workers)
-
-    async def _run_task(self, task: Awaitable):
-        async with self.semaphore:
-            return await task
-
-    async def run(self, tasks: List[Awaitable]):
-        return await asyncio.gather(*[self._run_task(task) for task in tasks])
-
-    async def map(self, fn: Callable[..., Awaitable], *iterables: Iterable):
-        tasks = [fn(*args) for args in zip(*iterables)]
-        return await self.run(tasks)
-
-
-if __name__ == "__main__":
-
-    async def test(x, y):
-        await asyncio.sleep(1)
-        print(x, y)
-        return x + y
-
-    result = asyncio.run(AsyncTaskPool(2).map(test, [1, 2, 3, 4], [5, 6, 7, 8]))
-    print(result)
+import asyncio
+from asyncio import Semaphore
+from typing import Awaitable, List, Iterable, Callable
+
+
+class AsyncTaskPool:
+    def __init__(self, max_workers: int = 10):
+        """
+
+        Args:
+            max_workers: 任务最大并发数
+        """
+        self.semaphore = Semaphore(max_workers)
+
+    async def _run_task(self, task: Awaitable):
+        async with self.semaphore:
+            return await task
+
+    async def run(self, tasks: List[Awaitable]):
+        return await asyncio.gather(*[self._run_task(task) for task in tasks])
+
+    async def map(self, fn: Callable[..., Awaitable], *iterables: Iterable):
+        tasks = [fn(*args) for args in zip(*iterables)]
+        return await self.run(tasks)
+
+
+if __name__ == "__main__":
+
+    async def test(x, y):
+        await asyncio.sleep(1)
+        print(x, y)
+        return x + y
+
+    result = asyncio.run(AsyncTaskPool(2).map(test, [1, 2, 3, 4], [5, 6, 7, 8]))
+    print(result)

re_common/v2/baselibrary/tools/data_processer/base.py

@@ -1,53 +1,53 @@
-from abc import ABC, abstractmethod
-from typing import List, Generator
-
-
-class BaseFileReader(ABC):
-
-    def __init__(self, batch_size: int = 10000):
-        self.batch_size = batch_size
-        self.read_model = 1
-
-    @abstractmethod
-    def list_files(self, path: str) -> List[str]:
-        """列出路径下所有目标文件"""
-        pass
-
-    @abstractmethod
-    def count_lines(self, file_path: str) -> int:
-        """统计文件行数"""
-        pass
-
-    @abstractmethod
-    def read_lines(self, file_path: str) -> Generator[List[str], None, None]:
-        """读取文件内容，返回批量数据"""
-        pass
-
-    @abstractmethod
-    def read_all(self, file_path: str) -> List[List[str]]:
-        """读取整个文件，默认按1000行分批"""
-        return [line for line in self.read_lines(file_path)]
-
-    def read_select(self, file_path: str) -> Generator[List[str], None, None]:
-        if self.read_model == 1:
-            for batch_data in self.read_lines(file_path):
-                yield batch_data
-        elif self.read_model == 2:
-            for batch_data in self.read_all(file_path):
-                yield batch_data
-        else:
-            raise Exception("模式选择错误")
-
-
-class BaseFileWriter(ABC):
-
-    def __init__(self, file_path: str, compress: bool = True, overwrite: bool = True, encoding: str = "utf-8"):
-        self.file_path = file_path
-        self.compress = compress
-        self.encoding = encoding
-        self.overwrite = overwrite
-
-    @abstractmethod
-    def write_lines(self, lines: List[str], file_path: str):
-        """写入多行文本到文件，支持压缩"""
-        pass
+from abc import ABC, abstractmethod
+from typing import List, Generator
+
+
+class BaseFileReader(ABC):
+
+    def __init__(self, batch_size: int = 10000):
+        self.batch_size = batch_size
+        self.read_model = 1
+
+    @abstractmethod
+    def list_files(self, path: str) -> List[str]:
+        """列出路径下所有目标文件"""
+        pass
+
+    @abstractmethod
+    def count_lines(self, file_path: str) -> int:
+        """统计文件行数"""
+        pass
+
+    @abstractmethod
+    def read_lines(self, file_path: str) -> Generator[List[str], None, None]:
+        """读取文件内容，返回批量数据"""
+        pass
+
+    @abstractmethod
+    def read_all(self, file_path: str) -> List[List[str]]:
+        """读取整个文件，默认按1000行分批"""
+        return [line for line in self.read_lines(file_path)]
+
+    def read_select(self, file_path: str) -> Generator[List[str], None, None]:
+        if self.read_model == 1:
+            for batch_data in self.read_lines(file_path):
+                yield batch_data
+        elif self.read_model == 2:
+            for batch_data in self.read_all(file_path):
+                yield batch_data
+        else:
+            raise Exception("模式选择错误")
+
+
+class BaseFileWriter(ABC):
+
+    def __init__(self, file_path: str, compress: bool = True, overwrite: bool = True, encoding: str = "utf-8"):
+        self.file_path = file_path
+        self.compress = compress
+        self.encoding = encoding
+        self.overwrite = overwrite
+
+    @abstractmethod
+    def write_lines(self, lines: List[str], file_path: str):
+        """写入多行文本到文件，支持压缩"""
+        pass