terryutils 1.0.6__tar.gz → 1.0.7__tar.gz

{terryutils-1.0.6 → terryutils-1.0.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: terryutils
-Version: 1.0.6
+Version: 1.0.7
 Summary: My personal python toolkit
 Author-email: Terry Zhu <596480606@qq.com>
 License: MIT

{terryutils-1.0.6 → terryutils-1.0.7}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "terryutils"               # pip install 用这个
-version = "1.0.6"
+version = "1.0.7"
 description = "My personal python toolkit"
 readme = "README.md"
 requires-python = ">=3.8"

terryutils-1.0.7/src/terryutils/mysql_pool_util.py

@@ -0,0 +1,63 @@
+from dbutils.pooled_db import PooledDB
+import pymysql
+from pymysql.cursors import DictCursor
+
+MYSQL_POOL = PooledDB(
+    creator=pymysql,
+    maxconnections=20,     # 最大连接数（重点）
+    mincached=2,           # 启动时创建
+    maxcached=10,          # 池中最大空闲
+    blocking=True,         # 连接耗尽时等待
+    host="localhost",
+    user="user",
+    password="password",
+    database="db",
+    port=3306,
+    charset="utf8mb4",
+    cursorclass=DictCursor,
+    autocommit=False
+)
+
+
+class MysqlClient:
+
+    def __init__(self, pool):
+        self.pool = pool
+
+    def _execute(self, fn):
+        conn = self.pool.connection()
+        try:
+            result = fn(conn)
+            conn.commit()
+            return result
+        except Exception:
+            conn.rollback()
+            raise
+        finally:
+            conn.close()  # ❗归还给 pool，不是真 close
+
+    def execute(self, sql, params=None):
+        def run(conn):
+            with conn.cursor() as cursor:
+                return cursor.execute(sql, params or ())
+        return self._execute(run)
+
+    def executemany(self, sql, params_list):
+        def run(conn):
+            with conn.cursor() as cursor:
+                return cursor.executemany(sql, params_list)
+        return self._execute(run)
+
+    def fetchone(self, sql, params=None):
+        def run(conn):
+            with conn.cursor() as cursor:
+                cursor.execute(sql, params or ())
+                return cursor.fetchone()
+        return self._execute(run)
+
+    def fetchall(self, sql, params=None):
+        def run(conn):
+            with conn.cursor() as cursor:
+                cursor.execute(sql, params or ())
+                return cursor.fetchall()
+        return self._execute(run)

terryutils-1.0.7/src/terryutils/task_framework.py

@@ -0,0 +1,222 @@
+import os
+import sys
+import time
+import asyncio
+import inspect
+import logging
+import traceback
+import threading
+from queue import Queue, Empty
+from dataclasses import dataclass, field
+from typing import Callable, Optional
+from logging.handlers import RotatingFileHandler
+
+log_dir = "logs"
+os.makedirs(log_dir, exist_ok=True)
+
+log_file = os.path.join(log_dir, "request_framework.log")
+
+formatter = logging.Formatter(
+    '%(asctime)s | %(levelname)-5s | %(threadName)s | %(message)s',
+    datefmt='%Y-%m-%d %H:%M:%S'
+)
+# --- 文件 handler ---
+file_handler = RotatingFileHandler(
+    log_file,
+    maxBytes=50 * 1024 * 1024,
+    backupCount=5,
+    encoding='utf-8'
+)
+file_handler.setFormatter(formatter)
+file_handler.setLevel(logging.INFO)
+
+# --- 控制台 handler ---
+console_handler = logging.StreamHandler()
+console_handler.setFormatter(formatter)
+console_handler.setLevel(logging.INFO)
+
+logger = logging.getLogger("RequestFramework")
+logger.setLevel(logging.INFO)
+
+# 防止重复 add
+if not logger.handlers:
+    logger.addHandler(file_handler)
+    logger.addHandler(console_handler)
+
+logger.propagate = False
+
+
+
+
+# --- 1. 任务实体：统一封装请求 ---
+@dataclass
+class RequestTask:
+    name: str  # 任务名称（日志用）
+    func: Callable  # 执行函数 (requests.get 或 sdk.call)
+    args: tuple = field(default_factory=tuple)
+    kwargs: dict = field(default_factory=dict)
+    callback: Optional[Callable] = None  # 成功回调
+    max_retries: int = 3  # 最大重试次数
+    retry_count: int = 0  # 当前重试计数
+    priority: int = 10  # 优先级（数字越小越高）
+
+
+# --- 2. 频率控制器：令牌桶思想 ---
+class RateLimiter:
+    def __init__(self, qps: int):
+        self.interval = 1.0 / qps
+        self.last_run = 0.0
+        self._lock = threading.Lock()
+
+    def wait(self):
+        with self._lock:
+            elapsed = time.time() - self.last_run
+            wait_time = self.interval - elapsed
+            if wait_time > 0:
+                time.sleep(wait_time)
+            self.last_run = time.time()
+
+
+# --- 3. 调度引擎 ---
+class RequestEngine:
+    def __init__(self, worker_num: int = 3, qps: int = 5):
+        self.task_queue = Queue()  # 待处理队列
+        self.error_queue = Queue()  # 错误队列
+        self.limiter = RateLimiter(qps)  # 频率控制
+        self.worker_num = worker_num
+        self.running = False
+
+    def add_task(self, task: RequestTask):
+        """外部提交任务接口"""
+        self.task_queue.put(task)
+
+    def start(self):
+        """启动引擎"""
+        self.running = True
+        for i in range(self.worker_num):
+            t = threading.Thread(target=self._worker_loop, name=f"Worker-{i}")
+            t.daemon = True
+            t.start()
+        logger.info(f"引擎启动：工作线程={self.worker_num}, QPS限制={1 / self.limiter.interval}")
+
+    def _worker_loop(self):
+        """工作线程主循环"""
+        while self.running:
+            try:
+                # 1. 获取任务 (阻塞1秒以便检查running状态)
+                task: RequestTask = self.task_queue.get(timeout=1)
+
+                # 2. 频率控制
+                self.limiter.wait()
+
+                # 3. 执行任务
+                self._execute(task)
+
+                self.task_queue.task_done()
+            except Empty:
+                continue
+
+    def _execute(self, task: RequestTask):
+        """执行逻辑：包含重试和异常处理"""
+        try:
+            logger.info(f"正在执行: {task.name}, 参数: {task.args}")
+            # 判断并处理异步函数
+            if inspect.iscoroutinefunction(task.func):
+                # 如果是 async def 定义的函数，启动一个临时的事件循环运行它
+                result = asyncio.run(task.func(*task.args, **task.kwargs))
+            else:
+                # 如果是普通的 def 定义的函数，直接调用
+                result = task.func(*task.args, **task.kwargs)
+
+            # 4. 执行回调
+            if task.callback:
+                task.callback(result)
+
+            logger.info(f"执行成功: {task.name}")
+
+        except Exception as e:
+            exc_type, exc_obj, tb = sys.exc_info()
+            filename = tb.tb_frame.f_code.co_filename
+            lineno = tb.tb_lineno
+            traceback.print_exc()
+            logger.error(
+                f"执行失败: {task.name}, "
+                f"{exc_type.__name__} at {filename}:{lineno}, "
+                f"错误信息: {e}"
+            )
+
+            # 5. 重试逻辑
+            if task.retry_count < task.max_retries:
+                task.retry_count += 1
+                wait_s = 2 ** task.retry_count  # 指数退避
+                logger.warning(f"任务 {task.name} 将在 {wait_s}s 后进行第 {task.retry_count} 次重试")
+
+                # 异步延迟重试：不阻塞当前线程，放入定时器重新入队
+                threading.Timer(wait_s, self.add_task, args=[task]).start()
+            else:
+                # 6. 最终失败，放入错误队列
+                logger.critical(f"任务 {task.name} 达到最大重试次数，放弃处理")
+                self.error_queue.put({"task": task, "error": str(e)})
+
+
+
+
+if __name__ == "__main__":
+    import requests
+
+    # 1. 初始化引擎（5个并发，每秒最多2个请求）
+    engine = RequestEngine(worker_num=3, qps=1)
+    # 2. 启动
+    engine.start()
+
+
+    # 示例 A: 封装 requests 请求
+    def get_weather(city):
+        resp = requests.get(f"https://httpbin.org/get?city={city}", timeout=5)
+        resp.raise_for_status()
+        return resp.json()
+
+    def callback_func(result):
+        print(result)
+
+    task1 = RequestTask(
+        name="天气查询-北京",
+        func=get_weather,
+        args=("Beijing",),
+        callback=callback_func # 成功后落库
+    )
+
+
+    # 示例 B: 封装 SDK 请求
+    class MySDK:
+        def upload_file(self, file_id):
+            if file_id == "bad_id": raise ValueError("SDK内部错误")
+            return {"status": "uploaded", "id": file_id}
+
+
+    sdk = MySDK()
+    task2 = RequestTask(
+        name="SDK上传-001",
+        func=sdk.upload_file,
+        kwargs={"file_id": "file_123"},
+        callback=callback_func
+    )
+
+    # 示例 C: 故意失败的任务（测试重试）
+    task3 = RequestTask(
+        name="必失败任务",
+        func=sdk.upload_file,
+        kwargs={"file_id": "bad_id"},
+        max_retries=2
+    )
+
+    # 3. 提交任务到队列
+    engine.add_task(task1)
+    engine.add_task(task2)
+    engine.add_task(task3)
+
+    # 保持主线程运行
+    try:
+        while True: time.sleep(1)
+    except KeyboardInterrupt:
+        print("退出中...")

{terryutils-1.0.6 → terryutils-1.0.7}/src/terryutils.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: terryutils
-Version: 1.0.6
+Version: 1.0.7
 Summary: My personal python toolkit
 Author-email: Terry Zhu <596480606@qq.com>
 License: MIT

{terryutils-1.0.6 → terryutils-1.0.7}/src/terryutils.egg-info/SOURCES.txt

@@ -3,7 +3,9 @@ pyproject.toml
 src/terryutils/__init__.py
 src/terryutils/date_util.py
 src/terryutils/log.py
+src/terryutils/mysql_pool_util.py
 src/terryutils/mysql_util.py
+src/terryutils/task_framework.py
 src/terryutils.egg-info/PKG-INFO
 src/terryutils.egg-info/SOURCES.txt
 src/terryutils.egg-info/dependency_links.txt