dotask 0.2.4__tar.gz → 0.3.0__tar.gz

{dotask-0.2.4 → dotask-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dotask
-Version: 0.2.4
+Version: 0.3.0
 Summary: 发布订阅注解方式
 Home-page: https://gitee.com/d-yz/task-manager
 Author: dyz
@@ -35,6 +35,11 @@ Dynamic: license-file
 3. 添加本地调用shell工具函数->可以很好的适配@task消费者
    1. max_concurrent:控制并发执行的shell数量
    2. 引入dotask.Shell 调用Shell(max_concurrent=?).local_shell_execute(cmd,callback)来使用
+### 0.3.x
+1. 添加令牌桶,通过`from dotask.util import TokenBucket`使用,可以更好的限制生产消费速率
+- 参数说明
+  - TokenBucket(capacity=1, rate=0),capacity:令牌桶容量,rate:每秒生成速率
+  - get_token(block=True),block:决定令牌桶无令牌时是阻塞还是返回false
 ## 快速开始
 ### 安装
 `pip install dotask`

{dotask-0.2.4 → dotask-0.3.0}/README.md

@@ -16,6 +16,11 @@
 3. 添加本地调用shell工具函数->可以很好的适配@task消费者
    1. max_concurrent:控制并发执行的shell数量
    2. 引入dotask.Shell 调用Shell(max_concurrent=?).local_shell_execute(cmd,callback)来使用
+### 0.3.x
+1. 添加令牌桶,通过`from dotask.util import TokenBucket`使用,可以更好的限制生产消费速率
+- 参数说明
+  - TokenBucket(capacity=1, rate=0),capacity:令牌桶容量,rate:每秒生成速率
+  - get_token(block=True),block:决定令牌桶无令牌时是阻塞还是返回false
 ## 快速开始
 ### 安装
 `pip install dotask`

{dotask-0.2.4 → dotask-0.3.0}/dotask/task.py

@@ -33,7 +33,7 @@ class TopicManager:
         q = self.get_topic_queue(topic)
         with self.get_topic_lock(topic):
             q.put(data)
-        logger.info(f"发布主题-[{topic}]:{data}")
+        logger.info(f"publish topic-[{topic}]:{data}")
 
     def consume(self, subscribe_topics: List[str], handler: Callable) -> None:
         subscribe_set: Set[str] = set(subscribe_topics)
@@ -51,7 +51,7 @@ class TopicManager:
                 except queue.Empty:
                     continue
                 except Exception as e:
-                    logger.error(f"消费[{topic}]出错：{e}")
+                    logger.error(f"Topic[{topic}] consumption error：{e}")
             time.sleep(0.01)
 
     def stop_all(self):
@@ -59,7 +59,7 @@ class TopicManager:
         with self.global_lock:
             for topic, q in self.topic_queues.items():
                 q.join()
-        logger.warn("\n 所有主题的生产/消费已停止")
+        logger.warning("\n all task is stopped!")
 
 # 初始化全局主题管理器
 topic_manager = TopicManager()
@@ -102,7 +102,7 @@ def task(
         # ------------------------------
         if role == "producer":
             if not topic:
-                raise ValueError("生产者必须指定topic参数（如topic='num'）")
+                raise ValueError("The producer must specify the 'topic' parameter (e.g., topic='num')")
             topics = [t.strip() for t in topic.split(",")]
 
             def producer_func():
@@ -111,9 +111,9 @@ def task(
                         produce_data = func(*fixed_args, **fixed_kwargs)
                         for t in topics:
                             topic_manager.publish(t, produce_data)
-                        time.sleep(0.5)
+                        time.sleep(0.1)
                 except Exception as e:
-                    print(f" 生产者[{thread_name}]出错：{e}")
+                    logger.error(f"producer[{thread_name}] error {e}")
 
             thread = threading.Thread(
                 target=producer_func,
@@ -128,7 +128,7 @@ def task(
         # ------------------------------
         elif role == "consumer":
             if not subscribe:
-                raise ValueError("消费者必须指定subscribe参数（如subscribe='num'）")
+                raise ValueError("The consumer must specify the 'subscribe' parameter (e.g., subscribe='num')")
             subscribe_topics = [t.strip() for t in subscribe.split(",")]
             # 解析消费后要发布的主题（多个用,分隔）
             publish_topics = [t.strip() for t in publish_after.split(",")] if publish_after else []
@@ -150,7 +150,7 @@ def task(
                                 # logger.info(f"消费后发布[{t}]：{consume_result}")
 
                     except Exception as e:
-                        logger.error(f"消费者[{thread_name}]处理出错：{e}")
+                        logger.error(f"Consumer [{thread_name}] failed to process message: {e}")
 
                 # 开始消费
                 topic_manager.consume(subscribe_topics, handler)
@@ -164,7 +164,7 @@ def task(
             return thread
 
         else:
-            raise ValueError(f"不支持的角色：{role}，可选：producer/consumer")
+            raise ValueError(f"Unsupported role: {role}, available options: producer/consumer")
 
     wrapper_result = wrapper()
     return wrapper_result

dotask-0.3.0/dotask/util/__init__.py

@@ -0,0 +1,3 @@
+from .logger import logger
+from .shell import Shell
+from .limit import TokenBucket

dotask-0.3.0/dotask/util/limit.py

@@ -0,0 +1,69 @@
+import time
+from threading import Lock
+
+
+class TokenBucket:
+    """
+    令牌桶限流
+    - capacity: 令牌桶最大容量
+    - rate: 令牌生成速率 (每秒生成多少个令牌)
+    """
+    def __init__(self,capacity:int,rate:float):
+        self.capacity = max(1,capacity)
+        self.rate = max(0.0001,rate)
+        # 当前令牌数量
+        self.tokens = capacity
+        # 上次填充令牌的时间
+        self.last_refill_time = time.time()
+        self.lock = Lock()
+
+    def get_token(self,block:bool=False) -> bool:
+        """
+        获取令牌
+        :param block: 是否阻塞等待令牌(True：阻塞直到获取到令牌，Flase：非阻塞，获取不到直接返回False)
+        :return: 是否成功获取令牌
+        """
+        with self.lock:
+            # 1.填充令牌（根据上次填充时间和速率计算应该补充的令牌数）
+            now = time.time()
+            time_elapsed = now - self.last_refill_time
+            # 新生成令牌数=时间间隔*令牌生成速率
+            new_tokens = time_elapsed * self.rate
+            # 取最小，防止令牌数超过桶的最大容量(防止超发)
+            self.tokens = min(self.capacity,self.tokens+new_tokens)
+            self.last_refill_time = now
+
+            # 2.尝试获取令牌
+            if self.tokens >= 1:
+                self.tokens -= 1
+                return True
+            elif block:
+                # 计算需要等待的时间
+                # 需要等待的时间=（需要的令牌数-当前令牌数）/令牌生成速率
+                wait_time = (1-self.tokens)/self.rate
+                time.sleep(wait_time)
+                # 安全消耗1个令牌,避免令牌数为负
+                self.tokens = max(0,self.tokens-1)
+                return True
+            else:
+                return False
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

{dotask-0.2.4 → dotask-0.3.0}/dotask/util/shell.py

@@ -4,34 +4,20 @@ import subprocess
 from typing import Optional,Callable
 
 class Shell:
-    _shell_semaphore:threading.Semaphore = None
-    _semaphore_lock = threading.Lock()
+    # _shell_semaphore:threading.Semaphore = None
+    # _semaphore_lock = threading.Lock()
 
-    def __init__(self,max_concurrent:int,global_sem=False):
-        self.global_sem = global_sem
-        if global_sem:
-            if max_concurrent <= 0:
-                raise ValueError("最大并发数必须大于0")
-            self._shell_semaphore = threading.Semaphore(max_concurrent)
-            self._max_concurrent = max_concurrent
-        else:
-            with Shell._semaphore_lock:
-                if Shell._shell_semaphore is None:
-                    if max_concurrent <= 0:
-                        raise ValueError("最大并发数必须大于0")
-                    Shell._shell_semaphore = threading.Semaphore(max_concurrent)
-                    self._max_concurrent = max_concurrent
-                else:
-                    if Shell._shell_semaphore._value != max_concurrent:
-                        logger.warning(f"全局信号量已存在（并发数：{Shell._shell_semaphore._value}），忽略传入的max_concurrent={max_concurrent}")
-                    self._max_concurrent = Shell._shell_semaphore._value
+    def __init__(self,max_concurrent:int=1):
+        if max_concurrent <= 0:
+            raise ValueError("Maximum concurrency must be greater than 0")
+        self._shell_semaphore = threading.Semaphore(max_concurrent)
+        self._max_concurrent = max_concurrent
 
     def local_shell_execute(self,cmd:str,callback: Optional[Callable[[bool, str], None]] = None) -> bool:
         def execute_and_wait():
-            semaphore = self._shell_semaphore if self.global_sem else Shell._shell_semaphore
-
-            with semaphore:
-                logger.debug(f"实例:{id(self)}-剩余并发名额: {semaphore._value} | 开始执行命令: {cmd[:50]}")
+            with self._shell_semaphore:
+                truncated_cmd = cmd[:50] + "..." if len(cmd) > 50 else cmd
+                logger.debug(f"Instance {id(self)}: Remaining concurrency quota {self._shell_semaphore._value} | Executing command: {truncated_cmd}")
                 proc = None
                 try:
                     # 创建并执行命令
@@ -48,15 +34,15 @@ class Shell:
 
                     # 处理结果
                     if proc.returncode == 0:
-                        logger.debug(f"命令执行成功: {cmd[:50]}")
+                        logger.debug(f"Command Succeeded: {truncated_cmd}")
                         self._callback(callback,success=True, result=stdout.strip())
                     else:
-                        error_msg = f"返回码：{proc.returncode}，错误信息：{stderr.strip()[:200]}"
-                        logger.error(f"命令执行失败: {cmd[:50]} → {error_msg}")
+                        error_msg = f"Code：{proc.returncode}，Error_msg：{stderr.strip()}"
+                        logger.error(f"Command Failed: {truncated_cmd} → {error_msg}")
                         self._callback(callback,success=False, result=error_msg)
 
                 except Exception as ex:
-                    logger.error(f"命令执行失败: {cmd[:50]} → {str(ex)}")
+                    logger.error(f"Command Failed: {truncated_cmd} → {str(ex)}")
                     self._callback(callback,success=False, result=str(ex))
                 finally:
                     if proc is not None:
@@ -69,17 +55,17 @@ class Shell:
 
         try:
             # 启动线程执行命令（守护线程，不阻塞主线程）
-            safe_cmd_name = cmd[:20].replace(' ', '_').replace('/', '_').replace('\\', '_').replace('|', '_')
+            safe_cmd_name = cmd[:20].replace(' ', '_').replace('/', '_').replace('\\', '_').replace('|', '_')+ "..." if len(cmd) > 20 else cmd
             exec_thread = threading.Thread(
                 target=execute_and_wait,
                 daemon=True,
                 name=f"CmdExecutor-{safe_cmd_name}"
             )
             exec_thread.start()
-            logger.debug(f"任务已提交: {cmd[:50]}")
+            logger.debug(f"Task submitted: {safe_cmd_name}")
             return True
         except Exception as e:
-            logger.error(f"任务提交失败: {cmd[:50]} → {str(e)}")
+            logger.error(f"Task submission failed: {cmd} → {str(e)}")
             self._callback(callback,success=False,result=str(e))
             return False
 
@@ -89,5 +75,5 @@ class Shell:
             try:
                 callback(success, result)
             except Exception as ex:
-                logger.error(f"回调函数执行失败: {str(ex)}")
+                logger.error(f"Callback function execution failed: {str(ex)}")
 

{dotask-0.2.4 → dotask-0.3.0}/dotask.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dotask
-Version: 0.2.4
+Version: 0.3.0
 Summary: 发布订阅注解方式
 Home-page: https://gitee.com/d-yz/task-manager
 Author: dyz
@@ -35,6 +35,11 @@ Dynamic: license-file
 3. 添加本地调用shell工具函数->可以很好的适配@task消费者
    1. max_concurrent:控制并发执行的shell数量
    2. 引入dotask.Shell 调用Shell(max_concurrent=?).local_shell_execute(cmd,callback)来使用
+### 0.3.x
+1. 添加令牌桶,通过`from dotask.util import TokenBucket`使用,可以更好的限制生产消费速率
+- 参数说明
+  - TokenBucket(capacity=1, rate=0),capacity:令牌桶容量,rate:每秒生成速率
+  - get_token(block=True),block:决定令牌桶无令牌时是阻塞还是返回false
 ## 快速开始
 ### 安装
 `pip install dotask`

{dotask-0.2.4 → dotask-0.3.0}/dotask.egg-info/SOURCES.txt

@@ -9,5 +9,6 @@ dotask.egg-info/SOURCES.txt
 dotask.egg-info/dependency_links.txt
 dotask.egg-info/top_level.txt
 dotask/util/__init__.py
+dotask/util/limit.py
 dotask/util/logger.py
 dotask/util/shell.py

{dotask-0.2.4 → dotask-0.3.0}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = dotask
-version = 0.2.4
+version = 0.3.0
 author = dyz
 author_email = 837701454@qq.com
 description = 发布订阅注解方式

dotask-0.2.4/dotask/util/__init__.py

@@ -1,2 +0,0 @@
-from .logger import logger
-from .shell import Shell