onestep 0.3.3a1__tar.gz → 0.3.4__tar.gz

{onestep-0.3.3a1 → onestep-0.3.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: onestep
-Version: 0.3.3a1
+Version: 0.3.4
 Summary: 
 Author: miclon
 Author-email: jcnd@163.com
@@ -10,11 +10,13 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Provides-Extra: redis
 Requires-Dist: asgiref (>=3.6.0,<4.0.0)
 Requires-Dist: blinker (>=1.5,<2.0)
 Requires-Dist: croniter (>=1.3.8,<2.0.0)
 Requires-Dist: usepy-plugin-rabbitmq (>=0.1.0,<0.2.0)
-Requires-Dist: usepy-plugin-redis (>=0.1.5,<0.2.0)
+Requires-Dist: usepy-plugin-redis (>=0.1.6,<0.2.0) ; extra == "redis"
 Description-Content-Type: text/markdown
 
 

{onestep-0.3.3a1 → onestep-0.3.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "onestep"
-version = "0.3.3a1"
+version = "0.3.4"
 description = ""
 authors = ["miclon <jcnd@163.com>"]
 readme = "README.md"
@@ -14,7 +14,10 @@ asgiref = "^3.6.0"
 blinker = "^1.5"
 croniter = "^1.3.8"
 usepy-plugin-rabbitmq = "^0.1.0"
-usepy-plugin-redis = "^0.1.5"
+usepy-plugin-redis = { version = "^0.1.6", optional = true }
+
+[tool.poetry.extras]
+redis = ["usepy-plugin-redis"]
 
 [tool.poetry.group.dev.dependencies]
 nacos-sdk-python = "^0.1.12"

{onestep-0.3.3a1 → onestep-0.3.4}/src/onestep/__init__.py

@@ -5,7 +5,7 @@ from .retry import (
 )
 from .broker import (
     BaseBroker, BaseConsumer, BaseLocalBroker, BaseLocalConsumer,
-    MemoryBroker, RabbitMQBroker, WebHookBroker, CronBroker
+    MemoryBroker, RabbitMQBroker, WebHookBroker, CronBroker, RedisStreamBroker
 )
 from .middleware import (
     BaseMiddleware, BaseConfigMiddleware,
@@ -19,7 +19,7 @@ from .exception import (
 
 __all__ = [
     'step',
-    
+
     # broker
     'BaseBroker',
     'BaseConsumer',
@@ -29,7 +29,8 @@ __all__ = [
     'RabbitMQBroker',
     'WebHookBroker',
     'CronBroker',
-    
+    'RedisStreamBroker',
+
     # retry
     'BaseRetry',
     'NeverRetry',
@@ -40,7 +41,7 @@ __all__ = [
     # error callback
     'BaseErrorCallback',
     'NackErrorCallBack',
-    
+
     # middleware
     'BaseMiddleware',
     'BaseConfigMiddleware',
@@ -48,7 +49,7 @@ __all__ = [
     'NacosConsumeConfigMiddleware',
     'RedisPublishConfigMiddleware',
     'RedisConsumeConfigMiddleware',
-    
+
     # exception
     'StopMiddleware',
     'DropMessage',

{onestep-0.3.3a1 → onestep-0.3.4}/src/onestep/broker/base.py

@@ -13,10 +13,11 @@ logger = logging.getLogger(__name__)
 
 class BaseBroker:
 
-    def __init__(self, name=None, queue=None, middlewares=None):
+    def __init__(self, name=None, queue=None, middlewares=None, once: bool = False):
         self.queue = queue
         self.name = name or "broker"
         self.middlewares = []
+        self.once = once
 
         if middlewares:
             for middleware in middlewares:
@@ -157,10 +158,13 @@ class BaseLocalBroker(BaseBroker):
 class BaseLocalConsumer(BaseConsumer):
 
     def _to_message(self, data: Any):
-        try:
-            message = json.loads(data)
-        except json.JSONDecodeError:
-            message = {"body": data}
+        if isinstance(data, (str, bytes, bytearray)):
+            try:
+                message = json.loads(data)
+            except json.JSONDecodeError:
+                message = {"body": data}
+        else:
+            message = data
         if not isinstance(message, dict):
             message = {"body": message}
         if "body" not in message:

{onestep-0.3.3a1 → onestep-0.3.4}/src/onestep/broker/rabbitmq.py

@@ -55,7 +55,7 @@ class RabbitMQBroker(BaseBroker):
         重发消息：先拒绝 再 重入
         
         :param message: 消息
-        :param is_source: 是否是源消息，True: 使用消息的最新数据重入当前队列，False: 使用消息的最新数据重入当前队列
+        :param is_source: 是否是原始消息，True: 使用原始消息重入当前队列，False: 使用消息的最新数据重入当前队列
         """
         if is_source:
             message.msg.reject(requeue=True)

{onestep-0.3.3a1 → onestep-0.3.4}/src/onestep/broker/redis.py

@@ -4,14 +4,17 @@ import uuid
 from queue import Queue
 from typing import Optional, Dict, Any
 
-from usepy_plugin_redis import useRedisStreamStore, RedisStreamMessage
+try:
+    from usepy_plugin_redis import useRedisStreamStore, RedisStreamMessage
+except ImportError:
+    ...
 
 from .base import BaseBroker, BaseConsumer, Message
 
 
 class RedisStreamBroker(BaseBroker):
     """ Redis Stream Broker """
-    
+
     def __init__(
             self,
             stream: str,
@@ -28,9 +31,9 @@ class RedisStreamBroker(BaseBroker):
         self.group = group
         self.prefetch = prefetch
         self.queue = Queue()
-        
+
         self.threads = []
-        
+
         self.client = useRedisStreamStore(
             stream=stream,
             group=group,
@@ -39,14 +42,14 @@ class RedisStreamBroker(BaseBroker):
             claim_interval=claim_interval,
             **(params or {})
         )
-    
+
     def _consume(self, *args, **kwargs):
         def callback(message):
             self.queue.put(message)
-        
+
         prefetch = kwargs.pop("prefetch", self.prefetch)
         self.client.start_consuming(consumer=uuid.uuid4().hex, callback=callback, prefetch=prefetch, **kwargs)
-    
+
     def consume(self, *args, **kwargs):
         daemon = kwargs.pop('daemon', True)
         thread = threading.Thread(target=self._consume, *args, **kwargs)
@@ -54,31 +57,31 @@ class RedisStreamBroker(BaseBroker):
         thread.start()
         self.threads.append(thread)
         return RedisStreamConsumer(self.queue)
-    
+
     def send(self, message: Any):
         """对消息进行预处理，然后再发送"""
         if not isinstance(message, Message):
             message = Message(body=message)
-        
+
         self.client.send({"_message": message.to_json()})
-    
+
     publish = send
-    
+
     def confirm(self, message: Message):
         self.client.ack(message.msg)
-    
+
     def reject(self, message: Message):
         self.client.reject(message.msg)
-    
+
     def requeue(self, message: Message, is_source=False):
         """
          重发消息：先拒绝 再 重入
  
          :param message: 消息
-         :param is_source: 是否是源消息，True: 使用消息的最新数据重入当前队列，False: 使用消息的最新数据重入当前队列
+         :param is_source: 是否是原始消息消息，True: 使用原始消息重入当前队列，False: 使用消息的最新数据重入当前队列
          """
         self.reject(message)
-        
+
         if is_source:
             self.client.send(message.msg.body)
         else:
@@ -86,7 +89,7 @@ class RedisStreamBroker(BaseBroker):
 
 
 class RedisStreamConsumer(BaseConsumer):
-    def _to_message(self, data: RedisStreamMessage):
+    def _to_message(self, data: "RedisStreamMessage"):
         if "_message" in data.body:
             # 来自 RedisStreamBroker.send 的消息，message.body 默认是存于 _message 字段中
             try:
@@ -96,5 +99,5 @@ class RedisStreamConsumer(BaseConsumer):
         else:
             # 来自 外部的消息 直接认为都是 message.body
             message = {"body": data.body}
-        
+
         yield Message(body=message.get("body"), extra=message.get("extra"), msg=data)

{onestep-0.3.3a1 → onestep-0.3.4}/src/onestep/broker/webhook.py

@@ -49,9 +49,6 @@ class WebHookBroker(BaseLocalBroker):
         self.path = path
         self.threads = []
 
-        self._create_server()
-        logger.debug(f"WebHookBroker: {self.host}:{self.port}{self.path}")
-
     def _create_server(self):
 
         if (self.host, self.port) not in self._servers:
@@ -69,6 +66,8 @@ class WebHookBroker(BaseLocalBroker):
         self.threads.append(thread)
 
     def consume(self, *args, **kwargs):
+        self._create_server()
+        logger.debug(f"WebHookBroker: {self.host}:{self.port}{self.path}")
         return WebHookConsumer(self.queue, *args, **kwargs)
 
     def shutdown(self):

{onestep-0.3.3a1 → onestep-0.3.4}/src/onestep/message.py

@@ -2,6 +2,7 @@ import json
 import sys
 import time
 import uuid
+from dataclasses import dataclass
 from traceback import TracebackException
 from typing import Optional, Any, Union
 
@@ -14,11 +15,15 @@ class MessageTracebackException(TracebackException):
         self.exc_value = exc_value
 
 
+@dataclass
 class Extra:
-    def __init__(self, task_id=None, publish_time=None, failure_count=0):
-        self.task_id = task_id or str(uuid.uuid4())
-        self.publish_time = publish_time or round(time.time(), 3)
-        self.failure_count = failure_count
+    task_id: str = None
+    publish_time: float = None
+    failure_count: int = 0
+
+    def __post_init__(self):
+        self.task_id = self.task_id or str(uuid.uuid4())
+        self.publish_time = self.publish_time or round(time.time(), 3)
 
     def to_dict(self):
         return {
@@ -30,9 +35,6 @@ class Extra:
     def __str__(self):
         return str(self.to_dict())
 
-    def __repr__(self):
-        return f"{self.__class__.__name__}({self.task_id}, {self.publish_time}, {self.failure_count})"
-
 
 class Message:
 

{onestep-0.3.3a1 → onestep-0.3.4}/src/onestep/onestep.py

@@ -8,7 +8,7 @@ from typing import Optional, List, Dict, Any, Callable, Union
 from .broker.base import BaseBroker
 from .exception import StopMiddleware
 from .message import Message
-from .retry import NeverRetry
+from .retry import TimesRetry
 from .signal import message_sent, started, stopped
 from .state import State
 from .worker import WorkerThread
@@ -29,7 +29,7 @@ class BaseOneStep:
                  to_broker: Union[BaseBroker, List[BaseBroker], None] = None,
                  workers: Optional[int] = None,
                  middlewares: Optional[List[Any]] = None,
-                 retry: Union[Callable, object] = NeverRetry(),
+                 retry: Union[Callable, object] = TimesRetry(),
                  error_callback: Optional[Union[Callable, object]] = None):
         self.group = group
         self.fn = fn
@@ -58,6 +58,7 @@ class BaseOneStep:
             f"broker must be BaseBroker or list or tuple, not {type(broker)}")
 
     def _add_consumer(self, broker):
+        """ 添加来源消费者 """
         for _ in range(self.workers):
             self.consumers[self.group].append(
                 WorkerThread(onestep=self, broker=broker)
@@ -131,6 +132,11 @@ class BaseOneStep:
                 logger.debug(f"middleware<{middleware}> is stopped，reason: {e}")
                 break
 
+    @classmethod
+    def is_shutdown(cls, group):
+        # check all broker
+        return all(broker._shutdown for broker in cls._find_consumers(group))
+
 
 def decorator_func_proxy(func):
     @functools.wraps(func)
@@ -182,7 +188,7 @@ class step:
                  to_broker: Union[BaseBroker, List[BaseBroker], None] = None,
                  workers: Optional[int] = None,
                  middlewares: Optional[List[Any]] = None,
-                 retry: Union[Callable, object] = NeverRetry(),
+                 retry: Union[Callable, object] = TimesRetry(),
                  error_callback: Optional[Union[Callable, object]] = None):
         self.params = {
             "group": group,
@@ -209,7 +215,7 @@ class step:
         started.send()
         if block:
             import time
-            while True:
+            while not BaseOneStep.is_shutdown(group):
                 time.sleep(1)
 
     @staticmethod

{onestep-0.3.3a1 → onestep-0.3.4}/src/onestep/retry.py

@@ -1,46 +1,56 @@
 from abc import ABC, abstractmethod
+from enum import Enum
 from typing import Optional, Tuple, Union, Type
 
 from .exception import RetryViaLocal, RetryViaQueue
 from .message import Message
 
 
+class RetryStatus(Enum):
+    CONTINUE = 1  # 继续（执行重试）
+    END_WITH_CALLBACK = 2  # 结束（执行回调）
+    END_IGNORE_CALLBACK = 3  # 结束（忽略回调）
+
+
 class BaseRetry(ABC):
-    
+
     @abstractmethod
-    def __call__(self, *args, **kwargs) -> Optional[bool]:
-        """True=继续（执行重试）， False=结束（执行回调），None=结束（忽略回调）"""
+    def __call__(self, *args, **kwargs) -> Optional[RetryStatus]:
         pass
 
 
 class NeverRetry(BaseRetry):
-    
-    def __call__(self, message) -> Optional[bool]:
-        return False
+
+    def __call__(self, message) -> Optional[RetryStatus]:
+        return RetryStatus.END_WITH_CALLBACK
 
 
 class AlwaysRetry(BaseRetry):
-    
-    def __call__(self, message) -> Optional[bool]:
-        return True
+
+    def __call__(self, message) -> Optional[RetryStatus]:
+        return RetryStatus.CONTINUE
 
 
 class TimesRetry(BaseRetry):
-    
+
     def __init__(self, times: int = 3):
         self.times = times
-    
-    def __call__(self, message) -> Optional[bool]:
-        return message.failure_count < self.times
+
+    def __call__(self, message) -> Optional[RetryStatus]:
+        if message.failure_count < self.times:
+            return RetryStatus.CONTINUE
+        return RetryStatus.END_WITH_CALLBACK
 
 
 class RetryIfException(BaseRetry):
-    
+
     def __init__(self, exceptions: Optional[Tuple[Union[Exception, Type]]] = Exception):
         self.exceptions = exceptions
-    
-    def __call__(self, message) -> Optional[bool]:
-        return isinstance(message.exception.exc_value, self.exceptions)  # noqa
+
+    def __call__(self, message) -> Optional[RetryStatus]:
+        if isinstance(message.exception.exc_value, self.exceptions):
+            return RetryStatus.CONTINUE
+        return RetryStatus.END_WITH_CALLBACK
 
 
 class AdvancedRetry(TimesRetry):
@@ -51,32 +61,33 @@ class AdvancedRetry(TimesRetry):
     3. 其他异常：如果异常不是 RetryViaLocal 或 RetryViaQueue 或 指定异常，则不重试，回调
     注：待重试的异常若继承自 RetryException，则可单独指定重试次数，否则默认为 3 次
     """
+
     def __init__(self, times: int = 3, exceptions: Optional[Tuple[Union[Exception, Type]]] = None):
         super().__init__(times=times)
         self.exceptions = (RetryViaLocal, RetryViaQueue) + (exceptions or ())
-    
-    def __call__(self, message: Message) -> Optional[bool]:
+
+    def __call__(self, message: Message) -> Optional[RetryStatus]:
         if isinstance(message.exception.exc_value, self.exceptions):
             max_retry_times = getattr(message.exception.exc_value, "times", None) or self.times
             if message.failure_count < max_retry_times:
                 if isinstance(message.exception.exc_value, RetryViaQueue):
-                    return None  # 入队重试，不回调
-                return True  # 本地重试，不回调
+                    return RetryStatus.END_IGNORE_CALLBACK
+                return RetryStatus.CONTINUE
             else:
-                return False  # 不重试了，回调
+                return RetryStatus.END_WITH_CALLBACK
         else:
-            return False  # 其他异常，回调
+            return RetryStatus.END_WITH_CALLBACK
 
 
 # error callback
 class BaseErrorCallback(ABC):
-    
+
     @abstractmethod
     def __call__(self, *args, **kwargs):
         pass
 
 
 class NackErrorCallBack(BaseErrorCallback):
-    
+
     def __call__(self, message):
         message.reject()

{onestep-0.3.3a1 → onestep-0.3.4}/src/onestep/worker.py

@@ -1,6 +1,7 @@
 """
 将指定的函数放入线程中运行
 """
+
 try:
     from collections import Iterable
 except ImportError:
@@ -12,6 +13,7 @@ from inspect import isasyncgenfunction
 
 from asgiref.sync import async_to_sync
 
+from .retry import RetryStatus
 from .broker import BaseBroker
 from .exception import DropMessage
 from .signal import message_received, message_consumed, message_error, message_drop
@@ -34,7 +36,7 @@ class WorkerThread(threading.Thread):
         self.broker = broker
         self.args = args
         self.kwargs = kwargs
-        self.__shutdown = False
+        self._shutdown = False
 
     def run(self):
         """线程执行包装过的`onestep`函数
@@ -43,11 +45,11 @@ class WorkerThread(threading.Thread):
         :return:
         """
 
-        while not self.__shutdown:
-            if self.__shutdown:
-                break
+        while not self._shutdown:
             # TODO：consume应当传入一些配置参数
             for result in self.broker.consume():
+                if self._shutdown:
+                    break
                 if result is None:
                     continue
                 messages = (
@@ -67,10 +69,14 @@ class WorkerThread(threading.Thread):
                         message_drop.send(self, message=message, reason=e)
                         logger.warning(f"{self.instance.name} dropped <{type(e).__name__}: {str(e)}>")
                         message.reject()
+                else:
+                    if self.broker.once:
+                        self.broker.shutdown()
+                        self._shutdown = True
 
     def shutdown(self):
         self.broker.shutdown()
-        self.__shutdown = True
+        self._shutdown = True
 
     def _run_instance(self, message):
         while True:
@@ -89,12 +95,13 @@ class WorkerThread(threading.Thread):
                     logger.error(f"{self.instance.name} run error <{type(e).__name__}: {str(e)}>")
                 message.set_exception()
 
-                retry_state = self.retry(message)
-                if retry_state:  # True=继续（执行重试）
+                retry_status = self.retry(message)
+                if retry_status is RetryStatus.CONTINUE:
                     continue
-                elif retry_state is False:  # False=结束（执行回调）
+                elif retry_status is RetryStatus.END_WITH_CALLBACK:
                     if self.error_callback:
                         self.error_callback(message)
                     return message.reject()
-                else:  # None=结束（忽略回调）
+                else:  # RetryStatus.END_IGNORE_CALLBACK
+                    # 由于是队列内重试，不会触发错误回调
                     return message.requeue()

{onestep-0.3.3a1 → onestep-0.3.4}/src/onestep/broker/cron.py

@@ -21,7 +21,6 @@ class CronBroker(BaseLocalBroker):
         self.itr = croniter(cron, datetime.now())
         self.next_fire_time = self.itr.get_next(datetime)
         self.kwargs = kwargs
-        self._scheduler()
 
     def _scheduler(self):
         if self.next_fire_time <= datetime.now():
@@ -32,6 +31,7 @@ class CronBroker(BaseLocalBroker):
         self._thread.start()
 
     def consume(self):
+        self._scheduler()
         return CronConsumer(self.queue)
 
     def shutdown(self):