onestep 0.4.1__tar.gz → 0.4.3__tar.gz

{onestep-0.4.1 → onestep-0.4.3}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: onestep
-Version: 0.4.1
+Version: 0.4.3
 Summary: 仅需一步，轻松实现分布式异步任务。
 Author: miclon
 Author-email: jcnd@163.com
@@ -11,6 +11,8 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Provides-Extra: redis
 Requires-Dist: asgiref (>=3.6.0,<4.0.0)
 Requires-Dist: blinker (>=1.5,<2.0)

{onestep-0.4.1 → onestep-0.4.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "onestep"
-version = "0.4.1"
+version = "0.4.3"
 description = "仅需一步，轻松实现分布式异步任务。"
 authors = ["miclon <jcnd@163.com>"]
 readme = "README.md"
@@ -28,6 +28,7 @@ nacos-sdk-python = "^0.1.12"
 redis = "^4.5.1"
 autopep8 = "^2.0.2"
 loguru = "^0.6.0"
+use-redis = "^0.1.92"
 
 
 [tool.poetry.group.test.dependencies]

{onestep-0.4.1 → onestep-0.4.3}/src/onestep/__init__.py

@@ -11,6 +11,7 @@ from .middleware import (
     BaseMiddleware, BaseConfigMiddleware,
     NacosPublishConfigMiddleware, NacosConsumeConfigMiddleware,
     RedisPublishConfigMiddleware, RedisConsumeConfigMiddleware,
+    UniqueMiddleware, MemoryUniqueMiddleware,
 )
 from .exception import (
     StopMiddleware, DropMessage,
@@ -48,6 +49,8 @@ __all__ = [
     'NacosConsumeConfigMiddleware',
     'RedisPublishConfigMiddleware',
     'RedisConsumeConfigMiddleware',
+    'UniqueMiddleware',
+    'MemoryUniqueMiddleware',
 
     # exception
     'StopMiddleware',
@@ -59,4 +62,4 @@ __all__ = [
     '__version__'
 ]
 
-__version__ = '0.4.0'
+__version__ = '0.4.3'

{onestep-0.4.1 → onestep-0.4.3}/src/onestep/broker/base.py

@@ -43,15 +43,15 @@ class BaseBroker:
     def add_middleware(self, middleware: BaseMiddleware):
         self.middlewares.append(middleware)
 
-    def send(self, message):
+    def send(self, message, *args, **kwargs):
         """对消息进行预处理，然后再发送"""
         if not isinstance(message, Message):
             message = self.message_cls(body=message)
         # TODO: 对消息发送进行N次重试，确保消息发送成功。
-        return self.publish(message.to_json())
+        return self.publish(message.to_json(), *args, **kwargs)
 
     @abc.abstractmethod
-    def publish(self, message: Any):
+    def publish(self, message: Any, *args, **kwargs):
         """
         将消息原样发布到 broker 中。如果当前Broker是Job的to_broker, 则必须实现此方法
         """

{onestep-0.4.1 → onestep-0.4.3}/src/onestep/broker/cron.py

@@ -14,14 +14,15 @@ logger = logging.getLogger(__name__)
 
 
 class CronBroker(MemoryBroker):
-    _thread = None
 
-    def __init__(self, cron, name=None, middlewares=None, body: Any = None, *args, **kwargs):
+    def __init__(self, cron, name=None, middlewares=None, body: Any = None, start_time=None, *args, **kwargs):
         super().__init__(name=name, middlewares=middlewares, *args, **kwargs)
         self.cron = cron
-        self.itr = croniter(cron, datetime.now())
+        self.start_time = start_time or datetime.now()
+        self.itr = croniter(cron, self.start_time)
         self.next_fire_time = self.itr.get_next(datetime)
         self.body = body
+        self._thread = None
 
     def _scheduler(self):
         if self.next_fire_time <= datetime.now():
@@ -36,7 +37,8 @@ class CronBroker(MemoryBroker):
         return CronConsumer(self, *args, **kwargs)
 
     def shutdown(self):
-        self._thread.cancel()
+        if self._thread:
+            self._thread.cancel()
 
 
 class CronConsumer(MemoryConsumer):

{onestep-0.4.1 → onestep-0.4.3}/src/onestep/broker/memory.py

@@ -36,7 +36,7 @@ class MemoryBroker(BaseBroker):
         super().__init__(*args, **kwargs)
         self.queue = Queue(maxsize)
 
-    def publish(self, message: Any):
+    def publish(self, message: Any, *args, **kwargs):
         try:
             self.queue.put_nowait(message)
         except FullException:

{onestep-0.4.1 → onestep-0.4.3}/src/onestep/broker/rabbitmq.py

@@ -3,17 +3,21 @@ import threading
 from queue import Queue
 from typing import Optional, Dict, Any
 
-import amqpstorm
+try:
+    import amqpstorm
+    from use_rabbitmq import useRabbitMQ as RabbitMQStore
+except ImportError:
+    amqpstorm = None
+    RabbitMQStore = None
 
 from .base import BaseBroker, BaseConsumer
-from use_rabbitmq import useRabbitMQ as RabbitMQStore
 from ..message import Message
 
 
 class _RabbitMQMessage(Message):
 
     @classmethod
-    def from_broker(cls, broker_message: amqpstorm.Message):
+    def from_broker(cls, broker_message):
         try:
             message = json.loads(broker_message.body)
         except json.JSONDecodeError:
@@ -30,20 +34,51 @@ class _RabbitMQMessage(Message):
 class RabbitMQBroker(BaseBroker):
     message_cls = _RabbitMQMessage
 
-    def __init__(self, queue_name, params: Optional[Dict] = None, prefetch: Optional[int] = 1, auto_create=True, *args,
-                 **kwargs):
+    def __init__(
+        self,
+        queue_name,
+        params: Optional[Dict] = None,
+        prefetch: Optional[int] = 1,
+        auto_create: Optional[bool]=True,
+        queue_params: Optional[Dict]=None,
+        *args,
+        **kwargs
+        ):
+        """
+        Initializes the RabbitMQ broker.
+
+        Args:
+            queue_name (str): The name of the queue.
+            params (Optional[Dict], optional): Parameters for RabbitMQStore. Defaults to None.
+            prefetch (Optional[int], optional): Number of messages to prefetch. Defaults to 1.
+            auto_create (Optional[bool], optional): Whether to automatically create the queue. Defaults to True.
+            queue_params (Optional[Dict], optional): Parameters for queue declaration. Defaults to None.
+            *args: Variable length argument list.
+            **kwargs: Arbitrary keyword arguments.
+
+        Attributes:
+            queue_name (str): The name of the queue.
+            queue (Queue): The queue instance.
+            client (RabbitMQStore): The RabbitMQ client instance.
+            prefetch (int): Number of messages to prefetch.
+            threads (list): List of threads.
+        """
+        
+        if RabbitMQStore is None:
+            raise ImportError("RabbitMQ dependencies not installed. Please install 'use-rabbitmq' package.")
+        
         super().__init__(*args, **kwargs)
         self.queue_name = queue_name
         self.queue = Queue()
         params = params or {}
         self.client = RabbitMQStore(**params)
         if auto_create:
-            self.client.declare_queue(self.queue_name)
+            self.client.declare_queue(self.queue_name, **(queue_params or {}))
         self.prefetch = prefetch
         self.threads = []
 
     def _consume(self, *args, **kwargs):
-        def callback(message: amqpstorm.Message):
+        def callback(message):
             self.queue.put(message)
 
         prefetch = kwargs.pop("prefetch", self.prefetch)
@@ -57,8 +92,9 @@ class RabbitMQBroker(BaseBroker):
         self.threads.append(thread)
         return RabbitMQConsumer(self)
 
-    def publish(self, message: Any):
-        self.client.send(self.queue_name, message)
+    def publish(self, message: Any, properties: Optional[dict] = None, **kwargs):
+        """发布消息"""
+        self.client.send(self.queue_name, message, properties=properties, **kwargs)
 
     def confirm(self, message: Message):
         """确认消息"""

{onestep-0.4.1 → onestep-0.4.3}/src/onestep/broker/redis/pubsub.py

@@ -59,12 +59,12 @@ class RedisPubSubBroker(BaseBroker):
         self.threads.append(thread)
         return RedisPubSubConsumer(self)
 
-    def send(self, message: Any):
+    def send(self, message: Any, *args, **kwargs):
         """Publish message to the Redis channel"""
         if not isinstance(message, Message):
             message = self.message_cls(body=message)
 
-        self.client.publish(self.channel, message.to_json())
+        self.client.publish(self.channel, message.to_json(), *args, **kwargs)
 
     publish = send
 

{onestep-0.4.1 → onestep-0.4.3}/src/onestep/broker/redis/stream.py

@@ -75,12 +75,12 @@ class RedisStreamBroker(BaseBroker):
         self.threads.append(thread)
         return RedisStreamConsumer(self)
 
-    def send(self, message: Any):
+    def send(self, message: Any, *args, **kwargs):
         """对消息进行预处理，然后再发送"""
         if not isinstance(message, Message):
             message = self.message_cls(body=message)
 
-        self.client.send({"_message": message.to_json()})
+        self.client.send({"_message": message.to_json()}, *args, **kwargs)
 
     publish = send
 

{onestep-0.4.1 → onestep-0.4.3}/src/onestep/broker/webhook.py

@@ -57,13 +57,15 @@ class WebHookBroker(MemoryBroker):
                 WebHookServer
             )
             self._servers[(self.host, self.port)] = hs
+            # 只有在创建新服务器时才启动线程
+            thread = threading.Thread(target=hs.serve_forever)
+            thread.daemon = True
+            thread.start()
+            self.threads.append(thread)
         else:
             hs = self._servers[(self.host, self.port)]
 
         WebHookServer.servers[(self.host, self.port)].append(Server(self.path, self.queue))
-        thread = threading.Thread(target=hs.serve_forever)
-        thread.start()
-        self.threads.append(thread)
 
     def consume(self, *args, **kwargs):
         self._create_server()
@@ -71,7 +73,7 @@ class WebHookBroker(MemoryBroker):
         return WebHookConsumer(self, *args, **kwargs)
 
     def shutdown(self):
-        hs = self._servers[(self.host, self.port)]
+        hs = self._servers.get((self.host, self.port))
         if hs:
             hs.shutdown()
         for thread in self.threads:

{onestep-0.4.1 → onestep-0.4.3}/src/onestep/cli.py

@@ -8,11 +8,17 @@ LOGFORMAT = "[%(asctime)s] [%(name)s] [%(levelname)s] %(message)s"
 
 
 def setup_logging():
-    logging.basicConfig(level=logging.DEBUG, format=LOGFORMAT, stream=sys.stdout)
+    # 设置全局日志级别为INFO，避免第三方库的DEBUG日志输出
+    logging.basicConfig(level=logging.INFO, format=LOGFORMAT, stream=sys.stdout)
 
     # exclude amqpstorm logs
     logging.getLogger("amqpstorm").setLevel(logging.CRITICAL)
-    return logging.getLogger("onestep")
+    
+    # 获取onestep的logger并设置为DEBUG级别以便调试
+    onestep_logger = logging.getLogger("onestep")
+    onestep_logger.setLevel(logging.DEBUG)
+    
+    return onestep_logger
 
 
 logger = setup_logging()

{onestep-0.4.1 → onestep-0.4.3}/src/onestep/middleware/unique.py

@@ -1,7 +1,7 @@
 import hashlib
 import json
 
-from . import BaseMiddleware
+from .base import BaseMiddleware
 from ..exception import DropMessage
 
 

{onestep-0.4.1 → onestep-0.4.3}/src/onestep/onestep.py

@@ -42,7 +42,7 @@ class BaseOneStep:
         self.workers = workers or DEFAULT_WORKERS
         self.worker_class = worker_class or DEFAULT_WORKER_CLASS
         if self.workers > MAX_WORKERS:
-            logger.warning(f"workers[{self.workers}] litter than {MAX_WORKERS}")
+            logger.warning(f"workers[{self.workers}] greater than {MAX_WORKERS}")
             self.workers = MAX_WORKERS
         self.middlewares = middlewares or []
 

{onestep-0.4.1 → onestep-0.4.3}/src/onestep/worker.py

@@ -169,6 +169,8 @@ class ThreadWorker(BaseWorker):
 
 
 class ThreadPoolWorker(BaseWorker):
+    broker_exit: Dict[BaseBroker, bool] = {}
+    broker_exit_lock = threading.Lock()
 
     def __init__(self, onestep, broker: BaseBroker, workers=None, *args, **kwargs):
         super().__init__(onestep, broker, *args, **kwargs)
@@ -191,7 +193,8 @@ class ThreadPoolWorker(BaseWorker):
                     self.shutdown()
                     break
             for message in self.receive_messages():
-                self.handle_message(message)
+                # 将消息处理提交到线程池中并发执行
+                self.executor.submit(self.handle_message, message)
 
     def shutdown(self):
         """关闭线程池 Worker"""