onestep 0.4.1__py3-none-any.whl → 0.4.3__py3-none-any.whl

onestep/__init__.py

@@ -11,6 +11,7 @@ from .middleware import (
     BaseMiddleware, BaseConfigMiddleware,
     NacosPublishConfigMiddleware, NacosConsumeConfigMiddleware,
     RedisPublishConfigMiddleware, RedisConsumeConfigMiddleware,
+    UniqueMiddleware, MemoryUniqueMiddleware,
 )
 from .exception import (
     StopMiddleware, DropMessage,
@@ -48,6 +49,8 @@ __all__ = [
     'NacosConsumeConfigMiddleware',
     'RedisPublishConfigMiddleware',
     'RedisConsumeConfigMiddleware',
+    'UniqueMiddleware',
+    'MemoryUniqueMiddleware',
 
     # exception
     'StopMiddleware',
@@ -59,4 +62,4 @@ __all__ = [
     '__version__'
 ]
 
-__version__ = '0.4.0'
+__version__ = '0.4.3'

onestep/broker/base.py

@@ -43,15 +43,15 @@ class BaseBroker:
     def add_middleware(self, middleware: BaseMiddleware):
         self.middlewares.append(middleware)
 
-    def send(self, message):
+    def send(self, message, *args, **kwargs):
         """对消息进行预处理，然后再发送"""
         if not isinstance(message, Message):
             message = self.message_cls(body=message)
         # TODO: 对消息发送进行N次重试，确保消息发送成功。
-        return self.publish(message.to_json())
+        return self.publish(message.to_json(), *args, **kwargs)
 
     @abc.abstractmethod
-    def publish(self, message: Any):
+    def publish(self, message: Any, *args, **kwargs):
         """
         将消息原样发布到 broker 中。如果当前Broker是Job的to_broker, 则必须实现此方法
         """

onestep/broker/cron.py

@@ -14,14 +14,15 @@ logger = logging.getLogger(__name__)
 
 
 class CronBroker(MemoryBroker):
-    _thread = None
 
-    def __init__(self, cron, name=None, middlewares=None, body: Any = None, *args, **kwargs):
+    def __init__(self, cron, name=None, middlewares=None, body: Any = None, start_time=None, *args, **kwargs):
         super().__init__(name=name, middlewares=middlewares, *args, **kwargs)
         self.cron = cron
-        self.itr = croniter(cron, datetime.now())
+        self.start_time = start_time or datetime.now()
+        self.itr = croniter(cron, self.start_time)
         self.next_fire_time = self.itr.get_next(datetime)
         self.body = body
+        self._thread = None
 
     def _scheduler(self):
         if self.next_fire_time <= datetime.now():
@@ -36,7 +37,8 @@ class CronBroker(MemoryBroker):
         return CronConsumer(self, *args, **kwargs)
 
     def shutdown(self):
-        self._thread.cancel()
+        if self._thread:
+            self._thread.cancel()
 
 
 class CronConsumer(MemoryConsumer):

onestep/broker/memory.py

@@ -36,7 +36,7 @@ class MemoryBroker(BaseBroker):
         super().__init__(*args, **kwargs)
         self.queue = Queue(maxsize)
 
-    def publish(self, message: Any):
+    def publish(self, message: Any, *args, **kwargs):
         try:
             self.queue.put_nowait(message)
         except FullException:

onestep/broker/rabbitmq.py

@@ -3,17 +3,21 @@ import threading
 from queue import Queue
 from typing import Optional, Dict, Any
 
-import amqpstorm
+try:
+    import amqpstorm
+    from use_rabbitmq import useRabbitMQ as RabbitMQStore
+except ImportError:
+    amqpstorm = None
+    RabbitMQStore = None
 
 from .base import BaseBroker, BaseConsumer
-from use_rabbitmq import useRabbitMQ as RabbitMQStore
 from ..message import Message
 
 
 class _RabbitMQMessage(Message):
 
     @classmethod
-    def from_broker(cls, broker_message: amqpstorm.Message):
+    def from_broker(cls, broker_message):
         try:
             message = json.loads(broker_message.body)
         except json.JSONDecodeError:
@@ -30,20 +34,51 @@ class _RabbitMQMessage(Message):
 class RabbitMQBroker(BaseBroker):
     message_cls = _RabbitMQMessage
 
-    def __init__(self, queue_name, params: Optional[Dict] = None, prefetch: Optional[int] = 1, auto_create=True, *args,
-                 **kwargs):
+    def __init__(
+        self,
+        queue_name,
+        params: Optional[Dict] = None,
+        prefetch: Optional[int] = 1,
+        auto_create: Optional[bool]=True,
+        queue_params: Optional[Dict]=None,
+        *args,
+        **kwargs
+        ):
+        """
+        Initializes the RabbitMQ broker.
+
+        Args:
+            queue_name (str): The name of the queue.
+            params (Optional[Dict], optional): Parameters for RabbitMQStore. Defaults to None.
+            prefetch (Optional[int], optional): Number of messages to prefetch. Defaults to 1.
+            auto_create (Optional[bool], optional): Whether to automatically create the queue. Defaults to True.
+            queue_params (Optional[Dict], optional): Parameters for queue declaration. Defaults to None.
+            *args: Variable length argument list.
+            **kwargs: Arbitrary keyword arguments.
+
+        Attributes:
+            queue_name (str): The name of the queue.
+            queue (Queue): The queue instance.
+            client (RabbitMQStore): The RabbitMQ client instance.
+            prefetch (int): Number of messages to prefetch.
+            threads (list): List of threads.
+        """
+        
+        if RabbitMQStore is None:
+            raise ImportError("RabbitMQ dependencies not installed. Please install 'use-rabbitmq' package.")
+        
         super().__init__(*args, **kwargs)
         self.queue_name = queue_name
         self.queue = Queue()
         params = params or {}
         self.client = RabbitMQStore(**params)
         if auto_create:
-            self.client.declare_queue(self.queue_name)
+            self.client.declare_queue(self.queue_name, **(queue_params or {}))
         self.prefetch = prefetch
         self.threads = []
 
     def _consume(self, *args, **kwargs):
-        def callback(message: amqpstorm.Message):
+        def callback(message):
             self.queue.put(message)
 
         prefetch = kwargs.pop("prefetch", self.prefetch)
@@ -57,8 +92,9 @@ class RabbitMQBroker(BaseBroker):
         self.threads.append(thread)
         return RabbitMQConsumer(self)
 
-    def publish(self, message: Any):
-        self.client.send(self.queue_name, message)
+    def publish(self, message: Any, properties: Optional[dict] = None, **kwargs):
+        """发布消息"""
+        self.client.send(self.queue_name, message, properties=properties, **kwargs)
 
     def confirm(self, message: Message):
         """确认消息"""

onestep/broker/redis/pubsub.py

@@ -59,12 +59,12 @@ class RedisPubSubBroker(BaseBroker):
         self.threads.append(thread)
         return RedisPubSubConsumer(self)
 
-    def send(self, message: Any):
+    def send(self, message: Any, *args, **kwargs):
         """Publish message to the Redis channel"""
         if not isinstance(message, Message):
             message = self.message_cls(body=message)
 
-        self.client.publish(self.channel, message.to_json())
+        self.client.publish(self.channel, message.to_json(), *args, **kwargs)
 
     publish = send
 

onestep/broker/redis/stream.py

@@ -75,12 +75,12 @@ class RedisStreamBroker(BaseBroker):
         self.threads.append(thread)
         return RedisStreamConsumer(self)
 
-    def send(self, message: Any):
+    def send(self, message: Any, *args, **kwargs):
         """对消息进行预处理，然后再发送"""
         if not isinstance(message, Message):
             message = self.message_cls(body=message)
 
-        self.client.send({"_message": message.to_json()})
+        self.client.send({"_message": message.to_json()}, *args, **kwargs)
 
     publish = send
 

onestep/broker/webhook.py

@@ -57,13 +57,15 @@ class WebHookBroker(MemoryBroker):
                 WebHookServer
             )
             self._servers[(self.host, self.port)] = hs
+            # 只有在创建新服务器时才启动线程
+            thread = threading.Thread(target=hs.serve_forever)
+            thread.daemon = True
+            thread.start()
+            self.threads.append(thread)
         else:
             hs = self._servers[(self.host, self.port)]
 
         WebHookServer.servers[(self.host, self.port)].append(Server(self.path, self.queue))
-        thread = threading.Thread(target=hs.serve_forever)
-        thread.start()
-        self.threads.append(thread)
 
     def consume(self, *args, **kwargs):
         self._create_server()
@@ -71,7 +73,7 @@ class WebHookBroker(MemoryBroker):
         return WebHookConsumer(self, *args, **kwargs)
 
     def shutdown(self):
-        hs = self._servers[(self.host, self.port)]
+        hs = self._servers.get((self.host, self.port))
         if hs:
             hs.shutdown()
         for thread in self.threads:

onestep/cli.py

@@ -8,11 +8,17 @@ LOGFORMAT = "[%(asctime)s] [%(name)s] [%(levelname)s] %(message)s"
 
 
 def setup_logging():
-    logging.basicConfig(level=logging.DEBUG, format=LOGFORMAT, stream=sys.stdout)
+    # 设置全局日志级别为INFO，避免第三方库的DEBUG日志输出
+    logging.basicConfig(level=logging.INFO, format=LOGFORMAT, stream=sys.stdout)
 
     # exclude amqpstorm logs
     logging.getLogger("amqpstorm").setLevel(logging.CRITICAL)
-    return logging.getLogger("onestep")
+    
+    # 获取onestep的logger并设置为DEBUG级别以便调试
+    onestep_logger = logging.getLogger("onestep")
+    onestep_logger.setLevel(logging.DEBUG)
+    
+    return onestep_logger
 
 
 logger = setup_logging()

onestep/middleware/unique.py

@@ -1,7 +1,7 @@
 import hashlib
 import json
 
-from . import BaseMiddleware
+from .base import BaseMiddleware
 from ..exception import DropMessage
 
 

onestep/onestep.py

@@ -42,7 +42,7 @@ class BaseOneStep:
         self.workers = workers or DEFAULT_WORKERS
         self.worker_class = worker_class or DEFAULT_WORKER_CLASS
         if self.workers > MAX_WORKERS:
-            logger.warning(f"workers[{self.workers}] litter than {MAX_WORKERS}")
+            logger.warning(f"workers[{self.workers}] greater than {MAX_WORKERS}")
             self.workers = MAX_WORKERS
         self.middlewares = middlewares or []
 

onestep/worker.py

@@ -169,6 +169,8 @@ class ThreadWorker(BaseWorker):
 
 
 class ThreadPoolWorker(BaseWorker):
+    broker_exit: Dict[BaseBroker, bool] = {}
+    broker_exit_lock = threading.Lock()
 
     def __init__(self, onestep, broker: BaseBroker, workers=None, *args, **kwargs):
         super().__init__(onestep, broker, *args, **kwargs)
@@ -191,7 +193,8 @@ class ThreadPoolWorker(BaseWorker):
                     self.shutdown()
                     break
             for message in self.receive_messages():
-                self.handle_message(message)
+                # 将消息处理提交到线程池中并发执行
+                self.executor.submit(self.handle_message, message)
 
     def shutdown(self):
         """关闭线程池 Worker"""

{onestep-0.4.1.dist-info → onestep-0.4.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: onestep
-Version: 0.4.1
+Version: 0.4.3
 Summary: 仅需一步，轻松实现分布式异步任务。
 Author: miclon
 Author-email: jcnd@163.com
@@ -11,6 +11,8 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Provides-Extra: redis
 Requires-Dist: asgiref (>=3.6.0,<4.0.0)
 Requires-Dist: blinker (>=1.5,<2.0)

onestep-0.4.3.dist-info/RECORD

@@ -0,0 +1,27 @@
+onestep/__init__.py,sha256=G-MOCgma1x23C6hAW6UIpsyCRFna-41SPyGX0CSaH3E,1517
+onestep/_utils.py,sha256=ySYmy-o2o3BurvLfuLWKGXR8TLwug6aTDNMvA3_QjKM,414
+onestep/broker/__init__.py,sha256=L-roNwPZVKsjytOlKCOlyAyvfejI28FyOj9uzfyXcrk,257
+onestep/broker/base.py,sha256=G_MzKIbpDSPDx2rx-Q2rwourbcI_faBXQdQZRxjgx8s,4192
+onestep/broker/cron.py,sha256=3CZGHftQKrNwzdKGtVG8uiPuDb3U5W1T6WI-fdY5ea4,1246
+onestep/broker/memory.py,sha256=ycOWWHhWhiuNPMDqsbxKbMh8rpt84FlRsJkINLJNX1U,2070
+onestep/broker/rabbitmq.py,sha256=c5rIkUprFlBIGZOdTI5_2tp6DmsdvD9u40Yp6-COQ-o,4301
+onestep/broker/redis/__init__.py,sha256=lqjvBKpMGf6I34nyJTEAq7XU2a8aQemhMGCbb5aByFI,236
+onestep/broker/redis/pubsub.py,sha256=m0QzBT--rftfTXSc6D3wTzfQrwUgv5mEEDnnER7nB9A,2708
+onestep/broker/redis/stream.py,sha256=yUOD-Z0gzU5zNjRW1ZxNwl35i2pbJKTw33xIj0ga3vY,3504
+onestep/broker/webhook.py,sha256=x8h6zGsgjGFQdzr-UVXW8vGGUM_XCllglSCBqXh4rtw,2501
+onestep/cli.py,sha256=vXrMAg7Nl0Ra79DraBrCtQq8xJC443AUEqFXe7rVNYo,2133
+onestep/exception.py,sha256=T-tqfRrJZT9Y85qe-1l-1PcHZzV2jxv-61HAhOc4AKE,753
+onestep/message.py,sha256=W37HKnIDxpd5UmtDbPTw2DTb3lfxu9Vs2KYIiCr5b2s,4057
+onestep/middleware/__init__.py,sha256=MP_45lqr4pecmbzQBnn2-AODQ0N_Fss8nl2SA7Zzljo,347
+onestep/middleware/base.py,sha256=adWQ_Lx2Nkaw4ySojIaa3lwUIUu97pHR_l5YMr0lrbw,975
+onestep/middleware/config.py,sha256=WZIvGXhpdSQRAFTbEWXcZdnhFcbvhGkLdkFIj-_QuZM,2438
+onestep/middleware/unique.py,sha256=L4F4NnpDO6hReHa21tUiiPH5GRMeRNgGgC0o6zgu5v0,1355
+onestep/onestep.py,sha256=PWAmjrrqXxnsh_CzA_pXET3Xe7jkKteWiIsfs5_gVaY,9373
+onestep/retry.py,sha256=fPO139hQrcMjXZuQ4QfFaQR57VEqv_EKS2WSz92Ayvc,3706
+onestep/signal.py,sha256=tz9bGzTFZeBviG_iaGLfy4OyFKlWat6szEI6kWEGfTA,337
+onestep/state.py,sha256=UVG91CXCabU64X6qgcO0S7RzbBP8ut0ID7aTMww94us,618
+onestep/worker.py,sha256=SLLaAjUA65i5PEHRbRzHaWLNokvrpPz1-BzSiZdbhqg,7029
+onestep-0.4.3.dist-info/METADATA,sha256=fGTZznrO5qUZ-lD_3Gh_3hARAVLRxYW0k-KmcIZKshU,2675
+onestep-0.4.3.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
+onestep-0.4.3.dist-info/entry_points.txt,sha256=ZfWnNQqiGujz2PPLjSlKPocOFRryL7Ot0vQ41TU1xw0,44
+onestep-0.4.3.dist-info/RECORD,,

{onestep-0.4.1.dist-info → onestep-0.4.3.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 1.9.0
+Generator: poetry-core 2.2.1
 Root-Is-Purelib: true
 Tag: py3-none-any

onestep-0.4.1.dist-info/RECORD

@@ -1,27 +0,0 @@
-onestep/__init__.py,sha256=F38-j9JgPMyUe0zBf-ao-f2ailidPI_DJQE4ddGCOX0,1417
-onestep/_utils.py,sha256=ySYmy-o2o3BurvLfuLWKGXR8TLwug6aTDNMvA3_QjKM,414
-onestep/broker/__init__.py,sha256=L-roNwPZVKsjytOlKCOlyAyvfejI28FyOj9uzfyXcrk,257
-onestep/broker/base.py,sha256=Ryo-0YekdZfpfSSqdmh7jsmNONc_GmlYFYadt_n5bds,4141
-onestep/broker/cron.py,sha256=wKOm4k0IBPgZIq5gHC6ITVFUZwgCQ91AXzgL1eFT1WE,1135
-onestep/broker/memory.py,sha256=X7DDiEOiJE3WtocEC_NtSNfL8Jpo7BVQVSG_BJfNGNQ,2053
-onestep/broker/rabbitmq.py,sha256=LFlcpOV36D6bme5nUVsZ3WzJVFQazlKDKBcA9aQONRM,2928
-onestep/broker/redis/__init__.py,sha256=lqjvBKpMGf6I34nyJTEAq7XU2a8aQemhMGCbb5aByFI,236
-onestep/broker/redis/pubsub.py,sha256=_p2bKqXu0gYjKz2DsS8UdpkN5dCy94xf_xYx5NFuKpY,2674
-onestep/broker/redis/stream.py,sha256=uQJncD93Fz8pdP_NdgyQ1UAXwIouh7AFA9_naXaeiJM,3470
-onestep/broker/webhook.py,sha256=Zg22QxKPqJzdxjBsglUcL7P1RcM8FltcHpHXJiJPYgM,2392
-onestep/cli.py,sha256=JU3xQCy0Zc8Ou6AH4j37_bqw5GQ7C2wll9MWQZFo5jY,1902
-onestep/exception.py,sha256=T-tqfRrJZT9Y85qe-1l-1PcHZzV2jxv-61HAhOc4AKE,753
-onestep/message.py,sha256=W37HKnIDxpd5UmtDbPTw2DTb3lfxu9Vs2KYIiCr5b2s,4057
-onestep/middleware/__init__.py,sha256=MP_45lqr4pecmbzQBnn2-AODQ0N_Fss8nl2SA7Zzljo,347
-onestep/middleware/base.py,sha256=adWQ_Lx2Nkaw4ySojIaa3lwUIUu97pHR_l5YMr0lrbw,975
-onestep/middleware/config.py,sha256=WZIvGXhpdSQRAFTbEWXcZdnhFcbvhGkLdkFIj-_QuZM,2438
-onestep/middleware/unique.py,sha256=e3nWHW2yxHFWT9PqiA96sOA1CZZVTbC3bOwUgxN9iaQ,1351
-onestep/onestep.py,sha256=IQGOfhfbneXCaINqMzO-5wjWpmE5dISEZQtmG3bZ3EU,9372
-onestep/retry.py,sha256=fPO139hQrcMjXZuQ4QfFaQR57VEqv_EKS2WSz92Ayvc,3706
-onestep/signal.py,sha256=tz9bGzTFZeBviG_iaGLfy4OyFKlWat6szEI6kWEGfTA,337
-onestep/state.py,sha256=UVG91CXCabU64X6qgcO0S7RzbBP8ut0ID7aTMww94us,618
-onestep/worker.py,sha256=8Bmbie-KrZxnHbne1RvNL3v8j1PObfd12Yp9jmzAFYg,6855
-onestep-0.4.1.dist-info/METADATA,sha256=WoZmHjd1LmL9MvamCVyUsEakOTmReNSg2FSBHBCqd2w,2573
-onestep-0.4.1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-onestep-0.4.1.dist-info/entry_points.txt,sha256=ZfWnNQqiGujz2PPLjSlKPocOFRryL7Ot0vQ41TU1xw0,44
-onestep-0.4.1.dist-info/RECORD,,