onestep 0.3.3__py3-none-any.whl → 0.3.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of onestep might be problematic. Click here for more details.
- onestep/__init__.py +6 -5
- onestep/broker/base.py +9 -5
- onestep/broker/cron.py +1 -1
- onestep/broker/rabbitmq.py +1 -1
- onestep/broker/redis.py +20 -17
- onestep/broker/webhook.py +2 -3
- onestep/message.py +9 -7
- onestep/onestep.py +10 -4
- onestep/retry.py +36 -25
- onestep/worker.py +16 -9
- {onestep-0.3.3.dist-info → onestep-0.3.4.dist-info}/METADATA +4 -2
- onestep-0.3.4.dist-info/RECORD +23 -0
- {onestep-0.3.3.dist-info → onestep-0.3.4.dist-info}/WHEEL +1 -1
- onestep-0.3.3.dist-info/RECORD +0 -23
onestep/__init__.py
CHANGED
|
@@ -5,7 +5,7 @@ from .retry import (
|
|
|
5
5
|
)
|
|
6
6
|
from .broker import (
|
|
7
7
|
BaseBroker, BaseConsumer, BaseLocalBroker, BaseLocalConsumer,
|
|
8
|
-
MemoryBroker, RabbitMQBroker, WebHookBroker, CronBroker
|
|
8
|
+
MemoryBroker, RabbitMQBroker, WebHookBroker, CronBroker, RedisStreamBroker
|
|
9
9
|
)
|
|
10
10
|
from .middleware import (
|
|
11
11
|
BaseMiddleware, BaseConfigMiddleware,
|
|
@@ -19,7 +19,7 @@ from .exception import (
|
|
|
19
19
|
|
|
20
20
|
__all__ = [
|
|
21
21
|
'step',
|
|
22
|
-
|
|
22
|
+
|
|
23
23
|
# broker
|
|
24
24
|
'BaseBroker',
|
|
25
25
|
'BaseConsumer',
|
|
@@ -29,7 +29,8 @@ __all__ = [
|
|
|
29
29
|
'RabbitMQBroker',
|
|
30
30
|
'WebHookBroker',
|
|
31
31
|
'CronBroker',
|
|
32
|
-
|
|
32
|
+
'RedisStreamBroker',
|
|
33
|
+
|
|
33
34
|
# retry
|
|
34
35
|
'BaseRetry',
|
|
35
36
|
'NeverRetry',
|
|
@@ -40,7 +41,7 @@ __all__ = [
|
|
|
40
41
|
# error callback
|
|
41
42
|
'BaseErrorCallback',
|
|
42
43
|
'NackErrorCallBack',
|
|
43
|
-
|
|
44
|
+
|
|
44
45
|
# middleware
|
|
45
46
|
'BaseMiddleware',
|
|
46
47
|
'BaseConfigMiddleware',
|
|
@@ -48,7 +49,7 @@ __all__ = [
|
|
|
48
49
|
'NacosConsumeConfigMiddleware',
|
|
49
50
|
'RedisPublishConfigMiddleware',
|
|
50
51
|
'RedisConsumeConfigMiddleware',
|
|
51
|
-
|
|
52
|
+
|
|
52
53
|
# exception
|
|
53
54
|
'StopMiddleware',
|
|
54
55
|
'DropMessage',
|
onestep/broker/base.py
CHANGED
|
@@ -13,10 +13,11 @@ logger = logging.getLogger(__name__)
|
|
|
13
13
|
|
|
14
14
|
class BaseBroker:
|
|
15
15
|
|
|
16
|
-
def __init__(self, name=None, queue=None, middlewares=None):
|
|
16
|
+
def __init__(self, name=None, queue=None, middlewares=None, once: bool = False):
|
|
17
17
|
self.queue = queue
|
|
18
18
|
self.name = name or "broker"
|
|
19
19
|
self.middlewares = []
|
|
20
|
+
self.once = once
|
|
20
21
|
|
|
21
22
|
if middlewares:
|
|
22
23
|
for middleware in middlewares:
|
|
@@ -157,10 +158,13 @@ class BaseLocalBroker(BaseBroker):
|
|
|
157
158
|
class BaseLocalConsumer(BaseConsumer):
|
|
158
159
|
|
|
159
160
|
def _to_message(self, data: Any):
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
161
|
+
if isinstance(data, (str, bytes, bytearray)):
|
|
162
|
+
try:
|
|
163
|
+
message = json.loads(data)
|
|
164
|
+
except json.JSONDecodeError:
|
|
165
|
+
message = {"body": data}
|
|
166
|
+
else:
|
|
167
|
+
message = data
|
|
164
168
|
if not isinstance(message, dict):
|
|
165
169
|
message = {"body": message}
|
|
166
170
|
if "body" not in message:
|
onestep/broker/cron.py
CHANGED
|
@@ -21,7 +21,6 @@ class CronBroker(BaseLocalBroker):
|
|
|
21
21
|
self.itr = croniter(cron, datetime.now())
|
|
22
22
|
self.next_fire_time = self.itr.get_next(datetime)
|
|
23
23
|
self.kwargs = kwargs
|
|
24
|
-
self._scheduler()
|
|
25
24
|
|
|
26
25
|
def _scheduler(self):
|
|
27
26
|
if self.next_fire_time <= datetime.now():
|
|
@@ -32,6 +31,7 @@ class CronBroker(BaseLocalBroker):
|
|
|
32
31
|
self._thread.start()
|
|
33
32
|
|
|
34
33
|
def consume(self):
|
|
34
|
+
self._scheduler()
|
|
35
35
|
return CronConsumer(self.queue)
|
|
36
36
|
|
|
37
37
|
def shutdown(self):
|
onestep/broker/rabbitmq.py
CHANGED
|
@@ -55,7 +55,7 @@ class RabbitMQBroker(BaseBroker):
|
|
|
55
55
|
重发消息:先拒绝 再 重入
|
|
56
56
|
|
|
57
57
|
:param message: 消息
|
|
58
|
-
:param is_source:
|
|
58
|
+
:param is_source: 是否是原始消息,True: 使用原始消息重入当前队列,False: 使用消息的最新数据重入当前队列
|
|
59
59
|
"""
|
|
60
60
|
if is_source:
|
|
61
61
|
message.msg.reject(requeue=True)
|
onestep/broker/redis.py
CHANGED
|
@@ -4,14 +4,17 @@ import uuid
|
|
|
4
4
|
from queue import Queue
|
|
5
5
|
from typing import Optional, Dict, Any
|
|
6
6
|
|
|
7
|
-
|
|
7
|
+
try:
|
|
8
|
+
from usepy_plugin_redis import useRedisStreamStore, RedisStreamMessage
|
|
9
|
+
except ImportError:
|
|
10
|
+
...
|
|
8
11
|
|
|
9
12
|
from .base import BaseBroker, BaseConsumer, Message
|
|
10
13
|
|
|
11
14
|
|
|
12
15
|
class RedisStreamBroker(BaseBroker):
|
|
13
16
|
""" Redis Stream Broker """
|
|
14
|
-
|
|
17
|
+
|
|
15
18
|
def __init__(
|
|
16
19
|
self,
|
|
17
20
|
stream: str,
|
|
@@ -28,9 +31,9 @@ class RedisStreamBroker(BaseBroker):
|
|
|
28
31
|
self.group = group
|
|
29
32
|
self.prefetch = prefetch
|
|
30
33
|
self.queue = Queue()
|
|
31
|
-
|
|
34
|
+
|
|
32
35
|
self.threads = []
|
|
33
|
-
|
|
36
|
+
|
|
34
37
|
self.client = useRedisStreamStore(
|
|
35
38
|
stream=stream,
|
|
36
39
|
group=group,
|
|
@@ -39,14 +42,14 @@ class RedisStreamBroker(BaseBroker):
|
|
|
39
42
|
claim_interval=claim_interval,
|
|
40
43
|
**(params or {})
|
|
41
44
|
)
|
|
42
|
-
|
|
45
|
+
|
|
43
46
|
def _consume(self, *args, **kwargs):
|
|
44
47
|
def callback(message):
|
|
45
48
|
self.queue.put(message)
|
|
46
|
-
|
|
49
|
+
|
|
47
50
|
prefetch = kwargs.pop("prefetch", self.prefetch)
|
|
48
51
|
self.client.start_consuming(consumer=uuid.uuid4().hex, callback=callback, prefetch=prefetch, **kwargs)
|
|
49
|
-
|
|
52
|
+
|
|
50
53
|
def consume(self, *args, **kwargs):
|
|
51
54
|
daemon = kwargs.pop('daemon', True)
|
|
52
55
|
thread = threading.Thread(target=self._consume, *args, **kwargs)
|
|
@@ -54,31 +57,31 @@ class RedisStreamBroker(BaseBroker):
|
|
|
54
57
|
thread.start()
|
|
55
58
|
self.threads.append(thread)
|
|
56
59
|
return RedisStreamConsumer(self.queue)
|
|
57
|
-
|
|
60
|
+
|
|
58
61
|
def send(self, message: Any):
|
|
59
62
|
"""对消息进行预处理,然后再发送"""
|
|
60
63
|
if not isinstance(message, Message):
|
|
61
64
|
message = Message(body=message)
|
|
62
|
-
|
|
65
|
+
|
|
63
66
|
self.client.send({"_message": message.to_json()})
|
|
64
|
-
|
|
67
|
+
|
|
65
68
|
publish = send
|
|
66
|
-
|
|
69
|
+
|
|
67
70
|
def confirm(self, message: Message):
|
|
68
71
|
self.client.ack(message.msg)
|
|
69
|
-
|
|
72
|
+
|
|
70
73
|
def reject(self, message: Message):
|
|
71
74
|
self.client.reject(message.msg)
|
|
72
|
-
|
|
75
|
+
|
|
73
76
|
def requeue(self, message: Message, is_source=False):
|
|
74
77
|
"""
|
|
75
78
|
重发消息:先拒绝 再 重入
|
|
76
79
|
|
|
77
80
|
:param message: 消息
|
|
78
|
-
:param is_source:
|
|
81
|
+
:param is_source: 是否是原始消息消息,True: 使用原始消息重入当前队列,False: 使用消息的最新数据重入当前队列
|
|
79
82
|
"""
|
|
80
83
|
self.reject(message)
|
|
81
|
-
|
|
84
|
+
|
|
82
85
|
if is_source:
|
|
83
86
|
self.client.send(message.msg.body)
|
|
84
87
|
else:
|
|
@@ -86,7 +89,7 @@ class RedisStreamBroker(BaseBroker):
|
|
|
86
89
|
|
|
87
90
|
|
|
88
91
|
class RedisStreamConsumer(BaseConsumer):
|
|
89
|
-
def _to_message(self, data: RedisStreamMessage):
|
|
92
|
+
def _to_message(self, data: "RedisStreamMessage"):
|
|
90
93
|
if "_message" in data.body:
|
|
91
94
|
# 来自 RedisStreamBroker.send 的消息,message.body 默认是存于 _message 字段中
|
|
92
95
|
try:
|
|
@@ -96,5 +99,5 @@ class RedisStreamConsumer(BaseConsumer):
|
|
|
96
99
|
else:
|
|
97
100
|
# 来自 外部的消息 直接认为都是 message.body
|
|
98
101
|
message = {"body": data.body}
|
|
99
|
-
|
|
102
|
+
|
|
100
103
|
yield Message(body=message.get("body"), extra=message.get("extra"), msg=data)
|
onestep/broker/webhook.py
CHANGED
|
@@ -49,9 +49,6 @@ class WebHookBroker(BaseLocalBroker):
|
|
|
49
49
|
self.path = path
|
|
50
50
|
self.threads = []
|
|
51
51
|
|
|
52
|
-
self._create_server()
|
|
53
|
-
logger.debug(f"WebHookBroker: {self.host}:{self.port}{self.path}")
|
|
54
|
-
|
|
55
52
|
def _create_server(self):
|
|
56
53
|
|
|
57
54
|
if (self.host, self.port) not in self._servers:
|
|
@@ -69,6 +66,8 @@ class WebHookBroker(BaseLocalBroker):
|
|
|
69
66
|
self.threads.append(thread)
|
|
70
67
|
|
|
71
68
|
def consume(self, *args, **kwargs):
|
|
69
|
+
self._create_server()
|
|
70
|
+
logger.debug(f"WebHookBroker: {self.host}:{self.port}{self.path}")
|
|
72
71
|
return WebHookConsumer(self.queue, *args, **kwargs)
|
|
73
72
|
|
|
74
73
|
def shutdown(self):
|
onestep/message.py
CHANGED
|
@@ -2,6 +2,7 @@ import json
|
|
|
2
2
|
import sys
|
|
3
3
|
import time
|
|
4
4
|
import uuid
|
|
5
|
+
from dataclasses import dataclass
|
|
5
6
|
from traceback import TracebackException
|
|
6
7
|
from typing import Optional, Any, Union
|
|
7
8
|
|
|
@@ -14,11 +15,15 @@ class MessageTracebackException(TracebackException):
|
|
|
14
15
|
self.exc_value = exc_value
|
|
15
16
|
|
|
16
17
|
|
|
18
|
+
@dataclass
|
|
17
19
|
class Extra:
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
20
|
+
task_id: str = None
|
|
21
|
+
publish_time: float = None
|
|
22
|
+
failure_count: int = 0
|
|
23
|
+
|
|
24
|
+
def __post_init__(self):
|
|
25
|
+
self.task_id = self.task_id or str(uuid.uuid4())
|
|
26
|
+
self.publish_time = self.publish_time or round(time.time(), 3)
|
|
22
27
|
|
|
23
28
|
def to_dict(self):
|
|
24
29
|
return {
|
|
@@ -30,9 +35,6 @@ class Extra:
|
|
|
30
35
|
def __str__(self):
|
|
31
36
|
return str(self.to_dict())
|
|
32
37
|
|
|
33
|
-
def __repr__(self):
|
|
34
|
-
return f"{self.__class__.__name__}({self.task_id}, {self.publish_time}, {self.failure_count})"
|
|
35
|
-
|
|
36
38
|
|
|
37
39
|
class Message:
|
|
38
40
|
|
onestep/onestep.py
CHANGED
|
@@ -8,7 +8,7 @@ from typing import Optional, List, Dict, Any, Callable, Union
|
|
|
8
8
|
from .broker.base import BaseBroker
|
|
9
9
|
from .exception import StopMiddleware
|
|
10
10
|
from .message import Message
|
|
11
|
-
from .retry import
|
|
11
|
+
from .retry import TimesRetry
|
|
12
12
|
from .signal import message_sent, started, stopped
|
|
13
13
|
from .state import State
|
|
14
14
|
from .worker import WorkerThread
|
|
@@ -29,7 +29,7 @@ class BaseOneStep:
|
|
|
29
29
|
to_broker: Union[BaseBroker, List[BaseBroker], None] = None,
|
|
30
30
|
workers: Optional[int] = None,
|
|
31
31
|
middlewares: Optional[List[Any]] = None,
|
|
32
|
-
retry: Union[Callable, object] =
|
|
32
|
+
retry: Union[Callable, object] = TimesRetry(),
|
|
33
33
|
error_callback: Optional[Union[Callable, object]] = None):
|
|
34
34
|
self.group = group
|
|
35
35
|
self.fn = fn
|
|
@@ -58,6 +58,7 @@ class BaseOneStep:
|
|
|
58
58
|
f"broker must be BaseBroker or list or tuple, not {type(broker)}")
|
|
59
59
|
|
|
60
60
|
def _add_consumer(self, broker):
|
|
61
|
+
""" 添加来源消费者 """
|
|
61
62
|
for _ in range(self.workers):
|
|
62
63
|
self.consumers[self.group].append(
|
|
63
64
|
WorkerThread(onestep=self, broker=broker)
|
|
@@ -131,6 +132,11 @@ class BaseOneStep:
|
|
|
131
132
|
logger.debug(f"middleware<{middleware}> is stopped,reason: {e}")
|
|
132
133
|
break
|
|
133
134
|
|
|
135
|
+
@classmethod
|
|
136
|
+
def is_shutdown(cls, group):
|
|
137
|
+
# check all broker
|
|
138
|
+
return all(broker._shutdown for broker in cls._find_consumers(group))
|
|
139
|
+
|
|
134
140
|
|
|
135
141
|
def decorator_func_proxy(func):
|
|
136
142
|
@functools.wraps(func)
|
|
@@ -182,7 +188,7 @@ class step:
|
|
|
182
188
|
to_broker: Union[BaseBroker, List[BaseBroker], None] = None,
|
|
183
189
|
workers: Optional[int] = None,
|
|
184
190
|
middlewares: Optional[List[Any]] = None,
|
|
185
|
-
retry: Union[Callable, object] =
|
|
191
|
+
retry: Union[Callable, object] = TimesRetry(),
|
|
186
192
|
error_callback: Optional[Union[Callable, object]] = None):
|
|
187
193
|
self.params = {
|
|
188
194
|
"group": group,
|
|
@@ -209,7 +215,7 @@ class step:
|
|
|
209
215
|
started.send()
|
|
210
216
|
if block:
|
|
211
217
|
import time
|
|
212
|
-
while
|
|
218
|
+
while not BaseOneStep.is_shutdown(group):
|
|
213
219
|
time.sleep(1)
|
|
214
220
|
|
|
215
221
|
@staticmethod
|
onestep/retry.py
CHANGED
|
@@ -1,46 +1,56 @@
|
|
|
1
1
|
from abc import ABC, abstractmethod
|
|
2
|
+
from enum import Enum
|
|
2
3
|
from typing import Optional, Tuple, Union, Type
|
|
3
4
|
|
|
4
5
|
from .exception import RetryViaLocal, RetryViaQueue
|
|
5
6
|
from .message import Message
|
|
6
7
|
|
|
7
8
|
|
|
9
|
+
class RetryStatus(Enum):
|
|
10
|
+
CONTINUE = 1 # 继续(执行重试)
|
|
11
|
+
END_WITH_CALLBACK = 2 # 结束(执行回调)
|
|
12
|
+
END_IGNORE_CALLBACK = 3 # 结束(忽略回调)
|
|
13
|
+
|
|
14
|
+
|
|
8
15
|
class BaseRetry(ABC):
|
|
9
|
-
|
|
16
|
+
|
|
10
17
|
@abstractmethod
|
|
11
|
-
def __call__(self, *args, **kwargs) -> Optional[
|
|
12
|
-
"""True=继续(执行重试), False=结束(执行回调),None=结束(忽略回调)"""
|
|
18
|
+
def __call__(self, *args, **kwargs) -> Optional[RetryStatus]:
|
|
13
19
|
pass
|
|
14
20
|
|
|
15
21
|
|
|
16
22
|
class NeverRetry(BaseRetry):
|
|
17
|
-
|
|
18
|
-
def __call__(self, message) -> Optional[
|
|
19
|
-
return
|
|
23
|
+
|
|
24
|
+
def __call__(self, message) -> Optional[RetryStatus]:
|
|
25
|
+
return RetryStatus.END_WITH_CALLBACK
|
|
20
26
|
|
|
21
27
|
|
|
22
28
|
class AlwaysRetry(BaseRetry):
|
|
23
|
-
|
|
24
|
-
def __call__(self, message) -> Optional[
|
|
25
|
-
return
|
|
29
|
+
|
|
30
|
+
def __call__(self, message) -> Optional[RetryStatus]:
|
|
31
|
+
return RetryStatus.CONTINUE
|
|
26
32
|
|
|
27
33
|
|
|
28
34
|
class TimesRetry(BaseRetry):
|
|
29
|
-
|
|
35
|
+
|
|
30
36
|
def __init__(self, times: int = 3):
|
|
31
37
|
self.times = times
|
|
32
|
-
|
|
33
|
-
def __call__(self, message) -> Optional[
|
|
34
|
-
|
|
38
|
+
|
|
39
|
+
def __call__(self, message) -> Optional[RetryStatus]:
|
|
40
|
+
if message.failure_count < self.times:
|
|
41
|
+
return RetryStatus.CONTINUE
|
|
42
|
+
return RetryStatus.END_WITH_CALLBACK
|
|
35
43
|
|
|
36
44
|
|
|
37
45
|
class RetryIfException(BaseRetry):
|
|
38
|
-
|
|
46
|
+
|
|
39
47
|
def __init__(self, exceptions: Optional[Tuple[Union[Exception, Type]]] = Exception):
|
|
40
48
|
self.exceptions = exceptions
|
|
41
|
-
|
|
42
|
-
def __call__(self, message) -> Optional[
|
|
43
|
-
|
|
49
|
+
|
|
50
|
+
def __call__(self, message) -> Optional[RetryStatus]:
|
|
51
|
+
if isinstance(message.exception.exc_value, self.exceptions):
|
|
52
|
+
return RetryStatus.CONTINUE
|
|
53
|
+
return RetryStatus.END_WITH_CALLBACK
|
|
44
54
|
|
|
45
55
|
|
|
46
56
|
class AdvancedRetry(TimesRetry):
|
|
@@ -51,32 +61,33 @@ class AdvancedRetry(TimesRetry):
|
|
|
51
61
|
3. 其他异常:如果异常不是 RetryViaLocal 或 RetryViaQueue 或 指定异常,则不重试,回调
|
|
52
62
|
注:待重试的异常若继承自 RetryException,则可单独指定重试次数,否则默认为 3 次
|
|
53
63
|
"""
|
|
64
|
+
|
|
54
65
|
def __init__(self, times: int = 3, exceptions: Optional[Tuple[Union[Exception, Type]]] = None):
|
|
55
66
|
super().__init__(times=times)
|
|
56
67
|
self.exceptions = (RetryViaLocal, RetryViaQueue) + (exceptions or ())
|
|
57
|
-
|
|
58
|
-
def __call__(self, message: Message) -> Optional[
|
|
68
|
+
|
|
69
|
+
def __call__(self, message: Message) -> Optional[RetryStatus]:
|
|
59
70
|
if isinstance(message.exception.exc_value, self.exceptions):
|
|
60
71
|
max_retry_times = getattr(message.exception.exc_value, "times", None) or self.times
|
|
61
72
|
if message.failure_count < max_retry_times:
|
|
62
73
|
if isinstance(message.exception.exc_value, RetryViaQueue):
|
|
63
|
-
return
|
|
64
|
-
return
|
|
74
|
+
return RetryStatus.END_IGNORE_CALLBACK
|
|
75
|
+
return RetryStatus.CONTINUE
|
|
65
76
|
else:
|
|
66
|
-
return
|
|
77
|
+
return RetryStatus.END_WITH_CALLBACK
|
|
67
78
|
else:
|
|
68
|
-
return
|
|
79
|
+
return RetryStatus.END_WITH_CALLBACK
|
|
69
80
|
|
|
70
81
|
|
|
71
82
|
# error callback
|
|
72
83
|
class BaseErrorCallback(ABC):
|
|
73
|
-
|
|
84
|
+
|
|
74
85
|
@abstractmethod
|
|
75
86
|
def __call__(self, *args, **kwargs):
|
|
76
87
|
pass
|
|
77
88
|
|
|
78
89
|
|
|
79
90
|
class NackErrorCallBack(BaseErrorCallback):
|
|
80
|
-
|
|
91
|
+
|
|
81
92
|
def __call__(self, message):
|
|
82
93
|
message.reject()
|
onestep/worker.py
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"""
|
|
2
2
|
将指定的函数放入线程中运行
|
|
3
3
|
"""
|
|
4
|
+
|
|
4
5
|
try:
|
|
5
6
|
from collections import Iterable
|
|
6
7
|
except ImportError:
|
|
@@ -12,6 +13,7 @@ from inspect import isasyncgenfunction
|
|
|
12
13
|
|
|
13
14
|
from asgiref.sync import async_to_sync
|
|
14
15
|
|
|
16
|
+
from .retry import RetryStatus
|
|
15
17
|
from .broker import BaseBroker
|
|
16
18
|
from .exception import DropMessage
|
|
17
19
|
from .signal import message_received, message_consumed, message_error, message_drop
|
|
@@ -34,7 +36,7 @@ class WorkerThread(threading.Thread):
|
|
|
34
36
|
self.broker = broker
|
|
35
37
|
self.args = args
|
|
36
38
|
self.kwargs = kwargs
|
|
37
|
-
self.
|
|
39
|
+
self._shutdown = False
|
|
38
40
|
|
|
39
41
|
def run(self):
|
|
40
42
|
"""线程执行包装过的`onestep`函数
|
|
@@ -43,11 +45,11 @@ class WorkerThread(threading.Thread):
|
|
|
43
45
|
:return:
|
|
44
46
|
"""
|
|
45
47
|
|
|
46
|
-
while not self.
|
|
47
|
-
if self.__shutdown:
|
|
48
|
-
break
|
|
48
|
+
while not self._shutdown:
|
|
49
49
|
# TODO:consume应当传入一些配置参数
|
|
50
50
|
for result in self.broker.consume():
|
|
51
|
+
if self._shutdown:
|
|
52
|
+
break
|
|
51
53
|
if result is None:
|
|
52
54
|
continue
|
|
53
55
|
messages = (
|
|
@@ -67,10 +69,14 @@ class WorkerThread(threading.Thread):
|
|
|
67
69
|
message_drop.send(self, message=message, reason=e)
|
|
68
70
|
logger.warning(f"{self.instance.name} dropped <{type(e).__name__}: {str(e)}>")
|
|
69
71
|
message.reject()
|
|
72
|
+
else:
|
|
73
|
+
if self.broker.once:
|
|
74
|
+
self.broker.shutdown()
|
|
75
|
+
self._shutdown = True
|
|
70
76
|
|
|
71
77
|
def shutdown(self):
|
|
72
78
|
self.broker.shutdown()
|
|
73
|
-
self.
|
|
79
|
+
self._shutdown = True
|
|
74
80
|
|
|
75
81
|
def _run_instance(self, message):
|
|
76
82
|
while True:
|
|
@@ -89,12 +95,13 @@ class WorkerThread(threading.Thread):
|
|
|
89
95
|
logger.error(f"{self.instance.name} run error <{type(e).__name__}: {str(e)}>")
|
|
90
96
|
message.set_exception()
|
|
91
97
|
|
|
92
|
-
|
|
93
|
-
if
|
|
98
|
+
retry_status = self.retry(message)
|
|
99
|
+
if retry_status is RetryStatus.CONTINUE:
|
|
94
100
|
continue
|
|
95
|
-
elif
|
|
101
|
+
elif retry_status is RetryStatus.END_WITH_CALLBACK:
|
|
96
102
|
if self.error_callback:
|
|
97
103
|
self.error_callback(message)
|
|
98
104
|
return message.reject()
|
|
99
|
-
else: #
|
|
105
|
+
else: # RetryStatus.END_IGNORE_CALLBACK
|
|
106
|
+
# 由于是队列内重试,不会触发错误回调
|
|
100
107
|
return message.requeue()
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: onestep
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.4
|
|
4
4
|
Summary:
|
|
5
5
|
Author: miclon
|
|
6
6
|
Author-email: jcnd@163.com
|
|
@@ -10,11 +10,13 @@ Classifier: Programming Language :: Python :: 3.8
|
|
|
10
10
|
Classifier: Programming Language :: Python :: 3.9
|
|
11
11
|
Classifier: Programming Language :: Python :: 3.10
|
|
12
12
|
Classifier: Programming Language :: Python :: 3.11
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
14
|
+
Provides-Extra: redis
|
|
13
15
|
Requires-Dist: asgiref (>=3.6.0,<4.0.0)
|
|
14
16
|
Requires-Dist: blinker (>=1.5,<2.0)
|
|
15
17
|
Requires-Dist: croniter (>=1.3.8,<2.0.0)
|
|
16
18
|
Requires-Dist: usepy-plugin-rabbitmq (>=0.1.0,<0.2.0)
|
|
17
|
-
Requires-Dist: usepy-plugin-redis (>=0.1.
|
|
19
|
+
Requires-Dist: usepy-plugin-redis (>=0.1.6,<0.2.0) ; extra == "redis"
|
|
18
20
|
Description-Content-Type: text/markdown
|
|
19
21
|
|
|
20
22
|
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
onestep/__init__.py,sha256=02PvqWR2mbLNLPoyTW6DIuWrmW6oh9GwRvyOLg8enBg,1418
|
|
2
|
+
onestep/_utils.py,sha256=ySYmy-o2o3BurvLfuLWKGXR8TLwug6aTDNMvA3_QjKM,414
|
|
3
|
+
onestep/broker/__init__.py,sha256=atvxgYy16nnSjzsXxX4Fm_zoPlqoHJK_INQeYVH8mRE,258
|
|
4
|
+
onestep/broker/base.py,sha256=JkfqHM0BweMQjV7qcVaiRzlNhR8VG4OcQNyGjkCM3qo,5219
|
|
5
|
+
onestep/broker/cron.py,sha256=gDkD0OymNMdYRxujg2sr3AwLEyAkxYGplhbkRV9Idu0,1058
|
|
6
|
+
onestep/broker/memory.py,sha256=ILDfYCbMuMmYcZW09jVCdkAg3IQcC-dHkGHIgd8Wh4Q,151
|
|
7
|
+
onestep/broker/rabbitmq.py,sha256=7xo8wGbFTD-6fpzP0lwh9uZIcdDh5ClioXBx9vXxLy0,2792
|
|
8
|
+
onestep/broker/redis.py,sha256=nE7TtQAQFiUcJdg72k5BhmbWsewrwNt1W4tznXImQPI,3304
|
|
9
|
+
onestep/broker/webhook.py,sha256=fqCLFtkUgsonWZBn9SKMKo5QItJsm8magB6P-sdwT2U,2408
|
|
10
|
+
onestep/exception.py,sha256=YePdLAFg2gHtzOdwmAtKEosdTCxr-xh5ZhRvofGrpDE,610
|
|
11
|
+
onestep/message.py,sha256=KbvqfqYugg1vzWocrQ7_eGJMGtyCxEbtraddZhJ-jr8,3819
|
|
12
|
+
onestep/middleware/__init__.py,sha256=MP_45lqr4pecmbzQBnn2-AODQ0N_Fss8nl2SA7Zzljo,347
|
|
13
|
+
onestep/middleware/base.py,sha256=adWQ_Lx2Nkaw4ySojIaa3lwUIUu97pHR_l5YMr0lrbw,975
|
|
14
|
+
onestep/middleware/config.py,sha256=WZIvGXhpdSQRAFTbEWXcZdnhFcbvhGkLdkFIj-_QuZM,2438
|
|
15
|
+
onestep/middleware/unique.py,sha256=e3nWHW2yxHFWT9PqiA96sOA1CZZVTbC3bOwUgxN9iaQ,1351
|
|
16
|
+
onestep/onestep.py,sha256=8SI6Wj_OyrpgeKNQIgBGw9elmaQe_UbDLr-0Upu4xx0,7554
|
|
17
|
+
onestep/retry.py,sha256=miv7fTeuCXD11kmUlIQ2-ESH_qO6KXRfQIjKQXg1P-E,3018
|
|
18
|
+
onestep/signal.py,sha256=o8KD58jL6lM35gDHVgr1lEspZqFdkMUNJAD-wuouUVo,293
|
|
19
|
+
onestep/state.py,sha256=UVG91CXCabU64X6qgcO0S7RzbBP8ut0ID7aTMww94us,618
|
|
20
|
+
onestep/worker.py,sha256=uTD6XnQx9l0zdCZn8hm3JucI-YdMZKyMVMKAZSVdnKM,4083
|
|
21
|
+
onestep-0.3.4.dist-info/METADATA,sha256=YkGpJvfEzTwbpV-8mEPdxtrOwsw51cwQ3o5wDpWelI0,2202
|
|
22
|
+
onestep-0.3.4.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
|
|
23
|
+
onestep-0.3.4.dist-info/RECORD,,
|
onestep-0.3.3.dist-info/RECORD
DELETED
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
onestep/__init__.py,sha256=q_4hdD6bfye2IESO2jRCME6L5gprUq9okcaib0PMgsk,1390
|
|
2
|
-
onestep/_utils.py,sha256=ySYmy-o2o3BurvLfuLWKGXR8TLwug6aTDNMvA3_QjKM,414
|
|
3
|
-
onestep/broker/__init__.py,sha256=atvxgYy16nnSjzsXxX4Fm_zoPlqoHJK_INQeYVH8mRE,258
|
|
4
|
-
onestep/broker/base.py,sha256=SVRn_wI_hal6_kHOS3cv0zEgAugTN_QIpPIE4QdVf2k,5063
|
|
5
|
-
onestep/broker/cron.py,sha256=63qubvFabFloRIAqgbYte5tLFi_S7xLzzoi58pQuLwY,1058
|
|
6
|
-
onestep/broker/memory.py,sha256=ILDfYCbMuMmYcZW09jVCdkAg3IQcC-dHkGHIgd8Wh4Q,151
|
|
7
|
-
onestep/broker/rabbitmq.py,sha256=Q7wWvxeVhWczLDF-Z9cdz3SJhbcZhMiTF-kySsWCPps,2798
|
|
8
|
-
onestep/broker/redis.py,sha256=IT5ztmrX4OrzUa1vwFuV2OScRp2fDeVXeNUq1HdbqgM,3345
|
|
9
|
-
onestep/broker/webhook.py,sha256=9GTY0Ttcdk6jF6xnbGtnRKGnRgnuXSN2AHlbREOzpCw,2409
|
|
10
|
-
onestep/exception.py,sha256=YePdLAFg2gHtzOdwmAtKEosdTCxr-xh5ZhRvofGrpDE,610
|
|
11
|
-
onestep/message.py,sha256=0WZ14EsmGaHrRNQ688yZUSpXDuUw2aXtPYc4NpiwZbA,3897
|
|
12
|
-
onestep/middleware/__init__.py,sha256=MP_45lqr4pecmbzQBnn2-AODQ0N_Fss8nl2SA7Zzljo,347
|
|
13
|
-
onestep/middleware/base.py,sha256=adWQ_Lx2Nkaw4ySojIaa3lwUIUu97pHR_l5YMr0lrbw,975
|
|
14
|
-
onestep/middleware/config.py,sha256=WZIvGXhpdSQRAFTbEWXcZdnhFcbvhGkLdkFIj-_QuZM,2438
|
|
15
|
-
onestep/middleware/unique.py,sha256=e3nWHW2yxHFWT9PqiA96sOA1CZZVTbC3bOwUgxN9iaQ,1351
|
|
16
|
-
onestep/onestep.py,sha256=3UjSlf3wyuWsRMwKAypulY9GVwubdpaqgZjKLYzUmho,7330
|
|
17
|
-
onestep/retry.py,sha256=IEqE9zHDEx9FgW1jnv29OdJJx-cVyfX6SZjz6HQR7Ic,2740
|
|
18
|
-
onestep/signal.py,sha256=o8KD58jL6lM35gDHVgr1lEspZqFdkMUNJAD-wuouUVo,293
|
|
19
|
-
onestep/state.py,sha256=UVG91CXCabU64X6qgcO0S7RzbBP8ut0ID7aTMww94us,618
|
|
20
|
-
onestep/worker.py,sha256=zsUe7W3IqN3_jLMMh87m48w3kBugOEJTlCRoPwW_DLA,3831
|
|
21
|
-
onestep-0.3.3.dist-info/METADATA,sha256=cVMaR4sPBdDI61-7Cd8Tgmer8hjQErgh5H9sWUoFKTE,2110
|
|
22
|
-
onestep-0.3.3.dist-info/WHEEL,sha256=d2fvjOD7sXsVzChCqf0Ty0JbHKBaLYwDbGQDwQTnJ50,88
|
|
23
|
-
onestep-0.3.3.dist-info/RECORD,,
|