onestep 0.3.7__py3-none-any.whl → 0.3.9__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of onestep might be problematic. Click here for more details.
- onestep/__init__.py +2 -4
- onestep/broker/__init__.py +2 -2
- onestep/broker/base.py +8 -71
- onestep/broker/cron.py +5 -5
- onestep/broker/memory.py +67 -4
- onestep/broker/rabbitmq.py +25 -18
- onestep/broker/redis/pubsub.py +23 -16
- onestep/broker/redis/stream.py +24 -18
- onestep/broker/webhook.py +4 -4
- onestep/exception.py +8 -0
- onestep/message.py +12 -10
- onestep/onestep.py +1 -0
- onestep/signal.py +1 -0
- onestep/worker.py +7 -3
- {onestep-0.3.7.dist-info → onestep-0.3.9.dist-info}/METADATA +5 -3
- onestep-0.3.9.dist-info/RECORD +27 -0
- {onestep-0.3.7.dist-info → onestep-0.3.9.dist-info}/WHEEL +1 -1
- onestep-0.3.7.dist-info/RECORD +0 -27
- {onestep-0.3.7.dist-info → onestep-0.3.9.dist-info}/entry_points.txt +0 -0
onestep/__init__.py
CHANGED
|
@@ -4,7 +4,7 @@ from .retry import (
|
|
|
4
4
|
NeverRetry, AlwaysRetry, TimesRetry, RetryIfException, AdvancedRetry
|
|
5
5
|
)
|
|
6
6
|
from .broker import (
|
|
7
|
-
BaseBroker, BaseConsumer,
|
|
7
|
+
BaseBroker, BaseConsumer,
|
|
8
8
|
MemoryBroker, RabbitMQBroker, WebHookBroker, CronBroker, RedisStreamBroker, RedisPubSubBroker
|
|
9
9
|
)
|
|
10
10
|
from .middleware import (
|
|
@@ -23,8 +23,6 @@ __all__ = [
|
|
|
23
23
|
# broker
|
|
24
24
|
'BaseBroker',
|
|
25
25
|
'BaseConsumer',
|
|
26
|
-
'BaseLocalBroker',
|
|
27
|
-
'BaseLocalConsumer',
|
|
28
26
|
'MemoryBroker',
|
|
29
27
|
'RabbitMQBroker',
|
|
30
28
|
'WebHookBroker',
|
|
@@ -61,4 +59,4 @@ __all__ = [
|
|
|
61
59
|
'__version__'
|
|
62
60
|
]
|
|
63
61
|
|
|
64
|
-
__version__ = '0.3.
|
|
62
|
+
__version__ = '0.3.9'
|
onestep/broker/__init__.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
from .base import (
|
|
2
|
-
BaseBroker, BaseConsumer
|
|
2
|
+
BaseBroker, BaseConsumer
|
|
3
3
|
)
|
|
4
|
-
from .memory import MemoryBroker
|
|
4
|
+
from .memory import MemoryBroker, MemoryConsumer
|
|
5
5
|
from .webhook import WebHookBroker
|
|
6
6
|
from .rabbitmq import RabbitMQBroker
|
|
7
7
|
from .redis import RedisStreamBroker, RedisPubSubBroker
|
onestep/broker/base.py
CHANGED
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
# -*- coding: utf-8 -*-
|
|
2
2
|
import abc
|
|
3
|
-
import json
|
|
4
3
|
import logging
|
|
5
|
-
from queue import Queue, Empty
|
|
4
|
+
from queue import Queue, Empty
|
|
6
5
|
from typing import Any, Optional, List, Callable
|
|
7
6
|
|
|
8
7
|
from onestep.middleware import BaseMiddleware
|
|
@@ -13,10 +12,11 @@ logger = logging.getLogger(__name__)
|
|
|
13
12
|
|
|
14
13
|
|
|
15
14
|
class BaseBroker:
|
|
15
|
+
message_cls = Message
|
|
16
16
|
|
|
17
17
|
def __init__(self,
|
|
18
18
|
name: Optional[str] = None,
|
|
19
|
-
queue: Optional[
|
|
19
|
+
queue: Optional[Queue] = None,
|
|
20
20
|
middlewares: Optional[List[BaseMiddleware]] = None,
|
|
21
21
|
once: bool = False,
|
|
22
22
|
cancel_consume: Optional[Callable] = None):
|
|
@@ -46,7 +46,7 @@ class BaseBroker:
|
|
|
46
46
|
def send(self, message):
|
|
47
47
|
"""对消息进行预处理,然后再发送"""
|
|
48
48
|
if not isinstance(message, Message):
|
|
49
|
-
message =
|
|
49
|
+
message = self.message_cls(body=message)
|
|
50
50
|
# TODO: 对消息发送进行N次重试,确保消息发送成功。
|
|
51
51
|
return self.publish(message.to_json())
|
|
52
52
|
|
|
@@ -112,80 +112,17 @@ class BaseBroker:
|
|
|
112
112
|
|
|
113
113
|
class BaseConsumer:
|
|
114
114
|
|
|
115
|
-
def __init__(self,
|
|
116
|
-
self.queue = queue
|
|
115
|
+
def __init__(self, broker: BaseBroker, *args, **kwargs):
|
|
116
|
+
self.queue = broker.queue
|
|
117
|
+
self.message_cls = broker.message_cls or Message
|
|
117
118
|
self.timeout = kwargs.pop("timeout", 1000)
|
|
118
119
|
|
|
119
|
-
@abc.abstractmethod
|
|
120
|
-
def _to_message(self, data: Any):
|
|
121
|
-
"""
|
|
122
|
-
转换消息内容到 Message , 则必须实现此方法
|
|
123
|
-
"""
|
|
124
|
-
raise NotImplementedError('Please implement in subclasses.')
|
|
125
|
-
|
|
126
120
|
def __next__(self):
|
|
127
121
|
try:
|
|
128
122
|
data = self.queue.get(timeout=self.timeout / 1000)
|
|
129
|
-
return self.
|
|
123
|
+
return self.message_cls.from_broker(broker_message=data)
|
|
130
124
|
except Empty:
|
|
131
125
|
return None
|
|
132
126
|
|
|
133
127
|
def __iter__(self):
|
|
134
128
|
return self
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
class BaseLocalBroker(BaseBroker):
|
|
138
|
-
|
|
139
|
-
def __init__(self, maxsize=0, *args, **kwargs):
|
|
140
|
-
super().__init__(*args, **kwargs)
|
|
141
|
-
self.queue = Queue(maxsize)
|
|
142
|
-
|
|
143
|
-
def publish(self, message: Any):
|
|
144
|
-
try:
|
|
145
|
-
self.queue.put_nowait(message)
|
|
146
|
-
except FullException:
|
|
147
|
-
logger.warning("CronBroker queue is full, skip this task, "
|
|
148
|
-
"you can increase maxsize with `maxsize` argument")
|
|
149
|
-
|
|
150
|
-
def consume(self, *args, **kwargs):
|
|
151
|
-
return BaseLocalConsumer(self.queue, *args, **kwargs)
|
|
152
|
-
|
|
153
|
-
def confirm(self, message: Message):
|
|
154
|
-
"""确认消息"""
|
|
155
|
-
pass
|
|
156
|
-
|
|
157
|
-
def reject(self, message: Message):
|
|
158
|
-
"""拒绝消息"""
|
|
159
|
-
pass
|
|
160
|
-
|
|
161
|
-
def requeue(self, message: Message, is_source=False):
|
|
162
|
-
"""重发消息:先拒绝 再 重入"""
|
|
163
|
-
if is_source:
|
|
164
|
-
self.publish(message.msg)
|
|
165
|
-
else:
|
|
166
|
-
self.send(message)
|
|
167
|
-
|
|
168
|
-
def __repr__(self):
|
|
169
|
-
return f"<{self.__class__.__name__} {self.name}>"
|
|
170
|
-
|
|
171
|
-
def __str__(self):
|
|
172
|
-
return self.name
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
class BaseLocalConsumer(BaseConsumer):
|
|
176
|
-
|
|
177
|
-
def _to_message(self, data: Any):
|
|
178
|
-
if isinstance(data, (str, bytes, bytearray)):
|
|
179
|
-
try:
|
|
180
|
-
message = json.loads(data)
|
|
181
|
-
except json.JSONDecodeError:
|
|
182
|
-
message = {"body": data}
|
|
183
|
-
else:
|
|
184
|
-
message = data
|
|
185
|
-
if not isinstance(message, dict):
|
|
186
|
-
message = {"body": message}
|
|
187
|
-
if "body" not in message:
|
|
188
|
-
# 来自 外部的消息 可能没有 body, 故直接认为都是 message.body
|
|
189
|
-
message = {"body": message}
|
|
190
|
-
|
|
191
|
-
return Message(body=message.get("body"), extra=message.get("extra"), msg=data)
|
onestep/broker/cron.py
CHANGED
|
@@ -8,12 +8,12 @@ from typing import Any
|
|
|
8
8
|
|
|
9
9
|
from croniter import croniter
|
|
10
10
|
|
|
11
|
-
from .
|
|
11
|
+
from .memory import MemoryBroker, MemoryConsumer
|
|
12
12
|
|
|
13
13
|
logger = logging.getLogger(__name__)
|
|
14
14
|
|
|
15
15
|
|
|
16
|
-
class CronBroker(
|
|
16
|
+
class CronBroker(MemoryBroker):
|
|
17
17
|
_thread = None
|
|
18
18
|
|
|
19
19
|
def __init__(self, cron, name=None, middlewares=None, body: Any = None, *args, **kwargs):
|
|
@@ -31,13 +31,13 @@ class CronBroker(BaseLocalBroker):
|
|
|
31
31
|
self._thread = threading.Timer(interval=1, function=self._scheduler)
|
|
32
32
|
self._thread.start()
|
|
33
33
|
|
|
34
|
-
def consume(self):
|
|
34
|
+
def consume(self, *args, **kwargs):
|
|
35
35
|
self._scheduler()
|
|
36
|
-
return CronConsumer(self
|
|
36
|
+
return CronConsumer(self, *args, **kwargs)
|
|
37
37
|
|
|
38
38
|
def shutdown(self):
|
|
39
39
|
self._thread.cancel()
|
|
40
40
|
|
|
41
41
|
|
|
42
|
-
class CronConsumer(
|
|
42
|
+
class CronConsumer(MemoryConsumer):
|
|
43
43
|
...
|
onestep/broker/memory.py
CHANGED
|
@@ -1,9 +1,72 @@
|
|
|
1
|
-
|
|
1
|
+
import logging
|
|
2
|
+
import json
|
|
3
|
+
from queue import Queue, Full as FullException
|
|
4
|
+
from typing import Any
|
|
2
5
|
|
|
6
|
+
from .base import BaseBroker, BaseConsumer
|
|
3
7
|
|
|
4
|
-
|
|
5
|
-
|
|
8
|
+
from ..message import Message
|
|
9
|
+
|
|
10
|
+
logger = logging.getLogger(__name__)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class MemoryMessage(Message):
|
|
14
|
+
|
|
15
|
+
@classmethod
|
|
16
|
+
def from_broker(cls, broker_message: Any):
|
|
17
|
+
if isinstance(broker_message, (str, bytes, bytearray)):
|
|
18
|
+
try:
|
|
19
|
+
message = json.loads(broker_message)
|
|
20
|
+
except json.JSONDecodeError:
|
|
21
|
+
message = {"body": broker_message}
|
|
22
|
+
else:
|
|
23
|
+
message = broker_message
|
|
24
|
+
if not isinstance(message, dict):
|
|
25
|
+
message = {"body": message}
|
|
26
|
+
if "body" not in message:
|
|
27
|
+
# 来自 外部的消息 可能没有 body, 故直接认为都是 message.body
|
|
28
|
+
message = {"body": message}
|
|
29
|
+
return cls(body=message.get("body"), extra=message.get("extra"), message=broker_message)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class MemoryBroker(BaseBroker):
|
|
33
|
+
message_cls = MemoryMessage
|
|
34
|
+
|
|
35
|
+
def __init__(self, maxsize=0, *args, **kwargs):
|
|
36
|
+
super().__init__(*args, **kwargs)
|
|
37
|
+
self.queue = Queue(maxsize)
|
|
38
|
+
|
|
39
|
+
def publish(self, message: Any):
|
|
40
|
+
try:
|
|
41
|
+
self.queue.put_nowait(message)
|
|
42
|
+
except FullException:
|
|
43
|
+
logger.warning("CronBroker queue is full, skip this task, "
|
|
44
|
+
"you can increase maxsize with `maxsize` argument")
|
|
45
|
+
|
|
46
|
+
def consume(self, *args, **kwargs):
|
|
47
|
+
return MemoryConsumer(self, *args, **kwargs)
|
|
48
|
+
|
|
49
|
+
def confirm(self, message: Message):
|
|
50
|
+
"""确认消息"""
|
|
51
|
+
pass
|
|
52
|
+
|
|
53
|
+
def reject(self, message: Message):
|
|
54
|
+
"""拒绝消息"""
|
|
55
|
+
pass
|
|
56
|
+
|
|
57
|
+
def requeue(self, message: Message, is_source=False):
|
|
58
|
+
"""重发消息:先拒绝 再 重入"""
|
|
59
|
+
if is_source:
|
|
60
|
+
self.publish(message.message)
|
|
61
|
+
else:
|
|
62
|
+
self.send(message)
|
|
63
|
+
|
|
64
|
+
def __repr__(self):
|
|
65
|
+
return f"<{self.__class__.__name__} {self.name}>"
|
|
66
|
+
|
|
67
|
+
def __str__(self):
|
|
68
|
+
return self.name
|
|
6
69
|
|
|
7
70
|
|
|
8
|
-
class MemoryConsumer(
|
|
71
|
+
class MemoryConsumer(BaseConsumer):
|
|
9
72
|
...
|
onestep/broker/rabbitmq.py
CHANGED
|
@@ -6,11 +6,29 @@ from typing import Optional, Dict, Any
|
|
|
6
6
|
import amqpstorm
|
|
7
7
|
|
|
8
8
|
from .base import BaseBroker, BaseConsumer
|
|
9
|
-
from
|
|
9
|
+
from use_rabbitmq import useRabbitMQ as RabbitMQStore
|
|
10
10
|
from ..message import Message
|
|
11
11
|
|
|
12
12
|
|
|
13
|
+
class _RabbitMQMessage(Message):
|
|
14
|
+
|
|
15
|
+
@classmethod
|
|
16
|
+
def from_broker(cls, broker_message: amqpstorm.Message):
|
|
17
|
+
try:
|
|
18
|
+
message = json.loads(broker_message.body)
|
|
19
|
+
except json.JSONDecodeError:
|
|
20
|
+
message = {"body": broker_message.body}
|
|
21
|
+
if not isinstance(message, dict):
|
|
22
|
+
message = {"body": message}
|
|
23
|
+
if "body" not in message:
|
|
24
|
+
# 来自 外部的消息 可能没有 body, 故直接认为都是 message.body
|
|
25
|
+
message = {"body": message}
|
|
26
|
+
|
|
27
|
+
return cls(body=message.get("body"), extra=message.get("extra"), message=broker_message)
|
|
28
|
+
|
|
29
|
+
|
|
13
30
|
class RabbitMQBroker(BaseBroker):
|
|
31
|
+
message_cls = _RabbitMQMessage
|
|
14
32
|
|
|
15
33
|
def __init__(self, queue_name, params: Optional[Dict] = None, prefetch: Optional[int] = 1, auto_create=True, *args,
|
|
16
34
|
**kwargs):
|
|
@@ -37,18 +55,18 @@ class RabbitMQBroker(BaseBroker):
|
|
|
37
55
|
thread.daemon = daemon
|
|
38
56
|
thread.start()
|
|
39
57
|
self.threads.append(thread)
|
|
40
|
-
return RabbitMQConsumer(self
|
|
58
|
+
return RabbitMQConsumer(self)
|
|
41
59
|
|
|
42
60
|
def publish(self, message: Any):
|
|
43
61
|
self.client.send(self.queue_name, message)
|
|
44
62
|
|
|
45
63
|
def confirm(self, message: Message):
|
|
46
64
|
"""确认消息"""
|
|
47
|
-
message.
|
|
65
|
+
message.message.ack()
|
|
48
66
|
|
|
49
67
|
def reject(self, message: Message):
|
|
50
68
|
"""拒绝消息"""
|
|
51
|
-
message.
|
|
69
|
+
message.message.reject(requeue=False)
|
|
52
70
|
|
|
53
71
|
def requeue(self, message: Message, is_source=False):
|
|
54
72
|
"""
|
|
@@ -58,9 +76,9 @@ class RabbitMQBroker(BaseBroker):
|
|
|
58
76
|
:param is_source: 是否是原始消息,True: 使用原始消息重入当前队列,False: 使用消息的最新数据重入当前队列
|
|
59
77
|
"""
|
|
60
78
|
if is_source:
|
|
61
|
-
message.
|
|
79
|
+
message.message.reject(requeue=True)
|
|
62
80
|
else:
|
|
63
|
-
message.
|
|
81
|
+
message.message.reject(requeue=False)
|
|
64
82
|
self.send(message)
|
|
65
83
|
|
|
66
84
|
def shutdown(self):
|
|
@@ -70,15 +88,4 @@ class RabbitMQBroker(BaseBroker):
|
|
|
70
88
|
|
|
71
89
|
|
|
72
90
|
class RabbitMQConsumer(BaseConsumer):
|
|
73
|
-
|
|
74
|
-
try:
|
|
75
|
-
message = json.loads(data.body)
|
|
76
|
-
except json.JSONDecodeError:
|
|
77
|
-
message = {"body": data.body}
|
|
78
|
-
if not isinstance(message, dict):
|
|
79
|
-
message = {"body": message}
|
|
80
|
-
if "body" not in message:
|
|
81
|
-
# 来自 外部的消息 可能没有 body, 故直接认为都是 message.body
|
|
82
|
-
message = {"body": message}
|
|
83
|
-
|
|
84
|
-
return Message(body=message.get("body"), extra=message.get("extra"), msg=data)
|
|
91
|
+
...
|
onestep/broker/redis/pubsub.py
CHANGED
|
@@ -4,15 +4,32 @@ from queue import Queue
|
|
|
4
4
|
from typing import Any
|
|
5
5
|
|
|
6
6
|
try:
|
|
7
|
-
from
|
|
7
|
+
from use_redis import useRedis
|
|
8
8
|
except ImportError:
|
|
9
9
|
...
|
|
10
10
|
|
|
11
11
|
from ..base import BaseBroker, BaseConsumer, Message
|
|
12
12
|
|
|
13
13
|
|
|
14
|
+
class _RedisPubSubMessage(Message):
|
|
15
|
+
|
|
16
|
+
@classmethod
|
|
17
|
+
def from_broker(cls, broker_message: Any):
|
|
18
|
+
if "channel" in broker_message:
|
|
19
|
+
try:
|
|
20
|
+
message = json.loads(broker_message.get("data")) # 已转换的 message
|
|
21
|
+
except (json.JSONDecodeError, TypeError):
|
|
22
|
+
message = {"body": broker_message.get("data")} # 未转换的 message
|
|
23
|
+
else:
|
|
24
|
+
# 来自 外部的消息 直接认为都是 message.body
|
|
25
|
+
message = {"body": broker_message.body}
|
|
26
|
+
|
|
27
|
+
yield cls(body=message.get("body"), extra=message.get("extra"), message=broker_message)
|
|
28
|
+
|
|
29
|
+
|
|
14
30
|
class RedisPubSubBroker(BaseBroker):
|
|
15
31
|
""" Redis PubSub Broker """
|
|
32
|
+
message_cls = _RedisPubSubMessage
|
|
16
33
|
|
|
17
34
|
def __init__(self, channel: str, *args, **kwargs):
|
|
18
35
|
super().__init__(*args, **kwargs)
|
|
@@ -40,14 +57,14 @@ class RedisPubSubBroker(BaseBroker):
|
|
|
40
57
|
thread.daemon = daemon
|
|
41
58
|
thread.start()
|
|
42
59
|
self.threads.append(thread)
|
|
43
|
-
return RedisPubSubConsumer(self
|
|
60
|
+
return RedisPubSubConsumer(self)
|
|
44
61
|
|
|
45
62
|
def send(self, message: Any):
|
|
46
63
|
"""Publish message to the Redis channel"""
|
|
47
64
|
if not isinstance(message, Message):
|
|
48
|
-
message =
|
|
65
|
+
message = self.message_cls(body=message)
|
|
49
66
|
|
|
50
|
-
|
|
67
|
+
self.client.publish(self.channel, message.to_json())
|
|
51
68
|
|
|
52
69
|
publish = send
|
|
53
70
|
|
|
@@ -67,20 +84,10 @@ class RedisPubSubBroker(BaseBroker):
|
|
|
67
84
|
self.reject(message)
|
|
68
85
|
|
|
69
86
|
if is_source:
|
|
70
|
-
self.client.publish(self.channel, message.
|
|
87
|
+
self.client.publish(self.channel, message.message['data'])
|
|
71
88
|
else:
|
|
72
89
|
self.send(message)
|
|
73
90
|
|
|
74
91
|
|
|
75
92
|
class RedisPubSubConsumer(BaseConsumer):
|
|
76
|
-
|
|
77
|
-
if "channel" in data:
|
|
78
|
-
try:
|
|
79
|
-
message = json.loads(data.get("data")) # 已转换的 message
|
|
80
|
-
except (json.JSONDecodeError, TypeError):
|
|
81
|
-
message = {"body": data.get("data")} # 未转换的 message
|
|
82
|
-
else:
|
|
83
|
-
# 来自 外部的消息 直接认为都是 message.body
|
|
84
|
-
message = {"body": data.body}
|
|
85
|
-
|
|
86
|
-
yield Message(body=message.get("body"), extra=message.get("extra"), msg=data)
|
|
93
|
+
...
|
onestep/broker/redis/stream.py
CHANGED
|
@@ -5,15 +5,32 @@ from queue import Queue
|
|
|
5
5
|
from typing import Optional, Dict, Any
|
|
6
6
|
|
|
7
7
|
try:
|
|
8
|
-
from
|
|
8
|
+
from use_redis import useRedisStreamStore, RedisStreamMessage
|
|
9
9
|
except ImportError:
|
|
10
10
|
...
|
|
11
11
|
|
|
12
12
|
from ..base import BaseBroker, BaseConsumer, Message
|
|
13
13
|
|
|
14
14
|
|
|
15
|
+
class _RedisStreamMessage(Message):
|
|
16
|
+
@classmethod
|
|
17
|
+
def from_broker(cls, broker_message: "RedisStreamMessage"):
|
|
18
|
+
if "_message" in broker_message.body:
|
|
19
|
+
# 来自 RedisStreamBroker.send 的消息,message.body 默认是存于 _message 字段中
|
|
20
|
+
try:
|
|
21
|
+
message = json.loads(broker_message.body.get("_message")) # 已转换的 message
|
|
22
|
+
except (json.JSONDecodeError, TypeError):
|
|
23
|
+
message = {"body": broker_message.body.get("_message")} # 未转换的 message
|
|
24
|
+
else:
|
|
25
|
+
# 来自 外部的消息 直接认为都是 message.body
|
|
26
|
+
message = {"body": broker_message.body}
|
|
27
|
+
|
|
28
|
+
yield cls(body=message.get("body"), extra=message.get("extra"), message=broker_message)
|
|
29
|
+
|
|
30
|
+
|
|
15
31
|
class RedisStreamBroker(BaseBroker):
|
|
16
32
|
""" Redis Stream Broker """
|
|
33
|
+
message_cls = _RedisStreamMessage
|
|
17
34
|
|
|
18
35
|
def __init__(
|
|
19
36
|
self,
|
|
@@ -56,22 +73,22 @@ class RedisStreamBroker(BaseBroker):
|
|
|
56
73
|
thread.daemon = daemon
|
|
57
74
|
thread.start()
|
|
58
75
|
self.threads.append(thread)
|
|
59
|
-
return RedisStreamConsumer(self
|
|
76
|
+
return RedisStreamConsumer(self)
|
|
60
77
|
|
|
61
78
|
def send(self, message: Any):
|
|
62
79
|
"""对消息进行预处理,然后再发送"""
|
|
63
80
|
if not isinstance(message, Message):
|
|
64
|
-
message =
|
|
81
|
+
message = self.message_cls(body=message)
|
|
65
82
|
|
|
66
83
|
self.client.send({"_message": message.to_json()})
|
|
67
84
|
|
|
68
85
|
publish = send
|
|
69
86
|
|
|
70
87
|
def confirm(self, message: Message):
|
|
71
|
-
self.client.ack(message.
|
|
88
|
+
self.client.ack(message.message)
|
|
72
89
|
|
|
73
90
|
def reject(self, message: Message):
|
|
74
|
-
self.client.reject(message.
|
|
91
|
+
self.client.reject(message.message)
|
|
75
92
|
|
|
76
93
|
def requeue(self, message: Message, is_source=False):
|
|
77
94
|
"""
|
|
@@ -83,21 +100,10 @@ class RedisStreamBroker(BaseBroker):
|
|
|
83
100
|
self.reject(message)
|
|
84
101
|
|
|
85
102
|
if is_source:
|
|
86
|
-
self.client.send(message.
|
|
103
|
+
self.client.send(message.message.body)
|
|
87
104
|
else:
|
|
88
105
|
self.send(message)
|
|
89
106
|
|
|
90
107
|
|
|
91
108
|
class RedisStreamConsumer(BaseConsumer):
|
|
92
|
-
|
|
93
|
-
if "_message" in data.body:
|
|
94
|
-
# 来自 RedisStreamBroker.send 的消息,message.body 默认是存于 _message 字段中
|
|
95
|
-
try:
|
|
96
|
-
message = json.loads(data.body.get("_message")) # 已转换的 message
|
|
97
|
-
except (json.JSONDecodeError, TypeError):
|
|
98
|
-
message = {"body": data.body.get("_message")} # 未转换的 message
|
|
99
|
-
else:
|
|
100
|
-
# 来自 外部的消息 直接认为都是 message.body
|
|
101
|
-
message = {"body": data.body}
|
|
102
|
-
|
|
103
|
-
yield Message(body=message.get("body"), extra=message.get("extra"), msg=data)
|
|
109
|
+
...
|
onestep/broker/webhook.py
CHANGED
|
@@ -3,7 +3,7 @@ import threading
|
|
|
3
3
|
import collections
|
|
4
4
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
5
5
|
|
|
6
|
-
from .
|
|
6
|
+
from .memory import MemoryBroker, MemoryConsumer
|
|
7
7
|
|
|
8
8
|
logger = logging.getLogger(__name__)
|
|
9
9
|
|
|
@@ -34,7 +34,7 @@ class WebHookServer(BaseHTTPRequestHandler):
|
|
|
34
34
|
self.wfile.write(b'{ "status": "ok" }')
|
|
35
35
|
|
|
36
36
|
|
|
37
|
-
class WebHookBroker(
|
|
37
|
+
class WebHookBroker(MemoryBroker):
|
|
38
38
|
_servers = {}
|
|
39
39
|
|
|
40
40
|
def __init__(self,
|
|
@@ -68,7 +68,7 @@ class WebHookBroker(BaseLocalBroker):
|
|
|
68
68
|
def consume(self, *args, **kwargs):
|
|
69
69
|
self._create_server()
|
|
70
70
|
logger.debug(f"WebHookBroker: {self.host}:{self.port}{self.path}")
|
|
71
|
-
return WebHookConsumer(self
|
|
71
|
+
return WebHookConsumer(self, *args, **kwargs)
|
|
72
72
|
|
|
73
73
|
def shutdown(self):
|
|
74
74
|
hs = self._servers[(self.host, self.port)]
|
|
@@ -78,5 +78,5 @@ class WebHookBroker(BaseLocalBroker):
|
|
|
78
78
|
thread.join()
|
|
79
79
|
|
|
80
80
|
|
|
81
|
-
class WebHookConsumer(
|
|
81
|
+
class WebHookConsumer(MemoryConsumer):
|
|
82
82
|
...
|
onestep/exception.py
CHANGED
onestep/message.py
CHANGED
|
@@ -42,12 +42,19 @@ class Message:
|
|
|
42
42
|
self,
|
|
43
43
|
body: Optional[Union[dict, Any]] = None,
|
|
44
44
|
extra: Optional[Union[dict, Extra]] = None,
|
|
45
|
-
|
|
45
|
+
message: Optional[Any] = None,
|
|
46
46
|
broker=None
|
|
47
47
|
):
|
|
48
|
+
""" Message
|
|
49
|
+
|
|
50
|
+
:param body: 解析后的消息体
|
|
51
|
+
:param extra: 额外信息
|
|
52
|
+
:param message: 原始消息体
|
|
53
|
+
:param broker: 当前消息所属的 broker
|
|
54
|
+
"""
|
|
48
55
|
self.body = body
|
|
49
56
|
self.extra = self._set_extra(extra)
|
|
50
|
-
self.
|
|
57
|
+
self.message = message
|
|
51
58
|
|
|
52
59
|
self.broker = broker
|
|
53
60
|
self._exception = None
|
|
@@ -134,11 +141,6 @@ class Message:
|
|
|
134
141
|
def __repr__(self):
|
|
135
142
|
return f"<{self.__class__.__name__} {self.body}>"
|
|
136
143
|
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
msg.x = 1
|
|
141
|
-
print(msg.x)
|
|
142
|
-
msg.qq1 = 1
|
|
143
|
-
del msg.qq1
|
|
144
|
-
print(msg.qq1)
|
|
144
|
+
@classmethod
|
|
145
|
+
def from_broker(cls, broker_message: Any):
|
|
146
|
+
return cls(body=broker_message, extra=None, message=broker_message, broker=None)
|
onestep/onestep.py
CHANGED
onestep/signal.py
CHANGED
onestep/worker.py
CHANGED
|
@@ -14,8 +14,8 @@ from asgiref.sync import async_to_sync
|
|
|
14
14
|
from .message import Message
|
|
15
15
|
from .retry import RetryStatus
|
|
16
16
|
from .broker import BaseBroker
|
|
17
|
-
from .
|
|
18
|
-
from .signal import message_received, message_consumed, message_error, message_drop
|
|
17
|
+
from . import exception
|
|
18
|
+
from .signal import message_received, message_consumed, message_error, message_drop, message_requeue
|
|
19
19
|
|
|
20
20
|
logger = logging.getLogger(__name__)
|
|
21
21
|
|
|
@@ -82,10 +82,14 @@ class BaseWorker:
|
|
|
82
82
|
message.confirm()
|
|
83
83
|
|
|
84
84
|
self.instance.after_emit("consume", message=message)
|
|
85
|
-
except DropMessage as e:
|
|
85
|
+
except (exception.DropMessage, exception.RejectMessage) as e:
|
|
86
86
|
message_drop.send(self, message=message, reason=e)
|
|
87
87
|
logger.warning(f"{self.instance.name} dropped <{type(e).__name__}: {str(e)}>")
|
|
88
88
|
message.reject()
|
|
89
|
+
except exception.RequeueMessage as e:
|
|
90
|
+
message_requeue.send(self, message=message, reason=e)
|
|
91
|
+
logger.warning(f"{self.instance.name} requeue <{type(e).__name__}: {str(e)}>")
|
|
92
|
+
message.requeue(is_source=True)
|
|
89
93
|
except Exception as e:
|
|
90
94
|
message_error.send(self, message=message, error=e)
|
|
91
95
|
if self.instance.state.debug:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: onestep
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.9
|
|
4
4
|
Summary:
|
|
5
5
|
Author: miclon
|
|
6
6
|
Author-email: jcnd@163.com
|
|
@@ -15,8 +15,8 @@ Provides-Extra: redis
|
|
|
15
15
|
Requires-Dist: asgiref (>=3.6.0,<4.0.0)
|
|
16
16
|
Requires-Dist: blinker (>=1.5,<2.0)
|
|
17
17
|
Requires-Dist: croniter (>=1.3.8,<2.0.0)
|
|
18
|
-
Requires-Dist:
|
|
19
|
-
Requires-Dist:
|
|
18
|
+
Requires-Dist: use-rabbitmq (>=0.1.6,<0.2.0)
|
|
19
|
+
Requires-Dist: use-redis (>=0.1.6,<0.2.0) ; extra == "redis"
|
|
20
20
|
Description-Content-Type: text/markdown
|
|
21
21
|
|
|
22
22
|
<div align=center><img src="https://onestep.code05.com/logo-3.svg" width="300"></div>
|
|
@@ -43,6 +43,8 @@ Description-Content-Type: text/markdown
|
|
|
43
43
|
- [x] WebHookBroker
|
|
44
44
|
- [x] RabbitMQBroker
|
|
45
45
|
- [x] RedisBroker
|
|
46
|
+
- [x] RedisStreamBroker
|
|
47
|
+
- [x] RedisPubSubBroker
|
|
46
48
|
- [ ] KafkaBroker
|
|
47
49
|
|
|
48
50
|
## 😋example
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
onestep/__init__.py,sha256=i2ggViIkU4GyPITpeRDv88PSMfqxXdWnyGpCwtc3I6c,1417
|
|
2
|
+
onestep/_utils.py,sha256=ySYmy-o2o3BurvLfuLWKGXR8TLwug6aTDNMvA3_QjKM,414
|
|
3
|
+
onestep/broker/__init__.py,sha256=L-roNwPZVKsjytOlKCOlyAyvfejI28FyOj9uzfyXcrk,257
|
|
4
|
+
onestep/broker/base.py,sha256=Ryo-0YekdZfpfSSqdmh7jsmNONc_GmlYFYadt_n5bds,4141
|
|
5
|
+
onestep/broker/cron.py,sha256=wKOm4k0IBPgZIq5gHC6ITVFUZwgCQ91AXzgL1eFT1WE,1135
|
|
6
|
+
onestep/broker/memory.py,sha256=X7DDiEOiJE3WtocEC_NtSNfL8Jpo7BVQVSG_BJfNGNQ,2053
|
|
7
|
+
onestep/broker/rabbitmq.py,sha256=LFlcpOV36D6bme5nUVsZ3WzJVFQazlKDKBcA9aQONRM,2928
|
|
8
|
+
onestep/broker/redis/__init__.py,sha256=lqjvBKpMGf6I34nyJTEAq7XU2a8aQemhMGCbb5aByFI,236
|
|
9
|
+
onestep/broker/redis/pubsub.py,sha256=_p2bKqXu0gYjKz2DsS8UdpkN5dCy94xf_xYx5NFuKpY,2674
|
|
10
|
+
onestep/broker/redis/stream.py,sha256=uQJncD93Fz8pdP_NdgyQ1UAXwIouh7AFA9_naXaeiJM,3470
|
|
11
|
+
onestep/broker/webhook.py,sha256=Zg22QxKPqJzdxjBsglUcL7P1RcM8FltcHpHXJiJPYgM,2392
|
|
12
|
+
onestep/cli.py,sha256=KfJFpccpfNrhULJmDymrW5bFBGXapdL57ESZdgaFbG8,1399
|
|
13
|
+
onestep/exception.py,sha256=T-tqfRrJZT9Y85qe-1l-1PcHZzV2jxv-61HAhOc4AKE,753
|
|
14
|
+
onestep/message.py,sha256=W37HKnIDxpd5UmtDbPTw2DTb3lfxu9Vs2KYIiCr5b2s,4057
|
|
15
|
+
onestep/middleware/__init__.py,sha256=MP_45lqr4pecmbzQBnn2-AODQ0N_Fss8nl2SA7Zzljo,347
|
|
16
|
+
onestep/middleware/base.py,sha256=adWQ_Lx2Nkaw4ySojIaa3lwUIUu97pHR_l5YMr0lrbw,975
|
|
17
|
+
onestep/middleware/config.py,sha256=WZIvGXhpdSQRAFTbEWXcZdnhFcbvhGkLdkFIj-_QuZM,2438
|
|
18
|
+
onestep/middleware/unique.py,sha256=e3nWHW2yxHFWT9PqiA96sOA1CZZVTbC3bOwUgxN9iaQ,1351
|
|
19
|
+
onestep/onestep.py,sha256=IQGOfhfbneXCaINqMzO-5wjWpmE5dISEZQtmG3bZ3EU,9372
|
|
20
|
+
onestep/retry.py,sha256=_bl_-0FEMai9KLZ0DkamYwUoUVPf6BjUDpn8SrfqQV4,3009
|
|
21
|
+
onestep/signal.py,sha256=tz9bGzTFZeBviG_iaGLfy4OyFKlWat6szEI6kWEGfTA,337
|
|
22
|
+
onestep/state.py,sha256=UVG91CXCabU64X6qgcO0S7RzbBP8ut0ID7aTMww94us,618
|
|
23
|
+
onestep/worker.py,sha256=A-XKvWz7ceRodMItjHKSN9DVTk3OoA0jsfmmBZ9z19U,6449
|
|
24
|
+
onestep-0.3.9.dist-info/METADATA,sha256=vHgxQhvMs0XjQ1JSKL0AjUddc0XPmk8tXlomZ51isuY,2522
|
|
25
|
+
onestep-0.3.9.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
|
|
26
|
+
onestep-0.3.9.dist-info/entry_points.txt,sha256=ZfWnNQqiGujz2PPLjSlKPocOFRryL7Ot0vQ41TU1xw0,44
|
|
27
|
+
onestep-0.3.9.dist-info/RECORD,,
|
onestep-0.3.7.dist-info/RECORD
DELETED
|
@@ -1,27 +0,0 @@
|
|
|
1
|
-
onestep/__init__.py,sha256=92LmJJSTLJrW9932IvoDT3YOPPCbGaqBSJClOwofxEw,1501
|
|
2
|
-
onestep/_utils.py,sha256=ySYmy-o2o3BurvLfuLWKGXR8TLwug6aTDNMvA3_QjKM,414
|
|
3
|
-
onestep/broker/__init__.py,sha256=0_qWPSIz09GSJutUuCcjHkfdqzLiP75j9Wdw0BOSvGQ,277
|
|
4
|
-
onestep/broker/base.py,sha256=NeLhGe70y6anToGnoc0OP9OrZ4t30arr6o6TgqtoEWc,5970
|
|
5
|
-
onestep/broker/cron.py,sha256=L8vC6NzSs4Wd_pH2zEpw9udOESf166zPiwU0YLhlUF4,1117
|
|
6
|
-
onestep/broker/memory.py,sha256=ILDfYCbMuMmYcZW09jVCdkAg3IQcC-dHkGHIgd8Wh4Q,151
|
|
7
|
-
onestep/broker/rabbitmq.py,sha256=7xo8wGbFTD-6fpzP0lwh9uZIcdDh5ClioXBx9vXxLy0,2792
|
|
8
|
-
onestep/broker/redis/__init__.py,sha256=lqjvBKpMGf6I34nyJTEAq7XU2a8aQemhMGCbb5aByFI,236
|
|
9
|
-
onestep/broker/redis/pubsub.py,sha256=Glbabpm0LeVlqlNL_i9Jem6K8Jc0IFxhYaWjet05XzM,2517
|
|
10
|
-
onestep/broker/redis/stream.py,sha256=MMLcw_ekKbYMePQ1s02NW1_znKTDWcOWpQDxEjnZ-0o,3304
|
|
11
|
-
onestep/broker/webhook.py,sha256=fqCLFtkUgsonWZBn9SKMKo5QItJsm8magB6P-sdwT2U,2408
|
|
12
|
-
onestep/cli.py,sha256=KfJFpccpfNrhULJmDymrW5bFBGXapdL57ESZdgaFbG8,1399
|
|
13
|
-
onestep/exception.py,sha256=5nd2iD2tWpewxLs4UKhxBCdN5cSiCwC1pO-Lpwm-0Dw,608
|
|
14
|
-
onestep/message.py,sha256=KbvqfqYugg1vzWocrQ7_eGJMGtyCxEbtraddZhJ-jr8,3819
|
|
15
|
-
onestep/middleware/__init__.py,sha256=MP_45lqr4pecmbzQBnn2-AODQ0N_Fss8nl2SA7Zzljo,347
|
|
16
|
-
onestep/middleware/base.py,sha256=adWQ_Lx2Nkaw4ySojIaa3lwUIUu97pHR_l5YMr0lrbw,975
|
|
17
|
-
onestep/middleware/config.py,sha256=WZIvGXhpdSQRAFTbEWXcZdnhFcbvhGkLdkFIj-_QuZM,2438
|
|
18
|
-
onestep/middleware/unique.py,sha256=e3nWHW2yxHFWT9PqiA96sOA1CZZVTbC3bOwUgxN9iaQ,1351
|
|
19
|
-
onestep/onestep.py,sha256=cUuBSHJPucXFu86wxDFkXeetBJS7Fxu9r1c3OFfsgS4,9329
|
|
20
|
-
onestep/retry.py,sha256=_bl_-0FEMai9KLZ0DkamYwUoUVPf6BjUDpn8SrfqQV4,3009
|
|
21
|
-
onestep/signal.py,sha256=o8KD58jL6lM35gDHVgr1lEspZqFdkMUNJAD-wuouUVo,293
|
|
22
|
-
onestep/state.py,sha256=UVG91CXCabU64X6qgcO0S7RzbBP8ut0ID7aTMww94us,618
|
|
23
|
-
onestep/worker.py,sha256=sHA_XQs5EEyzsgy42m--jUPSkhcBV40OZ31_HCpMAMU,6159
|
|
24
|
-
onestep-0.3.7.dist-info/METADATA,sha256=FCZlOp4kHkm4E4OSs6I03yROuW2hF0CrT7DOECULPHI,2483
|
|
25
|
-
onestep-0.3.7.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
|
|
26
|
-
onestep-0.3.7.dist-info/entry_points.txt,sha256=ZfWnNQqiGujz2PPLjSlKPocOFRryL7Ot0vQ41TU1xw0,44
|
|
27
|
-
onestep-0.3.7.dist-info/RECORD,,
|
|
File without changes
|