onestep 0.3.7__tar.gz → 0.3.9__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of onestep might be problematic. Click here for more details.

Files changed (27) hide show
  1. {onestep-0.3.7 → onestep-0.3.9}/PKG-INFO +5 -3
  2. {onestep-0.3.7 → onestep-0.3.9}/README.md +2 -0
  3. {onestep-0.3.7 → onestep-0.3.9}/pyproject.toml +12 -4
  4. {onestep-0.3.7 → onestep-0.3.9}/src/onestep/__init__.py +2 -4
  5. {onestep-0.3.7 → onestep-0.3.9}/src/onestep/broker/__init__.py +2 -2
  6. {onestep-0.3.7 → onestep-0.3.9}/src/onestep/broker/base.py +8 -71
  7. {onestep-0.3.7 → onestep-0.3.9}/src/onestep/broker/cron.py +5 -5
  8. onestep-0.3.9/src/onestep/broker/memory.py +72 -0
  9. {onestep-0.3.7 → onestep-0.3.9}/src/onestep/broker/rabbitmq.py +25 -18
  10. {onestep-0.3.7 → onestep-0.3.9}/src/onestep/broker/redis/pubsub.py +23 -16
  11. {onestep-0.3.7 → onestep-0.3.9}/src/onestep/broker/redis/stream.py +24 -18
  12. {onestep-0.3.7 → onestep-0.3.9}/src/onestep/broker/webhook.py +4 -4
  13. {onestep-0.3.7 → onestep-0.3.9}/src/onestep/exception.py +8 -0
  14. {onestep-0.3.7 → onestep-0.3.9}/src/onestep/message.py +12 -10
  15. {onestep-0.3.7 → onestep-0.3.9}/src/onestep/onestep.py +1 -0
  16. {onestep-0.3.7 → onestep-0.3.9}/src/onestep/signal.py +1 -0
  17. {onestep-0.3.7 → onestep-0.3.9}/src/onestep/worker.py +7 -3
  18. onestep-0.3.7/src/onestep/broker/memory.py +0 -9
  19. {onestep-0.3.7 → onestep-0.3.9}/src/onestep/_utils.py +0 -0
  20. {onestep-0.3.7 → onestep-0.3.9}/src/onestep/broker/redis/__init__.py +0 -0
  21. {onestep-0.3.7 → onestep-0.3.9}/src/onestep/cli.py +0 -0
  22. {onestep-0.3.7 → onestep-0.3.9}/src/onestep/middleware/__init__.py +0 -0
  23. {onestep-0.3.7 → onestep-0.3.9}/src/onestep/middleware/base.py +0 -0
  24. {onestep-0.3.7 → onestep-0.3.9}/src/onestep/middleware/config.py +0 -0
  25. {onestep-0.3.7 → onestep-0.3.9}/src/onestep/middleware/unique.py +0 -0
  26. {onestep-0.3.7 → onestep-0.3.9}/src/onestep/retry.py +0 -0
  27. {onestep-0.3.7 → onestep-0.3.9}/src/onestep/state.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: onestep
3
- Version: 0.3.7
3
+ Version: 0.3.9
4
4
  Summary:
5
5
  Author: miclon
6
6
  Author-email: jcnd@163.com
@@ -15,8 +15,8 @@ Provides-Extra: redis
15
15
  Requires-Dist: asgiref (>=3.6.0,<4.0.0)
16
16
  Requires-Dist: blinker (>=1.5,<2.0)
17
17
  Requires-Dist: croniter (>=1.3.8,<2.0.0)
18
- Requires-Dist: usepy-plugin-rabbitmq (>=0.1.0,<0.2.0)
19
- Requires-Dist: usepy-plugin-redis (>=0.1.6,<0.2.0) ; extra == "redis"
18
+ Requires-Dist: use-rabbitmq (>=0.1.6,<0.2.0)
19
+ Requires-Dist: use-redis (>=0.1.6,<0.2.0) ; extra == "redis"
20
20
  Description-Content-Type: text/markdown
21
21
 
22
22
  <div align=center><img src="https://onestep.code05.com/logo-3.svg" width="300"></div>
@@ -43,6 +43,8 @@ Description-Content-Type: text/markdown
43
43
  - [x] WebHookBroker
44
44
  - [x] RabbitMQBroker
45
45
  - [x] RedisBroker
46
+ - [x] RedisStreamBroker
47
+ - [x] RedisPubSubBroker
46
48
  - [ ] KafkaBroker
47
49
 
48
50
  ## 😋example
@@ -22,6 +22,8 @@
22
22
  - [x] WebHookBroker
23
23
  - [x] RabbitMQBroker
24
24
  - [x] RedisBroker
25
+ - [x] RedisStreamBroker
26
+ - [x] RedisPubSubBroker
25
27
  - [ ] KafkaBroker
26
28
 
27
29
  ## 😋example
@@ -1,6 +1,6 @@
1
1
  [tool.poetry]
2
2
  name = "onestep"
3
- version = "0.3.7"
3
+ version = "0.3.9"
4
4
  description = ""
5
5
  authors = ["miclon <jcnd@163.com>"]
6
6
  readme = "README.md"
@@ -17,11 +17,11 @@ python = "^3.8"
17
17
  asgiref = "^3.6.0"
18
18
  blinker = "^1.5"
19
19
  croniter = "^1.3.8"
20
- usepy-plugin-rabbitmq = "^0.1.0"
21
- usepy-plugin-redis = { version = "^0.1.6", optional = true }
20
+ use-rabbitmq = "^0.1.6"
21
+ use-redis = { version = "^0.1.6", optional = true }
22
22
 
23
23
  [tool.poetry.extras]
24
- redis = ["usepy-plugin-redis"]
24
+ redis = ["use-redis"]
25
25
 
26
26
  [tool.poetry.group.dev.dependencies]
27
27
  nacos-sdk-python = "^0.1.12"
@@ -36,6 +36,14 @@ amqpstorm = "^2.10.6"
36
36
  redis = "^4.5.1"
37
37
  httpx = "^0.23.3"
38
38
 
39
+ [tool.ruff]
40
+ ignore = [
41
+ "E501", # line too long, handled by black
42
+ ]
43
+
44
+ [tool.ruff.per-file-ignores]
45
+ "__init__.py" = ["F401"]
46
+
39
47
  [build-system]
40
48
  requires = ["poetry-core"]
41
49
  build-backend = "poetry.core.masonry.api"
@@ -4,7 +4,7 @@ from .retry import (
4
4
  NeverRetry, AlwaysRetry, TimesRetry, RetryIfException, AdvancedRetry
5
5
  )
6
6
  from .broker import (
7
- BaseBroker, BaseConsumer, BaseLocalBroker, BaseLocalConsumer,
7
+ BaseBroker, BaseConsumer,
8
8
  MemoryBroker, RabbitMQBroker, WebHookBroker, CronBroker, RedisStreamBroker, RedisPubSubBroker
9
9
  )
10
10
  from .middleware import (
@@ -23,8 +23,6 @@ __all__ = [
23
23
  # broker
24
24
  'BaseBroker',
25
25
  'BaseConsumer',
26
- 'BaseLocalBroker',
27
- 'BaseLocalConsumer',
28
26
  'MemoryBroker',
29
27
  'RabbitMQBroker',
30
28
  'WebHookBroker',
@@ -61,4 +59,4 @@ __all__ = [
61
59
  '__version__'
62
60
  ]
63
61
 
64
- __version__ = '0.3.7'
62
+ __version__ = '0.3.9'
@@ -1,7 +1,7 @@
1
1
  from .base import (
2
- BaseBroker, BaseConsumer, BaseLocalBroker, BaseLocalConsumer
2
+ BaseBroker, BaseConsumer
3
3
  )
4
- from .memory import MemoryBroker
4
+ from .memory import MemoryBroker, MemoryConsumer
5
5
  from .webhook import WebHookBroker
6
6
  from .rabbitmq import RabbitMQBroker
7
7
  from .redis import RedisStreamBroker, RedisPubSubBroker
@@ -1,8 +1,7 @@
1
1
  # -*- coding: utf-8 -*-
2
2
  import abc
3
- import json
4
3
  import logging
5
- from queue import Queue, Empty, Full as FullException
4
+ from queue import Queue, Empty
6
5
  from typing import Any, Optional, List, Callable
7
6
 
8
7
  from onestep.middleware import BaseMiddleware
@@ -13,10 +12,11 @@ logger = logging.getLogger(__name__)
13
12
 
14
13
 
15
14
  class BaseBroker:
15
+ message_cls = Message
16
16
 
17
17
  def __init__(self,
18
18
  name: Optional[str] = None,
19
- queue: Optional[str] = None,
19
+ queue: Optional[Queue] = None,
20
20
  middlewares: Optional[List[BaseMiddleware]] = None,
21
21
  once: bool = False,
22
22
  cancel_consume: Optional[Callable] = None):
@@ -46,7 +46,7 @@ class BaseBroker:
46
46
  def send(self, message):
47
47
  """对消息进行预处理,然后再发送"""
48
48
  if not isinstance(message, Message):
49
- message = Message(body=message)
49
+ message = self.message_cls(body=message)
50
50
  # TODO: 对消息发送进行N次重试,确保消息发送成功。
51
51
  return self.publish(message.to_json())
52
52
 
@@ -112,80 +112,17 @@ class BaseBroker:
112
112
 
113
113
  class BaseConsumer:
114
114
 
115
- def __init__(self, queue: Queue, *args, **kwargs):
116
- self.queue = queue
115
+ def __init__(self, broker: BaseBroker, *args, **kwargs):
116
+ self.queue = broker.queue
117
+ self.message_cls = broker.message_cls or Message
117
118
  self.timeout = kwargs.pop("timeout", 1000)
118
119
 
119
- @abc.abstractmethod
120
- def _to_message(self, data: Any):
121
- """
122
- 转换消息内容到 Message , 则必须实现此方法
123
- """
124
- raise NotImplementedError('Please implement in subclasses.')
125
-
126
120
  def __next__(self):
127
121
  try:
128
122
  data = self.queue.get(timeout=self.timeout / 1000)
129
- return self._to_message(data)
123
+ return self.message_cls.from_broker(broker_message=data)
130
124
  except Empty:
131
125
  return None
132
126
 
133
127
  def __iter__(self):
134
128
  return self
135
-
136
-
137
- class BaseLocalBroker(BaseBroker):
138
-
139
- def __init__(self, maxsize=0, *args, **kwargs):
140
- super().__init__(*args, **kwargs)
141
- self.queue = Queue(maxsize)
142
-
143
- def publish(self, message: Any):
144
- try:
145
- self.queue.put_nowait(message)
146
- except FullException:
147
- logger.warning("CronBroker queue is full, skip this task, "
148
- "you can increase maxsize with `maxsize` argument")
149
-
150
- def consume(self, *args, **kwargs):
151
- return BaseLocalConsumer(self.queue, *args, **kwargs)
152
-
153
- def confirm(self, message: Message):
154
- """确认消息"""
155
- pass
156
-
157
- def reject(self, message: Message):
158
- """拒绝消息"""
159
- pass
160
-
161
- def requeue(self, message: Message, is_source=False):
162
- """重发消息:先拒绝 再 重入"""
163
- if is_source:
164
- self.publish(message.msg)
165
- else:
166
- self.send(message)
167
-
168
- def __repr__(self):
169
- return f"<{self.__class__.__name__} {self.name}>"
170
-
171
- def __str__(self):
172
- return self.name
173
-
174
-
175
- class BaseLocalConsumer(BaseConsumer):
176
-
177
- def _to_message(self, data: Any):
178
- if isinstance(data, (str, bytes, bytearray)):
179
- try:
180
- message = json.loads(data)
181
- except json.JSONDecodeError:
182
- message = {"body": data}
183
- else:
184
- message = data
185
- if not isinstance(message, dict):
186
- message = {"body": message}
187
- if "body" not in message:
188
- # 来自 外部的消息 可能没有 body, 故直接认为都是 message.body
189
- message = {"body": message}
190
-
191
- return Message(body=message.get("body"), extra=message.get("extra"), msg=data)
@@ -8,12 +8,12 @@ from typing import Any
8
8
 
9
9
  from croniter import croniter
10
10
 
11
- from .base import BaseLocalBroker, BaseLocalConsumer
11
+ from .memory import MemoryBroker, MemoryConsumer
12
12
 
13
13
  logger = logging.getLogger(__name__)
14
14
 
15
15
 
16
- class CronBroker(BaseLocalBroker):
16
+ class CronBroker(MemoryBroker):
17
17
  _thread = None
18
18
 
19
19
  def __init__(self, cron, name=None, middlewares=None, body: Any = None, *args, **kwargs):
@@ -31,13 +31,13 @@ class CronBroker(BaseLocalBroker):
31
31
  self._thread = threading.Timer(interval=1, function=self._scheduler)
32
32
  self._thread.start()
33
33
 
34
- def consume(self):
34
+ def consume(self, *args, **kwargs):
35
35
  self._scheduler()
36
- return CronConsumer(self.queue)
36
+ return CronConsumer(self, *args, **kwargs)
37
37
 
38
38
  def shutdown(self):
39
39
  self._thread.cancel()
40
40
 
41
41
 
42
- class CronConsumer(BaseLocalConsumer):
42
+ class CronConsumer(MemoryConsumer):
43
43
  ...
@@ -0,0 +1,72 @@
1
+ import logging
2
+ import json
3
+ from queue import Queue, Full as FullException
4
+ from typing import Any
5
+
6
+ from .base import BaseBroker, BaseConsumer
7
+
8
+ from ..message import Message
9
+
10
+ logger = logging.getLogger(__name__)
11
+
12
+
13
+ class MemoryMessage(Message):
14
+
15
+ @classmethod
16
+ def from_broker(cls, broker_message: Any):
17
+ if isinstance(broker_message, (str, bytes, bytearray)):
18
+ try:
19
+ message = json.loads(broker_message)
20
+ except json.JSONDecodeError:
21
+ message = {"body": broker_message}
22
+ else:
23
+ message = broker_message
24
+ if not isinstance(message, dict):
25
+ message = {"body": message}
26
+ if "body" not in message:
27
+ # 来自 外部的消息 可能没有 body, 故直接认为都是 message.body
28
+ message = {"body": message}
29
+ return cls(body=message.get("body"), extra=message.get("extra"), message=broker_message)
30
+
31
+
32
+ class MemoryBroker(BaseBroker):
33
+ message_cls = MemoryMessage
34
+
35
+ def __init__(self, maxsize=0, *args, **kwargs):
36
+ super().__init__(*args, **kwargs)
37
+ self.queue = Queue(maxsize)
38
+
39
+ def publish(self, message: Any):
40
+ try:
41
+ self.queue.put_nowait(message)
42
+ except FullException:
43
+ logger.warning("CronBroker queue is full, skip this task, "
44
+ "you can increase maxsize with `maxsize` argument")
45
+
46
+ def consume(self, *args, **kwargs):
47
+ return MemoryConsumer(self, *args, **kwargs)
48
+
49
+ def confirm(self, message: Message):
50
+ """确认消息"""
51
+ pass
52
+
53
+ def reject(self, message: Message):
54
+ """拒绝消息"""
55
+ pass
56
+
57
+ def requeue(self, message: Message, is_source=False):
58
+ """重发消息:先拒绝 再 重入"""
59
+ if is_source:
60
+ self.publish(message.message)
61
+ else:
62
+ self.send(message)
63
+
64
+ def __repr__(self):
65
+ return f"<{self.__class__.__name__} {self.name}>"
66
+
67
+ def __str__(self):
68
+ return self.name
69
+
70
+
71
+ class MemoryConsumer(BaseConsumer):
72
+ ...
@@ -6,11 +6,29 @@ from typing import Optional, Dict, Any
6
6
  import amqpstorm
7
7
 
8
8
  from .base import BaseBroker, BaseConsumer
9
- from usepy_plugin_rabbitmq import useRabbitMQ as RabbitMQStore
9
+ from use_rabbitmq import useRabbitMQ as RabbitMQStore
10
10
  from ..message import Message
11
11
 
12
12
 
13
+ class _RabbitMQMessage(Message):
14
+
15
+ @classmethod
16
+ def from_broker(cls, broker_message: amqpstorm.Message):
17
+ try:
18
+ message = json.loads(broker_message.body)
19
+ except json.JSONDecodeError:
20
+ message = {"body": broker_message.body}
21
+ if not isinstance(message, dict):
22
+ message = {"body": message}
23
+ if "body" not in message:
24
+ # 来自 外部的消息 可能没有 body, 故直接认为都是 message.body
25
+ message = {"body": message}
26
+
27
+ return cls(body=message.get("body"), extra=message.get("extra"), message=broker_message)
28
+
29
+
13
30
  class RabbitMQBroker(BaseBroker):
31
+ message_cls = _RabbitMQMessage
14
32
 
15
33
  def __init__(self, queue_name, params: Optional[Dict] = None, prefetch: Optional[int] = 1, auto_create=True, *args,
16
34
  **kwargs):
@@ -37,18 +55,18 @@ class RabbitMQBroker(BaseBroker):
37
55
  thread.daemon = daemon
38
56
  thread.start()
39
57
  self.threads.append(thread)
40
- return RabbitMQConsumer(self.queue)
58
+ return RabbitMQConsumer(self)
41
59
 
42
60
  def publish(self, message: Any):
43
61
  self.client.send(self.queue_name, message)
44
62
 
45
63
  def confirm(self, message: Message):
46
64
  """确认消息"""
47
- message.msg.ack()
65
+ message.message.ack()
48
66
 
49
67
  def reject(self, message: Message):
50
68
  """拒绝消息"""
51
- message.msg.reject(requeue=False)
69
+ message.message.reject(requeue=False)
52
70
 
53
71
  def requeue(self, message: Message, is_source=False):
54
72
  """
@@ -58,9 +76,9 @@ class RabbitMQBroker(BaseBroker):
58
76
  :param is_source: 是否是原始消息,True: 使用原始消息重入当前队列,False: 使用消息的最新数据重入当前队列
59
77
  """
60
78
  if is_source:
61
- message.msg.reject(requeue=True)
79
+ message.message.reject(requeue=True)
62
80
  else:
63
- message.msg.reject(requeue=False)
81
+ message.message.reject(requeue=False)
64
82
  self.send(message)
65
83
 
66
84
  def shutdown(self):
@@ -70,15 +88,4 @@ class RabbitMQBroker(BaseBroker):
70
88
 
71
89
 
72
90
  class RabbitMQConsumer(BaseConsumer):
73
- def _to_message(self, data: amqpstorm.Message):
74
- try:
75
- message = json.loads(data.body)
76
- except json.JSONDecodeError:
77
- message = {"body": data.body}
78
- if not isinstance(message, dict):
79
- message = {"body": message}
80
- if "body" not in message:
81
- # 来自 外部的消息 可能没有 body, 故直接认为都是 message.body
82
- message = {"body": message}
83
-
84
- return Message(body=message.get("body"), extra=message.get("extra"), msg=data)
91
+ ...
@@ -4,15 +4,32 @@ from queue import Queue
4
4
  from typing import Any
5
5
 
6
6
  try:
7
- from usepy_plugin_redis import useRedis
7
+ from use_redis import useRedis
8
8
  except ImportError:
9
9
  ...
10
10
 
11
11
  from ..base import BaseBroker, BaseConsumer, Message
12
12
 
13
13
 
14
+ class _RedisPubSubMessage(Message):
15
+
16
+ @classmethod
17
+ def from_broker(cls, broker_message: Any):
18
+ if "channel" in broker_message:
19
+ try:
20
+ message = json.loads(broker_message.get("data")) # 已转换的 message
21
+ except (json.JSONDecodeError, TypeError):
22
+ message = {"body": broker_message.get("data")} # 未转换的 message
23
+ else:
24
+ # 来自 外部的消息 直接认为都是 message.body
25
+ message = {"body": broker_message.body}
26
+
27
+ yield cls(body=message.get("body"), extra=message.get("extra"), message=broker_message)
28
+
29
+
14
30
  class RedisPubSubBroker(BaseBroker):
15
31
  """ Redis PubSub Broker """
32
+ message_cls = _RedisPubSubMessage
16
33
 
17
34
  def __init__(self, channel: str, *args, **kwargs):
18
35
  super().__init__(*args, **kwargs)
@@ -40,14 +57,14 @@ class RedisPubSubBroker(BaseBroker):
40
57
  thread.daemon = daemon
41
58
  thread.start()
42
59
  self.threads.append(thread)
43
- return RedisPubSubConsumer(self.queue)
60
+ return RedisPubSubConsumer(self)
44
61
 
45
62
  def send(self, message: Any):
46
63
  """Publish message to the Redis channel"""
47
64
  if not isinstance(message, Message):
48
- message = Message(body=message)
65
+ message = self.message_cls(body=message)
49
66
 
50
- print(self.client.publish(self.channel, message.to_json()))
67
+ self.client.publish(self.channel, message.to_json())
51
68
 
52
69
  publish = send
53
70
 
@@ -67,20 +84,10 @@ class RedisPubSubBroker(BaseBroker):
67
84
  self.reject(message)
68
85
 
69
86
  if is_source:
70
- self.client.publish(self.channel, message.msg['data'])
87
+ self.client.publish(self.channel, message.message['data'])
71
88
  else:
72
89
  self.send(message)
73
90
 
74
91
 
75
92
  class RedisPubSubConsumer(BaseConsumer):
76
- def _to_message(self, data):
77
- if "channel" in data:
78
- try:
79
- message = json.loads(data.get("data")) # 已转换的 message
80
- except (json.JSONDecodeError, TypeError):
81
- message = {"body": data.get("data")} # 未转换的 message
82
- else:
83
- # 来自 外部的消息 直接认为都是 message.body
84
- message = {"body": data.body}
85
-
86
- yield Message(body=message.get("body"), extra=message.get("extra"), msg=data)
93
+ ...
@@ -5,15 +5,32 @@ from queue import Queue
5
5
  from typing import Optional, Dict, Any
6
6
 
7
7
  try:
8
- from usepy_plugin_redis import useRedisStreamStore, RedisStreamMessage
8
+ from use_redis import useRedisStreamStore, RedisStreamMessage
9
9
  except ImportError:
10
10
  ...
11
11
 
12
12
  from ..base import BaseBroker, BaseConsumer, Message
13
13
 
14
14
 
15
+ class _RedisStreamMessage(Message):
16
+ @classmethod
17
+ def from_broker(cls, broker_message: "RedisStreamMessage"):
18
+ if "_message" in broker_message.body:
19
+ # 来自 RedisStreamBroker.send 的消息,message.body 默认是存于 _message 字段中
20
+ try:
21
+ message = json.loads(broker_message.body.get("_message")) # 已转换的 message
22
+ except (json.JSONDecodeError, TypeError):
23
+ message = {"body": broker_message.body.get("_message")} # 未转换的 message
24
+ else:
25
+ # 来自 外部的消息 直接认为都是 message.body
26
+ message = {"body": broker_message.body}
27
+
28
+ yield cls(body=message.get("body"), extra=message.get("extra"), message=broker_message)
29
+
30
+
15
31
  class RedisStreamBroker(BaseBroker):
16
32
  """ Redis Stream Broker """
33
+ message_cls = _RedisStreamMessage
17
34
 
18
35
  def __init__(
19
36
  self,
@@ -56,22 +73,22 @@ class RedisStreamBroker(BaseBroker):
56
73
  thread.daemon = daemon
57
74
  thread.start()
58
75
  self.threads.append(thread)
59
- return RedisStreamConsumer(self.queue)
76
+ return RedisStreamConsumer(self)
60
77
 
61
78
  def send(self, message: Any):
62
79
  """对消息进行预处理,然后再发送"""
63
80
  if not isinstance(message, Message):
64
- message = Message(body=message)
81
+ message = self.message_cls(body=message)
65
82
 
66
83
  self.client.send({"_message": message.to_json()})
67
84
 
68
85
  publish = send
69
86
 
70
87
  def confirm(self, message: Message):
71
- self.client.ack(message.msg)
88
+ self.client.ack(message.message)
72
89
 
73
90
  def reject(self, message: Message):
74
- self.client.reject(message.msg)
91
+ self.client.reject(message.message)
75
92
 
76
93
  def requeue(self, message: Message, is_source=False):
77
94
  """
@@ -83,21 +100,10 @@ class RedisStreamBroker(BaseBroker):
83
100
  self.reject(message)
84
101
 
85
102
  if is_source:
86
- self.client.send(message.msg.body)
103
+ self.client.send(message.message.body)
87
104
  else:
88
105
  self.send(message)
89
106
 
90
107
 
91
108
  class RedisStreamConsumer(BaseConsumer):
92
- def _to_message(self, data: "RedisStreamMessage"):
93
- if "_message" in data.body:
94
- # 来自 RedisStreamBroker.send 的消息,message.body 默认是存于 _message 字段中
95
- try:
96
- message = json.loads(data.body.get("_message")) # 已转换的 message
97
- except (json.JSONDecodeError, TypeError):
98
- message = {"body": data.body.get("_message")} # 未转换的 message
99
- else:
100
- # 来自 外部的消息 直接认为都是 message.body
101
- message = {"body": data.body}
102
-
103
- yield Message(body=message.get("body"), extra=message.get("extra"), msg=data)
109
+ ...
@@ -3,7 +3,7 @@ import threading
3
3
  import collections
4
4
  from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
5
5
 
6
- from .base import BaseLocalBroker, BaseLocalConsumer
6
+ from .memory import MemoryBroker, MemoryConsumer
7
7
 
8
8
  logger = logging.getLogger(__name__)
9
9
 
@@ -34,7 +34,7 @@ class WebHookServer(BaseHTTPRequestHandler):
34
34
  self.wfile.write(b'{ "status": "ok" }')
35
35
 
36
36
 
37
- class WebHookBroker(BaseLocalBroker):
37
+ class WebHookBroker(MemoryBroker):
38
38
  _servers = {}
39
39
 
40
40
  def __init__(self,
@@ -68,7 +68,7 @@ class WebHookBroker(BaseLocalBroker):
68
68
  def consume(self, *args, **kwargs):
69
69
  self._create_server()
70
70
  logger.debug(f"WebHookBroker: {self.host}:{self.port}{self.path}")
71
- return WebHookConsumer(self.queue, *args, **kwargs)
71
+ return WebHookConsumer(self, *args, **kwargs)
72
72
 
73
73
  def shutdown(self):
74
74
  hs = self._servers[(self.host, self.port)]
@@ -78,5 +78,5 @@ class WebHookBroker(BaseLocalBroker):
78
78
  thread.join()
79
79
 
80
80
 
81
- class WebHookConsumer(BaseLocalConsumer):
81
+ class WebHookConsumer(MemoryConsumer):
82
82
  ...
@@ -25,3 +25,11 @@ class RetryInLocal(RetryException):
25
25
 
26
26
  class DropMessage(Exception):
27
27
  """从 Brokers 中 丢弃该消息"""
28
+
29
+
30
+ class RejectMessage(DropMessage):
31
+ """从 Brokers 中 丢弃该消息"""
32
+
33
+
34
+ class RequeueMessage(Exception):
35
+ """重新入队该消息"""
@@ -42,12 +42,19 @@ class Message:
42
42
  self,
43
43
  body: Optional[Union[dict, Any]] = None,
44
44
  extra: Optional[Union[dict, Extra]] = None,
45
- msg: Optional[Any] = None,
45
+ message: Optional[Any] = None,
46
46
  broker=None
47
47
  ):
48
+ """ Message
49
+
50
+ :param body: 解析后的消息体
51
+ :param extra: 额外信息
52
+ :param message: 原始消息体
53
+ :param broker: 当前消息所属的 broker
54
+ """
48
55
  self.body = body
49
56
  self.extra = self._set_extra(extra)
50
- self.msg = msg
57
+ self.message = message
51
58
 
52
59
  self.broker = broker
53
60
  self._exception = None
@@ -134,11 +141,6 @@ class Message:
134
141
  def __repr__(self):
135
142
  return f"<{self.__class__.__name__} {self.body}>"
136
143
 
137
-
138
- if __name__ == '__main__':
139
- msg = Message()
140
- msg.x = 1
141
- print(msg.x)
142
- msg.qq1 = 1
143
- del msg.qq1
144
- print(msg.qq1)
144
+ @classmethod
145
+ def from_broker(cls, broker_message: Any):
146
+ return cls(body=broker_message, extra=None, message=broker_message, broker=None)
@@ -241,6 +241,7 @@ class step:
241
241
  }
242
242
 
243
243
  def __call__(self, func, *_args, **_kwargs):
244
+ func.__step_params__ = self.params
244
245
  if iscoroutinefunction(func) or isasyncgenfunction(func):
245
246
  os = AsyncOneStep(fn=func, **self.params)
246
247
  else:
@@ -8,3 +8,4 @@ message_sent = signal("message_sent")
8
8
  message_consumed = signal("message_consumed")
9
9
  message_error = signal("message_error")
10
10
  message_drop = signal("message_drop")
11
+ message_requeue = signal("message_requeue")
@@ -14,8 +14,8 @@ from asgiref.sync import async_to_sync
14
14
  from .message import Message
15
15
  from .retry import RetryStatus
16
16
  from .broker import BaseBroker
17
- from .exception import DropMessage
18
- from .signal import message_received, message_consumed, message_error, message_drop
17
+ from . import exception
18
+ from .signal import message_received, message_consumed, message_error, message_drop, message_requeue
19
19
 
20
20
  logger = logging.getLogger(__name__)
21
21
 
@@ -82,10 +82,14 @@ class BaseWorker:
82
82
  message.confirm()
83
83
 
84
84
  self.instance.after_emit("consume", message=message)
85
- except DropMessage as e:
85
+ except (exception.DropMessage, exception.RejectMessage) as e:
86
86
  message_drop.send(self, message=message, reason=e)
87
87
  logger.warning(f"{self.instance.name} dropped <{type(e).__name__}: {str(e)}>")
88
88
  message.reject()
89
+ except exception.RequeueMessage as e:
90
+ message_requeue.send(self, message=message, reason=e)
91
+ logger.warning(f"{self.instance.name} requeue <{type(e).__name__}: {str(e)}>")
92
+ message.requeue(is_source=True)
89
93
  except Exception as e:
90
94
  message_error.send(self, message=message, error=e)
91
95
  if self.instance.state.debug:
@@ -1,9 +0,0 @@
1
- from .base import BaseLocalBroker, BaseLocalConsumer
2
-
3
-
4
- class MemoryBroker(BaseLocalBroker):
5
- ...
6
-
7
-
8
- class MemoryConsumer(BaseLocalConsumer):
9
- ...
File without changes
File without changes
File without changes
File without changes