queue-func 0.2.2__py3-none-any.whl → 0.2.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- queue_func/{broker/queue.py → backends/queue/broker.py} +5 -5
- queue_func/{factory/builders/queue.py → backends/queue/build.py} +3 -3
- queue_func/{config/queue.py → backends/queue/config.py} +2 -2
- queue_func/{consumer/queue.py → backends/queue/consumer.py} +3 -2
- queue_func/{producer/queue.py → backends/queue/producer.py} +2 -2
- queue_func/{shared_resource/queue.py → backends/queue/resource.py} +1 -1
- queue_func/{broker/rabbit_aio_pika.py → backends/rabbit_aio_pika/broker.py} +5 -5
- queue_func/{factory/builders/rabbit_aio_pika.py → backends/rabbit_aio_pika/build.py} +3 -3
- queue_func/{config/rabbit_aio_pika.py → backends/rabbit_aio_pika/config.py} +2 -2
- queue_func/{consumer/rabbit_aio_pika.py → backends/rabbit_aio_pika/consumer.py} +3 -2
- queue_func/{producer/rabbit_aio_pika.py → backends/rabbit_aio_pika/producer.py} +3 -3
- queue_func/{shared_resource/rabbit_aio_pika.py → backends/rabbit_aio_pika/resource.py} +1 -1
- queue_func/{broker/base.py → core/broker.py} +4 -4
- queue_func/{config/general.py → core/config.py} +9 -1
- queue_func/{consumer/base.py → core/consumer.py} +9 -1
- queue_func/{producer/base.py → core/producer.py} +1 -1
- queue_func/factory/builders/__init__.py +2 -2
- queue_func/factory/deco.py +3 -3
- queue_func/factory/registry.py +1 -1
- {queue_func-0.2.2.dist-info → queue_func-0.2.3.dist-info}/METADATA +38 -53
- queue_func-0.2.3.dist-info/RECORD +48 -0
- queue_func/config/utils.py +0 -15
- queue_func/shared_resource/__init__.py +0 -1
- queue_func-0.2.2.dist-info/RECORD +0 -50
- /queue_func/{broker → backends}/__init__.py +0 -0
- /queue_func/{config → backends/queue}/__init__.py +0 -0
- /queue_func/{consumer → backends/rabbit_aio_pika}/__init__.py +0 -0
- /queue_func/{producer → core}/__init__.py +0 -0
- /queue_func/{shared_resource/base.py → core/resource.py} +0 -0
- {queue_func-0.2.2.dist-info → queue_func-0.2.3.dist-info}/WHEEL +0 -0
- {queue_func-0.2.2.dist-info → queue_func-0.2.3.dist-info}/licenses/LICENSE +0 -0
|
@@ -4,13 +4,13 @@ import asyncio
|
|
|
4
4
|
import typing as t
|
|
5
5
|
|
|
6
6
|
from queue_func.serializer.base import Message
|
|
7
|
-
from queue_func.
|
|
8
|
-
from queue_func.broker
|
|
9
|
-
from queue_func.
|
|
10
|
-
from queue_func.
|
|
7
|
+
from queue_func.backends.queue.resource import Resource
|
|
8
|
+
from queue_func.core.broker import BaseBroker as _BaseBroker
|
|
9
|
+
from queue_func.backends.queue.producer import Producer
|
|
10
|
+
from queue_func.backends.queue.consumer import Consumer
|
|
11
11
|
|
|
12
12
|
if t.TYPE_CHECKING:
|
|
13
|
-
from queue_func.
|
|
13
|
+
from queue_func.backends.queue.config import Config
|
|
14
14
|
|
|
15
15
|
|
|
16
16
|
class Broker(_BaseBroker["Config", Message]):
|
|
@@ -6,7 +6,7 @@ from queue_func.factory.backend import Backend
|
|
|
6
6
|
from queue_func.factory.registry import register_backend
|
|
7
7
|
|
|
8
8
|
if t.TYPE_CHECKING:
|
|
9
|
-
from queue_func.broker
|
|
9
|
+
from queue_func.core.broker import BaseBroker
|
|
10
10
|
|
|
11
11
|
|
|
12
12
|
@register_backend(Backend.QUEUE)
|
|
@@ -14,7 +14,7 @@ def build_queue_broker(
|
|
|
14
14
|
coro: t.Callable[..., t.Coroutine],
|
|
15
15
|
**kwargs: t.Any) -> "BaseBroker[t.Any, t.Any]":
|
|
16
16
|
|
|
17
|
-
from queue_func.
|
|
18
|
-
from queue_func.
|
|
17
|
+
from queue_func.backends.queue.broker import Broker
|
|
18
|
+
from queue_func.backends.queue.config import Config
|
|
19
19
|
|
|
20
20
|
return Broker(Config(coro=coro, **kwargs))
|
|
@@ -3,8 +3,8 @@
|
|
|
3
3
|
import typing as t
|
|
4
4
|
from pydantic.dataclasses import dataclass as pyd_dataclass
|
|
5
5
|
|
|
6
|
-
from queue_func.config
|
|
7
|
-
from queue_func.config
|
|
6
|
+
from queue_func.core.config import Config as _Config
|
|
7
|
+
from queue_func.core.config import ExtraIgnoreConfig
|
|
8
8
|
from queue_func.serializer.base import BaseSerializer
|
|
9
9
|
from queue_func.serializer.raw import Serializer
|
|
10
10
|
|
|
@@ -4,9 +4,9 @@ import asyncio
|
|
|
4
4
|
import typing as t
|
|
5
5
|
|
|
6
6
|
from queue_func.serializer.base import Message
|
|
7
|
-
from queue_func.
|
|
7
|
+
from queue_func.backends.queue.resource import Resource
|
|
8
8
|
from queue_func.pool_executor.base import BasePoolExecutor
|
|
9
|
-
from queue_func.consumer
|
|
9
|
+
from queue_func.core.consumer import BaseConsumer as _BaseConsumer
|
|
10
10
|
|
|
11
11
|
|
|
12
12
|
class Consumer(_BaseConsumer[Message, Resource]):
|
|
@@ -14,6 +14,7 @@ class Consumer(_BaseConsumer[Message, Resource]):
|
|
|
14
14
|
async def _consume(self) -> None:
|
|
15
15
|
|
|
16
16
|
while (not self._stopped) or (not self._resource.queue.empty()):
|
|
17
|
+
self._mark_running()
|
|
17
18
|
try:
|
|
18
19
|
message: Message = await asyncio.wait_for(
|
|
19
20
|
self._resource.queue.get(),
|
|
@@ -4,8 +4,8 @@ import asyncio
|
|
|
4
4
|
|
|
5
5
|
from queue_func.errors import PublishTimeoutError
|
|
6
6
|
from queue_func.serializer.base import Message
|
|
7
|
-
from queue_func.producer
|
|
8
|
-
from queue_func.
|
|
7
|
+
from queue_func.core.producer import BaseProducer
|
|
8
|
+
from queue_func.backends.queue.resource import Resource
|
|
9
9
|
|
|
10
10
|
|
|
11
11
|
class Producer(BaseProducer[Message, Resource]):
|
|
@@ -2,13 +2,13 @@
|
|
|
2
2
|
|
|
3
3
|
import typing as t
|
|
4
4
|
|
|
5
|
-
from queue_func.broker
|
|
6
|
-
from queue_func.
|
|
7
|
-
from queue_func.
|
|
8
|
-
from queue_func.
|
|
5
|
+
from queue_func.core.broker import BaseBroker as _BaseBroker
|
|
6
|
+
from queue_func.backends.rabbit_aio_pika.consumer import Consumer
|
|
7
|
+
from queue_func.backends.rabbit_aio_pika.producer import Producer
|
|
8
|
+
from queue_func.backends.rabbit_aio_pika.resource import Resource
|
|
9
9
|
|
|
10
10
|
if t.TYPE_CHECKING:
|
|
11
|
-
from queue_func.
|
|
11
|
+
from queue_func.backends.rabbit_aio_pika.config import Config
|
|
12
12
|
|
|
13
13
|
|
|
14
14
|
class Broker(_BaseBroker["Config", bytes]):
|
|
@@ -6,7 +6,7 @@ from queue_func.factory.backend import Backend
|
|
|
6
6
|
from queue_func.factory.registry import register_backend
|
|
7
7
|
|
|
8
8
|
if t.TYPE_CHECKING:
|
|
9
|
-
from queue_func.broker
|
|
9
|
+
from queue_func.core.broker import BaseBroker
|
|
10
10
|
|
|
11
11
|
|
|
12
12
|
@register_backend(Backend.RABBIT_AIO_PIKA)
|
|
@@ -15,8 +15,8 @@ def build_rabbit_aio_pika_broker(
|
|
|
15
15
|
**kwargs: t.Any) -> "BaseBroker[t.Any, t.Any]":
|
|
16
16
|
|
|
17
17
|
try:
|
|
18
|
-
from queue_func.
|
|
19
|
-
from queue_func.
|
|
18
|
+
from queue_func.backends.rabbit_aio_pika.broker import Broker
|
|
19
|
+
from queue_func.backends.rabbit_aio_pika.config import Config
|
|
20
20
|
except ImportError as exc:
|
|
21
21
|
raise ImportError(
|
|
22
22
|
"Backend.RABBIT_AIO_PIKA requires aio-pika. "
|
|
@@ -4,8 +4,8 @@ from enum import unique, StrEnum, IntEnum
|
|
|
4
4
|
|
|
5
5
|
from pydantic.dataclasses import dataclass as pyd_dataclass
|
|
6
6
|
|
|
7
|
-
from queue_func.config
|
|
8
|
-
from queue_func.config
|
|
7
|
+
from queue_func.core.config import Config as _Config
|
|
8
|
+
from queue_func.core.config import ExtraIgnoreConfig
|
|
9
9
|
|
|
10
10
|
|
|
11
11
|
@unique
|
|
@@ -3,9 +3,9 @@
|
|
|
3
3
|
import asyncio
|
|
4
4
|
import typing as t
|
|
5
5
|
|
|
6
|
-
from queue_func.consumer
|
|
6
|
+
from queue_func.core.consumer import BaseConsumer as _BaseConsumer
|
|
7
7
|
from queue_func.pool_executor.base import BasePoolExecutor
|
|
8
|
-
from queue_func.
|
|
8
|
+
from queue_func.backends.rabbit_aio_pika.resource import Resource
|
|
9
9
|
|
|
10
10
|
if t.TYPE_CHECKING:
|
|
11
11
|
from aio_pika.abc import AbstractIncomingMessage, AbstractQueue
|
|
@@ -55,6 +55,7 @@ class Consumer(_BaseConsumer[bytes, Resource]):
|
|
|
55
55
|
await self._ensure_queue()
|
|
56
56
|
while not self._stopped:
|
|
57
57
|
async with self._queue.iterator(timeout=self._consume_timeout) as queue_iter:
|
|
58
|
+
self._mark_running()
|
|
58
59
|
try:
|
|
59
60
|
message: "AbstractIncomingMessage"
|
|
60
61
|
async for message in queue_iter:
|
|
@@ -5,10 +5,10 @@ import typing as t
|
|
|
5
5
|
|
|
6
6
|
import aio_pika
|
|
7
7
|
|
|
8
|
-
from queue_func.
|
|
8
|
+
from queue_func.backends.rabbit_aio_pika.config import DeliveryMode, ExchangeName
|
|
9
9
|
from queue_func.errors import PublishTimeoutError
|
|
10
|
-
from queue_func.producer
|
|
11
|
-
from queue_func.
|
|
10
|
+
from queue_func.core.producer import BaseProducer
|
|
11
|
+
from queue_func.backends.rabbit_aio_pika.resource import Resource
|
|
12
12
|
|
|
13
13
|
if t.TYPE_CHECKING:
|
|
14
14
|
from aio_pika.abc import AbstractChannel, AbstractExchange
|
|
@@ -6,7 +6,7 @@ import typing as t
|
|
|
6
6
|
import aio_pika
|
|
7
7
|
|
|
8
8
|
from queue_func.utils.deco import before
|
|
9
|
-
from queue_func.
|
|
9
|
+
from queue_func.core.resource import BaseResource as _Base
|
|
10
10
|
from queue_func.serializer.base import Message, BaseSerializer
|
|
11
11
|
|
|
12
12
|
if t.TYPE_CHECKING:
|
|
@@ -4,12 +4,12 @@ import abc
|
|
|
4
4
|
import typing as t
|
|
5
5
|
|
|
6
6
|
from queue_func.serializer.base import Message
|
|
7
|
-
from queue_func.
|
|
7
|
+
from queue_func.core.resource import Resource
|
|
8
8
|
|
|
9
9
|
if t.TYPE_CHECKING:
|
|
10
|
-
from queue_func.config
|
|
11
|
-
from queue_func.producer
|
|
12
|
-
from queue_func.consumer
|
|
10
|
+
from queue_func.core.config import Config
|
|
11
|
+
from queue_func.core.producer import BaseProducer
|
|
12
|
+
from queue_func.core.consumer import BaseConsumer
|
|
13
13
|
|
|
14
14
|
|
|
15
15
|
ConfigType = t.TypeVar("ConfigType", bound="Config")
|
|
@@ -2,15 +2,23 @@
|
|
|
2
2
|
|
|
3
3
|
import typing as t
|
|
4
4
|
|
|
5
|
+
from pydantic.config import ConfigDict
|
|
5
6
|
from pydantic.dataclasses import dataclass as pyd_dataclass
|
|
6
7
|
|
|
7
|
-
from queue_func.config.utils import ExtraIgnoreConfig
|
|
8
8
|
from queue_func.serializer.base import BaseSerializer
|
|
9
9
|
from queue_func.serializer.orjson_ser import Serializer
|
|
10
10
|
from queue_func.pool_executor.base import BasePoolExecutor
|
|
11
11
|
from queue_func.pool_executor.general import PoolExecutor
|
|
12
12
|
|
|
13
13
|
|
|
14
|
+
ExtraIgnoreConfig = ConfigDict(
|
|
15
|
+
extra="ignore",
|
|
16
|
+
validate_assignment=True,
|
|
17
|
+
arbitrary_types_allowed=True,
|
|
18
|
+
coerce_numbers_to_str=True,
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
|
|
14
22
|
@pyd_dataclass(frozen=True, kw_only=True, config=ExtraIgnoreConfig)
|
|
15
23
|
class Config:
|
|
16
24
|
|
|
@@ -8,7 +8,7 @@ import typing as t
|
|
|
8
8
|
from functools import wraps
|
|
9
9
|
|
|
10
10
|
from queue_func.serializer.base import Message
|
|
11
|
-
from queue_func.
|
|
11
|
+
from queue_func.core.resource import Resource
|
|
12
12
|
from queue_func.pool_executor.base import BasePoolExecutor
|
|
13
13
|
from queue_func.utils.hook import OnceHooks
|
|
14
14
|
|
|
@@ -24,6 +24,7 @@ class BaseConsumer(t.Generic[Message, Resource], metaclass=abc.ABCMeta):
|
|
|
24
24
|
"_startup",
|
|
25
25
|
"_shutdown",
|
|
26
26
|
"_stopped",
|
|
27
|
+
"_running",
|
|
27
28
|
"_logger",
|
|
28
29
|
)
|
|
29
30
|
|
|
@@ -42,6 +43,7 @@ class BaseConsumer(t.Generic[Message, Resource], metaclass=abc.ABCMeta):
|
|
|
42
43
|
self._pool_executor = self._init_pool_executor(pool_executor_cls, pool_maxsize)
|
|
43
44
|
self._consume_timeout = consume_timeout
|
|
44
45
|
self._stopped = False
|
|
46
|
+
self._running = asyncio.Event()
|
|
45
47
|
self._startup = OnceHooks()
|
|
46
48
|
self._shutdown = OnceHooks()
|
|
47
49
|
self._logger = logging.getLogger(__name__)
|
|
@@ -84,6 +86,12 @@ class BaseConsumer(t.Generic[Message, Resource], metaclass=abc.ABCMeta):
|
|
|
84
86
|
self._startup.register(self._run_until_stopped(self._consume))
|
|
85
87
|
self._register_shutdown()
|
|
86
88
|
await self._startup.start()
|
|
89
|
+
await self._running.wait()
|
|
90
|
+
|
|
91
|
+
def _mark_running(self) -> None:
|
|
92
|
+
"""消费者已经开始收消息。``consume`` 等到这里才返回。"""
|
|
93
|
+
|
|
94
|
+
self._running.set()
|
|
87
95
|
|
|
88
96
|
@abc.abstractmethod
|
|
89
97
|
async def _consume(self) -> None: # pragma: no cover
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
import abc
|
|
4
4
|
import typing as t
|
|
5
5
|
|
|
6
|
-
from queue_func.
|
|
6
|
+
from queue_func.core.resource import Resource
|
|
7
7
|
from queue_func.serializer.base import Message
|
|
8
8
|
from queue_func.utils.hook import OnceHooks
|
|
9
9
|
from queue_func.utils.waiter import IdleWaiter
|
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
"""Import builder modules so they register with :func:`register_backend`."""
|
|
4
4
|
|
|
5
|
-
from queue_func.
|
|
6
|
-
from queue_func.
|
|
5
|
+
from queue_func.backends.queue import build as queue
|
|
6
|
+
from queue_func.backends.rabbit_aio_pika import build as rabbit_aio_pika
|
|
7
7
|
|
|
8
8
|
__all__ = (
|
|
9
9
|
"queue",
|
queue_func/factory/deco.py
CHANGED
|
@@ -2,14 +2,14 @@
|
|
|
2
2
|
|
|
3
3
|
import typing as t
|
|
4
4
|
|
|
5
|
-
from queue_func.
|
|
5
|
+
from queue_func.backends.rabbit_aio_pika.config import DeliveryMode, ExchangeName
|
|
6
6
|
from queue_func.factory import builders as _builders # noqa: F401 — register backends
|
|
7
7
|
from queue_func.factory.backend import Backend
|
|
8
8
|
from queue_func.factory.registry import build_broker
|
|
9
9
|
|
|
10
10
|
if t.TYPE_CHECKING:
|
|
11
|
-
from queue_func.broker
|
|
12
|
-
from queue_func.config
|
|
11
|
+
from queue_func.core.broker import BaseBroker
|
|
12
|
+
from queue_func.core.config import Config as GeneralConfig
|
|
13
13
|
from queue_func.pool_executor.base import BasePoolExecutor
|
|
14
14
|
from queue_func.serializer.base import BaseSerializer
|
|
15
15
|
|
queue_func/factory/registry.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: queue-func
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.3
|
|
4
4
|
Summary: Async coroutine functions as queue-backed tasks (QueueFunc).
|
|
5
5
|
License: Apache-2.0
|
|
6
6
|
License-File: LICENSE
|
|
@@ -17,7 +17,7 @@ Provides-Extra: rabbit
|
|
|
17
17
|
Requires-Dist: aio-pika (>=9.6,<10.0) ; extra == "rabbit"
|
|
18
18
|
Requires-Dist: orjson
|
|
19
19
|
Requires-Dist: pydantic (>=2.14.1,<3.0.0)
|
|
20
|
-
Project-URL: Changelog, https://github.com/sekil9529/queue-func/blob/master/
|
|
20
|
+
Project-URL: Changelog, https://github.com/sekil9529/queue-func/blob/master/CHANGELOG.md
|
|
21
21
|
Project-URL: Documentation, https://github.com/sekil9529/queue-func#readme
|
|
22
22
|
Project-URL: Homepage, https://github.com/sekil9529/queue-func
|
|
23
23
|
Project-URL: Issues, https://github.com/sekil9529/queue-func/issues
|
|
@@ -33,30 +33,29 @@ Description-Content-Type: text/markdown
|
|
|
33
33
|
[](https://pypi.org/project/queue-func/)
|
|
34
34
|
[](https://pypi.org/project/queue-func/)
|
|
35
35
|
|
|
36
|
-
|
|
36
|
+
把一个 `async def` 收成队列任务:`push` 投递,消费者在同一个事件循环里执行。
|
|
37
37
|
|
|
38
|
-
|
|
38
|
+
仓库示例在 [examples/](examples/)。版本记录见 [CHANGELOG.md](CHANGELOG.md)。
|
|
39
39
|
|
|
40
|
-
>
|
|
40
|
+
> 需要 Python 3.12 及以上。任务必须是协程函数。
|
|
41
41
|
>
|
|
42
|
-
> RabbitMQ
|
|
42
|
+
> RabbitMQ 后端是可选依赖,底层用 [aio-pika](https://github.com/mosquito/aio-pika) `^9.6`。
|
|
43
43
|
|
|
44
44
|
## 特点
|
|
45
45
|
|
|
46
|
-
-
|
|
47
|
-
-
|
|
48
|
-
-
|
|
49
|
-
-
|
|
50
|
-
-
|
|
46
|
+
- 进程内 `asyncio.Queue` 和 RabbitMQ 两种后端,装饰器都是 `@queue`。
|
|
47
|
+
- 每个任务有自己的队列、序列化和消费并发上限。
|
|
48
|
+
- `QueueFuncRuntime` 把多段任务按下游先启动、上游后停止的顺序一起管。
|
|
49
|
+
- `start` 会等到消费者开始收消息再返回。
|
|
50
|
+
- 装饰之后仍可以直接 `await` 调用,不经过队列。
|
|
51
51
|
|
|
52
|
-
##
|
|
52
|
+
## 启动和停止
|
|
53
53
|
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
| `Backend.RABBIT_AIO_PIKA` | RabbitMQ,安装可选依赖 `rabbit` |
|
|
54
|
+
`broker.start_consumer()` 创建消费任务,并等到它开始收消息。内存队列是已经堵在 `get()` 上;RabbitMQ 是队列已经订阅上。
|
|
55
|
+
|
|
56
|
+
`stop` 先标上停止,再等手头这次收消息结束。空闲时最多再等一个 `consume_timeout`。还没取走的 RabbitMQ 消息留在代理上。
|
|
58
57
|
|
|
59
|
-
|
|
58
|
+
RabbitMQ 在消息交给消费池之后确认。处理函数出错不会把消息退回队列。`push` 会等发布确认,超时抛出 `queue_func.PublishTimeoutError`。
|
|
60
59
|
|
|
61
60
|
## 安装
|
|
62
61
|
|
|
@@ -70,7 +69,7 @@ pip install queue-func
|
|
|
70
69
|
pip install "queue-func[rabbit]"
|
|
71
70
|
```
|
|
72
71
|
|
|
73
|
-
|
|
72
|
+
从仓库安装:
|
|
74
73
|
|
|
75
74
|
```shell
|
|
76
75
|
git clone https://github.com/sekil9529/queue-func.git
|
|
@@ -80,6 +79,8 @@ poetry install -E rabbit
|
|
|
80
79
|
|
|
81
80
|
## 用法
|
|
82
81
|
|
|
82
|
+
从 `queue_func.factory` 引入 `queue`、`Backend`、`QueueFuncRuntime`。
|
|
83
|
+
|
|
83
84
|
### 进程内队列
|
|
84
85
|
|
|
85
86
|
```python
|
|
@@ -103,9 +104,9 @@ async def main() -> None:
|
|
|
103
104
|
asyncio.run(main())
|
|
104
105
|
```
|
|
105
106
|
|
|
106
|
-
`push` 就是 `broker.publish`。不传 `timeout` 时用配置里的 `publish_timeout
|
|
107
|
+
`push` 就是 `broker.publish`。不传 `timeout` 时用配置里的 `publish_timeout`;传入数字只对这一次有效。`None` 表示不限时。
|
|
107
108
|
|
|
108
|
-
|
|
109
|
+
直接调用:
|
|
109
110
|
|
|
110
111
|
```python
|
|
111
112
|
await process_order(1)
|
|
@@ -113,7 +114,7 @@ await process_order(1)
|
|
|
113
114
|
|
|
114
115
|
### 多段流水线
|
|
115
116
|
|
|
116
|
-
`QueueFuncRuntime.start()`
|
|
117
|
+
`QueueFuncRuntime.start()` 按列表反序启动,下游先就绪。`stop()` 先逐个停消费者,再停全部生产者。
|
|
117
118
|
|
|
118
119
|
```python
|
|
119
120
|
import asyncio
|
|
@@ -141,7 +142,7 @@ async def main() -> None:
|
|
|
141
142
|
asyncio.run(main())
|
|
142
143
|
```
|
|
143
144
|
|
|
144
|
-
|
|
145
|
+
三段链式示例:
|
|
145
146
|
|
|
146
147
|
```shell
|
|
147
148
|
poetry run python examples/demo_queue.py
|
|
@@ -149,11 +150,9 @@ poetry run python examples/demo_queue.py
|
|
|
149
150
|
|
|
150
151
|
### RabbitMQ
|
|
151
152
|
|
|
152
|
-
`url` 和 `queue_name` 必填。`enable_declare_queue=True` 时,队列不存在会创建。默认交换机名是 `""
|
|
153
|
+
`url` 和 `queue_name` 必填。`enable_declare_queue=True` 时,队列不存在会创建。默认交换机名是 `""`,路由键默认用队列名。已经存在的队列再声明时,持久化等参数必须和代理上的一致。
|
|
153
154
|
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
同一个 `QueueFunc` 的生产者和消费者可以分进程运行。只投递的进程把 `enable_consumer` 设为 `False`。
|
|
155
|
+
只投递、不消费的进程把 `enable_consumer` 设为 `False`。
|
|
157
156
|
|
|
158
157
|
```python
|
|
159
158
|
import asyncio
|
|
@@ -166,6 +165,7 @@ from queue_func.factory import Backend, queue
|
|
|
166
165
|
url="amqp://guest:guest@127.0.0.1/",
|
|
167
166
|
queue_name="process_order",
|
|
168
167
|
enable_declare_queue=True,
|
|
168
|
+
consume_timeout=0.5,
|
|
169
169
|
)
|
|
170
170
|
async def process_order(order_id: int) -> None:
|
|
171
171
|
print(order_id)
|
|
@@ -194,19 +194,20 @@ poetry run python examples/demo_rabbit_aio_pika.py
|
|
|
194
194
|
| 参数 | 作用 |
|
|
195
195
|
|------|------|
|
|
196
196
|
| `consumer_pool_maxsize` | 同时执行的协程数,默认 200 |
|
|
197
|
-
| `consume_timeout` |
|
|
197
|
+
| `consume_timeout` | 一次等消息的时间,默认 2 秒 |
|
|
198
198
|
| `publish_timeout` | 投递等待上限,默认不限 |
|
|
199
199
|
| `maxsize` | 仅内存队列,`asyncio.Queue` 容量,默认 200 |
|
|
200
|
-
| `serializer_cls` |
|
|
200
|
+
| `serializer_cls` | 队列中的数据结构。内存队列默认 `raw`,RabbitMQ 默认 orjson |
|
|
201
201
|
| `prefetch_count` | 仅 RabbitMQ,未确认消息窗口 |
|
|
202
202
|
| `delivery_mode` | 仅 RabbitMQ,默认持久化 |
|
|
203
203
|
| `exchange_name` / `routing_key` | 仅 RabbitMQ |
|
|
204
|
+
| `queue_durable` | 仅 RabbitMQ,声明队列时是否持久化,默认否 |
|
|
204
205
|
|
|
205
|
-
其余字段见 `queue_func.
|
|
206
|
+
其余字段见 `queue_func.backends.queue.config.Config` 和 `queue_func.backends.rabbit_aio_pika.config.Config`。
|
|
206
207
|
|
|
207
|
-
##
|
|
208
|
+
## 一条任务里有什么
|
|
208
209
|
|
|
209
|
-
|
|
210
|
+
`@queue` 装饰的协程就是一条队列。`push` 投到它自己这条队列上。
|
|
210
211
|
|
|
211
212
|
- **Broker** 组装生产者、消费者和双方共用的资源,并提供启停与投递。
|
|
212
213
|
- **Producer** 序列化后把数据写入队列。
|
|
@@ -225,36 +226,20 @@ flowchart TB
|
|
|
225
226
|
POOL --> FN["业务协程"]
|
|
226
227
|
```
|
|
227
228
|
|
|
228
|
-
##
|
|
229
|
-
|
|
230
|
-
```
|
|
231
|
-
queue_func/
|
|
232
|
-
├── broker/
|
|
233
|
-
├── producer/
|
|
234
|
-
├── consumer/
|
|
235
|
-
├── serializer/ # raw、orjson
|
|
236
|
-
├── shared_resource/
|
|
237
|
-
├── config/
|
|
238
|
-
├── factory/ # @queue、QueueFunc、QueueFuncRuntime
|
|
239
|
-
├── pool_executor/
|
|
240
|
-
└── utils/
|
|
241
|
-
examples/
|
|
242
|
-
├── demo_queue.py
|
|
243
|
-
└── demo_rabbit_aio_pika.py
|
|
244
|
-
```
|
|
245
|
-
|
|
246
|
-
## 开发与测试
|
|
229
|
+
## 测试
|
|
247
230
|
|
|
248
231
|
```shell
|
|
249
232
|
poetry install -E rabbit
|
|
250
|
-
pytest
|
|
233
|
+
poetry run pytest
|
|
251
234
|
```
|
|
252
235
|
|
|
253
|
-
|
|
236
|
+
单元测试使用假的 RabbitMQ 连接,不要求本机有代理。`examples/demo_rabbit_aio_pika.py` 需要本机 `guest` / `guest`、端口 `5672`。
|
|
237
|
+
|
|
238
|
+
`pytest.ini` 会写出覆盖率页面 `htmlcov/` 和 `report.html`。
|
|
254
239
|
|
|
255
240
|
## 版本
|
|
256
241
|
|
|
257
|
-
遵循[语义化版本](https://semver.org/lang/zh-CN/)。变更记录在 [
|
|
242
|
+
遵循[语义化版本](https://semver.org/lang/zh-CN/)。变更记录在 [CHANGELOG.md](CHANGELOG.md)。
|
|
258
243
|
|
|
259
244
|
## 许可证
|
|
260
245
|
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
queue_func/__init__.py,sha256=hL8i9WF-A8geNk7K4ZpMCxB0yIVSuNVRhGghksg-sk0,116
|
|
2
|
+
queue_func/backends/__init__.py,sha256=7DxiEpT1g6hMwW1hBF17OOFRJ3DW03giEu6rgvRCTHY,17
|
|
3
|
+
queue_func/backends/queue/__init__.py,sha256=7DxiEpT1g6hMwW1hBF17OOFRJ3DW03giEu6rgvRCTHY,17
|
|
4
|
+
queue_func/backends/queue/broker.py,sha256=PTgPTweogn943Tm758bYK7mIIu486n-MtYa0mFNNPls,1193
|
|
5
|
+
queue_func/backends/queue/build.py,sha256=jR2apz3v_VlWJdEAluAJKKaL3F8gQfmke4-jGqYEeT8,552
|
|
6
|
+
queue_func/backends/queue/config.py,sha256=qxxPARXjC7k-msjHy-uiOg5o2YrmX0Pc2d16_p_nG0s,503
|
|
7
|
+
queue_func/backends/queue/consumer.py,sha256=lp0savE9sww4vumbx_mZbs2CGc_vc_EZFqX1gBcUkik,1165
|
|
8
|
+
queue_func/backends/queue/producer.py,sha256=qT5HBYa_h39tVOVG7zBhOaTg-vwiDkTEpEJwGypqfAo,692
|
|
9
|
+
queue_func/backends/queue/resource.py,sha256=2YMlYoShP56iuYA_o8ekJbWUP1dxjU19rbf8tr19xRc,549
|
|
10
|
+
queue_func/backends/rabbit_aio_pika/__init__.py,sha256=7DxiEpT1g6hMwW1hBF17OOFRJ3DW03giEu6rgvRCTHY,17
|
|
11
|
+
queue_func/backends/rabbit_aio_pika/broker.py,sha256=Ud5ns6bwoch4Jr4HI0_uAqarQExRgRdFvoiWc3RKqDc,1837
|
|
12
|
+
queue_func/backends/rabbit_aio_pika/build.py,sha256=j7c6btcal4d2l-0crdnX2fqF5dxwxcBdSTICk7oB6Gw,799
|
|
13
|
+
queue_func/backends/rabbit_aio_pika/config.py,sha256=rG3J65-a15IZu2RFoe4gTAy_9h9jyZmlAYl9KV_YeI8,1553
|
|
14
|
+
queue_func/backends/rabbit_aio_pika/consumer.py,sha256=ORnZPKEjSa-AX5ZAPJsMPeCrxO-G3LdqMUbmp7Asq8w,3339
|
|
15
|
+
queue_func/backends/rabbit_aio_pika/producer.py,sha256=8sZ8Adrkjchx2iQ-5O9CZOuJDDeWvLNwbpD8gLbEkm4,2788
|
|
16
|
+
queue_func/backends/rabbit_aio_pika/resource.py,sha256=Giu1freER0sxIMu8v26dOJGzF9YA5tpG0JCCmYp2D-U,3376
|
|
17
|
+
queue_func/core/__init__.py,sha256=7DxiEpT1g6hMwW1hBF17OOFRJ3DW03giEu6rgvRCTHY,17
|
|
18
|
+
queue_func/core/broker.py,sha256=kKbSX01xmz2le2i201x0cLqPQ9NvWpyusyVIcaQVEO4,1583
|
|
19
|
+
queue_func/core/config.py,sha256=z-n6io0L2-050rvTMci9lgd0xx7fOVORyhmIEGeBX9s,1003
|
|
20
|
+
queue_func/core/consumer.py,sha256=aexWTGWP8Mm0owG6p54xW-pR_jIUL-I90fYQMlNh0wQ,3604
|
|
21
|
+
queue_func/core/producer.py,sha256=4NQbydOV5cDmatzmJeQT_HObCkUROgbxbs8MTECxpPw,1841
|
|
22
|
+
queue_func/core/resource.py,sha256=6QXZRVcySCwNXWwkmsHBMWtPnVevks2CdIAm3zbb7AA,491
|
|
23
|
+
queue_func/errors.py,sha256=3KPNWM0HxC-hwYdrt9UNGrTHLxWYVYOCNEKW_08z6o0,120
|
|
24
|
+
queue_func/factory/__init__.py,sha256=7sSCKz_-rx29nJuNFAZlvKro8PuruvZ2ArGcDFFKNb8,269
|
|
25
|
+
queue_func/factory/backend.py,sha256=QBDvYnpxRl1l8VEpw1wWV7O2dw-efeYgxSj9S9kPcjU,189
|
|
26
|
+
queue_func/factory/builders/__init__.py,sha256=kWQD8BnPKQ-k3s0lqj0lKzwr2KUp4wCp2bGeygEatJs,283
|
|
27
|
+
queue_func/factory/deco.py,sha256=tS74glThXZSoIEwLDSvS7dvXlFZyHpJGOPh8kgoW51E,3824
|
|
28
|
+
queue_func/factory/registry.py,sha256=vLhQLmONd6g34MeM6z9w157n5YAFYAhnBSS0D2aybhk,1027
|
|
29
|
+
queue_func/factory/runtime.py,sha256=A5CUSVRi4YtlRnroOc-FVeWEbPgEGQ8A-gR49ZgRz8c,958
|
|
30
|
+
queue_func/pool_executor/__init__.py,sha256=7DxiEpT1g6hMwW1hBF17OOFRJ3DW03giEu6rgvRCTHY,17
|
|
31
|
+
queue_func/pool_executor/base.py,sha256=reYgENOa9bRMBKfzvlPP_aTMPSXFCZaNStn6vPwnWak,585
|
|
32
|
+
queue_func/pool_executor/general.py,sha256=MzYZUKNom-xcoop_BBJpma2LPuV3pnnpyWKno5KWa-s,1110
|
|
33
|
+
queue_func/pool_executor/utils.py,sha256=FQR6nE2IVPk_rgfEzQEh6ZeelTc3iqeJegKZxN7EM0c,1042
|
|
34
|
+
queue_func/serializer/__init__.py,sha256=7DxiEpT1g6hMwW1hBF17OOFRJ3DW03giEu6rgvRCTHY,17
|
|
35
|
+
queue_func/serializer/base.py,sha256=yWVqIN_URJl59K2Pl_hWttuDYx5e6wTAy2MTnO26LSU,528
|
|
36
|
+
queue_func/serializer/orjson_ser.py,sha256=vnqS1m0YpD-LFjqqiTQGWkeZqk1l_bccdFxgSZ8sjnw,599
|
|
37
|
+
queue_func/serializer/raw.py,sha256=RqS2JBr3s0Wu0SAm4OMdpdI-hRk8zt-0y9wOA81e55E,523
|
|
38
|
+
queue_func/task/__init__.py,sha256=7DxiEpT1g6hMwW1hBF17OOFRJ3DW03giEu6rgvRCTHY,17
|
|
39
|
+
queue_func/task/base.py,sha256=JblRVlT5go9ztFXsv0-e25U7Lz27yUIOHe4oszCNBHA,649
|
|
40
|
+
queue_func/task/thread.py,sha256=OIGG0H_z_pDWYHvjSCX-PyvGovuob8wJaaUaUg6PD1M,443
|
|
41
|
+
queue_func/utils/__init__.py,sha256=7DxiEpT1g6hMwW1hBF17OOFRJ3DW03giEu6rgvRCTHY,17
|
|
42
|
+
queue_func/utils/deco.py,sha256=tIosjhN1TwEbUPS-UmZICaHidvN0Ab-s7nMbrpyKJi0,904
|
|
43
|
+
queue_func/utils/hook.py,sha256=3EULaLewhBlv7nqSbEWMi9nkV-oNaTy2JnFFvsMQR1E,1596
|
|
44
|
+
queue_func/utils/waiter.py,sha256=e0W1Rf_vHKkEHWH9Pm6ZnVUZS1gorEclzNX2YsWnReM,697
|
|
45
|
+
queue_func-0.2.3.dist-info/licenses/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
|
|
46
|
+
queue_func-0.2.3.dist-info/METADATA,sha256=wWacZ0HtqMxznokbvarvouP3uGqb6gJKf7uFb7v8UtA,7779
|
|
47
|
+
queue_func-0.2.3.dist-info/WHEEL,sha256=L-WvLdSBvJWHXv5zrQxDvNRnIgoBzLUpcbuiw19KCuQ,88
|
|
48
|
+
queue_func-0.2.3.dist-info/RECORD,,
|
queue_func/config/utils.py
DELETED
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
# coding: utf-8
|
|
2
|
-
|
|
3
|
-
from pydantic.config import ConfigDict
|
|
4
|
-
|
|
5
|
-
__all__ = (
|
|
6
|
-
"ExtraIgnoreConfig",
|
|
7
|
-
)
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
ExtraIgnoreConfig = ConfigDict(
|
|
11
|
-
extra="ignore",
|
|
12
|
-
validate_assignment=True,
|
|
13
|
-
arbitrary_types_allowed=True,
|
|
14
|
-
coerce_numbers_to_str=True,
|
|
15
|
-
)
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
# coding: utf-8
|
|
@@ -1,50 +0,0 @@
|
|
|
1
|
-
queue_func/__init__.py,sha256=hL8i9WF-A8geNk7K4ZpMCxB0yIVSuNVRhGghksg-sk0,116
|
|
2
|
-
queue_func/broker/__init__.py,sha256=7DxiEpT1g6hMwW1hBF17OOFRJ3DW03giEu6rgvRCTHY,17
|
|
3
|
-
queue_func/broker/base.py,sha256=M50JGF4Zou56MwZU0E_euXvxTlTfPUBlAnVvb6MPWPc,1593
|
|
4
|
-
queue_func/broker/queue.py,sha256=9sPnIMg0d_oaNCLj0W0JLe5-GgHr2BRKzGOAulxDxC8,1164
|
|
5
|
-
queue_func/broker/rabbit_aio_pika.py,sha256=mPTYvFI4TfWz89blGK8BLGQ8XK5AUy1A923CGkfcwXw,1808
|
|
6
|
-
queue_func/config/__init__.py,sha256=7DxiEpT1g6hMwW1hBF17OOFRJ3DW03giEu6rgvRCTHY,17
|
|
7
|
-
queue_func/config/general.py,sha256=mAcu9TQO4JWwICQZEw5QizNhCmdUIWLuy7_MKiC8ACc,858
|
|
8
|
-
queue_func/config/queue.py,sha256=kWVmkeoQ3Z8l-jZlNv0l8H2a7nYmihp_TL97aiWpdxU,507
|
|
9
|
-
queue_func/config/rabbit_aio_pika.py,sha256=C3RF4linmgPYhk10lDoNoVTOLm9ru1Et-TZY0U5Vdj0,1557
|
|
10
|
-
queue_func/config/utils.py,sha256=DOZnwPkjMsfBVfZVkHEEK1sa-ZVbPjn9gwHFAjTn2UA,263
|
|
11
|
-
queue_func/consumer/__init__.py,sha256=7DxiEpT1g6hMwW1hBF17OOFRJ3DW03giEu6rgvRCTHY,17
|
|
12
|
-
queue_func/consumer/base.py,sha256=hIalhAQX8DAYM6DoaSfIcfWmBjLu-We5iDyDpICoJbY,3353
|
|
13
|
-
queue_func/consumer/queue.py,sha256=S2Ivu2DgoA9MvfkIuKUobtNB8IDLBh8kYIHNwBGGdJ0,1129
|
|
14
|
-
queue_func/consumer/rabbit_aio_pika.py,sha256=lS1ZrivTeHJBJPaJ-JhUjdJbPT_iJTqDnU1D7CiAc1k,3299
|
|
15
|
-
queue_func/errors.py,sha256=3KPNWM0HxC-hwYdrt9UNGrTHLxWYVYOCNEKW_08z6o0,120
|
|
16
|
-
queue_func/factory/__init__.py,sha256=7sSCKz_-rx29nJuNFAZlvKro8PuruvZ2ArGcDFFKNb8,269
|
|
17
|
-
queue_func/factory/backend.py,sha256=QBDvYnpxRl1l8VEpw1wWV7O2dw-efeYgxSj9S9kPcjU,189
|
|
18
|
-
queue_func/factory/builders/__init__.py,sha256=0lFzZre7FIj2BYKV1VGQK9kgJxglV_75u6UYhMjE_wM,287
|
|
19
|
-
queue_func/factory/builders/queue.py,sha256=Yxa8ySiqZurFrCSl5m9OBS6yKv6MF-joyTepkYbDdio,534
|
|
20
|
-
queue_func/factory/builders/rabbit_aio_pika.py,sha256=6hSQmQjbw6HxC04FJactWDURHyBofDw3W2uucFtkR7U,781
|
|
21
|
-
queue_func/factory/deco.py,sha256=3Cul1YatwQ4C3eZTu4478g5L_Qwc3IXQjyEa1WD2z-M,3818
|
|
22
|
-
queue_func/factory/registry.py,sha256=RN8PgcfkpM1pDFCYqOab-AYq6379IVEyAurtkgtDHwM,1027
|
|
23
|
-
queue_func/factory/runtime.py,sha256=A5CUSVRi4YtlRnroOc-FVeWEbPgEGQ8A-gR49ZgRz8c,958
|
|
24
|
-
queue_func/pool_executor/__init__.py,sha256=7DxiEpT1g6hMwW1hBF17OOFRJ3DW03giEu6rgvRCTHY,17
|
|
25
|
-
queue_func/pool_executor/base.py,sha256=reYgENOa9bRMBKfzvlPP_aTMPSXFCZaNStn6vPwnWak,585
|
|
26
|
-
queue_func/pool_executor/general.py,sha256=MzYZUKNom-xcoop_BBJpma2LPuV3pnnpyWKno5KWa-s,1110
|
|
27
|
-
queue_func/pool_executor/utils.py,sha256=FQR6nE2IVPk_rgfEzQEh6ZeelTc3iqeJegKZxN7EM0c,1042
|
|
28
|
-
queue_func/producer/__init__.py,sha256=7DxiEpT1g6hMwW1hBF17OOFRJ3DW03giEu6rgvRCTHY,17
|
|
29
|
-
queue_func/producer/base.py,sha256=QijNslD-S3uIOMu8Y07pZ2Y0UednhpiV-Cm2xFvKwm0,1848
|
|
30
|
-
queue_func/producer/queue.py,sha256=ONl39mNwghcedpfLhxjAh6e5tMZXBqM1Iibwu5BBbNE,690
|
|
31
|
-
queue_func/producer/rabbit_aio_pika.py,sha256=5NQn0Xc1gsMdlhQp2iYCC9MKEE7ddu1WP5SvKA_Q_bE,2777
|
|
32
|
-
queue_func/serializer/__init__.py,sha256=7DxiEpT1g6hMwW1hBF17OOFRJ3DW03giEu6rgvRCTHY,17
|
|
33
|
-
queue_func/serializer/base.py,sha256=yWVqIN_URJl59K2Pl_hWttuDYx5e6wTAy2MTnO26LSU,528
|
|
34
|
-
queue_func/serializer/orjson_ser.py,sha256=vnqS1m0YpD-LFjqqiTQGWkeZqk1l_bccdFxgSZ8sjnw,599
|
|
35
|
-
queue_func/serializer/raw.py,sha256=RqS2JBr3s0Wu0SAm4OMdpdI-hRk8zt-0y9wOA81e55E,523
|
|
36
|
-
queue_func/shared_resource/__init__.py,sha256=7DxiEpT1g6hMwW1hBF17OOFRJ3DW03giEu6rgvRCTHY,17
|
|
37
|
-
queue_func/shared_resource/base.py,sha256=6QXZRVcySCwNXWwkmsHBMWtPnVevks2CdIAm3zbb7AA,491
|
|
38
|
-
queue_func/shared_resource/queue.py,sha256=oDbguyGIbUnkGIMdSqSPgEui-sOWKJtl3lhHyoJmprE,556
|
|
39
|
-
queue_func/shared_resource/rabbit_aio_pika.py,sha256=qCPSCiLJXsGXt8aaukkr0jNw_grYtoj0QDkwmmbFfys,3383
|
|
40
|
-
queue_func/task/__init__.py,sha256=7DxiEpT1g6hMwW1hBF17OOFRJ3DW03giEu6rgvRCTHY,17
|
|
41
|
-
queue_func/task/base.py,sha256=JblRVlT5go9ztFXsv0-e25U7Lz27yUIOHe4oszCNBHA,649
|
|
42
|
-
queue_func/task/thread.py,sha256=OIGG0H_z_pDWYHvjSCX-PyvGovuob8wJaaUaUg6PD1M,443
|
|
43
|
-
queue_func/utils/__init__.py,sha256=7DxiEpT1g6hMwW1hBF17OOFRJ3DW03giEu6rgvRCTHY,17
|
|
44
|
-
queue_func/utils/deco.py,sha256=tIosjhN1TwEbUPS-UmZICaHidvN0Ab-s7nMbrpyKJi0,904
|
|
45
|
-
queue_func/utils/hook.py,sha256=3EULaLewhBlv7nqSbEWMi9nkV-oNaTy2JnFFvsMQR1E,1596
|
|
46
|
-
queue_func/utils/waiter.py,sha256=e0W1Rf_vHKkEHWH9Pm6ZnVUZS1gorEclzNX2YsWnReM,697
|
|
47
|
-
queue_func-0.2.2.dist-info/licenses/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
|
|
48
|
-
queue_func-0.2.2.dist-info/METADATA,sha256=1NJVZTClBPcpFtTa_lgb-ACEbBuNa5UGeO46ShP4TTA,8269
|
|
49
|
-
queue_func-0.2.2.dist-info/WHEEL,sha256=L-WvLdSBvJWHXv5zrQxDvNRnIgoBzLUpcbuiw19KCuQ,88
|
|
50
|
-
queue_func-0.2.2.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|