async-task-kit 0.1.0__tar.gz → 0.1.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {async_task_kit-0.1.0 → async_task_kit-0.1.2}/PKG-INFO +2 -1
- async_task_kit-0.1.2/async_task_kit/consumer/__init__.py +6 -0
- async_task_kit-0.1.2/async_task_kit/consumer/base.py +11 -0
- async_task_kit-0.1.2/async_task_kit/consumer/coroutine.py +97 -0
- async_task_kit-0.1.2/async_task_kit/consumer/process.py +107 -0
- async_task_kit-0.1.2/async_task_kit/consumer/thread.py +107 -0
- async_task_kit-0.1.2/async_task_kit/core/__init__.py +0 -0
- async_task_kit-0.1.2/async_task_kit/core/processor.py +19 -0
- async_task_kit-0.1.2/async_task_kit/core/rabbitmq.py +229 -0
- async_task_kit-0.1.2/async_task_kit/utils/__init__.py +0 -0
- async_task_kit-0.1.2/async_task_kit/utils/env_loader.py +55 -0
- async_task_kit-0.1.2/async_task_kit/utils/logger.py +36 -0
- {async_task_kit-0.1.0 → async_task_kit-0.1.2}/async_task_kit.egg-info/PKG-INFO +2 -1
- async_task_kit-0.1.2/async_task_kit.egg-info/SOURCES.txt +20 -0
- {async_task_kit-0.1.0 → async_task_kit-0.1.2}/async_task_kit.egg-info/requires.txt +1 -0
- {async_task_kit-0.1.0 → async_task_kit-0.1.2}/pyproject.toml +9 -4
- async_task_kit-0.1.0/async_task_kit.egg-info/SOURCES.txt +0 -9
- {async_task_kit-0.1.0 → async_task_kit-0.1.2}/LICENSE +0 -0
- {async_task_kit-0.1.0 → async_task_kit-0.1.2}/README.md +0 -0
- {async_task_kit-0.1.0 → async_task_kit-0.1.2}/async_task_kit/__init__.py +0 -0
- {async_task_kit-0.1.0 → async_task_kit-0.1.2}/async_task_kit.egg-info/dependency_links.txt +0 -0
- {async_task_kit-0.1.0 → async_task_kit-0.1.2}/async_task_kit.egg-info/top_level.txt +0 -0
- {async_task_kit-0.1.0 → async_task_kit-0.1.2}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: async-task-kit
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.2
|
|
4
4
|
Summary: A powerful async task processing kit based on RabbitMQ with Coroutine, Thread, and Process support.
|
|
5
5
|
Author-email: realwrtoff <realwrtoff@gmail.com>
|
|
6
6
|
Classifier: Programming Language :: Python :: 3
|
|
@@ -10,6 +10,7 @@ Requires-Python: >=3.12
|
|
|
10
10
|
Description-Content-Type: text/markdown
|
|
11
11
|
License-File: LICENSE
|
|
12
12
|
Requires-Dist: aio-pika>=9.4.0
|
|
13
|
+
Requires-Dist: asyncpg>=0.31.0
|
|
13
14
|
Requires-Dist: python-dotenv>=1.0.0
|
|
14
15
|
Dynamic: license-file
|
|
15
16
|
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import json
|
|
3
|
+
import logging
|
|
4
|
+
|
|
5
|
+
from .base import BaseConsumer
|
|
6
|
+
from ..core.processor import TaskProcessor
|
|
7
|
+
from ..core.rabbitmq import RabbitMQ
|
|
8
|
+
|
|
9
|
+
logger = logging.getLogger(__name__)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class CoroutineConsumer(BaseConsumer):
|
|
13
|
+
def __init__(
|
|
14
|
+
self,
|
|
15
|
+
amqp_url: str,
|
|
16
|
+
queue_name: str,
|
|
17
|
+
processor: TaskProcessor,
|
|
18
|
+
concurrency: int = 1,
|
|
19
|
+
max_retry: int = 3,
|
|
20
|
+
retry_delay: int = 30,
|
|
21
|
+
):
|
|
22
|
+
self.amqp_url = amqp_url
|
|
23
|
+
self.queue_name = queue_name
|
|
24
|
+
self.processor = processor
|
|
25
|
+
self.concurrency = concurrency
|
|
26
|
+
self.max_retry = max_retry
|
|
27
|
+
self.retry_delay = retry_delay
|
|
28
|
+
|
|
29
|
+
self._rmq = RabbitMQ(amqp_url)
|
|
30
|
+
self._stop_event = asyncio.Event()
|
|
31
|
+
|
|
32
|
+
async def start(self):
|
|
33
|
+
await self._rmq.init()
|
|
34
|
+
logger.info(f"[CoroutineConsumer] start | queue={self.queue_name} | concurrency={self.concurrency}")
|
|
35
|
+
|
|
36
|
+
tasks = [asyncio.create_task(self._consume()) for _ in range(self.concurrency)]
|
|
37
|
+
await asyncio.gather(*tasks)
|
|
38
|
+
|
|
39
|
+
async def _consume(self):
|
|
40
|
+
while not self._stop_event.is_set():
|
|
41
|
+
try:
|
|
42
|
+
msg = await self._rmq.pop(self.queue_name)
|
|
43
|
+
if not msg:
|
|
44
|
+
continue
|
|
45
|
+
|
|
46
|
+
logger.info({"event": "task_received", "queue": self.queue_name})
|
|
47
|
+
task = json.loads(msg.body)
|
|
48
|
+
count_val = msg.headers.get("count", 0)
|
|
49
|
+
if isinstance(count_val, bytes):
|
|
50
|
+
retry = int(count_val.decode())
|
|
51
|
+
elif isinstance(count_val, (int, str)):
|
|
52
|
+
retry = int(count_val)
|
|
53
|
+
else:
|
|
54
|
+
retry = 0
|
|
55
|
+
|
|
56
|
+
result = await self.processor.process(task)
|
|
57
|
+
await self._rmq.ack(msg)
|
|
58
|
+
|
|
59
|
+
logger.info({"event": "task_ack", "queue": self.queue_name})
|
|
60
|
+
|
|
61
|
+
if result:
|
|
62
|
+
await self.processor.callback(task, result)
|
|
63
|
+
logger.info({"event": "task_success", "queue": self.queue_name})
|
|
64
|
+
else:
|
|
65
|
+
await self._retry(task, retry)
|
|
66
|
+
|
|
67
|
+
except Exception as e:
|
|
68
|
+
logger.error(f"consume error: {e}", exc_info=True)
|
|
69
|
+
logger.error({"event": "task_exception", "queue": self.queue_name,"error": str(e)})
|
|
70
|
+
await asyncio.sleep(0.5)
|
|
71
|
+
|
|
72
|
+
async def _retry(self, task, current_retry):
|
|
73
|
+
next_retry = current_retry + 1
|
|
74
|
+
|
|
75
|
+
# 超过最大重试次数 → 失败
|
|
76
|
+
if next_retry > self.max_retry:
|
|
77
|
+
logger.error({"event": "task_failed", "queue": self.queue_name})
|
|
78
|
+
return
|
|
79
|
+
|
|
80
|
+
# 重试入队
|
|
81
|
+
await self._rmq.push(
|
|
82
|
+
self.queue_name,
|
|
83
|
+
task,
|
|
84
|
+
count=next_retry,
|
|
85
|
+
seconds=self.retry_delay,
|
|
86
|
+
max_retry=self.max_retry
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
# 重试日志
|
|
90
|
+
logger.warning(
|
|
91
|
+
{"event": "task_retry", "queue": self.queue_name, "current_retry": current_retry,"next_retry": next_retry}
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
async def stop(self):
|
|
95
|
+
self._stop_event.set()
|
|
96
|
+
await self._rmq.close()
|
|
97
|
+
logger.info(f"[CoroutineConsumer] stopped | queue={self.queue_name}")
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import json
|
|
3
|
+
import logging
|
|
4
|
+
import multiprocessing
|
|
5
|
+
|
|
6
|
+
from .base import BaseConsumer
|
|
7
|
+
from ..core.processor import TaskProcessor
|
|
8
|
+
from ..core.rabbitmq import RabbitMQ
|
|
9
|
+
|
|
10
|
+
logger = logging.getLogger(__name__)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class ProcessConsumer(BaseConsumer):
|
|
14
|
+
def __init__(
|
|
15
|
+
self,
|
|
16
|
+
amqp_url: str,
|
|
17
|
+
queue_name: str,
|
|
18
|
+
processor: TaskProcessor,
|
|
19
|
+
concurrency: int = 1,
|
|
20
|
+
max_retry: int = 3,
|
|
21
|
+
retry_delay: int = 30,
|
|
22
|
+
):
|
|
23
|
+
self.amqp_url = amqp_url
|
|
24
|
+
self.queue_name = queue_name
|
|
25
|
+
self.processor = processor
|
|
26
|
+
self.concurrency = concurrency
|
|
27
|
+
self.max_retry = max_retry
|
|
28
|
+
self.retry_delay = retry_delay
|
|
29
|
+
|
|
30
|
+
self._stop_event = multiprocessing.Event()
|
|
31
|
+
|
|
32
|
+
async def start(self):
|
|
33
|
+
logger.info(f"[ProcessConsumer] start | queue={self.queue_name} | concurrency={self.concurrency}")
|
|
34
|
+
|
|
35
|
+
for _ in range(self.concurrency):
|
|
36
|
+
p = multiprocessing.Process(target=self._process_worker, daemon=True)
|
|
37
|
+
p.start()
|
|
38
|
+
|
|
39
|
+
while not self._stop_event.is_set():
|
|
40
|
+
await asyncio.sleep(1)
|
|
41
|
+
|
|
42
|
+
def _process_worker(self):
|
|
43
|
+
loop = asyncio.new_event_loop()
|
|
44
|
+
asyncio.set_event_loop(loop)
|
|
45
|
+
|
|
46
|
+
# 每个进程自己的 RMQ 客户端
|
|
47
|
+
rmq = RabbitMQ(self.amqp_url)
|
|
48
|
+
loop.run_until_complete(rmq.init())
|
|
49
|
+
|
|
50
|
+
try:
|
|
51
|
+
loop.run_until_complete(self._process_consume(rmq))
|
|
52
|
+
finally:
|
|
53
|
+
loop.run_until_complete(rmq.close())
|
|
54
|
+
loop.close()
|
|
55
|
+
|
|
56
|
+
async def _process_consume(self, rmq):
|
|
57
|
+
while not self._stop_event.is_set():
|
|
58
|
+
try:
|
|
59
|
+
msg = await rmq.pop(self.queue_name)
|
|
60
|
+
if not msg:
|
|
61
|
+
continue
|
|
62
|
+
|
|
63
|
+
logger.info({"event": "task_received", "queue": self.queue_name})
|
|
64
|
+
task = json.loads(msg.body)
|
|
65
|
+
retry = int(msg.headers.get("count", 0))
|
|
66
|
+
|
|
67
|
+
result = await self.processor.process(task)
|
|
68
|
+
await rmq.ack(msg)
|
|
69
|
+
|
|
70
|
+
logger.info({"event": "task_ack", "queue": self.queue_name})
|
|
71
|
+
|
|
72
|
+
if result:
|
|
73
|
+
await self.processor.callback(task, result)
|
|
74
|
+
logger.info({"event": "task_success", "queue": self.queue_name})
|
|
75
|
+
else:
|
|
76
|
+
await self._retry(rmq, task, retry)
|
|
77
|
+
|
|
78
|
+
except Exception as e:
|
|
79
|
+
logger.error(f"consume error: {e}", exc_info=True)
|
|
80
|
+
logger.error({"event": "task_exception", "queue": self.queue_name,"error": str(e)})
|
|
81
|
+
await asyncio.sleep(0.5)
|
|
82
|
+
|
|
83
|
+
async def _retry(self, rmq, task, current_retry):
|
|
84
|
+
next_retry = current_retry + 1
|
|
85
|
+
|
|
86
|
+
# 超过最大重试次数 → 最终失败
|
|
87
|
+
if next_retry > self.max_retry:
|
|
88
|
+
logger.error({"event": "task_failed", "queue": self.queue_name})
|
|
89
|
+
return
|
|
90
|
+
|
|
91
|
+
# 重试入队
|
|
92
|
+
await rmq.push(
|
|
93
|
+
self.queue_name,
|
|
94
|
+
task,
|
|
95
|
+
count=next_retry,
|
|
96
|
+
seconds=self.retry_delay,
|
|
97
|
+
max_retry=self.max_retry
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
# 重试日志
|
|
101
|
+
logger.warning(
|
|
102
|
+
{"event": "task_retry", "queue": self.queue_name, "current_retry": current_retry,"next_retry": next_retry}
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
async def stop(self):
|
|
106
|
+
self._stop_event.set()
|
|
107
|
+
logger.info(f"[ProcessConsumer] stopped | queue={self.queue_name}")
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import json
|
|
3
|
+
import logging
|
|
4
|
+
import threading
|
|
5
|
+
|
|
6
|
+
from .base import BaseConsumer
|
|
7
|
+
from ..core.processor import TaskProcessor
|
|
8
|
+
from ..core.rabbitmq import RabbitMQ
|
|
9
|
+
|
|
10
|
+
logger = logging.getLogger(__name__)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class ThreadConsumer(BaseConsumer):
|
|
14
|
+
def __init__(
|
|
15
|
+
self,
|
|
16
|
+
amqp_url: str,
|
|
17
|
+
queue_name: str,
|
|
18
|
+
processor: TaskProcessor,
|
|
19
|
+
concurrency: int = 1,
|
|
20
|
+
max_retry: int = 3,
|
|
21
|
+
retry_delay: int = 30,
|
|
22
|
+
):
|
|
23
|
+
self.amqp_url = amqp_url
|
|
24
|
+
self.queue_name = queue_name
|
|
25
|
+
self.processor = processor
|
|
26
|
+
self.concurrency = concurrency
|
|
27
|
+
self.max_retry = max_retry
|
|
28
|
+
self.retry_delay = retry_delay
|
|
29
|
+
|
|
30
|
+
self._stop_event = threading.Event()
|
|
31
|
+
|
|
32
|
+
async def start(self):
|
|
33
|
+
logger.info(f"[ThreadConsumer] start | queue={self.queue_name} | concurrency={self.concurrency}")
|
|
34
|
+
|
|
35
|
+
for _ in range(self.concurrency):
|
|
36
|
+
t = threading.Thread(target=self._thread_worker, daemon=True)
|
|
37
|
+
t.start()
|
|
38
|
+
|
|
39
|
+
while not self._stop_event.is_set():
|
|
40
|
+
await asyncio.sleep(1)
|
|
41
|
+
|
|
42
|
+
def _thread_worker(self):
|
|
43
|
+
loop = asyncio.new_event_loop()
|
|
44
|
+
asyncio.set_event_loop(loop)
|
|
45
|
+
|
|
46
|
+
# 每个线程独立的 RMQ 客户端
|
|
47
|
+
rmq = RabbitMQ(self.amqp_url)
|
|
48
|
+
loop.run_until_complete(rmq.init())
|
|
49
|
+
|
|
50
|
+
try:
|
|
51
|
+
loop.run_until_complete(self._thread_consume(rmq))
|
|
52
|
+
finally:
|
|
53
|
+
loop.run_until_complete(rmq.close())
|
|
54
|
+
loop.close()
|
|
55
|
+
|
|
56
|
+
async def _thread_consume(self, rmq):
|
|
57
|
+
while not self._stop_event.is_set():
|
|
58
|
+
try:
|
|
59
|
+
msg = await rmq.pop(self.queue_name)
|
|
60
|
+
if not msg:
|
|
61
|
+
continue
|
|
62
|
+
|
|
63
|
+
logger.info({"event": "task_received", "queue": self.queue_name})
|
|
64
|
+
task = json.loads(msg.body)
|
|
65
|
+
retry = int(msg.headers.get("count", 0))
|
|
66
|
+
|
|
67
|
+
result = await self.processor.process(task)
|
|
68
|
+
await rmq.ack(msg)
|
|
69
|
+
|
|
70
|
+
logger.info({"event": "task_ack", "queue": self.queue_name})
|
|
71
|
+
|
|
72
|
+
if result:
|
|
73
|
+
await self.processor.callback(task, result)
|
|
74
|
+
logger.info({"event": "task_success", "queue": self.queue_name})
|
|
75
|
+
else:
|
|
76
|
+
await self._retry(rmq, task, retry)
|
|
77
|
+
|
|
78
|
+
except Exception as e:
|
|
79
|
+
logger.error(f"consume error: {e}", exc_info=True)
|
|
80
|
+
logger.error({"event": "task_exception", "queue": self.queue_name, "error": str(e)})
|
|
81
|
+
await asyncio.sleep(0.5)
|
|
82
|
+
|
|
83
|
+
async def _retry(self, rmq, task, current_retry):
|
|
84
|
+
next_retry = current_retry + 1
|
|
85
|
+
|
|
86
|
+
# 超过最大重试次数 → 最终失败 task_failed
|
|
87
|
+
if next_retry > self.max_retry:
|
|
88
|
+
logger.error({"event": "task_failed", "queue": self.queue_name})
|
|
89
|
+
return
|
|
90
|
+
|
|
91
|
+
# 重试入队
|
|
92
|
+
await rmq.push(
|
|
93
|
+
self.queue_name,
|
|
94
|
+
task,
|
|
95
|
+
count=next_retry,
|
|
96
|
+
seconds=self.retry_delay,
|
|
97
|
+
max_retry=self.max_retry
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
# 重试日志
|
|
101
|
+
logger.warning(
|
|
102
|
+
{"event": "task_retry", "queue": self.queue_name, "current_retry": current_retry, "next_retry": next_retry}
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
async def stop(self):
|
|
106
|
+
self._stop_event.set()
|
|
107
|
+
logger.info(f"[ThreadConsumer] stopped | queue={self.queue_name}")
|
|
File without changes
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
from abc import ABC, abstractmethod
|
|
2
|
+
from typing import Any, Dict
|
|
3
|
+
|
|
4
|
+
from ..utils.env_loader import EnvLoader
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class TaskProcessor(ABC):
|
|
8
|
+
def __init__(self, task_id: str):
|
|
9
|
+
self.task_id = task_id
|
|
10
|
+
self.env = EnvLoader(task_id)
|
|
11
|
+
self.queue_name = self.env.get('queue_name', self.task_id)
|
|
12
|
+
self.concurrency = self.env.get_int('concurrency', 1)
|
|
13
|
+
|
|
14
|
+
@abstractmethod
|
|
15
|
+
async def process(self, task: Dict[str, Any]) -> Any:
|
|
16
|
+
pass
|
|
17
|
+
|
|
18
|
+
async def callback(self, task: Dict[str, Any], result: Any) -> None:
|
|
19
|
+
return
|
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import json
|
|
3
|
+
import logging
|
|
4
|
+
from abc import ABC, abstractmethod
|
|
5
|
+
from typing import Any, Optional
|
|
6
|
+
|
|
7
|
+
import aio_pika
|
|
8
|
+
from aio_pika import DeliveryMode, ExchangeType, Message
|
|
9
|
+
from aio_pika.exceptions import (
|
|
10
|
+
QueueEmpty,
|
|
11
|
+
ConnectionClosed,
|
|
12
|
+
ChannelClosed,
|
|
13
|
+
AMQPError
|
|
14
|
+
)
|
|
15
|
+
from aio_pika.pool import Pool
|
|
16
|
+
|
|
17
|
+
logger = logging.getLogger(__name__)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class BasicQueue(ABC):
|
|
21
|
+
@abstractmethod
|
|
22
|
+
async def init(self) -> None:
|
|
23
|
+
raise NotImplementedError
|
|
24
|
+
|
|
25
|
+
@abstractmethod
|
|
26
|
+
async def push(self, queue_name: str, data: Any, **kwargs: Any) -> Any:
|
|
27
|
+
raise NotImplementedError
|
|
28
|
+
|
|
29
|
+
@abstractmethod
|
|
30
|
+
async def pop(self, queue_name: str,** kwargs: Any) -> Any:
|
|
31
|
+
raise NotImplementedError
|
|
32
|
+
|
|
33
|
+
@abstractmethod
|
|
34
|
+
async def ack(self, message: Any) -> None:
|
|
35
|
+
raise NotImplementedError
|
|
36
|
+
|
|
37
|
+
@abstractmethod
|
|
38
|
+
async def nack(self, message: Any, requeue: bool = False) -> None:
|
|
39
|
+
raise NotImplementedError
|
|
40
|
+
|
|
41
|
+
@abstractmethod
|
|
42
|
+
async def close(self) -> None:
|
|
43
|
+
raise NotImplementedError
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class RabbitMQ(BasicQueue):
|
|
47
|
+
DEFAULT_POOL_SIZE = 2
|
|
48
|
+
DEFAULT_CHANNEL_SIZE = 10
|
|
49
|
+
DEFAULT_MAX_RETRY_COUNT = 3
|
|
50
|
+
DEFAULT_DEAD_LETTER_DAYS = 7
|
|
51
|
+
DEFAULT_DELAY_SECONDS = 30
|
|
52
|
+
DEFAULT_POP_TIMEOUT = 1.0
|
|
53
|
+
|
|
54
|
+
def __init__(
|
|
55
|
+
self,
|
|
56
|
+
rabbit_url: str,
|
|
57
|
+
pool_size: int = DEFAULT_POOL_SIZE,
|
|
58
|
+
channel_size: int = DEFAULT_CHANNEL_SIZE,
|
|
59
|
+
heartbeat: int = 60,
|
|
60
|
+
max_retry_count: int = DEFAULT_MAX_RETRY_COUNT,
|
|
61
|
+
):
|
|
62
|
+
self.rabbit_url = rabbit_url
|
|
63
|
+
self.pool_size = pool_size
|
|
64
|
+
self.channel_size = channel_size
|
|
65
|
+
self.heartbeat = heartbeat
|
|
66
|
+
self.max_retry_count = max_retry_count
|
|
67
|
+
|
|
68
|
+
self._connection_pool: Optional[Pool] = None
|
|
69
|
+
self._channel_pool: Optional[Pool] = None
|
|
70
|
+
self._init_lock = asyncio.Lock()
|
|
71
|
+
self._ready = asyncio.Event()
|
|
72
|
+
self._closed = False
|
|
73
|
+
|
|
74
|
+
async def init(self) -> None:
|
|
75
|
+
if self._ready.is_set() or self._closed:
|
|
76
|
+
return
|
|
77
|
+
|
|
78
|
+
async with self._init_lock:
|
|
79
|
+
if self._ready.is_set() or self._closed:
|
|
80
|
+
return
|
|
81
|
+
|
|
82
|
+
try:
|
|
83
|
+
self._connection_pool = Pool(
|
|
84
|
+
self._get_connection, max_size=self.pool_size
|
|
85
|
+
)
|
|
86
|
+
self._channel_pool = Pool(
|
|
87
|
+
self._get_channel, max_size=self.channel_size
|
|
88
|
+
)
|
|
89
|
+
self._ready.set()
|
|
90
|
+
logger.info(f"[RabbitMQ] 初始化成功 pool={self.pool_size} channel={self.channel_size}")
|
|
91
|
+
except Exception as e:
|
|
92
|
+
logger.error(f"[RabbitMQ] 初始化失败: {e}", exc_info=True)
|
|
93
|
+
raise
|
|
94
|
+
|
|
95
|
+
async def _get_connection(self) -> aio_pika.abc.AbstractRobustConnection:
|
|
96
|
+
try:
|
|
97
|
+
return await aio_pika.connect_robust(
|
|
98
|
+
self.rabbit_url, heartbeat=self.heartbeat
|
|
99
|
+
)
|
|
100
|
+
except AMQPError as e:
|
|
101
|
+
logger.error(f"[RabbitMQ] 连接失败: {e}")
|
|
102
|
+
raise
|
|
103
|
+
|
|
104
|
+
async def _get_channel(self) -> aio_pika.abc.AbstractChannel:
|
|
105
|
+
if self._closed:
|
|
106
|
+
raise RuntimeError("client closed")
|
|
107
|
+
|
|
108
|
+
async with self._connection_pool.acquire() as conn:
|
|
109
|
+
ch = await conn.channel()
|
|
110
|
+
await ch.set_qos(prefetch_count=1) # 只设置一次!
|
|
111
|
+
return ch
|
|
112
|
+
|
|
113
|
+
async def push(self, queue_name: str, data: Any,** kwargs: Any) -> Any:
|
|
114
|
+
if self._closed:
|
|
115
|
+
raise RuntimeError("client closed")
|
|
116
|
+
|
|
117
|
+
await self.init()
|
|
118
|
+
count = int(kwargs.get("count", 0))
|
|
119
|
+
durable = kwargs.get("durable", True)
|
|
120
|
+
|
|
121
|
+
async with self._channel_pool.acquire() as channel:
|
|
122
|
+
if 0 < count < self.max_retry_count:
|
|
123
|
+
rk = await self._setup_delay_queue(channel, queue_name,** kwargs)
|
|
124
|
+
elif count >= self.max_retry_count:
|
|
125
|
+
rk = await self._setup_dead_letter_queue(channel, queue_name, **kwargs)
|
|
126
|
+
else:
|
|
127
|
+
rk = await self._setup_normal_queue(channel, queue_name, durable)
|
|
128
|
+
|
|
129
|
+
msg = self._create_message(data, count)
|
|
130
|
+
await channel.default_exchange.publish(msg, routing_key=rk)
|
|
131
|
+
logger.debug(f"[PUSH] queue={queue_name} count={count}")
|
|
132
|
+
return msg
|
|
133
|
+
|
|
134
|
+
async def pop(self, queue_name: str,** kwargs: Any) -> Optional[aio_pika.abc.AbstractIncomingMessage]:
|
|
135
|
+
if self._closed:
|
|
136
|
+
return None
|
|
137
|
+
|
|
138
|
+
await self.init()
|
|
139
|
+
durable = kwargs.get("durable", True)
|
|
140
|
+
|
|
141
|
+
try:
|
|
142
|
+
async with self._channel_pool.acquire() as channel:
|
|
143
|
+
queue = await channel.declare_queue(queue_name, durable=durable)
|
|
144
|
+
return await asyncio.wait_for(
|
|
145
|
+
queue.get(), timeout=self.DEFAULT_POP_TIMEOUT
|
|
146
|
+
)
|
|
147
|
+
except (QueueEmpty, asyncio.TimeoutError):
|
|
148
|
+
return None
|
|
149
|
+
except (ConnectionClosed, ChannelClosed):
|
|
150
|
+
logger.warning("[POP] 连接断开,准备重连")
|
|
151
|
+
self._ready.clear()
|
|
152
|
+
return None
|
|
153
|
+
except Exception as e:
|
|
154
|
+
logger.error(f"[POP] 异常 queue={queue_name}: {e}", exc_info=True)
|
|
155
|
+
return None
|
|
156
|
+
|
|
157
|
+
async def ack(self, message: aio_pika.abc.AbstractIncomingMessage) -> None:
|
|
158
|
+
try:
|
|
159
|
+
if not message.processed:
|
|
160
|
+
await message.ack()
|
|
161
|
+
except Exception as e:
|
|
162
|
+
logger.warning(f"[ACK] 失败: {e}")
|
|
163
|
+
|
|
164
|
+
async def nack(self, message: aio_pika.abc.AbstractIncomingMessage, requeue: bool = False) -> None:
|
|
165
|
+
try:
|
|
166
|
+
if not message.processed:
|
|
167
|
+
await message.nack(requeue=requeue)
|
|
168
|
+
except Exception as e:
|
|
169
|
+
logger.warning(f"[NACK] 失败: {e}")
|
|
170
|
+
|
|
171
|
+
async def _setup_delay_queue(self, channel: aio_pika.abc.AbstractChannel, queue_name: str,** kwargs: Any) -> str:
|
|
172
|
+
durable = kwargs.get("durable", True)
|
|
173
|
+
ex_name = kwargs.get("exchange", "letter-exchange")
|
|
174
|
+
delay = int(kwargs.get("seconds", self.DEFAULT_DELAY_SECONDS))
|
|
175
|
+
ttl = delay * 1000
|
|
176
|
+
|
|
177
|
+
ex = await channel.declare_exchange(ex_name, ExchangeType.DIRECT, durable=durable)
|
|
178
|
+
queue = await channel.declare_queue(queue_name, durable=durable)
|
|
179
|
+
trans_rk = f"{queue_name}_trans_router"
|
|
180
|
+
await queue.bind(ex, routing_key=trans_rk)
|
|
181
|
+
|
|
182
|
+
retry_rk = f"{queue_name}_retry"
|
|
183
|
+
await channel.declare_queue(
|
|
184
|
+
retry_rk,
|
|
185
|
+
durable=durable,
|
|
186
|
+
arguments={
|
|
187
|
+
"x-message-ttl": ttl,
|
|
188
|
+
"x-dead-letter-exchange": ex_name,
|
|
189
|
+
"x-dead-letter-routing-key": trans_rk,
|
|
190
|
+
}
|
|
191
|
+
)
|
|
192
|
+
return retry_rk
|
|
193
|
+
|
|
194
|
+
async def _setup_dead_letter_queue(self, channel: aio_pika.abc.AbstractChannel, queue_name: str,** kwargs: Any) -> str:
|
|
195
|
+
days = int(kwargs.get("days", self.DEFAULT_DEAD_LETTER_DAYS))
|
|
196
|
+
dlq_rk = f"{queue_name}_dlq"
|
|
197
|
+
await channel.declare_queue(
|
|
198
|
+
dlq_rk,
|
|
199
|
+
durable=kwargs.get("durable", True),
|
|
200
|
+
arguments={"x-expires": days * 24 * 3600 * 1000}
|
|
201
|
+
)
|
|
202
|
+
return dlq_rk
|
|
203
|
+
|
|
204
|
+
async def _setup_normal_queue(self, channel: aio_pika.abc.AbstractChannel, queue_name: str, durable: bool) -> str:
|
|
205
|
+
await channel.declare_queue(queue_name, durable=durable)
|
|
206
|
+
return queue_name
|
|
207
|
+
|
|
208
|
+
@staticmethod
|
|
209
|
+
def _create_message(data: Any, count: int) -> Message:
|
|
210
|
+
return Message(
|
|
211
|
+
body=json.dumps(data, ensure_ascii=False).encode(),
|
|
212
|
+
headers={"count": count},
|
|
213
|
+
delivery_mode=DeliveryMode.PERSISTENT
|
|
214
|
+
)
|
|
215
|
+
|
|
216
|
+
async def close(self) -> None:
|
|
217
|
+
if self._closed:
|
|
218
|
+
return
|
|
219
|
+
self._closed = True
|
|
220
|
+
self._ready.clear()
|
|
221
|
+
|
|
222
|
+
try:
|
|
223
|
+
if self._channel_pool:
|
|
224
|
+
await self._channel_pool.close()
|
|
225
|
+
if self._connection_pool:
|
|
226
|
+
await self._connection_pool.close()
|
|
227
|
+
logger.info("[RabbitMQ] 已关闭")
|
|
228
|
+
except Exception as e:
|
|
229
|
+
logger.error(f"[关闭失败] {e}")
|
|
File without changes
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
import os
|
|
3
|
+
from dotenv import load_dotenv
|
|
4
|
+
|
|
5
|
+
load_dotenv()
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class EnvLoader:
|
|
9
|
+
def __init__(self, task_id: str | None = None):
|
|
10
|
+
self.prefix = f"{task_id.upper()}_" if task_id else ""
|
|
11
|
+
|
|
12
|
+
def _candidate_keys(self, key: str) -> list[str]:
|
|
13
|
+
uk = key.upper()
|
|
14
|
+
if self.prefix:
|
|
15
|
+
# 有任务前缀:优先前缀变量,其次全局公共变量
|
|
16
|
+
return [self.prefix + uk, uk]
|
|
17
|
+
# 无task_id,只查全局公共变量
|
|
18
|
+
return [uk]
|
|
19
|
+
|
|
20
|
+
def get(self, key: str, default: str | None = None) -> str | None:
|
|
21
|
+
for k in self._candidate_keys(key):
|
|
22
|
+
val = os.getenv(k)
|
|
23
|
+
if val is not None:
|
|
24
|
+
return val.strip()
|
|
25
|
+
return default
|
|
26
|
+
|
|
27
|
+
def get_int(self, key: str, default: int | None = None) -> int | None:
|
|
28
|
+
raw_val = None
|
|
29
|
+
for k in self._candidate_keys(key):
|
|
30
|
+
raw_val = os.getenv(k)
|
|
31
|
+
if raw_val is not None:
|
|
32
|
+
break
|
|
33
|
+
if raw_val is None:
|
|
34
|
+
return default
|
|
35
|
+
try:
|
|
36
|
+
return int(raw_val.strip())
|
|
37
|
+
except (ValueError, TypeError):
|
|
38
|
+
return default
|
|
39
|
+
|
|
40
|
+
def get_bool(self, key: str, default: bool = False) -> bool:
|
|
41
|
+
raw_val = None
|
|
42
|
+
for k in self._candidate_keys(key):
|
|
43
|
+
raw_val = os.getenv(k)
|
|
44
|
+
if raw_val is not None:
|
|
45
|
+
break
|
|
46
|
+
if raw_val is None:
|
|
47
|
+
return default
|
|
48
|
+
v = raw_val.strip().lower()
|
|
49
|
+
true_set = {"1", "true", "yes", "on", "y"}
|
|
50
|
+
false_set = {"0", "false", "no", "off", "n"}
|
|
51
|
+
if v in true_set:
|
|
52
|
+
return True
|
|
53
|
+
if v in false_set:
|
|
54
|
+
return False
|
|
55
|
+
return default
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import os
|
|
3
|
+
from logging.handlers import TimedRotatingFileHandler
|
|
4
|
+
|
|
5
|
+
def setup_logger(
|
|
6
|
+
name: str = "app",
|
|
7
|
+
log_dir: str = "logs",
|
|
8
|
+
backup_count: int = 7 * 24 # 7天 * 24小时
|
|
9
|
+
):
|
|
10
|
+
os.makedirs(log_dir, exist_ok=True)
|
|
11
|
+
logger = logging.getLogger(name)
|
|
12
|
+
logger.setLevel(logging.INFO)
|
|
13
|
+
logger.handlers.clear()
|
|
14
|
+
|
|
15
|
+
# 按小时切割,保留7天
|
|
16
|
+
file_handler = TimedRotatingFileHandler(
|
|
17
|
+
when="H",
|
|
18
|
+
interval=1,
|
|
19
|
+
backupCount=backup_count,
|
|
20
|
+
encoding="utf-8",
|
|
21
|
+
filename=f"{log_dir}/app.log"
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
# 可观测格式:结构化纯文本(对接日志平台最舒服)
|
|
25
|
+
formatter = logging.Formatter(
|
|
26
|
+
"%(asctime)s | %(levelname)-8s | %(name)s | %(message)s"
|
|
27
|
+
)
|
|
28
|
+
file_handler.setFormatter(formatter)
|
|
29
|
+
|
|
30
|
+
# 控制台输出
|
|
31
|
+
console = logging.StreamHandler()
|
|
32
|
+
console.setFormatter(formatter)
|
|
33
|
+
|
|
34
|
+
logger.addHandler(file_handler)
|
|
35
|
+
logger.addHandler(console)
|
|
36
|
+
return logger
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: async-task-kit
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.2
|
|
4
4
|
Summary: A powerful async task processing kit based on RabbitMQ with Coroutine, Thread, and Process support.
|
|
5
5
|
Author-email: realwrtoff <realwrtoff@gmail.com>
|
|
6
6
|
Classifier: Programming Language :: Python :: 3
|
|
@@ -10,6 +10,7 @@ Requires-Python: >=3.12
|
|
|
10
10
|
Description-Content-Type: text/markdown
|
|
11
11
|
License-File: LICENSE
|
|
12
12
|
Requires-Dist: aio-pika>=9.4.0
|
|
13
|
+
Requires-Dist: asyncpg>=0.31.0
|
|
13
14
|
Requires-Dist: python-dotenv>=1.0.0
|
|
14
15
|
Dynamic: license-file
|
|
15
16
|
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
pyproject.toml
|
|
4
|
+
async_task_kit/__init__.py
|
|
5
|
+
async_task_kit.egg-info/PKG-INFO
|
|
6
|
+
async_task_kit.egg-info/SOURCES.txt
|
|
7
|
+
async_task_kit.egg-info/dependency_links.txt
|
|
8
|
+
async_task_kit.egg-info/requires.txt
|
|
9
|
+
async_task_kit.egg-info/top_level.txt
|
|
10
|
+
async_task_kit/consumer/__init__.py
|
|
11
|
+
async_task_kit/consumer/base.py
|
|
12
|
+
async_task_kit/consumer/coroutine.py
|
|
13
|
+
async_task_kit/consumer/process.py
|
|
14
|
+
async_task_kit/consumer/thread.py
|
|
15
|
+
async_task_kit/core/__init__.py
|
|
16
|
+
async_task_kit/core/processor.py
|
|
17
|
+
async_task_kit/core/rabbitmq.py
|
|
18
|
+
async_task_kit/utils/__init__.py
|
|
19
|
+
async_task_kit/utils/env_loader.py
|
|
20
|
+
async_task_kit/utils/logger.py
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "async-task-kit"
|
|
7
|
-
version = "0.1.
|
|
7
|
+
version = "0.1.2"
|
|
8
8
|
description = "A powerful async task processing kit based on RabbitMQ with Coroutine, Thread, and Process support."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.12"
|
|
@@ -13,7 +13,8 @@ authors = [
|
|
|
13
13
|
]
|
|
14
14
|
dependencies = [
|
|
15
15
|
"aio-pika>=9.4.0",
|
|
16
|
-
"
|
|
16
|
+
"asyncpg>=0.31.0",
|
|
17
|
+
"python-dotenv>=1.0.0",
|
|
17
18
|
]
|
|
18
19
|
classifiers = [
|
|
19
20
|
"Programming Language :: Python :: 3",
|
|
@@ -21,5 +22,9 @@ classifiers = [
|
|
|
21
22
|
"Operating System :: OS Independent",
|
|
22
23
|
]
|
|
23
24
|
|
|
24
|
-
[
|
|
25
|
-
|
|
25
|
+
[dependency-groups]
|
|
26
|
+
dev = ["build", "twine"]
|
|
27
|
+
|
|
28
|
+
[tool.setuptools.packages.find]
|
|
29
|
+
where = ["."]
|
|
30
|
+
include = ["async_task_kit*"]
|
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
LICENSE
|
|
2
|
-
README.md
|
|
3
|
-
pyproject.toml
|
|
4
|
-
async_task_kit/__init__.py
|
|
5
|
-
async_task_kit.egg-info/PKG-INFO
|
|
6
|
-
async_task_kit.egg-info/SOURCES.txt
|
|
7
|
-
async_task_kit.egg-info/dependency_links.txt
|
|
8
|
-
async_task_kit.egg-info/requires.txt
|
|
9
|
-
async_task_kit.egg-info/top_level.txt
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|