async-task-kit 0.1.0__tar.gz → 0.1.1__tar.gz

{async_task_kit-0.1.0 → async_task_kit-0.1.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: async-task-kit
-Version: 0.1.0
+Version: 0.1.1
 Summary: A powerful async task processing kit based on RabbitMQ with Coroutine, Thread, and Process support.
 Author-email: realwrtoff <realwrtoff@gmail.com>
 Classifier: Programming Language :: Python :: 3

async_task_kit-0.1.1/async_task_kit/consumer/__init__.py

@@ -0,0 +1,6 @@
+from .base import BaseConsumer
+from .coroutine import CoroutineConsumer
+from .thread import ThreadConsumer
+from .process import ProcessConsumer
+
+__all__ = ["BaseConsumer", "CoroutineConsumer", "ThreadConsumer", "ProcessConsumer"]

async_task_kit-0.1.1/async_task_kit/consumer/base.py

@@ -0,0 +1,11 @@
+from abc import ABC, abstractmethod
+
+
+class BaseConsumer(ABC):
+    @abstractmethod
+    async def start(self):
+        pass
+
+    @abstractmethod
+    async def stop(self):
+        pass

async_task_kit-0.1.1/async_task_kit/consumer/coroutine.py

@@ -0,0 +1,97 @@
+import asyncio
+import json
+import logging
+
+from .base import BaseConsumer
+from ..core.processor import TaskProcessor
+from ..core.rabbitmq import RabbitMQ
+
+logger = logging.getLogger(__name__)
+
+
+class CoroutineConsumer(BaseConsumer):
+    def __init__(
+        self,
+        amqp_url: str,
+        queue_name: str,
+        processor: TaskProcessor,
+        concurrency: int = 1,
+        max_retry: int = 3,
+        retry_delay: int = 30,
+    ):
+        self.amqp_url = amqp_url
+        self.queue_name = queue_name
+        self.processor = processor
+        self.concurrency = concurrency
+        self.max_retry = max_retry
+        self.retry_delay = retry_delay
+
+        self._rmq = RabbitMQ(amqp_url)
+        self._stop_event = asyncio.Event()
+
+    async def start(self):
+        await self._rmq.init()
+        logger.info(f"[CoroutineConsumer] start | queue={self.queue_name} | concurrency={self.concurrency}")
+
+        tasks = [asyncio.create_task(self._consume()) for _ in range(self.concurrency)]
+        await asyncio.gather(*tasks)
+
+    async def _consume(self):
+        while not self._stop_event.is_set():
+            try:
+                msg = await self._rmq.pop(self.queue_name)
+                if not msg:
+                    continue
+
+                logger.info({"event": "task_received", "queue": self.queue_name})
+                task = json.loads(msg.body)
+                count_val = msg.headers.get("count", 0)
+                if isinstance(count_val, bytes):
+                    retry = int(count_val.decode())
+                elif isinstance(count_val, (int, str)):
+                    retry = int(count_val)
+                else:
+                    retry = 0
+
+                result = await self.processor.process(task)
+                await self._rmq.ack(msg)
+
+                logger.info({"event": "task_ack", "queue": self.queue_name})
+
+                if result:
+                    await self.processor.callback(task, result)
+                    logger.info({"event": "task_success", "queue": self.queue_name})
+                else:
+                    await self._retry(task, retry)
+
+            except Exception as e:
+                logger.error(f"consume error: {e}", exc_info=True)
+                logger.error({"event": "task_exception", "queue": self.queue_name,"error": str(e)})
+                await asyncio.sleep(0.5)
+
+    async def _retry(self, task, current_retry):
+        next_retry = current_retry + 1
+
+        # 超过最大重试次数 → 失败
+        if next_retry > self.max_retry:
+            logger.error({"event": "task_failed", "queue": self.queue_name})
+            return
+
+        # 重试入队
+        await self._rmq.push(
+            self.queue_name,
+            task,
+            count=next_retry,
+            seconds=self.retry_delay,
+            max_retry=self.max_retry
+        )
+
+        # 重试日志
+        logger.warning(
+            {"event": "task_retry", "queue": self.queue_name, "current_retry": current_retry,"next_retry": next_retry}
+        )
+
+    async def stop(self):
+        self._stop_event.set()
+        await self._rmq.close()
+        logger.info(f"[CoroutineConsumer] stopped | queue={self.queue_name}")

async_task_kit-0.1.1/async_task_kit/consumer/process.py

@@ -0,0 +1,107 @@
+import asyncio
+import json
+import logging
+import multiprocessing
+
+from .base import BaseConsumer
+from ..core.processor import TaskProcessor
+from ..core.rabbitmq import RabbitMQ
+
+logger = logging.getLogger(__name__)
+
+
+class ProcessConsumer(BaseConsumer):
+    def __init__(
+        self,
+        amqp_url: str,
+        queue_name: str,
+        processor: TaskProcessor,
+        concurrency: int = 1,
+        max_retry: int = 3,
+        retry_delay: int = 30,
+    ):
+        self.amqp_url = amqp_url
+        self.queue_name = queue_name
+        self.processor = processor
+        self.concurrency = concurrency
+        self.max_retry = max_retry
+        self.retry_delay = retry_delay
+
+        self._stop_event = multiprocessing.Event()
+
+    async def start(self):
+        logger.info(f"[ProcessConsumer] start | queue={self.queue_name} | concurrency={self.concurrency}")
+
+        for _ in range(self.concurrency):
+            p = multiprocessing.Process(target=self._process_worker, daemon=True)
+            p.start()
+
+        while not self._stop_event.is_set():
+            await asyncio.sleep(1)
+
+    def _process_worker(self):
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+
+        # 每个进程自己的 RMQ 客户端
+        rmq = RabbitMQ(self.amqp_url)
+        loop.run_until_complete(rmq.init())
+
+        try:
+            loop.run_until_complete(self._process_consume(rmq))
+        finally:
+            loop.run_until_complete(rmq.close())
+            loop.close()
+
+    async def _process_consume(self, rmq):
+        while not self._stop_event.is_set():
+            try:
+                msg = await rmq.pop(self.queue_name)
+                if not msg:
+                    continue
+
+                logger.info({"event": "task_received", "queue": self.queue_name})
+                task = json.loads(msg.body)
+                retry = int(msg.headers.get("count", 0))
+
+                result = await self.processor.process(task)
+                await rmq.ack(msg)
+
+                logger.info({"event": "task_ack", "queue": self.queue_name})
+
+                if result:
+                    await self.processor.callback(task, result)
+                    logger.info({"event": "task_success", "queue": self.queue_name})
+                else:
+                    await self._retry(rmq, task, retry)
+
+            except Exception as e:
+                logger.error(f"consume error: {e}", exc_info=True)
+                logger.error({"event": "task_exception", "queue": self.queue_name,"error": str(e)})
+                await asyncio.sleep(0.5)
+
+    async def _retry(self, rmq, task, current_retry):
+        next_retry = current_retry + 1
+
+        # 超过最大重试次数 → 最终失败
+        if next_retry > self.max_retry:
+            logger.error({"event": "task_failed", "queue": self.queue_name})
+            return
+
+        # 重试入队
+        await rmq.push(
+            self.queue_name,
+            task,
+            count=next_retry,
+            seconds=self.retry_delay,
+            max_retry=self.max_retry
+        )
+
+        # 重试日志
+        logger.warning(
+            {"event": "task_retry", "queue": self.queue_name, "current_retry": current_retry,"next_retry": next_retry}
+        )
+
+    async def stop(self):
+        self._stop_event.set()
+        logger.info(f"[ProcessConsumer] stopped | queue={self.queue_name}")

async_task_kit-0.1.1/async_task_kit/consumer/thread.py

@@ -0,0 +1,107 @@
+import asyncio
+import json
+import logging
+import threading
+
+from .base import BaseConsumer
+from ..core.processor import TaskProcessor
+from ..core.rabbitmq import RabbitMQ
+
+logger = logging.getLogger(__name__)
+
+
+class ThreadConsumer(BaseConsumer):
+    def __init__(
+        self,
+        amqp_url: str,
+        queue_name: str,
+        processor: TaskProcessor,
+        concurrency: int = 1,
+        max_retry: int = 3,
+        retry_delay: int = 30,
+    ):
+        self.amqp_url = amqp_url
+        self.queue_name = queue_name
+        self.processor = processor
+        self.concurrency = concurrency
+        self.max_retry = max_retry
+        self.retry_delay = retry_delay
+
+        self._stop_event = threading.Event()
+
+    async def start(self):
+        logger.info(f"[ThreadConsumer] start | queue={self.queue_name} | concurrency={self.concurrency}")
+
+        for _ in range(self.concurrency):
+            t = threading.Thread(target=self._thread_worker, daemon=True)
+            t.start()
+
+        while not self._stop_event.is_set():
+            await asyncio.sleep(1)
+
+    def _thread_worker(self):
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+
+        # 每个线程独立的 RMQ 客户端
+        rmq = RabbitMQ(self.amqp_url)
+        loop.run_until_complete(rmq.init())
+
+        try:
+            loop.run_until_complete(self._thread_consume(rmq))
+        finally:
+            loop.run_until_complete(rmq.close())
+            loop.close()
+
+    async def _thread_consume(self, rmq):
+        while not self._stop_event.is_set():
+            try:
+                msg = await rmq.pop(self.queue_name)
+                if not msg:
+                    continue
+
+                logger.info({"event": "task_received", "queue": self.queue_name})
+                task = json.loads(msg.body)
+                retry = int(msg.headers.get("count", 0))
+
+                result = await self.processor.process(task)
+                await rmq.ack(msg)
+
+                logger.info({"event": "task_ack", "queue": self.queue_name})
+
+                if result:
+                    await self.processor.callback(task, result)
+                    logger.info({"event": "task_success", "queue": self.queue_name})
+                else:
+                    await self._retry(rmq, task, retry)
+
+            except Exception as e:
+                logger.error(f"consume error: {e}", exc_info=True)
+                logger.error({"event": "task_exception", "queue": self.queue_name, "error": str(e)})
+                await asyncio.sleep(0.5)
+
+    async def _retry(self, rmq, task, current_retry):
+        next_retry = current_retry + 1
+
+        # 超过最大重试次数 → 最终失败 task_failed
+        if next_retry > self.max_retry:
+            logger.error({"event": "task_failed", "queue": self.queue_name})
+            return
+
+        # 重试入队
+        await rmq.push(
+            self.queue_name,
+            task,
+            count=next_retry,
+            seconds=self.retry_delay,
+            max_retry=self.max_retry
+        )
+
+        # 重试日志
+        logger.warning(
+            {"event": "task_retry", "queue": self.queue_name, "current_retry": current_retry, "next_retry": next_retry}
+        )
+
+    async def stop(self):
+        self._stop_event.set()
+        logger.info(f"[ThreadConsumer] stopped | queue={self.queue_name}")

async_task_kit-0.1.1/async_task_kit/core/processor.py

@@ -0,0 +1,19 @@
+from abc import ABC, abstractmethod
+from typing import Any, Dict
+
+from ..utils.env_loader import EnvLoader
+
+
+class TaskProcessor(ABC):
+    def __init__(self, task_id: str):
+        self.task_id = task_id
+        self.env = EnvLoader(task_id)
+        self.queue_name = self.env.get('queue_name', self.task_id)
+        self.concurrency = self.env.get_int('concurrency', 1)
+
+    @abstractmethod
+    async def process(self, task: Dict[str, Any]) -> Any:
+        pass
+
+    async def callback(self, task: Dict[str, Any], result: Any) -> None:
+        return

async_task_kit-0.1.1/async_task_kit/core/rabbitmq.py

@@ -0,0 +1,229 @@
+import asyncio
+import json
+import logging
+from abc import ABC, abstractmethod
+from typing import Any, Optional
+
+import aio_pika
+from aio_pika import DeliveryMode, ExchangeType, Message
+from aio_pika.exceptions import (
+    QueueEmpty,
+    ConnectionClosed,
+    ChannelClosed,
+    AMQPError
+)
+from aio_pika.pool import Pool
+
+logger = logging.getLogger(__name__)
+
+
+class BasicQueue(ABC):
+    @abstractmethod
+    async def init(self) -> None:
+        raise NotImplementedError
+
+    @abstractmethod
+    async def push(self, queue_name: str, data: Any, **kwargs: Any) -> Any:
+        raise NotImplementedError
+
+    @abstractmethod
+    async def pop(self, queue_name: str,** kwargs: Any) -> Any:
+        raise NotImplementedError
+
+    @abstractmethod
+    async def ack(self, message: Any) -> None:
+        raise NotImplementedError
+
+    @abstractmethod
+    async def nack(self, message: Any, requeue: bool = False) -> None:
+        raise NotImplementedError
+
+    @abstractmethod
+    async def close(self) -> None:
+        raise NotImplementedError
+
+
+class RabbitMQ(BasicQueue):
+    DEFAULT_POOL_SIZE = 2
+    DEFAULT_CHANNEL_SIZE = 10
+    DEFAULT_MAX_RETRY_COUNT = 3
+    DEFAULT_DEAD_LETTER_DAYS = 7
+    DEFAULT_DELAY_SECONDS = 30
+    DEFAULT_POP_TIMEOUT = 1.0
+
+    def __init__(
+        self,
+        rabbit_url: str,
+        pool_size: int = DEFAULT_POOL_SIZE,
+        channel_size: int = DEFAULT_CHANNEL_SIZE,
+        heartbeat: int = 60,
+        max_retry_count: int = DEFAULT_MAX_RETRY_COUNT,
+    ):
+        self.rabbit_url = rabbit_url
+        self.pool_size = pool_size
+        self.channel_size = channel_size
+        self.heartbeat = heartbeat
+        self.max_retry_count = max_retry_count
+
+        self._connection_pool: Optional[Pool] = None
+        self._channel_pool: Optional[Pool] = None
+        self._init_lock = asyncio.Lock()
+        self._ready = asyncio.Event()
+        self._closed = False
+
+    async def init(self) -> None:
+        if self._ready.is_set() or self._closed:
+            return
+
+        async with self._init_lock:
+            if self._ready.is_set() or self._closed:
+                return
+
+            try:
+                self._connection_pool = Pool(
+                    self._get_connection, max_size=self.pool_size
+                )
+                self._channel_pool = Pool(
+                    self._get_channel, max_size=self.channel_size
+                )
+                self._ready.set()
+                logger.info(f"[RabbitMQ] 初始化成功 pool={self.pool_size} channel={self.channel_size}")
+            except Exception as e:
+                logger.error(f"[RabbitMQ] 初始化失败: {e}", exc_info=True)
+                raise
+
+    async def _get_connection(self) -> aio_pika.abc.AbstractRobustConnection:
+        try:
+            return await aio_pika.connect_robust(
+                self.rabbit_url, heartbeat=self.heartbeat
+            )
+        except AMQPError as e:
+            logger.error(f"[RabbitMQ] 连接失败: {e}")
+            raise
+
+    async def _get_channel(self) -> aio_pika.abc.AbstractChannel:
+        if self._closed:
+            raise RuntimeError("client closed")
+
+        async with self._connection_pool.acquire() as conn:
+            ch = await conn.channel()
+            await ch.set_qos(prefetch_count=1)  # 只设置一次！
+            return ch
+
+    async def push(self, queue_name: str, data: Any,** kwargs: Any) -> Any:
+        if self._closed:
+            raise RuntimeError("client closed")
+
+        await self.init()
+        count = int(kwargs.get("count", 0))
+        durable = kwargs.get("durable", True)
+
+        async with self._channel_pool.acquire() as channel:
+            if 0 < count < self.max_retry_count:
+                rk = await self._setup_delay_queue(channel, queue_name,** kwargs)
+            elif count >= self.max_retry_count:
+                rk = await self._setup_dead_letter_queue(channel, queue_name, **kwargs)
+            else:
+                rk = await self._setup_normal_queue(channel, queue_name, durable)
+
+            msg = self._create_message(data, count)
+            await channel.default_exchange.publish(msg, routing_key=rk)
+            logger.debug(f"[PUSH] queue={queue_name} count={count}")
+            return msg
+
+    async def pop(self, queue_name: str,** kwargs: Any) -> Optional[aio_pika.abc.AbstractIncomingMessage]:
+        if self._closed:
+            return None
+
+        await self.init()
+        durable = kwargs.get("durable", True)
+
+        try:
+            async with self._channel_pool.acquire() as channel:
+                queue = await channel.declare_queue(queue_name, durable=durable)
+                return await asyncio.wait_for(
+                    queue.get(), timeout=self.DEFAULT_POP_TIMEOUT
+                )
+        except (QueueEmpty, asyncio.TimeoutError):
+            return None
+        except (ConnectionClosed, ChannelClosed):
+            logger.warning("[POP] 连接断开，准备重连")
+            self._ready.clear()
+            return None
+        except Exception as e:
+            logger.error(f"[POP] 异常 queue={queue_name}: {e}", exc_info=True)
+            return None
+
+    async def ack(self, message: aio_pika.abc.AbstractIncomingMessage) -> None:
+        try:
+            if not message.processed:
+                await message.ack()
+        except Exception as e:
+            logger.warning(f"[ACK] 失败: {e}")
+
+    async def nack(self, message: aio_pika.abc.AbstractIncomingMessage, requeue: bool = False) -> None:
+        try:
+            if not message.processed:
+                await message.nack(requeue=requeue)
+        except Exception as e:
+            logger.warning(f"[NACK] 失败: {e}")
+
+    async def _setup_delay_queue(self, channel: aio_pika.abc.AbstractChannel, queue_name: str,** kwargs: Any) -> str:
+        durable = kwargs.get("durable", True)
+        ex_name = kwargs.get("exchange", "letter-exchange")
+        delay = int(kwargs.get("seconds", self.DEFAULT_DELAY_SECONDS))
+        ttl = delay * 1000
+
+        ex = await channel.declare_exchange(ex_name, ExchangeType.DIRECT, durable=durable)
+        queue = await channel.declare_queue(queue_name, durable=durable)
+        trans_rk = f"{queue_name}_trans_router"
+        await queue.bind(ex, routing_key=trans_rk)
+
+        retry_rk = f"{queue_name}_retry"
+        await channel.declare_queue(
+            retry_rk,
+            durable=durable,
+            arguments={
+                "x-message-ttl": ttl,
+                "x-dead-letter-exchange": ex_name,
+                "x-dead-letter-routing-key": trans_rk,
+            }
+        )
+        return retry_rk
+
+    async def _setup_dead_letter_queue(self, channel: aio_pika.abc.AbstractChannel, queue_name: str,** kwargs: Any) -> str:
+        days = int(kwargs.get("days", self.DEFAULT_DEAD_LETTER_DAYS))
+        dlq_rk = f"{queue_name}_dlq"
+        await channel.declare_queue(
+            dlq_rk,
+            durable=kwargs.get("durable", True),
+            arguments={"x-expires": days * 24 * 3600 * 1000}
+        )
+        return dlq_rk
+
+    async def _setup_normal_queue(self, channel: aio_pika.abc.AbstractChannel, queue_name: str, durable: bool) -> str:
+        await channel.declare_queue(queue_name, durable=durable)
+        return queue_name
+
+    @staticmethod
+    def _create_message(data: Any, count: int) -> Message:
+        return Message(
+            body=json.dumps(data, ensure_ascii=False).encode(),
+            headers={"count": count},
+            delivery_mode=DeliveryMode.PERSISTENT
+        )
+
+    async def close(self) -> None:
+        if self._closed:
+            return
+        self._closed = True
+        self._ready.clear()
+
+        try:
+            if self._channel_pool:
+                await self._channel_pool.close()
+            if self._connection_pool:
+                await self._connection_pool.close()
+            logger.info("[RabbitMQ] 已关闭")
+        except Exception as e:
+            logger.error(f"[关闭失败] {e}")

async_task_kit-0.1.1/async_task_kit/utils/env_loader.py

@@ -0,0 +1,16 @@
+import os
+from dotenv import load_dotenv
+load_dotenv()
+
+class EnvLoader:
+    def __init__(self, task_id=None):
+        self.prefix = f"{task_id}_".upper() if task_id else ""
+
+    def get(self, key, default=None):
+        return os.getenv(f"{self.prefix}{key}", default)
+
+    def get_int(self, key, default=1):
+        try:
+            return int(os.getenv(f"{self.prefix}{key}", default))
+        except:
+            return default

async_task_kit-0.1.1/async_task_kit/utils/logger.py

@@ -0,0 +1,36 @@
+import logging
+import os
+from logging.handlers import TimedRotatingFileHandler
+
+def setup_logger(
+    name: str = "app",
+    log_dir: str = "logs",
+    backup_count: int = 7 * 24  # 7天 * 24小时
+):
+    os.makedirs(log_dir, exist_ok=True)
+    logger = logging.getLogger(name)
+    logger.setLevel(logging.INFO)
+    logger.handlers.clear()
+
+    # 按小时切割，保留7天
+    file_handler = TimedRotatingFileHandler(
+        when="H",
+        interval=1,
+        backupCount=backup_count,
+        encoding="utf-8",
+        filename=f"{log_dir}/app.log"
+    )
+
+    # 可观测格式：结构化纯文本（对接日志平台最舒服）
+    formatter = logging.Formatter(
+        "%(asctime)s | %(levelname)-8s | %(name)s | %(message)s"
+    )
+    file_handler.setFormatter(formatter)
+
+    # 控制台输出
+    console = logging.StreamHandler()
+    console.setFormatter(formatter)
+
+    logger.addHandler(file_handler)
+    logger.addHandler(console)
+    return logger

{async_task_kit-0.1.0 → async_task_kit-0.1.1}/async_task_kit.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: async-task-kit
-Version: 0.1.0
+Version: 0.1.1
 Summary: A powerful async task processing kit based on RabbitMQ with Coroutine, Thread, and Process support.
 Author-email: realwrtoff <realwrtoff@gmail.com>
 Classifier: Programming Language :: Python :: 3

async_task_kit-0.1.1/async_task_kit.egg-info/SOURCES.txt

@@ -0,0 +1,20 @@
+LICENSE
+README.md
+pyproject.toml
+async_task_kit/__init__.py
+async_task_kit.egg-info/PKG-INFO
+async_task_kit.egg-info/SOURCES.txt
+async_task_kit.egg-info/dependency_links.txt
+async_task_kit.egg-info/requires.txt
+async_task_kit.egg-info/top_level.txt
+async_task_kit/consumer/__init__.py
+async_task_kit/consumer/base.py
+async_task_kit/consumer/coroutine.py
+async_task_kit/consumer/process.py
+async_task_kit/consumer/thread.py
+async_task_kit/core/__init__.py
+async_task_kit/core/processor.py
+async_task_kit/core/rabbitmq.py
+async_task_kit/utils/__init__.py
+async_task_kit/utils/env_loader.py
+async_task_kit/utils/logger.py

{async_task_kit-0.1.0 → async_task_kit-0.1.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "async-task-kit"
-version = "0.1.0"
+version = "0.1.1"
 description = "A powerful async task processing kit based on RabbitMQ with Coroutine, Thread, and Process support."
 readme = "README.md"
 requires-python = ">=3.12"
@@ -21,5 +21,6 @@ classifiers = [
     "Operating System :: OS Independent",
 ]
 
-[tool.setuptools]
-packages = ["async_task_kit"]
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["async_task_kit*"]

async_task_kit-0.1.0/async_task_kit.egg-info/SOURCES.txt

@@ -1,9 +0,0 @@
-LICENSE
-README.md
-pyproject.toml
-async_task_kit/__init__.py
-async_task_kit.egg-info/PKG-INFO
-async_task_kit.egg-info/SOURCES.txt
-async_task_kit.egg-info/dependency_links.txt
-async_task_kit.egg-info/requires.txt
-async_task_kit.egg-info/top_level.txt