queue-func 0.1__py3-none-any.whl

queue_func/pool_executor/base.py

@@ -0,0 +1,31 @@
+# coding: utf-8
+
+import abc
+import asyncio
+import typing as t
+
+
+class BasePoolExecutor(metaclass=abc.ABCMeta):
+
+    __slots__ = (
+        "_maxsize",
+    )
+
+    def __init__(self, maxsize: int):
+
+        self._maxsize = maxsize
+
+    @abc.abstractmethod
+    async def submit(
+            self,
+            coro: t.Callable[..., t.Coroutine],
+            coro_args: tuple[t.Any, ...],
+            coro_kwargs: dict[str, t.Any],
+            **kwargs) -> asyncio.Future:
+
+        pass
+
+    @abc.abstractmethod
+    async def join(self) -> None:
+
+        pass

queue_func/pool_executor/general.py

@@ -0,0 +1,42 @@
+# coding: utf-8
+
+import asyncio
+import typing as t
+
+from queue_func.pool_executor.base import BasePoolExecutor
+from queue_func.pool_executor.utils import JoinableSemaphore
+
+
+class PoolExecutor(BasePoolExecutor):
+
+    __slots__ = (
+        "_sem",
+    )
+
+    def __init__(self, maxsize: int):
+        super().__init__(maxsize)
+        self._sem = JoinableSemaphore(self._maxsize)
+
+    async def submit(
+            self,
+            coro: t.Callable[..., t.Coroutine],
+            coro_args: tuple[t.Any, ...],
+            coro_kwargs: dict[str, t.Any],
+            **kwargs) -> asyncio.Future:
+
+        await self._sem.acquire()
+        return asyncio.gather(self._consume(coro, coro_args, coro_kwargs))
+
+    async def _consume(
+            self,
+            coro: t.Callable[..., t.Coroutine],
+            coro_args: tuple[t.Any, ...],
+            coro_kwargs: dict[str, t.Any]) -> None:
+        try:
+            await coro(*coro_args, **coro_kwargs)
+        finally:
+            self._sem.release()
+
+    async def join(self) -> None:
+
+        await self._sem.join()

queue_func/pool_executor/utils.py

@@ -0,0 +1,49 @@
+# coding: utf-8
+
+import asyncio
+
+
+class JoinableSemaphore:
+
+    __slots__ = (
+        "_maxsize",
+        "_count",
+        "_sem",
+        "_event",
+    )
+
+    def __init__(self, maxsize: int):
+
+        self._maxsize = maxsize
+        self._count = 0
+        self._sem = asyncio.Semaphore(self._maxsize)
+        self._event = self._init_event()
+
+    @staticmethod
+    def _init_event() -> "asyncio.Event":
+
+        event = asyncio.Event()
+        event.set()
+        return event
+
+    async def acquire(self) -> bool:
+
+        is_ok = await self._sem.acquire()
+        if is_ok:
+            self._count += 1
+            # block until set
+            self._event.clear()
+        return is_ok
+
+    def release(self) -> None:
+
+        self._sem.release()
+        self._count = max(0, self._count - 1)
+        if self._count == 0:
+            # not block at all
+            self._event.set()
+
+    async def join(self) -> None:
+
+        while self._count > 0:
+            await self._event.wait()

queue_func/producer/__init__.py

@@ -0,0 +1 @@
+# coding: utf-8

queue_func/producer/base.py

@@ -0,0 +1,36 @@
+# coding: utf-8
+
+import abc
+import typing as t
+
+from queue_func.message.base import BaseMessage, Packed
+
+
+class BaseProducer(t.Generic[Packed], metaclass=abc.ABCMeta):
+
+    __slots__ = (
+        "_message",
+    )
+
+    def __init__(self, message: BaseMessage[Packed]):
+
+        self._message = message
+
+    async def publish(
+            self,
+            coro_args: tuple[t.Any],
+            coro_kwargs: dict[str, t.Any] | None = None,
+            **kwargs) -> None:
+
+        message = self._message.pack(coro_args, coro_kwargs)
+        await self._publish(message, **kwargs)
+
+    @abc.abstractmethod
+    async def _publish(self, data: Packed, **kwargs) -> None:
+        pass
+
+    async def stop(self) -> None:
+        pass
+
+    async def join(self) -> None:
+        pass

queue_func/producer/queue.py

@@ -0,0 +1,30 @@
+# coding: utf-8
+import asyncio
+
+from queue_func.message.base import BaseMessage, Packed
+from queue_func.producer.base import BaseProducer
+
+
+class Producer(BaseProducer[Packed]):
+
+    __slots__ = (
+        "_queue",
+        "_publish_timeout",
+    )
+
+    def __init__(
+            self,
+            queue: asyncio.Queue[Packed],
+            publish_timeout: float | None,
+            message: BaseMessage[Packed]):
+
+        super().__init__(message)
+        self._queue = queue
+        self._publish_timeout = publish_timeout
+
+    async def _publish(self, data: Packed, **kwargs) -> None:
+
+        if timeout := self._publish_timeout:
+            await asyncio.wait_for(self._queue.put(data), timeout)
+            return None
+        await self._queue.put(data)

queue_func/serializer/__init__.py

@@ -0,0 +1 @@
+# coding: utf-8

queue_func/serializer/base.py

@@ -0,0 +1,38 @@
+# coding: utf-8
+
+import abc
+import typing as t
+
+__all__ = (
+    "BaseSerializer",
+)
+
+
+class BaseSerializer(metaclass=abc.ABCMeta):
+
+    @t.overload
+    def dumps(self, data: t.Any, return_bytes: t.Literal[True]) -> bytes: ...
+    @t.overload
+    def dumps(self, data: t.Any, return_bytes: t.Literal[False]) -> str: ...
+    @t.overload
+    def dumps(self, data: t.Any, return_bytes: bool) -> t.Union[str, bytes]: ...
+
+    def dumps(self, data: t.Any, return_bytes: bool = False) -> t.Union[str, bytes]:
+
+        if return_bytes:
+            return self._dumps_to_bytes(data)
+        return self._dumps_to_str(data)
+
+    @abc.abstractmethod
+    def _dumps_to_str(self, data: t.Any) -> str:
+        pass
+
+    @abc.abstractmethod
+    def _dumps_to_bytes(self, data: t.Any) -> bytes:
+
+        pass
+
+    @abc.abstractmethod
+    def loads(self, data: t.Any) -> t.Any:
+
+        pass

queue_func/serializer/json_ser.py

@@ -0,0 +1,25 @@
+# coding: utf-8
+
+import json
+import typing as t
+
+from queue_func.serializer.base import BaseSerializer
+from queue_func.serializer.utils import default as _default
+
+
+class JSONSerializer(BaseSerializer):
+
+    def __init__(self, default: t.Optional[t.Callable[[t.Any], t.Any]] = None):
+        self._default = default or _default
+
+    def _dumps_to_str(self, data: t.Any) -> str:
+
+        return json.dumps(data, default=self._default)
+
+    def _dumps_to_bytes(self, data: t.Any) -> bytes:
+
+        return self._dumps_to_str(data).encode("utf-8")
+
+    def loads(self, data: t.Union[str, bytes]) -> t.Any:
+
+        return json.loads(data)

queue_func/serializer/orjson_ser.py

@@ -0,0 +1,30 @@
+# coding: utf-8
+
+import typing as t
+
+import orjson
+
+from queue_func.serializer.base import BaseSerializer
+from queue_func.serializer.utils import default as _default
+
+__all__ = (
+    "OrJSONSerializer",
+)
+
+
+class OrJSONSerializer(BaseSerializer):
+
+    def __init__(self, default: t.Optional[t.Callable[[t.Any], t.Any]] = None):
+        self._default = default or _default
+
+    def _dumps_to_str(self, data: t.Any) -> str:
+
+        return self._dumps_to_bytes(data).decode("utf-8")
+
+    def _dumps_to_bytes(self, data: t.Any) -> bytes:
+
+        return orjson.dumps(data, default=self._default)
+
+    def loads(self, data: t.Union[str, bytes]) -> t.Any:
+
+        return orjson.loads(data)

queue_func/serializer/utils.py

@@ -0,0 +1,16 @@
+# coding: utf-8
+
+import typing as t
+from decimal import Decimal
+from datetime import datetime, date
+
+
+def default(o: t.Any) -> t.Any:
+
+    if isinstance(o, Decimal):
+        return str(o)
+    if isinstance(o, datetime):
+        return o.strftime("%Y-%m-%d %H:%M:%S")
+    if isinstance(o, date):
+        return o.strftime("%Y-%m-%d")
+    raise TypeError(f"Object of type {o.__class__.__name__} is not JSON serializable")

queue_func/task/__init__.py

@@ -0,0 +1 @@
+# coding: utf-8

queue_func/task/base.py

@@ -0,0 +1,28 @@
+# coding: utf-8
+
+from abc import ABCMeta, abstractmethod
+
+
+class BaseTask(metaclass=ABCMeta):
+
+    @abstractmethod
+    def running(self) -> bool:
+        """Return True if the task is currently executing."""
+
+        pass
+
+    @abstractmethod
+    def done(self) -> bool:
+        """Return True if the task was cancelled or finished executing."""
+
+        pass
+
+    @abstractmethod
+    def cancel(self) -> bool:
+        """Cancel the task if possible.
+
+        Returns True if the task was cancelled, False otherwise. A task
+        cannot be cancelled if it is running or has already completed.
+        """
+
+        pass

queue_func/task/thread.py

@@ -0,0 +1,26 @@
+# coding: utf-8
+
+import typing as t
+
+from .base import BaseTask
+
+if t.TYPE_CHECKING:
+    from concurrent.futures import Future
+
+
+class ThreadTask(BaseTask):
+
+    def __init__(self, fut: "Future"):
+        self._fut = fut
+
+    def running(self) -> bool:
+
+        return self._fut.running()
+
+    def done(self) -> bool:
+
+        return self._fut.done()
+
+    def cancel(self) -> bool:
+
+        return self._fut.cancel()

queue_func-0.1.dist-info/METADATA

@@ -0,0 +1,121 @@
+Metadata-Version: 2.4
+Name: queue-func
+Version: 0.1
+Summary: Async coroutine functions as queue-backed tasks (QueueFunc).
+License: Apache-2.0
+License-File: LICENSE
+Author: liuseki
+Author-email: liuseki@163.com
+Requires-Python: >=3.12,<4.0
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Programming Language :: Python :: 3.15
+Requires-Dist: orjson
+Requires-Dist: pydantic (>=2.14.1,<3.0.0)
+Description-Content-Type: text/markdown
+
+# queue-func
+
+**把 async 协程当成队列里的任务来投递与消费**——进程内用内存队列跑通链路，并为 RabbitMQ、Kafka 等后端预留同一套 Broker / Producer / Consumer 抽象。
+
+> 仅支持 **`async def`**，不支持普通同步函数。需要 **Python 3.12+**。
+
+## 关键概念
+
+1. **`@queue`**：把协程包成 `QueueFunc`，提供 `push(args=..., kwargs=...)` 与 `broker`。
+2. **消息**：`(args, kwargs)` 经 `Message` 打包；可选 JSON / orjson 序列化。
+3. **消费**：Consumer 从通道取消息，经池化 `await` 你的协程（`consumer_pool_maxsize` 限并发）。
+
+运行期的 `publish` / `consume` 在 Producer / Consumer 内完成；Broker 主要负责装配与生命周期，不做「万事中转」。
+
+## 快速开始
+
+```bash
+poetry install
+```
+
+单个任务：
+
+```python
+import asyncio
+
+from queue_func.factory import Backend, queue
+
+
+@queue(Backend.QUEUE, enable_message_serializer=False, consume_timeout=0.5, maxsize=1000)
+async def process_order(order_id: int):
+    ...
+
+
+async def main():
+    await process_order.broker.start_consumer()
+    await process_order.push(args=(42,))
+    await process_order.broker.stop_consumer()
+    await process_order.broker.join_consumer()
+
+
+asyncio.run(main())
+```
+
+- `push` → `broker.publish(coro_args=..., coro_kwargs=...)`
+- 仍可直接 `await process_order(1)` 做本地调用（不经队列）
+
+多个 `QueueFunc`（各自独立队列）用 **`QueueFuncRuntime`** 统一启停；`start()` 按注册列表**反序**启动 consumer（下游先就绪，便于链式 `push`）：
+
+```python
+from queue_func.factory import Backend, QueueFuncRuntime, queue
+
+# task_a 内 await task_b.push(...)，task_b 内 await task_c.push(...) 等
+
+
+async def main():
+    runtime = QueueFuncRuntime([task_a, task_b, task_c])
+    await runtime.start()
+    await task_a.push(args=("job-1",))
+    await runtime.stop()
+```
+
+可运行示例（a → b → c 链式投递 + 控制台日志）：
+
+```bash
+poetry run python examples/demo_queue.py
+```
+
+需要自行组装通道时，使用 `Config` + `queue_func.broker.queue.Broker`（见 `queue_func/config/`）。
+
+## 使用场景
+
+| 场景 | 说明 |
+|------|------|
+| 同进程解耦 + 控并发 | Handler 里 `push`，后台 consumer 跑重逻辑；`maxsize` / `publish_timeout` 做背压 |
+| 多段流水线 | 多个 `QueueFunc`，在协程里继续 `push` 到下一环；用 `QueueFuncRuntime` 管生命周期 |
+| 先单机、后换 MQ | 打开序列化后消息已是 bytes，后续可换 Broker 实现，协程签名尽量不变 |
+| 可测 | Producer / Consumer / `push` 副作用分开测；`tests/` 有集成示例 |
+
+**规划中**：`Backend` 除内存队列外的 RabbitMQ / Kafka（**当前仅 `Backend.QUEUE` 可用**）。
+
+**不太适合**：同步任务、分布式调度/结果后端/Celery 级平台、无界堆积不限流、纯 CPU 密集（应另配进程池或 worker）。
+
+## 项目结构
+
+```
+queue_func/
+  broker/ producer/ consumer/ message/ config/
+  factory/          # Backend、queue、QueueFunc、QueueFuncRuntime
+  pool_executor/ serializer/
+examples/demo_queue.py
+```
+
+## 开发与测试
+
+```bash
+pytest tests/ -q
+```
+
+## 许可证
+
+Apache-2.0
+

queue_func-0.1.dist-info/RECORD

@@ -0,0 +1,40 @@
+queue_func/__init__.py,sha256=7DxiEpT1g6hMwW1hBF17OOFRJ3DW03giEu6rgvRCTHY,17
+queue_func/broker/__init__.py,sha256=7DxiEpT1g6hMwW1hBF17OOFRJ3DW03giEu6rgvRCTHY,17
+queue_func/broker/base.py,sha256=Y0MMZGSHfm54LOWkso6OaP5UmRD0Gs94QYa6eyY8aHk,2192
+queue_func/broker/queue.py,sha256=yG2C_DQ5FhxTTPDw3lNI-jKLRAGEkQeSG750lzg2uoc,1353
+queue_func/config/__init__.py,sha256=7DxiEpT1g6hMwW1hBF17OOFRJ3DW03giEu6rgvRCTHY,17
+queue_func/config/general.py,sha256=_G-gfatX8Sfhb9vbEo-RPgOjcn_ctTgn5yDA0xdbZ2Y,1039
+queue_func/config/queue.py,sha256=A8pTbKUaee-653eeby4jqpxdwEwg6EowzPRrHh-1fRo,337
+queue_func/config/utils.py,sha256=DOZnwPkjMsfBVfZVkHEEK1sa-ZVbPjn9gwHFAjTn2UA,263
+queue_func/consumer/__init__.py,sha256=7DxiEpT1g6hMwW1hBF17OOFRJ3DW03giEu6rgvRCTHY,17
+queue_func/consumer/base.py,sha256=wuCPoB0zUgxhF3GBoKF33Zq43LJFIugzoiTRQeccNrE,2889
+queue_func/consumer/queue.py,sha256=bDf3qYCCTnT3CFazWdz3ivvWUsvqm0UpKa9qibrqi54,1579
+queue_func/factory/__init__.py,sha256=7sSCKz_-rx29nJuNFAZlvKro8PuruvZ2ArGcDFFKNb8,269
+queue_func/factory/backend.py,sha256=dWfxozQsm_uTaVyCgVRL8llUzTFQLmYeJRx3fjq1rng,148
+queue_func/factory/builders/__init__.py,sha256=149GwvM0jyWl-CmdjBb8XRH1LFibofGA_6MbxuaE1Mc,187
+queue_func/factory/builders/queue.py,sha256=Yxa8ySiqZurFrCSl5m9OBS6yKv6MF-joyTepkYbDdio,534
+queue_func/factory/deco.py,sha256=cD_aQssAGgvOgC9ioJNTLBT_JaHTnYYnKgNAXdJc100,2834
+queue_func/factory/registry.py,sha256=RN8PgcfkpM1pDFCYqOab-AYq6379IVEyAurtkgtDHwM,1027
+queue_func/factory/runtime.py,sha256=A5CUSVRi4YtlRnroOc-FVeWEbPgEGQ8A-gR49ZgRz8c,958
+queue_func/message/__init__.py,sha256=7DxiEpT1g6hMwW1hBF17OOFRJ3DW03giEu6rgvRCTHY,17
+queue_func/message/base.py,sha256=cVQwFDKWpX5v7MVYzPYko5LPHg9txsJ2vBqkHtjwyNo,877
+queue_func/message/general.py,sha256=GU9Enr1UMyM1cFV4yIdM0qVTFPlG3260AWIMxkfnC1U,842
+queue_func/pool_executor/__init__.py,sha256=7DxiEpT1g6hMwW1hBF17OOFRJ3DW03giEu6rgvRCTHY,17
+queue_func/pool_executor/base.py,sha256=reYgENOa9bRMBKfzvlPP_aTMPSXFCZaNStn6vPwnWak,585
+queue_func/pool_executor/general.py,sha256=MzYZUKNom-xcoop_BBJpma2LPuV3pnnpyWKno5KWa-s,1110
+queue_func/pool_executor/utils.py,sha256=FQR6nE2IVPk_rgfEzQEh6ZeelTc3iqeJegKZxN7EM0c,1042
+queue_func/producer/__init__.py,sha256=7DxiEpT1g6hMwW1hBF17OOFRJ3DW03giEu6rgvRCTHY,17
+queue_func/producer/base.py,sha256=oVJvFTz1nCGGSEl76Xj-_Z03_pxIZSveU2Zstk1pH-E,809
+queue_func/producer/queue.py,sha256=dlrTh88j0344PG5vhhobyKLrTG825H5dSZI3DW5HOfc,791
+queue_func/serializer/__init__.py,sha256=7DxiEpT1g6hMwW1hBF17OOFRJ3DW03giEu6rgvRCTHY,17
+queue_func/serializer/base.py,sha256=yBIVEZ3XImKOuFAqnlu5trrbV9pDvTC_2nF7tkBHawI,913
+queue_func/serializer/json_ser.py,sha256=S3HTxWDLSuyKi5g9SBLigNxUCpbmGyJwBqmOYqIcVMM,661
+queue_func/serializer/orjson_ser.py,sha256=NAYd74TCjg8lPhbcCrNNO_qVHEPt89SJqgR3j0Lk7xI,716
+queue_func/serializer/utils.py,sha256=ozUCUw6-8eMqYqljFcykoEb-NaLAEyBj3dj0YcAnuyY,436
+queue_func/task/__init__.py,sha256=7DxiEpT1g6hMwW1hBF17OOFRJ3DW03giEu6rgvRCTHY,17
+queue_func/task/base.py,sha256=JblRVlT5go9ztFXsv0-e25U7Lz27yUIOHe4oszCNBHA,649
+queue_func/task/thread.py,sha256=OIGG0H_z_pDWYHvjSCX-PyvGovuob8wJaaUaUg6PD1M,443
+queue_func-0.1.dist-info/licenses/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
+queue_func-0.1.dist-info/METADATA,sha256=xnUtrdkPN84tLNrQjRKxELL6O1C2KhIvscLIcwvWuKk,3876
+queue_func-0.1.dist-info/WHEEL,sha256=L-WvLdSBvJWHXv5zrQxDvNRnIgoBzLUpcbuiw19KCuQ,88
+queue_func-0.1.dist-info/RECORD,,

queue_func-0.1.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: poetry-core 2.5.0
+Root-Is-Purelib: true
+Tag: py3-none-any