flask-async-celery 0.1.0__py3-none-any.whl

flask_async_celery/__init__.py

@@ -0,0 +1,9 @@
+from .extension import AsyncCelery
+from .task import AsyncTask
+
+__all__ = [
+    "AsyncCelery",
+    "AsyncTask",
+]
+
+__version__ = "0.1.0"

flask_async_celery/bootstep.py

@@ -0,0 +1,84 @@
+from __future__ import annotations
+
+import logging
+
+from celery import bootsteps
+
+logger = logging.getLogger(__name__)
+
+
+class AsyncIOBootStep(bootsteps.StartStopStep):
+    """
+    Worker bootstep for the Flask Async Celery integration.
+
+    The Celery Pool bootstep is required so this component starts
+    after the worker pool has been created.
+
+    The actual asyncio event loop is owned by AsyncIOPool/
+    AsyncExecutor. This bootstep provides the worker-level hook
+    that we will use for lifecycle and backpressure handling.
+    """
+
+    requires = {"celery.worker.components:Pool"}
+
+    def __init__(self, worker, **kwargs):
+        self.worker = worker
+        super().__init__(worker, **kwargs)
+
+    def start(self, worker) -> None:
+        """
+        Called when the Celery worker starts.
+        """
+
+        pool = getattr(worker, "pool", None)
+
+        logger.info(
+            "AsyncIO worker bootstep started: pool=%s",
+            type(pool).__name__ if pool else None,
+        )
+
+        # The custom pool owns the AsyncExecutor.
+        #
+        # Keep a reference on the worker so other bootsteps/components
+        # can access it without importing the pool implementation.
+        if pool is not None and hasattr(pool, "executor"):
+            worker.async_executor = pool.async_executor
+
+        worker.asyncio_enabled = True
+
+    def stop(self, worker) -> None:
+        """
+        Called during normal worker shutdown.
+        """
+
+        logger.info("AsyncIO worker bootstep stopping")
+
+        worker.asyncio_enabled = False
+
+    def terminate(self, worker) -> None:
+        """
+        Called during forced worker termination.
+        """
+
+        logger.info("AsyncIO worker bootstep terminating")
+
+        worker.asyncio_enabled = False
+
+    def info(self, worker):
+        """
+        Expose basic information through worker inspection.
+        """
+
+        executor = getattr(worker, "async_executor", None)
+
+        if executor is None:
+            return {
+                "asyncio-enabled": False,
+            }
+
+        return {
+            "asyncio-enabled": True,
+            "asyncio-running": executor.running,
+            "asyncio-available": executor.available,
+            "asyncio-max-tasks": executor.max_tasks,
+        }

flask_async_celery/executor.py

@@ -0,0 +1,283 @@
+from __future__ import annotations
+
+import asyncio
+import logging
+import threading
+from concurrent.futures import Future
+from typing import Any
+from collections.abc import Coroutine
+
+
+logger = logging.getLogger(__name__)
+
+
+class AsyncExecutor:
+    """
+    Persistent asyncio executor.
+
+    One instance runs one asyncio event loop inside one
+    dedicated thread.
+
+    The executor itself is NOT a Celery worker pool.
+    It only knows how to execute coroutines concurrently.
+    """
+
+    def __init__(
+        self,
+        max_tasks: int = 20,
+        thread_name: str = "celery-asyncio",
+    ) -> None:
+
+        if max_tasks < 1:
+            raise ValueError(
+                "max_tasks must be greater than zero"
+            )
+
+        self.max_tasks = max_tasks
+        self.thread_name = thread_name
+
+        self.loop: asyncio.AbstractEventLoop | None = None
+
+        self.thread: threading.Thread | None = None
+
+        self._started = threading.Event()
+
+        self._stopping = threading.Event()
+
+        self._semaphore: asyncio.Semaphore | None = None
+
+        self._running = 0
+
+        self._running_lock = threading.Lock()
+
+    # ------------------------------------------------------------------
+    # Properties
+    # ------------------------------------------------------------------
+
+    @property
+    def running(self) -> int:
+        with self._running_lock:
+            return self._running
+
+    @property
+    def available(self) -> int:
+        return max(
+            0,
+            self.max_tasks - self.running,
+        )
+
+    @property
+    def is_running(self) -> bool:
+        return (
+            self.loop is not None
+            and self.loop.is_running()
+            and not self._stopping.is_set()
+        )
+
+    # ------------------------------------------------------------------
+    # Lifecycle
+    # ------------------------------------------------------------------
+
+    def start(self) -> None:
+        """
+        Start the dedicated asyncio thread.
+        """
+
+        if self.thread and self.thread.is_alive():
+            return
+
+        self._started.clear()
+        self._stopping.clear()
+
+        self.thread = threading.Thread(
+            target=self._run_loop,
+            name=self.thread_name,
+            daemon=True,
+        )
+
+        self.thread.start()
+
+        if not self._started.wait(timeout=10):
+            raise RuntimeError(
+                "AsyncIO event loop failed to start"
+            )
+
+        logger.info(
+            "AsyncIO executor started: "
+            "max_tasks=%s thread=%s",
+            self.max_tasks,
+            self.thread.name,
+        )
+
+    def _run_loop(self) -> None:
+        """
+        Runs inside the dedicated thread.
+        """
+
+        loop = asyncio.new_event_loop()
+
+        self.loop = loop
+
+        asyncio.set_event_loop(loop)
+
+        self._semaphore = asyncio.Semaphore(
+            self.max_tasks
+        )
+
+        self._started.set()
+
+        logger.info(
+            "AsyncIO event loop started"
+        )
+
+        try:
+            loop.run_forever()
+
+        finally:
+            self._shutdown_loop(loop)
+
+    def _shutdown_loop(
+        self,
+        loop: asyncio.AbstractEventLoop,
+    ) -> None:
+
+        logger.info(
+            "Shutting down AsyncIO event loop"
+        )
+
+        try:
+            pending = asyncio.all_tasks(loop)
+
+            for task in pending:
+                task.cancel()
+
+            if pending:
+                loop.run_until_complete(
+                    asyncio.gather(
+                        *pending,
+                        return_exceptions=True,
+                    )
+                )
+
+        except Exception:
+            logger.exception(
+                "Error while shutting down "
+                "AsyncIO tasks"
+            )
+
+        finally:
+            loop.close()
+
+            logger.info(
+                "AsyncIO event loop stopped"
+            )
+
+    # ------------------------------------------------------------------
+    # Submission
+    # ------------------------------------------------------------------
+
+    def submit(
+        self,
+        coroutine: Coroutine[Any, Any, Any],
+    ) -> Future:
+        """
+        Submit a coroutine to the persistent event loop.
+
+        Returns a concurrent.futures.Future.
+        """
+
+        if not self.is_running:
+            raise RuntimeError(
+                "AsyncIO executor is not running"
+            )
+
+        return asyncio.run_coroutine_threadsafe(
+            self._execute(coroutine),
+            self.loop,
+        )
+
+    async def _execute(
+        self,
+        coroutine: Coroutine[Any, Any, Any],
+    ) -> Any:
+
+        if self._semaphore is None:
+            raise RuntimeError(
+                "AsyncIO semaphore is not initialized"
+            )
+
+        async with self._semaphore:
+
+            self._increment_running()
+
+            try:
+                return await coroutine
+
+            finally:
+                self._decrement_running()
+
+    # ------------------------------------------------------------------
+    # Counters
+    # ------------------------------------------------------------------
+
+    def _increment_running(self) -> None:
+
+        with self._running_lock:
+            self._running += 1
+
+            logger.debug(
+                "AsyncIO tasks: %s/%s",
+                self._running,
+                self.max_tasks,
+            )
+
+    def _decrement_running(self) -> None:
+
+        with self._running_lock:
+            self._running -= 1
+
+            logger.debug(
+                "AsyncIO tasks: %s/%s",
+                self._running,
+                self.max_tasks,
+            )
+
+    # ------------------------------------------------------------------
+    # Shutdown
+    # ------------------------------------------------------------------
+
+    def shutdown(
+        self,
+        wait: bool = True,
+        timeout: float = 10,
+    ) -> None:
+
+        if self.loop is None:
+            return
+
+        if not self.loop.is_running():
+            return
+
+        self._stopping.set()
+
+        logger.info(
+            "Stopping AsyncIO executor"
+        )
+
+        self.loop.call_soon_threadsafe(
+            self.loop.stop
+        )
+
+        if wait and self.thread:
+
+            self.thread.join(
+                timeout=timeout
+            )
+
+            if self.thread.is_alive():
+
+                logger.warning(
+                    "AsyncIO thread did not stop "
+                    "within %.1f seconds",
+                    timeout,
+                )

flask_async_celery/extension.py

@@ -0,0 +1,111 @@
+from __future__ import annotations
+
+from typing import Any
+
+from celery import Celery
+from flask import Flask
+
+
+class AsyncCelery:
+    """
+    Flask integration for flask-async-celery.
+
+    Configures a Celery application to use AsyncIOPool and
+    optionally enables Celery consumer-side backpressure.
+    """
+
+    def __init__(
+        self,
+        app: Flask | None = None,
+        *,
+        celery: Celery | None = None,
+        broker_url: str | None = None,
+        result_backend: str | None = None,
+        max_tasks: int = 20,
+        disable_prefetch: bool = True,
+        **celery_options: Any,
+    ) -> None:
+        if max_tasks < 1:
+            raise ValueError("max_tasks must be greater than zero")
+
+        self.app: Flask | None = None
+        self.celery: Celery
+
+        self.max_tasks = max_tasks
+        self.disable_prefetch = disable_prefetch
+
+        if celery is not None:
+            self.celery = celery
+        else:
+            self.celery = Celery(
+                app.import_name if app is not None else "flask_async_celery",
+                broker=broker_url,
+                backend=result_backend,
+                **celery_options,
+            )
+
+        self._configure()
+
+        if app is not None:
+            self.init_app(app)
+
+    def init_app(self, app: Flask) -> None:
+        self.app = app
+
+        app.extensions["async_celery"] = self
+
+        app.config.setdefault(
+            "ASYNC_CELERY_MAX_TASKS",
+            self.max_tasks,
+        )
+
+        app.config.setdefault(
+            "ASYNC_CELERY_DISABLE_PREFETCH",
+            self.disable_prefetch,
+        )
+
+        self.max_tasks = int(
+            app.config["ASYNC_CELERY_MAX_TASKS"]
+        )
+
+        self.disable_prefetch = bool(
+            app.config["ASYNC_CELERY_DISABLE_PREFETCH"]
+        )
+
+        if self.max_tasks < 1:
+            raise ValueError(
+                "ASYNC_CELERY_MAX_TASKS must be greater than zero"
+            )
+
+        self._configure()
+
+    def _configure(self) -> None:
+        self.celery.conf.update(
+            worker_pool="flask_async_celery.pool:AsyncIOPool",
+            worker_concurrency=self.max_tasks,
+            worker_disable_prefetch=self.disable_prefetch,
+        )
+
+    def task(self, *args: Any, **kwargs: Any):
+        """
+        Register a Celery task.
+
+        Usage:
+
+            @celery.task
+            async def task():
+                ...
+
+        or:
+
+            @celery.task(base=AsyncTask)
+            async def task():
+                ...
+        """
+        return self.celery.task(*args, **kwargs)
+
+    def send_task(self, *args: Any, **kwargs: Any):
+        return self.celery.send_task(*args, **kwargs)
+
+    def __getattr__(self, name: str) -> Any:
+        return getattr(self.celery, name)

flask_async_celery/pool.py

@@ -0,0 +1,235 @@
+from __future__ import annotations
+import inspect
+import logging
+import os
+from concurrent.futures import Future, ThreadPoolExecutor, wait
+from typing import Any, Callable
+import time
+from celery.concurrency.base import BasePool, apply_target
+
+from .executor import AsyncExecutor
+from .task import reset_async_executor, set_async_executor
+
+logger = logging.getLogger(__name__)
+
+
+class ApplyResult:
+    """
+    Celery-compatible result wrapper around a Future.
+    """
+
+    def __init__(self, future: Future) -> None:
+        self.f = future
+        self.get = future.result
+
+    def wait(self, timeout: float | None = None) -> None:
+        wait([self.f], timeout)
+
+    def ready(self) -> bool:
+        return self.f.done()
+
+    def successful(self) -> bool:
+        if not self.f.done():
+            return False
+
+        return self.f.exception() is None
+
+
+class AsyncIOPool(BasePool):
+    """
+    Celery execution pool for asyncio tasks.
+
+    There are deliberately TWO execution layers:
+
+        Celery bridge threads
+                |
+                v
+        Celery trace function
+                |
+                v
+        AsyncIO executor
+                |
+                v
+        persistent asyncio event loop
+
+    The bridge threads are necessary because Celery's tracing
+    function is synchronous.
+
+    The asyncio executor is where async task bodies actually run.
+    """
+
+    signal_safe = False
+    is_green = False
+    body_can_be_buffer = True
+
+    def __init__(
+        self,
+        *args: Any,
+        max_tasks: int | None = None,
+        thread_name: str = "celery-asyncio",
+        **kwargs: Any,
+    ) -> None:
+        super().__init__(*args, **kwargs)
+
+        configured_limit = (
+            max_tasks
+            if max_tasks is not None
+            else self.limit
+        )
+
+        if configured_limit is None:
+            configured_limit = 20
+
+        self.max_tasks = int(configured_limit)
+
+        if self.max_tasks < 1:
+            raise ValueError(
+                "AsyncIOPool max_tasks must be greater than zero"
+            )
+
+        # Persistent asyncio event loop.
+        self.async_executor = AsyncExecutor(
+            max_tasks=self.max_tasks,
+            thread_name=thread_name,
+        )
+
+        # Celery tracing must NOT run on the asyncio event-loop
+        # thread. These threads execute Celery's synchronous trace
+        # function.
+        self.executor = ThreadPoolExecutor(
+            max_workers=self.max_tasks,
+            thread_name_prefix="celery-async-bridge",
+        )
+
+        logger.info(
+            "AsyncIOPool initialized: max_tasks=%s",
+            self.max_tasks,
+        )
+
+    @property
+    def num_processes(self) -> int:
+        return self.max_tasks
+
+    def on_start(self) -> None:
+        logger.info(
+            "Starting AsyncIOPool: max_tasks=%s",
+            self.max_tasks,
+        )
+
+        self.async_executor.start()
+
+        super().on_start()
+
+    def on_stop(self) -> None:
+        logger.info("Stopping AsyncIOPool")
+
+        self.executor.shutdown(
+            wait=True,
+            cancel_futures=False,
+        )
+
+        self.async_executor.shutdown(
+            wait=True,
+            timeout=10,
+        )
+
+        super().on_stop()
+
+    def on_terminate(self) -> None:
+        logger.warning("Terminating AsyncIOPool")
+
+        self.executor.shutdown(
+            wait=False,
+            cancel_futures=True,
+        )
+
+        self.async_executor.shutdown(
+            wait=False,
+        )
+
+        super().on_terminate()
+
+    def on_apply(
+        self,
+        target: Callable[..., Any],
+        args: tuple[Any, ...] | None = None,
+        kwargs: dict[str, Any] | None = None,
+        callback: Callable[..., Any] | None = None,
+        accept_callback: Callable[..., Any] | None = None,
+        **options: Any,
+    ) -> ApplyResult:
+        """
+        Submit Celery's tracing function to a bridge thread.
+
+        The bridge thread executes Celery's normal synchronous
+        tracing machinery.
+
+        AsyncTask then detects the coroutine and submits it to
+        the persistent asyncio event loop.
+        """
+
+        args = args or ()
+        kwargs = kwargs or {}
+
+        if not self.async_executor.is_running:
+            raise RuntimeError(
+                "AsyncIO executor is not running"
+            )
+
+        future = self.executor.submit(
+            self._run_celery_target,
+            target,
+            args,
+            kwargs,
+            callback,
+            accept_callback,
+        )
+
+        return ApplyResult(future)
+
+    def _run_celery_target(
+            self,
+            target: Callable[..., Any],
+            args: tuple[Any, ...],
+            kwargs: dict[str, Any],
+            callback: Callable[..., Any] | None,
+            accept_callback: Callable[..., Any] | None,
+    ) -> Any:
+        token = set_async_executor(self.async_executor)
+
+        try:
+            if accept_callback:
+                accept_callback(os.getpid(), time.monotonic())
+
+            result = target(*args, **kwargs)
+
+            # Async Celery target:
+            # run the coroutine on the persistent asyncio event loop.
+            if inspect.isawaitable(result):
+                future = self.async_executor.submit(result)
+                result = future.result()
+
+            if callback:
+                callback(result)
+
+            return result
+
+        except BaseException:
+            raise
+
+        finally:
+            reset_async_executor(token)
+
+    def _get_info(self) -> dict[str, Any]:
+        info = super()._get_info()
+
+        info.update(
+            {
+                "max-concurrency": self.max_tasks,
+                "asyncio-running": self.async_executor.running,
+                "asyncio-available": self.async_executor.available,
+                "bridge-threads": len(self.executor._threads),
+            }
+        )
+
+        return info

flask_async_celery/task.py

@@ -0,0 +1,63 @@
+from __future__ import annotations
+
+import inspect
+from contextvars import ContextVar
+from typing import Any
+
+from celery import Task
+
+
+_async_executor: ContextVar[Any | None] = ContextVar(
+    "flask_async_celery_executor",
+    default=None,
+)
+
+
+def set_async_executor(executor: Any):
+    return _async_executor.set(executor)
+
+
+def reset_async_executor(token) -> None:
+    _async_executor.reset(token)
+
+
+def get_async_executor() -> Any | None:
+    return _async_executor.get()
+
+
+class AsyncTask(Task):
+    abstract = True
+
+    def __call__(self, *args: Any, **kwargs: Any) -> Any:
+        if not inspect.iscoroutinefunction(self.run):
+            return super().__call__(*args, **kwargs)
+
+        executor = get_async_executor()
+        if executor is None:
+            raise RuntimeError(
+                "AsyncTask requires an AsyncExecutor. "
+                "Make sure the task is running inside AsyncIOPool."
+            )
+
+        # Capture the real Celery request while we're still on
+        # the Celery worker/bridge thread.
+        request = self.request
+
+        async def execute_with_request() -> Any:
+            # Restore the Celery request inside the asyncio thread.
+            self.push_request(**request.__dict__)
+
+            try:
+                return await self.run(*args, **kwargs)
+            finally:
+                self.pop_request()
+
+        coroutine = execute_with_request()
+
+        try:
+            future = executor.submit(coroutine)
+            return future.result()
+        except BaseException:
+            if inspect.iscoroutine(coroutine):
+                coroutine.close()
+            raise

flask_async_celery-0.1.0.dist-info/METADATA

@@ -0,0 +1,597 @@
+Metadata-Version: 2.4
+Name: flask-async-celery
+Version: 0.1.0
+Summary: AsyncIO execution pool for Celery with Flask integration
+Author: Mazhar Ali
+License: MIT
+Keywords: celery,asyncio,flask,async,tasks,redis
+Classifier: Development Status :: 3 - Alpha
+Classifier: Framework :: Flask
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Libraries
+Classifier: Topic :: System :: Distributed Computing
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: celery<5.7,>=5.6
+Requires-Dist: Flask>=2.3
+Provides-Extra: redis
+Requires-Dist: redis>=5.0; extra == "redis"
+Provides-Extra: test
+Requires-Dist: pytest>=8.0; extra == "test"
+Requires-Dist: pytest-asyncio>=0.24; extra == "test"
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.24; extra == "dev"
+Requires-Dist: build>=1.2; extra == "dev"
+Requires-Dist: twine>=5.0; extra == "dev"
+
+# Flask Async Celery
+
+Run `async def` Celery tasks on a persistent asyncio event loop with bounded concurrency, Flask integration, and Celery consumer backpressure.
+
+## Features
+
+* Persistent `asyncio` event loop in a dedicated thread per Celery worker process.
+* Run native `async def` Celery tasks.
+* Bounded asynchronous concurrency with `max_tasks`.
+* Celery request context propagation into the asyncio execution thread.
+* `self.retry()` support for async tasks.
+* Normal synchronous Celery tasks continue to work.
+* Redis consumer-side backpressure through Celery's `worker_disable_prefetch`.
+* Flask extension with simple configuration.
+* Graceful asyncio executor shutdown.
+* Compatible with Celery 5.6.x and Python 3.10+.
+
+## Architecture
+
+```text
+                         Redis
+                           │
+                           ▼
+                    Celery Consumer
+                           │
+                 worker_disable_prefetch
+                           │
+                           ▼
+                     AsyncIOPool
+                    max_tasks = N
+                           │
+                           ▼
+                    AsyncExecutor
+                  asyncio.Semaphore(N)
+                           │
+                    Persistent loop
+                           │
+              ┌────────────┼────────────┐
+              ▼            ▼            ▼
+          Async task    Async task    Async task
+```
+
+The package separates Celery's worker execution from asyncio execution:
+
+1. Celery receives and traces the task.
+2. `AsyncIOPool` bridges Celery execution into the asyncio executor.
+3. `AsyncExecutor` owns a persistent asyncio event loop.
+4. A semaphore limits active async tasks.
+5. Celery's Redis `worker_disable_prefetch` option can prevent the consumer from reserving work beyond the available pool capacity.
+
+## Requirements
+
+* Python 3.10+
+* Celery 5.6.x
+* Flask 2.3+
+* Redis when using the Redis broker/result backend and consumer backpressure.
+
+## Installation
+
+From PyPI:
+
+```bash
+pip install flask-async-celery
+```
+
+For Redis support:
+
+```bash
+pip install "flask-async-celery[redis]"
+```
+
+For development and testing:
+
+```bash
+pip install "flask-async-celery[test]"
+```
+
+## Basic Flask Setup
+
+```python
+import asyncio
+
+from flask import Flask
+
+from flask_async_celery import AsyncCelery, AsyncTask
+
+
+app = Flask(__name__)
+
+celery = AsyncCelery(
+    app,
+    broker_url="redis://127.0.0.1:6379/0",
+    result_backend="redis://127.0.0.1:6379/1",
+    max_tasks=5,
+    disable_prefetch=True,
+)
+
+
+@celery.task(base=AsyncTask)
+async def my_task(value):
+    await asyncio.sleep(1)
+    return value * 2
+```
+
+Send the task normally:
+
+```python
+result = my_task.delay(10)
+
+print(result.get(timeout=30))
+# 20
+```
+
+## Flask Configuration
+
+Configuration can be supplied through Flask:
+
+```python
+app.config["ASYNC_CELERY_MAX_TASKS"] = 10
+app.config["ASYNC_CELERY_DISABLE_PREFETCH"] = True
+
+celery = AsyncCelery(
+    app,
+    broker_url="redis://127.0.0.1:6379/0",
+    result_backend="redis://127.0.0.1:6379/1",
+)
+```
+
+Available settings:
+
+| Setting                         | Default | Description                                           |
+| ------------------------------- | ------: | ----------------------------------------------------- |
+| `ASYNC_CELERY_MAX_TASKS`        |    `20` | Maximum number of concurrently executing async tasks. |
+| `ASYNC_CELERY_DISABLE_PREFETCH` |  `True` | Enables Celery consumer-side backpressure.            |
+
+Constructor arguments can also be used directly:
+
+```python
+celery = AsyncCelery(
+    app,
+    max_tasks=10,
+    disable_prefetch=True,
+)
+```
+
+Flask configuration takes precedence over the constructor defaults when the extension is initialized.
+
+## Worker Configuration
+
+Run the worker using the package's custom pool:
+
+```bash
+celery -A your_app.celery worker \
+    -P flask_async_celery.pool:AsyncIOPool \
+    -c 5 \
+    --loglevel=INFO
+```
+
+For example:
+
+```bash
+celery -A your_app.celery worker \
+    -P flask_async_celery.pool:AsyncIOPool \
+    -c 10 \
+    --loglevel=INFO
+```
+
+The `-c` value should match the desired async concurrency.
+
+The custom pool exposes its configured concurrency through `num_processes`, allowing Celery's consumer to use the same capacity when consumer-side prefetch is disabled.
+
+## Concurrency
+
+Set the maximum number of simultaneously executing async tasks with:
+
+```python
+celery = AsyncCelery(
+    app,
+    max_tasks=5,
+)
+```
+
+With:
+
+```text
+max_tasks = 5
+```
+
+the asyncio executor allows at most five active coroutines at once.
+
+Additional work waits for an available execution slot.
+
+This is different from simply creating more threads. The package uses one persistent asyncio event loop and runs async coroutines concurrently on that loop.
+
+## Consumer Backpressure
+
+For Redis, Celery 5.6 supports:
+
+```python
+worker_disable_prefetch = True
+```
+
+The extension enables this by default:
+
+```python
+celery = AsyncCelery(
+    app,
+    max_tasks=5,
+    disable_prefetch=True,
+)
+```
+
+This provides two levels of protection:
+
+```text
+Celery Consumer
+      │
+      │ Don't reserve beyond available capacity
+      ▼
+ AsyncIOPool
+      │
+      │ max_tasks
+      ▼
+AsyncExecutor
+      │
+      │ semaphore
+      ▼
+asyncio tasks
+```
+
+You can disable the consumer-side behavior:
+
+```python
+celery = AsyncCelery(
+    app,
+    max_tasks=5,
+    disable_prefetch=False,
+)
+```
+
+When disabled, the asyncio executor still enforces its own concurrency limit.
+
+### Redis Requirement
+
+`worker_disable_prefetch` is intended for supported Redis worker configurations. If you use another broker, verify that your Celery version and broker transport support this feature before relying on consumer-side backpressure.
+
+The executor-level concurrency limit remains independent of consumer prefetch behavior.
+
+## Async Tasks
+
+Use `AsyncTask` as the base class for native async tasks:
+
+```python
+from flask_async_celery import AsyncTask
+
+
+@celery.task(base=AsyncTask)
+async def fetch_data():
+    await some_async_operation()
+    return "done"
+```
+
+The task can use normal Celery task features:
+
+```python
+@celery.task(
+    base=AsyncTask,
+    bind=True,
+    max_retries=3,
+)
+async def process_item(self, item_id):
+    try:
+        return await process(item_id)
+    except TemporaryError as exc:
+        raise self.retry(
+            exc=exc,
+            countdown=5,
+        )
+```
+
+The Celery request context is propagated from the Celery worker thread into the asyncio execution thread, so task information such as the task ID, retry count, delivery information, and retry context remains available.
+
+## Synchronous Tasks
+
+Normal synchronous tasks can still be used:
+
+```python
+@celery.task
+def sync_task(value):
+    return value * 2
+```
+
+The package does not require every task to be asynchronous.
+
+## Retries
+
+Async `self.retry()` is supported:
+
+```python
+@celery.task(
+    base=AsyncTask,
+    bind=True,
+    max_retries=3,
+)
+async def retrying_task(self):
+    if should_retry():
+        raise self.retry(countdown=5)
+
+    return "success"
+```
+
+The Celery task request is preserved when execution moves from the Celery worker thread to the asyncio event loop. This allows Celery retry metadata and delivery information to remain available to the async task.
+
+## Exceptions
+
+Exceptions raised by an async task propagate through the Celery execution path:
+
+```python
+@celery.task(base=AsyncTask)
+async def failing_task():
+    raise RuntimeError("something went wrong")
+```
+
+Celery remains responsible for task failure state, result handling, retry behavior, and worker-level task tracing.
+
+## Graceful Shutdown
+
+The asyncio executor runs in a dedicated daemon thread.
+
+When the pool stops, the executor:
+
+1. Stops accepting new work.
+2. Stops the asyncio event loop.
+3. Cancels pending asyncio tasks.
+4. Waits for the loop thread to terminate.
+5. Closes the event loop.
+
+## Public API
+
+The main public API is intentionally small:
+
+```python
+from flask_async_celery import AsyncCelery, AsyncTask
+```
+
+### `AsyncCelery`
+
+Flask integration and Celery configuration.
+
+```python
+AsyncCelery(
+    app=None,
+    *,
+    celery=None,
+    broker_url=None,
+    result_backend=None,
+    max_tasks=20,
+    disable_prefetch=True,
+)
+```
+
+### `AsyncTask`
+
+Base class for asynchronous Celery tasks:
+
+```python
+@celery.task(base=AsyncTask)
+async def my_task():
+    ...
+```
+
+## Development
+
+Clone the repository and install the project in editable mode:
+
+```bash
+git clone <repository-url>
+cd flask-async-celery
+
+pip install -e ".[test]"
+```
+
+Run the test suite:
+
+```bash
+pytest -v
+```
+
+The test suite covers:
+
+* asyncio executor concurrency
+* AsyncIOPool execution
+* async task exceptions
+* async retries
+* synchronous retries
+* real Celery worker integration
+* Redis consumer backpressure
+* Flask extension configuration
+* end-to-end Flask/Celery/async execution
+
+## Project Structure
+
+```text
+flask-async-celery/
+├── pyproject.toml
+├── README.md
+├── src/
+│   └── flask_async_celery/
+│       ├── __init__.py
+│       ├── extension.py
+│       ├── executor.py
+│       ├── bootstep.py
+│       ├── pool.py
+│       └── task.py
+└── test/
+    ├── conftest.py
+    ├── test_executor.py
+    ├── test_tasks.py
+    ├── test_backpressure.py
+    ├── test_celery_pool.py
+    ├── test_worker_integration.py
+    ├── test_extension.py
+    └── test_extension_integration.py
+```
+
+## Design Notes
+
+This package does not replace Celery's task tracing and lifecycle handling.
+
+Celery remains responsible for:
+
+* task delivery
+* task acknowledgment
+* retries
+* result state
+* task IDs
+* worker lifecycle
+* task tracing
+
+The package provides the asyncio execution layer and integrates it with Celery's pool interface.
+
+The asyncio event loop is persistent for the lifetime of the worker process rather than creating a new event loop for every task.
+
+### Why a Persistent Event Loop?
+
+Creating a new event loop for every task adds unnecessary setup and teardown overhead.
+
+Instead, each worker process owns one persistent asyncio event loop:
+
+```text
+Celery Worker Process
+│
+├── Celery Consumer
+├── Celery task execution
+├── bridge threads
+│
+└── asyncio event loop thread
+    ├── Task A
+    ├── Task B
+    └── Task C
+```
+
+Async tasks can therefore share the same event loop while still being bounded by `max_tasks`.
+
+### Why Request Propagation?
+
+Celery's request context is associated with the worker execution context.
+
+The asyncio event loop runs in a separate thread, so the package explicitly transfers the current Celery request into that execution context.
+
+This preserves information required by features such as:
+
+```python
+self.request.id
+self.request.retries
+self.request.delivery_info
+self.retry()
+```
+
+The request is pushed before async execution and removed afterward.
+
+## Limitations
+
+### Broker-specific Backpressure
+
+Consumer-side `worker_disable_prefetch` support depends on Celery and the broker transport.
+
+The asyncio executor's own `max_tasks` limit remains the final execution boundary.
+
+### Worker Pool
+
+The worker must use:
+
+```text
+flask_async_celery.pool:AsyncIOPool
+```
+
+for the package's asyncio execution model.
+
+### One Event Loop Per Worker Process
+
+Each worker process owns its own asyncio event loop and concurrency limit.
+
+For example:
+
+```bash
+celery -A your_app.celery worker \
+    -P flask_async_celery.pool:AsyncIOPool \
+    -c 5
+```
+
+creates a worker configuration with five execution slots.
+
+If you run multiple worker processes, each process has its own pool and event loop.
+
+## Testing
+
+Run the complete test suite:
+
+```bash
+pytest -v
+```
+
+The project currently tests the execution model with real Celery workers in addition to unit-level executor and pool tests.
+
+A successful test run should show all tests passing.
+
+## Building the Package
+
+Install the build tooling:
+
+```bash
+python -m pip install build
+```
+
+Build the source distribution and wheel:
+
+```bash
+python -m build
+```
+
+The generated files will be placed in:
+
+```text
+dist/
+```
+
+You can inspect the generated wheel with:
+
+```bash
+python -m pip install dist/*.whl
+```
+
+## Version
+
+Current version:
+
+```text
+0.1.0
+```
+
+## License
+
+Add the project's license information here.

flask_async_celery-0.1.0.dist-info/RECORD

@@ -0,0 +1,10 @@
+flask_async_celery/__init__.py,sha256=GzLJSzIzc8yRJqvgYLVEKXKonmjiEGnm84TZTTdB4Rw,136
+flask_async_celery/bootstep.py,sha256=b_FZs1h5Y7yyxEfeqWXnUf_1llps7MEseBBTr8vYwsU,2271
+flask_async_celery/executor.py,sha256=UkgVsnKo7hI4scqDzTD__NDwriWJeVmfeTmdbOwdCq4,6527
+flask_async_celery/extension.py,sha256=J0wTGCZNNJNv0ldaLV3NcbDfireHL2vwwQc2ltucH4w,2771
+flask_async_celery/pool.py,sha256=bLLDOSde5dBeEoeBKsEBFomDV8ssImppY3RQscW5HqI,5905
+flask_async_celery/task.py,sha256=VB56Br5J2A7BF8Epu3ilLKOQCyC9FDNsu1E7NgZX5lk,1633
+flask_async_celery-0.1.0.dist-info/METADATA,sha256=8vxvwlmlk__OcxT76elhuC4FNBNV6ClPD3wiKOC1kc4,13645
+flask_async_celery-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+flask_async_celery-0.1.0.dist-info/top_level.txt,sha256=RCoWK-DxiJ1nM8PZ-OvlEd-UqoX5tZscHm_cnjFww6s,19
+flask_async_celery-0.1.0.dist-info/RECORD,,

flask_async_celery-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

flask_async_celery-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+flask_async_celery