celery-fastapi 0.1.3__tar.gz → 0.1.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {celery_fastapi-0.1.3 → celery_fastapi-0.1.5}/LICENSE +0 -0
- {celery_fastapi-0.1.3 → celery_fastapi-0.1.5}/PKG-INFO +60 -11
- {celery_fastapi-0.1.3 → celery_fastapi-0.1.5}/README.md +58 -9
- {celery_fastapi-0.1.3 → celery_fastapi-0.1.5}/celery_fastapi/__init__.py +1 -1
- {celery_fastapi-0.1.3 → celery_fastapi-0.1.5}/celery_fastapi/app.py +2 -0
- {celery_fastapi-0.1.3 → celery_fastapi-0.1.5}/celery_fastapi/cli.py +1 -2
- {celery_fastapi-0.1.3 → celery_fastapi-0.1.5}/celery_fastapi/core.py +433 -2
- {celery_fastapi-0.1.3 → celery_fastapi-0.1.5}/celery_fastapi/py.typed +0 -0
- {celery_fastapi-0.1.3 → celery_fastapi-0.1.5}/celery_fastapi/server.py +0 -0
- {celery_fastapi-0.1.3 → celery_fastapi-0.1.5}/pyproject.toml +16 -4
|
File without changes
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: celery-fastapi
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.5
|
|
4
4
|
Summary: Automatic REST API generation for Celery tasks with FastAPI
|
|
5
5
|
License: MIT
|
|
6
6
|
License-File: LICENSE
|
|
@@ -35,7 +35,7 @@ Provides-Extra: server
|
|
|
35
35
|
Provides-Extra: standard
|
|
36
36
|
Provides-Extra: ujson
|
|
37
37
|
Provides-Extra: uvicorn
|
|
38
|
-
Requires-Dist: celery (>=5.3.0)
|
|
38
|
+
Requires-Dist: celery (>=5.3.0,<=5.6.3)
|
|
39
39
|
Requires-Dist: eventlet (>=0.33.0) ; extra == "eventlet"
|
|
40
40
|
Requires-Dist: fastapi (>=0.100.0)
|
|
41
41
|
Requires-Dist: gevent (>=23.0.0) ; extra == "gevent"
|
|
@@ -60,8 +60,12 @@ Description-Content-Type: text/markdown
|
|
|
60
60
|
[](https://github.com/karailker/celery-fastapi/actions/workflows/ci.yml)
|
|
61
61
|
[](https://badge.fury.io/py/celery-fastapi)
|
|
62
62
|
[](https://pypi.org/project/celery-fastapi/)
|
|
63
|
+
[](https://pepy.tech/projects/celery-fastapi)
|
|
63
64
|
[](https://opensource.org/licenses/MIT)
|
|
64
65
|
|
|
66
|
+

|
|
67
|
+
<!--  -->
|
|
68
|
+
|
|
65
69
|
Automatic REST API generation for Celery tasks with FastAPI. This package seamlessly bridges Celery and FastAPI, automatically creating REST endpoints for all your registered Celery tasks.
|
|
66
70
|
|
|
67
71
|
## Features
|
|
@@ -77,6 +81,9 @@ Automatic REST API generation for Celery tasks with FastAPI. This package seamle
|
|
|
77
81
|
- 🔒 **Production ready** - Full uvicorn/gunicorn support with SSL, workers, and all options
|
|
78
82
|
- ⚡ **Full Celery options** - All task options (countdown, eta, priority, etc.)
|
|
79
83
|
- 🔌 **Pool support** - Compatible with eventlet, gevent, prefork, and solo pools
|
|
84
|
+
- 🧮 **Batch execution** - Submit groups of tasks in a single request via `/tasks/batch`
|
|
85
|
+
- 🛡️ **Input validation** - Pydantic-driven validation on task_name/queue at trust boundary
|
|
86
|
+
- 🔌 **WebSocket streaming** - Live task status updates via `/tasks/{task_id}/ws`
|
|
80
87
|
|
|
81
88
|
## Requirements
|
|
82
89
|
|
|
@@ -160,25 +167,29 @@ uvicorn myapp:app --reload
|
|
|
160
167
|
|
|
161
168
|
```bash
|
|
162
169
|
# Start the server (development)
|
|
163
|
-
celery-fastapi serve
|
|
170
|
+
celery-fastapi serve examples.celery_app:celery_app --port 8000 --reload
|
|
164
171
|
|
|
165
172
|
# Production with multiple workers
|
|
166
|
-
celery-fastapi serve
|
|
173
|
+
celery-fastapi serve examples.celery_app:celery_app -w 4 --host 0.0.0.0
|
|
174
|
+
|
|
175
|
+
# With custom worker hostname (for health checks)
|
|
176
|
+
export CELERY_WORKER_HOSTNAME="celery@worker1"
|
|
177
|
+
celery-fastapi serve examples.celery_app:celery_app --port 8000
|
|
167
178
|
|
|
168
179
|
# With SSL
|
|
169
|
-
celery-fastapi serve
|
|
180
|
+
celery-fastapi serve examples.celery_app:celery_app --ssl-keyfile key.pem --ssl-certfile cert.pem
|
|
170
181
|
|
|
171
182
|
# Using gunicorn (production)
|
|
172
|
-
celery-fastapi serve-gunicorn
|
|
183
|
+
celery-fastapi serve-gunicorn examples.celery_app:celery_app -w 4 -k uvicorn.workers.UvicornWorker
|
|
173
184
|
|
|
174
185
|
# List available routes
|
|
175
|
-
celery-fastapi routes
|
|
186
|
+
celery-fastapi routes examples.celery_app:celery_app
|
|
176
187
|
|
|
177
188
|
# List registered tasks
|
|
178
|
-
celery-fastapi tasks
|
|
189
|
+
celery-fastapi tasks examples.celery_app:celery_app
|
|
179
190
|
|
|
180
191
|
# Show active workers
|
|
181
|
-
celery-fastapi workers
|
|
192
|
+
celery-fastapi workers examples.celery_app:celery_app
|
|
182
193
|
```
|
|
183
194
|
|
|
184
195
|
## API Endpoints
|
|
@@ -272,6 +283,44 @@ GET /queues
|
|
|
272
283
|
POST /purge
|
|
273
284
|
```
|
|
274
285
|
|
|
286
|
+
### Health Check and Monitoring
|
|
287
|
+
|
|
288
|
+
```bash
|
|
289
|
+
# Health check for local Celery worker
|
|
290
|
+
GET /healthz
|
|
291
|
+
|
|
292
|
+
# Response
|
|
293
|
+
{
|
|
294
|
+
"status": "healthy",
|
|
295
|
+
"celery_app": "example_tasks",
|
|
296
|
+
"broker_connected": true,
|
|
297
|
+
"worker_hostname": "celery@worker1",
|
|
298
|
+
"worker_online": true
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
# Ping local Celery worker
|
|
302
|
+
GET /ping
|
|
303
|
+
|
|
304
|
+
# Response
|
|
305
|
+
{
|
|
306
|
+
"worker_hostname": "celery@worker1",
|
|
307
|
+
"online": true,
|
|
308
|
+
"response": {"ok": "pong"}
|
|
309
|
+
}
|
|
310
|
+
```
|
|
311
|
+
|
|
312
|
+
**Note:** Health and ping endpoints automatically discover the local worker using:
|
|
313
|
+
1. `CELERY_WORKER_HOSTNAME` environment variable (recommended for custom hostnames)
|
|
314
|
+
2. Hostname matching (when worker and API share the same hostname)
|
|
315
|
+
3. Single worker fallback (when only one worker has this app's tasks)
|
|
316
|
+
|
|
317
|
+
**For custom worker hostnames**, set the environment variable:
|
|
318
|
+
```bash
|
|
319
|
+
export CELERY_WORKER_HOSTNAME="celery@worker1"
|
|
320
|
+
celery -A examples.celery_app worker --hostname worker1
|
|
321
|
+
celery-fastapi serve examples.celery_app:celery_app --port 8000
|
|
322
|
+
```
|
|
323
|
+
|
|
275
324
|
### List All Tasks
|
|
276
325
|
|
|
277
326
|
```bash
|
|
@@ -351,7 +400,7 @@ Commands:
|
|
|
351
400
|
workers Show active Celery workers
|
|
352
401
|
|
|
353
402
|
# Serve options (uvicorn)
|
|
354
|
-
celery-fastapi serve
|
|
403
|
+
celery-fastapi serve examples.celery_app:celery_app \
|
|
355
404
|
--host 0.0.0.0 \
|
|
356
405
|
--port 8000 \
|
|
357
406
|
--reload \
|
|
@@ -364,7 +413,7 @@ celery-fastapi serve myapp:celery_app \
|
|
|
364
413
|
--forwarded-allow-ips '*'
|
|
365
414
|
|
|
366
415
|
# Serve options (gunicorn)
|
|
367
|
-
celery-fastapi serve-gunicorn
|
|
416
|
+
celery-fastapi serve-gunicorn examples.celery_app:celery_app \
|
|
368
417
|
--bind 0.0.0.0:8000 \
|
|
369
418
|
--workers 4 \
|
|
370
419
|
--worker-class uvicorn.workers.UvicornWorker \
|
|
@@ -3,8 +3,12 @@
|
|
|
3
3
|
[](https://github.com/karailker/celery-fastapi/actions/workflows/ci.yml)
|
|
4
4
|
[](https://badge.fury.io/py/celery-fastapi)
|
|
5
5
|
[](https://pypi.org/project/celery-fastapi/)
|
|
6
|
+
[](https://pepy.tech/projects/celery-fastapi)
|
|
6
7
|
[](https://opensource.org/licenses/MIT)
|
|
7
8
|
|
|
9
|
+

|
|
10
|
+
<!--  -->
|
|
11
|
+
|
|
8
12
|
Automatic REST API generation for Celery tasks with FastAPI. This package seamlessly bridges Celery and FastAPI, automatically creating REST endpoints for all your registered Celery tasks.
|
|
9
13
|
|
|
10
14
|
## Features
|
|
@@ -20,6 +24,9 @@ Automatic REST API generation for Celery tasks with FastAPI. This package seamle
|
|
|
20
24
|
- 🔒 **Production ready** - Full uvicorn/gunicorn support with SSL, workers, and all options
|
|
21
25
|
- ⚡ **Full Celery options** - All task options (countdown, eta, priority, etc.)
|
|
22
26
|
- 🔌 **Pool support** - Compatible with eventlet, gevent, prefork, and solo pools
|
|
27
|
+
- 🧮 **Batch execution** - Submit groups of tasks in a single request via `/tasks/batch`
|
|
28
|
+
- 🛡️ **Input validation** - Pydantic-driven validation on task_name/queue at trust boundary
|
|
29
|
+
- 🔌 **WebSocket streaming** - Live task status updates via `/tasks/{task_id}/ws`
|
|
23
30
|
|
|
24
31
|
## Requirements
|
|
25
32
|
|
|
@@ -103,25 +110,29 @@ uvicorn myapp:app --reload
|
|
|
103
110
|
|
|
104
111
|
```bash
|
|
105
112
|
# Start the server (development)
|
|
106
|
-
celery-fastapi serve
|
|
113
|
+
celery-fastapi serve examples.celery_app:celery_app --port 8000 --reload
|
|
107
114
|
|
|
108
115
|
# Production with multiple workers
|
|
109
|
-
celery-fastapi serve
|
|
116
|
+
celery-fastapi serve examples.celery_app:celery_app -w 4 --host 0.0.0.0
|
|
117
|
+
|
|
118
|
+
# With custom worker hostname (for health checks)
|
|
119
|
+
export CELERY_WORKER_HOSTNAME="celery@worker1"
|
|
120
|
+
celery-fastapi serve examples.celery_app:celery_app --port 8000
|
|
110
121
|
|
|
111
122
|
# With SSL
|
|
112
|
-
celery-fastapi serve
|
|
123
|
+
celery-fastapi serve examples.celery_app:celery_app --ssl-keyfile key.pem --ssl-certfile cert.pem
|
|
113
124
|
|
|
114
125
|
# Using gunicorn (production)
|
|
115
|
-
celery-fastapi serve-gunicorn
|
|
126
|
+
celery-fastapi serve-gunicorn examples.celery_app:celery_app -w 4 -k uvicorn.workers.UvicornWorker
|
|
116
127
|
|
|
117
128
|
# List available routes
|
|
118
|
-
celery-fastapi routes
|
|
129
|
+
celery-fastapi routes examples.celery_app:celery_app
|
|
119
130
|
|
|
120
131
|
# List registered tasks
|
|
121
|
-
celery-fastapi tasks
|
|
132
|
+
celery-fastapi tasks examples.celery_app:celery_app
|
|
122
133
|
|
|
123
134
|
# Show active workers
|
|
124
|
-
celery-fastapi workers
|
|
135
|
+
celery-fastapi workers examples.celery_app:celery_app
|
|
125
136
|
```
|
|
126
137
|
|
|
127
138
|
## API Endpoints
|
|
@@ -215,6 +226,44 @@ GET /queues
|
|
|
215
226
|
POST /purge
|
|
216
227
|
```
|
|
217
228
|
|
|
229
|
+
### Health Check and Monitoring
|
|
230
|
+
|
|
231
|
+
```bash
|
|
232
|
+
# Health check for local Celery worker
|
|
233
|
+
GET /healthz
|
|
234
|
+
|
|
235
|
+
# Response
|
|
236
|
+
{
|
|
237
|
+
"status": "healthy",
|
|
238
|
+
"celery_app": "example_tasks",
|
|
239
|
+
"broker_connected": true,
|
|
240
|
+
"worker_hostname": "celery@worker1",
|
|
241
|
+
"worker_online": true
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
# Ping local Celery worker
|
|
245
|
+
GET /ping
|
|
246
|
+
|
|
247
|
+
# Response
|
|
248
|
+
{
|
|
249
|
+
"worker_hostname": "celery@worker1",
|
|
250
|
+
"online": true,
|
|
251
|
+
"response": {"ok": "pong"}
|
|
252
|
+
}
|
|
253
|
+
```
|
|
254
|
+
|
|
255
|
+
**Note:** Health and ping endpoints automatically discover the local worker using:
|
|
256
|
+
1. `CELERY_WORKER_HOSTNAME` environment variable (recommended for custom hostnames)
|
|
257
|
+
2. Hostname matching (when worker and API share the same hostname)
|
|
258
|
+
3. Single worker fallback (when only one worker has this app's tasks)
|
|
259
|
+
|
|
260
|
+
**For custom worker hostnames**, set the environment variable:
|
|
261
|
+
```bash
|
|
262
|
+
export CELERY_WORKER_HOSTNAME="celery@worker1"
|
|
263
|
+
celery -A examples.celery_app worker --hostname worker1
|
|
264
|
+
celery-fastapi serve examples.celery_app:celery_app --port 8000
|
|
265
|
+
```
|
|
266
|
+
|
|
218
267
|
### List All Tasks
|
|
219
268
|
|
|
220
269
|
```bash
|
|
@@ -294,7 +343,7 @@ Commands:
|
|
|
294
343
|
workers Show active Celery workers
|
|
295
344
|
|
|
296
345
|
# Serve options (uvicorn)
|
|
297
|
-
celery-fastapi serve
|
|
346
|
+
celery-fastapi serve examples.celery_app:celery_app \
|
|
298
347
|
--host 0.0.0.0 \
|
|
299
348
|
--port 8000 \
|
|
300
349
|
--reload \
|
|
@@ -307,7 +356,7 @@ celery-fastapi serve myapp:celery_app \
|
|
|
307
356
|
--forwarded-allow-ips '*'
|
|
308
357
|
|
|
309
358
|
# Serve options (gunicorn)
|
|
310
|
-
celery-fastapi serve-gunicorn
|
|
359
|
+
celery-fastapi serve-gunicorn examples.celery_app:celery_app \
|
|
311
360
|
--bind 0.0.0.0:8000 \
|
|
312
361
|
--workers 4 \
|
|
313
362
|
--worker-class uvicorn.workers.UvicornWorker \
|
|
@@ -85,6 +85,7 @@ def create_app(
|
|
|
85
85
|
prefix: str = "",
|
|
86
86
|
include_status_endpoints: bool = True,
|
|
87
87
|
fastapi_kwargs: dict[str, Any] | None = None,
|
|
88
|
+
rate_limit: int | None = None,
|
|
88
89
|
) -> FastAPI:
|
|
89
90
|
"""
|
|
90
91
|
Create a FastAPI application with Celery task endpoints.
|
|
@@ -143,6 +144,7 @@ def create_app(
|
|
|
143
144
|
fastapi_app=fastapi_app,
|
|
144
145
|
prefix=prefix,
|
|
145
146
|
include_status_endpoints=include_status_endpoints,
|
|
147
|
+
rate_limit=rate_limit,
|
|
146
148
|
)
|
|
147
149
|
|
|
148
150
|
# Register all routes
|
|
@@ -799,7 +799,7 @@ def workers(
|
|
|
799
799
|
|
|
800
800
|
@app.callback()
|
|
801
801
|
def main(
|
|
802
|
-
version: Annotated[
|
|
802
|
+
version: Annotated[ # noqa: ARG001
|
|
803
803
|
bool | None,
|
|
804
804
|
typer.Option(
|
|
805
805
|
"--version",
|
|
@@ -816,7 +816,6 @@ def main(
|
|
|
816
816
|
Generate FastAPI endpoints for your Celery tasks automatically.
|
|
817
817
|
Supports uvicorn and gunicorn for production deployment.
|
|
818
818
|
"""
|
|
819
|
-
pass
|
|
820
819
|
|
|
821
820
|
|
|
822
821
|
if __name__ == "__main__":
|
|
@@ -1,14 +1,24 @@
|
|
|
1
1
|
"""Core functionality for Celery FastAPI."""
|
|
2
2
|
|
|
3
|
+
import asyncio
|
|
3
4
|
import inspect
|
|
5
|
+
import time
|
|
6
|
+
from collections import defaultdict, deque
|
|
4
7
|
from collections.abc import Callable
|
|
5
8
|
from datetime import datetime
|
|
6
9
|
from typing import Any, get_type_hints
|
|
7
10
|
|
|
8
11
|
from celery import Celery
|
|
9
12
|
from celery.result import AsyncResult
|
|
10
|
-
from fastapi import
|
|
11
|
-
|
|
13
|
+
from fastapi import (
|
|
14
|
+
FastAPI,
|
|
15
|
+
HTTPException,
|
|
16
|
+
Query,
|
|
17
|
+
Request,
|
|
18
|
+
WebSocket,
|
|
19
|
+
WebSocketDisconnect,
|
|
20
|
+
)
|
|
21
|
+
from pydantic import BaseModel, Field, ValidationInfo, create_model, field_validator
|
|
12
22
|
|
|
13
23
|
# Celery execution options - shared fields for all task payloads
|
|
14
24
|
CELERY_OPTIONS_FIELDS: dict[str, Any] = {
|
|
@@ -142,6 +152,20 @@ class GenericTaskPayload(BaseModel):
|
|
|
142
152
|
}
|
|
143
153
|
}
|
|
144
154
|
|
|
155
|
+
@field_validator("task_name", "queue")
|
|
156
|
+
@classmethod
|
|
157
|
+
def _validate_names(cls, value: str, info: ValidationInfo) -> str:
|
|
158
|
+
# Trust boundary: reject control chars / injection attempts.
|
|
159
|
+
if not value or not value.strip():
|
|
160
|
+
raise ValueError("must be a non-empty string")
|
|
161
|
+
if any(ord(c) < 0x20 for c in value):
|
|
162
|
+
raise ValueError("control characters are not allowed")
|
|
163
|
+
if not all(c.isalnum() or c in "._-" for c in value):
|
|
164
|
+
raise ValueError("only alphanumeric, '.', '_', '-' allowed")
|
|
165
|
+
if info.field_name == "task_name" and len(value) > 255:
|
|
166
|
+
raise ValueError("task_name exceeds 255 character limit")
|
|
167
|
+
return value
|
|
168
|
+
|
|
145
169
|
|
|
146
170
|
def _python_type_to_json_type(py_type: type) -> str:
|
|
147
171
|
"""Convert Python type to JSON schema type string."""
|
|
@@ -271,6 +295,95 @@ class TaskRevokePayload(BaseModel):
|
|
|
271
295
|
)
|
|
272
296
|
|
|
273
297
|
|
|
298
|
+
class BatchTaskItem(BaseModel):
|
|
299
|
+
"""A single task to execute as part of a batch."""
|
|
300
|
+
|
|
301
|
+
task_name: str = Field(description="Full task name (e.g., 'myapp.tasks.add')")
|
|
302
|
+
queue: str | None = Field(default=None, description="Queue override")
|
|
303
|
+
args: list[Any] = Field(
|
|
304
|
+
default_factory=list, description="Positional arguments for the task"
|
|
305
|
+
)
|
|
306
|
+
kwargs: dict[str, Any] = Field(
|
|
307
|
+
default_factory=dict, description="Keyword arguments for the task"
|
|
308
|
+
)
|
|
309
|
+
countdown: float | None = Field(default=None, description="Seconds to wait")
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
class BatchTaskRequest(BaseModel):
|
|
313
|
+
"""Payload for batch task execution."""
|
|
314
|
+
|
|
315
|
+
tasks: list[BatchTaskItem] = Field(
|
|
316
|
+
description="List of tasks to execute as a group"
|
|
317
|
+
)
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
class BatchTaskResponse(BaseModel):
|
|
321
|
+
"""Response model for batch task submission."""
|
|
322
|
+
|
|
323
|
+
group_id: str = Field(description="Group ID for the submitted batch")
|
|
324
|
+
task_ids: list[str] = Field(description="Individual task IDs in the group")
|
|
325
|
+
task_count: int = Field(description="Number of tasks submitted")
|
|
326
|
+
status: str = Field(default="PENDING", description="Group status")
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
class HealthCheckResponse(BaseModel):
|
|
330
|
+
"""Response model for health check endpoint."""
|
|
331
|
+
|
|
332
|
+
status: str = Field(description="Overall health status (healthy/unhealthy)")
|
|
333
|
+
celery_app: str = Field(description="Celery application name")
|
|
334
|
+
broker_connected: bool = Field(description="Whether broker connection is active")
|
|
335
|
+
worker_hostname: str | None = Field(
|
|
336
|
+
default=None, description="Target worker hostname"
|
|
337
|
+
)
|
|
338
|
+
worker_online: bool = Field(description="Whether the target worker is online")
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
class PingResponse(BaseModel):
|
|
342
|
+
"""Response model for worker ping endpoint."""
|
|
343
|
+
|
|
344
|
+
worker_hostname: str | None = Field(
|
|
345
|
+
default=None, description="Target worker hostname"
|
|
346
|
+
)
|
|
347
|
+
online: bool = Field(description="Whether the worker responded to ping")
|
|
348
|
+
response: dict[str, str] | None = Field(
|
|
349
|
+
default=None, description="Ping response from the worker"
|
|
350
|
+
)
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
class RateLimiter:
|
|
354
|
+
"""Simple in-memory sliding-window rate limiter.
|
|
355
|
+
|
|
356
|
+
Tracks request counts per client key (defaults to client IP) within a
|
|
357
|
+
fixed time window. When the limit is exceeded, `check` raises HTTP 429.
|
|
358
|
+
|
|
359
|
+
ponytail: in-memory, single-process. Replace with Redis-backed limiter
|
|
360
|
+
(e.g., ``limits`` package) when running multiple uvicorn workers.
|
|
361
|
+
"""
|
|
362
|
+
|
|
363
|
+
def __init__(self, limit: int, window_seconds: int = 60) -> None:
|
|
364
|
+
self.limit = limit
|
|
365
|
+
self.window_seconds = window_seconds
|
|
366
|
+
self._hits: dict[str, deque[float]] = defaultdict(deque)
|
|
367
|
+
|
|
368
|
+
def check(self, key: str) -> None:
|
|
369
|
+
"""Record a hit for ``key``; raise HTTPException(429) if over limit."""
|
|
370
|
+
now = time.monotonic()
|
|
371
|
+
window = self._hits[key]
|
|
372
|
+
cutoff = now - self.window_seconds
|
|
373
|
+
|
|
374
|
+
# Drop hits outside the window
|
|
375
|
+
while window and window[0] < cutoff:
|
|
376
|
+
window.popleft()
|
|
377
|
+
|
|
378
|
+
if len(window) >= self.limit:
|
|
379
|
+
raise HTTPException(
|
|
380
|
+
status_code=429,
|
|
381
|
+
detail=f"Rate limit exceeded: {self.limit} requests per {self.window_seconds}s",
|
|
382
|
+
)
|
|
383
|
+
|
|
384
|
+
window.append(now)
|
|
385
|
+
|
|
386
|
+
|
|
274
387
|
class CeleryFastAPIBridge:
|
|
275
388
|
"""
|
|
276
389
|
Bridge class that connects Celery tasks to FastAPI endpoints.
|
|
@@ -303,6 +416,7 @@ class CeleryFastAPIBridge:
|
|
|
303
416
|
prefix: str = "",
|
|
304
417
|
include_status_endpoints: bool = True,
|
|
305
418
|
task_filter: Callable[[str], bool] | None = None,
|
|
419
|
+
rate_limit: int | None = None,
|
|
306
420
|
) -> None:
|
|
307
421
|
"""
|
|
308
422
|
Initialize the Celery FastAPI Bridge.
|
|
@@ -315,12 +429,15 @@ class CeleryFastAPIBridge:
|
|
|
315
429
|
include_status_endpoints: Whether to include task status and listing endpoints.
|
|
316
430
|
task_filter: Optional callable to filter which tasks to expose.
|
|
317
431
|
Takes task name, returns True to include, False to exclude.
|
|
432
|
+
rate_limit: Optional maximum number of requests per minute per client.
|
|
433
|
+
If set, requests exceeding the limit get HTTP 429.
|
|
318
434
|
"""
|
|
319
435
|
self.celery_app = celery_app
|
|
320
436
|
self.fastapi_app = fastapi_app or FastAPI()
|
|
321
437
|
self.prefix = prefix.rstrip("/")
|
|
322
438
|
self.include_status_endpoints = include_status_endpoints
|
|
323
439
|
self.task_filter = task_filter or (lambda name: not name.startswith("celery."))
|
|
440
|
+
self.rate_limiter = RateLimiter(rate_limit) if rate_limit else None
|
|
324
441
|
self._registered = False
|
|
325
442
|
|
|
326
443
|
# Store the registered task names from THIS app only
|
|
@@ -380,6 +497,7 @@ class CeleryFastAPIBridge:
|
|
|
380
497
|
|
|
381
498
|
# Create the endpoint handler
|
|
382
499
|
async def run_task(
|
|
500
|
+
request: Request,
|
|
383
501
|
payload: PayloadModel, # type: ignore[valid-type]
|
|
384
502
|
task_name_override: str | None = Query(
|
|
385
503
|
default=None,
|
|
@@ -393,6 +511,11 @@ class CeleryFastAPIBridge:
|
|
|
393
511
|
),
|
|
394
512
|
) -> TaskResponse:
|
|
395
513
|
"""Execute a Celery task asynchronously."""
|
|
514
|
+
# Rate limit check (per client IP)
|
|
515
|
+
if self.rate_limiter is not None:
|
|
516
|
+
client_ip = request.client.host if request.client else "unknown"
|
|
517
|
+
self.rate_limiter.check(client_ip)
|
|
518
|
+
|
|
396
519
|
# Determine actual task name and queue
|
|
397
520
|
actual_task_name = task_name_override or task_name
|
|
398
521
|
actual_queue = (
|
|
@@ -557,6 +680,39 @@ class CeleryFastAPIBridge:
|
|
|
557
680
|
|
|
558
681
|
return result.result
|
|
559
682
|
|
|
683
|
+
@self.fastapi_app.websocket(f"{self.prefix}/tasks/{{task_id}}/ws")
|
|
684
|
+
async def stream_task_status(websocket: WebSocket, task_id: str) -> None:
|
|
685
|
+
"""
|
|
686
|
+
Stream task status updates via WebSocket.
|
|
687
|
+
|
|
688
|
+
Polls the task state every 0.5s and sends JSON frames until the
|
|
689
|
+
task reaches a terminal state (SUCCESS/FAILURE/REVOKED), then closes.
|
|
690
|
+
|
|
691
|
+
ponytail: polling-based. For high-throughput streams, replace with
|
|
692
|
+
Celery events (celery_app.events.Receiver) or Redis pub/sub.
|
|
693
|
+
"""
|
|
694
|
+
await websocket.accept()
|
|
695
|
+
result = AsyncResult(task_id, app=self.celery_app)
|
|
696
|
+
try:
|
|
697
|
+
while True:
|
|
698
|
+
state = result.state
|
|
699
|
+
frame: dict[str, Any] = {
|
|
700
|
+
"task_id": task_id,
|
|
701
|
+
"state": state,
|
|
702
|
+
"ready": result.ready(),
|
|
703
|
+
}
|
|
704
|
+
if result.ready():
|
|
705
|
+
if result.failed():
|
|
706
|
+
frame["error"] = str(result.traceback or "task failed")
|
|
707
|
+
else:
|
|
708
|
+
frame["result"] = result.result
|
|
709
|
+
await websocket.send_json(frame)
|
|
710
|
+
break
|
|
711
|
+
await websocket.send_json(frame)
|
|
712
|
+
await asyncio.sleep(0.5)
|
|
713
|
+
except WebSocketDisconnect:
|
|
714
|
+
pass
|
|
715
|
+
|
|
560
716
|
@self.fastapi_app.get(
|
|
561
717
|
f"{self.prefix}/tasks",
|
|
562
718
|
response_model=TaskListResponse,
|
|
@@ -672,6 +828,208 @@ class CeleryFastAPIBridge:
|
|
|
672
828
|
inspector = self.celery_app.control.inspect()
|
|
673
829
|
return {"queues": inspector.active_queues() or {}}
|
|
674
830
|
|
|
831
|
+
def _find_local_worker() -> str | None:
|
|
832
|
+
"""
|
|
833
|
+
Find the Celery worker running on the same host.
|
|
834
|
+
|
|
835
|
+
Celery worker hostnames follow the pattern: prefix@hostname
|
|
836
|
+
where prefix can be any format (e.g., celery, appname_uuid, etc.).
|
|
837
|
+
|
|
838
|
+
Matching strategy:
|
|
839
|
+
1. CELERY_WORKER_HOSTNAME environment variable (RECOMMENDED)
|
|
840
|
+
Set this to the exact worker name when starting both worker and API
|
|
841
|
+
Example: export CELERY_WORKER_HOSTNAME="celery@worker1"
|
|
842
|
+
|
|
843
|
+
2. Hostname matching (works when worker and API share same hostname):
|
|
844
|
+
- Exact match: worker hostname part equals socket.gethostname()
|
|
845
|
+
- Partial match: handles FQDN vs short hostname
|
|
846
|
+
|
|
847
|
+
3. Single worker fallback (only when one worker exists for this app)
|
|
848
|
+
|
|
849
|
+
Note: When using custom --hostname, always set CELERY_WORKER_HOSTNAME
|
|
850
|
+
to ensure correct worker discovery.
|
|
851
|
+
|
|
852
|
+
Examples:
|
|
853
|
+
- Standard: celery@myhost (auto-discovered)
|
|
854
|
+
- Custom: celery@worker1 (needs CELERY_WORKER_HOSTNAME="celery@worker1")
|
|
855
|
+
- UUID: myapp_abc123@myhost (auto-discovered if hostname matches)
|
|
856
|
+
"""
|
|
857
|
+
import os
|
|
858
|
+
import socket
|
|
859
|
+
|
|
860
|
+
# Strategy 1: Check for explicit environment variable (RECOMMENDED)
|
|
861
|
+
env_worker = os.environ.get("CELERY_WORKER_HOSTNAME")
|
|
862
|
+
if env_worker:
|
|
863
|
+
try:
|
|
864
|
+
inspector = self.celery_app.control.inspect(timeout=1.0)
|
|
865
|
+
env_ping_response: dict[str, Any] = inspector.ping() or {}
|
|
866
|
+
if env_worker in env_ping_response:
|
|
867
|
+
return str(env_worker)
|
|
868
|
+
# If env var is set but worker not found, log and continue
|
|
869
|
+
except Exception: # noqa: BLE001
|
|
870
|
+
pass
|
|
871
|
+
|
|
872
|
+
local_hostname = socket.gethostname()
|
|
873
|
+
|
|
874
|
+
try:
|
|
875
|
+
inspector = self.celery_app.control.inspect(timeout=1.0)
|
|
876
|
+
|
|
877
|
+
# Get all active workers
|
|
878
|
+
ping_response: dict[str, Any] = inspector.ping() or {}
|
|
879
|
+
if not ping_response:
|
|
880
|
+
return None
|
|
881
|
+
|
|
882
|
+
# Strategy 2: Hostname-based matching
|
|
883
|
+
# First pass: exact hostname match
|
|
884
|
+
for worker_name in ping_response:
|
|
885
|
+
if "@" in worker_name:
|
|
886
|
+
_, worker_host = worker_name.rsplit("@", 1)
|
|
887
|
+
if worker_host == local_hostname:
|
|
888
|
+
return str(worker_name)
|
|
889
|
+
elif worker_name == local_hostname:
|
|
890
|
+
return str(worker_name)
|
|
891
|
+
|
|
892
|
+
# Second pass: partial hostname match (FQDN vs short name)
|
|
893
|
+
for worker_name in ping_response:
|
|
894
|
+
if "@" in worker_name:
|
|
895
|
+
_, worker_host = worker_name.rsplit("@", 1)
|
|
896
|
+
if (
|
|
897
|
+
local_hostname in worker_host
|
|
898
|
+
or worker_host in local_hostname
|
|
899
|
+
):
|
|
900
|
+
return str(worker_name)
|
|
901
|
+
|
|
902
|
+
# Strategy 3: Filter by app tasks and check for single worker
|
|
903
|
+
registered: dict[str, list[str]] = inspector.registered() or {}
|
|
904
|
+
app_workers = []
|
|
905
|
+
for worker_name in ping_response:
|
|
906
|
+
worker_tasks = registered.get(worker_name, [])
|
|
907
|
+
if any(task in self._app_task_names for task in worker_tasks):
|
|
908
|
+
app_workers.append(worker_name)
|
|
909
|
+
|
|
910
|
+
# If only one worker has this app's tasks, assume it's local
|
|
911
|
+
if len(app_workers) == 1:
|
|
912
|
+
return str(app_workers[0])
|
|
913
|
+
|
|
914
|
+
# If multiple workers with same tasks exist and hostname doesn't match,
|
|
915
|
+
# we cannot determine which is local without CELERY_WORKER_HOSTNAME
|
|
916
|
+
return None
|
|
917
|
+
except Exception: # noqa: BLE001
|
|
918
|
+
return None
|
|
919
|
+
|
|
920
|
+
@self.fastapi_app.get(
|
|
921
|
+
f"{self.prefix}/healthz",
|
|
922
|
+
response_model=HealthCheckResponse,
|
|
923
|
+
tags=["health"],
|
|
924
|
+
summary="Health check",
|
|
925
|
+
)
|
|
926
|
+
async def health_check(
|
|
927
|
+
worker: str | None = Query(
|
|
928
|
+
default=None,
|
|
929
|
+
description=(
|
|
930
|
+
"Explicit worker hostname to check (e.g., 'celery@worker1'). "
|
|
931
|
+
"If provided, bypasses local worker discovery."
|
|
932
|
+
),
|
|
933
|
+
),
|
|
934
|
+
) -> HealthCheckResponse:
|
|
935
|
+
"""
|
|
936
|
+
Check the health of a Celery worker.
|
|
937
|
+
|
|
938
|
+
If a `worker` query parameter is provided, checks that specific worker.
|
|
939
|
+
Otherwise, automatically discovers the worker running on the same host by
|
|
940
|
+
matching the system hostname with active Celery worker hostnames.
|
|
941
|
+
|
|
942
|
+
Returns:
|
|
943
|
+
Health status including broker connection and worker availability.
|
|
944
|
+
|
|
945
|
+
This endpoint is useful for:
|
|
946
|
+
- Kubernetes/Docker health probes (sidecar pattern)
|
|
947
|
+
- Load balancer health checks
|
|
948
|
+
- Monitoring systems
|
|
949
|
+
"""
|
|
950
|
+
local_worker = worker # Use explicit worker if provided
|
|
951
|
+
try:
|
|
952
|
+
broker_connected = True
|
|
953
|
+
if local_worker is None:
|
|
954
|
+
local_worker = _find_local_worker()
|
|
955
|
+
# When a specific worker is requested, verify it's online via inspection.
|
|
956
|
+
# In environments without active workers, this will be False.
|
|
957
|
+
inspector = self.celery_app.control.inspect(
|
|
958
|
+
destination=[local_worker] if local_worker else None
|
|
959
|
+
)
|
|
960
|
+
ping_response = inspector.ping() or {}
|
|
961
|
+
worker_online = local_worker in ping_response if local_worker else False
|
|
962
|
+
|
|
963
|
+
except Exception: # noqa: BLE001
|
|
964
|
+
broker_connected = False
|
|
965
|
+
local_worker = None
|
|
966
|
+
worker_online = False
|
|
967
|
+
|
|
968
|
+
# Determine overall health status
|
|
969
|
+
status = "healthy" if broker_connected and worker_online else "unhealthy"
|
|
970
|
+
|
|
971
|
+
return HealthCheckResponse(
|
|
972
|
+
status=status,
|
|
973
|
+
celery_app=self.celery_app.main,
|
|
974
|
+
broker_connected=broker_connected,
|
|
975
|
+
worker_hostname=local_worker,
|
|
976
|
+
worker_online=worker_online,
|
|
977
|
+
)
|
|
978
|
+
|
|
979
|
+
@self.fastapi_app.get(
|
|
980
|
+
f"{self.prefix}/ping",
|
|
981
|
+
response_model=PingResponse,
|
|
982
|
+
tags=["health"],
|
|
983
|
+
summary="Ping worker",
|
|
984
|
+
)
|
|
985
|
+
async def ping_worker(
|
|
986
|
+
worker: str | None = Query(
|
|
987
|
+
default=None,
|
|
988
|
+
description=(
|
|
989
|
+
"Explicit worker hostname to ping (e.g., 'celery@worker1'). "
|
|
990
|
+
"If provided, bypasses local worker discovery."
|
|
991
|
+
),
|
|
992
|
+
),
|
|
993
|
+
) -> PingResponse:
|
|
994
|
+
"""
|
|
995
|
+
Ping a Celery worker.
|
|
996
|
+
|
|
997
|
+
If a `worker` query parameter is provided, pings that specific worker.
|
|
998
|
+
Otherwise, automatically discovers the local Celery worker by matching
|
|
999
|
+
the system hostname with active Celery worker hostnames.
|
|
1000
|
+
|
|
1001
|
+
Returns:
|
|
1002
|
+
Ping response from the discovered or specified worker.
|
|
1003
|
+
|
|
1004
|
+
This endpoint is useful for:
|
|
1005
|
+
- Checking specific worker responsiveness
|
|
1006
|
+
- Diagnosing connection issues
|
|
1007
|
+
- Verifying worker health in containerized deployments
|
|
1008
|
+
"""
|
|
1009
|
+
local_worker = worker # Use explicit worker if provided
|
|
1010
|
+
if not local_worker:
|
|
1011
|
+
local_worker = _find_local_worker()
|
|
1012
|
+
|
|
1013
|
+
if not local_worker:
|
|
1014
|
+
# No worker found or specified
|
|
1015
|
+
return PingResponse(
|
|
1016
|
+
worker_hostname=None,
|
|
1017
|
+
online=False,
|
|
1018
|
+
response=None,
|
|
1019
|
+
)
|
|
1020
|
+
|
|
1021
|
+
# Ping the specific worker
|
|
1022
|
+
inspector = self.celery_app.control.inspect(destination=[local_worker])
|
|
1023
|
+
ping_response = inspector.ping() or {}
|
|
1024
|
+
|
|
1025
|
+
worker_response = ping_response.get(local_worker)
|
|
1026
|
+
|
|
1027
|
+
return PingResponse(
|
|
1028
|
+
worker_hostname=local_worker,
|
|
1029
|
+
online=worker_response is not None,
|
|
1030
|
+
response=worker_response,
|
|
1031
|
+
)
|
|
1032
|
+
|
|
675
1033
|
@self.fastapi_app.post(
|
|
676
1034
|
f"{self.prefix}/purge",
|
|
677
1035
|
tags=["workers"],
|
|
@@ -744,6 +1102,79 @@ class CeleryFastAPIBridge:
|
|
|
744
1102
|
result = self.celery_app.send_task(payload.task_name, **send_options)
|
|
745
1103
|
return TaskResponse(task_id=result.id, status="PENDING")
|
|
746
1104
|
|
|
1105
|
+
@self.fastapi_app.post(
|
|
1106
|
+
f"{self.prefix}/tasks/batch",
|
|
1107
|
+
response_model=BatchTaskResponse,
|
|
1108
|
+
tags=["tasks"],
|
|
1109
|
+
summary="Execute multiple tasks in a group",
|
|
1110
|
+
)
|
|
1111
|
+
async def batch_execute_tasks(
|
|
1112
|
+
payload: BatchTaskRequest,
|
|
1113
|
+
) -> BatchTaskResponse:
|
|
1114
|
+
"""
|
|
1115
|
+
Execute multiple Celery tasks as a single group.
|
|
1116
|
+
|
|
1117
|
+
The tasks are submitted as a Celery group, enabling coordinated
|
|
1118
|
+
execution and result collection.
|
|
1119
|
+
|
|
1120
|
+
Returns:
|
|
1121
|
+
Group ID, individual task IDs, and count of submitted tasks.
|
|
1122
|
+
"""
|
|
1123
|
+
from celery import group
|
|
1124
|
+
|
|
1125
|
+
# Validate: at least one task required
|
|
1126
|
+
if not payload.tasks:
|
|
1127
|
+
raise HTTPException(status_code=422, detail="No tasks provided")
|
|
1128
|
+
|
|
1129
|
+
# Build Celery signatures for each task
|
|
1130
|
+
signatures = []
|
|
1131
|
+
for item in payload.tasks:
|
|
1132
|
+
if not item.task_name:
|
|
1133
|
+
raise HTTPException(
|
|
1134
|
+
status_code=422, detail="task_name is required for each task"
|
|
1135
|
+
)
|
|
1136
|
+
sig = self.celery_app.signature(
|
|
1137
|
+
item.task_name,
|
|
1138
|
+
args=item.args,
|
|
1139
|
+
kwargs=item.kwargs,
|
|
1140
|
+
queue=item.queue,
|
|
1141
|
+
)
|
|
1142
|
+
signatures.append(sig)
|
|
1143
|
+
|
|
1144
|
+
# Submit the group
|
|
1145
|
+
job = group(*signatures)
|
|
1146
|
+
result = job.apply_async()
|
|
1147
|
+
|
|
1148
|
+
task_ids = [r.id for r in result.children or []]
|
|
1149
|
+
|
|
1150
|
+
return BatchTaskResponse(
|
|
1151
|
+
group_id=result.id,
|
|
1152
|
+
task_ids=task_ids,
|
|
1153
|
+
task_count=len(payload.tasks),
|
|
1154
|
+
status="PENDING",
|
|
1155
|
+
)
|
|
1156
|
+
|
|
1157
|
+
@self.fastapi_app.post(
|
|
1158
|
+
f"{self.prefix}/tasks/batch/revoke",
|
|
1159
|
+
summary="Revoke multiple tasks",
|
|
1160
|
+
)
|
|
1161
|
+
async def batch_revoke_tasks(
|
|
1162
|
+
payload: dict[str, list[str]],
|
|
1163
|
+
) -> dict[str, Any]:
|
|
1164
|
+
"""
|
|
1165
|
+
Revoke multiple tasks by their task IDs.
|
|
1166
|
+
|
|
1167
|
+
Task IDs should be provided in the `task_ids` field.
|
|
1168
|
+
"""
|
|
1169
|
+
task_ids = payload.get("task_ids", [])
|
|
1170
|
+
if not task_ids:
|
|
1171
|
+
raise HTTPException(status_code=422, detail="task_ids is required")
|
|
1172
|
+
|
|
1173
|
+
for task_id in task_ids:
|
|
1174
|
+
self.celery_app.control.revoke(task_id)
|
|
1175
|
+
|
|
1176
|
+
return {"status": "revoked", "task_ids": task_ids, "count": len(task_ids)}
|
|
1177
|
+
|
|
747
1178
|
def get_registered_routes(self) -> list[dict[str, str]]:
|
|
748
1179
|
"""
|
|
749
1180
|
Get a list of all registered routes.
|
|
File without changes
|
|
File without changes
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[tool.poetry]
|
|
2
2
|
name = "celery-fastapi"
|
|
3
|
-
version = "0.1.
|
|
3
|
+
version = "0.1.5" # placeholder; overridden by poetry-dynamic-versioning at build time
|
|
4
4
|
description = "Automatic REST API generation for Celery tasks with FastAPI"
|
|
5
5
|
authors = ["ilkerkara <ilkerkara@outlook.com.tr>"]
|
|
6
6
|
license = "MIT"
|
|
@@ -29,7 +29,7 @@ packages = [{ include = "celery_fastapi" }]
|
|
|
29
29
|
[tool.poetry.dependencies]
|
|
30
30
|
python = "^3.11"
|
|
31
31
|
fastapi = ">=0.100.0"
|
|
32
|
-
celery = ">=5.3.0"
|
|
32
|
+
celery = ">=5.3.0,<=5.6.3"
|
|
33
33
|
pydantic = ">=2.0.0"
|
|
34
34
|
|
|
35
35
|
# Server dependencies (optional)
|
|
@@ -107,8 +107,20 @@ rich = ">=13.0.0"
|
|
|
107
107
|
celery-fastapi = "celery_fastapi.cli:app"
|
|
108
108
|
|
|
109
109
|
[build-system]
|
|
110
|
-
requires = ["poetry-core>=1.0.0"]
|
|
111
|
-
build-backend = "
|
|
110
|
+
requires = ["poetry-core>=1.0.0", "poetry-dynamic-versioning>=1.0.0,<2.0.0"]
|
|
111
|
+
build-backend = "poetry_dynamic_versioning.backend"
|
|
112
|
+
|
|
113
|
+
[tool.poetry.requires-plugins]
|
|
114
|
+
poetry-dynamic-versioning = { version = ">=1.0.0,<2.0.0", extras = ["plugin"] }
|
|
115
|
+
|
|
116
|
+
[tool.poetry-dynamic-versioning]
|
|
117
|
+
enable = false
|
|
118
|
+
vcs = "git"
|
|
119
|
+
style = "semver"
|
|
120
|
+
|
|
121
|
+
[tool.poetry-dynamic-versioning.substitution]
|
|
122
|
+
# Sync __version__ in source with the tag-derived version at build time.
|
|
123
|
+
files = ["celery_fastapi/__init__.py"]
|
|
112
124
|
|
|
113
125
|
[tool.ruff]
|
|
114
126
|
target-version = "py311"
|