pytaskq 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pytaskq/__init__.py +6 -0
- pytaskq/__main__.py +4 -0
- pytaskq/api/router.py +15 -0
- pytaskq/api/v1/auth.py +235 -0
- pytaskq/api/v1/dashboard.py +99 -0
- pytaskq/api/v1/dlq.py +114 -0
- pytaskq/api/v1/system.py +68 -0
- pytaskq/api/v1/tasks.py +99 -0
- pytaskq/app.py +324 -0
- pytaskq/cli.py +250 -0
- pytaskq/core/auth.py +535 -0
- pytaskq/core/config.py +52 -0
- pytaskq/core/dependencies.py +106 -0
- pytaskq/core/exceptions.py +12 -0
- pytaskq/core/idempotency.py +45 -0
- pytaskq/core/queueing.py +282 -0
- pytaskq/core/redis_client.py +19 -0
- pytaskq/main.py +124 -0
- pytaskq/net.py +67 -0
- pytaskq/schemas/requests.py +32 -0
- pytaskq/services/dashboard_service.py +42 -0
- pytaskq/static/assets/index-Biu-vuAP.js +9 -0
- pytaskq/static/assets/index-mgpfRVrS.css +1 -0
- pytaskq/static/favicon.svg +1 -0
- pytaskq/static/index.html +15 -0
- pytaskq/task_registery.py +144 -0
- pytaskq/tracing.py +98 -0
- pytaskq/worker.py +636 -0
- pytaskq-0.3.0.dist-info/METADATA +262 -0
- pytaskq-0.3.0.dist-info/RECORD +34 -0
- pytaskq-0.3.0.dist-info/WHEEL +5 -0
- pytaskq-0.3.0.dist-info/entry_points.txt +2 -0
- pytaskq-0.3.0.dist-info/licenses/LICENSE +21 -0
- pytaskq-0.3.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
from fastapi import Request
|
|
2
|
+
from fastapi.responses import JSONResponse
|
|
3
|
+
from redis.exceptions import ConnectionError as RedisConnectionError
|
|
4
|
+
|
|
5
|
+
async def redis_connection_error_handler(request: Request, exc: RedisConnectionError):
|
|
6
|
+
return JSONResponse(
|
|
7
|
+
status_code=503,
|
|
8
|
+
content={
|
|
9
|
+
"error": "Redis is unavailable",
|
|
10
|
+
"detail": "The task queue service is temporarily down. Please try again shortly.",
|
|
11
|
+
},
|
|
12
|
+
)
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import hashlib
|
|
3
|
+
from fastapi import Request, HTTPException
|
|
4
|
+
from fastapi.responses import JSONResponse
|
|
5
|
+
from pytaskq.core import redis_client
|
|
6
|
+
from pytaskq.core.config import settings
|
|
7
|
+
from pytaskq.schemas.requests import TaskRequest
|
|
8
|
+
|
|
9
|
+
def _resolve_idempotency_key(req: Request, body: TaskRequest) -> str | None:
|
|
10
|
+
return req.headers.get("idempotency-key") or body.idempotency_key
|
|
11
|
+
|
|
12
|
+
async def check_idempotency(req: Request, body: TaskRequest, tenant_id: str = "global") -> JSONResponse | None:
|
|
13
|
+
idempotency_key = _resolve_idempotency_key(req, body)
|
|
14
|
+
if idempotency_key:
|
|
15
|
+
redis_key = f"idempotency:global:{idempotency_key}"
|
|
16
|
+
claimed = await redis_client.r.set(redis_key, "pending", nx=True, ex=settings.IDEMPOTENCY_PENDING_TTL)
|
|
17
|
+
if claimed:
|
|
18
|
+
return None
|
|
19
|
+
|
|
20
|
+
cached = await redis_client.r.get(redis_key)
|
|
21
|
+
if cached and cached != "pending":
|
|
22
|
+
return JSONResponse(content=json.loads(cached))
|
|
23
|
+
|
|
24
|
+
raise HTTPException(
|
|
25
|
+
status_code=409,
|
|
26
|
+
detail="A request with this Idempotency-Key is already being processed.",
|
|
27
|
+
headers={"Retry-After": str(settings.IDEMPOTENCY_PENDING_TTL)},
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
raw = f"{body.task_name}:{json.dumps(body.args, sort_keys=True)}:global"
|
|
31
|
+
content_hash = hashlib.sha256(raw.encode()).hexdigest()[:16]
|
|
32
|
+
dedup_key = f"dedup:{content_hash}"
|
|
33
|
+
|
|
34
|
+
if not await redis_client.r.set(dedup_key, "1", nx=True, ex=settings.DEDUP_TTL):
|
|
35
|
+
raise HTTPException(
|
|
36
|
+
status_code=409,
|
|
37
|
+
detail="Duplicate request detected. Please try again shortly.",
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
return None
|
|
41
|
+
|
|
42
|
+
async def store_idempotency_response(idempotency_key: str | None, response_data: dict, tenant_id: str = "global"):
|
|
43
|
+
if idempotency_key:
|
|
44
|
+
redis_key = f"idempotency:global:{idempotency_key}"
|
|
45
|
+
await redis_client.r.set(redis_key, json.dumps(response_data), ex=settings.IDEMPOTENCY_TTL)
|
pytaskq/core/queueing.py
ADDED
|
@@ -0,0 +1,282 @@
|
|
|
1
|
+
"""
|
|
2
|
+
pytaskq.core.queueing — the one place where tasks enter Redis
|
|
3
|
+
==============================================================
|
|
4
|
+
|
|
5
|
+
The SDK (`task.delay()`), the REST API (`POST /task/enqueue`) and the worker
|
|
6
|
+
(webhook follow-ups) all enqueue through `enqueue()`, so every task gets:
|
|
7
|
+
|
|
8
|
+
* a `task:{id}` status hash from the moment it is queued (lookups never
|
|
9
|
+
return "not found" for a task that is merely waiting),
|
|
10
|
+
* an entry in the `tasks:recent` history list (powers the dashboard table),
|
|
11
|
+
* consistent counters and a pub/sub nudge for live dashboards.
|
|
12
|
+
|
|
13
|
+
Every function takes the Redis connection explicitly so callers (and tests)
|
|
14
|
+
decide which client is used.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import json
|
|
20
|
+
import time
|
|
21
|
+
import uuid
|
|
22
|
+
from typing import Any, Iterable, Optional
|
|
23
|
+
|
|
24
|
+
from pytaskq.core.config import settings
|
|
25
|
+
|
|
26
|
+
PRIORITIES = ("high", "default", "low")
|
|
27
|
+
|
|
28
|
+
# Task lifecycle states stored in task:{id} → status
|
|
29
|
+
QUEUED = "Queued"
|
|
30
|
+
SCHEDULED = "Scheduled"
|
|
31
|
+
PROCESSING = "Processing"
|
|
32
|
+
RETRY_SCHEDULED = "RetryScheduled"
|
|
33
|
+
SUCCESS = "Success"
|
|
34
|
+
FAILED = "Failed"
|
|
35
|
+
DEAD_LETTER = "DeadLetter"
|
|
36
|
+
CANCELLED = "Cancelled"
|
|
37
|
+
|
|
38
|
+
FINAL_STATES = {SUCCESS, FAILED, DEAD_LETTER, CANCELLED}
|
|
39
|
+
ALL_STATES = (QUEUED, SCHEDULED, PROCESSING, RETRY_SCHEDULED, SUCCESS, FAILED, DEAD_LETTER, CANCELLED)
|
|
40
|
+
|
|
41
|
+
RECENT_KEY = "tasks:recent"
|
|
42
|
+
REGISTRY_KEY = "pytaskq:registry"
|
|
43
|
+
EVENTS_CHANNEL = "events:global"
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def task_key(task_id: str) -> str:
|
|
47
|
+
return f"task:{task_id}"
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def queue_key(priority: str) -> str:
|
|
51
|
+
return f"queue:{priority if priority in PRIORITIES else 'default'}"
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def encode_result(result: Any) -> str:
|
|
55
|
+
"""Store strings as-is, everything else as JSON (falls back to str())."""
|
|
56
|
+
if isinstance(result, str):
|
|
57
|
+
return result
|
|
58
|
+
try:
|
|
59
|
+
return json.dumps(result, default=str)
|
|
60
|
+
except (TypeError, ValueError):
|
|
61
|
+
return str(result)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _clean(mapping: dict) -> dict:
|
|
65
|
+
"""Redis hashes can't store None — drop those fields."""
|
|
66
|
+
return {k: v for k, v in mapping.items() if v is not None}
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def new_payload(
|
|
70
|
+
task_name: str,
|
|
71
|
+
args: Optional[Iterable[Any]] = None,
|
|
72
|
+
kwargs: Optional[dict] = None,
|
|
73
|
+
*,
|
|
74
|
+
priority: str = "default",
|
|
75
|
+
on_success_url: Optional[str] = None,
|
|
76
|
+
max_retries: Optional[int] = None,
|
|
77
|
+
timeout: Optional[float] = None,
|
|
78
|
+
task_id: Optional[str] = None,
|
|
79
|
+
trace_carrier: Optional[dict] = None,
|
|
80
|
+
) -> dict:
|
|
81
|
+
if priority not in PRIORITIES:
|
|
82
|
+
raise ValueError(f"priority must be one of {PRIORITIES}, got {priority!r}")
|
|
83
|
+
payload = {
|
|
84
|
+
"task_id": task_id or str(uuid.uuid4()),
|
|
85
|
+
"task_name": task_name,
|
|
86
|
+
"args": list(args or []),
|
|
87
|
+
"kwargs": dict(kwargs or {}),
|
|
88
|
+
"retry_count": 0,
|
|
89
|
+
"fence_token": 0,
|
|
90
|
+
"priority": priority,
|
|
91
|
+
"on_success_url": on_success_url,
|
|
92
|
+
}
|
|
93
|
+
if max_retries is not None:
|
|
94
|
+
payload["max_retries"] = int(max_retries)
|
|
95
|
+
if timeout is not None:
|
|
96
|
+
payload["timeout"] = float(timeout)
|
|
97
|
+
if trace_carrier:
|
|
98
|
+
payload["trace_carrier"] = trace_carrier
|
|
99
|
+
return payload
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _meta(payload: dict, raw: str, status: str, now: float, delay_seconds: float = 0) -> dict:
|
|
103
|
+
return _clean({
|
|
104
|
+
"task_id": payload["task_id"],
|
|
105
|
+
"task_name": payload["task_name"],
|
|
106
|
+
"status": status,
|
|
107
|
+
"priority": payload.get("priority", "default"),
|
|
108
|
+
"created_at": now,
|
|
109
|
+
"scheduled_for": now + delay_seconds if delay_seconds > 0 else None,
|
|
110
|
+
"args": json.dumps(payload.get("args", []), default=str),
|
|
111
|
+
"kwargs": json.dumps(payload.get("kwargs", {}), default=str),
|
|
112
|
+
"retry_count": payload.get("retry_count", 0),
|
|
113
|
+
"payload": raw,
|
|
114
|
+
})
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
async def record(r, payload: dict, status: str = QUEUED) -> None:
|
|
118
|
+
"""Create the status hash + history entry for a payload pushed elsewhere (e.g. cron Lua)."""
|
|
119
|
+
raw = json.dumps(payload)
|
|
120
|
+
task_id = payload["task_id"]
|
|
121
|
+
async with r.pipeline(transaction=True) as pipe:
|
|
122
|
+
pipe.hset(task_key(task_id), mapping=_meta(payload, raw, status, time.time()))
|
|
123
|
+
pipe.expire(task_key(task_id), int(settings.RESULT_TTL))
|
|
124
|
+
pipe.lpush(RECENT_KEY, task_id)
|
|
125
|
+
pipe.ltrim(RECENT_KEY, 0, max(1, settings.HISTORY_LIMIT) - 1)
|
|
126
|
+
pipe.publish(EVENTS_CHANNEL, "update")
|
|
127
|
+
await pipe.execute()
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
async def enqueue(r, payload: dict, delay_seconds: float = 0, created_by: Optional[str] = None) -> dict:
|
|
131
|
+
"""Push a task payload onto its priority queue (or the delayed set)."""
|
|
132
|
+
task_id = payload["task_id"]
|
|
133
|
+
priority = payload.get("priority", "default")
|
|
134
|
+
raw = json.dumps(payload)
|
|
135
|
+
now = time.time()
|
|
136
|
+
delay_seconds = max(0.0, float(delay_seconds or 0))
|
|
137
|
+
status = SCHEDULED if delay_seconds > 0 else QUEUED
|
|
138
|
+
meta = _meta(payload, raw, status, now, delay_seconds)
|
|
139
|
+
if created_by:
|
|
140
|
+
meta["created_by"] = created_by
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
async with r.pipeline(transaction=True) as pipe:
|
|
144
|
+
if delay_seconds > 0:
|
|
145
|
+
pipe.zadd("delayed_tasks", {raw: now + delay_seconds})
|
|
146
|
+
pipe.incr("stats:delayed")
|
|
147
|
+
else:
|
|
148
|
+
pipe.lpush(queue_key(priority), raw)
|
|
149
|
+
pipe.incr("stats:pending")
|
|
150
|
+
pipe.hset(task_key(task_id), mapping=meta)
|
|
151
|
+
pipe.expire(task_key(task_id), int(settings.RESULT_TTL + delay_seconds))
|
|
152
|
+
pipe.lpush(RECENT_KEY, task_id)
|
|
153
|
+
pipe.ltrim(RECENT_KEY, 0, max(1, settings.HISTORY_LIMIT) - 1)
|
|
154
|
+
pipe.publish(EVENTS_CHANNEL, "update")
|
|
155
|
+
await pipe.execute()
|
|
156
|
+
|
|
157
|
+
result = {"task_id": task_id, "status": status.lower()}
|
|
158
|
+
if delay_seconds > 0:
|
|
159
|
+
result["execute_in_seconds"] = delay_seconds
|
|
160
|
+
return result
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
# Atomically pull a waiting task out of its queue or the delayed set.
|
|
164
|
+
# KEYS: queue list, delayed zset, task hash, stats:pending, stats:delayed
|
|
165
|
+
# ARGV: exact payload JSON, cancelled timestamp
|
|
166
|
+
CANCEL_SCRIPT = """
|
|
167
|
+
local removed = redis.call('LREM', KEYS[1], 1, ARGV[1])
|
|
168
|
+
if removed > 0 then
|
|
169
|
+
redis.call('DECR', KEYS[4])
|
|
170
|
+
else
|
|
171
|
+
removed = redis.call('ZREM', KEYS[2], ARGV[1])
|
|
172
|
+
if removed > 0 then
|
|
173
|
+
redis.call('DECR', KEYS[5])
|
|
174
|
+
end
|
|
175
|
+
end
|
|
176
|
+
if removed > 0 then
|
|
177
|
+
redis.call('HSET', KEYS[3], 'status', 'Cancelled', 'finished_at', ARGV[2])
|
|
178
|
+
return 1
|
|
179
|
+
end
|
|
180
|
+
return 0
|
|
181
|
+
"""
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
async def cancel(r, task_id: str, by: Optional[str] = None) -> tuple[bool, str]:
|
|
185
|
+
"""
|
|
186
|
+
Cancel a task that has not started yet. Returns (cancelled, status).
|
|
187
|
+
Running or finished tasks cannot be cancelled.
|
|
188
|
+
"""
|
|
189
|
+
meta = await r.hgetall(task_key(task_id))
|
|
190
|
+
if not meta:
|
|
191
|
+
return False, "NotFound"
|
|
192
|
+
status = meta.get("status", "")
|
|
193
|
+
payload = meta.get("payload")
|
|
194
|
+
if status in FINAL_STATES or status == PROCESSING or not payload:
|
|
195
|
+
return False, status
|
|
196
|
+
ok = await r.eval(
|
|
197
|
+
CANCEL_SCRIPT, 5,
|
|
198
|
+
queue_key(meta.get("priority", "default")), "delayed_tasks", task_key(task_id),
|
|
199
|
+
"stats:pending", "stats:delayed",
|
|
200
|
+
payload, str(time.time()),
|
|
201
|
+
)
|
|
202
|
+
if ok:
|
|
203
|
+
if by:
|
|
204
|
+
await r.hset(task_key(task_id), "cancelled_by", by)
|
|
205
|
+
await r.publish(EVENTS_CHANNEL, "update")
|
|
206
|
+
return True, CANCELLED
|
|
207
|
+
# Payload was picked up between the read and the script — report fresh state
|
|
208
|
+
return False, (await r.hget(task_key(task_id), "status")) or status
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def public_view(meta: dict) -> dict:
|
|
212
|
+
"""Task hash minus internal fields, with JSON fields decoded for clients."""
|
|
213
|
+
out = {k: v for k, v in meta.items() if k != "payload"}
|
|
214
|
+
for field in ("args", "kwargs"):
|
|
215
|
+
if field in out:
|
|
216
|
+
try:
|
|
217
|
+
out[field] = json.loads(out[field])
|
|
218
|
+
except (TypeError, ValueError):
|
|
219
|
+
pass
|
|
220
|
+
for field in ("created_at", "scheduled_for", "started_at", "finished_at", "duration_ms"):
|
|
221
|
+
if field in out:
|
|
222
|
+
try:
|
|
223
|
+
out[field] = float(out[field])
|
|
224
|
+
except (TypeError, ValueError):
|
|
225
|
+
pass
|
|
226
|
+
if "retry_count" in out:
|
|
227
|
+
try:
|
|
228
|
+
out["retry_count"] = int(out["retry_count"])
|
|
229
|
+
except (TypeError, ValueError):
|
|
230
|
+
pass
|
|
231
|
+
return out
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
async def recent_tasks(r, limit: int = 50, status: Optional[str] = None, task_name: Optional[str] = None) -> list[dict]:
|
|
235
|
+
"""Newest-first task history, optionally filtered by status / task name."""
|
|
236
|
+
limit = max(1, min(limit, settings.HISTORY_LIMIT))
|
|
237
|
+
ids = await r.lrange(RECENT_KEY, 0, settings.HISTORY_LIMIT - 1)
|
|
238
|
+
out: list[dict] = []
|
|
239
|
+
seen = set()
|
|
240
|
+
batch = 100
|
|
241
|
+
for start in range(0, len(ids), batch):
|
|
242
|
+
chunk = [i for i in ids[start:start + batch] if i not in seen]
|
|
243
|
+
seen.update(chunk)
|
|
244
|
+
async with r.pipeline(transaction=False) as pipe:
|
|
245
|
+
for tid in chunk:
|
|
246
|
+
pipe.hgetall(task_key(tid))
|
|
247
|
+
metas = await pipe.execute()
|
|
248
|
+
for meta in metas:
|
|
249
|
+
if not meta:
|
|
250
|
+
continue # expired
|
|
251
|
+
if status and meta.get("status", "").lower() != status.lower():
|
|
252
|
+
continue
|
|
253
|
+
if task_name and meta.get("task_name") != task_name:
|
|
254
|
+
continue
|
|
255
|
+
out.append(public_view(meta))
|
|
256
|
+
if len(out) >= limit:
|
|
257
|
+
return out
|
|
258
|
+
return out
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
async def registered_tasks(r) -> list[dict]:
|
|
262
|
+
"""Tasks advertised by running workers (or the dashboard's --app)."""
|
|
263
|
+
raw = await r.hgetall(REGISTRY_KEY)
|
|
264
|
+
tasks = []
|
|
265
|
+
for name, spec in sorted(raw.items()):
|
|
266
|
+
try:
|
|
267
|
+
info = json.loads(spec)
|
|
268
|
+
except ValueError:
|
|
269
|
+
info = {}
|
|
270
|
+
info["name"] = name
|
|
271
|
+
tasks.append(info)
|
|
272
|
+
return tasks
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
async def publish_registry(r, specs: dict[str, dict]) -> None:
|
|
276
|
+
"""Replace the advertised task registry with `specs` (name → JSON-able spec)."""
|
|
277
|
+
async with r.pipeline(transaction=True) as pipe:
|
|
278
|
+
pipe.delete(REGISTRY_KEY)
|
|
279
|
+
if specs:
|
|
280
|
+
pipe.hset(REGISTRY_KEY, mapping={k: json.dumps(v, default=str) for k, v in specs.items()})
|
|
281
|
+
pipe.publish(EVENTS_CHANNEL, "update")
|
|
282
|
+
await pipe.execute()
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
from redis import asyncio as redis
|
|
2
|
+
from pytaskq.core.config import settings
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def make_client() -> "redis.Redis":
|
|
6
|
+
"""Build the shared async Redis client from settings (REDIS_URL wins)."""
|
|
7
|
+
common = dict(decode_responses=True, health_check_interval=30)
|
|
8
|
+
if settings.REDIS_URL:
|
|
9
|
+
return redis.Redis.from_url(settings.REDIS_URL, **common)
|
|
10
|
+
return redis.Redis(
|
|
11
|
+
host=settings.REDIS_HOST,
|
|
12
|
+
port=settings.REDIS_PORT,
|
|
13
|
+
db=settings.REDIS_DB,
|
|
14
|
+
password=settings.REDIS_PASSWORD or None,
|
|
15
|
+
**common,
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
r = make_client()
|
pytaskq/main.py
ADDED
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import os
|
|
3
|
+
from contextlib import asynccontextmanager
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
from fastapi import FastAPI
|
|
7
|
+
from fastapi.middleware.cors import CORSMiddleware
|
|
8
|
+
from fastapi.responses import FileResponse, HTMLResponse, JSONResponse
|
|
9
|
+
from fastapi.staticfiles import StaticFiles
|
|
10
|
+
from redis.exceptions import ConnectionError as RedisConnectionError
|
|
11
|
+
|
|
12
|
+
from pytaskq import __version__
|
|
13
|
+
from pytaskq.api.router import api_router
|
|
14
|
+
from pytaskq.core.config import settings
|
|
15
|
+
from pytaskq.core.exceptions import redis_connection_error_handler
|
|
16
|
+
|
|
17
|
+
logging.basicConfig(
|
|
18
|
+
level=logging.INFO,
|
|
19
|
+
format="%(asctime)s [%(levelname)s] [%(processName)s] %(message)s",
|
|
20
|
+
datefmt="%Y-%m-%d %H:%M:%S",
|
|
21
|
+
)
|
|
22
|
+
logger = logging.getLogger("api")
|
|
23
|
+
|
|
24
|
+
# Resolve relative to this file so the API works from any working directory
|
|
25
|
+
STATIC_DIR = Path(__file__).resolve().parent / "static"
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
async def _publish_app_registry():
|
|
29
|
+
"""If PYTASKQ_APP is set (e.g. `pytaskq dashboard --app x:queue`), show its tasks right away."""
|
|
30
|
+
spec = os.environ.get("PYTASKQ_APP", "")
|
|
31
|
+
if not spec:
|
|
32
|
+
return
|
|
33
|
+
try:
|
|
34
|
+
from pytaskq.cli import load_app
|
|
35
|
+
await load_app(spec).publish()
|
|
36
|
+
logger.info(f"Published task registry for {spec}")
|
|
37
|
+
except Exception as e: # never block startup on this
|
|
38
|
+
logger.warning(f"Could not publish task registry for {spec}: {e}")
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
async def _prepare_auth():
|
|
42
|
+
"""Create the first admin from env vars, or print the one-time setup code."""
|
|
43
|
+
from pytaskq.core import auth, redis_client
|
|
44
|
+
if not settings.auth_enabled:
|
|
45
|
+
logger.warning("Sign-in is DISABLED (PYTASKQ_AUTH=off). Anyone who can reach this server has full control.")
|
|
46
|
+
return
|
|
47
|
+
try:
|
|
48
|
+
r = redis_client.r
|
|
49
|
+
user = await auth.bootstrap_from_env(r)
|
|
50
|
+
if user:
|
|
51
|
+
logger.info(f"Created admin '{user.username}' from PYTASKQ_ADMIN_USER.")
|
|
52
|
+
code = await auth.ensure_setup_code(r)
|
|
53
|
+
if code:
|
|
54
|
+
bar = "=" * 64
|
|
55
|
+
logger.warning(
|
|
56
|
+
f"\n{bar}\n PyTaskQ first-run setup\n\n"
|
|
57
|
+
f" Open the dashboard and enter this setup code to create the admin:\n\n"
|
|
58
|
+
f" {code}\n\n"
|
|
59
|
+
f" (Lost it? Run: pytaskq users setup-code)\n{bar}"
|
|
60
|
+
)
|
|
61
|
+
except Exception as e: # Redis down at startup must not crash the API
|
|
62
|
+
logger.warning(f"Could not prepare sign-in yet: {e}")
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@asynccontextmanager
|
|
66
|
+
async def lifespan(app: FastAPI):
|
|
67
|
+
await _publish_app_registry()
|
|
68
|
+
await _prepare_auth()
|
|
69
|
+
yield
|
|
70
|
+
from pytaskq.core import redis_client
|
|
71
|
+
await redis_client.r.aclose()
|
|
72
|
+
|
|
73
|
+
from opentelemetry import trace
|
|
74
|
+
provider = trace.get_tracer_provider()
|
|
75
|
+
if hasattr(provider, "force_flush"):
|
|
76
|
+
provider.force_flush()
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
app = FastAPI(
|
|
80
|
+
title="PyTaskQ",
|
|
81
|
+
description="Distributed async task queue API",
|
|
82
|
+
version=__version__,
|
|
83
|
+
docs_url=None if settings.ENVIRONMENT == "production" else "/docs",
|
|
84
|
+
redoc_url=None if settings.ENVIRONMENT == "production" else "/redoc",
|
|
85
|
+
lifespan=lifespan,
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
app.add_middleware(
|
|
89
|
+
CORSMiddleware,
|
|
90
|
+
allow_origins=settings.allowed_origins_list,
|
|
91
|
+
allow_credentials=True,
|
|
92
|
+
allow_methods=["GET", "POST", "PATCH", "DELETE"],
|
|
93
|
+
allow_headers=["Authorization", "Content-Type"],
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
if STATIC_DIR.is_dir():
|
|
97
|
+
app.mount("/static", StaticFiles(directory=str(STATIC_DIR)), name="static")
|
|
98
|
+
|
|
99
|
+
app.add_exception_handler(RedisConnectionError, redis_connection_error_handler)
|
|
100
|
+
|
|
101
|
+
app.include_router(api_router)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
@app.get("/health", tags=["ops"])
|
|
105
|
+
async def health_check():
|
|
106
|
+
"""Used by Docker health checks and load balancers to verify service health."""
|
|
107
|
+
try:
|
|
108
|
+
from pytaskq.core import redis_client
|
|
109
|
+
await redis_client.r.ping()
|
|
110
|
+
return {"status": "ok", "redis": "connected", "version": __version__}
|
|
111
|
+
except Exception:
|
|
112
|
+
return JSONResponse(status_code=503, content={"status": "degraded", "redis": "unreachable"})
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
@app.get("/", include_in_schema=False)
|
|
116
|
+
def homepage():
|
|
117
|
+
index = STATIC_DIR / "index.html"
|
|
118
|
+
if index.is_file():
|
|
119
|
+
return FileResponse(str(index))
|
|
120
|
+
return HTMLResponse(
|
|
121
|
+
"<h1>PyTaskQ API is running</h1><p>The dashboard bundle was not found. "
|
|
122
|
+
"Build it with <code>cd frontend && npm install && npm run build</code>, "
|
|
123
|
+
"or use the API docs at <a href='/docs'>/docs</a>.</p>"
|
|
124
|
+
)
|
pytaskq/net.py
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""
|
|
2
|
+
pytaskq.net — safe outbound HTTP helpers for task handlers
|
|
3
|
+
===========================================================
|
|
4
|
+
|
|
5
|
+
Task handlers often fetch user-supplied URLs. Without a guard a caller could make
|
|
6
|
+
workers request internal services (localhost, 169.254.169.254 cloud metadata,
|
|
7
|
+
RFC1918 ranges, ...). `assert_safe_url` resolves the hostname and rejects any URL
|
|
8
|
+
whose DNS points at a non-public address; `SAFE_OPENER` re-validates every redirect
|
|
9
|
+
hop so http://public.host/ can't bounce a worker behind the firewall.
|
|
10
|
+
|
|
11
|
+
Use it in your own tasks:
|
|
12
|
+
|
|
13
|
+
from pytaskq.net import assert_safe_url, SAFE_OPENER
|
|
14
|
+
assert_safe_url(url)
|
|
15
|
+
with SAFE_OPENER.open(url, timeout=10) as resp: ...
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
import ipaddress
|
|
19
|
+
import socket
|
|
20
|
+
import urllib.parse
|
|
21
|
+
import urllib.request
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class UnsafeURL(Exception):
|
|
25
|
+
"""Raised when a user-supplied URL targets a non-public address."""
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def assert_safe_url(url: str) -> None:
|
|
29
|
+
parsed = urllib.parse.urlparse(url)
|
|
30
|
+
if parsed.scheme not in ("http", "https"):
|
|
31
|
+
raise UnsafeURL(f"Only http/https URLs are allowed, got: {parsed.scheme!r}")
|
|
32
|
+
host = parsed.hostname
|
|
33
|
+
if not host:
|
|
34
|
+
raise UnsafeURL("URL has no hostname")
|
|
35
|
+
if host in ("localhost",) or host.endswith((".localhost", ".local", ".internal")):
|
|
36
|
+
raise UnsafeURL(f"Access to internal host {host!r} is not allowed")
|
|
37
|
+
try:
|
|
38
|
+
addr_infos = socket.getaddrinfo(host, None)
|
|
39
|
+
except socket.gaierror:
|
|
40
|
+
raise UnsafeURL(f"Cannot resolve host: {host}")
|
|
41
|
+
for info in addr_infos:
|
|
42
|
+
ip = ipaddress.ip_address(info[4][0])
|
|
43
|
+
if (
|
|
44
|
+
ip.is_private
|
|
45
|
+
or ip.is_loopback
|
|
46
|
+
or ip.is_link_local # 169.254.0.0/16 — cloud metadata lives here
|
|
47
|
+
or ip.is_reserved
|
|
48
|
+
or ip.is_multicast
|
|
49
|
+
or ip.is_unspecified
|
|
50
|
+
):
|
|
51
|
+
raise UnsafeURL(f"Access to non-public address {ip} is not allowed")
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class SafeRedirectHandler(urllib.request.HTTPRedirectHandler):
|
|
55
|
+
"""Re-validates every redirect target before following it."""
|
|
56
|
+
|
|
57
|
+
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
|
58
|
+
assert_safe_url(newurl)
|
|
59
|
+
return super().redirect_request(req, fp, code, msg, headers, newurl)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
SAFE_OPENER = urllib.request.build_opener(SafeRedirectHandler())
|
|
63
|
+
|
|
64
|
+
# Backwards-compatible private aliases
|
|
65
|
+
_assert_safe_url = assert_safe_url
|
|
66
|
+
_SafeRedirectHandler = SafeRedirectHandler
|
|
67
|
+
_SAFE_OPENER = SAFE_OPENER
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
from typing import Literal, Any, Dict, List, Optional
|
|
2
|
+
from pydantic import BaseModel, Field
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class Taskloader(BaseModel):
|
|
6
|
+
"""A task payload as it travels through Redis."""
|
|
7
|
+
task_id: str
|
|
8
|
+
task_name: str
|
|
9
|
+
args: List[Any] = Field(default_factory=list)
|
|
10
|
+
kwargs: Dict[str, Any] = Field(default_factory=dict)
|
|
11
|
+
retry_count: int = 0
|
|
12
|
+
task_type: str = "io"
|
|
13
|
+
fence_token: int = 0
|
|
14
|
+
prev_delay: float = 1.0
|
|
15
|
+
priority: str = "default" # preserved across retries
|
|
16
|
+
max_retries: Optional[int] = None # per-task override (else registry / settings)
|
|
17
|
+
timeout: Optional[float] = None # per-task override (else registry / settings)
|
|
18
|
+
trace_carrier: dict = Field(default_factory=dict)
|
|
19
|
+
on_success_url: Optional[str] = None # If set, worker enqueues a webhook delivery task on success
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class Taskresult(BaseModel):
|
|
23
|
+
task_id: str
|
|
24
|
+
status: str
|
|
25
|
+
result: str
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class TaskRequest(BaseModel):
|
|
29
|
+
task_name: str
|
|
30
|
+
args: List[Any] = Field(default_factory=list)
|
|
31
|
+
priority: Literal["high", "default", "low"] = "default"
|
|
32
|
+
idempotency_key: Optional[str] = None
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import time
|
|
2
|
+
import json
|
|
3
|
+
from pytaskq.core import redis_client
|
|
4
|
+
from pytaskq.core.config import settings
|
|
5
|
+
|
|
6
|
+
async def _notify_tenant(tenant_id: str, event_type: str) -> None:
|
|
7
|
+
await redis_client.r.publish("events:global", event_type)
|
|
8
|
+
|
|
9
|
+
async def _get_dashboard_data(tenant_id: str = "global") -> dict:
|
|
10
|
+
pending = max(0, int(await redis_client.r.get("stats:pending") or 0))
|
|
11
|
+
processing = max(0, int(await redis_client.r.get("stats:processing") or 0))
|
|
12
|
+
delayed = max(0, int(await redis_client.r.get("stats:delayed") or 0))
|
|
13
|
+
dlq_count = await redis_client.r.llen("dlq")
|
|
14
|
+
completed = int(await redis_client.r.get("stats:completed") or 0)
|
|
15
|
+
failed = int(await redis_client.r.get("stats:failed") or 0)
|
|
16
|
+
queue_high = await redis_client.r.llen("queue:high")
|
|
17
|
+
queue_default = await redis_client.r.llen("queue:default")
|
|
18
|
+
queue_low = await redis_client.r.llen("queue:low")
|
|
19
|
+
current_minute = int(time.time() / 60)
|
|
20
|
+
rate_used = int(await redis_client.r.get(f"rate_limit:global:{current_minute}") or 0)
|
|
21
|
+
# Cap what we push to dashboards; the full list is at GET /dlq
|
|
22
|
+
dlq_tasks = [json.loads(t) for t in await redis_client.r.lrange("dlq", 0, 99)]
|
|
23
|
+
worker_count = await redis_client.r.zcount("active_workers", time.time(), "+inf")
|
|
24
|
+
return {
|
|
25
|
+
"metrics": {
|
|
26
|
+
"pending": pending,
|
|
27
|
+
"processing": processing,
|
|
28
|
+
"delayed": delayed,
|
|
29
|
+
"dlq": dlq_count,
|
|
30
|
+
"completed_total": completed,
|
|
31
|
+
"failed_total": failed,
|
|
32
|
+
"queue_high": queue_high,
|
|
33
|
+
"queue_default": queue_default,
|
|
34
|
+
"queue_low": queue_low,
|
|
35
|
+
"queue_total": queue_high + queue_default + queue_low,
|
|
36
|
+
"queue_capacity": settings.QUEUE_CAPACITY,
|
|
37
|
+
"rate_limit_used": rate_used,
|
|
38
|
+
"rate_limit_max": settings.RATE_LIMIT_PER_MINUTE,
|
|
39
|
+
"worker_count": worker_count,
|
|
40
|
+
},
|
|
41
|
+
"dlq": dlq_tasks,
|
|
42
|
+
}
|